From ca00d3d3b06d7df8409be1abecc74f89fdd6a597 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Dec 2025 16:23:05 -0600
Subject: [PATCH 0001/1000] Tunred libcafcuda back on

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 328a375880..77f106a3f2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -60,7 +60,7 @@ option(CAF_ENABLE_QT6_EXAMPLES "Build examples with the Qt6 framework" OFF)
 option(CAF_ENABLE_ROBOT_TESTS "Add the Robot tests to CTest " OFF)
 option(CAF_ENABLE_RUNTIME_CHECKS "Build CAF with extra runtime assertions" OFF)
 option(CAF_USE_STD_FORMAT "Enable std::format support" OFF)
-option(CAF_ENABLE_CUDA "Build caf with cuda support" OFF)
+option(CAF_ENABLE_CUDA "Build caf with cuda support" ON)
 
 # -- CAF options that are on by default ----------------------------------------
 

From 73b3f9eee5637e82853608ebce4b3be15a5e78ad Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Dec 2025 16:24:54 -0600
Subject: [PATCH 0002/1000] Initial commit.

---
 .../control-layer/launch_response_token.hpp   | 67 +++++++++++++++++++
 .../caf/cuda/control-layer/launch_token.hpp   | 44 ++++++++++++
 libcaf_cuda/caf/cuda/control-layer/token.hpp  | 28 ++++++++
 3 files changed, 139 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/token.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
new file mode 100644
index 0000000000..4d40440076
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -0,0 +1,67 @@
+#pragma once
+#include "caf/scheduler/token.hpp"
+#include "caf/scheduler/launch_token.hpp"
+#include <caf/actor.hpp>
+#include "caf/cuda/all.hpp"
+#include <atomic>
+
+namespace caf::cuda {
+
+class launch_response_token : public token {
+public:
+    // Construct manually
+    launch_response_token(caf::actor receiver,
+                          nd_range range,
+                          int memory_usage,
+                          std::string id)
+        : receiver_(std::move(receiver)),
+          range_(std::move(range)),
+          memory_usage_(memory_usage),
+          id_(std::move(id)),
+          released_(false) {}
+
+    // Construct from a launch_token
+    launch_response_token(caf::actor receiver, const launch_token& token)
+        : receiver_(std::move(receiver)),
+          range_(token.getRange()),
+          memory_usage_(token.getMemoryUsage()),
+          id_(token.getId()),
+          released_(false) {}
+
+    ~launch_response_token() {
+        release();
+    }
+
+    int getType() override { return 2; }
+
+    const nd_range& getRange() const { return range_; }
+    int getMemoryUsage() const { return memory_usage_; }
+
+    // Return requested number of CUDA blocks
+    int getBlocks() const {
+        return static_cast<int>(
+            range_.getGridDimX() *
+            range_.getGridDimY() *
+            range_.getGridDimZ()
+        );
+    }
+
+    const std::string& getId() const { return id_; }
+
+    void release() {
+        bool expected = false;
+        if (released_.compare_exchange_strong(expected, true)) {
+            caf::anon_send(receiver_, id_, getBlocks());
+        }
+    }
+
+private:
+    caf::actor receiver_;
+    nd_range range_;
+    int memory_usage_;
+    std::string id_;
+    std::atomic<bool> released_;
+};
+
+} // namespace caf::cuda
+
diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
new file mode 100644
index 0000000000..0a870acedb
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
@@ -0,0 +1,44 @@
+#pragma once
+#include "caf/scheduler/token.hpp"
+#include "caf/cuda/all.hpp"
+#include <string>
+
+namespace caf::cuda {
+
+class launch_token : public token {
+public:
+    launch_token(program_ptr prog,
+                 nd_range range,
+                 int memory_usage,
+                 std::string id)
+        : program_(std::move(prog)),
+          range_(std::move(range)),
+          memory_usage_(memory_usage),
+          id_(std::move(id)) {}
+
+    int getType() override { return 1; }
+
+    const program_ptr& getProgram() const { return program_; }
+    const nd_range& getRange() const { return range_; }
+    int getMemoryUsage() const { return memory_usage_; }
+
+    // Return requested number of CUDA blocks (gridDimX * gridDimY * gridDimZ)
+    int getBlocks() const {
+        return static_cast<int>(
+            range_.getGridDimX() *
+            range_.getGridDimY() *
+            range_.getGridDimZ()
+        );
+    }
+
+    const std::string& getId() const { return id_; }
+
+private:
+    program_ptr program_;
+    nd_range range_;
+    int memory_usage_;
+    std::string id_;
+};
+
+} // namespace caf::cuda
+
diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
new file mode 100644
index 0000000000..3894868e84
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -0,0 +1,28 @@
+#pragma once
+#include <atomic>
+#include "caf/all.hpp"
+
+namespace caf::cuda {
+
+// Base token interface
+class token : public caf::ref_counted {
+public:
+    virtual ~token() = default;
+    virtual int getType() = 0;
+
+protected:
+    std::atomic<size_t> ref_count_{0};
+
+    friend void intrusive_ptr_add_ref(const token* p) noexcept {
+        p->ref_count_.fetch_add(1, std::memory_order_relaxed);
+    }
+
+    friend void intrusive_ptr_release(const token* p) noexcept {
+        if (p->ref_count_.fetch_sub(1, std::memory_order_acq_rel) == 1)
+            delete p;
+    }
+};
+
+using token_ptr = caf::intrusive_ptr<token>;
+
+} // namespace caf::cuda

From cf6d5a8ca85a2bacfe2f589e9fe57e55e878ccdc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Dec 2025 16:32:42 -0600
Subject: [PATCH 0003/1000] Initial commit.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
new file mode 100644
index 0000000000..5db45023db
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -0,0 +1,8 @@
+#pragma once
+#include "caf/cuda/control-layer/token.hpp"
+#include "caf/cuda/control-layer/launch_token.hpp"
+#include "caf/cuda/control-layer/launch_response_token.hpp"
+
+
+
+

From c0c85877560a5bcbd325405830eacd3dd3f56de3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Dec 2025 16:34:31 -0600
Subject: [PATCH 0004/1000] Added a new header include.

---
 libcaf_cuda/caf/cuda/global.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index f9c81ca912..483203dc51 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -13,6 +13,7 @@
 // CAF type ID registration
 #include <caf/type_id.hpp>
 #include <caf/anon_mail.hpp>
+#include "caf/cuda/control-layer/all-control-layer.hpp"
 
 //a strange fix required in order to get the .so files to become viewable for binaries
 //linking against them, if this is not defined with classes you want viewable then

From ce871de4ce2e948ddfdec0f7367024a63602189f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Dec 2025 16:57:16 -0600
Subject: [PATCH 0005/1000] Initial commit.

---
 .../caf/cuda/control-layer/behavior.hpp       | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/behavior.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
new file mode 100644
index 0000000000..8a494c668a
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
@@ -0,0 +1,26 @@
+#include "caf/cuda/control-layer/launch_token.hpp"
+#include "caf/cuda/control-layer/launch_response_token.hpp"
+
+
+
+//this class is meant to provide an interface so that 
+//the actor can change behavior at runtime
+//normally I would say use become 
+//however that is too much boiler plate 
+//so instead we can just build a behavior
+//abstract class
+//and create a dispatch table
+
+namespace caf::cuda {
+class scheduler_actor_behavior {
+
+public:
+
+	virtual void schedule() = 0;
+	virtual void receive(launch_token token) = 0;
+
+};
+
+
+} //namespace caf::cuda
+

From d526b808ccf23e488f778e0cadfe30ee61df97af Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 16:04:36 -0600
Subject: [PATCH 0006/1000] Added token type identifiers.

---
 libcaf_cuda/caf/cuda/control-layer/token.hpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index 3894868e84..614bc56428 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -2,6 +2,13 @@
 #include <atomic>
 #include "caf/all.hpp"
 
+//types of tokens
+#define LAUNCH 1
+#define LAUNCH_RESPONSE 2
+#define BEHAVIOR 3
+
+
+
 namespace caf::cuda {
 
 // Base token interface

From eedaaddf2c9f40104c5bd96621bd834e04d0b938 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 16:07:04 -0600
Subject: [PATCH 0007/1000] Fixed include paths and adjusted get type methods
 to return the appropriate type identifier in token.hpp.

---
 .../caf/cuda/control-layer/launch_response_token.hpp        | 6 +++---
 libcaf_cuda/caf/cuda/control-layer/launch_token.hpp         | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
index 4d40440076..c99af10c69 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -1,6 +1,6 @@
 #pragma once
-#include "caf/scheduler/token.hpp"
-#include "caf/scheduler/launch_token.hpp"
+#include "caf/cuda/control-layer/token.hpp"
+#include "caf/cuda/control-layer/launch_token.hpp"
 #include <caf/actor.hpp>
 #include "caf/cuda/all.hpp"
 #include <atomic>
@@ -32,7 +32,7 @@ class launch_response_token : public token {
         release();
     }
 
-    int getType() override { return 2; }
+    int getType() override { return LAUNCH_RESPONSE; }
 
     const nd_range& getRange() const { return range_; }
     int getMemoryUsage() const { return memory_usage_; }
diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
index 0a870acedb..e14f64858f 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
@@ -1,5 +1,5 @@
 #pragma once
-#include "caf/scheduler/token.hpp"
+#include "caf/cuda/control-layer/token.hpp"
 #include "caf/cuda/all.hpp"
 #include <string>
 
@@ -16,7 +16,7 @@ class launch_token : public token {
           memory_usage_(memory_usage),
           id_(std::move(id)) {}
 
-    int getType() override { return 1; }
+    int getType() override { return LAUNCH; }
 
     const program_ptr& getProgram() const { return program_; }
     const nd_range& getRange() const { return range_; }

From 4e914a38a35ee5e8e126446d34d1af31d5c7365e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 16:09:34 -0600
Subject: [PATCH 0008/1000] Fixed include paths.

---
 libcaf_cuda/caf/cuda/control-layer/token.hpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index 614bc56428..e6fafe9bff 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -1,6 +1,8 @@
 #pragma once
 #include <atomic>
-#include "caf/all.hpp"
+#include "caf/cuda/all.hpp"
+#include <caf/intrusive_ptr.hpp>
+#include <caf/ref_counted.hpp>
 
 //types of tokens
 #define LAUNCH 1

From 6bf09bc1325cd3e9fc73c74615fed2f228d2e6ba Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 16:13:57 -0600
Subject: [PATCH 0009/1000] Initial commit.

---
 .../caf/cuda/control-layer/behavior_token.hpp | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp
new file mode 100644
index 0000000000..a38439bee6
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp
@@ -0,0 +1,23 @@
+#include "caf/cuda/control-layer/token.hpp"
+#include <string>
+
+
+namespace caf::cuda {
+
+	class behavior_token : token {
+	
+		public:
+			behavior_token(String behavior) :
+				behavior_(behavior) {}
+
+			int getType() override {return BEHAVIOR;}
+			String getBehavior() {return behavior_;}
+		private:
+			String behavior_;
+
+
+	};
+
+
+
+}//namespace caf::cuda

From df0e7949b9212eb02449d885be135cece75cc109 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 16:20:35 -0600
Subject: [PATCH 0010/1000] Fixed syntax error.

---
 libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp
index a38439bee6..acf49f940d 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp
@@ -7,13 +7,13 @@ namespace caf::cuda {
 	class behavior_token : token {
 	
 		public:
-			behavior_token(String behavior) :
+			behavior_token(std::string behavior) :
 				behavior_(behavior) {}
 
 			int getType() override {return BEHAVIOR;}
 			String getBehavior() {return behavior_;}
 		private:
-			String behavior_;
+			std::string behavior_;
 
 
 	};

From d2e271047e225f1013ac64fccf5bb8243eb7979c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 16:30:50 -0600
Subject: [PATCH 0011/1000] Initial commit.

---
 .../caf/cuda/control-layer/behavior_table.hpp | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
new file mode 100644
index 0000000000..215b12b696
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
@@ -0,0 +1,34 @@
+#include "caf/cuda/control-layer/token.hpp"
+#include "caf/cuda/control-layer/behavior.hpp"
+#include "caf/cuda/control-layer/behavior_token.hpp"
+#include <unordered_map>
+#include <string>
+
+namespace caf::cuda {
+
+class behavior_table {
+public:
+    behavior_table() {
+        // TODO insert behaviors here
+    }
+
+    /*
+     * Given a behavior_token, return a scheduler_actor_behavior*.
+     * Returns nullptr if nothing is found.
+     */
+    scheduler_actor_behavior* getBehavior(behavior_token tok) {
+
+        auto it = table.find(tok.getBehavior());
+
+        if (it != table.end())
+            return it->second;
+
+        return nullptr;
+    }
+
+private:
+    std::unordered_map<std::string, scheduler_actor_behavior*> table;
+};
+
+} // namespace caf::cuda
+

From 5b7057bc76f98c84fecb671e4486d89ca3eb3c6b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 16:33:10 -0600
Subject: [PATCH 0012/1000] Updated message serialization types.

---
 libcaf_cuda/caf/cuda/global.hpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index 483203dc51..7012581b36 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -201,4 +201,7 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<int>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<float>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<double>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<char>)
-
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::ndrange)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::program_ptr)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::token_ptr)
+//TODO may need to add launch and launch response token pointer

From 8de63c37f3f5dd282e1949511297a9232fa48ce9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 16:48:02 -0600
Subject: [PATCH 0013/1000] Initial commit.

---
 .../caf/cuda/control-layer/scheduler_actor.hpp | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
new file mode 100644
index 0000000000..74e7e7b6d3
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
@@ -0,0 +1,18 @@
+#pragma once
+#include <caf/all.hpp>
+#include "caf/cuda/control-layer/launch_token.hpp"
+#include "caf/cuda/control-layer/launch_response_token.hpp"
+#include <queue>
+
+/*
+ * The scheduler actor 
+ * meant to make scheduling decisions using s/r/r ipc
+ */
+
+struct scheduler_actor_state {
+	std::queue<token_ptr> queue;
+}
+
+
+caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>);
+

From c6292b9870ce7ce06b145f99fb5ad733333e6d73 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 17:06:05 -0600
Subject: [PATCH 0014/1000] Refactoring to fix circular dependencies.

---
 libcaf_cuda/caf/cuda/control-layer/behavior.hpp | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
index 8a494c668a..76be6ca082 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
@@ -1,3 +1,4 @@
+#pragma once 
 #include "caf/cuda/control-layer/launch_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 
@@ -12,15 +13,16 @@
 //and create a dispatch table
 
 namespace caf::cuda {
-class scheduler_actor_behavior {
 
-public:
+class launch_token;
 
-	virtual void schedule() = 0;
-	virtual void receive(launch_token token) = 0;
+class scheduler_actor_behavior {
+public:
+    virtual ~scheduler_actor_behavior() = default;
 
+    virtual void schedule() = 0;
+    virtual void receive(class scheduler_actor_state* state, const launch_token& tok) = 0;
 };
 
-
-} //namespace caf::cuda
+} // namespace caf::cuda
 

From 188543450fa543022a83f3f274f110ed21d9fef7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 17:11:21 -0600
Subject: [PATCH 0015/1000] More refactors.

---
 libcaf_cuda/caf/cuda/control-layer/behavior.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
index 76be6ca082..fb240e0eb4 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
@@ -1,7 +1,7 @@
 #pragma once 
 #include "caf/cuda/control-layer/launch_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
-
+#include "caf/cuda/control-layer/scheduler_actor_state.hpp"
 
 
 //this class is meant to provide an interface so that 
@@ -21,7 +21,7 @@ class scheduler_actor_behavior {
     virtual ~scheduler_actor_behavior() = default;
 
     virtual void schedule() = 0;
-    virtual void receive(class scheduler_actor_state* state, const launch_token& tok) = 0;
+    virtual void receive(scheduler_actor_state* state, const token_ptr& tok) = 0;
 };
 
 } // namespace caf::cuda

From e0a77f10371543585416df70e9d4eeb6a4af26f7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 17:12:56 -0600
Subject: [PATCH 0016/1000] Refactored to avoid circular dependencies, added
 pragma once and have a return all behaviors method now.

---
 .../caf/cuda/control-layer/behavior_table.hpp | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
index 215b12b696..f89dab96d4 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
@@ -1,33 +1,33 @@
+#pragma once
 #include "caf/cuda/control-layer/token.hpp"
 #include "caf/cuda/control-layer/behavior.hpp"
 #include "caf/cuda/control-layer/behavior_token.hpp"
 #include <unordered_map>
 #include <string>
 
+
 namespace caf::cuda {
 
+class scheduler_actor_behavior;
+class behavior_token;
+
 class behavior_table {
 public:
-    behavior_table() {
-        // TODO insert behaviors here
-    }
-
-    /*
-     * Given a behavior_token, return a scheduler_actor_behavior*.
-     * Returns nullptr if nothing is found.
-     */
-    scheduler_actor_behavior* getBehavior(behavior_token tok) {
+    behavior_table() = default;
 
-        auto it = table.find(tok.getBehavior());
-
-        if (it != table.end())
-            return it->second;
+    void add(const std::string& name, scheduler_actor_behavior* beh) {
+        table_[name] = beh;
+    }
 
-        return nullptr;
+    scheduler_actor_behavior* get(const behavior_token& tok) const {
+        auto it = table_.find(tok.name());
+        return it != table_.end() ? it->second : nullptr;
     }
 
+    auto& all_behaviors() { return table_; }
+
 private:
-    std::unordered_map<std::string, scheduler_actor_behavior*> table;
+    std::unordered_map<std::string, scheduler_actor_behavior*> table_;
 };
 
 } // namespace caf::cuda

From 6ba434d335aa9a01588edaf5aacfa3c09dd39507 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 17:14:39 -0600
Subject: [PATCH 0017/1000] Initial commit.

---
 .../control-layer/scheduler_actor_state.hpp     | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
new file mode 100644
index 0000000000..7696d6bce2
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
@@ -0,0 +1,17 @@
+#pragma once
+#include <queue>
+#include "caf/cuda/control-layer/behavior_table.hpp"
+#include "caf/cuda/control-layer/behavior.hpp"
+#include "caf/cuda/control-layer/behavior_token.hpp"
+#include "caf/cuda/control-layer/token.hpp"
+
+namespace caf::cuda {
+
+struct scheduler_actor_state {
+    scheduler_actor_behavior* current_behavior = nullptr;
+    behavior_table table;
+    std::queue<token_ptr> queue;
+};
+
+} // namespace caf::cuda
+

From 34e588c01d578eb8ca38abc4b525a87639b6b293 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 17:16:24 -0600
Subject: [PATCH 0018/1000] Implemented template.

---
 .../control-layer/green_light_behavior.hpp    | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
new file mode 100644
index 0000000000..834af2f4db
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
@@ -0,0 +1,29 @@
+#pragma once
+#include "caf/cuda/control-layer/behavior.hpp"
+#include "caf/cuda/control-layer/launch_token.hpp"
+
+namespace caf::cuda {
+
+class green_light_behavior : public scheduler_actor_behavior {
+public:
+    void schedule() override {
+        // scheduling logic, if needed
+    }
+
+    void receive(scheduler_actor_state* state, const token_ptr& tok) override {
+        // flush the queue
+        while (!state->queue.empty()) {
+            auto queued = state->queue.front();
+            state->queue.pop();
+            // respond to queued token (demo: just print)
+        }
+
+        // handle current token immediately
+        // (demo: just print)
+    }
+};
+
+inline green_light_behavior GREEN_BEHAVIOR;
+
+} // namespace caf::cuda
+

From 787bed07ccd17ed5b3eed9430d618f563812876f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 17:17:13 -0600
Subject: [PATCH 0019/1000] Initial commit.

---
 .../cuda/control-layer/red_light_behavior.hpp | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
new file mode 100644
index 0000000000..ca55b7de4e
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
@@ -0,0 +1,20 @@
+#pragma once
+#include "caf/cuda/control-layer/behavior.hpp"
+
+namespace caf::cuda {
+
+class red_light_behavior : public scheduler_actor_behavior {
+public:
+    void schedule() override {
+        // scheduling logic, if needed
+    }
+
+    void receive(scheduler_actor_state* state, const token_ptr& tok) override {
+        state->queue.push(tok); // enqueue everything
+    }
+};
+
+inline red_light_behavior RED_BEHAVIOR;
+
+} // namespace caf::cuda
+

From b84343a97258d9bfa6bc22dc1cf68c3c3ee81f64 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 17:20:51 -0600
Subject: [PATCH 0020/1000] Initial commit.

---
 .../caf/cuda/control-layer/scheduler_actor.hpp      | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
index 74e7e7b6d3..b999bdfc30 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
@@ -2,6 +2,9 @@
 #include <caf/all.hpp>
 #include "caf/cuda/control-layer/launch_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
+#include "caf/cuda/control-layer/behavior.hpp"
+#include "caf/cuda/control-layer/behavior_table.hpp"
+#include "caf/cuda/control-layer/behavior_token.hpp"
 #include <queue>
 
 /*
@@ -9,10 +12,6 @@
  * meant to make scheduling decisions using s/r/r ipc
  */
 
-struct scheduler_actor_state {
-	std::queue<token_ptr> queue;
-}
-
-
-caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>);
-
+namespace caf::cuda {
+caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state> * self);
+}//namespace caf::cuda

From d94bf46576e11f32ddd0ba382cd6a4aead428452 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 17:25:05 -0600
Subject: [PATCH 0021/1000] Initial commit.

---
 .../src/control-layer/scheduler_actor.cpp     | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 libcaf_cuda/src/control-layer/scheduler_actor.cpp

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
new file mode 100644
index 0000000000..653fc7013a
--- /dev/null
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -0,0 +1,37 @@
+#include "caf/cuda/control-layer/scheduler_actor.hpp"
+#include "caf/cuda/control-layer/green_light_behavior.hpp"
+#include "caf/cuda/control-layer/red_light_behavior.hpp"
+
+/*
+ * This class is meant to handle actor GPU scheduling via s/r/r IPC 
+ * it has nothing to do with the scheduler class, that is kernel laye
+ */ 
+
+namespace caf::cuda {
+
+caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self) {
+
+    // populate the table
+    self->state().table.add("green", &GREEN_BEHAVIOR);
+    self->state().table.add("red", &RED_BEHAVIOR);
+
+    // default behavior
+    self->state().current_behavior = self->state().table.get(behavior_token("green"));
+
+    return {
+        [=](const token_ptr& tok) {
+            self->state().current_behavior->receive(&self->state(), tok);
+        },
+        [=](const behavior_token& tok) {
+            auto* next = self->state().table.get(tok);
+            if (next)
+                self->state().current_behavior = next;  // swap behavior
+        },
+        [=](schedule_request&) {
+            self->state().current_behavior->schedule();
+        }
+    };
+}
+
+} // namespace caf::cuda
+

From 3e3487da976078f1a61fb411c2cbd915b57f3fec Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 17:26:45 -0600
Subject: [PATCH 0022/1000] Updated file to build the scheduler actor.

---
 libcaf_cuda/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/CMakeLists.txt b/libcaf_cuda/CMakeLists.txt
index e664b2a6b2..594e084e8f 100644
--- a/libcaf_cuda/CMakeLists.txt
+++ b/libcaf_cuda/CMakeLists.txt
@@ -22,6 +22,7 @@ set(LIBCAF_CUDA_SRCS
   src/test.cpp
   src/scheduler.cpp
   src/streampool.cpp
+  src/control-layer/scheduler_actor.cpp
 )
 
 # -- add targets ---------------------------------------------------------------

From b7942b682683c0b9833da18e8c35b6a23bfc5b9f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 17:34:29 -0600
Subject: [PATCH 0023/1000] Removed control layer include and move unsafe
 message declarations to all-control-layer.hpp

---
 libcaf_cuda/caf/cuda/global.hpp | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index 7012581b36..41d94cfe2f 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -13,7 +13,6 @@
 // CAF type ID registration
 #include <caf/type_id.hpp>
 #include <caf/anon_mail.hpp>
-#include "caf/cuda/control-layer/all-control-layer.hpp"
 
 //a strange fix required in order to get the .so files to become viewable for binaries
 //linking against them, if this is not defined with classes you want viewable then
@@ -201,7 +200,5 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<int>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<float>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<double>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<char>)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::ndrange)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::nd_range)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::program_ptr)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::token_ptr)
-//TODO may need to add launch and launch response token pointer

From 6d665c25f1f12b2c5820adea0d0bb9e5eeee8661 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 17:34:56 -0600
Subject: [PATCH 0024/1000] Added unsafe message declarations.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 5db45023db..09b7903b25 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -4,5 +4,7 @@
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 
 
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::token_ptr)
+//TODO may need to add launch and launch response token pointer
 
 

From 807a34e28a4372679c1da7c8ada6ac073976bc82 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 17:36:28 -0600
Subject: [PATCH 0025/1000] Made ref count mutable.

---
 libcaf_cuda/caf/cuda/control-layer/token.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index e6fafe9bff..b3ae75da1c 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -20,7 +20,7 @@ class token : public caf::ref_counted {
     virtual int getType() = 0;
 
 protected:
-    std::atomic<size_t> ref_count_{0};
+    mutable std::atomic<size_t> ref_count_{0};
 
     friend void intrusive_ptr_add_ref(const token* p) noexcept {
         p->ref_count_.fetch_add(1, std::memory_order_relaxed);

From 3fce39537f72bc333b8f53feccabe1bce3fe87ce Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 17:43:29 -0600
Subject: [PATCH 0026/1000] Added more unsafe message tags.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 5 +++++
 libcaf_cuda/src/control-layer/scheduler_actor.cpp        | 3 ---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 09b7903b25..968d10e6fe 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -4,7 +4,12 @@
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 
 
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::token_ptr)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::behavior_token)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::behavior_token_ptr)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_token)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_response_token)
 //TODO may need to add launch and launch response token pointer
 
 
diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 653fc7013a..09f54fef21 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -26,9 +26,6 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self)
             auto* next = self->state().table.get(tok);
             if (next)
                 self->state().current_behavior = next;  // swap behavior
-        },
-        [=](schedule_request&) {
-            self->state().current_behavior->schedule();
         }
     };
 }

From 5527fbef58b47205ff1ca65c27bb9f2f9b1ea39f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 18:02:13 -0600
Subject: [PATCH 0027/1000] Shuffled things around to break circular
 dependecies to get everything to compile.

---
 .../cuda/control-layer/all-control-layer.hpp  | 21 +++++++++++++---
 .../caf/cuda/control-layer/behavior_token.hpp | 24 +++++++++----------
 2 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 968d10e6fe..79aef82166 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -1,15 +1,30 @@
 #pragma once
+#include "caf/cuda/all.hpp"
 #include "caf/cuda/control-layer/token.hpp"
 #include "caf/cuda/control-layer/launch_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
+#include "caf/cuda/control-layer/behavior_token.hpp"
+#include "caf/cuda/control-layer/behavior.hpp"
+#include "caf/cuda/control-layer/scheduler_actor.hpp"
 
+CAF_BEGIN_TYPE_ID_BLOCK(cuda_control, caf::first_custom_type_id + 200)
+// You can also use  caf::id_block::core::end  instead of the +200 if you prefer
 
+  CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::token))
+CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::token>))  
+CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::behavior_token))
+  CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::behavior_token>)) // ← and this one too if you ever use it
+	CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::launch_token))
+  CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::launch_response_token))
+
+  // Very commonly needed as well – add them now so you don’t hit the error later
+
+CAF_END_TYPE_ID_BLOCK(cuda_control)
+
+// Optional but harmless – keep your old macros (they silence the “unsafe” warning)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::token_ptr)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::behavior_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::behavior_token_ptr)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_response_token)
-//TODO may need to add launch and launch response token pointer
-
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp
index acf49f940d..ed4929c515 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp
@@ -1,23 +1,23 @@
+#pragma once
 #include "caf/cuda/control-layer/token.hpp"
 #include <string>
 
-
 namespace caf::cuda {
 
-	class behavior_token : token {
-	
-		public:
-			behavior_token(std::string behavior) :
-				behavior_(behavior) {}
+class behavior_token : public token {
+public:
+    explicit behavior_token(std::string n) : name_(std::move(n)) {}
 
-			int getType() override {return BEHAVIOR;}
-			String getBehavior() {return behavior_;}
-		private:
-			std::string behavior_;
+    const std::string& name() const { return name_; }
 
+    // override getType() from token
+    int getType() override { return BEHAVIOR; }
 
-	};
+private:
+    std::string name_;
+};
 
+using behavior_token_ptr = caf::intrusive_ptr<behavior_token>;
 
+} // namespace caf::cuda
 
-}//namespace caf::cuda

From 6f7ef95f0342cd91c5a0b7abfa6092e97d4ab19a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Dec 2025 18:02:28 -0600
Subject: [PATCH 0028/1000] Added missing includes and other stuff.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 09f54fef21..ba477359c5 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -1,3 +1,4 @@
+#include "caf/cuda/control-layer/all-control-layer.hpp"
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
 #include "caf/cuda/control-layer/green_light_behavior.hpp"
 #include "caf/cuda/control-layer/red_light_behavior.hpp"

From 1420d86d3e7b29876905e3037eec2d8436c62bc3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Dec 2025 15:50:19 -0600
Subject: [PATCH 0029/1000] Initial commit.

---
 libcaf_cuda/caf/cuda/manager_confg.hpp | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/manager_confg.hpp

diff --git a/libcaf_cuda/caf/cuda/manager_confg.hpp b/libcaf_cuda/caf/cuda/manager_confg.hpp
new file mode 100644
index 0000000000..bfa85c1338
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/manager_confg.hpp
@@ -0,0 +1,20 @@
+/*
+ * A class that is meant to enable users to configure 
+ * the caf cuda library
+ */
+
+namespace caf::cuda {
+
+class manager_config {
+public:
+    manager_config() : scheduler_on(false) {}      // initialize the bool
+    manager_config(bool scheduler) : scheduler_on(scheduler) {}
+
+    bool getSchedulerOn() const { return scheduler_on; } // should be const
+
+private:
+    bool scheduler_on;
+};
+
+} // namespace caf::cuda
+

From 808de7a71c17bde7f5b340e748d1acaa923b211a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Dec 2025 16:05:17 -0600
Subject: [PATCH 0030/1000] Fixed spelling error.

---
 libcaf_cuda/caf/cuda/{manager_confg.hpp => manager_config.hpp} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename libcaf_cuda/caf/cuda/{manager_confg.hpp => manager_config.hpp} (100%)

diff --git a/libcaf_cuda/caf/cuda/manager_confg.hpp b/libcaf_cuda/caf/cuda/manager_config.hpp
similarity index 100%
rename from libcaf_cuda/caf/cuda/manager_confg.hpp
rename to libcaf_cuda/caf/cuda/manager_config.hpp

From 82c7ed70219072993f08c58c8cb61d786f0e5779 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Dec 2025 16:19:04 -0600
Subject: [PATCH 0031/1000] Moved static methods out of the header file and
 into the cpp file

---
 libcaf_cuda/caf/cuda/manager.hpp | 42 ++++++++++++--------------------
 1 file changed, 16 insertions(+), 26 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index df9d84580e..9443b0543c 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -20,6 +20,8 @@
 #include "caf/cuda/program.hpp"
 #include "caf/cuda/actor_facade.hpp"
 #include "caf/cuda/platform.hpp"
+#include "caf/cuda/manager_config.hpp"
+#include "caf/cuda/control-layer/scheduler_actor.hpp"
 
 //A class that just acts as a user interface
 //and a system initialization for cuda 
@@ -40,36 +42,22 @@ class actor_facade;
 class CAF_CUDA_EXPORT manager {
 public:
   /// Initializes the singleton. Must be called exactly once before get().
-  static void init(caf::actor_system& sys) {
-    std::lock_guard<std::mutex> guard(mutex_);
-    if (instance_) {
-      	//return;
-      throw std::runtime_error("CUDA manager already initialized");
-    }
-    CHECK_CUDA(cuInit(0));
-    CUcontext ctx = nullptr;
-    cuCtxGetCurrent(&ctx);
-    //std::cout << "Before cuCtxCreate, context: " << ctx << std::endl;
-    instance_ = new manager(sys);
-    //caf::core::init_global_meta_objects();
-     caf::init_global_meta_objects<caf::id_block::cuda>();
-  }
+  static void init(caf::actor_system& sys);
+
+
+
+
+ /// Initializes the singleton. Must be called exactly once before get().
+  static void init(caf::actor_system& sys,manager_config config);
 
   /// Returns the singleton instance. Crashes if not yet initialized.
-  static manager& get() {
-    std::lock_guard<std::mutex> guard(mutex_);
-    if (!instance_) {
-      throw std::runtime_error("CUDA manager used before initialization\n  Please place caf::cuda::manager::init() at the top of CAF_MAIN\n");
-    }
-    return *instance_;
-  }
+  static manager& get();
 
   /// Deletes the singleton if needed (optional).
-  static void shutdown() {
-    std::lock_guard<std::mutex> guard(mutex_);
-    delete instance_;
-    instance_ = nullptr;
-  }
+  //deletes the scheduler actor as well if it exists
+  static void shutdown();
+  caf::actor get_scheduler_actor();
+  
 
   // Prevent copy/assignment
   manager(const manager&) = delete;
@@ -202,6 +190,8 @@ class CAF_CUDA_EXPORT manager {
 
   static manager* instance_;
   static std::mutex mutex_;
+  bool scheduler_on = false;
+  caf::actor scheduler_actor;
 };
 
 } // namespace caf::cuda

From ac1935231bf515e6f968ccb3a483a9b249845da5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Dec 2025 16:19:58 -0600
Subject: [PATCH 0032/1000] Moved static methods out of the header file and
 into the cpp file. Change was made to make code more easy to read and less
 likely to enter a circular dependency.

---
 libcaf_cuda/src/manager.cpp | 108 ++++++++++++++++++++++++++++++++++++
 1 file changed, 108 insertions(+)

diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 3f4138fb09..cb8dc13ecb 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -2,6 +2,8 @@
 #include <stdexcept>
 #include <mutex>
 #include <map>
+#include "caf/cuda/control-layer/scheduler_actor.hpp"
+#include "caf/cuda/manager_config.hpp"
 
 
 namespace caf::cuda {
@@ -10,6 +12,112 @@ namespace caf::cuda {
 	std::mutex manager::mutex_;
 
 
+namespace caf::cuda {
+
+// --------------------------------
+// Static members
+// --------------------------------
+manager* manager::instance_ = nullptr;
+std::mutex manager::mutex_;
+
+// --------------------------------
+// Static init (no config)
+// --------------------------------
+void manager::init(caf::actor_system& sys) {
+    std::lock_guard<std::mutex> guard(mutex_);
+
+    if (instance_) {
+        throw std::runtime_error("CUDA manager already initialized");
+    }
+
+    CHECK_CUDA(cuInit(0));
+
+    CUcontext ctx = nullptr;
+    cuCtxGetCurrent(&ctx);
+
+    instance_ = new manager(sys);
+
+    caf::init_global_meta_objects<caf::id_block::cuda>();
+}
+
+// --------------------------------
+// Static init (with config)
+// --------------------------------
+void manager::init(caf::actor_system& sys, manager_config config) {
+    std::lock_guard<std::mutex> guard(mutex_);
+
+    if (instance_) {
+        throw std::runtime_error("CUDA manager already initialized");
+    }
+
+    CHECK_CUDA(cuInit(0));
+
+    CUcontext ctx = nullptr;
+    cuCtxGetCurrent(&ctx);
+
+    instance_ = new manager(sys);
+
+    caf::init_global_meta_objects<caf::id_block::cuda>();
+
+    instance_->scheduler_on = config.getSchedulerOn();
+
+    if (instance_->scheduler_on) {
+        instance_->scheduler_actor =
+            sys.spawn(actor_from_state<scheduler_actor_state>);
+    }
+}
+
+// --------------------------------
+// Static get()
+// --------------------------------
+manager& manager::get() {
+    std::lock_guard<std::mutex> guard(mutex_);
+
+    if (!instance_) {
+        throw std::runtime_error(
+            "CUDA manager used before initialization.\n"
+            "Call caf::cuda::manager::init() inside CAF_MAIN."
+        );
+    }
+
+    return *instance_;
+}
+
+// --------------------------------
+// Static shutdown()
+// --------------------------------
+void manager::shutdown() {
+    std::lock_guard<std::mutex> guard(mutex_);
+
+    if (!instance_)
+        return;
+
+    if (instance_->scheduler_on) {
+        anon_send_exit(
+            instance_->scheduler_actor,
+            caf::exit_reason::user_shutdown
+        );
+    }
+
+    delete instance_;
+    instance_ = nullptr;
+}
+
+// --------------------------------
+// Static getter for scheduler actor
+// --------------------------------
+caf::actor manager::get_scheduler_actor() {
+    std::lock_guard<std::mutex> guard(mutex_);
+
+    if (!instance_) {
+        throw std::runtime_error("CUDA manager not initialized");
+    }
+
+    return instance_->scheduler_actor;
+}
+
+
+
 
 
 device_ptr manager::find_device(std::size_t) const {

From 35f51b68b63fc77b2b5bbcb94b43b27a6768dca1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Dec 2025 16:27:33 -0600
Subject: [PATCH 0033/1000] Added a pragma once.

---
 libcaf_cuda/caf/cuda/manager_config.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/manager_config.hpp b/libcaf_cuda/caf/cuda/manager_config.hpp
index bfa85c1338..06fa0b8282 100644
--- a/libcaf_cuda/caf/cuda/manager_config.hpp
+++ b/libcaf_cuda/caf/cuda/manager_config.hpp
@@ -1,3 +1,4 @@
+#pragma once
 /*
  * A class that is meant to enable users to configure 
  * the caf cuda library

From 47b7dbbd423193cf15be5c7b1808e3f08909af8d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Dec 2025 16:29:00 -0600
Subject: [PATCH 0034/1000] Fixed compiler error.

---
 libcaf_cuda/src/manager.cpp | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index cb8dc13ecb..a6487e9a3a 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -12,13 +12,7 @@ namespace caf::cuda {
 	std::mutex manager::mutex_;
 
 
-namespace caf::cuda {
 
-// --------------------------------
-// Static members
-// --------------------------------
-manager* manager::instance_ = nullptr;
-std::mutex manager::mutex_;
 
 // --------------------------------
 // Static init (no config)
@@ -249,7 +243,7 @@ program_ptr manager::create_program_from_fatbin(const std::string& filename,
 // Returns true on success; on failure prints log and returns false
 bool manager::compile_nvrtc_program(const char* source, CUdevice device, std::vector<char>& ptx_out) {
 
-	return caf::cuda::compile_nvrtc_program(source,device,ptx_out);
+	return compile_nvrtc_program(source,device,ptx_out);
 }
 
 

From faf35bc973e68ea9da25945656c0f3a9e01608f4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Dec 2025 16:48:35 -0600
Subject: [PATCH 0035/1000] Renamed scheduler actor varialbe to
 scheduler_actor_handle Change was made to avoid naming conflicts with
 scheduler_actor behavior.

---
 libcaf_cuda/caf/cuda/manager.hpp |  2 +-
 libcaf_cuda/src/manager.cpp      | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index 9443b0543c..2987307967 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -191,7 +191,7 @@ class CAF_CUDA_EXPORT manager {
   static manager* instance_;
   static std::mutex mutex_;
   bool scheduler_on = false;
-  caf::actor scheduler_actor;
+  caf::actor scheduler_actor_handle;
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index a6487e9a3a..5ce14b7173 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -56,8 +56,8 @@ void manager::init(caf::actor_system& sys, manager_config config) {
     instance_->scheduler_on = config.getSchedulerOn();
 
     if (instance_->scheduler_on) {
-        instance_->scheduler_actor =
-            sys.spawn(actor_from_state<scheduler_actor_state>);
+        instance_->scheduler_actor_handle =
+            sys.spawn(scheduler_actor);
     }
 }
 
@@ -88,7 +88,7 @@ void manager::shutdown() {
 
     if (instance_->scheduler_on) {
         anon_send_exit(
-            instance_->scheduler_actor,
+            instance_->scheduler_actor_handle,
             caf::exit_reason::user_shutdown
         );
     }
@@ -101,13 +101,14 @@ void manager::shutdown() {
 // Static getter for scheduler actor
 // --------------------------------
 caf::actor manager::get_scheduler_actor() {
-    std::lock_guard<std::mutex> guard(mutex_);
+    	//this is a read only data no need for lock
+	//std::lock_guard<std::mutex> guard(mutex_);
 
     if (!instance_) {
         throw std::runtime_error("CUDA manager not initialized");
     }
 
-    return instance_->scheduler_actor;
+    return instance_->scheduler_actor_handle;
 }
 
 

From 74247cd8cbf3fbc3e29c6cf0681a310c0ddb2136 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Dec 2025 16:55:15 -0600
Subject: [PATCH 0036/1000] Added a reply_actor handle. Change was made to
 ensure the scheduler actor can reply to the required actor.

---
 libcaf_cuda/caf/cuda/control-layer/launch_token.hpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
index e14f64858f..c432efdd55 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
@@ -1,4 +1,5 @@
 #pragma once
+#include <caf/all.hpp>
 #include "caf/cuda/control-layer/token.hpp"
 #include "caf/cuda/all.hpp"
 #include <string>
@@ -10,17 +11,20 @@ class launch_token : public token {
     launch_token(program_ptr prog,
                  nd_range range,
                  int memory_usage,
-                 std::string id)
+                 std::string id,
+		 caf::actor receiver)
         : program_(std::move(prog)),
           range_(std::move(range)),
           memory_usage_(memory_usage),
-          id_(std::move(id)) {}
+          id_(std::move(id)),
+          reply_handle_(receiver){}
 
     int getType() override { return LAUNCH; }
 
     const program_ptr& getProgram() const { return program_; }
     const nd_range& getRange() const { return range_; }
     int getMemoryUsage() const { return memory_usage_; }
+    caf::actor getReplyActor() {return reply_handle_;}
 
     // Return requested number of CUDA blocks (gridDimX * gridDimY * gridDimZ)
     int getBlocks() const {
@@ -38,6 +42,7 @@ class launch_token : public token {
     nd_range range_;
     int memory_usage_;
     std::string id_;
+    caf::actor reply_handle_;
 };
 
 } // namespace caf::cuda

From 46258bf45a3ff8ba9d8940d5d8f55e4926b658cf Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Dec 2025 16:58:51 -0600
Subject: [PATCH 0037/1000] Updated message dispatch to the scheduler, to be
 urgent and not use depreciated API.

---
 libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
index c99af10c69..96e0bca819 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -51,7 +51,7 @@ class launch_response_token : public token {
     void release() {
         bool expected = false;
         if (released_.compare_exchange_strong(expected, true)) {
-            caf::anon_send(receiver_, id_, getBlocks());
+            caf::anon_mail(id_, getBlocks()).urgent().send(receiver_);
         }
     }
 

From f525777b11aefe715b4ffd8254815d4874e3facf Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Dec 2025 17:07:59 -0600
Subject: [PATCH 0038/1000] Moved implementation out of the headers and into
 the cpp files to reduce software bloat and chances of circular dependencies.

---
 .../control-layer/green_light_behavior.hpp    | 20 ++-------
 .../cuda/control-layer/red_light_behavior.hpp | 12 ++---
 .../control-layer/red_light_green_light.cpp   | 45 +++++++++++++++++++
 3 files changed, 53 insertions(+), 24 deletions(-)
 create mode 100644 libcaf_cuda/src/control-layer/red_light_green_light.cpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
index 834af2f4db..665b5336c3 100644
--- a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
@@ -6,24 +6,12 @@ namespace caf::cuda {
 
 class green_light_behavior : public scheduler_actor_behavior {
 public:
-    void schedule() override {
-        // scheduling logic, if needed
-    }
-
-    void receive(scheduler_actor_state* state, const token_ptr& tok) override {
-        // flush the queue
-        while (!state->queue.empty()) {
-            auto queued = state->queue.front();
-            state->queue.pop();
-            // respond to queued token (demo: just print)
-        }
-
-        // handle current token immediately
-        // (demo: just print)
-    }
+    void schedule() override;
+    void receive(scheduler_actor_state* state, const token_ptr& tok) override;
 };
 
-inline green_light_behavior GREEN_BEHAVIOR;
+// Declare externally defined instance
+extern green_light_behavior GREEN_BEHAVIOR;
 
 } // namespace caf::cuda
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
index ca55b7de4e..f4436f8110 100644
--- a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
@@ -5,16 +5,12 @@ namespace caf::cuda {
 
 class red_light_behavior : public scheduler_actor_behavior {
 public:
-    void schedule() override {
-        // scheduling logic, if needed
-    }
-
-    void receive(scheduler_actor_state* state, const token_ptr& tok) override {
-        state->queue.push(tok); // enqueue everything
-    }
+    void schedule() override;
+    void receive(scheduler_actor_state* state, const token_ptr& tok) override;
 };
 
-inline red_light_behavior RED_BEHAVIOR;
+// Declare externally defined instance
+extern red_light_behavior RED_BEHAVIOR;
 
 } // namespace caf::cuda
 
diff --git a/libcaf_cuda/src/control-layer/red_light_green_light.cpp b/libcaf_cuda/src/control-layer/red_light_green_light.cpp
new file mode 100644
index 0000000000..5559d9810e
--- /dev/null
+++ b/libcaf_cuda/src/control-layer/red_light_green_light.cpp
@@ -0,0 +1,45 @@
+#include "caf/cuda/control-layer/red_light_behavior.hpp"
+#include "caf/cuda/control-layer/green_light_behavior.hpp"
+#include <iostream>
+
+namespace caf::cuda {
+
+// Define the member functions for red_light_behavior
+void red_light_behavior::schedule() {
+    // scheduling logic if needed
+}
+
+void red_light_behavior::receive(scheduler_actor_state* state,
+                                 const token_ptr& tok) {
+    state->queue.push(tok);
+}
+
+// Define the global instance
+red_light_behavior RED_BEHAVIOR;
+
+// -------------------------------------------
+
+// Define the member functions for green_light_behavior
+void green_light_behavior::schedule() {
+    // scheduling logic if needed
+}
+
+void green_light_behavior::receive(scheduler_actor_state* state,
+                                   const token_ptr& tok) {
+    // flush the queue
+    while (!state->queue.empty()) {
+        auto queued = state->queue.front();
+        state->queue.pop();
+        // respond to queued token (demo: just print)
+        std::cout << "Processing queued token\n";
+    }
+
+    // handle current token immediately
+    std::cout << "Processing current token\n";
+}
+
+// Define the global instance
+green_light_behavior GREEN_BEHAVIOR;
+
+} // namespace caf::cuda
+

From f8e3f112e3d3f76af7f3a339c787ecd0613d4d24 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Dec 2025 17:39:57 -0600
Subject: [PATCH 0039/1000] Revert "Moved implementation out of the headers and
 into the cpp files to reduce software bloat and chances of circular
 dependencies."

This reverts commit f525777b11aefe715b4ffd8254815d4874e3facf.
---
 .../control-layer/green_light_behavior.hpp    | 20 +++++++--
 .../cuda/control-layer/red_light_behavior.hpp | 12 +++--
 .../control-layer/red_light_green_light.cpp   | 45 -------------------
 3 files changed, 24 insertions(+), 53 deletions(-)
 delete mode 100644 libcaf_cuda/src/control-layer/red_light_green_light.cpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
index 665b5336c3..834af2f4db 100644
--- a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
@@ -6,12 +6,24 @@ namespace caf::cuda {
 
 class green_light_behavior : public scheduler_actor_behavior {
 public:
-    void schedule() override;
-    void receive(scheduler_actor_state* state, const token_ptr& tok) override;
+    void schedule() override {
+        // scheduling logic, if needed
+    }
+
+    void receive(scheduler_actor_state* state, const token_ptr& tok) override {
+        // flush the queue
+        while (!state->queue.empty()) {
+            auto queued = state->queue.front();
+            state->queue.pop();
+            // respond to queued token (demo: just print)
+        }
+
+        // handle current token immediately
+        // (demo: just print)
+    }
 };
 
-// Declare externally defined instance
-extern green_light_behavior GREEN_BEHAVIOR;
+inline green_light_behavior GREEN_BEHAVIOR;
 
 } // namespace caf::cuda
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
index f4436f8110..ca55b7de4e 100644
--- a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
@@ -5,12 +5,16 @@ namespace caf::cuda {
 
 class red_light_behavior : public scheduler_actor_behavior {
 public:
-    void schedule() override;
-    void receive(scheduler_actor_state* state, const token_ptr& tok) override;
+    void schedule() override {
+        // scheduling logic, if needed
+    }
+
+    void receive(scheduler_actor_state* state, const token_ptr& tok) override {
+        state->queue.push(tok); // enqueue everything
+    }
 };
 
-// Declare externally defined instance
-extern red_light_behavior RED_BEHAVIOR;
+inline red_light_behavior RED_BEHAVIOR;
 
 } // namespace caf::cuda
 
diff --git a/libcaf_cuda/src/control-layer/red_light_green_light.cpp b/libcaf_cuda/src/control-layer/red_light_green_light.cpp
deleted file mode 100644
index 5559d9810e..0000000000
--- a/libcaf_cuda/src/control-layer/red_light_green_light.cpp
+++ /dev/null
@@ -1,45 +0,0 @@
-#include "caf/cuda/control-layer/red_light_behavior.hpp"
-#include "caf/cuda/control-layer/green_light_behavior.hpp"
-#include <iostream>
-
-namespace caf::cuda {
-
-// Define the member functions for red_light_behavior
-void red_light_behavior::schedule() {
-    // scheduling logic if needed
-}
-
-void red_light_behavior::receive(scheduler_actor_state* state,
-                                 const token_ptr& tok) {
-    state->queue.push(tok);
-}
-
-// Define the global instance
-red_light_behavior RED_BEHAVIOR;
-
-// -------------------------------------------
-
-// Define the member functions for green_light_behavior
-void green_light_behavior::schedule() {
-    // scheduling logic if needed
-}
-
-void green_light_behavior::receive(scheduler_actor_state* state,
-                                   const token_ptr& tok) {
-    // flush the queue
-    while (!state->queue.empty()) {
-        auto queued = state->queue.front();
-        state->queue.pop();
-        // respond to queued token (demo: just print)
-        std::cout << "Processing queued token\n";
-    }
-
-    // handle current token immediately
-    std::cout << "Processing current token\n";
-}
-
-// Define the global instance
-green_light_behavior GREEN_BEHAVIOR;
-
-} // namespace caf::cuda
-

From f011d3589e6ccd391583209dcff12ca0cdc27f16 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 14:42:29 -0600
Subject: [PATCH 0040/1000] Moved implementation out of the hpp file to fix
 potiental messy codebase down the road.

---
 libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
index ca55b7de4e..7c8d0c4249 100644
--- a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
@@ -9,12 +9,11 @@ class red_light_behavior : public scheduler_actor_behavior {
         // scheduling logic, if needed
     }
 
-    void receive(scheduler_actor_state* state, const token_ptr& tok) override {
-        state->queue.push(tok); // enqueue everything
-    }
+    void receive(scheduler_actor_state* state, const token_ptr& tok) override;
+
+    ~red_light_behavior() noexcept override;
 };
 
-inline red_light_behavior RED_BEHAVIOR;
 
 } // namespace caf::cuda
 

From aa91e90630c7f0c0ea42166abd88d53904389cce Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 14:42:44 -0600
Subject: [PATCH 0041/1000] Moved implementation out of the hpp file to fix
 potiental messy codebase down the road.

---
 .../src/control-layer/red_light_behavior.cpp     | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 libcaf_cuda/src/control-layer/red_light_behavior.cpp

diff --git a/libcaf_cuda/src/control-layer/red_light_behavior.cpp b/libcaf_cuda/src/control-layer/red_light_behavior.cpp
new file mode 100644
index 0000000000..72fe5348b7
--- /dev/null
+++ b/libcaf_cuda/src/control-layer/red_light_behavior.cpp
@@ -0,0 +1,16 @@
+#pragma once
+#include "caf/cuda/control-layer/red_light_behavior.hpp"
+
+namespace caf::cuda {
+
+    void red_light_behavior::schedule() override {
+        // scheduling logic, if needed
+    }
+
+    void red_light_behavior::receive(scheduler_actor_state* state, const token_ptr& tok) override {
+        state->queue.push(tok); // enqueue everything
+    }
+
+    red_light_behavior::~red_light_behavior() noexcept = default;
+
+} // namespace caf::cuda

From 6df2cb45355787b3f2c4164c1353482a8e13d1b7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 14:44:03 -0600
Subject: [PATCH 0042/1000] Updated code to reflect changes in red_light
 behavior being stored in a cpp file instead of an hpp file.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index ba477359c5..894234b4ce 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -13,8 +13,9 @@ namespace caf::cuda {
 caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self) {
 
     // populate the table
+    static red_light_behavior red_behavior;
     self->state().table.add("green", &GREEN_BEHAVIOR);
-    self->state().table.add("red", &RED_BEHAVIOR);
+    self->state().table.add("red", &red_behavior);
 
     // default behavior
     self->state().current_behavior = self->state().table.get(behavior_token("green"));

From 0182b1525cb11bf161ef858da70cd0d45d1d1a9b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 14:45:37 -0600
Subject: [PATCH 0043/1000] Added red_light_behavior.

---
 libcaf_cuda/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/CMakeLists.txt b/libcaf_cuda/CMakeLists.txt
index 594e084e8f..8da5836d26 100644
--- a/libcaf_cuda/CMakeLists.txt
+++ b/libcaf_cuda/CMakeLists.txt
@@ -23,6 +23,7 @@ set(LIBCAF_CUDA_SRCS
   src/scheduler.cpp
   src/streampool.cpp
   src/control-layer/scheduler_actor.cpp
+  src/control-layer/red_light_behavior.cpp
 )
 
 # -- add targets ---------------------------------------------------------------

From f05bc6168166142ac32cb0a584355655f7c4beac Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 14:57:02 -0600
Subject: [PATCH 0044/1000] Updated includes to attempt to break circular
 dependencies.

---
 libcaf_cuda/caf/cuda/manager.hpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index 2987307967..5ffc3a3b95 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -21,7 +21,6 @@
 #include "caf/cuda/actor_facade.hpp"
 #include "caf/cuda/platform.hpp"
 #include "caf/cuda/manager_config.hpp"
-#include "caf/cuda/control-layer/scheduler_actor.hpp"
 
 //A class that just acts as a user interface
 //and a system initialization for cuda 

From 6a4b36d89f791c74ea7e92b55e29392bebe556c9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 14:57:54 -0600
Subject: [PATCH 0045/1000] Removed all from includes to break circular
 dependencies.

---
 libcaf_cuda/caf/cuda/control-layer/token.hpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index b3ae75da1c..342c18cb7c 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -1,6 +1,5 @@
 #pragma once
 #include <atomic>
-#include "caf/cuda/all.hpp"
 #include <caf/intrusive_ptr.hpp>
 #include <caf/ref_counted.hpp>
 

From 3b885b825aa50b99a184bccda779237c8ce073c2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 15:02:45 -0600
Subject: [PATCH 0046/1000] Refactor some of the code to break circular
 dependencies.

---
 .../caf/cuda/control-layer/behavior_table.hpp | 11 +---------
 .../control-layer/launch_response_token.hpp   | 18 ++++-------------
 .../caf/cuda/control-layer/launch_token.hpp   | 20 +++++++------------
 .../control-layer/scheduler_actor_state.hpp   |  6 ++----
 4 files changed, 14 insertions(+), 41 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
index f89dab96d4..61081c6d1f 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
@@ -1,34 +1,25 @@
 #pragma once
 #include "caf/cuda/control-layer/token.hpp"
-#include "caf/cuda/control-layer/behavior.hpp"
 #include "caf/cuda/control-layer/behavior_token.hpp"
 #include <unordered_map>
 #include <string>
 
-
 namespace caf::cuda {
-
-class scheduler_actor_behavior;
+class scheduler_actor_behavior;  // Forward decl
 class behavior_token;
 
 class behavior_table {
 public:
     behavior_table() = default;
-
     void add(const std::string& name, scheduler_actor_behavior* beh) {
         table_[name] = beh;
     }
-
     scheduler_actor_behavior* get(const behavior_token& tok) const {
         auto it = table_.find(tok.name());
         return it != table_.end() ? it->second : nullptr;
     }
-
     auto& all_behaviors() { return table_; }
-
 private:
     std::unordered_map<std::string, scheduler_actor_behavior*> table_;
 };
-
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
index 96e0bca819..b6bf94ba0b 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -1,12 +1,12 @@
 #pragma once
 #include "caf/cuda/control-layer/token.hpp"
-#include "caf/cuda/control-layer/launch_token.hpp"
-#include <caf/actor.hpp>
-#include "caf/cuda/all.hpp"
+#include "caf/cuda/control-layer/launch_token.hpp"  // Full include for constructor param
+#include <caf/actor.hpp>  // For caf::actor
+#include "caf/cuda/nd_range.hpp"  // For nd_range
 #include <atomic>
+#include <string>  // For std::string (if not from elsewhere)
 
 namespace caf::cuda {
-
 class launch_response_token : public token {
 public:
     // Construct manually
@@ -19,7 +19,6 @@ class launch_response_token : public token {
           memory_usage_(memory_usage),
           id_(std::move(id)),
           released_(false) {}
-
     // Construct from a launch_token
     launch_response_token(caf::actor receiver, const launch_token& token)
         : receiver_(std::move(receiver)),
@@ -27,16 +26,12 @@ class launch_response_token : public token {
           memory_usage_(token.getMemoryUsage()),
           id_(token.getId()),
           released_(false) {}
-
     ~launch_response_token() {
         release();
     }
-
     int getType() override { return LAUNCH_RESPONSE; }
-
     const nd_range& getRange() const { return range_; }
     int getMemoryUsage() const { return memory_usage_; }
-
     // Return requested number of CUDA blocks
     int getBlocks() const {
         return static_cast<int>(
@@ -45,16 +40,13 @@ class launch_response_token : public token {
             range_.getGridDimZ()
         );
     }
-
     const std::string& getId() const { return id_; }
-
     void release() {
         bool expected = false;
         if (released_.compare_exchange_strong(expected, true)) {
             caf::anon_mail(id_, getBlocks()).urgent().send(receiver_);
         }
     }
-
 private:
     caf::actor receiver_;
     nd_range range_;
@@ -62,6 +54,4 @@ class launch_response_token : public token {
     std::string id_;
     std::atomic<bool> released_;
 };
-
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
index c432efdd55..39b9f5277a 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
@@ -1,10 +1,11 @@
 #pragma once
-#include <caf/all.hpp>
+#include <caf/all.hpp>  // For caf::actor, caf::intrusive_ptr 
 #include "caf/cuda/control-layer/token.hpp"
-#include "caf/cuda/all.hpp"
-#include <string>
+#include "caf/cuda/nd_range.hpp"  // For nd_range (full def needed for member)
 
 namespace caf::cuda {
+class program; 
+using program_ptr = caf::intrusive_ptr<program>;  
 
 class launch_token : public token {
 public:
@@ -12,20 +13,17 @@ class launch_token : public token {
                  nd_range range,
                  int memory_usage,
                  std::string id,
-		 caf::actor receiver)
+                 caf::actor receiver)
         : program_(std::move(prog)),
           range_(std::move(range)),
           memory_usage_(memory_usage),
           id_(std::move(id)),
-          reply_handle_(receiver){}
-
+          reply_handle_(receiver) {}  // Fixed missing )
     int getType() override { return LAUNCH; }
-
     const program_ptr& getProgram() const { return program_; }
     const nd_range& getRange() const { return range_; }
     int getMemoryUsage() const { return memory_usage_; }
-    caf::actor getReplyActor() {return reply_handle_;}
-
+    caf::actor getReplyActor() { return reply_handle_; }
     // Return requested number of CUDA blocks (gridDimX * gridDimY * gridDimZ)
     int getBlocks() const {
         return static_cast<int>(
@@ -34,9 +32,7 @@ class launch_token : public token {
             range_.getGridDimZ()
         );
     }
-
     const std::string& getId() const { return id_; }
-
 private:
     program_ptr program_;
     nd_range range_;
@@ -44,6 +40,4 @@ class launch_token : public token {
     std::string id_;
     caf::actor reply_handle_;
 };
-
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
index 7696d6bce2..c9fb97d680 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
@@ -1,17 +1,15 @@
 #pragma once
 #include <queue>
 #include "caf/cuda/control-layer/behavior_table.hpp"
-#include "caf/cuda/control-layer/behavior.hpp"
-#include "caf/cuda/control-layer/behavior_token.hpp"
+#include "caf/cuda/control-layer/behavior_token.hpp"  // Still needed for behavior_token
 #include "caf/cuda/control-layer/token.hpp"
 
 namespace caf::cuda {
+class scheduler_actor_behavior; 
 
 struct scheduler_actor_state {
     scheduler_actor_behavior* current_behavior = nullptr;
     behavior_table table;
     std::queue<token_ptr> queue;
 };
-
 } // namespace caf::cuda
-

From 1dc5434cbe5e2c6eb39feef3e060685b55447cf5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 15:11:48 -0600
Subject: [PATCH 0047/1000] Fixed syntax error with schedule method.

---
 libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
index 7c8d0c4249..dd07a40dae 100644
--- a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
@@ -5,9 +5,7 @@ namespace caf::cuda {
 
 class red_light_behavior : public scheduler_actor_behavior {
 public:
-    void schedule() override {
-        // scheduling logic, if needed
-    }
+    void schedule() override;
 
     void receive(scheduler_actor_state* state, const token_ptr& tok) override;
 

From 799db18ec208b9bde986db44911f3cbbf2857f57 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 15:14:08 -0600
Subject: [PATCH 0048/1000] Added instrusive pointer methods to this class, Not
 sure why it was removed in the first place but its back now.

---
 libcaf_cuda/caf/cuda/program.hpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/program.hpp b/libcaf_cuda/caf/cuda/program.hpp
index 4ef7baa76d..2ab76b2d26 100644
--- a/libcaf_cuda/caf/cuda/program.hpp
+++ b/libcaf_cuda/caf/cuda/program.hpp
@@ -3,6 +3,7 @@
 #include <string>
 #include <vector>
 #include <unordered_map>
+#include <atomic>
 
 #include <caf/ref_counted.hpp>
 #include "caf/cuda/global.hpp"
@@ -25,6 +26,15 @@ class CAF_CUDA_EXPORT program : public caf::ref_counted {
   /// @throws std::runtime_error if the kernel was not loaded for the device.
   CUfunction get_kernel(int device_id);
 
+    friend void intrusive_ptr_add_ref(const program* p) noexcept {
+        p->ref_count_.fetch_add(1, std::memory_order_relaxed);
+    }
+    friend void intrusive_ptr_release(const program* p) noexcept {
+        if (p->ref_count_.fetch_sub(1, std::memory_order_acq_rel) == 1)
+            delete p;
+    }
+
+
 private:
   /// Internal helper to load the kernel modules on all devices.
   void load_kernels(bool is_fatbin);
@@ -32,6 +42,7 @@ class CAF_CUDA_EXPORT program : public caf::ref_counted {
   std::string name_;                       ///< Name of the kernel
   std::vector<char> binary_;               ///< The binary or PTX of the program
   std::unordered_map<int, CUfunction> kernels_; ///< Device ID -> CUfunction mapping
+  mutable std::atomic<size_t> ref_count_{0};
 };
 
 /// Alias for an intrusive pointer to a program

From 48b287588cebc3c548f66761d19fbbb04694ce1a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 15:15:13 -0600
Subject: [PATCH 0049/1000] Include program.hpp This include is added since
 compiler needs to know about the deconstructor at compile time.

---
 libcaf_cuda/caf/cuda/control-layer/launch_token.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
index 39b9f5277a..7c6c8ccd8c 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
@@ -1,8 +1,10 @@
 #pragma once
 #include <caf/all.hpp>  // For caf::actor, caf::intrusive_ptr 
+#include "caf/cuda/program.hpp"
 #include "caf/cuda/control-layer/token.hpp"
 #include "caf/cuda/nd_range.hpp"  // For nd_range (full def needed for member)
 
+
 namespace caf::cuda {
 class program; 
 using program_ptr = caf::intrusive_ptr<program>;  

From 4b25b657464d03a7bb07f709536b987c1224be36 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 15:44:25 -0600
Subject: [PATCH 0050/1000] Commented out program ptr instrusive pointer logic.
 CHange was made since for some reason program pointer acting as instrusive
 pointer causes segfaults with inappropriate deleting, will need to fix this
 later but it works for now.

---
 libcaf_cuda/caf/cuda/program.hpp | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/program.hpp b/libcaf_cuda/caf/cuda/program.hpp
index 2ab76b2d26..cc6fbbf252 100644
--- a/libcaf_cuda/caf/cuda/program.hpp
+++ b/libcaf_cuda/caf/cuda/program.hpp
@@ -27,11 +27,16 @@ class CAF_CUDA_EXPORT program : public caf::ref_counted {
   CUfunction get_kernel(int device_id);
 
     friend void intrusive_ptr_add_ref(const program* p) noexcept {
-        p->ref_count_.fetch_add(1, std::memory_order_relaxed);
+        //p->ref_count_.fetch_add(1, std::memory_order_relaxed);
     }
     friend void intrusive_ptr_release(const program* p) noexcept {
-        if (p->ref_count_.fetch_sub(1, std::memory_order_acq_rel) == 1)
-            delete p;
+        if (p->ref_count_.fetch_sub(1, std::memory_order_acq_rel) == 1) {
+	   //WARNING TURNING THIS ON FOR SOME REASON, CAUSES SEGFAUTLS
+	   //I HAVE NO IDEA WHY 
+	   //TODO FIX THIS 
+	   // std::cout<< "Deleting\n";
+           // delete p;
+	}
     }
 
 

From c1eac26d97422d9a8e3060733fbd4d745262e7b0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 15:45:37 -0600
Subject: [PATCH 0051/1000] Fixed syntax errors.

---
 libcaf_cuda/src/control-layer/red_light_behavior.cpp | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/red_light_behavior.cpp b/libcaf_cuda/src/control-layer/red_light_behavior.cpp
index 72fe5348b7..8128ccda16 100644
--- a/libcaf_cuda/src/control-layer/red_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/red_light_behavior.cpp
@@ -1,13 +1,11 @@
-#pragma once
 #include "caf/cuda/control-layer/red_light_behavior.hpp"
 
 namespace caf::cuda {
 
-    void red_light_behavior::schedule() override {
-        // scheduling logic, if needed
+    void red_light_behavior::schedule() {
     }
 
-    void red_light_behavior::receive(scheduler_actor_state* state, const token_ptr& tok) override {
+    void red_light_behavior::receive(scheduler_actor_state* state, const token_ptr& tok) {
         state->queue.push(tok); // enqueue everything
     }
 

From bf565585efaf360f491b8c8242cbb4ee2ff92737 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 15:49:13 -0600
Subject: [PATCH 0052/1000] Moved implementation logic out of the hpp file.
 Change was made to ensure code is cleaner and does not cause future errors.

---
 .../control-layer/green_light_behavior.hpp    | 18 ++------------
 .../control-layer/green_light_behavior.cpp    | 24 +++++++++++++++++++
 2 files changed, 26 insertions(+), 16 deletions(-)
 create mode 100644 libcaf_cuda/src/control-layer/green_light_behavior.cpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
index 834af2f4db..7f62d2c372 100644
--- a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
@@ -1,29 +1,15 @@
 #pragma once
 #include "caf/cuda/control-layer/behavior.hpp"
-#include "caf/cuda/control-layer/launch_token.hpp"
 
 namespace caf::cuda {
 
 class green_light_behavior : public scheduler_actor_behavior {
 public:
-    void schedule() override {
-        // scheduling logic, if needed
-    }
+    void schedule() override;
 
-    void receive(scheduler_actor_state* state, const token_ptr& tok) override {
-        // flush the queue
-        while (!state->queue.empty()) {
-            auto queued = state->queue.front();
-            state->queue.pop();
-            // respond to queued token (demo: just print)
-        }
-
-        // handle current token immediately
-        // (demo: just print)
-    }
+    void receive(scheduler_actor_state* state, const token_ptr& tok) override; 
 };
 
-inline green_light_behavior GREEN_BEHAVIOR;
 
 } // namespace caf::cuda
 
diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
new file mode 100644
index 0000000000..14bc201e85
--- /dev/null
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -0,0 +1,24 @@
+#include "caf/cuda/control-layer/green_light_behavior.hpp"
+
+namespace caf::cuda {
+
+    void green_light_behavior::schedule() {
+    //TODO IMPLEMENT
+    }
+
+    void green_light_behavior::receive(scheduler_actor_state* state, const token_ptr& tok)    {
+        // flush the queue
+        while (!state->queue.empty()) {
+            auto queued = state->queue.front();
+            state->queue.pop();
+            // respond to queued token (demo: just print)
+        }
+
+        // handle current token immediately
+        // (demo: just print)
+    }
+};
+
+
+} // namespace caf::cuda
+

From 7a00009be5d4f35ce04701df727e211245245e4d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 15:50:52 -0600
Subject: [PATCH 0053/1000] Updated cmakelists to include greenlight behavior.

---
 libcaf_cuda/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/CMakeLists.txt b/libcaf_cuda/CMakeLists.txt
index 8da5836d26..e6a0ef7b6e 100644
--- a/libcaf_cuda/CMakeLists.txt
+++ b/libcaf_cuda/CMakeLists.txt
@@ -24,6 +24,7 @@ set(LIBCAF_CUDA_SRCS
   src/streampool.cpp
   src/control-layer/scheduler_actor.cpp
   src/control-layer/red_light_behavior.cpp
+  src/control-layer/green_light_behavior.cpp
 )
 
 # -- add targets ---------------------------------------------------------------

From edd15cd053f4af1da37ac8a2ba38e8119496b77a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 15:52:11 -0600
Subject: [PATCH 0054/1000] Fixed syntax error.

---
 libcaf_cuda/src/control-layer/green_light_behavior.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index 14bc201e85..a5bbec6b38 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -17,8 +17,6 @@ namespace caf::cuda {
         // handle current token immediately
         // (demo: just print)
     }
-};
-
 
 } // namespace caf::cuda
 

From 074bca71fe5ac25431e43e5ba7e8c7db5a26b276 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 15:53:49 -0600
Subject: [PATCH 0055/1000] Updated scheudler actor to reflect changes made to
 green light behavior.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 894234b4ce..f8b18deb26 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -14,7 +14,9 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self)
 
     // populate the table
     static red_light_behavior red_behavior;
-    self->state().table.add("green", &GREEN_BEHAVIOR);
+    static green_light_behavior green_behavior;
+
+    self->state().table.add("green", &green_behavior);
     self->state().table.add("red", &red_behavior);
 
     // default behavior

From def08baca7b16d7103bfa2a6855f4fd262889ded Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 16:08:49 -0600
Subject: [PATCH 0056/1000] Added stubs for cleanup and init methods.

---
 libcaf_cuda/caf/cuda/control-layer/behavior.hpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
index fb240e0eb4..2ea21c0ad1 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
@@ -14,6 +14,7 @@
 
 namespace caf::cuda {
 
+struct scheduler_actor_state;
 class launch_token;
 
 class scheduler_actor_behavior {
@@ -22,6 +23,16 @@ class scheduler_actor_behavior {
 
     virtual void schedule() = 0;
     virtual void receive(scheduler_actor_state* state, const token_ptr& tok) = 0;
+    
+    //define what to do when transitioning into the state
+    virtual void init(scheduler_actor_state * state) {
+    	//If the behavior decides not to overide do nothing
+    }
+
+    //define what to do when transitioning out of the state
+    virtual void cleanup(scheduler_actor_state * state) { 
+    	//If the behavior decides not to overide do nothing
+    }
 };
 
 } // namespace caf::cuda

From 9a7f92ef45e90044ba90879f373cacc8ddcc66ec Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 16:12:12 -0600
Subject: [PATCH 0057/1000] ADded a self_handle attribute.

---
 libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
index c9fb97d680..afff543ce9 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
@@ -8,6 +8,7 @@ namespace caf::cuda {
 class scheduler_actor_behavior; 
 
 struct scheduler_actor_state {
+    caf::actor  self_handle;
     scheduler_actor_behavior* current_behavior = nullptr;
     behavior_table table;
     std::queue<token_ptr> queue;

From 78c73d391929c508c326e60eafb044716a443473 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 16:13:19 -0600
Subject: [PATCH 0058/1000] Renamed self_handle to self.

---
 libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
index afff543ce9..2498a6a710 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
@@ -8,7 +8,7 @@ namespace caf::cuda {
 class scheduler_actor_behavior; 
 
 struct scheduler_actor_state {
-    caf::actor  self_handle;
+    caf::actor  self;
     scheduler_actor_behavior* current_behavior = nullptr;
     behavior_table table;
     std::queue<token_ptr> queue;

From 3252bf5d80d8e5a75b4cd902b2070fcba158d17f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 16:14:36 -0600
Subject: [PATCH 0059/1000] Updated behavior to add a self_reference on
 startup.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index f8b18deb26..a29ee6972b 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -12,6 +12,9 @@ namespace caf::cuda {
 
 caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self) {
 
+    //add its self reference
+    self -> state().self = self;
+
     // populate the table
     static red_light_behavior red_behavior;
     static green_light_behavior green_behavior;

From cf78d9b4fe643bb2de5ff32348f939f3423f1044 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 16:17:37 -0600
Subject: [PATCH 0060/1000] Implemented new message handler for testing.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index a29ee6972b..aa082b2c9d 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -2,6 +2,7 @@
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
 #include "caf/cuda/control-layer/green_light_behavior.hpp"
 #include "caf/cuda/control-layer/red_light_behavior.hpp"
+#include <string>
 
 /*
  * This class is meant to handle actor GPU scheduling via s/r/r IPC 
@@ -33,7 +34,10 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self)
             auto* next = self->state().table.get(tok);
             if (next)
                 self->state().current_behavior = next;  // swap behavior
-        }
+        },
+	[=](std::string word) {
+		std::cout << "Received message " << word << "\n";
+	}
     };
 }
 

From ded769bbd36087b45b22f2c07d13cd62ffa59eda Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 16:23:32 -0600
Subject: [PATCH 0061/1000] Added token_factory.cpp

---
 libcaf_cuda/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/CMakeLists.txt b/libcaf_cuda/CMakeLists.txt
index e6a0ef7b6e..20f395b573 100644
--- a/libcaf_cuda/CMakeLists.txt
+++ b/libcaf_cuda/CMakeLists.txt
@@ -25,6 +25,7 @@ set(LIBCAF_CUDA_SRCS
   src/control-layer/scheduler_actor.cpp
   src/control-layer/red_light_behavior.cpp
   src/control-layer/green_light_behavior.cpp
+  src/control-layer/token_factory.cpp
 )
 
 # -- add targets ---------------------------------------------------------------

From 144be9626aaa73a9261a2c0b11efda8251145dc2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 16:23:49 -0600
Subject: [PATCH 0062/1000] initial commit.

---
 .../caf/cuda/control-layer/token_factory.hpp  | 27 ++++++++++++++++
 .../src/control-layer/token_factory.cpp       | 32 +++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
 create mode 100644 libcaf_cuda/src/control-layer/token_factory.cpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
new file mode 100644
index 0000000000..49812fbb81
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
@@ -0,0 +1,27 @@
+#pragma once
+
+#include "caf/cuda/control-layer/token.hpp"
+#include "caf/cuda/control-layer/launch_token.hpp"
+#include "caf/cuda/control-layer/launch_response_token.hpp"
+#include "caf/cuda/control-layer/behavior_token.hpp"
+#include <caf/actor.hpp>
+#include <string>
+
+namespace caf::cuda {
+
+/// Creates a launch_token (used by users when submitting kernels)
+token_ptr make_launch_token(program_ptr prog,
+                            nd_range range,
+                            int memory_usage,
+                            std::string id,
+                            actor reply_to);
+
+/// Creates a launch_response_token (created internally by the scheduler
+/// when it accepts a kernel launch request)
+token_ptr make_launch_response_token(actor scheduler_or_proxy,
+                                     const launch_token& orig);
+
+/// Creates a behavior_token (special — returns its own strong ptr type)
+behavior_token_ptr make_behavior_token(std::string name);
+
+} // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/token_factory.cpp b/libcaf_cuda/src/control-layer/token_factory.cpp
new file mode 100644
index 0000000000..90d81453ed
--- /dev/null
+++ b/libcaf_cuda/src/control-layer/token_factory.cpp
@@ -0,0 +1,32 @@
+#include "caf/cuda/control-layer/token_factory.hpp"
+#include "caf/cuda/control-layer/launch_token.hpp"
+#include "caf/cuda/control-layer/launch_response_token.hpp"
+#include "caf/cuda/control-layer/behavior_token.hpp"
+
+namespace caf::cuda {
+
+token_ptr make_launch_token(program_ptr prog,
+                            nd_range range,
+                            int memory_usage,
+                            std::string id,
+                            actor reply_to)
+{
+    return caf::make_counted<launch_token>(std::move(prog),
+                                           std::move(range),
+                                           memory_usage,
+                                           std::move(id),
+                                           reply_to);
+}
+
+token_ptr make_launch_response_token(actor receiver,
+                                     const launch_token& orig)
+{
+    return caf::make_counted<launch_response_token>(receiver, orig);
+}
+
+behavior_token_ptr make_behavior_token(std::string name)
+{
+    return caf::make_counted<behavior_token>(std::move(name));
+}
+
+} // namespace caf::cuda

From e09aa1b862c9babb2d2f1d7be0456218d8d67dd8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 16:34:39 -0600
Subject: [PATCH 0063/1000] Added stub for init method.

---
 libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
index dd07a40dae..4d1425e515 100644
--- a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
@@ -8,7 +8,8 @@ class red_light_behavior : public scheduler_actor_behavior {
     void schedule() override;
 
     void receive(scheduler_actor_state* state, const token_ptr& tok) override;
-
+    void init(scheduler_actor_state* state);
+    //void cleanup(scheduler_actor_state* state);
     ~red_light_behavior() noexcept override;
 };
 

From 8fb62163354e51b3655eef9abe66fb6e776f60fc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 16:35:05 -0600
Subject: [PATCH 0064/1000] Implemented init method.

---
 libcaf_cuda/src/control-layer/red_light_behavior.cpp | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/red_light_behavior.cpp b/libcaf_cuda/src/control-layer/red_light_behavior.cpp
index 8128ccda16..a4e1976366 100644
--- a/libcaf_cuda/src/control-layer/red_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/red_light_behavior.cpp
@@ -1,4 +1,6 @@
+#include "caf/cuda/control-layer/all-control-layer.hpp"
 #include "caf/cuda/control-layer/red_light_behavior.hpp"
+#include "caf/cuda/control-layer/token_factory.hpp"
 
 namespace caf::cuda {
 
@@ -10,5 +12,15 @@ namespace caf::cuda {
     }
 
     red_light_behavior::~red_light_behavior() noexcept = default;
+    
+    void red_light_behavior::init(scheduler_actor_state * state) {
+	   std::cout << "RED LIGHT\n";
+	   behavior_token_ptr green_light = make_behavior_token("green");
+
+	   //send a request to change behavior to green light after 5 seconds
+    	   anon_mail(green_light)
+		   .delay(std::chrono::seconds(5))
+		   .send(state -> self);	
+    }
 
 } // namespace caf::cuda

From 6e369536441d72872550ba1f51bf6d6d296dddf1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 16:37:50 -0600
Subject: [PATCH 0065/1000] Added token factory header to include.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 79aef82166..cee11883b4 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -6,6 +6,7 @@
 #include "caf/cuda/control-layer/behavior_token.hpp"
 #include "caf/cuda/control-layer/behavior.hpp"
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
+#include "caf/cuda/control-layer/token_factory.hpp"
 
 CAF_BEGIN_TYPE_ID_BLOCK(cuda_control, caf::first_custom_type_id + 200)
 // You can also use  caf::id_block::core::end  instead of the +200 if you prefer

From 54c2ead43c119fa1eaee94acb0abdb0ec9e35d0e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 16:38:03 -0600
Subject: [PATCH 0066/1000] Added init stub.

---
 libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
index 7f62d2c372..2469262a5b 100644
--- a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
@@ -8,6 +8,7 @@ class green_light_behavior : public scheduler_actor_behavior {
     void schedule() override;
 
     void receive(scheduler_actor_state* state, const token_ptr& tok) override; 
+    void init(scheduler_actor_state* state); 
 };
 
 

From 2fae812e28081a72bcff526572cee077ee5b7135 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Dec 2025 16:38:18 -0600
Subject: [PATCH 0067/1000] Implemented init method.

---
 .../src/control-layer/green_light_behavior.cpp     | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index a5bbec6b38..8baeb059ab 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -1,5 +1,7 @@
+#include "caf/cuda/control-layer/all-control-layer.hpp"
 #include "caf/cuda/control-layer/green_light_behavior.hpp"
 
+
 namespace caf::cuda {
 
     void green_light_behavior::schedule() {
@@ -18,5 +20,17 @@ namespace caf::cuda {
         // (demo: just print)
     }
 
+   void green_light_behavior::init(scheduler_actor_state * state) {
+	   std::cout << "GREEN LIGHT\n";
+	   behavior_token_ptr red_light = make_behavior_token("red");
+
+	   //send a request to change behavior to green light after 5 seconds
+    	   anon_mail(red_light)
+		   .delay(std::chrono::seconds(5))
+		   .send(state -> self);	
+    }
+
+
+
 } // namespace caf::cuda
 

From 5a3301438c6c236dd8e980158069946e03fdccce Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 12 Dec 2025 12:36:50 -0600
Subject: [PATCH 0068/1000] Implemented init and cleanup methods to behavior
 swap.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index aa082b2c9d..c8218e4e9a 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -32,8 +32,13 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self)
         },
         [=](const behavior_token& tok) {
             auto* next = self->state().table.get(tok);
-            if (next)
-                self->state().current_behavior = next;  // swap behavior
+            if (next) {
+                
+		    self->state().current_behavior -> cleanup(&self->state()); //cleanup current behavior
+		    self->state().current_behavior = next;  // swap behavior
+		    self->state().current_behavior -> init(&self->state()); //init new current behavior
+		
+	    }
         },
 	[=](std::string word) {
 		std::cout << "Received message " << word << "\n";

From 7329b3852892206fdb2a5e31999895f5106334e5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 12 Dec 2025 12:54:34 -0600
Subject: [PATCH 0069/1000] Updated greenlight behavior to reply to all
 messsages in the queue.

---
 .../control-layer/green_light_behavior.cpp    | 21 ++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index 8baeb059ab..faa54a7778 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -9,15 +9,22 @@ namespace caf::cuda {
     }
 
     void green_light_behavior::receive(scheduler_actor_state* state, const token_ptr& tok)    {
-        // flush the queue
-        while (!state->queue.empty()) {
-            auto queued = state->queue.front();
-            state->queue.pop();
-            // respond to queued token (demo: just print)
+         // flush the queue
+    while (!state->queue.empty()) {
+        token_ptr queued = state->queue.front();
+        state->queue.pop();
+
+        if (queued->getType() == LAUNCH) {
+            // manually downcast raw pointer
+            caf::intrusive_ptr<launch_token> ltok(static_cast<launch_token*>(queued.get()));
+
+            // dereference to pass reference to factory function
+            token_ptr response = make_launch_response_token(state->self, *ltok);
+            anon_mail(response).send(ltok->getReplyActor());
         }
+    }
+
 
-        // handle current token immediately
-        // (demo: just print)
     }
 
    void green_light_behavior::init(scheduler_actor_state * state) {

From 5af9c1893e147dcdfd78114eebdb4ff7ecacbd64 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 12 Dec 2025 12:56:59 -0600
Subject: [PATCH 0070/1000] Implemented the reply message to be a test message
 for now.

---
 libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
index b6bf94ba0b..b366ffa414 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -44,7 +44,10 @@ class launch_response_token : public token {
     void release() {
         bool expected = false;
         if (released_.compare_exchange_strong(expected, true)) {
-            caf::anon_mail(id_, getBlocks()).urgent().send(receiver_);
+            	//the real message commented out for testing 
+		//caf::anon_mail(id_, getBlocks()).urgent().send(receiver_);
+		//test message
+		caf::anon_mail("Hello world from me").urgent().send(receiver_);
         }
     }
 private:

From a7dc9d54bcffafe1001e805e18c04d72ccb63502 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 12 Dec 2025 12:59:15 -0600
Subject: [PATCH 0071/1000] Initial commit.

---
 .../red-ligh-green-light/CMakeLists.txt       |  44 +
 .../red-ligh-green-light/compile_kernels.sh   |  18 +
 .../red-ligh-green-light/main.test.cpp        | 755 ++++++++++++++++++
 .../red-ligh-green-light/mmul.cu              |  16 +
 4 files changed, 833 insertions(+)
 create mode 100644 libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
 create mode 100644 libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/mmul.cu

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/CMakeLists.txt b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/compile_kernels.sh b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/compile_kernels.sh
new file mode 100755
index 0000000000..586196454e
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/compile_kernels.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+
+# Compile genMatrix.cu to fatbin in current directory
+#nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
+#echo "Generated generate_random_matrix.fatbin"
+
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
new file mode 100644
index 0000000000..5cc0e2944e
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -0,0 +1,755 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+// Define a custom type ID block for custom actors
+CAF_ADD_ATOM(cuda,shared_mem)
+
+
+
+const char* kernel_code = R"(
+extern "C" __global__
+void compare_strings(const char* a, const char* b, int* result, int * length) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx < * length) {
+        result[idx] = (a[idx] == b[idx]) ? 1 : 0;
+    }
+}
+)";
+
+const char* matrixMulKernel2 = R"(
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int *N_val) {
+    int N = *N_val;
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+)";
+
+
+const char* matrixMulKernel = R"(
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    //printf("%d\n",N);
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+)";
+
+
+
+
+#include <chrono>
+#include <iostream>
+
+// Extend your actor state to keep the start time
+struct mmul_actor_state {
+  static inline const char* name = "my_actor";
+  int last_N = 0; // example state variable
+  int id = rand(); // an actor id
+  // per-actor timing start
+  std::chrono::high_resolution_clock::time_point start_time;
+  int times = 0;
+};
+
+
+
+
+//commands classes used to launch kernels 
+using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
+using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
+
+using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
+
+mmulCommand mmul;
+matrixGenCommand randomMatrix;
+mmulAsyncCommand mmulAsync;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
+  return {
+    // 1st handler: Just int N, and who to send the matrices to
+    [=](int N, std::vector<caf::actor> receivers) {
+
+	/*
+	 * Unfortuanley libraries such as curand cannot be linked with cubins 
+	 * making it incompatable with this software for right now
+	 * its not really random, just a matrix filled with 5's
+	 */
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
+        //create the program and configure the dimesnions of the kernel
+        auto program = mgr.create_program_from_fatbin("../generate_random_matrix.fatbin","generate_random_matrix");
+	int THREADS = 256;
+	int BLOCKS = (N*N + THREADS - 1) / THREADS;
+  	caf::cuda::nd_range dim(BLOCKS,1, 1, THREADS,1, 1);
+
+	//tag the arguments so that caf::cuda knows what to do with them	
+         auto arg1 = caf::cuda::create_out_arg(N*N); //output buffer indicate its size, caf::cuda will handle the rest
+          auto arg2 = caf::cuda::create_in_arg(N*N); //matrix size
+          auto arg3 = caf::cuda::create_in_arg(1234); //seed
+	  auto arg4 = caf::cuda::create_in_arg(9999); //max valux
+	  
+
+
+	  //launch kernels and collect their outputs
+	  auto tempA = randomMatrix.run(program,dim, self -> state().id,arg1,arg2,arg3,arg4);
+	  auto tempB = randomMatrix.run(program,dim, self -> state().id,arg1,arg2,arg3,arg4);
+	  std::vector<int> matrixA =  caf::cuda::extract_vector<int>(tempA);
+	  std::vector<int> matrixB = caf::cuda::extract_vector<int>(tempB);
+
+
+
+	  //cpu code
+	  //std::vector<int> matrixA(N*N);
+	  //std::vector<int> matrixB(N*N);
+
+	  // std::generate(matrixA.begin(), matrixA.end(), []() { return rand() % 10; });
+	   //std::generate(matrixB.begin(), matrixB.end(), []() { return rand() % 10; });
+
+
+	  //broadcast the result out to receviers.
+	  for (auto actor: receivers) {
+	  
+		  self->mail(matrixA,matrixB,N).send(actor);
+	  }
+
+    },
+
+    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+    [=](const std::vector<int> matrixA,
+        const std::vector<int> matrixB, int N) {
+ 
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //create program and dims   
+  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+  const int THREADS = 32;
+  const int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    //create args
+    auto arg1 = caf::cuda::create_in_arg(matrixA);
+    auto arg2 = caf::cuda::create_in_arg(matrixB);
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    auto tempC = mmul.run(program,dims,self -> state().id,arg1,arg2,arg3,arg4);
+    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+    //verify its own result 
+    self -> mail(matrixA,matrixB,matrixC,N).send(self);
+
+    },
+
+    // 3rd handler: CPU atom + matrices + N
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB, const std::vector<int> matrixC, int N) {
+       
+	 std::vector<int> result(N*N);
+
+	 serial_matrix_multiply(matrixA,matrixB,result,N);
+
+	 if (result == matrixC) {
+	 
+		 std::cout << "actor with id " <<  self->state().id << " references match\n";
+	 
+	 }
+
+	 else {
+	    std::cout << "actor with id " <<  self->state().id << " references did not match\n";
+	 }
+
+	 self-> quit();
+
+    }
+  };
+}
+
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_actor_fun));
+  }
+
+  // Actor 0 generates matrices and broadcasts to others 
+  caf::anon_mail(matrix_size, actors).send(actors[0]);
+
+   sys.await_all_actors_done();
+}
+
+
+// Stateful actor behavior
+caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
+  return {
+    // 1st handler: Just int N, and who to send the matrices to
+    [=](int N, std::vector<caf::actor> receivers) {
+
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
+        //create the program and configure the dimesnions of the kernel
+        auto program = mgr.create_program_from_fatbin("../generate_random_matrix.fatbin","generate_random_matrix");
+	int THREADS = 256;
+	int BLOCKS = (N*N + THREADS - 1) / THREADS;
+  	caf::cuda::nd_range dim(BLOCKS,1, 1, THREADS,1, 1);
+
+	//tag the arguments so that caf::cuda knows what to do with them	
+         auto arg1 = caf::cuda::create_out_arg(N*N); //output buffer indicate its size, caf::cuda will handle the rest
+          auto arg2 = caf::cuda::create_in_arg(N*N); //matrix size
+          auto arg3 = caf::cuda::create_in_arg(rand()); //seed
+	  auto arg4 = caf::cuda::create_in_arg(9999); //max valux
+	  
+          auto arg3B = caf::cuda::create_in_arg(rand()); //seed
+	  int device_number= 74; //arbitary number to show that 
+				 //can give illusion of selecting gpus that are
+				 //not there
+
+
+	  //launch kernels and collect their outputs
+	  auto tempA = randomMatrix.run_async(program,dim, self -> state().id,0,device_number,arg1,arg2,arg3,arg4);
+	  auto tempB = randomMatrix.run_async(program,dim, self -> state().id,0,device_number,arg1,arg2,arg3B,arg4);
+	  caf::cuda::mem_ptr<int> matrixA =  std::get<0>(tempA);
+	  caf::cuda::mem_ptr<int> matrixB = std::get<0>(tempB);
+
+	  //ensure the data is actually done being worked on
+	  matrixA -> synchronize();
+	  matrixB -> synchronize();
+
+
+
+
+	  //cpu code
+	  //std::vector<int> matrixA(N*N);
+	  //std::vector<int> matrixB(N*N);
+
+	  // std::generate(matrixA.begin(), matrixA.end(), []() { return rand() % 10; });
+	   //std::generate(matrixB.begin(), matrixB.end(), []() { return rand() % 10; });
+
+
+	  std::cout << "Broadcasting\n";
+	  //broadcast the result out to receviers.
+	  for (auto actor: receivers) {
+	  
+		  self->mail(3,matrixA,matrixB,N,device_number).send(actor);
+	  }
+
+    },
+
+    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+    [=](const caf::cuda::mem_ptr<int> matrixA,
+        const caf::cuda::mem_ptr<int> matrixB, int N,int device_number) {
+ 
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //create program and dims   
+  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+  const int THREADS = 32;
+  const int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    //create args
+    auto arg1 = matrixA;
+    auto arg2 = matrixB;
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+
+    auto tempC = mmulAsync.run(program,dims,self -> state().id,0,device_number,arg1,arg2,arg3,arg4);
+
+    std::vector<int> matrix1 = matrixA -> copy_to_host();
+    std::vector<int> matrix2 = matrixB -> copy_to_host();    
+    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC,2);
+
+    //verify its own result 
+    self -> mail(matrix1,matrix2,matrixC,N).send(self);
+
+    },
+
+ // 3nd handler: GPU atom + matrices + N, launches a kenrel using shared memory and sends its result to itself for verification
+    [=](int x,const caf::cuda::mem_ptr<int> matrixA,
+        const caf::cuda::mem_ptr<int> matrixB, int N,int device_number) {
+ 
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //create program and dims   
+  auto program = mgr.create_program_from_cubin("../shared_mmul.cubin","matrixMul");
+  const int THREADS = 32;
+  const int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+  int shared_mem = 8192; //we need 8KB of shared memory here
+    //create args
+    auto arg1 = matrixA;
+    auto arg2 = matrixB;
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+
+    auto tempC = mmulAsync.run(program,dims,self -> state().id,shared_mem,device_number,arg1,arg2,arg3,arg4);
+
+    std::vector<int> matrix1 = matrixA -> copy_to_host();
+    std::vector<int> matrix2 = matrixB -> copy_to_host();    
+    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC,2);
+
+    //verify its own result 
+    self -> mail(matrix1,matrix2,matrixC,N).send(self);
+
+    },
+
+
+
+    // 3rd handler: CPU atom + matrices + N
+    [=](const std::vector<int>& matrixA,
+    const std::vector<int> &matrixB,
+    const std::vector<int> &matrixC, int N) {
+
+    std::vector<int> result(N * N);
+
+    serial_matrix_multiply(matrixA, matrixB, result, N);
+
+    if (result == matrixC) {
+        std::cout << "actor with id " << self->state().id << " references match\n";
+    }
+    else {
+        std::cout << "actor with id " << self->state().id << " references did not match\n";
+
+    }
+
+
+    /*
+    auto print_matrix = [N](const std::vector<int>& mat, const std::string& name) {
+            std::cout << name << ":\n";
+            for (int i = 0; i < N; ++i) {
+                for (int j = 0; j < N; ++j) {
+                    std::cout << mat[i * N + j] << " ";
+                }
+                std::cout << "\n";
+            }
+            std::cout << std::endl;
+        };
+
+        print_matrix(matrixA, "Matrix A");
+        print_matrix(matrixB, "Matrix B");
+        print_matrix(result, "Result Matrix");
+        print_matrix(matrixC, "GPU Result Matrix");
+	*/
+    self->quit();
+    }
+  };
+}
+
+
+void run_async_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_async_actor_fun));
+  }
+
+  // Actor 0 generates matrices and broadcasts to others 
+  caf::anon_mail(matrix_size, actors).send(actors[0]);
+
+   sys.await_all_actors_done();
+}
+
+
+//--------------------------------Perfomance tests 
+
+// Perf-version of the actor: each actor generates a matrix and sends to itself
+caf::behavior mmul_async_actor_fun_perf(caf::stateful_actor<mmul_actor_state>* self) {
+  return {
+    // 1) start: generate matrices and send them to self
+    [=](int N) {
+      // store start time in actor state (no locks)
+      self->state().start_time = std::chrono::high_resolution_clock::now();
+      self->state().last_N = N;
+
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+      // use the generator fatbin (as in your code)
+      auto program = mgr.create_program_from_fatbin("../generate_random_matrix.fatbin",
+                                                    "generate_random_matrix");
+
+      int THREADS = 256;
+      int BLOCKS = (N * N + THREADS - 1) / THREADS;
+      caf::cuda::nd_range dim(BLOCKS, 1, 1, THREADS, 1, 1);
+
+      // prepare args (same as your existing code)
+      auto arg_out = caf::cuda::create_out_arg(N * N);
+      auto arg_size = caf::cuda::create_in_arg(N * N);
+      auto arg_seed = caf::cuda::create_in_arg(rand());
+      auto arg_max = caf::cuda::create_in_arg(9999);
+
+      int device_number = rand()%2;
+
+      // launch generator(s) asynchronously and get mem_ptrs back
+      // (we follow your earlier style: run_async returns tuple of mem_ptrs)
+      auto tA = randomMatrix.run_async(program, dim, self->state().id,0,device_number,arg_out, arg_size, arg_seed, arg_max);
+      auto tB = randomMatrix.run_async(program, dim, self->state().id,0,device_number, arg_out, arg_size, arg_seed, arg_max);
+
+      // Extract the mem_ptrs (assume index 0 holds the buffer)
+      auto matA_ptr = std::get<0>(tA);
+      auto matB_ptr = std::get<0>(tB);
+
+      // ensure kernels are done and data is ready
+      //since we are sending to ourself no need to synchronize, since actors
+      //get their own stream
+      //if (matA_ptr) matA_ptr->synchronize();
+      //if (matB_ptr) matB_ptr->synchronize();
+
+      // send the mem_ptrs to ourselves to trigger the multiply step
+      // (we send device buffers, N)
+      for (int i =0;i < 20;i++)
+      self->mail(matA_ptr, matB_ptr, N).send(self);
+    },
+
+    // 2) multiply: receive mem_ptrs, run the mmul kernel, measure time, print, quit
+    [=](const caf::cuda::mem_ptr<int> matA,
+        const caf::cuda::mem_ptr<int> matB,
+        int N) {
+
+      // prepare mmul program + dims (same as your code)
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+      auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+      const int THREADS = 32;
+      const int BLOCKS = (N + THREADS - 1) / THREADS;
+      caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+      // create arguments; use device pointers directly (your style)
+      auto arg1 = matA;
+      auto arg2 = matB;
+      auto arg3 = caf::cuda::create_out_arg(N * N);
+      auto arg4 = caf::cuda::create_in_arg(N);
+
+      // Synchronous launch (blocks until kernel finishes and output is collected).
+      // This represents "actor is done with its result".
+      auto start = std::chrono::high_resolution_clock::now();
+      auto out_bufs = mmulAsync.run(program, dims, self->state().id,0,matA ->deviceNumber(), arg1, arg2, arg3, arg4);
+      auto end = std::chrono::high_resolution_clock::now();
+
+      // compute per-actor latency from the generation start stored in state
+      double actor_latency_ms =
+        std::chrono::duration<double, std::milli>(end - self->state().start_time).count();
+
+      // Print per-actor latency (actor id included)
+      std::cout << "[PERF] Actor id=" << self->state().id
+                << " N=" << N
+                << " latency=" << actor_latency_ms << " ms\n";
+
+      if (self -> state().times++ == 19) {
+      // Done for this actor; exit
+      self->quit();
+      }
+    }
+  };
+}
+
+// Driver: spawn actors, start timer, tell each actor to generate/send-to-self, wait, print total time
+void run_async_mmul_perf_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  // spawn actors
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_async_actor_fun_perf));
+  }
+
+  // Total runtime start
+  auto total_start = std::chrono::high_resolution_clock::now();
+
+  // Tell every actor to generate a matrix and route it to itself
+  for (auto& a : actors) {
+    // send N to the actor (actor will generate and self-send)
+    caf::anon_mail(matrix_size).send(a);
+  }
+
+  // wait for all actors to finish
+  sys.await_all_actors_done();
+
+  // Total runtime end & print
+  auto total_end = std::chrono::high_resolution_clock::now();
+  double total_ms = std::chrono::duration<double, std::milli>(total_end - total_start).count();
+  std::cout << "[PERF] Total runtime for " << num_actors << " actors: " << total_ms << " ms\n";
+}
+
+
+
+// ---------------------------
+// Shared-memory perf actor
+// ---------------------------
+caf::behavior mmul_shared_async_actor_fun_perf(caf::stateful_actor<mmul_actor_state>* self) {
+  return {
+    // 1) start: generate matrices and send them to self
+    [=](int N) {
+      auto& st = self->state();
+      st.start_time = std::chrono::high_resolution_clock::now();
+      st.last_N = N;
+
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+      // generator (same as before)
+      auto gen_prog = mgr.create_program_from_fatbin(
+          "../generate_random_matrix.fatbin", "generate_random_matrix");
+
+      const int GEN_THREADS = 256;
+      const int GEN_BLOCKS = (N * N + GEN_THREADS - 1) / GEN_THREADS;
+      caf::cuda::nd_range gen_dim(GEN_BLOCKS, 1, 1, GEN_THREADS, 1, 1);
+
+      auto arg_out = caf::cuda::create_out_arg(N * N);
+      auto arg_size = caf::cuda::create_in_arg(N * N);
+      auto arg_seed = caf::cuda::create_in_arg(rand());
+      auto arg_max  = caf::cuda::create_in_arg(9999);
+
+      // choose device (keep consistent across generator and shared kernel)
+      int device_number = rand() % 2; // or any device selection strategy
+
+      // generate device buffers asynchronously (shared_mem for generator = 0)
+      auto tA = randomMatrix.run_async(gen_prog, gen_dim, st.id, 0, device_number,
+                                       arg_out, arg_size, arg_seed, arg_max);
+      auto tB = randomMatrix.run_async(gen_prog, gen_dim, st.id, 0, device_number,
+                                       arg_out, arg_size, arg_seed, arg_max);
+
+      auto matA_ptr = std::get<0>(tA);
+      auto matB_ptr = std::get<0>(tB);
+
+      //since we send to ourselves we dont need to synchronize
+      //each actor gets its own stream
+      //if (matA_ptr) matA_ptr->synchronize();
+      //if (matB_ptr) matB_ptr->synchronize();
+
+      // send mem_ptrs + N + device_number to self for the shared-memory multiply
+     for(int i = 0; i < 20;i++)
+      self->mail(matA_ptr, matB_ptr, N, device_number).send(self);
+    },
+
+    // 2) multiply with shared memory: receive mem_ptrs, run shared kernel, measure, quit
+    [=](const caf::cuda::mem_ptr<int> matA,
+        const caf::cuda::mem_ptr<int> matB,
+        int N,
+        int device_number) {
+
+      auto& st = self->state();
+
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+      // shared-memory kernel binary
+      auto shared_prog = mgr.create_program_from_cubin("../shared_mmul.cubin", "matrixMul");
+
+      const int THREADS = 32;
+      const int BLOCKS = (N + THREADS - 1) / THREADS;
+      caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+      // prepare args (use device pointers / wrapper args as your API expects)
+      auto arg1 = matA;
+      auto arg2 = matB;
+      auto arg3 = caf::cuda::create_out_arg(N * N);
+      auto arg4 = caf::cuda::create_in_arg(N);
+
+      // choose shared memory amount for this launch (bytes)
+      // adjust as needed for your kernel (here example: 8KB)
+      const int shared_mem_bytes = 8 * 1024;
+
+      // synchronous launch that returns when outputs are ready
+      auto launch_start = std::chrono::high_resolution_clock::now();
+      // NOTE: shared_mem comes BEFORE device_number in your API
+      auto out_bufs = mmulAsync.run(shared_prog,
+                                    dims,
+                                    st.id,
+                                    shared_mem_bytes,         // <-- shared memory
+                                    device_number,            // <-- device number (same device)
+                                    arg1, arg2, arg3, arg4);
+      auto launch_end = std::chrono::high_resolution_clock::now();
+
+      // per-actor latency measured from generation start stored in state
+      double actor_latency_ms =
+        std::chrono::duration<double, std::milli>(launch_end - st.start_time).count();
+
+      std::cout << "[PERF][SHARED] Actor id=" << st.id
+                << " N=" << N
+                << " shared_mem=" << shared_mem_bytes
+                << " latency=" << actor_latency_ms << " ms\n";
+
+      
+      if (self -> state().times++ == 19) {
+      // Done for this actor; exit
+      self->quit();
+      }
+     }
+  };
+}
+
+// Driver for shared-memory perf test
+void run_shared_mmul_perf_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  // spawn actors
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_shared_async_actor_fun_perf));
+  }
+
+  // Total runtime start
+  auto total_start = std::chrono::high_resolution_clock::now();
+
+  // Tell every actor to generate a matrix and handle it with the shared kernel
+  for (auto& a : actors) {
+    caf::anon_mail(matrix_size).send(a);
+  }
+
+  // wait for all actors to finish
+  sys.await_all_actors_done();
+
+  // Total runtime end & print
+  auto total_end = std::chrono::high_resolution_clock::now();
+  double total_ms = std::chrono::duration<double, std::milli>(total_end - total_start).count();
+  std::cout << "[PERF][SHARED] Total runtime for " << num_actors << " actors: "
+            << total_ms << " ms\n";
+}
+
+
+//-----------------------------------BenchMark Tests
+
+
+// Benchmark driver for the "async (no-shared)" perf test
+void benchmark_async_perf_all(caf::actor_system& sys) {
+  const std::vector<int> actor_counts = {1, 50, 200};
+  const std::vector<int> matrix_sizes = {1024, 2048, 4096};
+
+  std::cout << "=== Async (no-shared) benchmark ===\n";
+  for (int size : matrix_sizes) {
+    for (int num_actors : actor_counts) {
+      std::cout << "[RUN] matrix_size=" << size
+                << " actors=" << num_actors
+                << "  -- starting\n" << std::flush;
+
+      auto t0 = std::chrono::high_resolution_clock::now();
+      // This function blocks until all actors finish and prints per-actor latencies.
+      run_async_mmul_perf_test(sys, size, num_actors);
+      auto t1 = std::chrono::high_resolution_clock::now();
+
+      double total_ms = std::chrono::duration<double, std::milli>(t1 - t0).count();
+      std::cout << "[RESULT] async  matrix_size=" << size
+                << " actors=" << num_actors
+                << " total_time_ms=" << total_ms << "\n\n" << std::flush;
+    }
+  }
+  std::cout << "=== Async (no-shared) benchmark complete ===\n\n";
+}
+
+// Benchmark driver for the "shared-memory" perf test
+void benchmark_shared_perf_all(caf::actor_system& sys) {
+  const std::vector<int> actor_counts = {1, 50, 200};
+  const std::vector<int> matrix_sizes = {1024, 2048, 4096};
+
+  std::cout << "=== Shared-memory benchmark ===\n";
+  for (int size : matrix_sizes) {
+    for (int num_actors : actor_counts) {
+      std::cout << "[RUN] matrix_size=" << size
+                << " actors=" << num_actors
+                << "  -- starting\n" << std::flush;
+
+      auto t0 = std::chrono::high_resolution_clock::now();
+      // This function blocks until all actors finish and prints per-actor latencies.
+      run_shared_mmul_perf_test(sys, size, num_actors);
+      auto t1 = std::chrono::high_resolution_clock::now();
+
+      double total_ms = std::chrono::duration<double, std::milli>(t1 - t0).count();
+      std::cout << "[RESULT] shared matrix_size=" << size
+                << " actors=" << num_actors
+                << " total_time_ms=" << total_ms << "\n\n" << std::flush;
+    }
+  }
+  std::cout << "=== Shared-memory benchmark complete ===\n\n";
+}
+
+
+
+
+void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager::init(sys);
+
+  run_mmul_test(sys,100,4000);
+  //run_async_mmul_test(sys,100,1);
+  //run_async_mmul_perf_test(sys,1024,200);
+
+  // run the async (no-shared) suite:
+  //benchmark_async_perf_all(sys);
+
+  // run the shared-memory suite:
+  //benchmark_shared_perf_all(sys);
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/mmul.cu b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From 604d687a80da5719acda0a0a8fb73f50175819f9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 12 Dec 2025 13:16:49 -0600
Subject: [PATCH 0072/1000] Updated the test to try and test the control layer.

---
 .../red-ligh-green-light/main.test.cpp        | 630 ++----------------
 1 file changed, 41 insertions(+), 589 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index 5cc0e2944e..5404b3b62e 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -1,5 +1,6 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
 #include <iostream>
 #include <iomanip>
 #include <vector>
@@ -21,51 +22,6 @@ CAF_ADD_ATOM(cuda,shared_mem)
 
 
 
-const char* kernel_code = R"(
-extern "C" __global__
-void compare_strings(const char* a, const char* b, int* result, int * length) {
-    int idx = blockIdx.x * blockDim.x + threadIdx.x;
-    if (idx < * length) {
-        result[idx] = (a[idx] == b[idx]) ? 1 : 0;
-    }
-}
-)";
-
-const char* matrixMulKernel2 = R"(
-extern "C" __global__
-void matrixMul(const int* a, const int* b, int* c, int *N_val) {
-    int N = *N_val;
-    int row = blockIdx.y * blockDim.y + threadIdx.y;
-    int col = blockIdx.x * blockDim.x + threadIdx.x;
-    if (row < N && col < N) {
-        int temp = 0;
-        for (int k = 0; k < N; ++k) {
-            temp += a[row * N + k] * b[k * N + col];
-        }
-        c[row * N + col] = temp;
-    }
-}
-)";
-
-
-const char* matrixMulKernel = R"(
-extern "C" __global__
-void matrixMul(const int* a, const int* b, int* c, int N) {
-    //printf("%d\n",N);
-    int row = blockIdx.y * blockDim.y + threadIdx.y;
-    int col = blockIdx.x * blockDim.x + threadIdx.x;
-    if (row < N && col < N) {
-        int temp = 0;
-        for (int k = 0; k < N; ++k) {
-            temp += a[row * N + k] * b[k * N + col];
-        }
-        c[row * N + col] = temp;
-    }
-}
-)";
-
-
-
 
 #include <chrono>
 #include <iostream>
@@ -73,11 +29,15 @@ void matrixMul(const int* a, const int* b, int* c, int N) {
 // Extend your actor state to keep the start time
 struct mmul_actor_state {
   static inline const char* name = "my_actor";
-  int last_N = 0; // example state variable
+  int N = 1024; // example state variable
   int id = rand(); // an actor id
   // per-actor timing start
   std::chrono::high_resolution_clock::time_point start_time;
   int times = 0;
+  caf::cuda::program_ptr program = caf::cuda::manager::get().create_program_from_cubin("../mmul.cubin","matrixMul");
+  int THREADS = 32;
+  int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS,BLOCKS,1,THREADS,THREADS,THREADS);
 };
 
 
@@ -116,54 +76,36 @@ void serial_matrix_multiply(const std::vector<int>& a,
 
 // Stateful actor behavior
 caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
-  return {
-    // 1st handler: Just int N, and who to send the matrices to
-    [=](int N, std::vector<caf::actor> receivers) {
-
-	/*
-	 * Unfortuanley libraries such as curand cannot be linked with cubins 
-	 * making it incompatable with this software for right now
-	 * its not really random, just a matrix filled with 5's
-	 */
-        caf::cuda::manager& mgr = caf::cuda::manager::get();
-        //create the program and configure the dimesnions of the kernel
-        auto program = mgr.create_program_from_fatbin("../generate_random_matrix.fatbin","generate_random_matrix");
-	int THREADS = 256;
-	int BLOCKS = (N*N + THREADS - 1) / THREADS;
-  	caf::cuda::nd_range dim(BLOCKS,1, 1, THREADS,1, 1);
-
-	//tag the arguments so that caf::cuda knows what to do with them	
-         auto arg1 = caf::cuda::create_out_arg(N*N); //output buffer indicate its size, caf::cuda will handle the rest
-          auto arg2 = caf::cuda::create_in_arg(N*N); //matrix size
-          auto arg3 = caf::cuda::create_in_arg(1234); //seed
-	  auto arg4 = caf::cuda::create_in_arg(9999); //max valux
-	  
-
-
-	  //launch kernels and collect their outputs
-	  auto tempA = randomMatrix.run(program,dim, self -> state().id,arg1,arg2,arg3,arg4);
-	  auto tempB = randomMatrix.run(program,dim, self -> state().id,arg1,arg2,arg3,arg4);
-	  std::vector<int> matrixA =  caf::cuda::extract_vector<int>(tempA);
-	  std::vector<int> matrixB = caf::cuda::extract_vector<int>(tempB);
-
-
-
-	  //cpu code
-	  //std::vector<int> matrixA(N*N);
-	  //std::vector<int> matrixB(N*N);
-
-	  // std::generate(matrixA.begin(), matrixA.end(), []() { return rand() % 10; });
-	   //std::generate(matrixB.begin(), matrixB.end(), []() { return rand() % 10; });
-
+  
 
-	  //broadcast the result out to receviers.
-	  for (auto actor: receivers) {
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+	
+  	caf::actor scheduler = mgr.get_scheduler_actor();
+	token_ptr launch_token = make_launch_token(self -> program,
+			self -> dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+	  [=] (token_ptr launch_response_token) {
+	  
+		  //assume N = 1024
+		  int N = self -> state().N;
+		  std::vector<int> matrixA;
+		  maxtrixA.reserve(N);
+		  std::vector<int> matrixB;
+		  matrixB.reserve(N);
+
+		  self -> mail(matrixA,matrixB,N).send(self);
+	 
+		 //token should drop out of scope now, triggering a response 
 	  
-		  self->mail(matrixA,matrixB,N).send(actor);
 	  }
 
-    },
-
     // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
     [=](const std::vector<int> matrixA,
         const std::vector<int> matrixB, int N) {
@@ -223,520 +165,30 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
     return;
   }
 
+  while (true) {
   // Spawn num_actors actors running the mmul behavior
   std::vector<caf::actor> actors;
   actors.reserve(num_actors);
   for (int i = 0; i < num_actors; ++i) {
     actors.push_back(sys.spawn(mmul_actor_fun));
   }
-
-  // Actor 0 generates matrices and broadcasts to others 
-  caf::anon_mail(matrix_size, actors).send(actors[0]);
-
-   sys.await_all_actors_done();
-}
-
-
-// Stateful actor behavior
-caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
-  return {
-    // 1st handler: Just int N, and who to send the matrices to
-    [=](int N, std::vector<caf::actor> receivers) {
-
-        caf::cuda::manager& mgr = caf::cuda::manager::get();
-        //create the program and configure the dimesnions of the kernel
-        auto program = mgr.create_program_from_fatbin("../generate_random_matrix.fatbin","generate_random_matrix");
-	int THREADS = 256;
-	int BLOCKS = (N*N + THREADS - 1) / THREADS;
-  	caf::cuda::nd_range dim(BLOCKS,1, 1, THREADS,1, 1);
-
-	//tag the arguments so that caf::cuda knows what to do with them	
-         auto arg1 = caf::cuda::create_out_arg(N*N); //output buffer indicate its size, caf::cuda will handle the rest
-          auto arg2 = caf::cuda::create_in_arg(N*N); //matrix size
-          auto arg3 = caf::cuda::create_in_arg(rand()); //seed
-	  auto arg4 = caf::cuda::create_in_arg(9999); //max valux
-	  
-          auto arg3B = caf::cuda::create_in_arg(rand()); //seed
-	  int device_number= 74; //arbitary number to show that 
-				 //can give illusion of selecting gpus that are
-				 //not there
-
-
-	  //launch kernels and collect their outputs
-	  auto tempA = randomMatrix.run_async(program,dim, self -> state().id,0,device_number,arg1,arg2,arg3,arg4);
-	  auto tempB = randomMatrix.run_async(program,dim, self -> state().id,0,device_number,arg1,arg2,arg3B,arg4);
-	  caf::cuda::mem_ptr<int> matrixA =  std::get<0>(tempA);
-	  caf::cuda::mem_ptr<int> matrixB = std::get<0>(tempB);
-
-	  //ensure the data is actually done being worked on
-	  matrixA -> synchronize();
-	  matrixB -> synchronize();
-
-
-
-
-	  //cpu code
-	  //std::vector<int> matrixA(N*N);
-	  //std::vector<int> matrixB(N*N);
-
-	  // std::generate(matrixA.begin(), matrixA.end(), []() { return rand() % 10; });
-	   //std::generate(matrixB.begin(), matrixB.end(), []() { return rand() % 10; });
-
-
-	  std::cout << "Broadcasting\n";
-	  //broadcast the result out to receviers.
-	  for (auto actor: receivers) {
-	  
-		  self->mail(3,matrixA,matrixB,N,device_number).send(actor);
-	  }
-
-    },
-
-    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const caf::cuda::mem_ptr<int> matrixA,
-        const caf::cuda::mem_ptr<int> matrixB, int N,int device_number) {
- 
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //create program and dims   
-  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    //create args
-    auto arg1 = matrixA;
-    auto arg2 = matrixB;
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
-
-
-    auto tempC = mmulAsync.run(program,dims,self -> state().id,0,device_number,arg1,arg2,arg3,arg4);
-
-    std::vector<int> matrix1 = matrixA -> copy_to_host();
-    std::vector<int> matrix2 = matrixB -> copy_to_host();    
-    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC,2);
-
-    //verify its own result 
-    self -> mail(matrix1,matrix2,matrixC,N).send(self);
-
-    },
-
- // 3nd handler: GPU atom + matrices + N, launches a kenrel using shared memory and sends its result to itself for verification
-    [=](int x,const caf::cuda::mem_ptr<int> matrixA,
-        const caf::cuda::mem_ptr<int> matrixB, int N,int device_number) {
- 
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //create program and dims   
-  auto program = mgr.create_program_from_cubin("../shared_mmul.cubin","matrixMul");
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  int shared_mem = 8192; //we need 8KB of shared memory here
-    //create args
-    auto arg1 = matrixA;
-    auto arg2 = matrixB;
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
-
-
-    auto tempC = mmulAsync.run(program,dims,self -> state().id,shared_mem,device_number,arg1,arg2,arg3,arg4);
-
-    std::vector<int> matrix1 = matrixA -> copy_to_host();
-    std::vector<int> matrix2 = matrixB -> copy_to_host();    
-    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC,2);
-
-    //verify its own result 
-    self -> mail(matrix1,matrix2,matrixC,N).send(self);
-
-    },
-
-
-
-    // 3rd handler: CPU atom + matrices + N
-    [=](const std::vector<int>& matrixA,
-    const std::vector<int> &matrixB,
-    const std::vector<int> &matrixC, int N) {
-
-    std::vector<int> result(N * N);
-
-    serial_matrix_multiply(matrixA, matrixB, result, N);
-
-    if (result == matrixC) {
-        std::cout << "actor with id " << self->state().id << " references match\n";
-    }
-    else {
-        std::cout << "actor with id " << self->state().id << " references did not match\n";
-
-    }
-
-
-    /*
-    auto print_matrix = [N](const std::vector<int>& mat, const std::string& name) {
-            std::cout << name << ":\n";
-            for (int i = 0; i < N; ++i) {
-                for (int j = 0; j < N; ++j) {
-                    std::cout << mat[i * N + j] << " ";
-                }
-                std::cout << "\n";
-            }
-            std::cout << std::endl;
-        };
-
-        print_matrix(matrixA, "Matrix A");
-        print_matrix(matrixB, "Matrix B");
-        print_matrix(result, "Result Matrix");
-        print_matrix(matrixC, "GPU Result Matrix");
-	*/
-    self->quit();
-    }
-  };
-}
-
-
-void run_async_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_async_actor_fun));
+  
+  sleep(1000);
   }
 
-  // Actor 0 generates matrices and broadcasts to others 
-  caf::anon_mail(matrix_size, actors).send(actors[0]);
+  //caf::anon_mail(matrix_size, actors).send(actors[0]);
 
    sys.await_all_actors_done();
 }
 
 
-//--------------------------------Perfomance tests 
-
-// Perf-version of the actor: each actor generates a matrix and sends to itself
-caf::behavior mmul_async_actor_fun_perf(caf::stateful_actor<mmul_actor_state>* self) {
-  return {
-    // 1) start: generate matrices and send them to self
-    [=](int N) {
-      // store start time in actor state (no locks)
-      self->state().start_time = std::chrono::high_resolution_clock::now();
-      self->state().last_N = N;
-
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
-      // use the generator fatbin (as in your code)
-      auto program = mgr.create_program_from_fatbin("../generate_random_matrix.fatbin",
-                                                    "generate_random_matrix");
-
-      int THREADS = 256;
-      int BLOCKS = (N * N + THREADS - 1) / THREADS;
-      caf::cuda::nd_range dim(BLOCKS, 1, 1, THREADS, 1, 1);
-
-      // prepare args (same as your existing code)
-      auto arg_out = caf::cuda::create_out_arg(N * N);
-      auto arg_size = caf::cuda::create_in_arg(N * N);
-      auto arg_seed = caf::cuda::create_in_arg(rand());
-      auto arg_max = caf::cuda::create_in_arg(9999);
-
-      int device_number = rand()%2;
-
-      // launch generator(s) asynchronously and get mem_ptrs back
-      // (we follow your earlier style: run_async returns tuple of mem_ptrs)
-      auto tA = randomMatrix.run_async(program, dim, self->state().id,0,device_number,arg_out, arg_size, arg_seed, arg_max);
-      auto tB = randomMatrix.run_async(program, dim, self->state().id,0,device_number, arg_out, arg_size, arg_seed, arg_max);
-
-      // Extract the mem_ptrs (assume index 0 holds the buffer)
-      auto matA_ptr = std::get<0>(tA);
-      auto matB_ptr = std::get<0>(tB);
-
-      // ensure kernels are done and data is ready
-      //since we are sending to ourself no need to synchronize, since actors
-      //get their own stream
-      //if (matA_ptr) matA_ptr->synchronize();
-      //if (matB_ptr) matB_ptr->synchronize();
-
-      // send the mem_ptrs to ourselves to trigger the multiply step
-      // (we send device buffers, N)
-      for (int i =0;i < 20;i++)
-      self->mail(matA_ptr, matB_ptr, N).send(self);
-    },
-
-    // 2) multiply: receive mem_ptrs, run the mmul kernel, measure time, print, quit
-    [=](const caf::cuda::mem_ptr<int> matA,
-        const caf::cuda::mem_ptr<int> matB,
-        int N) {
-
-      // prepare mmul program + dims (same as your code)
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
-      auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-      const int THREADS = 32;
-      const int BLOCKS = (N + THREADS - 1) / THREADS;
-      caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-      // create arguments; use device pointers directly (your style)
-      auto arg1 = matA;
-      auto arg2 = matB;
-      auto arg3 = caf::cuda::create_out_arg(N * N);
-      auto arg4 = caf::cuda::create_in_arg(N);
-
-      // Synchronous launch (blocks until kernel finishes and output is collected).
-      // This represents "actor is done with its result".
-      auto start = std::chrono::high_resolution_clock::now();
-      auto out_bufs = mmulAsync.run(program, dims, self->state().id,0,matA ->deviceNumber(), arg1, arg2, arg3, arg4);
-      auto end = std::chrono::high_resolution_clock::now();
-
-      // compute per-actor latency from the generation start stored in state
-      double actor_latency_ms =
-        std::chrono::duration<double, std::milli>(end - self->state().start_time).count();
-
-      // Print per-actor latency (actor id included)
-      std::cout << "[PERF] Actor id=" << self->state().id
-                << " N=" << N
-                << " latency=" << actor_latency_ms << " ms\n";
-
-      if (self -> state().times++ == 19) {
-      // Done for this actor; exit
-      self->quit();
-      }
-    }
-  };
-}
-
-// Driver: spawn actors, start timer, tell each actor to generate/send-to-self, wait, print total time
-void run_async_mmul_perf_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  // spawn actors
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_async_actor_fun_perf));
-  }
-
-  // Total runtime start
-  auto total_start = std::chrono::high_resolution_clock::now();
-
-  // Tell every actor to generate a matrix and route it to itself
-  for (auto& a : actors) {
-    // send N to the actor (actor will generate and self-send)
-    caf::anon_mail(matrix_size).send(a);
-  }
-
-  // wait for all actors to finish
-  sys.await_all_actors_done();
-
-  // Total runtime end & print
-  auto total_end = std::chrono::high_resolution_clock::now();
-  double total_ms = std::chrono::duration<double, std::milli>(total_end - total_start).count();
-  std::cout << "[PERF] Total runtime for " << num_actors << " actors: " << total_ms << " ms\n";
-}
-
-
-
-// ---------------------------
-// Shared-memory perf actor
-// ---------------------------
-caf::behavior mmul_shared_async_actor_fun_perf(caf::stateful_actor<mmul_actor_state>* self) {
-  return {
-    // 1) start: generate matrices and send them to self
-    [=](int N) {
-      auto& st = self->state();
-      st.start_time = std::chrono::high_resolution_clock::now();
-      st.last_N = N;
-
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
-      // generator (same as before)
-      auto gen_prog = mgr.create_program_from_fatbin(
-          "../generate_random_matrix.fatbin", "generate_random_matrix");
-
-      const int GEN_THREADS = 256;
-      const int GEN_BLOCKS = (N * N + GEN_THREADS - 1) / GEN_THREADS;
-      caf::cuda::nd_range gen_dim(GEN_BLOCKS, 1, 1, GEN_THREADS, 1, 1);
-
-      auto arg_out = caf::cuda::create_out_arg(N * N);
-      auto arg_size = caf::cuda::create_in_arg(N * N);
-      auto arg_seed = caf::cuda::create_in_arg(rand());
-      auto arg_max  = caf::cuda::create_in_arg(9999);
-
-      // choose device (keep consistent across generator and shared kernel)
-      int device_number = rand() % 2; // or any device selection strategy
-
-      // generate device buffers asynchronously (shared_mem for generator = 0)
-      auto tA = randomMatrix.run_async(gen_prog, gen_dim, st.id, 0, device_number,
-                                       arg_out, arg_size, arg_seed, arg_max);
-      auto tB = randomMatrix.run_async(gen_prog, gen_dim, st.id, 0, device_number,
-                                       arg_out, arg_size, arg_seed, arg_max);
-
-      auto matA_ptr = std::get<0>(tA);
-      auto matB_ptr = std::get<0>(tB);
-
-      //since we send to ourselves we dont need to synchronize
-      //each actor gets its own stream
-      //if (matA_ptr) matA_ptr->synchronize();
-      //if (matB_ptr) matB_ptr->synchronize();
-
-      // send mem_ptrs + N + device_number to self for the shared-memory multiply
-     for(int i = 0; i < 20;i++)
-      self->mail(matA_ptr, matB_ptr, N, device_number).send(self);
-    },
-
-    // 2) multiply with shared memory: receive mem_ptrs, run shared kernel, measure, quit
-    [=](const caf::cuda::mem_ptr<int> matA,
-        const caf::cuda::mem_ptr<int> matB,
-        int N,
-        int device_number) {
-
-      auto& st = self->state();
-
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
-      // shared-memory kernel binary
-      auto shared_prog = mgr.create_program_from_cubin("../shared_mmul.cubin", "matrixMul");
-
-      const int THREADS = 32;
-      const int BLOCKS = (N + THREADS - 1) / THREADS;
-      caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-      // prepare args (use device pointers / wrapper args as your API expects)
-      auto arg1 = matA;
-      auto arg2 = matB;
-      auto arg3 = caf::cuda::create_out_arg(N * N);
-      auto arg4 = caf::cuda::create_in_arg(N);
-
-      // choose shared memory amount for this launch (bytes)
-      // adjust as needed for your kernel (here example: 8KB)
-      const int shared_mem_bytes = 8 * 1024;
-
-      // synchronous launch that returns when outputs are ready
-      auto launch_start = std::chrono::high_resolution_clock::now();
-      // NOTE: shared_mem comes BEFORE device_number in your API
-      auto out_bufs = mmulAsync.run(shared_prog,
-                                    dims,
-                                    st.id,
-                                    shared_mem_bytes,         // <-- shared memory
-                                    device_number,            // <-- device number (same device)
-                                    arg1, arg2, arg3, arg4);
-      auto launch_end = std::chrono::high_resolution_clock::now();
-
-      // per-actor latency measured from generation start stored in state
-      double actor_latency_ms =
-        std::chrono::duration<double, std::milli>(launch_end - st.start_time).count();
-
-      std::cout << "[PERF][SHARED] Actor id=" << st.id
-                << " N=" << N
-                << " shared_mem=" << shared_mem_bytes
-                << " latency=" << actor_latency_ms << " ms\n";
-
-      
-      if (self -> state().times++ == 19) {
-      // Done for this actor; exit
-      self->quit();
-      }
-     }
-  };
-}
-
-// Driver for shared-memory perf test
-void run_shared_mmul_perf_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  // spawn actors
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_shared_async_actor_fun_perf));
-  }
-
-  // Total runtime start
-  auto total_start = std::chrono::high_resolution_clock::now();
-
-  // Tell every actor to generate a matrix and handle it with the shared kernel
-  for (auto& a : actors) {
-    caf::anon_mail(matrix_size).send(a);
-  }
-
-  // wait for all actors to finish
-  sys.await_all_actors_done();
-
-  // Total runtime end & print
-  auto total_end = std::chrono::high_resolution_clock::now();
-  double total_ms = std::chrono::duration<double, std::milli>(total_end - total_start).count();
-  std::cout << "[PERF][SHARED] Total runtime for " << num_actors << " actors: "
-            << total_ms << " ms\n";
-}
-
-
-//-----------------------------------BenchMark Tests
-
-
-// Benchmark driver for the "async (no-shared)" perf test
-void benchmark_async_perf_all(caf::actor_system& sys) {
-  const std::vector<int> actor_counts = {1, 50, 200};
-  const std::vector<int> matrix_sizes = {1024, 2048, 4096};
-
-  std::cout << "=== Async (no-shared) benchmark ===\n";
-  for (int size : matrix_sizes) {
-    for (int num_actors : actor_counts) {
-      std::cout << "[RUN] matrix_size=" << size
-                << " actors=" << num_actors
-                << "  -- starting\n" << std::flush;
-
-      auto t0 = std::chrono::high_resolution_clock::now();
-      // This function blocks until all actors finish and prints per-actor latencies.
-      run_async_mmul_perf_test(sys, size, num_actors);
-      auto t1 = std::chrono::high_resolution_clock::now();
-
-      double total_ms = std::chrono::duration<double, std::milli>(t1 - t0).count();
-      std::cout << "[RESULT] async  matrix_size=" << size
-                << " actors=" << num_actors
-                << " total_time_ms=" << total_ms << "\n\n" << std::flush;
-    }
-  }
-  std::cout << "=== Async (no-shared) benchmark complete ===\n\n";
-}
-
-// Benchmark driver for the "shared-memory" perf test
-void benchmark_shared_perf_all(caf::actor_system& sys) {
-  const std::vector<int> actor_counts = {1, 50, 200};
-  const std::vector<int> matrix_sizes = {1024, 2048, 4096};
-
-  std::cout << "=== Shared-memory benchmark ===\n";
-  for (int size : matrix_sizes) {
-    for (int num_actors : actor_counts) {
-      std::cout << "[RUN] matrix_size=" << size
-                << " actors=" << num_actors
-                << "  -- starting\n" << std::flush;
-
-      auto t0 = std::chrono::high_resolution_clock::now();
-      // This function blocks until all actors finish and prints per-actor latencies.
-      run_shared_mmul_perf_test(sys, size, num_actors);
-      auto t1 = std::chrono::high_resolution_clock::now();
-
-      double total_ms = std::chrono::duration<double, std::milli>(t1 - t0).count();
-      std::cout << "[RESULT] shared matrix_size=" << size
-                << " actors=" << num_actors
-                << " total_time_ms=" << total_ms << "\n\n" << std::flush;
-    }
-  }
-  std::cout << "=== Shared-memory benchmark complete ===\n\n";
-}
-
-
-
 
 void caf_main(caf::actor_system& sys) {
-  caf::cuda::manager::init(sys);
+  
+	
+
+	caf::cuda::manager_config man_config(true); //turns the scheduler on
+	caf::cuda::manager::init(sys,man_config);
 
   run_mmul_test(sys,100,4000);
   //run_async_mmul_test(sys,100,1);

From ad1f0acbb3413cf445df0e242a3348cfc697aaaa Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 12 Dec 2025 13:24:23 -0600
Subject: [PATCH 0073/1000] Fixed syntax errors.

---
 .../red-ligh-green-light/main.test.cpp        | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index 5404b3b62e..487ac7de7b 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -9,6 +9,7 @@
 #include <algorithm>
 #include <numeric>
 #include <random>
+#include <unistd.h>
 #include "caf/actor_registry.hpp"
 //#include <caf/atoms.hpp>
 
@@ -81,8 +82,8 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 	
   	caf::actor scheduler = mgr.get_scheduler_actor();
-	token_ptr launch_token = make_launch_token(self -> program,
-			self -> dims,
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
+			self -> state().dims,
 			0,
 			"hello",
 			self
@@ -91,20 +92,20 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
 
 	return {
 
-	  [=] (token_ptr launch_response_token) {
+	  [=] (caf::cuda::token_ptr launch_response_token) {
 	  
 		  //assume N = 1024
 		  int N = self -> state().N;
-		  std::vector<int> matrixA;
-		  maxtrixA.reserve(N);
-		  std::vector<int> matrixB;
-		  matrixB.reserve(N);
+		  std::vector<int> matrix1;
+		  matrix1.reserve(N);
+		  std::vector<int> matrix2;
+		  matrix2.reserve(N);
 
-		  self -> mail(matrixA,matrixB,N).send(self);
+		  self -> mail(matrix1,matrix2,N).send(self);
 	 
 		 //token should drop out of scope now, triggering a response 
 	  
-	  }
+	  },
 
     // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
     [=](const std::vector<int> matrixA,
@@ -173,7 +174,7 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
     actors.push_back(sys.spawn(mmul_actor_fun));
   }
   
-  sleep(1000);
+  sleep(1);
   }
 
   //caf::anon_mail(matrix_size, actors).send(actors[0]);

From c3f68347e897cda3593bef483e6334bbb49c2b92 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 12 Dec 2025 13:39:20 -0600
Subject: [PATCH 0074/1000] Moved the global export out of the global file, to
 ensure control layer can have access to it.

---
 libcaf_cuda/caf/cuda/global.hpp        | 11 +++--------
 libcaf_cuda/caf/cuda/global_export.hpp | 11 +++++++++++
 2 files changed, 14 insertions(+), 8 deletions(-)
 create mode 100644 libcaf_cuda/caf/cuda/global_export.hpp

diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index 41d94cfe2f..28c056502a 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -13,15 +13,10 @@
 // CAF type ID registration
 #include <caf/type_id.hpp>
 #include <caf/anon_mail.hpp>
+#include "caf/cuda/global_export.hpp"
+
+
 
-//a strange fix required in order to get the .so files to become viewable for binaries
-//linking against them, if this is not defined with classes you want viewable then
-//the linker will complain
-#if defined(_MSC_VER)
-  #define CAF_CUDA_EXPORT __declspec(dllexport)
-#else
-  #define CAF_CUDA_EXPORT __attribute__((visibility("default")))
-#endif
 
 //helper function to check errors
 void inline check(CUresult result, const char* msg) {
diff --git a/libcaf_cuda/caf/cuda/global_export.hpp b/libcaf_cuda/caf/cuda/global_export.hpp
new file mode 100644
index 0000000000..1bcd8719c2
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/global_export.hpp
@@ -0,0 +1,11 @@
+#pragma once
+//a strange fix required in order to get the .so files to become viewable for binaries
+//linking against them, if this is not defined with classes you want viewable then
+//the linker will complain
+#if defined(_MSC_VER)
+  #define CAF_CUDA_EXPORT __declspec(dllexport)
+#else
+  #define CAF_CUDA_EXPORT __attribute__((visibility("default")))
+#endif
+
+

From a589fdbf366be9c98d1e9b1e98d8f7b7394481d0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 12 Dec 2025 13:43:30 -0600
Subject: [PATCH 0075/1000] Added CAF_CUDA_EXPORT tags to classes and functions
 so that the linker can properly see these classes.

---
 .../caf/cuda/control-layer/launch_response_token.hpp       | 3 ++-
 libcaf_cuda/caf/cuda/control-layer/launch_token.hpp        | 3 ++-
 libcaf_cuda/caf/cuda/control-layer/token.hpp               | 3 ++-
 libcaf_cuda/caf/cuda/control-layer/token_factory.hpp       | 7 ++++---
 4 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
index b366ffa414..1864a89a60 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -3,11 +3,12 @@
 #include "caf/cuda/control-layer/launch_token.hpp"  // Full include for constructor param
 #include <caf/actor.hpp>  // For caf::actor
 #include "caf/cuda/nd_range.hpp"  // For nd_range
+#include "caf/cuda/global_export.hpp"
 #include <atomic>
 #include <string>  // For std::string (if not from elsewhere)
 
 namespace caf::cuda {
-class launch_response_token : public token {
+class CAF_CUDA_EXPORT launch_response_token : public token {
 public:
     // Construct manually
     launch_response_token(caf::actor receiver,
diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
index 7c6c8ccd8c..634a821f4c 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
@@ -3,13 +3,14 @@
 #include "caf/cuda/program.hpp"
 #include "caf/cuda/control-layer/token.hpp"
 #include "caf/cuda/nd_range.hpp"  // For nd_range (full def needed for member)
+#include "caf/cuda/global_export.hpp"
 
 
 namespace caf::cuda {
 class program; 
 using program_ptr = caf::intrusive_ptr<program>;  
 
-class launch_token : public token {
+class CAF_CUDA_EXPORT launch_token : public token {
 public:
     launch_token(program_ptr prog,
                  nd_range range,
diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index 342c18cb7c..9c1c21e535 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -2,6 +2,7 @@
 #include <atomic>
 #include <caf/intrusive_ptr.hpp>
 #include <caf/ref_counted.hpp>
+#include "caf/cuda/global_export.hpp"
 
 //types of tokens
 #define LAUNCH 1
@@ -13,7 +14,7 @@
 namespace caf::cuda {
 
 // Base token interface
-class token : public caf::ref_counted {
+class CAF_CUDA_EXPORT token : public caf::ref_counted {
 public:
     virtual ~token() = default;
     virtual int getType() = 0;
diff --git a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
index 49812fbb81..c6d607dbe7 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
@@ -1,5 +1,6 @@
 #pragma once
 
+#include "caf/cuda/global_export.hpp" //here to export files
 #include "caf/cuda/control-layer/token.hpp"
 #include "caf/cuda/control-layer/launch_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
@@ -10,7 +11,7 @@
 namespace caf::cuda {
 
 /// Creates a launch_token (used by users when submitting kernels)
-token_ptr make_launch_token(program_ptr prog,
+CAF_CUDA_EXPORT token_ptr make_launch_token(program_ptr prog,
                             nd_range range,
                             int memory_usage,
                             std::string id,
@@ -18,10 +19,10 @@ token_ptr make_launch_token(program_ptr prog,
 
 /// Creates a launch_response_token (created internally by the scheduler
 /// when it accepts a kernel launch request)
-token_ptr make_launch_response_token(actor scheduler_or_proxy,
+CAF_CUDA_EXPORT token_ptr make_launch_response_token(actor scheduler_or_proxy,
                                      const launch_token& orig);
 
 /// Creates a behavior_token (special — returns its own strong ptr type)
-behavior_token_ptr make_behavior_token(std::string name);
+CAF_CUDA_EXPORT behavior_token_ptr make_behavior_token(std::string name);
 
 } // namespace caf::cuda

From 0c035387463de34956b9f14cd0f75c3abdd4276f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 12 Dec 2025 13:50:52 -0600
Subject: [PATCH 0076/1000] Updated downcasting to prevent segfault

---
 .../src/control-layer/green_light_behavior.cpp | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index faa54a7778..d81ec512e9 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -14,15 +14,17 @@ namespace caf::cuda {
         token_ptr queued = state->queue.front();
         state->queue.pop();
 
-        if (queued->getType() == LAUNCH) {
-            // manually downcast raw pointer
-            caf::intrusive_ptr<launch_token> ltok(static_cast<launch_token*>(queued.get()));
-
-            // dereference to pass reference to factory function
-            token_ptr response = make_launch_response_token(state->self, *ltok);
-            anon_mail(response).send(ltok->getReplyActor());
-        }
+ if (queued->getType() == LAUNCH) {
+      // safe: we've checked the runtime type
+      caf::cuda::launch_token& lt = static_cast<caf::cuda::launch_token&>(*queued);
+
+      // create the response using a reference to the existing object
+      caf::cuda::token_ptr response = make_launch_response_token(state->self, lt);
+
+      // send response to the reply actor stored in launch_token
+      anon_mail(response).send(lt.getReplyActor());
     }
+  }
 
 
     }

From c8ec755ff492f33b98076fdcfb6f66b0a531a5de Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 13 Dec 2025 10:17:36 -0600
Subject: [PATCH 0077/1000] Modified code to test certain aspects.

---
 .../control-layer-tests/red-ligh-green-light/main.test.cpp  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index 487ac7de7b..4d17219f21 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -96,9 +96,9 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
 	  
 		  //assume N = 1024
 		  int N = self -> state().N;
-		  std::vector<int> matrix1;
+		  std::vector<int> matrix1(N*N);
 		  matrix1.reserve(N);
-		  std::vector<int> matrix2;
+		  std::vector<int> matrix2(N*N);
 		  matrix2.reserve(N);
 
 		  self -> mail(matrix1,matrix2,N).send(self);
@@ -191,7 +191,7 @@ void caf_main(caf::actor_system& sys) {
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	caf::cuda::manager::init(sys,man_config);
 
-  run_mmul_test(sys,100,4000);
+  run_mmul_test(sys,100,1);
   //run_async_mmul_test(sys,100,1);
   //run_async_mmul_perf_test(sys,1024,200);
 

From 9ad15733968c6c840de39d2529d2013a3ca297d6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 13 Dec 2025 12:45:05 -0600
Subject: [PATCH 0078/1000] Moved allocation over to heap allocation.

---
 libcaf_cuda/src/control-layer/token_factory.cpp | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/token_factory.cpp b/libcaf_cuda/src/control-layer/token_factory.cpp
index 90d81453ed..614a8a3ce5 100644
--- a/libcaf_cuda/src/control-layer/token_factory.cpp
+++ b/libcaf_cuda/src/control-layer/token_factory.cpp
@@ -5,28 +5,31 @@
 
 namespace caf::cuda {
 
+// Use manual heap allocation with intrusive_ptr
 token_ptr make_launch_token(program_ptr prog,
                             nd_range range,
                             int memory_usage,
                             std::string id,
                             actor reply_to)
 {
-    return caf::make_counted<launch_token>(std::move(prog),
-                                           std::move(range),
-                                           memory_usage,
-                                           std::move(id),
-                                           reply_to);
+    return token_ptr(
+        new launch_token(std::move(prog),
+                         std::move(range),
+                         memory_usage,
+                         std::move(id),
+                         reply_to));
 }
 
 token_ptr make_launch_response_token(actor receiver,
                                      const launch_token& orig)
 {
-    return caf::make_counted<launch_response_token>(receiver, orig);
+    return token_ptr(new launch_response_token(receiver, orig));
 }
 
 behavior_token_ptr make_behavior_token(std::string name)
 {
-    return caf::make_counted<behavior_token>(std::move(name));
+    return behavior_token_ptr(new behavior_token(std::move(name)));
 }
 
 } // namespace caf::cuda
+

From f8f52ed72e17c9d6334daaa87a2c40b431859ba9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 13 Dec 2025 13:49:36 -0600
Subject: [PATCH 0079/1000] Successfully replicated crash its because
 cuda_control is not a register id block with caf.

---
 .../red-ligh-green-light/main.test.cpp        | 125 ++++++++++++++++--
 1 file changed, 116 insertions(+), 9 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index 4d17219f21..fd95dcb5ff 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -93,7 +93,8 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
 	return {
 
 	  [=] (caf::cuda::token_ptr launch_response_token) {
-	  
+	 
+		  std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
 		  //assume N = 1024
 		  int N = self -> state().N;
 		  std::vector<int> matrix1(N*N);
@@ -101,6 +102,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
 		  std::vector<int> matrix2(N*N);
 		  matrix2.reserve(N);
 
+		  std::cout << "GPU ACTOR sending data to compute\n";
 		  self -> mail(matrix1,matrix2,N).send(self);
 	 
 		 //token should drop out of scope now, triggering a response 
@@ -112,6 +114,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
         const std::vector<int> matrixB, int N) {
  
 
+		  std::cout << "GPU ACTOR  computing\n";
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 
   //create program and dims   
@@ -129,6 +132,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
     auto tempC = mmul.run(program,dims,self -> state().id,arg1,arg2,arg3,arg4);
     std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
 
+		  std::cout << "GPU ACTOR done  computing\n";
     //verify its own result 
     self -> mail(matrixA,matrixB,matrixC,N).send(self);
 
@@ -138,6 +142,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
     [=](const std::vector<int>& matrixA,
         const std::vector<int>& matrixB, const std::vector<int> matrixC, int N) {
        
+		  std::cout << "GPU ACTOR  verfiying\n";
 	 std::vector<int> result(N*N);
 
 	 serial_matrix_multiply(matrixA,matrixB,result,N);
@@ -181,6 +186,113 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
 
    sys.await_all_actors_done();
 }
+// Stateful actor behavior
+// tests if mem_ptr can be sent correctly
+caf::behavior mmul_actor_fun2(caf::stateful_actor<mmul_actor_state>* self) {
+  
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+/*		
+  caf::cuda::token_ptr bad_launch_token = caf::cuda::make_launch_token(self ->state().program,
+			self -> state().dims,
+			0,
+			"hello",
+			self
+			);
+
+	self -> mail(bad_launch_token).send(scheduler);
+			*/
+
+  	caf::actor scheduler = mgr.get_scheduler_actor();
+	caf::cuda::mem_ptr<int> launch_token = caf::cuda::make_mem_ptr(16);
+			
+	self -> mail(launch_token).send(scheduler);
+
+	self -> mail("Stay alive\n").send(self);
+
+
+
+	return {
+
+		[=] (std::string hello){
+		while(1) {
+	
+		std::cout << "Staying alive\n";
+		sleep(1);
+	}	
+		},
+	  [=] (caf::cuda::token_ptr launch_response_token) {
+	 
+		  std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+		  //assume N = 1024
+		  int N = self -> state().N;
+		  std::vector<int> matrix1(N*N);
+		  matrix1.reserve(N);
+		  std::vector<int> matrix2(N*N);
+		  matrix2.reserve(N);
+
+		  std::cout << "GPU ACTOR sending data to compute\n";
+		  self -> mail(matrix1,matrix2,N).send(self);
+	 
+		 //token should drop out of scope now, triggering a response 
+	  
+	  },
+
+    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+    [=](const std::vector<int> matrixA,
+        const std::vector<int> matrixB, int N) {
+ 
+
+		  std::cout << "GPU ACTOR  computing\n";
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //create program and dims   
+  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+  const int THREADS = 32;
+  const int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    //create args
+    auto arg1 = caf::cuda::create_in_arg(matrixA);
+    auto arg2 = caf::cuda::create_in_arg(matrixB);
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    auto tempC = mmul.run(program,dims,self -> state().id,arg1,arg2,arg3,arg4);
+    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+		  std::cout << "GPU ACTOR done  computing\n";
+    //verify its own result 
+    self -> mail(matrixA,matrixB,matrixC,N).send(self);
+
+    },
+
+    // 3rd handler: CPU atom + matrices + N
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB, const std::vector<int> matrixC, int N) {
+       
+		  std::cout << "GPU ACTOR  verfiying\n";
+	 std::vector<int> result(N*N);
+
+	 serial_matrix_multiply(matrixA,matrixB,result,N);
+
+	 if (result == matrixC) {
+	 
+		 std::cout << "actor with id " <<  self->state().id << " references match\n";
+	 
+	 }
+
+	 else {
+	    std::cout << "actor with id " <<  self->state().id << " references did not match\n";
+	 }
+
+	 self-> quit();
+
+    }
+  };
+}
+
+
 
 
 
@@ -191,15 +303,10 @@ void caf_main(caf::actor_system& sys) {
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	caf::cuda::manager::init(sys,man_config);
 
-  run_mmul_test(sys,100,1);
-  //run_async_mmul_test(sys,100,1);
-  //run_async_mmul_perf_test(sys,1024,200);
-
-  // run the async (no-shared) suite:
-  //benchmark_async_perf_all(sys);
+	 //caf::init_global_meta_objects<caf::id_block::cuda_control>();
 
-  // run the shared-memory suite:
-  //benchmark_shared_perf_all(sys);
+	sys.spawn(mmul_actor_fun2);
+  //run_mmul_test(sys,100,1);
 }
 
 

From e7c8272b09a6435c5fb8a770d09c585bd14e2c70 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 10:24:55 -0600
Subject: [PATCH 0080/1000] Added default constructors to be complaint with
 caf's type id system.

---
 .../caf/cuda/control-layer/behavior_token.hpp |  3 +-
 .../control-layer/launch_response_token.hpp   | 10 ++++--
 .../caf/cuda/control-layer/launch_token.hpp   |  9 +++--
 libcaf_cuda/caf/cuda/control-layer/token.hpp  | 35 +++++++++++++------
 4 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp
index ed4929c515..45a6b95e59 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior_token.hpp
@@ -8,10 +8,11 @@ class behavior_token : public token {
 public:
     explicit behavior_token(std::string n) : name_(std::move(n)) {}
 
+    behavior_token() = default;
     const std::string& name() const { return name_; }
 
     // override getType() from token
-    int getType() override { return BEHAVIOR; }
+    int getType() const override { return BEHAVIOR; }
 
 private:
     std::string name_;
diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
index 1864a89a60..27f5f056f6 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -10,8 +10,12 @@
 namespace caf::cuda {
 class CAF_CUDA_EXPORT launch_response_token : public token {
 public:
-    // Construct manually
-    launch_response_token(caf::actor receiver,
+
+	//only here to be complaint with CAFS type if system DO NOT USE 
+	launch_response_token() = default;
+
+    // Construct manually    
+launch_response_token(caf::actor receiver,
                           nd_range range,
                           int memory_usage,
                           std::string id)
@@ -30,7 +34,7 @@ class CAF_CUDA_EXPORT launch_response_token : public token {
     ~launch_response_token() {
         release();
     }
-    int getType() override { return LAUNCH_RESPONSE; }
+    int getType() const override { return LAUNCH_RESPONSE; }
     const nd_range& getRange() const { return range_; }
     int getMemoryUsage() const { return memory_usage_; }
     // Return requested number of CUDA blocks
diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
index 634a821f4c..65b2557037 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
@@ -12,7 +12,12 @@ using program_ptr = caf::intrusive_ptr<program>;
 
 class CAF_CUDA_EXPORT launch_token : public token {
 public:
-    launch_token(program_ptr prog,
+
+	//only here to be complaint with CAFS type if system DO NOT USE 
+	launch_token() = default;
+    
+	
+	launch_token(program_ptr prog,
                  nd_range range,
                  int memory_usage,
                  std::string id,
@@ -22,7 +27,7 @@ class CAF_CUDA_EXPORT launch_token : public token {
           memory_usage_(memory_usage),
           id_(std::move(id)),
           reply_handle_(receiver) {}  // Fixed missing )
-    int getType() override { return LAUNCH; }
+    int getType() const override { return LAUNCH; }
     const program_ptr& getProgram() const { return program_; }
     const nd_range& getRange() const { return range_; }
     int getMemoryUsage() const { return memory_usage_; }
diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index 9c1c21e535..224dde049b 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -3,6 +3,7 @@
 #include <caf/intrusive_ptr.hpp>
 #include <caf/ref_counted.hpp>
 #include "caf/cuda/global_export.hpp"
+#include <iostream>
 
 //types of tokens
 #define LAUNCH 1
@@ -16,20 +17,32 @@ namespace caf::cuda {
 // Base token interface
 class CAF_CUDA_EXPORT token : public caf::ref_counted {
 public:
-    virtual ~token() = default;
-    virtual int getType() = 0;
+   virtual ~token() {
+    // Print ref count when destructor runs
+    size_t count = ref_count.load(std::memory_order_acquire);
+    std::cout << "token object getting deleted, ref_count = " << count << "\n";
+}
 
-protected:
-    mutable std::atomic<size_t> ref_count_{0};
+    //should only be used by caf's type id system
+    token() = default;
 
-    friend void intrusive_ptr_add_ref(const token* p) noexcept {
-        p->ref_count_.fetch_add(1, std::memory_order_relaxed);
-    }
+    virtual int getType() const {return -1;}
 
-    friend void intrusive_ptr_release(const token* p) noexcept {
-        if (p->ref_count_.fetch_sub(1, std::memory_order_acq_rel) == 1)
-            delete p;
-    }
+protected:
+ mutable std::atomic<size_t> ref_count{0}; // start at 1 for make_counted
+
+friend  void intrusive_ptr_add_ref(const token* p) noexcept {
+    std::cout << "[add_ref] old count = " << p->ref_count << "\n";
+    ++p->ref_count;
+    std::cout << "[add_ref] new count = " << p->ref_count << "\n";
+}
+
+friend void intrusive_ptr_release(const token* p) noexcept {
+    std::cout << "[release] old count = " << p->ref_count << "\n";
+    --p->ref_count;
+    std::cout << "[release] new count = " << p->ref_count << "\n";
+    if (p->ref_count == 0) std::cout << "token object getting deleted\n";
+}
 };
 
 using token_ptr = caf::intrusive_ptr<token>;

From 716985ed45af8aeae28e192ab8dcb6fa35499054 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 10:32:32 -0600
Subject: [PATCH 0081/1000] Commeneted out some type id systems to conform to
 cafs message id system.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index cee11883b4..b512d8faae 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -13,10 +13,10 @@ CAF_BEGIN_TYPE_ID_BLOCK(cuda_control, caf::first_custom_type_id + 200)
 
   CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::token))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::token>))  
-CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::behavior_token))
+//CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::behavior_token))
   CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::behavior_token>)) // ← and this one too if you ever use it
-	CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::launch_token))
-  CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::launch_response_token))
+	//CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::launch_token))
+  //CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::launch_response_token))
 
   // Very commonly needed as well – add them now so you don’t hit the error later
 

From 3a8fe05375db8ec28139711d53ef89b2fc544f98 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 10:53:15 -0600
Subject: [PATCH 0082/1000] Changed registered message types to adhere to cafs
 message passing policy.

---
 .../cuda/control-layer/all-control-layer.hpp  | 34 ++++++++++++-------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index b512d8faae..ebb055da45 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -1,31 +1,41 @@
 #pragma once
+
 #include "caf/cuda/all.hpp"
+
+// Control-layer object types (ref-counted, non-copyable)
 #include "caf/cuda/control-layer/token.hpp"
 #include "caf/cuda/control-layer/launch_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 #include "caf/cuda/control-layer/behavior_token.hpp"
+
+// Control-layer logic
 #include "caf/cuda/control-layer/behavior.hpp"
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
 #include "caf/cuda/control-layer/token_factory.hpp"
 
-CAF_BEGIN_TYPE_ID_BLOCK(cuda_control, caf::first_custom_type_id + 200)
-// You can also use  caf::id_block::core::end  instead of the +200 if you prefer
+// -----------------------------------------------------------------------------
+// Type ID block
+//   Register ONLY the types that may appear by value in caf::message.
+//   Ref-counted control objects are passed exclusively via intrusive_ptr.
+// -----------------------------------------------------------------------------
 
-  CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::token))
-CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::token>))  
-//CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::behavior_token))
-  CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::behavior_token>)) // ← and this one too if you ever use it
-	//CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::launch_token))
-  //CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::launch_response_token))
+CAF_BEGIN_TYPE_ID_BLOCK(cuda_control, caf::first_custom_type_id + 200)
 
-  // Very commonly needed as well – add them now so you don’t hit the error later
+CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::token>))
+CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::behavior_token>))
+CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_token>))
+CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_response_token>))
 
 CAF_END_TYPE_ID_BLOCK(cuda_control)
 
-// Optional but harmless – keep your old macros (they silence the “unsafe” warning)
+// -----------------------------------------------------------------------------
+// Unsafe message types
+//   These are explicitly local-only, non-serializable control objects.
+//   CAF will reject any attempt to send them over the network.
+// -----------------------------------------------------------------------------
+
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::token)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::token_ptr)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::behavior_token)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::behavior_token_ptr)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_response_token)
+

From b7c9b5ae11e4148996e6b00dae3b525d1526a8e2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 10:56:14 -0600
Subject: [PATCH 0083/1000] Unregistered all types to ensure caf does not
 attempt to serialize them.

---
 .../cuda/control-layer/all-control-layer.hpp  | 35 +++++++++----------
 1 file changed, 16 insertions(+), 19 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index ebb055da45..ab643e5c1e 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -2,7 +2,7 @@
 
 #include "caf/cuda/all.hpp"
 
-// Control-layer object types (ref-counted, non-copyable)
+// Control-layer object types (local-only, ref-counted)
 #include "caf/cuda/control-layer/token.hpp"
 #include "caf/cuda/control-layer/launch_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
@@ -14,24 +14,15 @@
 #include "caf/cuda/control-layer/token_factory.hpp"
 
 // -----------------------------------------------------------------------------
-// Type ID block
-//   Register ONLY the types that may appear by value in caf::message.
-//   Ref-counted control objects are passed exclusively via intrusive_ptr.
-// -----------------------------------------------------------------------------
-
-CAF_BEGIN_TYPE_ID_BLOCK(cuda_control, caf::first_custom_type_id + 200)
-
-CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::token>))
-CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::behavior_token>))
-CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_token>))
-CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_response_token>))
-
-CAF_END_TYPE_ID_BLOCK(cuda_control)
-
-// -----------------------------------------------------------------------------
-// Unsafe message types
-//   These are explicitly local-only, non-serializable control objects.
-//   CAF will reject any attempt to send them over the network.
+// NO TYPE ID BLOCK
+//
+// These control-layer types are:
+//   - local-only
+//   - never serialized
+//   - never sent over the network
+//   - ref-counted and non-copyable
+//
+// Therefore they must NOT be registered.
 // -----------------------------------------------------------------------------
 
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::token)
@@ -39,3 +30,9 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::behavior_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_response_token)
 
+// And most importantly:
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::token>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::behavior_token>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_token>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_response_token>)
+

From f7a9223eaa245e7ec70579469badcf82d81f2b99 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 11:00:39 -0600
Subject: [PATCH 0084/1000] Fixed type id compiler issue.

---
 .../cuda/control-layer/all-control-layer.hpp  | 27 ++++++++++---------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index ab643e5c1e..5dac47b8af 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -2,27 +2,31 @@
 
 #include "caf/cuda/all.hpp"
 
-// Control-layer object types (local-only, ref-counted)
+// Control-layer object types
 #include "caf/cuda/control-layer/token.hpp"
 #include "caf/cuda/control-layer/launch_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 #include "caf/cuda/control-layer/behavior_token.hpp"
 
-// Control-layer logic
 #include "caf/cuda/control-layer/behavior.hpp"
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
 #include "caf/cuda/control-layer/token_factory.hpp"
 
 // -----------------------------------------------------------------------------
-// NO TYPE ID BLOCK
-//
-// These control-layer types are:
-//   - local-only
-//   - never serialized
-//   - never sent over the network
-//   - ref-counted and non-copyable
-//
-// Therefore they must NOT be registered.
+// Type IDs (required for typed behaviors)
+// -----------------------------------------------------------------------------
+
+CAF_BEGIN_TYPE_ID_BLOCK(cuda_control, caf::first_custom_type_id + 200)
+
+CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::token>))
+CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::behavior_token>))
+CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_token>))
+CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_response_token>))
+
+CAF_END_TYPE_ID_BLOCK(cuda_control)
+
+// -----------------------------------------------------------------------------
+// Unsafe: explicitly local-only, never serialized
 // -----------------------------------------------------------------------------
 
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::token)
@@ -30,7 +34,6 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::behavior_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_response_token)
 
-// And most importantly:
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::behavior_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_token>)

From 5db5bd5e1aaa80a6996dee1fa414ef2d3ae3c0dc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 11:01:30 -0600
Subject: [PATCH 0085/1000] Added proper initiliazation for control layer
 messages.

---
 libcaf_cuda/src/manager.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 5ce14b7173..34a84cffb2 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -4,6 +4,7 @@
 #include <map>
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
 #include "caf/cuda/manager_config.hpp"
+#include "caf/cuda/control-layer/all-control-layer.hpp"
 
 
 namespace caf::cuda {
@@ -52,6 +53,7 @@ void manager::init(caf::actor_system& sys, manager_config config) {
     instance_ = new manager(sys);
 
     caf::init_global_meta_objects<caf::id_block::cuda>();
+    caf::init_global_meta_objects<caf::id_block::cuda_control>();
 
     instance_->scheduler_on = config.getSchedulerOn();
 

From 32410e02a6b72ddc0f42e7353a3a4ec21c1e0e08 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 11:03:09 -0600
Subject: [PATCH 0086/1000] Changed lambda to take in behavior token ptr
 instead of behavior token. THis change was made to ensure that the scheduler
 is compliant with cafs messaging types.

---
 .../src/control-layer/scheduler_actor.cpp     | 31 ++++++++++++++++---
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index c8218e4e9a..0bfd095afe 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -3,6 +3,7 @@
 #include "caf/cuda/control-layer/green_light_behavior.hpp"
 #include "caf/cuda/control-layer/red_light_behavior.hpp"
 #include <string>
+#include <iostream>
 
 /*
  * This class is meant to handle actor GPU scheduling via s/r/r IPC 
@@ -28,10 +29,13 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self)
 
     return {
         [=](const token_ptr& tok) {
-            self->state().current_behavior->receive(&self->state(), tok);
+           
+
+		std::cout << "Received token\n";
+		self->state().current_behavior->receive(&self->state(), tok);
         },
-        [=](const behavior_token& tok) {
-            auto* next = self->state().table.get(tok);
+        [=](const caf::cuda::behavior_token_ptr& tok) {
+            auto* next = self->state().table.get(*tok);
             if (next) {
                 
 		    self->state().current_behavior -> cleanup(&self->state()); //cleanup current behavior
@@ -42,7 +46,26 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self)
         },
 	[=](std::string word) {
 		std::cout << "Received message " << word << "\n";
-	}
+	},
+	 [=](caf::cuda::mem_ptr<int> token) {
+            if (!token) {
+                std::cout << "Received null mem_ptr\n";
+                return;
+            }
+
+            if (token->is_scalar()) {
+                std::cout << "Received mem_ptr with scalar value: "
+                          << *token->host_scalar_ptr() << "\n";
+            } else {
+                std::cout << "Received mem_ptr with "
+                          << token->size() << " elements\n";
+                // Optional: print fake data if testing copy_to_host
+               // auto host_data = token->copy_to_host();
+               // for (auto v : host_data) std::cout << v << " ";
+                //std::cout << "\n";
+            }
+        }
+
     };
 }
 

From d075d7f54a1e135c76dae17a69802eec9ea07e2e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 11:04:17 -0600
Subject: [PATCH 0087/1000] Uncommented out init and cleanup behavior
 implementations.

---
 .../control-layer/green_light_behavior.cpp    | 23 ++++++++++++++++---
 .../src/control-layer/red_light_behavior.cpp  |  2 +-
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index d81ec512e9..550d697157 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -1,6 +1,6 @@
 #include "caf/cuda/control-layer/all-control-layer.hpp"
 #include "caf/cuda/control-layer/green_light_behavior.hpp"
-
+#include <iostream>
 
 namespace caf::cuda {
 
@@ -9,7 +9,21 @@ namespace caf::cuda {
     }
 
     void green_light_behavior::receive(scheduler_actor_state* state, const token_ptr& tok)    {
-         // flush the queue
+         
+	    std::cout << "Hello from green light processing token\n";
+
+	caf::cuda::launch_token& launch = static_cast<caf::cuda::launch_token&>(*tok);
+
+      // create the response using a reference to the existing object
+      caf::cuda::token_ptr r = make_launch_response_token(state->self, launch);
+
+      // send response to the reply actor stored in launch_token
+      anon_mail(r).send(launch.getReplyActor());
+
+
+      std::cout << "GREEN LIGHT DONE PROCESSING TOKEN\n";
+
+	    // flush the queue
     while (!state->queue.empty()) {
         token_ptr queued = state->queue.front();
         state->queue.pop();
@@ -31,12 +45,15 @@ namespace caf::cuda {
 
    void green_light_behavior::init(scheduler_actor_state * state) {
 	   std::cout << "GREEN LIGHT\n";
+
+	   
 	   behavior_token_ptr red_light = make_behavior_token("red");
 
 	   //send a request to change behavior to green light after 5 seconds
     	   anon_mail(red_light)
 		   .delay(std::chrono::seconds(5))
-		   .send(state -> self);	
+		   .send(state -> self);
+	   
     }
 
 
diff --git a/libcaf_cuda/src/control-layer/red_light_behavior.cpp b/libcaf_cuda/src/control-layer/red_light_behavior.cpp
index a4e1976366..bad3a83c0c 100644
--- a/libcaf_cuda/src/control-layer/red_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/red_light_behavior.cpp
@@ -20,7 +20,7 @@ namespace caf::cuda {
 	   //send a request to change behavior to green light after 5 seconds
     	   anon_mail(green_light)
 		   .delay(std::chrono::seconds(5))
-		   .send(state -> self);	
+		   .send(state -> self);
     }
 
 } // namespace caf::cuda

From 5eb8a128ef9f8348ed09e4399cb0cdd4c4924b44 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 11:05:30 -0600
Subject: [PATCH 0088/1000] Made it so upon initialization init is called on
 the first behavior.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 0bfd095afe..63b1ed4d6f 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -26,6 +26,7 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self)
 
     // default behavior
     self->state().current_behavior = self->state().table.get(behavior_token("green"));
+    self->state().current_behavior -> init(&self->state());
 
     return {
         [=](const token_ptr& tok) {

From 81eca2bb2936f42eb8b89b5536537fcfbf46ef58 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 11:12:05 -0600
Subject: [PATCH 0089/1000] Updated reference counting to actually work again.

---
 libcaf_cuda/caf/cuda/control-layer/token.hpp | 28 +++++++++++---------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index 224dde049b..9691bf9ee9 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -19,7 +19,7 @@ class CAF_CUDA_EXPORT token : public caf::ref_counted {
 public:
    virtual ~token() {
     // Print ref count when destructor runs
-    size_t count = ref_count.load(std::memory_order_acquire);
+    size_t count = ref_count_.load(std::memory_order_acquire);
     std::cout << "token object getting deleted, ref_count = " << count << "\n";
 }
 
@@ -29,22 +29,24 @@ class CAF_CUDA_EXPORT token : public caf::ref_counted {
     virtual int getType() const {return -1;}
 
 protected:
- mutable std::atomic<size_t> ref_count{0}; // start at 1 for make_counted
 
-friend  void intrusive_ptr_add_ref(const token* p) noexcept {
-    std::cout << "[add_ref] old count = " << p->ref_count << "\n";
-    ++p->ref_count;
-    std::cout << "[add_ref] new count = " << p->ref_count << "\n";
-}
+ friend void intrusive_ptr_add_ref(token* p) noexcept {
+    p->ref_count_.fetch_add(1, std::memory_order_relaxed);
+  }
+
+  friend void intrusive_ptr_release(token* p) noexcept {
+    if (p->ref_count_.fetch_sub(1, std::memory_order_acq_rel) == 1)
+      delete p;
+  
+  }
+
+private:
+ mutable std::atomic<size_t> ref_count_{1};
+
 
-friend void intrusive_ptr_release(const token* p) noexcept {
-    std::cout << "[release] old count = " << p->ref_count << "\n";
-    --p->ref_count;
-    std::cout << "[release] new count = " << p->ref_count << "\n";
-    if (p->ref_count == 0) std::cout << "token object getting deleted\n";
-}
 };
 
+
 using token_ptr = caf::intrusive_ptr<token>;
 
 } // namespace caf::cuda

From 6b74a97c3d74d2daa245afdec56ba81022100110 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 11:15:22 -0600
Subject: [PATCH 0090/1000] Updated token ref count to start at zero so that it
 would delete correctly as in the previous implementation it would trigger a
 memory leak since tokens wouldnt be deleted.

---
 libcaf_cuda/caf/cuda/control-layer/token.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index 9691bf9ee9..d0360533ca 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -41,7 +41,7 @@ class CAF_CUDA_EXPORT token : public caf::ref_counted {
   }
 
 private:
- mutable std::atomic<size_t> ref_count_{1};
+ mutable std::atomic<size_t> ref_count_{0};
 
 
 };

From 785e930e71e3db166b77bf8f98e4f72fdceabfe7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 11:19:24 -0600
Subject: [PATCH 0091/1000] Added a test mem ref factory function, leaving it
 for now and should be deleted later.

---
 libcaf_cuda/caf/cuda/control-layer/token_factory.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
index c6d607dbe7..66f2b78827 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
@@ -7,6 +7,7 @@
 #include "caf/cuda/control-layer/behavior_token.hpp"
 #include <caf/actor.hpp>
 #include <string>
+#include "caf/cuda/mem_ref.hpp"
 
 namespace caf::cuda {
 
@@ -24,5 +25,6 @@ CAF_CUDA_EXPORT token_ptr make_launch_response_token(actor scheduler_or_proxy,
 
 /// Creates a behavior_token (special — returns its own strong ptr type)
 CAF_CUDA_EXPORT behavior_token_ptr make_behavior_token(std::string name);
+CAF_CUDA_EXPORT mem_ptr<int> make_mem_ptr(size_t num_elements);
 
 } // namespace caf::cuda

From 1def0cf319769118e84c1645ea984c4c3cd881a6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 11:19:50 -0600
Subject: [PATCH 0092/1000] Added a test mem ref factory function, leaving it
 for now and should be deleted later.

---
 libcaf_cuda/src/control-layer/token_factory.cpp | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/token_factory.cpp b/libcaf_cuda/src/control-layer/token_factory.cpp
index 614a8a3ce5..1e67a40c52 100644
--- a/libcaf_cuda/src/control-layer/token_factory.cpp
+++ b/libcaf_cuda/src/control-layer/token_factory.cpp
@@ -31,5 +31,19 @@ behavior_token_ptr make_behavior_token(std::string name)
     return behavior_token_ptr(new behavior_token(std::move(name)));
 }
 
+/// Factory function: create a mem_ptr<int> with fake data
+ mem_ptr<int> make_mem_ptr(size_t num_elements = 16) {
+    if (num_elements == 1) {
+        // scalar test
+        return mem_ptr<int>(new mem_ref<int>(42, /*access=*/0));
+    } else {
+        // create a device-like mem_ref with scalar backing for simplicity
+        // normally this would allocate GPU memory, here just fake values
+        auto ptr = mem_ptr<int>(new mem_ref<int>(0, /*CUdeviceptr*/0, /*access=*/0));
+        return ptr;
+    }
+}
+
+
 } // namespace caf::cuda
 

From d2aeb0dd21c96d8ed20c53cf3b84d984158f627c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 11:20:23 -0600
Subject: [PATCH 0093/1000] No major changes.

---
 libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
index b999bdfc30..d72d868608 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
@@ -6,6 +6,7 @@
 #include "caf/cuda/control-layer/behavior_table.hpp"
 #include "caf/cuda/control-layer/behavior_token.hpp"
 #include <queue>
+#include "caf/cuda/global_export.hpp"
 
 /*
  * The scheduler actor 
@@ -13,5 +14,5 @@
  */
 
 namespace caf::cuda {
-caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state> * self);
+caf::behavior CAF_CUDA_EXPORT scheduler_actor(caf::stateful_actor<scheduler_actor_state> * self);
 }//namespace caf::cuda

From 06368e30de209087e0e4449ef8f9216ee6524268 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 14 Dec 2025 11:20:41 -0600
Subject: [PATCH 0094/1000] Red light green light scheduling test passes.

---
 .../control-layer-tests/red-ligh-green-light/main.test.cpp    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index fd95dcb5ff..785d85f662 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -305,8 +305,8 @@ void caf_main(caf::actor_system& sys) {
 
 	 //caf::init_global_meta_objects<caf::id_block::cuda_control>();
 
-	sys.spawn(mmul_actor_fun2);
-  //run_mmul_test(sys,100,1);
+//	sys.spawn(mmul_actor_fun2);
+  run_mmul_test(sys,100,1);
 }
 
 

From 30d7b2358150fd80bb71c4a71abd26b641901bd2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 6 Jan 2026 13:29:08 -0600
Subject: [PATCH 0095/1000] Added a shutdown to the test.

---
 .../red-ligh-green-light/main.test.cpp                    | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index 785d85f662..d7e58e5450 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -171,7 +171,9 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
     return;
   }
 
-  while (true) {
+  int limit = 5;
+
+  for (int i = 0; i < limit; i++) {
   // Spawn num_actors actors running the mmul behavior
   std::vector<caf::actor> actors;
   actors.reserve(num_actors);
@@ -182,6 +184,10 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
   sleep(1);
   }
 
+  std::cout << "Beginning Shutdown in 2 seconds\n";
+  sleep(2);
+  caf::cuda::manager::shutdown();
+
   //caf::anon_mail(matrix_size, actors).send(actors[0]);
 
    sys.await_all_actors_done();

From e463e3a52fe3464485aadcaad8511d593b47bcec Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 6 Jan 2026 13:39:21 -0600
Subject: [PATCH 0096/1000] Commented out print statement on deconstructor.

---
 libcaf_cuda/caf/cuda/control-layer/token.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index d0360533ca..a64336b5d2 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -20,7 +20,7 @@ class CAF_CUDA_EXPORT token : public caf::ref_counted {
    virtual ~token() {
     // Print ref count when destructor runs
     size_t count = ref_count_.load(std::memory_order_acquire);
-    std::cout << "token object getting deleted, ref_count = " << count << "\n";
+    //std::cout << "token object getting deleted, ref_count = " << count << "\n";
 }
 
     //should only be used by caf's type id system

From adbcd5614ba212ccffda27378cd9432ac78f6bb8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 6 Jan 2026 13:45:09 -0600
Subject: [PATCH 0097/1000] Removed redundant print statements.

---
 libcaf_cuda/src/control-layer/green_light_behavior.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index 550d697157..8a1980d2e7 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -10,7 +10,7 @@ namespace caf::cuda {
 
     void green_light_behavior::receive(scheduler_actor_state* state, const token_ptr& tok)    {
          
-	    std::cout << "Hello from green light processing token\n";
+	    //std::cout << "Hello from green light processing token\n";
 
 	caf::cuda::launch_token& launch = static_cast<caf::cuda::launch_token&>(*tok);
 
@@ -21,7 +21,7 @@ namespace caf::cuda {
       anon_mail(r).send(launch.getReplyActor());
 
 
-      std::cout << "GREEN LIGHT DONE PROCESSING TOKEN\n";
+      //std::cout << "GREEN LIGHT DONE PROCESSING TOKEN\n";
 
 	    // flush the queue
     while (!state->queue.empty()) {

From 2bd1c752d5532b45ecd2d226c436c88a5175aeb6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 6 Jan 2026 14:00:07 -0600
Subject: [PATCH 0098/1000] Made some modifications to the test.

---
 .../red-ligh-green-light/main.test.cpp        | 28 +++++++++++--------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index d7e58e5450..8a9c496e1c 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -94,7 +94,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
 
 	  [=] (caf::cuda::token_ptr launch_response_token) {
 	 
-		  std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
 		  //assume N = 1024
 		  int N = self -> state().N;
 		  std::vector<int> matrix1(N*N);
@@ -102,7 +102,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
 		  std::vector<int> matrix2(N*N);
 		  matrix2.reserve(N);
 
-		  std::cout << "GPU ACTOR sending data to compute\n";
+		  //std::cout << "GPU ACTOR sending data to compute\n";
 		  self -> mail(matrix1,matrix2,N).send(self);
 	 
 		 //token should drop out of scope now, triggering a response 
@@ -110,11 +110,11 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
 	  },
 
     // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const std::vector<int> matrixA,
-        const std::vector<int> matrixB, int N) {
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB, int N) {
  
 
-		  std::cout << "GPU ACTOR  computing\n";
+		  //std::cout << "GPU ACTOR  computing\n";
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 
   //create program and dims   
@@ -140,7 +140,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
 
     // 3rd handler: CPU atom + matrices + N
     [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB, const std::vector<int> matrixC, int N) {
+        const std::vector<int>& matrixB, const std::vector<int>& matrixC, int N) {
        
 		  std::cout << "GPU ACTOR  verfiying\n";
 	 std::vector<int> result(N*N);
@@ -171,7 +171,7 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
     return;
   }
 
-  int limit = 5;
+  int limit = 1;
 
   for (int i = 0; i < limit; i++) {
   // Spawn num_actors actors running the mmul behavior
@@ -180,12 +180,16 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
   for (int i = 0; i < num_actors; ++i) {
     actors.push_back(sys.spawn(mmul_actor_fun));
   }
-  
-  sleep(1);
+ 
+  std::cout << "Sleeping\n"; 
+  std::this_thread::sleep_for(std::chrono::seconds(1));
+  std::cout << "Done Sleeping\n"; 
   }
 
-  std::cout << "Beginning Shutdown in 2 seconds\n";
-  sleep(2);
+  std::cout << "Beginning Shutdown in 10 seconds\n";
+  std::cout << "Sleeping\n"; 
+  std::this_thread::sleep_for(std::chrono::seconds(10));
+  std::cout << "Done Sleeping\n"; 
   caf::cuda::manager::shutdown();
 
   //caf::anon_mail(matrix_size, actors).send(actors[0]);
@@ -312,7 +316,7 @@ void caf_main(caf::actor_system& sys) {
 	 //caf::init_global_meta_objects<caf::id_block::cuda_control>();
 
 //	sys.spawn(mmul_actor_fun2);
-  run_mmul_test(sys,100,1);
+  run_mmul_test(sys,100,10);
 }
 
 

From d251ca3da6fc68de1a149ae057f90e0c88eb0ae9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 6 Jan 2026 14:00:35 -0600
Subject: [PATCH 0099/1000] Removed print statement.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 63b1ed4d6f..6f9547577f 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -46,7 +46,7 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self)
 	    }
         },
 	[=](std::string word) {
-		std::cout << "Received message " << word << "\n";
+		//std::cout << "Received message " << word << "\n";
 	},
 	 [=](caf::cuda::mem_ptr<int> token) {
             if (!token) {

From 9f8740b8855852ce4dd3245800924368bdce7b9e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 6 Jan 2026 14:12:43 -0600
Subject: [PATCH 0100/1000] Initial commit.

---
 .../control-layer/memory_transfer_token.hpp   | 41 +++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
new file mode 100644
index 0000000000..e85d4ae24b
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
@@ -0,0 +1,41 @@
+#include "caf/cuda/control-layer/token.hpp"
+
+
+/*
+ * This token represents transfer of memory 
+ * from either host or device 
+ * all size should be in bytes
+ */
+
+
+//direction defines
+#define H2D 1
+#define D2H 2
+
+
+namespace caf::cuda {
+
+class memory_transfer_token : token {
+
+	public:
+		memory_transfer_token(int size,int direction):
+			size_(size),
+			direction_(direction) {}
+
+		//only here to ensure that caf can copy the object for message
+		//passing do not use
+		memory_transfer_token() = default;
+
+		virtual int getType() const {return MEMORY;}
+		int getSize() const {return size_;}
+		int direction const {return direction_;}
+	private:
+		int size_;
+		int direction_; 
+
+
+}//memory transfer token class
+
+
+}//caf cuda namespace
+

From 9dd3dcb10541f9cb609321d2e9dee7320fee7863 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 6 Jan 2026 14:15:30 -0600
Subject: [PATCH 0101/1000] Updated all to include memory transfer token and
 enable message passing with it.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 5dac47b8af..e10a3c74e7 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -7,6 +7,7 @@
 #include "caf/cuda/control-layer/launch_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 #include "caf/cuda/control-layer/behavior_token.hpp"
+#include "caf/cuda/control-layer/memory_transfer_token.hpp"
 
 #include "caf/cuda/control-layer/behavior.hpp"
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
@@ -22,6 +23,7 @@ CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::behavior_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_response_token>))
+CAF_ADD_TYPE_ID(cuda_control, caf::instrusive_ptr<caf::cuda::memory_transfer_token>)
 
 CAF_END_TYPE_ID_BLOCK(cuda_control)
 
@@ -33,9 +35,10 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::behavior_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_response_token)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_transfer_token)
 
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::behavior_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_response_token>)
-
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::instrusive_ptr<caf::cuda::memory_transfer_token>)

From 56b00a0f485e9ca32f467eb3978127c25a270152 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 10:27:43 -0600
Subject: [PATCH 0102/1000] Added a memory token type id.

---
 libcaf_cuda/caf/cuda/control-layer/token.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index a64336b5d2..954910dd8f 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -9,7 +9,7 @@
 #define LAUNCH 1
 #define LAUNCH_RESPONSE 2
 #define BEHAVIOR 3
-
+#define MEMORY 4
 
 
 namespace caf::cuda {

From f308715957934360c224a65f16b9b69d09ab11ed Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 10:34:10 -0600
Subject: [PATCH 0103/1000] Implemented an exit actor to coordinate shutdown.

---
 .../red-ligh-green-light/main.test.cpp        | 32 +++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index 8a9c496e1c..79dc645891 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -11,6 +11,8 @@
 #include <random>
 #include <unistd.h>
 #include "caf/actor_registry.hpp"
+#include <chrono>
+#include <iostream>
 //#include <caf/atoms.hpp>
 
 
@@ -18,14 +20,40 @@
 using namespace caf;
 using namespace std::chrono_literals;
 
+
+struct exit_actor_state {
+	int completed = 0;
+};
+
+
+caf::behavior exit_actor(caf::stateful_actor<exit_actor_state>* self,int limit) {
+
+
+	return {
+		[=](int num_completed) {
+			self->state().completed += num_completed;
+			if (self->state().completed >= limit) {
+			
+				caf::cuda::manager::shutdown();
+				self->quit();
+			}
+		}
+	};
+
+
+}
+
+
+
+
+
+
 // Define a custom type ID block for custom actors
 CAF_ADD_ATOM(cuda,shared_mem)
 
 
 
 
-#include <chrono>
-#include <iostream>
 
 // Extend your actor state to keep the start time
 struct mmul_actor_state {

From 530b2d0130563ea96dbae7b8e54ef8fa41401227 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 10:36:02 -0600
Subject: [PATCH 0104/1000] Changed diretion to getDirection() to fix syntax
 error.

---
 libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
index e85d4ae24b..50befb3c2a 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
@@ -28,7 +28,7 @@ class memory_transfer_token : token {
 
 		virtual int getType() const {return MEMORY;}
 		int getSize() const {return size_;}
-		int direction const {return direction_;}
+		int getDirection() const {return direction_;}
 	private:
 		int size_;
 		int direction_; 

From 7b4becd793fb0f2ab77cd3c56a8586e007279583 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 10:36:42 -0600
Subject: [PATCH 0105/1000] Added semicolon to fix syntax error.

---
 libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
index 50befb3c2a..b1d97b1cf5 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
@@ -34,7 +34,7 @@ class memory_transfer_token : token {
 		int direction_; 
 
 
-}//memory transfer token class
+};//memory transfer token class
 
 
 }//caf cuda namespace

From a2913a75b52b40f9905f2c9d12fcc9d9a8db51a3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 10:43:01 -0600
Subject: [PATCH 0106/1000] Fixed typos.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index e10a3c74e7..5c05540c67 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -23,7 +23,7 @@ CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::behavior_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_response_token>))
-CAF_ADD_TYPE_ID(cuda_control, caf::instrusive_ptr<caf::cuda::memory_transfer_token>)
+CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_transfer_token>))
 
 CAF_END_TYPE_ID_BLOCK(cuda_control)
 
@@ -41,4 +41,4 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::behavior_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_response_token>)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::instrusive_ptr<caf::cuda::memory_transfer_token>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::memory_transfer_token>)

From 6ab6f3141952bf6c26a0af340ee49db263b11a3b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 10:49:29 -0600
Subject: [PATCH 0107/1000] Fixed syntax error reguarding inheritance scope.:

---
 libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
index b1d97b1cf5..a9a8a52805 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
@@ -15,7 +15,7 @@
 
 namespace caf::cuda {
 
-class memory_transfer_token : token {
+class CAF_CUDA_EXPORT memory_transfer_token : public token {
 
 	public:
 		memory_transfer_token(int size,int direction):

From 77c2c59105c89b97bacd45d7c81fb2cd468e0978 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 10:52:26 -0600
Subject: [PATCH 0108/1000] Removed mmul_actor_func2 This change was made since
 the bug in the code was found it no longer needs to exist and thus pollutes
 the codebase.

---
 .../red-ligh-green-light/main.test.cpp        | 108 ------------------
 1 file changed, 108 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index 79dc645891..6e6371568d 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -224,114 +224,6 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
 
    sys.await_all_actors_done();
 }
-// Stateful actor behavior
-// tests if mem_ptr can be sent correctly
-caf::behavior mmul_actor_fun2(caf::stateful_actor<mmul_actor_state>* self) {
-  
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-/*		
-  caf::cuda::token_ptr bad_launch_token = caf::cuda::make_launch_token(self ->state().program,
-			self -> state().dims,
-			0,
-			"hello",
-			self
-			);
-
-	self -> mail(bad_launch_token).send(scheduler);
-			*/
-
-  	caf::actor scheduler = mgr.get_scheduler_actor();
-	caf::cuda::mem_ptr<int> launch_token = caf::cuda::make_mem_ptr(16);
-			
-	self -> mail(launch_token).send(scheduler);
-
-	self -> mail("Stay alive\n").send(self);
-
-
-
-	return {
-
-		[=] (std::string hello){
-		while(1) {
-	
-		std::cout << "Staying alive\n";
-		sleep(1);
-	}	
-		},
-	  [=] (caf::cuda::token_ptr launch_response_token) {
-	 
-		  std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-		  //assume N = 1024
-		  int N = self -> state().N;
-		  std::vector<int> matrix1(N*N);
-		  matrix1.reserve(N);
-		  std::vector<int> matrix2(N*N);
-		  matrix2.reserve(N);
-
-		  std::cout << "GPU ACTOR sending data to compute\n";
-		  self -> mail(matrix1,matrix2,N).send(self);
-	 
-		 //token should drop out of scope now, triggering a response 
-	  
-	  },
-
-    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const std::vector<int> matrixA,
-        const std::vector<int> matrixB, int N) {
- 
-
-		  std::cout << "GPU ACTOR  computing\n";
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //create program and dims   
-  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    //create args
-    auto arg1 = caf::cuda::create_in_arg(matrixA);
-    auto arg2 = caf::cuda::create_in_arg(matrixB);
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
-
-    auto tempC = mmul.run(program,dims,self -> state().id,arg1,arg2,arg3,arg4);
-    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-		  std::cout << "GPU ACTOR done  computing\n";
-    //verify its own result 
-    self -> mail(matrixA,matrixB,matrixC,N).send(self);
-
-    },
-
-    // 3rd handler: CPU atom + matrices + N
-    [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB, const std::vector<int> matrixC, int N) {
-       
-		  std::cout << "GPU ACTOR  verfiying\n";
-	 std::vector<int> result(N*N);
-
-	 serial_matrix_multiply(matrixA,matrixB,result,N);
-
-	 if (result == matrixC) {
-	 
-		 std::cout << "actor with id " <<  self->state().id << " references match\n";
-	 
-	 }
-
-	 else {
-	    std::cout << "actor with id " <<  self->state().id << " references did not match\n";
-	 }
-
-	 self-> quit();
-
-    }
-  };
-}
-
-
-
 
 
 void caf_main(caf::actor_system& sys) {

From c94c66c20a6014e8e8104426996d640d1ff02d5d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 11:00:40 -0600
Subject: [PATCH 0109/1000] Integrated exit actor within test.

---
 .../red-ligh-green-light/main.test.cpp        | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index 6e6371568d..3f02bd69d4 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -26,12 +26,14 @@ struct exit_actor_state {
 };
 
 
-caf::behavior exit_actor(caf::stateful_actor<exit_actor_state>* self,int limit) {
+caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
 
 
 	return {
 		[=](int num_completed) {
 			self->state().completed += num_completed;
+			
+			std::cout << "Actors finished is " << self->state().completed << "\n";
 			if (self->state().completed >= limit) {
 			
 				caf::cuda::manager::shutdown();
@@ -104,7 +106,7 @@ void serial_matrix_multiply(const std::vector<int>& a,
 
 
 // Stateful actor behavior
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor) {
   
 
   caf::cuda::manager& mgr = caf::cuda::manager::get();
@@ -185,6 +187,8 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
 	    std::cout << "actor with id " <<  self->state().id << " references did not match\n";
 	 }
 
+	 //signal to the exit actor we are quitting and exit
+	 self->mail(1).send(exit_actor);
 	 self-> quit();
 
     }
@@ -201,27 +205,34 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
 
   int limit = 1;
 
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+
   for (int i = 0; i < limit; i++) {
   // Spawn num_actors actors running the mmul behavior
   std::vector<caf::actor> actors;
   actors.reserve(num_actors);
   for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_actor_fun));
+    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor));
   }
- 
+
+ /* 
   std::cout << "Sleeping\n"; 
   std::this_thread::sleep_for(std::chrono::seconds(1));
   std::cout << "Done Sleeping\n"; 
+  */
   }
 
+  /*
   std::cout << "Beginning Shutdown in 10 seconds\n";
   std::cout << "Sleeping\n"; 
   std::this_thread::sleep_for(std::chrono::seconds(10));
   std::cout << "Done Sleeping\n"; 
+  
+  
   caf::cuda::manager::shutdown();
 
   //caf::anon_mail(matrix_size, actors).send(actors[0]);
-
+*/
    sys.await_all_actors_done();
 }
 

From e0d3be94d0282ab6d6d79b3322a4a7f3475facd4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 11:17:58 -0600
Subject: [PATCH 0110/1000] Added an N parameter to the mmul_actor_fun This
 change was made to fix a silent bug where the matrix size would always
 default to 1024x1024 making tunning matrix size impossible.

---
 .../red-ligh-green-light/main.test.cpp        | 76 ++++++++++---------
 1 file changed, 40 insertions(+), 36 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index 3f02bd69d4..069fcca8fe 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -106,8 +106,11 @@ void serial_matrix_multiply(const std::vector<int>& a,
 
 
 // Stateful actor behavior
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor) {
-  
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
+ 
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
 
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 	
@@ -170,26 +173,40 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 
     // 3rd handler: CPU atom + matrices + N
     [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB, const std::vector<int>& matrixC, int N) {
-       
-		  std::cout << "GPU ACTOR  verfiying\n";
-	 std::vector<int> result(N*N);
+    const std::vector<int>& matrixB,
+    const std::vector<int>& matrixC,
+    int N) {
 
-	 serial_matrix_multiply(matrixA,matrixB,result,N);
+  using clock = std::chrono::high_resolution_clock;
 
-	 if (result == matrixC) {
-	 
-		 std::cout << "actor with id " <<  self->state().id << " references match\n";
-	 
-	 }
+  auto start = clock::now();
 
-	 else {
-	    std::cout << "actor with id " <<  self->state().id << " references did not match\n";
-	 }
+  std::cout << "GPU ACTOR verifying\n";
+
+  std::vector<int> result(N * N);
+
+  serial_matrix_multiply(matrixA, matrixB, result, N);
+
+  if (result == matrixC) {
+    std::cout << "actor with id " << self->state().id
+              << " references match\n";
+  } else {
+    std::cout << "actor with id " << self->state().id
+              << " references did not match\n";
+  }
 
-	 //signal to the exit actor we are quitting and exit
-	 self->mail(1).send(exit_actor);
-	 self-> quit();
+  auto end = clock::now();
+
+  auto ms =
+    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[TIMING] verification took "
+            << ms << " ms (actor id "
+            << self->state().id << ")\n";
+
+  // signal exit actor and quit
+  self->mail(1).send(exit_actor);
+  self->quit();
 
     }
   };
@@ -212,27 +229,14 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
   std::vector<caf::actor> actors;
   actors.reserve(num_actors);
   for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor));
+    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
   }
 
- /* 
-  std::cout << "Sleeping\n"; 
-  std::this_thread::sleep_for(std::chrono::seconds(1));
-  std::cout << "Done Sleeping\n"; 
-  */
-  }
 
-  /*
-  std::cout << "Beginning Shutdown in 10 seconds\n";
-  std::cout << "Sleeping\n"; 
-  std::this_thread::sleep_for(std::chrono::seconds(10));
-  std::cout << "Done Sleeping\n"; 
-  
-  
-  caf::cuda::manager::shutdown();
+ // std::cout << actors.size() << "\n";
+
+  }
 
-  //caf::anon_mail(matrix_size, actors).send(actors[0]);
-*/
    sys.await_all_actors_done();
 }
 
@@ -247,7 +251,7 @@ void caf_main(caf::actor_system& sys) {
 	 //caf::init_global_meta_objects<caf::id_block::cuda_control>();
 
 //	sys.spawn(mmul_actor_fun2);
-  run_mmul_test(sys,100,10);
+  run_mmul_test(sys,10,10);
 }
 
 

From 6912b4302b76eb60090f14e2c3cd952955e3554a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 11:24:34 -0600
Subject: [PATCH 0111/1000] Wrote red light green light test and commented out
 redundant print statements.

---
 .../red-ligh-green-light/main.test.cpp        | 48 ++++++++++++++++---
 1 file changed, 41 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index 069fcca8fe..e51d5a818e 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -165,7 +165,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
     auto tempC = mmul.run(program,dims,self -> state().id,arg1,arg2,arg3,arg4);
     std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
 
-		  std::cout << "GPU ACTOR done  computing\n";
+		  //std::cout << "GPU ACTOR done  computing\n";
     //verify its own result 
     self -> mail(matrixA,matrixB,matrixC,N).send(self);
 
@@ -181,7 +181,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 
   auto start = clock::now();
 
-  std::cout << "GPU ACTOR verifying\n";
+  //std::cout << "GPU ACTOR verifying\n";
 
   std::vector<int> result(N * N);
 
@@ -241,17 +241,51 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
 }
 
 
+
+//this test will spawn more actors over time to demonstrate 
+//changing scheduling algorithims at runtime 
+void run_red_light_green_light_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  std::cout << "Starting RED LIGHT GREEN LIGHT TEST\n";
+  int limit = 10;
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+
+  for (int i = 0; i < limit; i++) {
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
+  }
+
+
+  sleep(1);
+ // std::cout << actors.size() << "\n";
+
+  }
+
+   sys.await_all_actors_done();
+}
+
+
+
 void caf_main(caf::actor_system& sys) {
   
 	
 
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	caf::cuda::manager::init(sys,man_config);
-
-	 //caf::init_global_meta_objects<caf::id_block::cuda_control>();
-
-//	sys.spawn(mmul_actor_fun2);
-  run_mmul_test(sys,10,10);
+       	run_mmul_test(sys,10,1000);
+	
+	//tests will delete the old manager so will have to reinit if you do this 
+	//in conjunction with each other	
+	caf::cuda::manager::init(sys,man_config);
+	run_red_light_green_light_test(sys,10,1000);
 }
 
 

From 39aaf46a72a2f23aaf11c5980c1cd0de6e6266ec Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 11:27:23 -0600
Subject: [PATCH 0112/1000] Fixed exit actor parameter on line 256 to fix issue
 where manager would be shutdown prematurely.

---
 .../control-layer-tests/red-ligh-green-light/main.test.cpp  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index e51d5a818e..1caa9f73f0 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -253,7 +253,7 @@ void run_red_light_green_light_test(caf::actor_system& sys, int matrix_size, int
   std::cout << "Starting RED LIGHT GREEN LIGHT TEST\n";
   int limit = 10;
 
-  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors * limit);
 
   for (int i = 0; i < limit; i++) {
   // Spawn num_actors actors running the mmul behavior
@@ -280,11 +280,11 @@ void caf_main(caf::actor_system& sys) {
 
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	caf::cuda::manager::init(sys,man_config);
-       	run_mmul_test(sys,10,1000);
+       	//run_mmul_test(sys,10,1000);
 	
 	//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	
-	caf::cuda::manager::init(sys,man_config);
+	//caf::cuda::manager::init(sys,man_config);
 	run_red_light_green_light_test(sys,10,1000);
 }
 

From db23b87f42543941567e7d72c11d5764e44fa455 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 11:27:55 -0600
Subject: [PATCH 0113/1000] COmmented out redundant print statement.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 6f9547577f..66ecc22c3f 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -32,7 +32,7 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self)
         [=](const token_ptr& tok) {
            
 
-		std::cout << "Received token\n";
+	//	std::cout << "Received token\n";
 		self->state().current_behavior->receive(&self->state(), tok);
         },
         [=](const caf::cuda::behavior_token_ptr& tok) {

From 0424ac52252f3e6bcd96fcdbf6fe34e15c43d729 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 11:36:36 -0600
Subject: [PATCH 0114/1000] Initial commit, Copying everything over from
 red_light_green_light_test.

---
 .../memory_test/CMakeLists.txt                |  44 +++
 .../memory_test/compile_kernels.sh            |  18 ++
 .../memory_test/main.test.cpp                 | 294 ++++++++++++++++++
 .../control-layer-tests/memory_test/mmul.cu   |  16 +
 4 files changed, 372 insertions(+)
 create mode 100644 libcaf_cuda/tests/control-layer-tests/memory_test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/control-layer-tests/memory_test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp
 create mode 100644 libcaf_cuda/tests/control-layer-tests/memory_test/mmul.cu

diff --git a/libcaf_cuda/tests/control-layer-tests/memory_test/CMakeLists.txt b/libcaf_cuda/tests/control-layer-tests/memory_test/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/memory_test/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/control-layer-tests/memory_test/compile_kernels.sh b/libcaf_cuda/tests/control-layer-tests/memory_test/compile_kernels.sh
new file mode 100755
index 0000000000..586196454e
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/memory_test/compile_kernels.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+
+# Compile genMatrix.cu to fatbin in current directory
+#nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
+#echo "Generated generate_random_matrix.fatbin"
+
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp
new file mode 100644
index 0000000000..1caa9f73f0
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp
@@ -0,0 +1,294 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include <unistd.h>
+#include "caf/actor_registry.hpp"
+#include <chrono>
+#include <iostream>
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+struct exit_actor_state {
+	int completed = 0;
+};
+
+
+caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
+
+
+	return {
+		[=](int num_completed) {
+			self->state().completed += num_completed;
+			
+			std::cout << "Actors finished is " << self->state().completed << "\n";
+			if (self->state().completed >= limit) {
+			
+				caf::cuda::manager::shutdown();
+				self->quit();
+			}
+		}
+	};
+
+
+}
+
+
+
+
+
+
+// Define a custom type ID block for custom actors
+CAF_ADD_ATOM(cuda,shared_mem)
+
+
+
+
+
+// Extend your actor state to keep the start time
+struct mmul_actor_state {
+  static inline const char* name = "my_actor";
+  int N = 1024; // example state variable
+  int id = rand(); // an actor id
+  // per-actor timing start
+  std::chrono::high_resolution_clock::time_point start_time;
+  int times = 0;
+  caf::cuda::program_ptr program = caf::cuda::manager::get().create_program_from_cubin("../mmul.cubin","matrixMul");
+  int THREADS = 32;
+  int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS,BLOCKS,1,THREADS,THREADS,THREADS);
+};
+
+
+
+
+//commands classes used to launch kernels 
+using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
+using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
+
+using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
+
+mmulCommand mmul;
+matrixGenCommand randomMatrix;
+mmulAsyncCommand mmulAsync;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
+ 
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+	
+  	caf::actor scheduler = mgr.get_scheduler_actor();
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
+			self -> state().dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+	  [=] (caf::cuda::token_ptr launch_response_token) {
+	 
+		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+		  //assume N = 1024
+		  int N = self -> state().N;
+		  std::vector<int> matrix1(N*N);
+		  matrix1.reserve(N);
+		  std::vector<int> matrix2(N*N);
+		  matrix2.reserve(N);
+
+		  //std::cout << "GPU ACTOR sending data to compute\n";
+		  self -> mail(matrix1,matrix2,N).send(self);
+	 
+		 //token should drop out of scope now, triggering a response 
+	  
+	  },
+
+    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB, int N) {
+ 
+
+		  //std::cout << "GPU ACTOR  computing\n";
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //create program and dims   
+  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+  const int THREADS = 32;
+  const int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    //create args
+    auto arg1 = caf::cuda::create_in_arg(matrixA);
+    auto arg2 = caf::cuda::create_in_arg(matrixB);
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    auto tempC = mmul.run(program,dims,self -> state().id,arg1,arg2,arg3,arg4);
+    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+		  //std::cout << "GPU ACTOR done  computing\n";
+    //verify its own result 
+    self -> mail(matrixA,matrixB,matrixC,N).send(self);
+
+    },
+
+    // 3rd handler: CPU atom + matrices + N
+    [=](const std::vector<int>& matrixA,
+    const std::vector<int>& matrixB,
+    const std::vector<int>& matrixC,
+    int N) {
+
+  using clock = std::chrono::high_resolution_clock;
+
+  auto start = clock::now();
+
+  //std::cout << "GPU ACTOR verifying\n";
+
+  std::vector<int> result(N * N);
+
+  serial_matrix_multiply(matrixA, matrixB, result, N);
+
+  if (result == matrixC) {
+    std::cout << "actor with id " << self->state().id
+              << " references match\n";
+  } else {
+    std::cout << "actor with id " << self->state().id
+              << " references did not match\n";
+  }
+
+  auto end = clock::now();
+
+  auto ms =
+    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[TIMING] verification took "
+            << ms << " ms (actor id "
+            << self->state().id << ")\n";
+
+  // signal exit actor and quit
+  self->mail(1).send(exit_actor);
+  self->quit();
+
+    }
+  };
+}
+
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  int limit = 1;
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+
+  for (int i = 0; i < limit; i++) {
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
+  }
+
+
+ // std::cout << actors.size() << "\n";
+
+  }
+
+   sys.await_all_actors_done();
+}
+
+
+
+//this test will spawn more actors over time to demonstrate 
+//changing scheduling algorithims at runtime 
+void run_red_light_green_light_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  std::cout << "Starting RED LIGHT GREEN LIGHT TEST\n";
+  int limit = 10;
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors * limit);
+
+  for (int i = 0; i < limit; i++) {
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
+  }
+
+
+  sleep(1);
+ // std::cout << actors.size() << "\n";
+
+  }
+
+   sys.await_all_actors_done();
+}
+
+
+
+void caf_main(caf::actor_system& sys) {
+  
+	
+
+	caf::cuda::manager_config man_config(true); //turns the scheduler on
+	caf::cuda::manager::init(sys,man_config);
+       	//run_mmul_test(sys,10,1000);
+	
+	//tests will delete the old manager so will have to reinit if you do this 
+	//in conjunction with each other	
+	//caf::cuda::manager::init(sys,man_config);
+	run_red_light_green_light_test(sys,10,1000);
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/control-layer-tests/memory_test/mmul.cu b/libcaf_cuda/tests/control-layer-tests/memory_test/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/memory_test/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From 3b0ead253a14052986569f220cbe03098f1cb559 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 11:39:30 -0600
Subject: [PATCH 0115/1000] Added a stub for factory of memory token.

---
 libcaf_cuda/caf/cuda/control-layer/token_factory.hpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
index 66f2b78827..8ba9eff159 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
@@ -25,6 +25,15 @@ CAF_CUDA_EXPORT token_ptr make_launch_response_token(actor scheduler_or_proxy,
 
 /// Creates a behavior_token (special — returns its own strong ptr type)
 CAF_CUDA_EXPORT behavior_token_ptr make_behavior_token(std::string name);
+
+//creats a memory transfer token
+CAF_CUDA_EXPORT token_ptr make_memory_token(int size, int direction);
+
+
+//do not use this, for testing only 
 CAF_CUDA_EXPORT mem_ptr<int> make_mem_ptr(size_t num_elements);
 
+
+
+
 } // namespace caf::cuda

From 5a7b81b5f8e552505c924241a7fb38d6478a0435 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 11:41:46 -0600
Subject: [PATCH 0116/1000] Implemented factory function for memory transfer
 requests.

---
 libcaf_cuda/src/control-layer/token_factory.cpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/token_factory.cpp b/libcaf_cuda/src/control-layer/token_factory.cpp
index 1e67a40c52..65cb449935 100644
--- a/libcaf_cuda/src/control-layer/token_factory.cpp
+++ b/libcaf_cuda/src/control-layer/token_factory.cpp
@@ -31,6 +31,15 @@ behavior_token_ptr make_behavior_token(std::string name)
     return behavior_token_ptr(new behavior_token(std::move(name)));
 }
 
+
+token_ptr make_memory_token(int size,int direction) {
+
+	return token_ptr(new memory_transfer_token(size,direction));
+
+}
+
+
+
 /// Factory function: create a mem_ptr<int> with fake data
  mem_ptr<int> make_mem_ptr(size_t num_elements = 16) {
     if (num_elements == 1) {

From 35d877c30323d24dfece55cc2bf8924bd474bef6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 11:44:25 -0600
Subject: [PATCH 0117/1000] Added pragma once.

---
 libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
index a9a8a52805..3db00c457c 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
@@ -1,3 +1,4 @@
+#pragma once
 #include "caf/cuda/control-layer/token.hpp"
 
 

From 1306efcd536cc0b59c6873b313e42dfbe982d90f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 7 Jan 2026 11:45:21 -0600
Subject: [PATCH 0118/1000] Added memory_transfer token include.

---
 libcaf_cuda/caf/cuda/control-layer/token_factory.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
index 8ba9eff159..94765321f3 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
@@ -5,6 +5,7 @@
 #include "caf/cuda/control-layer/launch_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 #include "caf/cuda/control-layer/behavior_token.hpp"
+#include "caf/cuda/control-layer/memory_transfer_token.hpp"
 #include <caf/actor.hpp>
 #include <string>
 #include "caf/cuda/mem_ref.hpp"

From 4fae055a5ce3cee49f0a31b893ce53b5b3f35e90 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 13:40:21 -0600
Subject: [PATCH 0119/1000] Added 2 helper stubs to reduce boilerplate code
 from exisiting and so that it can be in once method.

---
 libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
index 2469262a5b..c24ea89624 100644
--- a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
@@ -9,6 +9,8 @@ class green_light_behavior : public scheduler_actor_behavior {
 
     void receive(scheduler_actor_state* state, const token_ptr& tok) override; 
     void init(scheduler_actor_state* state); 
+    void process_launch_token(const token_ptr& tok);
+    void process_memory_transfer_token(const token_ptr& tok);
 };
 
 

From 7400d10e3e6c0df41356fe5ac857090aeb31c705 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 13:47:02 -0600
Subject: [PATCH 0120/1000] Added a reply Actor attribute and getter method, as
 well as updated constructor to include this this change was made since the
 scheduler actor needs someone to send a response token to.

---
 .../caf/cuda/control-layer/memory_transfer_token.hpp       | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
index 3db00c457c..8a4bb381aa 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
@@ -19,9 +19,10 @@ namespace caf::cuda {
 class CAF_CUDA_EXPORT memory_transfer_token : public token {
 
 	public:
-		memory_transfer_token(int size,int direction):
+		memory_transfer_token(int size,int direction,caf::actor replyActor):
 			size_(size),
-			direction_(direction) {}
+			direction_(direction),
+       			replyActor_(replyActor)	{}
 
 		//only here to ensure that caf can copy the object for message
 		//passing do not use
@@ -30,9 +31,11 @@ class CAF_CUDA_EXPORT memory_transfer_token : public token {
 		virtual int getType() const {return MEMORY;}
 		int getSize() const {return size_;}
 		int getDirection() const {return direction_;}
+		caf::actor getReplyActor() const {return replyActor_;}
 	private:
 		int size_;
 		int direction_; 
+		caf::actor replyActor_;
 
 
 };//memory transfer token class

From 42c1f59b14e12c281ef93d8c1710695eb2f4407a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 13:47:32 -0600
Subject: [PATCH 0121/1000] changed stub to reflect changes in memory transfer
 token constructor.

---
 libcaf_cuda/caf/cuda/control-layer/token_factory.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
index 94765321f3..ea23800162 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
@@ -28,7 +28,7 @@ CAF_CUDA_EXPORT token_ptr make_launch_response_token(actor scheduler_or_proxy,
 CAF_CUDA_EXPORT behavior_token_ptr make_behavior_token(std::string name);
 
 //creats a memory transfer token
-CAF_CUDA_EXPORT token_ptr make_memory_token(int size, int direction);
+CAF_CUDA_EXPORT token_ptr make_memory_token(int size, int direction,caf::actor replyActor);
 
 
 //do not use this, for testing only 

From ec0ddbbe3707666be72ede3fe4e92741c386bf52 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 13:48:18 -0600
Subject: [PATCH 0122/1000] Changed make_memory_token to reflect changes made
 in its constructor.

---
 libcaf_cuda/src/control-layer/token_factory.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/token_factory.cpp b/libcaf_cuda/src/control-layer/token_factory.cpp
index 65cb449935..f7754255a2 100644
--- a/libcaf_cuda/src/control-layer/token_factory.cpp
+++ b/libcaf_cuda/src/control-layer/token_factory.cpp
@@ -32,9 +32,9 @@ behavior_token_ptr make_behavior_token(std::string name)
 }
 
 
-token_ptr make_memory_token(int size,int direction) {
+token_ptr make_memory_token(int size,int direction,caf::actor replyActor) {
 
-	return token_ptr(new memory_transfer_token(size,direction));
+	return token_ptr(new memory_transfer_token(size,direction,replyActor));
 
 }
 

From e79193f1af772efaa9617807f08aea448061a0ab Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 13:53:14 -0600
Subject: [PATCH 0123/1000] Updated stubs to include self parameter.

---
 libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
index c24ea89624..4d4b070785 100644
--- a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
@@ -9,8 +9,8 @@ class green_light_behavior : public scheduler_actor_behavior {
 
     void receive(scheduler_actor_state* state, const token_ptr& tok) override; 
     void init(scheduler_actor_state* state); 
-    void process_launch_token(const token_ptr& tok);
-    void process_memory_transfer_token(const token_ptr& tok);
+    void process_launch_token(const token_ptr& tok,caf::actor self);
+    void process_memory_transfer_token(const token_ptr& tok, caf::actor self);
 };
 
 

From 3fd0579e60596a51e3a3931c5f80a229088f7857 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 13:56:33 -0600
Subject: [PATCH 0124/1000] Implemented process_launch_token helper method.

---
 .../control-layer/green_light_behavior.cpp    | 53 +++++++++++++++----
 1 file changed, 43 insertions(+), 10 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index 8a1980d2e7..c4ff28d17e 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -10,8 +10,8 @@ namespace caf::cuda {
 
     void green_light_behavior::receive(scheduler_actor_state* state, const token_ptr& tok)    {
          
-	    //std::cout << "Hello from green light processing token\n";
 
+	if (tok -> getType() == LAUNCH) {
 	caf::cuda::launch_token& launch = static_cast<caf::cuda::launch_token&>(*tok);
 
       // create the response using a reference to the existing object
@@ -19,24 +19,32 @@ namespace caf::cuda {
 
       // send response to the reply actor stored in launch_token
       anon_mail(r).send(launch.getReplyActor());
+	
+     }
 
+	else if (tok -> getType() == MEMORY) {
+	
 
-      //std::cout << "GREEN LIGHT DONE PROCESSING TOKEN\n";
+	
+	}
 
-	    // flush the queue
+	//this may cause an issue if a message is never received then 
+	//we may never end up dequeueing certain requests 
+	//may lead to a deadlock scenario?
     while (!state->queue.empty()) {
         token_ptr queued = state->queue.front();
         state->queue.pop();
 
- if (queued->getType() == LAUNCH) {
-      // safe: we've checked the runtime type
-      caf::cuda::launch_token& lt = static_cast<caf::cuda::launch_token&>(*queued);
+ 
+	if (queued->getType() == LAUNCH) {
+      	// safe: we've checked the runtime type
+      	caf::cuda::launch_token& lt = static_cast<caf::cuda::launch_token&>(*queued);
 
-      // create the response using a reference to the existing object
-      caf::cuda::token_ptr response = make_launch_response_token(state->self, lt);
+      	// create the response using a reference to the existing object
+      	caf::cuda::token_ptr response = make_launch_response_token(state->self, lt);
 
-      // send response to the reply actor stored in launch_token
-      anon_mail(response).send(lt.getReplyActor());
+      	// send response to the reply actor stored in launch_token
+      	anon_mail(response).send(lt.getReplyActor());
     }
   }
 
@@ -56,6 +64,31 @@ namespace caf::cuda {
 	   
     }
 
+//--------------------------------------------------
+// process_launch_token
+//--------------------------------------------------
+void green_light_behavior::process_launch_token(const token_ptr& tok,caf::actor self) {
+    caf::cuda::launch_token& launch =
+        static_cast<caf::cuda::launch_token&>(*tok);
+
+    // Create response token using the existing launch token
+    caf::cuda::token_ptr response =
+        make_launch_response_token(self, launch);
+
+    // Send response to the actor that requested the launch
+    anon_mail(response).send(launch.getReplyActor());
+}
+
+//--------------------------------------------------
+// process_memory_transfer_token
+//--------------------------------------------------
+void green_light_behavior::process_memory_transfer_token(const token_ptr& tok,caf::actor self) {
+    caf::cuda::memory_transfer_token& mem =
+        static_cast<caf::cuda::memory_transfer_token&>(*tok);
+
+    // For now, green light allows immediate forwarding
+    anon_mail(tok).send(mem.getReplyActor());
+}
 
 
 } // namespace caf::cuda

From 94400d8a058d124030c64dbda18bb41d2418fbcd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 14:05:24 -0600
Subject: [PATCH 0125/1000] Implemented memory_response_token.

---
 .../control-layer/memory_response_token.hpp   | 71 +++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp
new file mode 100644
index 0000000000..25099a3aa2
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp
@@ -0,0 +1,71 @@
+#pragma once
+
+//this class is meant to act as a response issued by the scheduler
+//actor authorizing memory transfer as needed 
+
+#pragma once
+
+#include "caf/cuda/control-layer/token.hpp"
+#include "caf/cuda/control-layer/memory_transfer_token.hpp"
+#include "caf/cuda/global_export.hpp"
+
+#include <caf/actor.hpp>
+#include <atomic>
+
+namespace caf::cuda {
+
+class CAF_CUDA_EXPORT memory_response_token : public token {
+public:
+    // Required by CAF – do not use directly
+    memory_response_token() = default;
+
+    // Construct from memory_transfer_token
+    memory_response_token(caf::actor receiver,
+                          const memory_transfer_token& token)
+        : receiver_(std::move(receiver)),
+          size_(token.getSize()),
+          direction_(token.getDirection()),
+          released_(false) {}
+
+    ~memory_response_token() {
+        release();
+    }
+
+    int getType() const override {
+        return MEMORY_RESPONSE;
+    }
+
+    int getSize() const {
+        return size_;
+    }
+
+    int getDirection() const {
+        return direction_;
+    }
+
+    void release() {
+        bool expected = false;
+
+        // ONLY send if we successfully transition false → true
+        if (!released_.compare_exchange_strong(expected, true)) {
+            return; // already released → do nothing
+        }
+
+        // Real message (commented for testing)
+        // caf::anon_mail(size_, direction_).urgent().send(receiver_);
+
+        // Test message
+        caf::anon_mail("Hello world from memory response")
+            .urgent()
+            .send(receiver_);
+    }
+
+private:
+    caf::actor receiver_;
+    int size_;
+    int direction_;
+    std::atomic<bool> released_;
+};
+
+} // namespace caf::cuda
+

From a71ee8a809bc0f7dbcc99b2e9310ced73a40a313 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 14:07:32 -0600
Subject: [PATCH 0126/1000] Implemented process_memory token helper.

---
 .../src/control-layer/green_light_behavior.cpp       | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index c4ff28d17e..1e45338c9e 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -82,14 +82,20 @@ void green_light_behavior::process_launch_token(const token_ptr& tok,caf::actor
 //--------------------------------------------------
 // process_memory_transfer_token
 //--------------------------------------------------
-void green_light_behavior::process_memory_transfer_token(const token_ptr& tok,caf::actor self) {
+void green_light_behavior::process_memory_transfer_token(const token_ptr& tok,
+                                                         caf::actor self) {
     caf::cuda::memory_transfer_token& mem =
         static_cast<caf::cuda::memory_transfer_token&>(*tok);
 
-    // For now, green light allows immediate forwarding
-    anon_mail(tok).send(mem.getReplyActor());
+    // Create memory response token
+    caf::cuda::token_ptr response =
+        make_memory_response_token(self, mem);
+
+    // Send response to requesting actor
+    anon_mail(response).send(mem.getReplyActor());
 }
 
 
+
 } // namespace caf::cuda
 

From da89e9742260d403fea474e0c2709c29a934eb2b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 14:09:23 -0600
Subject: [PATCH 0127/1000] Added a stub to create memory response token.

---
 libcaf_cuda/caf/cuda/control-layer/token_factory.hpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
index ea23800162..7aac2df6bd 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
@@ -6,6 +6,7 @@
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 #include "caf/cuda/control-layer/behavior_token.hpp"
 #include "caf/cuda/control-layer/memory_transfer_token.hpp"
+#include "caf/cuda/control-layer/memory_response_token.hpp"
 #include <caf/actor.hpp>
 #include <string>
 #include "caf/cuda/mem_ref.hpp"
@@ -30,6 +31,10 @@ CAF_CUDA_EXPORT behavior_token_ptr make_behavior_token(std::string name);
 //creats a memory transfer token
 CAF_CUDA_EXPORT token_ptr make_memory_token(int size, int direction,caf::actor replyActor);
 
+CAF_CUDA_EXPORT token_ptr make_memory_response_token(actor receiver,
+                                     const memory_transfer_token& orig);
+
+
 
 //do not use this, for testing only 
 CAF_CUDA_EXPORT mem_ptr<int> make_mem_ptr(size_t num_elements);

From f18ca5edb55d8de50db34094adb7520ef8291419 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 14:11:00 -0600
Subject: [PATCH 0128/1000] Implemented memory transfer token response.

---
 libcaf_cuda/src/control-layer/token_factory.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/token_factory.cpp b/libcaf_cuda/src/control-layer/token_factory.cpp
index f7754255a2..83a622dda6 100644
--- a/libcaf_cuda/src/control-layer/token_factory.cpp
+++ b/libcaf_cuda/src/control-layer/token_factory.cpp
@@ -2,6 +2,7 @@
 #include "caf/cuda/control-layer/launch_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 #include "caf/cuda/control-layer/behavior_token.hpp"
+#include "caf/cuda/control-layer/memory_response_token.hpp"
 
 namespace caf::cuda {
 
@@ -38,6 +39,10 @@ token_ptr make_memory_token(int size,int direction,caf::actor replyActor) {
 
 }
 
+token_ptr make_memory_response_token(actor receiver,
+                                     const memory_transfer_token& orig) {
+    return token_ptr(new memory_response_token(receiver, orig));
+}
 
 
 /// Factory function: create a mem_ptr<int> with fake data

From b9d3ac7f35a9d4c9b5e9f509b1ff3c240a9dc6d1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 14:13:54 -0600
Subject: [PATCH 0129/1000] Updated receive to use helper functions.

---
 .../control-layer/green_light_behavior.cpp    | 54 +++++++------------
 1 file changed, 19 insertions(+), 35 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index 1e45338c9e..ea3e73524f 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -7,49 +7,33 @@ namespace caf::cuda {
     void green_light_behavior::schedule() {
     //TODO IMPLEMENT
     }
+void green_light_behavior::receive(scheduler_actor_state* state,
+                                   const token_ptr& tok) {
 
-    void green_light_behavior::receive(scheduler_actor_state* state, const token_ptr& tok)    {
-         
-
-	if (tok -> getType() == LAUNCH) {
-	caf::cuda::launch_token& launch = static_cast<caf::cuda::launch_token&>(*tok);
-
-      // create the response using a reference to the existing object
-      caf::cuda::token_ptr r = make_launch_response_token(state->self, launch);
-
-      // send response to the reply actor stored in launch_token
-      anon_mail(r).send(launch.getReplyActor());
-	
-     }
-
-	else if (tok -> getType() == MEMORY) {
-	
-
-	
-	}
+    if (tok->getType() == LAUNCH) {
+        process_launch_token(tok, state->self);
+    }
+    else if (tok->getType() == MEMORY) {
+        process_memory_transfer_token(tok, state->self);
+    }
 
-	//this may cause an issue if a message is never received then 
-	//we may never end up dequeueing certain requests 
-	//may lead to a deadlock scenario?
+    //this may cause an issue if a message is never received then 
+    //we may never end up dequeueing certain requests 
+    //may lead to a deadlock scenario?
     while (!state->queue.empty()) {
         token_ptr queued = state->queue.front();
         state->queue.pop();
 
- 
-	if (queued->getType() == LAUNCH) {
-      	// safe: we've checked the runtime type
-      	caf::cuda::launch_token& lt = static_cast<caf::cuda::launch_token&>(*queued);
-
-      	// create the response using a reference to the existing object
-      	caf::cuda::token_ptr response = make_launch_response_token(state->self, lt);
-
-      	// send response to the reply actor stored in launch_token
-      	anon_mail(response).send(lt.getReplyActor());
+        if (queued->getType() == LAUNCH) {
+            // safe: we've checked the runtime type
+            process_launch_token(queued, state->self);
+        }
+        else if (queued->getType() == MEMORY) {
+            process_memory_transfer_token(queued, state->self);
+        }
     }
-  }
-
+}
 
-    }
 
    void green_light_behavior::init(scheduler_actor_state * state) {
 	   std::cout << "GREEN LIGHT\n";

From 63d000b02283fb634cd50359d513216f472e8aac Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 14:23:55 -0600
Subject: [PATCH 0130/1000] Added a memory respone macro.

---
 libcaf_cuda/caf/cuda/control-layer/token.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index 954910dd8f..681a2c3e40 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -10,7 +10,7 @@
 #define LAUNCH_RESPONSE 2
 #define BEHAVIOR 3
 #define MEMORY 4
-
+#define MEMORY_RESPONSE 5
 
 namespace caf::cuda {
 

From 692c51d08a965c2e2da4b3e8794884ec95d4921b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 14:26:15 -0600
Subject: [PATCH 0131/1000] Updated to include memory_response_token.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 5c05540c67..83e8557568 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -8,6 +8,7 @@
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 #include "caf/cuda/control-layer/behavior_token.hpp"
 #include "caf/cuda/control-layer/memory_transfer_token.hpp"
+#include "caf/cuda/control-layer/memory_response_token.hpp"
 
 #include "caf/cuda/control-layer/behavior.hpp"
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
@@ -24,6 +25,7 @@ CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::behavior_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_response_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_transfer_token>))
+CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_response_token>))
 
 CAF_END_TYPE_ID_BLOCK(cuda_control)
 
@@ -36,9 +38,11 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::behavior_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_response_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_transfer_token)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_response_token)
 
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::behavior_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_response_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::memory_transfer_token>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::memory_response_token>)

From 3103bc037c8dbe3c8ab6a16324953da990bfca52 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 14:35:53 -0600
Subject: [PATCH 0132/1000] Updated test to use a memory transfer token.

---
 .../memory_test/main.test.cpp                  | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp
index 1caa9f73f0..d08450d79e 100644
--- a/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp
@@ -115,6 +115,14 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 	
   	caf::actor scheduler = mgr.get_scheduler_actor();
+	
+
+	//send a memory transfer token
+	int bytes = N*N * sizeof(int);
+	caf::cuda::token_ptr memory_token = caf::cuda::make_memory_token(bytes,H2D,self);
+	self -> mail(memory_token).send(scheduler);	
+
+	//send a launch token
 	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
 			self -> state().dims,
 			0,
@@ -125,8 +133,9 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 
 	return {
 
-	  [=] (caf::cuda::token_ptr launch_response_token) {
-	 
+	  [=] (caf::cuda::token_ptr response_token) {
+	
+		 if (response_token -> getType() == LAUNCH_RESPONSE) {
 		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
 		  //assume N = 1024
 		  int N = self -> state().N;
@@ -138,8 +147,11 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 		  //std::cout << "GPU ACTOR sending data to compute\n";
 		  self -> mail(matrix1,matrix2,N).send(self);
 	 
+		 }
+		 else {
+			 std::cout << "Got a memory response token\n";
+		 }
 		 //token should drop out of scope now, triggering a response 
-	  
 	  },
 
     // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification

From 264924a16b44711eca3780e57070a01761860e1f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 14:40:00 -0600
Subject: [PATCH 0133/1000] Memory transfer token test passes in that tokens
 can be message passed and deconstructor will send a message.

---
 .../tests/control-layer-tests/memory_test/main.test.cpp       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp
index d08450d79e..15e3883a1a 100644
--- a/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp
@@ -292,12 +292,12 @@ void caf_main(caf::actor_system& sys) {
 
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	caf::cuda::manager::init(sys,man_config);
-       	//run_mmul_test(sys,10,1000);
+       	run_mmul_test(sys,10,10);
 	
 	//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	
 	//caf::cuda::manager::init(sys,man_config);
-	run_red_light_green_light_test(sys,10,1000);
+	//run_red_light_green_light_test(sys,10,1000);
 }
 
 

From 357972d9c1f9a2c70e018e5b733899fc17ee0573 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 14:40:22 -0600
Subject: [PATCH 0134/1000] Added back test print statement.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 66ecc22c3f..64e0a2697d 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -46,7 +46,7 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self)
 	    }
         },
 	[=](std::string word) {
-		//std::cout << "Received message " << word << "\n";
+		std::cout << "Received message " << word << "\n";
 	},
 	 [=](caf::cuda::mem_ptr<int> token) {
             if (!token) {

From 77dff612dbf52e3f1f786ca85d2b2657023d4501 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 14:50:37 -0600
Subject: [PATCH 0135/1000] Initial commit.

---
 libcaf_cuda/caf/cuda/memory_command.hpp | 50 +++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/memory_command.hpp

diff --git a/libcaf_cuda/caf/cuda/memory_command.hpp b/libcaf_cuda/caf/cuda/memory_command.hpp
new file mode 100644
index 0000000000..29b115b129
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/memory_command.hpp
@@ -0,0 +1,50 @@
+#pragma once
+
+#include <utility>
+
+#include <caf/ref_counted.hpp>
+#include <caf/intrusive_ptr.hpp>
+
+#include "caf/cuda/platform.hpp"
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/mem_ref.hpp"
+#include "caf/cuda/types.hpp"
+
+namespace caf::cuda {
+
+// ===========================================================================
+// MEMORY COMMAND (single transfer)
+// ===========================================================================
+template <typename T>
+class memory_command : public caf::ref_counted {
+public:
+  using result_type = mem_ptr<raw_t<T>>;
+
+  // -------------------------------------------------------------------------
+  // Constructor
+  // -------------------------------------------------------------------------
+  memory_command(int device_number,
+                 int stream_id,
+                 T arg)
+      : stream_id_(stream_id),
+        arg_(std::move(arg)) {
+
+    dev_ = platform::create()->schedule(stream_id_, device_number);
+  }
+
+  // -------------------------------------------------------------------------
+  // Execute memory transfer
+  // -------------------------------------------------------------------------
+  result_type enqueue() {
+    CUstream stream = dev_->get_stream_for_actor(stream_id_);
+    return dev_->make_arg(arg_, stream);
+  }
+
+private:
+  int stream_id_;
+  device_ptr dev_;
+  T arg_;
+};
+
+} // namespace caf::cuda
+

From d0c65266a2f791e0a85fb438d2a15606d84c2062 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 8 Jan 2026 14:59:55 -0600
Subject: [PATCH 0136/1000] Implemented transfer_memory method.

---
 libcaf_cuda/caf/cuda/command_runner.hpp | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index 8e0301bde6..28f6469cde 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "caf/cuda/command.hpp"
+#include "caf/cuda/memory_command.hpp"
 #include "caf/cuda/program.hpp"
 #include "caf/cuda/nd_range.hpp"
 #include "caf/cuda/platform.hpp"
@@ -130,6 +131,20 @@ class command_runner {
       return cmd->base_enqueue();
   }
 
+    // -------------------------------------------------------------------------
+    // MEMORY TRANSFER
+    // Single transfer per command, returns device buffer
+    // -------------------------------------------------------------------------
+    template <typename T>
+    mem_ptr<raw_t<T>> transfer_memory(int device_number,
+                                      int stream_id,
+                                      T arg)
+    {
+        // stack-allocate memory_command and execute transfer
+        memory_command<T> cmd(device_number, stream_id, std::move(arg));
+        return cmd.enqueue();
+    }
+
   // -------------------------------
   // Destroy streams for a given actor ID
   // -------------------------------

From 2fb140eeeb05107c59631723f834fb82c9fa7712 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 9 Jan 2026 14:01:34 -0600
Subject: [PATCH 0137/1000] Changed compile_nvrtc program to call the helper
 fucntion in helpers.cpp This change was made to fix an issue where the method
 would mistakingly call managers compile_nvrtc program again creating an
 infinite recursion error.

---
 libcaf_cuda/src/manager.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 34a84cffb2..5482db62fa 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -246,7 +246,7 @@ program_ptr manager::create_program_from_fatbin(const std::string& filename,
 // Returns true on success; on failure prints log and returns false
 bool manager::compile_nvrtc_program(const char* source, CUdevice device, std::vector<char>& ptx_out) {
 
-	return compile_nvrtc_program(source,device,ptx_out);
+	return caf::cuda::compile_nvrtc_program(source,device,ptx_out);
 }
 
 

From db41fc36d73d3f64094cea8925436a59d123000e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 9 Jan 2026 14:48:08 -0600
Subject: [PATCH 0138/1000] Updated memory_transfer test to successfully pass.

---
 libcaf_cuda/tests/unit-test/main.test.cpp | 84 ++++++++++++++++++++++-
 1 file changed, 83 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/unit-test/main.test.cpp b/libcaf_cuda/tests/unit-test/main.test.cpp
index 8c76373ce2..785c17bb1d 100644
--- a/libcaf_cuda/tests/unit-test/main.test.cpp
+++ b/libcaf_cuda/tests/unit-test/main.test.cpp
@@ -683,6 +683,87 @@ void test_compare_strings([[maybe_unused]] caf::actor_system& sys) {
   assert(vectors_equal(result, expected));
 }
 
+
+// Test for command_runner.hpp: memory transfer via transfer_memory
+void test_command_runner_memory_transfer([[maybe_unused]] caf::actor_system& sys) {
+    using namespace caf::cuda;
+    auto& mgr = manager::get();
+    auto dev = mgr.find_device(0);
+
+    int device_number = 0;
+    int stream_id = 1;
+
+    command_runner<> runner; // no template args needed for transfer_memory
+
+    // Host data
+    std::vector<int> in_data = {1, 2, 3};
+    std::vector<int> io_data = {10, 20, 30};
+    int out_size = 3;
+
+    // Wrap args
+    in<int> in_arg(in_data);
+    in_out<int> io_arg(io_data);
+    out<int> out_arg(out_size);
+
+    // Transfer host -> device
+    caf::cuda::mem_ptr<int> d_in =
+    runner.transfer_memory(device_number, stream_id, in_arg);
+    caf::cuda::mem_ptr<int> d_io =
+    runner.transfer_memory(device_number, stream_id, io_arg);
+
+    caf::cuda::mem_ptr<int> d_out =
+    runner.transfer_memory(device_number, stream_id, out_arg);
+    
+    // Do a dummy kernel on device that increments each element (optional)
+    const char* kernel_src = R"(
+    extern "C" __global__
+    void increment_kernel(int* inout, int* out, int size) {
+        int idx = blockIdx.x * blockDim.x + threadIdx.x;
+        
+	if (idx < size) {
+	    inout[idx] += 1;    // in_out
+            out[idx] = idx+1;   // out
+        }
+    }
+    )";
+
+    program_ptr prog;
+    try {
+        prog = mgr.create_program(kernel_src, "increment_kernel", dev);
+    } catch (const std::exception& e) {
+        std::cerr << "Skipping test_command_runner_memory_transfer due to create_program failure: "
+                  << e.what() << std::endl;
+        return;
+    }
+
+    nd_range dims(out_size, 1, 1, out_size, 1, 1);
+
+    // Run kernel through command_runner using transferred memory
+    command_runner<in_out<int>, out<int>,in<int>> kernel_runner;
+    auto outputs = kernel_runner.run(prog, dims, 1 /* actor_id */,
+                                     io_arg, out_arg,
+				     in<int>{out_size});
+
+    // Copy back to host
+    std::vector<int> h_io  = caf::cuda::extract_vector<int>(outputs,0); 
+    std::vector<int> h_out = caf::cuda::extract_vector<int>(outputs,1);
+
+    // Verify memory transfer
+    //assert(vectors_equal(h_in, in_data));               // in: unchanged
+    for (size_t i = 0; i < h_io.size(); ++i)
+        assert(h_io[i] == io_data[i]+1);               // in_out: incremented
+    for (size_t i = 0; i < h_out.size(); ++i)
+        assert(h_out[i] == i+1);                       // out: set by kernel
+}
+
+
+
+
+
+
+
+
+
 // Structure to hold test information
 struct Test {
     std::string name;
@@ -708,7 +789,8 @@ const std::vector<Test> tests = {
     {"test_vector_addition", test_vector_addition},
     {"test_invalid_kernel_params", test_invalid_kernel_params},
     {"test_stream_async_execution", test_stream_async_execution},
-    {"test_compare_strings", test_compare_strings}
+    {"test_compare_strings", test_compare_strings},
+    {"test_command_runner_memory_transfer", test_command_runner_memory_transfer}
 };
 
 // Function to run a single test and report its result

From 6f72990d1472270f8d704270d47658f68b34cede Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 9 Jan 2026 15:12:25 -0600
Subject: [PATCH 0139/1000] Added wall clock time metrics.

---
 .../red-ligh-green-light/main.test.cpp        | 65 +++++++++++++------
 .../tests/custom-actors-test/main.test.cpp    | 24 +++++--
 2 files changed, 64 insertions(+), 25 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index 1caa9f73f0..2e3bfb5322 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -177,9 +177,9 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
     const std::vector<int>& matrixC,
     int N) {
 
-  using clock = std::chrono::high_resolution_clock;
+  //using clock = std::chrono::high_resolution_clock;
 
-  auto start = clock::now();
+ // auto start = clock::now();
 
   //std::cout << "GPU ACTOR verifying\n";
 
@@ -195,14 +195,14 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
               << " references did not match\n";
   }
 
-  auto end = clock::now();
+ // auto end = clock::now();
 
-  auto ms =
-    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+  //auto ms =
+    //std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
 
-  std::cout << "[TIMING] verification took "
-            << ms << " ms (actor id "
-            << self->state().id << ")\n";
+  //std::cout << "[TIMING] verification took "
+    //        << ms << " ms (actor id "
+      //      << self->state().id << ")\n";
 
   // signal exit actor and quit
   self->mail(1).send(exit_actor);
@@ -213,6 +213,8 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 }
 
 
+#include <chrono>
+#include <iostream>
 
 void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
   if (num_actors < 1) {
@@ -220,28 +222,49 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
     return;
   }
 
+  std::cout << "Starting run mmul test with matrix_size: "
+            << matrix_size << " and num_actors " << num_actors << "\n";
+
   int limit = 1;
 
-  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+  // ------------------------------------
+  // Start timing
+  // ------------------------------------
+  auto start = std::chrono::steady_clock::now();
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
 
   for (int i = 0; i < limit; i++) {
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
+    // Spawn num_actors actors running the mmul behavior
+    std::vector<caf::actor> actors;
+    actors.reserve(num_actors);
+
+    for (int j = 0; j < num_actors; ++j) {
+      actors.push_back(
+        sys.spawn(mmul_actor_fun, exit_actor, matrix_size)
+      );
+    }
   }
 
+  // Wait for all actors to finish
+  sys.await_all_actors_done();
 
- // std::cout << actors.size() << "\n";
+  // ------------------------------------
+  // Stop timing
+  // ------------------------------------
+  auto end = std::chrono::steady_clock::now();
+  auto duration_ms =
+      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
 
-  }
-
-   sys.await_all_actors_done();
+  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
+            << ", actors=" << num_actors
+            << ", iterations=" << limit
+            << ", time=" << duration_ms << " ms\n";
 }
 
 
 
+
 //this test will spawn more actors over time to demonstrate 
 //changing scheduling algorithims at runtime 
 void run_red_light_green_light_test(caf::actor_system& sys, int matrix_size, int num_actors) {
@@ -280,12 +303,12 @@ void caf_main(caf::actor_system& sys) {
 
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	caf::cuda::manager::init(sys,man_config);
-       	//run_mmul_test(sys,10,1000);
+       	run_mmul_test(sys,10,250);
 	
 	//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	
-	//caf::cuda::manager::init(sys,man_config);
-	run_red_light_green_light_test(sys,10,1000);
+//	caf::cuda::manager::init(sys,man_config);
+//	run_red_light_green_light_test(sys,10,1000);
 }
 
 
diff --git a/libcaf_cuda/tests/custom-actors-test/main.test.cpp b/libcaf_cuda/tests/custom-actors-test/main.test.cpp
index 5cc0e2944e..5add57debf 100644
--- a/libcaf_cuda/tests/custom-actors-test/main.test.cpp
+++ b/libcaf_cuda/tests/custom-actors-test/main.test.cpp
@@ -216,13 +216,17 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
 }
 
 
-
 void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
   if (num_actors < 1) {
     std::cerr << "[ERROR] Number of actors must be >= 1\n";
     return;
   }
 
+  // ------------------------------------
+  // Start timing
+  // ------------------------------------
+  auto start = std::chrono::steady_clock::now();
+
   // Spawn num_actors actors running the mmul behavior
   std::vector<caf::actor> actors;
   actors.reserve(num_actors);
@@ -230,10 +234,22 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
     actors.push_back(sys.spawn(mmul_actor_fun));
   }
 
-  // Actor 0 generates matrices and broadcasts to others 
+  // Actor 0 generates matrices and broadcasts to others
   caf::anon_mail(matrix_size, actors).send(actors[0]);
 
-   sys.await_all_actors_done();
+  // Wait for all actors to finish
+  sys.await_all_actors_done();
+
+  // ------------------------------------
+  // Stop timing
+  // ------------------------------------
+  auto end = std::chrono::steady_clock::now();
+  auto duration_ms =
+      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
+            << ", actors=" << num_actors
+            << ", time=" << duration_ms << " ms\n";
 }
 
 
@@ -738,7 +754,7 @@ void benchmark_shared_perf_all(caf::actor_system& sys) {
 void caf_main(caf::actor_system& sys) {
   caf::cuda::manager::init(sys);
 
-  run_mmul_test(sys,100,4000);
+  run_mmul_test(sys,10,250);
   //run_async_mmul_test(sys,100,1);
   //run_async_mmul_perf_test(sys,1024,200);
 

From 208328c2ea763d3ed3316dde87c7398491ad7abe Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 9 Jan 2026 15:35:12 -0600
Subject: [PATCH 0140/1000] Implemented scaling tests.

---
 .../red-ligh-green-light/main.test.cpp        | 38 ++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index 2e3bfb5322..b2505c84f3 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -296,6 +296,38 @@ void run_red_light_green_light_test(caf::actor_system& sys, int matrix_size, int
 }
 
 
+void run_mmul_scaling_tests(caf::actor_system& sys,caf::cuda::manager_config man_config) {
+  const int min_size   = 10;
+  const int max_size   = 1024;
+  const int min_actors = 1;
+  const int max_actors = 1024;
+
+  // Matrix sizes: 10, 32, 64, 128, ..., 1024
+  std::vector<int> matrix_sizes = {10};
+  for (int s = 32; s <= max_size; s *= 2)
+    matrix_sizes.push_back(s);
+
+  // Actor counts: 1, 2, 4, 8, ..., 1024
+  std::vector<int> actor_counts;
+  for (int a = min_actors; a <= max_actors; a *= 2)
+    actor_counts.push_back(a);
+
+  std::cout << "=== MMUL Scaling Tests ===\n";
+
+  for (int size : matrix_sizes) {
+    for (int actors : actor_counts) {
+      std::cout << "\n[RUN] matrix_size=" << size
+                << ", actors=" << actors << "\n";
+
+      run_mmul_test(sys, size, actors);
+	caf::cuda::manager::init(sys,man_config);
+    }
+  }
+
+  std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
+}
+
+
 
 void caf_main(caf::actor_system& sys) {
   
@@ -303,12 +335,16 @@ void caf_main(caf::actor_system& sys) {
 
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	caf::cuda::manager::init(sys,man_config);
-       	run_mmul_test(sys,10,250);
+       	//run_mmul_test(sys,10,250);
 	
 	//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	
 //	caf::cuda::manager::init(sys,man_config);
 //	run_red_light_green_light_test(sys,10,1000);
+	
+	run_mmul_scaling_tests(sys,man_config);
+
+
 }
 
 

From 2f2f10ccb0e47538c7d6e4a24f3ccfbe4a8a2d0b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 9 Jan 2026 15:35:42 -0600
Subject: [PATCH 0141/1000] Added scaling test.

---
 .../tests/custom-actors-test/main.test.cpp    | 34 ++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/custom-actors-test/main.test.cpp b/libcaf_cuda/tests/custom-actors-test/main.test.cpp
index 5add57debf..91aa42f304 100644
--- a/libcaf_cuda/tests/custom-actors-test/main.test.cpp
+++ b/libcaf_cuda/tests/custom-actors-test/main.test.cpp
@@ -749,12 +749,42 @@ void benchmark_shared_perf_all(caf::actor_system& sys) {
 }
 
 
+void run_mmul_scaling_tests(caf::actor_system& sys) {
+  const int min_size   = 10;
+  const int max_size   = 1024;
+  const int min_actors = 1;
+  const int max_actors = 1024;
+
+  // Matrix sizes: 10, 32, 64, 128, ..., 1024
+  std::vector<int> matrix_sizes = {10};
+  for (int s = 32; s <= max_size; s *= 2)
+    matrix_sizes.push_back(s);
+
+  // Actor counts: 1, 2, 4, 8, ..., 1024
+  std::vector<int> actor_counts;
+  for (int a = min_actors; a <= max_actors; a *= 2)
+    actor_counts.push_back(a);
+
+  std::cout << "=== MMUL Scaling Tests ===\n";
+
+  for (int size : matrix_sizes) {
+    for (int actors : actor_counts) {
+      std::cout << "\n[RUN] matrix_size=" << size
+                << ", actors=" << actors << "\n";
+
+      run_mmul_test(sys, size, actors);
+    }
+  }
+
+  std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
+}
+
 
 
 void caf_main(caf::actor_system& sys) {
   caf::cuda::manager::init(sys);
 
-  run_mmul_test(sys,10,250);
+  //run_mmul_test(sys,10,250);
   //run_async_mmul_test(sys,100,1);
   //run_async_mmul_perf_test(sys,1024,200);
 
@@ -763,6 +793,8 @@ void caf_main(caf::actor_system& sys) {
 
   // run the shared-memory suite:
   //benchmark_shared_perf_all(sys);
+
+  run_mmul_scaling_tests(sys);
 }
 
 

From cba9f82f7fe38a6bc6eaa12bad26ecdede16e7f4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 12 Jan 2026 16:07:06 -0600
Subject: [PATCH 0142/1000] Added device number and stream id attributes to
 constructor,along with getter methods. Change was made so that it can enable
 the scheduler actor to pick the device and stream id of the GPU actor without
 the GPU actor knowing.

---
 .../control-layer/launch_response_token.hpp     | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
index 27f5f056f6..a37e629245 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -23,20 +23,29 @@ launch_response_token(caf::actor receiver,
           range_(std::move(range)),
           memory_usage_(memory_usage),
           id_(std::move(id)),
-          released_(false) {}
+          released_(false),
+	  device_number(0),
+	  stream_id(0) {}
+
     // Construct from a launch_token
-    launch_response_token(caf::actor receiver, const launch_token& token)
+    launch_response_token(caf::actor receiver, const launch_token& token,int device_num,int streamId)
         : receiver_(std::move(receiver)),
           range_(token.getRange()),
           memory_usage_(token.getMemoryUsage()),
           id_(token.getId()),
-          released_(false) {}
+          released_(false),
+	  device_number(device_num),
+	  stream_id(streamId) {}
+
     ~launch_response_token() {
         release();
     }
     int getType() const override { return LAUNCH_RESPONSE; }
     const nd_range& getRange() const { return range_; }
     int getMemoryUsage() const { return memory_usage_; }
+    int getDeviceNumber() const {return device_number;}
+    int getStreamId() const {return stream_id;}
+
     // Return requested number of CUDA blocks
     int getBlocks() const {
         return static_cast<int>(
@@ -61,5 +70,7 @@ launch_response_token(caf::actor receiver,
     int memory_usage_;
     std::string id_;
     std::atomic<bool> released_;
+    int device_number;
+    int stream_id;
 };
 } // namespace caf::cuda

From 3a00c79c8be03be5c36b1002de8aa9ba5b08ad02 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 12 Jan 2026 16:10:44 -0600
Subject: [PATCH 0143/1000] Added device number and stream id attributes to
 constructor,along with getter methods. Change was made so that it can enable
 the scheduler actor to pick the device and stream id of the GPU actor without
 the GPU actor knowing.

---
 .../cuda/control-layer/memory_response_token.hpp   | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp
index 25099a3aa2..92e8654d55 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp
@@ -21,11 +21,16 @@ class CAF_CUDA_EXPORT memory_response_token : public token {
 
     // Construct from memory_transfer_token
     memory_response_token(caf::actor receiver,
-                          const memory_transfer_token& token)
+                          const memory_transfer_token& token,
+			  int device_num,
+			  int streamId)
         : receiver_(std::move(receiver)),
           size_(token.getSize()),
           direction_(token.getDirection()),
-          released_(false) {}
+          released_(false),
+       	  device_number(device_num),
+	  stream_id(streamId) {}
+
 
     ~memory_response_token() {
         release();
@@ -43,6 +48,9 @@ class CAF_CUDA_EXPORT memory_response_token : public token {
         return direction_;
     }
 
+    int getDeviceNumber() const { return device_number;}
+    int getStreamId() const {return stream_id;}
+
     void release() {
         bool expected = false;
 
@@ -65,6 +73,8 @@ class CAF_CUDA_EXPORT memory_response_token : public token {
     int size_;
     int direction_;
     std::atomic<bool> released_;
+    int device_number;
+    int stream_id;
 };
 
 } // namespace caf::cuda

From 7d597989206c682b148c8b4e55da510a3c63bb21 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 12 Jan 2026 16:14:34 -0600
Subject: [PATCH 0144/1000] Updated factory functions to reflect changes in the
 response tokens constructor.

---
 libcaf_cuda/caf/cuda/control-layer/token_factory.hpp |  8 ++++++--
 libcaf_cuda/src/control-layer/token_factory.cpp      | 10 +++++++---
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
index 7aac2df6bd..2609a6ac55 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
@@ -23,7 +23,9 @@ CAF_CUDA_EXPORT token_ptr make_launch_token(program_ptr prog,
 /// Creates a launch_response_token (created internally by the scheduler
 /// when it accepts a kernel launch request)
 CAF_CUDA_EXPORT token_ptr make_launch_response_token(actor scheduler_or_proxy,
-                                     const launch_token& orig);
+                                     const launch_token& orig,
+				     int device_number,
+				     int stream_id);
 
 /// Creates a behavior_token (special — returns its own strong ptr type)
 CAF_CUDA_EXPORT behavior_token_ptr make_behavior_token(std::string name);
@@ -32,7 +34,9 @@ CAF_CUDA_EXPORT behavior_token_ptr make_behavior_token(std::string name);
 CAF_CUDA_EXPORT token_ptr make_memory_token(int size, int direction,caf::actor replyActor);
 
 CAF_CUDA_EXPORT token_ptr make_memory_response_token(actor receiver,
-                                     const memory_transfer_token& orig);
+                                     const memory_transfer_token& orig,
+				     int device_number,
+				     int stream_id);
 
 
 
diff --git a/libcaf_cuda/src/control-layer/token_factory.cpp b/libcaf_cuda/src/control-layer/token_factory.cpp
index 83a622dda6..1fd1dd9a7a 100644
--- a/libcaf_cuda/src/control-layer/token_factory.cpp
+++ b/libcaf_cuda/src/control-layer/token_factory.cpp
@@ -22,9 +22,11 @@ token_ptr make_launch_token(program_ptr prog,
 }
 
 token_ptr make_launch_response_token(actor receiver,
-                                     const launch_token& orig)
+                                     const launch_token& orig,
+				     int device_number,
+				     int stream_id)
 {
-    return token_ptr(new launch_response_token(receiver, orig));
+    return token_ptr(new launch_response_token(receiver, orig,device_number,stream_id));
 }
 
 behavior_token_ptr make_behavior_token(std::string name)
@@ -40,7 +42,9 @@ token_ptr make_memory_token(int size,int direction,caf::actor replyActor) {
 }
 
 token_ptr make_memory_response_token(actor receiver,
-                                     const memory_transfer_token& orig) {
+                                     const memory_transfer_token& orig,
+				     int device_number,
+				     int stream_id) {
     return token_ptr(new memory_response_token(receiver, orig));
 }
 

From 1e4faaba963fffd4b92638d4461c0d2e969cc159 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 12 Jan 2026 16:19:33 -0600
Subject: [PATCH 0145/1000] Implemented changes to include a device_number
 argument in the scheduler actor and scheduler actor state. Change is being
 made so that the scheduler actor can seemless transition devices by changing
 integers or sending messages in the future.

---
 libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp     | 2 +-
 .../caf/cuda/control-layer/scheduler_actor_state.hpp       | 1 +
 libcaf_cuda/src/control-layer/scheduler_actor.cpp          | 7 +++++--
 libcaf_cuda/src/manager.cpp                                | 2 +-
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
index d72d868608..eda0f22726 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
@@ -14,5 +14,5 @@
  */
 
 namespace caf::cuda {
-caf::behavior CAF_CUDA_EXPORT scheduler_actor(caf::stateful_actor<scheduler_actor_state> * self);
+caf::behavior CAF_CUDA_EXPORT scheduler_actor(caf::stateful_actor<scheduler_actor_state> * self,int device_number);
 }//namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
index 2498a6a710..5d41f4c374 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
@@ -12,5 +12,6 @@ struct scheduler_actor_state {
     scheduler_actor_behavior* current_behavior = nullptr;
     behavior_table table;
     std::queue<token_ptr> queue;
+    int device_number;
 };
 } // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 64e0a2697d..a9082835e0 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -12,11 +12,14 @@
 
 namespace caf::cuda {
 
-caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self) {
+caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,int device_number) {
 
     //add its self reference
     self -> state().self = self;
 
+    //set device number
+    self -> state().device_number = device_number
+
     // populate the table
     static red_light_behavior red_behavior;
     static green_light_behavior green_behavior;
@@ -46,7 +49,7 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self)
 	    }
         },
 	[=](std::string word) {
-		std::cout << "Received message " << word << "\n";
+	//	std::cout << "Received message " << word << "\n";
 	},
 	 [=](caf::cuda::mem_ptr<int> token) {
             if (!token) {
diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 5482db62fa..8889a89fd9 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -59,7 +59,7 @@ void manager::init(caf::actor_system& sys, manager_config config) {
 
     if (instance_->scheduler_on) {
         instance_->scheduler_actor_handle =
-            sys.spawn(scheduler_actor);
+            sys.spawn(scheduler_actor,0);
     }
 }
 

From 50942eafb317bdb32b3ad835de84e2f899ec8706 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 12 Jan 2026 16:21:59 -0600
Subject: [PATCH 0146/1000] Commneted out red_light scheduler transition,
 Change was made temporarly since green_light_scheduler dequeue is needed for
 performance analysis.

---
 libcaf_cuda/src/control-layer/green_light_behavior.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index ea3e73524f..adf6b9ff3e 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -42,10 +42,11 @@ void green_light_behavior::receive(scheduler_actor_state* state,
 	   behavior_token_ptr red_light = make_behavior_token("red");
 
 	   //send a request to change behavior to green light after 5 seconds
-    	   anon_mail(red_light)
+    	   /*
+	   anon_mail(red_light)
 		   .delay(std::chrono::seconds(5))
 		   .send(state -> self);
-	   
+	   */
     }
 
 //--------------------------------------------------

From 53539e11d495b7cdb1fc73688069a9f9fbaf6f37 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 12 Jan 2026 16:27:14 -0600
Subject: [PATCH 0147/1000] Made changes to methods to reflect changes made in
 response tokens constructors and factory functions.

---
 .../control-layer/green_light_behavior.cpp    | 21 +++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index adf6b9ff3e..7315ea24c1 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -11,10 +11,14 @@ void green_light_behavior::receive(scheduler_actor_state* state,
                                    const token_ptr& tok) {
 
     if (tok->getType() == LAUNCH) {
-        process_launch_token(tok, state->self);
+	//use 0 as stream id for now, eventually will have to figure out
+	//stream load balancing 
+        process_launch_token(tok, state->self,state -> device_number,0);
     }
     else if (tok->getType() == MEMORY) {
-        process_memory_transfer_token(tok, state->self);
+	//use 0 as stream id for now, eventually will have to figure out
+	//stream load balancing 
+        process_memory_transfer_token(tok, state->self,state -> device_number,0);
     }
 
     //this may cause an issue if a message is never received then 
@@ -52,13 +56,16 @@ void green_light_behavior::receive(scheduler_actor_state* state,
 //--------------------------------------------------
 // process_launch_token
 //--------------------------------------------------
-void green_light_behavior::process_launch_token(const token_ptr& tok,caf::actor self) {
+void green_light_behavior::process_launch_token(const token_ptr& tok,
+		caf::actor self,
+		int device_number,
+		int stream_id) {
     caf::cuda::launch_token& launch =
         static_cast<caf::cuda::launch_token&>(*tok);
 
     // Create response token using the existing launch token
     caf::cuda::token_ptr response =
-        make_launch_response_token(self, launch);
+        make_launch_response_token(self, launch,device_number,stream_id);
 
     // Send response to the actor that requested the launch
     anon_mail(response).send(launch.getReplyActor());
@@ -68,13 +75,15 @@ void green_light_behavior::process_launch_token(const token_ptr& tok,caf::actor
 // process_memory_transfer_token
 //--------------------------------------------------
 void green_light_behavior::process_memory_transfer_token(const token_ptr& tok,
-                                                         caf::actor self) {
+                                                         caf::actor self,
+							 int device_number,
+							 int stream_id) {
     caf::cuda::memory_transfer_token& mem =
         static_cast<caf::cuda::memory_transfer_token&>(*tok);
 
     // Create memory response token
     caf::cuda::token_ptr response =
-        make_memory_response_token(self, mem);
+        make_memory_response_token(self, mem,device_number,stream_id);
 
     // Send response to requesting actor
     anon_mail(response).send(mem.getReplyActor());

From aaac9a0bb46474abe16b598f73f906198987c339 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 12 Jan 2026 16:29:55 -0600
Subject: [PATCH 0148/1000] Fixed syntax error.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index a9082835e0..7d9e54db01 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -18,7 +18,7 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,i
     self -> state().self = self;
 
     //set device number
-    self -> state().device_number = device_number
+    self -> state().device_number = device_number;
 
     // populate the table
     static red_light_behavior red_behavior;

From 5868fa9e83203fe597c0cc5053a101ad2057d7b8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 12 Jan 2026 16:30:51 -0600
Subject: [PATCH 0149/1000] Updated stubs to include device number and stream
 id, so that the scheduler can begin to make more intellgent scheduler
 decisions on where to send a request.

---
 libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
index 4d4b070785..9559889484 100644
--- a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
@@ -9,8 +9,8 @@ class green_light_behavior : public scheduler_actor_behavior {
 
     void receive(scheduler_actor_state* state, const token_ptr& tok) override; 
     void init(scheduler_actor_state* state); 
-    void process_launch_token(const token_ptr& tok,caf::actor self);
-    void process_memory_transfer_token(const token_ptr& tok, caf::actor self);
+    void process_launch_token(const token_ptr& tok,caf::actor self,int device_number,int stream_id);
+    void process_memory_transfer_token(const token_ptr& tok, caf::actor self,int device_number,int stream_id);
 };
 
 

From edd2045a15049d71752e2c5eea87c634b01676f2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 12 Jan 2026 16:33:58 -0600
Subject: [PATCH 0150/1000] Fixed compiler syntax errors.

---
 libcaf_cuda/src/control-layer/green_light_behavior.cpp | 4 ++--
 libcaf_cuda/src/control-layer/token_factory.cpp        | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index 7315ea24c1..e0e4e2ebba 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -30,10 +30,10 @@ void green_light_behavior::receive(scheduler_actor_state* state,
 
         if (queued->getType() == LAUNCH) {
             // safe: we've checked the runtime type
-            process_launch_token(queued, state->self);
+            process_launch_token(tok, state->self,state -> device_number,0);
         }
         else if (queued->getType() == MEMORY) {
-            process_memory_transfer_token(queued, state->self);
+            process_memory_transfer_token(tok, state->self,state -> device_number,0);
         }
     }
 }
diff --git a/libcaf_cuda/src/control-layer/token_factory.cpp b/libcaf_cuda/src/control-layer/token_factory.cpp
index 1fd1dd9a7a..6a796ce023 100644
--- a/libcaf_cuda/src/control-layer/token_factory.cpp
+++ b/libcaf_cuda/src/control-layer/token_factory.cpp
@@ -45,7 +45,7 @@ token_ptr make_memory_response_token(actor receiver,
                                      const memory_transfer_token& orig,
 				     int device_number,
 				     int stream_id) {
-    return token_ptr(new memory_response_token(receiver, orig));
+    return token_ptr(new memory_response_token(receiver, orig,device_number,stream_id));
 }
 
 

From 46f524a0469226728eb876df3bbf106b82ba6932 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 12 Jan 2026 16:49:09 -0600
Subject: [PATCH 0151/1000] Implemented run,run_async and transfer_memory
 overloads using launch_response_token and memory_transfer_token.

---
 libcaf_cuda/caf/cuda/command_runner.hpp | 74 +++++++++++++++++++++++++
 1 file changed, 74 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index 28f6469cde..8e92288f21 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -5,6 +5,8 @@
 #include "caf/cuda/program.hpp"
 #include "caf/cuda/nd_range.hpp"
 #include "caf/cuda/platform.hpp"
+#include "caf/cuda/control-layer/launch_response_token.hpp"
+#include "caf/cuda/control-layer/memory_response_token.hpp"
 
 namespace caf::cuda {
 
@@ -76,6 +78,32 @@ class command_runner {
       return cmd->enqueue();
   }
 
+
+  // -------------------------------
+
+  // Synchronous run using launch_response_token
+
+  // stream comes from token
+
+  // -------------------------------
+
+  template <class... Us>
+  auto run(program_ptr program,
+         nd_range dims,
+         launch_response_token token,
+         Us&&... xs)
+  {
+    return run(std::move(program),
+               std::move(dims),
+               /* actor_id = */ token.getStreamId(),
+               /* shared_memory = */ 0,
+               /* device_number = */ token.getDeviceNumber(),
+               std::forward<Us>(xs)...);
+}
+
+
+
+
   // -------------------------------
   // Asynchronous run: actor_id only
   // returns a tuple of mem_ptrs
@@ -131,6 +159,31 @@ class command_runner {
       return cmd->base_enqueue();
   }
 
+
+
+   // -------------------------------
+   // Asynchronous run using launch_response_token
+   // stream comes from token
+
+   // -------------------------------
+    template <class... Us>
+    auto run_async(program_ptr program,
+               nd_range dims,
+               launch_response_token token,
+               Us&&... xs)
+    {
+    	return run_async(std::move(program),
+                     std::move(dims),
+                     /* actor_id = */ token.getStreamId(),
+                     /* shared_memory = */ 0,
+                     /* device_number = */ token.getDeviceNumber(),
+                      std::forward<Us>(xs)...);
+
+    }
+
+
+
+
     // -------------------------------------------------------------------------
     // MEMORY TRANSFER
     // Single transfer per command, returns device buffer
@@ -145,6 +198,21 @@ class command_runner {
         return cmd.enqueue();
     }
 
+   // -------------------------------------------------------------------------
+    // MEMORY TRANSFER
+    // Single transfer per command, returns device buffer
+    // can transfer memory with a response token
+    // -------------------------------------------------------------------------
+    template <typename T>
+    mem_ptr<raw_t<T>> transfer_memory(memory_response_token token,
+                                      T arg)
+    {
+        // stack-allocate memory_command and execute transfer
+        return transfer_memory(token.getDeviceNumber(),token.getStreamId(),arg); 
+    }
+
+
+
   // -------------------------------
   // Destroy streams for a given actor ID
   // -------------------------------
@@ -152,6 +220,12 @@ class command_runner {
       auto plat = platform::create();
       plat->release_streams_for_actor(actor_id);
   }
+
+
+
+
+
+
 };
 
 } // namespace caf::cuda

From f0673968728f49eded05cf9d6a5a470ded6f25b3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 13 Jan 2026 13:25:00 -0600
Subject: [PATCH 0152/1000] Initial commit, copied over from memory_test.

---
 .../CMakeLists.txt                            |  44 +++
 .../compile_kernels.sh                        |  18 ++
 .../main.test.cpp                             | 306 ++++++++++++++++++
 .../command-runner-integration-test/mmul.cu   |  16 +
 4 files changed, 384 insertions(+)
 create mode 100644 libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
 create mode 100644 libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/mmul.cu

diff --git a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/CMakeLists.txt b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/compile_kernels.sh b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/compile_kernels.sh
new file mode 100755
index 0000000000..586196454e
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/compile_kernels.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+
+# Compile genMatrix.cu to fatbin in current directory
+#nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
+#echo "Generated generate_random_matrix.fatbin"
+
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
new file mode 100644
index 0000000000..15e3883a1a
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
@@ -0,0 +1,306 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include <unistd.h>
+#include "caf/actor_registry.hpp"
+#include <chrono>
+#include <iostream>
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+struct exit_actor_state {
+	int completed = 0;
+};
+
+
+caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
+
+
+	return {
+		[=](int num_completed) {
+			self->state().completed += num_completed;
+			
+			std::cout << "Actors finished is " << self->state().completed << "\n";
+			if (self->state().completed >= limit) {
+			
+				caf::cuda::manager::shutdown();
+				self->quit();
+			}
+		}
+	};
+
+
+}
+
+
+
+
+
+
+// Define a custom type ID block for custom actors
+CAF_ADD_ATOM(cuda,shared_mem)
+
+
+
+
+
+// Extend your actor state to keep the start time
+struct mmul_actor_state {
+  static inline const char* name = "my_actor";
+  int N = 1024; // example state variable
+  int id = rand(); // an actor id
+  // per-actor timing start
+  std::chrono::high_resolution_clock::time_point start_time;
+  int times = 0;
+  caf::cuda::program_ptr program = caf::cuda::manager::get().create_program_from_cubin("../mmul.cubin","matrixMul");
+  int THREADS = 32;
+  int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS,BLOCKS,1,THREADS,THREADS,THREADS);
+};
+
+
+
+
+//commands classes used to launch kernels 
+using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
+using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
+
+using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
+
+mmulCommand mmul;
+matrixGenCommand randomMatrix;
+mmulAsyncCommand mmulAsync;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
+ 
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+	
+  	caf::actor scheduler = mgr.get_scheduler_actor();
+	
+
+	//send a memory transfer token
+	int bytes = N*N * sizeof(int);
+	caf::cuda::token_ptr memory_token = caf::cuda::make_memory_token(bytes,H2D,self);
+	self -> mail(memory_token).send(scheduler);	
+
+	//send a launch token
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
+			self -> state().dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+	  [=] (caf::cuda::token_ptr response_token) {
+	
+		 if (response_token -> getType() == LAUNCH_RESPONSE) {
+		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+		  //assume N = 1024
+		  int N = self -> state().N;
+		  std::vector<int> matrix1(N*N);
+		  matrix1.reserve(N);
+		  std::vector<int> matrix2(N*N);
+		  matrix2.reserve(N);
+
+		  //std::cout << "GPU ACTOR sending data to compute\n";
+		  self -> mail(matrix1,matrix2,N).send(self);
+	 
+		 }
+		 else {
+			 std::cout << "Got a memory response token\n";
+		 }
+		 //token should drop out of scope now, triggering a response 
+	  },
+
+    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB, int N) {
+ 
+
+		  //std::cout << "GPU ACTOR  computing\n";
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //create program and dims   
+  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+  const int THREADS = 32;
+  const int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    //create args
+    auto arg1 = caf::cuda::create_in_arg(matrixA);
+    auto arg2 = caf::cuda::create_in_arg(matrixB);
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    auto tempC = mmul.run(program,dims,self -> state().id,arg1,arg2,arg3,arg4);
+    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+		  //std::cout << "GPU ACTOR done  computing\n";
+    //verify its own result 
+    self -> mail(matrixA,matrixB,matrixC,N).send(self);
+
+    },
+
+    // 3rd handler: CPU atom + matrices + N
+    [=](const std::vector<int>& matrixA,
+    const std::vector<int>& matrixB,
+    const std::vector<int>& matrixC,
+    int N) {
+
+  using clock = std::chrono::high_resolution_clock;
+
+  auto start = clock::now();
+
+  //std::cout << "GPU ACTOR verifying\n";
+
+  std::vector<int> result(N * N);
+
+  serial_matrix_multiply(matrixA, matrixB, result, N);
+
+  if (result == matrixC) {
+    std::cout << "actor with id " << self->state().id
+              << " references match\n";
+  } else {
+    std::cout << "actor with id " << self->state().id
+              << " references did not match\n";
+  }
+
+  auto end = clock::now();
+
+  auto ms =
+    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[TIMING] verification took "
+            << ms << " ms (actor id "
+            << self->state().id << ")\n";
+
+  // signal exit actor and quit
+  self->mail(1).send(exit_actor);
+  self->quit();
+
+    }
+  };
+}
+
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  int limit = 1;
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+
+  for (int i = 0; i < limit; i++) {
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
+  }
+
+
+ // std::cout << actors.size() << "\n";
+
+  }
+
+   sys.await_all_actors_done();
+}
+
+
+
+//this test will spawn more actors over time to demonstrate 
+//changing scheduling algorithims at runtime 
+void run_red_light_green_light_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  std::cout << "Starting RED LIGHT GREEN LIGHT TEST\n";
+  int limit = 10;
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors * limit);
+
+  for (int i = 0; i < limit; i++) {
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
+  }
+
+
+  sleep(1);
+ // std::cout << actors.size() << "\n";
+
+  }
+
+   sys.await_all_actors_done();
+}
+
+
+
+void caf_main(caf::actor_system& sys) {
+  
+	
+
+	caf::cuda::manager_config man_config(true); //turns the scheduler on
+	caf::cuda::manager::init(sys,man_config);
+       	run_mmul_test(sys,10,10);
+	
+	//tests will delete the old manager so will have to reinit if you do this 
+	//in conjunction with each other	
+	//caf::cuda::manager::init(sys,man_config);
+	//run_red_light_green_light_test(sys,10,1000);
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/mmul.cu b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From 46f45079cb7228e87e19fac0df7c8c24cfeb028a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 13 Jan 2026 13:43:11 -0600
Subject: [PATCH 0153/1000] Added a kernel_launch_token typedef to make pointer
 more readable.

---
 .../caf/cuda/control-layer/launch_response_token.hpp        | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
index a37e629245..5d235650e3 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -73,4 +73,10 @@ launch_response_token(caf::actor receiver,
     int device_number;
     int stream_id;
 };
+
+
+using kernel_launch_token = caf::instrusive_ptr<launch_response_token>;
+
+
+
 } // namespace caf::cuda

From 36947c0267bad468f8c6feda7a116954cae16170 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 13 Jan 2026 13:45:36 -0600
Subject: [PATCH 0154/1000] Added a memory_token typedef to improve
 readability.

---
 libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp
index 92e8654d55..9cbfce07e5 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp
@@ -77,5 +77,9 @@ class CAF_CUDA_EXPORT memory_response_token : public token {
     int stream_id;
 };
 
+
+using memory_token = caf::instrusive_ptr<memory_response_token>;
+
+
 } // namespace caf::cuda
 

From 1ec7013e1d4d00a51e2e307a0bde488aa8e8129c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 13 Jan 2026 13:47:29 -0600
Subject: [PATCH 0155/1000] Fixed spelling mistake of intrusive_ptr.

---
 libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp | 2 +-
 libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
index 5d235650e3..7ef251378b 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -75,7 +75,7 @@ launch_response_token(caf::actor receiver,
 };
 
 
-using kernel_launch_token = caf::instrusive_ptr<launch_response_token>;
+using kernel_launch_token = caf::intrusive_ptr<launch_response_token>;
 
 
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp
index 9cbfce07e5..23120f6e26 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp
@@ -78,7 +78,7 @@ class CAF_CUDA_EXPORT memory_response_token : public token {
 };
 
 
-using memory_token = caf::instrusive_ptr<memory_response_token>;
+using memory_token = caf::intrusive_ptr<memory_response_token>;
 
 
 } // namespace caf::cuda

From 68451abf2b717c850f32bbe92fa8f5120be7911a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 13 Jan 2026 13:50:59 -0600
Subject: [PATCH 0156/1000] Changed the token command options from take token
 object parameters to the respective token pointer parameters. This change was
 made to reduce the boilerplate code and burden on the application programmer
 required by typecasting and dereferencing a pointer, it just made no sense.

---
 libcaf_cuda/caf/cuda/command_runner.hpp | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index 8e92288f21..0e988c3930 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -90,14 +90,14 @@ class command_runner {
   template <class... Us>
   auto run(program_ptr program,
          nd_range dims,
-         launch_response_token token,
+         kernel_launch_token token,
          Us&&... xs)
   {
     return run(std::move(program),
                std::move(dims),
-               /* actor_id = */ token.getStreamId(),
+               /* actor_id = */ token -> getStreamId(),
                /* shared_memory = */ 0,
-               /* device_number = */ token.getDeviceNumber(),
+               /* device_number = */ token -> getDeviceNumber(),
                std::forward<Us>(xs)...);
 }
 
@@ -169,14 +169,14 @@ class command_runner {
     template <class... Us>
     auto run_async(program_ptr program,
                nd_range dims,
-               launch_response_token token,
+               kernel_launch_token token,
                Us&&... xs)
     {
     	return run_async(std::move(program),
                      std::move(dims),
-                     /* actor_id = */ token.getStreamId(),
+                     /* actor_id = */ token ->getStreamId(),
                      /* shared_memory = */ 0,
-                     /* device_number = */ token.getDeviceNumber(),
+                     /* device_number = */ token -> getDeviceNumber(),
                       std::forward<Us>(xs)...);
 
     }
@@ -204,11 +204,11 @@ class command_runner {
     // can transfer memory with a response token
     // -------------------------------------------------------------------------
     template <typename T>
-    mem_ptr<raw_t<T>> transfer_memory(memory_response_token token,
+    mem_ptr<raw_t<T>> transfer_memory(memory_token token,
                                       T arg)
     {
         // stack-allocate memory_command and execute transfer
-        return transfer_memory(token.getDeviceNumber(),token.getStreamId(),arg); 
+        return transfer_memory(token -> getDeviceNumber(),token -> getStreamId(),arg); 
     }
 
 

From 8867401186362a6d03d82315058dbfba0b39b60c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 13 Jan 2026 14:19:36 -0600
Subject: [PATCH 0157/1000] Reverted back to object references for now. Change
 is being made since intrusive pointers cant be typecasted safely like smart
 pointers, so will need to come up with a better way of doing this, but for
 now this works.

---
 libcaf_cuda/caf/cuda/command_runner.hpp | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index 0e988c3930..e46137e4e1 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -90,14 +90,14 @@ class command_runner {
   template <class... Us>
   auto run(program_ptr program,
          nd_range dims,
-         kernel_launch_token token,
+         launch_response_token& token,
          Us&&... xs)
   {
     return run(std::move(program),
                std::move(dims),
-               /* actor_id = */ token -> getStreamId(),
+               /* actor_id = */ token.getStreamId(),
                /* shared_memory = */ 0,
-               /* device_number = */ token -> getDeviceNumber(),
+               /* device_number = */ token.getDeviceNumber(),
                std::forward<Us>(xs)...);
 }
 
@@ -169,14 +169,14 @@ class command_runner {
     template <class... Us>
     auto run_async(program_ptr program,
                nd_range dims,
-               kernel_launch_token token,
+               launch_response_token& token,
                Us&&... xs)
     {
     	return run_async(std::move(program),
                      std::move(dims),
-                     /* actor_id = */ token ->getStreamId(),
+                     /* actor_id = */ token.getStreamId(),
                      /* shared_memory = */ 0,
-                     /* device_number = */ token -> getDeviceNumber(),
+                     /* device_number = */ token.getDeviceNumber(),
                       std::forward<Us>(xs)...);
 
     }
@@ -204,11 +204,11 @@ class command_runner {
     // can transfer memory with a response token
     // -------------------------------------------------------------------------
     template <typename T>
-    mem_ptr<raw_t<T>> transfer_memory(memory_token token,
+    mem_ptr<raw_t<T>> transfer_memory(memory_response_token& token,
                                       T arg)
     {
         // stack-allocate memory_command and execute transfer
-        return transfer_memory(token -> getDeviceNumber(),token -> getStreamId(),arg); 
+        return transfer_memory(token.getDeviceNumber(),token.getStreamId(),arg); 
     }
 
 

From 7e61c687178cd95b30be0a8d7f8540c8e08a9031 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 13 Jan 2026 14:22:06 -0600
Subject: [PATCH 0158/1000] Updated test to use command_runner
 launch_response_token parameters.

---
 .../main.test.cpp                             | 50 +++----------------
 1 file changed, 8 insertions(+), 42 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
index 15e3883a1a..da6fd80ce4 100644
--- a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
@@ -116,12 +116,6 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 	
   	caf::actor scheduler = mgr.get_scheduler_actor();
 	
-
-	//send a memory transfer token
-	int bytes = N*N * sizeof(int);
-	caf::cuda::token_ptr memory_token = caf::cuda::make_memory_token(bytes,H2D,self);
-	self -> mail(memory_token).send(scheduler);	
-
 	//send a launch token
 	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
 			self -> state().dims,
@@ -145,7 +139,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 		  matrix2.reserve(N);
 
 		  //std::cout << "GPU ACTOR sending data to compute\n";
-		  self -> mail(matrix1,matrix2,N).send(self);
+		  self -> mail(matrix1,matrix2,response_token,N).send(self);
 	 
 		 }
 		 else {
@@ -156,11 +150,15 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 
     // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
     [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB, int N) {
+        const std::vector<int>& matrixB,
+	const caf::cuda::token_ptr& kToken, int N) {
  
 
+	    caf::cuda::launch_response_token& kt =
+		    static_cast<caf::cuda::launch_response_token&>(*kToken);
+
 		  //std::cout << "GPU ACTOR  computing\n";
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
+	    caf::cuda::manager& mgr = caf::cuda::manager::get();
 
   //create program and dims   
   auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
@@ -174,7 +172,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
     auto arg3 = caf::cuda::create_out_arg(N*N);
     auto arg4 = caf::cuda::create_in_arg(N);
 
-    auto tempC = mmul.run(program,dims,self -> state().id,arg1,arg2,arg3,arg4);
+    auto tempC = mmul.run(program,dims,kt,arg1,arg2,arg3,arg4);
     std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
 
 		  //std::cout << "GPU ACTOR done  computing\n";
@@ -254,37 +252,6 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
 
 
 
-//this test will spawn more actors over time to demonstrate 
-//changing scheduling algorithims at runtime 
-void run_red_light_green_light_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  std::cout << "Starting RED LIGHT GREEN LIGHT TEST\n";
-  int limit = 10;
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors * limit);
-
-  for (int i = 0; i < limit; i++) {
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
-  }
-
-
-  sleep(1);
- // std::cout << actors.size() << "\n";
-
-  }
-
-   sys.await_all_actors_done();
-}
-
-
 
 void caf_main(caf::actor_system& sys) {
   
@@ -297,7 +264,6 @@ void caf_main(caf::actor_system& sys) {
 	//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	
 	//caf::cuda::manager::init(sys,man_config);
-	//run_red_light_green_light_test(sys,10,1000);
 }
 
 

From 6f648419dee4e2a3b033aff9b3361eb38162364b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 13 Jan 2026 14:46:33 -0600
Subject: [PATCH 0159/1000] Implemented response_token to be an abstract class
 of response tokens. Change was made to abstract away common features both
 classes had and provide a common interface.

---
 .../control-layer/launch_response_token.hpp   | 84 ++++++++++---------
 .../control-layer/memory_response_token.hpp   | 42 +++-------
 .../caf/cuda/control-layer/response_token.hpp | 45 ++++++++++
 3 files changed, 100 insertions(+), 71 deletions(-)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/response_token.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
index 7ef251378b..132ebca399 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -1,50 +1,54 @@
 #pragma once
-#include "caf/cuda/control-layer/token.hpp"
-#include "caf/cuda/control-layer/launch_token.hpp"  // Full include for constructor param
-#include <caf/actor.hpp>  // For caf::actor
-#include "caf/cuda/nd_range.hpp"  // For nd_range
+#include "caf/cuda/control-layer/response_token.hpp"
+#include "caf/cuda/control-layer/launch_token.hpp"
 #include "caf/cuda/global_export.hpp"
+
+#include <caf/actor.hpp>
 #include <atomic>
-#include <string>  // For std::string (if not from elsewhere)
+#include <string>
+#include "caf/cuda/nd_range.hpp"
 
 namespace caf::cuda {
-class CAF_CUDA_EXPORT launch_response_token : public token {
-public:
 
-	//only here to be complaint with CAFS type if system DO NOT USE 
-	launch_response_token() = default;
+// -----------------------------------------------------------------------------
+// Launch response token returned after a kernel launch request
+// -----------------------------------------------------------------------------
+class CAF_CUDA_EXPORT launch_response_token : public response_token {
+public:
+    // Only here to be compliant with CAF's type system – DO NOT USE directly
+    launch_response_token() = default;
 
-    // Construct manually    
-launch_response_token(caf::actor receiver,
+    // Construct manually
+    launch_response_token(caf::actor receiver,
                           nd_range range,
                           int memory_usage,
-                          std::string id)
-        : receiver_(std::move(receiver)),
+                          std::string id,
+                          int device_num = 0,
+                          int stream_id = 0)
+        : response_token(std::move(receiver), device_num, stream_id, memory_usage),
           range_(std::move(range)),
-          memory_usage_(memory_usage),
           id_(std::move(id)),
-          released_(false),
-	  device_number(0),
-	  stream_id(0) {}
+          released_(false) {}
 
     // Construct from a launch_token
-    launch_response_token(caf::actor receiver, const launch_token& token,int device_num,int streamId)
-        : receiver_(std::move(receiver)),
+    launch_response_token(caf::actor receiver,
+                          const launch_token& token,
+                          int device_num,
+                          int stream_id)
+        : response_token(std::move(receiver), device_num, stream_id, token.getMemoryUsage()),
           range_(token.getRange()),
-          memory_usage_(token.getMemoryUsage()),
           id_(token.getId()),
-          released_(false),
-	  device_number(device_num),
-	  stream_id(streamId) {}
+          released_(false) {}
 
     ~launch_response_token() {
         release();
     }
+
     int getType() const override { return LAUNCH_RESPONSE; }
+
     const nd_range& getRange() const { return range_; }
-    int getMemoryUsage() const { return memory_usage_; }
-    int getDeviceNumber() const {return device_number;}
-    int getStreamId() const {return stream_id;}
+
+    const std::string& getId() const { return id_; }
 
     // Return requested number of CUDA blocks
     int getBlocks() const {
@@ -54,29 +58,29 @@ launch_response_token(caf::actor receiver,
             range_.getGridDimZ()
         );
     }
-    const std::string& getId() const { return id_; }
-    void release() {
+
+    void release() override {
         bool expected = false;
-        if (released_.compare_exchange_strong(expected, true)) {
-            	//the real message commented out for testing 
-		//caf::anon_mail(id_, getBlocks()).urgent().send(receiver_);
-		//test message
-		caf::anon_mail("Hello world from me").urgent().send(receiver_);
+
+        // ONLY send if we successfully transition false → true
+        if (!released_.compare_exchange_strong(expected, true)) {
+            return; // already released → do nothing
         }
+
+        // Real message (commented for testing)
+        // caf::anon_mail(id_, memorySize()).urgent().send(receiver_);
+
+        // Test message
+        caf::anon_mail("Hello world from launch response").urgent().send(receiver_);
     }
+
 private:
-    caf::actor receiver_;
     nd_range range_;
-    int memory_usage_;
     std::string id_;
     std::atomic<bool> released_;
-    int device_number;
-    int stream_id;
 };
 
-
 using kernel_launch_token = caf::intrusive_ptr<launch_response_token>;
 
-
-
 } // namespace caf::cuda
+
diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp
index 23120f6e26..aacdcd27a2 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_response_token.hpp
@@ -1,11 +1,5 @@
 #pragma once
-
-//this class is meant to act as a response issued by the scheduler
-//actor authorizing memory transfer as needed 
-
-#pragma once
-
-#include "caf/cuda/control-layer/token.hpp"
+#include "caf/cuda/control-layer/response_token.hpp"
 #include "caf/cuda/control-layer/memory_transfer_token.hpp"
 #include "caf/cuda/global_export.hpp"
 
@@ -14,7 +8,10 @@
 
 namespace caf::cuda {
 
-class CAF_CUDA_EXPORT memory_response_token : public token {
+// -----------------------------------------------------------------------------
+// Memory response issued by the scheduler / actor authorizing memory transfer
+// -----------------------------------------------------------------------------
+class CAF_CUDA_EXPORT memory_response_token : public response_token {
 public:
     // Required by CAF – do not use directly
     memory_response_token() = default;
@@ -22,15 +19,11 @@ class CAF_CUDA_EXPORT memory_response_token : public token {
     // Construct from memory_transfer_token
     memory_response_token(caf::actor receiver,
                           const memory_transfer_token& token,
-			  int device_num,
-			  int streamId)
-        : receiver_(std::move(receiver)),
-          size_(token.getSize()),
+                          int device_num,
+                          int stream_id)
+        : response_token(std::move(receiver), device_num, stream_id, token.getSize()),
           direction_(token.getDirection()),
-          released_(false),
-       	  device_number(device_num),
-	  stream_id(streamId) {}
-
+          released_(false) {}
 
     ~memory_response_token() {
         release();
@@ -40,18 +33,11 @@ class CAF_CUDA_EXPORT memory_response_token : public token {
         return MEMORY_RESPONSE;
     }
 
-    int getSize() const {
-        return size_;
-    }
-
     int getDirection() const {
         return direction_;
     }
 
-    int getDeviceNumber() const { return device_number;}
-    int getStreamId() const {return stream_id;}
-
-    void release() {
+    void release() override {
         bool expected = false;
 
         // ONLY send if we successfully transition false → true
@@ -60,7 +46,7 @@ class CAF_CUDA_EXPORT memory_response_token : public token {
         }
 
         // Real message (commented for testing)
-        // caf::anon_mail(size_, direction_).urgent().send(receiver_);
+        // caf::anon_mail(memorySize(), direction_).urgent().send(receiver_);
 
         // Test message
         caf::anon_mail("Hello world from memory response")
@@ -69,17 +55,11 @@ class CAF_CUDA_EXPORT memory_response_token : public token {
     }
 
 private:
-    caf::actor receiver_;
-    int size_;
     int direction_;
     std::atomic<bool> released_;
-    int device_number;
-    int stream_id;
 };
 
-
 using memory_token = caf::intrusive_ptr<memory_response_token>;
 
-
 } // namespace caf::cuda
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/response_token.hpp
new file mode 100644
index 0000000000..2dcdeaa49e
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/response_token.hpp
@@ -0,0 +1,45 @@
+#pragma once
+#include "caf/cuda/control-layer/token.hpp"
+#include <caf/actor.hpp>
+#include "caf/cuda/global_export.hpp"
+
+namespace caf::cuda {
+
+// -----------------------------------------------------------------------------
+// Base class for all response tokens (memory, launch, etc.)
+// Abstracts common attributes like device, stream, and memory usage
+// -----------------------------------------------------------------------------
+class CAF_CUDA_EXPORT response_token : public token {
+public:
+   
+    response_token() = default;
+
+    response_token(caf::actor receiver, int device_num, int stream_id, int memory_size = 0)
+        : receiver_(std::move(receiver)),
+          device_number_(device_num),
+          stream_id_(stream_id),
+          memory_size_(memory_size) {}
+
+    virtual ~response_token() = default;
+
+    // Common getters
+    int getDeviceNumber() const { return device_number_; }
+    int getStreamId() const { return stream_id_; }
+    int memorySize() const { return memory_size_; }
+    const caf::actor& getReceiver() const { return receiver_; }
+
+    // Pure virtual: children must implement release()
+    virtual void release() = 0;
+
+protected:
+    caf::actor receiver_;
+    int device_number_{0};
+    int stream_id_{0};
+    int memory_size_{0};  // abstracted memory usage / size
+};
+
+// Typedef for convenience
+using response_token_ptr = caf::intrusive_ptr<response_token>;
+
+} // namespace caf::cuda
+

From 6616f0786243aed8aadfafc4f349c7d0f10acabe Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 13 Jan 2026 14:47:48 -0600
Subject: [PATCH 0160/1000] Include response token.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 83e8557568..6b76325f31 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -5,6 +5,7 @@
 // Control-layer object types
 #include "caf/cuda/control-layer/token.hpp"
 #include "caf/cuda/control-layer/launch_token.hpp"
+#include "caf/cuda/control-layer/response_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 #include "caf/cuda/control-layer/behavior_token.hpp"
 #include "caf/cuda/control-layer/memory_transfer_token.hpp"
@@ -24,6 +25,7 @@ CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::behavior_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_response_token>))
+CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::response_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_transfer_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_response_token>))
 
@@ -39,10 +41,12 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_response_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_transfer_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_response_token)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::response_token)
 
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::behavior_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_response_token>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::response_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::memory_transfer_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::memory_response_token>)

From 322972bc47cfad98f85f413fa0cf88f8189f41c5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 13 Jan 2026 14:55:10 -0600
Subject: [PATCH 0161/1000] Updated token_factories to make  response_token_ptr
 rather than token_ptr. CHange is being made so that downcasting can occur
 safely.

---
 libcaf_cuda/caf/cuda/control-layer/token_factory.hpp | 4 ++--
 libcaf_cuda/src/control-layer/token_factory.cpp      | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
index 2609a6ac55..2f5d37deb4 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
@@ -22,7 +22,7 @@ CAF_CUDA_EXPORT token_ptr make_launch_token(program_ptr prog,
 
 /// Creates a launch_response_token (created internally by the scheduler
 /// when it accepts a kernel launch request)
-CAF_CUDA_EXPORT token_ptr make_launch_response_token(actor scheduler_or_proxy,
+CAF_CUDA_EXPORT response_token_ptr make_launch_response_token(actor scheduler_or_proxy,
                                      const launch_token& orig,
 				     int device_number,
 				     int stream_id);
@@ -33,7 +33,7 @@ CAF_CUDA_EXPORT behavior_token_ptr make_behavior_token(std::string name);
 //creats a memory transfer token
 CAF_CUDA_EXPORT token_ptr make_memory_token(int size, int direction,caf::actor replyActor);
 
-CAF_CUDA_EXPORT token_ptr make_memory_response_token(actor receiver,
+CAF_CUDA_EXPORT response_token_ptr make_memory_response_token(actor receiver,
                                      const memory_transfer_token& orig,
 				     int device_number,
 				     int stream_id);
diff --git a/libcaf_cuda/src/control-layer/token_factory.cpp b/libcaf_cuda/src/control-layer/token_factory.cpp
index 6a796ce023..44a7a2112e 100644
--- a/libcaf_cuda/src/control-layer/token_factory.cpp
+++ b/libcaf_cuda/src/control-layer/token_factory.cpp
@@ -21,12 +21,12 @@ token_ptr make_launch_token(program_ptr prog,
                          reply_to));
 }
 
-token_ptr make_launch_response_token(actor receiver,
+response_token_ptr make_launch_response_token(actor receiver,
                                      const launch_token& orig,
 				     int device_number,
 				     int stream_id)
 {
-    return token_ptr(new launch_response_token(receiver, orig,device_number,stream_id));
+    return response_token_ptr(new launch_response_token(receiver, orig,device_number,stream_id));
 }
 
 behavior_token_ptr make_behavior_token(std::string name)
@@ -41,11 +41,11 @@ token_ptr make_memory_token(int size,int direction,caf::actor replyActor) {
 
 }
 
-token_ptr make_memory_response_token(actor receiver,
+response_token_ptr make_memory_response_token(actor receiver,
                                      const memory_transfer_token& orig,
 				     int device_number,
 				     int stream_id) {
-    return token_ptr(new memory_response_token(receiver, orig,device_number,stream_id));
+    return response_token_ptr(new memory_response_token(receiver, orig,device_number,stream_id));
 }
 
 

From e93a212c9bed9a561d2949dd5885ebdf33611b30 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 13 Jan 2026 14:56:22 -0600
Subject: [PATCH 0162/1000] Updated process token methods to use response_token
 type instead of token Change is being made to gain additional access to
 methods without the need to downcast.

---
 libcaf_cuda/src/control-layer/green_light_behavior.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index e0e4e2ebba..8fe2680898 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -64,7 +64,7 @@ void green_light_behavior::process_launch_token(const token_ptr& tok,
         static_cast<caf::cuda::launch_token&>(*tok);
 
     // Create response token using the existing launch token
-    caf::cuda::token_ptr response =
+    caf::cuda::response_token_ptr response =
         make_launch_response_token(self, launch,device_number,stream_id);
 
     // Send response to the actor that requested the launch
@@ -82,7 +82,7 @@ void green_light_behavior::process_memory_transfer_token(const token_ptr& tok,
         static_cast<caf::cuda::memory_transfer_token&>(*tok);
 
     // Create memory response token
-    caf::cuda::token_ptr response =
+    caf::cuda::response_token_ptr response =
         make_memory_response_token(self, mem,device_number,stream_id);
 
     // Send response to requesting actor

From 6abd620ae852fe8534f1497e1bdb9e9dce4120bc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 13 Jan 2026 15:04:42 -0600
Subject: [PATCH 0163/1000] Updated token methods to use a response_token_ptr
 instead of objects. Change was made to prevent lots of boilerplate code from
 occuring due to static typecasting.

---
 libcaf_cuda/caf/cuda/command_runner.hpp | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index e46137e4e1..ca5bd03a1f 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -5,6 +5,7 @@
 #include "caf/cuda/program.hpp"
 #include "caf/cuda/nd_range.hpp"
 #include "caf/cuda/platform.hpp"
+#include "caf/cuda/control-layer/response_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 #include "caf/cuda/control-layer/memory_response_token.hpp"
 
@@ -90,14 +91,14 @@ class command_runner {
   template <class... Us>
   auto run(program_ptr program,
          nd_range dims,
-         launch_response_token& token,
+         const response_token_ptr& token,
          Us&&... xs)
   {
     return run(std::move(program),
                std::move(dims),
-               /* actor_id = */ token.getStreamId(),
+               /* actor_id = */ token ->getStreamId(),
                /* shared_memory = */ 0,
-               /* device_number = */ token.getDeviceNumber(),
+               /* device_number = */ token -> getDeviceNumber(),
                std::forward<Us>(xs)...);
 }
 
@@ -169,14 +170,14 @@ class command_runner {
     template <class... Us>
     auto run_async(program_ptr program,
                nd_range dims,
-               launch_response_token& token,
+               const response_token_ptr& token,
                Us&&... xs)
     {
     	return run_async(std::move(program),
                      std::move(dims),
-                     /* actor_id = */ token.getStreamId(),
+                     /* actor_id = */ token -> getStreamId(),
                      /* shared_memory = */ 0,
-                     /* device_number = */ token.getDeviceNumber(),
+                     /* device_number = */ token -> getDeviceNumber(),
                       std::forward<Us>(xs)...);
 
     }
@@ -204,11 +205,11 @@ class command_runner {
     // can transfer memory with a response token
     // -------------------------------------------------------------------------
     template <typename T>
-    mem_ptr<raw_t<T>> transfer_memory(memory_response_token& token,
+    mem_ptr<raw_t<T>> transfer_memory(const response_token_ptr& token,
                                       T arg)
     {
         // stack-allocate memory_command and execute transfer
-        return transfer_memory(token.getDeviceNumber(),token.getStreamId(),arg); 
+        return transfer_memory(token -> getDeviceNumber(),token -> getStreamId(),arg); 
     }
 
 

From 6cb134b6df4e3a18fffbdf2ba1ed781c5747d795 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 13 Jan 2026 15:05:20 -0600
Subject: [PATCH 0164/1000] Updated test to reflect changes made to command
 runner using response_token_ptr as parameters instead of token objects
 themselves.

---
 .../main.test.cpp                                | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
index da6fd80ce4..e77f2a681c 100644
--- a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
@@ -127,9 +127,9 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 
 	return {
 
-	  [=] (caf::cuda::token_ptr response_token) {
+	  [=] (caf::cuda::response_token_ptr res_token) {
 	
-		 if (response_token -> getType() == LAUNCH_RESPONSE) {
+		 if (res_token -> getType() == LAUNCH_RESPONSE) {
 		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
 		  //assume N = 1024
 		  int N = self -> state().N;
@@ -139,7 +139,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 		  matrix2.reserve(N);
 
 		  //std::cout << "GPU ACTOR sending data to compute\n";
-		  self -> mail(matrix1,matrix2,response_token,N).send(self);
+		  self -> mail(matrix1,matrix2,res_token,N).send(self);
 	 
 		 }
 		 else {
@@ -151,11 +151,13 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
     // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
     [=](const std::vector<int>& matrixA,
         const std::vector<int>& matrixB,
-	const caf::cuda::token_ptr& kToken, int N) {
+	const caf::cuda::response_token_ptr& res_token, int N) {
  
 
-	    caf::cuda::launch_response_token& kt =
-		    static_cast<caf::cuda::launch_response_token&>(*kToken);
+	    //caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
+
+	    //caf::cuda::launch_response_token& kt =
+	//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
 
 		  //std::cout << "GPU ACTOR  computing\n";
 	    caf::cuda::manager& mgr = caf::cuda::manager::get();
@@ -172,7 +174,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
     auto arg3 = caf::cuda::create_out_arg(N*N);
     auto arg4 = caf::cuda::create_in_arg(N);
 
-    auto tempC = mmul.run(program,dims,kt,arg1,arg2,arg3,arg4);
+    auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
     std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
 
 		  //std::cout << "GPU ACTOR done  computing\n";

From 0b3a843dd243b197842b9d6868dd43d0c86acd41 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 14 Jan 2026 14:33:11 -0600
Subject: [PATCH 0165/1000] Changed mmul_actor_fun message callback parameter
 to response_token_ptr type to coinside with the changes made to the scheduler
 actor and the token factory tokens.

---
 .../control-layer-tests/red-ligh-green-light/main.test.cpp      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
index b2505c84f3..dcae54f1f7 100644
--- a/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/red-ligh-green-light/main.test.cpp
@@ -125,7 +125,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 
 	return {
 
-	  [=] (caf::cuda::token_ptr launch_response_token) {
+	  [=] (caf::cuda::response_token_ptr launch_response_token) {
 	 
 		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
 		  //assume N = 1024

From bd536933793cec2c67ad67eb57c973c5f7ab4e91 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 14 Jan 2026 14:34:03 -0600
Subject: [PATCH 0166/1000] Changed mmul_actor_fun message callback parameter
 to response_token_ptr type to coinside with the changes made to the scheduler
 actor and the token factory tokens.

---
 .../tests/control-layer-tests/memory_test/main.test.cpp       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp
index 15e3883a1a..19a9d4cb9b 100644
--- a/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/memory_test/main.test.cpp
@@ -133,9 +133,9 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 
 	return {
 
-	  [=] (caf::cuda::token_ptr response_token) {
+	  [=] (caf::cuda::response_token_ptr launch_response_token) {
 	
-		 if (response_token -> getType() == LAUNCH_RESPONSE) {
+		 if (launch_response_token -> getType() == LAUNCH_RESPONSE) {
 		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
 		  //assume N = 1024
 		  int N = self -> state().N;

From af8a3debe12616033da07891c71ad46cbdbd567c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 14 Jan 2026 14:59:12 -0600
Subject: [PATCH 0167/1000] Implemented mmul_actor_async_fun and
 test_mmul_async to test to see if the response tokens worked correctly.

---
 .../main.test.cpp                             | 146 +++++++++++++++++-
 1 file changed, 145 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
index e77f2a681c..333b66d30f 100644
--- a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
@@ -252,6 +252,150 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
    sys.await_all_actors_done();
 }
 
+// Stateful actor behavior
+caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
+ 
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+	
+  	caf::actor scheduler = mgr.get_scheduler_actor();
+	
+	//send a launch token
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
+			self -> state().dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+	  [=] (caf::cuda::response_token_ptr res_token) {
+	
+		 if (res_token -> getType() == LAUNCH_RESPONSE) {
+		  int N = self -> state().N;
+
+		  caf::cuda::command_runner<in_out<int>> mem_transfer_command;
+
+
+		  std::vector<int> matrix1(N*N);
+		  std::vector<int> matrix2(N*N);
+
+		  caf::cuda::mem_ptr<int> matrixA = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix1});
+		  caf::cuda::mem_ptr<int> matrixB = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix2});
+
+		  //std::cout << "GPU ACTOR sending data to compute\n";
+		  self -> mail(matrixA,matrixB,res_token,N).send(self);
+	 
+		 }
+		 else {
+			 //std::cout << "Got a memory response token\n";
+		 }
+		 //token should drop out of scope now, triggering a response 
+	  },
+
+    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+    [=](const caf::cuda::mem_ptr<int>& matrixA,
+        const caf::cuda::mem_ptr<int>& matrixB,
+	const caf::cuda::response_token_ptr& res_token, int N) {
+ 
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+	//create program and dims   
+ 	auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+  	const int THREADS = 32;
+  	const int BLOCKS = (N + THREADS - 1) / THREADS;
+  	caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    //create args
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    auto temp = mmulAsync.run_async(program,dims,res_token,matrixA,matrixB,arg3,arg4);
+    caf::cuda::mem_ptr<int> matrixC = std::get<2>(temp);
+
+    std::vector<int> matrix1 = matrixA -> copy_to_host();
+    std::vector<int> matrix2 = matrixB -> copy_to_host();
+    std::vector<int> matrix3 = matrixC -> copy_to_host();
+
+		  //std::cout << "GPU ACTOR done  computing\n";
+    //verify its own result 
+    self -> mail(matrix1,matrix2,matrix3,N).send(self);
+
+    },
+
+    // 3rd handler: CPU atom + matrices + N
+    [=](const std::vector<int>& matrixA,
+    const std::vector<int>& matrixB,
+    const std::vector<int>& matrixC,
+    int N) {
+
+  using clock = std::chrono::high_resolution_clock;
+
+  auto start = clock::now();
+
+  //std::cout << "GPU ACTOR verifying\n";
+
+  std::vector<int> result(N * N);
+
+  serial_matrix_multiply(matrixA, matrixB, result, N);
+
+  if (result == matrixC) {
+    std::cout << "actor with id " << self->state().id
+              << " references match\n";
+  } else {
+    std::cout << "actor with id " << self->state().id
+              << " references did not match\n";
+  }
+
+  auto end = clock::now();
+
+  auto ms =
+    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[TIMING] verification took "
+            << ms << " ms (actor id "
+            << self->state().id << ")\n";
+
+  // signal exit actor and quit
+  self->mail(1).send(exit_actor);
+  self->quit();
+
+    }
+  };
+}
+
+
+
+void run_async_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  int limit = 1;
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+
+  for (int i = 0; i < limit; i++) {
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_async_actor_fun,exit_actor,matrix_size));
+  }
+
+
+ // std::cout << actors.size() << "\n";
+
+  }
+
+   sys.await_all_actors_done();
+}
 
 
 
@@ -261,7 +405,7 @@ void caf_main(caf::actor_system& sys) {
 
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	caf::cuda::manager::init(sys,man_config);
-       	run_mmul_test(sys,10,10);
+       	run_async_mmul_test(sys,10,1000);
 	
 	//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	

From 4b76f80dfb63962edad7f38c256eb411e2f6ee46 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 21 Jan 2026 13:46:47 -0600
Subject: [PATCH 0168/1000] Implemented std::vector<token_ptr> as acceptable
 message type.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 6b76325f31..6e744289e2 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -28,6 +28,7 @@ CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_response_tok
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::response_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_transfer_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_response_token>))
+CAF_ADD_TYPE_ID(cuda_control, (std::vector<caf::intrusive_ptr<caf::cuda::token>>))
 
 CAF_END_TYPE_ID_BLOCK(cuda_control)
 
@@ -50,3 +51,4 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_response_toke
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::response_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::memory_transfer_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::memory_response_token>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<caf::intrusive_ptr<caf::cuda::token>>)

From 7aa5a30576625ab6daeb44ad73949e6091165e3b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 21 Jan 2026 13:55:16 -0600
Subject: [PATCH 0169/1000] Initial commit, copied over from command runner
 test.

---
 .../bulk-token-test/CMakeLists.txt            |  44 ++
 .../bulk-token-test/compile_kernels.sh        |  18 +
 .../bulk-token-test/main.test.cpp             | 418 ++++++++++++++++++
 .../bulk-token-test/mmul.cu                   |  16 +
 .../bulk-token-test/mmul.cubin                | Bin 0 -> 5608 bytes
 5 files changed, 496 insertions(+)
 create mode 100644 libcaf_cuda/tests/control-layer-tests/bulk-token-test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/control-layer-tests/bulk-token-test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
 create mode 100644 libcaf_cuda/tests/control-layer-tests/bulk-token-test/mmul.cu
 create mode 100644 libcaf_cuda/tests/control-layer-tests/bulk-token-test/mmul.cubin

diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/CMakeLists.txt b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/compile_kernels.sh b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/compile_kernels.sh
new file mode 100755
index 0000000000..586196454e
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/compile_kernels.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+
+# Compile genMatrix.cu to fatbin in current directory
+#nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
+#echo "Generated generate_random_matrix.fatbin"
+
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
new file mode 100644
index 0000000000..333b66d30f
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
@@ -0,0 +1,418 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include <unistd.h>
+#include "caf/actor_registry.hpp"
+#include <chrono>
+#include <iostream>
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+struct exit_actor_state {
+	int completed = 0;
+};
+
+
+caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
+
+
+	return {
+		[=](int num_completed) {
+			self->state().completed += num_completed;
+			
+			std::cout << "Actors finished is " << self->state().completed << "\n";
+			if (self->state().completed >= limit) {
+			
+				caf::cuda::manager::shutdown();
+				self->quit();
+			}
+		}
+	};
+
+
+}
+
+
+
+
+
+
+// Define a custom type ID block for custom actors
+CAF_ADD_ATOM(cuda,shared_mem)
+
+
+
+
+
+// Extend your actor state to keep the start time
+struct mmul_actor_state {
+  static inline const char* name = "my_actor";
+  int N = 1024; // example state variable
+  int id = rand(); // an actor id
+  // per-actor timing start
+  std::chrono::high_resolution_clock::time_point start_time;
+  int times = 0;
+  caf::cuda::program_ptr program = caf::cuda::manager::get().create_program_from_cubin("../mmul.cubin","matrixMul");
+  int THREADS = 32;
+  int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS,BLOCKS,1,THREADS,THREADS,THREADS);
+};
+
+
+
+
+//commands classes used to launch kernels 
+using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
+using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
+
+using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
+
+mmulCommand mmul;
+matrixGenCommand randomMatrix;
+mmulAsyncCommand mmulAsync;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
+ 
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+	
+  	caf::actor scheduler = mgr.get_scheduler_actor();
+	
+	//send a launch token
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
+			self -> state().dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+	  [=] (caf::cuda::response_token_ptr res_token) {
+	
+		 if (res_token -> getType() == LAUNCH_RESPONSE) {
+		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+		  //assume N = 1024
+		  int N = self -> state().N;
+		  std::vector<int> matrix1(N*N);
+		  matrix1.reserve(N);
+		  std::vector<int> matrix2(N*N);
+		  matrix2.reserve(N);
+
+		  //std::cout << "GPU ACTOR sending data to compute\n";
+		  self -> mail(matrix1,matrix2,res_token,N).send(self);
+	 
+		 }
+		 else {
+			 std::cout << "Got a memory response token\n";
+		 }
+		 //token should drop out of scope now, triggering a response 
+	  },
+
+    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+	const caf::cuda::response_token_ptr& res_token, int N) {
+ 
+
+	    //caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
+
+	    //caf::cuda::launch_response_token& kt =
+	//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
+
+		  //std::cout << "GPU ACTOR  computing\n";
+	    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //create program and dims   
+  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+  const int THREADS = 32;
+  const int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    //create args
+    auto arg1 = caf::cuda::create_in_arg(matrixA);
+    auto arg2 = caf::cuda::create_in_arg(matrixB);
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+		  //std::cout << "GPU ACTOR done  computing\n";
+    //verify its own result 
+    self -> mail(matrixA,matrixB,matrixC,N).send(self);
+
+    },
+
+    // 3rd handler: CPU atom + matrices + N
+    [=](const std::vector<int>& matrixA,
+    const std::vector<int>& matrixB,
+    const std::vector<int>& matrixC,
+    int N) {
+
+  using clock = std::chrono::high_resolution_clock;
+
+  auto start = clock::now();
+
+  //std::cout << "GPU ACTOR verifying\n";
+
+  std::vector<int> result(N * N);
+
+  serial_matrix_multiply(matrixA, matrixB, result, N);
+
+  if (result == matrixC) {
+    std::cout << "actor with id " << self->state().id
+              << " references match\n";
+  } else {
+    std::cout << "actor with id " << self->state().id
+              << " references did not match\n";
+  }
+
+  auto end = clock::now();
+
+  auto ms =
+    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[TIMING] verification took "
+            << ms << " ms (actor id "
+            << self->state().id << ")\n";
+
+  // signal exit actor and quit
+  self->mail(1).send(exit_actor);
+  self->quit();
+
+    }
+  };
+}
+
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  int limit = 1;
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+
+  for (int i = 0; i < limit; i++) {
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
+  }
+
+
+ // std::cout << actors.size() << "\n";
+
+  }
+
+   sys.await_all_actors_done();
+}
+
+// Stateful actor behavior
+caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
+ 
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+	
+  	caf::actor scheduler = mgr.get_scheduler_actor();
+	
+	//send a launch token
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
+			self -> state().dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+	  [=] (caf::cuda::response_token_ptr res_token) {
+	
+		 if (res_token -> getType() == LAUNCH_RESPONSE) {
+		  int N = self -> state().N;
+
+		  caf::cuda::command_runner<in_out<int>> mem_transfer_command;
+
+
+		  std::vector<int> matrix1(N*N);
+		  std::vector<int> matrix2(N*N);
+
+		  caf::cuda::mem_ptr<int> matrixA = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix1});
+		  caf::cuda::mem_ptr<int> matrixB = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix2});
+
+		  //std::cout << "GPU ACTOR sending data to compute\n";
+		  self -> mail(matrixA,matrixB,res_token,N).send(self);
+	 
+		 }
+		 else {
+			 //std::cout << "Got a memory response token\n";
+		 }
+		 //token should drop out of scope now, triggering a response 
+	  },
+
+    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+    [=](const caf::cuda::mem_ptr<int>& matrixA,
+        const caf::cuda::mem_ptr<int>& matrixB,
+	const caf::cuda::response_token_ptr& res_token, int N) {
+ 
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+	//create program and dims   
+ 	auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+  	const int THREADS = 32;
+  	const int BLOCKS = (N + THREADS - 1) / THREADS;
+  	caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    //create args
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    auto temp = mmulAsync.run_async(program,dims,res_token,matrixA,matrixB,arg3,arg4);
+    caf::cuda::mem_ptr<int> matrixC = std::get<2>(temp);
+
+    std::vector<int> matrix1 = matrixA -> copy_to_host();
+    std::vector<int> matrix2 = matrixB -> copy_to_host();
+    std::vector<int> matrix3 = matrixC -> copy_to_host();
+
+		  //std::cout << "GPU ACTOR done  computing\n";
+    //verify its own result 
+    self -> mail(matrix1,matrix2,matrix3,N).send(self);
+
+    },
+
+    // 3rd handler: CPU atom + matrices + N
+    [=](const std::vector<int>& matrixA,
+    const std::vector<int>& matrixB,
+    const std::vector<int>& matrixC,
+    int N) {
+
+  using clock = std::chrono::high_resolution_clock;
+
+  auto start = clock::now();
+
+  //std::cout << "GPU ACTOR verifying\n";
+
+  std::vector<int> result(N * N);
+
+  serial_matrix_multiply(matrixA, matrixB, result, N);
+
+  if (result == matrixC) {
+    std::cout << "actor with id " << self->state().id
+              << " references match\n";
+  } else {
+    std::cout << "actor with id " << self->state().id
+              << " references did not match\n";
+  }
+
+  auto end = clock::now();
+
+  auto ms =
+    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[TIMING] verification took "
+            << ms << " ms (actor id "
+            << self->state().id << ")\n";
+
+  // signal exit actor and quit
+  self->mail(1).send(exit_actor);
+  self->quit();
+
+    }
+  };
+}
+
+
+
+void run_async_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  int limit = 1;
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+
+  for (int i = 0; i < limit; i++) {
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_async_actor_fun,exit_actor,matrix_size));
+  }
+
+
+ // std::cout << actors.size() << "\n";
+
+  }
+
+   sys.await_all_actors_done();
+}
+
+
+
+void caf_main(caf::actor_system& sys) {
+  
+	
+
+	caf::cuda::manager_config man_config(true); //turns the scheduler on
+	caf::cuda::manager::init(sys,man_config);
+       	run_async_mmul_test(sys,10,1000);
+	
+	//tests will delete the old manager so will have to reinit if you do this 
+	//in conjunction with each other	
+	//caf::cuda::manager::init(sys,man_config);
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/mmul.cu b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+
diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/mmul.cubin b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/mmul.cubin
new file mode 100644
index 0000000000000000000000000000000000000000..901c492f65180ca6d4013bb2a448df2f41f8fb45
GIT binary patch
literal 5608
zcmeHL&1+oO6+iDoZ${(MdrxCo9$8W)JF=NTkeoEMS!7{MV=o$`&_xI(#Fk>I;}Xlm
zn~8ZAfs9%xWK##a%C30jW$7OfQfiFwDugb)RD=jk$ii(X1QIg&{m#AbV@A#*Z7E&k
zg5JI7o{xLJ?;XAU!iyJN$N6zZ%y8v>xoOv&&K)yAzhUMJV;AI{JSQH$N5m0X?zX$V
zq_?(?RW{$cZeX1+ciUT6Q(4}+xqNMF^M)+<Hqzem^|fAdExq*S8#!WRtJ_{nHm;6r
zzJ6n?+gsb}oh|QQ-B^F~SC==Fwd)(QoNT<YJi4(qyaPo@wf1`N+KsJ$!+?_N|7BAy
zch+Eq46FP7Ei#dM{P%fmdr~`u?CJjaq5VJTzJF-{uezUQO@V#}%SK$t@7E1(fU#uB
zxaRQ0e~WWe-|6>TQXTB{TUA|(aR1pt?9eKc=~FGgg^v;7f9U3P{z_HG04?Irf5NPZ
z^c$u1WNG~qpV=(_PkqgQzg87WyW-4=H!~;xabt)lO_4{FIqXUd_+$<{QXPVMCh^|v
zK>VkDQj0%Vl|OR(h{(?|>u}v_Tw6JXH3V=}?Ht>=Y3D^dx9oh@&d=!#8|-ZO9Wnfl
z7=A|#zaxg<5yS6@;fKx2@7nWWpM?#-Yx!Nv?^=G>@<Z{dr||I*6k72!elX}u#^Z<a
z|K$O98yUlLjqrCbl6Ss9;bA$DP-H)nnOj!h)0kT!U>n7g8H`ACoWWn4ct#q7nY<p#
z(zZe}=Kj7S4;G|O_%)Hv`Xi6>m$p5Rh36`Mg?LZ#n;nsdkzC4KQs0x$B5}7NAN;Cs
z#t(Qve!y<hgzgr95J^217vjx%kazJNkw0mBGV<cociC$OdxMG`M1CV3b7|4w1IRP{
zKH<(4=nL3Q{2}>6=Y448zoa@_`P-E->O1&Y<%^q6iFQ-^oAMK>P~XmZ$iE-S>6=wg
zUO^Gjo?XIIBEPgS?2FS-3wyBlh2rBR8t}Nm0i*y=`(DzvjPpNN`ztKO&iQ|;{&Oh+
z!jEv?-$4JKNM98)`sVnLfxj2YPIm?hV0`)`kvk%%(}qiO)erjabZen}iIgG!ZU}fj
zl&>xPcfu#k%YpcP7yQ%8-=KZUKZ!)M^3TBDw66GR;t`LX?vYSC3HjSRFP_fnPkEr9
zQu?~eTL8Xp8a3%R9pI~lzWbCn8wxGT<xLg#wLS1pgc=~?*B8KV?5(Q&72r)@HT(|X
zW1)6TrKkR*_63FgDt|oGOTfxsO6F#mj!l)n3VF4{KJ|CA@CWh%^e5&)Kd1h=4f+!y
zuO!t+{XI{}!23nKRQ~P?`Qu5T4S(dEPGgm?@j(6{-k~g7{uce&DD-d9-X@CL;!mkQ
z^@&X0{m9+v9u0*TK<+;i&sgoF{Fqmu!Jl%V&-4_Zm{y<Rc%nXbx^tn-S$gEhQc|tr
z+7!(X)wdMyb>@Tdf03W1_^y*b2v{!~Psq>mqmYMuY}xbLw12vYhw{(XW`PdW|Eu7y
zssB`->Jx-=*7DyK@Q>^Lss2!S!aR=XpF7LYXLkMs$Y<3*HlEK~e$-2xRw|FO{)G6W
zpAFS>oL0kgARp}g3;k~a@)`}gOXclG&QdZH&j8l=YW~#gOpqC#w;Q=jNqvg>p!H+$
z(?>HH_OQRORSgc_c4U7gk57&K#HYT`fZG0v{%Fb}@6Smv9%S-%SD{~Xg7&Ka_HWBY
zDEIDHH1N0D-)SHFi}ow_YxJipX!+<;ibp((dR2cSUx)h7@uL6ihq=Bc9~e*6KL`D<
z{if8<wBMkAQ32xH%)g*GPwUCVyXaq~`Z&Y<GyO*Wqw-Ln%z#{8t%(0R`;(~;(_eUA
zslRM8KTUmVd{iFlfgLWFSI&RZn|^5OjrNx6P3=K_GGNMo5_t4iT1XiBpdP20FGW9U
z7WUt9fe(V?z-zti-X`40>p}BvzQ+Ef@zwmAuQJ~AdA<bUBH+Q^KeYcN|LV-w9AD%c
z?hE!`V~@rU`8RYw-Df@5<Dq;-KGNM;KC*t09@+h_`N@1N^0V1wKT-eQTP^P6mpl6Y
z;C<TX1H$HC7BAID<AHh~%D-bJe=+||KA8K7@tr8{r%#9<?H_TGzuM1B{i8+x(tW9L
zqm-}g&qqu7#rmvNa2*@_cT4Y$Uzqm{>zC=S{z5$?p2z4Pb6>Dtg9)ZfR<Cgyho;`~
z{s4UzAI*2j2mLtviJ_02IGMEl1Md^`TlNR?tN#xc07pXlQaq0G!m<4BMKtUFH#ETZ
zheXEoy+r-(dv1^XHa>9@ctiGV!@Spkw)y9#<3hZi_W=6e3<AEl#!Wxu`^)EhOJTg{
zP#;VuQy<!Id`%y%7t}+Xj*U?}^`G%?p`ZH4#NXTx3!o1IqaWXIE9kEkJPhG~`+h@z
zJVLzs|G}!z{*E4C?2i*q^nGgHt9VbkBlh6=`l@|TL;v4(#nlJFr|>%Txpa%haF~92
z5wPAb^zR3eG;=&i{s*wC@*jLyn)VmSkd$*4`peL+=x_Y;7Hj`;{EpVV$VX*jtnGHb
z%ig2cQ@&#s4=v%lcAqr-<z$ix@wNNE!8|IHsJ0c_v2@uqO8cZijFQPi%YW7~W;SZ>
zXTmU_{|<iq3el08Nqxk|GRpsF?7@eJ&PR6X*fUf9&Fs(S+rS{j4~;AKpx9z4j$0XD
zw*PDF(I?hitjy?G{<lgFdz}8yfSe?zs4X)myOI5`iRUAJi+$osgJg_r_J7h6N|$%-
bMmA^q^Gxz(dRnr4!Y!a9n@9P+?|I(>-yt?^

literal 0
HcmV?d00001


From f072d5d740c95623a47000ae589aaf079e9d56b0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 21 Jan 2026 14:06:56 -0600
Subject: [PATCH 0170/1000] Copied over from red-light-green-light instead
 since that is what was benchedmarked.

---
 .../bulk-token-test/main.test.cpp             | 253 +++++++-----------
 1 file changed, 94 insertions(+), 159 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
index 333b66d30f..dcae54f1f7 100644
--- a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
@@ -115,8 +115,6 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 	
   	caf::actor scheduler = mgr.get_scheduler_actor();
-	
-	//send a launch token
 	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
 			self -> state().dims,
 			0,
@@ -127,9 +125,8 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 
 	return {
 
-	  [=] (caf::cuda::response_token_ptr res_token) {
-	
-		 if (res_token -> getType() == LAUNCH_RESPONSE) {
+	  [=] (caf::cuda::response_token_ptr launch_response_token) {
+	 
 		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
 		  //assume N = 1024
 		  int N = self -> state().N;
@@ -139,28 +136,19 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 		  matrix2.reserve(N);
 
 		  //std::cout << "GPU ACTOR sending data to compute\n";
-		  self -> mail(matrix1,matrix2,res_token,N).send(self);
+		  self -> mail(matrix1,matrix2,N).send(self);
 	 
-		 }
-		 else {
-			 std::cout << "Got a memory response token\n";
-		 }
 		 //token should drop out of scope now, triggering a response 
+	  
 	  },
 
     // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
     [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB,
-	const caf::cuda::response_token_ptr& res_token, int N) {
+        const std::vector<int>& matrixB, int N) {
  
 
-	    //caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
-
-	    //caf::cuda::launch_response_token& kt =
-	//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
-
 		  //std::cout << "GPU ACTOR  computing\n";
-	    caf::cuda::manager& mgr = caf::cuda::manager::get();
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
 
   //create program and dims   
   auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
@@ -174,7 +162,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
     auto arg3 = caf::cuda::create_out_arg(N*N);
     auto arg4 = caf::cuda::create_in_arg(N);
 
-    auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+    auto tempC = mmul.run(program,dims,self -> state().id,arg1,arg2,arg3,arg4);
     std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
 
 		  //std::cout << "GPU ACTOR done  computing\n";
@@ -189,9 +177,9 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
     const std::vector<int>& matrixC,
     int N) {
 
-  using clock = std::chrono::high_resolution_clock;
+  //using clock = std::chrono::high_resolution_clock;
 
-  auto start = clock::now();
+ // auto start = clock::now();
 
   //std::cout << "GPU ACTOR verifying\n";
 
@@ -207,14 +195,14 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
               << " references did not match\n";
   }
 
-  auto end = clock::now();
+ // auto end = clock::now();
 
-  auto ms =
-    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+  //auto ms =
+    //std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
 
-  std::cout << "[TIMING] verification took "
-            << ms << " ms (actor id "
-            << self->state().id << ")\n";
+  //std::cout << "[TIMING] verification took "
+    //        << ms << " ms (actor id "
+      //      << self->state().id << ")\n";
 
   // signal exit actor and quit
   self->mail(1).send(exit_actor);
@@ -225,6 +213,8 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 }
 
 
+#include <chrono>
+#include <iostream>
 
 void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
   if (num_actors < 1) {
@@ -232,164 +222,72 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
     return;
   }
 
+  std::cout << "Starting run mmul test with matrix_size: "
+            << matrix_size << " and num_actors " << num_actors << "\n";
+
   int limit = 1;
 
-  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+  // ------------------------------------
+  // Start timing
+  // ------------------------------------
+  auto start = std::chrono::steady_clock::now();
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
 
   for (int i = 0; i < limit; i++) {
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
+    // Spawn num_actors actors running the mmul behavior
+    std::vector<caf::actor> actors;
+    actors.reserve(num_actors);
+
+    for (int j = 0; j < num_actors; ++j) {
+      actors.push_back(
+        sys.spawn(mmul_actor_fun, exit_actor, matrix_size)
+      );
+    }
   }
 
+  // Wait for all actors to finish
+  sys.await_all_actors_done();
 
- // std::cout << actors.size() << "\n";
-
-  }
+  // ------------------------------------
+  // Stop timing
+  // ------------------------------------
+  auto end = std::chrono::steady_clock::now();
+  auto duration_ms =
+      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
 
-   sys.await_all_actors_done();
+  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
+            << ", actors=" << num_actors
+            << ", iterations=" << limit
+            << ", time=" << duration_ms << " ms\n";
 }
 
-// Stateful actor behavior
-caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
- 
-
-	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-	
-  	caf::actor scheduler = mgr.get_scheduler_actor();
-	
-	//send a launch token
-	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
-			self -> state().dims,
-			0,
-			"hello",
-			self
-			);
-	self -> mail(launch_token).send(scheduler);
-
-	return {
-
-	  [=] (caf::cuda::response_token_ptr res_token) {
-	
-		 if (res_token -> getType() == LAUNCH_RESPONSE) {
-		  int N = self -> state().N;
-
-		  caf::cuda::command_runner<in_out<int>> mem_transfer_command;
-
-
-		  std::vector<int> matrix1(N*N);
-		  std::vector<int> matrix2(N*N);
-
-		  caf::cuda::mem_ptr<int> matrixA = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix1});
-		  caf::cuda::mem_ptr<int> matrixB = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix2});
-
-		  //std::cout << "GPU ACTOR sending data to compute\n";
-		  self -> mail(matrixA,matrixB,res_token,N).send(self);
-	 
-		 }
-		 else {
-			 //std::cout << "Got a memory response token\n";
-		 }
-		 //token should drop out of scope now, triggering a response 
-	  },
-
-    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const caf::cuda::mem_ptr<int>& matrixA,
-        const caf::cuda::mem_ptr<int>& matrixB,
-	const caf::cuda::response_token_ptr& res_token, int N) {
- 
-	caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-	//create program and dims   
- 	auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-  	const int THREADS = 32;
-  	const int BLOCKS = (N + THREADS - 1) / THREADS;
-  	caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    //create args
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
-
-    auto temp = mmulAsync.run_async(program,dims,res_token,matrixA,matrixB,arg3,arg4);
-    caf::cuda::mem_ptr<int> matrixC = std::get<2>(temp);
-
-    std::vector<int> matrix1 = matrixA -> copy_to_host();
-    std::vector<int> matrix2 = matrixB -> copy_to_host();
-    std::vector<int> matrix3 = matrixC -> copy_to_host();
-
-		  //std::cout << "GPU ACTOR done  computing\n";
-    //verify its own result 
-    self -> mail(matrix1,matrix2,matrix3,N).send(self);
-
-    },
-
-    // 3rd handler: CPU atom + matrices + N
-    [=](const std::vector<int>& matrixA,
-    const std::vector<int>& matrixB,
-    const std::vector<int>& matrixC,
-    int N) {
-
-  using clock = std::chrono::high_resolution_clock;
-
-  auto start = clock::now();
-
-  //std::cout << "GPU ACTOR verifying\n";
-
-  std::vector<int> result(N * N);
-
-  serial_matrix_multiply(matrixA, matrixB, result, N);
 
-  if (result == matrixC) {
-    std::cout << "actor with id " << self->state().id
-              << " references match\n";
-  } else {
-    std::cout << "actor with id " << self->state().id
-              << " references did not match\n";
-  }
 
-  auto end = clock::now();
 
-  auto ms =
-    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-  std::cout << "[TIMING] verification took "
-            << ms << " ms (actor id "
-            << self->state().id << ")\n";
-
-  // signal exit actor and quit
-  self->mail(1).send(exit_actor);
-  self->quit();
-
-    }
-  };
-}
-
-
-
-void run_async_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+//this test will spawn more actors over time to demonstrate 
+//changing scheduling algorithims at runtime 
+void run_red_light_green_light_test(caf::actor_system& sys, int matrix_size, int num_actors) {
   if (num_actors < 1) {
     std::cerr << "[ERROR] Number of actors must be >= 1\n";
     return;
   }
 
-  int limit = 1;
+  std::cout << "Starting RED LIGHT GREEN LIGHT TEST\n";
+  int limit = 10;
 
-  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors * limit);
 
   for (int i = 0; i < limit; i++) {
   // Spawn num_actors actors running the mmul behavior
   std::vector<caf::actor> actors;
   actors.reserve(num_actors);
   for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_async_actor_fun,exit_actor,matrix_size));
+    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
   }
 
 
+  sleep(1);
  // std::cout << actors.size() << "\n";
 
   }
@@ -398,6 +296,38 @@ void run_async_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors
 }
 
 
+void run_mmul_scaling_tests(caf::actor_system& sys,caf::cuda::manager_config man_config) {
+  const int min_size   = 10;
+  const int max_size   = 1024;
+  const int min_actors = 1;
+  const int max_actors = 1024;
+
+  // Matrix sizes: 10, 32, 64, 128, ..., 1024
+  std::vector<int> matrix_sizes = {10};
+  for (int s = 32; s <= max_size; s *= 2)
+    matrix_sizes.push_back(s);
+
+  // Actor counts: 1, 2, 4, 8, ..., 1024
+  std::vector<int> actor_counts;
+  for (int a = min_actors; a <= max_actors; a *= 2)
+    actor_counts.push_back(a);
+
+  std::cout << "=== MMUL Scaling Tests ===\n";
+
+  for (int size : matrix_sizes) {
+    for (int actors : actor_counts) {
+      std::cout << "\n[RUN] matrix_size=" << size
+                << ", actors=" << actors << "\n";
+
+      run_mmul_test(sys, size, actors);
+	caf::cuda::manager::init(sys,man_config);
+    }
+  }
+
+  std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
+}
+
+
 
 void caf_main(caf::actor_system& sys) {
   
@@ -405,11 +335,16 @@ void caf_main(caf::actor_system& sys) {
 
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	caf::cuda::manager::init(sys,man_config);
-       	run_async_mmul_test(sys,10,1000);
+       	//run_mmul_test(sys,10,250);
 	
 	//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	
-	//caf::cuda::manager::init(sys,man_config);
+//	caf::cuda::manager::init(sys,man_config);
+//	run_red_light_green_light_test(sys,10,1000);
+	
+	run_mmul_scaling_tests(sys,man_config);
+
+
 }
 
 

From a7dce0768c035c16acdbeba58c6c05fa05c4acc7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 21 Jan 2026 14:26:07 -0600
Subject: [PATCH 0171/1000] Updated exit actor to spawn every actor and create
 a bulk token or std::vector of token_ptr's.

---
 .../bulk-token-test/main.test.cpp             | 51 +++++++++++++++++--
 1 file changed, 47 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
index dcae54f1f7..715ae720e2 100644
--- a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
@@ -26,7 +26,47 @@ struct exit_actor_state {
 };
 
 
-caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
+caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit,int matrix_size) {
+
+  int N = matrix_size;
+  caf::cuda::program_ptr program = caf::cuda::manager::get().create_program_from_cubin("../mmul.cubin","matrixMul");
+  int THREADS = 32;
+  int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS,BLOCKS,1,THREADS,THREADS,THREADS);
+
+  // ------------------------------------
+  // Start timing
+  // ------------------------------------
+  //  auto start = std::chrono::steady_clock::now();
+ 
+       caf::cuda::manager& mgr = caf::cuda::manager::get();
+	
+  	caf::actor scheduler = mgr.get_scheduler_actor();
+ 
+  
+    caf::actor exit_actor = self;
+    // Spawn num_actors actors running the mmul behavior
+    std::vector<caf::actor> actors;
+    actors.reserve(num_actors);
+    std::vector<caf::cuda::token_ptr> tokens(limit);
+    int num_actors = limit;
+   
+
+    for (int j = 0; j < num_actors; ++j) {
+       caf::actor a = sys.spawn(mmul_actor_fun, exit_actor, matrix_size);
+       actors.push_back(a);
+       caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(program,
+			dims,
+			0,
+			"hello",
+			a
+			);
+
+       tokens.emplace_back(launch_token);
+  }
+
+	self -> mail(tokens).send(scheduler);
+
 
 
 	return {
@@ -121,7 +161,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 			"hello",
 			self
 			);
-	self -> mail(launch_token).send(scheduler);
+//	self -> mail(launch_token).send(scheduler);
 
 	return {
 
@@ -166,8 +206,11 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
     std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
 
 		  //std::cout << "GPU ACTOR done  computing\n";
-    //verify its own result 
-    self -> mail(matrixA,matrixB,matrixC,N).send(self);
+  
+    //do not verify result just exit 
+    self->mail(1).send(exit_actor);
+    self->quit();
+
 
     },
 

From 267a8b368f6350855175b3927c39a7f9a2282278 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 21 Jan 2026 14:26:52 -0600
Subject: [PATCH 0172/1000] Updated mmul test to spawn exit actor.

---
 .../bulk-token-test/main.test.cpp                  | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
index 715ae720e2..402f5ac50e 100644
--- a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
@@ -275,19 +275,7 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
   // ------------------------------------
   auto start = std::chrono::steady_clock::now();
 
-  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-  for (int i = 0; i < limit; i++) {
-    // Spawn num_actors actors running the mmul behavior
-    std::vector<caf::actor> actors;
-    actors.reserve(num_actors);
-
-    for (int j = 0; j < num_actors; ++j) {
-      actors.push_back(
-        sys.spawn(mmul_actor_fun, exit_actor, matrix_size)
-      );
-    }
-  }
+  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors,matrix_size);
 
   // Wait for all actors to finish
   sys.await_all_actors_done();

From 7a27ca1caced8f644815ea1e95fb03ac01200fad Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 21 Jan 2026 14:36:19 -0600
Subject: [PATCH 0173/1000] Fixed syntax errors.

---
 .../bulk-token-test/main.test.cpp             | 160 ++++++++----------
 1 file changed, 66 insertions(+), 94 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
index 402f5ac50e..0ea00f5664 100644
--- a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
@@ -26,67 +26,6 @@ struct exit_actor_state {
 };
 
 
-caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit,int matrix_size) {
-
-  int N = matrix_size;
-  caf::cuda::program_ptr program = caf::cuda::manager::get().create_program_from_cubin("../mmul.cubin","matrixMul");
-  int THREADS = 32;
-  int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS,BLOCKS,1,THREADS,THREADS,THREADS);
-
-  // ------------------------------------
-  // Start timing
-  // ------------------------------------
-  //  auto start = std::chrono::steady_clock::now();
- 
-       caf::cuda::manager& mgr = caf::cuda::manager::get();
-	
-  	caf::actor scheduler = mgr.get_scheduler_actor();
- 
-  
-    caf::actor exit_actor = self;
-    // Spawn num_actors actors running the mmul behavior
-    std::vector<caf::actor> actors;
-    actors.reserve(num_actors);
-    std::vector<caf::cuda::token_ptr> tokens(limit);
-    int num_actors = limit;
-   
-
-    for (int j = 0; j < num_actors; ++j) {
-       caf::actor a = sys.spawn(mmul_actor_fun, exit_actor, matrix_size);
-       actors.push_back(a);
-       caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(program,
-			dims,
-			0,
-			"hello",
-			a
-			);
-
-       tokens.emplace_back(launch_token);
-  }
-
-	self -> mail(tokens).send(scheduler);
-
-
-
-	return {
-		[=](int num_completed) {
-			self->state().completed += num_completed;
-			
-			std::cout << "Actors finished is " << self->state().completed << "\n";
-			if (self->state().completed >= limit) {
-			
-				caf::cuda::manager::shutdown();
-				self->quit();
-			}
-		}
-	};
-
-
-}
-
-
-
 
 
 
@@ -256,6 +195,72 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 }
 
 
+caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit,int matrix_size) {
+
+  int N = matrix_size;
+  caf::cuda::program_ptr program = caf::cuda::manager::get().create_program_from_cubin("../mmul.cubin","matrixMul");
+  int THREADS = 32;
+  int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS,BLOCKS,1,THREADS,THREADS,THREADS);
+
+  // ------------------------------------
+  // Start timing
+  // ------------------------------------
+  //  auto start = std::chrono::steady_clock::now();
+ 
+       caf::cuda::manager& mgr = caf::cuda::manager::get();
+	
+  	caf::actor scheduler = mgr.get_scheduler_actor();
+
+  
+    int num_actors = limit;
+    caf::actor exit_actor = self;
+    // Spawn num_actors actors running the mmul behavior
+    std::vector<caf::actor> actors;
+    actors.reserve(num_actors);
+    std::vector<caf::cuda::token_ptr> tokens(num_actors);
+   
+
+    for (int j = 0; j < num_actors; ++j) {
+       caf::actor a = self -> spawn(mmul_actor_fun, exit_actor, matrix_size);
+       actors.push_back(a);
+       caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(program,
+			dims,
+			0,
+			"hello",
+			a
+			);
+
+       tokens.emplace_back(launch_token);
+  }
+
+	self -> mail(tokens).send(scheduler);
+
+
+
+	return {
+		[=](int num_completed) {
+			self->state().completed += num_completed;
+			
+			std::cout << "Actors finished is " << self->state().completed << "\n";
+			if (self->state().completed >= limit) {
+			
+				caf::cuda::manager::shutdown();
+				self->quit();
+			}
+		}
+	};
+
+
+}
+
+
+
+
+
+
+
+
 #include <chrono>
 #include <iostream>
 
@@ -294,39 +299,6 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
 }
 
 
-
-
-//this test will spawn more actors over time to demonstrate 
-//changing scheduling algorithims at runtime 
-void run_red_light_green_light_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  std::cout << "Starting RED LIGHT GREEN LIGHT TEST\n";
-  int limit = 10;
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors * limit);
-
-  for (int i = 0; i < limit; i++) {
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
-  }
-
-
-  sleep(1);
- // std::cout << actors.size() << "\n";
-
-  }
-
-   sys.await_all_actors_done();
-}
-
-
 void run_mmul_scaling_tests(caf::actor_system& sys,caf::cuda::manager_config man_config) {
   const int min_size   = 10;
   const int max_size   = 1024;

From 907444289316ee2ff7f7d7dece87b0639ab89971 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 21 Jan 2026 15:03:51 -0600
Subject: [PATCH 0174/1000] Added some profiling to the exit actor.

---
 .../bulk-token-test/main.test.cpp             | 111 ++++++++----------
 1 file changed, 52 insertions(+), 59 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
index 0ea00f5664..c3d41d9f11 100644
--- a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
@@ -91,18 +91,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 	//set the value of N correctly to overide the base option.	
 	self->state().N = N;
 
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-	
-  	caf::actor scheduler = mgr.get_scheduler_actor();
-	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
-			self -> state().dims,
-			0,
-			"hello",
-			self
-			);
-//	self -> mail(launch_token).send(scheduler);
-
-	return {
+  	return {
 
 	  [=] (caf::cuda::response_token_ptr launch_response_token) {
 	 
@@ -195,63 +184,67 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 }
 
 
-caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit,int matrix_size) {
+caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,
+                             int limit,
+                             int matrix_size) {
 
-  int N = matrix_size;
-  caf::cuda::program_ptr program = caf::cuda::manager::get().create_program_from_cubin("../mmul.cubin","matrixMul");
-  int THREADS = 32;
-  int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS,BLOCKS,1,THREADS,THREADS,THREADS);
+    int N = matrix_size;
+    caf::cuda::program_ptr program = caf::cuda::manager::get()
+                                        .create_program_from_cubin("../mmul.cubin",
+                                                                   "matrixMul");
+    int THREADS = 32;
+    int BLOCKS = (N + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS, BLOCKS, 1, THREADS, THREADS, THREADS);
 
-  // ------------------------------------
-  // Start timing
-  // ------------------------------------
-  //  auto start = std::chrono::steady_clock::now();
- 
-       caf::cuda::manager& mgr = caf::cuda::manager::get();
-	
-  	caf::actor scheduler = mgr.get_scheduler_actor();
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+    caf::actor scheduler = mgr.get_scheduler_actor();
+    caf::actor exit_actor = self;
 
-  
     int num_actors = limit;
-    caf::actor exit_actor = self;
-    // Spawn num_actors actors running the mmul behavior
     std::vector<caf::actor> actors;
     actors.reserve(num_actors);
-    std::vector<caf::cuda::token_ptr> tokens(num_actors);
-   
 
-    for (int j = 0; j < num_actors; ++j) {
-       caf::actor a = self -> spawn(mmul_actor_fun, exit_actor, matrix_size);
-       actors.push_back(a);
-       caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(program,
-			dims,
-			0,
-			"hello",
-			a
-			);
-
-       tokens.emplace_back(launch_token);
-  }
-
-	self -> mail(tokens).send(scheduler);
+    std::vector<caf::cuda::token_ptr> tokens;
+    tokens.reserve(num_actors);
 
+    // --------------------------
+    // Start timing for token creation + send
+    // --------------------------
+    auto t_start = std::chrono::steady_clock::now();
 
+    for (int j = 0; j < num_actors; ++j) {
+        caf::actor a = self->spawn(mmul_actor_fun, exit_actor, matrix_size);
+        actors.push_back(a);
 
-	return {
-		[=](int num_completed) {
-			self->state().completed += num_completed;
-			
-			std::cout << "Actors finished is " << self->state().completed << "\n";
-			if (self->state().completed >= limit) {
-			
-				caf::cuda::manager::shutdown();
-				self->quit();
-			}
-		}
-	};
+        caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+            program, dims, 0, "hello", a);
 
+        tokens.emplace_back(std::move(launch_token));
+    }
 
+    self->mail(tokens).send(scheduler);
+
+    auto t_end = std::chrono::steady_clock::now();
+    auto elapsed_us = std::chrono::duration_cast<std::chrono::microseconds>(t_end - t_start).count();
+
+    std::cout << "[EXIT] token creation + send took "
+              << elapsed_us << " us for "
+              << tokens.size() << " actors\n";
+
+    // --------------------------
+    // Return the exit actor behavior as before
+    // --------------------------
+    return {
+        [=](int num_completed) {
+            self->state().completed += num_completed;
+
+            std::cout << "Actors finished is " << self->state().completed << "\n";
+            if (self->state().completed >= limit) {
+                caf::cuda::manager::shutdown();
+                self->quit();
+            }
+        }
+    };
 }
 
 
@@ -338,14 +331,14 @@ void caf_main(caf::actor_system& sys) {
 
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	caf::cuda::manager::init(sys,man_config);
-       	//run_mmul_test(sys,10,250);
+       	run_mmul_test(sys,512,512);
 	
 	//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	
 //	caf::cuda::manager::init(sys,man_config);
 //	run_red_light_green_light_test(sys,10,1000);
 	
-	run_mmul_scaling_tests(sys,man_config);
+	//run_mmul_scaling_tests(sys,man_config);
 
 
 }

From 1540e6a0ba1bf306f89a2adaa8c40a8d0b403ad5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 21 Jan 2026 15:14:39 -0600
Subject: [PATCH 0175/1000] Added timestamps.

---
 .../bulk-token-test/main.test.cpp               | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
index c3d41d9f11..4271674dd9 100644
--- a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
@@ -140,6 +140,14 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
     self->quit();
 
 
+     // print simple timestamp
+    auto now = std::chrono::system_clock::now();
+    std::time_t t = std::chrono::system_clock::to_time_t(now);
+    std::cout << "[GPU ACTOR] actor " << self->state().id
+              << " calling self->quit() at "
+              << std::ctime(&t); // prints human-readable time
+
+
     },
 
     // 3rd handler: CPU atom + matrices + N
@@ -188,6 +196,15 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,
                              int limit,
                              int matrix_size) {
 
+ // print simple timestamp
+    auto now = std::chrono::system_clock::now();
+    std::time_t t = std::chrono::system_clock::to_time_t(now);
+    std::cout << "[exit] actor "
+              << " starting at "
+              << std::ctime(&t); // prints human-readable time
+
+
+
     int N = matrix_size;
     caf::cuda::program_ptr program = caf::cuda::manager::get()
                                         .create_program_from_cubin("../mmul.cubin",

From fff36886b7b26c9e3f328ff14e71704a8c887ff3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 21 Jan 2026 16:39:31 -0600
Subject: [PATCH 0176/1000] Added more benchmarks.

---
 .../bulk-token-test/main.test.cpp             | 113 ++++++++++++------
 1 file changed, 74 insertions(+), 39 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
index 4271674dd9..d303c476f0 100644
--- a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
@@ -42,12 +42,7 @@ struct mmul_actor_state {
   int N = 1024; // example state variable
   int id = rand(); // an actor id
   // per-actor timing start
-  std::chrono::high_resolution_clock::time_point start_time;
   int times = 0;
-  caf::cuda::program_ptr program = caf::cuda::manager::get().create_program_from_cubin("../mmul.cubin","matrixMul");
-  int THREADS = 32;
-  int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS,BLOCKS,1,THREADS,THREADS,THREADS);
 };
 
 
@@ -138,16 +133,15 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
     //do not verify result just exit 
     self->mail(1).send(exit_actor);
     self->quit();
-
-
-     // print simple timestamp
-    auto now = std::chrono::system_clock::now();
-    std::time_t t = std::chrono::system_clock::to_time_t(now);
-    std::cout << "[GPU ACTOR] actor " << self->state().id
-              << " calling self->quit() at "
-              << std::ctime(&t); // prints human-readable time
-
-
+ // print timestamp in milliseconds
+   // auto now = std::chrono::system_clock::now();
+   // auto ms_since_epoch = std::chrono::duration_cast<std::chrono::milliseconds>(
+    //                          now.time_since_epoch())
+      //                        .count();
+
+//    std::cout << "[GPU ACTOR] actor " << self->state().id
+  //            << " calling self->quit() at "
+    //          << ms_since_epoch << " ms since epoch\n";
     },
 
     // 3rd handler: CPU atom + matrices + N
@@ -183,6 +177,21 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
     //        << ms << " ms (actor id "
       //      << self->state().id << ")\n";
 
+
+
+  /*
+   // print timestamp in milliseconds
+    auto now = std::chrono::system_clock::now();
+    auto ms_since_epoch = std::chrono::duration_cast<std::chrono::milliseconds>(
+                              now.time_since_epoch())
+                              .count();
+
+    std::cout << "[GPU ACTOR] actor " << self->state().id
+              << " calling self->quit() at "
+              << ms_since_epoch << " ms since epoch\n";
+
+	      */
+
   // signal exit actor and quit
   self->mail(1).send(exit_actor);
   self->quit();
@@ -196,14 +205,15 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,
                              int limit,
                              int matrix_size) {
 
- // print simple timestamp
-    auto now = std::chrono::system_clock::now();
-    std::time_t t = std::chrono::system_clock::to_time_t(now);
-    std::cout << "[exit] actor "
-              << " starting at "
-              << std::ctime(&t); // prints human-readable time
-
+ // print timestamp in milliseconds
+   // auto now = std::chrono::system_clock::now();
+   // auto ms_since_epoch = std::chrono::duration_cast<std::chrono::milliseconds>(
+     //                         now.time_since_epoch())
+       //                       .count();
 
+    //std::cout << "[exit] actor "
+      //        << " starting at "
+        //      << ms_since_epoch << " ms since epoch\n";
 
     int N = matrix_size;
     caf::cuda::program_ptr program = caf::cuda::manager::get()
@@ -227,27 +237,52 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,
     // --------------------------
     // Start timing for token creation + send
     // --------------------------
-    auto t_start = std::chrono::steady_clock::now();
-
-    for (int j = 0; j < num_actors; ++j) {
-        caf::actor a = self->spawn(mmul_actor_fun, exit_actor, matrix_size);
-        actors.push_back(a);
-
-        caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-            program, dims, 0, "hello", a);
+   auto t_start_all = std::chrono::steady_clock::now();
+
+long long total_spawn_us = 0;
+long long total_token_us = 0;
+
+for (int j = 0; j < num_actors; ++j) {
+    auto t_spawn_start = std::chrono::steady_clock::now();
+    caf::actor a = self->spawn(mmul_actor_fun, exit_actor, matrix_size);
+    actors.push_back(a);
+    auto t_spawn_end = std::chrono::steady_clock::now();
+
+    total_spawn_us +=
+        std::chrono::duration_cast<std::chrono::microseconds>(
+            t_spawn_end - t_spawn_start).count();
+
+    auto t_token_start = std::chrono::steady_clock::now();
+    caf::cuda::token_ptr launch_token =
+        caf::cuda::make_launch_token(program, dims, 0, "hello", a);
+    tokens.emplace_back(std::move(launch_token));
+    auto t_token_end = std::chrono::steady_clock::now();
+
+    total_token_us +=
+        std::chrono::duration_cast<std::chrono::microseconds>(
+            t_token_end - t_token_start).count();
+}
 
-        tokens.emplace_back(std::move(launch_token));
-    }
 
-    self->mail(tokens).send(scheduler);
+// --------------------------
+// Time sending all tokens
+// --------------------------
+auto t_send_start = std::chrono::steady_clock::now();
+self->mail(tokens).send(scheduler);
+auto t_send_end = std::chrono::steady_clock::now();
+auto send_ms = std::chrono::duration_cast<std::chrono::milliseconds>(t_send_end - t_send_start).count();
 
-    auto t_end = std::chrono::steady_clock::now();
-    auto elapsed_us = std::chrono::duration_cast<std::chrono::microseconds>(t_end - t_start).count();
+auto t_end_all = std::chrono::steady_clock::now();
+auto total_ms = std::chrono::duration_cast<std::chrono::milliseconds>(t_end_all - t_start_all).count();
 
-    std::cout << "[EXIT] token creation + send took "
-              << elapsed_us << " us for "
-              << tokens.size() << " actors\n";
+std::cout << "[EXIT] total spawn time: "
+          << total_spawn_us / 1000.0 << " ms\n";
+std::cout << "[EXIT] total launch token creation time: "
+          << total_token_us / 1000.0 << " ms\n";
 
+std::cout << "[EXIT] sending tokens took: " << send_ms << " ms\n";
+std::cout << "[EXIT] total elapsed time (spawn + token + send): " << total_ms << " ms for "
+          << num_actors << " actors\n";
     // --------------------------
     // Return the exit actor behavior as before
     // --------------------------
@@ -255,7 +290,7 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,
         [=](int num_completed) {
             self->state().completed += num_completed;
 
-            std::cout << "Actors finished is " << self->state().completed << "\n";
+            //std::cout << "Actors finished is " << self->state().completed << "\n";
             if (self->state().completed >= limit) {
                 caf::cuda::manager::shutdown();
                 self->quit();

From 345cb4024b5a0754e5564a7f8cee9e53278bdf63 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 21 Jan 2026 16:51:30 -0600
Subject: [PATCH 0177/1000] Added even more benchmarks.

---
 .../bulk-token-test/main.test.cpp             | 146 ++++++++++--------
 1 file changed, 83 insertions(+), 63 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
index d303c476f0..dba6834ba1 100644
--- a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
@@ -23,6 +23,8 @@ using namespace std::chrono_literals;
 
 struct exit_actor_state {
 	int completed = 0;
+    std::chrono::steady_clock::time_point start_time;
+
 };
 
 
@@ -205,23 +207,20 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,
                              int limit,
                              int matrix_size) {
 
- // print timestamp in milliseconds
-   // auto now = std::chrono::system_clock::now();
-   // auto ms_since_epoch = std::chrono::duration_cast<std::chrono::milliseconds>(
-     //                         now.time_since_epoch())
-       //                       .count();
-
-    //std::cout << "[exit] actor "
-      //        << " starting at "
-        //      << ms_since_epoch << " ms since epoch\n";
+    // ------------------------------------
+    // Record exit actor start time
+    // ------------------------------------
+    self->state().start_time = std::chrono::steady_clock::now();
 
     int N = matrix_size;
     caf::cuda::program_ptr program = caf::cuda::manager::get()
-                                        .create_program_from_cubin("../mmul.cubin",
-                                                                   "matrixMul");
+        .create_program_from_cubin("../mmul.cubin", "matrixMul");
+
     int THREADS = 32;
     int BLOCKS = (N + THREADS - 1) / THREADS;
-    caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS, BLOCKS, 1, THREADS, THREADS, THREADS);
+    caf::cuda::nd_range dims(
+        BLOCKS, BLOCKS, 1,
+        THREADS, THREADS, THREADS);
 
     caf::cuda::manager& mgr = caf::cuda::manager::get();
     caf::actor scheduler = mgr.get_scheduler_actor();
@@ -234,64 +233,84 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,
     std::vector<caf::cuda::token_ptr> tokens;
     tokens.reserve(num_actors);
 
-    // --------------------------
-    // Start timing for token creation + send
-    // --------------------------
-   auto t_start_all = std::chrono::steady_clock::now();
-
-long long total_spawn_us = 0;
-long long total_token_us = 0;
-
-for (int j = 0; j < num_actors; ++j) {
-    auto t_spawn_start = std::chrono::steady_clock::now();
-    caf::actor a = self->spawn(mmul_actor_fun, exit_actor, matrix_size);
-    actors.push_back(a);
-    auto t_spawn_end = std::chrono::steady_clock::now();
-
-    total_spawn_us +=
-        std::chrono::duration_cast<std::chrono::microseconds>(
-            t_spawn_end - t_spawn_start).count();
-
-    auto t_token_start = std::chrono::steady_clock::now();
-    caf::cuda::token_ptr launch_token =
-        caf::cuda::make_launch_token(program, dims, 0, "hello", a);
-    tokens.emplace_back(std::move(launch_token));
-    auto t_token_end = std::chrono::steady_clock::now();
-
-    total_token_us +=
-        std::chrono::duration_cast<std::chrono::microseconds>(
-            t_token_end - t_token_start).count();
-}
+    // ------------------------------------
+    // Timing accumulators
+    // ------------------------------------
+    auto t_start_all = std::chrono::steady_clock::now();
+
+    long long total_spawn_us = 0;
+    long long total_token_us = 0;
+
+    // ------------------------------------
+    // Spawn actors + create launch tokens
+    // ------------------------------------
+    for (int j = 0; j < num_actors; ++j) {
+
+        auto t_spawn_start = std::chrono::steady_clock::now();
+        caf::actor a = self->spawn(mmul_actor_fun, exit_actor, matrix_size);
+        actors.push_back(a);
+        auto t_spawn_end = std::chrono::steady_clock::now();
+
+        total_spawn_us += std::chrono::duration_cast<
+            std::chrono::microseconds>(t_spawn_end - t_spawn_start).count();
+
+        auto t_token_start = std::chrono::steady_clock::now();
+        caf::cuda::token_ptr launch_token =
+            caf::cuda::make_launch_token(program, dims, 0, "hello", a);
+        tokens.emplace_back(std::move(launch_token));
+        auto t_token_end = std::chrono::steady_clock::now();
 
+        total_token_us += std::chrono::duration_cast<
+            std::chrono::microseconds>(t_token_end - t_token_start).count();
+    }
+
+    // ------------------------------------
+    // Send tokens to scheduler
+    // ------------------------------------
+    auto t_send_start = std::chrono::steady_clock::now();
+    self->mail(tokens).send(scheduler);
+    auto t_send_end = std::chrono::steady_clock::now();
+
+    auto send_ms = std::chrono::duration_cast<
+        std::chrono::milliseconds>(t_send_end - t_send_start).count();
+
+    auto t_end_all = std::chrono::steady_clock::now();
+    auto total_ms = std::chrono::duration_cast<
+        std::chrono::milliseconds>(t_end_all - t_start_all).count();
+
+    // ------------------------------------
+    // Print setup timings
+    // ------------------------------------
+    std::cout << "[EXIT] total spawn time: "
+              << total_spawn_us / 1000.0 << " ms\n";
+
+    std::cout << "[EXIT] total launch token creation time: "
+              << total_token_us / 1000.0 << " ms\n";
 
-// --------------------------
-// Time sending all tokens
-// --------------------------
-auto t_send_start = std::chrono::steady_clock::now();
-self->mail(tokens).send(scheduler);
-auto t_send_end = std::chrono::steady_clock::now();
-auto send_ms = std::chrono::duration_cast<std::chrono::milliseconds>(t_send_end - t_send_start).count();
-
-auto t_end_all = std::chrono::steady_clock::now();
-auto total_ms = std::chrono::duration_cast<std::chrono::milliseconds>(t_end_all - t_start_all).count();
-
-std::cout << "[EXIT] total spawn time: "
-          << total_spawn_us / 1000.0 << " ms\n";
-std::cout << "[EXIT] total launch token creation time: "
-          << total_token_us / 1000.0 << " ms\n";
-
-std::cout << "[EXIT] sending tokens took: " << send_ms << " ms\n";
-std::cout << "[EXIT] total elapsed time (spawn + token + send): " << total_ms << " ms for "
-          << num_actors << " actors\n";
-    // --------------------------
-    // Return the exit actor behavior as before
-    // --------------------------
+    std::cout << "[EXIT] sending tokens took: "
+              << send_ms << " ms\n";
+
+    std::cout << "[EXIT] total elapsed time (spawn + token + send): "
+              << total_ms << " ms for "
+              << num_actors << " actors\n";
+
+    // ------------------------------------
+    // Exit actor behavior
+    // ------------------------------------
     return {
         [=](int num_completed) {
             self->state().completed += num_completed;
 
-            //std::cout << "Actors finished is " << self->state().completed << "\n";
             if (self->state().completed >= limit) {
+
+                auto end_time = std::chrono::steady_clock::now();
+                auto lifetime_ms = std::chrono::duration_cast<
+                    std::chrono::milliseconds>(
+                        end_time - self->state().start_time).count();
+
+                std::cout << "[EXIT] exit actor lifetime: "
+                          << lifetime_ms << " ms\n";
+
                 caf::cuda::manager::shutdown();
                 self->quit();
             }
@@ -306,6 +325,7 @@ std::cout << "[EXIT] total elapsed time (spawn + token + send): " << total_ms <<
 
 
 
+
 #include <chrono>
 #include <iostream>
 

From 6da92151a835255847627c67b39a896817f777ef Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 21 Jan 2026 17:09:10 -0600
Subject: [PATCH 0178/1000] Saving for now.

---
 .../bulk-token-test/main.test.cpp              | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
index dba6834ba1..5ebdae1ec0 100644
--- a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
@@ -82,7 +82,7 @@ void serial_matrix_multiply(const std::vector<int>& a,
 
 
 // Stateful actor behavior
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N, const std::vector<int>& matrix1, const std::vector<int>& matrix2) {
  
 
 	//set the value of N correctly to overide the base option.	
@@ -95,10 +95,6 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::ac
 		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
 		  //assume N = 1024
 		  int N = self -> state().N;
-		  std::vector<int> matrix1(N*N);
-		  matrix1.reserve(N);
-		  std::vector<int> matrix2(N*N);
-		  matrix2.reserve(N);
 
 		  //std::cout << "GPU ACTOR sending data to compute\n";
 		  self -> mail(matrix1,matrix2,N).send(self);
@@ -210,9 +206,15 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,
     // ------------------------------------
     // Record exit actor start time
     // ------------------------------------
-    self->state().start_time = std::chrono::steady_clock::now();
-
+    
     int N = matrix_size;
+			  std::vector<int> matrix1(N*N);
+		  std::vector<int> matrix2(N*N);
+
+
+	
+   self->state().start_time = std::chrono::steady_clock::now();
+
     caf::cuda::program_ptr program = caf::cuda::manager::get()
         .create_program_from_cubin("../mmul.cubin", "matrixMul");
 
@@ -247,7 +249,7 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,
     for (int j = 0; j < num_actors; ++j) {
 
         auto t_spawn_start = std::chrono::steady_clock::now();
-        caf::actor a = self->spawn(mmul_actor_fun, exit_actor, matrix_size);
+        caf::actor a = self->spawn(mmul_actor_fun, exit_actor, matrix_size,matrix1,matrix2);
         actors.push_back(a);
         auto t_spawn_end = std::chrono::steady_clock::now();
 

From 046d9c3f0d841ab0dcf94ff8b5c86e055d6ee9c8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 21 Jan 2026 17:13:27 -0600
Subject: [PATCH 0179/1000] Adeded an std::vector<token_ptr> message handler.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 7d9e54db01..9f3dfb9c49 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -48,6 +48,19 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,i
 		
 	    }
         },
+
+	[=](std::vector<token_ptr> tokens) {
+	
+		for (int i =0; i<tokens.size();i++) {
+		
+		self->state().current_behavior->receive(&self->state(), tokens[i]);
+
+		
+		}
+
+	},
+
+
 	[=](std::string word) {
 	//	std::cout << "Received message " << word << "\n";
 	},

From e134be7b025f51f45c136a364e190b9906704f6d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 22 Jan 2026 18:10:46 -0600
Subject: [PATCH 0180/1000] Refactored behavior classes by making actor state
 part of their constructor, and moved green lights process token helpers up to
 behavior. CHanges were made to make the code easier to maintain and benefit
 more from oop principles.

---
 .../caf/cuda/control-layer/behavior.hpp       | 34 +++++++++++--------
 .../control-layer/green_light_behavior.hpp    |  9 ++---
 .../cuda/control-layer/red_light_behavior.hpp | 10 +++---
 3 files changed, 26 insertions(+), 27 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
index 2ea21c0ad1..c13bb7c3d8 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
@@ -1,7 +1,12 @@
-#pragma once 
+#pragma once
 #include "caf/cuda/control-layer/launch_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
+#include "caf/cuda/control-layer/memory_transfer_token.hpp"
 #include "caf/cuda/control-layer/scheduler_actor_state.hpp"
+#include "caf/cuda/control-layer/token.hpp"
+#include "caf/cuda/control-layer/token_factory.hpp"
+
+
 
 
 //this class is meant to provide an interface so that 
@@ -14,25 +19,24 @@
 
 namespace caf::cuda {
 
-struct scheduler_actor_state;
-class launch_token;
-
 class scheduler_actor_behavior {
 public:
+    explicit scheduler_actor_behavior(scheduler_actor_state& state)
+        : state_(state) {}
     virtual ~scheduler_actor_behavior() = default;
 
+    virtual void on_enter() {}
+    virtual void on_exit() {}
+
     virtual void schedule() = 0;
-    virtual void receive(scheduler_actor_state* state, const token_ptr& tok) = 0;
-    
-    //define what to do when transitioning into the state
-    virtual void init(scheduler_actor_state * state) {
-    	//If the behavior decides not to overide do nothing
-    }
-
-    //define what to do when transitioning out of the state
-    virtual void cleanup(scheduler_actor_state * state) { 
-    	//If the behavior decides not to overide do nothing
-    }
+    virtual void receive(const token_ptr& tok) = 0;
+
+protected:
+    scheduler_actor_state& state_;
+
+    // Default implementation (immediate response) – takes token_ptr and casts internally
+    virtual void process_launch_token(const token_ptr& tok, int stream_id);
+    virtual void process_memory_transfer_token(const token_ptr& tok, int stream_id);
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
index 9559889484..c26adadc28 100644
--- a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
@@ -5,14 +5,11 @@ namespace caf::cuda {
 
 class green_light_behavior : public scheduler_actor_behavior {
 public:
+    explicit green_light_behavior(scheduler_actor_state& state);
+    void on_enter() override;
     void schedule() override;
+    void receive(const token_ptr& tok) override;
 
-    void receive(scheduler_actor_state* state, const token_ptr& tok) override; 
-    void init(scheduler_actor_state* state); 
-    void process_launch_token(const token_ptr& tok,caf::actor self,int device_number,int stream_id);
-    void process_memory_transfer_token(const token_ptr& tok, caf::actor self,int device_number,int stream_id);
 };
 
-
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
index 4d1425e515..74db486e09 100644
--- a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
@@ -5,14 +5,12 @@ namespace caf::cuda {
 
 class red_light_behavior : public scheduler_actor_behavior {
 public:
+    explicit red_light_behavior(scheduler_actor_state& state);
+    void on_enter() override;
     void schedule() override;
-
-    void receive(scheduler_actor_state* state, const token_ptr& tok) override;
-    void init(scheduler_actor_state* state);
-    //void cleanup(scheduler_actor_state* state);
+    void receive(const token_ptr& tok) override;
     ~red_light_behavior() noexcept override;
-};
 
+};
 
 } // namespace caf::cuda
-

From b5f927220611122bb34cf79d98cdaba635e5e5af Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 22 Jan 2026 18:11:09 -0600
Subject: [PATCH 0181/1000] Refactored behavior classes by making actor state
 part of their constructor, and moved green lights process token helpers up to
 behavior. CHanges were made to make the code easier to maintain and benefit
 more from oop principles.

---
 libcaf_cuda/src/control-layer/behavior.cpp    |  19 ++++
 .../control-layer/green_light_behavior.cpp    | 103 ++++++------------
 .../src/control-layer/red_light_behavior.cpp  |  40 ++++---
 .../src/control-layer/scheduler_actor.cpp     |  84 ++++++--------
 4 files changed, 108 insertions(+), 138 deletions(-)
 create mode 100644 libcaf_cuda/src/control-layer/behavior.cpp

diff --git a/libcaf_cuda/src/control-layer/behavior.cpp b/libcaf_cuda/src/control-layer/behavior.cpp
new file mode 100644
index 0000000000..3b27e6fbe1
--- /dev/null
+++ b/libcaf_cuda/src/control-layer/behavior.cpp
@@ -0,0 +1,19 @@
+#include "caf/cuda/control-layer/all-control-layer.hpp"
+#include "caf/cuda/control-layer/behavior.hpp"
+#include "caf/all.hpp"
+
+namespace caf::cuda {
+
+void scheduler_actor_behavior::process_launch_token(const token_ptr& tok, int stream_id) {
+    const auto& launch = static_cast<const launch_token&>(*tok);
+    auto response = make_launch_response_token(state_.self, launch, state_.device_number, stream_id);
+    anon_mail(response).send(launch.getReplyActor());
+}
+
+void scheduler_actor_behavior::process_memory_transfer_token(const token_ptr& tok, int stream_id) {
+    const auto& mem = static_cast<const memory_transfer_token&>(*tok);
+    auto response = make_memory_response_token(state_.self, mem, state_.device_number, stream_id);
+    anon_mail(response).send(mem.getReplyActor());
+}
+
+} // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index 8fe2680898..ade904ea40 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -4,92 +4,51 @@
 
 namespace caf::cuda {
 
-    void green_light_behavior::schedule() {
+green_light_behavior::green_light_behavior(scheduler_actor_state& state)
+    : scheduler_actor_behavior(state) {}
+
+void green_light_behavior::on_enter() {
+    std::cout << "GREEN LIGHT\n";
+
+    behavior_token_ptr red_light = make_behavior_token("red");
+    //send a request to change behavior to red light after 5 seconds
+    /*
+    anon_mail(red_light)
+            .delay(std::chrono::seconds(5))
+            .send(state_.self);
+    */
+}
+
+void green_light_behavior::schedule() {
     //TODO IMPLEMENT
-    }
-void green_light_behavior::receive(scheduler_actor_state* state,
-                                   const token_ptr& tok) {
+}
 
+void green_light_behavior::receive(const token_ptr& tok) {
     if (tok->getType() == LAUNCH) {
-	//use 0 as stream id for now, eventually will have to figure out
-	//stream load balancing 
-        process_launch_token(tok, state->self,state -> device_number,0);
+        //use 0 as stream id for now, eventually will have to figure out
+        //stream load balancing
+        process_launch_token(tok, 0);
     }
     else if (tok->getType() == MEMORY) {
-	//use 0 as stream id for now, eventually will have to figure out
-	//stream load balancing 
-        process_memory_transfer_token(tok, state->self,state -> device_number,0);
+        //use 0 as stream id for now, eventually will have to figure out
+        //stream load balancing
+        process_memory_transfer_token(tok, 0);
     }
-
-    //this may cause an issue if a message is never received then 
-    //we may never end up dequeueing certain requests 
+    //this may cause an issue if a message is never received then
+    //we may never end up dequeueing certain requests
     //may lead to a deadlock scenario?
-    while (!state->queue.empty()) {
-        token_ptr queued = state->queue.front();
-        state->queue.pop();
-
+    while (!state_.queue.empty()) {
+        token_ptr queued = state_.queue.front();
+        state_.queue.pop();
         if (queued->getType() == LAUNCH) {
             // safe: we've checked the runtime type
-            process_launch_token(tok, state->self,state -> device_number,0);
+            process_launch_token(queued, 0);
         }
         else if (queued->getType() == MEMORY) {
-            process_memory_transfer_token(tok, state->self,state -> device_number,0);
+            process_memory_transfer_token(queued, 0);
         }
     }
 }
 
 
-   void green_light_behavior::init(scheduler_actor_state * state) {
-	   std::cout << "GREEN LIGHT\n";
-
-	   
-	   behavior_token_ptr red_light = make_behavior_token("red");
-
-	   //send a request to change behavior to green light after 5 seconds
-    	   /*
-	   anon_mail(red_light)
-		   .delay(std::chrono::seconds(5))
-		   .send(state -> self);
-	   */
-    }
-
-//--------------------------------------------------
-// process_launch_token
-//--------------------------------------------------
-void green_light_behavior::process_launch_token(const token_ptr& tok,
-		caf::actor self,
-		int device_number,
-		int stream_id) {
-    caf::cuda::launch_token& launch =
-        static_cast<caf::cuda::launch_token&>(*tok);
-
-    // Create response token using the existing launch token
-    caf::cuda::response_token_ptr response =
-        make_launch_response_token(self, launch,device_number,stream_id);
-
-    // Send response to the actor that requested the launch
-    anon_mail(response).send(launch.getReplyActor());
-}
-
-//--------------------------------------------------
-// process_memory_transfer_token
-//--------------------------------------------------
-void green_light_behavior::process_memory_transfer_token(const token_ptr& tok,
-                                                         caf::actor self,
-							 int device_number,
-							 int stream_id) {
-    caf::cuda::memory_transfer_token& mem =
-        static_cast<caf::cuda::memory_transfer_token&>(*tok);
-
-    // Create memory response token
-    caf::cuda::response_token_ptr response =
-        make_memory_response_token(self, mem,device_number,stream_id);
-
-    // Send response to requesting actor
-    anon_mail(response).send(mem.getReplyActor());
-}
-
-
-
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/src/control-layer/red_light_behavior.cpp b/libcaf_cuda/src/control-layer/red_light_behavior.cpp
index bad3a83c0c..bc4dbe9e2c 100644
--- a/libcaf_cuda/src/control-layer/red_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/red_light_behavior.cpp
@@ -1,26 +1,36 @@
 #include "caf/cuda/control-layer/all-control-layer.hpp"
 #include "caf/cuda/control-layer/red_light_behavior.hpp"
 #include "caf/cuda/control-layer/token_factory.hpp"
+#include <iostream>
 
 namespace caf::cuda {
 
-    void red_light_behavior::schedule() {
-    }
+red_light_behavior::red_light_behavior(scheduler_actor_state& state)
+    : scheduler_actor_behavior(state) {}
 
-    void red_light_behavior::receive(scheduler_actor_state* state, const token_ptr& tok) {
-        state->queue.push(tok); // enqueue everything
-    }
+void red_light_behavior::schedule() {}
 
-    red_light_behavior::~red_light_behavior() noexcept = default;
-    
-    void red_light_behavior::init(scheduler_actor_state * state) {
-	   std::cout << "RED LIGHT\n";
-	   behavior_token_ptr green_light = make_behavior_token("green");
+void red_light_behavior::receive(const token_ptr& tok) {
+    state_.queue.push(tok); // enqueue everything
+}
 
-	   //send a request to change behavior to green light after 5 seconds
-    	   anon_mail(green_light)
-		   .delay(std::chrono::seconds(5))
-		   .send(state -> self);
-    }
+red_light_behavior::~red_light_behavior() noexcept = default;
+
+void red_light_behavior::on_enter() {
+    std::cout << "RED LIGHT\n";
+    behavior_token_ptr green_light = make_behavior_token("green");
+    //send a request to change behavior to green light after 5 seconds
+    anon_mail(green_light)
+            .delay(std::chrono::seconds(5))
+            .send(state_.self);
+}
+
+void red_light_behavior::process_launch_token(const token_ptr& tok, int stream_id) {
+    // Override to do nothing, as red light should not process
+}
+
+void red_light_behavior::process_memory_transfer_token(const token_ptr& tok, int stream_id) {
+    // Override to do nothing
+}
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 9f3dfb9c49..c16fb2435a 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -6,70 +6,54 @@
 #include <iostream>
 
 /*
- * This class is meant to handle actor GPU scheduling via s/r/r IPC 
- * it has nothing to do with the scheduler class, that is kernel laye
- */ 
-
+ * This class is meant to handle actor GPU scheduling via s/r/r IPC
+ * it has nothing to do with the scheduler class, that is kernel layer
+ */
 namespace caf::cuda {
 
-caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,int device_number) {
-
-    //add its self reference
-    self -> state().self = self;
+caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self, int device_number) {
+    auto& state = self->state();
 
-    //set device number
-    self -> state().device_number = device_number;
+    // add self reference
+    state.self = self;
 
-    // populate the table
-    static red_light_behavior red_behavior;
-    static green_light_behavior green_behavior;
+    // set device number
+    state.device_number = device_number;
 
-    self->state().table.add("green", &green_behavior);
-    self->state().table.add("red", &red_behavior);
+    // populate the behavior table
+    state.table.add("green", new green_light_behavior(state));
+    state.table.add("red",   new red_light_behavior(state));
 
     // default behavior
-    self->state().current_behavior = self->state().table.get(behavior_token("green"));
-    self->state().current_behavior -> init(&self->state());
+    state.current_behavior = state.table.get(behavior_token("green"));
+    state.current_behavior->on_enter();
 
     return {
         [=](const token_ptr& tok) {
-           
-
-	//	std::cout << "Received token\n";
-		self->state().current_behavior->receive(&self->state(), tok);
+            // std::cout << "Received token\n";
+            state.current_behavior->receive(tok);
         },
         [=](const caf::cuda::behavior_token_ptr& tok) {
-            auto* next = self->state().table.get(*tok);
+            auto* next = state.table.get(*tok);
             if (next) {
-                
-		    self->state().current_behavior -> cleanup(&self->state()); //cleanup current behavior
-		    self->state().current_behavior = next;  // swap behavior
-		    self->state().current_behavior -> init(&self->state()); //init new current behavior
-		
-	    }
+                state.current_behavior->on_exit();   // cleanup current behavior
+                state.current_behavior = next;       // swap behavior
+                state.current_behavior->on_enter();  // init new current behavior
+            }
         },
-
-	[=](std::vector<token_ptr> tokens) {
-	
-		for (int i =0; i<tokens.size();i++) {
-		
-		self->state().current_behavior->receive(&self->state(), tokens[i]);
-
-		
-		}
-
-	},
-
-
-	[=](std::string word) {
-	//	std::cout << "Received message " << word << "\n";
-	},
-	 [=](caf::cuda::mem_ptr<int> token) {
+        [=](std::vector<token_ptr> tokens) {
+            for (size_t i = 0; i < tokens.size(); ++i) {
+                state.current_behavior->receive(tokens[i]);
+            }
+        },
+        [=](std::string word) {
+            // std::cout << "Received message " << word << "\n";
+        },
+        [=](caf::cuda::mem_ptr<int> token) {
             if (!token) {
                 std::cout << "Received null mem_ptr\n";
                 return;
             }
-
             if (token->is_scalar()) {
                 std::cout << "Received mem_ptr with scalar value: "
                           << *token->host_scalar_ptr() << "\n";
@@ -77,14 +61,12 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,i
                 std::cout << "Received mem_ptr with "
                           << token->size() << " elements\n";
                 // Optional: print fake data if testing copy_to_host
-               // auto host_data = token->copy_to_host();
-               // for (auto v : host_data) std::cout << v << " ";
-                //std::cout << "\n";
+                // auto host_data = token->copy_to_host();
+                // for (auto v : host_data) std::cout << v << " ";
+                // std::cout << "\n";
             }
         }
-
     };
 }
 
 } // namespace caf::cuda
-

From c5f00865a9e64caf5a601fa79c71528ab8f6ec6c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 22 Jan 2026 18:12:15 -0600
Subject: [PATCH 0182/1000] Added behavior.cpp to build.

---
 libcaf_cuda/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/CMakeLists.txt b/libcaf_cuda/CMakeLists.txt
index 20f395b573..cd68c65efa 100644
--- a/libcaf_cuda/CMakeLists.txt
+++ b/libcaf_cuda/CMakeLists.txt
@@ -22,6 +22,7 @@ set(LIBCAF_CUDA_SRCS
   src/test.cpp
   src/scheduler.cpp
   src/streampool.cpp
+  src/control-layer/behavior.cpp
   src/control-layer/scheduler_actor.cpp
   src/control-layer/red_light_behavior.cpp
   src/control-layer/green_light_behavior.cpp

From 7fa9af306d52a5ee1237654bc8768c28e10eed0e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 22 Jan 2026 18:15:50 -0600
Subject: [PATCH 0183/1000] Made getReplyActor declare return a const Change is
 made to fix compiler errors and be more c++ program stylish.

---
 libcaf_cuda/caf/cuda/control-layer/launch_token.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
index 65b2557037..38d5f1d7da 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
@@ -31,7 +31,7 @@ class CAF_CUDA_EXPORT launch_token : public token {
     const program_ptr& getProgram() const { return program_; }
     const nd_range& getRange() const { return range_; }
     int getMemoryUsage() const { return memory_usage_; }
-    caf::actor getReplyActor() { return reply_handle_; }
+    caf::actor getReplyActor() const { return reply_handle_; }
     // Return requested number of CUDA blocks (gridDimX * gridDimY * gridDimZ)
     int getBlocks() const {
         return static_cast<int>(

From dfd72c9fd3bef24e07726f8c6325944f33a6a5f2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 22 Jan 2026 18:24:21 -0600
Subject: [PATCH 0184/1000] Added deconstructor that will delete behaviors on
 deconstruction.

---
 libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
index 61081c6d1f..02219f60c0 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
@@ -14,6 +14,15 @@ class behavior_table {
     void add(const std::string& name, scheduler_actor_behavior* beh) {
         table_[name] = beh;
     }
+ 
+    // Destructor: clean up all owned behaviors
+    ~behavior_table() {
+        for (auto& [name, beh] : table_) {
+            delete beh;
+        }
+        table_.clear();  // optional, but good hygiene
+    }
+ 
     scheduler_actor_behavior* get(const behavior_token& tok) const {
         auto it = table_.find(tok.name());
         return it != table_.end() ? it->second : nullptr;

From 2864cff2e60e424bf2739167a93401a4004714f0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 22 Jan 2026 18:28:48 -0600
Subject: [PATCH 0185/1000] Added behavior_table.cpp to build.

---
 libcaf_cuda/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/CMakeLists.txt b/libcaf_cuda/CMakeLists.txt
index cd68c65efa..45fa0cad16 100644
--- a/libcaf_cuda/CMakeLists.txt
+++ b/libcaf_cuda/CMakeLists.txt
@@ -23,6 +23,7 @@ set(LIBCAF_CUDA_SRCS
   src/scheduler.cpp
   src/streampool.cpp
   src/control-layer/behavior.cpp
+  src/control-layer/behavior_table.cpp
   src/control-layer/scheduler_actor.cpp
   src/control-layer/red_light_behavior.cpp
   src/control-layer/green_light_behavior.cpp

From 5d914c0475dffdb1a68838f4894ca67ca660eb03 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 22 Jan 2026 18:32:06 -0600
Subject: [PATCH 0186/1000] Refactored behavior_table into hpp and cpp files
 Change was made to ensure deletion of objects could occur correctly.

---
 .../caf/cuda/control-layer/behavior_table.hpp | 26 +++++++------------
 1 file changed, 10 insertions(+), 16 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
index 02219f60c0..210a2991a9 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
@@ -1,34 +1,28 @@
 #pragma once
-#include "caf/cuda/control-layer/token.hpp"
-#include "caf/cuda/control-layer/behavior_token.hpp"
 #include <unordered_map>
 #include <string>
 
 namespace caf::cuda {
-class scheduler_actor_behavior;  // Forward decl
+
+class scheduler_actor_behavior; // Forward declaration is fine here
 class behavior_token;
 
 class behavior_table {
 public:
     behavior_table() = default;
+
+    ~behavior_table();  // ← Declaration only
+
     void add(const std::string& name, scheduler_actor_behavior* beh) {
         table_[name] = beh;
     }
- 
-    // Destructor: clean up all owned behaviors
-    ~behavior_table() {
-        for (auto& [name, beh] : table_) {
-            delete beh;
-        }
-        table_.clear();  // optional, but good hygiene
-    }
- 
-    scheduler_actor_behavior* get(const behavior_token& tok) const {
-        auto it = table_.find(tok.name());
-        return it != table_.end() ? it->second : nullptr;
-    }
+
+    scheduler_actor_behavior* get(const behavior_token& tok) const;
     auto& all_behaviors() { return table_; }
+    const auto& all_behaviors() const { return table_; }
+
 private:
     std::unordered_map<std::string, scheduler_actor_behavior*> table_;
 };
+
 } // namespace caf::cuda

From 373584d74a8514c927e9b1a4bf3fb7e8e4f05d5d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 22 Jan 2026 18:32:44 -0600
Subject: [PATCH 0187/1000] Fixed syntax and compiler errors.

---
 libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp | 4 ++++
 libcaf_cuda/src/control-layer/scheduler_actor.cpp         | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
index 74db486e09..c5c00057c1 100644
--- a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
@@ -11,6 +11,10 @@ class red_light_behavior : public scheduler_actor_behavior {
     void receive(const token_ptr& tok) override;
     ~red_light_behavior() noexcept override;
 
+protected:
+    virtual void process_launch_token(const token_ptr& tok, int stream_id);
+    virtual void process_memory_transfer_token(const token_ptr& tok, int stream_id);
+
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index c16fb2435a..2f2e822f72 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -33,7 +33,7 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
             // std::cout << "Received token\n";
             state.current_behavior->receive(tok);
         },
-        [=](const caf::cuda::behavior_token_ptr& tok) {
+        [&state](const caf::cuda::behavior_token_ptr& tok) {
             auto* next = state.table.get(*tok);
             if (next) {
                 state.current_behavior->on_exit();   // cleanup current behavior

From 160edf7461e99ff37922e38a0d7ae43f12894523 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 22 Jan 2026 18:46:19 -0600
Subject: [PATCH 0188/1000] Changed things in reguards to how behaviors are
 allocated and deallocated to fix segfault and memory leak issues.

---
 .../src/control-layer/behavior_table.cpp      | 24 +++++++++++++++++++
 .../src/control-layer/scheduler_actor.cpp     |  8 +++++--
 2 files changed, 30 insertions(+), 2 deletions(-)
 create mode 100644 libcaf_cuda/src/control-layer/behavior_table.cpp

diff --git a/libcaf_cuda/src/control-layer/behavior_table.cpp b/libcaf_cuda/src/control-layer/behavior_table.cpp
new file mode 100644
index 0000000000..1ea2286a4f
--- /dev/null
+++ b/libcaf_cuda/src/control-layer/behavior_table.cpp
@@ -0,0 +1,24 @@
+#include "caf/cuda/control-layer/behavior_table.hpp"
+
+// IMPORTANT: Include the full definition of the base class
+#include "caf/cuda/control-layer/behavior.hpp"  
+#include "caf/cuda/control-layer/behavior_token.hpp"  // ← full type here
+
+// If your derived classes have important cleanup, also include them if needed
+#include "caf/cuda/control-layer/green_light_behavior.hpp"
+#include "caf/cuda/control-layer/red_light_behavior.hpp"
+
+namespace caf::cuda {
+
+
+scheduler_actor_behavior* behavior_table::get(const behavior_token& tok) const {
+    auto it = table_.find(tok.name());
+    return it != table_.end() ? it->second : nullptr;
+}
+
+
+behavior_table::~behavior_table() {
+    table_.clear();  // optional but clean
+}
+
+} // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 2f2e822f72..109aad3b59 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -20,9 +20,13 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
     // set device number
     state.device_number = device_number;
 
+    static red_light_behavior red_behavior(state);
+    static green_light_behavior green_behavior(state);
+
+
     // populate the behavior table
-    state.table.add("green", new green_light_behavior(state));
-    state.table.add("red",   new red_light_behavior(state));
+    state.table.add("red", &red_behavior);
+    state.table.add("green",   &green_behavior);
 
     // default behavior
     state.current_behavior = state.table.get(behavior_token("green"));

From 8a3e199bcc35bc86f3924beed49d019c2738ffe0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 23 Jan 2026 14:38:38 -0600
Subject: [PATCH 0189/1000] Implemented attributes to describe certain device
 properities, getter methods for these properities and an init method to be
 called once upon constructor startup. These changes are being implemented
 since we need some specific GPU information required at runtime to be able to
 do scheduling and device is meant to be an abstraction for the GPU, so it
 should go in here.

---
 libcaf_cuda/caf/cuda/device.hpp | 76 ++++++++++++++++++++++++++++++++-
 1 file changed, 75 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 99058753ba..2a16609b92 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -33,7 +33,8 @@ class device : public caf::ref_counted {
       id_(id),
       name_(name),
       stream_table_(context, stream_pool_size) {
-  }
+ 	      init_device_properties();
+      }
 
   ~device() {
     check(cuCtxDestroy(context_), "cuCtxDestroy");
@@ -53,6 +54,37 @@ class device : public caf::ref_counted {
 
   CUcontext getContext(int) { return context_; }
 
+
+  // Number of streaming multiprocessors (SMs)
+  int num_sms() const noexcept { return sm_count_; }
+
+  // Warp size (usually 32)
+  int warp_size() const noexcept { return warp_size_; }
+
+  // Maximum threads per SM
+  int max_threads_per_sm() const noexcept { return max_threads_per_sm_; }
+
+  // Derived: warps per SM
+  int warps_per_sm() const noexcept { return warps_per_sm_; }
+
+  // Derived: total warps on the device
+  int total_warps() const noexcept { return total_warps_; }
+
+  // Total device memory in bytes
+  std::size_t total_memory_bytes() const noexcept { return total_mem_bytes_; }
+
+  // Convenience: total memory in megabytes
+  double total_memory_mb() const noexcept { return static_cast<double>(total_mem_bytes_) / (1024.0 * 1024.0); }
+
+  // Short human-readable device summary
+  std::string device_summary() const {
+	  return std::string(name_) + " (id=" + std::to_string(id_) + ") - SMs: " + std::to_string(sm_count_) +
+		  ", warp_size: " + std::to_string(warp_size_) +
+		  ", max_threads/SM: " + std::to_string(max_threads_per_sm_) +
+		  ", total_mem(MB): " + std::to_string(total_memory_mb());
+  }
+
+
   //returns the CUStream associated with the actor id 
   CUstream get_stream_for_actor(int actor_id) {
     return stream_table_.get_stream(actor_id);
@@ -254,6 +286,48 @@ class device : public caf::ref_counted {
   DeviceStreamTable stream_table_;
   std::mutex stream_mutex_;
 
+  // Cached GPU properties (queried once during construction)
+  int sm_count_ = 0;
+  int warp_size_ = 0;
+  int max_threads_per_sm_ = 0;
+  int warps_per_sm_ = 0;
+  int total_warps_ = 0;
+  std::size_t total_mem_bytes_ = 0;
+
+  // Initialize and cache device properties. Called once from constructor.
+  void init_device_properties() {
+	  CUresult res;
+	  int tmp = 0;
+
+	  // Number of SMs
+	  res = cuDeviceGetAttribute(&tmp, CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, device_);
+	  if (res == CUDA_SUCCESS) sm_count_ = tmp;
+	  else { const char* n = nullptr; cuGetErrorName(res, &n); throw std::runtime_error(std::string("cuDeviceGetAttribute(MULTIPROCESSOR_COUNT) failed: ") + (n ? n : "unknown")); }
+
+	  // Warp size
+	  res = cuDeviceGetAttribute(&tmp, CU_DEVICE_ATTRIBUTE_WARP_SIZE, device_);
+	  if (res == CUDA_SUCCESS) warp_size_ = tmp;
+	  else { const char* n = nullptr; cuGetErrorName(res, &n); throw std::runtime_error(std::string("cuDeviceGetAttribute(WARP_SIZE) failed: ") + (n ? n : "unknown")); }
+
+	  // Max threads per SM
+	  res = cuDeviceGetAttribute(&tmp, CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR, device_);
+	  if (res == CUDA_SUCCESS) max_threads_per_sm_ = tmp;
+	  else { const char* n = nullptr; cuGetErrorName(res, &n); throw std::runtime_error(std::string("cuDeviceGetAttribute(MAX_THREADS_PER_MULTIPROCESSOR) failed: ") + (n ? n : "unknown")); }
+
+	  // Derived values
+	  if (warp_size_ > 0 && max_threads_per_sm_ > 0) {
+		  warps_per_sm_ = std::max(1, max_threads_per_sm_ / warp_size_);
+		  total_warps_ = warps_per_sm_ * sm_count_;
+	  }
+
+	  // Total memory
+	  size_t bytes = 0;
+	  res = cuDeviceTotalMem(&bytes, device_);
+	  if (res == CUDA_SUCCESS) total_mem_bytes_ = bytes;
+	  else { const char* n = nullptr; cuGetErrorName(res, &n); throw std::runtime_error(std::string("cuDeviceTotalMem failed: ") + (n ? n : "unknown")); }
+  }
+
+
   // === Memory handling ===
   
   //----------------------------------------------

From c1e1ce7fa8edf3292e1e535143e01f0141c9518a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 23 Jan 2026 14:48:34 -0600
Subject: [PATCH 0190/1000] Implemented test for devices GPU specific device
 properities and a test driver.

---
 libcaf_cuda/tests/inspect-test/main.test.cpp | 142 +++++++++++++++++++
 1 file changed, 142 insertions(+)
 create mode 100644 libcaf_cuda/tests/inspect-test/main.test.cpp

diff --git a/libcaf_cuda/tests/inspect-test/main.test.cpp b/libcaf_cuda/tests/inspect-test/main.test.cpp
new file mode 100644
index 0000000000..3b1f408e54
--- /dev/null
+++ b/libcaf_cuda/tests/inspect-test/main.test.cpp
@@ -0,0 +1,142 @@
+/*
+ * A file full of caf cuda tests that can only be verified by
+ * inspection
+ */
+
+#include <caf/all.hpp>
+#include <caf/actor_system.hpp>
+#include <caf/cuda/manager.hpp>
+#include <caf/cuda/helpers.hpp>
+#include <caf/cuda/types.hpp>
+#include <caf/cuda/command_runner.hpp>
+#include <caf/cuda/mem_ref.hpp>
+#include <caf/cuda/device.hpp>
+#include <caf/cuda/streampool.hpp>
+#include <caf/cuda/program.hpp>
+#include <caf/cuda/nd_range.hpp>
+#include <caf/cuda/platform.hpp>
+#include <cassert>
+#include <vector>
+#include <string>
+#include <cuda.h>
+#include <stdexcept>
+#include <iostream>
+
+
+// Structure to hold test information
+struct Test {
+    std::string name;
+    void (*function)(caf::actor_system&);
+};
+
+// Print all device properties using the device getters we added.
+void test_device_display_info([[maybe_unused]] caf::actor_system& sys) {
+    using namespace caf::cuda;
+
+    try {
+        auto& mgr = manager::get();
+        auto dev = mgr.find_device(0);
+        if (!dev) {
+            std::cerr << "[test_device_display_info] No device found with id 0 — skipping\n";
+            return;
+        }
+
+        std::cout << "==== Device Summary (short) ====\n";
+        std::cout << dev->device_summary() << "\n\n";
+
+        std::cout << "==== Device Detailed Properties ====\n";
+        std::cout << "Device name: " << dev->name() << "\n";
+        std::cout << "Device ID: " << dev->getId() << "\n";
+        std::cout << "CUdevice handle: " << dev->getDevice() << "\n";
+        std::cout << "CUcontext handle: " << dev->getContext() << "\n\n";
+
+        std::cout << "Number of SMs: " << dev->num_sms() << "\n";
+        std::cout << "Warp size: " << dev->warp_size() << "\n";
+        std::cout << "Max threads per SM: " << dev->max_threads_per_sm() << "\n";
+        std::cout << "Warps per SM (derived): " << dev->warps_per_sm() << "\n";
+        std::cout << "Total warps on device (derived): " << dev->total_warps() << "\n";
+        std::cout << "Total device memory (bytes): " << dev->total_memory_bytes() << "\n";
+        std::cout << "Total device memory (MB): " << dev->total_memory_mb() << "\n";
+
+        std::cout << "\n[test_device_display_info] Finished printing device info.\n";
+    } catch (const std::exception& e) {
+        std::cerr << "[test_device_display_info] ERROR: " << e.what() << "\n";
+    } catch (...) {
+        std::cerr << "[test_device_display_info] ERROR: unknown exception\n";
+    }
+}
+
+// Return codes: 0 = PASS, 1 = SKIPPED, 2 = FAIL
+int run_test(const Test& test, caf::actor_system& sys) {
+    std::cout << "Running test: " << test.name << "... ";
+    try {
+        test.function(sys);
+        std::cout << "DONE\n";
+        return 0;
+    } catch (const std::exception& e) {
+        std::string msg = e.what();
+        if (msg.find("Skipping") != std::string::npos || msg.find("skip") != std::string::npos) {
+            std::cout << "SKIPPED: " << msg << "\n";
+            return 1;
+        }
+        std::cout << "FAILED: " << msg << "\n";
+        return 2;
+    } catch (...) {
+        std::cout << "FAILED: Unknown error\n";
+        return 2;
+    }
+}
+
+// Test registry — add more tests here as needed.
+const std::vector<Test> tests = {
+    {"test_device_display_info", test_device_display_info}
+};
+
+// CAF main function to run tests
+void caf_main(caf::actor_system& sys) {
+    // Initialize CUDA manager
+    try {
+        caf::cuda::manager::init(sys);
+        std::cout << "CUDA manager initialized successfully\n";
+    } catch (const std::exception& e) {
+        std::cerr << "Failed to initialize CUDA manager: " << e.what() << "\n";
+        return;
+    }
+
+    // Run tests
+    std::cout << "\nStarting unit tests...\n\n";
+    int passed = 0;
+    int skipped = 0;
+    int failed = 0;
+
+    for (const auto& test : tests) {
+        int status = run_test(test, sys);
+        if (status == 0) ++passed;
+        else if (status == 1) ++skipped;
+        else ++failed;
+    }
+
+    // Shutdown CUDA manager
+    try {
+        caf::cuda::manager::shutdown();
+        std::cout << "\nCUDA manager shutdown successfully\n";
+    } catch (const std::exception& e) {
+        std::cerr << "Failed to shutdown CUDA manager: " << e.what() << "\n";
+    }
+
+    // Summary
+    std::cout << "\nTest Summary:\n";
+    std::cout << "Total tests listed: " << tests.size() << "\n";
+    std::cout << "Passed: " << passed << "\n";
+    std::cout << "Skipped: " << skipped << "\n";
+    std::cout << "Failed: " << failed << "\n";
+
+    if (failed > 0) {
+        // Throw to indicate failure to the test runner / CI environment if present.
+        throw std::runtime_error("One or more tests failed");
+    }
+}
+
+// Register caf_main
+CAF_MAIN()
+

From ec4fcc50538c39a8612ca2a4dc8796c2ede99df7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 23 Jan 2026 14:52:21 -0600
Subject: [PATCH 0191/1000] Added support test files for build automation.

---
 libcaf_cuda/tests/inspect-test/CMakeLists.txt | 44 ++++++++++++++++
 .../tests/inspect-test/compile_kernels.sh     | 18 +++++++
 libcaf_cuda/tests/inspect-test/genMatrix.cu   | 43 ++++++++++++++++
 libcaf_cuda/tests/inspect-test/mmul.cu        | 46 +++++++++++++++++
 libcaf_cuda/tests/inspect-test/shared_mmul.cu | 51 +++++++++++++++++++
 5 files changed, 202 insertions(+)
 create mode 100644 libcaf_cuda/tests/inspect-test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/inspect-test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/inspect-test/genMatrix.cu
 create mode 100644 libcaf_cuda/tests/inspect-test/mmul.cu
 create mode 100644 libcaf_cuda/tests/inspect-test/shared_mmul.cu

diff --git a/libcaf_cuda/tests/inspect-test/CMakeLists.txt b/libcaf_cuda/tests/inspect-test/CMakeLists.txt
new file mode 100644
index 0000000000..463e22ef5e
--- /dev/null
+++ b/libcaf_cuda/tests/inspect-test/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++17 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/inspect-test/compile_kernels.sh b/libcaf_cuda/tests/inspect-test/compile_kernels.sh
new file mode 100755
index 0000000000..4198b92e5b
--- /dev/null
+++ b/libcaf_cuda/tests/inspect-test/compile_kernels.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+
+# Compile genMatrix.cu to fatbin in current directory
+nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
+echo "Generated generate_random_matrix.fatbin"
+
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/inspect-test/genMatrix.cu b/libcaf_cuda/tests/inspect-test/genMatrix.cu
new file mode 100644
index 0000000000..123748e0a7
--- /dev/null
+++ b/libcaf_cuda/tests/inspect-test/genMatrix.cu
@@ -0,0 +1,43 @@
+
+#include <curand_kernel.h>
+//generate_random_matrix
+extern "C" __global__
+void generate_random_matrix(int* matrix, int total_elements, int seed, int max_val) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= total_elements) return;
+
+    curandState state;
+    curand_init((unsigned long long)seed, idx, 0, &state);
+
+    unsigned int r = curand(&state);
+    matrix[idx] = r % max_val;
+}
+
+
+extern "C" __global__
+void generate_random_matrix_float(float* matrix, int total_elements, int seed, float max_val) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= total_elements) return;
+
+    curandState state;
+    curand_init((unsigned long long)seed, idx, 0, &state);
+
+    unsigned int r = curand(&state);
+    matrix[idx] = (float)(r % (unsigned int)max_val);
+}
+
+
+extern "C" __global__
+void generate_random_matrix_double(double* matrix, int total_elements, int seed, double max_val) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= total_elements) return;
+
+    curandState state;
+    curand_init((unsigned long long)seed, idx, 0, &state);
+
+    unsigned int r = curand(&state);
+    matrix[idx] = (double)(r % (unsigned int)max_val);
+}
+
+
+
diff --git a/libcaf_cuda/tests/inspect-test/mmul.cu b/libcaf_cuda/tests/inspect-test/mmul.cu
new file mode 100644
index 0000000000..28b899b9f9
--- /dev/null
+++ b/libcaf_cuda/tests/inspect-test/mmul.cu
@@ -0,0 +1,46 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+
+extern "C" __global__
+void matrixMulFloat(const float* a, const float* b, float* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (row < N && col < N) {
+        float temp = 0.0f;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+extern "C" __global__
+void matrixMulDouble(const double* a, const double* b, double* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (row < N && col < N) {
+        double temp = 0.0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+
+
diff --git a/libcaf_cuda/tests/inspect-test/shared_mmul.cu b/libcaf_cuda/tests/inspect-test/shared_mmul.cu
new file mode 100644
index 0000000000..85c361ed93
--- /dev/null
+++ b/libcaf_cuda/tests/inspect-test/shared_mmul.cu
@@ -0,0 +1,51 @@
+extern "C" __global__
+void matrixMul(const int* __restrict__ a,
+               const int* __restrict__ b,
+               int* __restrict__ c,
+               int N)
+{
+    const int TILE = 32;
+    int row = blockIdx.y * blockDim.y + threadIdx.y; // global row in C
+    int col = blockIdx.x * blockDim.x + threadIdx.x; // global col in C
+
+    __shared__ int s_a[TILE * TILE];
+    __shared__ int s_b[TILE * TILE];
+
+    int acc = 0;
+
+    // Sweep tiles across the K dimension
+    for (int i = 0; i < N; i += TILE) {
+
+        // Each thread loads one element into shared memory (with bounds checks)
+        int aCol = i + threadIdx.x;
+        int bRow = i + threadIdx.y;
+
+        // s_a[y, x] = a[row, aCol] if in range, else 0
+        if (row < N && aCol < N)
+            s_a[threadIdx.y * TILE + threadIdx.x] = a[row * N + aCol];
+        else
+            s_a[threadIdx.y * TILE + threadIdx.x] = 0;
+
+        // s_b[y, x] = b[bRow, col] if in range, else 0
+        if (bRow < N && col < N)
+            s_b[threadIdx.y * TILE + threadIdx.x] = b[bRow * N + col];
+        else
+            s_b[threadIdx.y * TILE + threadIdx.x] = 0;
+
+        __syncthreads();
+
+        // Compute partial dot product for this tile
+        #pragma unroll
+        for (int k = 0; k < TILE; ++k) {
+            acc += s_a[threadIdx.y * TILE + k] *
+                   s_b[k * TILE + threadIdx.x];
+        }
+
+        __syncthreads();
+    }
+
+    // Final write (guarded)
+    if (row < N && col < N)
+        c[row * N + col] = acc;
+}
+

From 57acadf94e85f46deaf14586acfde4ad21e7dede Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 23 Jan 2026 15:03:42 -0600
Subject: [PATCH 0192/1000] Implemented get_num_blocks helper function.

---
 libcaf_cuda/caf/cuda/nd_range.hpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/nd_range.hpp b/libcaf_cuda/caf/cuda/nd_range.hpp
index f8be8edc74..a18b2bf656 100644
--- a/libcaf_cuda/caf/cuda/nd_range.hpp
+++ b/libcaf_cuda/caf/cuda/nd_range.hpp
@@ -51,6 +51,12 @@ class nd_range {
   const dim_vec& getGridDims() const { return gridDim; }
   const dim_vec& getBlockDims() const { return blockDim; }
 
+  // Returns total number of threads per block
+  [[nodiscard]] constexpr size_t get_num_threads() const noexcept {
+	  return blockDim[0] * blockDim[1] * blockDim[2];
+  }
+
+
   ~nd_range() {
   //no-op
   }

From 31d7fa430a4bc17d2529316e9945944763ccf010 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 23 Jan 2026 15:08:04 -0600
Subject: [PATCH 0193/1000] Fixed compiler error.

---
 libcaf_cuda/caf/cuda/nd_range.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/nd_range.hpp b/libcaf_cuda/caf/cuda/nd_range.hpp
index a18b2bf656..142b0b7c0b 100644
--- a/libcaf_cuda/caf/cuda/nd_range.hpp
+++ b/libcaf_cuda/caf/cuda/nd_range.hpp
@@ -52,7 +52,7 @@ class nd_range {
   const dim_vec& getBlockDims() const { return blockDim; }
 
   // Returns total number of threads per block
-  [[nodiscard]] constexpr size_t get_num_threads() const noexcept {
+  [[nodiscard]]  size_t get_num_threads() const noexcept {
 	  return blockDim[0] * blockDim[1] * blockDim[2];
   }
 

From c6653ec5eb0a75a7de60e057e515bc830eab9f37 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 23 Jan 2026 15:18:49 -0600
Subject: [PATCH 0194/1000] Moved max_active_blocks_per_sm to the device.cpp
 file This change was made since accessing the program.hpp inside the header
 of device.hpp is bad practice and would likely introduce circular
 dependencies.

---
 libcaf_cuda/caf/cuda/device.hpp |  8 ++++++++
 libcaf_cuda/src/device.cpp      | 21 +++++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 libcaf_cuda/src/device.cpp

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 2a16609b92..b00b86a6ba 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -85,6 +85,14 @@ class device : public caf::ref_counted {
   }
 
 
+  //given a program/kernel and dimesions 
+  //returns the max blocks that can be on an SM 
+  int max_active_blocks_per_sm(const program_ptr& prog, const nd_range& range,
+		  size_t dynamic_smem_bytes = 0) const; 
+
+
+
+
   //returns the CUStream associated with the actor id 
   CUstream get_stream_for_actor(int actor_id) {
     return stream_table_.get_stream(actor_id);
diff --git a/libcaf_cuda/src/device.cpp b/libcaf_cuda/src/device.cpp
new file mode 100644
index 0000000000..744d7418c9
--- /dev/null
+++ b/libcaf_cuda/src/device.cpp
@@ -0,0 +1,21 @@
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/program.hpp"
+
+namespace caf::cuda {
+
+	int device::max_active_blocks_per_sm(const program_ptr& prog, const nd_range& range,
+			size_t dynamic_smem_bytes) const {
+		CUfunction kernel = prog->get_kernel(id_); // full type known here
+		int block_size = static_cast<int>(range.get_num_threads());
+		int active_blocks = 0;
+		cuOccupancyMaxActiveBlocksPerMultiprocessor(&active_blocks, kernel, block_size, dynamic_smem_bytes);
+		return active_blocks;
+	}
+
+
+
+}//namespace caf cuda
+
+
+
+

From 05c9f219d6ddd849a6c4daed5e8633f88966f366 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 23 Jan 2026 15:19:21 -0600
Subject: [PATCH 0195/1000] Added device.cpp to list.

---
 libcaf_cuda/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/CMakeLists.txt b/libcaf_cuda/CMakeLists.txt
index 45fa0cad16..32d0283718 100644
--- a/libcaf_cuda/CMakeLists.txt
+++ b/libcaf_cuda/CMakeLists.txt
@@ -18,6 +18,7 @@ set(LIBCAF_CUDA_SRCS
   src/manager.cpp
   src/platform.cpp
   src/program.cpp
+  src/device.cpp
   src/helpers.cpp
   src/test.cpp
   src/scheduler.cpp

From a1e8de1d310e90c22b3009e24b5e397cdac41795 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 23 Jan 2026 16:55:37 -0600
Subject: [PATCH 0196/1000] IMplemented test_print_mmul_occupancy.

---
 libcaf_cuda/tests/inspect-test/main.test.cpp | 71 +++++++++++++++++++-
 1 file changed, 70 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/inspect-test/main.test.cpp b/libcaf_cuda/tests/inspect-test/main.test.cpp
index 3b1f408e54..d2aa858cd6 100644
--- a/libcaf_cuda/tests/inspect-test/main.test.cpp
+++ b/libcaf_cuda/tests/inspect-test/main.test.cpp
@@ -66,6 +66,74 @@ void test_device_display_info([[maybe_unused]] caf::actor_system& sys) {
     }
 }
 
+
+void test_print_mmul_occupancy([[maybe_unused]] caf::actor_system& sys) {
+  using namespace caf::cuda;
+
+  auto& mgr = manager::get();
+  auto dev = mgr.find_device(0);
+
+  if (!dev) {
+    std::cerr << "[mmul occupancy] No CUDA device found\n";
+    return;
+  }
+
+  program_ptr prog;
+  try {
+    prog = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+  } catch (const std::exception& e) {
+    std::cerr << "[mmul occupancy] Failed to load cubin: "
+              << e.what() << "\n";
+    return;
+  }
+
+  // Example problem size (does not affect occupancy directly)
+  constexpr int N = 1024;
+  constexpr int THREADS = 32;
+  const int BLOCKS = (N + THREADS - 1) / THREADS;
+
+  nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1
+  );
+
+  try {
+    int blocks_per_sm =
+        dev->max_active_blocks_per_sm(prog, dims);
+
+    std::cout << "\n[mmul occupancy]\n";
+    std::cout << "Device ID              : " << dev->getId() << "\n";
+    std::cout << "Kernel                  : matrixMul\n";
+    std::cout << "Threads per block       : "
+              << dims.getBlockDimX() * dims.getBlockDimY() * dims.getBlockDimZ()
+              << "\n";
+    std::cout << "Block dims              : ("
+              << dims.getBlockDimX() << ", "
+              << dims.getBlockDimY() << ", "
+              << dims.getBlockDimZ() << ")\n";
+    std::cout << "Grid dims               : ("
+              << dims.getGridDimX() << ", "
+              << dims.getGridDimY() << ", "
+              << dims.getGridDimZ() << ")\n";
+    std::cout << "Max active blocks / SM  : "
+              << blocks_per_sm << "\n";
+    std::cout << "Total active blocks     : "
+              << blocks_per_sm * dev->num_sms() << "\n\n";
+
+  } catch (const std::exception& e) {
+    std::cerr << "[mmul occupancy] Error querying occupancy: "
+              << e.what() << "\n";
+  }
+}
+
+
+
+
+
+
+
+
+
 // Return codes: 0 = PASS, 1 = SKIPPED, 2 = FAIL
 int run_test(const Test& test, caf::actor_system& sys) {
     std::cout << "Running test: " << test.name << "... ";
@@ -89,7 +157,8 @@ int run_test(const Test& test, caf::actor_system& sys) {
 
 // Test registry — add more tests here as needed.
 const std::vector<Test> tests = {
-    {"test_device_display_info", test_device_display_info}
+    {"test_device_display_info", test_device_display_info},
+    {"test_print_mmul_occupancy", test_print_mmul_occupancy}
 };
 
 // CAF main function to run tests

From 0b2e73ab51297222157e9cb7fee45bc51f1ac545 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 23 Jan 2026 16:57:00 -0600
Subject: [PATCH 0197/1000] Added CAF_CUDA_EXPORT to device.hpp so that other
 files can see it when being linked This change is being made so I can run
 test drivers on some of its methods It would be nice if this did not need to
 be added since the user should never have to interact with the device class
 in the first place, hopefully someday this change will be reversed.

---
 libcaf_cuda/caf/cuda/device.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index b00b86a6ba..7bd4dbf9c8 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -23,7 +23,7 @@
 
 namespace caf::cuda {
 
-class device : public caf::ref_counted {
+class CAF_CUDA_EXPORT device : public caf::ref_counted {
 public:
   using device_ptr = caf::intrusive_ptr<device>;
 

From 9d0bab5544e3128d7a05d241e28020ebfa772b89 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 23 Jan 2026 17:30:20 -0600
Subject: [PATCH 0198/1000] Initial commit.

---
 .../cuda/control-layer/core_usage_behavior.hpp   | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
new file mode 100644
index 0000000000..51bd602f99
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -0,0 +1,16 @@
+#pragma once
+#include "caf/cuda/control-layer/behavior.hpp"
+
+namespace caf::cuda {
+
+class core_usage_behavior : public scheduler_actor_behavior {
+public:
+    explicit core_usage_behavior(scheduler_actor_state& state);
+    void on_enter() override;
+    void schedule() override;
+    void receive(const token_ptr& tok) override;
+
+};
+
+} // namespace caf::cuda
+

From 0e08dabf1c66ebc885278eeb4ec02c5c65f24b8e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 23 Jan 2026 17:50:52 -0600
Subject: [PATCH 0199/1000] Initial commit.

---
 .../caf/cuda/control-layer/kernel_graph.hpp   | 61 +++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
new file mode 100644
index 0000000000..ce20ea412f
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
@@ -0,0 +1,61 @@
+#pragma once
+#include <vector>
+#include "caf/cuda/control-layer/token.hpp"
+
+/* Meant to represent a directed acylic graph for kernel operations
+ * operations or token_ptrs are meant to come in order and declared with a dependency number to indicate dependency 
+ */
+
+// status codes
+#define WAITING 0
+#define READY 1
+#define ERROR 2
+
+namespace caf::cuda {
+
+class kernel_graph {
+public:
+    // for caf's messaging system do not use
+    kernel_graph() = default;
+
+    kernel_graph(int device_number,
+                 int stream_id,
+                 int dependency_number)
+        : device_number_(device_number),
+          stream_id_(stream_id),
+          dependency_number_(dependency_number) {}
+
+    // returns the next operation/token_ptr that can be dequeued
+    token_ptr peek() const {
+        if (operations.empty())
+            return nullptr;
+        return operations.front();
+    }
+
+    void add_operation(token_ptr operation) {
+        operations.push_back(operation);
+    }
+
+    // removes the operation and returns it
+    token_ptr getOperation() {
+        if (operations.empty())
+            return nullptr;
+
+        token_ptr op = operations.front();
+        operations.erase(operations.begin());
+        return op;
+    }
+
+    bool empty() const {
+        return operations.empty();
+    }
+
+private:
+    int device_number_;
+    int stream_id_;
+    int dependency_number_;
+    std::vector<token_ptr> operations;
+};
+
+} // namespace caf::cuda
+

From dcfc216752f6b2a809688fc7e05e4279fa6f79e2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 12:16:49 -0600
Subject: [PATCH 0200/1000] Initial commit.

---
 .../control-layer-unit-tests/CMakeLists.txt   |  44 +++++
 .../compile_kernels.sh                        |  18 ++
 .../control-layer-unit-tests/genMatrix.cu     |  43 +++++
 .../control-layer-unit-tests/main.test.cpp    | 157 ++++++++++++++++++
 .../tests/control-layer-unit-tests/mmul.cu    |  46 +++++
 .../tests/control-layer-unit-tests/mmul.cubin | Bin 0 -> 14312 bytes
 .../control-layer-unit-tests/shared_mmul.cu   |  51 ++++++
 7 files changed, 359 insertions(+)
 create mode 100644 libcaf_cuda/tests/control-layer-unit-tests/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/control-layer-unit-tests/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/control-layer-unit-tests/genMatrix.cu
 create mode 100644 libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
 create mode 100644 libcaf_cuda/tests/control-layer-unit-tests/mmul.cu
 create mode 100644 libcaf_cuda/tests/control-layer-unit-tests/mmul.cubin
 create mode 100644 libcaf_cuda/tests/control-layer-unit-tests/shared_mmul.cu

diff --git a/libcaf_cuda/tests/control-layer-unit-tests/CMakeLists.txt b/libcaf_cuda/tests/control-layer-unit-tests/CMakeLists.txt
new file mode 100644
index 0000000000..463e22ef5e
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-unit-tests/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++17 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/control-layer-unit-tests/compile_kernels.sh b/libcaf_cuda/tests/control-layer-unit-tests/compile_kernels.sh
new file mode 100755
index 0000000000..4198b92e5b
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-unit-tests/compile_kernels.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+
+# Compile genMatrix.cu to fatbin in current directory
+nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
+echo "Generated generate_random_matrix.fatbin"
+
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/control-layer-unit-tests/genMatrix.cu b/libcaf_cuda/tests/control-layer-unit-tests/genMatrix.cu
new file mode 100644
index 0000000000..123748e0a7
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-unit-tests/genMatrix.cu
@@ -0,0 +1,43 @@
+
+#include <curand_kernel.h>
+//generate_random_matrix
+extern "C" __global__
+void generate_random_matrix(int* matrix, int total_elements, int seed, int max_val) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= total_elements) return;
+
+    curandState state;
+    curand_init((unsigned long long)seed, idx, 0, &state);
+
+    unsigned int r = curand(&state);
+    matrix[idx] = r % max_val;
+}
+
+
+extern "C" __global__
+void generate_random_matrix_float(float* matrix, int total_elements, int seed, float max_val) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= total_elements) return;
+
+    curandState state;
+    curand_init((unsigned long long)seed, idx, 0, &state);
+
+    unsigned int r = curand(&state);
+    matrix[idx] = (float)(r % (unsigned int)max_val);
+}
+
+
+extern "C" __global__
+void generate_random_matrix_double(double* matrix, int total_elements, int seed, double max_val) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= total_elements) return;
+
+    curandState state;
+    curand_init((unsigned long long)seed, idx, 0, &state);
+
+    unsigned int r = curand(&state);
+    matrix[idx] = (double)(r % (unsigned int)max_val);
+}
+
+
+
diff --git a/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp b/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
new file mode 100644
index 0000000000..a3ba4b6e31
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
@@ -0,0 +1,157 @@
+/*
+ * A file full of caf cuda unit tests focused on the control layer of caf cuda
+ */
+
+
+
+#include <caf/all.hpp>
+#include <caf/actor_system.hpp>
+#include "caf/cuda/kernel_graph.hpp"
+#include <iostream>
+#include <vector>
+#include <string>
+#include <stdexcept>
+
+using namespace caf::cuda;
+
+// Structure to hold test information
+struct Test {
+    std::string name;
+    void (*function)(caf::actor_system&);
+};
+
+
+/*
+ * kernel_graph_tests.cpp
+ *
+ * Unit tests for caf::cuda::kernel_graph
+ *
+ * These tests are lightweight and do not depend on launching kernels or
+ * initializing the CUDA manager. They exercise the queue semantics:
+ *   - peek() on an empty graph returns nullptr
+ *   - add_operation() makes the graph non-empty
+ *   - getOperation() pops the front element and restores empty state
+ *
+ * We deliberately use nullptr token_ptr instances so these tests do not
+ * require constructing concrete token subclasses or having complex
+ * control-layer dependencies present.
+ */
+
+
+
+// --- Tests ---
+
+// 1) Validate empty graph behavior
+void test_kernel_graph_empty([[maybe_unused]] caf::actor_system& sys) {
+    kernel_graph g(/*device*/0, /*stream*/0, /*dependency*/0);
+
+    // peek on empty graph -> should be nullptr
+    auto p = g.peek();
+    if (p != nullptr) {
+        std::cerr << "[test_kernel_graph_empty] ERROR: peek() on empty graph returned non-null\n";
+        throw std::runtime_error("peek() returned non-null on empty graph");
+    }
+
+    // getOperation on empty graph -> should be nullptr and graph remains empty
+    auto r = g.getOperation();
+    if (r != nullptr) {
+        std::cerr << "[test_kernel_graph_empty] ERROR: getOperation() on empty graph returned non-null\n";
+        throw std::runtime_error("getOperation() returned non-null on empty graph");
+    }
+
+    if (!g.empty()) {
+        std::cerr << "[test_kernel_graph_empty] ERROR: graph reports non-empty after no ops\n";
+        throw std::runtime_error("graph not empty after no ops");
+    }
+
+    std::cout << "[test_kernel_graph_empty] OK\n";
+}
+
+// 2) Validate push / pop semantics using nullptr token_ptr
+void test_kernel_graph_push_pop([[maybe_unused]] caf::actor_system& sys) {
+    kernel_graph g(/*device*/1, /*stream*/2, /*dependency*/3);
+
+    // initially empty
+    if (!g.empty()) {
+        throw std::runtime_error("graph unexpectedly non-empty at start");
+    }
+
+    // create a token_ptr that is intentionally null to avoid heavy dependencies
+    token_ptr t = nullptr;
+
+    // add operation
+    g.add_operation(t);
+
+    // now graph should be non-empty
+    if (g.empty()) {
+        std::cerr << "[test_kernel_graph_push_pop] ERROR: graph empty after add_operation\n";
+        throw std::runtime_error("graph empty after add_operation");
+    }
+
+    // peek should return the front element (nullptr in this test) but graph must not be empty
+    auto pe = g.peek();
+    // peek can be nullptr as we added a nullptr item — just ensure graph remains non-empty
+    if (g.empty()) {
+        std::cerr << "[test_kernel_graph_push_pop] ERROR: graph became empty after peek()\n";
+        throw std::runtime_error("graph empty after peek");
+    }
+
+    // getOperation should remove and return the element (nullptr expected)
+    auto popped = g.getOperation();
+    (void)popped; // we don't inspect the token itself in this test
+
+    // after popping, graph should be empty again
+    if (!g.empty()) {
+        std::cerr << "[test_kernel_graph_push_pop] ERROR: graph not empty after getOperation\n";
+        throw std::runtime_error("graph not empty after getOperation");
+    }
+
+    std::cout << "[test_kernel_graph_push_pop] OK\n";
+}
+
+// Register tests
+const std::vector<Test> tests = {
+    {"test_kernel_graph_empty", test_kernel_graph_empty},
+    {"test_kernel_graph_push_pop", test_kernel_graph_push_pop}
+};
+
+// Run a single test and return status code:
+// 0 = PASS, 1 = SKIPPED, 2 = FAIL
+int run_test(const Test& test, caf::actor_system& sys) {
+    std::cout << "Running test: " << test.name << "... ";
+    try {
+        test.function(sys);
+        std::cout << "PASSED\n";
+        return 0;
+    } catch (const std::exception& e) {
+        std::cout << "FAILED: " << e.what() << "\n";
+        return 2;
+    } catch (...) {
+        std::cout << "FAILED: Unknown error\n";
+        return 2;
+    }
+}
+
+void caf_main(caf::actor_system& sys) {
+    std::cout << "\nStarting kernel_graph unit tests...\n\n";
+    int passed = 0;
+    int failed = 0;
+
+    for (const auto& test : tests) {
+        int status = run_test(test, sys);
+        if (status == 0) ++passed;
+        else ++failed;
+    }
+
+    std::cout << "\nTest Summary:\n";
+    std::cout << "Total tests listed: " << tests.size() << "\n";
+    std::cout << "Passed: " << passed << "\n";
+    std::cout << "Failed: " << failed << "\n";
+
+    if (failed > 0) {
+        throw std::runtime_error("One or more kernel_graph tests failed");
+    }
+}
+
+CAF_MAIN()
+
diff --git a/libcaf_cuda/tests/control-layer-unit-tests/mmul.cu b/libcaf_cuda/tests/control-layer-unit-tests/mmul.cu
new file mode 100644
index 0000000000..28b899b9f9
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-unit-tests/mmul.cu
@@ -0,0 +1,46 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+
+extern "C" __global__
+void matrixMulFloat(const float* a, const float* b, float* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (row < N && col < N) {
+        float temp = 0.0f;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+extern "C" __global__
+void matrixMulDouble(const double* a, const double* b, double* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (row < N && col < N) {
+        double temp = 0.0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+
+
diff --git a/libcaf_cuda/tests/control-layer-unit-tests/mmul.cubin b/libcaf_cuda/tests/control-layer-unit-tests/mmul.cubin
new file mode 100644
index 0000000000000000000000000000000000000000..3f3c92d77fb962f198fab9a9d0dc03ac08d0e95e
GIT binary patch
literal 14312
zcmeHOO=w)#6~6OkG#ZUZGf%RtNU|I&^4O?2NNy66>>^BR2iioYi&EN>Sc>f`;Mn3x
zF%uISiIhM}8++TXTH}RRZ3^ACWD$f3DR@yzmW~i1X}WM53W1iW-}jyS{$?g_<ovWW
zdZ2ml+<VWt=l`DbX7u7WzI7}p6z=bJb_MPa?t*{jZT2A#+;8_|J;&Uvd)O86Ip7M;
zO)oDluS6^J3wVm}&p&Ih?oTZ*o;%fY)8{TsKXdNPc{jasy0tR>?EFgfOzZLI&OUnn
zxrMW*lj*J#%Zu~T=~G>&o<4tWd1e0G%8}05{65Fdo}XXovQ0YPzDqjOWfMP{-=hn8
z4YgC&U>Blp`qb%#=YIIqnP~o5s3balwqsyEKLVw4)%?>d&zwJZ2gc|yLtdaS%s5Xi
z%|k|)Y~NL#sR8IXse#k+cEzSMU9@YH$(HSo_+K}k547&LqZlr_ddK8XHNLB3{B?~t
zI>vhuvPD<x7@yX7*fD-a<NY1uKhk)qWBk_|@9P-94<mzN%Kxfo;kO!3`HaV#pfebz
z{KVs1xSF0ipLrEs=>^^!wYAmNDOcHATb-)t(K(Jk(7}xj#57A3qcnvNBT#xTNHB<h
zr~54>RP5ms_q*}=?054H_AAX)9KB}(f29NVOfsdx%=19z8SAT<pWqs-=QA&^PoA+p
zdB%Er!vxo0ee4-rpFCrI^88?lr-~~*RMI(As#YAebFwh*ilgJMbeB0!(TzBFYaoFG
z2Vc2$PXZTQB?p6q;NspbSNeL1t6gcl;(m$y36Lyx*_UqdMechGitn~>`AgXMo7h8F
z5LCkI%p{(nFkTgZ5B+__-zWTi%HNOp`@_1!X8vyY1!wpLXZQtY_yuS91!wpLXZWzO
z@B_a;?CD{{4?KS0@dJ+^cziHEbT2-N2x3aX{K1R9L;wCA0bGRXn~xH}ML7074eWgE
zXkbSJI~w?sG!R@wqGCc~3G_z8z49Rn4m@tL*4t>f(dRuQ2XUXO1Mabbdtw+FkJlmm
zEpnb24c8?87V55fQIP8iUhQ>n?sql9Pdc}>aH~lA&5Ok%58OcWGn_AK{tT*@w;S&9
zBss1Y-Cr6mxCr{d7r!w+;4R_<?rHI4#kB>Zn+;cMxd2bLALt!_#kr6P*%`gCRSH;R
zZQa}=ekq3!dV{T=fD&%K4*Eu~M0n|>=RYcWePRC3Mt?~>d-@m0ddTnAJD?Bzpfzp>
z4fRcWksG7FmyUwodcz&OP%gUfIyX*vt`i=2?t2~vKVhp<ri5EJKh%5}^?@+pO;Eu+
z<y%rSZTmz1F&;=uNB>^re&a0wgl}QLKVkjPox99J!|NlNe+Tn#G~C*96%63|;BM!x
zg@Ef7cMBpDzEXDwTO$DvO7RQ+*OrGW?jt|{F6YNJKSFsn_m|d|QFz7lrntV0LJAk@
zZE*c?8@*b(|761*j_Orb@v`#zN#Nt0CPndgH{j|p$qByd5_-da|HIKxc?i?$1e$AD
z4|zz8`0w7G$fNa=znszoe}|*N+8~ybhI`o+Brhc({^b6=ywXQ4l~4SKRc=QbZhaQ}
zLmv}(PWeye75}C`=@<Hg{N+S`@elc@JihqP=+F31>52cLS|Y#ruT|U;k1zhso|0eu
z*HihY3gCCR;vVzs7xq&h)kGhX2ljtBCr>KB<T3iOKOBxi`LpyPd7zJ3k1u&L{$ui_
z^fny!uhr?_@`s-Uut%86d+I=959#m54DFrH%N|2j>O)1_`cce7A1D0&lQY1trvCUA
z;?eti)%*$TtJZ4JeuxL?qnzmD82zhUVLZ#eSLk0uX*?jl4@bix6>Rbs0{Qbvzy3Y?
z^I$6PCg~4LUK!;o{SW*u`t?%-SYMZGnm;mMjEx~&Ex%#ETs}a)I2@I$toKFl#uI?U
zLF$rYuK5!1%av6;Zr%I`^7o8uM!m{+edT(A`3Cv3#PvRZCi)(telQ<I6{Dqmw;yny
zQ+gt-{lnCL3$Ac{6q)FsWcVfJd-7YtQspn^Z{({b9075@f__qaZj`WpIklI?Uz)F^
zPxx<~pOk+jzK`ERV&5|PH~(jLvH3-x-#9{h3(n-PQv(UU{QIr8e6TTx_zedWe{KE>
z_R8c3<yZJW6-xSMZzVJixl_fwe{W1<zl?vrsr;z?VEz#VaXvMFok9MEKhmQ}-{epI
zcQW;F#s3E5JCiRqnXiy9Cp`W&`e#jBiQZNE=TMmVe~JE|$xmJV+5AsVd!=5WKT*By
z{v>_M-^RvZKJ#bETdpx5%3r}>*s4`#h5t9z8_lRc_Se|oQvY69P0vG~cbn#~$_4&>
zroZa#6Jg|^<h=gQ66%vjbl$GA!0_iM@?+|c*XocT`p%t~wKo663x?PqbzWM2tSP@V
zU@z#KS7NW4*b}VtGLui_ubF%Sdm>*9Q#bM-<>%~q$@5&tq0vYFV!rrStY5`vwMGY#
zk;|7-`M(W)N&nSB>R(QDHGuu<l?lMAPn0hjUjMI3|FkdTx!u0md_j7u?=~%8Q1x-W
z7oQi}@6@b2j#_K=V=~Nq@d)|rQ9VWee7!f3u}{$IOY1Ml*KC!GoGzKZQICSXCo$k*
z)MI7l?>@rUS&vmptj7ccKZr_ceuMsFf9rLYKZ2<0D6jFq-~53v?Rf>;M7vV|y21RC
zKTpC|PY<P&zOa8zKKN@IFQ^B>&%stj{#6mb`xAcp%cg(okGhjTJa$t0JE09M--c1q
zsorHh%y?+wjR^Ur5n%6w7f=rWz}M^G_vK|5h}H-A+?a8X$NhqjhvR^C{?I<JH{3`v
z4|;(A-};Xr8bR;m^#%QP>pxzT^ZNcE+kb2}O1JGdXpc7f^uJ_3*n{=|3B*L4kHU++
zZT-hk>W}0va=nU|r2nX>|4=;2-;tT&?{R!GzAZn;{YPVq+hzNY6LJ6XSIOVYgmt_A
zV^slS`snOGMy;NC-{R$9YcSw)@uPf)c7XYl^UAmT>JAA^@O#X^V=Ao#Zv=>sVc`!l
zK7>D@bffu0jGvn3tzTgN-Bnj{NcxM6PnBy<>nDp?Kcw}_*XxXLpP#uNUtTaC6MoTO
zt}XAat9eN14W;re7J=Vimw`Bs`Z|_Z@x0FZ)!uJv|3%j0d+G{MttbEO@~Ip(enr0z
zNgfLH^bbViqjX0Vnd@_)=ldtlBR{d;CI8X~`~mC7_5s!*w|NQc$Ld1ndg*(g)vJV2
zpU3>`{JuUz{BR(_Zxjx;LeZB!kiXWJN9(FDh(ASr4yN`~|1pA>3LwDwN%2$T`7D0c
z8>pXm*WJB-Uive8hKarXmpZ~PfS=)^JuyDwA!kp(Ys=$x5$AgN$APF);p+pPhvGM5
z?*;mU=|A<)jC~h~UoJCVlvA~z`G?h8+z<XZ<@sNv{KHmXaev|Ot?tD%&vdsrhyAM3
zpZr7op!_@H@vk}H_h~EXOK`ukkKilQ;Aii?F)Sxdx9;uvfX7F?gstA*w*Da(&v+kE
zpT%<Z2dEbhM1v~GMell}a3C5DM=>uW%71DqcsYjiG&m5|hUgC}YHWUZcML<GFlub^
z=-JwQ5&gzYVjtDBoX_?j&}R@?IOx2>JoMd1T^ap3z$5*XSNhwy<k)Xq&i5O=^!IGP
zaWdC$45#rbeZpUJ`RA@Q|M>bWiGSq>#6K_ZPjP(6Uy%RTmXW`Rz<K2xU!M_HJej@I
z{FRB1L-arMC#&~K5AjJ4Na*?cg75<Klf{SS7y4f&znoz_SbV1SDB^+2Oz36(KaC%Y
zH~McT-Xss=GtNJY_h26Rl^XT@ARdS58)-foN%egtpgqfXVO~LT{Sx89Bp&2%`>M=O
zvakGWUxoJOnDwjW`UD=gZlXU}r+?J+f+?9#{SE7xe=Wb;`8SjA7Yskl`D?a+fj=^S
zl77Yer+A}N%Rfg(m`~)NZ_K6VvHA-;&$InSY_HT`5brtv+mq23-=~>B*!e{J_G>HY
zTm43R{=k>yx$`rVKc;+s0>8o+jPgJ6SDr_^)BLhv@+bYo^-R6N`ds?S_XDu!9_mN-
zzs`6q_tRbCcnw=&ov``~=r6X9{2lbMzK{9D);9`aG~n|GsvNI>;wwL>zi8lhL>@MK
z>}Ia=_(8tE5dS@@moojuP2$JzlcREx>WKTBTt5N&>_0fK{Z&ujroT`>u&XV9m&BU%
zk9fFke<1ys{q20%kM-rU@yGgY2Klu&_x>39aTn*M|2OB<A1wWc{va<;XMZ4C??db0
zXS@ADd%y5`!#n*)=KGIN)qiCA4aHNY-`VLukpFi259WikKDmAW(Tw{K)#r22-|hPk
zthfGy>+hid0KWAf!vF032k2S<Vg1Fn`;TVae;7Z>dFJaa=p)~MG~@n*>yb}${pP3W
zKbmp>k?gm<{-YWB{zLe4;J3Yg*k1nude(oC-uflTgL?G=I1uZ>JLo@}asNSjs0W*I
z|6wrRhxz`)-~h1oAL+dHAB3Ub&)R=L-`0PK{v7uA{fFr9^dE6O`@hnEARev%pncYv
zkMjKo<4666?0E_H^!<lm`kU`R42FOC{(~_6sg>_P=#PANu`izAX8!?v>px=rZT27V
zN9#X`ztexvp4NX*ALu^-TmK>bUV{B@-+y4e^&e)BZTBCrhxH$Z|CQ@MAiwn=l>aNx
ze<1Wf)5Y(u{4cO_bY?yDGky1!PPKvhZE83CKl+8gZ$oGU9p8QE^;o)vo-TW$_T8JU
zN#1*6g?d8Vp8&yjnk4X=no4|^(Kguzy039DUf0<z^J!hOyX=Qz89mu#;bZpHfq&WK
zw=(#Aao3Ljn@_;6=sOVSviN-E(~gf{QSj>6f3*vK*ae^e2Z8rT9rz!4{F|hnx$I?v
zfB%f}?*pH9X}{j~3y=5->W{p~_~eDx3;3f5?LJWPL|-uZxKzKnYM1{VjCaHTvAsod
zS^RGLdl1Oo@a=^f_~WF%y6Nve%yEe~cf8cC`hPiDwYO!8F^Dv=nyj#gU*Oga-|?j*
zt}H%8>cIaPYr5gD9uj&EA9n7*C*S-<X7Yb>hL(I@^H1?TwjcK-UTmiX!9RTD11r9_
J<861pe*@J{*f;<H

literal 0
HcmV?d00001

diff --git a/libcaf_cuda/tests/control-layer-unit-tests/shared_mmul.cu b/libcaf_cuda/tests/control-layer-unit-tests/shared_mmul.cu
new file mode 100644
index 0000000000..85c361ed93
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-unit-tests/shared_mmul.cu
@@ -0,0 +1,51 @@
+extern "C" __global__
+void matrixMul(const int* __restrict__ a,
+               const int* __restrict__ b,
+               int* __restrict__ c,
+               int N)
+{
+    const int TILE = 32;
+    int row = blockIdx.y * blockDim.y + threadIdx.y; // global row in C
+    int col = blockIdx.x * blockDim.x + threadIdx.x; // global col in C
+
+    __shared__ int s_a[TILE * TILE];
+    __shared__ int s_b[TILE * TILE];
+
+    int acc = 0;
+
+    // Sweep tiles across the K dimension
+    for (int i = 0; i < N; i += TILE) {
+
+        // Each thread loads one element into shared memory (with bounds checks)
+        int aCol = i + threadIdx.x;
+        int bRow = i + threadIdx.y;
+
+        // s_a[y, x] = a[row, aCol] if in range, else 0
+        if (row < N && aCol < N)
+            s_a[threadIdx.y * TILE + threadIdx.x] = a[row * N + aCol];
+        else
+            s_a[threadIdx.y * TILE + threadIdx.x] = 0;
+
+        // s_b[y, x] = b[bRow, col] if in range, else 0
+        if (bRow < N && col < N)
+            s_b[threadIdx.y * TILE + threadIdx.x] = b[bRow * N + col];
+        else
+            s_b[threadIdx.y * TILE + threadIdx.x] = 0;
+
+        __syncthreads();
+
+        // Compute partial dot product for this tile
+        #pragma unroll
+        for (int k = 0; k < TILE; ++k) {
+            acc += s_a[threadIdx.y * TILE + k] *
+                   s_b[k * TILE + threadIdx.x];
+        }
+
+        __syncthreads();
+    }
+
+    // Final write (guarded)
+    if (row < N && col < N)
+        c[row * N + col] = acc;
+}
+

From 6d552dd12035d6d88638332dac3abdfa13cff691 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 12:18:00 -0600
Subject: [PATCH 0201/1000] Fixed syntax errors.

---
 libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp b/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
index a3ba4b6e31..d90c1dc950 100644
--- a/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
@@ -6,7 +6,7 @@
 
 #include <caf/all.hpp>
 #include <caf/actor_system.hpp>
-#include "caf/cuda/kernel_graph.hpp"
+#include "caf/cuda/control-layer/kernel_graph.hpp"
 #include <iostream>
 #include <vector>
 #include <string>
@@ -98,7 +98,7 @@ void test_kernel_graph_push_pop([[maybe_unused]] caf::actor_system& sys) {
 
     // getOperation should remove and return the element (nullptr expected)
     auto popped = g.getOperation();
-    (void)popped; // we don't inspect the token itself in this test
+    //(void)popped; // we don't inspect the token itself in this test
 
     // after popping, graph should be empty again
     if (!g.empty()) {

From 706140d8f75eb82988c8b7bf3f8322dd1d5d03bc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 12:40:26 -0600
Subject: [PATCH 0202/1000] Initial commit.

---
 .../heuristic_function.hpp                    | 40 +++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
new file mode 100644
index 0000000000..c88db24d70
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
@@ -0,0 +1,40 @@
+#pragma once
+
+#include "caf/cuda/program.hpp"
+#include "caf/cuda/nd_range.hpp"
+
+#include <unordered_map>
+#include <string>
+
+namespace caf::cuda {
+
+// Abstract heuristic function object.
+// Subclasses estimate the "cost" of launching a kernel based on
+// the program and execution configuration.
+class heuristic_function {
+public:
+  /// Default constructor
+  heuristic_function() = default;
+
+  /// Copy constructor from another heuristic_function
+  /// Copies all internal heuristic values 
+  heuristic_function(const heuristic_function& other)
+    : values_(other.values__) {}
+
+  /// Virtual destructor (required for polymorphic base classes)
+  virtual ~heuristic_function() = default;
+
+  /// Abstract cost function
+  /// @param prog Kernel program
+  /// @param range Kernel execution configuration
+  /// @return Cost metric (interpretation left to implementation)
+  virtual int getCost(const program_ptr& prog,
+                      const nd_range& range) const = 0;
+
+protected:
+  /// Heuristic-specific values
+  std::unordered_map<std::string, int> values_;
+};
+
+} // namespace caf::cuda
+

From 9042ec278e94f01bdc574e8a4e06d27344f49ba4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 12:45:05 -0600
Subject: [PATCH 0203/1000] Implemented getName method.

---
 libcaf_cuda/caf/cuda/program.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/program.hpp b/libcaf_cuda/caf/cuda/program.hpp
index cc6fbbf252..8cbd220ea9 100644
--- a/libcaf_cuda/caf/cuda/program.hpp
+++ b/libcaf_cuda/caf/cuda/program.hpp
@@ -39,6 +39,7 @@ class CAF_CUDA_EXPORT program : public caf::ref_counted {
 	}
     }
 
+    std::string getName() {return name_;}
 
 private:
   /// Internal helper to load the kernel modules on all devices.

From a994c8a75e4979b1622e6fd3451f47bf22dd8315 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 12:53:51 -0600
Subject: [PATCH 0204/1000] Implemented to_string method.

---
 libcaf_cuda/caf/cuda/nd_range.hpp | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/nd_range.hpp b/libcaf_cuda/caf/cuda/nd_range.hpp
index 142b0b7c0b..5fc3986536 100644
--- a/libcaf_cuda/caf/cuda/nd_range.hpp
+++ b/libcaf_cuda/caf/cuda/nd_range.hpp
@@ -58,7 +58,23 @@ class nd_range {
 
 
   ~nd_range() {
-  //no-op
+	  //no-op
+  }
+
+
+  // Returns a stable string representation of grid + block dims
+
+  [[nodiscard]] std::string to_string() const {
+	  std::ostringstream oss;
+	  oss << "grid("
+		  << gridDim[0] << ","
+		  << gridDim[1] << ","
+		  << gridDim[2] << ")"
+		  << "_block("
+		  << blockDim[0] << ","
+		  << blockDim[1] << ","
+		  << blockDim[2] << ")";
+	  return oss.str();
   }
 
 

From bcdcdfdad79944d08f7980f1bb0078275d9cc6b7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 12:56:05 -0600
Subject: [PATCH 0205/1000] Initial commit.

---
 .../core_heuristic_function.hpp               | 77 +++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
new file mode 100644
index 0000000000..5eaaf450ee
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
@@ -0,0 +1,77 @@
+#pragma once
+
+#include "caf/cuda/program.hpp"
+#include "caf/cuda/nd_range.hpp"
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/control-layer/scheduler-functions.hpp"
+
+#include <unordered_map>
+#include <string>
+
+namespace caf::cuda {
+
+// Concrete heuristic based on GPU occupancy / device properties
+class cost_heuristic_function : public heuristic_function {
+public:
+  /// Construct with device (no prior state)
+  explicit cost_heuristic_function(device_ptr dev)
+    : dev_(dev) {
+    init_device_properties();
+  }
+
+  /// Construct with device + copy state from another heuristic
+  cost_heuristic_function(device_ptr dev,
+                          const heuristic_function& other)
+    : heuristic_function(other),
+      dev_(dev) {
+    init_device_properties();
+  }
+
+  /// Virtual destructor
+  ~cost_heuristic_function() override = default;
+
+  /// Returns cached or computed cost
+  int getCost(const program_ptr& prog,
+              const nd_range& range) const override {
+
+    const std::string key =
+      prog->getName() + "_" + range.to_string();
+
+    auto it = values_.find(key);
+    if (it != values_.end())
+      return it->second;
+
+    // Compute cost (occupancy-based for now)
+    int cost = dev_->max_active_blocks_per_sm(prog, range);
+
+    values_[key] = cost;
+    return cost;
+  }
+
+private:
+  void init_device_properties() {
+    sm_count_            = dev_->sm_count();
+    warp_size_           = dev_->warp_size();
+    max_threads_per_sm_  = dev_->max_threads_per_sm();
+    warps_per_sm_        = max_threads_per_sm_ / warp_size_;
+    total_warps_         = sm_count_ * warps_per_sm_;
+    total_mem_bytes_     = dev_->total_global_mem();
+  }
+
+private:
+  device_ptr dev_;
+
+  // Mutable because getCost() is logically const but caches values
+  mutable std::unordered_map<std::string, int> values_;
+
+  // Cached GPU properties
+  int sm_count_ = 0;
+  int warp_size_ = 0;
+  int max_threads_per_sm_ = 0;
+  int warps_per_sm_ = 0;
+  int total_warps_ = 0;
+  std::size_t total_mem_bytes_ = 0;
+};
+
+} // namespace caf::cuda
+

From f50e4b08d3b9b4595a8dd4ea6e215a802168c1e0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 13:11:46 -0600
Subject: [PATCH 0206/1000] Implemented an error code macro. This change is
 made since the functions are meant to catch device specific errors and handle
 them for the actor, shielding them from them.

---
 .../control-layer/scheduler-functions/heuristic_function.hpp  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
index c88db24d70..afacd70ad3 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
@@ -6,6 +6,10 @@
 #include <unordered_map>
 #include <string>
 
+
+#define ERROR_CODE -999999
+
+
 namespace caf::cuda {
 
 // Abstract heuristic function object.

From a58a569757484d31e7996b4422f913b6d56b0466 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 13:18:13 -0600
Subject: [PATCH 0207/1000] Added heuristic function includes.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 6e744289e2..0af2728f51 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -14,6 +14,8 @@
 #include "caf/cuda/control-layer/behavior.hpp"
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
 #include "caf/cuda/control-layer/token_factory.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp"
 
 // -----------------------------------------------------------------------------
 // Type IDs (required for typed behaviors)

From e7e14aef45e33f628be62ef74d99f944c182b6ce Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 13:23:10 -0600
Subject: [PATCH 0208/1000] Fixed syntax errors.

---
 .../control-layer/scheduler-functions/heuristic_function.hpp    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
index afacd70ad3..51364a9c34 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
@@ -23,7 +23,7 @@ class heuristic_function {
   /// Copy constructor from another heuristic_function
   /// Copies all internal heuristic values 
   heuristic_function(const heuristic_function& other)
-    : values_(other.values__) {}
+    : values_(other.values_) {}
 
   /// Virtual destructor (required for polymorphic base classes)
   virtual ~heuristic_function() = default;

From 1caab862a8922c1fabdb9018033e7706cac1f4d4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 13:32:20 -0600
Subject: [PATCH 0209/1000] Fixed syntax errors.

---
 .../control-layer/scheduler-functions/heuristic_function.hpp    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
index 51364a9c34..66a1b623fc 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
@@ -33,7 +33,7 @@ class heuristic_function {
   /// @param range Kernel execution configuration
   /// @return Cost metric (interpretation left to implementation)
   virtual int getCost(const program_ptr& prog,
-                      const nd_range& range) const = 0;
+                      const nd_range& range) = 0;
 
 protected:
   /// Heuristic-specific values

From aa8f7a1e159986583a6f2bb1527f5f7c62b6c675 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 13:32:48 -0600
Subject: [PATCH 0210/1000] Renamed class to match file name, and made the
 getCost method catch any errors related to touching the GPU.

---
 .../core_heuristic_function.hpp               | 88 +++++++------------
 1 file changed, 31 insertions(+), 57 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
index 5eaaf450ee..7103c679a4 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
@@ -1,76 +1,50 @@
 #pragma once
 
-#include "caf/cuda/program.hpp"
-#include "caf/cuda/nd_range.hpp"
-#include "caf/cuda/device.hpp"
-#include "caf/cuda/control-layer/scheduler-functions.hpp"
-
-#include <unordered_map>
 #include <string>
+#include <exception>
+
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/nd_range.hpp"
+#include "caf/cuda/program.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp"
 
 namespace caf::cuda {
 
-// Concrete heuristic based on GPU occupancy / device properties
-class cost_heuristic_function : public heuristic_function {
+class core_heuristic_function : public heuristic_function {
 public:
-  /// Construct with device (no prior state)
-  explicit cost_heuristic_function(device_ptr dev)
-    : dev_(dev) {
-    init_device_properties();
-  }
+  explicit core_heuristic_function(device_ptr dev)
+    : dev_(dev) {}
 
-  /// Construct with device + copy state from another heuristic
-  cost_heuristic_function(device_ptr dev,
+  // Copy from ANY heuristic_function
+  core_heuristic_function(device_ptr dev,
                           const heuristic_function& other)
     : heuristic_function(other),
-      dev_(dev) {
-    init_device_properties();
-  }
+      dev_(dev) {}
 
-  /// Virtual destructor
-  ~cost_heuristic_function() override = default;
-
-  /// Returns cached or computed cost
   int getCost(const program_ptr& prog,
-              const nd_range& range) const override {
-
-    const std::string key =
-      prog->getName() + "_" + range.to_string();
-
-    auto it = values_.find(key);
-    if (it != values_.end())
-      return it->second;
-
-    // Compute cost (occupancy-based for now)
-    int cost = dev_->max_active_blocks_per_sm(prog, range);
-
-    values_[key] = cost;
-    return cost;
-  }
-
-private:
-  void init_device_properties() {
-    sm_count_            = dev_->sm_count();
-    warp_size_           = dev_->warp_size();
-    max_threads_per_sm_  = dev_->max_threads_per_sm();
-    warps_per_sm_        = max_threads_per_sm_ / warp_size_;
-    total_warps_         = sm_count_ * warps_per_sm_;
-    total_mem_bytes_     = dev_->total_global_mem();
+              const nd_range& range) override {
+    try {
+      const std::string key =
+        prog->getName() + range.to_string();
+
+      auto it = values_.find(key);
+      if (it != values_.end())
+        return it->second;
+
+      int cost = dev_->max_active_blocks_per_sm(prog, range);
+      values_[key] = cost;
+      return cost;
+    }
+    catch (const std::exception&) {
+      return ERROR_CODE;
+    }
+    catch (...) {
+      return ERROR_CODE;
+    }
   }
 
 private:
   device_ptr dev_;
-
-  // Mutable because getCost() is logically const but caches values
-  mutable std::unordered_map<std::string, int> values_;
-
-  // Cached GPU properties
-  int sm_count_ = 0;
-  int warp_size_ = 0;
-  int max_threads_per_sm_ = 0;
-  int warps_per_sm_ = 0;
-  int total_warps_ = 0;
-  std::size_t total_mem_bytes_ = 0;
 };
 
 } // namespace caf::cuda

From 9170bda2cc5736fd7cf2da374d0a2ed630bbf4d4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 13:37:47 -0600
Subject: [PATCH 0211/1000] Added kernel_graph include.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 0af2728f51..5696e6c307 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -16,6 +16,8 @@
 #include "caf/cuda/control-layer/token_factory.hpp"
 #include "caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp"
 #include "caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp"
+#include "caf/cuda/control-layer/kernel_graph.hpp"
+
 
 // -----------------------------------------------------------------------------
 // Type IDs (required for typed behaviors)

From b360a6ff2cee5bae88673659a49f2ce811a05e66 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 13:40:01 -0600
Subject: [PATCH 0212/1000] Updated driver to include cost heuristic function
 tests.

---
 .../control-layer-unit-tests/main.test.cpp    | 79 ++++++++++++++++++-
 1 file changed, 78 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp b/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
index d90c1dc950..da709e6241 100644
--- a/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
@@ -5,6 +5,8 @@
 
 
 #include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
 #include <caf/actor_system.hpp>
 #include "caf/cuda/control-layer/kernel_graph.hpp"
 #include <iostream>
@@ -109,10 +111,85 @@ void test_kernel_graph_push_pop([[maybe_unused]] caf::actor_system& sys) {
     std::cout << "[test_kernel_graph_push_pop] OK\n";
 }
 
+
+// 3) Test core_heuristic_function
+void test_core_heuristic_function([[maybe_unused]] caf::actor_system& sys) {
+    std::cout << "\n[core_heuristic_function test]\n";
+
+    caf::cuda::manager::init(sys);
+    auto& mgr = manager::get();
+    auto dev = mgr.find_device(0);
+
+    if (!dev) {
+        std::cerr << "No CUDA device found, skipping test\n";
+        return;
+    }
+
+    // Create two programs
+    auto prog1 = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+    auto prog2 = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    // nd_ranges
+    nd_range r1(32, 32, 1, 32, 32, 1);
+    nd_range r2(16, 16, 1, 16, 16, 1);
+
+    core_heuristic_function h(dev);
+
+    // ------------------------------------------------------------
+    std::cout << "\n[Test] same program + same nd_range\n";
+    int cost1 = h.getCost(prog1, r1);
+    int cost2 = h.getCost(prog1, r1);
+
+    std::cout << "cost #1: " << cost1 << "\n";
+    std::cout << "cost #2: " << cost2 << "\n";
+
+    // ------------------------------------------------------------
+    std::cout << "\n[Test] same program + different nd_range\n";
+    int cost3 = h.getCost(prog1, r2);
+
+    std::cout << "cost r1: " << cost1 << "\n";
+    std::cout << "cost r2: " << cost3 << "\n";
+
+    // ------------------------------------------------------------
+    std::cout << "\n[Test] different program_ptr + same nd_range\n";
+    int cost4 = h.getCost(prog2, r1);
+
+    std::cout << "prog1 cost: " << cost1 << "\n";
+    std::cout << "prog2 cost: " << cost4 << "\n";
+    std::cout << "(program_ptr identity differs: " << (&(*prog1) != &(*prog2)) << ")\n";
+
+    // ------------------------------------------------------------
+    std::cout << "\n[Test] cache stability (repeated calls)\n";
+    int cost5 = h.getCost(prog1, r1);
+    int cost6 = h.getCost(prog1, r1);
+
+    std::cout << "repeat #1: " << cost5 << "\n";
+    std::cout << "repeat #2: " << cost6 << "\n";
+
+    // ------------------------------------------------------------
+    std::cout << "\n[Test] copy constructor preserves cache\n";
+    core_heuristic_function h_copy(dev, h);
+
+    int cost7 = h_copy.getCost(prog1, r1);
+    int cost8 = h_copy.getCost(prog1, r2);
+
+    std::cout << "original r1: " << cost1 << ", copied r1: " << cost7 << "\n";
+    std::cout << "original r2: " << cost3 << ", copied r2: " << cost8 << "\n";
+
+    std::cout << "\n[core_heuristic_function test complete]\n";
+    caf::cuda::manager::shutdown();
+}
+
+
+
+
+
+
 // Register tests
 const std::vector<Test> tests = {
     {"test_kernel_graph_empty", test_kernel_graph_empty},
-    {"test_kernel_graph_push_pop", test_kernel_graph_push_pop}
+    {"test_kernel_graph_push_pop", test_kernel_graph_push_pop},
+    {"test_core_heuristic_function", test_core_heuristic_function}
 };
 
 // Run a single test and return status code:

From cddcf923db01f474529a93834b6e800d951d2d67 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 13:43:51 -0600
Subject: [PATCH 0213/1000] Updated test_core_heuristic_function to operator
 using asserts rather than print by inspection.

---
 .../control-layer-unit-tests/main.test.cpp    | 50 +++++++------------
 1 file changed, 18 insertions(+), 32 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp b/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
index da709e6241..96655283a7 100644
--- a/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
@@ -13,6 +13,7 @@
 #include <vector>
 #include <string>
 #include <stdexcept>
+#include <cassert>
 
 using namespace caf::cuda;
 
@@ -113,70 +114,54 @@ void test_kernel_graph_push_pop([[maybe_unused]] caf::actor_system& sys) {
 
 
 // 3) Test core_heuristic_function
-void test_core_heuristic_function([[maybe_unused]] caf::actor_system& sys) {
-    std::cout << "\n[core_heuristic_function test]\n";
+#include <cassert>  // for assert
 
+void test_core_heuristic_function([[maybe_unused]] caf::actor_system& sys) {
     caf::cuda::manager::init(sys);
     auto& mgr = manager::get();
     auto dev = mgr.find_device(0);
 
     if (!dev) {
-        std::cerr << "No CUDA device found, skipping test\n";
+        // Skip test if no CUDA device
         return;
     }
 
-    // Create two programs
     auto prog1 = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
     auto prog2 = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
-    // nd_ranges
     nd_range r1(32, 32, 1, 32, 32, 1);
     nd_range r2(16, 16, 1, 16, 16, 1);
 
     core_heuristic_function h(dev);
 
-    // ------------------------------------------------------------
-    std::cout << "\n[Test] same program + same nd_range\n";
+    // Same program + same nd_range: cost should be consistent
     int cost1 = h.getCost(prog1, r1);
     int cost2 = h.getCost(prog1, r1);
+    assert(cost1 == cost2);
 
-    std::cout << "cost #1: " << cost1 << "\n";
-    std::cout << "cost #2: " << cost2 << "\n";
-
-    // ------------------------------------------------------------
-    std::cout << "\n[Test] same program + different nd_range\n";
+    // Same program + different nd_range: cost should differ or be independent
     int cost3 = h.getCost(prog1, r2);
+    assert(cost1 != ERROR_CODE);
+    assert(cost3 != ERROR_CODE);
 
-    std::cout << "cost r1: " << cost1 << "\n";
-    std::cout << "cost r2: " << cost3 << "\n";
-
-    // ------------------------------------------------------------
-    std::cout << "\n[Test] different program_ptr + same nd_range\n";
+    // Different program + same nd_range: cost may differ
     int cost4 = h.getCost(prog2, r1);
+    assert(cost4 != ERROR_CODE);
 
-    std::cout << "prog1 cost: " << cost1 << "\n";
-    std::cout << "prog2 cost: " << cost4 << "\n";
-    std::cout << "(program_ptr identity differs: " << (&(*prog1) != &(*prog2)) << ")\n";
-
-    // ------------------------------------------------------------
-    std::cout << "\n[Test] cache stability (repeated calls)\n";
+    // Cache stability: repeated calls give same result
     int cost5 = h.getCost(prog1, r1);
     int cost6 = h.getCost(prog1, r1);
+    assert(cost5 == cost6);
 
-    std::cout << "repeat #1: " << cost5 << "\n";
-    std::cout << "repeat #2: " << cost6 << "\n";
-
-    // ------------------------------------------------------------
-    std::cout << "\n[Test] copy constructor preserves cache\n";
+    // Copy constructor preserves cache
     core_heuristic_function h_copy(dev, h);
-
     int cost7 = h_copy.getCost(prog1, r1);
     int cost8 = h_copy.getCost(prog1, r2);
 
-    std::cout << "original r1: " << cost1 << ", copied r1: " << cost7 << "\n";
-    std::cout << "original r2: " << cost3 << ", copied r2: " << cost8 << "\n";
+    // Should match the original costs
+    assert(cost7 == cost1);
+    assert(cost8 == cost3);
 
-    std::cout << "\n[core_heuristic_function test complete]\n";
     caf::cuda::manager::shutdown();
 }
 
@@ -185,6 +170,7 @@ void test_core_heuristic_function([[maybe_unused]] caf::actor_system& sys) {
 
 
 
+
 // Register tests
 const std::vector<Test> tests = {
     {"test_kernel_graph_empty", test_kernel_graph_empty},

From 9a9e456c715b7581ecd2e07c50423d929480ec67 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 24 Jan 2026 13:51:19 -0600
Subject: [PATCH 0214/1000] Updated CMakelists  to grab every cpp file in src
 directory automatically instead of manually having to type each one.

---
 libcaf_cuda/CMakeLists.txt | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/libcaf_cuda/CMakeLists.txt b/libcaf_cuda/CMakeLists.txt
index 32d0283718..8a95fcf535 100644
--- a/libcaf_cuda/CMakeLists.txt
+++ b/libcaf_cuda/CMakeLists.txt
@@ -14,21 +14,8 @@ file(GLOB_RECURSE CAF_CUDA_HEADERS "caf/*.hpp")
 
 
 # list cpp files excluding platform-dependent files
-set(LIBCAF_CUDA_SRCS
-  src/manager.cpp
-  src/platform.cpp
-  src/program.cpp
-  src/device.cpp
-  src/helpers.cpp
-  src/test.cpp
-  src/scheduler.cpp
-  src/streampool.cpp
-  src/control-layer/behavior.cpp
-  src/control-layer/behavior_table.cpp
-  src/control-layer/scheduler_actor.cpp
-  src/control-layer/red_light_behavior.cpp
-  src/control-layer/green_light_behavior.cpp
-  src/control-layer/token_factory.cpp
+file(GLOB_RECURSE LIBCAF_CUDA_SRCS
+    src/*.cpp
 )
 
 # -- add targets ---------------------------------------------------------------

From 6a61b36f75a52c815a920c17388e9ffc47552b2a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 27 Jan 2026 14:01:18 -0600
Subject: [PATCH 0215/1000] Updated state to include std::vector of kernel
 graphs, so that scheduler actor can map operation dependencies correctly.

---
 .../caf/cuda/control-layer/scheduler_actor_state.hpp       | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
index 5d41f4c374..192d42c79a 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
@@ -1,8 +1,10 @@
 #pragma once
 #include <queue>
+#include <vector>
 #include "caf/cuda/control-layer/behavior_table.hpp"
-#include "caf/cuda/control-layer/behavior_token.hpp"  // Still needed for behavior_token
+#include "caf/cuda/control-layer/behavior_token.hpp" 
 #include "caf/cuda/control-layer/token.hpp"
+#include "caf/cuda/control-layer/kernel_graph.hpp"
 
 namespace caf::cuda {
 class scheduler_actor_behavior; 
@@ -11,7 +13,8 @@ struct scheduler_actor_state {
     caf::actor  self;
     scheduler_actor_behavior* current_behavior = nullptr;
     behavior_table table;
-    std::queue<token_ptr> queue;
+    std::queue<token_ptr> queue; // here for legacy prototype schedulers
+    std::vector<kernel_graph> operations; //more modern dependency based data structure 
     int device_number;
 };
 } // namespace caf::cuda

From 93cb13378f29612457030c60b3dc8ac1d9acec09 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 27 Jan 2026 14:12:11 -0600
Subject: [PATCH 0216/1000] Added a token_ptr overide to the cost function,
 that just calls the implemented getCost This change was made to encapsulate
 type casting, so its not everywhere in the codebase.

---
 .../scheduler-functions/heuristic_function.hpp        | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
index 66a1b623fc..594b69b5cc 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
@@ -33,7 +33,16 @@ class heuristic_function {
   /// @param range Kernel execution configuration
   /// @return Cost metric (interpretation left to implementation)
   virtual int getCost(const program_ptr& prog,
-                      const nd_range& range) = 0;
+		  const nd_range& range) = 0;
+
+
+  /// Token-based cost function (default implementation)
+  virtual int getCost(const token_ptr& tok) {
+	  // Assume type checking is done elsewhere
+	  const auto& launch = static_cast<const launch_token&>(*tok);
+	  return getCost(launch.getProgram(), launch.getRange());
+  }
+
 
 protected:
   /// Heuristic-specific values

From 405c4d255fb28c251326ea9572c837436daad72a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 27 Jan 2026 14:16:58 -0600
Subject: [PATCH 0217/1000] Updated header file to include dev_ptr and
 heuristic functor.

---
 .../caf/cuda/control-layer/core_usage_behavior.hpp  | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index 51bd602f99..d8d1aaf330 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -1,5 +1,7 @@
 #pragma once
 #include "caf/cuda/control-layer/behavior.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/core_usage_heuristic.hpp"
+#include "caf/cuda/device.hpp"
 
 namespace caf::cuda {
 
@@ -9,6 +11,17 @@ class core_usage_behavior : public scheduler_actor_behavior {
     void on_enter() override;
     void schedule() override;
     void receive(const token_ptr& tok) override;
+    core_usage_behavior() {
+	    init_state();
+    }
+
+private:
+   dev_ptr device_;
+   core_heuristic_function heuristic;
+    
+    
+    void init_state();
+
 
 };
 

From 2d6f303ce93c30cf75babd23f6e674592f7aa52b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 27 Jan 2026 14:19:59 -0600
Subject: [PATCH 0218/1000] Added a virtual reclaim method that will be invoked
 once resources have been reclaimed.

---
 libcaf_cuda/caf/cuda/control-layer/behavior.hpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
index c13bb7c3d8..1b983f317d 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
@@ -31,6 +31,12 @@ class scheduler_actor_behavior {
     virtual void schedule() = 0;
     virtual void receive(const token_ptr& tok) = 0;
 
+    virtual void reclaim(int value, int memory_returned,int runtime) {
+	    //default implementation is to do nothing, this should be overidden
+	    //by children classes 
+    }
+
+
 protected:
     scheduler_actor_state& state_;
 

From 43a8b99fa81d1cb3078e13654fd21785bb422e4b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 27 Jan 2026 14:24:49 -0600
Subject: [PATCH 0219/1000] Updated header to have more private attributes and
 override the reclaim method.

---
 libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index d8d1aaf330..ad679b5508 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -11,6 +11,7 @@ class core_usage_behavior : public scheduler_actor_behavior {
     void on_enter() override;
     void schedule() override;
     void receive(const token_ptr& tok) override;
+    void reclaim(int value /*blocks consumed*/,int memory_returned,int time) override; 
     core_usage_behavior() {
 	    init_state();
     }
@@ -18,7 +19,8 @@ class core_usage_behavior : public scheduler_actor_behavior {
 private:
    dev_ptr device_;
    core_heuristic_function heuristic;
-    
+   int total_SM;
+   int available_SM; 
     
     void init_state();
 

From 70bb629aa2ddbad7a5d7d85997653b9d858a21b4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 27 Jan 2026 14:36:04 -0600
Subject: [PATCH 0220/1000] Refactored tokens to have a superclass called
 request token. Change was made to not have to put as much stuff in token.

---
 .../caf/cuda/control-layer/launch_token.hpp   | 84 ++++++++++---------
 .../control-layer/memory_transfer_token.hpp   | 59 +++++++------
 .../caf/cuda/control-layer/request_token.hpp  | 29 +++++++
 3 files changed, 102 insertions(+), 70 deletions(-)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/request_token.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
index 38d5f1d7da..702898696d 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
@@ -1,51 +1,55 @@
 #pragma once
-#include <caf/all.hpp>  // For caf::actor, caf::intrusive_ptr 
+
+#include <caf/all.hpp>
 #include "caf/cuda/program.hpp"
-#include "caf/cuda/control-layer/token.hpp"
-#include "caf/cuda/nd_range.hpp"  // For nd_range (full def needed for member)
+#include "caf/cuda/control-layer/request_token.hpp"
+#include "caf/cuda/nd_range.hpp"
 #include "caf/cuda/global_export.hpp"
 
-
 namespace caf::cuda {
-class program; 
-using program_ptr = caf::intrusive_ptr<program>;  
 
-class CAF_CUDA_EXPORT launch_token : public token {
+class CAF_CUDA_EXPORT launch_token : public request_token {
 public:
+  // CAF compliance only
+  launch_token() = default;
+
+  launch_token(program_ptr prog,
+               nd_range range,
+               int memory_usage,
+               std::string id,
+               caf::actor receiver,
+               int dependency = INDEPENDENT)
+    : request_token(dependency),
+      program_(std::move(prog)),
+      range_(std::move(range)),
+      memory_usage_(memory_usage),
+      id_(std::move(id)),
+      reply_handle_(receiver) {}
+
+  int getType() const override { return LAUNCH; }
+
+  const program_ptr& getProgram() const { return program_; }
+  const nd_range& getRange() const { return range_; }
+  int getMemoryUsage() const { return memory_usage_; }
+  caf::actor getReplyActor() const { return reply_handle_; }
+
+  int getBlocks() const {
+    return static_cast<int>(
+      range_.getGridDimX() *
+      range_.getGridDimY() *
+      range_.getGridDimZ()
+    );
+  }
+
+  const std::string& getId() const { return id_; }
 
-	//only here to be complaint with CAFS type if system DO NOT USE 
-	launch_token() = default;
-    
-	
-	launch_token(program_ptr prog,
-                 nd_range range,
-                 int memory_usage,
-                 std::string id,
-                 caf::actor receiver)
-        : program_(std::move(prog)),
-          range_(std::move(range)),
-          memory_usage_(memory_usage),
-          id_(std::move(id)),
-          reply_handle_(receiver) {}  // Fixed missing )
-    int getType() const override { return LAUNCH; }
-    const program_ptr& getProgram() const { return program_; }
-    const nd_range& getRange() const { return range_; }
-    int getMemoryUsage() const { return memory_usage_; }
-    caf::actor getReplyActor() const { return reply_handle_; }
-    // Return requested number of CUDA blocks (gridDimX * gridDimY * gridDimZ)
-    int getBlocks() const {
-        return static_cast<int>(
-            range_.getGridDimX() *
-            range_.getGridDimY() *
-            range_.getGridDimZ()
-        );
-    }
-    const std::string& getId() const { return id_; }
 private:
-    program_ptr program_;
-    nd_range range_;
-    int memory_usage_;
-    std::string id_;
-    caf::actor reply_handle_;
+  program_ptr program_;
+  nd_range range_;
+  int memory_usage_;
+  std::string id_;
+  caf::actor reply_handle_;
 };
+
 } // namespace caf::cuda
+
diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
index 8a4bb381aa..5b92e0d8cd 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_transfer_token.hpp
@@ -1,7 +1,6 @@
 #pragma once
-#include "caf/cuda/control-layer/token.hpp"
-
 
+#include "caf/cuda/control-layer/request_token.hpp"
 /*
  * This token represents transfer of memory 
  * from either host or device 
@@ -9,37 +8,37 @@
  */
 
 
-//direction defines
+// direction defines
 #define H2D 1
 #define D2H 2
 
-
 namespace caf::cuda {
 
-class CAF_CUDA_EXPORT memory_transfer_token : public token {
-
-	public:
-		memory_transfer_token(int size,int direction,caf::actor replyActor):
-			size_(size),
-			direction_(direction),
-       			replyActor_(replyActor)	{}
-
-		//only here to ensure that caf can copy the object for message
-		//passing do not use
-		memory_transfer_token() = default;
-
-		virtual int getType() const {return MEMORY;}
-		int getSize() const {return size_;}
-		int getDirection() const {return direction_;}
-		caf::actor getReplyActor() const {return replyActor_;}
-	private:
-		int size_;
-		int direction_; 
-		caf::actor replyActor_;
-
-
-};//memory transfer token class
-
-
-}//caf cuda namespace
+class CAF_CUDA_EXPORT memory_transfer_token : public request_token {
+public:
+  memory_transfer_token(int size,
+                        int direction,
+                        caf::actor replyActor,
+                        int dependency = INDEPENDENT)
+    : request_token(dependency),
+      size_(size),
+      direction_(direction),
+      replyActor_(replyActor) {}
+
+  // CAF compliance
+  memory_transfer_token() = default;
+
+  int getType() const override { return MEMORY; }
+
+  int getSize() const { return size_; }
+  int getDirection() const { return direction_; }
+  caf::actor getReplyActor() const { return replyActor_; }
+
+private:
+  int size_;
+  int direction_;
+  caf::actor replyActor_;
+};
+
+} // namespace caf::cuda
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/request_token.hpp b/libcaf_cuda/caf/cuda/control-layer/request_token.hpp
new file mode 100644
index 0000000000..6a71c03127
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/request_token.hpp
@@ -0,0 +1,29 @@
+#pragma once
+
+#include "caf/cuda/control-layer/token.hpp"
+
+
+#define INDEPENDENT -1
+
+
+namespace caf::cuda {
+
+class CAF_CUDA_EXPORT request_token : public token {
+public:
+  request_token(int dependency = INDEPENDENT)
+    : dependency_(dependency) {}
+
+  // Required for CAF message passing
+  request_token() = default;
+
+  int getDependency() const { return dependency_; }
+  bool isIndependent() const { return dependency_ == INDEPENDENT; }
+
+private:
+  int dependency_ = INDEPENDENT;
+};
+
+using request_token_ptr = caf::intrusive_ptr<request_token>;
+
+} // namespace caf::cuda
+

From 3231d131a315d8991913368cdafbab063302f127 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 27 Jan 2026 14:46:47 -0600
Subject: [PATCH 0221/1000] Added comments.

---
 libcaf_cuda/caf/cuda/control-layer/request_token.hpp | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/request_token.hpp b/libcaf_cuda/caf/cuda/control-layer/request_token.hpp
index 6a71c03127..86e4433bbf 100644
--- a/libcaf_cuda/caf/cuda/control-layer/request_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/request_token.hpp
@@ -1,9 +1,17 @@
 #pragma once
+/*
+ * Meant to be superclass of all tokens send to the scheduler actor
+ * unfortunately got cut mostly due to the fact that 
+ * it requires too much refactoring then has time to do
+ * Will hopefully come back to this class
+ */
+
+
 
 #include "caf/cuda/control-layer/token.hpp"
 
 
-#define INDEPENDENT -1
+//#define INDEPENDENT -1
 
 
 namespace caf::cuda {

From dd352beee0ed30af94c3f24026da475ec66611ca Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 27 Jan 2026 14:50:21 -0600
Subject: [PATCH 0222/1000] Added optional parameter to tokens constructor This
 change is made so it is possible to declare depenedencies. This should be in
 request token, but unforuntanly it will take too much refactoring to fix for
 right now, so will acrue some tech debt here. FIX LATER PLEASE .

---
 libcaf_cuda/caf/cuda/control-layer/token.hpp | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index 681a2c3e40..465f3ebf52 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -12,6 +12,9 @@
 #define MEMORY 4
 #define MEMORY_RESPONSE 5
 
+//dependency tags
+#define INDEPENDENT -1
+
 namespace caf::cuda {
 
 // Base token interface
@@ -24,10 +27,18 @@ class CAF_CUDA_EXPORT token : public caf::ref_counted {
 }
 
     //should only be used by caf's type id system
-    token() = default;
+  //  token() = default;
+
+    explicit  token(int dependency = INDEPENDENT)
+    : dependency_(dependency) {}
 
     virtual int getType() const {return -1;}
 
+  // Dependency API
+  virtual int getDependency() const { return dependency_; }
+  bool isIndependent() const { return dependency_ == INDEPENDENT; }
+
+
 protected:
 
  friend void intrusive_ptr_add_ref(token* p) noexcept {
@@ -42,6 +53,7 @@ class CAF_CUDA_EXPORT token : public caf::ref_counted {
 
 private:
  mutable std::atomic<size_t> ref_count_{0};
+ int dependency_ = INDEPENDENT;
 
 
 };

From 4857f225792e780365ea4212d700b9373542cffc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 27 Jan 2026 14:58:56 -0600
Subject: [PATCH 0223/1000] Update launch_token and memory_token factory
 functions to use optional dependency number argument. Change was made to
 align with changes made to token constructor and trying to get dependency
 numbers to be integrated.

---
 .../caf/cuda/control-layer/token_factory.hpp    |  8 ++++++--
 libcaf_cuda/src/control-layer/token_factory.cpp | 17 +++++++++++------
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
index 2f5d37deb4..fff5508c25 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
@@ -18,7 +18,8 @@ CAF_CUDA_EXPORT token_ptr make_launch_token(program_ptr prog,
                             nd_range range,
                             int memory_usage,
                             std::string id,
-                            actor reply_to);
+                            actor reply_to,
+			    int dependency = INDEPENDENT);
 
 /// Creates a launch_response_token (created internally by the scheduler
 /// when it accepts a kernel launch request)
@@ -31,7 +32,10 @@ CAF_CUDA_EXPORT response_token_ptr make_launch_response_token(actor scheduler_or
 CAF_CUDA_EXPORT behavior_token_ptr make_behavior_token(std::string name);
 
 //creats a memory transfer token
-CAF_CUDA_EXPORT token_ptr make_memory_token(int size, int direction,caf::actor replyActor);
+CAF_CUDA_EXPORT token_ptr make_memory_token(int size,
+	       	int direction,
+		caf::actor replyActor,
+		int dependency = INDEPENDENT);
 
 CAF_CUDA_EXPORT response_token_ptr make_memory_response_token(actor receiver,
                                      const memory_transfer_token& orig,
diff --git a/libcaf_cuda/src/control-layer/token_factory.cpp b/libcaf_cuda/src/control-layer/token_factory.cpp
index 44a7a2112e..f1b2d4ce2c 100644
--- a/libcaf_cuda/src/control-layer/token_factory.cpp
+++ b/libcaf_cuda/src/control-layer/token_factory.cpp
@@ -11,14 +11,16 @@ token_ptr make_launch_token(program_ptr prog,
                             nd_range range,
                             int memory_usage,
                             std::string id,
-                            actor reply_to)
+                            actor reply_to,
+                            int dependency)
 {
     return token_ptr(
         new launch_token(std::move(prog),
                          std::move(range),
                          memory_usage,
                          std::move(id),
-                         reply_to));
+                         reply_to,
+                         dependency));
 }
 
 response_token_ptr make_launch_response_token(actor receiver,
@@ -35,10 +37,13 @@ behavior_token_ptr make_behavior_token(std::string name)
 }
 
 
-token_ptr make_memory_token(int size,int direction,caf::actor replyActor) {
-
-	return token_ptr(new memory_transfer_token(size,direction,replyActor));
-
+token_ptr make_memory_token(int size,
+                            int direction,
+                            caf::actor replyActor,
+                            int dependency)
+{
+    return token_ptr(
+        new memory_transfer_token(size, direction, replyActor, dependency));
 }
 
 response_token_ptr make_memory_response_token(actor receiver,

From 200325e76593e75b73a14fb3b8fb9ee786913d90 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 28 Jan 2026 17:06:39 -0600
Subject: [PATCH 0224/1000] Initial commit.

---
 .../src/control-layer/core_usage_behavior.cpp | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 libcaf_cuda/src/control-layer/core_usage_behavior.cpp

diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
new file mode 100644
index 0000000000..f51fa1cc9d
--- /dev/null
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -0,0 +1,30 @@
+#include "caf/cuda/control-layer/all-control-layer.hpp"
+#include "caf/cuda/control-layer/core_usage_behavior.hpp"
+
+namespace caf::cuda {
+
+core_usage_behavior::core_usage__behavior(scheduler_actor_state& state)
+    : scheduler_actor_behavior(state) {}
+
+
+
+void core_usage_behavior::schedule() {
+    //TODO IMPLEMENT
+}
+
+void core_usage_behavior::receive(const token_ptr& tok) {
+    if (tok->getType() == LAUNCH) {
+        //use 0 as stream id for now, eventually will have to figure out
+        //stream load balancing
+        process_launch_token(tok, 0);
+    }
+    else if (tok->getType() == MEMORY) {
+        //use 0 as stream id for now, eventually will have to figure out
+        //stream load balancing
+        process_memory_transfer_token(tok, 0);
+    }
+    
+}
+
+
+} // namespace caf::cuda

From 2a65586a0d76aac3e19902a8af30552918852fe9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 28 Jan 2026 17:14:59 -0600
Subject: [PATCH 0225/1000] Implemented init_state method.

---
 .../src/control-layer/core_usage_behavior.cpp        | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index f51fa1cc9d..25c4e4d5d8 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -1,5 +1,7 @@
 #include "caf/cuda/control-layer/all-control-layer.hpp"
 #include "caf/cuda/control-layer/core_usage_behavior.hpp"
+#include "caf/cuda/manager.hpp"
+#include "caf/cuda/device.hpp"
 
 namespace caf::cuda {
 
@@ -27,4 +29,14 @@ void core_usage_behavior::receive(const token_ptr& tok) {
 }
 
 
+void core_usage_behavior::init_state() {
+
+	dev = manager::get.find_device(state-> device_number);
+	heuristic = core_heuristic_function(dev);
+	total_SM =  dev -> num_sms();
+	available_SM = total_sm;
+	available_memory = static_cast<int>(dev -> total_memory_bytes());
+
+}
+
 } // namespace caf::cuda

From b38bd9939c49b29ddb7c9fa84c3c5789afe342e4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 28 Jan 2026 17:16:32 -0600
Subject: [PATCH 0226/1000] Updated include to include core_usage_behavior.hpp

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 5696e6c307..ec7e9aa9d9 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -17,7 +17,7 @@
 #include "caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp"
 #include "caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp"
 #include "caf/cuda/control-layer/kernel_graph.hpp"
-
+#include "caf/cuda/control-layer/core_usage_behavior.hpp"
 
 // -----------------------------------------------------------------------------
 // Type IDs (required for typed behaviors)

From 11c6e8ce9a25ed4a254e7f1570f2d2838e1b0497 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 28 Jan 2026 17:27:20 -0600
Subject: [PATCH 0227/1000] Fixed compiler errors.

---
 .../caf/cuda/control-layer/core_usage_behavior.hpp     | 10 ++++------
 libcaf_cuda/src/control-layer/core_usage_behavior.cpp  |  4 +++-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index ad679b5508..9dd07124cb 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -1,6 +1,6 @@
 #pragma once
 #include "caf/cuda/control-layer/behavior.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/core_usage_heuristic.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp"
 #include "caf/cuda/device.hpp"
 
 namespace caf::cuda {
@@ -12,16 +12,14 @@ class core_usage_behavior : public scheduler_actor_behavior {
     void schedule() override;
     void receive(const token_ptr& tok) override;
     void reclaim(int value /*blocks consumed*/,int memory_returned,int time) override; 
-    core_usage_behavior() {
-	    init_state();
-    }
 
 private:
-   dev_ptr device_;
+   device_ptr device_;
    core_heuristic_function heuristic;
    int total_SM;
    int available_SM; 
-    
+   int available_memory; //in bytes 
+
     void init_state();
 
 
diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 25c4e4d5d8..de316b5260 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -6,7 +6,9 @@
 namespace caf::cuda {
 
 core_usage_behavior::core_usage__behavior(scheduler_actor_state& state)
-    : scheduler_actor_behavior(state) {}
+    : scheduler_actor_behavior(state) {
+	    init_state();
+    }
 
 
 

From 31bab6fd9194dd5d7ecb4712410fe26efa68493d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 28 Jan 2026 17:30:59 -0600
Subject: [PATCH 0228/1000] Updated parameters of reclaim method.

---
 libcaf_cuda/caf/cuda/control-layer/behavior.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
index 1b983f317d..7662444e84 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
@@ -31,7 +31,7 @@ class scheduler_actor_behavior {
     virtual void schedule() = 0;
     virtual void receive(const token_ptr& tok) = 0;
 
-    virtual void reclaim(int value, int memory_returned,int runtime) {
+    virtual void reclaim(int value, int memory_returned,int runtime,int dependency) {
 	    //default implementation is to do nothing, this should be overidden
 	    //by children classes 
     }

From 6f558772a33b4861f90f90ca413d98e37f10a1db Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 28 Jan 2026 17:32:22 -0600
Subject: [PATCH 0229/1000] Updated reclaim to include a dependency number.

---
 libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index 9dd07124cb..3de135bff0 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -11,7 +11,7 @@ class core_usage_behavior : public scheduler_actor_behavior {
     void on_enter() override;
     void schedule() override;
     void receive(const token_ptr& tok) override;
-    void reclaim(int value /*blocks consumed*/,int memory_returned,int time) override; 
+    void reclaim(int value /*blocks consumed*/,int memory_returned,int time,int dependency) override; 
 
 private:
    device_ptr device_;

From 22a6d7b921c72f0e331a4f8544fbaaeeb998886b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 28 Jan 2026 18:43:55 -0600
Subject: [PATCH 0230/1000] Updated header file with some private attributes
 and helper functions along with notes and documentation.

---
 .../caf/cuda/control-layer/core_usage_behavior.hpp  | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index 3de135bff0..9bf8b7ad8f 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -1,7 +1,9 @@
 #pragma once
 #include "caf/cuda/control-layer/behavior.hpp"
+#include "caf/cuda/control-layer/kernel_graph.hpp"
 #include "caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp"
 #include "caf/cuda/device.hpp"
+#include <unordered_map>
 
 namespace caf::cuda {
 
@@ -19,8 +21,15 @@ class core_usage_behavior : public scheduler_actor_behavior {
    int total_SM;
    int available_SM; 
    int available_memory; //in bytes 
-
-    void init_state();
+   std::unordered_map<int,kernel_graph> graphs;
+   std::vector<kernel_graph> independent_graphs;
+   std::vector<kernel_graph> best_graphs; //should contain top 5-10 best selections ideally or something along the lines
+
+   void init_state();
+   void create_new_graph(); //this should either add to indepedent or graphs data structure
+			    //note to self use std::move for cheap copies
+   
+   void rank(); //this should rank the graphs (high to low) for best canidates
 
 
 };

From 5079f33fa5ed6ee1def26602f830603f86cc91ec Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 13:25:07 -0600
Subject: [PATCH 0231/1000] Added a num_streams attribute.

---
 libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
index 192d42c79a..09a5b87a8f 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
@@ -16,5 +16,6 @@ struct scheduler_actor_state {
     std::queue<token_ptr> queue; // here for legacy prototype schedulers
     std::vector<kernel_graph> operations; //more modern dependency based data structure 
     int device_number;
+    int num_streams; // number of streams that can be used by the scheduler
 };
 } // namespace caf::cuda

From 5c6fca9ba6171b676d72b6ca8e1a41f4fa07acee Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 13:27:36 -0600
Subject: [PATCH 0232/1000] Added a default value for number of streams.

---
 libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
index 09a5b87a8f..d9617d6dc8 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
@@ -16,6 +16,6 @@ struct scheduler_actor_state {
     std::queue<token_ptr> queue; // here for legacy prototype schedulers
     std::vector<kernel_graph> operations; //more modern dependency based data structure 
     int device_number;
-    int num_streams; // number of streams that can be used by the scheduler
+    int num_streams = 32; // number of streams that can be used by the scheduler
 };
 } // namespace caf::cuda

From 49607ad0aedc2d5a5c42484c1d4ecfabebd0095f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 13:44:36 -0600
Subject: [PATCH 0233/1000] Implemented create_new_graph method and fixed
 syntax errors.

---
 .../control-layer/core_usage_behavior.hpp     | 12 ++++-
 .../src/control-layer/core_usage_behavior.cpp | 54 ++++++++++++++++---
 2 files changed, 57 insertions(+), 9 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index 9bf8b7ad8f..dc1a2d099f 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -18,19 +18,29 @@ class core_usage_behavior : public scheduler_actor_behavior {
 private:
    device_ptr device_;
    core_heuristic_function heuristic;
+  
+   //tracking the resources of the device 
    int total_SM;
    int available_SM; 
    int available_memory; //in bytes 
+   int num_tokens = 0;
+   
+   int num_streams = state -> num_streams;
+   int current_stream = 0;
+
+
+   //data structures to manage dependencies
    std::unordered_map<int,kernel_graph> graphs;
    std::vector<kernel_graph> independent_graphs;
    std::vector<kernel_graph> best_graphs; //should contain top 5-10 best selections ideally or something along the lines
 
    void init_state();
-   void create_new_graph(); //this should either add to indepedent or graphs data structure
+   void create_new_graph(token_ptr& token); //this should either add to indepedent or graphs data structure
 			    //note to self use std::move for cheap copies
    
    void rank(); //this should rank the graphs (high to low) for best canidates
 
+   int get_next_stream(); // this should return the next stream based on some decisions
 
 };
 
diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index de316b5260..78ff814e94 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -5,11 +5,25 @@
 
 namespace caf::cuda {
 
-core_usage_behavior::core_usage__behavior(scheduler_actor_state& state)
+core_usage_behavior::core_usage_behavior(scheduler_actor_state& state)
     : scheduler_actor_behavior(state) {
 	    init_state();
     }
 
+void core_usage_behavior::init_state() {
+
+	dev = manager::get().find_device(state-> device_number);
+	heuristic = core_heuristic_function(device_);
+	total_SM =  device_ -> num_sms();
+	available_SM = total_SM;
+	available_memory = static_cast<int>(device_ -> total_memory_bytes());
+	num_streams = state -> num_streams;
+
+}
+
+void core_usage_behavior::on_enter() {
+	//TODO implement 
+}
 
 
 void core_usage_behavior::schedule() {
@@ -20,7 +34,8 @@ void core_usage_behavior::receive(const token_ptr& tok) {
     if (tok->getType() == LAUNCH) {
         //use 0 as stream id for now, eventually will have to figure out
         //stream load balancing
-        process_launch_token(tok, 0);
+    	create_new_graph(token);
+	schedule();	
     }
     else if (tok->getType() == MEMORY) {
         //use 0 as stream id for now, eventually will have to figure out
@@ -31,14 +46,37 @@ void core_usage_behavior::receive(const token_ptr& tok) {
 }
 
 
-void core_usage_behavior::init_state() {
 
-	dev = manager::get.find_device(state-> device_number);
-	heuristic = core_heuristic_function(dev);
-	total_SM =  dev -> num_sms();
-	available_SM = total_sm;
-	available_memory = static_cast<int>(dev -> total_memory_bytes());
+int core_usage_behavior::get_next_stream() { return current_stream++ % num_streams;}
+
+void core_usage_behavior::create_new_graph(token_ptr& token) {
+
+	//check for independence first 
+	if (token -> isIndependent()) {
+		
+        	kernel_graph new_graph(state -> device_number, get_next_stream());
+		
+		new_graph.add_operation(token);
+		independent_graphs.push_back(std::move(new_graph));
+		return;
+	}
+	//check if we have seen the operation already 
+	else if (graphs.contains(token -> getDependency())) {
+		graphs[token-> getDependency()].add_operation(token);
+	}
+
+	//at this point this is the first time we are seeing this add to hashmap
+	else {
+	
+        	kernel_graph new_graph(state -> device_number, get_next_stream());	
+		new_graph.add_operation(token);
+		graphs[token -> getDependency()] = std::move(new_graph);
+
+	}
+
 
 }
 
+
+
 } // namespace caf::cuda

From d025ab0d8b957794729ff3b6c763be7b36bbb70e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 13:45:12 -0600
Subject: [PATCH 0234/1000] Fixed initialization errors.

---
 libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index dc1a2d099f..30441ebf77 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -25,7 +25,7 @@ class core_usage_behavior : public scheduler_actor_behavior {
    int available_memory; //in bytes 
    int num_tokens = 0;
    
-   int num_streams = state -> num_streams;
+   int num_streams = 0;
    int current_stream = 0;
 
 

From 13cb622a8898b2d3596e10d8648adb63e6302878 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 13:51:40 -0600
Subject: [PATCH 0235/1000] Added core usage behavior to the scheduler actor.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 109aad3b59..ff49f3d6ca 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -2,6 +2,7 @@
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
 #include "caf/cuda/control-layer/green_light_behavior.hpp"
 #include "caf/cuda/control-layer/red_light_behavior.hpp"
+#include "caf/cuda/control-layer/core_usage_behavior.hpp"
 #include <string>
 #include <iostream>
 
@@ -22,11 +23,12 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
 
     static red_light_behavior red_behavior(state);
     static green_light_behavior green_behavior(state);
-
+    static core_usage_behavior core_behavior(state);
 
     // populate the behavior table
     state.table.add("red", &red_behavior);
     state.table.add("green",   &green_behavior);
+    state.table.add("core_usage",   &core_behavior);
 
     // default behavior
     state.current_behavior = state.table.get(behavior_token("green"));

From cc6d391e9e455b637bcdf1d88ba556ab90a6c816 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 13:54:59 -0600
Subject: [PATCH 0236/1000] Initial commit copied over from
 command_runner_tests.

---
 .../core_usage_behavior_tests/CMakeLists.txt  |  44 ++
 .../compile_kernels.sh                        |  18 +
 .../core_usage_behavior_tests/main.test.cpp   | 418 ++++++++++++++++++
 .../core_usage_behavior_tests/mmul.cu         |  16 +
 4 files changed, 496 insertions(+)
 create mode 100644 libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
 create mode 100644 libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/mmul.cu

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/CMakeLists.txt b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/compile_kernels.sh b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/compile_kernels.sh
new file mode 100755
index 0000000000..586196454e
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/compile_kernels.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+
+# Compile genMatrix.cu to fatbin in current directory
+#nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
+#echo "Generated generate_random_matrix.fatbin"
+
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
new file mode 100644
index 0000000000..f5de194eb7
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -0,0 +1,418 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include <unistd.h>
+#include "caf/actor_registry.hpp"
+#include <chrono>
+#include <iostream>
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+struct exit_actor_state {
+	int completed = 0;
+};
+
+
+caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
+
+
+	return {
+		[=](int num_completed) {
+			self->state().completed += num_completed;
+			
+			std::cout << "Actors finished is " << self->state().completed << "\n";
+			if (self->state().completed >= limit) {
+			
+				caf::cuda::manager::shutdown();
+				self->quit();
+			}
+		}
+	};
+
+
+}
+
+
+
+
+
+
+// Define a custom type ID block for custom actors
+CAF_ADD_ATOM(cuda,shared_mem)
+
+
+
+
+
+// Extend your actor state to keep the start time
+struct mmul_actor_state {
+  static inline const char* name = "my_actor";
+  int N = 1024; // example state variable
+  int id = rand(); // an actor id
+  // per-actor timing start
+  std::chrono::high_resolution_clock::time_point start_time;
+  int times = 0;
+  caf::cuda::program_ptr program = caf::cuda::manager::get().create_program_from_cubin("../mmul.cubin","matrixMul");
+  int THREADS = 32;
+  int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS,BLOCKS,1,THREADS,THREADS,THREADS);
+};
+
+
+
+
+//commands classes used to launch kernels 
+using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
+using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
+
+using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
+
+mmulCommand mmul;
+matrixGenCommand randomMatrix;
+mmulAsyncCommand mmulAsync;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
+ 
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+	
+  	caf::actor scheduler = mgr.get_scheduler_actor();
+	
+	//send a launch token
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
+			self -> state().dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+	  [=] (caf::cuda::response_token_ptr res_token) {
+	
+		 if (res_token -> getType() == LAUNCH_RESPONSE) {
+		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+		  //assume N = 1024
+		  int N = self -> state().N;
+		  std::vector<int> matrix1(N*N);
+		  matrix1.reserve(N);
+		  std::vector<int> matrix2(N*N);
+		  matrix2.reserve(N);
+
+		  //std::cout << "GPU ACTOR sending data to compute\n";
+		  self -> mail(matrix1,matrix2,res_token,N).send(self);
+	 
+		 }
+		 else {
+			 std::cout << "Got a memory response token\n";
+		 }
+		 //token should drop out of scope now, triggering a response 
+	  },
+
+    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+	const caf::cuda::response_token_ptr& res_token, int N) {
+ 
+
+	    //caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
+
+	    //caf::cuda::launch_response_token& kt =
+	//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
+
+		  //std::cout << "GPU ACTOR  computing\n";
+	    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //create program and dims   
+  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+  const int THREADS = 32;
+  const int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    //create args
+    auto arg1 = caf::cuda::create_in_arg(matrixA);
+    auto arg2 = caf::cuda::create_in_arg(matrixB);
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+		  //std::cout << "GPU ACTOR done  computing\n";
+    //verify its own result 
+    self -> mail(matrixA,matrixB,matrixC,N).send(self);
+
+    },
+
+    // 3rd handler: CPU atom + matrices + N
+    [=](const std::vector<int>& matrixA,
+    const std::vector<int>& matrixB,
+    const std::vector<int>& matrixC,
+    int N) {
+
+  using clock = std::chrono::high_resolution_clock;
+
+  auto start = clock::now();
+
+  //std::cout << "GPU ACTOR verifying\n";
+
+  std::vector<int> result(N * N);
+
+  serial_matrix_multiply(matrixA, matrixB, result, N);
+
+  if (result == matrixC) {
+    std::cout << "actor with id " << self->state().id
+              << " references match\n";
+  } else {
+    std::cout << "actor with id " << self->state().id
+              << " references did not match\n";
+  }
+
+  auto end = clock::now();
+
+  auto ms =
+    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[TIMING] verification took "
+            << ms << " ms (actor id "
+            << self->state().id << ")\n";
+
+  // signal exit actor and quit
+  self->mail(1).send(exit_actor);
+  self->quit();
+
+    }
+  };
+}
+
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  int limit = 1;
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+
+  for (int i = 0; i < limit; i++) {
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
+  }
+
+
+ // std::cout << actors.size() << "\n";
+
+  }
+
+   sys.await_all_actors_done();
+}
+
+// Stateful actor behavior
+caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
+ 
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+	
+  	caf::actor scheduler = mgr.get_scheduler_actor();
+	
+	//send a launch token
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
+			self -> state().dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+	  [=] (caf::cuda::response_token_ptr res_token) {
+	
+		 if (res_token -> getType() == LAUNCH_RESPONSE) {
+		  int N = self -> state().N;
+
+		  caf::cuda::command_runner<in_out<int>> mem_transfer_command;
+
+
+		  std::vector<int> matrix1(N*N);
+		  std::vector<int> matrix2(N*N);
+
+		  caf::cuda::mem_ptr<int> matrixA = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix1});
+		  caf::cuda::mem_ptr<int> matrixB = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix2});
+
+		  //std::cout << "GPU ACTOR sending data to compute\n";
+		  self -> mail(matrixA,matrixB,res_token,N).send(self);
+	 
+		 }
+		 else {
+			 //std::cout << "Got a memory response token\n";
+		 }
+		 //token should drop out of scope now, triggering a response 
+	  },
+
+    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+    [=](const caf::cuda::mem_ptr<int>& matrixA,
+        const caf::cuda::mem_ptr<int>& matrixB,
+	const caf::cuda::response_token_ptr& res_token, int N) {
+ 
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+	//create program and dims   
+ 	auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+  	const int THREADS = 32;
+  	const int BLOCKS = (N + THREADS - 1) / THREADS;
+  	caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    //create args
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    auto temp = mmulAsync.run_async(program,dims,res_token,matrixA,matrixB,arg3,arg4);
+    caf::cuda::mem_ptr<int> matrixC = std::get<2>(temp);
+
+    std::vector<int> matrix1 = matrixA -> copy_to_host();
+    std::vector<int> matrix2 = matrixB -> copy_to_host();
+    std::vector<int> matrix3 = matrixC -> copy_to_host();
+
+		  //std::cout << "GPU ACTOR done  computing\n";
+    //verify its own result 
+    self -> mail(matrix1,matrix2,matrix3,N).send(self);
+
+    },
+
+    // 3rd handler: CPU atom + matrices + N
+    [=](const std::vector<int>& matrixA,
+    const std::vector<int>& matrixB,
+    const std::vector<int>& matrixC,
+    int N) {
+
+  using clock = std::chrono::high_resolution_clock;
+
+  auto start = clock::now();
+
+  //std::cout << "GPU ACTOR verifying\n";
+
+  std::vector<int> result(N * N);
+
+  serial_matrix_multiply(matrixA, matrixB, result, N);
+
+  if (result == matrixC) {
+    std::cout << "actor with id " << self->state().id
+              << " references match\n";
+  } else {
+    std::cout << "actor with id " << self->state().id
+              << " references did not match\n";
+  }
+
+  auto end = clock::now();
+
+  auto ms =
+    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[TIMING] verification took "
+            << ms << " ms (actor id "
+            << self->state().id << ")\n";
+
+  // signal exit actor and quit
+  self->mail(1).send(exit_actor);
+  self->quit();
+
+    }
+  };
+}
+
+
+
+void run_async_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  int limit = 1;
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+
+  for (int i = 0; i < limit; i++) {
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_async_actor_fun,exit_actor,matrix_size));
+  }
+
+
+ // std::cout << actors.size() << "\n";
+
+  }
+
+   sys.await_all_actors_done();
+}
+
+
+
+void caf_main(caf::actor_system& sys) {
+  
+	
+
+	caf::cuda::manager_config man_config(true); //turns the scheduler on
+	caf::cuda::manager::init(sys,man_config);
+       	run_async_mmul_test(sys,10,500);
+	
+	//tests will delete the old manager so will have to reinit if you do this 
+	//in conjunction with each other	
+	//caf::cuda::manager::init(sys,man_config);
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/mmul.cu b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From 0a853ae85181eca49ef0bccf17aecb27a4688013 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 14:01:16 -0600
Subject: [PATCH 0237/1000] Implemented dummy_schedule.

---
 .../control-layer/core_usage_behavior.hpp     |  1 +
 .../src/control-layer/core_usage_behavior.cpp | 48 +++++++++++++++++++
 2 files changed, 49 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index 30441ebf77..12f3f919eb 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -41,6 +41,7 @@ class core_usage_behavior : public scheduler_actor_behavior {
    void rank(); //this should rank the graphs (high to low) for best canidates
 
    int get_next_stream(); // this should return the next stream based on some decisions
+   void dummy_schedule(); //just a test scheduler algorthim do not use seriously
 
 };
 
diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 78ff814e94..88d3594dd7 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -23,11 +23,15 @@ void core_usage_behavior::init_state() {
 
 void core_usage_behavior::on_enter() {
 	//TODO implement 
+	//
+
 }
 
 
 void core_usage_behavior::schedule() {
     //TODO IMPLEMENT
+
+	dummy_schedule();
 }
 
 void core_usage_behavior::receive(const token_ptr& tok) {
@@ -79,4 +83,48 @@ void core_usage_behavior::create_new_graph(token_ptr& token) {
 
 
 
+void core_usage_behavior::dummy_schedule() {
+    /*
+     * 1. Drain independent graphs fully.
+     *    These can be erased once empty.
+     */
+    for (auto it = independent_graphs.begin();
+         it != independent_graphs.end(); ) {
+
+        kernel_graph& graph = *it;
+
+        while (!graph.empty()) {
+            token_ptr tok = graph.getOperation();
+            if (!tok)
+                break;
+
+            if (tok->getType() == LAUNCH) {
+                process_launch_token(tok, graph.get_stream_id());
+            }
+            // ignore other token types for now
+        }
+
+        // independent graphs can be deleted once drained
+        it = independent_graphs.erase(it);
+    }
+
+    /*
+     * 2. Drain dependency graphs, but DO NOT delete them.
+     */
+    for (auto& [dep, graph] : graphs) {
+        while (!graph.empty()) {
+            token_ptr tok = graph.getOperation();
+            if (!tok)
+                break;
+
+            if (tok->getType() == LAUNCH) {
+                process_launch_token(tok, graph.get_stream_id());
+            }
+        }
+    }
+}
+
+
+
+
 } // namespace caf::cuda

From df8cf9b37fe5f5e8ebc88920aae8960c2ccceda3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 14:15:11 -0600
Subject: [PATCH 0238/1000] Updated tests by removing out problematic variables
 of mmul_actor_state including program_ptr and ndrange these would cause
 silient bugs that would drastically reduce performance.

---
 .../core_usage_behavior_tests/main.test.cpp   | 355 ++++++------------
 1 file changed, 110 insertions(+), 245 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index f5de194eb7..289696edf2 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -59,21 +59,20 @@ CAF_ADD_ATOM(cuda,shared_mem)
 
 // Extend your actor state to keep the start time
 struct mmul_actor_state {
-  static inline const char* name = "my_actor";
-  int N = 1024; // example state variable
-  int id = rand(); // an actor id
-  // per-actor timing start
+  static inline const char* name = "mmul_actor";
+
+  int N = 0;
+  int id = rand();
+
+  // timing / bookkeeping only
   std::chrono::high_resolution_clock::time_point start_time;
   int times = 0;
-  caf::cuda::program_ptr program = caf::cuda::manager::get().create_program_from_cubin("../mmul.cubin","matrixMul");
-  int THREADS = 32;
-  int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS,BLOCKS,1,THREADS,THREADS,THREADS);
 };
 
 
 
 
+
 //commands classes used to launch kernels 
 using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
 using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
@@ -106,166 +105,25 @@ void serial_matrix_multiply(const std::vector<int>& a,
 
 
 // Stateful actor behavior
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
- 
+caf::behavior mmul_actor_fun(
+    caf::stateful_actor<mmul_actor_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims)
+{
 
 	//set the value of N correctly to overide the base option.	
 	self->state().N = N;
 
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-	
-  	caf::actor scheduler = mgr.get_scheduler_actor();
-	
-	//send a launch token
-	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
-			self -> state().dims,
-			0,
-			"hello",
-			self
-			);
-	self -> mail(launch_token).send(scheduler);
-
-	return {
-
-	  [=] (caf::cuda::response_token_ptr res_token) {
-	
-		 if (res_token -> getType() == LAUNCH_RESPONSE) {
-		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-		  //assume N = 1024
-		  int N = self -> state().N;
-		  std::vector<int> matrix1(N*N);
-		  matrix1.reserve(N);
-		  std::vector<int> matrix2(N*N);
-		  matrix2.reserve(N);
-
-		  //std::cout << "GPU ACTOR sending data to compute\n";
-		  self -> mail(matrix1,matrix2,res_token,N).send(self);
-	 
-		 }
-		 else {
-			 std::cout << "Got a memory response token\n";
-		 }
-		 //token should drop out of scope now, triggering a response 
-	  },
-
-    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB,
-	const caf::cuda::response_token_ptr& res_token, int N) {
- 
-
-	    //caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
-
-	    //caf::cuda::launch_response_token& kt =
-	//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
-
-		  //std::cout << "GPU ACTOR  computing\n";
-	    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //create program and dims   
-  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    //create args
-    auto arg1 = caf::cuda::create_in_arg(matrixA);
-    auto arg2 = caf::cuda::create_in_arg(matrixB);
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
-
-    auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-		  //std::cout << "GPU ACTOR done  computing\n";
-    //verify its own result 
-    self -> mail(matrixA,matrixB,matrixC,N).send(self);
-
-    },
-
-    // 3rd handler: CPU atom + matrices + N
-    [=](const std::vector<int>& matrixA,
-    const std::vector<int>& matrixB,
-    const std::vector<int>& matrixC,
-    int N) {
-
-  using clock = std::chrono::high_resolution_clock;
-
-  auto start = clock::now();
-
-  //std::cout << "GPU ACTOR verifying\n";
-
-  std::vector<int> result(N * N);
-
-  serial_matrix_multiply(matrixA, matrixB, result, N);
-
-  if (result == matrixC) {
-    std::cout << "actor with id " << self->state().id
-              << " references match\n";
-  } else {
-    std::cout << "actor with id " << self->state().id
-              << " references did not match\n";
-  }
-
-  auto end = clock::now();
-
-  auto ms =
-    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-  std::cout << "[TIMING] verification took "
-            << ms << " ms (actor id "
-            << self->state().id << ")\n";
-
-  // signal exit actor and quit
-  self->mail(1).send(exit_actor);
-  self->quit();
-
-    }
-  };
-}
-
-
-
-void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  int limit = 1;
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
-
-  for (int i = 0; i < limit; i++) {
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
-  }
-
-
- // std::cout << actors.size() << "\n";
-
-  }
-
-   sys.await_all_actors_done();
-}
-
-// Stateful actor behavior
-caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
- 
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
+	caf::actor scheduler = mgr.get_scheduler_actor();
 
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-	
-  	caf::actor scheduler = mgr.get_scheduler_actor();
-	
 	//send a launch token
-	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
-			self -> state().dims,
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+			program,
+			dims,
 			0,
 			"hello",
 			self
@@ -274,138 +132,145 @@ caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self,c
 
 	return {
 
-	  [=] (caf::cuda::response_token_ptr res_token) {
-	
-		 if (res_token -> getType() == LAUNCH_RESPONSE) {
-		  int N = self -> state().N;
+		[=] (caf::cuda::response_token_ptr res_token) {
+
+			if (res_token -> getType() == LAUNCH_RESPONSE) {
+				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+				//assume N = 1024
+				int N = self -> state().N;
+				std::vector<int> matrix1(N*N);
+				matrix1.reserve(N);
+				std::vector<int> matrix2(N*N);
+				matrix2.reserve(N);
 
-		  caf::cuda::command_runner<in_out<int>> mem_transfer_command;
+				//std::cout << "GPU ACTOR sending data to compute\n";
+				self -> mail(matrix1,matrix2,res_token,N).send(self);
 
+			}
+			else {
+				std::cout << "Got a memory response token\n";
+			}
+			//token should drop out of scope now, triggering a response 
+		},
 
-		  std::vector<int> matrix1(N*N);
-		  std::vector<int> matrix2(N*N);
+			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+			[=](const std::vector<int>& matrixA,
+					const std::vector<int>& matrixB,
+					const caf::cuda::response_token_ptr& res_token, int N) {
 
-		  caf::cuda::mem_ptr<int> matrixA = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix1});
-		  caf::cuda::mem_ptr<int> matrixB = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix2});
 
-		  //std::cout << "GPU ACTOR sending data to compute\n";
-		  self -> mail(matrixA,matrixB,res_token,N).send(self);
-	 
-		 }
-		 else {
-			 //std::cout << "Got a memory response token\n";
-		 }
-		 //token should drop out of scope now, triggering a response 
-	  },
+				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
 
-    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const caf::cuda::mem_ptr<int>& matrixA,
-        const caf::cuda::mem_ptr<int>& matrixB,
-	const caf::cuda::response_token_ptr& res_token, int N) {
- 
-	caf::cuda::manager& mgr = caf::cuda::manager::get();
+				//caf::cuda::launch_response_token& kt =
+				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
 
-	//create program and dims   
- 	auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-  	const int THREADS = 32;
-  	const int BLOCKS = (N + THREADS - 1) / THREADS;
-  	caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+				//std::cout << "GPU ACTOR  computing\n";
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-    //create args
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
+				//create program and dims   
+				auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+				const int THREADS = 32;
+				const int BLOCKS = (N + THREADS - 1) / THREADS;
+				caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-    auto temp = mmulAsync.run_async(program,dims,res_token,matrixA,matrixB,arg3,arg4);
-    caf::cuda::mem_ptr<int> matrixC = std::get<2>(temp);
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrixA);
+				auto arg2 = caf::cuda::create_in_arg(matrixB);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
 
-    std::vector<int> matrix1 = matrixA -> copy_to_host();
-    std::vector<int> matrix2 = matrixB -> copy_to_host();
-    std::vector<int> matrix3 = matrixC -> copy_to_host();
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
 
-		  //std::cout << "GPU ACTOR done  computing\n";
-    //verify its own result 
-    self -> mail(matrix1,matrix2,matrix3,N).send(self);
+				//std::cout << "GPU ACTOR done  computing\n";
+				//verify its own result 
+				self -> mail(matrixA,matrixB,matrixC,N).send(self);
 
-    },
+			},
 
-    // 3rd handler: CPU atom + matrices + N
-    [=](const std::vector<int>& matrixA,
-    const std::vector<int>& matrixB,
-    const std::vector<int>& matrixC,
-    int N) {
+					// 3rd handler: CPU atom + matrices + N
+					[=](const std::vector<int>& matrixA,
+							const std::vector<int>& matrixB,
+							const std::vector<int>& matrixC,
+							int N) {
 
-  using clock = std::chrono::high_resolution_clock;
+						using clock = std::chrono::high_resolution_clock;
 
-  auto start = clock::now();
+						auto start = clock::now();
 
-  //std::cout << "GPU ACTOR verifying\n";
+						//std::cout << "GPU ACTOR verifying\n";
 
-  std::vector<int> result(N * N);
+						std::vector<int> result(N * N);
 
-  serial_matrix_multiply(matrixA, matrixB, result, N);
+						serial_matrix_multiply(matrixA, matrixB, result, N);
 
-  if (result == matrixC) {
-    std::cout << "actor with id " << self->state().id
-              << " references match\n";
-  } else {
-    std::cout << "actor with id " << self->state().id
-              << " references did not match\n";
-  }
+						if (result == matrixC) {
+							std::cout << "actor with id " << self->state().id
+								<< " references match\n";
+						} else {
+							std::cout << "actor with id " << self->state().id
+								<< " references did not match\n";
+						}
 
-  auto end = clock::now();
+						auto end = clock::now();
 
-  auto ms =
-    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+						auto ms =
+							std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
 
-  std::cout << "[TIMING] verification took "
-            << ms << " ms (actor id "
-            << self->state().id << ")\n";
+						std::cout << "[TIMING] verification took "
+							<< ms << " ms (actor id "
+							<< self->state().id << ")\n";
 
-  // signal exit actor and quit
-  self->mail(1).send(exit_actor);
-  self->quit();
+						// signal exit actor and quit
+						self->mail(1).send(exit_actor);
+						self->quit();
 
-    }
-  };
+					}
+	};
 }
 
 
 
-void run_async_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
   if (num_actors < 1) {
     std::cerr << "[ERROR] Number of actors must be >= 1\n";
     return;
   }
 
-  int limit = 1;
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
-
-  for (int i = 0; i < limit; i++) {
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_async_actor_fun,exit_actor,matrix_size));
-  }
+  // CREATE ONCE
+  auto program =
+      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
+  const int THREADS = 32;
+  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1);
 
- // std::cout << actors.size() << "\n";
+  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
 
+  for (int i = 0; i < num_actors; ++i) {
+    sys.spawn(
+        mmul_actor_fun,
+        exit_actor,
+        matrix_size,
+        program,
+        dims);
   }
 
-   sys.await_all_actors_done();
+  sys.await_all_actors_done();
 }
 
 
-
 void caf_main(caf::actor_system& sys) {
   
 	
 
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	caf::cuda::manager::init(sys,man_config);
-       	run_async_mmul_test(sys,10,500);
+       	run_mmul_test(sys,10,500);
 	
 	//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	

From 6362edc0307873466e5f24662dc970cc290185f0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 14:18:15 -0600
Subject: [PATCH 0239/1000] Implemented reclaim stub for now to fix linker
 error.

---
 libcaf_cuda/src/control-layer/core_usage_behavior.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 88d3594dd7..54adc83420 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -27,6 +27,13 @@ void core_usage_behavior::on_enter() {
 
 }
 
+void core_usage_behavior::reclaim(
+    int /*value*/,
+    int /*memory_returned*/,
+    int /*time*/,
+    int /*dependency*/) {
+    // intentionally empty for now
+}
 
 void core_usage_behavior::schedule() {
     //TODO IMPLEMENT

From b9312bc018989aedc88b3a0b3b072237895a1348 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 14:24:13 -0600
Subject: [PATCH 0240/1000] Implemented a default constructor.

---
 libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp | 1 +
 libcaf_cuda/src/control-layer/core_usage_behavior.cpp      | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index 12f3f919eb..8794faff98 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -14,6 +14,7 @@ class core_usage_behavior : public scheduler_actor_behavior {
     void schedule() override;
     void receive(const token_ptr& tok) override;
     void reclaim(int value /*blocks consumed*/,int memory_returned,int time,int dependency) override; 
+    ~core_usage_behavior() override;   // 👈 ADD THIS
 
 private:
    device_ptr device_;
diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 54adc83420..6c18fade69 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -10,9 +10,11 @@ core_usage_behavior::core_usage_behavior(scheduler_actor_state& state)
 	    init_state();
     }
 
+core_usage_behavior::~core_usage_behavior() = default;
+
 void core_usage_behavior::init_state() {
 
-	dev = manager::get().find_device(state-> device_number);
+	device_ = manager::get().find_device(state-> device_number);
 	heuristic = core_heuristic_function(device_);
 	total_SM =  device_ -> num_sms();
 	available_SM = total_SM;

From c002544cf4a2ec0ba43fd33ed6f6ac1d10268374 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 14:37:00 -0600
Subject: [PATCH 0241/1000] Implemented stream_id() method.

---
 libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
index ce20ea412f..76a7d96908 100644
--- a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
@@ -50,6 +50,10 @@ class kernel_graph {
         return operations.empty();
     }
 
+    int stream_id() const noexcept { return stream_id_; }
+
+
+
 private:
     int device_number_;
     int stream_id_;

From aa99cb822eb6791c2fa6edab0e24a25ae5ca6e7d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 14:41:15 -0600
Subject: [PATCH 0242/1000] Fixed compiler errors.

---
 .../control-layer/core_usage_behavior.hpp     |  11 +-
 .../src/control-layer/core_usage_behavior.cpp | 133 ++++++------------
 2 files changed, 49 insertions(+), 95 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index 8794faff98..18c2791e52 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -4,6 +4,7 @@
 #include "caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp"
 #include "caf/cuda/device.hpp"
 #include <unordered_map>
+#include <optional>
 
 namespace caf::cuda {
 
@@ -14,12 +15,16 @@ class core_usage_behavior : public scheduler_actor_behavior {
     void schedule() override;
     void receive(const token_ptr& tok) override;
     void reclaim(int value /*blocks consumed*/,int memory_returned,int time,int dependency) override; 
-    ~core_usage_behavior() override;   // 👈 ADD THIS
+    ~core_usage_behavior() override;  
 
 private:
    device_ptr device_;
-   core_heuristic_function heuristic;
+   //core_heuristic_function heuristic;
   
+   std::optional<core_heuristic_function> heuristic;
+
+
+
    //tracking the resources of the device 
    int total_SM;
    int available_SM; 
@@ -36,7 +41,7 @@ class core_usage_behavior : public scheduler_actor_behavior {
    std::vector<kernel_graph> best_graphs; //should contain top 5-10 best selections ideally or something along the lines
 
    void init_state();
-   void create_new_graph(token_ptr& token); //this should either add to indepedent or graphs data structure
+   void create_new_graph(const token_ptr& token); //this should either add to indepedent or graphs data structure
 			    //note to self use std::move for cheap copies
    
    void rank(); //this should rank the graphs (high to low) for best canidates
diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 6c18fade69..2232831bc0 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -7,133 +7,82 @@ namespace caf::cuda {
 
 core_usage_behavior::core_usage_behavior(scheduler_actor_state& state)
     : scheduler_actor_behavior(state) {
-	    init_state();
-    }
-
-core_usage_behavior::~core_usage_behavior() = default;
+    init_state();
+}
 
 void core_usage_behavior::init_state() {
-
-	device_ = manager::get().find_device(state-> device_number);
-	heuristic = core_heuristic_function(device_);
-	total_SM =  device_ -> num_sms();
-	available_SM = total_SM;
-	available_memory = static_cast<int>(device_ -> total_memory_bytes());
-	num_streams = state -> num_streams;
-
+    device_ = manager::get().find_device(state_.device_number);
+    heuristic.emplace(device_);
+    total_SM = device_->num_sms();
+    available_SM = total_SM;
+    available_memory = static_cast<int>(device_->total_memory_bytes());
+    num_streams = state_.num_streams;
 }
 
 void core_usage_behavior::on_enter() {
-	//TODO implement 
-	//
-
+    // TODO implement
 }
 
-void core_usage_behavior::reclaim(
-    int /*value*/,
-    int /*memory_returned*/,
-    int /*time*/,
-    int /*dependency*/) {
+void core_usage_behavior::reclaim(int, int, int, int) {
     // intentionally empty for now
 }
 
 void core_usage_behavior::schedule() {
-    //TODO IMPLEMENT
-
-	dummy_schedule();
+    dummy_schedule();
 }
 
 void core_usage_behavior::receive(const token_ptr& tok) {
     if (tok->getType() == LAUNCH) {
-        //use 0 as stream id for now, eventually will have to figure out
-        //stream load balancing
-    	create_new_graph(token);
-	schedule();	
-    }
-    else if (tok->getType() == MEMORY) {
-        //use 0 as stream id for now, eventually will have to figure out
-        //stream load balancing
+        create_new_graph(tok);
+        schedule();
+    } else if (tok->getType() == MEMORY) {
         process_memory_transfer_token(tok, 0);
     }
-    
 }
 
-
-
-int core_usage_behavior::get_next_stream() { return current_stream++ % num_streams;}
-
-void core_usage_behavior::create_new_graph(token_ptr& token) {
-
-	//check for independence first 
-	if (token -> isIndependent()) {
-		
-        	kernel_graph new_graph(state -> device_number, get_next_stream());
-		
-		new_graph.add_operation(token);
-		independent_graphs.push_back(std::move(new_graph));
-		return;
-	}
-	//check if we have seen the operation already 
-	else if (graphs.contains(token -> getDependency())) {
-		graphs[token-> getDependency()].add_operation(token);
-	}
-
-	//at this point this is the first time we are seeing this add to hashmap
-	else {
-	
-        	kernel_graph new_graph(state -> device_number, get_next_stream());	
-		new_graph.add_operation(token);
-		graphs[token -> getDependency()] = std::move(new_graph);
-
-	}
-
-
+int core_usage_behavior::get_next_stream() {
+    return current_stream++ % num_streams;
 }
 
-
+void core_usage_behavior::create_new_graph(const token_ptr& tok) {
+    if (tok->isIndependent()) {
+        kernel_graph new_graph(state_.device_number, get_next_stream());
+        new_graph.add_operation(tok);
+        independent_graphs.push_back(std::move(new_graph));
+        return;
+    }
+    else if (graphs.contains(tok->getDependency())) {
+        graphs[tok->getDependency()].add_operation(tok);
+    }
+    else {
+        kernel_graph new_graph(state_.device_number, get_next_stream());
+        new_graph.add_operation(tok);
+        graphs[tok->getDependency()] = std::move(new_graph);
+    }
+}
 
 void core_usage_behavior::dummy_schedule() {
-    /*
-     * 1. Drain independent graphs fully.
-     *    These can be erased once empty.
-     */
-    for (auto it = independent_graphs.begin();
-         it != independent_graphs.end(); ) {
-
+    // Drain independent graphs fully
+    for (auto it = independent_graphs.begin(); it != independent_graphs.end(); ) {
         kernel_graph& graph = *it;
-
         while (!graph.empty()) {
             token_ptr tok = graph.getOperation();
-            if (!tok)
-                break;
-
-            if (tok->getType() == LAUNCH) {
-                process_launch_token(tok, graph.get_stream_id());
-            }
-            // ignore other token types for now
+            if (!tok) break;
+            if (tok->getType() == LAUNCH)
+                process_launch_token(tok, graph.stream_id());
         }
-
-        // independent graphs can be deleted once drained
         it = independent_graphs.erase(it);
     }
 
-    /*
-     * 2. Drain dependency graphs, but DO NOT delete them.
-     */
+    // Drain dependency graphs, do not delete
     for (auto& [dep, graph] : graphs) {
         while (!graph.empty()) {
             token_ptr tok = graph.getOperation();
-            if (!tok)
-                break;
-
-            if (tok->getType() == LAUNCH) {
-                process_launch_token(tok, graph.get_stream_id());
-            }
+            if (!tok) break;
+            if (tok->getType() == LAUNCH)
+                process_launch_token(tok, graph.stream_id());
         }
     }
 }
 
-
-
-
 } // namespace caf::cuda

From 94578cab1ab548d60d76e65a87217441ab3dba20 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 14:42:41 -0600
Subject: [PATCH 0243/1000] Added deconstructor to fix linker error.

---
 libcaf_cuda/src/control-layer/core_usage_behavior.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 2232831bc0..e3dddd32bf 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -10,6 +10,10 @@ core_usage_behavior::core_usage_behavior(scheduler_actor_state& state)
     init_state();
 }
 
+core_usage_behavior::~core_usage_behavior(){
+
+}
+
 void core_usage_behavior::init_state() {
     device_ = manager::get().find_device(state_.device_number);
     heuristic.emplace(device_);

From b744c2af095ac62c2bf4b06207bfce0dff5a441f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 14:45:58 -0600
Subject: [PATCH 0244/1000] Added message handler to the test driver to change
 to neccesary scheduler.

---
 .../core_usage_behavior_tests/main.test.cpp                  | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index 289696edf2..fc332d12c8 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -239,6 +239,11 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
 
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 
+  //change the scheduler to core_usage
+  anon_mail(
+	caf::cuda::behavior_token("core_usage")
+	).send(mgr.get_scheduler_actor());
+
   // CREATE ONCE
   auto program =
       mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");

From 30e06fcdb5d8929b424dbba035df99ccd97c2daf Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 14:49:58 -0600
Subject: [PATCH 0245/1000] Added logs to scheduler actor behavior change to
 give feedback on changing behavior for now.

---
 .../src/control-layer/scheduler_actor.cpp     | 27 +++++++++++++------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index ff49f3d6ca..9c52a2830e 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -39,14 +39,25 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
             // std::cout << "Received token\n";
             state.current_behavior->receive(tok);
         },
-        [&state](const caf::cuda::behavior_token_ptr& tok) {
-            auto* next = state.table.get(*tok);
-            if (next) {
-                state.current_behavior->on_exit();   // cleanup current behavior
-                state.current_behavior = next;       // swap behavior
-                state.current_behavior->on_enter();  // init new current behavior
-            }
-        },
+        [&state](const caf::cuda::behavior_token_ptr& tok) -> bool {
+    auto* next = state.table.get(*tok);
+    if (next) {
+        if (next != state.current_behavior) {
+            state.current_behavior->on_exit();   // cleanup current behavior
+            state.current_behavior = next;       // swap behavior
+            state.current_behavior->on_enter();  // init new behavior
+            std::cout << "[INFO] Behavior changed to: " << tok->name() << "\n";
+            return true; // behavior changed
+        } else {
+            std::cout << "[INFO] Behavior already active: " << tok->name() << "\n";
+            return false; // behavior was already current
+        }
+    } else {
+        std::cout << "[WARN] No behavior found for token: " << tok->name() << "\n";
+        return false; // no change
+    }
+	}
+	,
         [=](std::vector<token_ptr> tokens) {
             for (size_t i = 0; i < tokens.size(); ++i) {
                 state.current_behavior->receive(tokens[i]);

From 998b46df4cdfe8f58f4910c1a90e6ba8240ee8f6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 29 Jan 2026 14:56:27 -0600
Subject: [PATCH 0246/1000] Updated message dispatcher to send the correct
 object.

---
 .../control-layer-tests/core_usage_behavior_tests/main.test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index fc332d12c8..37c729e455 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -241,7 +241,7 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
 
   //change the scheduler to core_usage
   anon_mail(
-	caf::cuda::behavior_token("core_usage")
+	caf::cuda::make_behavior_token("core_usage")
 	).send(mgr.get_scheduler_actor());
 
   // CREATE ONCE

From 12b2225cebce8ff3135b37e64013be8515af11f9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 30 Jan 2026 14:32:12 -0600
Subject: [PATCH 0247/1000] Implemented getCost(token_ptr) method since
 overloads with same name are hidden in c++ and thats not what I want here.

---
 .../scheduler-functions/core_heuristic_function.hpp       | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
index 7103c679a4..08114469a4 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
@@ -21,6 +21,14 @@ class core_heuristic_function : public heuristic_function {
     : heuristic_function(other),
       dev_(dev) {}
 
+
+  //mostly here to please the c++ compiler
+  int getCost(const token_ptr& tok) override {
+    return heuristic_function::getCost(tok); // call base
+  }
+
+
+
   int getCost(const program_ptr& prog,
               const nd_range& range) override {
     try {

From 64851d31b14d6e847d909893b000c976eaf56c0b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 30 Jan 2026 14:35:34 -0600
Subject: [PATCH 0248/1000] Implemented getCost(token_ptr) method since
 overloads with same name are hidden in c++ and thats not what I want here.

---
 libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index 18c2791e52..45e1dde556 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -38,13 +38,13 @@ class core_usage_behavior : public scheduler_actor_behavior {
    //data structures to manage dependencies
    std::unordered_map<int,kernel_graph> graphs;
    std::vector<kernel_graph> independent_graphs;
-   std::vector<kernel_graph> best_graphs; //should contain top 5-10 best selections ideally or something along the lines
+   std::vector<kernel_graph*> best_graphs; //should contain top 5-10 best selections ideally or something along the lines
 
    void init_state();
    void create_new_graph(const token_ptr& token); //this should either add to indepedent or graphs data structure
 			    //note to self use std::move for cheap copies
    
-   void rank(); //this should rank the graphs (high to low) for best canidates
+   void rank(std::size_t); //this should rank the graphs (high to low) for best canidates
 
    int get_next_stream(); // this should return the next stream based on some decisions
    void dummy_schedule(); //just a test scheduler algorthim do not use seriously

From 213bd3a6b333608924106a03043a95a1cc0e239f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 30 Jan 2026 14:36:06 -0600
Subject: [PATCH 0249/1000] Implemented rank method here.

---
 .../src/control-layer/core_usage_behavior.cpp | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index e3dddd32bf..fe0b3c2450 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -89,4 +89,50 @@ void core_usage_behavior::dummy_schedule() {
     }
 }
 
+void core_usage_behavior::rank(std::size_t max_best = 5) {
+    best_graphs.clear();
+
+    struct candidate {
+        int cost;
+        kernel_graph* graph;
+    };
+
+    std::vector<candidate> candidates;
+    candidates.reserve(graphs.size());
+
+    // Step 1: gather all candidate graphs
+    for (auto& [dep, graph] : graphs) {
+        if (graph.empty())
+            continue;
+
+        token_ptr tok = graph.peek();  // non-destructive
+        if (!tok || tok->getType() != LAUNCH)
+            continue;
+
+        int cost = heuristic->getCost(tok);
+        if (cost == ERROR_CODE)
+            continue;
+
+        candidates.push_back({cost, &graph});
+    }
+
+    if (candidates.empty())
+        return;
+
+    // Step 2: sort by ascending cost (cheap kernels first)
+    std::sort(candidates.begin(), candidates.end(),
+              [](const candidate& a, const candidate& b) {
+                  return a.cost < b.cost;
+              });
+
+    // Step 3: keep top N candidates
+    const std::size_t limit = std::min(max_best, candidates.size());
+    for (std::size_t i = 0; i < limit; ++i) {
+        best_graphs.push_back(candidates[i].graph);
+    }
+}
+
+
+
+
 } // namespace caf::cuda

From 292e17a3edf1d6d90873075152fa13326864a42c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 30 Jan 2026 14:42:00 -0600
Subject: [PATCH 0250/1000] Added set_Status and get_status methods

---
 libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
index 76a7d96908..b1ffa6ac67 100644
--- a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
@@ -25,6 +25,14 @@ class kernel_graph {
           stream_id_(stream_id),
           dependency_number_(dependency_number) {}
 
+
+        // Convenience constructor for independent graphs
+    kernel_graph(int device_number, int stream_id)
+        : device_number_(device_number),
+          stream_id_(stream_id),
+          dependency_number_(INDEPENDENT) {}
+
+
     // returns the next operation/token_ptr that can be dequeued
     token_ptr peek() const {
         if (operations.empty())
@@ -51,13 +59,15 @@ class kernel_graph {
     }
 
     int stream_id() const noexcept { return stream_id_; }
-
+    void set_status(int s) noexcept {status = s;}
+    int get_status() const noexcept {return status;}
 
 
 private:
     int device_number_;
     int stream_id_;
     int dependency_number_;
+    int status = READY;
     std::vector<token_ptr> operations;
 };
 

From 270d62809895b64073c6be72abf257ca2939b7e7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 30 Jan 2026 14:56:35 -0600
Subject: [PATCH 0251/1000] Updated the receive method to dispatch a token if
 there is aviable resources to do so.

---
 .../src/control-layer/core_usage_behavior.cpp | 23 +++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index fe0b3c2450..c29caa9d23 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -31,14 +31,25 @@ void core_usage_behavior::reclaim(int, int, int, int) {
     // intentionally empty for now
 }
 
-void core_usage_behavior::schedule() {
-    dummy_schedule();
+void core_usage_behavior::schedule() {	
+	//dummy_schedule();
 }
 
 void core_usage_behavior::receive(const token_ptr& tok) {
     if (tok->getType() == LAUNCH) {
         create_new_graph(tok);
-        schedule();
+	if (available_SM -  heuristic->getCost(tok) < 0) {schedule();}
+	else {
+		if (tok->isIndependent()) 
+		{
+			process_launch_token(tok,get_next_stream());
+			return;
+		}
+
+		int stream = graphs[tok->getDependency()].stream_id();
+		process_launch_token(tok,stream);
+	}
+
     } else if (tok->getType() == MEMORY) {
         process_memory_transfer_token(tok, 0);
     }
@@ -90,7 +101,11 @@ void core_usage_behavior::dummy_schedule() {
 }
 
 void core_usage_behavior::rank(std::size_t max_best = 5) {
-    best_graphs.clear();
+   
+   //TODO INCORPORATE GRAPH STATUS INTO THIS EVENTUALLY
+   //AND FIGURE OUT A WAY TO CUT OUT EMPTY GRAPHS	
+	
+   best_graphs.clear();
 
     struct candidate {
         int cost;

From a5d72e38552ed5aa7947bf3c682637139c30aa07 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 30 Jan 2026 15:04:31 -0600
Subject: [PATCH 0252/1000] Implemented a process_launch token method override
 to can do bookeeping a bit easier.

---
 .../caf/cuda/control-layer/core_usage_behavior.hpp       | 4 ++++
 libcaf_cuda/src/control-layer/core_usage_behavior.cpp    | 9 +++++++++
 2 files changed, 13 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index 45e1dde556..33e698ba02 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -17,6 +17,10 @@ class core_usage_behavior : public scheduler_actor_behavior {
     void reclaim(int value /*blocks consumed*/,int memory_returned,int time,int dependency) override; 
     ~core_usage_behavior() override;  
 
+
+protected:
+   void process_launch_token(const token_ptr& tok, int stream_id) override;
+
 private:
    device_ptr device_;
    //core_heuristic_function heuristic;
diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index c29caa9d23..4d7e88bb34 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -35,6 +35,15 @@ void core_usage_behavior::schedule() {
 	//dummy_schedule();
 }
 
+
+void core_usage_behavior::process_launch_token(const token_ptr& tok,int stream_id )  {
+
+	scheduler_actor_behavior::process_launch_token(tok,stream_id);
+	available_SM -= heuristic->getCost(tok);
+
+}
+
+
 void core_usage_behavior::receive(const token_ptr& tok) {
     if (tok->getType() == LAUNCH) {
         create_new_graph(tok);

From a6e222863117c09554c3d6301105f6a823bb08dd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 30 Jan 2026 16:35:35 -0600
Subject: [PATCH 0253/1000] Implemented get_num_blocks method.

---
 libcaf_cuda/caf/cuda/nd_range.hpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/nd_range.hpp b/libcaf_cuda/caf/cuda/nd_range.hpp
index 5fc3986536..ca7830ff48 100644
--- a/libcaf_cuda/caf/cuda/nd_range.hpp
+++ b/libcaf_cuda/caf/cuda/nd_range.hpp
@@ -56,6 +56,11 @@ class nd_range {
 	  return blockDim[0] * blockDim[1] * blockDim[2];
   }
 
+ //get number of blocks in total
+ size_t get_num_blocks() const noexcept {
+	  return gridDim[0] * gridDim[1] * gridDim[2];
+ }
+
 
   ~nd_range() {
 	  //no-op

From 59dcd6065528bd75b9c859c527251f23950211f8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 30 Jan 2026 16:44:45 -0600
Subject: [PATCH 0254/1000] Created sm_usage_heuristic function.

---
 .../cuda/control-layer/all-control-layer.hpp  |  1 +
 .../sm_usage_heuristic.hpp                    | 58 +++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index ec7e9aa9d9..99c10f2ad8 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -16,6 +16,7 @@
 #include "caf/cuda/control-layer/token_factory.hpp"
 #include "caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp"
 #include "caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp"
 #include "caf/cuda/control-layer/kernel_graph.hpp"
 #include "caf/cuda/control-layer/core_usage_behavior.hpp"
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
new file mode 100644
index 0000000000..b4050126c3
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
@@ -0,0 +1,58 @@
+#pragma once
+
+#include <cmath>
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/nd_range.hpp"
+#include "caf/cuda/program.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp"
+
+namespace caf::cuda {
+
+class sm_usage_heuristic : public heuristic_function {
+public:
+  explicit sm_usage_heuristic(device_ptr dev)
+    : dev_(dev) {}
+
+  int getCost(const program_ptr& prog,
+              const nd_range& range) override {
+    try {
+      const std::string key =
+        prog->getName() + range.to_string();
+
+      auto it = values_.find(key);
+      if (it != values_.end())
+        return it->second;
+
+      int total_blocks =
+        static_cast<int>(range.get_num_blocks());
+
+      int blocks_per_sm =
+        dev_->max_active_blocks_per_sm(prog, range);
+
+      if (blocks_per_sm <= 0)
+        return ERROR_CODE;
+
+      int sms_needed =
+        (total_blocks + blocks_per_sm - 1) / blocks_per_sm; // ceil
+
+      int sms_used =
+        std::min(dev_->num_sms(), sms_needed);
+
+      values_[key] = sms_used;
+      return sms_used;
+    }
+    catch (...) {
+      return ERROR_CODE;
+    }
+  }
+
+  int getCost(const token_ptr& tok) override {
+    return heuristic_function::getCost(tok);
+  }
+
+private:
+  device_ptr dev_;
+};
+
+} // namespace caf::cuda
+

From cc72ce195464a9e256485a9ae8ec42eb2ca55b3c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 30 Jan 2026 16:48:59 -0600
Subject: [PATCH 0255/1000] Changed class core_usage_behavior to
 sm_usage_heuritic This change was made due to the discrepencies in metrics,
 the behavior looks at SM's will the core_usage_behavior looked at how many
 blocks could fit into 1 SM, which would lead to bad decisions being made.

---
 libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index 33e698ba02..e2fea7655b 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -1,7 +1,8 @@
 #pragma once
 #include "caf/cuda/control-layer/behavior.hpp"
 #include "caf/cuda/control-layer/kernel_graph.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp"
+//#include "caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp"
 #include "caf/cuda/device.hpp"
 #include <unordered_map>
 #include <optional>
@@ -25,7 +26,7 @@ class core_usage_behavior : public scheduler_actor_behavior {
    device_ptr device_;
    //core_heuristic_function heuristic;
   
-   std::optional<core_heuristic_function> heuristic;
+   std::optional<sm_usage_heuristic> heuristic;
 
 
 

From e5ec9ff8371213bb2ccecc0bc6dbb45ecc4d0f28 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 30 Jan 2026 16:51:11 -0600
Subject: [PATCH 0256/1000] Added documentation.

---
 .../scheduler-functions/core_heuristic_function.hpp          | 5 +++++
 .../control-layer/scheduler-functions/sm_usage_heuristic.hpp | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
index 08114469a4..0cb082a3c5 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
@@ -7,6 +7,11 @@
 #include "caf/cuda/nd_range.hpp"
 #include "caf/cuda/program.hpp"
 #include "caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp"
+/*
+ * Was originally supposed to estimate core usage 
+ * but it just tells how much blocks can fit into 1 SM
+ */
+
 
 namespace caf::cuda {
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
index b4050126c3..106faa3d97 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
@@ -6,6 +6,11 @@
 #include "caf/cuda/program.hpp"
 #include "caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp"
 
+/*
+ * will return a value indicating 
+ * how much blocks a kernel with dimensions will consume
+ */
+
 namespace caf::cuda {
 
 class sm_usage_heuristic : public heuristic_function {

From 7a9e778d6e2b73a8631a9b0f31356a0d8310ce16 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 30 Jan 2026 16:54:34 -0600
Subject: [PATCH 0257/1000] Updated the receive method.

---
 libcaf_cuda/src/control-layer/core_usage_behavior.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 4d7e88bb34..13636ad0e5 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -47,8 +47,10 @@ void core_usage_behavior::process_launch_token(const token_ptr& tok,int stream_i
 void core_usage_behavior::receive(const token_ptr& tok) {
     if (tok->getType() == LAUNCH) {
         create_new_graph(tok);
-	if (available_SM -  heuristic->getCost(tok) < 0) {schedule();}
-	else {
+	
+	//if we have the resources to dispatch, just do it right away 
+	if (available_SM -  heuristic->getCost(tok) > 0)
+	 {
 		if (tok->isIndependent()) 
 		{
 			process_launch_token(tok,get_next_stream());

From d584706ef0b905e8b3ba94b01b922c601c0bcafd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 30 Jan 2026 17:10:58 -0600
Subject: [PATCH 0258/1000] Implemented schedule method.

---
 .../src/control-layer/core_usage_behavior.cpp | 38 ++++++++++++++++++-
 1 file changed, 36 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 13636ad0e5..3678be6ed6 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -31,11 +31,45 @@ void core_usage_behavior::reclaim(int, int, int, int) {
     // intentionally empty for now
 }
 
-void core_usage_behavior::schedule() {	
-	//dummy_schedule();
+void core_usage_behavior::schedule() {
+  
+  //if there is only 2 kernels to consider then rerank  	
+  if (best_graphs.size() <= 2) {
+        rank(5);
+        if (best_graphs.empty())
+            return;
+    }
+
+
+    // Greedy: largest-cost first
+    for (int i = static_cast<int>(best_graphs.size()) - 1; i >= 0; --i) {
+        kernel_graph* graph = best_graphs[i];
+        if (!graph || graph->empty())
+            continue;
+
+        token_ptr tok = graph->peek();
+        if (!tok || tok->getType() != LAUNCH)
+            continue;
+
+        int cost = heuristic->getCost(tok);
+        if (cost == ERROR_CODE)
+            continue;
+
+        if (available_SM >= cost) {
+            tok = graph->getOperation();
+            process_launch_token(tok, graph->stream_id());
+            best_graphs.erase(best_graphs.begin() + i);
+        }
+    }
+
+    //again re-rank if we have less than 2 operations to consider
+    if (best_graphs.size() <= 2) {
+        rank(5);
+    }
 }
 
 
+
 void core_usage_behavior::process_launch_token(const token_ptr& tok,int stream_id )  {
 
 	scheduler_actor_behavior::process_launch_token(tok,stream_id);

From 89180c35ff27d96e3d235070c3720049b14c2f5b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 30 Jan 2026 17:17:14 -0600
Subject: [PATCH 0259/1000] Implemented reclaim method.

---
 .../src/control-layer/core_usage_behavior.cpp | 83 ++++++++++---------
 1 file changed, 45 insertions(+), 38 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 3678be6ed6..6170af4c40 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -27,54 +27,25 @@ void core_usage_behavior::on_enter() {
     // TODO implement
 }
 
-void core_usage_behavior::reclaim(int, int, int, int) {
-    // intentionally empty for now
-}
-
-void core_usage_behavior::schedule() {
-  
-  //if there is only 2 kernels to consider then rerank  	
-  if (best_graphs.size() <= 2) {
-        rank(5);
-        if (best_graphs.empty())
-            return;
-    }
+void core_usage_behavior::reclaim(int blocks_consumed,
+	       	int memory_returned,
+	       	int time,
+	       	int dependency_number) {
 
+	available_SM += blocks_consumed;
+	available_memory+= memory_returned;
+	//will eventually do something with the dependency number and stalling or maybe not
+	schedule();
 
-    // Greedy: largest-cost first
-    for (int i = static_cast<int>(best_graphs.size()) - 1; i >= 0; --i) {
-        kernel_graph* graph = best_graphs[i];
-        if (!graph || graph->empty())
-            continue;
-
-        token_ptr tok = graph->peek();
-        if (!tok || tok->getType() != LAUNCH)
-            continue;
-
-        int cost = heuristic->getCost(tok);
-        if (cost == ERROR_CODE)
-            continue;
-
-        if (available_SM >= cost) {
-            tok = graph->getOperation();
-            process_launch_token(tok, graph->stream_id());
-            best_graphs.erase(best_graphs.begin() + i);
-        }
-    }
-
-    //again re-rank if we have less than 2 operations to consider
-    if (best_graphs.size() <= 2) {
-        rank(5);
-    }
 }
 
 
 
+
 void core_usage_behavior::process_launch_token(const token_ptr& tok,int stream_id )  {
 
 	scheduler_actor_behavior::process_launch_token(tok,stream_id);
 	available_SM -= heuristic->getCost(tok);
-
 }
 
 
@@ -193,6 +164,42 @@ void core_usage_behavior::rank(std::size_t max_best = 5) {
 }
 
 
+void core_usage_behavior::schedule() {
+  
+  //if there is only 2 kernels to consider then rerank  	
+  if (best_graphs.size() <= 2) {
+        rank(5);
+        if (best_graphs.empty())
+            return;
+    }
+
+
+    // Greedy: largest-cost first
+    for (int i = static_cast<int>(best_graphs.size()) - 1; i >= 0; --i) {
+        kernel_graph* graph = best_graphs[i];
+        if (!graph || graph->empty())
+            continue;
+
+        token_ptr tok = graph->peek();
+        if (!tok || tok->getType() != LAUNCH)
+            continue;
+
+        int cost = heuristic->getCost(tok);
+        if (cost == ERROR_CODE)
+            continue;
+
+        if (available_SM >= cost) {
+            tok = graph->getOperation();
+            process_launch_token(tok, graph->stream_id());
+            best_graphs.erase(best_graphs.begin() + i);
+        }
+    }
+
+    //again re-rank if we have less than 2 operations to consider
+    if (best_graphs.size() <= 2) {
+        rank(5);
+    }
+}
 
 
 } // namespace caf::cuda

From c6efbed80ede9a1e391876edded8dfaa4d9cde14 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 31 Jan 2026 11:22:36 -0600
Subject: [PATCH 0260/1000] Added scaling test.

---
 .../core_usage_behavior_tests/main.test.cpp   | 119 +++++++++++++++++-
 1 file changed, 114 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index 37c729e455..fca86a57c7 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -269,14 +269,123 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
 }
 
 
+void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  /*
+  //change the scheduler to core_usage
+  anon_mail(
+	caf::cuda::make_behavior_token("core_usage")
+	).send(mgr.get_scheduler_actor());
+
+  */
+
+
+  // CREATE ONCE
+  auto program =
+      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+  const int THREADS = 32;
+  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1);
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+  for (int i = 0; i < num_actors; ++i) {
+    sys.spawn(
+        mmul_actor_fun,
+        exit_actor,
+        matrix_size,
+        program,
+        dims);
+  }
+
+  sys.await_all_actors_done();
+}
+
+
+
+template <class Fn>
+double time_run(Fn&& fn) {
+  auto start = std::chrono::steady_clock::now();
+  fn();
+  auto end = std::chrono::steady_clock::now();
+  std::chrono::duration<double> elapsed = end - start;
+  return elapsed.count();
+}
+
+
+
+void run_mmul_scaling_tests(caf::actor_system& sys,
+                            caf::cuda::manager_config man_config) {
+  const int min_size   = 10;
+  const int max_size   = 1024;
+  const int min_actors = 1;
+  const int max_actors = 1024;
+
+  // Matrix sizes: 10, 32, 64, 128, ..., 1024
+  std::vector<int> matrix_sizes = {10};
+  for (int s = 32; s <= max_size; s *= 2)
+    matrix_sizes.push_back(s);
+
+  // Actor counts: 1, 2, 4, ..., 1024
+  std::vector<int> actor_counts;
+  for (int a = min_actors; a <= max_actors; a *= 2)
+    actor_counts.push_back(a);
+
+  std::cout << "=== MMUL Scaling Tests ===\n";
+  std::cout << "Columns:\n";
+  std::cout << "matrix_size actors scheduler_time(s) no_scheduler_time(s)\n";
+
+  for (int size : matrix_sizes) {
+    for (int actors : actor_counts) {
+
+      std::cout << "\n[RUN] matrix_size=" << size
+                << ", actors=" << actors << "\n";
+
+      /* ---------------- Scheduler enabled ---------------- */
+      caf::cuda::manager::init(sys, man_config);
+
+      double sched_time = time_run([&] {
+        run_mmul_test(sys, size, actors);
+      });
+
+      /* ---------------- No scheduler ---------------- */
+      caf::cuda::manager::init(sys, man_config);
+
+      double no_sched_time = time_run([&] {
+        run_mmul_test_no_scheduler(sys, size, actors);
+      });
+
+      std::cout << std::fixed << std::setprecision(6)
+                << "RESULT "
+                << size << " "
+                << actors << " "
+                << sched_time << " "
+                << no_sched_time << "\n";
+    }
+  }
+
+  std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
+}
+
+
+
+
+
 void caf_main(caf::actor_system& sys) {
   
-	
-
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
-	caf::cuda::manager::init(sys,man_config);
-       	run_mmul_test(sys,10,500);
-	
+	//caf::cuda::manager::init(sys,man_config);
+       	//run_mmul_test(sys,512,512);	
+	run_mmul_scaling_tests(sys,man_config);
 	//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	
 	//caf::cuda::manager::init(sys,man_config);

From 0a2154221ca9bdf712070a94652e2e09b5ebc497 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 31 Jan 2026 11:25:51 -0600
Subject: [PATCH 0261/1000] Updated test to have better logs.

---
 .../core_usage_behavior_tests/main.test.cpp   | 39 ++++++++++---------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index fca86a57c7..23c80946e2 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -17,6 +17,7 @@
 
 
 
+
 using namespace caf;
 using namespace std::chrono_literals;
 
@@ -310,8 +311,6 @@ void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num
   sys.await_all_actors_done();
 }
 
-
-
 template <class Fn>
 double time_run(Fn&& fn) {
   auto start = std::chrono::steady_clock::now();
@@ -321,11 +320,8 @@ double time_run(Fn&& fn) {
   return elapsed.count();
 }
 
-
-
 void run_mmul_scaling_tests(caf::actor_system& sys,
                             caf::cuda::manager_config man_config) {
-  const int min_size   = 10;
   const int max_size   = 1024;
   const int min_actors = 1;
   const int max_actors = 1024;
@@ -335,40 +331,50 @@ void run_mmul_scaling_tests(caf::actor_system& sys,
   for (int s = 32; s <= max_size; s *= 2)
     matrix_sizes.push_back(s);
 
-  // Actor counts: 1, 2, 4, ..., 1024
+  // Actor counts: 1, 2, 4, 8, ..., 1024
   std::vector<int> actor_counts;
   for (int a = min_actors; a <= max_actors; a *= 2)
     actor_counts.push_back(a);
 
   std::cout << "=== MMUL Scaling Tests ===\n";
-  std::cout << "Columns:\n";
-  std::cout << "matrix_size actors scheduler_time(s) no_scheduler_time(s)\n";
+  std::cout << "Format:\n";
+  std::cout << "scheduler matrix_size actors time_seconds\n";
 
   for (int size : matrix_sizes) {
     for (int actors : actor_counts) {
 
-      std::cout << "\n[RUN] matrix_size=" << size
-                << ", actors=" << actors << "\n";
-
-      /* ---------------- Scheduler enabled ---------------- */
+      /* ================= Scheduler enabled ================= */
       caf::cuda::manager::init(sys, man_config);
 
+      std::cout << "\n[RUN] scheduler=core_usage "
+                << "matrix_size=" << size
+                << " actors=" << actors << "\n";
+
       double sched_time = time_run([&] {
         run_mmul_test(sys, size, actors);
       });
 
-      /* ---------------- No scheduler ---------------- */
+      std::cout << std::fixed << std::setprecision(6)
+                << "RESULT core_usage "
+                << size << " "
+                << actors << " "
+                << sched_time << "\n";
+
+      /* ================= No scheduler ================= */
       caf::cuda::manager::init(sys, man_config);
 
+      std::cout << "\n[RUN] scheduler=none "
+                << "matrix_size=" << size
+                << " actors=" << actors << "\n";
+
       double no_sched_time = time_run([&] {
         run_mmul_test_no_scheduler(sys, size, actors);
       });
 
       std::cout << std::fixed << std::setprecision(6)
-                << "RESULT "
+                << "RESULT none "
                 << size << " "
                 << actors << " "
-                << sched_time << " "
                 << no_sched_time << "\n";
     }
   }
@@ -377,9 +383,6 @@ void run_mmul_scaling_tests(caf::actor_system& sys,
 }
 
 
-
-
-
 void caf_main(caf::actor_system& sys) {
   
 	caf::cuda::manager_config man_config(true); //turns the scheduler on

From 3182c2d8a06d6c316d1a80c5a5fb8749090fc48f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 31 Jan 2026 11:33:07 -0600
Subject: [PATCH 0262/1000] Added a no verify actor.

---
 .../core_usage_behavior_tests/main.test.cpp   | 100 ++++++++++++++++++
 1 file changed, 100 insertions(+)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index 23c80946e2..70a8b302f4 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -232,6 +232,106 @@ caf::behavior mmul_actor_fun(
 
 
 
+
+
+// this actor will not verify its results
+// great for performance analysis 
+caf::behavior mmul_actor_fun_no_verify(
+    caf::stateful_actor<mmul_actor_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims)
+{
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+	caf::actor scheduler = mgr.get_scheduler_actor();
+
+	//send a launch token
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+			program,
+			dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+		[=] (caf::cuda::response_token_ptr res_token) {
+
+			if (res_token -> getType() == LAUNCH_RESPONSE) {
+				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+				//assume N = 1024
+				int N = self -> state().N;
+				std::vector<int> matrix1(N*N);
+				matrix1.reserve(N);
+				std::vector<int> matrix2(N*N);
+				matrix2.reserve(N);
+
+				//std::cout << "GPU ACTOR sending data to compute\n";
+				self -> mail(matrix1,matrix2,res_token,N).send(self);
+
+			}
+			else {
+				std::cout << "Got a memory response token\n";
+			}
+			//token should drop out of scope now, triggering a response 
+		},
+
+			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+			[=](const std::vector<int>& matrixA,
+					const std::vector<int>& matrixB,
+					const caf::cuda::response_token_ptr& res_token, int N) {
+
+
+				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
+
+				//caf::cuda::launch_response_token& kt =
+				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
+
+				//std::cout << "GPU ACTOR  computing\n";
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+				//create program and dims   
+				auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+				const int THREADS = 32;
+				const int BLOCKS = (N + THREADS - 1) / THREADS;
+				caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrixA);
+				auto arg2 = caf::cuda::create_in_arg(matrixB);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
+
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				//std::cout << "GPU ACTOR done  computing\n";
+				// signal exit actor and quit
+				self->mail(1).send(exit_actor);
+				self->quit();
+
+			}
+
+				
+	};
+}
+
+
+
+
+
+
+
+
+
 void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
   if (num_actors < 1) {
     std::cerr << "[ERROR] Number of actors must be >= 1\n";

From ccd76d6e2c92ee4dc58936b3c9411072a450ae6f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 31 Jan 2026 11:54:27 -0600
Subject: [PATCH 0263/1000] Added name features for logging.

---
 libcaf_cuda/caf/cuda/control-layer/behavior.hpp             | 4 +++-
 libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp  | 2 +-
 libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp | 2 +-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
index 7662444e84..d6bbe0279a 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
@@ -5,7 +5,7 @@
 #include "caf/cuda/control-layer/scheduler_actor_state.hpp"
 #include "caf/cuda/control-layer/token.hpp"
 #include "caf/cuda/control-layer/token_factory.hpp"
-
+#include <string>
 
 
 
@@ -36,6 +36,8 @@ class scheduler_actor_behavior {
 	    //by children classes 
     }
 
+    virtual std::string name() const {return "No name\n";}
+
 
 protected:
     scheduler_actor_state& state_;
diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index e2fea7655b..6c86ba9d36 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -17,7 +17,7 @@ class core_usage_behavior : public scheduler_actor_behavior {
     void receive(const token_ptr& tok) override;
     void reclaim(int value /*blocks consumed*/,int memory_returned,int time,int dependency) override; 
     ~core_usage_behavior() override;  
-
+    std::string name() const override {return "core_usage\n";}
 
 protected:
    void process_launch_token(const token_ptr& tok, int stream_id) override;
diff --git a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
index c26adadc28..eb14717c47 100644
--- a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
@@ -9,7 +9,7 @@ class green_light_behavior : public scheduler_actor_behavior {
     void on_enter() override;
     void schedule() override;
     void receive(const token_ptr& tok) override;
-
+    std::string name() const override {return "green\n";}
 };
 
 } // namespace caf::cuda

From 822629c06fff1a7fc86ca7ea375f2058f99e2be4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 31 Jan 2026 12:00:27 -0600
Subject: [PATCH 0264/1000] FIxed capture of callback functions to ensure that
 the handle of state was not stale when changing behavirs.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 9c52a2830e..ebf7b257d6 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -35,21 +35,21 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
     state.current_behavior->on_enter();
 
     return {
-        [=](const token_ptr& tok) {
+        [&](const token_ptr& tok) {
             // std::cout << "Received token\n";
             state.current_behavior->receive(tok);
         },
-        [&state](const caf::cuda::behavior_token_ptr& tok) -> bool {
+        
+   [&state](const caf::cuda::behavior_token_ptr& tok) -> bool {
     auto* next = state.table.get(*tok);
     if (next) {
         if (next != state.current_behavior) {
             state.current_behavior->on_exit();   // cleanup current behavior
             state.current_behavior = next;       // swap behavior
             state.current_behavior->on_enter();  // init new behavior
-            std::cout << "[INFO] Behavior changed to: " << tok->name() << "\n";
-            return true; // behavior changed
+	    return true; // behavior changed
         } else {
-            std::cout << "[INFO] Behavior already active: " << tok->name() << "\n";
+            std::cout << "[INFO] Behavior already active: " << state.current_behavior->name() << "\n";
             return false; // behavior was already current
         }
     } else {

From 11c072c6ef9f8696599406adb25dbceee5afb57a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 31 Jan 2026 12:12:09 -0600
Subject: [PATCH 0265/1000] Updated scaling tests to use no verify for now.

---
 .../core_usage_behavior_tests/main.test.cpp   | 26 ++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index 70a8b302f4..b288142702 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -358,12 +358,21 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
   caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
 
   for (int i = 0; i < num_actors; ++i) {
+    /*
     sys.spawn(
         mmul_actor_fun,
         exit_actor,
         matrix_size,
         program,
         dims);
+    */
+      sys.spawn(
+        mmul_actor_fun_no_verify,
+        exit_actor,
+        matrix_size,
+        program,
+        dims);
+    
   }
 
   sys.await_all_actors_done();
@@ -400,12 +409,23 @@ void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num
   caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
 
   for (int i = 0; i < num_actors; ++i) {
-    sys.spawn(
+    
+	sys.spawn(
+        mmul_actor_fun_no_verify,
+        exit_actor,
+        matrix_size,
+        program,
+        dims);
+    
+  
+	/*  
+sys.spawn(
         mmul_actor_fun,
         exit_actor,
         matrix_size,
         program,
         dims);
+  */
   }
 
   sys.await_all_actors_done();
@@ -486,8 +506,8 @@ void run_mmul_scaling_tests(caf::actor_system& sys,
 void caf_main(caf::actor_system& sys) {
   
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
-	//caf::cuda::manager::init(sys,man_config);
-       	//run_mmul_test(sys,512,512);	
+//	caf::cuda::manager::init(sys,man_config);
+  //     	run_mmul_test(sys,512,10);	
 	run_mmul_scaling_tests(sys,man_config);
 	//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	

From 674edd63cb90079523d35427ed4919c098ee310b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 31 Jan 2026 12:50:55 -0600
Subject: [PATCH 0266/1000] Updated launch response token to encapsulate values
 require to return a reclaim message.

---
 .../control-layer/launch_response_token.hpp   | 60 +++++++++++++++----
 .../caf/cuda/control-layer/token_factory.hpp  | 13 ++--
 .../src/control-layer/token_factory.cpp       | 17 ++++--
 3 files changed, 69 insertions(+), 21 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
index 132ebca399..8db2103bf1 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -15,30 +15,46 @@ namespace caf::cuda {
 // -----------------------------------------------------------------------------
 class CAF_CUDA_EXPORT launch_response_token : public response_token {
 public:
-    // Only here to be compliant with CAF's type system – DO NOT USE directly
+    // Default constructor – only for CAF compliance
     launch_response_token() = default;
 
-    // Construct manually
+    // Construct manually (full control over reclaim fields)
     launch_response_token(caf::actor receiver,
                           nd_range range,
                           int memory_usage,
                           std::string id,
                           int device_num = 0,
-                          int stream_id = 0)
+                          int stream_id = 0,
+                          int reclaim_value = 0,
+                          int reclaim_memory_returned = 0,
+                          int reclaim_runtime = 0,
+                          int reclaim_dependency = 0)
         : response_token(std::move(receiver), device_num, stream_id, memory_usage),
           range_(std::move(range)),
           id_(std::move(id)),
-          released_(false) {}
+          released_(false),
+          reclaim_value_(reclaim_value),
+          reclaim_memory_returned_(reclaim_memory_returned),
+          reclaim_runtime_(reclaim_runtime),
+          reclaim_dependency_(reclaim_dependency) {}
 
     // Construct from a launch_token
+    // Memory returned and dependency are copied from launch_token
     launch_response_token(caf::actor receiver,
                           const launch_token& token,
                           int device_num,
-                          int stream_id)
+                          int stream_id,
+                          int reclaim_value = 0,
+                          int reclaim_runtime = 0)
         : response_token(std::move(receiver), device_num, stream_id, token.getMemoryUsage()),
           range_(token.getRange()),
           id_(token.getId()),
-          released_(false) {}
+          released_(false),
+          reclaim_value_(reclaim_value),
+          reclaim_memory_returned_(token.getMemoryUsage()), // copy from launch_token
+          reclaim_runtime_(reclaim_runtime),
+          reclaim_dependency_(token.getDependency())        // copy from launch_token
+    {}
 
     ~launch_response_token() {
         release();
@@ -59,25 +75,43 @@ class CAF_CUDA_EXPORT launch_response_token : public response_token {
         );
     }
 
+    // Release and send reclaim information exactly once
     void release() override {
         bool expected = false;
 
-        // ONLY send if we successfully transition false → true
         if (!released_.compare_exchange_strong(expected, true)) {
-            return; // already released → do nothing
+            return; // already released
         }
 
-        // Real message (commented for testing)
-        // caf::anon_mail(id_, memorySize()).urgent().send(receiver_);
-
-        // Test message
-        caf::anon_mail("Hello world from launch response").urgent().send(receiver_);
+        try {
+            // Send a message containing all four reclaim fields
+            caf::anon_mail(
+                           reclaim_value_,
+                           reclaim_memory_returned_,
+                           reclaim_runtime_,
+                           reclaim_dependency_).urgent().send(receiver_);
+        } catch (...) {
+            // swallow exceptions — destructor safe
+        }
     }
 
+    // Accessors for reclaim fields
+    int reclaim_value() const { return reclaim_value_; }
+    int reclaim_memory_returned() const { return reclaim_memory_returned_; }
+    int reclaim_runtime() const { return reclaim_runtime_; }
+    int reclaim_dependency() const { return reclaim_dependency_; }
+
+
 private:
     nd_range range_;
     std::string id_;
     std::atomic<bool> released_;
+
+    // Reclaim fields
+    int reclaim_value_ = 0;
+    int reclaim_memory_returned_ = 0;
+    int reclaim_runtime_ = 0;
+    int reclaim_dependency_ = 0;
 };
 
 using kernel_launch_token = caf::intrusive_ptr<launch_response_token>;
diff --git a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
index fff5508c25..a2bc57f2b5 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
@@ -23,10 +23,15 @@ CAF_CUDA_EXPORT token_ptr make_launch_token(program_ptr prog,
 
 /// Creates a launch_response_token (created internally by the scheduler
 /// when it accepts a kernel launch request)
-CAF_CUDA_EXPORT response_token_ptr make_launch_response_token(actor scheduler_or_proxy,
-                                     const launch_token& orig,
-				     int device_number,
-				     int stream_id);
+CAF_CUDA_EXPORT response_token_ptr make_launch_response_token(
+    actor receiver,
+    const launch_token& orig,
+    int device_number,
+    int stream_id,
+    int reclaim_value = 0,      // optional
+    int reclaim_runtime = 0     // optional
+);
+
 
 /// Creates a behavior_token (special — returns its own strong ptr type)
 CAF_CUDA_EXPORT behavior_token_ptr make_behavior_token(std::string name);
diff --git a/libcaf_cuda/src/control-layer/token_factory.cpp b/libcaf_cuda/src/control-layer/token_factory.cpp
index f1b2d4ce2c..fd16496140 100644
--- a/libcaf_cuda/src/control-layer/token_factory.cpp
+++ b/libcaf_cuda/src/control-layer/token_factory.cpp
@@ -24,13 +24,22 @@ token_ptr make_launch_token(program_ptr prog,
 }
 
 response_token_ptr make_launch_response_token(actor receiver,
-                                     const launch_token& orig,
-				     int device_number,
-				     int stream_id)
+                                              const launch_token& orig,
+                                              int device_number,
+                                              int stream_id,
+                                              int reclaim_value,
+                                              int reclaim_runtime)
 {
-    return response_token_ptr(new launch_response_token(receiver, orig,device_number,stream_id));
+    return response_token_ptr(
+        new launch_response_token(receiver,
+                                  orig,
+                                  device_number,
+                                  stream_id,
+                                  reclaim_value,
+                                  reclaim_runtime));
 }
 
+
 behavior_token_ptr make_behavior_token(std::string name)
 {
     return behavior_token_ptr(new behavior_token(std::move(name)));

From 9de100d448b7bb80b5cfc600b82650e06c58af76 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 31 Jan 2026 12:56:28 -0600
Subject: [PATCH 0267/1000] Added reclaim message handler.

---
 .../src/control-layer/scheduler_actor.cpp     | 30 +++++++------------
 1 file changed, 11 insertions(+), 19 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index ebf7b257d6..8f4991cf86 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -47,6 +47,7 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
             state.current_behavior->on_exit();   // cleanup current behavior
             state.current_behavior = next;       // swap behavior
             state.current_behavior->on_enter();  // init new behavior
+            std::cout << "[INFO] Behavior changed to: " << state.current_behavior->name() << "\n";
 	    return true; // behavior changed
         } else {
             std::cout << "[INFO] Behavior already active: " << state.current_behavior->name() << "\n";
@@ -58,31 +59,22 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
     }
 	}
 	,
-        [=](std::vector<token_ptr> tokens) {
+        [&](std::vector<token_ptr> tokens) {
             for (size_t i = 0; i < tokens.size(); ++i) {
                 state.current_behavior->receive(tokens[i]);
             }
         },
+
+	//can send the scheduler a message if you want 
+	//it is more than happy to print it out for you 
         [=](std::string word) {
-            // std::cout << "Received message " << word << "\n";
+             std::cout << "Received message " << word << "\n";
         },
-        [=](caf::cuda::mem_ptr<int> token) {
-            if (!token) {
-                std::cout << "Received null mem_ptr\n";
-                return;
-            }
-            if (token->is_scalar()) {
-                std::cout << "Received mem_ptr with scalar value: "
-                          << *token->host_scalar_ptr() << "\n";
-            } else {
-                std::cout << "Received mem_ptr with "
-                          << token->size() << " elements\n";
-                // Optional: print fake data if testing copy_to_host
-                // auto host_data = token->copy_to_host();
-                // for (auto v : host_data) std::cout << v << " ";
-                // std::cout << "\n";
-            }
-        }
+	
+	//message handler for reclaim
+	[&](int value, int memory,int runtime,int dependency) {	
+		state.current_behavior->reclaim(value,memory,runtime,dependency);
+	}
     };
 }
 

From 5e3b5e236c968e25c5f1e7cd316ac09d87b84d9d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 31 Jan 2026 12:59:07 -0600
Subject: [PATCH 0268/1000] Update process launch token to encapsulate blocks
 consumed in the token so it can be returned on reclaimed.

---
 .../src/control-layer/core_usage_behavior.cpp      | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 6170af4c40..2447c61b49 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -25,6 +25,7 @@ void core_usage_behavior::init_state() {
 
 void core_usage_behavior::on_enter() {
     // TODO implement
+	//std::cout << "Hello\n";
 }
 
 void core_usage_behavior::reclaim(int blocks_consumed,
@@ -44,13 +45,20 @@ void core_usage_behavior::reclaim(int blocks_consumed,
 
 void core_usage_behavior::process_launch_token(const token_ptr& tok,int stream_id )  {
 
-	scheduler_actor_behavior::process_launch_token(tok,stream_id);
-	available_SM -= heuristic->getCost(tok);
+	int cost = heuristic -> getCost(tok);
+
+	const auto& launch = static_cast<const launch_token&>(*tok);
+	auto response = make_launch_response_token(state_.self, launch, state_.device_number, stream_id,cost);
+	anon_mail(response).send(launch.getReplyActor());
+	available_SM -= cost;
 }
 
 
 void core_usage_behavior::receive(const token_ptr& tok) {
-    if (tok->getType() == LAUNCH) {
+
+	//std::cout <<"YARRRRRRRRRRRRRRRRRRRRR\n ";
+    
+	if (tok->getType() == LAUNCH) {
         create_new_graph(tok);
 	
 	//if we have the resources to dispatch, just do it right away 

From 086092a49dae3254466ea4808a7908bd03c71aa3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 31 Jan 2026 13:35:01 -0600
Subject: [PATCH 0269/1000] Updated max_sms block method to correctly push and
 pop cuda context and added more error handling checks These changes were made
 to fix an invalid cuda context error when calling cumaxoccupancy leading to
 invalid numbers being read all the time, also error checks are there to be
 more fault tolerant.

---
 libcaf_cuda/src/device.cpp | 71 +++++++++++++++++++++++++++++++++-----
 1 file changed, 62 insertions(+), 9 deletions(-)

diff --git a/libcaf_cuda/src/device.cpp b/libcaf_cuda/src/device.cpp
index 744d7418c9..36257e63ee 100644
--- a/libcaf_cuda/src/device.cpp
+++ b/libcaf_cuda/src/device.cpp
@@ -1,21 +1,74 @@
 #include "caf/cuda/device.hpp"
 #include "caf/cuda/program.hpp"
+#include <iostream>
+#include <cuda.h>
 
 namespace caf::cuda {
 
-	int device::max_active_blocks_per_sm(const program_ptr& prog, const nd_range& range,
-			size_t dynamic_smem_bytes) const {
-		CUfunction kernel = prog->get_kernel(id_); // full type known here
-		int block_size = static_cast<int>(range.get_num_threads());
-		int active_blocks = 0;
-		cuOccupancyMaxActiveBlocksPerMultiprocessor(&active_blocks, kernel, block_size, dynamic_smem_bytes);
-		return active_blocks;
-	}
+int device::max_active_blocks_per_sm(const program_ptr& prog,
+                                     const nd_range& range,
+                                     size_t dynamic_smem_bytes) const {
 
+//	std::cout << "Hello???\n";
+   try {
+        if (!context_) {
+            std::cerr << "[ERROR] Device context is null for device id " << id_ << "\n";
+            return 0;
+        }
 
+        CUfunction kernel = prog->get_kernel(id_);
+        if (!kernel) {
+            std::cerr << "[ERROR] Kernel handle is null for id: " << id_ << "\n";
+            return 0;
+        }
 
-}//namespace caf cuda
+        int block_size = static_cast<int>(range.get_num_threads());
+        if (block_size <= 0) {
+            std::cerr << "[ERROR] Block size <= 0. Block dims: "
+                      << range.getBlockDimX() << "x"
+                      << range.getBlockDimY() << "x"
+                      << range.getBlockDimZ() << "\n";
+            return 0;
+        }
 
+        // Push the device context for this thread
+        CUresult res = cuCtxPushCurrent(context_);
+        if (res != CUDA_SUCCESS) {
+            const char* errStr = nullptr;
+            cuGetErrorString(res, &errStr);
+            std::cerr << "[ERROR] cuCtxPushCurrent failed: "
+                      << res << " (" << (errStr ? errStr : "Unknown error") << ")\n";
+            return 0;
+        }
 
+        int active_blocks = 0;
+        res = cuOccupancyMaxActiveBlocksPerMultiprocessor(&active_blocks,
+                                                          kernel,
+                                                          block_size,
+                                                          dynamic_smem_bytes);
 
+        if (res != CUDA_SUCCESS) {
+            const char* errStr = nullptr;
+            cuGetErrorString(res, &errStr);
+            std::cerr << "[ERROR] cuOccupancyMaxActiveBlocksPerMultiprocessor failed: "
+                      << res << " (" << (errStr ? errStr : "Unknown error") << ")\n";
+            active_blocks = 0;
+        }
+
+        CUcontext popped_ctx = nullptr;
+        cuCtxPopCurrent(&popped_ctx);
+
+        return active_blocks;
+
+    } catch (const std::exception& e) {
+        std::cerr << "[EXCEPTION] std::exception caught: " << e.what() << "\n";
+        return 0;
+    } catch (...) {
+        std::cerr << "[EXCEPTION] Unknown exception caught while computing occupancy.\n";
+        return 0;
+    }
+
+}
+
+} // namespace caf::cuda
 

From cd4a6f82fad8e71cdceae780931ba2dd22dbce3f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 31 Jan 2026 13:36:33 -0600
Subject: [PATCH 0270/1000] Updated condition in receive to ensure that the
 first avaible kernel will alwasy be scheduled if necessary to prevent a
 deadlock scenario where the scheduler just would not respond.

---
 libcaf_cuda/src/control-layer/core_usage_behavior.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 2447c61b49..68f90a4370 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -33,6 +33,7 @@ void core_usage_behavior::reclaim(int blocks_consumed,
 	       	int time,
 	       	int dependency_number) {
 
+	std::cout << "blocks is " <<  blocks_consumed << "\n";
 	available_SM += blocks_consumed;
 	available_memory+= memory_returned;
 	//will eventually do something with the dependency number and stalling or maybe not
@@ -62,7 +63,7 @@ void core_usage_behavior::receive(const token_ptr& tok) {
         create_new_graph(tok);
 	
 	//if we have the resources to dispatch, just do it right away 
-	if (available_SM -  heuristic->getCost(tok) > 0)
+	if (available_SM -  heuristic->getCost(tok) >= 0)
 	 {
 		if (tok->isIndependent()) 
 		{

From 98e5f8ab5100b46ce11a4dd90cc7ff054f0ba5bc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 31 Jan 2026 13:50:47 -0600
Subject: [PATCH 0271/1000] Updated rank to include independent graphs as well
 This change was made to fix a bug where since rank did not include
 independent graphs the scheduler would refuse to schedule independent
 operations causing a deadlock.

---
 .../src/control-layer/core_usage_behavior.cpp | 48 ++++++++++++-------
 1 file changed, 30 insertions(+), 18 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 68f90a4370..00a686fc81 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -33,7 +33,7 @@ void core_usage_behavior::reclaim(int blocks_consumed,
 	       	int time,
 	       	int dependency_number) {
 
-	std::cout << "blocks is " <<  blocks_consumed << "\n";
+	//std::cout << "blocks is " <<  blocks_consumed << "\n";
 	available_SM += blocks_consumed;
 	available_memory+= memory_returned;
 	//will eventually do something with the dependency number and stalling or maybe not
@@ -126,21 +126,18 @@ void core_usage_behavior::dummy_schedule() {
 }
 
 void core_usage_behavior::rank(std::size_t max_best = 5) {
-   
-   //TODO INCORPORATE GRAPH STATUS INTO THIS EVENTUALLY
-   //AND FIGURE OUT A WAY TO CUT OUT EMPTY GRAPHS	
-	
-   best_graphs.clear();
+    best_graphs.clear();
 
+    //TODO FIGURE OUT A WAY TO PUT STATUS IN HERE AND 
+    //CUT OUT EMPTY GRAPHS
     struct candidate {
         int cost;
         kernel_graph* graph;
     };
 
     std::vector<candidate> candidates;
-    candidates.reserve(graphs.size());
 
-    // Step 1: gather all candidate graphs
+    // Step 1: gather all candidate graphs from dependent graphs
     for (auto& [dep, graph] : graphs) {
         if (graph.empty())
             continue;
@@ -156,6 +153,22 @@ void core_usage_behavior::rank(std::size_t max_best = 5) {
         candidates.push_back({cost, &graph});
     }
 
+    // Step 1b: gather all candidate graphs from independent_graphs
+    for (auto& graph : independent_graphs) {
+        if (graph.empty())
+            continue;
+
+        token_ptr tok = graph.peek();
+        if (!tok || tok->getType() != LAUNCH)
+            continue;
+
+        int cost = heuristic->getCost(tok);
+        if (cost == ERROR_CODE)
+            continue;
+
+        candidates.push_back({cost, &graph});
+    }
+
     if (candidates.empty())
         return;
 
@@ -173,28 +186,28 @@ void core_usage_behavior::rank(std::size_t max_best = 5) {
 }
 
 
+
 void core_usage_behavior::schedule() {
-  
-  //if there is only 2 kernels to consider then rerank  	
-  if (best_graphs.size() <= 2) {
+    // If there are only 2 kernels to consider then rerank
+    if (best_graphs.size() <= 2) {
         rank(5);
-        if (best_graphs.empty())
+        if (best_graphs.empty()) {
             return;
+        }
     }
 
-
     // Greedy: largest-cost first
     for (int i = static_cast<int>(best_graphs.size()) - 1; i >= 0; --i) {
         kernel_graph* graph = best_graphs[i];
-        if (!graph || graph->empty())
+        if (!graph || graph->empty()) 
             continue;
 
         token_ptr tok = graph->peek();
-        if (!tok || tok->getType() != LAUNCH)
+        if (!tok || tok->getType() != LAUNCH) 
             continue;
 
         int cost = heuristic->getCost(tok);
-        if (cost == ERROR_CODE)
+        if (cost == ERROR_CODE) 
             continue;
 
         if (available_SM >= cost) {
@@ -204,11 +217,10 @@ void core_usage_behavior::schedule() {
         }
     }
 
-    //again re-rank if we have less than 2 operations to consider
+    // Re-rank if we have less than 2 operations left
     if (best_graphs.size() <= 2) {
         rank(5);
     }
 }
 
-
 } // namespace caf::cuda

From 729e18839ae8fcc3ed1287f17640d5d9c779ea3f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 31 Jan 2026 14:08:02 -0600
Subject: [PATCH 0272/1000] Added actor that does not use the scheduler at all.

---
 .../core_usage_behavior_tests/main.test.cpp   | 69 ++++++++++++++++---
 1 file changed, 61 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index b288142702..3ba2f01bf7 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -299,11 +299,6 @@ caf::behavior mmul_actor_fun_no_verify(
 				caf::cuda::manager& mgr = caf::cuda::manager::get();
 
 				//create program and dims   
-				auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-				const int THREADS = 32;
-				const int BLOCKS = (N + THREADS - 1) / THREADS;
-				caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
 				//create args
 				auto arg1 = caf::cuda::create_in_arg(matrixA);
 				auto arg2 = caf::cuda::create_in_arg(matrixB);
@@ -326,6 +321,64 @@ caf::behavior mmul_actor_fun_no_verify(
 
 
 
+// Stateful actor behavior
+// this actor does not invoke the scheduler at all 
+caf::behavior mmul_actor_fun_no_schedule(
+    caf::stateful_actor<mmul_actor_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims) {
+
+    self->state().N = N;
+
+    std::vector<int> matrix1(N * N);
+    std::vector<int> matrix2(N * N);
+
+    // send initial mail to self
+    self->mail(matrix1, matrix2, N).send(self);
+
+    return {
+        // GPU atom + matrices + N
+        [=](const std::vector<int>& matrixA,
+            const std::vector<int>& matrixB,
+            int N_local) {   // avoid shadowing outer N
+            caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+            auto arg1 = caf::cuda::create_in_arg(matrixA);
+            auto arg2 = caf::cuda::create_in_arg(matrixB);
+            auto arg3 = caf::cuda::create_out_arg(N_local * N_local);
+            auto arg4 = caf::cuda::create_in_arg(N_local);
+
+            auto tempC = mmul.run(program, dims, self->state().id, arg1, arg2, arg3, arg4);
+            std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+            self->quit();
+        },
+
+        // CPU verification
+        [=](const std::vector<int>& matrixA,
+            const std::vector<int>& matrixB,
+            const std::vector<int>& matrixC,
+            int N_local) {
+
+            std::vector<int> result(N_local * N_local);
+            serial_matrix_multiply(matrixA, matrixB, result, N_local);
+
+            if (result == matrixC) {
+                std::cout << "actor with id " << self->state().id << " references match\n";
+            } else {
+                std::cout << "actor with id " << self->state().id << " references did not match\n";
+            }
+
+            self->quit();
+        }
+    };
+}
+
+
+
+
 
 
 
@@ -506,9 +559,9 @@ void run_mmul_scaling_tests(caf::actor_system& sys,
 void caf_main(caf::actor_system& sys) {
   
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
-//	caf::cuda::manager::init(sys,man_config);
-  //     	run_mmul_test(sys,512,10);	
-	run_mmul_scaling_tests(sys,man_config);
+	caf::cuda::manager::init(sys,man_config);
+       //	run_mmul_test(sys,256,1000);	
+//	run_mmul_scaling_tests(sys,man_config);
 	//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	
 	//caf::cuda::manager::init(sys,man_config);

From 8a52603ef176e7e312e6013f576b3c1c183acb9c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 31 Jan 2026 14:18:47 -0600
Subject: [PATCH 0273/1000] Fixed deadlock in no scheduler actor.

---
 .../core_usage_behavior_tests/main.test.cpp   | 174 ++++++++++++------
 1 file changed, 114 insertions(+), 60 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index 3ba2f01bf7..95fa90b995 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -343,7 +343,10 @@ caf::behavior mmul_actor_fun_no_schedule(
         [=](const std::vector<int>& matrixA,
             const std::vector<int>& matrixB,
             int N_local) {   // avoid shadowing outer N
-            caf::cuda::manager& mgr = caf::cuda::manager::get();
+            
+	    
+    	    std::cout << "Hello\n";
+	    caf::cuda::manager& mgr = caf::cuda::manager::get();
 
             auto arg1 = caf::cuda::create_in_arg(matrixA);
             auto arg2 = caf::cuda::create_in_arg(matrixB);
@@ -353,6 +356,7 @@ caf::behavior mmul_actor_fun_no_schedule(
             auto tempC = mmul.run(program, dims, self->state().id, arg1, arg2, arg3, arg4);
             std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
 
+	    self->mail(1).send(exit_actor);
             self->quit();
         },
 
@@ -484,6 +488,37 @@ sys.spawn(
   sys.await_all_actors_done();
 }
 
+void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, int num_actors) {
+    if (num_actors < 1) {
+        std::cerr << "[ERROR] Number of actors must be >= 1\n";
+        return;
+    }
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    // CREATE ONCE
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    const int THREADS = 32;
+    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    for (int i = 0; i < num_actors; ++i) {
+        sys.spawn(
+            mmul_actor_fun_no_schedule,
+            exit_actor,
+            matrix_size,
+            program,
+            dims
+        );
+    }
+
+    sys.await_all_actors_done();
+}
+
+
 template <class Fn>
 double time_run(Fn&& fn) {
   auto start = std::chrono::steady_clock::now();
@@ -492,76 +527,95 @@ double time_run(Fn&& fn) {
   std::chrono::duration<double> elapsed = end - start;
   return elapsed.count();
 }
-
 void run_mmul_scaling_tests(caf::actor_system& sys,
                             caf::cuda::manager_config man_config) {
-  const int max_size   = 1024;
-  const int min_actors = 1;
-  const int max_actors = 1024;
-
-  // Matrix sizes: 10, 32, 64, 128, ..., 1024
-  std::vector<int> matrix_sizes = {10};
-  for (int s = 32; s <= max_size; s *= 2)
-    matrix_sizes.push_back(s);
-
-  // Actor counts: 1, 2, 4, 8, ..., 1024
-  std::vector<int> actor_counts;
-  for (int a = min_actors; a <= max_actors; a *= 2)
-    actor_counts.push_back(a);
-
-  std::cout << "=== MMUL Scaling Tests ===\n";
-  std::cout << "Format:\n";
-  std::cout << "scheduler matrix_size actors time_seconds\n";
-
-  for (int size : matrix_sizes) {
-    for (int actors : actor_counts) {
-
-      /* ================= Scheduler enabled ================= */
-      caf::cuda::manager::init(sys, man_config);
-
-      std::cout << "\n[RUN] scheduler=core_usage "
-                << "matrix_size=" << size
-                << " actors=" << actors << "\n";
-
-      double sched_time = time_run([&] {
-        run_mmul_test(sys, size, actors);
-      });
-
-      std::cout << std::fixed << std::setprecision(6)
-                << "RESULT core_usage "
-                << size << " "
-                << actors << " "
-                << sched_time << "\n";
-
-      /* ================= No scheduler ================= */
-      caf::cuda::manager::init(sys, man_config);
-
-      std::cout << "\n[RUN] scheduler=none "
-                << "matrix_size=" << size
-                << " actors=" << actors << "\n";
-
-      double no_sched_time = time_run([&] {
-        run_mmul_test_no_scheduler(sys, size, actors);
-      });
-
-      std::cout << std::fixed << std::setprecision(6)
-                << "RESULT none "
-                << size << " "
-                << actors << " "
-                << no_sched_time << "\n";
+    const int max_size   = 1024;
+    const int min_actors = 1;
+    const int max_actors = 1024;
+
+    std::vector<int> matrix_sizes = {10};
+    for (int s = 32; s <= max_size; s *= 2)
+        matrix_sizes.push_back(s);
+
+    std::vector<int> actor_counts;
+    for (int a = min_actors; a <= max_actors; a *= 2)
+        actor_counts.push_back(a);
+
+    std::cout << "=== MMUL Scaling Tests ===\n";
+    std::cout << "Format:\n";
+    std::cout << "scheduler matrix_size actors time_seconds\n";
+
+    for (int size : matrix_sizes) {
+        for (int actors : actor_counts) {
+
+            /* ================= Scheduler-enabled (core_usage) ================= */
+            caf::cuda::manager::init(sys, man_config); // green-light scheduler enabled
+            std::cout << "\n[RUN] scheduler=core_usage "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double core_usage_time = time_run([&] {
+                run_mmul_test(sys, size, actors); // uses mmul_actor_fun_no_verify
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT core_usage "
+                      << size << " "
+                      << actors << " "
+                      << core_usage_time << "\n";
+
+            caf::cuda::manager::shutdown(); // make sure manager is cleaned up
+
+            /* ================= Scheduler-disabled actor (still uses green-light) ================= */
+            caf::cuda::manager::init(sys, man_config); // init with scheduler
+            std::cout << "\n[RUN] scheduler=green_light_only "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double green_light_time = time_run([&] {
+                run_mmul_test_no_scheduler(sys, size, actors); // your previous "no scheduler" actor
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT green_light_only "
+                      << size << " "
+                      << actors << " "
+                      << green_light_time << "\n";
+
+            caf::cuda::manager::shutdown();
+
+            /* ================= No scheduler at all actor ================= */
+            caf::cuda::manager_config no_sched_config(false); // disable scheduler
+            caf::cuda::manager::init(sys, no_sched_config);
+            std::cout << "\n[RUN] scheduler=none "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double no_scheduler_time = time_run([&] {
+                run_mmul_test_no_scheduler_actor(sys, size, actors); // mmul_actor_fun_no_schedule
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT none "
+                      << size << " "
+                      << actors << " "
+                      << no_scheduler_time << "\n";
+
+            caf::cuda::manager::shutdown();
+        }
     }
-  }
 
-  std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
+    std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
 }
 
 
+
 void caf_main(caf::actor_system& sys) {
   
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
-	caf::cuda::manager::init(sys,man_config);
+//	caf::cuda::manager::init(sys,man_config);
        //	run_mmul_test(sys,256,1000);	
-//	run_mmul_scaling_tests(sys,man_config);
+	run_mmul_scaling_tests(sys,man_config);
 	//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	
 	//caf::cuda::manager::init(sys,man_config);

From 7612e46c17d38dac29c2d9277bf82bf9ae07aba6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 1 Feb 2026 10:37:35 -0600
Subject: [PATCH 0274/1000] Added batch testing.

---
 .../core_usage_behavior_tests/main.test.cpp   | 132 +++++++++++++++++-
 1 file changed, 129 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index 95fa90b995..be392fef29 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -607,16 +607,142 @@ void run_mmul_scaling_tests(caf::actor_system& sys,
 
     std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
 }
+void run_mmul_mixed_batch_one_mode(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    bool use_scheduler_actor,
+    bool use_core_usage_behavior,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get(); // SAFE NOW
+
+    if (use_scheduler_actor && use_core_usage_behavior) {
+        anon_mail(caf::cuda::make_behavior_token("core_usage"))
+            .send(mgr.get_scheduler_actor());
+    }
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    std::mt19937 rng(123456);
+    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+
+    const int THREADS = 32;
+
+    for (int i = 0; i < num_actors; ++i) {
+        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
+        int BLOCKS = (N + THREADS - 1) / THREADS;
+
+        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+        if (use_scheduler_actor) {
+            sys.spawn(
+                mmul_actor_fun_no_verify,
+                exit_actor,
+                N,
+                program,
+                dims);
+        } else {
+            sys.spawn(
+                mmul_actor_fun_no_schedule,
+                exit_actor,
+                N,
+                program,
+                dims);
+        }
+    }
+
+    sys.await_all_actors_done();
+}
+
+
+
+
+void run_mmul_mixed_batch_comparison(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors)
+{
+    std::cout << "\n=== MMUL Mixed-Size Batch Comparison ===\n";
+    std::cout << "scheduler actors sizes time_seconds\n\n";
+
+    /* ================= core_usage ================= */
+    {
+        caf::cuda::manager_config cfg(true);
+        caf::cuda::manager::init(sys, cfg);
+
+        double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode(
+                sys, sizes, num_actors,
+                /*use_scheduler_actor=*/true,
+                /*use_core_usage_behavior=*/true);
+        });
+
+        std::cout << "RESULT core_usage "
+                  << num_actors << " "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    /* ================= green-light only ================= */
+    {
+        caf::cuda::manager_config cfg(true);
+        caf::cuda::manager::init(sys, cfg);
+
+        double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode(
+                sys, sizes, num_actors,
+                /*use_scheduler_actor=*/true,
+                /*use_core_usage_behavior=*/false);
+        });
+
+        std::cout << "RESULT green_light_only "
+                  << num_actors << " "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    /* ================= no scheduler ================= */
+    {
+        caf::cuda::manager_config cfg(false);
+        caf::cuda::manager::init(sys, cfg);
+
+        double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode(
+                sys, sizes, num_actors,
+                /*use_scheduler_actor=*/false,
+                /*use_core_usage_behavior=*/false);
+        });
+
+        std::cout << "RESULT none "
+                  << num_actors << " "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    std::cout << "\n=== Comparison Complete ===\n";
+}
 
 
 
 void caf_main(caf::actor_system& sys) {
   
-	caf::cuda::manager_config man_config(true); //turns the scheduler on
+//	caf::cuda::manager_config man_config(true); //turns the scheduler on
 //	caf::cuda::manager::init(sys,man_config);
        //	run_mmul_test(sys,256,1000);	
-	run_mmul_scaling_tests(sys,man_config);
-	//tests will delete the old manager so will have to reinit if you do this 
+//	run_mmul_scaling_tests(sys,man_config);
+
+   std::vector<int> sizes = {32, 64, 128, 256, 512, 1024};
+    const int num_actors = 200;
+ // Option A: round-robin distribution (deterministic)
+    run_mmul_mixed_batch_comparison(sys, sizes, num_actors);    
+
+//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	
 	//caf::cuda::manager::init(sys,man_config);
 }

From 9932a6af954950645464e70d3e202d9e7a6db9bd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 1 Feb 2026 11:39:41 -0600
Subject: [PATCH 0275/1000] Added batch tests for core usage scheduler.

---
 .../core_usage_behavior_tests/main.test.cpp   | 94 +++++++++++++++++++
 1 file changed, 94 insertions(+)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index be392fef29..fdd3d3dfd3 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -730,6 +730,100 @@ void run_mmul_mixed_batch_comparison(
 
 
 
+void test_core_usage_uniform_mmul(
+    caf::actor_system& sys,
+    int matrix_size,
+    int num_actors)
+{
+    std::cout << "\n[TEST] core_usage uniform matrix size\n";
+    std::cout << "N=" << matrix_size
+              << " actors=" << num_actors << "\n";
+
+    caf::cuda::manager_config cfg(true);
+    caf::cuda::manager::init(sys, cfg);
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    // force core_usage behavior
+    anon_mail(caf::cuda::make_behavior_token("core_usage"))
+        .send(mgr.get_scheduler_actor());
+
+    auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    const int THREADS = 32;
+    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(
+        BLOCKS, BLOCKS, 1,
+        THREADS, THREADS, 1);
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    for (int i = 0; i < num_actors; ++i) {
+        sys.spawn(
+            mmul_actor_fun_no_verify,
+            exit_actor,
+            matrix_size,
+            program,
+            dims);
+    }
+
+    sys.await_all_actors_done();
+    caf::cuda::manager::shutdown();
+
+    std::cout << "[PASS] core_usage uniform test complete\n";
+}
+
+
+
+void test_core_usage_mixed_mmul(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors)
+{
+    std::cout << "\n[TEST] core_usage mixed matrix sizes\n";
+    std::cout << "actors=" << num_actors << "\n";
+
+    caf::cuda::manager_config cfg(true);
+    caf::cuda::manager::init(sys, cfg);
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    anon_mail(caf::cuda::make_behavior_token("core_usage"))
+        .send(mgr.get_scheduler_actor());
+
+    auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    const int THREADS = 32;
+
+    for (int i = 0; i < num_actors; ++i) {
+        int N = sizes[i % sizes.size()];
+
+        int BLOCKS = (N + THREADS - 1) / THREADS;
+        caf::cuda::nd_range dims(
+            BLOCKS, BLOCKS, 1,
+            THREADS, THREADS, 1);
+
+        sys.spawn(
+            mmul_actor_fun_no_verify,
+            exit_actor,
+            N,
+            program,
+            dims);
+    }
+
+    sys.await_all_actors_done();
+    caf::cuda::manager::shutdown();
+
+    std::cout << "[PASS] core_usage mixed-size test complete\n";
+}
+
+
+
+
 void caf_main(caf::actor_system& sys) {
   
 //	caf::cuda::manager_config man_config(true); //turns the scheduler on

From 3a13402d390524b7c1da6d10d40abcc0fe1b6aa6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 1 Feb 2026 12:39:37 -0600
Subject: [PATCH 0276/1000] Updated references to best graphs to fix a memory
 error where it would reference a garabge kernel graph.

---
 .../control-layer/core_usage_behavior.hpp     |  4 +-
 .../caf/cuda/control-layer/kernel_graph.hpp   | 19 ++++
 .../src/control-layer/core_usage_behavior.cpp | 96 +++++++++++--------
 3 files changed, 75 insertions(+), 44 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index 6c86ba9d36..68598606a6 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -43,7 +43,7 @@ class core_usage_behavior : public scheduler_actor_behavior {
    //data structures to manage dependencies
    std::unordered_map<int,kernel_graph> graphs;
    std::vector<kernel_graph> independent_graphs;
-   std::vector<kernel_graph*> best_graphs; //should contain top 5-10 best selections ideally or something along the lines
+   std::vector<graph_ref> best_graphs; //should contain top 5-10 best selections ideally or something along the lines
 
    void init_state();
    void create_new_graph(const token_ptr& token); //this should either add to indepedent or graphs data structure
@@ -53,7 +53,7 @@ class core_usage_behavior : public scheduler_actor_behavior {
 
    int get_next_stream(); // this should return the next stream based on some decisions
    void dummy_schedule(); //just a test scheduler algorthim do not use seriously
-
+   kernel_graph* resolve(const graph_ref& ref);
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
index b1ffa6ac67..f272f59b65 100644
--- a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
@@ -71,5 +71,24 @@ class kernel_graph {
     std::vector<token_ptr> operations;
 };
 
+
+
+struct graph_ref {
+    enum class kind_t {
+        dependent,
+        independent
+    };
+
+    kind_t kind;
+
+    // Only valid if kind == dependent
+    int dependency = -1;
+
+    // Only valid if kind == independent
+    std::size_t index = 0;
+};
+
+
+
 } // namespace caf::cuda
 
diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 00a686fc81..f5da9cead7 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -132,95 +132,107 @@ void core_usage_behavior::rank(std::size_t max_best = 5) {
     //CUT OUT EMPTY GRAPHS
     struct candidate {
         int cost;
-        kernel_graph* graph;
+        graph_ref ref;
     };
-
     std::vector<candidate> candidates;
 
-    // Step 1: gather all candidate graphs from dependent graphs
+    // Dependent graphs
     for (auto& [dep, graph] : graphs) {
-        if (graph.empty())
-            continue;
+        if (graph.empty()) continue;
 
-        token_ptr tok = graph.peek();  // non-destructive
-        if (!tok || tok->getType() != LAUNCH)
-            continue;
+        token_ptr tok = graph.peek();
+        if (!tok || tok->getType() != LAUNCH) continue;
 
         int cost = heuristic->getCost(tok);
-        if (cost == ERROR_CODE)
-            continue;
+        if (cost == ERROR_CODE) continue;
 
-        candidates.push_back({cost, &graph});
+        candidates.push_back({cost, graph_ref{graph_ref::kind_t::dependent, dep}});
     }
 
-    // Step 1b: gather all candidate graphs from independent_graphs
-    for (auto& graph : independent_graphs) {
-        if (graph.empty())
-            continue;
+    // Independent graphs
+    for (std::size_t i = 0; i < independent_graphs.size(); ++i) {
+        auto& graph = independent_graphs[i];
+        if (graph.empty()) continue;
 
         token_ptr tok = graph.peek();
-        if (!tok || tok->getType() != LAUNCH)
-            continue;
+        if (!tok || tok->getType() != LAUNCH) continue;
 
         int cost = heuristic->getCost(tok);
-        if (cost == ERROR_CODE)
-            continue;
+        if (cost == ERROR_CODE) continue;
 
-        candidates.push_back({cost, &graph});
+        candidates.push_back({cost, graph_ref{graph_ref::kind_t::independent, -1, i}});
     }
 
-    if (candidates.empty())
-        return;
+    if (candidates.empty()) return;
 
-    // Step 2: sort by ascending cost (cheap kernels first)
+    // Sort by ascending cost
     std::sort(candidates.begin(), candidates.end(),
               [](const candidate& a, const candidate& b) {
                   return a.cost < b.cost;
               });
 
-    // Step 3: keep top N candidates
-    const std::size_t limit = std::min(max_best, candidates.size());
+    // Keep top N
+    const auto limit = std::min(max_best, candidates.size());
     for (std::size_t i = 0; i < limit; ++i) {
-        best_graphs.push_back(candidates[i].graph);
+        best_graphs.push_back(candidates[i].ref);
     }
 }
 
 
+kernel_graph* core_usage_behavior::resolve(const graph_ref& ref) {
+    switch (ref.kind) {
+        case graph_ref::kind_t::dependent: {
+            auto it = graphs.find(ref.dependency);
+            if (it == graphs.end()) return nullptr;
+            return &it->second;
+        }
+        case graph_ref::kind_t::independent: {
+            if (ref.index >= independent_graphs.size()) return nullptr;
+            return &independent_graphs[ref.index];
+        }
+    }
+    return nullptr;
+}
+
+
+
 
 void core_usage_behavior::schedule() {
-    // If there are only 2 kernels to consider then rerank
-    if (best_graphs.size() <= 2) {
-        rank(5);
+    // Re-rank if we have no best graphs
+    if (best_graphs.empty()) {
+        rank(); // uses default max_best
         if (best_graphs.empty()) {
-            return;
+            return; // nothing to schedule
         }
     }
 
-    // Greedy: largest-cost first
+    // iterate backwards so we can safely erase elements
     for (int i = static_cast<int>(best_graphs.size()) - 1; i >= 0; --i) {
-        kernel_graph* graph = best_graphs[i];
-        if (!graph || graph->empty()) 
+        kernel_graph* graph = resolve(best_graphs[i]);
+        if (!graph || graph->empty()) {
+            best_graphs.erase(best_graphs.begin() + i);
             continue;
+        }
 
         token_ptr tok = graph->peek();
-        if (!tok || tok->getType() != LAUNCH) 
-            continue;
+        if (!tok || tok->getType() != LAUNCH) {
+            continue; // leave the graph in best_graphs for next round
+        }
 
         int cost = heuristic->getCost(tok);
-        if (cost == ERROR_CODE) 
-            continue;
+        if (cost == ERROR_CODE) continue;
 
         if (available_SM >= cost) {
             tok = graph->getOperation();
             process_launch_token(tok, graph->stream_id());
+
+            // **remove graph immediately after picking it**
             best_graphs.erase(best_graphs.begin() + i);
         }
     }
-
-    // Re-rank if we have less than 2 operations left
-    if (best_graphs.size() <= 2) {
-        rank(5);
-    }
 }
 
+
+
+
 } // namespace caf::cuda

From 0a3fa89fc7c283d28f35b150b5e0d426c4718ffa Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Feb 2026 09:55:39 -0600
Subject: [PATCH 0277/1000] Added a uniform batch size comparsion test.

---
 .../core_usage_behavior_tests/main.test.cpp   | 96 ++++++++++++++++---
 1 file changed, 85 insertions(+), 11 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index fdd3d3dfd3..cd500b3c6b 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -34,7 +34,7 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int lim
 		[=](int num_completed) {
 			self->state().completed += num_completed;
 			
-			std::cout << "Actors finished is " << self->state().completed << "\n";
+			//std::cout << "Actors finished is " << self->state().completed << "\n";
 			if (self->state().completed >= limit) {
 			
 				caf::cuda::manager::shutdown();
@@ -345,7 +345,7 @@ caf::behavior mmul_actor_fun_no_schedule(
             int N_local) {   // avoid shadowing outer N
             
 	    
-    	    std::cout << "Hello\n";
+    	    //std::cout << "Hello\n";
 	    caf::cuda::manager& mgr = caf::cuda::manager::get();
 
             auto arg1 = caf::cuda::create_in_arg(matrixA);
@@ -567,7 +567,9 @@ void run_mmul_scaling_tests(caf::actor_system& sys,
             caf::cuda::manager::shutdown(); // make sure manager is cleaned up
 
             /* ================= Scheduler-disabled actor (still uses green-light) ================= */
-            caf::cuda::manager::init(sys, man_config); // init with scheduler
+
+/*
+	    caf::cuda::manager::init(sys, man_config); // init with scheduler
             std::cout << "\n[RUN] scheduler=green_light_only "
                       << "matrix_size=" << size
                       << " actors=" << actors << "\n";
@@ -584,6 +586,7 @@ void run_mmul_scaling_tests(caf::actor_system& sys,
 
             caf::cuda::manager::shutdown();
 
+	    */
             /* ================= No scheduler at all actor ================= */
             caf::cuda::manager_config no_sched_config(false); // disable scheduler
             caf::cuda::manager::init(sys, no_sched_config);
@@ -688,15 +691,18 @@ void run_mmul_mixed_batch_comparison(
     }
 
     /* ================= green-light only ================= */
+   
+    /*
     {
+	    std::cout << "Starting green_light tests\n";
         caf::cuda::manager_config cfg(true);
         caf::cuda::manager::init(sys, cfg);
 
         double t = time_run([&] {
             run_mmul_mixed_batch_one_mode(
                 sys, sizes, num_actors,
-                /*use_scheduler_actor=*/true,
-                /*use_core_usage_behavior=*/false);
+               true,
+                false);
         });
 
         std::cout << "RESULT green_light_only "
@@ -706,6 +712,8 @@ void run_mmul_mixed_batch_comparison(
         caf::cuda::manager::shutdown();
     }
 
+    */
+	
     /* ================= no scheduler ================= */
     {
         caf::cuda::manager_config cfg(false);
@@ -822,19 +830,85 @@ void test_core_usage_mixed_mmul(
 }
 
 
+void run_mmul_fixed_256_batch_comparison(
+    caf::actor_system& sys,
+    int num_actors)
+{
+    // All actors run the same matrix size: 256
+    std::vector<int> sizes(num_actors, 256);
+
+    std::cout << "\n=== MMUL Fixed-Size (256) Batch Comparison ===\n";
+    std::cout << "scheduler actors size time_seconds\n\n";
+
+    /* ================= core_usage ================= */
+    {
+        caf::cuda::manager_config cfg(true);
+        caf::cuda::manager::init(sys, cfg);
+
+        double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode(
+                sys,
+                sizes,
+                num_actors,
+                /*use_scheduler_actor=*/true,
+                /*use_core_usage_behavior=*/true);
+        });
+
+        std::cout << "RESULT core_usage "
+                  << num_actors << " 256 "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    /* ================= no scheduler ================= */
+    {
+        caf::cuda::manager_config cfg(false);
+        caf::cuda::manager::init(sys, cfg);
+
+        double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode(
+                sys,
+                sizes,
+                num_actors,
+                /*use_scheduler_actor=*/false,
+                /*use_core_usage_behavior=*/false);
+        });
+
+        std::cout << "RESULT none "
+                  << num_actors << " 256 "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    std::cout << "\n=== Fixed-256 Comparison Complete ===\n";
+}
+
+
+
 
 
 void caf_main(caf::actor_system& sys) {
   
-//	caf::cuda::manager_config man_config(true); //turns the scheduler on
-//	caf::cuda::manager::init(sys,man_config);
-       //	run_mmul_test(sys,256,1000);	
-//	run_mmul_scaling_tests(sys,man_config);
+	//caf::cuda::manager_config man_config(true); //turns the scheduler on
+	//caf::cuda::manager::init(sys,man_config);
+        // run_mmul_test(sys,10,64);	
+	//run_mmul_scaling_tests(sys,man_config);
 
    std::vector<int> sizes = {32, 64, 128, 256, 512, 1024};
     const int num_actors = 200;
- // Option A: round-robin distribution (deterministic)
-    run_mmul_mixed_batch_comparison(sys, sizes, num_actors);    
+    //run_mmul_mixed_batch_comparison(sys, sizes, num_actors);    
+
+    run_mmul_fixed_256_batch_comparison(sys, /*num_actors=*/200);
+
+
+	 //test_core_usage_uniform_mmul(sys, 256, 1000);
+
+  //std::vector<int> sizes = {32, 64, 128, 256, 512, 1024};
+    //test_core_usage_mixed_mmul(sys, sizes, 200);
+
+
 
 //tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	

From 51ea15889a18b6266602fa43430bd6b8c33dda53 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Feb 2026 10:01:10 -0600
Subject: [PATCH 0278/1000] Added a profiler class to help with profiling.'

---
 .../scheduler-functions/profiler.hpp          | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/scheduler-functions/profiler.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/profiler.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/profiler.hpp
new file mode 100644
index 0000000000..f44d160d34
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/profiler.hpp
@@ -0,0 +1,24 @@
+#include <chrono>
+#include <iostream>
+
+
+namespace caf::cuda {
+struct scoped_timer {
+    const char* name;
+    std::chrono::steady_clock::time_point start;
+
+    explicit scoped_timer(const char* n)
+        : name(n), start(std::chrono::steady_clock::now()) {}
+
+    ~scoped_timer() {
+        auto end = std::chrono::steady_clock::now();
+        auto us =
+            std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
+
+        std::cout << "[PROFILE] " << name << " took "
+                  << us << " us\n";
+    }
+};
+
+} //namespace caf::cuda
+

From fddc96018d60852bbf73d311a67fbd0e1bdbda5c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Feb 2026 10:02:32 -0600
Subject: [PATCH 0279/1000] Includeed profiler.hpp.

---
 libcaf_cuda/src/control-layer/core_usage_behavior.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index f5da9cead7..246f6abaf1 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -1,5 +1,6 @@
 #include "caf/cuda/control-layer/all-control-layer.hpp"
 #include "caf/cuda/control-layer/core_usage_behavior.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/profiler.hpp"
 #include "caf/cuda/manager.hpp"
 #include "caf/cuda/device.hpp"
 

From f36f99d2326c8bcd3b6d5e48c6a83bdaeff61e22 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Feb 2026 10:38:21 -0600
Subject: [PATCH 0280/1000] Added profiling on methods as well as upgraded the
 total sms to be 16 times what it originally was to reduce overhead incurred
 by scheduling.

---
 .../src/control-layer/core_usage_behavior.cpp | 73 +++++++++++--------
 1 file changed, 43 insertions(+), 30 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 246f6abaf1..acc950308f 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -18,7 +18,7 @@ core_usage_behavior::~core_usage_behavior(){
 void core_usage_behavior::init_state() {
     device_ = manager::get().find_device(state_.device_number);
     heuristic.emplace(device_);
-    total_SM = device_->num_sms();
+    total_SM = device_->num_sms() * 16;
     available_SM = total_SM;
     available_memory = static_cast<int>(device_->total_memory_bytes());
     num_streams = state_.num_streams;
@@ -126,11 +126,11 @@ void core_usage_behavior::dummy_schedule() {
     }
 }
 
-void core_usage_behavior::rank(std::size_t max_best = 5) {
+void core_usage_behavior::rank(std::size_t max_best=5) {
+    scoped_timer timer("core_usage_behavior::rank");
+
     best_graphs.clear();
 
-    //TODO FIGURE OUT A WAY TO PUT STATUS IN HERE AND 
-    //CUT OUT EMPTY GRAPHS
     struct candidate {
         int cost;
         graph_ref ref;
@@ -147,7 +147,8 @@ void core_usage_behavior::rank(std::size_t max_best = 5) {
         int cost = heuristic->getCost(tok);
         if (cost == ERROR_CODE) continue;
 
-        candidates.push_back({cost, graph_ref{graph_ref::kind_t::dependent, dep}});
+        candidates.push_back({cost,
+            graph_ref{graph_ref::kind_t::dependent, dep}});
     }
 
     // Independent graphs
@@ -161,18 +162,17 @@ void core_usage_behavior::rank(std::size_t max_best = 5) {
         int cost = heuristic->getCost(tok);
         if (cost == ERROR_CODE) continue;
 
-        candidates.push_back({cost, graph_ref{graph_ref::kind_t::independent, -1, i}});
+        candidates.push_back({cost,
+            graph_ref{graph_ref::kind_t::independent, -1, i}});
     }
 
     if (candidates.empty()) return;
 
-    // Sort by ascending cost
     std::sort(candidates.begin(), candidates.end(),
               [](const candidate& a, const candidate& b) {
                   return a.cost < b.cost;
               });
 
-    // Keep top N
     const auto limit = std::min(max_best, candidates.size());
     for (std::size_t i = 0; i < limit; ++i) {
         best_graphs.push_back(candidates[i].ref);
@@ -180,7 +180,10 @@ void core_usage_behavior::rank(std::size_t max_best = 5) {
 }
 
 
+
 kernel_graph* core_usage_behavior::resolve(const graph_ref& ref) {
+    //scoped_timer timer("core_usage_behavior::resolve");
+
     switch (ref.kind) {
         case graph_ref::kind_t::dependent: {
             auto it = graphs.find(ref.dependency);
@@ -198,42 +201,52 @@ kernel_graph* core_usage_behavior::resolve(const graph_ref& ref) {
 
 
 
+
 void core_usage_behavior::schedule() {
-    // Re-rank if we have no best graphs
+    scoped_timer timer("core_usage_behavior::schedule");
+
     if (best_graphs.empty()) {
-        rank(); // uses default max_best
+        rank(20); // profiled independently
         if (best_graphs.empty()) {
-            return; // nothing to schedule
+            return;
         }
     }
 
-    // iterate backwards so we can safely erase elements
-    for (int i = static_cast<int>(best_graphs.size()) - 1; i >= 0; --i) {
-        kernel_graph* graph = resolve(best_graphs[i]);
-        if (!graph || graph->empty()) {
-            best_graphs.erase(best_graphs.begin() + i);
-            continue;
-        }
+   for (std::size_t i = 0; i < best_graphs.size(); ) {
+    kernel_graph* graph = resolve(best_graphs[i]);
+    if (!graph || graph->empty()) {
+        best_graphs.erase(best_graphs.begin() + i);
+        continue; // stay at same index
+    }
 
-        token_ptr tok = graph->peek();
-        if (!tok || tok->getType() != LAUNCH) {
-            continue; // leave the graph in best_graphs for next round
-        }
+    token_ptr tok = graph->peek();
+    if (!tok || tok->getType() != LAUNCH) {
+        ++i;
+        continue;
+    }
 
-        int cost = heuristic->getCost(tok);
-        if (cost == ERROR_CODE) continue;
+    int cost = heuristic->getCost(tok);
+    if (cost == ERROR_CODE) {
+        ++i;
+        continue;
+    }
 
-        if (available_SM >= cost) {
-            tok = graph->getOperation();
-            process_launch_token(tok, graph->stream_id());
+    if (available_SM >= cost) {
+        tok = graph->getOperation();
+        process_launch_token(tok, graph->stream_id());
 
-            // **remove graph immediately after picking it**
-            best_graphs.erase(best_graphs.begin() + i);
-        }
+        best_graphs.erase(best_graphs.begin() + i);
+        continue; // stay at same index
     }
+
+    ++i;
+
+   }
+
 }
 
 
 
 
+
 } // namespace caf::cuda

From 1c2db18f6d453964215c6f2c4ca1a899dd0d6d93 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Feb 2026 10:53:28 -0600
Subject: [PATCH 0281/1000] Added a pragma once declaration.

---
 .../caf/cuda/control-layer/scheduler-functions/profiler.hpp      | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/profiler.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/profiler.hpp
index f44d160d34..def0278f46 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/profiler.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/profiler.hpp
@@ -1,3 +1,4 @@
+#pragma once
 #include <chrono>
 #include <iostream>
 

From 2514a658e3b31eb0dd2db1f13cc71aa4f54dd639 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Feb 2026 11:01:10 -0600
Subject: [PATCH 0282/1000] Added a hashValue of program class and added a
 getter method. Change is being made since we want faster hashing than strings
 since concatinating strings is expensive and wasteful compared to using
 integers.

---
 libcaf_cuda/caf/cuda/program.hpp | 7 +++++++
 libcaf_cuda/src/program.cpp      | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/program.hpp b/libcaf_cuda/caf/cuda/program.hpp
index 8cbd220ea9..18f075533b 100644
--- a/libcaf_cuda/caf/cuda/program.hpp
+++ b/libcaf_cuda/caf/cuda/program.hpp
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <string>
+#include <functional>
 #include <vector>
 #include <unordered_map>
 #include <atomic>
@@ -41,6 +42,9 @@ class CAF_CUDA_EXPORT program : public caf::ref_counted {
 
     std::string getName() {return name_;}
 
+    int getHash() const {return hashValue;}
+
+
 private:
   /// Internal helper to load the kernel modules on all devices.
   void load_kernels(bool is_fatbin);
@@ -49,6 +53,9 @@ class CAF_CUDA_EXPORT program : public caf::ref_counted {
   std::vector<char> binary_;               ///< The binary or PTX of the program
   std::unordered_map<int, CUfunction> kernels_; ///< Device ID -> CUfunction mapping
   mutable std::atomic<size_t> ref_count_{0};
+  std::hash<std::string> hasher;
+  int hashValue = 0;
+
 };
 
 /// Alias for an intrusive pointer to a program
diff --git a/libcaf_cuda/src/program.cpp b/libcaf_cuda/src/program.cpp
index 5f18a62789..70382142c4 100644
--- a/libcaf_cuda/src/program.cpp
+++ b/libcaf_cuda/src/program.cpp
@@ -3,7 +3,7 @@
 namespace caf::cuda {
 
 program::program(std::string name, std::vector<char> binary, bool is_fatbin)
-    : name_(std::move(name)), binary_(std::move(binary)) {
+    : name_(std::move(name)), binary_(std::move(binary),hashValue(hasher(name_) {
   load_kernels(is_fatbin);
 }
 

From 07dfed7f25b8691cadbaebd1267503d5640b4e0d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Feb 2026 11:07:21 -0600
Subject: [PATCH 0283/1000] Fixed syntax errors.

---
 libcaf_cuda/src/program.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/program.cpp b/libcaf_cuda/src/program.cpp
index 70382142c4..817d8d0706 100644
--- a/libcaf_cuda/src/program.cpp
+++ b/libcaf_cuda/src/program.cpp
@@ -3,7 +3,7 @@
 namespace caf::cuda {
 
 program::program(std::string name, std::vector<char> binary, bool is_fatbin)
-    : name_(std::move(name)), binary_(std::move(binary),hashValue(hasher(name_) {
+    : name_(std::move(name)), binary_(std::move(binary)), hashValue(hasher(name_)) {
   load_kernels(is_fatbin);
 }
 

From 62bb4d4d09b6f4994936c56f929df9934d25a0a8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Feb 2026 11:15:24 -0600
Subject: [PATCH 0284/1000] Added a hashvalue to nd_range Change is being made
 to prevent expensive construction and string operations overhead.

---
 libcaf_cuda/caf/cuda/nd_range.hpp | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/nd_range.hpp b/libcaf_cuda/caf/cuda/nd_range.hpp
index ca7830ff48..0153d96477 100644
--- a/libcaf_cuda/caf/cuda/nd_range.hpp
+++ b/libcaf_cuda/caf/cuda/nd_range.hpp
@@ -4,6 +4,8 @@
 #include <cstddef>
 #include <stdexcept>
 #include <iostream>
+#include <string>
+#include <functional>
 
 namespace caf::cuda {
 
@@ -19,7 +21,11 @@ class nd_range {
                 static_cast<size_t>(gridZ)},
         blockDim{static_cast<size_t>(blockX),
                  static_cast<size_t>(blockY),
-                 static_cast<size_t>(blockZ)} {}
+                 static_cast<size_t>(blockZ)} {
+		 
+			 computeHash();
+		 
+		 }
 
 
   // Constructor from vectors
@@ -60,7 +66,10 @@ class nd_range {
  size_t get_num_blocks() const noexcept {
 	  return gridDim[0] * gridDim[1] * gridDim[2];
  }
-
+ // Returns the precomputed hash
+  [[nodiscard]] size_t getHash() const noexcept {
+    return hashValue_;
+  }
 
   ~nd_range() {
 	  //no-op
@@ -86,8 +95,20 @@ class nd_range {
 
 private:
   // Dimensions are stored in order of x, y, z
-  dim_vec gridDim{3};
-  dim_vec blockDim{3};
+  dim_vec gridDim{3,0};
+  dim_vec blockDim{3,0};
+  size_t hashValue_{0}; // store precomputed hash
+
+
+   // Precompute hash from to_string
+  void computeHash() {
+    std::hash<std::string> hasher;
+    hashValue_ = hasher(to_string());
+  }
+
+
+
+
 };
 
 } // namespace caf::cuda

From 579cf94535dea9f037aa96f2e481549470863b43 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Feb 2026 11:22:56 -0600
Subject: [PATCH 0285/1000] Updated hashmap to use ints as key and changed the
 getCost methods of children accordingly. This change is being made in an
 effort to reduce the overhead induced by calling the getCost method, as
 currently it is expensive due to constructing strings needlessly.

---
 .../core_heuristic_function.hpp               | 39 +++++++-------
 .../heuristic_function.hpp                    |  2 +-
 .../sm_usage_heuristic.hpp                    | 54 ++++++++++---------
 3 files changed, 50 insertions(+), 45 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
index 0cb082a3c5..b4535eb635 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
@@ -35,26 +35,27 @@ class core_heuristic_function : public heuristic_function {
 
 
   int getCost(const program_ptr& prog,
-              const nd_range& range) override {
-    try {
-      const std::string key =
-        prog->getName() + range.to_string();
-
-      auto it = values_.find(key);
-      if (it != values_.end())
-        return it->second;
-
-      int cost = dev_->max_active_blocks_per_sm(prog, range);
-      values_[key] = cost;
-      return cost;
-    }
-    catch (const std::exception&) {
-      return ERROR_CODE;
-    }
-    catch (...) {
-      return ERROR_CODE;
-    }
+            const nd_range& range) override {
+  try {
+    // Use integer hashes instead of concatenated strings
+    int key = prog->getHash() ^ static_cast<int>(range.getHash());
+
+    auto it = values_.find(key);
+    if (it != values_.end())
+      return it->second;
+
+    int cost = dev_->max_active_blocks_per_sm(prog, range);
+    values_[key] = cost;
+    return cost;
   }
+  catch (const std::exception&) {
+    return ERROR_CODE;
+  }
+  catch (...) {
+    return ERROR_CODE;
+  }
+}
+
 
 private:
   device_ptr dev_;
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
index 594b69b5cc..c2d684de20 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
@@ -46,7 +46,7 @@ class heuristic_function {
 
 protected:
   /// Heuristic-specific values
-  std::unordered_map<std::string, int> values_;
+  std::unordered_map<int, int> values_;
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
index 106faa3d97..adbe272786 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
@@ -5,6 +5,7 @@
 #include "caf/cuda/nd_range.hpp"
 #include "caf/cuda/program.hpp"
 #include "caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/profiler.hpp"
 
 /*
  * will return a value indicating 
@@ -18,38 +19,41 @@ class sm_usage_heuristic : public heuristic_function {
   explicit sm_usage_heuristic(device_ptr dev)
     : dev_(dev) {}
 
-  int getCost(const program_ptr& prog,
-              const nd_range& range) override {
-    try {
-      const std::string key =
-        prog->getName() + range.to_string();
+int getCost(const program_ptr& prog,
+            const nd_range& range) override {
 
-      auto it = values_.find(key);
-      if (it != values_.end())
-        return it->second;
+  scoped_timer timer("sm_usage_heuristic::getCost");
+  try {
+    int key = prog->getHash() ^ static_cast<int>(range.getHash());
 
-      int total_blocks =
-        static_cast<int>(range.get_num_blocks());
+    auto it = values_.find(key);
+    if (it != values_.end())
+      return it->second;
 
-      int blocks_per_sm =
-        dev_->max_active_blocks_per_sm(prog, range);
+    int total_blocks = static_cast<int>(range.get_num_blocks());
+    int blocks_per_sm = dev_->max_active_blocks_per_sm(prog, range);
 
-      if (blocks_per_sm <= 0)
-        return ERROR_CODE;
-
-      int sms_needed =
-        (total_blocks + blocks_per_sm - 1) / blocks_per_sm; // ceil
-
-      int sms_used =
-        std::min(dev_->num_sms(), sms_needed);
-
-      values_[key] = sms_used;
-      return sms_used;
-    }
-    catch (...) {
+    if (blocks_per_sm <= 0) {
+      std::cout << "blocks_per_sm = " << blocks_per_sm << "\n"; 
+      std::cout << "blocks is less than zero\n";
       return ERROR_CODE;
     }
+
+    int sms_needed = (total_blocks + blocks_per_sm - 1) / blocks_per_sm; // ceil
+    int sms_used = std::min(dev_->num_sms(), sms_needed);
+
+    values_[key] = sms_used;
+    return sms_used;
+  }
+  catch (const std::exception& e) {
+    std::cerr << "Caught std::exception: " << e.what() << "\n";
+    return ERROR_CODE;
+  }
+  catch (...) {
+    std::cerr << "Caught unknown exception!\n";
+    return ERROR_CODE;
   }
+}
 
   int getCost(const token_ptr& tok) override {
     return heuristic_function::getCost(tok);

From cf8ab7f2ce93212f04739600278fcc4dc43f692e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Feb 2026 11:24:05 -0600
Subject: [PATCH 0286/1000] Removed profiler.

---
 .../control-layer/scheduler-functions/sm_usage_heuristic.hpp     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
index adbe272786..6fb5112602 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
@@ -22,7 +22,6 @@ class sm_usage_heuristic : public heuristic_function {
 int getCost(const program_ptr& prog,
             const nd_range& range) override {
 
-  scoped_timer timer("sm_usage_heuristic::getCost");
   try {
     int key = prog->getHash() ^ static_cast<int>(range.getHash());
 

From 6ebb46d0e833d50c78595b5eb77f097e765a3452 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 3 Feb 2026 12:11:32 -0600
Subject: [PATCH 0287/1000] Added a bulk token transfer test.

---
 .../core_usage_behavior_tests/main.test.cpp   | 128 ++++++++++++++----
 1 file changed, 105 insertions(+), 23 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index cd500b3c6b..10b217fe51 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -241,7 +241,9 @@ caf::behavior mmul_actor_fun_no_verify(
     caf::actor exit_actor,
     int N,
     caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims)
+    caf::cuda::nd_range dims,
+    bool request
+    )
 {
 
 	//set the value of N correctly to overide the base option.	
@@ -251,16 +253,17 @@ caf::behavior mmul_actor_fun_no_verify(
 
 	caf::actor scheduler = mgr.get_scheduler_actor();
 
-	//send a launch token
-	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-			program,
-			dims,
-			0,
-			"hello",
-			self
-			);
-	self -> mail(launch_token).send(scheduler);
-
+	if (request) {
+		//send a launch token
+		caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+				program,
+				dims,
+				0,
+				"hello",
+				self
+				);
+		self -> mail(launch_token).send(scheduler);
+	}
 	return {
 
 		[=] (caf::cuda::response_token_ptr res_token) {
@@ -306,6 +309,10 @@ caf::behavior mmul_actor_fun_no_verify(
 				auto arg4 = caf::cuda::create_in_arg(N);
 
 				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+			
+			
+				//mask the transfer back to the cpu for scheduler
+				res_token -> release();
 				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
 
 				//std::cout << "GPU ACTOR done  computing\n";
@@ -428,7 +435,8 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
         exit_actor,
         matrix_size,
         program,
-        dims);
+        dims,
+	true);
     
   }
 
@@ -472,7 +480,8 @@ void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num
         exit_actor,
         matrix_size,
         program,
-        dims);
+        dims,
+	true);
     
   
 	/*  
@@ -646,7 +655,8 @@ void run_mmul_mixed_batch_one_mode(
                 exit_actor,
                 N,
                 program,
-                dims);
+                dims,
+		true);
         } else {
             sys.spawn(
                 mmul_actor_fun_no_schedule,
@@ -663,6 +673,66 @@ void run_mmul_mixed_batch_one_mode(
 
 
 
+void run_mmul_mixed_batch_one_mode_bulk(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get(); // SAFE NOW
+
+        anon_mail(caf::cuda::make_behavior_token("core_usage"))
+            .send(mgr.get_scheduler_actor());
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    std::vector<caf::cuda::token_ptr> tokens(num_actors);
+
+    std::mt19937 rng(123456);
+    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+
+    const int THREADS = 32;
+
+    for (int i = 0; i < num_actors; ++i) {
+        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
+        int BLOCKS = (N + THREADS - 1) / THREADS;
+
+	caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+	caf::actor a = sys.spawn(
+			mmul_actor_fun_no_verify,
+			exit_actor,
+			N,
+			program,
+			dims,
+			false);
+
+	tokens[i] = caf::cuda::make_launch_token(
+			program,
+			dims,
+			0 /*this should not be 0 but its fine for now*/,
+			"hello",
+			a);    
+    }
+
+    
+        anon_mail(tokens)
+            .send(mgr.get_scheduler_actor());
+
+
+    sys.await_all_actors_done();
+}
+
+
+
+
+
+
+
+
+
+
 void run_mmul_mixed_batch_comparison(
     caf::actor_system& sys,
     const std::vector<int>& sizes,
@@ -676,13 +746,22 @@ void run_mmul_mixed_batch_comparison(
         caf::cuda::manager_config cfg(true);
         caf::cuda::manager::init(sys, cfg);
 
-        double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode(
-                sys, sizes, num_actors,
-                /*use_scheduler_actor=*/true,
-                /*use_core_usage_behavior=*/true);
+	
+        //double t = time_run([&] {
+          //  run_mmul_mixed_batch_one_mode(
+            //    sys, sizes, num_actors,
+              //  /*use_scheduler_actor=*/true,
+               // /*use_core_usage_behavior=*/true);
+       // });
+
+	double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode_bulk(
+                sys, sizes, num_actors);
         });
 
+
+
+
         std::cout << "RESULT core_usage "
                   << num_actors << " "
                   << t << "\n";
@@ -773,7 +852,8 @@ void test_core_usage_uniform_mmul(
             exit_actor,
             matrix_size,
             program,
-            dims);
+            dims,
+	    false);
     }
 
     sys.await_all_actors_done();
@@ -820,7 +900,8 @@ void test_core_usage_mixed_mmul(
             exit_actor,
             N,
             program,
-            dims);
+            dims,
+	    false);
     }
 
     sys.await_all_actors_done();
@@ -898,9 +979,10 @@ void caf_main(caf::actor_system& sys) {
 
    std::vector<int> sizes = {32, 64, 128, 256, 512, 1024};
     const int num_actors = 200;
-    //run_mmul_mixed_batch_comparison(sys, sizes, num_actors);    
+    run_mmul_mixed_batch_comparison(sys, sizes, num_actors);    
 
-    run_mmul_fixed_256_batch_comparison(sys, /*num_actors=*/200);
+    run_mmul_mixed_batch_one_mode_bulk(sys,sizes,num_actors);
+    //run_mmul_fixed_256_batch_comparison(sys, /*num_actors=*/200);
 
 
 	 //test_core_usage_uniform_mmul(sys, 256, 1000);

From 320d18dc4b6e223ee9638fb58fac08476a1c1b56 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 3 Feb 2026 12:19:33 -0600
Subject: [PATCH 0288/1000] Updated no verify to not message itself, Change was
 made to reduce some overhead.

---
 .../core_usage_behavior_tests/main.test.cpp   | 28 +++++++++++++++++--
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index 10b217fe51..8047a7d93b 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -277,8 +277,30 @@ caf::behavior mmul_actor_fun_no_verify(
 				std::vector<int> matrix2(N*N);
 				matrix2.reserve(N);
 
+
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+				//create program and dims   
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrix1);
+				auto arg2 = caf::cuda::create_in_arg(matrix2);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
+
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+			
+			
+				//mask the transfer back to the cpu for scheduler
+				res_token -> release();
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				//std::cout << "GPU ACTOR done  computing\n";
+				// signal exit actor and quit
+				self->mail(1).send(exit_actor);
+				self->quit();
+
 				//std::cout << "GPU ACTOR sending data to compute\n";
-				self -> mail(matrix1,matrix2,res_token,N).send(self);
+			//	self -> mail(matrix1,matrix2,res_token,N).send(self);
 
 			}
 			else {
@@ -978,10 +1000,10 @@ void caf_main(caf::actor_system& sys) {
 	//run_mmul_scaling_tests(sys,man_config);
 
    std::vector<int> sizes = {32, 64, 128, 256, 512, 1024};
-    const int num_actors = 200;
+    const int num_actors = 1000;
     run_mmul_mixed_batch_comparison(sys, sizes, num_actors);    
 
-    run_mmul_mixed_batch_one_mode_bulk(sys,sizes,num_actors);
+    //run_mmul_mixed_batch_one_mode_bulk(sys,sizes,num_actors);
     //run_mmul_fixed_256_batch_comparison(sys, /*num_actors=*/200);
 
 

From ae5e97d3d19de483fc11c47ded867372215e86ed Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 3 Feb 2026 15:16:39 -0600
Subject: [PATCH 0289/1000] Implemented a single usage behavior that only
 allows one kernel at a time, This scheduler is mostly useful for testing
 certain things.

---
 .../control-layer/single_usage_behavior.hpp   |  54 +++++++
 .../control-layer/single_usage_behavior.cpp   | 147 ++++++++++++++++++
 2 files changed, 201 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/single_usage_behavior.hpp
 create mode 100644 libcaf_cuda/src/control-layer/single_usage_behavior.cpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/single_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/single_usage_behavior.hpp
new file mode 100644
index 0000000000..fa6ad36d80
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/single_usage_behavior.hpp
@@ -0,0 +1,54 @@
+#pragma once
+
+#include "caf/cuda/control-layer/behavior.hpp"
+#include "caf/cuda/control-layer/kernel_graph.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp"
+#include "caf/cuda/device.hpp"
+
+#include <unordered_map>
+#include <vector>
+#include <optional>
+
+namespace caf::cuda {
+
+class single_usage_behavior : public scheduler_actor_behavior {
+public:
+    explicit single_usage_behavior(scheduler_actor_state& state);
+    ~single_usage_behavior() override;
+
+    void on_enter() override;
+    void schedule() override;
+    void receive(const token_ptr& tok) override;
+    void reclaim(int blocks_consumed,
+                 int memory_returned,
+                 int time,
+                 int dependency_number) override;
+
+    std::string name() const override { return "single_usage"; }
+
+protected:
+    void process_launch_token(const token_ptr& tok, int stream_id) override;
+
+private:
+    void init_state();
+    void create_new_graph(const token_ptr& tok);
+    int get_next_stream();
+
+    device_ptr device_;
+    std::optional<sm_usage_heuristic> heuristic;  // optional — kept for logging/real cost reporting
+
+    // State for single-kernel-at-a-time scheduling
+    bool gpu_available = true;                    // true = GPU is idle and can accept a kernel
+
+    // Queues for pending operations (still respect dependencies)
+    std::unordered_map<int, kernel_graph> graphs;           // dependency → graph
+    std::vector<kernel_graph> independent_graphs;           // no dependency
+
+    // Stream management (even in serial mode, streams can be useful)
+    int num_streams = 0;
+    int current_stream = 0;
+
+    int64_t available_memory = 0;
+};
+
+} // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/single_usage_behavior.cpp b/libcaf_cuda/src/control-layer/single_usage_behavior.cpp
new file mode 100644
index 0000000000..32daf9a870
--- /dev/null
+++ b/libcaf_cuda/src/control-layer/single_usage_behavior.cpp
@@ -0,0 +1,147 @@
+#include "caf/cuda/control-layer/all-control-layer.hpp"
+#include "caf/cuda/control-layer/single_usage_behavior.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/profiler.hpp"
+#include "caf/cuda/manager.hpp"
+#include "caf/cuda/device.hpp"
+
+namespace caf::cuda {
+
+single_usage_behavior::single_usage_behavior(scheduler_actor_state& state)
+    : scheduler_actor_behavior(state) {
+    init_state();
+}
+
+single_usage_behavior::~single_usage_behavior() = default;
+
+void single_usage_behavior::init_state() {
+    device_ = manager::get().find_device(state_.device_number);
+    // We can still keep the heuristic if you want to log real SM usage later,
+    // but we won't use it for scheduling decisions
+    heuristic.emplace(device_);
+
+    // For single-usage we treat the whole GPU as occupied or free
+    gpu_available = true;           // initially free
+    available_memory = static_cast<int64_t>(device_->total_memory_bytes());
+}
+
+void single_usage_behavior::on_enter() {
+    schedule();  // try to launch something right away if tokens already waiting
+}
+
+void single_usage_behavior::reclaim(int blocks_consumed,
+                                    int memory_returned,
+                                    int time,
+                                    int dependency_number) {
+    // GPU is now free again
+    gpu_available = true;
+    available_memory += memory_returned;
+
+    // Optional: could log real usage
+    // std::cout << "Reclaimed: " << blocks_consumed << " SMs, " << time << " μs\n";
+
+    schedule();  // try to launch the next one immediately
+}
+
+void single_usage_behavior::process_launch_token(const token_ptr& tok, int stream_id) {
+    scoped_timer timer("single_usage_behavior::process_launch_token");
+
+    // For pure single-usage mode we usually don't care about the heuristic cost
+    // but we can still compute it for logging / debugging
+    int reported_cost = heuristic->getCost(tok);
+
+    const auto& launch = static_cast<const launch_token&>(*tok);
+    auto response = make_launch_response_token(
+        state_.self,
+        launch,
+        state_.device_number,
+        stream_id,
+        reported_cost  // report real cost even if we don't use it for decision
+    );
+
+    anon_mail(response).send(launch.getReplyActor());
+
+    // Mark GPU as busy
+    gpu_available = false;
+}
+
+void single_usage_behavior::receive(const token_ptr& tok) {
+
+    if (tok->getType() == LAUNCH) {
+        create_new_graph(tok);
+
+        // Try to dispatch immediately if GPU is currently free
+        if (gpu_available) {
+            schedule();
+        }
+    }
+    else if (tok->getType() == MEMORY) {
+        // For pure kernel serialisation testing you can often just forward memory ops
+        // without blocking — or implement strict ordering if needed
+        process_memory_transfer_token(tok, 0);
+    }
+}
+
+void single_usage_behavior::create_new_graph(const token_ptr& tok) {
+    if (tok->isIndependent()) {
+        kernel_graph g(state_.device_number, get_next_stream());
+        g.add_operation(tok);
+        independent_graphs.push_back(std::move(g));
+    }
+    else {
+        int dep = tok->getDependency();
+        if (!graphs.contains(dep)) {
+            kernel_graph g(state_.device_number, get_next_stream());
+            graphs[dep] = std::move(g);
+        }
+        graphs[dep].add_operation(tok);
+    }
+}
+
+int single_usage_behavior::get_next_stream() {
+    return current_stream++ % num_streams;
+}
+
+// ────────────────────────────────────────────────
+// The only real scheduling logic — find and launch ONE kernel if possible
+// ────────────────────────────────────────────────
+void single_usage_behavior::schedule() {
+    if (!gpu_available) {
+        return;  // GPU still busy → do nothing
+    }
+
+    token_ptr next = nullptr;
+    int stream_id = -1;
+
+    // 1. Prefer independent kernels (they have no dependencies → lowest risk)
+    for (auto it = independent_graphs.begin(); it != independent_graphs.end(); ++it) {
+        if (!it->empty()) {
+            next = it->getOperation();
+            stream_id = it->stream_id();
+            if (next && next->getType() == LAUNCH) {
+                // found one → launch and remove empty graph if needed
+                if (it->empty()) {
+                    independent_graphs.erase(it);
+                }
+                goto launch;
+            }
+        }
+    }
+
+    // 2. Otherwise take the first non-empty dependent graph (FIFO-ish)
+    for (auto& [dep, graph] : graphs) {
+        if (!graph.empty()) {
+            next = graph.getOperation();
+            stream_id = graph.stream_id();
+            if (next && next->getType() == LAUNCH) {
+                goto launch;
+            }
+        }
+    }
+
+    return;  // nothing ready to run
+
+launch:
+    process_launch_token(next, stream_id);
+}
+
+} // namespace caf::cuda

From ada2db9d386ec5279f7270ac19f09d21c5900ebe Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 3 Feb 2026 15:19:18 -0600
Subject: [PATCH 0290/1000] Integrated single usage behavior into the
 scheduler.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 8f4991cf86..d799b7d1dd 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -3,6 +3,7 @@
 #include "caf/cuda/control-layer/green_light_behavior.hpp"
 #include "caf/cuda/control-layer/red_light_behavior.hpp"
 #include "caf/cuda/control-layer/core_usage_behavior.hpp"
+#include "caf/cuda/control-layer/single_usage_behavior.hpp"
 #include <string>
 #include <iostream>
 
@@ -24,16 +25,20 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
     static red_light_behavior red_behavior(state);
     static green_light_behavior green_behavior(state);
     static core_usage_behavior core_behavior(state);
+    static single_usage_behavior single_behavior(state);
 
     // populate the behavior table
     state.table.add("red", &red_behavior);
     state.table.add("green",   &green_behavior);
     state.table.add("core_usage",   &core_behavior);
+    state.table.add("single_usage",   &single_behavior);
 
     // default behavior
     state.current_behavior = state.table.get(behavior_token("green"));
     state.current_behavior->on_enter();
 
+
+
     return {
         [&](const token_ptr& tok) {
             // std::cout << "Received token\n";

From cf4128ce7d2e95cd3515b8ef2a6e42a4daab457b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 3 Feb 2026 15:24:23 -0600
Subject: [PATCH 0291/1000] Initial commmit, copied main.test and CMakelists
 over from core_usage_test.

---
 .../fault-tolerance-test/CMakeLists.txt       |   44 +
 .../fault-tolerance-test/compile_kernels.sh   |   18 +
 .../fault-tolerance-test/faulty_kernels.cu    |   47 +
 .../fault-tolerance-test/main.test.cpp        | 1025 +++++++++++++++++
 4 files changed, 1134 insertions(+)
 create mode 100644 libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/faulty_kernels.cu
 create mode 100644 libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp

diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/CMakeLists.txt b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/compile_kernels.sh b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/compile_kernels.sh
new file mode 100755
index 0000000000..cf7385147a
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/compile_kernels.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin faulty_kernels.cu -o fault.cubin
+echo "Generated mmul.cubin"
+
+# Compile genMatrix.cu to fatbin in current directory
+#nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
+#echo "Generated generate_random_matrix.fatbin"
+
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/faulty_kernels.cu b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/faulty_kernels.cu
new file mode 100644
index 0000000000..c7b80e2c20
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/faulty_kernels.cu
@@ -0,0 +1,47 @@
+#include <cuda_runtime.h>
+#include <curand.h>
+#include <curand_kernel.h>
+#include <iostream>
+#include <cstdlib>
+
+// Error checking macro
+#define CUDA_CHECK(err) if (err != cudaSuccess) { std::cerr << "CUDA Error: " << cudaGetErrorString(err) << std::endl; exit(1); }
+
+// Step 1: Initialize denominators with ~50% zeros using cuRAND
+__global__ void init_denominators(float* denominators, int n, unsigned long long seed) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+
+    curandState state;
+    curand_init(seed, idx, 0, &state);
+    float rand_val = curand_uniform(&state);
+    denominators[idx] = (rand_val < 0.5f) ? 0.0f : 1.0f;  // ~50% chance of zero
+}
+
+// Step 2: Perform division (potential div by zero -> Inf)
+__global__ void perform_division(float* numerators, float* denominators, float* results, int n) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    results[idx] = numerators[idx] / denominators[idx];  // Triggers Inf if denominator == 0
+}
+
+// Step 3: Simple reduction to sum results (propagates Inf if present)
+__global__ void sum_results(float* results, float* final_sum, int n) {
+    extern __shared__ float sdata[];
+    int tid = threadIdx.x;
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+
+    sdata[tid] = (idx < n) ? results[idx] : 0.0f;
+    __syncthreads();
+
+    for (int s = blockDim.x / 2; s > 0; s >>= 1) {
+        if (tid < s) {
+            sdata[tid] += sdata[tid + s];
+        }
+        __syncthreads();
+    }
+
+    if (tid == 0) {
+        atomicAdd(final_sum, sdata[0]);
+    }
+}
diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
new file mode 100644
index 0000000000..b406dca545
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
@@ -0,0 +1,1025 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include <unistd.h>
+#include "caf/actor_registry.hpp"
+#include <chrono>
+#include <iostream>
+//#include <caf/atoms.hpp>
+
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+struct exit_actor_state {
+	int completed = 0;
+};
+
+
+caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
+
+
+	return {
+		[=](int num_completed) {
+			self->state().completed += num_completed;
+			
+			//std::cout << "Actors finished is " << self->state().completed << "\n";
+			if (self->state().completed >= limit) {
+			
+				caf::cuda::manager::shutdown();
+				self->quit();
+			}
+		}
+	};
+
+
+}
+
+
+
+
+
+
+// Define a custom type ID block for custom actors
+CAF_ADD_ATOM(cuda,shared_mem)
+
+
+
+
+
+// Extend your actor state to keep the start time
+struct mmul_actor_state {
+  static inline const char* name = "mmul_actor";
+
+  int N = 0;
+  int id = rand();
+
+  // timing / bookkeeping only
+  std::chrono::high_resolution_clock::time_point start_time;
+  int times = 0;
+};
+
+
+
+
+
+//commands classes used to launch kernels 
+using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
+using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
+
+using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
+
+mmulCommand mmul;
+matrixGenCommand randomMatrix;
+mmulAsyncCommand mmulAsync;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun(
+    caf::stateful_actor<mmul_actor_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims)
+{
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+	caf::actor scheduler = mgr.get_scheduler_actor();
+
+	//send a launch token
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+			program,
+			dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+		[=] (caf::cuda::response_token_ptr res_token) {
+
+			if (res_token -> getType() == LAUNCH_RESPONSE) {
+				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+				//assume N = 1024
+				int N = self -> state().N;
+				std::vector<int> matrix1(N*N);
+				matrix1.reserve(N);
+				std::vector<int> matrix2(N*N);
+				matrix2.reserve(N);
+
+				//std::cout << "GPU ACTOR sending data to compute\n";
+				self -> mail(matrix1,matrix2,res_token,N).send(self);
+
+			}
+			else {
+				std::cout << "Got a memory response token\n";
+			}
+			//token should drop out of scope now, triggering a response 
+		},
+
+			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+			[=](const std::vector<int>& matrixA,
+					const std::vector<int>& matrixB,
+					const caf::cuda::response_token_ptr& res_token, int N) {
+
+
+				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
+
+				//caf::cuda::launch_response_token& kt =
+				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
+
+				//std::cout << "GPU ACTOR  computing\n";
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+				//create program and dims   
+				auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+				const int THREADS = 32;
+				const int BLOCKS = (N + THREADS - 1) / THREADS;
+				caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrixA);
+				auto arg2 = caf::cuda::create_in_arg(matrixB);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
+
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				//std::cout << "GPU ACTOR done  computing\n";
+				//verify its own result 
+				self -> mail(matrixA,matrixB,matrixC,N).send(self);
+
+			},
+
+					// 3rd handler: CPU atom + matrices + N
+					[=](const std::vector<int>& matrixA,
+							const std::vector<int>& matrixB,
+							const std::vector<int>& matrixC,
+							int N) {
+
+						using clock = std::chrono::high_resolution_clock;
+
+						auto start = clock::now();
+
+						//std::cout << "GPU ACTOR verifying\n";
+
+						std::vector<int> result(N * N);
+
+						serial_matrix_multiply(matrixA, matrixB, result, N);
+
+						if (result == matrixC) {
+							std::cout << "actor with id " << self->state().id
+								<< " references match\n";
+						} else {
+							std::cout << "actor with id " << self->state().id
+								<< " references did not match\n";
+						}
+
+						auto end = clock::now();
+
+						auto ms =
+							std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+						std::cout << "[TIMING] verification took "
+							<< ms << " ms (actor id "
+							<< self->state().id << ")\n";
+
+						// signal exit actor and quit
+						self->mail(1).send(exit_actor);
+						self->quit();
+
+					}
+	};
+}
+
+
+
+
+
+// this actor will not verify its results
+// great for performance analysis 
+caf::behavior mmul_actor_fun_no_verify(
+    caf::stateful_actor<mmul_actor_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims,
+    bool request
+    )
+{
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+	caf::actor scheduler = mgr.get_scheduler_actor();
+
+	if (request) {
+		//send a launch token
+		caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+				program,
+				dims,
+				0,
+				"hello",
+				self
+				);
+		self -> mail(launch_token).send(scheduler);
+	}
+	return {
+
+		[=] (caf::cuda::response_token_ptr res_token) {
+
+			if (res_token -> getType() == LAUNCH_RESPONSE) {
+				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+				//assume N = 1024
+				int N = self -> state().N;
+				std::vector<int> matrix1(N*N);
+				matrix1.reserve(N);
+				std::vector<int> matrix2(N*N);
+				matrix2.reserve(N);
+
+
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+				//create program and dims   
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrix1);
+				auto arg2 = caf::cuda::create_in_arg(matrix2);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
+
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+			
+			
+				//mask the transfer back to the cpu for scheduler
+				res_token -> release();
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				//std::cout << "GPU ACTOR done  computing\n";
+				// signal exit actor and quit
+				self->mail(1).send(exit_actor);
+				self->quit();
+
+				//std::cout << "GPU ACTOR sending data to compute\n";
+			//	self -> mail(matrix1,matrix2,res_token,N).send(self);
+
+			}
+			else {
+				std::cout << "Got a memory response token\n";
+			}
+			//token should drop out of scope now, triggering a response 
+		},
+
+			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+			[=](const std::vector<int>& matrixA,
+					const std::vector<int>& matrixB,
+					const caf::cuda::response_token_ptr& res_token, int N) {
+
+
+				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
+
+				//caf::cuda::launch_response_token& kt =
+				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
+
+				//std::cout << "GPU ACTOR  computing\n";
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+				//create program and dims   
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrixA);
+				auto arg2 = caf::cuda::create_in_arg(matrixB);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
+
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+			
+			
+				//mask the transfer back to the cpu for scheduler
+				res_token -> release();
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				//std::cout << "GPU ACTOR done  computing\n";
+				// signal exit actor and quit
+				self->mail(1).send(exit_actor);
+				self->quit();
+
+			}
+
+				
+	};
+}
+
+
+
+// Stateful actor behavior
+// this actor does not invoke the scheduler at all 
+caf::behavior mmul_actor_fun_no_schedule(
+    caf::stateful_actor<mmul_actor_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims) {
+
+    self->state().N = N;
+
+    std::vector<int> matrix1(N * N);
+    std::vector<int> matrix2(N * N);
+
+    // send initial mail to self
+    self->mail(matrix1, matrix2, N).send(self);
+
+    return {
+        // GPU atom + matrices + N
+        [=](const std::vector<int>& matrixA,
+            const std::vector<int>& matrixB,
+            int N_local) {   // avoid shadowing outer N
+            
+	    
+    	    //std::cout << "Hello\n";
+	    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+            auto arg1 = caf::cuda::create_in_arg(matrixA);
+            auto arg2 = caf::cuda::create_in_arg(matrixB);
+            auto arg3 = caf::cuda::create_out_arg(N_local * N_local);
+            auto arg4 = caf::cuda::create_in_arg(N_local);
+
+            auto tempC = mmul.run(program, dims, self->state().id, arg1, arg2, arg3, arg4);
+            std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+	    self->mail(1).send(exit_actor);
+            self->quit();
+        },
+
+        // CPU verification
+        [=](const std::vector<int>& matrixA,
+            const std::vector<int>& matrixB,
+            const std::vector<int>& matrixC,
+            int N_local) {
+
+            std::vector<int> result(N_local * N_local);
+            serial_matrix_multiply(matrixA, matrixB, result, N_local);
+
+            if (result == matrixC) {
+                std::cout << "actor with id " << self->state().id << " references match\n";
+            } else {
+                std::cout << "actor with id " << self->state().id << " references did not match\n";
+            }
+
+            self->quit();
+        }
+    };
+}
+
+
+
+
+
+
+
+
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //change the scheduler to core_usage
+  anon_mail(
+	caf::cuda::make_behavior_token("core_usage")
+	).send(mgr.get_scheduler_actor());
+
+  // CREATE ONCE
+  auto program =
+      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+  const int THREADS = 32;
+  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1);
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+  for (int i = 0; i < num_actors; ++i) {
+    /*
+    sys.spawn(
+        mmul_actor_fun,
+        exit_actor,
+        matrix_size,
+        program,
+        dims);
+    */
+      sys.spawn(
+        mmul_actor_fun_no_verify,
+        exit_actor,
+        matrix_size,
+        program,
+        dims,
+	true);
+    
+  }
+
+  sys.await_all_actors_done();
+}
+
+
+void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  /*
+  //change the scheduler to core_usage
+  anon_mail(
+	caf::cuda::make_behavior_token("core_usage")
+	).send(mgr.get_scheduler_actor());
+
+  */
+
+
+  // CREATE ONCE
+  auto program =
+      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+  const int THREADS = 32;
+  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1);
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+  for (int i = 0; i < num_actors; ++i) {
+    
+	sys.spawn(
+        mmul_actor_fun_no_verify,
+        exit_actor,
+        matrix_size,
+        program,
+        dims,
+	true);
+    
+  
+	/*  
+sys.spawn(
+        mmul_actor_fun,
+        exit_actor,
+        matrix_size,
+        program,
+        dims);
+  */
+  }
+
+  sys.await_all_actors_done();
+}
+
+void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, int num_actors) {
+    if (num_actors < 1) {
+        std::cerr << "[ERROR] Number of actors must be >= 1\n";
+        return;
+    }
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    // CREATE ONCE
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    const int THREADS = 32;
+    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    for (int i = 0; i < num_actors; ++i) {
+        sys.spawn(
+            mmul_actor_fun_no_schedule,
+            exit_actor,
+            matrix_size,
+            program,
+            dims
+        );
+    }
+
+    sys.await_all_actors_done();
+}
+
+
+template <class Fn>
+double time_run(Fn&& fn) {
+  auto start = std::chrono::steady_clock::now();
+  fn();
+  auto end = std::chrono::steady_clock::now();
+  std::chrono::duration<double> elapsed = end - start;
+  return elapsed.count();
+}
+void run_mmul_scaling_tests(caf::actor_system& sys,
+                            caf::cuda::manager_config man_config) {
+    const int max_size   = 1024;
+    const int min_actors = 1;
+    const int max_actors = 1024;
+
+    std::vector<int> matrix_sizes = {10};
+    for (int s = 32; s <= max_size; s *= 2)
+        matrix_sizes.push_back(s);
+
+    std::vector<int> actor_counts;
+    for (int a = min_actors; a <= max_actors; a *= 2)
+        actor_counts.push_back(a);
+
+    std::cout << "=== MMUL Scaling Tests ===\n";
+    std::cout << "Format:\n";
+    std::cout << "scheduler matrix_size actors time_seconds\n";
+
+    for (int size : matrix_sizes) {
+        for (int actors : actor_counts) {
+
+            /* ================= Scheduler-enabled (core_usage) ================= */
+            caf::cuda::manager::init(sys, man_config); // green-light scheduler enabled
+            std::cout << "\n[RUN] scheduler=core_usage "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double core_usage_time = time_run([&] {
+                run_mmul_test(sys, size, actors); // uses mmul_actor_fun_no_verify
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT core_usage "
+                      << size << " "
+                      << actors << " "
+                      << core_usage_time << "\n";
+
+            caf::cuda::manager::shutdown(); // make sure manager is cleaned up
+
+            /* ================= Scheduler-disabled actor (still uses green-light) ================= */
+
+/*
+	    caf::cuda::manager::init(sys, man_config); // init with scheduler
+            std::cout << "\n[RUN] scheduler=green_light_only "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double green_light_time = time_run([&] {
+                run_mmul_test_no_scheduler(sys, size, actors); // your previous "no scheduler" actor
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT green_light_only "
+                      << size << " "
+                      << actors << " "
+                      << green_light_time << "\n";
+
+            caf::cuda::manager::shutdown();
+
+	    */
+            /* ================= No scheduler at all actor ================= */
+            caf::cuda::manager_config no_sched_config(false); // disable scheduler
+            caf::cuda::manager::init(sys, no_sched_config);
+            std::cout << "\n[RUN] scheduler=none "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double no_scheduler_time = time_run([&] {
+                run_mmul_test_no_scheduler_actor(sys, size, actors); // mmul_actor_fun_no_schedule
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT none "
+                      << size << " "
+                      << actors << " "
+                      << no_scheduler_time << "\n";
+
+            caf::cuda::manager::shutdown();
+        }
+    }
+
+    std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
+}
+void run_mmul_mixed_batch_one_mode(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    bool use_scheduler_actor,
+    bool use_core_usage_behavior,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get(); // SAFE NOW
+
+    if (use_scheduler_actor && use_core_usage_behavior) {
+        anon_mail(caf::cuda::make_behavior_token("core_usage"))
+            .send(mgr.get_scheduler_actor());
+    }
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    std::mt19937 rng(123456);
+    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+
+    const int THREADS = 32;
+
+    for (int i = 0; i < num_actors; ++i) {
+        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
+        int BLOCKS = (N + THREADS - 1) / THREADS;
+
+        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+        if (use_scheduler_actor) {
+            sys.spawn(
+                mmul_actor_fun_no_verify,
+                exit_actor,
+                N,
+                program,
+                dims,
+		true);
+        } else {
+            sys.spawn(
+                mmul_actor_fun_no_schedule,
+                exit_actor,
+                N,
+                program,
+                dims);
+        }
+    }
+
+    sys.await_all_actors_done();
+}
+
+
+
+
+void run_mmul_mixed_batch_one_mode_bulk(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get(); // SAFE NOW
+
+        anon_mail(caf::cuda::make_behavior_token("core_usage"))
+            .send(mgr.get_scheduler_actor());
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    std::vector<caf::cuda::token_ptr> tokens(num_actors);
+
+    std::mt19937 rng(123456);
+    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+
+    const int THREADS = 32;
+
+    for (int i = 0; i < num_actors; ++i) {
+        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
+        int BLOCKS = (N + THREADS - 1) / THREADS;
+
+	caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+	caf::actor a = sys.spawn(
+			mmul_actor_fun_no_verify,
+			exit_actor,
+			N,
+			program,
+			dims,
+			false);
+
+	tokens[i] = caf::cuda::make_launch_token(
+			program,
+			dims,
+			0 /*this should not be 0 but its fine for now*/,
+			"hello",
+			a);    
+    }
+
+    
+        anon_mail(tokens)
+            .send(mgr.get_scheduler_actor());
+
+
+    sys.await_all_actors_done();
+}
+
+
+
+
+
+
+
+
+
+
+void run_mmul_mixed_batch_comparison(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors)
+{
+    std::cout << "\n=== MMUL Mixed-Size Batch Comparison ===\n";
+    std::cout << "scheduler actors sizes time_seconds\n\n";
+
+    /* ================= core_usage ================= */
+    {
+        caf::cuda::manager_config cfg(true);
+        caf::cuda::manager::init(sys, cfg);
+
+	
+        //double t = time_run([&] {
+          //  run_mmul_mixed_batch_one_mode(
+            //    sys, sizes, num_actors,
+              //  /*use_scheduler_actor=*/true,
+               // /*use_core_usage_behavior=*/true);
+       // });
+
+	double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode_bulk(
+                sys, sizes, num_actors);
+        });
+
+
+
+
+        std::cout << "RESULT core_usage "
+                  << num_actors << " "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    /* ================= green-light only ================= */
+   
+    /*
+    {
+	    std::cout << "Starting green_light tests\n";
+        caf::cuda::manager_config cfg(true);
+        caf::cuda::manager::init(sys, cfg);
+
+        double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode(
+                sys, sizes, num_actors,
+               true,
+                false);
+        });
+
+        std::cout << "RESULT green_light_only "
+                  << num_actors << " "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    */
+	
+    /* ================= no scheduler ================= */
+    {
+        caf::cuda::manager_config cfg(false);
+        caf::cuda::manager::init(sys, cfg);
+
+        double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode(
+                sys, sizes, num_actors,
+                /*use_scheduler_actor=*/false,
+                /*use_core_usage_behavior=*/false);
+        });
+
+        std::cout << "RESULT none "
+                  << num_actors << " "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    std::cout << "\n=== Comparison Complete ===\n";
+}
+
+
+
+void test_core_usage_uniform_mmul(
+    caf::actor_system& sys,
+    int matrix_size,
+    int num_actors)
+{
+    std::cout << "\n[TEST] core_usage uniform matrix size\n";
+    std::cout << "N=" << matrix_size
+              << " actors=" << num_actors << "\n";
+
+    caf::cuda::manager_config cfg(true);
+    caf::cuda::manager::init(sys, cfg);
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    // force core_usage behavior
+    anon_mail(caf::cuda::make_behavior_token("core_usage"))
+        .send(mgr.get_scheduler_actor());
+
+    auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    const int THREADS = 32;
+    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(
+        BLOCKS, BLOCKS, 1,
+        THREADS, THREADS, 1);
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    for (int i = 0; i < num_actors; ++i) {
+        sys.spawn(
+            mmul_actor_fun_no_verify,
+            exit_actor,
+            matrix_size,
+            program,
+            dims,
+	    false);
+    }
+
+    sys.await_all_actors_done();
+    caf::cuda::manager::shutdown();
+
+    std::cout << "[PASS] core_usage uniform test complete\n";
+}
+
+
+
+void test_core_usage_mixed_mmul(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors)
+{
+    std::cout << "\n[TEST] core_usage mixed matrix sizes\n";
+    std::cout << "actors=" << num_actors << "\n";
+
+    caf::cuda::manager_config cfg(true);
+    caf::cuda::manager::init(sys, cfg);
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    anon_mail(caf::cuda::make_behavior_token("core_usage"))
+        .send(mgr.get_scheduler_actor());
+
+    auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    const int THREADS = 32;
+
+    for (int i = 0; i < num_actors; ++i) {
+        int N = sizes[i % sizes.size()];
+
+        int BLOCKS = (N + THREADS - 1) / THREADS;
+        caf::cuda::nd_range dims(
+            BLOCKS, BLOCKS, 1,
+            THREADS, THREADS, 1);
+
+        sys.spawn(
+            mmul_actor_fun_no_verify,
+            exit_actor,
+            N,
+            program,
+            dims,
+	    false);
+    }
+
+    sys.await_all_actors_done();
+    caf::cuda::manager::shutdown();
+
+    std::cout << "[PASS] core_usage mixed-size test complete\n";
+}
+
+
+void run_mmul_fixed_256_batch_comparison(
+    caf::actor_system& sys,
+    int num_actors)
+{
+    // All actors run the same matrix size: 256
+    std::vector<int> sizes(num_actors, 256);
+
+    std::cout << "\n=== MMUL Fixed-Size (256) Batch Comparison ===\n";
+    std::cout << "scheduler actors size time_seconds\n\n";
+
+    /* ================= core_usage ================= */
+    {
+        caf::cuda::manager_config cfg(true);
+        caf::cuda::manager::init(sys, cfg);
+
+        double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode(
+                sys,
+                sizes,
+                num_actors,
+                /*use_scheduler_actor=*/true,
+                /*use_core_usage_behavior=*/true);
+        });
+
+        std::cout << "RESULT core_usage "
+                  << num_actors << " 256 "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    /* ================= no scheduler ================= */
+    {
+        caf::cuda::manager_config cfg(false);
+        caf::cuda::manager::init(sys, cfg);
+
+        double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode(
+                sys,
+                sizes,
+                num_actors,
+                /*use_scheduler_actor=*/false,
+                /*use_core_usage_behavior=*/false);
+        });
+
+        std::cout << "RESULT none "
+                  << num_actors << " 256 "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    std::cout << "\n=== Fixed-256 Comparison Complete ===\n";
+}
+
+
+
+
+
+void caf_main(caf::actor_system& sys) {
+  
+	//caf::cuda::manager_config man_config(true); //turns the scheduler on
+	//caf::cuda::manager::init(sys,man_config);
+        // run_mmul_test(sys,10,64);	
+	//run_mmul_scaling_tests(sys,man_config);
+
+   std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048,4096};
+    const int num_actors = 1000;
+    run_mmul_mixed_batch_comparison(sys, sizes, num_actors);    
+
+    //run_mmul_mixed_batch_one_mode_bulk(sys,sizes,num_actors);
+    //run_mmul_fixed_256_batch_comparison(sys, /*num_actors=*/200);
+
+
+	 //test_core_usage_uniform_mmul(sys, 256, 1000);
+
+  //std::vector<int> sizes = {32, 64, 128, 256, 512, 1024};
+    //test_core_usage_mixed_mmul(sys, sizes, 200);
+
+
+
+//tests will delete the old manager so will have to reinit if you do this 
+	//in conjunction with each other	
+	//caf::cuda::manager::init(sys,man_config);
+}
+
+
+
+
+CAF_MAIN()

From 8f6c806e2d4768a040c2f086cbc709c46ebb2c31 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 3 Feb 2026 15:43:52 -0600
Subject: [PATCH 0292/1000] Added name method to both classes. Change is being
 made so actors can identify response tokens easier.

---
 libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp | 3 ++-
 libcaf_cuda/caf/cuda/control-layer/response_token.hpp        | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
index 8db2103bf1..ccad9be3b5 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -65,7 +65,8 @@ class CAF_CUDA_EXPORT launch_response_token : public response_token {
     const nd_range& getRange() const { return range_; }
 
     const std::string& getId() const { return id_; }
-
+    const std::string& name() const override {return id_;}
+    
     // Return requested number of CUDA blocks
     int getBlocks() const {
         return static_cast<int>(
diff --git a/libcaf_cuda/caf/cuda/control-layer/response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/response_token.hpp
index 2dcdeaa49e..5e71a43cfe 100644
--- a/libcaf_cuda/caf/cuda/control-layer/response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/response_token.hpp
@@ -27,6 +27,7 @@ class CAF_CUDA_EXPORT response_token : public token {
     int getStreamId() const { return stream_id_; }
     int memorySize() const { return memory_size_; }
     const caf::actor& getReceiver() const { return receiver_; }
+    virtual const std::string& name() const { return default_name;}
 
     // Pure virtual: children must implement release()
     virtual void release() = 0;
@@ -36,6 +37,7 @@ class CAF_CUDA_EXPORT response_token : public token {
     int device_number_{0};
     int stream_id_{0};
     int memory_size_{0};  // abstracted memory usage / size
+    std::string default_name = "unknown";
 };
 
 // Typedef for convenience

From adf6b2f034159aa96ffac08b5818b55148638e76 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Feb 2026 11:13:41 -0600
Subject: [PATCH 0293/1000] Updated test to have actor code in it (not done)

---
 .../fault-tolerance-test/main.test.cpp        | 1139 +++--------------
 1 file changed, 188 insertions(+), 951 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
index b406dca545..733d10cf30 100644
--- a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
@@ -27,384 +27,185 @@ struct exit_actor_state {
 };
 
 
-caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
+// --- command runner types (put near top of file) -------------------------
+using initCommand =
+  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::in<int>, caf::cuda::in<unsigned long long>>;
 
+using divCommand  =
+  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, caf::cuda::in<int>>;
 
-	return {
-		[=](int num_completed) {
-			self->state().completed += num_completed;
-			
-			//std::cout << "Actors finished is " << self->state().completed << "\n";
-			if (self->state().completed >= limit) {
-			
-				caf::cuda::manager::shutdown();
-				self->quit();
-			}
-		}
-	};
-
-
-}
-
-
-
-
-
-
-// Define a custom type ID block for custom actors
-CAF_ADD_ATOM(cuda,shared_mem)
+using sumCommand  =
+  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, caf::cuda::in<int>>;
 
+// single instances (can be file-global)
+static initCommand init_cmd;
+static divCommand  div_cmd;
+static sumCommand  sum_cmd;
 
+// --- pipeline actor state (device buffers persist here) ------------------
+struct pipeline_actor_state {
+    int id = rand();
 
-
-
-// Extend your actor state to keep the start time
-struct mmul_actor_state {
-  static inline const char* name = "mmul_actor";
-
-  int N = 0;
-  int id = rand();
-
-  // timing / bookkeeping only
-  std::chrono::high_resolution_clock::time_point start_time;
-  int times = 0;
+    // device-side buffers that must persist across stages:
+    caf::cuda::mem_ptr<float> d_denoms;
+    caf::cuda::mem_ptr<float> d_results;
+    caf::cuda::mem_ptr<float> d_sum;
 };
 
-
-
-
-
-//commands classes used to launch kernels 
-using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
-using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
-
-using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
-
-mmulCommand mmul;
-matrixGenCommand randomMatrix;
-mmulAsyncCommand mmulAsync;
-
-
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-  
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
-    }
-  }
-}
-
-
-
-
-// Stateful actor behavior
-caf::behavior mmul_actor_fun(
-    caf::stateful_actor<mmul_actor_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims)
+// --- corrected pipeline_actor -------------------------------------------
+behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
+                        actor supervisor,
+                        program_ptr p1,
+                        program_ptr p2,
+                        program_ptr p3,
+                        int n)
 {
+    // host-side scratch (only used for post-stage2 NaN/Inf detection)
+    std::vector<float> h_results;
 
-	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
-
-	caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-	caf::actor scheduler = mgr.get_scheduler_actor();
-
-	//send a launch token
-	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-			program,
-			dims,
-			0,
-			"hello",
-			self
-			);
-	self -> mail(launch_token).send(scheduler);
-
-	return {
-
-		[=] (caf::cuda::response_token_ptr res_token) {
-
-			if (res_token -> getType() == LAUNCH_RESPONSE) {
-				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-				//assume N = 1024
-				int N = self -> state().N;
-				std::vector<int> matrix1(N*N);
-				matrix1.reserve(N);
-				std::vector<int> matrix2(N*N);
-				matrix2.reserve(N);
-
-				//std::cout << "GPU ACTOR sending data to compute\n";
-				self -> mail(matrix1,matrix2,res_token,N).send(self);
-
-			}
-			else {
-				std::cout << "Got a memory response token\n";
-			}
-			//token should drop out of scope now, triggering a response 
-		},
-
-			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-			[=](const std::vector<int>& matrixA,
-					const std::vector<int>& matrixB,
-					const caf::cuda::response_token_ptr& res_token, int N) {
-
-
-				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
-
-				//caf::cuda::launch_response_token& kt =
-				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
-
-				//std::cout << "GPU ACTOR  computing\n";
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-				const int THREADS = 32;
-				const int BLOCKS = (N + THREADS - 1) / THREADS;
-				caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrixA);
-				auto arg2 = caf::cuda::create_in_arg(matrixB);
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
-
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				//std::cout << "GPU ACTOR done  computing\n";
-				//verify its own result 
-				self -> mail(matrixA,matrixB,matrixC,N).send(self);
-
-			},
-
-					// 3rd handler: CPU atom + matrices + N
-					[=](const std::vector<int>& matrixA,
-							const std::vector<int>& matrixB,
-							const std::vector<int>& matrixC,
-							int N) {
-
-						using clock = std::chrono::high_resolution_clock;
-
-						auto start = clock::now();
-
-						//std::cout << "GPU ACTOR verifying\n";
-
-						std::vector<int> result(N * N);
-
-						serial_matrix_multiply(matrixA, matrixB, result, N);
-
-						if (result == matrixC) {
-							std::cout << "actor with id " << self->state().id
-								<< " references match\n";
-						} else {
-							std::cout << "actor with id " << self->state().id
-								<< " references did not match\n";
-						}
-
-						auto end = clock::now();
-
-						auto ms =
-							std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-						std::cout << "[TIMING] verification took "
-							<< ms << " ms (actor id "
-							<< self->state().id << ")\n";
-
-						// signal exit actor and quit
-						self->mail(1).send(exit_actor);
-						self->quit();
-
-					}
-	};
-}
-
-
-
-
-
-// this actor will not verify its results
-// great for performance analysis 
-caf::behavior mmul_actor_fun_no_verify(
-    caf::stateful_actor<mmul_actor_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims,
-    bool request
-    )
-{
-
-	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
-
-	caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-	caf::actor scheduler = mgr.get_scheduler_actor();
+    // scheduler from manager
+    caf::actor scheduler = caf::cuda::manager::get().get_scheduler_actor();
 
-	if (request) {
-		//send a launch token
-		caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-				program,
-				dims,
-				0,
-				"hello",
-				self
-				);
-		self -> mail(launch_token).send(scheduler);
-	}
-	return {
-
-		[=] (caf::cuda::response_token_ptr res_token) {
-
-			if (res_token -> getType() == LAUNCH_RESPONSE) {
-				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-				//assume N = 1024
-				int N = self -> state().N;
-				std::vector<int> matrix1(N*N);
-				matrix1.reserve(N);
-				std::vector<int> matrix2(N*N);
-				matrix2.reserve(N);
-
-
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrix1);
-				auto arg2 = caf::cuda::create_in_arg(matrix2);
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
-
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-			
-			
-				//mask the transfer back to the cpu for scheduler
-				res_token -> release();
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				//std::cout << "GPU ACTOR done  computing\n";
-				// signal exit actor and quit
-				self->mail(1).send(exit_actor);
-				self->quit();
-
-				//std::cout << "GPU ACTOR sending data to compute\n";
-			//	self -> mail(matrix1,matrix2,res_token,N).send(self);
-
-			}
-			else {
-				std::cout << "Got a memory response token\n";
-			}
-			//token should drop out of scope now, triggering a response 
-		},
-
-			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-			[=](const std::vector<int>& matrixA,
-					const std::vector<int>& matrixB,
-					const caf::cuda::response_token_ptr& res_token, int N) {
-
-
-				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
-
-				//caf::cuda::launch_response_token& kt =
-				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
-
-				//std::cout << "GPU ACTOR  computing\n";
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrixA);
-				auto arg2 = caf::cuda::create_in_arg(matrixB);
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
-
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-			
-			
-				//mask the transfer back to the cpu for scheduler
-				res_token -> release();
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				//std::cout << "GPU ACTOR done  computing\n";
-				// signal exit actor and quit
-				self->mail(1).send(exit_actor);
-				self->quit();
-
-			}
-
-				
-	};
-}
-
-
-
-// Stateful actor behavior
-// this actor does not invoke the scheduler at all 
-caf::behavior mmul_actor_fun_no_schedule(
-    caf::stateful_actor<mmul_actor_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims) {
-
-    self->state().N = N;
+    // nd_range used for all stages (adapt to your kernels as needed)
+    caf::cuda::nd_range range{
+        {(n + 255) / 256, 1, 1},
+        {256, 1, 1}
+    };
 
-    std::vector<int> matrix1(N * N);
-    std::vector<int> matrix2(N * N);
+    // helper to create and send a launch token
+    auto launch = [&](program_ptr prog, const std::string& stage) {
+        auto tok = make_launch_token(
+            prog,
+            range,
+            /*memory_usage=*/static_cast<int>(sizeof(float) * n),
+            stage,
+            self,
+            self->state().id  // dependency/demo id
+        );
+        anon_mail(tok).send(scheduler);
+    };
 
-    // send initial mail to self
-    self->mail(matrix1, matrix2, N).send(self);
+    // fire all three tokens (scheduler will reply with response_token on grants)
+    launch(p1, "stage1");
+    launch(p2, "stage2");
+    launch(p3, "stage3");
 
     return {
-        // GPU atom + matrices + N
-        [=](const std::vector<int>& matrixA,
-            const std::vector<int>& matrixB,
-            int N_local) {   // avoid shadowing outer N
-            
-	    
-    	    //std::cout << "Hello\n";
-	    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-            auto arg1 = caf::cuda::create_in_arg(matrixA);
-            auto arg2 = caf::cuda::create_in_arg(matrixB);
-            auto arg3 = caf::cuda::create_out_arg(N_local * N_local);
-            auto arg4 = caf::cuda::create_in_arg(N_local);
 
-            auto tempC = mmul.run(program, dims, self->state().id, arg1, arg2, arg3, arg4);
-            std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-	    self->mail(1).send(exit_actor);
-            self->quit();
+        // optional start message (kept for compatibility)
+        [=](const std::string& msg) {
+            if (msg == "start") {
+                // no-op (we already launched tokens above)
+            }
         },
 
-        // CPU verification
-        [=](const std::vector<int>& matrixA,
-            const std::vector<int>& matrixB,
-            const std::vector<int>& matrixC,
-            int N_local) {
+        // handle response tokens by name — opaque to reclaim payload
+        [=](caf::cuda::response_token_ptr res_token) mutable {
+
+            const auto& stage = res_token->name();
+
+            // --------------------- Stage 1: init_denominators ---------------------
+            if (stage == "stage1") {
+                // allocate device buffer for denominators (persist in state)
+                
+		caf::caf::out<float> buffer = caf::cuda::create_out_arg_with_size<float>(n);
+                self->state().d_denoms = init_cmd.transfer_memory(res_token,bufer);
+
+                // run kernel on the stream/device from res_token
+                // kernel signature: (float* denominators, int n, unsigned long long seed)
+                init_cmd.run(
+                    p1,
+                    range,
+                    res_token,                            // uses token's stream/device
+                    self->state().d_denoms,               // device buffer
+                    caf::cuda::create_in_arg(n),          // n
+                    caf::cuda::create_in_arg(1234ULL)     // seed
+                );
+
+                // stage1 intentionally no checks — data may contain zeros
+                return;
+            }
 
-            std::vector<int> result(N_local * N_local);
-            serial_matrix_multiply(matrixA, matrixB, result, N_local);
+            // --------------------- Stage 2: perform_division ---------------------
+            if (stage == "stage2") {
+                // allocate device buffer for results (persist in state)
+                self->state().d_results = caf::cuda::create_out_arg<float>(n);
+
+                // create a host numerators vector (all ones)
+                std::vector<float> h_nums(n, 1.0f);
+
+                // transfer numerators to device on the token's stream/device
+                // transfer_memory returns a caf::cuda::mem_ptr<float>
+                auto d_nums = div_cmd.transfer_memory(res_token, in_out{h_nums});
+
+                // run division kernel on the token's stream/device:
+                // kernel signature: (float* numerators, float* denominators, float* results, int n)
+                div_cmd.run(
+                    p2,
+                    range,
+                    res_token,
+                    d_nums,
+                    self->state().d_denoms,
+                    self->state().d_results,
+                    caf::cuda::create_in_arg(n)
+                );
+
+                // extract the device results back to host for verification.
+                // extract_vector will synchronize as needed.
+                h_results = self->state().d_results -> copy_to_host();
+
+                // check for NaN/Inf AFTER the kernel finished
+                bool fault = false;
+                for (float v : h_results) {
+                    if (!std::isfinite(v)) {
+                        fault = true;
+                        break;
+                    }
+                }
+
+                if (fault) {
+                    // inform supervisor and exit;
+                    anon_mail(std::string("crash")).send(supervisor);
+                    self->quit();
+                    return;
+                }
+
+                // stage2 passed — keep d_results in state for stage3
+                return;
+            }
 
-            if (result == matrixC) {
-                std::cout << "actor with id " << self->state().id << " references match\n";
-            } else {
-                std::cout << "actor with id " << self->state().id << " references did not match\n";
+            // --------------------- Stage 3: sum_results --------------------------
+            if (stage == "stage3") {
+                // allocate device scalar for sum result
+                self->state().d_sum = caf::cuda::create_out_arg<float>(1);
+
+                // run reduction on the token's stream/device:
+                // kernel signature: (float* results, float* final_sum, int n)
+                sum_cmd.run(
+                    p3,
+                    range,
+                    res_token,
+                    self->state().d_results,
+                    self->state().d_sum,
+                    caf::cuda::create_in_arg(n)
+                );
+
+                // extract final scalar
+		
+		std::vector<float> buf = self->state().d_sum -> copy_to_host();
+                float final_sum = buf[0];
+                std::cout << "[pipeline] completed, sum = " << final_sum << "\n";
+
+                // successful completion -> tell supervisor to tear everything down
+                anon_mail(std::string("done")).send(supervisor);
+
+                // quit the pipeline actor
+                self->quit();
+                return;
             }
 
-            self->quit();
+            // unknown stage: ignore or log
+            std::cerr << "[pipeline] received unknown response token: " << stage << "\n";
         }
     };
 }
@@ -412,614 +213,50 @@ caf::behavior mmul_actor_fun_no_schedule(
 
 
 
-
-
-
-
-
-
-void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //change the scheduler to core_usage
-  anon_mail(
-	caf::cuda::make_behavior_token("core_usage")
-	).send(mgr.get_scheduler_actor());
-
-  // CREATE ONCE
-  auto program =
-      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-  const int THREADS = 32;
-  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(
-      BLOCKS, BLOCKS, 1,
-      THREADS, THREADS, 1);
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-  for (int i = 0; i < num_actors; ++i) {
-    /*
-    sys.spawn(
-        mmul_actor_fun,
-        exit_actor,
-        matrix_size,
-        program,
-        dims);
-    */
-      sys.spawn(
-        mmul_actor_fun_no_verify,
-        exit_actor,
-        matrix_size,
-        program,
-        dims,
-	true);
-    
-  }
-
-  sys.await_all_actors_done();
-}
-
-
-void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  /*
-  //change the scheduler to core_usage
-  anon_mail(
-	caf::cuda::make_behavior_token("core_usage")
-	).send(mgr.get_scheduler_actor());
-
-  */
-
-
-  // CREATE ONCE
-  auto program =
-      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-  const int THREADS = 32;
-  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(
-      BLOCKS, BLOCKS, 1,
-      THREADS, THREADS, 1);
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-  for (int i = 0; i < num_actors; ++i) {
-    
-	sys.spawn(
-        mmul_actor_fun_no_verify,
-        exit_actor,
-        matrix_size,
-        program,
-        dims,
-	true);
-    
-  
-	/*  
-sys.spawn(
-        mmul_actor_fun,
-        exit_actor,
-        matrix_size,
-        program,
-        dims);
-  */
-  }
-
-  sys.await_all_actors_done();
-}
-
-void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, int num_actors) {
-    if (num_actors < 1) {
-        std::cerr << "[ERROR] Number of actors must be >= 1\n";
-        return;
-    }
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-    // CREATE ONCE
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    const int THREADS = 32;
-    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    for (int i = 0; i < num_actors; ++i) {
-        sys.spawn(
-            mmul_actor_fun_no_schedule,
-            exit_actor,
-            matrix_size,
-            program,
-            dims
-        );
-    }
-
-    sys.await_all_actors_done();
-}
-
-
-template <class Fn>
-double time_run(Fn&& fn) {
-  auto start = std::chrono::steady_clock::now();
-  fn();
-  auto end = std::chrono::steady_clock::now();
-  std::chrono::duration<double> elapsed = end - start;
-  return elapsed.count();
-}
-void run_mmul_scaling_tests(caf::actor_system& sys,
-                            caf::cuda::manager_config man_config) {
-    const int max_size   = 1024;
-    const int min_actors = 1;
-    const int max_actors = 1024;
-
-    std::vector<int> matrix_sizes = {10};
-    for (int s = 32; s <= max_size; s *= 2)
-        matrix_sizes.push_back(s);
-
-    std::vector<int> actor_counts;
-    for (int a = min_actors; a <= max_actors; a *= 2)
-        actor_counts.push_back(a);
-
-    std::cout << "=== MMUL Scaling Tests ===\n";
-    std::cout << "Format:\n";
-    std::cout << "scheduler matrix_size actors time_seconds\n";
-
-    for (int size : matrix_sizes) {
-        for (int actors : actor_counts) {
-
-            /* ================= Scheduler-enabled (core_usage) ================= */
-            caf::cuda::manager::init(sys, man_config); // green-light scheduler enabled
-            std::cout << "\n[RUN] scheduler=core_usage "
-                      << "matrix_size=" << size
-                      << " actors=" << actors << "\n";
-
-            double core_usage_time = time_run([&] {
-                run_mmul_test(sys, size, actors); // uses mmul_actor_fun_no_verify
-            });
-
-            std::cout << std::fixed << std::setprecision(6)
-                      << "RESULT core_usage "
-                      << size << " "
-                      << actors << " "
-                      << core_usage_time << "\n";
-
-            caf::cuda::manager::shutdown(); // make sure manager is cleaned up
-
-            /* ================= Scheduler-disabled actor (still uses green-light) ================= */
-
-/*
-	    caf::cuda::manager::init(sys, man_config); // init with scheduler
-            std::cout << "\n[RUN] scheduler=green_light_only "
-                      << "matrix_size=" << size
-                      << " actors=" << actors << "\n";
-
-            double green_light_time = time_run([&] {
-                run_mmul_test_no_scheduler(sys, size, actors); // your previous "no scheduler" actor
-            });
-
-            std::cout << std::fixed << std::setprecision(6)
-                      << "RESULT green_light_only "
-                      << size << " "
-                      << actors << " "
-                      << green_light_time << "\n";
-
-            caf::cuda::manager::shutdown();
-
-	    */
-            /* ================= No scheduler at all actor ================= */
-            caf::cuda::manager_config no_sched_config(false); // disable scheduler
-            caf::cuda::manager::init(sys, no_sched_config);
-            std::cout << "\n[RUN] scheduler=none "
-                      << "matrix_size=" << size
-                      << " actors=" << actors << "\n";
-
-            double no_scheduler_time = time_run([&] {
-                run_mmul_test_no_scheduler_actor(sys, size, actors); // mmul_actor_fun_no_schedule
-            });
-
-            std::cout << std::fixed << std::setprecision(6)
-                      << "RESULT none "
-                      << size << " "
-                      << actors << " "
-                      << no_scheduler_time << "\n";
-
-            caf::cuda::manager::shutdown();
-        }
-    }
-
-    std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
-}
-void run_mmul_mixed_batch_one_mode(
-    caf::actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors,
-    bool use_scheduler_actor,
-    bool use_core_usage_behavior,
-    bool randomize = false)
+behavior supervisor_actor(event_based_actor* self,
+                          actor_system& system,
+                          program_ptr p1,
+                          program_ptr p2,
+                          program_ptr p3,
+                          int n)
 {
-    caf::cuda::manager& mgr = caf::cuda::manager::get(); // SAFE NOW
-
-    if (use_scheduler_actor && use_core_usage_behavior) {
-        anon_mail(caf::cuda::make_behavior_token("core_usage"))
-            .send(mgr.get_scheduler_actor());
-    }
-
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    std::mt19937 rng(123456);
-    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
-
-    const int THREADS = 32;
-
-    for (int i = 0; i < num_actors; ++i) {
-        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
-        int BLOCKS = (N + THREADS - 1) / THREADS;
+    auto spawn_pipeline = [&]() {
+        auto p = self->spawn(
+            pipeline_actor,
+            self,
+            p1,
+            p2,
+            p3,
+            n
+        );
+        anon_send(p, std::string("start"));
+    };
 
-        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+    spawn_pipeline();
 
-        if (use_scheduler_actor) {
-            sys.spawn(
-                mmul_actor_fun_no_verify,
-                exit_actor,
-                N,
-                program,
-                dims,
-		true);
-        } else {
-            sys.spawn(
-                mmul_actor_fun_no_schedule,
-                exit_actor,
-                N,
-                program,
-                dims);
+    return {
+        [=](const std::string& msg) {
+            if (msg == "crash") {
+                aout(self) << "Pipeline crashed — restarting\n";
+                spawn_pipeline();
+            }
+            else if (msg == "done") {
+                aout(self) << "Pipeline completed — shutting down\n";
+                caf::cuda::manager::shutdown();
+                system.shutdown();
+            }
         }
-    }
-
-    sys.await_all_actors_done();
-}
-
-
-
-
-void run_mmul_mixed_batch_one_mode_bulk(
-    caf::actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors,
-    bool randomize = false)
-{
-    caf::cuda::manager& mgr = caf::cuda::manager::get(); // SAFE NOW
-
-        anon_mail(caf::cuda::make_behavior_token("core_usage"))
-            .send(mgr.get_scheduler_actor());
-
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    std::vector<caf::cuda::token_ptr> tokens(num_actors);
-
-    std::mt19937 rng(123456);
-    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
-
-    const int THREADS = 32;
-
-    for (int i = 0; i < num_actors; ++i) {
-        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
-        int BLOCKS = (N + THREADS - 1) / THREADS;
-
-	caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-	caf::actor a = sys.spawn(
-			mmul_actor_fun_no_verify,
-			exit_actor,
-			N,
-			program,
-			dims,
-			false);
-
-	tokens[i] = caf::cuda::make_launch_token(
-			program,
-			dims,
-			0 /*this should not be 0 but its fine for now*/,
-			"hello",
-			a);    
-    }
-
-    
-        anon_mail(tokens)
-            .send(mgr.get_scheduler_actor());
-
-
-    sys.await_all_actors_done();
-}
-
-
-
-
-
-
-
-
-
-
-void run_mmul_mixed_batch_comparison(
-    caf::actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors)
-{
-    std::cout << "\n=== MMUL Mixed-Size Batch Comparison ===\n";
-    std::cout << "scheduler actors sizes time_seconds\n\n";
-
-    /* ================= core_usage ================= */
-    {
-        caf::cuda::manager_config cfg(true);
-        caf::cuda::manager::init(sys, cfg);
-
-	
-        //double t = time_run([&] {
-          //  run_mmul_mixed_batch_one_mode(
-            //    sys, sizes, num_actors,
-              //  /*use_scheduler_actor=*/true,
-               // /*use_core_usage_behavior=*/true);
-       // });
-
-	double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode_bulk(
-                sys, sizes, num_actors);
-        });
-
-
-
-
-        std::cout << "RESULT core_usage "
-                  << num_actors << " "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    /* ================= green-light only ================= */
-   
-    /*
-    {
-	    std::cout << "Starting green_light tests\n";
-        caf::cuda::manager_config cfg(true);
-        caf::cuda::manager::init(sys, cfg);
-
-        double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode(
-                sys, sizes, num_actors,
-               true,
-                false);
-        });
-
-        std::cout << "RESULT green_light_only "
-                  << num_actors << " "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    */
-	
-    /* ================= no scheduler ================= */
-    {
-        caf::cuda::manager_config cfg(false);
-        caf::cuda::manager::init(sys, cfg);
-
-        double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode(
-                sys, sizes, num_actors,
-                /*use_scheduler_actor=*/false,
-                /*use_core_usage_behavior=*/false);
-        });
-
-        std::cout << "RESULT none "
-                  << num_actors << " "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    std::cout << "\n=== Comparison Complete ===\n";
-}
-
-
-
-void test_core_usage_uniform_mmul(
-    caf::actor_system& sys,
-    int matrix_size,
-    int num_actors)
-{
-    std::cout << "\n[TEST] core_usage uniform matrix size\n";
-    std::cout << "N=" << matrix_size
-              << " actors=" << num_actors << "\n";
-
-    caf::cuda::manager_config cfg(true);
-    caf::cuda::manager::init(sys, cfg);
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-    // force core_usage behavior
-    anon_mail(caf::cuda::make_behavior_token("core_usage"))
-        .send(mgr.get_scheduler_actor());
-
-    auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    const int THREADS = 32;
-    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-    caf::cuda::nd_range dims(
-        BLOCKS, BLOCKS, 1,
-        THREADS, THREADS, 1);
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    for (int i = 0; i < num_actors; ++i) {
-        sys.spawn(
-            mmul_actor_fun_no_verify,
-            exit_actor,
-            matrix_size,
-            program,
-            dims,
-	    false);
-    }
-
-    sys.await_all_actors_done();
-    caf::cuda::manager::shutdown();
-
-    std::cout << "[PASS] core_usage uniform test complete\n";
-}
-
-
-
-void test_core_usage_mixed_mmul(
-    caf::actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors)
-{
-    std::cout << "\n[TEST] core_usage mixed matrix sizes\n";
-    std::cout << "actors=" << num_actors << "\n";
-
-    caf::cuda::manager_config cfg(true);
-    caf::cuda::manager::init(sys, cfg);
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-    anon_mail(caf::cuda::make_behavior_token("core_usage"))
-        .send(mgr.get_scheduler_actor());
-
-    auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    const int THREADS = 32;
-
-    for (int i = 0; i < num_actors; ++i) {
-        int N = sizes[i % sizes.size()];
-
-        int BLOCKS = (N + THREADS - 1) / THREADS;
-        caf::cuda::nd_range dims(
-            BLOCKS, BLOCKS, 1,
-            THREADS, THREADS, 1);
-
-        sys.spawn(
-            mmul_actor_fun_no_verify,
-            exit_actor,
-            N,
-            program,
-            dims,
-	    false);
-    }
-
-    sys.await_all_actors_done();
-    caf::cuda::manager::shutdown();
-
-    std::cout << "[PASS] core_usage mixed-size test complete\n";
-}
-
-
-void run_mmul_fixed_256_batch_comparison(
-    caf::actor_system& sys,
-    int num_actors)
-{
-    // All actors run the same matrix size: 256
-    std::vector<int> sizes(num_actors, 256);
-
-    std::cout << "\n=== MMUL Fixed-Size (256) Batch Comparison ===\n";
-    std::cout << "scheduler actors size time_seconds\n\n";
-
-    /* ================= core_usage ================= */
-    {
-        caf::cuda::manager_config cfg(true);
-        caf::cuda::manager::init(sys, cfg);
-
-        double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode(
-                sys,
-                sizes,
-                num_actors,
-                /*use_scheduler_actor=*/true,
-                /*use_core_usage_behavior=*/true);
-        });
-
-        std::cout << "RESULT core_usage "
-                  << num_actors << " 256 "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    /* ================= no scheduler ================= */
-    {
-        caf::cuda::manager_config cfg(false);
-        caf::cuda::manager::init(sys, cfg);
-
-        double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode(
-                sys,
-                sizes,
-                num_actors,
-                /*use_scheduler_actor=*/false,
-                /*use_core_usage_behavior=*/false);
-        });
-
-        std::cout << "RESULT none "
-                  << num_actors << " 256 "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    std::cout << "\n=== Fixed-256 Comparison Complete ===\n";
+    };
 }
 
 
 
 
 
-void caf_main(caf::actor_system& sys) {
-  
-	//caf::cuda::manager_config man_config(true); //turns the scheduler on
-	//caf::cuda::manager::init(sys,man_config);
-        // run_mmul_test(sys,10,64);	
-	//run_mmul_scaling_tests(sys,man_config);
-
-   std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048,4096};
-    const int num_actors = 1000;
-    run_mmul_mixed_batch_comparison(sys, sizes, num_actors);    
-
-    //run_mmul_mixed_batch_one_mode_bulk(sys,sizes,num_actors);
-    //run_mmul_fixed_256_batch_comparison(sys, /*num_actors=*/200);
 
 
-	 //test_core_usage_uniform_mmul(sys, 256, 1000);
 
-  //std::vector<int> sizes = {32, 64, 128, 256, 512, 1024};
-    //test_core_usage_mixed_mmul(sys, sizes, 200);
-
-
-
-//tests will delete the old manager so will have to reinit if you do this 
-	//in conjunction with each other	
-	//caf::cuda::manager::init(sys,man_config);
-}
 
 
 
 
-CAF_MAIN()

From d04ca0793657206f997681c38d05f48ff575900d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Feb 2026 11:28:03 -0600
Subject: [PATCH 0294/1000] Fixed syntax errors.

---
 .../fault-tolerance-test/main.test.cpp        | 51 +++++++++----------
 1 file changed, 24 insertions(+), 27 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
index 733d10cf30..19c273c001 100644
--- a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
@@ -29,13 +29,13 @@ struct exit_actor_state {
 
 // --- command runner types (put near top of file) -------------------------
 using initCommand =
-  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::in<int>, caf::cuda::in<unsigned long long>>;
+  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, in<int>, in<unsigned long long>>;
 
 using divCommand  =
-  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, caf::cuda::in<int>>;
+  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, in<int>>;
 
 using sumCommand  =
-  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, caf::cuda::in<int>>;
+  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, in<int>>;
 
 // single instances (can be file-global)
 static initCommand init_cmd;
@@ -55,9 +55,9 @@ struct pipeline_actor_state {
 // --- corrected pipeline_actor -------------------------------------------
 behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
                         actor supervisor,
-                        program_ptr p1,
-                        program_ptr p2,
-                        program_ptr p3,
+                        caf::cuda::program_ptr p1,
+                        caf::cuda::program_ptr p2,
+                        caf::cuda::program_ptr p3,
                         int n)
 {
     // host-side scratch (only used for post-stage2 NaN/Inf detection)
@@ -73,7 +73,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
     };
 
     // helper to create and send a launch token
-    auto launch = [&](program_ptr prog, const std::string& stage) {
+    auto launch = [&](caf::cuda::program_ptr prog, const std::string& stage) {
         auto tok = make_launch_token(
             prog,
             range,
@@ -92,13 +92,6 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 
     return {
 
-        // optional start message (kept for compatibility)
-        [=](const std::string& msg) {
-            if (msg == "start") {
-                // no-op (we already launched tokens above)
-            }
-        },
-
         // handle response tokens by name — opaque to reclaim payload
         [=](caf::cuda::response_token_ptr res_token) mutable {
 
@@ -108,12 +101,12 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
             if (stage == "stage1") {
                 // allocate device buffer for denominators (persist in state)
                 
-		caf::caf::out<float> buffer = caf::cuda::create_out_arg_with_size<float>(n);
-                self->state().d_denoms = init_cmd.transfer_memory(res_token,bufer);
+		out<float> buffer = caf::cuda::create_out_arg_with_size<float>(n);
+                self->state().d_denoms = init_cmd.transfer_memory(res_token,buffer);
 
                 // run kernel on the stream/device from res_token
                 // kernel signature: (float* denominators, int n, unsigned long long seed)
-                init_cmd.run(
+                init_cmd.run_async(
                     p1,
                     range,
                     res_token,                            // uses token's stream/device
@@ -129,14 +122,16 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
             // --------------------- Stage 2: perform_division ---------------------
             if (stage == "stage2") {
                 // allocate device buffer for results (persist in state)
-                self->state().d_results = caf::cuda::create_out_arg<float>(n);
+		    std::vector<float> buffer1(n); 
+
+		self->state().d_results = div_cmd.transfer_memory(res_token,out<float>{buffer1});
 
                 // create a host numerators vector (all ones)
                 std::vector<float> h_nums(n, 1.0f);
 
                 // transfer numerators to device on the token's stream/device
                 // transfer_memory returns a caf::cuda::mem_ptr<float>
-                auto d_nums = div_cmd.transfer_memory(res_token, in_out{h_nums});
+                auto d_nums = div_cmd.transfer_memory(res_token, in_out<float>{h_nums});
 
                 // run division kernel on the token's stream/device:
                 // kernel signature: (float* numerators, float* denominators, float* results, int n)
@@ -177,8 +172,11 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
             // --------------------- Stage 3: sum_results --------------------------
             if (stage == "stage3") {
                 // allocate device scalar for sum result
-                self->state().d_sum = caf::cuda::create_out_arg<float>(1);
+                
+		    std::vector<float> buffer1(1); 
 
+		self->state().d_sum = div_cmd.transfer_memory(res_token,out<float>{buffer1});
+		    
                 // run reduction on the token's stream/device:
                 // kernel signature: (float* results, float* final_sum, int n)
                 sum_cmd.run(
@@ -215,9 +213,9 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 
 behavior supervisor_actor(event_based_actor* self,
                           actor_system& system,
-                          program_ptr p1,
-                          program_ptr p2,
-                          program_ptr p3,
+                          caf::cuda::program_ptr p1,
+                          caf::cuda::program_ptr p2,
+                          caf::cuda::program_ptr p3,
                           int n)
 {
     auto spawn_pipeline = [&]() {
@@ -229,7 +227,6 @@ behavior supervisor_actor(event_based_actor* self,
             p3,
             n
         );
-        anon_send(p, std::string("start"));
     };
 
     spawn_pipeline();
@@ -237,13 +234,13 @@ behavior supervisor_actor(event_based_actor* self,
     return {
         [=](const std::string& msg) {
             if (msg == "crash") {
-                aout(self) << "Pipeline crashed — restarting\n";
+		std::cout << "Pipeline crashed — restarting\n";
                 spawn_pipeline();
             }
             else if (msg == "done") {
-                aout(self) << "Pipeline completed — shutting down\n";
+		std::cout << "Pipeline completed — shutting down\n";
                 caf::cuda::manager::shutdown();
-                system.shutdown();
+		self -> quit();
             }
         }
     };

From d1fb8120b3e0b796f69ba3ece87097d619a5e321 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Feb 2026 11:44:48 -0600
Subject: [PATCH 0295/1000] Added extern c to kernel to provent the nvcc from
 mangaling names making them unreadable by caf cuda.

---
 .../fault-tolerance-test/faulty_kernels.cu               | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/faulty_kernels.cu b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/faulty_kernels.cu
index c7b80e2c20..479b90711a 100644
--- a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/faulty_kernels.cu
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/faulty_kernels.cu
@@ -4,11 +4,8 @@
 #include <iostream>
 #include <cstdlib>
 
-// Error checking macro
-#define CUDA_CHECK(err) if (err != cudaSuccess) { std::cerr << "CUDA Error: " << cudaGetErrorString(err) << std::endl; exit(1); }
-
 // Step 1: Initialize denominators with ~50% zeros using cuRAND
-__global__ void init_denominators(float* denominators, int n, unsigned long long seed) {
+extern "C" __global__ void init_denominators(float* denominators, int n, unsigned long long seed) {
     int idx = blockIdx.x * blockDim.x + threadIdx.x;
     if (idx >= n) return;
 
@@ -19,14 +16,14 @@ __global__ void init_denominators(float* denominators, int n, unsigned long long
 }
 
 // Step 2: Perform division (potential div by zero -> Inf)
-__global__ void perform_division(float* numerators, float* denominators, float* results, int n) {
+extern "C" __global__ void perform_division(float* numerators, float* denominators, float* results, int n) {
     int idx = blockIdx.x * blockDim.x + threadIdx.x;
     if (idx >= n) return;
     results[idx] = numerators[idx] / denominators[idx];  // Triggers Inf if denominator == 0
 }
 
 // Step 3: Simple reduction to sum results (propagates Inf if present)
-__global__ void sum_results(float* results, float* final_sum, int n) {
+extern "C" __global__ void sum_results(float* results, float* final_sum, int n) {
     extern __shared__ float sdata[];
     int tid = threadIdx.x;
     int idx = blockIdx.x * blockDim.x + threadIdx.x;

From e344499d776b432f702070e7871b1a6717eaf783 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Feb 2026 14:05:21 -0600
Subject: [PATCH 0296/1000] Changed num streams from 0 to 1. THis change is
 being made to fix an unintentional division by zero error causes by
 get_next_stream method.

---
 libcaf_cuda/caf/cuda/control-layer/single_usage_behavior.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/single_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/single_usage_behavior.hpp
index fa6ad36d80..58b55f78c8 100644
--- a/libcaf_cuda/caf/cuda/control-layer/single_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/single_usage_behavior.hpp
@@ -45,7 +45,7 @@ class single_usage_behavior : public scheduler_actor_behavior {
     std::vector<kernel_graph> independent_graphs;           // no dependency
 
     // Stream management (even in serial mode, streams can be useful)
-    int num_streams = 0;
+    int num_streams = 1;
     int current_stream = 0;
 
     int64_t available_memory = 0;

From 4f0e078c45e9f377505a2795a76d2cc380831ba3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Feb 2026 14:45:16 -0600
Subject: [PATCH 0297/1000] Fixed supervisor actor running into strange memory
 issues when restarting the pipeline actor.

---
 .../fault-tolerance-test/main.test.cpp        | 76 +++++++++++++------
 1 file changed, 52 insertions(+), 24 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
index 19c273c001..4fabc96d29 100644
--- a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
@@ -210,38 +210,39 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 
 
 
+void supervisor_handle_msg(event_based_actor* self,
+                           caf::cuda::program_ptr p1,
+                           caf::cuda::program_ptr p2,
+                           caf::cuda::program_ptr p3,
+                           int n,
+                           const std::string& msg) {
+    if (msg == "crash") {
+        std::cout << "[supervisor] Pipeline crashed — restarting\n";
+        self->system().spawn(pipeline_actor, self, p1, p2, p3, n);
+    } else if (msg == "done") {
+        std::cout << "[supervisor] Pipeline completed — shutting down\n";
+        caf::cuda::manager::shutdown();
+        self->quit();
+    } else {
+        std::cerr << "[supervisor] Unknown message: " << msg << "\n";
+    }
+}
+
+
+
 
 behavior supervisor_actor(event_based_actor* self,
-                          actor_system& system,
                           caf::cuda::program_ptr p1,
                           caf::cuda::program_ptr p2,
                           caf::cuda::program_ptr p3,
-                          int n)
-{
-    auto spawn_pipeline = [&]() {
-        auto p = self->spawn(
-            pipeline_actor,
-            self,
-            p1,
-            p2,
-            p3,
-            n
-        );
-    };
-
-    spawn_pipeline();
+                          int n) {
+    // Spawn first pipeline safely
+    self->system().spawn(pipeline_actor, self, p1, p2, p3, n);
 
+    // Behavior: just route string messages to the helper
     return {
         [=](const std::string& msg) {
-            if (msg == "crash") {
-		std::cout << "Pipeline crashed — restarting\n";
-                spawn_pipeline();
-            }
-            else if (msg == "done") {
-		std::cout << "Pipeline completed — shutting down\n";
-                caf::cuda::manager::shutdown();
-		self -> quit();
-            }
+            supervisor_handle_msg(self, p1, p2, p3, n, msg);
         }
     };
 }
@@ -249,6 +250,33 @@ behavior supervisor_actor(event_based_actor* self,
 
 
 
+void caf_main(caf::actor_system& sys) {
+
+
+
+        caf::cuda::manager_config man_config(true); //turns the scheduler on
+        caf::cuda::manager::init(sys,man_config);
+
+	//change the scheduler to core_usage
+	anon_mail(
+        caf::cuda::make_behavior_token("single_usage")
+        ).send(caf::cuda::manager::get().get_scheduler_actor());
+
+
+	caf::cuda::program_ptr p1 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","init_denominators");
+	caf::cuda::program_ptr p2 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","perform_division");
+	caf::cuda::program_ptr p3 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","sum_results");
+
+	sys.spawn(supervisor_actor,p1,p2,p3,32);
+	sys.await_all_actors_done();
+
+
+}
+
+
+
+
+CAF_MAIN()
 
 
 

From 168fe6818493916e15fd0c2f21e6d79e175034ca Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Feb 2026 14:57:40 -0600
Subject: [PATCH 0298/1000] Changed n = 1 to the caf main, since when it was 32
 the probably of getting a zero denominator was too high causing infinite
 crash loops.

---
 .../fault-tolerance-test/main.test.cpp               | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
index 4fabc96d29..06c6ee0e0d 100644
--- a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
@@ -100,7 +100,13 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
             // --------------------- Stage 1: init_denominators ---------------------
             if (stage == "stage1") {
                 // allocate device buffer for denominators (persist in state)
-                
+               
+		    unsigned long long seed = static_cast<unsigned long long>(
+    std::chrono::high_resolution_clock::now().time_since_epoch().count()
+);
+
+
+
 		out<float> buffer = caf::cuda::create_out_arg_with_size<float>(n);
                 self->state().d_denoms = init_cmd.transfer_memory(res_token,buffer);
 
@@ -112,7 +118,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
                     res_token,                            // uses token's stream/device
                     self->state().d_denoms,               // device buffer
                     caf::cuda::create_in_arg(n),          // n
-                    caf::cuda::create_in_arg(1234ULL)     // seed
+                    caf::cuda::create_in_arg(seed)     // seed
                 );
 
                 // stage1 intentionally no checks — data may contain zeros
@@ -267,7 +273,7 @@ void caf_main(caf::actor_system& sys) {
 	caf::cuda::program_ptr p2 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","perform_division");
 	caf::cuda::program_ptr p3 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","sum_results");
 
-	sys.spawn(supervisor_actor,p1,p2,p3,32);
+	sys.spawn(supervisor_actor,p1,p2,p3,1);
 	sys.await_all_actors_done();
 
 

From cf83e83f7c8ec01b52213af2887ed20197b5acf6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Feb 2026 14:58:48 -0600
Subject: [PATCH 0299/1000] Changed sum results kernel to not used shared
 memory since, did not need to use it and was crashing my program. THis is a
 fault tolerance test not a performance test.

---
 .../fault-tolerance-test/faulty_kernels.cu      | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/faulty_kernels.cu b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/faulty_kernels.cu
index 479b90711a..06ec34c594 100644
--- a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/faulty_kernels.cu
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/faulty_kernels.cu
@@ -24,21 +24,10 @@ extern "C" __global__ void perform_division(float* numerators, float* denominato
 
 // Step 3: Simple reduction to sum results (propagates Inf if present)
 extern "C" __global__ void sum_results(float* results, float* final_sum, int n) {
-    extern __shared__ float sdata[];
-    int tid = threadIdx.x;
     int idx = blockIdx.x * blockDim.x + threadIdx.x;
 
-    sdata[tid] = (idx < n) ? results[idx] : 0.0f;
-    __syncthreads();
-
-    for (int s = blockDim.x / 2; s > 0; s >>= 1) {
-        if (tid < s) {
-            sdata[tid] += sdata[tid + s];
-        }
-        __syncthreads();
-    }
-
-    if (tid == 0) {
-        atomicAdd(final_sum, sdata[0]);
+    if (idx < n) {
+        // atomic add each element directly to the final sum
+        atomicAdd(final_sum, results[idx]);
     }
 }

From 0ea1b296288915958d95133ca6a260f768c41c93 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Feb 2026 14:59:24 -0600
Subject: [PATCH 0300/1000] Added logging to its methods, since this is a test
 behavior anyways.

---
 libcaf_cuda/src/control-layer/single_usage_behavior.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/single_usage_behavior.cpp b/libcaf_cuda/src/control-layer/single_usage_behavior.cpp
index 32daf9a870..3863c684de 100644
--- a/libcaf_cuda/src/control-layer/single_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/single_usage_behavior.cpp
@@ -32,6 +32,8 @@ void single_usage_behavior::reclaim(int blocks_consumed,
                                     int memory_returned,
                                     int time,
                                     int dependency_number) {
+   
+    std::cout << "reclaiming\n";
     // GPU is now free again
     gpu_available = true;
     available_memory += memory_returned;

From a34c635186d3cc932cebfa73f4a2d229526023a6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 5 Feb 2026 12:30:30 -0600
Subject: [PATCH 0301/1000] Initial commit.

---
 .../multilevel_usage_behavior.hpp             |  77 +++++++
 .../multilevel_usage_behavior.cpp             | 194 ++++++++++++++++++
 2 files changed, 271 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
 create mode 100644 libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
new file mode 100644
index 0000000000..b440060c02
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
@@ -0,0 +1,77 @@
+#pragma once
+
+#include "caf/cuda/control-layer/behavior.hpp"
+#include "caf/cuda/control-layer/kernel_graph.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp"
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/manager.hpp"
+#include "caf/cuda/control-layer/profiler.hpp"
+
+#include <deque>
+#include <optional>
+#include <algorithm>
+
+namespace caf::cuda {
+
+// Multilevel queue scheduling behavior (low / medium / high)
+// - Graphs are classified by the *next operation's* cost relative to total_SM
+// - Classification thresholds:
+//     low:    cost <= total_SM
+//     medium: cost <= 16 * total_SM
+//     high:   cost >  16 * total_SM
+// - schedule() will try to drain low first, then medium, then high.
+// - When a graph has work dispatched it is removed from the queues and
+//   not re-inserted. reclaim(...) can push graphs back into queues by
+//   looking up the dependency number and re-evaluating the front op.
+
+class multilevel_usage_behavior : public scheduler_actor_behavior {
+public:
+    explicit multilevel_usage_behavior(scheduler_actor_state& state);
+    ~multilevel_usage_behavior() override;
+
+    void on_enter() override;
+    void schedule() override;
+    void receive(const token_ptr& tok) override;
+
+    // reclaim: called when resources are returned; dependency_number
+    // allows this behavior to find the graph that might now be ready
+    void reclaim(int blocks_consumed, int memory_returned, int time, int dependency_number) override;
+
+    std::string name() const override { return "multilevel_usage\n"; }
+
+protected:
+    void process_launch_token(const token_ptr& tok, int stream_id) override;
+
+private:
+    device_ptr device_;
+    std::optional<sm_usage_heuristic> heuristic;
+
+    int total_SM = 0;
+    int available_SM = 0;
+    int available_memory = 0; // bytes
+    int num_streams = 0;
+    int current_stream = 0;
+
+    // multilevel queues of graph_refs
+    std::deque<graph_ref> low_queue;
+    std::deque<graph_ref> med_queue;
+    std::deque<graph_ref> high_queue;
+
+    void init_state();
+    void create_new_graph(const token_ptr& token);
+
+    int get_next_stream();
+
+    // classify & enqueue a graph reference based on its next op cost
+    void enqueue_graph_by_cost(const graph_ref& ref);
+
+    // attempt to dispatch as many graphs from q as possible (front-first)
+    void try_dispatch_queue(std::deque<graph_ref>& q);
+
+    kernel_graph* resolve(const graph_ref& ref);
+};
+
+
+
+} // namespace caf::cuda
+
diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
new file mode 100644
index 0000000000..9552b20402
--- /dev/null
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -0,0 +1,194 @@
+#include "caf/cuda/control-layer/multilevel_usage_behavior.hpp"
+
+namespace caf::cuda {
+
+
+multilevel_usage_behavior::multilevel_usage_behavior(scheduler_actor_state& state)
+    : scheduler_actor_behavior(state) {
+    init_state();
+}
+
+multilevel_usage_behavior::~multilevel_usage_behavior() {}
+
+void multilevel_usage_behavior::init_state() {
+    device_ = manager::get().find_device(state_.device_number);
+    heuristic.emplace(device_);
+    total_SM = device_->num_sms() * 16; // preserve previous semantics
+    available_SM = total_SM;
+    available_memory = static_cast<int>(device_->total_memory_bytes());
+    num_streams = state_.num_streams;
+}
+
+void multilevel_usage_behavior::on_enter() {
+    // nothing for now
+}
+
+void multilevel_usage_behavior::process_launch_token(const token_ptr& tok, int stream_id) {
+    scoped_timer timer("multilevel_usage_behavior::process_launch_token");
+
+    int cost = heuristic->getCost(tok);
+
+    const auto& launch = static_cast<const launch_token&>(*tok);
+    auto response = make_launch_response_token(state_.self, launch, state_.device_number, stream_id, cost);
+    anon_mail(response).send(launch.getReplyActor());
+    available_SM -= cost;
+}
+
+void multilevel_usage_behavior::receive(const token_ptr& tok) {
+    scoped_timer timer("multilevel_usage_behavior::receive");
+
+    if (tok->getType() == LAUNCH) {
+        create_new_graph(tok);
+        // after creating a graph we do not eagerly dispatch here; schedule() will
+        // be responsible for draining the queues in the desired order
+    } else if (tok->getType() == MEMORY) {
+        process_memory_transfer_token(tok, 0);
+    }
+}
+
+int multilevel_usage_behavior::get_next_stream() {
+    return current_stream++ % num_streams;
+}
+
+void multilevel_usage_behavior::create_new_graph(const token_ptr& tok) {
+    // create graph similar to core_usage_behavior but also enqueue by cost
+    if (tok->isIndependent()) {
+        kernel_graph new_graph(state_.device_number, get_next_stream());
+        new_graph.add_operation(tok);
+        independent_graphs.push_back(std::move(new_graph));
+        // reference to the newly added independent graph
+        graph_ref ref{graph_ref::kind_t::independent, -1, independent_graphs.size() - 1};
+        enqueue_graph_by_cost(ref);
+        return;
+    }
+
+    int dep = tok->getDependency();
+    if (graphs.contains(dep)) {
+        graphs[dep].add_operation(tok);
+        // If graph already existed, ensure it's enqueued only if not currently in any queue
+        // For simplicity we enqueue it — caller reclaim/schedule will ensure duplicates don't cause re-dispatch
+        graph_ref ref{graph_ref::kind_t::dependent, dep};
+        enqueue_graph_by_cost(ref);
+    } else {
+        kernel_graph new_graph(state_.device_number, get_next_stream());
+        new_graph.add_operation(tok);
+        graphs[dep] = std::move(new_graph);
+        graph_ref ref{graph_ref::kind_t::dependent, dep};
+        enqueue_graph_by_cost(ref);
+    }
+}
+
+void multilevel_usage_behavior::enqueue_graph_by_cost(const graph_ref& ref) {
+    kernel_graph* g = resolve(ref);
+    if (!g || g->empty()) return;
+    token_ptr tok = g->peek();
+    if (!tok || tok->getType() != LAUNCH) return;
+
+    int cost = heuristic->getCost(tok);
+    if (cost == ERROR_CODE) return;
+
+    // classification thresholds
+    const long long medium_threshold = 16LL * static_cast<long long>(total_SM);
+
+    if (cost <= total_SM) {
+        low_queue.push_back(ref);
+    } else if (cost <= medium_threshold) {
+        med_queue.push_back(ref);
+    } else {
+        high_queue.push_back(ref);
+    }
+}
+
+void multilevel_usage_behavior::try_dispatch_queue(std::deque<graph_ref>& q) {
+    // Dispatch front-first while resources allow
+    while (!q.empty()) {
+        graph_ref ref = q.front();
+        kernel_graph* g = resolve(ref);
+        if (!g || g->empty()) {
+            q.pop_front();
+            continue;
+        }
+
+        token_ptr tok = g->peek();
+        if (!tok || tok->getType() != LAUNCH) {
+            // not a launch op at front => remove and continue
+            q.pop_front();
+            continue;
+        }
+
+        int cost = heuristic->getCost(tok);
+        if (cost == ERROR_CODE) {
+            q.pop_front();
+            continue;
+        }
+
+        if (available_SM >= cost) {
+            // we have enough resources, dispatch
+            q.pop_front();
+            tok = g->getOperation();
+            process_launch_token(tok, g->stream_id());
+
+            // do not reinsert; when that graph becomes ready again it will be
+            // re-enqueued by reclaim(...)
+        } else {
+            // not enough resources for this graph; stop trying this queue
+            break;
+        }
+    }
+}
+
+void multilevel_usage_behavior::schedule() {
+    scoped_timer timer("multilevel_usage_behavior::schedule");
+
+    // Prioritize low, then medium, then high
+    try_dispatch_queue(low_queue);
+    try_dispatch_queue(med_queue);
+    try_dispatch_queue(high_queue);
+}
+
+void multilevel_usage_behavior::reclaim(int blocks_consumed,
+                                       int memory_returned,
+                                       int time,
+                                       int dependency_number) {
+    // update available resources
+    available_SM += blocks_consumed;
+    available_memory += memory_returned;
+
+    // If this reclaim call references a dependent graph, re-enqueue its graph
+    if (dependency_number != INDEPENDENT) {
+        if (graphs.contains(dependency_number)) {
+            graph_ref ref{graph_ref::kind_t::dependent, dependency_number};
+            enqueue_graph_by_cost(ref);
+        }
+    } else {
+        // dependency_number < 0: we don't have a direct index for independent graphs here.
+        // As a best-effort, enqueue any independent graphs that are non-empty
+        //for (std::size_t i = 0; i < independent_graphs.size(); ++i) {
+          ///  if (!independent_graphs[i].empty()) {
+             //   graph_ref ref{graph_ref::kind_t::independent, -1, i};
+               // enqueue_graph_by_cost(ref);
+            //}
+       // }
+    }
+
+    // After re-enqueue, attempt to schedule immediately
+    schedule();
+}
+
+kernel_graph* multilevel_usage_behavior::resolve(const graph_ref& ref) {
+    switch (ref.kind) {
+        case graph_ref::kind_t::dependent: {
+            auto it = graphs.find(ref.dependency);
+            if (it == graphs.end()) return nullptr;
+            return &it->second;
+        }
+        case graph_ref::kind_t::independent: {
+            if (ref.index >= independent_graphs.size()) return nullptr;
+            return &independent_graphs[ref.index];
+        }
+    }
+    return nullptr;
+}
+
+} // namespace caf::cuda
+

From 4e9e05b2dfdf28a95efc70036be63c6ffb3de9b0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 5 Feb 2026 12:42:10 -0600
Subject: [PATCH 0302/1000] Fixed compiler errors, squashed logic bugs related
 to a deadlock occuring right at the start (see receive method and how it
 would never start scheduling) and added some documentation.

---
 .../caf/cuda/control-layer/multilevel_usage_behavior.hpp | 7 ++++++-
 .../src/control-layer/multilevel_usage_behavior.cpp      | 9 ++++++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
index b440060c02..c04fc904e2 100644
--- a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
@@ -5,11 +5,12 @@
 #include "caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp"
 #include "caf/cuda/device.hpp"
 #include "caf/cuda/manager.hpp"
-#include "caf/cuda/control-layer/profiler.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/profiler.hpp"
 
 #include <deque>
 #include <optional>
 #include <algorithm>
+#include <unordered_map>
 
 namespace caf::cuda {
 
@@ -52,6 +53,10 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
     int num_streams = 0;
     int current_stream = 0;
 
+    //tracking graphs
+    std::unordered_map<int,kernel_graph> graphs;
+    std::vector<kernel_graph> independent_graphs;
+
     // multilevel queues of graph_refs
     std::deque<graph_ref> low_queue;
     std::deque<graph_ref> med_queue;
diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 9552b20402..e91f8460b6 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -1,4 +1,5 @@
 #include "caf/cuda/control-layer/multilevel_usage_behavior.hpp"
+#include "caf/cuda/control-layer/all-control-layer.hpp"
 
 namespace caf::cuda {
 
@@ -39,8 +40,7 @@ void multilevel_usage_behavior::receive(const token_ptr& tok) {
 
     if (tok->getType() == LAUNCH) {
         create_new_graph(tok);
-        // after creating a graph we do not eagerly dispatch here; schedule() will
-        // be responsible for draining the queues in the desired order
+	schedule();
     } else if (tok->getType() == MEMORY) {
         process_memory_transfer_token(tok, 0);
     }
@@ -67,7 +67,10 @@ void multilevel_usage_behavior::create_new_graph(const token_ptr& tok) {
         graphs[dep].add_operation(tok);
         // If graph already existed, ensure it's enqueued only if not currently in any queue
         // For simplicity we enqueue it — caller reclaim/schedule will ensure duplicates don't cause re-dispatch
-        graph_ref ref{graph_ref::kind_t::dependent, dep};
+        //this may lead in an error where dependencies are triggered before they are ready
+	//however since each graph gets a designated stream for now 
+	//this will not until that happens 
+	graph_ref ref{graph_ref::kind_t::dependent, dep};
         enqueue_graph_by_cost(ref);
     } else {
         kernel_graph new_graph(state_.device_number, get_next_stream());

From 2375b2006f6e267af507952c90c5deb49aed8232 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 5 Feb 2026 12:45:12 -0600
Subject: [PATCH 0303/1000] Addeded multilevel scheduler to the behavior table.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index d799b7d1dd..89a65e90bf 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -4,6 +4,7 @@
 #include "caf/cuda/control-layer/red_light_behavior.hpp"
 #include "caf/cuda/control-layer/core_usage_behavior.hpp"
 #include "caf/cuda/control-layer/single_usage_behavior.hpp"
+#include "caf/cuda/control-layer/multilevel_usage_behavior.hpp"
 #include <string>
 #include <iostream>
 
@@ -26,12 +27,13 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
     static green_light_behavior green_behavior(state);
     static core_usage_behavior core_behavior(state);
     static single_usage_behavior single_behavior(state);
+    static multilevel_usage_behavior multi_behavior(state);
 
     // populate the behavior table
     state.table.add("red", &red_behavior);
     state.table.add("green",   &green_behavior);
     state.table.add("core_usage",   &core_behavior);
-    state.table.add("single_usage",   &single_behavior);
+    state.table.add("multilevel",   &multi_behavior);
 
     // default behavior
     state.current_behavior = state.table.get(behavior_token("green"));

From 2419a28e90c1b83c7463eea810c6b4190d87a33e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 5 Feb 2026 12:49:24 -0600
Subject: [PATCH 0304/1000] Updated test_core_usage mixed and uniform to have a
 string parameter that indicates what kind of scheduler you want Change was
 made to make the tests applicable to multiple schedulers rather than just
 one.

---
 .../core_usage_behavior_tests/main.test.cpp          | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index 8047a7d93b..facf62de3d 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -842,7 +842,8 @@ void run_mmul_mixed_batch_comparison(
 void test_core_usage_uniform_mmul(
     caf::actor_system& sys,
     int matrix_size,
-    int num_actors)
+    int num_actors,
+    std::string scheduler_behavior)
 {
     std::cout << "\n[TEST] core_usage uniform matrix size\n";
     std::cout << "N=" << matrix_size
@@ -854,7 +855,7 @@ void test_core_usage_uniform_mmul(
     caf::cuda::manager& mgr = caf::cuda::manager::get();
 
     // force core_usage behavior
-    anon_mail(caf::cuda::make_behavior_token("core_usage"))
+    anon_mail(caf::cuda::make_behavior_token(scheduler_behavior))
         .send(mgr.get_scheduler_actor());
 
     auto program =
@@ -889,7 +890,8 @@ void test_core_usage_uniform_mmul(
 void test_core_usage_mixed_mmul(
     caf::actor_system& sys,
     const std::vector<int>& sizes,
-    int num_actors)
+    int num_actors,
+    std::string scheduler_behavior)
 {
     std::cout << "\n[TEST] core_usage mixed matrix sizes\n";
     std::cout << "actors=" << num_actors << "\n";
@@ -899,7 +901,7 @@ void test_core_usage_mixed_mmul(
 
     caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-    anon_mail(caf::cuda::make_behavior_token("core_usage"))
+    anon_mail(caf::cuda::make_behavior_token(scheduler_behavior))
         .send(mgr.get_scheduler_actor());
 
     auto program =
@@ -999,7 +1001,7 @@ void caf_main(caf::actor_system& sys) {
         // run_mmul_test(sys,10,64);	
 	//run_mmul_scaling_tests(sys,man_config);
 
-   std::vector<int> sizes = {32, 64, 128, 256, 512, 1024};
+   std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048,4096};
     const int num_actors = 1000;
     run_mmul_mixed_batch_comparison(sys, sizes, num_actors);    
 

From a924afb92ea4463984a11126bcf84aa14ca6a9a4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 5 Feb 2026 12:59:54 -0600
Subject: [PATCH 0305/1000] Updated core usage test of both mixed and uniform
 mmul to ensure actors actually send a request token. Change was made to fix a
 deadlock that would ensure since actors never sent a message the scheduler
 never did anything to fix it.

---
 .../core_usage_behavior_tests/main.test.cpp            | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index facf62de3d..860bebae65 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -876,7 +876,7 @@ void test_core_usage_uniform_mmul(
             matrix_size,
             program,
             dims,
-	    false);
+	    true);
     }
 
     sys.await_all_actors_done();
@@ -925,7 +925,7 @@ void test_core_usage_mixed_mmul(
             N,
             program,
             dims,
-	    false);
+	    true);
     }
 
     sys.await_all_actors_done();
@@ -1001,15 +1001,19 @@ void caf_main(caf::actor_system& sys) {
         // run_mmul_test(sys,10,64);	
 	//run_mmul_scaling_tests(sys,man_config);
 
+	/*
    std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048,4096};
     const int num_actors = 1000;
     run_mmul_mixed_batch_comparison(sys, sizes, num_actors);    
 
+    */
+
     //run_mmul_mixed_batch_one_mode_bulk(sys,sizes,num_actors);
     //run_mmul_fixed_256_batch_comparison(sys, /*num_actors=*/200);
 
 
-	 //test_core_usage_uniform_mmul(sys, 256, 1000);
+	 test_core_usage_uniform_mmul(sys, 256, 1000,"multilevel");
+	 //test_core_usage_mixed_mmul(sys, 256, 1000,"multilevel");
 
   //std::vector<int> sizes = {32, 64, 128, 256, 512, 1024};
     //test_core_usage_mixed_mmul(sys, sizes, 200);

From 58537e52bdb951abcefcfaeb437f52d27081f895 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 6 Feb 2026 14:54:38 -0600
Subject: [PATCH 0306/1000] Added vector of actors and a boolean flag to the
 state Change is being made so that other actors can load balance with each
 other via message passing.

---
 .../caf/cuda/control-layer/scheduler_actor_state.hpp      | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
index d9617d6dc8..b7f21640a4 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
@@ -16,6 +16,12 @@ struct scheduler_actor_state {
     std::queue<token_ptr> queue; // here for legacy prototype schedulers
     std::vector<kernel_graph> operations; //more modern dependency based data structure 
     int device_number;
-    int num_streams = 32; // number of streams that can be used by the scheduler
+    int num_streams = 150; // number of streams that can be used by the scheduler
+    std::vector<caf::actor> schedulers; //the other scheduler actors in the system, an actor for a 
+			     //specific GPU can be accessed via there corrosponding 
+			     //device number in the std::vector
+   bool multiple_gpus = false; //flag that will indicate to the actor whether or not there is
+			       //multiple GPUs 
+
 };
 } // namespace caf::cuda

From f6a11e2aeeea302d351a7a7dde3419f2e5e1a635 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 6 Feb 2026 15:15:38 -0600
Subject: [PATCH 0307/1000] Added methods for behaviors to handle load
 balancing between multiple GPUs

---
 .../caf/cuda/control-layer/behavior.hpp       | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
index d6bbe0279a..8eba5f5129 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
@@ -38,6 +38,29 @@ class scheduler_actor_behavior {
 
     virtual std::string name() const {return "No name\n";}
 
+    //this method is meant to be a handler for when
+    //another scheduler actor queries for more work
+    virtual void handle_load_balance_request(int device_number) {
+    
+	    //default action is to do nothing and not particpate in load balancing
+	    //whether of not a scheduler wants to participate in load balancing 
+	    //and what actions it should take is a policy decision
+    }
+
+
+    //method is meant to handle work being sent over from another scheduler actor
+   virtual void receive_work(std::vector<kernel_graph> work_graphs) {
+     //ideally this should not default to do nothing 
+    //however I do not have the time implement this on every existing behavior 
+    //as of right now
+    //so be warned if you do not implement an override and request work to do be done
+    //this will end in a deadlock  
+    
+    
+    }
+
+
+
 
 protected:
     scheduler_actor_state& state_;

From a280d2f6cf3e7539c1f520d7c2936f5522ccca51 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 6 Feb 2026 15:23:37 -0600
Subject: [PATCH 0308/1000] Implemented get num_devices method into platform.
 Change was made since the manager may want to know the number of GPUs on the
 system.

---
 libcaf_cuda/caf/cuda/platform.hpp | 4 ++++
 libcaf_cuda/src/platform.cpp      | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/platform.hpp b/libcaf_cuda/caf/cuda/platform.hpp
index 263e7649d3..26e61aea99 100644
--- a/libcaf_cuda/caf/cuda/platform.hpp
+++ b/libcaf_cuda/caf/cuda/platform.hpp
@@ -46,6 +46,10 @@ class CAF_CUDA_EXPORT platform : public ref_counted {
   //releases a stream for an actor
   void release_streams_for_actor(int actor_id);
 
+  //returns how many devices are currently on the GPU
+  int get_num_devices();
+
+
 private:
   platform();
   ~platform();
diff --git a/libcaf_cuda/src/platform.cpp b/libcaf_cuda/src/platform.cpp
index d7d925e56b..175dfe1808 100644
--- a/libcaf_cuda/src/platform.cpp
+++ b/libcaf_cuda/src/platform.cpp
@@ -83,6 +83,8 @@ device_ptr platform::getDevice(int id) {
   return devices_[id];
 }
 
+int platform::get_num_devices() { return devices_.size();}
+
 scheduler* platform::get_scheduler() {
   return scheduler_.get();
 }

From fa9d4e4f0f1054532ef0824fc3d5a83c310268c4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 6 Feb 2026 15:57:33 -0600
Subject: [PATCH 0309/1000] Updated managers create and shutdown schedule actor
 process to accomodate multiple scheduler actors.Furthermore updated scheduler
 actor to be able to understand if there is multiple scheduler actors and have
 a message handler to other scheduler actors. These changes were made to
 prepare for upcomming multi-gpu scheduling.

---
 .../cuda/control-layer/scheduler_actor.hpp    |  2 +-
 libcaf_cuda/caf/cuda/manager.hpp              |  2 +
 .../src/control-layer/scheduler_actor.cpp     | 12 +++-
 libcaf_cuda/src/manager.cpp                   | 57 +++++++++++++++++--
 4 files changed, 66 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
index eda0f22726..80c4e8b8dd 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
@@ -14,5 +14,5 @@
  */
 
 namespace caf::cuda {
-caf::behavior CAF_CUDA_EXPORT scheduler_actor(caf::stateful_actor<scheduler_actor_state> * self,int device_number);
+caf::behavior CAF_CUDA_EXPORT scheduler_actor(caf::stateful_actor<scheduler_actor_state> * self,int device_number,bool multi_gpu);
 }//namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index 5ffc3a3b95..3b20f5b907 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -186,11 +186,13 @@ class CAF_CUDA_EXPORT manager {
   //helper to compile a nvrtc program
   bool compile_nvrtc_program(const char* source, CUdevice device, std::vector<char>& ptx_out);
 
+  void init_scheduler_actors(caf::actor_system&);
 
   static manager* instance_;
   static std::mutex mutex_;
   bool scheduler_on = false;
   caf::actor scheduler_actor_handle;
+  std::vector<caf::actor> scheduler_actors;
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 89a65e90bf..079c81446d 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -14,7 +14,7 @@
  */
 namespace caf::cuda {
 
-caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self, int device_number) {
+caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self, int device_number,bool multi_gpu) {
     auto& state = self->state();
 
     // add self reference
@@ -23,6 +23,9 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
     // set device number
     state.device_number = device_number;
 
+    //check if multiple gpus
+    state.multiple_gpus = multi_gpu;
+
     static red_light_behavior red_behavior(state);
     static green_light_behavior green_behavior(state);
     static core_usage_behavior core_behavior(state);
@@ -81,6 +84,13 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
 	//message handler for reclaim
 	[&](int value, int memory,int runtime,int dependency) {	
 		state.current_behavior->reclaim(value,memory,runtime,dependency);
+	},
+
+	//handler sent to set the scheduler actors 
+	//do not send a message more than once
+	//or else undefined behavior
+	[&](std::vector<caf::actor> s) {
+		state.schedulers = s;
 	}
     };
 }
diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 8889a89fd9..7f54d5e012 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -58,11 +58,42 @@ void manager::init(caf::actor_system& sys, manager_config config) {
     instance_->scheduler_on = config.getSchedulerOn();
 
     if (instance_->scheduler_on) {
-        instance_->scheduler_actor_handle =
-            sys.spawn(scheduler_actor,0);
+        
+	    
+	    instance_ -> init_scheduler_actors(sys);
+	    //instance_->scheduler_actor_handle =
+          //  sys.spawn(scheduler_actor,0);
     }
 }
 
+
+
+void manager::init_scheduler_actors(caf::actor_system& sys) {
+
+	int num_devices = platform_ -> get_num_devices();
+
+	bool multi_gpu = num_devices > 1;
+	for (int i = 0; i < num_devices; i++) {
+	
+		instance_ -> scheduler_actors.push_back( sys.spawn(scheduler_actor,i,multi_gpu));
+	
+	}
+
+	//if there is multiple GPUs send every scheduler actor contact information 
+	//about the other on	
+	if (num_devices > 1) {
+	
+		for (int i = 0; i < num_devices; i++) {
+			anon_mail(scheduler_actors).send(instance_ -> scheduler_actors[i]);
+		}
+	
+	}
+
+}
+
+
+
+
 // --------------------------------
 // Static get()
 // --------------------------------
@@ -89,10 +120,23 @@ void manager::shutdown() {
         return;
 
     if (instance_->scheduler_on) {
-        anon_send_exit(
-            instance_->scheduler_actor_handle,
+        
+	for (int i = 0; i < instance_ ->  platform_ -> get_num_devices(); i++) {
+	    
+//	anon_send_exit(
+  //          instance_->scheduler_actor_handle,
+    //        caf::exit_reason::user_shutdown
+      //  );
+     
+	anon_send_exit(
+            instance_->scheduler_actors[i],
             caf::exit_reason::user_shutdown
         );
+
+
+
+    }
+
     }
 
     delete instance_;
@@ -102,15 +146,18 @@ void manager::shutdown() {
 // --------------------------------
 // Static getter for scheduler actor
 // --------------------------------
+// this is legacy code, do not use
+// only exists to be backwards compatable with tests 
 caf::actor manager::get_scheduler_actor() {
     	//this is a read only data no need for lock
 	//std::lock_guard<std::mutex> guard(mutex_);
 
+
     if (!instance_) {
         throw std::runtime_error("CUDA manager not initialized");
     }
 
-    return instance_->scheduler_actor_handle;
+    return instance_->scheduler_actors[0];
 }
 
 

From 00a427b8f86ee4130ffcfa27cf23f0da6a32266c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 8 Feb 2026 09:31:18 -0600
Subject: [PATCH 0310/1000] Iniital commit, copied straight from multilevel.

---
 .../cuda/control-layer/pressure_scheduler.hpp | 82 +++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp b/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
new file mode 100644
index 0000000000..c04fc904e2
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
@@ -0,0 +1,82 @@
+#pragma once
+
+#include "caf/cuda/control-layer/behavior.hpp"
+#include "caf/cuda/control-layer/kernel_graph.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp"
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/manager.hpp"
+#include "caf/cuda/control-layer/scheduler-functions/profiler.hpp"
+
+#include <deque>
+#include <optional>
+#include <algorithm>
+#include <unordered_map>
+
+namespace caf::cuda {
+
+// Multilevel queue scheduling behavior (low / medium / high)
+// - Graphs are classified by the *next operation's* cost relative to total_SM
+// - Classification thresholds:
+//     low:    cost <= total_SM
+//     medium: cost <= 16 * total_SM
+//     high:   cost >  16 * total_SM
+// - schedule() will try to drain low first, then medium, then high.
+// - When a graph has work dispatched it is removed from the queues and
+//   not re-inserted. reclaim(...) can push graphs back into queues by
+//   looking up the dependency number and re-evaluating the front op.
+
+class multilevel_usage_behavior : public scheduler_actor_behavior {
+public:
+    explicit multilevel_usage_behavior(scheduler_actor_state& state);
+    ~multilevel_usage_behavior() override;
+
+    void on_enter() override;
+    void schedule() override;
+    void receive(const token_ptr& tok) override;
+
+    // reclaim: called when resources are returned; dependency_number
+    // allows this behavior to find the graph that might now be ready
+    void reclaim(int blocks_consumed, int memory_returned, int time, int dependency_number) override;
+
+    std::string name() const override { return "multilevel_usage\n"; }
+
+protected:
+    void process_launch_token(const token_ptr& tok, int stream_id) override;
+
+private:
+    device_ptr device_;
+    std::optional<sm_usage_heuristic> heuristic;
+
+    int total_SM = 0;
+    int available_SM = 0;
+    int available_memory = 0; // bytes
+    int num_streams = 0;
+    int current_stream = 0;
+
+    //tracking graphs
+    std::unordered_map<int,kernel_graph> graphs;
+    std::vector<kernel_graph> independent_graphs;
+
+    // multilevel queues of graph_refs
+    std::deque<graph_ref> low_queue;
+    std::deque<graph_ref> med_queue;
+    std::deque<graph_ref> high_queue;
+
+    void init_state();
+    void create_new_graph(const token_ptr& token);
+
+    int get_next_stream();
+
+    // classify & enqueue a graph reference based on its next op cost
+    void enqueue_graph_by_cost(const graph_ref& ref);
+
+    // attempt to dispatch as many graphs from q as possible (front-first)
+    void try_dispatch_queue(std::deque<graph_ref>& q);
+
+    kernel_graph* resolve(const graph_ref& ref);
+};
+
+
+
+} // namespace caf::cuda
+

From 3c1876de547eb2a9108efe34c47b2e3b3e5a4245 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 8 Feb 2026 09:54:02 -0600
Subject: [PATCH 0311/1000] Updated hpp file.

---
 .../cuda/control-layer/pressure_scheduler.hpp | 89 ++++++++++++++-----
 1 file changed, 68 insertions(+), 21 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp b/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
index c04fc904e2..d3edfb9761 100644
--- a/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
@@ -12,23 +12,22 @@
 #include <algorithm>
 #include <unordered_map>
 
+
+#define LOW 14
+#define MEDIUM 15
+#define HIGH 16
+
+
 namespace caf::cuda {
 
-// Multilevel queue scheduling behavior (low / medium / high)
-// - Graphs are classified by the *next operation's* cost relative to total_SM
-// - Classification thresholds:
-//     low:    cost <= total_SM
-//     medium: cost <= 16 * total_SM
-//     high:   cost >  16 * total_SM
-// - schedule() will try to drain low first, then medium, then high.
 // - When a graph has work dispatched it is removed from the queues and
 //   not re-inserted. reclaim(...) can push graphs back into queues by
 //   looking up the dependency number and re-evaluating the front op.
 
-class multilevel_usage_behavior : public scheduler_actor_behavior {
+class pressure_scheduler : public scheduler_actor_behavior {
 public:
-    explicit multilevel_usage_behavior(scheduler_actor_state& state);
-    ~multilevel_usage_behavior() override;
+    explicit pressure_scheduler(scheduler_actor_state& state);
+    ~pressure_scheduler() override;
 
     void on_enter() override;
     void schedule() override;
@@ -36,9 +35,9 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
 
     // reclaim: called when resources are returned; dependency_number
     // allows this behavior to find the graph that might now be ready
-    void reclaim(int blocks_consumed, int memory_returned, int time, int dependency_number) override;
+    void reclaim(int resources_consumed, int memory_returned, int time, int dependency_number) override;
 
-    std::string name() const override { return "multilevel_usage\n"; }
+    std::string name() const override { return "pressure_scheduler\n"; }
 
 protected:
     void process_launch_token(const token_ptr& tok, int stream_id) override;
@@ -46,26 +45,53 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
 private:
     device_ptr device_;
     std::optional<sm_usage_heuristic> heuristic;
+    void init_state();
+
 
-    int total_SM = 0;
-    int available_SM = 0;
+    // resource values of the GPU
+    int total_SM = 0; //can be used for proportional resource consumption
     int available_memory = 0; // bytes
     int num_streams = 0;
     int current_stream = 0;
 
+
+    //threshold values
+    //should inited with int_state method
+    int resource_threshold; //if exceeded do not dispatch kernel
+    int resource_pressure; // tracks resources in use, if low we should dispatch heavy kernels, if high dispatch light kernels 
+    
+    int low_concurreny_threshold; //if we under this immediately accept any work
+				  //of if multiple gpus, seek out work
+
+    int high_concurrency_threshold; //if we are above this, enqueue any work
+				    //since could flood GPU with requests
+
+    int current_concurreny; //number to assign how much kernels on the GPU
+			    //values should be in proportion to how much 
+			    //resources a kernel intends to consume 
+
+    int compute_bound_pressure; //determines if we should favor compute or memory bound kernels when seeking work to dispatch 
+
+
+    //Methods and data structures that organize and dispatch kernels
+ 
+
     //tracking graphs
     std::unordered_map<int,kernel_graph> graphs;
     std::vector<kernel_graph> independent_graphs;
 
-    // multilevel queues of graph_refs
-    std::deque<graph_ref> low_queue;
-    std::deque<graph_ref> med_queue;
-    std::deque<graph_ref> high_queue;
+    // multilevel queues of graph_refs of 
+    // graphs whose next kernel is compute bound 
+    std::deque<graph_ref> low_compute_queue;
+    std::deque<graph_ref> med_compute_queue;
+    std::deque<graph_ref> high_compute_queue;
 
-    void init_state();
-    void create_new_graph(const token_ptr& token);
 
-    int get_next_stream();
+    // multilevel queues of graph_refs of 
+    // graphs whose next kernel is memory bound 
+    std::deque<graph_ref> low_memory_queue;
+    std::deque<graph_ref> med_memory_queue;
+    std::deque<graph_ref> high_memory_queue;
 
     // classify & enqueue a graph reference based on its next op cost
     void enqueue_graph_by_cost(const graph_ref& ref);
@@ -73,7 +99,28 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
     // attempt to dispatch as many graphs from q as possible (front-first)
     void try_dispatch_queue(std::deque<graph_ref>& q);
 
+    void dispatch_prefer_compute();
+    void dispatch_prefer_memory();
+
     kernel_graph* resolve(const graph_ref& ref);
+
+    void create_new_graph(const token_ptr& token);
+
+    int get_next_stream();
+
+
+    //methods that return some value of resources consumed 
+
+    //returns integer code signalling low medium or high
+    //should be used in combination of thresholds to decide how much 
+    //pressure it puts on a dimension of the GPU 
+    //(concurreny,memory vs compute bound, resource)
+    int get_resource_pressure(int blocks_consumed);
+
+    //helps determine how much concurrent work a kernel is going to use
+    int get_concurrency_pressure(int blocks_consumed); 
+    
+    
 };
 
 

From d40c22ebbef992aaa43e3e82a0d6f4a75fd1034e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 8 Feb 2026 10:09:53 -0600
Subject: [PATCH 0312/1000] Intial commit.

---
 .../src/control-layer/pressure_scheduler.cpp  | 272 ++++++++++++++++++
 1 file changed, 272 insertions(+)
 create mode 100644 libcaf_cuda/src/control-layer/pressure_scheduler.cpp

diff --git a/libcaf_cuda/src/control-layer/pressure_scheduler.cpp b/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
new file mode 100644
index 0000000000..c02b54116d
--- /dev/null
+++ b/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
@@ -0,0 +1,272 @@
+#include "caf/cuda/control-layer/pressure_scheduler.hpp"
+#include "caf/cuda/control-layer/all-control-layer.hpp"
+#include <iostream>
+
+namespace caf::cuda {
+
+//TODO overhaul this, this sucks 
+
+
+pressure_scheduler::pressure_scheduler(scheduler_actor_state& state)
+    : scheduler_actor_behavior(state) {
+    init_state();
+}
+
+pressure_scheduler::~pressure_scheduler() = default;
+
+void pressure_scheduler::init_state() {
+    device_ = manager::get().find_device(state_.device_number);
+    heuristic.emplace(device_);
+
+    total_SM = device_->num_sms();
+    available_memory = static_cast<int>(device_->total_memory_bytes());
+    num_streams = state_.num_streams;
+
+    // thresholds (tunable)
+    resource_threshold = total_SM * 16; // arbitrary guard
+
+    // concurrency thresholds: small, medium, high
+    low_concurreny_threshold = 1; // 0..1 treated as low
+    high_concurrency_threshold = device_->num_sms() * 4; // e.g., 4 kernels/SM
+
+    // initial accounting
+    current_concurreny = 0;
+    resource_pressure = 0;
+}
+
+void pressure_scheduler::on_enter() {
+    // nothing special at enter
+}
+
+void pressure_scheduler::receive(const token_ptr& tok) {
+    if (tok->getType() == LAUNCH) {
+        create_new_graph(tok);
+        schedule();
+    } else if (tok->getType() == MEMORY) {
+        // For now, treat memory transfer tokens as independent graphs
+        create_new_graph(tok);
+        schedule();
+    } else {
+        // other token types: ignore or extend later
+        create_new_graph(tok);
+    }
+}
+
+void pressure_scheduler::schedule() {
+    // Determine inflight level using current_concurreny and thresholds
+    int level;
+    if (current_concurreny >= high_concurrency_threshold) {
+        level = HIGH;
+    } else if (current_concurreny >= low_concurreny_threshold) {
+        level = MEDIUM;
+    } else {
+        level = LOW;
+    }
+
+    // Hard cutoff: if HIGH, do not dispatch further work
+    if (level == HIGH) {
+        // Intentionally idle until reclaim reduces concurrency
+        return;
+    }
+
+    // For Phase 1: everything classified as memory-bound. Prefer memory dispatch.
+    dispatch_prefer_memory();
+}
+
+void pressure_scheduler::process_launch_token(const token_ptr& tok, int stream_id) {
+    // Use the sm_usage_heuristic to compute a conservative cost
+    int cost = heuristic->getCost(tok);
+    if (cost == ERROR_CODE) return;
+
+    // Update accounting similar to multilevel behavior
+    // In multilevel the available resource was decremented by cost; keep that
+    available_memory = std::max(0, available_memory - 0); // placeholder: token-based memory not yet available
+
+    // Update concurrency accounting (heuristic returns SMS used; map to concurrency units)
+    int concurrency_units = std::max(1, cost);
+    current_concurreny += concurrency_units;
+    resource_pressure += cost;
+
+    // Build and send launch response similar to multilevel behavior
+    if (tok->getType() == LAUNCH) {
+        const auto& launch = static_cast<const launch_token&>(*tok);
+        auto response = make_launch_response_token(state_.self, launch, state_.device_number, stream_id, cost);
+        anon_mail(response).send(launch.getReplyActor());
+    } else {
+        // For non-launch tokens we simply log for now and assume a single unit of work
+        std::cerr << "[pressure_scheduler] dispatched non-launch token on stream " << stream_id << "
+";
+    }
+}
+
+void pressure_scheduler::reclaim(int blocks_consumed, int memory_returned, int /*time*/, int dependency_number) {
+    // Update available resources
+    // blocks_consumed represents how many concurrency units to free
+    current_concurreny = std::max(0, current_concurreny - blocks_consumed);
+    resource_pressure = std::max(0, resource_pressure - blocks_consumed);
+    available_memory = std::min(available_memory + memory_returned, static_cast<int>(device_->total_memory_bytes()));
+
+    // If dependent graph may now be ready, re-enqueue
+    if (dependency_number != INDEPENDENT) {
+        if (graphs.contains(dependency_number)) {
+            graph_ref ref{graph_ref::kind_t::dependent, dependency_number, 0};
+            enqueue_graph_by_cost(ref);
+        }
+    } else {
+        // Re-enqueue all independent graphs that are non-empty (best-effort)
+        for (std::size_t i = 0; i < independent_graphs.size(); ++i) {
+            if (!independent_graphs[i].empty()) {
+                graph_ref ref{graph_ref::kind_t::independent, -1, static_cast<std::size_t>(i)};
+                enqueue_graph_by_cost(ref);
+            }
+        }
+    }
+
+    // Attempt to schedule after resources freed
+    schedule();
+}
+
+void pressure_scheduler::enqueue_graph_by_cost(const graph_ref& ref) {
+    kernel_graph* g = resolve(ref);
+    if (!g || g->empty()) return;
+
+    token_ptr tok = g->peek();
+    if (!tok) return;
+
+    // For now, assume only launch tokens are costed; non-launch tokens are light
+    int cost = ERROR_CODE;
+    if (tok->getType() == LAUNCH) {
+        cost = heuristic->getCost(tok);
+        if (cost == ERROR_CODE) return;
+    } else {
+        cost = 1; // small cost for memory/other tokens
+    }
+
+    // Place into memory-bound queues (phase 1: everything is memory-bound)
+    const long long medium_threshold = 16LL * static_cast<long long>(total_SM);
+    if (cost <= total_SM) {
+        low_memory_queue.push_back(ref);
+    } else if (cost <= medium_threshold) {
+        med_memory_queue.push_back(ref);
+    } else {
+        high_memory_queue.push_back(ref);
+    }
+}
+
+void pressure_scheduler::try_dispatch_queue(std::deque<graph_ref>& q) {
+    while (!q.empty()) {
+        graph_ref ref = q.front();
+        kernel_graph* g = resolve(ref);
+        if (!g || g->empty()) {
+            q.pop_front();
+            continue;
+        }
+
+        token_ptr tok = g->peek();
+        if (!tok) {
+            q.pop_front();
+            continue;
+        }
+
+        int cost = ERROR_CODE;
+        if (tok->getType() == LAUNCH) {
+            cost = heuristic->getCost(tok);
+            if (cost == ERROR_CODE) {
+                q.pop_front();
+                continue;
+            }
+        } else {
+            cost = 1;
+        }
+
+        // If launching this would exceed the resource_threshold, stop for this queue
+        if (resource_pressure + cost > resource_threshold) {
+            break;
+        }
+
+        // If adding concurrency would exceed high_concurrency_threshold, stop
+        if (current_concurreny + cost > high_concurrency_threshold) {
+            break;
+        }
+
+        // Otherwise dispatch
+        q.pop_front();
+        token_ptr op = g->getOperation();
+        if (!op) continue;
+
+        int stream = get_next_stream();
+        process_launch_token(op, stream);
+
+        // If graph still has ops, re-enqueue for future
+        if (!g->empty()) {
+            enqueue_graph_by_cost(ref);
+        } else {
+            // If dependent, we leave removal to reclaim/gc logic elsewhere
+        }
+    }
+}
+
+void pressure_scheduler::dispatch_prefer_compute() {
+    // Not used in Phase 1, but keep implementation symmetric
+    try_dispatch_queue(low_compute_queue);
+    try_dispatch_queue(med_compute_queue);
+    try_dispatch_queue(high_compute_queue);
+}
+
+void pressure_scheduler::dispatch_prefer_memory() {
+    // Prefer low -> med -> high memory queues
+    try_dispatch_queue(low_memory_queue);
+    try_dispatch_queue(med_memory_queue);
+    try_dispatch_queue(high_memory_queue);
+}
+
+kernel_graph* pressure_scheduler::resolve(const graph_ref& ref) {
+    if (ref.kind == graph_ref::kind_t::independent) {
+        if (ref.index < independent_graphs.size()) return &independent_graphs[ref.index];
+        return nullptr;
+    }
+    auto it = graphs.find(ref.dependency);
+    if (it == graphs.end()) return nullptr;
+    return &it->second;
+}
+
+void pressure_scheduler::create_new_graph(const token_ptr& token) {
+    if (token->isIndependent()) {
+        kernel_graph g(state_.device_number, get_next_stream());
+        g.add_operation(token);
+        independent_graphs.push_back(std::move(g));
+        graph_ref ref{graph_ref::kind_t::independent, -1, independent_graphs.size() - 1};
+        enqueue_graph_by_cost(ref);
+        return;
+    }
+
+    int dep = token->getDependency();
+    if (graphs.contains(dep)) {
+        graphs[dep].add_operation(token);
+        graph_ref ref{graph_ref::kind_t::dependent, dep, 0};
+        enqueue_graph_by_cost(ref);
+    } else {
+        kernel_graph new_graph(state_.device_number, get_next_stream());
+        new_graph.add_operation(token);
+        graphs[dep] = std::move(new_graph);
+        graph_ref ref{graph_ref::kind_t::dependent, dep, 0};
+        enqueue_graph_by_cost(ref);
+    }
+}
+
+int pressure_scheduler::get_next_stream() {
+    int s = current_stream++ % std::max(1, num_streams);
+    return s;
+}
+
+int pressure_scheduler::get_resource_pressure(int blocks_consumed) {
+    // Phase 1: simple unit cost model; integrate sm_usage_heuristic later if needed
+    return blocks_consumed > 0 ? blocks_consumed : 1;
+}
+
+int pressure_scheduler::get_concurrency_pressure(int blocks_consumed) {
+    return get_resource_pressure(blocks_consumed);
+}
+
+} // namespace caf::cuda
+

From eef11afd4858e36651fde40fd3a820d1e5e5296b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 8 Feb 2026 10:18:49 -0600
Subject: [PATCH 0313/1000] Added message tags for kernel graph and std::Vector
 kernel_graph These changes are being made so scheduler actors can move them
 around to load balance across each other.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 99c10f2ad8..1c1801d6ed 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -34,6 +34,8 @@ CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::response_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_transfer_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_response_token>))
 CAF_ADD_TYPE_ID(cuda_control, (std::vector<caf::intrusive_ptr<caf::cuda::token>>))
+CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::kernel_graph))
+CAF_ADD_TYPE_ID(cuda_control, (std::vector<caf::cuda::kernel_graph>))
 
 CAF_END_TYPE_ID_BLOCK(cuda_control)
 
@@ -48,7 +50,6 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_response_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_transfer_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_response_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::response_token)
-
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::behavior_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_token>)
@@ -57,3 +58,7 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::response_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::memory_transfer_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::memory_response_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<caf::intrusive_ptr<caf::cuda::token>>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::kernel_graph)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<caf::cuda::kernel_graph>)
+
+

From fafc13816f1ee5c2c7fb2132bacecc988dd5f0ca Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 8 Feb 2026 10:23:47 -0600
Subject: [PATCH 0314/1000] Implemented message handlers to load balancing
 between scheduler actors.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 079c81446d..73a17aaca9 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -5,6 +5,7 @@
 #include "caf/cuda/control-layer/core_usage_behavior.hpp"
 #include "caf/cuda/control-layer/single_usage_behavior.hpp"
 #include "caf/cuda/control-layer/multilevel_usage_behavior.hpp"
+#include "caf/cuda/control-layer/kernel_graph.hpp"
 #include <string>
 #include <iostream>
 
@@ -91,7 +92,19 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
 	//or else undefined behavior
 	[&](std::vector<caf::actor> s) {
 		state.schedulers = s;
-	}
+	},
+
+
+
+	//message handler for a request for work from another scheduler actor
+	[&](int device_number) {
+		state.current_behavior -> handle_load_balance_request(device_number);
+	},
+
+	//message handler for work being transfered over from another scheduler actor
+	[&](std::vector<kernel_graph> work_graphs) {
+		state.current_behavior -> receive_work(work_graphs);
+	},
     };
 }
 

From 382a13eca09e9bba4f02bfc958862460482cb60b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 8 Feb 2026 10:28:50 -0600
Subject: [PATCH 0315/1000] Added a transfer type.

---
 libcaf_cuda/caf/cuda/control-layer/token.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index 465f3ebf52..5cc2786d35 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -11,6 +11,7 @@
 #define BEHAVIOR 3
 #define MEMORY 4
 #define MEMORY_RESPONSE 5
+#define TRANSFER 6
 
 //dependency tags
 #define INDEPENDENT -1

From d5adc481d2367fcb30ce45feaf75a889e9c5e237 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 8 Feb 2026 10:52:42 -0600
Subject: [PATCH 0316/1000] Defined return payload objects and put them in the
 neccesary header files. THese changes are being made to ensure that different
 kinds of return messages can be sent to the scheduler actor without the need
 to change the interfact over and over again.

---
 .../cuda/control-layer/all-control-layer.hpp  |  1 +
 .../control-layer/return_payloads/ack.hpp     | 28 +++++++++++++++++++
 .../return_payloads/all_return_payloads.hpp   |  2 ++
 .../return_payloads/transfer_ack.hpp          | 20 +++++++++++++
 4 files changed, 51 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/return_payloads/all_return_payloads.hpp
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/return_payloads/transfer_ack.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 1c1801d6ed..a123fee17f 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "caf/cuda/all.hpp"
+#include "caf/cuda/control-layer/return_payloads/all_return_payloads.hpp"
 
 // Control-layer object types
 #include "caf/cuda/control-layer/token.hpp"
diff --git a/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp b/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
new file mode 100644
index 0000000000..2e3c005c4b
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
@@ -0,0 +1,28 @@
+#pragma once
+#include "caf/cuda/global_export.hpp"
+
+namespace caf::cuda {
+
+// -----------------------------------------------------------------------------
+// ACK type codes (stable ABI, no enum churn)
+// -----------------------------------------------------------------------------
+#define CAF_CUDA_ACK_TRANSFER  1
+#define CAF_CUDA_ACK_LAUNCH    2
+#define CAF_CUDA_ACK_MEMORY    3
+
+// -----------------------------------------------------------------------------
+// Base ACK payload
+// -----------------------------------------------------------------------------
+class CAF_CUDA_EXPORT ack {
+public:
+    explicit ack(int type) : type_(type) {}
+    virtual ~ack() = default;
+
+    int getType() const { return type_; }
+
+private:
+    int type_;
+};
+
+} // namespace caf::cuda
+
diff --git a/libcaf_cuda/caf/cuda/control-layer/return_payloads/all_return_payloads.hpp b/libcaf_cuda/caf/cuda/control-layer/return_payloads/all_return_payloads.hpp
new file mode 100644
index 0000000000..34d8dbea3d
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/return_payloads/all_return_payloads.hpp
@@ -0,0 +1,2 @@
+#include "caf/cuda/control-layer/return_payloads/ack.hpp"
+#include "caf/cuda/control-layer/return_payloads/transfer_ack.hpp"
diff --git a/libcaf_cuda/caf/cuda/control-layer/return_payloads/transfer_ack.hpp b/libcaf_cuda/caf/cuda/control-layer/return_payloads/transfer_ack.hpp
new file mode 100644
index 0000000000..4f32c037a8
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/return_payloads/transfer_ack.hpp
@@ -0,0 +1,20 @@
+#pragma once
+#include "caf/cuda/control-layer/return_payloads/ack.hpp"
+
+namespace caf::cuda {
+
+class CAF_CUDA_EXPORT transfer_ack final : public ack {
+public:
+    explicit transfer_ack(int dependency)
+      : ack(CAF_CUDA_ACK_TRANSFER),
+        dependency_(dependency) {}
+
+    int dependency() const { return dependency_; }
+
+private:
+    int dependency_;
+};
+
+} //namespace caf::cuda
+
+

From 53f6aedf65f99e2e1119827b2917c20b55e6f5fe Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 8 Feb 2026 11:03:05 -0600
Subject: [PATCH 0317/1000] Added default constructors for caf's messaging
 system.

---
 libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp    | 3 +++
 .../caf/cuda/control-layer/return_payloads/transfer_ack.hpp   | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp b/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
index 2e3c005c4b..03abe5a4f7 100644
--- a/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
@@ -15,6 +15,9 @@ namespace caf::cuda {
 // -----------------------------------------------------------------------------
 class CAF_CUDA_EXPORT ack {
 public:
+   
+   //for caf messaging system do not use	
+    ack() = default;
     explicit ack(int type) : type_(type) {}
     virtual ~ack() = default;
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/return_payloads/transfer_ack.hpp b/libcaf_cuda/caf/cuda/control-layer/return_payloads/transfer_ack.hpp
index 4f32c037a8..107ed8c6d6 100644
--- a/libcaf_cuda/caf/cuda/control-layer/return_payloads/transfer_ack.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/return_payloads/transfer_ack.hpp
@@ -5,6 +5,10 @@ namespace caf::cuda {
 
 class CAF_CUDA_EXPORT transfer_ack final : public ack {
 public:
+    
+     //for caf's messaging system
+     transfer_ack() = default;
+	
     explicit transfer_ack(int dependency)
       : ack(CAF_CUDA_ACK_TRANSFER),
         dependency_(dependency) {}

From 136b403f2147958f18a6bab1906fb13ce13378a2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 8 Feb 2026 11:03:22 -0600
Subject: [PATCH 0318/1000] Initial commit.

---
 .../caf/cuda/control-layer/transfer_token.hpp | 89 +++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp b/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp
new file mode 100644
index 0000000000..ed833da354
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp
@@ -0,0 +1,89 @@
+#pragma once
+#include "caf/cuda/control-layer/response_token.hpp"
+#include "caf/cuda/control-layer/launch_token.hpp"
+#include "caf/cuda/control-layer/return_payloads/transfer_ack.hpp"
+#include "caf/cuda/global_export.hpp"
+
+#include <caf/actor.hpp>
+#include <atomic>
+#include <string>
+#include "caf/cuda/nd_range.hpp"
+
+namespace caf::cuda {
+
+// -----------------------------------------------------------------------------
+// Transfer response token returned after a kernel transfer request
+// -----------------------------------------------------------------------------
+class CAF_CUDA_EXPORT transfer_token : public response_token {
+public:
+    // Default constructor – only for CAF compliance
+    transfer_token() = default;
+
+    // Construct manually (full control over dependency number)
+    transfer_token(caf::actor receiver,
+                   nd_range range,
+                   int memory_usage,
+                   std::string id,
+                   int device_num = 0,
+                   int stream_id = 0,
+                   int dependency_number = 0)
+        : response_token(std::move(receiver), device_num, stream_id, memory_usage),
+          range_(std::move(range)),
+          id_(std::move(id)),
+          released_(false),
+          dependency_number_(dependency_number) {}
+
+    // Construct from a launch_token
+    transfer_token(caf::actor receiver,
+                   const launch_token& token,
+                   int device_num,
+                   int stream_id)
+        : response_token(std::move(receiver),
+                         device_num,
+                         stream_id,
+                         token.getMemoryUsage()),
+          range_(token.getRange()),
+          id_(token.getId()),
+          released_(false),
+          dependency_number_(token.getDependency()) {}
+
+    ~transfer_token() override {
+        release();
+    }
+
+    int getType() const override { return LAUNCH_RESPONSE; }
+
+    const nd_range& getRange() const { return range_; }
+    const std::string& getId() const { return id_; }
+    const std::string& name() const override { return id_; }
+
+    // -------------------------------------------------------------------------
+    // Release and send transfer_ack as base ack (explicit upcast)
+    // -------------------------------------------------------------------------
+    void release() override {
+        bool expected = false;
+        if (!released_.compare_exchange_strong(expected, true))
+            return;
+
+        try {
+            // Create concrete transfer_ack
+            transfer_ack ack_obj{dependency_number_};
+
+            // Upcast explicitly to ack reference before sending
+            const ack& base_ack = ack_obj;
+            caf::anon_mail(base_ack).urgent().send(receiver_);
+
+        } catch (...) {
+            // destructor-safe
+        }
+    }
+
+private:
+    nd_range range_;
+    std::string id_;
+    std::atomic<bool> released_{false};
+    int dependency_number_{0};
+};
+
+} // namespace caf::cuda
+

From d0d2bd8271280e942eb2fc0490dcf5e859b265d7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 8 Feb 2026 11:12:23 -0600
Subject: [PATCH 0319/1000] Updated behavior and message handlers, and message
 type tags, to include the use of return payload object called ack This change
 is being made since with different return message types, we can just downcast
 a payload object based on type rather than keep changing the interface over
 and over again.

---
 .../caf/cuda/control-layer/all-control-layer.hpp     |  9 ++++++++-
 libcaf_cuda/caf/cuda/control-layer/behavior.hpp      | 12 ++++++++++++
 libcaf_cuda/src/control-layer/scheduler_actor.cpp    |  9 +++++++++
 3 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index a123fee17f..8fc7b99168 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -11,6 +11,7 @@
 #include "caf/cuda/control-layer/behavior_token.hpp"
 #include "caf/cuda/control-layer/memory_transfer_token.hpp"
 #include "caf/cuda/control-layer/memory_response_token.hpp"
+#include "caf/cuda/control-layer/transfer_token.hpp"
 
 #include "caf/cuda/control-layer/behavior.hpp"
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
@@ -34,9 +35,12 @@ CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_response_tok
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::response_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_transfer_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_response_token>))
+CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::transfer_token>))
 CAF_ADD_TYPE_ID(cuda_control, (std::vector<caf::intrusive_ptr<caf::cuda::token>>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::kernel_graph))
 CAF_ADD_TYPE_ID(cuda_control, (std::vector<caf::cuda::kernel_graph>))
+CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::ack))
+CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::transfer_ack))
 
 CAF_END_TYPE_ID_BLOCK(cuda_control)
 
@@ -51,15 +55,18 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_response_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_transfer_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_response_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::response_token)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::transfer_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::behavior_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::launch_response_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::response_token>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::transfer_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::memory_transfer_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::memory_response_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<caf::intrusive_ptr<caf::cuda::token>>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::kernel_graph)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<caf::cuda::kernel_graph>)
-
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::ack)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::transfer_ack)
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
index 8eba5f5129..5a435f7db8 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
@@ -5,6 +5,8 @@
 #include "caf/cuda/control-layer/scheduler_actor_state.hpp"
 #include "caf/cuda/control-layer/token.hpp"
 #include "caf/cuda/control-layer/token_factory.hpp"
+#include "caf/cuda/control-layer/return_payloads/ack.hpp"
+
 #include <string>
 
 
@@ -36,6 +38,16 @@ class scheduler_actor_behavior {
 	    //by children classes 
     }
 
+
+
+    //this is here to ensure that payloads on return can conform to an interface
+    //rather than changing the interface to accomidate every scheduling need
+    virtual void reclaim(ack payload) {
+	    //default implementation is to do nothing, this should be overidden
+	    //by children classes 
+    }
+
+
     virtual std::string name() const {return "No name\n";}
 
     //this method is meant to be a handler for when
diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 73a17aaca9..3ee715fe82 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -87,6 +87,15 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
 		state.current_behavior->reclaim(value,memory,runtime,dependency);
 	},
 
+
+	//message handler for reclaim
+	[&](ack payload) {	
+		state.current_behavior->reclaim(payload);
+	},
+
+
+
+
 	//handler sent to set the scheduler actors 
 	//do not send a message more than once
 	//or else undefined behavior

From a7161a7c1a29923509ddf6499ffab8204c466daa Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 8 Feb 2026 11:46:23 -0600
Subject: [PATCH 0320/1000] Added a optional ceiling parameter to
 sm_usage_heuritsic

---
 .../scheduler-functions/sm_usage_heuristic.hpp        | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
index 6fb5112602..319c459335 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
@@ -16,8 +16,8 @@ namespace caf::cuda {
 
 class sm_usage_heuristic : public heuristic_function {
 public:
-  explicit sm_usage_heuristic(device_ptr dev)
-    : dev_(dev) {}
+  explicit sm_usage_heuristic(device_ptr dev,bool ceil = true)
+    : dev_(dev),ceil_(ceil) {}
 
 int getCost(const program_ptr& prog,
             const nd_range& range) override {
@@ -39,7 +39,11 @@ int getCost(const program_ptr& prog,
     }
 
     int sms_needed = (total_blocks + blocks_per_sm - 1) / blocks_per_sm; // ceil
-    int sms_used = std::min(dev_->num_sms(), sms_needed);
+    int sms_used = sms_needed;
+    if (ceil_) {
+	    sms_used = std::min(dev_->num_sms(), sms_needed);
+    }
+
 
     values_[key] = sms_used;
     return sms_used;
@@ -60,6 +64,7 @@ int getCost(const program_ptr& prog,
 
 private:
   device_ptr dev_;
+  bool ceil_ = true;
 };
 
 } // namespace caf::cuda

From b4c2d68fe294ccd167f7b141bdcf55ad1799b7cf Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 8 Feb 2026 12:18:36 -0600
Subject: [PATCH 0321/1000] Updated schedule to dispatch work based on a
 resource pressure model where low SM usage kernsl will be take up less
 pressure than ones that dont take up as much resources.

---
 .../cuda/control-layer/pressure_scheduler.hpp |  14 +-
 .../src/control-layer/pressure_scheduler.cpp  | 145 ++++++------------
 2 files changed, 53 insertions(+), 106 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp b/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
index d3edfb9761..92b2c18fdf 100644
--- a/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
@@ -60,16 +60,18 @@ class pressure_scheduler : public scheduler_actor_behavior {
     int resource_threshold; //if exceeded do not dispatch kernel
     int resource_pressure; // tracks resources in use, if low we should dispatch heavy kernels, if high dispatch light kernels 
     
-    int low_concurreny_threshold; //if we under this immediately accept any work
+    double low_concurreny_threshold; //if we under this immediately accept any work
 				  //of if multiple gpus, seek out work
 
-    int high_concurrency_threshold; //if we are above this, enqueue any work
+    double high_concurrency_threshold; //if we are above this, enqueue any work
 				    //since could flood GPU with requests
 
-    int current_concurreny; //number to assign how much kernels on the GPU
+    double current_concurreny; //number to assign how much kernels on the GPU
 			    //values should be in proportion to how much 
 			    //resources a kernel intends to consume 
 
+    double current_sm_pressure;
+    
     int compute_bound_pressure; //determines if we should favor compute or memory bound kernels when seeking work to dispatch 
 
 
@@ -116,10 +118,8 @@ class pressure_scheduler : public scheduler_actor_behavior {
     //pressure it puts on a dimension of the GPU 
     //(concurreny,memory vs compute bound, resource)
     int get_resource_pressure(int blocks_consumed);
-
-    //helps determine how much concurrent work a kernel is going to use
-    int get_concurrency_pressure(int blocks_consumed); 
-    
+    int get_concurrency_pressure(int);
+    int get_pressure_level(double); 
     
 };
 
diff --git a/libcaf_cuda/src/control-layer/pressure_scheduler.cpp b/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
index c02b54116d..1a24c0e5e9 100644
--- a/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
+++ b/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
@@ -4,9 +4,6 @@
 
 namespace caf::cuda {
 
-//TODO overhaul this, this sucks 
-
-
 pressure_scheduler::pressure_scheduler(scheduler_actor_state& state)
     : scheduler_actor_behavior(state) {
     init_state();
@@ -16,22 +13,20 @@ pressure_scheduler::~pressure_scheduler() = default;
 
 void pressure_scheduler::init_state() {
     device_ = manager::get().find_device(state_.device_number);
-    heuristic.emplace(device_);
+    heuristic.emplace(device_, false); // turn off ceiling functionality
 
     total_SM = device_->num_sms();
     available_memory = static_cast<int>(device_->total_memory_bytes());
     num_streams = state_.num_streams;
 
     // thresholds (tunable)
-    resource_threshold = total_SM * 16; // arbitrary guard
+    resource_threshold = 1.0; // normalized SM pressure threshold (1.0 = fully loaded)
 
-    // concurrency thresholds: small, medium, high
-    low_concurreny_threshold = 1; // 0..1 treated as low
-    high_concurrency_threshold = device_->num_sms() * 4; // e.g., 4 kernels/SM
+    low_concurreny_threshold = 0.25;  // low SM pressure fraction
+    high_concurrency_threshold = 0.75; // high SM pressure fraction
 
     // initial accounting
-    current_concurreny = 0;
-    resource_pressure = 0;
+    current_sm_pressure = 0.0;
 }
 
 void pressure_scheduler::on_enter() {
@@ -43,86 +38,62 @@ void pressure_scheduler::receive(const token_ptr& tok) {
         create_new_graph(tok);
         schedule();
     } else if (tok->getType() == MEMORY) {
-        // For now, treat memory transfer tokens as independent graphs
-        create_new_graph(tok);
-        schedule();
+        process_memory_transfer_token(tok, 0);
     } else {
-        // other token types: ignore or extend later
         create_new_graph(tok);
     }
 }
 
-void pressure_scheduler::schedule() {
-    // Determine inflight level using current_concurreny and thresholds
-    int level;
-    if (current_concurreny >= high_concurrency_threshold) {
-        level = HIGH;
-    } else if (current_concurreny >= low_concurreny_threshold) {
-        level = MEDIUM;
-    } else {
-        level = LOW;
-    }
+int pressure_scheduler::get_pressure_level(double sm_ratio) {
+    if (sm_ratio >= high_concurrency_threshold) return HIGH;
+    if (sm_ratio >= low_concurreny_threshold) return MEDIUM;
+    return LOW;
+}
 
-    // Hard cutoff: if HIGH, do not dispatch further work
-    if (level == HIGH) {
-        // Intentionally idle until reclaim reduces concurrency
+void pressure_scheduler::schedule() {
+    // Hard cutoff: if total SM pressure is above HIGH threshold, do not dispatch further work
+    if (current_sm_pressure >= high_concurrency_threshold) {
         return;
     }
 
-    // For Phase 1: everything classified as memory-bound. Prefer memory dispatch.
+    // Phase 1: everything is memory-bound; prioritize memory queues
     dispatch_prefer_memory();
 }
 
 void pressure_scheduler::process_launch_token(const token_ptr& tok, int stream_id) {
-    // Use the sm_usage_heuristic to compute a conservative cost
-    int cost = heuristic->getCost(tok);
-    if (cost == ERROR_CODE) return;
+    int sm_used = heuristic->getCost(tok);
+    if (sm_used == ERROR_CODE) return;
 
-    // Update accounting similar to multilevel behavior
-    // In multilevel the available resource was decremented by cost; keep that
-    available_memory = std::max(0, available_memory - 0); // placeholder: token-based memory not yet available
+    double sm_ratio = static_cast<double>(sm_used) / total_SM;
 
-    // Update concurrency accounting (heuristic returns SMS used; map to concurrency units)
-    int concurrency_units = std::max(1, cost);
-    current_concurreny += concurrency_units;
-    resource_pressure += cost;
+    // Update SM pressure accounting
+    current_sm_pressure += sm_ratio;
 
-    // Build and send launch response similar to multilevel behavior
+    // Send launch response if needed
     if (tok->getType() == LAUNCH) {
         const auto& launch = static_cast<const launch_token&>(*tok);
-        auto response = make_launch_response_token(state_.self, launch, state_.device_number, stream_id, cost);
+        auto response = make_launch_response_token(state_.self, launch, state_.device_number, stream_id, sm_used);
         anon_mail(response).send(launch.getReplyActor());
     } else {
-        // For non-launch tokens we simply log for now and assume a single unit of work
-        std::cerr << "[pressure_scheduler] dispatched non-launch token on stream " << stream_id << "
-";
+        std::cerr << "[pressure_scheduler] dispatched non-launch token on stream " << stream_id << "\n";
     }
 }
 
-void pressure_scheduler::reclaim(int blocks_consumed, int memory_returned, int /*time*/, int dependency_number) {
-    // Update available resources
-    // blocks_consumed represents how many concurrency units to free
-    current_concurreny = std::max(0, current_concurreny - blocks_consumed);
-    resource_pressure = std::max(0, resource_pressure - blocks_consumed);
+void pressure_scheduler::reclaim(int sm_used, int memory_returned, int /*time*/, int dependency_number) {
+    double sm_ratio = static_cast<double>(sm_used) / total_SM;
+    current_sm_pressure = std::max(0.0, current_sm_pressure - sm_ratio);
     available_memory = std::min(available_memory + memory_returned, static_cast<int>(device_->total_memory_bytes()));
 
-    // If dependent graph may now be ready, re-enqueue
     if (dependency_number != INDEPENDENT) {
         if (graphs.contains(dependency_number)) {
             graph_ref ref{graph_ref::kind_t::dependent, dependency_number, 0};
             enqueue_graph_by_cost(ref);
         }
-    } else {
-        // Re-enqueue all independent graphs that are non-empty (best-effort)
-        for (std::size_t i = 0; i < independent_graphs.size(); ++i) {
-            if (!independent_graphs[i].empty()) {
-                graph_ref ref{graph_ref::kind_t::independent, -1, static_cast<std::size_t>(i)};
-                enqueue_graph_by_cost(ref);
-            }
-        }
     }
+    //do nothing if it was INDEPEDENDATN FOR NOW
+    //WILL NEED TO CLEANUP LATER
+
 
-    // Attempt to schedule after resources freed
     schedule();
 }
 
@@ -133,23 +104,20 @@ void pressure_scheduler::enqueue_graph_by_cost(const graph_ref& ref) {
     token_ptr tok = g->peek();
     if (!tok) return;
 
-    // For now, assume only launch tokens are costed; non-launch tokens are light
-    int cost = ERROR_CODE;
+    int sm_used = 1; // default small cost
     if (tok->getType() == LAUNCH) {
-        cost = heuristic->getCost(tok);
-        if (cost == ERROR_CODE) return;
-    } else {
-        cost = 1; // small cost for memory/other tokens
+        sm_used = heuristic->getCost(tok);
+        if (sm_used == ERROR_CODE) return;
     }
 
-    // Place into memory-bound queues (phase 1: everything is memory-bound)
-    const long long medium_threshold = 16LL * static_cast<long long>(total_SM);
-    if (cost <= total_SM) {
-        low_memory_queue.push_back(ref);
-    } else if (cost <= medium_threshold) {
-        med_memory_queue.push_back(ref);
-    } else {
-        high_memory_queue.push_back(ref);
+    double sm_ratio = static_cast<double>(sm_used) / total_SM;
+    int level = get_pressure_level(sm_ratio);
+
+    // For now, everything is memory-bound; place in appropriate memory queue
+    switch(level) {
+        case LOW:    low_memory_queue.push_back(ref); break;
+        case MEDIUM: med_memory_queue.push_back(ref); break;
+        case HIGH:   high_memory_queue.push_back(ref); break;
     }
 }
 
@@ -168,28 +136,14 @@ void pressure_scheduler::try_dispatch_queue(std::deque<graph_ref>& q) {
             continue;
         }
 
-        int cost = ERROR_CODE;
-        if (tok->getType() == LAUNCH) {
-            cost = heuristic->getCost(tok);
-            if (cost == ERROR_CODE) {
-                q.pop_front();
-                continue;
-            }
-        } else {
-            cost = 1;
-        }
+        int sm_used = (tok->getType() == LAUNCH) ? heuristic->getCost(tok) : 1;
+        if (sm_used == ERROR_CODE) { q.pop_front(); continue; }
 
-        // If launching this would exceed the resource_threshold, stop for this queue
-        if (resource_pressure + cost > resource_threshold) {
-            break;
-        }
+        double sm_ratio = static_cast<double>(sm_used) / total_SM;
 
-        // If adding concurrency would exceed high_concurrency_threshold, stop
-        if (current_concurreny + cost > high_concurrency_threshold) {
-            break;
-        }
+        // Dispatch cutoff: don't exceed 100% SM usage
+        if (current_sm_pressure + sm_ratio > resource_threshold) break;
 
-        // Otherwise dispatch
         q.pop_front();
         token_ptr op = g->getOperation();
         if (!op) continue;
@@ -197,24 +151,19 @@ void pressure_scheduler::try_dispatch_queue(std::deque<graph_ref>& q) {
         int stream = get_next_stream();
         process_launch_token(op, stream);
 
-        // If graph still has ops, re-enqueue for future
         if (!g->empty()) {
             enqueue_graph_by_cost(ref);
-        } else {
-            // If dependent, we leave removal to reclaim/gc logic elsewhere
         }
     }
 }
 
 void pressure_scheduler::dispatch_prefer_compute() {
-    // Not used in Phase 1, but keep implementation symmetric
     try_dispatch_queue(low_compute_queue);
     try_dispatch_queue(med_compute_queue);
     try_dispatch_queue(high_compute_queue);
 }
 
 void pressure_scheduler::dispatch_prefer_memory() {
-    // Prefer low -> med -> high memory queues
     try_dispatch_queue(low_memory_queue);
     try_dispatch_queue(med_memory_queue);
     try_dispatch_queue(high_memory_queue);
@@ -255,12 +204,10 @@ void pressure_scheduler::create_new_graph(const token_ptr& token) {
 }
 
 int pressure_scheduler::get_next_stream() {
-    int s = current_stream++ % std::max(1, num_streams);
-    return s;
+    return current_stream++ % std::max(1, num_streams);
 }
 
 int pressure_scheduler::get_resource_pressure(int blocks_consumed) {
-    // Phase 1: simple unit cost model; integrate sm_usage_heuristic later if needed
     return blocks_consumed > 0 ? blocks_consumed : 1;
 }
 

From 5d5b298994851ebef8a99f6a4ba37b65aca579ed Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Feb 2026 08:32:21 -0600
Subject: [PATCH 0322/1000] Changed get_pressure_level to get_bucket_level.

---
 .../cuda/control-layer/pressure_scheduler.hpp   |  2 +-
 .../src/control-layer/pressure_scheduler.cpp    | 17 +++++++++++------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp b/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
index 92b2c18fdf..0fde750b41 100644
--- a/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
@@ -119,7 +119,7 @@ class pressure_scheduler : public scheduler_actor_behavior {
     //(concurreny,memory vs compute bound, resource)
     int get_resource_pressure(int blocks_consumed);
     int get_concurrency_pressure(int);
-    int get_pressure_level(double); 
+    int get_bucket_level(double); 
     
 };
 
diff --git a/libcaf_cuda/src/control-layer/pressure_scheduler.cpp b/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
index 1a24c0e5e9..feaf1e3368 100644
--- a/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
+++ b/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
@@ -44,7 +44,7 @@ void pressure_scheduler::receive(const token_ptr& tok) {
     }
 }
 
-int pressure_scheduler::get_pressure_level(double sm_ratio) {
+int pressure_scheduler::get_bucket_level(double sm_ratio) {
     if (sm_ratio >= high_concurrency_threshold) return HIGH;
     if (sm_ratio >= low_concurreny_threshold) return MEDIUM;
     return LOW;
@@ -111,7 +111,7 @@ void pressure_scheduler::enqueue_graph_by_cost(const graph_ref& ref) {
     }
 
     double sm_ratio = static_cast<double>(sm_used) / total_SM;
-    int level = get_pressure_level(sm_ratio);
+    int level = get_bucket_level(sm_ratio);
 
     // For now, everything is memory-bound; place in appropriate memory queue
     switch(level) {
@@ -141,8 +141,7 @@ void pressure_scheduler::try_dispatch_queue(std::deque<graph_ref>& q) {
 
         double sm_ratio = static_cast<double>(sm_used) / total_SM;
 
-        // Dispatch cutoff: don't exceed 100% SM usage
-        if (current_sm_pressure + sm_ratio > resource_threshold) break;
+        if (current_sm_pressure + sm_ratio > high_concurrency_threshold) break;
 
         q.pop_front();
         token_ptr op = g->getOperation();
@@ -211,8 +210,14 @@ int pressure_scheduler::get_resource_pressure(int blocks_consumed) {
     return blocks_consumed > 0 ? blocks_consumed : 1;
 }
 
-int pressure_scheduler::get_concurrency_pressure(int blocks_consumed) {
-    return get_resource_pressure(blocks_consumed);
+int pressure_scheduler::get_concurrency_pressure(int sm_used) {
+    double ratio = double(sm_used) / double(total_SM);
+
+    if (ratio < 0.10) return 1;
+    if (ratio < 0.25) return 2;
+    if (ratio < 0.50) return 4;
+    if (ratio < 0.75) return 8;
+    return 16;
 }
 
 } // namespace caf::cuda

From d74b057a76ab994e8cd37621f7923f89f18a9fa8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Feb 2026 08:37:05 -0600
Subject: [PATCH 0323/1000] Updated get_bucket_level to not rely on
 get_concurrency_threshold.

---
 libcaf_cuda/src/control-layer/pressure_scheduler.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/pressure_scheduler.cpp b/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
index feaf1e3368..923b0b9458 100644
--- a/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
+++ b/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
@@ -45,9 +45,9 @@ void pressure_scheduler::receive(const token_ptr& tok) {
 }
 
 int pressure_scheduler::get_bucket_level(double sm_ratio) {
-    if (sm_ratio >= high_concurrency_threshold) return HIGH;
-    if (sm_ratio >= low_concurreny_threshold) return MEDIUM;
-    return LOW;
+    if (sm_ratio >= 0.75) return HIGH;    // big kernel
+    if (sm_ratio >= 0.40) return MEDIUM;  // mid-sized
+    return LOW;                           // narrow
 }
 
 void pressure_scheduler::schedule() {

From cdd3925377842ebff3bae8745c0cbc8634c87a42 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Feb 2026 08:42:20 -0600
Subject: [PATCH 0324/1000] Added inline documentation.

---
 libcaf_cuda/src/control-layer/pressure_scheduler.cpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/pressure_scheduler.cpp b/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
index 923b0b9458..68172d7db3 100644
--- a/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
+++ b/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
@@ -22,8 +22,8 @@ void pressure_scheduler::init_state() {
     // thresholds (tunable)
     resource_threshold = 1.0; // normalized SM pressure threshold (1.0 = fully loaded)
 
-    low_concurreny_threshold = 0.25;  // low SM pressure fraction
-    high_concurrency_threshold = 0.75; // high SM pressure fraction
+    low_concurreny_threshold = 5.0;  // low SM pressure fraction
+    high_concurrency_threshold = 15.0; // high SM pressure fraction
 
     // initial accounting
     current_sm_pressure = 0.0;
@@ -206,6 +206,10 @@ int pressure_scheduler::get_next_stream() {
     return current_stream++ % std::max(1, num_streams);
 }
 
+
+
+//these methods were drafted in original design and were meant to be used
+//now its unclear what they are supposed to do 
 int pressure_scheduler::get_resource_pressure(int blocks_consumed) {
     return blocks_consumed > 0 ? blocks_consumed : 1;
 }

From 052d1dd778d00360e69645a4a300c6b1be2bcdd2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Feb 2026 08:44:00 -0600
Subject: [PATCH 0325/1000] Added pressure_scheduler behavior.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 3ee715fe82..b339843a26 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -5,6 +5,7 @@
 #include "caf/cuda/control-layer/core_usage_behavior.hpp"
 #include "caf/cuda/control-layer/single_usage_behavior.hpp"
 #include "caf/cuda/control-layer/multilevel_usage_behavior.hpp"
+#include "caf/cuda/control-layer/pressure_scheduler.hpp"
 #include "caf/cuda/control-layer/kernel_graph.hpp"
 #include <string>
 #include <iostream>
@@ -27,17 +28,21 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
     //check if multiple gpus
     state.multiple_gpus = multi_gpu;
 
+    //static declarations may cause issues when expanding to
+    //multiple GPUs
     static red_light_behavior red_behavior(state);
     static green_light_behavior green_behavior(state);
     static core_usage_behavior core_behavior(state);
     static single_usage_behavior single_behavior(state);
     static multilevel_usage_behavior multi_behavior(state);
+    static pressure_scheduler pressure(state);
 
     // populate the behavior table
     state.table.add("red", &red_behavior);
     state.table.add("green",   &green_behavior);
     state.table.add("core_usage",   &core_behavior);
     state.table.add("multilevel",   &multi_behavior);
+    state.table.add("pressure",   &pressure);
 
     // default behavior
     state.current_behavior = state.table.get(behavior_token("green"));

From 881a5647b9d72c7907da26a96cc99f75fb062092 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Feb 2026 10:01:58 -0600
Subject: [PATCH 0326/1000] Implemented stubs to send the scheduler actor
 messages.

---
 libcaf_cuda/caf/cuda/manager.hpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index 3b20f5b907..b5728c904a 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -21,6 +21,9 @@
 #include "caf/cuda/actor_facade.hpp"
 #include "caf/cuda/platform.hpp"
 #include "caf/cuda/manager_config.hpp"
+#include "caf/cuda/control-layer/token.hpp"
+#include "caf/cuda/control-layer/behavior_token.hpp"
+
 
 //A class that just acts as a user interface
 //and a system initialization for cuda 
@@ -174,6 +177,12 @@ class CAF_CUDA_EXPORT manager {
 
   device_ptr find_device(int id);
 
+  //methods used to send scheduler actors messages
+  void send_scheduler_actor_message(token_ptr token,int device_number);
+  void send_scheduler_actor_message(std::vector<token_ptr> tokens,int device_number);
+  void send_scheduler_actor_message(behavior_token_ptr token,int device_number);
+
+
 private:
   explicit manager(caf::actor_system& sys)
     : system_(sys), platform_(platform::create()) {

From 1f89609af91461f18f2c674fae7752a9ac34380a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Feb 2026 10:11:20 -0600
Subject: [PATCH 0327/1000] updated send_scheduler_message

---
 libcaf_cuda/caf/cuda/manager.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index b5728c904a..369f6471d2 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -178,8 +178,8 @@ class CAF_CUDA_EXPORT manager {
   device_ptr find_device(int id);
 
   //methods used to send scheduler actors messages
-  void send_scheduler_actor_message(token_ptr token,int device_number);
-  void send_scheduler_actor_message(std::vector<token_ptr> tokens,int device_number);
+  void send_scheduler_actor_message(token_ptr token,int device_number = -1);
+  void send_scheduler_actor_message(std::vector<token_ptr> tokens,int device_number = -1);
   void send_scheduler_actor_message(behavior_token_ptr token,int device_number);
 
 

From 3d324d5e68e3e04432bc65f98aa775e404ca9598 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Feb 2026 10:15:14 -0600
Subject: [PATCH 0328/1000] Implemented send_scheduler_actor message for token
 ptrs.

---
 libcaf_cuda/src/manager.cpp | 53 +++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 7f54d5e012..1a1042f329 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -296,6 +296,59 @@ bool manager::compile_nvrtc_program(const char* source, CUdevice device, std::ve
 	return caf::cuda::compile_nvrtc_program(source,device,ptx_out);
 }
 
+// ---------------------------------------------
+// Send single token
+// ---------------------------------------------
+void manager::send_scheduler_actor_message(token_ptr token, int device_number) {
+    if (!scheduler_on || scheduler_actors.empty())
+        return;
+
+    int num_devices = static_cast<int>(scheduler_actors.size());
+    int target = -1;
+
+    if (device_number != -1) {
+        // Explicit device
+        if (device_number >= num_devices)
+            return; // silently discard
+        target = device_number;
+    } else {
+        // No device specified
+        if (!token->isIndependent()) {
+            target = token->getDependency() % num_devices;
+            if (target < 0)
+                target += num_devices;
+        } else {
+            target = rand() % num_devices;
+        }
+    }
+
+    anon_mail(token).send(scheduler_actors[target]);
+}
+
+// ---------------------------------------------
+// Send vector of tokens
+// ---------------------------------------------
+void manager::send_scheduler_actor_message(std::vector<token_ptr> tokens,
+                                           int device_number) {
+    if (!scheduler_on || scheduler_actors.empty() || tokens.empty())
+        return;
+
+    int num_devices = static_cast<int>(scheduler_actors.size());
+    int target = -1;
+
+    if (device_number != -1) {
+        // Explicit device
+        if (device_number >= num_devices)
+            return; // silently discard
+        target = device_number;
+    } else {
+        // No device specified → random
+        target = rand() % num_devices;
+    }
+
+    anon_mail(std::move(tokens)).send(scheduler_actors[target]);
+}
+
 
  
 

From 0686d737b1f18d10e927cb1a8b6174fd861deaa0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Feb 2026 10:20:06 -0600
Subject: [PATCH 0329/1000] Implemented behavior_token_ptr
 send_scheduler_actor_message.

---
 libcaf_cuda/src/manager.cpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 1a1042f329..2b86f1343f 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -349,7 +349,18 @@ void manager::send_scheduler_actor_message(std::vector<token_ptr> tokens,
     anon_mail(std::move(tokens)).send(scheduler_actors[target]);
 }
 
+void manager::send_scheduler_actor_message(behavior_token_ptr token, int device_number) {
+    if (!scheduler_on || scheduler_actors.empty())
+        return;
+
+    int num_devices = static_cast<int>(scheduler_actors.size());
 
+    // Drop if device number is invalid
+    if (device_number < 0 || device_number >= num_devices)
+        return;
+
+    anon_mail(token).send(scheduler_actors[device_number]);
+}
  
 
 } // namespace caf::cuda

From 68389b88cc6329603e105b8e567a29180bae1a5b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Feb 2026 10:22:49 -0600
Subject: [PATCH 0330/1000] Implemented another send_scheduler_message method.

---
 libcaf_cuda/caf/cuda/manager.hpp | 1 +
 libcaf_cuda/src/manager.cpp      | 7 ++++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index 369f6471d2..06e22c7c4b 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -181,6 +181,7 @@ class CAF_CUDA_EXPORT manager {
   void send_scheduler_actor_message(token_ptr token,int device_number = -1);
   void send_scheduler_actor_message(std::vector<token_ptr> tokens,int device_number = -1);
   void send_scheduler_actor_message(behavior_token_ptr token,int device_number);
+  void send_scheduler_actor_message(std::string behavior,int device_number);
 
 
 private:
diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 2b86f1343f..eb0740b246 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -361,6 +361,11 @@ void manager::send_scheduler_actor_message(behavior_token_ptr token, int device_
 
     anon_mail(token).send(scheduler_actors[device_number]);
 }
- 
+
+void manager::send_scheduler_actor_message(std::string behavior, int device_number) {
+    auto token = caf::cuda::make_behavior_token(std::move(behavior));
+    send_scheduler_actor_message(token, device_number);
+}
+
 
 } // namespace caf::cuda

From 68c6bb5e10def728add851ce57333604400dc170 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Feb 2026 10:29:22 -0600
Subject: [PATCH 0331/1000] Added a timer type.

---
 libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp b/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
index 03abe5a4f7..b1d8ed2e64 100644
--- a/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
@@ -9,6 +9,7 @@ namespace caf::cuda {
 #define CAF_CUDA_ACK_TRANSFER  1
 #define CAF_CUDA_ACK_LAUNCH    2
 #define CAF_CUDA_ACK_MEMORY    3
+#define TIMER 4
 
 // -----------------------------------------------------------------------------
 // Base ACK payload

From a9f054b6a896ccd6394760bd0c5d2977fdd8fb25 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Feb 2026 10:43:50 -0600
Subject: [PATCH 0332/1000] Implemented stubs for multi GPU load balancing.

---
 .../multilevel_usage_behavior.hpp             | 29 +++++++++------
 .../multilevel_usage_behavior.cpp             | 35 +++++++++++++++----
 2 files changed, 47 insertions(+), 17 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
index c04fc904e2..4a860d50fc 100644
--- a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
@@ -16,14 +16,6 @@ namespace caf::cuda {
 
 // Multilevel queue scheduling behavior (low / medium / high)
 // - Graphs are classified by the *next operation's* cost relative to total_SM
-// - Classification thresholds:
-//     low:    cost <= total_SM
-//     medium: cost <= 16 * total_SM
-//     high:   cost >  16 * total_SM
-// - schedule() will try to drain low first, then medium, then high.
-// - When a graph has work dispatched it is removed from the queues and
-//   not re-inserted. reclaim(...) can push graphs back into queues by
-//   looking up the dependency number and re-evaluating the front op.
 
 class multilevel_usage_behavior : public scheduler_actor_behavior {
 public:
@@ -38,20 +30,37 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
     // allows this behavior to find the graph that might now be ready
     void reclaim(int blocks_consumed, int memory_returned, int time, int dependency_number) override;
 
+    //more improved version of reclaim, meant for when we need to dispatch transfer
+    //tokens
+    void reclaim(ack return_msg) override;
+
     std::string name() const override { return "multilevel_usage\n"; }
 
+
+    //multi GPU load balancing methods
+    //by default this scheduler behavior will try to load balance
+    //across all gpus 
+    void handle_load_balance_request(int device_number) override;
+    void receive_work(std::vector<kernel_graph> work_graphs) override;
+    
+
+
 protected:
     void process_launch_token(const token_ptr& tok, int stream_id) override;
 
 private:
     device_ptr device_;
-    std::optional<sm_usage_heuristic> heuristic;
+    std::optional<sm_usage_heuristic> heuristic; //we also clamp results since
+						 //it leads to more concurrent work
+
 
-    int total_SM = 0;
+    int total_SM = 0; //this is not really total_SM anymore, more like a threshold 
     int available_SM = 0;
     int available_memory = 0; // bytes
     int num_streams = 0;
     int current_stream = 0;
+    int low_threshold = 0; //this is used to check if we should request more work
+			   //or not
 
     //tracking graphs
     std::unordered_map<int,kernel_graph> graphs;
diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index e91f8460b6..9578bca89b 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -14,10 +14,11 @@ multilevel_usage_behavior::~multilevel_usage_behavior() {}
 void multilevel_usage_behavior::init_state() {
     device_ = manager::get().find_device(state_.device_number);
     heuristic.emplace(device_);
-    total_SM = device_->num_sms() * 16; // preserve previous semantics
+    total_SM = device_->num_sms() * 16;
     available_SM = total_SM;
     available_memory = static_cast<int>(device_->total_memory_bytes());
     num_streams = state_.num_streams;
+    low_threshold = total_SM / 6;
 }
 
 void multilevel_usage_behavior::on_enter() {
@@ -25,7 +26,7 @@ void multilevel_usage_behavior::on_enter() {
 }
 
 void multilevel_usage_behavior::process_launch_token(const token_ptr& tok, int stream_id) {
-    scoped_timer timer("multilevel_usage_behavior::process_launch_token");
+    //scoped_timer timer("multilevel_usage_behavior::process_launch_token");
 
     int cost = heuristic->getCost(tok);
 
@@ -36,7 +37,7 @@ void multilevel_usage_behavior::process_launch_token(const token_ptr& tok, int s
 }
 
 void multilevel_usage_behavior::receive(const token_ptr& tok) {
-    scoped_timer timer("multilevel_usage_behavior::receive");
+   // scoped_timer timer("multilevel_usage_behavior::receive");
 
     if (tok->getType() == LAUNCH) {
         create_new_graph(tok);
@@ -141,12 +142,12 @@ void multilevel_usage_behavior::try_dispatch_queue(std::deque<graph_ref>& q) {
 }
 
 void multilevel_usage_behavior::schedule() {
-    scoped_timer timer("multilevel_usage_behavior::schedule");
-
+   // scoped_timer timer("multilevel_usage_behavior::schedule");
+    
     // Prioritize low, then medium, then high
-    try_dispatch_queue(low_queue);
-    try_dispatch_queue(med_queue);
     try_dispatch_queue(high_queue);
+    try_dispatch_queue(med_queue);
+    try_dispatch_queue(low_queue);
 }
 
 void multilevel_usage_behavior::reclaim(int blocks_consumed,
@@ -178,6 +179,13 @@ void multilevel_usage_behavior::reclaim(int blocks_consumed,
     schedule();
 }
 
+void multilevel_usage_behavior::reclaim(ack return_msg) {
+
+	//TODO IMPLEMENT TIMER ACK AND TRANSFER ACK 
+
+}
+
+
 kernel_graph* multilevel_usage_behavior::resolve(const graph_ref& ref) {
     switch (ref.kind) {
         case graph_ref::kind_t::dependent: {
@@ -193,5 +201,18 @@ kernel_graph* multilevel_usage_behavior::resolve(const graph_ref& ref) {
     return nullptr;
 }
 
+
+    //multi GPU load balancing methods
+    void multilevel_usage_behavior::handle_load_balance_request(int device_number) {
+	    //TODO IMPLEMENT
+
+    }
+    void multilevel_usage_behavior::receive_work(std::vector<kernel_graph> work_graphs) {  
+	    //TODO IMPLEMENT
+    
+    }
+
+
+
 } // namespace caf::cuda
 

From 6138456cb18ba2f00d71198344f14c8ddb95bc6b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Feb 2026 10:56:37 -0600
Subject: [PATCH 0333/1000] Implemented some logic required for load balancing.

---
 .../multilevel_usage_behavior.hpp             |  2 ++
 .../multilevel_usage_behavior.cpp             | 21 ++++++++++++++++++-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
index 4a860d50fc..09d327d193 100644
--- a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
@@ -83,6 +83,8 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
     void try_dispatch_queue(std::deque<graph_ref>& q);
 
     kernel_graph* resolve(const graph_ref& ref);
+
+    void send_timed_msg();
 };
 
 
diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 9578bca89b..7b7ff97ba0 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -22,7 +22,20 @@ void multilevel_usage_behavior::init_state() {
 }
 
 void multilevel_usage_behavior::on_enter() {
-    // nothing for now
+
+	//trigger load balancing mechanisms
+	if (state_.multiple_gpus) {	
+		send_timed_msg();
+	}
+
+}
+
+void multilevel_usage_behavior::send_timed_msg() {
+
+
+	anon_mail(ack(TIMER)).delay(std::chrono::seconds(2)).send(state_.self);
+
+
 }
 
 void multilevel_usage_behavior::process_launch_token(const token_ptr& tok, int stream_id) {
@@ -183,6 +196,12 @@ void multilevel_usage_behavior::reclaim(ack return_msg) {
 
 	//TODO IMPLEMENT TIMER ACK AND TRANSFER ACK 
 
+	if (return_msg.getType() == TIMER) {
+
+		//check if load balance logic goes here
+	     send_timed_msg();	
+	}
+
 }
 
 

From f3a3d264ac93038c0b6514d8322147b566c4e119 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 13:31:41 -0600
Subject: [PATCH 0334/1000] No major changes.

---
 libcaf_cuda/caf/cuda/control-layer/launch_token.hpp       | 2 ++
 libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp | 4 +++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
index 702898696d..021ea43193 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_token.hpp
@@ -43,6 +43,8 @@ class CAF_CUDA_EXPORT launch_token : public request_token {
 
   const std::string& getId() const { return id_; }
 
+
+
 private:
   program_ptr program_;
   nd_range range_;
diff --git a/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp b/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
index 0fde750b41..1407b209b4 100644
--- a/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
@@ -120,7 +120,9 @@ class pressure_scheduler : public scheduler_actor_behavior {
     int get_resource_pressure(int blocks_consumed);
     int get_concurrency_pressure(int);
     int get_bucket_level(double); 
-    
+
+    double clamp_sm_ratio(double) const;
+
 };
 
 

From 9e79b04a2c56b0604b0defdb778692db95c29ee0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 13:40:09 -0600
Subject: [PATCH 0335/1000] Updated kernel graph to include timestamps as well
 as canMove and markMoved methods Change was made to make it so the
 kernel_graph can track whether or not it should be moved across actors or
 not, since we do not want these to be moving back and fourth unecessarly.

---
 .../caf/cuda/control-layer/kernel_graph.hpp   | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
index f272f59b65..e8df199d3e 100644
--- a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
@@ -1,6 +1,8 @@
 #pragma once
 #include <vector>
 #include "caf/cuda/control-layer/token.hpp"
+#include <chrono>
+
 
 /* Meant to represent a directed acylic graph for kernel operations
  * operations or token_ptrs are meant to come in order and declared with a dependency number to indicate dependency 
@@ -15,6 +17,8 @@ namespace caf::cuda {
 
 class kernel_graph {
 public:
+    using clock_t = std::chrono::steady_clock;
+
     // for caf's messaging system do not use
     kernel_graph() = default;
 
@@ -54,6 +58,22 @@ class kernel_graph {
         return op;
     }
 
+     // Returns true if enough time has passed to allow movement
+    bool canMove(std::chrono::seconds min_interval = std::chrono::seconds{2}) const noexcept {
+        return (clock_t::now() - last_move_) >= min_interval;
+    }
+
+    // Call when the graph is actually moved / rescheduled
+    void markMoved() noexcept {
+        last_move_ = clock_t::now();
+    }
+
+    // Optional: force-disable movement (useful for debugging)
+    void disableMove() noexcept {
+        last_move_ = clock_t::now() + std::chrono::hours{24};
+    }
+
+
     bool empty() const {
         return operations.empty();
     }
@@ -69,6 +89,8 @@ class kernel_graph {
     int dependency_number_;
     int status = READY;
     std::vector<token_ptr> operations;
+    clock_t::time_point last_move_ = clock_t::time_point::min();
+
 };
 
 

From f492a795fc1cd44f18077a0c4c60e7dea81e5891 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 13:42:03 -0600
Subject: [PATCH 0336/1000] No major changes.

---
 libcaf_cuda/src/control-layer/behavior.cpp    |   1 +
 .../src/control-layer/core_usage_behavior.cpp |   5 +-
 .../control-layer/green_light_behavior.cpp    |   6 +-
 .../src/control-layer/pressure_scheduler.cpp  | 126 ++++++++++++------
 4 files changed, 98 insertions(+), 40 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/behavior.cpp b/libcaf_cuda/src/control-layer/behavior.cpp
index 3b27e6fbe1..842ade8642 100644
--- a/libcaf_cuda/src/control-layer/behavior.cpp
+++ b/libcaf_cuda/src/control-layer/behavior.cpp
@@ -10,6 +10,7 @@ void scheduler_actor_behavior::process_launch_token(const token_ptr& tok, int st
     anon_mail(response).send(launch.getReplyActor());
 }
 
+
 void scheduler_actor_behavior::process_memory_transfer_token(const token_ptr& tok, int stream_id) {
     const auto& mem = static_cast<const memory_transfer_token&>(*tok);
     auto response = make_memory_response_token(state_.self, mem, state_.device_number, stream_id);
diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index acc950308f..411d434e52 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -47,6 +47,9 @@ void core_usage_behavior::reclaim(int blocks_consumed,
 
 void core_usage_behavior::process_launch_token(const token_ptr& tok,int stream_id )  {
 
+    
+	scoped_timer timer("core_usage_behavior::process_launch_token");
+	
 	int cost = heuristic -> getCost(tok);
 
 	const auto& launch = static_cast<const launch_token&>(*tok);
@@ -58,7 +61,7 @@ void core_usage_behavior::process_launch_token(const token_ptr& tok,int stream_i
 
 void core_usage_behavior::receive(const token_ptr& tok) {
 
-	//std::cout <<"YARRRRRRRRRRRRRRRRRRRRR\n ";
+    scoped_timer timer("core_usage_behavior::receive");
     
 	if (tok->getType() == LAUNCH) {
         create_new_graph(tok);
diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index ade904ea40..627aa0019b 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -24,7 +24,11 @@ void green_light_behavior::schedule() {
 }
 
 void green_light_behavior::receive(const token_ptr& tok) {
-    if (tok->getType() == LAUNCH) {
+   
+
+//	std::cout << "Green light receive\n";
+
+     if (tok->getType() == LAUNCH) {
         //use 0 as stream id for now, eventually will have to figure out
         //stream load balancing
         process_launch_token(tok, 0);
diff --git a/libcaf_cuda/src/control-layer/pressure_scheduler.cpp b/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
index 68172d7db3..3509d36bd5 100644
--- a/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
+++ b/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
@@ -1,6 +1,7 @@
 #include "caf/cuda/control-layer/pressure_scheduler.hpp"
 #include "caf/cuda/control-layer/all-control-layer.hpp"
 #include <iostream>
+#include <algorithm>
 
 namespace caf::cuda {
 
@@ -19,11 +20,12 @@ void pressure_scheduler::init_state() {
     available_memory = static_cast<int>(device_->total_memory_bytes());
     num_streams = state_.num_streams;
 
-    // thresholds (tunable)
-    resource_threshold = 1.0; // normalized SM pressure threshold (1.0 = fully loaded)
+    // policy thresholds (tunable)
+    resource_threshold = 1.0;
 
-    low_concurreny_threshold = 5.0;  // low SM pressure fraction
-    high_concurrency_threshold = 15.0; // high SM pressure fraction
+    // concurrency pressure thresholds (absolute units)
+    low_concurreny_threshold  = 25.0;
+    high_concurrency_threshold = 75.0;
 
     // initial accounting
     current_sm_pressure = 0.0;
@@ -44,19 +46,33 @@ void pressure_scheduler::receive(const token_ptr& tok) {
     }
 }
 
+/* -------------------------------------------------------------------------- */
+/* Helper utilities                                                            */
+/* -------------------------------------------------------------------------- */
+
+double pressure_scheduler::clamp_sm_ratio(double raw_ratio) const {
+    const double max_ratio = 0.20 * high_concurrency_threshold;
+    return std::min(raw_ratio, max_ratio);
+}
+
 int pressure_scheduler::get_bucket_level(double sm_ratio) {
-    if (sm_ratio >= 0.75) return HIGH;    // big kernel
-    if (sm_ratio >= 0.40) return MEDIUM;  // mid-sized
+    // sm_ratio here is already clamped
+    if (sm_ratio >= 10.0) return HIGH;    // wide kernel
+    if (sm_ratio >=  6.0) return MEDIUM;  // mid-sized
     return LOW;                           // narrow
 }
 
+/* -------------------------------------------------------------------------- */
+/* Scheduling                                                                  */
+/* -------------------------------------------------------------------------- */
+
 void pressure_scheduler::schedule() {
-    // Hard cutoff: if total SM pressure is above HIGH threshold, do not dispatch further work
+    // Hard cutoff: if concurrency pressure is too high, wait
     if (current_sm_pressure >= high_concurrency_threshold) {
         return;
     }
 
-    // Phase 1: everything is memory-bound; prioritize memory queues
+    // Phase 1: everything treated as memory-bound for now
     dispatch_prefer_memory();
 }
 
@@ -64,25 +80,40 @@ void pressure_scheduler::process_launch_token(const token_ptr& tok, int stream_i
     int sm_used = heuristic->getCost(tok);
     if (sm_used == ERROR_CODE) return;
 
-    double sm_ratio = static_cast<double>(sm_used) / total_SM;
+    double raw_ratio = static_cast<double>(sm_used) / total_SM;
+    double sm_ratio  = clamp_sm_ratio(raw_ratio);
 
-    // Update SM pressure accounting
+    // Update pressure accounting
     current_sm_pressure += sm_ratio;
 
-    // Send launch response if needed
     if (tok->getType() == LAUNCH) {
         const auto& launch = static_cast<const launch_token&>(*tok);
-        auto response = make_launch_response_token(state_.self, launch, state_.device_number, stream_id, sm_used);
+        auto response = make_launch_response_token(
+            state_.self,
+            launch,
+            state_.device_number,
+            stream_id,
+            sm_used
+        );
         anon_mail(response).send(launch.getReplyActor());
     } else {
-        std::cerr << "[pressure_scheduler] dispatched non-launch token on stream " << stream_id << "\n";
+        std::cerr << "[pressure_scheduler] dispatched non-launch token on stream "
+                  << stream_id << "\n";
     }
 }
 
-void pressure_scheduler::reclaim(int sm_used, int memory_returned, int /*time*/, int dependency_number) {
-    double sm_ratio = static_cast<double>(sm_used) / total_SM;
+void pressure_scheduler::reclaim(int sm_used,
+                                 int memory_returned,
+                                 int /*time*/,
+                                 int dependency_number) {
+    double raw_ratio = static_cast<double>(sm_used) / total_SM;
+    double sm_ratio  = clamp_sm_ratio(raw_ratio);
+
     current_sm_pressure = std::max(0.0, current_sm_pressure - sm_ratio);
-    available_memory = std::min(available_memory + memory_returned, static_cast<int>(device_->total_memory_bytes()));
+    available_memory = std::min(
+        available_memory + memory_returned,
+        static_cast<int>(device_->total_memory_bytes())
+    );
 
     if (dependency_number != INDEPENDENT) {
         if (graphs.contains(dependency_number)) {
@@ -90,13 +121,14 @@ void pressure_scheduler::reclaim(int sm_used, int memory_returned, int /*time*/,
             enqueue_graph_by_cost(ref);
         }
     }
-    //do nothing if it was INDEPEDENDATN FOR NOW
-    //WILL NEED TO CLEANUP LATER
-
 
     schedule();
 }
 
+/* -------------------------------------------------------------------------- */
+/* Queueing                                                                    */
+/* -------------------------------------------------------------------------- */
+
 void pressure_scheduler::enqueue_graph_by_cost(const graph_ref& ref) {
     kernel_graph* g = resolve(ref);
     if (!g || g->empty()) return;
@@ -104,17 +136,17 @@ void pressure_scheduler::enqueue_graph_by_cost(const graph_ref& ref) {
     token_ptr tok = g->peek();
     if (!tok) return;
 
-    int sm_used = 1; // default small cost
+    int sm_used = 1;
     if (tok->getType() == LAUNCH) {
         sm_used = heuristic->getCost(tok);
         if (sm_used == ERROR_CODE) return;
     }
 
-    double sm_ratio = static_cast<double>(sm_used) / total_SM;
+    double raw_ratio = static_cast<double>(sm_used) / total_SM;
+    double sm_ratio  = clamp_sm_ratio(raw_ratio);
     int level = get_bucket_level(sm_ratio);
 
-    // For now, everything is memory-bound; place in appropriate memory queue
-    switch(level) {
+    switch (level) {
         case LOW:    low_memory_queue.push_back(ref); break;
         case MEDIUM: med_memory_queue.push_back(ref); break;
         case HIGH:   high_memory_queue.push_back(ref); break;
@@ -136,12 +168,20 @@ void pressure_scheduler::try_dispatch_queue(std::deque<graph_ref>& q) {
             continue;
         }
 
-        int sm_used = (tok->getType() == LAUNCH) ? heuristic->getCost(tok) : 1;
-        if (sm_used == ERROR_CODE) { q.pop_front(); continue; }
+        int sm_used = (tok->getType() == LAUNCH)
+                        ? heuristic->getCost(tok)
+                        : 1;
+
+        if (sm_used == ERROR_CODE) {
+            q.pop_front();
+            continue;
+        }
 
-        double sm_ratio = static_cast<double>(sm_used) / total_SM;
+        double raw_ratio = static_cast<double>(sm_used) / total_SM;
+        double sm_ratio  = clamp_sm_ratio(raw_ratio);
 
-        if (current_sm_pressure + sm_ratio > high_concurrency_threshold) break;
+        if (current_sm_pressure + sm_ratio > high_concurrency_threshold)
+            break;
 
         q.pop_front();
         token_ptr op = g->getOperation();
@@ -163,16 +203,22 @@ void pressure_scheduler::dispatch_prefer_compute() {
 }
 
 void pressure_scheduler::dispatch_prefer_memory() {
-    try_dispatch_queue(low_memory_queue);
-    try_dispatch_queue(med_memory_queue);
     try_dispatch_queue(high_memory_queue);
+    try_dispatch_queue(med_memory_queue);
+    try_dispatch_queue(low_memory_queue);
 }
 
+/* -------------------------------------------------------------------------- */
+/* Graph management                                                            */
+/* -------------------------------------------------------------------------- */
+
 kernel_graph* pressure_scheduler::resolve(const graph_ref& ref) {
     if (ref.kind == graph_ref::kind_t::independent) {
-        if (ref.index < independent_graphs.size()) return &independent_graphs[ref.index];
+        if (ref.index < independent_graphs.size())
+            return &independent_graphs[ref.index];
         return nullptr;
     }
+
     auto it = graphs.find(ref.dependency);
     if (it == graphs.end()) return nullptr;
     return &it->second;
@@ -183,7 +229,12 @@ void pressure_scheduler::create_new_graph(const token_ptr& token) {
         kernel_graph g(state_.device_number, get_next_stream());
         g.add_operation(token);
         independent_graphs.push_back(std::move(g));
-        graph_ref ref{graph_ref::kind_t::independent, -1, independent_graphs.size() - 1};
+
+        graph_ref ref{
+            graph_ref::kind_t::independent,
+            -1,
+            independent_graphs.size() - 1
+        };
         enqueue_graph_by_cost(ref);
         return;
     }
@@ -191,25 +242,24 @@ void pressure_scheduler::create_new_graph(const token_ptr& token) {
     int dep = token->getDependency();
     if (graphs.contains(dep)) {
         graphs[dep].add_operation(token);
-        graph_ref ref{graph_ref::kind_t::dependent, dep, 0};
-        enqueue_graph_by_cost(ref);
     } else {
         kernel_graph new_graph(state_.device_number, get_next_stream());
         new_graph.add_operation(token);
         graphs[dep] = std::move(new_graph);
-        graph_ref ref{graph_ref::kind_t::dependent, dep, 0};
-        enqueue_graph_by_cost(ref);
     }
+
+    graph_ref ref{graph_ref::kind_t::dependent, dep, 0};
+    enqueue_graph_by_cost(ref);
 }
 
 int pressure_scheduler::get_next_stream() {
     return current_stream++ % std::max(1, num_streams);
 }
 
+/* -------------------------------------------------------------------------- */
+/* Legacy / unused hooks                                                       */
+/* -------------------------------------------------------------------------- */
 
-
-//these methods were drafted in original design and were meant to be used
-//now its unclear what they are supposed to do 
 int pressure_scheduler::get_resource_pressure(int blocks_consumed) {
     return blocks_consumed > 0 ? blocks_consumed : 1;
 }

From 9f826db2a6f7c9881ef1378f0904a299862300ab Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 13:43:09 -0600
Subject: [PATCH 0337/1000] Deleted irrelevant file.

---
 libcaf_cuda/tests/unit-test/mmul-actors.cpp | 125 --------------------
 1 file changed, 125 deletions(-)
 delete mode 100644 libcaf_cuda/tests/unit-test/mmul-actors.cpp

diff --git a/libcaf_cuda/tests/unit-test/mmul-actors.cpp b/libcaf_cuda/tests/unit-test/mmul-actors.cpp
deleted file mode 100644
index 9865fb5da4..0000000000
--- a/libcaf_cuda/tests/unit-test/mmul-actors.cpp
+++ /dev/null
@@ -1,125 +0,0 @@
-//file not in use since I am not wasting time messing around with cmake
-
-
-#include "main.test.hpp"
-
-const char* matrixMulKernel = R"(
-extern "C" __global__
-void matrixMul(const int* a, const int* b, int* c, int N) {
-    int row = blockIdx.y * blockDim.y + threadIdx.y;
-    int col = blockIdx.x * blockDim.x + threadIdx.x;
-
-    if (row < N && col < N) {
-        int temp = 0;
-        for (int k = 0; k < N; ++k) {
-            temp += a[row * N + k] * b[k * N + col];
-        }
-        c[row * N + col] = temp;
-    }
-}
-)";
-
-
-
-// Check result on the CPU
-void verify_result(vector<int> &a, vector<int> &b, vector<int> &c, int N) {
-  // For every row...
-  for (int i = 0; i < N; i++) {
-    // For every column...
-    for (int j = 0; j < N; j++) {
-      // For every element in the row-column pair
-      int tmp = 0;
-      for (int k = 0; k < N; k++) {
-        // Accumulate the partial results
-        tmp += a[i * N + k] * b[k * N + j];
-      }
-
-      // Check against the CPU result
-      assert(tmp == c[i * N + j]);
-    }
-  }
-}
-void test_mmul(caf::actor_system& sys) {
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  // Matrix dimension (N x N)
-  int N = 1024;
-  int THREADS = 32;
-  int BLOCKS = N / THREADS;
-
-  // Setup kernel launch configuration
-  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  // Spawn CUDA actor for matrix multiplication kernel
-  auto gpuActor = mgr.spawn(matrixMulKernel, "matrixMul", dim,
-                            in<int>{}, in<int>{}, out<int>{}, in<int>{});
-
-  // Allocate and initialize host matrices
-  std::vector<int> h_a(N * N);
-  std::vector<int> h_b(N * N);
-  std::vector<int> h_c(N * N, 0);  // initialized to 0
-  std::vector<int> h_n(1, N);
-
-  std::generate(h_a.begin(), h_a.end(), []() { return rand() % 10; });
-  std::generate(h_b.begin(), h_b.end(), []() { return rand() % 10; });
-
-  // Compose device arguments
-  auto arg1 = caf::cuda::create_in_arg(h_a);
-  auto arg2 = caf::cuda::create_in_arg(h_b);
-  auto arg3 = caf::cuda::create_out_arg(h_c);
-  auto arg4 = caf::cuda::create_in_arg(h_n);
-
-  // Spawn an actor to send the message and receive the result
-  sys.spawn([=](caf::event_based_actor* self_actor) {
-    self_actor->mail(gpuActor, arg1, arg2, arg3, arg4)
-      .request(gpuActor, 30s).then(
-        [=](const std::vector<output_buffer>& outputs) {
-          std::vector<int> result;
-          bool got_output = false;
-
-          // Extract the result from outputs
-          for (const auto& out : outputs) {
-            std::visit([&](const auto& vec) {
-              if constexpr (std::is_same_v<std::decay_t<decltype(vec)>, std::vector<int>>) {
-                result = vec;
-                got_output = true;
-              }
-            }, out.data);
-          }
-
-          if (!got_output) {
-            aout(self_actor) << "No output data received!\n";
-          } else {
-            aout(self_actor) << "Verifying result..." << std::endl;
-
-            // Verify GPU result against CPU computation
-            std::vector<int> expected(N * N);
-            for (int i = 0; i < N; ++i) {
-              for (int j = 0; j < N; ++j) {
-                int tmp = 0;
-                for (int k = 0; k < N; ++k) {
-                  tmp += h_a[i * N + k] * h_b[k * N + j];
-                }
-                expected[i * N + j] = tmp;
-              }
-            }
-
-            bool success = std::equal(result.begin(), result.end(), expected.begin());
-            if (success) {
-              aout(self_actor) << "Matrix multiplication result verified successfully!\n";
-            } else {
-              aout(self_actor) << "Mismatch found in matrix multiplication results!\n";
-            }
-          }
-
-          self_actor->quit();
-        }
-      );
-  });
-
-  std::this_thread::sleep_for(std::chrono::seconds(5)); // Wait for actor to complete
-}
-
-
-
-

From c6b944a6e5e4e3c5df4999e6af0c420de2756cd0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 13:58:13 -0600
Subject: [PATCH 0338/1000] Updated kernel_graph constructor to set a default
 initial time. THis change was made to fix a bug that occured where an
 overflow error from the difference between clock::min and now() would cause
 the kernel graph into believing it could never be moved.

---
 libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
index e8df199d3e..5a1ca82b44 100644
--- a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
@@ -27,14 +27,16 @@ class kernel_graph {
                  int dependency_number)
         : device_number_(device_number),
           stream_id_(stream_id),
-          dependency_number_(dependency_number) {}
+          dependency_number_(dependency_number),
+          last_move_(clock_t::now() - std::chrono::seconds{10})	{}
 
 
         // Convenience constructor for independent graphs
     kernel_graph(int device_number, int stream_id)
         : device_number_(device_number),
           stream_id_(stream_id),
-          dependency_number_(INDEPENDENT) {}
+          dependency_number_(INDEPENDENT),
+          last_move_(clock_t::now() - std::chrono::seconds{10})	{}
 
 
     // returns the next operation/token_ptr that can be dequeued
@@ -89,8 +91,7 @@ class kernel_graph {
     int dependency_number_;
     int status = READY;
     std::vector<token_ptr> operations;
-    clock_t::time_point last_move_ = clock_t::time_point::min();
-
+    clock_t::time_point last_move_;
 };
 
 

From ddf6225a77fef88ce6f71083a3b467156cd716d0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 13:58:34 -0600
Subject: [PATCH 0339/1000] Added unit tests for kernel_graphs timing and move
 methods.

---
 .../control-layer-unit-tests/main.test.cpp    | 60 +++++++++++++++++--
 1 file changed, 56 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp b/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
index 96655283a7..bf069ea9fc 100644
--- a/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-unit-tests/main.test.cpp
@@ -13,6 +13,8 @@
 #include <vector>
 #include <string>
 #include <stdexcept>
+#include <chrono>
+#include <thread>
 #include <cassert>
 
 using namespace caf::cuda;
@@ -113,8 +115,54 @@ void test_kernel_graph_push_pop([[maybe_unused]] caf::actor_system& sys) {
 }
 
 
+void test_kernel_graph_can_move_initial([[maybe_unused]] caf::actor_system& sys) {
+    kernel_graph g(/*device*/0, /*stream*/0);
+
+    // Fresh graph should be allowed to move immediately
+    if (!g.canMove()) {
+        throw std::runtime_error("fresh kernel_graph cannot move");
+    }
+
+    std::cout << "[test_kernel_graph_can_move_initial] OK\n";
+}
+
+void test_kernel_graph_mark_moved_blocks([[maybe_unused]] caf::actor_system& sys) {
+    kernel_graph g(/*device*/0, /*stream*/0);
+
+    // First move allowed
+    if (!g.canMove()) {
+        throw std::runtime_error("kernel_graph cannot move initially");
+    }
+
+    // Mark as moved
+    g.markMoved();
+
+    // Immediately after marking, movement should be blocked
+    if (g.canMove()) {
+        throw std::runtime_error("kernel_graph canMove() returned true too soon after markMoved()");
+    }
+
+    std::cout << "[test_kernel_graph_mark_moved_blocks] OK\n";
+}
+
+void test_kernel_graph_can_move_after_delay([[maybe_unused]] caf::actor_system& sys) {
+    kernel_graph g(/*device*/0, /*stream*/0);
+
+    g.markMoved();
+
+    // Sleep slightly longer than the default 2s threshold
+    std::this_thread::sleep_for(std::chrono::milliseconds(2100));
+
+    if (!g.canMove()) {
+        throw std::runtime_error("kernel_graph still blocked after delay");
+    }
+
+    std::cout << "[test_kernel_graph_can_move_after_delay] OK\n";
+}
+
+
+
 // 3) Test core_heuristic_function
-#include <cassert>  // for assert
 
 void test_core_heuristic_function([[maybe_unused]] caf::actor_system& sys) {
     caf::cuda::manager::init(sys);
@@ -173,9 +221,13 @@ void test_core_heuristic_function([[maybe_unused]] caf::actor_system& sys) {
 
 // Register tests
 const std::vector<Test> tests = {
-    {"test_kernel_graph_empty", test_kernel_graph_empty},
-    {"test_kernel_graph_push_pop", test_kernel_graph_push_pop},
-    {"test_core_heuristic_function", test_core_heuristic_function}
+	{"test_kernel_graph_empty", test_kernel_graph_empty},
+	{"test_kernel_graph_push_pop", test_kernel_graph_push_pop},
+	{"test_kernel_graph_can_move_initial", test_kernel_graph_can_move_initial},
+	{"test_kernel_graph_mark_moved_blocks", test_kernel_graph_mark_moved_blocks},
+	{"test_kernel_graph_can_move_after_delay", test_kernel_graph_can_move_after_delay},
+	{"test_core_heuristic_function", test_core_heuristic_function}
+
 };
 
 // Run a single test and return status code:

From fdf52d582ef21e0eabf33280c7d3dfb26602d5df Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 14:09:19 -0600
Subject: [PATCH 0340/1000] Implemented transfer token factory.

---
 .../caf/cuda/control-layer/token_factory.hpp      |  9 +++++++++
 libcaf_cuda/src/control-layer/token_factory.cpp   | 15 +++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
index a2bc57f2b5..c7fa6dee98 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token_factory.hpp
@@ -49,6 +49,15 @@ CAF_CUDA_EXPORT response_token_ptr make_memory_response_token(actor receiver,
 
 
 
+/// Creates a transfer_token from a launch_token
+CAF_CUDA_EXPORT response_token_ptr make_transfer_token(
+    caf::actor receiver,
+    const launch_token& orig,
+    int device_number,
+    int stream_id
+);
+
+
 //do not use this, for testing only 
 CAF_CUDA_EXPORT mem_ptr<int> make_mem_ptr(size_t num_elements);
 
diff --git a/libcaf_cuda/src/control-layer/token_factory.cpp b/libcaf_cuda/src/control-layer/token_factory.cpp
index fd16496140..993ecd21fb 100644
--- a/libcaf_cuda/src/control-layer/token_factory.cpp
+++ b/libcaf_cuda/src/control-layer/token_factory.cpp
@@ -3,6 +3,7 @@
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 #include "caf/cuda/control-layer/behavior_token.hpp"
 #include "caf/cuda/control-layer/memory_response_token.hpp"
+#include "caf/cuda/control-layer/transfer_token.hpp"
 
 namespace caf::cuda {
 
@@ -62,6 +63,20 @@ response_token_ptr make_memory_response_token(actor receiver,
     return response_token_ptr(new memory_response_token(receiver, orig,device_number,stream_id));
 }
 
+response_token_ptr make_transfer_token(caf::actor receiver,
+                                       const launch_token& orig,
+                                       int device_number,
+                                       int stream_id)
+{
+    return response_token_ptr(
+        new transfer_token(receiver,
+                           orig,
+                           device_number,
+                           stream_id));
+}
+
+
+
 
 /// Factory function: create a mem_ptr<int> with fake data
  mem_ptr<int> make_mem_ptr(size_t num_elements = 16) {

From d133b98a938f461e963499bcb78c0d4989ec8704 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 14:10:56 -0600
Subject: [PATCH 0341/1000] Added header to remove static error message from
 caf.

---
 libcaf_cuda/src/control-layer/token_factory.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/src/control-layer/token_factory.cpp b/libcaf_cuda/src/control-layer/token_factory.cpp
index 993ecd21fb..e28a8c37ae 100644
--- a/libcaf_cuda/src/control-layer/token_factory.cpp
+++ b/libcaf_cuda/src/control-layer/token_factory.cpp
@@ -4,6 +4,7 @@
 #include "caf/cuda/control-layer/behavior_token.hpp"
 #include "caf/cuda/control-layer/memory_response_token.hpp"
 #include "caf/cuda/control-layer/transfer_token.hpp"
+#include "caf/cuda/control-layer/all-control-layer.hpp"
 
 namespace caf::cuda {
 

From e35294c024aaab4e6971ec0ceff21038d168dac5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 14:13:45 -0600
Subject: [PATCH 0342/1000] Added single usage back to the scheduler actor.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index b339843a26..111becbb88 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -43,6 +43,7 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
     state.table.add("core_usage",   &core_behavior);
     state.table.add("multilevel",   &multi_behavior);
     state.table.add("pressure",   &pressure);
+    state.table.add("single_usage",   &single_behavior);
 
     // default behavior
     state.current_behavior = state.table.get(behavior_token("green"));

From 91cb608804187e3b9fd34ae3f9c8ce5837785d0a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 14:14:24 -0600
Subject: [PATCH 0343/1000] Removed logging.

---
 libcaf_cuda/src/control-layer/green_light_behavior.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index 627aa0019b..f52644a660 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -8,7 +8,7 @@ green_light_behavior::green_light_behavior(scheduler_actor_state& state)
     : scheduler_actor_behavior(state) {}
 
 void green_light_behavior::on_enter() {
-    std::cout << "GREEN LIGHT\n";
+    //std::cout << "GREEN LIGHT\n";
 
     behavior_token_ptr red_light = make_behavior_token("red");
     //send a request to change behavior to red light after 5 seconds

From 6ee990760b19280f84d768080eff1566a58d36a7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 14:17:36 -0600
Subject: [PATCH 0344/1000] Removed profling.

---
 libcaf_cuda/src/control-layer/single_usage_behavior.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/single_usage_behavior.cpp b/libcaf_cuda/src/control-layer/single_usage_behavior.cpp
index 3863c684de..d1a98b560d 100644
--- a/libcaf_cuda/src/control-layer/single_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/single_usage_behavior.cpp
@@ -45,7 +45,7 @@ void single_usage_behavior::reclaim(int blocks_consumed,
 }
 
 void single_usage_behavior::process_launch_token(const token_ptr& tok, int stream_id) {
-    scoped_timer timer("single_usage_behavior::process_launch_token");
+    //scoped_timer timer("single_usage_behavior::process_launch_token");
 
     // For pure single-usage mode we usually don't care about the heuristic cost
     // but we can still compute it for logging / debugging

From cbf9f25cb0027afacc88189e971f1b3f34b6e6bc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 14:50:44 -0600
Subject: [PATCH 0345/1000] Chaned default behavior to single_usage_behavior
 for now. This change is being made to temporarily patch a race condition
 where a return msg could beat the initial state change causing the return
 message to be dropped and nothing will occur.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 111becbb88..37c5ec7d06 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -46,7 +46,8 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
     state.table.add("single_usage",   &single_behavior);
 
     // default behavior
-    state.current_behavior = state.table.get(behavior_token("green"));
+    //state.current_behavior = state.table.get(behavior_token("green"));
+    state.current_behavior = state.table.get(behavior_token("single_usage"));
     state.current_behavior->on_enter();
 
 
@@ -90,6 +91,8 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
 	
 	//message handler for reclaim
 	[&](int value, int memory,int runtime,int dependency) {	
+
+                //std::cout << "Received reclaim request\n";
 		state.current_behavior->reclaim(value,memory,runtime,dependency);
 	},
 

From 6c94def8c85a51805cfb24357cdb77373bfef2cb Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 15:06:37 -0600
Subject: [PATCH 0346/1000] Implemented get_num_devices.

---
 libcaf_cuda/caf/cuda/manager.hpp | 2 ++
 libcaf_cuda/src/manager.cpp      | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index 06e22c7c4b..44f15e7b3c 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -177,6 +177,8 @@ class CAF_CUDA_EXPORT manager {
 
   device_ptr find_device(int id);
 
+  int get_num_devices();
+
   //methods used to send scheduler actors messages
   void send_scheduler_actor_message(token_ptr token,int device_number = -1);
   void send_scheduler_actor_message(std::vector<token_ptr> tokens,int device_number = -1);
diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index eb0740b246..56898f92fc 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -66,7 +66,7 @@ void manager::init(caf::actor_system& sys, manager_config config) {
     }
 }
 
-
+int manager::get_num_devices() {return platform_ -> get_num_devices();}
 
 void manager::init_scheduler_actors(caf::actor_system& sys) {
 

From c4908c84ff619740a6f85eef500b1f2c5682d3cb Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 15:09:15 -0600
Subject: [PATCH 0347/1000] Implemented num_devices attribute.

---
 .../cuda/control-layer/multilevel_usage_behavior.hpp |  2 ++
 .../src/control-layer/multilevel_usage_behavior.cpp  | 12 ++++++++++++
 2 files changed, 14 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
index 09d327d193..ec9cbc546d 100644
--- a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
@@ -46,7 +46,9 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
 
 
 protected:
+    int num_devices;
     void process_launch_token(const token_ptr& tok, int stream_id) override;
+   
 
 private:
     device_ptr device_;
diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 7b7ff97ba0..2cedc5ecf2 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -19,6 +19,7 @@ void multilevel_usage_behavior::init_state() {
     available_memory = static_cast<int>(device_->total_memory_bytes());
     num_streams = state_.num_streams;
     low_threshold = total_SM / 6;
+    num_devices = manager::get().get_num_devices();
 }
 
 void multilevel_usage_behavior::on_enter() {
@@ -202,6 +203,7 @@ void multilevel_usage_behavior::reclaim(ack return_msg) {
 	     send_timed_msg();	
 	}
 
+
 }
 
 
@@ -225,6 +227,16 @@ kernel_graph* multilevel_usage_behavior::resolve(const graph_ref& ref) {
     void multilevel_usage_behavior::handle_load_balance_request(int device_number) {
 	    //TODO IMPLEMENT
 
+
+	    //we should only accept to transfer work if we are busy as is 
+	    //since otherwise whats the point?
+	    if (available_SM >= low_threshold) {
+	    
+	    
+	    } 
+
+
+
     }
     void multilevel_usage_behavior::receive_work(std::vector<kernel_graph> work_graphs) {  
 	    //TODO IMPLEMENT

From 0228ef488a15f4f03c6a96006e280d11413dc9ae Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 15:18:39 -0600
Subject: [PATCH 0348/1000] Implemented dependency to device number tracking
 data structure and methods.

---
 .../multilevel_usage_behavior.hpp             | 13 +++-
 .../multilevel_usage_behavior.cpp             | 69 +++++++++++--------
 2 files changed, 54 insertions(+), 28 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
index ec9cbc546d..88fcc506a2 100644
--- a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
@@ -11,6 +11,7 @@
 #include <optional>
 #include <algorithm>
 #include <unordered_map>
+#include <unordered_set>
 
 namespace caf::cuda {
 
@@ -48,7 +49,7 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
 protected:
     int num_devices;
     void process_launch_token(const token_ptr& tok, int stream_id) override;
-   
+       
 
 private:
     device_ptr device_;
@@ -64,6 +65,10 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
     int low_threshold = 0; //this is used to check if we should request more work
 			   //or not
 
+    // dependency -> device mapping
+    std::unordered_map<int, int> dependency_device_map;
+
+
     //tracking graphs
     std::unordered_map<int,kernel_graph> graphs;
     std::vector<kernel_graph> independent_graphs;
@@ -87,6 +92,12 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
     kernel_graph* resolve(const graph_ref& ref);
 
     void send_timed_msg();
+
+    void add_dependency_to_device(int dependency_number, int device_number);
+    void remove_dependency(int dependency_number);
+    int get_device_for_dependency(int dependency_number) const;
+
+
 };
 
 
diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 2cedc5ecf2..118d73064d 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -200,7 +200,7 @@ void multilevel_usage_behavior::reclaim(ack return_msg) {
 	if (return_msg.getType() == TIMER) {
 
 		//check if load balance logic goes here
-	     send_timed_msg();	
+		send_timed_msg();	
 	}
 
 
@@ -208,42 +208,57 @@ void multilevel_usage_behavior::reclaim(ack return_msg) {
 
 
 kernel_graph* multilevel_usage_behavior::resolve(const graph_ref& ref) {
-    switch (ref.kind) {
-        case graph_ref::kind_t::dependent: {
-            auto it = graphs.find(ref.dependency);
-            if (it == graphs.end()) return nullptr;
-            return &it->second;
-        }
-        case graph_ref::kind_t::independent: {
-            if (ref.index >= independent_graphs.size()) return nullptr;
-            return &independent_graphs[ref.index];
-        }
-    }
-    return nullptr;
+	switch (ref.kind) {
+		case graph_ref::kind_t::dependent: {
+							   auto it = graphs.find(ref.dependency);
+							   if (it == graphs.end()) return nullptr;
+							   return &it->second;
+						   }
+		case graph_ref::kind_t::independent: {
+							     if (ref.index >= independent_graphs.size()) return nullptr;
+							     return &independent_graphs[ref.index];
+						     }
+	}
+	return nullptr;
 }
 
 
-    //multi GPU load balancing methods
-    void multilevel_usage_behavior::handle_load_balance_request(int device_number) {
-	    //TODO IMPLEMENT
+//multi GPU load balancing methods
+void multilevel_usage_behavior::handle_load_balance_request(int device_number) {
+	//TODO IMPLEMENT
 
 
-	    //we should only accept to transfer work if we are busy as is 
-	    //since otherwise whats the point?
-	    if (available_SM >= low_threshold) {
-	    
-	    
-	    } 
+	//we should only accept to transfer work if we are busy as is 
+	//since otherwise whats the point?
+	if (available_SM >= low_threshold) {
 
 
+	} 
 
-    }
-    void multilevel_usage_behavior::receive_work(std::vector<kernel_graph> work_graphs) {  
-	    //TODO IMPLEMENT
-    
-    }
 
 
+}
+void multilevel_usage_behavior::receive_work(std::vector<kernel_graph> work_graphs) {  
+	//TODO IMPLEMENT
+
+}
+
+void multilevel_usage_behavior::add_dependency_to_device(int dependency_number, int device_number) {
+	// Always override existing value
+	dependency_device_map[dependency_number] = device_number;
+}
+
+void multilevel_usage_behavior::remove_dependency(int dependency_number) {
+	dependency_device_map.erase(dependency_number);
+}
+
+int multilevel_usage_behavior::get_device_for_dependency(int dependency_number) const {
+	auto it = dependency_device_map.find(dependency_number);
+	if (it != dependency_device_map.end()) {
+		return it->second;
+	}
+	return -1; // not found
+}
 
 } // namespace caf::cuda
 

From cbd16de88b0b0228e6315e88be97b9edcb4817ee Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 15:29:25 -0600
Subject: [PATCH 0349/1000] Updated receive method to check if multiple gpus
 and if the dependency number is actually on the correct GPU.

---
 .../control-layer/multilevel_usage_behavior.cpp   | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 118d73064d..1802c1f364 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -78,6 +78,21 @@ void multilevel_usage_behavior::create_new_graph(const token_ptr& tok) {
     }
 
     int dep = tok->getDependency();
+    
+    //check to ensure that the token is not found on another device
+    if (state_.multiple_gpus) {
+	    int dev_num = get_device_for_dependency(dep);
+
+	    //found elsewhere
+	    if (dev_num != state_.device_number && dev_num != -1) {
+	    
+		    anon_mail(tok).send(state_.schedulers[dev_num]);
+		    return;
+	    }
+    }
+    
+    
+    
     if (graphs.contains(dep)) {
         graphs[dep].add_operation(tok);
         // If graph already existed, ensure it's enqueued only if not currently in any queue

From 536c1d0b1d8739b02c4b6957291a170dc1bd10b7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 10 Feb 2026 15:43:23 -0600
Subject: [PATCH 0350/1000] implemented handle load balance request.

---
 .../multilevel_usage_behavior.cpp             | 65 +++++++++++++++++--
 1 file changed, 59 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 1802c1f364..4048b9c892 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -240,19 +240,72 @@ kernel_graph* multilevel_usage_behavior::resolve(const graph_ref& ref) {
 
 //multi GPU load balancing methods
 void multilevel_usage_behavior::handle_load_balance_request(int device_number) {
-	//TODO IMPLEMENT
-
+    
+    // Only transfer work if we are busy
+    if (available_SM < low_threshold) {
+        return; // GPU not busy enough, do nothing
+    }
 
-	//we should only accept to transfer work if we are busy as is 
-	//since otherwise whats the point?
-	if (available_SM >= low_threshold) {
+    std::vector<kernel_graph> work_to_transfer;
+
+    // ---- Helper lambda to collect transferable graphs from a queue ----
+    auto collect_graphs_from_queue = [&](std::deque<graph_ref>& q, std::size_t max_count) {
+        std::size_t collected = 0;
+
+        for (auto it = q.begin(); it != q.end() && collected < max_count;) {
+            graph_ref ref = *it;
+            kernel_graph* g = resolve(ref);
+
+            if (!g || g->empty() || !g->canMove()) {
+                ++it;
+                continue; // skip invalid or non-movable graphs
+            }
+
+            // Move the graph into the transfer vector
+            work_to_transfer.push_back(std::move(*g));
+            ++collected;
+
+            // Clean up local structures
+            if (ref.kind == graph_ref::kind_t::dependent) {
+                remove_dependency(ref.dependency);
+                graphs.erase(ref.dependency);
+            } else { // independent
+                if (ref.index < independent_graphs.size()) {
+		    //TODO come up with a better way to clean this up
+                    independent_graphs[ref.index] = kernel_graph(); // reset empty
+                }
+            }
+
+            // Remove from queue
+            it = q.erase(it);
+        }
+    };
 
+    // ---- Step 1: transfer independent graphs first ----
+    std::size_t max_independent = independent_graphs.size() / 2;
+    std::size_t transferred_independent = 0;
 
-	} 
+    for (std::size_t i = 0; i < independent_graphs.size() && transferred_independent < max_independent; ++i) {
+        kernel_graph& g = independent_graphs[i];
+        if (g.empty() || !g.canMove()) continue;
 
+        work_to_transfer.push_back(std::move(g));
+        independent_graphs[i] = kernel_graph(); // reset
+        ++transferred_independent;
+    }
 
+    // ---- Step 2: transfer from high and medium queues ----
+    std::size_t max_high_med = (high_queue.size() + med_queue.size()) / 2;
+    collect_graphs_from_queue(high_queue, max_high_med);
+    collect_graphs_from_queue(med_queue, max_high_med);
 
+    // ---- Step 3: send if we have anything ----
+    if (!work_to_transfer.empty()) {
+        anon_mail(work_to_transfer).send(state_.schedulers[device_number]);
+    }
 }
+
+
 void multilevel_usage_behavior::receive_work(std::vector<kernel_graph> work_graphs) {  
 	//TODO IMPLEMENT
 

From dba65f05bfb285a3252f29b91c1a0caafa9fd393 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 11 Feb 2026 13:42:29 -0600
Subject: [PATCH 0351/1000] Added get_dependency_number and isIndependent.
 Changes were made to help with book keeping of kernel graphs.

---
 libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
index 5a1ca82b44..4f78b99002 100644
--- a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
@@ -84,6 +84,14 @@ class kernel_graph {
     void set_status(int s) noexcept {status = s;}
     int get_status() const noexcept {return status;}
 
+     bool is_independent() const noexcept {
+        return dependency_number_ == INDEPENDENT;
+    }
+
+    int get_dependency_number() const noexcept {
+        return dependency_number_;
+    }
+
 
 private:
     int device_number_;

From 8c63acade0bdf904f96e9c9af020e70885d4ec69 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 11 Feb 2026 14:14:11 -0600
Subject: [PATCH 0352/1000] Added pragma once.

---
 .../cuda/control-layer/return_payloads/all_return_payloads.hpp   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/return_payloads/all_return_payloads.hpp b/libcaf_cuda/caf/cuda/control-layer/return_payloads/all_return_payloads.hpp
index 34d8dbea3d..1c97dfc8fc 100644
--- a/libcaf_cuda/caf/cuda/control-layer/return_payloads/all_return_payloads.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/return_payloads/all_return_payloads.hpp
@@ -1,2 +1,3 @@
+#pragma once
 #include "caf/cuda/control-layer/return_payloads/ack.hpp"
 #include "caf/cuda/control-layer/return_payloads/transfer_ack.hpp"

From a7c50482354fc9ee7bc094da4fabcece73b75e31 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 11 Feb 2026 14:23:15 -0600
Subject: [PATCH 0353/1000] Implemented a helper method called
 dispatch_transfer_token to be available to all child behaviors and updated
 some header files inclusions.

---
 .../caf/cuda/control-layer/behavior.hpp       |  5 +++--
 libcaf_cuda/src/control-layer/behavior.cpp    | 20 +++++++++++++++++++
 .../src/control-layer/behavior_table.cpp      |  1 +
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
index 5a435f7db8..6717bb6f2f 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
@@ -2,10 +2,10 @@
 #include "caf/cuda/control-layer/launch_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 #include "caf/cuda/control-layer/memory_transfer_token.hpp"
+#include "caf/cuda/control-layer/transfer_token.hpp"
 #include "caf/cuda/control-layer/scheduler_actor_state.hpp"
 #include "caf/cuda/control-layer/token.hpp"
-#include "caf/cuda/control-layer/token_factory.hpp"
-#include "caf/cuda/control-layer/return_payloads/ack.hpp"
+#include "caf/cuda/control-layer/return_payloads/all_return_payloads.hpp"
 
 #include <string>
 
@@ -80,6 +80,7 @@ class scheduler_actor_behavior {
     // Default implementation (immediate response) – takes token_ptr and casts internally
     virtual void process_launch_token(const token_ptr& tok, int stream_id);
     virtual void process_memory_transfer_token(const token_ptr& tok, int stream_id);
+    virtual void dispatch_transfer_token(const token_ptr& tok, int stream_id);
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/behavior.cpp b/libcaf_cuda/src/control-layer/behavior.cpp
index 842ade8642..bdb0f72246 100644
--- a/libcaf_cuda/src/control-layer/behavior.cpp
+++ b/libcaf_cuda/src/control-layer/behavior.cpp
@@ -1,5 +1,6 @@
 #include "caf/cuda/control-layer/all-control-layer.hpp"
 #include "caf/cuda/control-layer/behavior.hpp"
+#include "caf/cuda/control-layer/return_payloads/all_return_payloads.hpp"
 #include "caf/all.hpp"
 
 namespace caf::cuda {
@@ -17,4 +18,23 @@ void scheduler_actor_behavior::process_memory_transfer_token(const token_ptr& to
     anon_mail(response).send(mem.getReplyActor());
 }
 
+
+void scheduler_actor_behavior::dispatch_transfer_token(const token_ptr& tok, int stream_id) {
+    
+   //for right now only assumes launch tokens
+   //anything else is undefined behavior 
+    const auto& launch = static_cast<const launch_token&>(*tok);
+    
+     response_token_ptr transfer = make_transfer_token(
+            state_.self,                    // ack should return to this scheduler
+            launch,                    // original launch token
+            state_.device_number,           // new device
+            stream_id        // stream we stored it under
+        );
+    
+    anon_mail(transfer).send(launch.getReplyActor());
+}
+
+
+
 } // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/behavior_table.cpp b/libcaf_cuda/src/control-layer/behavior_table.cpp
index 1ea2286a4f..c9e11d0118 100644
--- a/libcaf_cuda/src/control-layer/behavior_table.cpp
+++ b/libcaf_cuda/src/control-layer/behavior_table.cpp
@@ -7,6 +7,7 @@
 // If your derived classes have important cleanup, also include them if needed
 #include "caf/cuda/control-layer/green_light_behavior.hpp"
 #include "caf/cuda/control-layer/red_light_behavior.hpp"
+#include "caf/cuda/control-layer/all-control-layer.hpp"
 
 namespace caf::cuda {
 

From affbf5b1d248459b2a87b3dc04ede55e0284cad8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 11 Feb 2026 14:23:56 -0600
Subject: [PATCH 0354/1000] Implemented receive work method.

---
 .../multilevel_usage_behavior.cpp             | 59 ++++++++++++++++++-
 1 file changed, 57 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 4048b9c892..2808298b79 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -306,11 +306,66 @@ void multilevel_usage_behavior::handle_load_balance_request(int device_number) {
 }
 
 
-void multilevel_usage_behavior::receive_work(std::vector<kernel_graph> work_graphs) {  
-	//TODO IMPLEMENT
+void multilevel_usage_behavior::receive_work(std::vector<kernel_graph> work_graphs) {
+    for (auto& g : work_graphs) {
 
+        if (g.empty() || !g.canMove())
+            continue;
+
+        token_ptr tok = g.peek();
+        if (!tok)
+            continue;
+
+        int dep = tok->getDependency();
+
+        // ============================
+        // Independent Graph
+        // ============================
+        if (dep == INDEPENDENT) {
+
+            g.markMoved();
+
+            independent_graphs.push_back(std::move(g));
+
+            graph_ref ref;
+            ref.kind = graph_ref::kind_t::independent;
+            ref.index = independent_graphs.size() - 1;
+
+            enqueue_graph_by_cost(ref);
+            continue;
+        }
+
+        // ============================
+        // Dependent Graph
+        // ============================
+
+        // We are taking ownership of this dependency
+        add_dependency_to_device(dep, state_.device_number);
+
+        g.markMoved();
+
+        // Store or merge into graphs map
+        auto it = graphs.find(dep);
+        if (it == graphs.end()) {
+            graphs.emplace(dep, std::move(g));
+        } else {
+            // Merge operations into existing graph
+            while (!g.empty()) {
+                token_ptr op = g.getOperation();
+                if (!op) break;
+                it->second.add_operation(op);
+            }
+        }
+
+        // Send transfer token to actor so it migrates dependencies
+	dispatch_transfer_token(tok,g.stream_id());
+   }
+
+    // Only independent graphs were enqueued here
+    schedule();
 }
 
+
 void multilevel_usage_behavior::add_dependency_to_device(int dependency_number, int device_number) {
 	// Always override existing value
 	dependency_device_map[dependency_number] = device_number;

From 4e379dba24b943af868db98cf04bde1531596c67 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 11 Feb 2026 14:39:33 -0600
Subject: [PATCH 0355/1000] Updatded reclaim signature to use a reference to
 prevent object slicing.

---
 libcaf_cuda/caf/cuda/control-layer/behavior.hpp                 | 2 +-
 .../caf/cuda/control-layer/multilevel_usage_behavior.hpp        | 2 +-
 libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
index 6717bb6f2f..57ed52f8ce 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
@@ -42,7 +42,7 @@ class scheduler_actor_behavior {
 
     //this is here to ensure that payloads on return can conform to an interface
     //rather than changing the interface to accomidate every scheduling need
-    virtual void reclaim(ack payload) {
+    virtual void reclaim(ack& payload) {
 	    //default implementation is to do nothing, this should be overidden
 	    //by children classes 
     }
diff --git a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
index 88fcc506a2..916e23ebc2 100644
--- a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
@@ -33,7 +33,7 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
 
     //more improved version of reclaim, meant for when we need to dispatch transfer
     //tokens
-    void reclaim(ack return_msg) override;
+    void reclaim(ack& return_msg) override;
 
     std::string name() const override { return "multilevel_usage\n"; }
 
diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 2808298b79..008023b156 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -208,7 +208,7 @@ void multilevel_usage_behavior::reclaim(int blocks_consumed,
     schedule();
 }
 
-void multilevel_usage_behavior::reclaim(ack return_msg) {
+void multilevel_usage_behavior::reclaim(ack& return_msg) {
 
 	//TODO IMPLEMENT TIMER ACK AND TRANSFER ACK 
 

From 9a262d41f43e329dbe56bb883e9d01cee2c310ee Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 11 Feb 2026 14:41:51 -0600
Subject: [PATCH 0356/1000] Implemented process transfer_ack method.

---
 .../multilevel_usage_behavior.hpp             |  2 +-
 .../multilevel_usage_behavior.cpp             | 23 +++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
index 916e23ebc2..743169b365 100644
--- a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
@@ -34,6 +34,7 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
     //more improved version of reclaim, meant for when we need to dispatch transfer
     //tokens
     void reclaim(ack& return_msg) override;
+    void process_transfer_ack(ack& msg); 
 
     std::string name() const override { return "multilevel_usage\n"; }
 
@@ -43,7 +44,6 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
     //across all gpus 
     void handle_load_balance_request(int device_number) override;
     void receive_work(std::vector<kernel_graph> work_graphs) override;
-    
 
 
 protected:
diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 008023b156..124f4a9a31 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -221,6 +221,29 @@ void multilevel_usage_behavior::reclaim(ack& return_msg) {
 
 }
 
+void multilevel_usage_behavior::process_transfer_ack(ack& msg) {
+    // We already verified type before calling this
+    auto& transfer = static_cast<transfer_ack&>(msg);
+
+    int dep = transfer.dependency();
+
+    // Check if we still own this dependency
+    auto it = graphs.find(dep);
+    if (it == graphs.end())
+        return;
+
+    graph_ref ref;
+    ref.kind = graph_ref::kind_t::dependent;
+    ref.dependency = dep;
+
+    enqueue_graph_by_cost(ref);
+
+    schedule();
+}
+
+
+
+
 
 kernel_graph* multilevel_usage_behavior::resolve(const graph_ref& ref) {
 	switch (ref.kind) {

From f8db15c349ddd6ef9ed37f7414838653846d844c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 11 Feb 2026 14:44:29 -0600
Subject: [PATCH 0357/1000] Updated reclaim(ack) to handle transfer ack.

---
 libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 124f4a9a31..864a7fa164 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -218,6 +218,9 @@ void multilevel_usage_behavior::reclaim(ack& return_msg) {
 		send_timed_msg();	
 	}
 
+	else if (return_msg.getType() == CAF_CUDA_ACK_TRANSFER) {
+		process_transfer_ack(return_msg);
+	}
 
 }
 

From 7c5a6a268c38216dd46049a5e065a225770ab9bd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 11 Feb 2026 14:50:23 -0600
Subject: [PATCH 0358/1000] Implemented request_load_balance helper method.

---
 .../multilevel_usage_behavior.hpp             |  1 +
 .../multilevel_usage_behavior.cpp             | 21 ++++++++++++++++++-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
index 743169b365..1bce6b9deb 100644
--- a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
@@ -44,6 +44,7 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
     //across all gpus 
     void handle_load_balance_request(int device_number) override;
     void receive_work(std::vector<kernel_graph> work_graphs) override;
+    void request_load_balance();
 
 
 protected:
diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 864a7fa164..82317864b3 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -214,7 +214,7 @@ void multilevel_usage_behavior::reclaim(ack& return_msg) {
 
 	if (return_msg.getType() == TIMER) {
 
-		//check if load balance logic goes here
+			
 		send_timed_msg();	
 	}
 
@@ -245,6 +245,25 @@ void multilevel_usage_behavior::process_transfer_ack(ack& msg) {
 }
 
 
+void multilevel_usage_behavior::request_load_balance() {
+    if (!state_.multiple_gpus)
+        return;
+
+    // Only request work if we're underutilized
+    if (available_SM >= low_threshold)
+        return;
+
+    int my_device = state_.device_number;
+
+    for (int i = 0; i < num_devices; ++i) {
+        // Skip sending to self
+        if (i == my_device)
+            continue;
+
+        // Send our device number to other scheduler actors
+        anon_mail(my_device).urgent().send(state_.schedulers[i]);
+    }
+}
 
 
 

From 5a765211b9efc1def839bc588a557b23965f2902 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 11 Feb 2026 14:53:08 -0600
Subject: [PATCH 0359/1000] Updated message of ack to move ownership to caf to
 ensure that the lifetime of the object does not expire before reaching the
 scheduler actor.

---
 libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp b/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp
index ed833da354..d564240707 100644
--- a/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp
@@ -71,7 +71,8 @@ class CAF_CUDA_EXPORT transfer_token : public response_token {
 
             // Upcast explicitly to ack reference before sending
             const ack& base_ack = ack_obj;
-            caf::anon_mail(base_ack).urgent().send(receiver_);
+	    caf::anon_mail(std::move(ack_obj)).urgent().send(receiver_);
+
 
         } catch (...) {
             // destructor-safe

From 5ceca853b1226bcdfa1cf63e7dbac4ac4b8fce14 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 11 Feb 2026 14:55:45 -0600
Subject: [PATCH 0360/1000] Initial commit, copied over from
 core_usage_behavior_tests

---
 .../load-balancing-test/CMakeLists.txt        |   44 +
 .../load-balancing-test/compile_kernels.sh    |   18 +
 .../load-balancing-test/main.test.cpp         | 1044 +++++++++++++++++
 .../load-balancing-test/mmul.cu               |   16 +
 4 files changed, 1122 insertions(+)
 create mode 100644 libcaf_cuda/tests/control-layer-tests/load-balancing-test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/control-layer-tests/load-balancing-test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
 create mode 100644 libcaf_cuda/tests/control-layer-tests/load-balancing-test/mmul.cu

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/CMakeLists.txt b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/compile_kernels.sh b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/compile_kernels.sh
new file mode 100755
index 0000000000..586196454e
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/compile_kernels.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+
+# Compile genMatrix.cu to fatbin in current directory
+#nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
+#echo "Generated generate_random_matrix.fatbin"
+
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
new file mode 100644
index 0000000000..568dd032d9
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -0,0 +1,1044 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include <unistd.h>
+#include "caf/actor_registry.hpp"
+#include <chrono>
+#include <iostream>
+//#include <caf/atoms.hpp>
+
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+struct exit_actor_state {
+	int completed = 0;
+};
+
+
+caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
+
+
+	return {
+		[=](int num_completed) {
+			self->state().completed += num_completed;
+			
+			//std::cout << "Actors finished is " << self->state().completed << "\n";
+			if (self->state().completed >= limit) {
+			
+				caf::cuda::manager::shutdown();
+				self->quit();
+			}
+		}
+	};
+
+
+}
+
+
+
+
+
+
+// Define a custom type ID block for custom actors
+CAF_ADD_ATOM(cuda,shared_mem)
+
+
+
+
+
+// Extend your actor state to keep the start time
+struct mmul_actor_state {
+  static inline const char* name = "mmul_actor";
+
+  int N = 0;
+  int id = rand();
+
+  // timing / bookkeeping only
+  std::chrono::high_resolution_clock::time_point start_time;
+  int times = 0;
+};
+
+
+
+
+
+//commands classes used to launch kernels 
+using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
+using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
+
+using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
+
+mmulCommand mmul;
+matrixGenCommand randomMatrix;
+mmulAsyncCommand mmulAsync;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun(
+    caf::stateful_actor<mmul_actor_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims)
+{
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+	caf::actor scheduler = mgr.get_scheduler_actor();
+
+	//send a launch token
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+			program,
+			dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+		[=] (caf::cuda::response_token_ptr res_token) {
+
+			if (res_token -> getType() == LAUNCH_RESPONSE) {
+				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+				//assume N = 1024
+				int N = self -> state().N;
+				std::vector<int> matrix1(N*N);
+				matrix1.reserve(N);
+				std::vector<int> matrix2(N*N);
+				matrix2.reserve(N);
+
+				//std::cout << "GPU ACTOR sending data to compute\n";
+				self -> mail(matrix1,matrix2,res_token,N).send(self);
+
+			}
+			else {
+				std::cout << "Got a memory response token\n";
+			}
+			//token should drop out of scope now, triggering a response 
+		},
+
+			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+			[=](const std::vector<int>& matrixA,
+					const std::vector<int>& matrixB,
+					const caf::cuda::response_token_ptr& res_token, int N) {
+
+
+				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
+
+				//caf::cuda::launch_response_token& kt =
+				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
+
+				//std::cout << "GPU ACTOR  computing\n";
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+				//create program and dims   
+				auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+				const int THREADS = 32;
+				const int BLOCKS = (N + THREADS - 1) / THREADS;
+				caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrixA);
+				auto arg2 = caf::cuda::create_in_arg(matrixB);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
+
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				//std::cout << "GPU ACTOR done  computing\n";
+				//verify its own result 
+				self -> mail(matrixA,matrixB,matrixC,N).send(self);
+
+			},
+
+					// 3rd handler: CPU atom + matrices + N
+					[=](const std::vector<int>& matrixA,
+							const std::vector<int>& matrixB,
+							const std::vector<int>& matrixC,
+							int N) {
+
+						using clock = std::chrono::high_resolution_clock;
+
+						auto start = clock::now();
+
+						//std::cout << "GPU ACTOR verifying\n";
+
+						std::vector<int> result(N * N);
+
+						serial_matrix_multiply(matrixA, matrixB, result, N);
+
+						if (result == matrixC) {
+							std::cout << "actor with id " << self->state().id
+								<< " references match\n";
+						} else {
+							std::cout << "actor with id " << self->state().id
+								<< " references did not match\n";
+						}
+
+						auto end = clock::now();
+
+						auto ms =
+							std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+						std::cout << "[TIMING] verification took "
+							<< ms << " ms (actor id "
+							<< self->state().id << ")\n";
+
+						// signal exit actor and quit
+						self->mail(1).send(exit_actor);
+						self->quit();
+
+					}
+	};
+}
+
+
+
+
+
+// this actor will not verify its results
+// great for performance analysis 
+caf::behavior mmul_actor_fun_no_verify(
+    caf::stateful_actor<mmul_actor_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims,
+    bool request
+    )
+{
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+	caf::actor scheduler = mgr.get_scheduler_actor();
+
+	if (request) {
+		//send a launch token
+		caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+				program,
+				dims,
+				0,
+				"hello",
+				self
+				);
+		self -> mail(launch_token).send(scheduler);
+	}
+	return {
+
+		[=] (caf::cuda::response_token_ptr res_token) {
+
+			if (res_token -> getType() == LAUNCH_RESPONSE) {
+				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+				//assume N = 1024
+				int N = self -> state().N;
+				std::vector<int> matrix1(N*N);
+				matrix1.reserve(N);
+				std::vector<int> matrix2(N*N);
+				matrix2.reserve(N);
+
+
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+				//create program and dims   
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrix1);
+				auto arg2 = caf::cuda::create_in_arg(matrix2);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
+
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+			
+			
+				//mask the transfer back to the cpu for scheduler
+				res_token -> release();
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				//std::cout << "GPU ACTOR done  computing\n";
+				// signal exit actor and quit
+				self->mail(1).send(exit_actor);
+				self->quit();
+
+				//std::cout << "GPU ACTOR sending data to compute\n";
+			//	self -> mail(matrix1,matrix2,res_token,N).send(self);
+
+			}
+			else {
+				std::cout << "Got a memory response token\n";
+			}
+			//token should drop out of scope now, triggering a response 
+		},
+
+			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+			[=](const std::vector<int>& matrixA,
+					const std::vector<int>& matrixB,
+					const caf::cuda::response_token_ptr& res_token, int N) {
+
+
+				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
+
+				//caf::cuda::launch_response_token& kt =
+				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
+
+				//std::cout << "GPU ACTOR  computing\n";
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+				//create program and dims   
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrixA);
+				auto arg2 = caf::cuda::create_in_arg(matrixB);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
+
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+			
+			
+				//mask the transfer back to the cpu for scheduler
+				res_token -> release();
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				//std::cout << "GPU ACTOR done  computing\n";
+				// signal exit actor and quit
+				self->mail(1).send(exit_actor);
+				self->quit();
+
+			}
+
+				
+	};
+}
+
+
+
+// Stateful actor behavior
+// this actor does not invoke the scheduler at all 
+caf::behavior mmul_actor_fun_no_schedule(
+    caf::stateful_actor<mmul_actor_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims) {
+
+    self->state().N = N;
+
+    std::vector<int> matrix1(N * N);
+    std::vector<int> matrix2(N * N);
+
+    // send initial mail to self
+    self->mail(matrix1, matrix2, N).send(self);
+
+    return {
+        // GPU atom + matrices + N
+        [=](const std::vector<int>& matrixA,
+            const std::vector<int>& matrixB,
+            int N_local) {   // avoid shadowing outer N
+            
+	    
+    	    //std::cout << "Hello\n";
+	    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+            auto arg1 = caf::cuda::create_in_arg(matrixA);
+            auto arg2 = caf::cuda::create_in_arg(matrixB);
+            auto arg3 = caf::cuda::create_out_arg(N_local * N_local);
+            auto arg4 = caf::cuda::create_in_arg(N_local);
+
+            auto tempC = mmul.run(program, dims, self->state().id, arg1, arg2, arg3, arg4);
+            std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+	    self->mail(1).send(exit_actor);
+            self->quit();
+        },
+
+        // CPU verification
+        [=](const std::vector<int>& matrixA,
+            const std::vector<int>& matrixB,
+            const std::vector<int>& matrixC,
+            int N_local) {
+
+            std::vector<int> result(N_local * N_local);
+            serial_matrix_multiply(matrixA, matrixB, result, N_local);
+
+            if (result == matrixC) {
+                std::cout << "actor with id " << self->state().id << " references match\n";
+            } else {
+                std::cout << "actor with id " << self->state().id << " references did not match\n";
+            }
+
+            self->quit();
+        }
+    };
+}
+
+
+
+
+
+
+
+
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //change the scheduler to core_usage
+  anon_mail(
+	caf::cuda::make_behavior_token("core_usage")
+	).send(mgr.get_scheduler_actor());
+
+  // CREATE ONCE
+  auto program =
+      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+  const int THREADS = 32;
+  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1);
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+  for (int i = 0; i < num_actors; ++i) {
+    /*
+    sys.spawn(
+        mmul_actor_fun,
+        exit_actor,
+        matrix_size,
+        program,
+        dims);
+    */
+      sys.spawn(
+        mmul_actor_fun_no_verify,
+        exit_actor,
+        matrix_size,
+        program,
+        dims,
+	true);
+    
+  }
+
+  sys.await_all_actors_done();
+}
+
+
+void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  /*
+  //change the scheduler to core_usage
+  anon_mail(
+	caf::cuda::make_behavior_token("core_usage")
+	).send(mgr.get_scheduler_actor());
+
+  */
+
+
+  // CREATE ONCE
+  auto program =
+      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+  const int THREADS = 32;
+  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1);
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+  for (int i = 0; i < num_actors; ++i) {
+    
+	sys.spawn(
+        mmul_actor_fun_no_verify,
+        exit_actor,
+        matrix_size,
+        program,
+        dims,
+	true);
+    
+  
+  }
+
+  sys.await_all_actors_done();
+}
+
+void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, int num_actors) {
+    if (num_actors < 1) {
+        std::cerr << "[ERROR] Number of actors must be >= 1\n";
+        return;
+    }
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    // CREATE ONCE
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    const int THREADS = 32;
+    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    for (int i = 0; i < num_actors; ++i) {
+        sys.spawn(
+            mmul_actor_fun_no_schedule,
+            exit_actor,
+            matrix_size,
+            program,
+            dims
+        );
+    }
+
+    sys.await_all_actors_done();
+}
+
+
+template <class Fn>
+double time_run(Fn&& fn) {
+  auto start = std::chrono::steady_clock::now();
+  fn();
+  auto end = std::chrono::steady_clock::now();
+  std::chrono::duration<double> elapsed = end - start;
+  return elapsed.count();
+}
+void run_mmul_scaling_tests(caf::actor_system& sys,
+                            caf::cuda::manager_config man_config) {
+    const int max_size   = 1024;
+    const int min_actors = 1;
+    const int max_actors = 1024;
+
+    std::vector<int> matrix_sizes = {10};
+    for (int s = 32; s <= max_size; s *= 2)
+        matrix_sizes.push_back(s);
+
+    std::vector<int> actor_counts;
+    for (int a = min_actors; a <= max_actors; a *= 2)
+        actor_counts.push_back(a);
+
+    std::cout << "=== MMUL Scaling Tests ===\n";
+    std::cout << "Format:\n";
+    std::cout << "scheduler matrix_size actors time_seconds\n";
+
+    for (int size : matrix_sizes) {
+        for (int actors : actor_counts) {
+
+            /* ================= Scheduler-enabled (core_usage) ================= */
+            caf::cuda::manager::init(sys, man_config); // green-light scheduler enabled
+            std::cout << "\n[RUN] scheduler=core_usage "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double core_usage_time = time_run([&] {
+                run_mmul_test(sys, size, actors); // uses mmul_actor_fun_no_verify
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT core_usage "
+                      << size << " "
+                      << actors << " "
+                      << core_usage_time << "\n";
+
+            caf::cuda::manager::shutdown(); // make sure manager is cleaned up
+
+            /* ================= Scheduler-disabled actor (still uses green-light) ================= */
+
+/*
+	    caf::cuda::manager::init(sys, man_config); // init with scheduler
+            std::cout << "\n[RUN] scheduler=green_light_only "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double green_light_time = time_run([&] {
+                run_mmul_test_no_scheduler(sys, size, actors); // your previous "no scheduler" actor
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT green_light_only "
+                      << size << " "
+                      << actors << " "
+                      << green_light_time << "\n";
+
+            caf::cuda::manager::shutdown();
+
+	    */
+            /* ================= No scheduler at all actor ================= */
+            caf::cuda::manager_config no_sched_config(false); // disable scheduler
+            caf::cuda::manager::init(sys, no_sched_config);
+            std::cout << "\n[RUN] scheduler=none "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double no_scheduler_time = time_run([&] {
+                run_mmul_test_no_scheduler_actor(sys, size, actors); // mmul_actor_fun_no_schedule
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT none "
+                      << size << " "
+                      << actors << " "
+                      << no_scheduler_time << "\n";
+
+            caf::cuda::manager::shutdown();
+        }
+    }
+
+    std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
+}
+void run_mmul_mixed_batch_one_mode(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    bool use_scheduler_actor,
+    bool use_core_usage_behavior,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get(); // SAFE NOW
+
+    if (use_scheduler_actor && use_core_usage_behavior) {
+        anon_mail(caf::cuda::make_behavior_token("core_usage"))
+            .send(mgr.get_scheduler_actor());
+    }
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    std::mt19937 rng(123456);
+    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+
+    const int THREADS = 32;
+
+    for (int i = 0; i < num_actors; ++i) {
+        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
+        int BLOCKS = (N + THREADS - 1) / THREADS;
+
+        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+        if (use_scheduler_actor) {
+            sys.spawn(
+                mmul_actor_fun_no_verify,
+                exit_actor,
+                N,
+                program,
+                dims,
+		true);
+        } else {
+            sys.spawn(
+                mmul_actor_fun_no_schedule,
+                exit_actor,
+                N,
+                program,
+                dims);
+        }
+    }
+
+    sys.await_all_actors_done();
+}
+
+
+
+
+void run_mmul_mixed_batch_one_mode_bulk(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get(); // SAFE NOW
+
+       // anon_mail(caf::cuda::make_behavior_token("core_usage"))
+         //  .send(mgr.get_scheduler_actor());
+
+
+        anon_mail(caf::cuda::make_behavior_token("multilevel"))
+            .send(mgr.get_scheduler_actor());
+
+	//anon_mail(caf::cuda::make_behavior_token("pressure"))
+          //  .send(mgr.get_scheduler_actor());
+
+
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    std::vector<caf::cuda::token_ptr> tokens(num_actors);
+
+    std::mt19937 rng(123456);
+    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+
+    const int THREADS = 32;
+
+    auto t_start = std::chrono::steady_clock::now();
+
+for (int i = 0; i < num_actors; ++i) {
+    int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
+    int BLOCKS = (N + THREADS - 1) / THREADS;
+
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1,
+                             THREADS, THREADS, 1);
+
+    caf::actor a = sys.spawn(
+        mmul_actor_fun_no_verify,
+        exit_actor,
+        N,
+        program,
+        dims,
+        false
+    );
+
+    tokens[i] = caf::cuda::make_launch_token(
+        program,
+        dims,
+        0, // FIXME later
+        "hello",
+        a
+    );
+}
+
+auto t_end = std::chrono::steady_clock::now();
+
+auto us = std::chrono::duration_cast<std::chrono::microseconds>(
+              t_end - t_start
+          ).count();
+
+std::cout << "Actor spawn + token creation loop took "
+          << us << " us\n";
+
+    
+        anon_mail(tokens)
+            .send(mgr.get_scheduler_actor());
+
+
+    sys.await_all_actors_done();
+}
+
+
+
+
+
+
+
+
+
+
+void run_mmul_mixed_batch_comparison(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors)
+{
+    std::cout << "\n=== MMUL Mixed-Size Batch Comparison ===\n";
+    std::cout << "scheduler actors sizes time_seconds\n\n";
+
+    /* ================= core_usage ================= */
+    {
+        caf::cuda::manager_config cfg(true);
+        caf::cuda::manager::init(sys, cfg);
+
+	
+        //double t = time_run([&] {
+          //  run_mmul_mixed_batch_one_mode(
+            //    sys, sizes, num_actors,
+              //  /*use_scheduler_actor=*/true,
+               // /*use_core_usage_behavior=*/true);
+       // });
+
+	double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode_bulk(
+                sys, sizes, num_actors);
+        });
+
+
+
+
+        std::cout << "RESULT core_usage "
+                  << num_actors << " "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    /* ================= green-light only ================= */
+   
+    /*
+    {
+	    std::cout << "Starting green_light tests\n";
+        caf::cuda::manager_config cfg(true);
+        caf::cuda::manager::init(sys, cfg);
+
+        double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode(
+                sys, sizes, num_actors,
+               true,
+                false);
+        });
+
+        std::cout << "RESULT green_light_only "
+                  << num_actors << " "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    */
+	
+    /* ================= no scheduler ================= */
+    {
+        caf::cuda::manager_config cfg(false);
+        caf::cuda::manager::init(sys, cfg);
+
+        double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode(
+                sys, sizes, num_actors,
+                /*use_scheduler_actor=*/false,
+                /*use_core_usage_behavior=*/false);
+        });
+
+        std::cout << "RESULT none "
+                  << num_actors << " "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    std::cout << "\n=== Comparison Complete ===\n";
+}
+
+
+
+void test_core_usage_uniform_mmul(
+    caf::actor_system& sys,
+    int matrix_size,
+    int num_actors,
+    std::string scheduler_behavior)
+{
+    std::cout << "\n[TEST] core_usage uniform matrix size\n";
+    std::cout << "N=" << matrix_size
+              << " actors=" << num_actors << "\n";
+
+    caf::cuda::manager_config cfg(true);
+    caf::cuda::manager::init(sys, cfg);
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    // force core_usage behavior
+    anon_mail(caf::cuda::make_behavior_token(scheduler_behavior))
+        .send(mgr.get_scheduler_actor());
+
+    auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    const int THREADS = 32;
+    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(
+        BLOCKS, BLOCKS, 1,
+        THREADS, THREADS, 1);
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    for (int i = 0; i < num_actors; ++i) {
+        sys.spawn(
+            mmul_actor_fun_no_verify,
+            exit_actor,
+            matrix_size,
+            program,
+            dims,
+	    true);
+    }
+
+    sys.await_all_actors_done();
+    caf::cuda::manager::shutdown();
+
+    std::cout << "[PASS] core_usage uniform test complete\n";
+}
+
+
+
+void test_core_usage_mixed_mmul(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    std::string scheduler_behavior)
+{
+    std::cout << "\n[TEST] core_usage mixed matrix sizes\n";
+    std::cout << "actors=" << num_actors << "\n";
+
+    caf::cuda::manager_config cfg(true);
+    caf::cuda::manager::init(sys, cfg);
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    anon_mail(caf::cuda::make_behavior_token(scheduler_behavior))
+        .send(mgr.get_scheduler_actor());
+
+    auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    const int THREADS = 32;
+
+    for (int i = 0; i < num_actors; ++i) {
+        int N = sizes[i % sizes.size()];
+
+        int BLOCKS = (N + THREADS - 1) / THREADS;
+        caf::cuda::nd_range dims(
+            BLOCKS, BLOCKS, 1,
+            THREADS, THREADS, 1);
+
+        sys.spawn(
+            mmul_actor_fun_no_verify,
+            exit_actor,
+            N,
+            program,
+            dims,
+	    true);
+    }
+
+    sys.await_all_actors_done();
+    caf::cuda::manager::shutdown();
+
+    std::cout << "[PASS] core_usage mixed-size test complete\n";
+}
+
+
+void run_mmul_fixed_256_batch_comparison(
+    caf::actor_system& sys,
+    int num_actors)
+{
+    // All actors run the same matrix size: 256
+    std::vector<int> sizes(num_actors, 256);
+
+    std::cout << "\n=== MMUL Fixed-Size (256) Batch Comparison ===\n";
+    std::cout << "scheduler actors size time_seconds\n\n";
+
+    /* ================= core_usage ================= */
+    {
+        caf::cuda::manager_config cfg(true);
+        caf::cuda::manager::init(sys, cfg);
+
+        double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode(
+                sys,
+                sizes,
+                num_actors,
+                /*use_scheduler_actor=*/true,
+                /*use_core_usage_behavior=*/true);
+        });
+
+        std::cout << "RESULT core_usage "
+                  << num_actors << " 256 "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    /* ================= no scheduler ================= */
+    {
+        caf::cuda::manager_config cfg(false);
+        caf::cuda::manager::init(sys, cfg);
+
+        double t = time_run([&] {
+            run_mmul_mixed_batch_one_mode(
+                sys,
+                sizes,
+                num_actors,
+                /*use_scheduler_actor=*/false,
+                /*use_core_usage_behavior=*/false);
+        });
+
+        std::cout << "RESULT none "
+                  << num_actors << " 256 "
+                  << t << "\n";
+
+        caf::cuda::manager::shutdown();
+    }
+
+    std::cout << "\n=== Fixed-256 Comparison Complete ===\n";
+}
+
+
+
+
+
+void caf_main(caf::actor_system& sys) {
+  
+	//caf::cuda::manager_config man_config(true); //turns the scheduler on
+	//caf::cuda::manager::init(sys,man_config);
+        // run_mmul_test(sys,10,64);	
+	//run_mmul_scaling_tests(sys,man_config);
+
+   std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048};
+    const int num_actors = 2000;
+    run_mmul_mixed_batch_comparison(sys, sizes, num_actors);    
+
+
+    //run_mmul_mixed_batch_one_mode_bulk(sys,sizes,num_actors);
+    //run_mmul_fixed_256_batch_comparison(sys, /*num_actors=*/200);
+
+
+	// test_core_usage_uniform_mmul(sys, 256, 1000,"multilevel");
+
+  //std::vector<int> sizes = {32, 64, 128, 256, 512, 1024};
+    //test_core_usage_mixed_mmul(sys, sizes, 200,"multilevel");
+
+
+
+//tests will delete the old manager so will have to reinit if you do this 
+	//in conjunction with each other	
+	//caf::cuda::manager::init(sys,man_config);
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/mmul.cu b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From 776ccf908333b06bfab0046bc293b11f9378f262 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 11 Feb 2026 15:12:52 -0600
Subject: [PATCH 0361/1000] Cleared out a lot of redudant code and implemented
 run_load_balance_test.

---
 .../load-balancing-test/main.test.cpp         | 442 +-----------------
 1 file changed, 20 insertions(+), 422 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index 568dd032d9..36b5452b6c 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -550,162 +550,25 @@ double time_run(Fn&& fn) {
   std::chrono::duration<double> elapsed = end - start;
   return elapsed.count();
 }
-void run_mmul_scaling_tests(caf::actor_system& sys,
-                            caf::cuda::manager_config man_config) {
-    const int max_size   = 1024;
-    const int min_actors = 1;
-    const int max_actors = 1024;
-
-    std::vector<int> matrix_sizes = {10};
-    for (int s = 32; s <= max_size; s *= 2)
-        matrix_sizes.push_back(s);
-
-    std::vector<int> actor_counts;
-    for (int a = min_actors; a <= max_actors; a *= 2)
-        actor_counts.push_back(a);
-
-    std::cout << "=== MMUL Scaling Tests ===\n";
-    std::cout << "Format:\n";
-    std::cout << "scheduler matrix_size actors time_seconds\n";
-
-    for (int size : matrix_sizes) {
-        for (int actors : actor_counts) {
-
-            /* ================= Scheduler-enabled (core_usage) ================= */
-            caf::cuda::manager::init(sys, man_config); // green-light scheduler enabled
-            std::cout << "\n[RUN] scheduler=core_usage "
-                      << "matrix_size=" << size
-                      << " actors=" << actors << "\n";
-
-            double core_usage_time = time_run([&] {
-                run_mmul_test(sys, size, actors); // uses mmul_actor_fun_no_verify
-            });
-
-            std::cout << std::fixed << std::setprecision(6)
-                      << "RESULT core_usage "
-                      << size << " "
-                      << actors << " "
-                      << core_usage_time << "\n";
-
-            caf::cuda::manager::shutdown(); // make sure manager is cleaned up
-
-            /* ================= Scheduler-disabled actor (still uses green-light) ================= */
-
-/*
-	    caf::cuda::manager::init(sys, man_config); // init with scheduler
-            std::cout << "\n[RUN] scheduler=green_light_only "
-                      << "matrix_size=" << size
-                      << " actors=" << actors << "\n";
-
-            double green_light_time = time_run([&] {
-                run_mmul_test_no_scheduler(sys, size, actors); // your previous "no scheduler" actor
-            });
-
-            std::cout << std::fixed << std::setprecision(6)
-                      << "RESULT green_light_only "
-                      << size << " "
-                      << actors << " "
-                      << green_light_time << "\n";
-
-            caf::cuda::manager::shutdown();
-
-	    */
-            /* ================= No scheduler at all actor ================= */
-            caf::cuda::manager_config no_sched_config(false); // disable scheduler
-            caf::cuda::manager::init(sys, no_sched_config);
-            std::cout << "\n[RUN] scheduler=none "
-                      << "matrix_size=" << size
-                      << " actors=" << actors << "\n";
-
-            double no_scheduler_time = time_run([&] {
-                run_mmul_test_no_scheduler_actor(sys, size, actors); // mmul_actor_fun_no_schedule
-            });
-
-            std::cout << std::fixed << std::setprecision(6)
-                      << "RESULT none "
-                      << size << " "
-                      << actors << " "
-                      << no_scheduler_time << "\n";
-
-            caf::cuda::manager::shutdown();
-        }
-    }
-
-    std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
-}
-void run_mmul_mixed_batch_one_mode(
-    caf::actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors,
-    bool use_scheduler_actor,
-    bool use_core_usage_behavior,
-    bool randomize = false)
-{
-    caf::cuda::manager& mgr = caf::cuda::manager::get(); // SAFE NOW
-
-    if (use_scheduler_actor && use_core_usage_behavior) {
-        anon_mail(caf::cuda::make_behavior_token("core_usage"))
-            .send(mgr.get_scheduler_actor());
-    }
-
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    std::mt19937 rng(123456);
-    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
-
-    const int THREADS = 32;
-
-    for (int i = 0; i < num_actors; ++i) {
-        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
-        int BLOCKS = (N + THREADS - 1) / THREADS;
-
-        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-        if (use_scheduler_actor) {
-            sys.spawn(
-                mmul_actor_fun_no_verify,
-                exit_actor,
-                N,
-                program,
-                dims,
-		true);
-        } else {
-            sys.spawn(
-                mmul_actor_fun_no_schedule,
-                exit_actor,
-                N,
-                program,
-                dims);
-        }
-    }
-
-    sys.await_all_actors_done();
-}
 
 
-
-
-void run_mmul_mixed_batch_one_mode_bulk(
+//this test is meant to demonstrate the fact that scheduler actors can 
+//migrate work to correct load imbalance
+//the sizes should be large enough such that the tests exceed 4-5 seconds in total
+//otherwise the schedulers wont care to do this fast enough
+void run_load_balance_test(
     caf::actor_system& sys,
     const std::vector<int>& sizes,
     int num_actors,
     bool randomize = false)
 {
-    caf::cuda::manager& mgr = caf::cuda::manager::get(); // SAFE NOW
-
-       // anon_mail(caf::cuda::make_behavior_token("core_usage"))
-         //  .send(mgr.get_scheduler_actor());
-
-
-        anon_mail(caf::cuda::make_behavior_token("multilevel"))
-            .send(mgr.get_scheduler_actor());
-
-	//anon_mail(caf::cuda::make_behavior_token("pressure"))
-          //  .send(mgr.get_scheduler_actor());
+    caf::cuda::manager& mgr = caf::cuda::manager::get(); 
 
 
+    //set the behaviors of each scheduler actor
+    for (int i = 0; i < mgr.get_num_devices();i++) {
+	    mgr.send_scheduler_actor_message("multilevel",i);
+    }
 
     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
@@ -739,7 +602,7 @@ for (int i = 0; i < num_actors; ++i) {
     tokens[i] = caf::cuda::make_launch_token(
         program,
         dims,
-        0, // FIXME later
+        0, // memory usage is zero for now, we still do not track it at all
         "hello",
         a
     );
@@ -754,288 +617,23 @@ auto us = std::chrono::duration_cast<std::chrono::microseconds>(
 std::cout << "Actor spawn + token creation loop took "
           << us << " us\n";
 
-    
-        anon_mail(tokens)
-            .send(mgr.get_scheduler_actor());
-
-
-    sys.await_all_actors_done();
-}
-
-
-
-
-
-
-
-
-
-
-void run_mmul_mixed_batch_comparison(
-    caf::actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors)
-{
-    std::cout << "\n=== MMUL Mixed-Size Batch Comparison ===\n";
-    std::cout << "scheduler actors sizes time_seconds\n\n";
-
-    /* ================= core_usage ================= */
-    {
-        caf::cuda::manager_config cfg(true);
-        caf::cuda::manager::init(sys, cfg);
-
-	
-        //double t = time_run([&] {
-          //  run_mmul_mixed_batch_one_mode(
-            //    sys, sizes, num_actors,
-              //  /*use_scheduler_actor=*/true,
-               // /*use_core_usage_behavior=*/true);
-       // });
-
-	double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode_bulk(
-                sys, sizes, num_actors);
-        });
-
-
-
-
-        std::cout << "RESULT core_usage "
-                  << num_actors << " "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    /* ================= green-light only ================= */
    
-    /*
-    {
-	    std::cout << "Starting green_light tests\n";
-        caf::cuda::manager_config cfg(true);
-        caf::cuda::manager::init(sys, cfg);
-
-        double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode(
-                sys, sizes, num_actors,
-               true,
-                false);
-        });
-
-        std::cout << "RESULT green_light_only "
-                  << num_actors << " "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    */
-	
-    /* ================= no scheduler ================= */
-    {
-        caf::cuda::manager_config cfg(false);
-        caf::cuda::manager::init(sys, cfg);
-
-        double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode(
-                sys, sizes, num_actors,
-                /*use_scheduler_actor=*/false,
-                /*use_core_usage_behavior=*/false);
-        });
-
-        std::cout << "RESULT none "
-                  << num_actors << " "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    std::cout << "\n=== Comparison Complete ===\n";
-}
-
-
-
-void test_core_usage_uniform_mmul(
-    caf::actor_system& sys,
-    int matrix_size,
-    int num_actors,
-    std::string scheduler_behavior)
-{
-    std::cout << "\n[TEST] core_usage uniform matrix size\n";
-    std::cout << "N=" << matrix_size
-              << " actors=" << num_actors << "\n";
-
-    caf::cuda::manager_config cfg(true);
-    caf::cuda::manager::init(sys, cfg);
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-    // force core_usage behavior
-    anon_mail(caf::cuda::make_behavior_token(scheduler_behavior))
-        .send(mgr.get_scheduler_actor());
-
-    auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    const int THREADS = 32;
-    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-    caf::cuda::nd_range dims(
-        BLOCKS, BLOCKS, 1,
-        THREADS, THREADS, 1);
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    for (int i = 0; i < num_actors; ++i) {
-        sys.spawn(
-            mmul_actor_fun_no_verify,
-            exit_actor,
-            matrix_size,
-            program,
-            dims,
-	    true);
-    }
-
-    sys.await_all_actors_done();
-    caf::cuda::manager::shutdown();
-
-    std::cout << "[PASS] core_usage uniform test complete\n";
+	  //send the tokens to only 1 GPU and let them
+	  //figure out that there is a load imbalance 
+	  mgr.send_scheduler_actor_message(tokens);
+          sys.await_all_actors_done();
 }
 
 
-
-void test_core_usage_mixed_mmul(
-    caf::actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors,
-    std::string scheduler_behavior)
-{
-    std::cout << "\n[TEST] core_usage mixed matrix sizes\n";
-    std::cout << "actors=" << num_actors << "\n";
-
-    caf::cuda::manager_config cfg(true);
-    caf::cuda::manager::init(sys, cfg);
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-    anon_mail(caf::cuda::make_behavior_token(scheduler_behavior))
-        .send(mgr.get_scheduler_actor());
-
-    auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    const int THREADS = 32;
-
-    for (int i = 0; i < num_actors; ++i) {
-        int N = sizes[i % sizes.size()];
-
-        int BLOCKS = (N + THREADS - 1) / THREADS;
-        caf::cuda::nd_range dims(
-            BLOCKS, BLOCKS, 1,
-            THREADS, THREADS, 1);
-
-        sys.spawn(
-            mmul_actor_fun_no_verify,
-            exit_actor,
-            N,
-            program,
-            dims,
-	    true);
-    }
-
-    sys.await_all_actors_done();
-    caf::cuda::manager::shutdown();
-
-    std::cout << "[PASS] core_usage mixed-size test complete\n";
-}
-
-
-void run_mmul_fixed_256_batch_comparison(
-    caf::actor_system& sys,
-    int num_actors)
-{
-    // All actors run the same matrix size: 256
-    std::vector<int> sizes(num_actors, 256);
-
-    std::cout << "\n=== MMUL Fixed-Size (256) Batch Comparison ===\n";
-    std::cout << "scheduler actors size time_seconds\n\n";
-
-    /* ================= core_usage ================= */
-    {
-        caf::cuda::manager_config cfg(true);
-        caf::cuda::manager::init(sys, cfg);
-
-        double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode(
-                sys,
-                sizes,
-                num_actors,
-                /*use_scheduler_actor=*/true,
-                /*use_core_usage_behavior=*/true);
-        });
-
-        std::cout << "RESULT core_usage "
-                  << num_actors << " 256 "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    /* ================= no scheduler ================= */
-    {
-        caf::cuda::manager_config cfg(false);
-        caf::cuda::manager::init(sys, cfg);
-
-        double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode(
-                sys,
-                sizes,
-                num_actors,
-                /*use_scheduler_actor=*/false,
-                /*use_core_usage_behavior=*/false);
-        });
-
-        std::cout << "RESULT none "
-                  << num_actors << " 256 "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    std::cout << "\n=== Fixed-256 Comparison Complete ===\n";
-}
-
-
-
-
-
 void caf_main(caf::actor_system& sys) {
   
-	//caf::cuda::manager_config man_config(true); //turns the scheduler on
-	//caf::cuda::manager::init(sys,man_config);
-        // run_mmul_test(sys,10,64);	
-	//run_mmul_scaling_tests(sys,man_config);
-
-   std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048};
-    const int num_actors = 2000;
-    run_mmul_mixed_batch_comparison(sys, sizes, num_actors);    
-
-
-    //run_mmul_mixed_batch_one_mode_bulk(sys,sizes,num_actors);
-    //run_mmul_fixed_256_batch_comparison(sys, /*num_actors=*/200);
-
-
-	// test_core_usage_uniform_mmul(sys, 256, 1000,"multilevel");
-
-  //std::vector<int> sizes = {32, 64, 128, 256, 512, 1024};
-    //test_core_usage_mixed_mmul(sys, sizes, 200,"multilevel");
-
+	caf::cuda::manager_config man_config(true); //turns the scheduler on
+	caf::cuda::manager::init(sys,man_config);
+	std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048,4096};
+	const int num_actors = 2000;
+	run_load_balance_test(sys,sizes,num_actors);
 
 
-//tests will delete the old manager so will have to reinit if you do this 
-	//in conjunction with each other	
-	//caf::cuda::manager::init(sys,man_config);
 }
 
 

From 7a6de13cd6496054da6b7d867d1deaa0f420687b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 11 Feb 2026 16:14:21 -0600
Subject: [PATCH 0362/1000] Moved behavior initilization responsiblty to
 behavior table, This change was made to fix issues with static declarations
 of behaviors causing race conditions if multiple scheduler actors were
 created.

---
 .../caf/cuda/control-layer/behavior_table.hpp | 12 ++++++--
 .../control-layer/scheduler_actor_state.hpp   |  3 +-
 .../src/control-layer/behavior_table.cpp      | 28 +++++++++++++++----
 .../src/control-layer/scheduler_actor.cpp     | 25 +++--------------
 4 files changed, 37 insertions(+), 31 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
index 210a2991a9..d6c993fad1 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
@@ -6,12 +6,18 @@ namespace caf::cuda {
 
 class scheduler_actor_behavior; // Forward declaration is fine here
 class behavior_token;
+class scheduler_actor_state;
 
 class behavior_table {
 public:
-    behavior_table() = default;
-
-    ~behavior_table();  // ← Declaration only
+       	behavior_table() = default;
+      // Constructor that creates all default behaviors for a given actor state
+    explicit behavior_table(scheduler_actor_state& state);
+    
+
+ 
+    ~behavior_table();
+        
 
     void add(const std::string& name, scheduler_actor_behavior* beh) {
         table_[name] = beh;
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
index b7f21640a4..cf3dc19a3b 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
@@ -12,7 +12,8 @@ class scheduler_actor_behavior;
 struct scheduler_actor_state {
     caf::actor  self;
     scheduler_actor_behavior* current_behavior = nullptr;
-    behavior_table table;
+    std::unique_ptr<behavior_table> table;
+    //behavior_table table;
     std::queue<token_ptr> queue; // here for legacy prototype schedulers
     std::vector<kernel_graph> operations; //more modern dependency based data structure 
     int device_number;
diff --git a/libcaf_cuda/src/control-layer/behavior_table.cpp b/libcaf_cuda/src/control-layer/behavior_table.cpp
index c9e11d0118..2503ea91dc 100644
--- a/libcaf_cuda/src/control-layer/behavior_table.cpp
+++ b/libcaf_cuda/src/control-layer/behavior_table.cpp
@@ -1,16 +1,36 @@
 #include "caf/cuda/control-layer/behavior_table.hpp"
 
-// IMPORTANT: Include the full definition of the base class
 #include "caf/cuda/control-layer/behavior.hpp"  
-#include "caf/cuda/control-layer/behavior_token.hpp"  // ← full type here
+#include "caf/cuda/control-layer/behavior_token.hpp"  
 
 // If your derived classes have important cleanup, also include them if needed
 #include "caf/cuda/control-layer/green_light_behavior.hpp"
 #include "caf/cuda/control-layer/red_light_behavior.hpp"
+#include "caf/cuda/control-layer/core_usage_behavior.hpp"
+#include "caf/cuda/control-layer/single_usage_behavior.hpp"
+#include "caf/cuda/control-layer/multilevel_usage_behavior.hpp"
+#include "caf/cuda/control-layer/pressure_scheduler.hpp"
 #include "caf/cuda/control-layer/all-control-layer.hpp"
 
 namespace caf::cuda {
 
+ // Constructor that creates all default behaviors for a given actor state
+    behavior_table::behavior_table(scheduler_actor_state& state) {
+        // dynamically allocate behaviors and add to table
+        add("red",          new red_light_behavior(state));
+        add("green",        new green_light_behavior(state));
+        add("core_usage",   new core_usage_behavior(state));
+        add("single_usage", new single_usage_behavior(state));
+        add("multilevel",   new multilevel_usage_behavior(state));
+        add("pressure",     new pressure_scheduler(state));
+    }
+
+
+    behavior_table::~behavior_table() {
+        for (auto& [name, beh] : table_)
+            delete beh;  // clean up all behaviors on destruction
+    }
+
 
 scheduler_actor_behavior* behavior_table::get(const behavior_token& tok) const {
     auto it = table_.find(tok.name());
@@ -18,8 +38,4 @@ scheduler_actor_behavior* behavior_table::get(const behavior_token& tok) const {
 }
 
 
-behavior_table::~behavior_table() {
-    table_.clear();  // optional but clean
-}
-
 } // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 37c5ec7d06..4926aa963b 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -28,26 +28,9 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
     //check if multiple gpus
     state.multiple_gpus = multi_gpu;
 
-    //static declarations may cause issues when expanding to
-    //multiple GPUs
-    static red_light_behavior red_behavior(state);
-    static green_light_behavior green_behavior(state);
-    static core_usage_behavior core_behavior(state);
-    static single_usage_behavior single_behavior(state);
-    static multilevel_usage_behavior multi_behavior(state);
-    static pressure_scheduler pressure(state);
-
-    // populate the behavior table
-    state.table.add("red", &red_behavior);
-    state.table.add("green",   &green_behavior);
-    state.table.add("core_usage",   &core_behavior);
-    state.table.add("multilevel",   &multi_behavior);
-    state.table.add("pressure",   &pressure);
-    state.table.add("single_usage",   &single_behavior);
-
-    // default behavior
-    //state.current_behavior = state.table.get(behavior_token("green"));
-    state.current_behavior = state.table.get(behavior_token("single_usage"));
+   // default behavior
+    state.table = std::make_unique<behavior_table>(state);
+    state.current_behavior = state.table -> get(behavior_token("single_usage"));
     state.current_behavior->on_enter();
 
 
@@ -59,7 +42,7 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
         },
         
    [&state](const caf::cuda::behavior_token_ptr& tok) -> bool {
-    auto* next = state.table.get(*tok);
+    auto* next = state.table -> get(*tok);
     if (next) {
         if (next != state.current_behavior) {
             state.current_behavior->on_exit();   // cleanup current behavior

From 7a814f384d5936598a9596969b96c065b25ec5aa Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 11 Feb 2026 16:34:24 -0600
Subject: [PATCH 0363/1000] Added new threshold variable and tweaked
 conditionals to fix an issue where scheduler actors would gasley conclude
 that it did not need to load balance across multiple GPUs

---
 .../multilevel_usage_behavior.hpp             |  1 +
 .../multilevel_usage_behavior.cpp             | 26 ++++++++++++++-----
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
index 1bce6b9deb..72fba04272 100644
--- a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
@@ -66,6 +66,7 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
     int low_threshold = 0; //this is used to check if we should request more work
 			   //or not
 
+    int transfer_threshold =0; //check if we should transfer work or not
     // dependency -> device mapping
     std::unordered_map<int, int> dependency_device_map;
 
diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 82317864b3..dadbefc6e7 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -19,11 +19,13 @@ void multilevel_usage_behavior::init_state() {
     available_memory = static_cast<int>(device_->total_memory_bytes());
     num_streams = state_.num_streams;
     low_threshold = total_SM / 6;
+    transfer_threshold = total_SM / 2;
     num_devices = manager::get().get_num_devices();
 }
 
 void multilevel_usage_behavior::on_enter() {
 
+	//std::cout << "scheduler actor with device number " << state_.device_number << " Says hello\n";
 	//trigger load balancing mechanisms
 	if (state_.multiple_gpus) {	
 		send_timed_msg();
@@ -212,9 +214,12 @@ void multilevel_usage_behavior::reclaim(ack& return_msg) {
 
 	//TODO IMPLEMENT TIMER ACK AND TRANSFER ACK 
 
+	//std::cout << "scheduler actor with device number " << state_.device_number << " got an ack\n";
 	if (return_msg.getType() == TIMER) {
 
-			
+	//std::cout << "scheduler actor with device number " << state_.device_number << " got a timer ack\n";
+	
+		request_load_balance();	
 		send_timed_msg();	
 	}
 
@@ -246,13 +251,20 @@ void multilevel_usage_behavior::process_transfer_ack(ack& msg) {
 
 
 void multilevel_usage_behavior::request_load_balance() {
-    if (!state_.multiple_gpus)
-        return;
+	//std::cout << "Scheduler with device number " << state_.device_number << "is requesting load balance\n";  
+  if (!state_.multiple_gpus)  {
 
-    // Only request work if we're underutilized
-    if (available_SM >= low_threshold)
         return;
+  }
+    // Only request work if we're underutilized
 
+    int busy_SM = total_SM - available_SM;
+    if (busy_SM > low_threshold)
+    {
+	 // std::cout << "Returning from since too busy\n";
+        return;
+    }
+    //std::cout << "Hello from request load_balance\n";
     int my_device = state_.device_number;
 
     for (int i = 0; i < num_devices; ++i) {
@@ -287,7 +299,9 @@ kernel_graph* multilevel_usage_behavior::resolve(const graph_ref& ref) {
 void multilevel_usage_behavior::handle_load_balance_request(int device_number) {
     
     // Only transfer work if we are busy
-    if (available_SM < low_threshold) {
+
+    int free_SM = total_SM -  available_SM;
+    if (free_SM < transfer_threshold) {
         return; // GPU not busy enough, do nothing
     }
 

From dc78f524472878154de6f9e652379d406a39ceb7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Feb 2026 13:48:52 -0600
Subject: [PATCH 0364/1000] Implemented pipeline actor and made it so that it
 can handle transfer tokens if needed\n

---
 .../load-balancing-test/main.test.cpp         | 327 +++++++++++++++++-
 1 file changed, 326 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index 36b5452b6c..541f914e98 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -34,7 +34,7 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int lim
 		[=](int num_completed) {
 			self->state().completed += num_completed;
 			
-			//std::cout << "Actors finished is " << self->state().completed << "\n";
+		//	std::cout << "Actors finished is " << self->state().completed << "\n";
 			if (self->state().completed >= limit) {
 			
 				caf::cuda::manager::shutdown();
@@ -625,6 +625,331 @@ std::cout << "Actor spawn + token creation loop took "
 }
 
 
+//-------------------------------------try load balancing with actors with dependencies
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+// --- command runner types (put near top of file) -------------------------
+using initCommand =
+  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, in<int>, in<unsigned long long>>;
+
+using divCommand  =
+  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, in<int>>;
+
+using sumCommand  =
+  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, in<int>>;
+
+// single instances (can be file-global)
+static initCommand init_cmd;
+static divCommand  div_cmd;
+static sumCommand  sum_cmd;
+
+// --- pipeline actor state (device buffers persist here) ------------------
+struct pipeline_actor_state {
+    int id = rand();
+
+    // device-side buffers that must persist across stages:
+    caf::cuda::mem_ptr<float> d_denoms;
+    caf::cuda::mem_ptr<float> d_results;
+    caf::cuda::mem_ptr<float> d_sum;
+};
+
+// --- corrected pipeline_actor -------------------------------------------
+behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
+                        actor supervisor,
+                        caf::cuda::program_ptr p1,
+                        caf::cuda::program_ptr p2,
+                        caf::cuda::program_ptr p3,
+                        int n)
+{
+    // host-side scratch (only used for post-stage2 NaN/Inf detection)
+    std::vector<float> h_results;
+
+
+    // nd_range used for all stages (adapt to your kernels as needed)
+    caf::cuda::nd_range range{
+        {(n + 255) / 256, 1, 1},
+        {256, 1, 1}
+    };
+
+    // helper to create and send a launch token
+    auto launch = [&](caf::cuda::program_ptr prog, const std::string& stage) {
+        auto tok = make_launch_token(
+            prog,
+            range,
+            /*memory_usage=*/static_cast<int>(sizeof(float) * n),
+            stage,
+            self,
+            self->state().id  // dependency/demo id
+        );
+
+	//do not specifiy a device number to send it to, let it figure it out
+	caf::cuda::manager::get().send_scheduler_actor_message(tok);
+    };
+
+    // fire all three tokens (scheduler will reply with response_token on grants)
+    launch(p1, "stage1");
+    launch(p2, "stage2");
+    launch(p3, "stage3");
+
+    return {
+
+        // handle response tokens by name — opaque to reclaim payload
+        [=](caf::cuda::response_token_ptr res_token) mutable {
+
+            const auto& stage = res_token->name();
+
+	    if (res_token->getType() == LAUNCH) {
+
+		    // --------------------- Stage 1: init_denominators ---------------------
+		    if (stage == "stage1") {
+			    // allocate device buffer for denominators (persist in state)
+
+			    std::cout << "Starting stage 1\n";
+			    unsigned long long seed = static_cast<unsigned long long>(
+					    std::chrono::high_resolution_clock::now().time_since_epoch().count()
+					    );
+
+
+
+			    out<float> buffer = caf::cuda::create_out_arg_with_size<float>(n);
+			    self->state().d_denoms = init_cmd.transfer_memory(res_token,buffer);
+
+			    // run kernel on the stream/device from res_token
+			    // kernel signature: (float* denominators, int n, unsigned long long seed)
+			    init_cmd.run_async(
+					    p1,
+					    range,
+					    res_token,                            // uses token's stream/device
+					    self->state().d_denoms,               // device buffer
+					    caf::cuda::create_in_arg(n),          // n
+					    caf::cuda::create_in_arg(seed)     // seed
+					    );
+
+			    std::cout << "Finished stage 1\n";
+			    // stage1 intentionally no checks — data may contain zeros
+			    return;
+		    }
+
+		    // --------------------- Stage 2: perform_division ---------------------
+		    if (stage == "stage2") {
+			    // allocate device buffer for results (persist in state)
+			    std::vector<float> buffer1(n);
+
+			    std::cout << "Starting stage 2\n";
+			    self->state().d_results = div_cmd.transfer_memory(res_token,out<float>{buffer1});
+
+			    // create a host numerators vector (all ones)
+			    std::vector<float> h_nums(n, 1.0f);
+
+			    // transfer numerators to device on the token's stream/device
+			    // transfer_memory returns a caf::cuda::mem_ptr<float>
+			    auto d_nums = div_cmd.transfer_memory(res_token, in_out<float>{h_nums});
+
+			    // run division kernel on the token's stream/device:
+			    // kernel signature: (float* numerators, float* denominators, float* results, int n)
+			    div_cmd.run(
+					    p2,
+					    range,
+					    res_token,
+					    d_nums,
+					    self->state().d_denoms,
+					    self->state().d_results,
+					    caf::cuda::create_in_arg(n)
+				       );
+
+
+			    //there could be a division by zero in here 
+			    //but this is a load balancing test
+			    //not a fault test,
+			    //go see the fault tolerance test to see how thats handled	
+
+			    /*
+			    // extract the device results back to host for verification.
+			    // extract_vector will synchronize as needed.
+			    h_results = self->state().d_results -> copy_to_host();
+
+			    // check for NaN/Inf AFTER the kernel finished
+			    bool fault = false;
+			    for (float v : h_results) {
+				    if (!std::isfinite(v)) {
+					    fault = true;
+					    break;
+				    }
+			    }
+
+			    if (fault) {
+				    // inform supervisor and exit;
+				    anon_mail(std::string("crash")).send(supervisor);
+				    self->quit();
+				    return;
+			    }
+
+			    // stage2 passed — keep d_results in state for stage3
+			    */
+			    return;
+		    }
+
+		    // --------------------- Stage 3: sum_results --------------------------
+		    if (stage == "stage3") {
+			    // allocate device scalar for sum result
+
+			    //std::cout << "Starting stage 3\n";
+			    std::vector<float> buffer1(1);
+
+			    self->state().d_sum = div_cmd.transfer_memory(res_token,out<float>{buffer1});
+
+			    // run reduction on the token's stream/device:
+			    // kernel signature: (float* results, float* final_sum, int n)
+			    sum_cmd.run(
+					    p3,
+					    range,
+					    res_token,
+					    self->state().d_results,
+					    self->state().d_sum,
+					    caf::cuda::create_in_arg(n)
+				       );
+
+			    // extract final scalar
+
+			    std::vector<float> buf = self->state().d_sum -> copy_to_host();
+			    float final_sum = buf[0];
+			    std::cout << "[pipeline] completed, sum = " << final_sum << "\n";
+
+			    anon_mail(1).send(supervisor);
+
+			    // quit the pipeline actor
+			    self->quit();
+			    return;
+		    }
+	    }
+
+	    else if (res_token->getType() == TRANSFER) {
+	   
+		    std::cout << "Got a transfer token\n";
+
+		   if (stage == "stage1") {
+			   res_token->release(); // no dependencies at this point clear to continue
+		   } 
+
+		   else if (stage == "stage2") {
+		  
+			   //at this point the d_results needs to be transfer over to the other device
+			    
+			    in_out<float> temp_buffer{self->state().d_results -> copy_to_host()};
+
+			    self->state().d_results = div_cmd.transfer_memory(res_token,temp_buffer);
+			    //all done
+			    res_token->release();
+		   
+		   }
+
+		   else if (stage == "stage3") {
+		   
+			   //at this point d_results needs to be copied over to the new GPU   
+			    self->state().d_results = div_cmd.transfer_memory(res_token,in_out<float>{self->state().d_results->copy_to_host()});
+			    //all done
+			    res_token->release();
+		   }
+
+		   else { 
+			   std::cout << "Error unrecognized transfer token\n";
+		   }
+		    
+	    
+	    }
+
+            // unknown stage: ignore or log
+            std::cerr << "[pipeline] received unknown response token: " << stage << "\n";
+	}
+    };
+}
+
+
+//this test is meant to demonstrate the fact that scheduler actors can 
+//migrate work to correct load imbalance
+//the sizes should be large enough such that the tests exceed 4-5 seconds in total
+//otherwise the schedulers wont care to do this fast enough
+void run_load_balance_test_with_dependencies(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get(); 
+
+
+    //set the behaviors of each scheduler actor
+    for (int i = 0; i < mgr.get_num_devices();i++) {
+	    mgr.send_scheduler_actor_message("multilevel",i);
+    }
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    std::vector<caf::cuda::token_ptr> tokens(num_actors);
+
+    std::mt19937 rng(123456);
+    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+
+    const int THREADS = 32;
+
+    auto t_start = std::chrono::steady_clock::now();
+
+for (int i = 0; i < num_actors; ++i) {
+    int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
+    int BLOCKS = (N + THREADS - 1) / THREADS;
+
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1,
+                             THREADS, THREADS, 1);
+
+    caf::actor a = sys.spawn(
+        mmul_actor_fun_no_verify,
+        exit_actor,
+        N,
+        program,
+        dims,
+        false
+    );
+
+    tokens[i] = caf::cuda::make_launch_token(
+        program,
+        dims,
+        0, // memory usage is zero for now, we still do not track it at all
+        "hello",
+        a
+    );
+}
+
+auto t_end = std::chrono::steady_clock::now();
+
+auto us = std::chrono::duration_cast<std::chrono::microseconds>(
+              t_end - t_start
+          ).count();
+
+std::cout << "Actor spawn + token creation loop took "
+          << us << " us\n";
+
+   
+	  //send the tokens to only 1 GPU and let them
+	  //figure out that there is a load imbalance 
+	  mgr.send_scheduler_actor_message(tokens);
+          sys.await_all_actors_done();
+}
+
+
+
+
+
+
+
+
+
+
 void caf_main(caf::actor_system& sys) {
   
 	caf::cuda::manager_config man_config(true); //turns the scheduler on

From 17f848ed65dbb1f50f84bae4294671d1e991765e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Feb 2026 13:56:48 -0600
Subject: [PATCH 0365/1000] Implemented a test load balance with dependencies
 test.

---
 .../load-balancing-test/main.test.cpp         | 63 ++++++-------------
 1 file changed, 19 insertions(+), 44 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index 541f914e98..19d9b1f5be 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -875,7 +875,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 //otherwise the schedulers wont care to do this fast enough
 void run_load_balance_test_with_dependencies(
     caf::actor_system& sys,
-    const std::vector<int>& sizes,
+    const int N,
     int num_actors,
     bool randomize = false)
 {
@@ -890,54 +890,22 @@ void run_load_balance_test_with_dependencies(
     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
     caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+    caf::cuda::program_ptr p1 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","init_denominators");
+    caf::cuda::program_ptr p2 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","perform_division");
+    caf::cuda::program_ptr p3 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","sum_results");
 
-    std::vector<caf::cuda::token_ptr> tokens(num_actors);
-
-    std::mt19937 rng(123456);
-    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
 
-    const int THREADS = 32;
 
-    auto t_start = std::chrono::steady_clock::now();
 
-for (int i = 0; i < num_actors; ++i) {
-    int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
-    int BLOCKS = (N + THREADS - 1) / THREADS;
 
-    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1,
-                             THREADS, THREADS, 1);
 
-    caf::actor a = sys.spawn(
-        mmul_actor_fun_no_verify,
-        exit_actor,
-        N,
-        program,
-        dims,
-        false
-    );
+    auto t_start = std::chrono::steady_clock::now();
 
-    tokens[i] = caf::cuda::make_launch_token(
-        program,
-        dims,
-        0, // memory usage is zero for now, we still do not track it at all
-        "hello",
-        a
-    );
+for (int i = 0; i < num_actors; ++i) {
+	sys.spawn(pipeline_actor, exit_actor, p1, p2, p3, n);	
 }
-
-auto t_end = std::chrono::steady_clock::now();
-
-auto us = std::chrono::duration_cast<std::chrono::microseconds>(
-              t_end - t_start
-          ).count();
-
-std::cout << "Actor spawn + token creation loop took "
-          << us << " us\n";
-
-   
-	  //send the tokens to only 1 GPU and let them
-	  //figure out that there is a load imbalance 
-	  mgr.send_scheduler_actor_message(tokens);
+  
+	  //this time the gpu actors can figure out how to send tokens to the correct GPU scheduler
           sys.await_all_actors_done();
 }
 
@@ -954,9 +922,16 @@ void caf_main(caf::actor_system& sys) {
   
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	caf::cuda::manager::init(sys,man_config);
-	std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048,4096};
-	const int num_actors = 2000;
-	run_load_balance_test(sys,sizes,num_actors);
+	
+	
+	//no dependencies
+//	std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048,4096};
+//	const int num_actors = 2000;
+//	run_load_balance_test(sys,sizes,num_actors);
+
+
+	//dependencies
+	run_load_balance_test_with_dependencies(sys,1000,2000)
 
 
 }

From 806ecbe3667ee0bf6d3230bc7f8def5c23645432 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Feb 2026 13:58:24 -0600
Subject: [PATCH 0366/1000] Fixed syntax errors.

---
 .../control-layer-tests/load-balancing-test/main.test.cpp     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index 19d9b1f5be..b23fa86ce7 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -875,7 +875,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 //otherwise the schedulers wont care to do this fast enough
 void run_load_balance_test_with_dependencies(
     caf::actor_system& sys,
-    const int N,
+    const int n,
     int num_actors,
     bool randomize = false)
 {
@@ -931,7 +931,7 @@ void caf_main(caf::actor_system& sys) {
 
 
 	//dependencies
-	run_load_balance_test_with_dependencies(sys,1000,2000)
+	run_load_balance_test_with_dependencies(sys,1000,2000);
 
 
 }

From 57fe74e4a3b15d6fbee655a78592a1762f300034 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Feb 2026 14:15:35 -0600
Subject: [PATCH 0367/1000] FIxed type identifier on pipeline actor.

---
 .../control-layer-tests/load-balancing-test/main.test.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index b23fa86ce7..1d311e9c38 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -702,7 +702,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 
             const auto& stage = res_token->name();
 
-	    if (res_token->getType() == LAUNCH) {
+	    if (res_token->getType() == LAUNCH_RESPONSE) {
 
 		    // --------------------- Stage 1: init_denominators ---------------------
 		    if (stage == "stage1") {
@@ -833,6 +833,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 
 		   if (stage == "stage1") {
 			   res_token->release(); // no dependencies at this point clear to continue
+		   	   return;
 		   } 
 
 		   else if (stage == "stage2") {
@@ -844,7 +845,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 			    self->state().d_results = div_cmd.transfer_memory(res_token,temp_buffer);
 			    //all done
 			    res_token->release();
-		   
+		   	    return;
 		   }
 
 		   else if (stage == "stage3") {
@@ -853,6 +854,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 			    self->state().d_results = div_cmd.transfer_memory(res_token,in_out<float>{self->state().d_results->copy_to_host()});
 			    //all done
 			    res_token->release();
+			    return;
 		   }
 
 		   else { 
@@ -931,7 +933,7 @@ void caf_main(caf::actor_system& sys) {
 
 
 	//dependencies
-	run_load_balance_test_with_dependencies(sys,1000,2000);
+	run_load_balance_test_with_dependencies(sys,1000,1);
 
 
 }

From a9d17f2c8a7d5e2b064580bcd092f117660467bb Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Feb 2026 14:27:09 -0600
Subject: [PATCH 0368/1000] removed mutex from manager get method. THis change
 is being made since manager is meant ot be accessed in parallel with ease and
 convience.

---
 libcaf_cuda/src/manager.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 56898f92fc..cc356f9b2f 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -98,7 +98,7 @@ void manager::init_scheduler_actors(caf::actor_system& sys) {
 // Static get()
 // --------------------------------
 manager& manager::get() {
-    std::lock_guard<std::mutex> guard(mutex_);
+    //std::lock_guard<std::mutex> guard(mutex_);
 
     if (!instance_) {
         throw std::runtime_error(

From d624737dd1c569936f1366dcd0f965cdcaf99b04 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Feb 2026 14:37:28 -0600
Subject: [PATCH 0369/1000] Flipped condition on line 325 to fix an issue where
 multiple messages would be sent to a scheduler but it would be done so in a
 non deterministic way, resulting in dependencies ending up on the wrong
 device, creating a total mess.

---
 libcaf_cuda/src/manager.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index cc356f9b2f..c492725596 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -313,7 +313,7 @@ void manager::send_scheduler_actor_message(token_ptr token, int device_number) {
         target = device_number;
     } else {
         // No device specified
-        if (!token->isIndependent()) {
+        if (token->isIndependent()) {
             target = token->getDependency() % num_devices;
             if (target < 0)
                 target += num_devices;

From 4870e81216d3143ff0310e63ab7ee33952d586ad Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Feb 2026 14:51:27 -0600
Subject: [PATCH 0370/1000] Updated dependency test to force a transfer
 request.

---
 .../load-balancing-test/main.test.cpp           | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index 1d311e9c38..f5df37ae7a 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -687,7 +687,12 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
         );
 
 	//do not specifiy a device number to send it to, let it figure it out
-	caf::cuda::manager::get().send_scheduler_actor_message(tok);
+	//caf::cuda::manager::get().send_scheduler_actor_message(tok);
+    
+	//forcefully send to the first scheduler actor
+	caf::actor scheduler = caf::cuda::manager::get().get_scheduler_actor();
+	anon_mail(tok).send(scheduler);
+
     };
 
     // fire all three tokens (scheduler will reply with response_token on grants)
@@ -708,7 +713,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 		    if (stage == "stage1") {
 			    // allocate device buffer for denominators (persist in state)
 
-			    std::cout << "Starting stage 1\n";
+			    //std::cout << "Starting stage 1\n";
 			    unsigned long long seed = static_cast<unsigned long long>(
 					    std::chrono::high_resolution_clock::now().time_since_epoch().count()
 					    );
@@ -729,7 +734,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 					    caf::cuda::create_in_arg(seed)     // seed
 					    );
 
-			    std::cout << "Finished stage 1\n";
+			    //std::cout << "Finished stage 1\n";
 			    // stage1 intentionally no checks — data may contain zeros
 			    return;
 		    }
@@ -739,7 +744,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 			    // allocate device buffer for results (persist in state)
 			    std::vector<float> buffer1(n);
 
-			    std::cout << "Starting stage 2\n";
+			    //std::cout << "Starting stage 2\n";
 			    self->state().d_results = div_cmd.transfer_memory(res_token,out<float>{buffer1});
 
 			    // create a host numerators vector (all ones)
@@ -817,7 +822,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 
 			    std::vector<float> buf = self->state().d_sum -> copy_to_host();
 			    float final_sum = buf[0];
-			    std::cout << "[pipeline] completed, sum = " << final_sum << "\n";
+			    //std::cout << "[pipeline] completed, sum = " << final_sum << "\n";
 
 			    anon_mail(1).send(supervisor);
 
@@ -933,7 +938,7 @@ void caf_main(caf::actor_system& sys) {
 
 
 	//dependencies
-	run_load_balance_test_with_dependencies(sys,1000,1);
+	run_load_balance_test_with_dependencies(sys,10000,5000);
 
 
 }

From 1834c5418c663561b8cbefbddd980834250c2872 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Feb 2026 15:09:33 -0600
Subject: [PATCH 0371/1000] Changed get type to return TRANSFER instead of
 LAUNCH_RESPONSE. Change was made to fix a bug where transfer tokens woul dbe
 mistaken as launch response tokens.

---
 libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp b/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp
index d564240707..9c57875a1e 100644
--- a/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp
@@ -51,7 +51,7 @@ class CAF_CUDA_EXPORT transfer_token : public response_token {
         release();
     }
 
-    int getType() const override { return LAUNCH_RESPONSE; }
+    int getType() const override { return TRANSFER; }
 
     const nd_range& getRange() const { return range_; }
     const std::string& getId() const { return id_; }

From 8860ba351074192d7aad4507d67b60686077c3de Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Feb 2026 15:10:11 -0600
Subject: [PATCH 0372/1000] Added a temp transfer_ack handler to scheduler
 actor. Change is being made since upcasting and downcasting does not work
 with cafs static id typecasting system.

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 4926aa963b..626d73d7a1 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -106,6 +106,13 @@ caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self,
 	[&](std::vector<kernel_graph> work_graphs) {
 		state.current_behavior -> receive_work(work_graphs);
 	},
+	
+	//TEMPORARY FIX SINCE CAF TYPE ID IS STATIC SO POLYMORPHISM WONT WORK HERE
+	//TODO FIGURE OUT A WAY FOR ACK AND ITS CHILDREN TO BE 1 SINGLE CLASS AND
+	//DOWNCASTED EASILY
+	[&](transfer_ack payload) {
+		state.current_behavior->reclaim(static_cast<ack&>(payload));
+	}
     };
 }
 

From a747c442d3515c809cecd35c469104971a0a6722 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Feb 2026 15:12:29 -0600
Subject: [PATCH 0373/1000] Added a todo.

---
 .../control-layer-tests/load-balancing-test/main.test.cpp      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index f5df37ae7a..dad71d9172 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -844,7 +844,8 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 		   else if (stage == "stage2") {
 		  
 			   //at this point the d_results needs to be transfer over to the other device
-			    
+			   
+			  //TODO FIX SEGFAULT TRIGGERED BY THIS LINE 
 			    in_out<float> temp_buffer{self->state().d_results -> copy_to_host()};
 
 			    self->state().d_results = div_cmd.transfer_memory(res_token,temp_buffer);

From 243ba970c65f5bde621a5fd2d24829e1b4713de0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 13 Feb 2026 09:39:47 -0600
Subject: [PATCH 0374/1000] Changed methods such as isIndependent to be virtual
 and only implemented in token and dependency to be a protected attribute.
 Change was made to fix an error where due to double attributes being used,
 isIndependent would always evalute to true reguardless of what the dependency
 number is.

---
 libcaf_cuda/caf/cuda/control-layer/request_token.hpp | 6 ++----
 libcaf_cuda/caf/cuda/control-layer/token.hpp         | 4 ++--
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/request_token.hpp b/libcaf_cuda/caf/cuda/control-layer/request_token.hpp
index 86e4433bbf..af4444f807 100644
--- a/libcaf_cuda/caf/cuda/control-layer/request_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/request_token.hpp
@@ -19,16 +19,14 @@ namespace caf::cuda {
 class CAF_CUDA_EXPORT request_token : public token {
 public:
   request_token(int dependency = INDEPENDENT)
-    : dependency_(dependency) {}
+    : token(dependency) {}
 
   // Required for CAF message passing
   request_token() = default;
 
-  int getDependency() const { return dependency_; }
-  bool isIndependent() const { return dependency_ == INDEPENDENT; }
 
 private:
-  int dependency_ = INDEPENDENT;
+
 };
 
 using request_token_ptr = caf::intrusive_ptr<request_token>;
diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index 5cc2786d35..43ef888416 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -41,7 +41,8 @@ class CAF_CUDA_EXPORT token : public caf::ref_counted {
 
 
 protected:
-
+ int dependency_ = INDEPENDENT;
+ 
  friend void intrusive_ptr_add_ref(token* p) noexcept {
     p->ref_count_.fetch_add(1, std::memory_order_relaxed);
   }
@@ -54,7 +55,6 @@ class CAF_CUDA_EXPORT token : public caf::ref_counted {
 
 private:
  mutable std::atomic<size_t> ref_count_{0};
- int dependency_ = INDEPENDENT;
 
 
 };

From a3ec54739e6c0f30adcb4533046f839cf5632f83 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 13 Feb 2026 09:54:50 -0600
Subject: [PATCH 0375/1000] Moved mmul_async_actor_fun over

---
 .../load-balancing-test/main.test.cpp         | 258 +++++++++++++++++-
 1 file changed, 254 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index dad71d9172..bfd2a08833 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -22,6 +22,52 @@ using namespace caf;
 using namespace std::chrono_literals;
 
 
+
+
+
+struct mmul_actor_state {
+  static inline const char* name = "my_actor";
+  int last_N = 0; // example state variable
+  int id = rand(); // an actor id
+  int times = 0;
+};
+
+
+
+
+//commands classes used to launch kernels
+using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
+using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
+
+using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
+
+mmulCommand mmul;
+matrixGenCommand randomMatrix;
+mmulAsyncCommand mmulAsync;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+
+
+
+
+
 struct exit_actor_state {
 	int completed = 0;
 };
@@ -651,10 +697,13 @@ static sumCommand  sum_cmd;
 struct pipeline_actor_state {
     int id = rand();
 
+    int finished_stage = 0;
+
     // device-side buffers that must persist across stages:
     caf::cuda::mem_ptr<float> d_denoms;
     caf::cuda::mem_ptr<float> d_results;
     caf::cuda::mem_ptr<float> d_sum;
+    
 };
 
 // --- corrected pipeline_actor -------------------------------------------
@@ -736,6 +785,8 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 
 			    //std::cout << "Finished stage 1\n";
 			    // stage1 intentionally no checks — data may contain zeros
+			    
+			    self->state().finished_stage++;
 			    return;
 		    }
 
@@ -756,6 +807,21 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 
 			    // run division kernel on the token's stream/device:
 			    // kernel signature: (float* numerators, float* denominators, float* results, int n)
+			    
+			    if (self->state().d_denoms == nullptr) {
+			    
+				    std::cout << "Error with pipeline actor d_denoms is nullptr\n";
+			    
+			    }
+
+
+			    if (d_nums == nullptr) {
+			    
+				    std::cout << "Error with pipeline actor d_denoms is nullptr\n";
+			    
+			    }
+
+			    
 			    div_cmd.run(
 					    p2,
 					    range,
@@ -795,6 +861,8 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 
 			    // stage2 passed — keep d_results in state for stage3
 			    */
+
+			    self->state().finished_stage++;
 			    return;
 		    }
 
@@ -827,6 +895,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 			    anon_mail(1).send(supervisor);
 
 			    // quit the pipeline actor
+			    self->state().finished_stage++;
 			    self->quit();
 			    return;
 		    }
@@ -845,10 +914,20 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 		  
 			   //at this point the d_results needs to be transfer over to the other device
 			   
-			  //TODO FIX SEGFAULT TRIGGERED BY THIS LINE 
-			    in_out<float> temp_buffer{self->state().d_results -> copy_to_host()};
+		           std::cout << "Transfering at stage 2\n";
+			      if (self->state().d_denoms == nullptr) {
+			    
+				    std::cout << "Error with pipeline actor d_denoms is nullptr during transfer\n";
+				    std::cout << "Completed stage is " << self->state().finished_stage << "\n";
+			    
+			    }
+			   
+			   
+			   
+			   //TODO FIX SEGFAULT TRIGGERED BY THIS LINE 
+			    in_out<float> temp_buffer{self->state().d_denoms -> copy_to_host()};
 
-			    self->state().d_results = div_cmd.transfer_memory(res_token,temp_buffer);
+			    self->state().d_denoms = div_cmd.transfer_memory(res_token,temp_buffer);
 			    //all done
 			    res_token->release();
 		   	    return;
@@ -877,6 +956,177 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 }
 
 
+
+
+
+// Stateful actor behavior
+caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
+  return {
+    // 1st handler: Just int N, and who to send the matrices to
+    [=](int N, std::vector<caf::actor> receivers) {
+
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
+        //create the program and configure the dimesnions of the kernel
+        auto program = mgr.create_program_from_fatbin("../generate_random_matrix.fatbin","generate_random_matrix");
+        int THREADS = 256;
+        int BLOCKS = (N*N + THREADS - 1) / THREADS;
+        caf::cuda::nd_range dim(BLOCKS,1, 1, THREADS,1, 1);
+
+        //tag the arguments so that caf::cuda knows what to do with them
+         auto arg1 = caf::cuda::create_out_arg(N*N); //output buffer indicate its size, caf::cuda will handle the rest
+          auto arg2 = caf::cuda::create_in_arg(N*N); //matrix size
+          auto arg3 = caf::cuda::create_in_arg(rand()); //seed
+          auto arg4 = caf::cuda::create_in_arg(9999); //max valux
+
+          auto arg3B = caf::cuda::create_in_arg(rand()); //seed
+          int device_number= 74; //arbitary number to show that
+                                 //can give illusion of selecting gpus that are
+                                 //not there
+
+
+          //launch kernels and collect their outputs
+          auto tempA = randomMatrix.run_async(program,dim, self -> state().id,0,device_number,arg1,arg2,arg3,arg4);
+          auto tempB = randomMatrix.run_async(program,dim, self -> state().id,0,device_number,arg1,arg2,arg3B,arg4);
+          caf::cuda::mem_ptr<int> matrixA =  std::get<0>(tempA);
+          caf::cuda::mem_ptr<int> matrixB = std::get<0>(tempB);
+
+          //ensure the data is actually done being worked on
+          matrixA -> synchronize();
+          matrixB -> synchronize();
+
+
+
+
+          //cpu code
+          //std::vector<int> matrixA(N*N);
+          //std::vector<int> matrixB(N*N);
+
+          // std::generate(matrixA.begin(), matrixA.end(), []() { return rand() % 10; });
+           //std::generate(matrixB.begin(), matrixB.end(), []() { return rand() % 10; });
+
+
+          std::cout << "Broadcasting\n";
+          //broadcast the result out to receviers.
+          for (auto actor: receivers) {
+
+                  self->mail(3,matrixA,matrixB,N,device_number).send(actor);
+          }
+
+    },
+
+    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+    [=](const caf::cuda::mem_ptr<int> matrixA,
+        const caf::cuda::mem_ptr<int> matrixB, int N,int device_number) {
+
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //create program and dims
+  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+  const int THREADS = 32;
+  const int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    //create args
+    auto arg1 = matrixA;
+    auto arg2 = matrixB;
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+
+    auto tempC = mmulAsync.run(program,dims,self -> state().id,0,device_number,arg1,arg2,arg3,arg4);
+
+    std::vector<int> matrix1 = matrixA -> copy_to_host();
+    std::vector<int> matrix2 = matrixB -> copy_to_host();
+    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC,2);
+
+    //verify its own result
+    self -> mail(matrix1,matrix2,matrixC,N).send(self);
+
+    },
+
+ // 3nd handler: GPU atom + matrices + N, launches a kenrel using shared memory and sends its result to itself for verification
+    [=](int x,const caf::cuda::mem_ptr<int> matrixA,
+        const caf::cuda::mem_ptr<int> matrixB, int N,int device_number) {
+
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //create program and dims
+  auto program = mgr.create_program_from_cubin("../shared_mmul.cubin","matrixMul");
+  const int THREADS = 32;
+  const int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+  int shared_mem = 8192; //we need 8KB of shared memory here
+    //create args
+    auto arg1 = matrixA;
+    auto arg2 = matrixB;
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+
+    auto tempC = mmulAsync.run(program,dims,self -> state().id,shared_mem,device_number,arg1,arg2,arg3,arg4);
+
+    std::vector<int> matrix1 = matrixA -> copy_to_host();
+    std::vector<int> matrix2 = matrixB -> copy_to_host();
+    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC,2);
+
+    //verify its own result
+    self -> mail(matrix1,matrix2,matrixC,N).send(self);
+
+    },
+
+
+
+    // 3rd handler: CPU atom + matrices + N
+    [=](const std::vector<int>& matrixA,
+    const std::vector<int> &matrixB,
+    const std::vector<int> &matrixC, int N) {
+
+    std::vector<int> result(N * N);
+
+    serial_matrix_multiply(matrixA, matrixB, result, N);
+
+    if (result == matrixC) {
+        std::cout << "actor with id " << self->state().id << " references match\n";
+    }
+    else {
+        std::cout << "actor with id " << self->state().id << " references did not match\n";
+
+    }
+
+
+    /*
+    auto print_matrix = [N](const std::vector<int>& mat, const std::string& name) {
+            std::cout << name << ":\n";
+            for (int i = 0; i < N; ++i) {
+                for (int j = 0; j < N; ++j) {
+                    std::cout << mat[i * N + j] << " ";
+                }
+                std::cout << "\n";
+            }
+            std::cout << std::endl;
+        };
+
+        print_matrix(matrixA, "Matrix A");
+        print_matrix(matrixB, "Matrix B");
+        print_matrix(result, "Result Matrix");
+        print_matrix(matrixC, "GPU Result Matrix");
+        */
+    self->quit();
+    }
+  };
+}
+
+
+
+
+
+
+
+
+
 //this test is meant to demonstrate the fact that scheduler actors can 
 //migrate work to correct load imbalance
 //the sizes should be large enough such that the tests exceed 4-5 seconds in total
@@ -939,7 +1189,7 @@ void caf_main(caf::actor_system& sys) {
 
 
 	//dependencies
-	run_load_balance_test_with_dependencies(sys,10000,5000);
+	run_load_balance_test_with_dependencies(sys,10000,10000);
 
 
 }

From cb03de399d4af0ec5c31bd1cc6d5ab4e4b9a3962 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 13 Feb 2026 10:04:50 -0600
Subject: [PATCH 0376/1000] Reverted conditional on line 316 since tokens were
 falsely evaluating as indepedent.

---
 libcaf_cuda/src/manager.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index c492725596..cc356f9b2f 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -313,7 +313,7 @@ void manager::send_scheduler_actor_message(token_ptr token, int device_number) {
         target = device_number;
     } else {
         // No device specified
-        if (token->isIndependent()) {
+        if (!token->isIndependent()) {
             target = token->getDependency() % num_devices;
             if (target < 0)
                 target += num_devices;

From 7f44033505c9337ddc66441b79cfae5c813228fe Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 13 Feb 2026 10:06:44 -0600
Subject: [PATCH 0377/1000] Removed redundant code.

---
 .../load-balancing-test/main.test.cpp         | 500 +-----------------
 1 file changed, 3 insertions(+), 497 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index bfd2a08833..a445412c6b 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -22,52 +22,6 @@ using namespace caf;
 using namespace std::chrono_literals;
 
 
-
-
-
-struct mmul_actor_state {
-  static inline const char* name = "my_actor";
-  int last_N = 0; // example state variable
-  int id = rand(); // an actor id
-  int times = 0;
-};
-
-
-
-
-//commands classes used to launch kernels
-using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
-using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
-
-using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
-
-mmulCommand mmul;
-matrixGenCommand randomMatrix;
-mmulAsyncCommand mmulAsync;
-
-
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
-    }
-  }
-}
-
-
-
-
-
-
 struct exit_actor_state {
 	int completed = 0;
 };
@@ -150,454 +104,6 @@ void serial_matrix_multiply(const std::vector<int>& a,
 
 
 
-
-// Stateful actor behavior
-caf::behavior mmul_actor_fun(
-    caf::stateful_actor<mmul_actor_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims)
-{
-
-	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
-
-	caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-	caf::actor scheduler = mgr.get_scheduler_actor();
-
-	//send a launch token
-	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-			program,
-			dims,
-			0,
-			"hello",
-			self
-			);
-	self -> mail(launch_token).send(scheduler);
-
-	return {
-
-		[=] (caf::cuda::response_token_ptr res_token) {
-
-			if (res_token -> getType() == LAUNCH_RESPONSE) {
-				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-				//assume N = 1024
-				int N = self -> state().N;
-				std::vector<int> matrix1(N*N);
-				matrix1.reserve(N);
-				std::vector<int> matrix2(N*N);
-				matrix2.reserve(N);
-
-				//std::cout << "GPU ACTOR sending data to compute\n";
-				self -> mail(matrix1,matrix2,res_token,N).send(self);
-
-			}
-			else {
-				std::cout << "Got a memory response token\n";
-			}
-			//token should drop out of scope now, triggering a response 
-		},
-
-			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-			[=](const std::vector<int>& matrixA,
-					const std::vector<int>& matrixB,
-					const caf::cuda::response_token_ptr& res_token, int N) {
-
-
-				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
-
-				//caf::cuda::launch_response_token& kt =
-				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
-
-				//std::cout << "GPU ACTOR  computing\n";
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-				const int THREADS = 32;
-				const int BLOCKS = (N + THREADS - 1) / THREADS;
-				caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrixA);
-				auto arg2 = caf::cuda::create_in_arg(matrixB);
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
-
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				//std::cout << "GPU ACTOR done  computing\n";
-				//verify its own result 
-				self -> mail(matrixA,matrixB,matrixC,N).send(self);
-
-			},
-
-					// 3rd handler: CPU atom + matrices + N
-					[=](const std::vector<int>& matrixA,
-							const std::vector<int>& matrixB,
-							const std::vector<int>& matrixC,
-							int N) {
-
-						using clock = std::chrono::high_resolution_clock;
-
-						auto start = clock::now();
-
-						//std::cout << "GPU ACTOR verifying\n";
-
-						std::vector<int> result(N * N);
-
-						serial_matrix_multiply(matrixA, matrixB, result, N);
-
-						if (result == matrixC) {
-							std::cout << "actor with id " << self->state().id
-								<< " references match\n";
-						} else {
-							std::cout << "actor with id " << self->state().id
-								<< " references did not match\n";
-						}
-
-						auto end = clock::now();
-
-						auto ms =
-							std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-						std::cout << "[TIMING] verification took "
-							<< ms << " ms (actor id "
-							<< self->state().id << ")\n";
-
-						// signal exit actor and quit
-						self->mail(1).send(exit_actor);
-						self->quit();
-
-					}
-	};
-}
-
-
-
-
-
-// this actor will not verify its results
-// great for performance analysis 
-caf::behavior mmul_actor_fun_no_verify(
-    caf::stateful_actor<mmul_actor_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims,
-    bool request
-    )
-{
-
-	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
-
-	caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-	caf::actor scheduler = mgr.get_scheduler_actor();
-
-	if (request) {
-		//send a launch token
-		caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-				program,
-				dims,
-				0,
-				"hello",
-				self
-				);
-		self -> mail(launch_token).send(scheduler);
-	}
-	return {
-
-		[=] (caf::cuda::response_token_ptr res_token) {
-
-			if (res_token -> getType() == LAUNCH_RESPONSE) {
-				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-				//assume N = 1024
-				int N = self -> state().N;
-				std::vector<int> matrix1(N*N);
-				matrix1.reserve(N);
-				std::vector<int> matrix2(N*N);
-				matrix2.reserve(N);
-
-
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrix1);
-				auto arg2 = caf::cuda::create_in_arg(matrix2);
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
-
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-			
-			
-				//mask the transfer back to the cpu for scheduler
-				res_token -> release();
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				//std::cout << "GPU ACTOR done  computing\n";
-				// signal exit actor and quit
-				self->mail(1).send(exit_actor);
-				self->quit();
-
-				//std::cout << "GPU ACTOR sending data to compute\n";
-			//	self -> mail(matrix1,matrix2,res_token,N).send(self);
-
-			}
-			else {
-				std::cout << "Got a memory response token\n";
-			}
-			//token should drop out of scope now, triggering a response 
-		},
-
-			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-			[=](const std::vector<int>& matrixA,
-					const std::vector<int>& matrixB,
-					const caf::cuda::response_token_ptr& res_token, int N) {
-
-
-				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
-
-				//caf::cuda::launch_response_token& kt =
-				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
-
-				//std::cout << "GPU ACTOR  computing\n";
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrixA);
-				auto arg2 = caf::cuda::create_in_arg(matrixB);
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
-
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-			
-			
-				//mask the transfer back to the cpu for scheduler
-				res_token -> release();
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				//std::cout << "GPU ACTOR done  computing\n";
-				// signal exit actor and quit
-				self->mail(1).send(exit_actor);
-				self->quit();
-
-			}
-
-				
-	};
-}
-
-
-
-// Stateful actor behavior
-// this actor does not invoke the scheduler at all 
-caf::behavior mmul_actor_fun_no_schedule(
-    caf::stateful_actor<mmul_actor_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims) {
-
-    self->state().N = N;
-
-    std::vector<int> matrix1(N * N);
-    std::vector<int> matrix2(N * N);
-
-    // send initial mail to self
-    self->mail(matrix1, matrix2, N).send(self);
-
-    return {
-        // GPU atom + matrices + N
-        [=](const std::vector<int>& matrixA,
-            const std::vector<int>& matrixB,
-            int N_local) {   // avoid shadowing outer N
-            
-	    
-    	    //std::cout << "Hello\n";
-	    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-            auto arg1 = caf::cuda::create_in_arg(matrixA);
-            auto arg2 = caf::cuda::create_in_arg(matrixB);
-            auto arg3 = caf::cuda::create_out_arg(N_local * N_local);
-            auto arg4 = caf::cuda::create_in_arg(N_local);
-
-            auto tempC = mmul.run(program, dims, self->state().id, arg1, arg2, arg3, arg4);
-            std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-	    self->mail(1).send(exit_actor);
-            self->quit();
-        },
-
-        // CPU verification
-        [=](const std::vector<int>& matrixA,
-            const std::vector<int>& matrixB,
-            const std::vector<int>& matrixC,
-            int N_local) {
-
-            std::vector<int> result(N_local * N_local);
-            serial_matrix_multiply(matrixA, matrixB, result, N_local);
-
-            if (result == matrixC) {
-                std::cout << "actor with id " << self->state().id << " references match\n";
-            } else {
-                std::cout << "actor with id " << self->state().id << " references did not match\n";
-            }
-
-            self->quit();
-        }
-    };
-}
-
-
-
-
-
-
-
-
-
-
-void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //change the scheduler to core_usage
-  anon_mail(
-	caf::cuda::make_behavior_token("core_usage")
-	).send(mgr.get_scheduler_actor());
-
-  // CREATE ONCE
-  auto program =
-      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-  const int THREADS = 32;
-  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(
-      BLOCKS, BLOCKS, 1,
-      THREADS, THREADS, 1);
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-  for (int i = 0; i < num_actors; ++i) {
-    /*
-    sys.spawn(
-        mmul_actor_fun,
-        exit_actor,
-        matrix_size,
-        program,
-        dims);
-    */
-      sys.spawn(
-        mmul_actor_fun_no_verify,
-        exit_actor,
-        matrix_size,
-        program,
-        dims,
-	true);
-    
-  }
-
-  sys.await_all_actors_done();
-}
-
-
-void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  /*
-  //change the scheduler to core_usage
-  anon_mail(
-	caf::cuda::make_behavior_token("core_usage")
-	).send(mgr.get_scheduler_actor());
-
-  */
-
-
-  // CREATE ONCE
-  auto program =
-      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-  const int THREADS = 32;
-  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(
-      BLOCKS, BLOCKS, 1,
-      THREADS, THREADS, 1);
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-  for (int i = 0; i < num_actors; ++i) {
-    
-	sys.spawn(
-        mmul_actor_fun_no_verify,
-        exit_actor,
-        matrix_size,
-        program,
-        dims,
-	true);
-    
-  
-  }
-
-  sys.await_all_actors_done();
-}
-
-void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, int num_actors) {
-    if (num_actors < 1) {
-        std::cerr << "[ERROR] Number of actors must be >= 1\n";
-        return;
-    }
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-    // CREATE ONCE
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    const int THREADS = 32;
-    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    for (int i = 0; i < num_actors; ++i) {
-        sys.spawn(
-            mmul_actor_fun_no_schedule,
-            exit_actor,
-            matrix_size,
-            program,
-            dims
-        );
-    }
-
-    sys.await_all_actors_done();
-}
-
-
-template <class Fn>
-double time_run(Fn&& fn) {
-  auto start = std::chrono::steady_clock::now();
-  fn();
-  auto end = std::chrono::steady_clock::now();
-  std::chrono::duration<double> elapsed = end - start;
-  return elapsed.count();
-}
-
-
 //this test is meant to demonstrate the fact that scheduler actors can 
 //migrate work to correct load imbalance
 //the sizes should be large enough such that the tests exceed 4-5 seconds in total
@@ -736,11 +242,11 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
         );
 
 	//do not specifiy a device number to send it to, let it figure it out
-	//caf::cuda::manager::get().send_scheduler_actor_message(tok);
+	caf::cuda::manager::get().send_scheduler_actor_message(tok);
     
 	//forcefully send to the first scheduler actor
-	caf::actor scheduler = caf::cuda::manager::get().get_scheduler_actor();
-	anon_mail(tok).send(scheduler);
+	//caf::actor scheduler = caf::cuda::manager::get().get_scheduler_actor();
+	//anon_mail(tok).send(scheduler);
 
     };
 

From aad4cb81dd61b16c3dc973f94ea10f7e7f12012c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 13 Feb 2026 10:10:40 -0600
Subject: [PATCH 0378/1000] Undoing previous commit.

---
 .../load-balancing-test/main.test.cpp         | 500 +++++++++++++++++-
 1 file changed, 497 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index a445412c6b..bfd2a08833 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -22,6 +22,52 @@ using namespace caf;
 using namespace std::chrono_literals;
 
 
+
+
+
+struct mmul_actor_state {
+  static inline const char* name = "my_actor";
+  int last_N = 0; // example state variable
+  int id = rand(); // an actor id
+  int times = 0;
+};
+
+
+
+
+//commands classes used to launch kernels
+using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
+using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
+
+using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
+
+mmulCommand mmul;
+matrixGenCommand randomMatrix;
+mmulAsyncCommand mmulAsync;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+
+
+
+
+
 struct exit_actor_state {
 	int completed = 0;
 };
@@ -104,6 +150,454 @@ void serial_matrix_multiply(const std::vector<int>& a,
 
 
 
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun(
+    caf::stateful_actor<mmul_actor_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims)
+{
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+	caf::actor scheduler = mgr.get_scheduler_actor();
+
+	//send a launch token
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+			program,
+			dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+		[=] (caf::cuda::response_token_ptr res_token) {
+
+			if (res_token -> getType() == LAUNCH_RESPONSE) {
+				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+				//assume N = 1024
+				int N = self -> state().N;
+				std::vector<int> matrix1(N*N);
+				matrix1.reserve(N);
+				std::vector<int> matrix2(N*N);
+				matrix2.reserve(N);
+
+				//std::cout << "GPU ACTOR sending data to compute\n";
+				self -> mail(matrix1,matrix2,res_token,N).send(self);
+
+			}
+			else {
+				std::cout << "Got a memory response token\n";
+			}
+			//token should drop out of scope now, triggering a response 
+		},
+
+			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+			[=](const std::vector<int>& matrixA,
+					const std::vector<int>& matrixB,
+					const caf::cuda::response_token_ptr& res_token, int N) {
+
+
+				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
+
+				//caf::cuda::launch_response_token& kt =
+				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
+
+				//std::cout << "GPU ACTOR  computing\n";
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+				//create program and dims   
+				auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+				const int THREADS = 32;
+				const int BLOCKS = (N + THREADS - 1) / THREADS;
+				caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrixA);
+				auto arg2 = caf::cuda::create_in_arg(matrixB);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
+
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				//std::cout << "GPU ACTOR done  computing\n";
+				//verify its own result 
+				self -> mail(matrixA,matrixB,matrixC,N).send(self);
+
+			},
+
+					// 3rd handler: CPU atom + matrices + N
+					[=](const std::vector<int>& matrixA,
+							const std::vector<int>& matrixB,
+							const std::vector<int>& matrixC,
+							int N) {
+
+						using clock = std::chrono::high_resolution_clock;
+
+						auto start = clock::now();
+
+						//std::cout << "GPU ACTOR verifying\n";
+
+						std::vector<int> result(N * N);
+
+						serial_matrix_multiply(matrixA, matrixB, result, N);
+
+						if (result == matrixC) {
+							std::cout << "actor with id " << self->state().id
+								<< " references match\n";
+						} else {
+							std::cout << "actor with id " << self->state().id
+								<< " references did not match\n";
+						}
+
+						auto end = clock::now();
+
+						auto ms =
+							std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+						std::cout << "[TIMING] verification took "
+							<< ms << " ms (actor id "
+							<< self->state().id << ")\n";
+
+						// signal exit actor and quit
+						self->mail(1).send(exit_actor);
+						self->quit();
+
+					}
+	};
+}
+
+
+
+
+
+// this actor will not verify its results
+// great for performance analysis 
+caf::behavior mmul_actor_fun_no_verify(
+    caf::stateful_actor<mmul_actor_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims,
+    bool request
+    )
+{
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+	caf::actor scheduler = mgr.get_scheduler_actor();
+
+	if (request) {
+		//send a launch token
+		caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+				program,
+				dims,
+				0,
+				"hello",
+				self
+				);
+		self -> mail(launch_token).send(scheduler);
+	}
+	return {
+
+		[=] (caf::cuda::response_token_ptr res_token) {
+
+			if (res_token -> getType() == LAUNCH_RESPONSE) {
+				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+				//assume N = 1024
+				int N = self -> state().N;
+				std::vector<int> matrix1(N*N);
+				matrix1.reserve(N);
+				std::vector<int> matrix2(N*N);
+				matrix2.reserve(N);
+
+
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+				//create program and dims   
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrix1);
+				auto arg2 = caf::cuda::create_in_arg(matrix2);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
+
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+			
+			
+				//mask the transfer back to the cpu for scheduler
+				res_token -> release();
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				//std::cout << "GPU ACTOR done  computing\n";
+				// signal exit actor and quit
+				self->mail(1).send(exit_actor);
+				self->quit();
+
+				//std::cout << "GPU ACTOR sending data to compute\n";
+			//	self -> mail(matrix1,matrix2,res_token,N).send(self);
+
+			}
+			else {
+				std::cout << "Got a memory response token\n";
+			}
+			//token should drop out of scope now, triggering a response 
+		},
+
+			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+			[=](const std::vector<int>& matrixA,
+					const std::vector<int>& matrixB,
+					const caf::cuda::response_token_ptr& res_token, int N) {
+
+
+				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
+
+				//caf::cuda::launch_response_token& kt =
+				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
+
+				//std::cout << "GPU ACTOR  computing\n";
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+				//create program and dims   
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrixA);
+				auto arg2 = caf::cuda::create_in_arg(matrixB);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
+
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+			
+			
+				//mask the transfer back to the cpu for scheduler
+				res_token -> release();
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				//std::cout << "GPU ACTOR done  computing\n";
+				// signal exit actor and quit
+				self->mail(1).send(exit_actor);
+				self->quit();
+
+			}
+
+				
+	};
+}
+
+
+
+// Stateful actor behavior
+// this actor does not invoke the scheduler at all 
+caf::behavior mmul_actor_fun_no_schedule(
+    caf::stateful_actor<mmul_actor_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims) {
+
+    self->state().N = N;
+
+    std::vector<int> matrix1(N * N);
+    std::vector<int> matrix2(N * N);
+
+    // send initial mail to self
+    self->mail(matrix1, matrix2, N).send(self);
+
+    return {
+        // GPU atom + matrices + N
+        [=](const std::vector<int>& matrixA,
+            const std::vector<int>& matrixB,
+            int N_local) {   // avoid shadowing outer N
+            
+	    
+    	    //std::cout << "Hello\n";
+	    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+            auto arg1 = caf::cuda::create_in_arg(matrixA);
+            auto arg2 = caf::cuda::create_in_arg(matrixB);
+            auto arg3 = caf::cuda::create_out_arg(N_local * N_local);
+            auto arg4 = caf::cuda::create_in_arg(N_local);
+
+            auto tempC = mmul.run(program, dims, self->state().id, arg1, arg2, arg3, arg4);
+            std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+	    self->mail(1).send(exit_actor);
+            self->quit();
+        },
+
+        // CPU verification
+        [=](const std::vector<int>& matrixA,
+            const std::vector<int>& matrixB,
+            const std::vector<int>& matrixC,
+            int N_local) {
+
+            std::vector<int> result(N_local * N_local);
+            serial_matrix_multiply(matrixA, matrixB, result, N_local);
+
+            if (result == matrixC) {
+                std::cout << "actor with id " << self->state().id << " references match\n";
+            } else {
+                std::cout << "actor with id " << self->state().id << " references did not match\n";
+            }
+
+            self->quit();
+        }
+    };
+}
+
+
+
+
+
+
+
+
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //change the scheduler to core_usage
+  anon_mail(
+	caf::cuda::make_behavior_token("core_usage")
+	).send(mgr.get_scheduler_actor());
+
+  // CREATE ONCE
+  auto program =
+      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+  const int THREADS = 32;
+  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1);
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+  for (int i = 0; i < num_actors; ++i) {
+    /*
+    sys.spawn(
+        mmul_actor_fun,
+        exit_actor,
+        matrix_size,
+        program,
+        dims);
+    */
+      sys.spawn(
+        mmul_actor_fun_no_verify,
+        exit_actor,
+        matrix_size,
+        program,
+        dims,
+	true);
+    
+  }
+
+  sys.await_all_actors_done();
+}
+
+
+void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  /*
+  //change the scheduler to core_usage
+  anon_mail(
+	caf::cuda::make_behavior_token("core_usage")
+	).send(mgr.get_scheduler_actor());
+
+  */
+
+
+  // CREATE ONCE
+  auto program =
+      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+  const int THREADS = 32;
+  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1);
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+  for (int i = 0; i < num_actors; ++i) {
+    
+	sys.spawn(
+        mmul_actor_fun_no_verify,
+        exit_actor,
+        matrix_size,
+        program,
+        dims,
+	true);
+    
+  
+  }
+
+  sys.await_all_actors_done();
+}
+
+void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, int num_actors) {
+    if (num_actors < 1) {
+        std::cerr << "[ERROR] Number of actors must be >= 1\n";
+        return;
+    }
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    // CREATE ONCE
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    const int THREADS = 32;
+    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    for (int i = 0; i < num_actors; ++i) {
+        sys.spawn(
+            mmul_actor_fun_no_schedule,
+            exit_actor,
+            matrix_size,
+            program,
+            dims
+        );
+    }
+
+    sys.await_all_actors_done();
+}
+
+
+template <class Fn>
+double time_run(Fn&& fn) {
+  auto start = std::chrono::steady_clock::now();
+  fn();
+  auto end = std::chrono::steady_clock::now();
+  std::chrono::duration<double> elapsed = end - start;
+  return elapsed.count();
+}
+
+
 //this test is meant to demonstrate the fact that scheduler actors can 
 //migrate work to correct load imbalance
 //the sizes should be large enough such that the tests exceed 4-5 seconds in total
@@ -242,11 +736,11 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
         );
 
 	//do not specifiy a device number to send it to, let it figure it out
-	caf::cuda::manager::get().send_scheduler_actor_message(tok);
+	//caf::cuda::manager::get().send_scheduler_actor_message(tok);
     
 	//forcefully send to the first scheduler actor
-	//caf::actor scheduler = caf::cuda::manager::get().get_scheduler_actor();
-	//anon_mail(tok).send(scheduler);
+	caf::actor scheduler = caf::cuda::manager::get().get_scheduler_actor();
+	anon_mail(tok).send(scheduler);
 
     };
 

From 02e2fe81bf9422d841f05a26710843924d34e0f3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 13 Feb 2026 10:45:02 -0600
Subject: [PATCH 0379/1000] Wrote basic version of mmul_async_actor and deleted
 a lot of redundant code.

---
 .../load-balancing-test/main.test.cpp         | 488 +++++++++---------
 1 file changed, 239 insertions(+), 249 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index bfd2a08833..1b0b36771c 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -97,60 +97,6 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int lim
 
 
 
-// Define a custom type ID block for custom actors
-CAF_ADD_ATOM(cuda,shared_mem)
-
-
-
-
-
-// Extend your actor state to keep the start time
-struct mmul_actor_state {
-  static inline const char* name = "mmul_actor";
-
-  int N = 0;
-  int id = rand();
-
-  // timing / bookkeeping only
-  std::chrono::high_resolution_clock::time_point start_time;
-  int times = 0;
-};
-
-
-
-
-
-//commands classes used to launch kernels 
-using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
-using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
-
-using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
-
-mmulCommand mmul;
-matrixGenCommand randomMatrix;
-mmulAsyncCommand mmulAsync;
-
-
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-  
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
-    }
-  }
-}
-
-
-
-
 // Stateful actor behavior
 caf::behavior mmul_actor_fun(
     caf::stateful_actor<mmul_actor_state>* self,
@@ -160,8 +106,6 @@ caf::behavior mmul_actor_fun(
     caf::cuda::nd_range dims)
 {
 
-	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
 
 	caf::cuda::manager& mgr = caf::cuda::manager::get();
 
@@ -184,7 +128,6 @@ caf::behavior mmul_actor_fun(
 			if (res_token -> getType() == LAUNCH_RESPONSE) {
 				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
 				//assume N = 1024
-				int N = self -> state().N;
 				std::vector<int> matrix1(N*N);
 				matrix1.reserve(N);
 				std::vector<int> matrix2(N*N);
@@ -293,7 +236,6 @@ caf::behavior mmul_actor_fun_no_verify(
 {
 
 	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
 
 	caf::cuda::manager& mgr = caf::cuda::manager::get();
 
@@ -317,7 +259,6 @@ caf::behavior mmul_actor_fun_no_verify(
 			if (res_token -> getType() == LAUNCH_RESPONSE) {
 				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
 				//assume N = 1024
-				int N = self -> state().N;
 				std::vector<int> matrix1(N*N);
 				matrix1.reserve(N);
 				std::vector<int> matrix2(N*N);
@@ -396,196 +337,6 @@ caf::behavior mmul_actor_fun_no_verify(
 
 
 
-// Stateful actor behavior
-// this actor does not invoke the scheduler at all 
-caf::behavior mmul_actor_fun_no_schedule(
-    caf::stateful_actor<mmul_actor_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims) {
-
-    self->state().N = N;
-
-    std::vector<int> matrix1(N * N);
-    std::vector<int> matrix2(N * N);
-
-    // send initial mail to self
-    self->mail(matrix1, matrix2, N).send(self);
-
-    return {
-        // GPU atom + matrices + N
-        [=](const std::vector<int>& matrixA,
-            const std::vector<int>& matrixB,
-            int N_local) {   // avoid shadowing outer N
-            
-	    
-    	    //std::cout << "Hello\n";
-	    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-            auto arg1 = caf::cuda::create_in_arg(matrixA);
-            auto arg2 = caf::cuda::create_in_arg(matrixB);
-            auto arg3 = caf::cuda::create_out_arg(N_local * N_local);
-            auto arg4 = caf::cuda::create_in_arg(N_local);
-
-            auto tempC = mmul.run(program, dims, self->state().id, arg1, arg2, arg3, arg4);
-            std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-	    self->mail(1).send(exit_actor);
-            self->quit();
-        },
-
-        // CPU verification
-        [=](const std::vector<int>& matrixA,
-            const std::vector<int>& matrixB,
-            const std::vector<int>& matrixC,
-            int N_local) {
-
-            std::vector<int> result(N_local * N_local);
-            serial_matrix_multiply(matrixA, matrixB, result, N_local);
-
-            if (result == matrixC) {
-                std::cout << "actor with id " << self->state().id << " references match\n";
-            } else {
-                std::cout << "actor with id " << self->state().id << " references did not match\n";
-            }
-
-            self->quit();
-        }
-    };
-}
-
-
-
-
-
-
-
-
-
-
-void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //change the scheduler to core_usage
-  anon_mail(
-	caf::cuda::make_behavior_token("core_usage")
-	).send(mgr.get_scheduler_actor());
-
-  // CREATE ONCE
-  auto program =
-      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-  const int THREADS = 32;
-  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(
-      BLOCKS, BLOCKS, 1,
-      THREADS, THREADS, 1);
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-  for (int i = 0; i < num_actors; ++i) {
-    /*
-    sys.spawn(
-        mmul_actor_fun,
-        exit_actor,
-        matrix_size,
-        program,
-        dims);
-    */
-      sys.spawn(
-        mmul_actor_fun_no_verify,
-        exit_actor,
-        matrix_size,
-        program,
-        dims,
-	true);
-    
-  }
-
-  sys.await_all_actors_done();
-}
-
-
-void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  /*
-  //change the scheduler to core_usage
-  anon_mail(
-	caf::cuda::make_behavior_token("core_usage")
-	).send(mgr.get_scheduler_actor());
-
-  */
-
-
-  // CREATE ONCE
-  auto program =
-      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-  const int THREADS = 32;
-  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(
-      BLOCKS, BLOCKS, 1,
-      THREADS, THREADS, 1);
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-  for (int i = 0; i < num_actors; ++i) {
-    
-	sys.spawn(
-        mmul_actor_fun_no_verify,
-        exit_actor,
-        matrix_size,
-        program,
-        dims,
-	true);
-    
-  
-  }
-
-  sys.await_all_actors_done();
-}
-
-void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, int num_actors) {
-    if (num_actors < 1) {
-        std::cerr << "[ERROR] Number of actors must be >= 1\n";
-        return;
-    }
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-    // CREATE ONCE
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    const int THREADS = 32;
-    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    for (int i = 0; i < num_actors; ++i) {
-        sys.spawn(
-            mmul_actor_fun_no_schedule,
-            exit_actor,
-            matrix_size,
-            program,
-            dims
-        );
-    }
-
-    sys.await_all_actors_done();
-}
 
 
 template <class Fn>
@@ -1169,7 +920,246 @@ for (int i = 0; i < num_actors; ++i) {
 
 
 
+struct mmul_async_actor_state {
+  static inline const char* name = "mmul_actor";
+
+  int N = 0;
+  int id = rand();
+
+  // timing / bookkeeping only
+  std::chrono::high_resolution_clock::time_point start_time;
+  int times = 0;
+
+  // --- mmul_async state (added) -------------------------------------------
+  caf::cuda::mem_ptr<int> d_genA;                // device buffer for generated A
+  caf::cuda::mem_ptr<int> d_genB;                // device buffer for generated B
+  bool have_genA = false;
+  bool have_genB = false;
+};
+
 
+caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_async_actor_state>* self) {
+  return {
+
+    // ------------------------------------------------------------------
+    // 1) Initial request: generate two matrices
+    // ------------------------------------------------------------------
+    [=](int N) {
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+      self->state().N = N;
+      self->state().have_genA = false;
+      self->state().have_genB = false;
+      self->state().d_genA = nullptr;
+      self->state().d_genB = nullptr;
+
+      // Explicit generator launch configuration
+      const int THREADS = 256;
+      const int BLOCKS  = (N * N + THREADS - 1) / THREADS;
+      caf::cuda::nd_range gen_range(BLOCKS, 1, 1,
+                                    THREADS, 1, 1);
+
+      auto gen_program =
+        mgr.create_program_from_fatbin(
+          "../generate_random_matrix.fatbin",
+          "generate_random_matrix");
+
+      auto send_launch = [&](const std::string& name) {
+        auto tok = caf::cuda::make_launch_token(
+                     gen_program,
+                     gen_range,
+                     sizeof(int) * N * N,
+                     name,
+                     self
+                   );
+        anon_mail(tok).send(mgr.get_scheduler_actor());
+      };
+
+      send_launch("genA");
+      send_launch("genB");
+    },
+
+    // ------------------------------------------------------------------
+    // 2) Handle scheduler response tokens
+    // ------------------------------------------------------------------
+    [=](caf::cuda::response_token_ptr res_token) mutable {
+
+      const auto type = res_token->getType();
+      const auto name = res_token->name();
+      int N = self->state().N;
+
+      // --------------------------------------------------------------
+      // TRANSFER handling
+      // --------------------------------------------------------------
+      if (type == TRANSFER) {
+
+        if (name == "genA" && self->state().d_genA) {
+          auto host_copy = self->state().d_genA->copy_to_host();
+          self->state().d_genA =
+            randomMatrix.transfer_memory(
+              res_token,
+              in_out<int>{host_copy});
+          res_token->release();
+          return;
+        }
+
+        if (name == "genB" && self->state().d_genB) {
+          auto host_copy = self->state().d_genB->copy_to_host();
+          self->state().d_genB =
+            randomMatrix.transfer_memory(
+              res_token,
+              in_out<int>{host_copy});
+          
+	  //since GenA comes before GenB we have to transfer over both memories to enusure
+	  //they are both on the same GPU
+	  auto host_copy_2 = self->state().d_genA->copy_to_host();
+          self->state().d_genA =
+            randomMatrix.transfer_memory(
+              res_token,
+              in_out<int>{host_copy_2});
+
+	  
+	  
+	  res_token->release();
+          return;
+        }
+
+        return;
+      }
+
+      // --------------------------------------------------------------
+      // LAUNCH_RESPONSE handling
+      // --------------------------------------------------------------
+      if (type != LAUNCH_RESPONSE)
+        return;
+
+      // ----------------------------
+      // Generator completion
+      // ----------------------------
+      if (name == "genA" || name == "genB") {
+
+        auto out_arg     = caf::cuda::create_out_arg(N * N);
+        auto size_arg    = caf::cuda::create_in_arg(N * N);
+        auto seed_arg    = caf::cuda::create_in_arg(rand());
+        auto maxval_arg  = caf::cuda::create_in_arg(9999);
+
+        const int THREADS = 256;
+        const int BLOCKS  = (N * N + THREADS - 1) / THREADS;
+        caf::cuda::nd_range gen_range(BLOCKS,1,1,
+                                      THREADS,1,1);
+
+        auto gen_program =
+          caf::cuda::manager::get().create_program_from_fatbin(
+            "../generate_random_matrix.fatbin",
+            "generate_random_matrix");
+
+        auto result =
+          randomMatrix.run_async(
+            gen_program,
+            gen_range,
+            res_token,
+            out_arg,
+            size_arg,
+            seed_arg,
+            maxval_arg);
+
+        auto device_buffer = std::get<0>(result);
+
+        if (name == "genA") {
+          self->state().d_genA = device_buffer;
+          self->state().have_genA = true;
+        } else {
+          self->state().d_genB = device_buffer;
+          self->state().have_genB = true;
+        }
+
+        res_token->release();
+
+        // ----------------------------------------------------------
+        // If BOTH matrices exist → request mmul launch
+        // ----------------------------------------------------------
+        if (self->state().have_genA &&
+            self->state().have_genB) {
+
+          const int THREADS_M = 32;
+          int BLOCKS_M = (N + THREADS_M - 1) / THREADS_M;
+
+          caf::cuda::nd_range mmul_range(
+            BLOCKS_M, BLOCKS_M, 1,
+            THREADS_M, THREADS_M, 1);
+
+          auto mmul_program =
+            caf::cuda::manager::get()
+              .create_program_from_cubin(
+                "../mmul.cubin",
+                "matrixMul");
+
+          auto mmul_token =
+            caf::cuda::make_launch_token(
+              mmul_program,
+              mmul_range,
+              sizeof(int) * N * N,
+              "mmul",
+              self);
+
+          anon_mail(mmul_token)
+            .send(caf::cuda::manager::get()
+                    .get_scheduler_actor());
+        }
+
+        return;
+      }
+
+      // ----------------------------
+      // mmul completion
+      // ----------------------------
+      if (name == "mmul") {
+
+        const int THREADS_M = 32;
+        int BLOCKS_M = (N + THREADS_M - 1) / THREADS_M;
+
+        caf::cuda::nd_range mmul_range(
+          BLOCKS_M, BLOCKS_M, 1,
+          THREADS_M, THREADS_M, 1);
+
+        auto mmul_program =
+          caf::cuda::manager::get()
+            .create_program_from_cubin(
+              "../mmul.cubin",
+              "matrixMul");
+
+        auto outC = caf::cuda::create_out_arg(N * N);
+        auto inN  = caf::cuda::create_in_arg(N);
+
+        auto result =
+          mmulAsync.run(
+            mmul_program,
+            mmul_range,
+            res_token,
+            self->state().d_genA,
+            self->state().d_genB,
+            outC,
+            inN);
+
+        std::vector<int> matrixC =
+          caf::cuda::extract_vector<int>(result, 2);
+
+        std::vector<int> matrixA =
+          self->state().d_genA->copy_to_host();
+
+        std::vector<int> matrixB =
+          self->state().d_genB->copy_to_host();
+
+        self->state().have_genA = false;
+        self->state().have_genB = false;
+        self->state().d_genA.reset();
+        self->state().d_genB.reset();
+
+        res_token->release();
+      }
+    }
+  };
+}
 
 
 

From e37bf15c9831aafe5db00025bb51724a28ca4297 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 13 Feb 2026 13:12:13 -0600
Subject: [PATCH 0380/1000] Implemented
 run_load_balancing_test_large_dependencies.

---
 .../load-balancing-test/main.test.cpp         | 213 ++++--------------
 1 file changed, 47 insertions(+), 166 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index 1b0b36771c..a6c52104bf 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -710,170 +710,6 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
 
 
 
-// Stateful actor behavior
-caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
-  return {
-    // 1st handler: Just int N, and who to send the matrices to
-    [=](int N, std::vector<caf::actor> receivers) {
-
-        caf::cuda::manager& mgr = caf::cuda::manager::get();
-        //create the program and configure the dimesnions of the kernel
-        auto program = mgr.create_program_from_fatbin("../generate_random_matrix.fatbin","generate_random_matrix");
-        int THREADS = 256;
-        int BLOCKS = (N*N + THREADS - 1) / THREADS;
-        caf::cuda::nd_range dim(BLOCKS,1, 1, THREADS,1, 1);
-
-        //tag the arguments so that caf::cuda knows what to do with them
-         auto arg1 = caf::cuda::create_out_arg(N*N); //output buffer indicate its size, caf::cuda will handle the rest
-          auto arg2 = caf::cuda::create_in_arg(N*N); //matrix size
-          auto arg3 = caf::cuda::create_in_arg(rand()); //seed
-          auto arg4 = caf::cuda::create_in_arg(9999); //max valux
-
-          auto arg3B = caf::cuda::create_in_arg(rand()); //seed
-          int device_number= 74; //arbitary number to show that
-                                 //can give illusion of selecting gpus that are
-                                 //not there
-
-
-          //launch kernels and collect their outputs
-          auto tempA = randomMatrix.run_async(program,dim, self -> state().id,0,device_number,arg1,arg2,arg3,arg4);
-          auto tempB = randomMatrix.run_async(program,dim, self -> state().id,0,device_number,arg1,arg2,arg3B,arg4);
-          caf::cuda::mem_ptr<int> matrixA =  std::get<0>(tempA);
-          caf::cuda::mem_ptr<int> matrixB = std::get<0>(tempB);
-
-          //ensure the data is actually done being worked on
-          matrixA -> synchronize();
-          matrixB -> synchronize();
-
-
-
-
-          //cpu code
-          //std::vector<int> matrixA(N*N);
-          //std::vector<int> matrixB(N*N);
-
-          // std::generate(matrixA.begin(), matrixA.end(), []() { return rand() % 10; });
-           //std::generate(matrixB.begin(), matrixB.end(), []() { return rand() % 10; });
-
-
-          std::cout << "Broadcasting\n";
-          //broadcast the result out to receviers.
-          for (auto actor: receivers) {
-
-                  self->mail(3,matrixA,matrixB,N,device_number).send(actor);
-          }
-
-    },
-
-    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const caf::cuda::mem_ptr<int> matrixA,
-        const caf::cuda::mem_ptr<int> matrixB, int N,int device_number) {
-
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //create program and dims
-  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    //create args
-    auto arg1 = matrixA;
-    auto arg2 = matrixB;
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
-
-
-    auto tempC = mmulAsync.run(program,dims,self -> state().id,0,device_number,arg1,arg2,arg3,arg4);
-
-    std::vector<int> matrix1 = matrixA -> copy_to_host();
-    std::vector<int> matrix2 = matrixB -> copy_to_host();
-    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC,2);
-
-    //verify its own result
-    self -> mail(matrix1,matrix2,matrixC,N).send(self);
-
-    },
-
- // 3nd handler: GPU atom + matrices + N, launches a kenrel using shared memory and sends its result to itself for verification
-    [=](int x,const caf::cuda::mem_ptr<int> matrixA,
-        const caf::cuda::mem_ptr<int> matrixB, int N,int device_number) {
-
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //create program and dims
-  auto program = mgr.create_program_from_cubin("../shared_mmul.cubin","matrixMul");
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  int shared_mem = 8192; //we need 8KB of shared memory here
-    //create args
-    auto arg1 = matrixA;
-    auto arg2 = matrixB;
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
-
-
-    auto tempC = mmulAsync.run(program,dims,self -> state().id,shared_mem,device_number,arg1,arg2,arg3,arg4);
-
-    std::vector<int> matrix1 = matrixA -> copy_to_host();
-    std::vector<int> matrix2 = matrixB -> copy_to_host();
-    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC,2);
-
-    //verify its own result
-    self -> mail(matrix1,matrix2,matrixC,N).send(self);
-
-    },
-
-
-
-    // 3rd handler: CPU atom + matrices + N
-    [=](const std::vector<int>& matrixA,
-    const std::vector<int> &matrixB,
-    const std::vector<int> &matrixC, int N) {
-
-    std::vector<int> result(N * N);
-
-    serial_matrix_multiply(matrixA, matrixB, result, N);
-
-    if (result == matrixC) {
-        std::cout << "actor with id " << self->state().id << " references match\n";
-    }
-    else {
-        std::cout << "actor with id " << self->state().id << " references did not match\n";
-
-    }
-
-
-    /*
-    auto print_matrix = [N](const std::vector<int>& mat, const std::string& name) {
-            std::cout << name << ":\n";
-            for (int i = 0; i < N; ++i) {
-                for (int j = 0; j < N; ++j) {
-                    std::cout << mat[i * N + j] << " ";
-                }
-                std::cout << "\n";
-            }
-            std::cout << std::endl;
-        };
-
-        print_matrix(matrixA, "Matrix A");
-        print_matrix(matrixB, "Matrix B");
-        print_matrix(result, "Result Matrix");
-        print_matrix(matrixC, "GPU Result Matrix");
-        */
-    self->quit();
-    }
-  };
-}
-
-
-
-
-
 
 
 
@@ -882,6 +718,8 @@ caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self)
 //migrate work to correct load imbalance
 //the sizes should be large enough such that the tests exceed 4-5 seconds in total
 //otherwise the schedulers wont care to do this fast enough
+//As it turns out pipeline actor does not do enough work in order to convince the GPUs
+//that it should even attempt to migrate it
 void run_load_balance_test_with_dependencies(
     caf::actor_system& sys,
     const int n,
@@ -938,7 +776,11 @@ struct mmul_async_actor_state {
 };
 
 
-caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_async_actor_state>* self) {
+
+//we intentionally send to only 1 actor to force load balancing and also
+//see what happens if an actor gets a request that it is not responsible for
+caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_async_actor_state>* self,
+		caf::actor exit_actor) {
   return {
 
     // ------------------------------------------------------------------
@@ -1156,6 +998,7 @@ caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_async_actor_state>*
         self->state().d_genB.reset();
 
         res_token->release();
+        self->mail(1).send(exit_actor);
       }
     }
   };
@@ -1164,6 +1007,44 @@ caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_async_actor_state>*
 
 
 
+//this test is meant to demonstrate the fact that scheduler actors can 
+//migrate work to correct load imbalance
+//the sizes should be large enough such that the tests exceed 4-5 seconds in total
+//otherwise the schedulers wont care to do this fast enough
+void run_load_balance_test_with_large_dependencies(
+    caf::actor_system& sys,
+    const int n,
+    int num_actors,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get(); 
+
+
+    //set the behaviors of each scheduler actor
+    for (int i = 0; i < mgr.get_num_devices();i++) {
+	    mgr.send_scheduler_actor_message("multilevel",i);
+    }
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+
+
+
+
+
+    auto t_start = std::chrono::steady_clock::now();
+
+for (int i = 0; i < num_actors; ++i) {
+	caf::actor a = sys.spawn(mmul_async_actor_fun, exit_actor);
+	anon_mail(n).send(a);	
+}
+  
+	  //this time the gpu actors can figure out how to send tokens to the correct GPU scheduler
+          sys.await_all_actors_done();
+}
+
 
 
 void caf_main(caf::actor_system& sys) {
@@ -1179,7 +1060,7 @@ void caf_main(caf::actor_system& sys) {
 
 
 	//dependencies
-	run_load_balance_test_with_dependencies(sys,10000,10000);
+	run_load_balance_test_with_large_dependencies(sys,1024,2000);
 
 
 }

From 7d8de40fa316416504155ba5b910fcc7bd72f34e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 13 Feb 2026 13:34:08 -0600
Subject: [PATCH 0381/1000] Added try catch to mmul_async_actor_fun.

---
 .../load-balancing-test/main.test.cpp         | 314 ++++++++----------
 1 file changed, 147 insertions(+), 167 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index a6c52104bf..ad65a7e2d5 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -80,7 +80,7 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int lim
 		[=](int num_completed) {
 			self->state().completed += num_completed;
 			
-		//	std::cout << "Actors finished is " << self->state().completed << "\n";
+			std::cout << "Actors finished is " << self->state().completed << "\n";
 			if (self->state().completed >= limit) {
 			
 				caf::cuda::manager::shutdown();
@@ -824,183 +824,164 @@ caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_async_actor_state>*
     // ------------------------------------------------------------------
     // 2) Handle scheduler response tokens
     // ------------------------------------------------------------------
-    [=](caf::cuda::response_token_ptr res_token) mutable {
-
-      const auto type = res_token->getType();
-      const auto name = res_token->name();
-      int N = self->state().N;
+[=](caf::cuda::response_token_ptr res_token) mutable {
+  try {
+    const auto type = res_token->getType();
+    const auto name = res_token->name();
+    int N = self->state().N;
+
+    // ----------------------------
+    // TRANSFER handling
+    // ----------------------------
+    if (type == TRANSFER) {
+      if (name == "genA" && self->state().d_genA) {
+        auto host_copy = self->state().d_genA->copy_to_host();
+        self->state().d_genA =
+          randomMatrix.transfer_memory(res_token, in_out<int>{host_copy});
+      }
 
-      // --------------------------------------------------------------
-      // TRANSFER handling
-      // --------------------------------------------------------------
-      if (type == TRANSFER) {
+      if (name == "genB" && self->state().d_genB) {
+        // Transfer B
+        auto host_copyB = self->state().d_genB->copy_to_host();
+        self->state().d_genB =
+          randomMatrix.transfer_memory(res_token, in_out<int>{host_copyB});
 
-        if (name == "genA" && self->state().d_genA) {
-          auto host_copy = self->state().d_genA->copy_to_host();
+        // ALSO transfer A to the same device as B
+        if (self->state().d_genA) {
+          auto host_copyA = self->state().d_genA->copy_to_host();
           self->state().d_genA =
-            randomMatrix.transfer_memory(
-              res_token,
-              in_out<int>{host_copy});
-          res_token->release();
-          return;
+            randomMatrix.transfer_memory(res_token, in_out<int>{host_copyA});
         }
+      }
 
-        if (name == "genB" && self->state().d_genB) {
-          auto host_copy = self->state().d_genB->copy_to_host();
-          self->state().d_genB =
-            randomMatrix.transfer_memory(
-              res_token,
-              in_out<int>{host_copy});
-          
-	  //since GenA comes before GenB we have to transfer over both memories to enusure
-	  //they are both on the same GPU
-	  auto host_copy_2 = self->state().d_genA->copy_to_host();
-          self->state().d_genA =
-            randomMatrix.transfer_memory(
-              res_token,
-              in_out<int>{host_copy_2});
-
-	  
-	  
-	  res_token->release();
-          return;
-        }
+      res_token->release();
+      return;
+    }
 
-        return;
-      }
+    // ----------------------------
+    // LAUNCH_RESPONSE handling
+    // ----------------------------
+    if (type != LAUNCH_RESPONSE)
+      return;
 
-      // --------------------------------------------------------------
-      // LAUNCH_RESPONSE handling
-      // --------------------------------------------------------------
-      if (type != LAUNCH_RESPONSE)
-        return;
-
-      // ----------------------------
-      // Generator completion
-      // ----------------------------
-      if (name == "genA" || name == "genB") {
-
-        auto out_arg     = caf::cuda::create_out_arg(N * N);
-        auto size_arg    = caf::cuda::create_in_arg(N * N);
-        auto seed_arg    = caf::cuda::create_in_arg(rand());
-        auto maxval_arg  = caf::cuda::create_in_arg(9999);
-
-        const int THREADS = 256;
-        const int BLOCKS  = (N * N + THREADS - 1) / THREADS;
-        caf::cuda::nd_range gen_range(BLOCKS,1,1,
-                                      THREADS,1,1);
-
-        auto gen_program =
-          caf::cuda::manager::get().create_program_from_fatbin(
-            "../generate_random_matrix.fatbin",
-            "generate_random_matrix");
-
-        auto result =
-          randomMatrix.run_async(
-            gen_program,
-            gen_range,
-            res_token,
-            out_arg,
-            size_arg,
-            seed_arg,
-            maxval_arg);
-
-        auto device_buffer = std::get<0>(result);
-
-        if (name == "genA") {
-          self->state().d_genA = device_buffer;
-          self->state().have_genA = true;
-        } else {
-          self->state().d_genB = device_buffer;
-          self->state().have_genB = true;
-        }
+    // Generator completion (genA / genB)
+    if (name == "genA" || name == "genB") {
+      auto out_arg  = caf::cuda::create_out_arg(N * N);
+      auto size_arg = caf::cuda::create_in_arg(N * N);
+      auto seed_arg = caf::cuda::create_in_arg(rand());
+      auto maxval_arg = caf::cuda::create_in_arg(9999);
 
-        res_token->release();
-
-        // ----------------------------------------------------------
-        // If BOTH matrices exist → request mmul launch
-        // ----------------------------------------------------------
-        if (self->state().have_genA &&
-            self->state().have_genB) {
-
-          const int THREADS_M = 32;
-          int BLOCKS_M = (N + THREADS_M - 1) / THREADS_M;
-
-          caf::cuda::nd_range mmul_range(
-            BLOCKS_M, BLOCKS_M, 1,
-            THREADS_M, THREADS_M, 1);
-
-          auto mmul_program =
-            caf::cuda::manager::get()
-              .create_program_from_cubin(
-                "../mmul.cubin",
-                "matrixMul");
-
-          auto mmul_token =
-            caf::cuda::make_launch_token(
-              mmul_program,
-              mmul_range,
-              sizeof(int) * N * N,
-              "mmul",
-              self);
-
-          anon_mail(mmul_token)
-            .send(caf::cuda::manager::get()
-                    .get_scheduler_actor());
-        }
+      const int THREADS = 256;
+      const int BLOCKS  = (N * N + THREADS - 1) / THREADS;
 
-        return;
-      }
+      caf::cuda::nd_range gen_range(BLOCKS,1,1, THREADS,1,1);
 
-      // ----------------------------
-      // mmul completion
-      // ----------------------------
-      if (name == "mmul") {
-
-        const int THREADS_M = 32;
-        int BLOCKS_M = (N + THREADS_M - 1) / THREADS_M;
-
-        caf::cuda::nd_range mmul_range(
-          BLOCKS_M, BLOCKS_M, 1,
-          THREADS_M, THREADS_M, 1);
-
-        auto mmul_program =
-          caf::cuda::manager::get()
-            .create_program_from_cubin(
-              "../mmul.cubin",
-              "matrixMul");
-
-        auto outC = caf::cuda::create_out_arg(N * N);
-        auto inN  = caf::cuda::create_in_arg(N);
-
-        auto result =
-          mmulAsync.run(
-            mmul_program,
-            mmul_range,
-            res_token,
-            self->state().d_genA,
-            self->state().d_genB,
-            outC,
-            inN);
-
-        std::vector<int> matrixC =
-          caf::cuda::extract_vector<int>(result, 2);
-
-        std::vector<int> matrixA =
-          self->state().d_genA->copy_to_host();
-
-        std::vector<int> matrixB =
-          self->state().d_genB->copy_to_host();
-
-        self->state().have_genA = false;
-        self->state().have_genB = false;
-        self->state().d_genA.reset();
-        self->state().d_genB.reset();
-
-        res_token->release();
-        self->mail(1).send(exit_actor);
+      auto gen_program =
+        caf::cuda::manager::get().create_program_from_fatbin(
+          "../generate_random_matrix.fatbin",
+          "generate_random_matrix");
+
+      auto result = randomMatrix.run_async(
+        gen_program, gen_range, res_token,
+        out_arg, size_arg, seed_arg, maxval_arg);
+
+      auto device_buffer = std::get<0>(result);
+
+      if (name == "genA") {
+	      self->state().d_genA = device_buffer;
+      	      self->state().have_genA = true;
+      
       }
+      else {
+	      self->state().d_genB = device_buffer;
+      	      self->state().have_genB = true;
+      }
+
+      // After handling genA / genB completion
+if (self->state().have_genA && self->state().have_genB) {
+    const int THREADS_M = 32;
+    int BLOCKS_M = (N + THREADS_M - 1) / THREADS_M;
+
+    caf::cuda::nd_range mmul_range(
+      BLOCKS_M, BLOCKS_M, 1,
+      THREADS_M, THREADS_M, 1
+    );
+
+    auto mmul_program =
+      caf::cuda::manager::get().create_program_from_cubin(
+        "../mmul.cubin",
+        "matrixMul"
+      );
+
+    // Create a launch token for mmul
+    auto mmul_token = caf::cuda::make_launch_token(
+      mmul_program,
+      mmul_range,
+      sizeof(int) * N * N,
+      "mmul",
+      self
+    );
+
+    // Send the launch token to the scheduler actor
+    anon_mail(mmul_token)
+      .send(caf::cuda::manager::get().get_scheduler_actor());
+  }
+
+
+      res_token->release();
+      return;
+    }
+
+    // ----------------------------
+    // mmul completion / kernel launch
+    // ----------------------------
+    if (name == "mmul") {
+      const int THREADS_M = 32;
+      int BLOCKS_M = (N + THREADS_M - 1) / THREADS_M;
+
+      caf::cuda::nd_range mmul_range(BLOCKS_M, BLOCKS_M, 1,
+                                    THREADS_M, THREADS_M, 1);
+
+      auto mmul_program =
+        caf::cuda::manager::get().create_program_from_cubin(
+          "../mmul.cubin",
+          "matrixMul");
+
+      auto outC = caf::cuda::create_out_arg(N * N);
+      auto inN  = caf::cuda::create_in_arg(N);
+
+      auto result = mmulAsync.run(
+        mmul_program,
+        mmul_range,
+        res_token,
+        self->state().d_genA,
+        self->state().d_genB,
+        outC,
+        inN);
+
+      std::vector<int> matrixC = caf::cuda::extract_vector<int>(result, 2);
+
+      self->state().have_genA = false;
+      self->state().have_genB = false;
+      self->state().d_genA.reset();
+      self->state().d_genB.reset();
+
+      res_token->release();
+      self->mail(1).send(exit_actor);
     }
+
+  } catch (std::exception& e) {
+    std::cerr << "*** Caught exception: " << e.what() << "\n";
+
+    if (self->state().d_genA)
+      std::cerr << "d_genA deviceID: " << self->state().d_genA->deviceID() << "\n";
+    if (self->state().d_genB)
+      std::cerr << "d_genB deviceID: " << self->state().d_genB->deviceID() << "\n";
+    if (res_token)
+      std::cerr << "res_token deviceID: " << res_token->getDeviceNumber() << "\n";
+  }
+}
+
   };
 }
 
@@ -1025,7 +1006,6 @@ void run_load_balance_test_with_large_dependencies(
 	    mgr.send_scheduler_actor_message("multilevel",i);
     }
 
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
     caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
 

From 65603d5d128f0f9e8d04c88f9f664c07c8dde031 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 13 Feb 2026 13:50:19 -0600
Subject: [PATCH 0382/1000] fixed issue in mmul_async_actor where dependency
 number was not declared in request tokens and as such tricked the scheduler
 into thinking jobs were independent.

---
 .../load-balancing-test/main.test.cpp                | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index ad65a7e2d5..8813c73de1 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -812,7 +812,9 @@ caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_async_actor_state>*
                      gen_range,
                      sizeof(int) * N * N,
                      name,
-                     self
+                     self,
+		     self->state().id //use this for dependency number
+				      //required for scheduler actor internal bookeeping
                    );
         anon_mail(tok).send(mgr.get_scheduler_actor());
       };
@@ -838,6 +840,7 @@ caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_async_actor_state>*
         auto host_copy = self->state().d_genA->copy_to_host();
         self->state().d_genA =
           randomMatrix.transfer_memory(res_token, in_out<int>{host_copy});
+	std::cout << "Moved genA over to new device\n";
       }
 
       if (name == "genB" && self->state().d_genB) {
@@ -852,6 +855,8 @@ caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_async_actor_state>*
           self->state().d_genA =
             randomMatrix.transfer_memory(res_token, in_out<int>{host_copyA});
         }
+
+	std::cout << "Moved genA and genB over to new device\n";
       }
 
       res_token->release();
@@ -919,7 +924,8 @@ if (self->state().have_genA && self->state().have_genB) {
       mmul_range,
       sizeof(int) * N * N,
       "mmul",
-      self
+      self,
+      self ->state().id //needed to help track depedencies
     );
 
     // Send the launch token to the scheduler actor
@@ -1040,7 +1046,7 @@ void caf_main(caf::actor_system& sys) {
 
 
 	//dependencies
-	run_load_balance_test_with_large_dependencies(sys,1024,2000);
+	run_load_balance_test_with_large_dependencies(sys,1024,1);
 
 
 }

From 7a816d691b18ceba30e2b4da06a4545bf4d6e8ff Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 13 Feb 2026 14:11:32 -0600
Subject: [PATCH 0383/1000] Fixed low threshold conditional and added the
 ability to collect low queue jobs as well since if a job was tagged as low it
 would not have been shared with the other scheduler but in reality it should
 have since its work to distrubute.

---
 .../src/control-layer/multilevel_usage_behavior.cpp   | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index dadbefc6e7..241fdc92d6 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -301,7 +301,7 @@ void multilevel_usage_behavior::handle_load_balance_request(int device_number) {
     // Only transfer work if we are busy
 
     int free_SM = total_SM -  available_SM;
-    if (free_SM < transfer_threshold) {
+    if (free_SM > transfer_threshold) {
         return; // GPU not busy enough, do nothing
     }
 
@@ -354,9 +354,12 @@ void multilevel_usage_behavior::handle_load_balance_request(int device_number) {
     }
 
     // ---- Step 2: transfer from high and medium queues ----
-    std::size_t max_high_med = (high_queue.size() + med_queue.size()) / 2;
-    collect_graphs_from_queue(high_queue, max_high_med);
-    collect_graphs_from_queue(med_queue, max_high_med);
+    std::size_t mid_high = high_queue.size() / 2;
+    std::size_t mid_med = med_queue.size() / 2;
+    std::size_t mid_low = low_queue.size() / 2;
+    collect_graphs_from_queue(high_queue, mid_high);
+    collect_graphs_from_queue(med_queue, mid_med);
+    collect_graphs_from_queue(low_queue, mid_low);
 
     // ---- Step 3: send if we have anything ----
     if (!work_to_transfer.empty()) {

From 6ad9de40e2a1de74d5446b864dc13a1479bb7431 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 13 Feb 2026 14:32:45 -0600
Subject: [PATCH 0384/1000] FIxed transfer threshold again.

---
 libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 241fdc92d6..7f8280050b 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -300,7 +300,7 @@ void multilevel_usage_behavior::handle_load_balance_request(int device_number) {
     
     // Only transfer work if we are busy
 
-    int free_SM = total_SM -  available_SM;
+    int free_SM = available_SM;
     if (free_SM > transfer_threshold) {
         return; // GPU not busy enough, do nothing
     }

From 0db5a8778f9069d11b18d94519f146c1042dfb9f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 13 Feb 2026 14:53:52 -0600
Subject: [PATCH 0385/1000] Fixed memory transfer logic.

---
 .../load-balancing-test/main.test.cpp         | 55 ++++++++++---------
 1 file changed, 30 insertions(+), 25 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index 8813c73de1..f5ac3354c5 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -835,34 +835,33 @@ caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_async_actor_state>*
     // ----------------------------
     // TRANSFER handling
     // ----------------------------
-    if (type == TRANSFER) {
-      if (name == "genA" && self->state().d_genA) {
-        auto host_copy = self->state().d_genA->copy_to_host();
-        self->state().d_genA =
-          randomMatrix.transfer_memory(res_token, in_out<int>{host_copy});
-	std::cout << "Moved genA over to new device\n";
-      }
+if (type == TRANSFER) {
 
-      if (name == "genB" && self->state().d_genB) {
-        // Transfer B
-        auto host_copyB = self->state().d_genB->copy_to_host();
-        self->state().d_genB =
-          randomMatrix.transfer_memory(res_token, in_out<int>{host_copyB});
+  std::cout << "TRANSFER for " << name << std::endl;
 
-        // ALSO transfer A to the same device as B
-        if (self->state().d_genA) {
-          auto host_copyA = self->state().d_genA->copy_to_host();
-          self->state().d_genA =
-            randomMatrix.transfer_memory(res_token, in_out<int>{host_copyA});
-        }
+  // If mmul is moving, move BOTH matrices
+  if (name == "mmul") {
 
-	std::cout << "Moved genA and genB over to new device\n";
-      }
+    if (self->state().d_genA) {
+      auto host_copyA = self->state().d_genA->copy_to_host();
+      self->state().d_genA =
+        randomMatrix.transfer_memory(res_token, in_out<int>{host_copyA});
+    }
 
-      res_token->release();
-      return;
+    if (self->state().d_genB) {
+      auto host_copyB = self->state().d_genB->copy_to_host();
+      self->state().d_genB =
+        randomMatrix.transfer_memory(res_token, in_out<int>{host_copyB});
     }
 
+    std::cout << "Moved genA and genB for mmul\n";
+  }
+
+  res_token->release();
+  return;
+}
+
+
     // ----------------------------
     // LAUNCH_RESPONSE handling
     // ----------------------------
@@ -983,9 +982,15 @@ if (self->state().have_genA && self->state().have_genB) {
       std::cerr << "d_genA deviceID: " << self->state().d_genA->deviceID() << "\n";
     if (self->state().d_genB)
       std::cerr << "d_genB deviceID: " << self->state().d_genB->deviceID() << "\n";
-    if (res_token)
+    if (res_token) {
       std::cerr << "res_token deviceID: " << res_token->getDeviceNumber() << "\n";
-  }
+      std::cerr << "res_token name: " << res_token->name() << "\n";
+    
+    }
+    
+    }
+
+
 }
 
   };
@@ -1046,7 +1051,7 @@ void caf_main(caf::actor_system& sys) {
 
 
 	//dependencies
-	run_load_balance_test_with_large_dependencies(sys,1024,1);
+	run_load_balance_test_with_large_dependencies(sys,1024,2000);
 
 
 }

From 83f85cc8a263ebdb59ca69df229df149c51b7db5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 13 Feb 2026 15:04:37 -0600
Subject: [PATCH 0386/1000] Added add_dependency_to_device on line 331. THis
 change was made to fix a subtle error where dependencies would not be tracked
 correctly so if a dependency was sent ot the wrong actor it would believe it
 had never seen it before resulting in an incorrect flow of operations between
 or withing a gpu actor.

---
 libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 7f8280050b..713e9f9367 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -328,6 +328,7 @@ void multilevel_usage_behavior::handle_load_balance_request(int device_number) {
             if (ref.kind == graph_ref::kind_t::dependent) {
                 remove_dependency(ref.dependency);
                 graphs.erase(ref.dependency);
+		add_dependency_to_device(ref.dependency,device_number);
             } else { // independent
                 if (ref.index < independent_graphs.size()) {
 		    //TODO come up with a better way to clean this up

From 1b720d39619c4ba751db3f827a826571d96f1d73 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 13 Feb 2026 15:12:09 -0600
Subject: [PATCH 0387/1000] Fixed memory transfer issues.

---
 .../control-layer-tests/load-balancing-test/main.test.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index f5ac3354c5..3dafb5e6c6 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -80,7 +80,7 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int lim
 		[=](int num_completed) {
 			self->state().completed += num_completed;
 			
-			std::cout << "Actors finished is " << self->state().completed << "\n";
+//			std::cout << "Actors finished is " << self->state().completed << "\n";
 			if (self->state().completed >= limit) {
 			
 				caf::cuda::manager::shutdown();
@@ -837,7 +837,7 @@ caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_async_actor_state>*
     // ----------------------------
 if (type == TRANSFER) {
 
-  std::cout << "TRANSFER for " << name << std::endl;
+  //std::cout << "TRANSFER for " << name << std::endl;
 
   // If mmul is moving, move BOTH matrices
   if (name == "mmul") {
@@ -854,7 +854,9 @@ if (type == TRANSFER) {
         randomMatrix.transfer_memory(res_token, in_out<int>{host_copyB});
     }
 
-    std::cout << "Moved genA and genB for mmul\n";
+    if (res_token->getDeviceNumber() == 1) {
+    //std::cout << "Moved genA and genB for mmul\n";
+  } 
   }
 
   res_token->release();

From adf2d67930100362331c38753d797081c12ace4b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Feb 2026 10:09:22 -0600
Subject: [PATCH 0388/1000] Updated kernels in directory and compile script to
 keep in line with the tests.

---
 .../load-balancing-test/compile_kernels.sh    |  7 ++--
 .../load-balancing-test/fault.cu              | 33 +++++++++++++++++++
 .../load-balancing-test/genMatrix.cu          | 16 +++++++++
 3 files changed, 54 insertions(+), 2 deletions(-)
 create mode 100644 libcaf_cuda/tests/control-layer-tests/load-balancing-test/fault.cu
 create mode 100644 libcaf_cuda/tests/control-layer-tests/load-balancing-test/genMatrix.cu

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/compile_kernels.sh b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/compile_kernels.sh
index 586196454e..1bcba9f066 100755
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/compile_kernels.sh
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/compile_kernels.sh
@@ -9,10 +9,13 @@ echo "Using NVCC arch flag: $SM_ARCH"
 # Compile mmul.cu to cubin in current directory
 nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
 echo "Generated mmul.cubin"
+#compile fault.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin fault.cu -o fault.cubin
+echo "Generated fault.cubin"
 
 # Compile genMatrix.cu to fatbin in current directory
-#nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
-#echo "Generated generate_random_matrix.fatbin"
+nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
+echo "Generated generate_random_matrix.fatbin"
 
 echo "All kernels compiled successfully!"
 
diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/fault.cu b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/fault.cu
new file mode 100644
index 0000000000..06ec34c594
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/fault.cu
@@ -0,0 +1,33 @@
+#include <cuda_runtime.h>
+#include <curand.h>
+#include <curand_kernel.h>
+#include <iostream>
+#include <cstdlib>
+
+// Step 1: Initialize denominators with ~50% zeros using cuRAND
+extern "C" __global__ void init_denominators(float* denominators, int n, unsigned long long seed) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+
+    curandState state;
+    curand_init(seed, idx, 0, &state);
+    float rand_val = curand_uniform(&state);
+    denominators[idx] = (rand_val < 0.5f) ? 0.0f : 1.0f;  // ~50% chance of zero
+}
+
+// Step 2: Perform division (potential div by zero -> Inf)
+extern "C" __global__ void perform_division(float* numerators, float* denominators, float* results, int n) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    results[idx] = numerators[idx] / denominators[idx];  // Triggers Inf if denominator == 0
+}
+
+// Step 3: Simple reduction to sum results (propagates Inf if present)
+extern "C" __global__ void sum_results(float* results, float* final_sum, int n) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (idx < n) {
+        // atomic add each element directly to the final sum
+        atomicAdd(final_sum, results[idx]);
+    }
+}
diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/genMatrix.cu b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/genMatrix.cu
new file mode 100644
index 0000000000..98189eb4d0
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/genMatrix.cu
@@ -0,0 +1,16 @@
+
+#include <curand_kernel.h>
+//generate_random_matrix
+extern "C" __global__
+void generate_random_matrix(int* matrix, int total_elements, int seed, int max_val) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= total_elements) return;
+
+    curandState state;
+    curand_init((unsigned long long)seed, idx, 0, &state);
+
+    unsigned int r = curand(&state);
+    matrix[idx] = r % max_val;
+}
+
+

From 6c76c92f76573c187f1ec16860c205da2129a37e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Feb 2026 10:10:07 -0600
Subject: [PATCH 0389/1000] No major changes.

---
 .../control-layer-tests/fault-tolerance-test/main.test.cpp  | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
index 06c6ee0e0d..70a4323957 100644
--- a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
@@ -100,7 +100,8 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
             // --------------------- Stage 1: init_denominators ---------------------
             if (stage == "stage1") {
                 // allocate device buffer for denominators (persist in state)
-               
+
+     	           std::cout << "Starting stage 1\n";		    
 		    unsigned long long seed = static_cast<unsigned long long>(
     std::chrono::high_resolution_clock::now().time_since_epoch().count()
 );
@@ -121,6 +122,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
                     caf::cuda::create_in_arg(seed)     // seed
                 );
 
+		std::cout << "Finished stage 1\n";
                 // stage1 intentionally no checks — data may contain zeros
                 return;
             }
@@ -130,6 +132,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
                 // allocate device buffer for results (persist in state)
 		    std::vector<float> buffer1(n); 
 
+     	           std::cout << "Starting stage 2\n";		    
 		self->state().d_results = div_cmd.transfer_memory(res_token,out<float>{buffer1});
 
                 // create a host numerators vector (all ones)
@@ -179,6 +182,7 @@ behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
             if (stage == "stage3") {
                 // allocate device scalar for sum result
                 
+     	           std::cout << "Starting stage 3\n";		    
 		    std::vector<float> buffer1(1); 
 
 		self->state().d_sum = div_cmd.transfer_memory(res_token,out<float>{buffer1});

From 3d7f162b9f5e50bd4d18afd79c93f211e1d5a045 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Feb 2026 10:10:37 -0600
Subject: [PATCH 0390/1000] No major changes.

---
 libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp b/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
index b1d8ed2e64..ac861bf9de 100644
--- a/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
@@ -11,6 +11,7 @@ namespace caf::cuda {
 #define CAF_CUDA_ACK_MEMORY    3
 #define TIMER 4
 
+
 // -----------------------------------------------------------------------------
 // Base ACK payload
 // -----------------------------------------------------------------------------

From 3ec3e79b0b3c34bad9f5ae65f02e45a161eb0fd8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Feb 2026 11:28:03 -0600
Subject: [PATCH 0391/1000] Modified scaling tests.

---
 .../core_usage_behavior_tests/main.test.cpp   | 23 ++++++++++---------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index 860bebae65..7ee728e8a3 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -426,9 +426,9 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
 
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-  //change the scheduler to core_usage
+  //change the scheduler to mulitlevle_usage
   anon_mail(
-	caf::cuda::make_behavior_token("core_usage")
+	caf::cuda::make_behavior_token("multilevel")
 	).send(mgr.get_scheduler_actor());
 
   // CREATE ONCE
@@ -482,6 +482,7 @@ void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num
 
   */
 
+    mgr.send_scheduler_actor_message("green",0);
 
   // CREATE ONCE
   auto program =
@@ -527,6 +528,8 @@ void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, i
 
     caf::cuda::manager& mgr = caf::cuda::manager::get();
 
+    mgr.send_scheduler_actor_message("green",0);
+
     // CREATE ONCE
     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
@@ -560,7 +563,7 @@ double time_run(Fn&& fn) {
 }
 void run_mmul_scaling_tests(caf::actor_system& sys,
                             caf::cuda::manager_config man_config) {
-    const int max_size   = 1024;
+    const int max_size   = 2048;
     const int min_actors = 1;
     const int max_actors = 1024;
 
@@ -580,8 +583,8 @@ void run_mmul_scaling_tests(caf::actor_system& sys,
         for (int actors : actor_counts) {
 
             /* ================= Scheduler-enabled (core_usage) ================= */
-            caf::cuda::manager::init(sys, man_config); // green-light scheduler enabled
-            std::cout << "\n[RUN] scheduler=core_usage "
+            caf::cuda::manager::init(sys, man_config); // scheduler enabled
+            std::cout << "\n[RUN] scheduler=multilevel_usage "
                       << "matrix_size=" << size
                       << " actors=" << actors << "\n";
 
@@ -597,9 +600,8 @@ void run_mmul_scaling_tests(caf::actor_system& sys,
 
             caf::cuda::manager::shutdown(); // make sure manager is cleaned up
 
-            /* ================= Scheduler-disabled actor (still uses green-light) ================= */
+            /* ================= Scheduler-disabled actor ( uses green-light) ================= */
 
-/*
 	    caf::cuda::manager::init(sys, man_config); // init with scheduler
             std::cout << "\n[RUN] scheduler=green_light_only "
                       << "matrix_size=" << size
@@ -617,7 +619,6 @@ void run_mmul_scaling_tests(caf::actor_system& sys,
 
             caf::cuda::manager::shutdown();
 
-	    */
             /* ================= No scheduler at all actor ================= */
             caf::cuda::manager_config no_sched_config(false); // disable scheduler
             caf::cuda::manager::init(sys, no_sched_config);
@@ -996,10 +997,10 @@ void run_mmul_fixed_256_batch_comparison(
 
 void caf_main(caf::actor_system& sys) {
   
-	//caf::cuda::manager_config man_config(true); //turns the scheduler on
+	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	//caf::cuda::manager::init(sys,man_config);
         // run_mmul_test(sys,10,64);	
-	//run_mmul_scaling_tests(sys,man_config);
+	run_mmul_scaling_tests(sys,man_config);
 
 	/*
    std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048,4096};
@@ -1012,7 +1013,7 @@ void caf_main(caf::actor_system& sys) {
     //run_mmul_fixed_256_batch_comparison(sys, /*num_actors=*/200);
 
 
-	 test_core_usage_uniform_mmul(sys, 256, 1000,"multilevel");
+	// test_core_usage_uniform_mmul(sys, 256, 1000,"multilevel");
 	 //test_core_usage_mixed_mmul(sys, 256, 1000,"multilevel");
 
   //std::vector<int> sizes = {32, 64, 128, 256, 512, 1024};

From 5462ec92ccf734c74029840e5437ee0744a62342 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 23 Feb 2026 16:00:35 -0600
Subject: [PATCH 0392/1000] Implemented get_avaiable_memory methods.

---
 libcaf_cuda/caf/cuda/device.hpp |  6 ++++++
 libcaf_cuda/src/device.cpp      | 24 ++++++++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 7bd4dbf9c8..135bacf667 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -76,6 +76,12 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   // Convenience: total memory in megabytes
   double total_memory_mb() const noexcept { return static_cast<double>(total_mem_bytes_) / (1024.0 * 1024.0); }
 
+  //total free memory on the device
+  std::size_t available_memory_bytes() const;
+  // Convenience: returns available memory in megabytes
+  double available_memory_mb() const;
+
+
   // Short human-readable device summary
   std::string device_summary() const {
 	  return std::string(name_) + " (id=" + std::to_string(id_) + ") - SMs: " + std::to_string(sm_count_) +
diff --git a/libcaf_cuda/src/device.cpp b/libcaf_cuda/src/device.cpp
index 36257e63ee..4167497b2e 100644
--- a/libcaf_cuda/src/device.cpp
+++ b/libcaf_cuda/src/device.cpp
@@ -69,6 +69,30 @@ int device::max_active_blocks_per_sm(const program_ptr& prog,
     }
 
 }
+// Returns the currently available memory on this device in bytes
+std::size_t device::available_memory_bytes() const {
+    CUcontext prev_ctx = nullptr;
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+
+    size_t free_bytes = 0;
+    size_t total_bytes = 0;
+    CUresult res = cuMemGetInfo(&free_bytes, &total_bytes);
+
+    CHECK_CUDA(cuCtxPopCurrent(&prev_ctx));
+
+    if (res != CUDA_SUCCESS) {
+        const char* err_name = nullptr;
+        cuGetErrorName(res, &err_name);
+        throw std::runtime_error(std::string("cuMemGetInfo failed: ") + (err_name ? err_name : "unknown error"));
+    }
+
+    return free_bytes;
+}
+
+// Convenience: returns available memory in megabytes
+double device::available_memory_mb() const {
+    return static_cast<double>(available_memory_bytes()) / (1024.0 * 1024.0);
+}
 
 } // namespace caf::cuda
 

From ccbc929ff08f36d88835e171ef843879b1525450 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 23 Feb 2026 16:29:55 -0600
Subject: [PATCH 0393/1000] Implemented inital headers of memory actor, its
 payload object and updated control layer header to include it.

---
 .../cuda/control-layer/all-control-layer.hpp  |  8 +++++
 .../memory_actor/memory_actor.hpp             | 30 ++++++++++++++++
 .../memory_actor/memory_request_token.hpp     | 36 +++++++++++++++++++
 3 files changed, 74 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_request_token.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 8fc7b99168..70dd94f9ff 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -13,6 +13,7 @@
 #include "caf/cuda/control-layer/memory_response_token.hpp"
 #include "caf/cuda/control-layer/transfer_token.hpp"
 
+//scheduler actor includes
 #include "caf/cuda/control-layer/behavior.hpp"
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
 #include "caf/cuda/control-layer/token_factory.hpp"
@@ -22,6 +23,11 @@
 #include "caf/cuda/control-layer/kernel_graph.hpp"
 #include "caf/cuda/control-layer/core_usage_behavior.hpp"
 
+//memory actor includes
+#include "caf/cuda/control-layer/memory_actor/memory_actor.hpp"
+#include "caf/cuda/control-layer/memory_actor/memory_request_token.hpp"
+
+
 // -----------------------------------------------------------------------------
 // Type IDs (required for typed behaviors)
 // -----------------------------------------------------------------------------
@@ -35,6 +41,7 @@ CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::launch_response_tok
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::response_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_transfer_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_response_token>))
+CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::memory_request_token))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::transfer_token>))
 CAF_ADD_TYPE_ID(cuda_control, (std::vector<caf::intrusive_ptr<caf::cuda::token>>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::kernel_graph))
@@ -53,6 +60,7 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::behavior_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_response_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_transfer_token)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_request_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_response_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::response_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::transfer_token)
diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
new file mode 100644
index 0000000000..37596355d4
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
@@ -0,0 +1,30 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <queue>
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/global_export.hpp"
+#include "caf/cuda/control-layer/memory_actor/memory_request_token.hpp"
+
+/*
+ * The memory actor 
+ * meant to help prevent out of memory errors by employing s/r/r IPC 
+ * it will reply to a request once it believes there is enough memory to serve it
+ * as of right now still is a best effort service as due to fragmentation
+ * replying still may result in an out of memory error
+ */
+
+namespace caf::cuda {
+
+struct memory_actor_state {
+
+	std::queue<memory_request_token> requests;
+	std::vector<device_ptr> devices;
+	int num_devices;
+
+};
+
+caf::behavior CAF_CUDA_EXPORT memory_actor(caf::stateful_actor<memory_actor_state> * self,
+		int num_devices);
+}
+
diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_request_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_request_token.hpp
new file mode 100644
index 0000000000..f83aa0ae6e
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_request_token.hpp
@@ -0,0 +1,36 @@
+#pragma once
+
+
+/*
+ * This token represents a request for memory allocation
+ * on a specific device. All sizes are in bytes.
+ */
+
+namespace caf::cuda {
+
+class CAF_CUDA_EXPORT memory_request_token {
+public:
+  memory_request_token(int size,
+                       int device_number,
+                       caf::actor replyActor)
+    : 
+      size_(size),
+      device_number_(device_number),
+      replyActor_(replyActor) {}
+
+  // CAF compliance
+  memory_request_token() = default;
+
+  int getType() const { return MEMORY; }
+
+  int getSize() const { return size_; }
+  int getDeviceNumber() const { return device_number_; }
+  caf::actor getReplyActor() const { return replyActor_; }
+
+private:
+  int size_;
+  int device_number_;
+  caf::actor replyActor_;
+};
+
+} // namespace caf::cuda

From f25fcee35dfc867b2e072717220991fdedcb11b2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 24 Feb 2026 12:15:00 -0600
Subject: [PATCH 0394/1000] Initial commit, mostly copied over from
 scheduler_actor.cpp

---
 .../memory_actor/memory_actor.cpp             | 107 ++++++++++++++++++
 1 file changed, 107 insertions(+)
 create mode 100644 libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp

diff --git a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
new file mode 100644
index 0000000000..2c156b0491
--- /dev/null
+++ b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
@@ -0,0 +1,107 @@
+#include "caf/cuda/control-layer/all-control-layer.hpp"
+#include <string>
+#include <iostream>
+
+/*
+ * This class is meant to handle memory managment/coordination  via s/r/r IPC
+ * it has nothing to do with the mem_ptr<T> (yet), that is kernel layer
+ */
+namespace caf::cuda {
+
+caf::behavior scheduler_actor(caf::stateful_actor<memory_actor_state>* self, int num_devices) {
+
+   // set device number
+    state.device_number = device_number;
+
+    //check if multiple gpus
+    state.multiple_gpus = multi_gpu;
+
+   // default behavior
+    state.table = std::make_unique<behavior_table>(state);
+    state.current_behavior = state.table -> get(behavior_token("single_usage"));
+    state.current_behavior->on_enter();
+
+
+
+    return {
+        [&](const token_ptr& tok) {
+            // std::cout << "Received token\n";
+            state.current_behavior->receive(tok);
+        },
+        
+   [&state](const caf::cuda::behavior_token_ptr& tok) -> bool {
+    auto* next = state.table -> get(*tok);
+    if (next) {
+        if (next != state.current_behavior) {
+            state.current_behavior->on_exit();   // cleanup current behavior
+            state.current_behavior = next;       // swap behavior
+            state.current_behavior->on_enter();  // init new behavior
+            std::cout << "[INFO] Behavior changed to: " << state.current_behavior->name() << "\n";
+	    return true; // behavior changed
+        } else {
+            std::cout << "[INFO] Behavior already active: " << state.current_behavior->name() << "\n";
+            return false; // behavior was already current
+        }
+    } else {
+        std::cout << "[WARN] No behavior found for token: " << tok->name() << "\n";
+        return false; // no change
+    }
+	}
+	,
+        [&](std::vector<token_ptr> tokens) {
+            for (size_t i = 0; i < tokens.size(); ++i) {
+                state.current_behavior->receive(tokens[i]);
+            }
+        },
+
+	//can send the scheduler a message if you want 
+	//it is more than happy to print it out for you 
+        [=](std::string word) {
+             std::cout << "Received message " << word << "\n";
+        },
+	
+	//message handler for reclaim
+	[&](int value, int memory,int runtime,int dependency) {	
+
+                //std::cout << "Received reclaim request\n";
+		state.current_behavior->reclaim(value,memory,runtime,dependency);
+	},
+
+
+	//message handler for reclaim
+	[&](ack payload) {	
+		state.current_behavior->reclaim(payload);
+	},
+
+
+
+
+	//handler sent to set the scheduler actors 
+	//do not send a message more than once
+	//or else undefined behavior
+	[&](std::vector<caf::actor> s) {
+		state.schedulers = s;
+	},
+
+
+
+	//message handler for a request for work from another scheduler actor
+	[&](int device_number) {
+		state.current_behavior -> handle_load_balance_request(device_number);
+	},
+
+	//message handler for work being transfered over from another scheduler actor
+	[&](std::vector<kernel_graph> work_graphs) {
+		state.current_behavior -> receive_work(work_graphs);
+	},
+	
+	//TEMPORARY FIX SINCE CAF TYPE ID IS STATIC SO POLYMORPHISM WONT WORK HERE
+	//TODO FIGURE OUT A WAY FOR ACK AND ITS CHILDREN TO BE 1 SINGLE CLASS AND
+	//DOWNCASTED EASILY
+	[&](transfer_ack payload) {
+		state.current_behavior->reclaim(static_cast<ack&>(payload));
+	}
+    };
+}
+
+} // namespace caf::cuda

From 08a59bfc080dd81ffc990127f499b6927b69235d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 24 Feb 2026 12:18:38 -0600
Subject: [PATCH 0395/1000] Updated requests to be an std::vector of
 std::queues, change is being made to accomodate ability to handle multiple
 GPU's at once.

---
 .../caf/cuda/control-layer/memory_actor/memory_actor.hpp        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
index 37596355d4..f6049d508c 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
@@ -18,7 +18,7 @@ namespace caf::cuda {
 
 struct memory_actor_state {
 
-	std::queue<memory_request_token> requests;
+	std::vector<std::queue<memory_request_token>> requests;
 	std::vector<device_ptr> devices;
 	int num_devices;
 

From 7ff4f12206ada02c115d9dc6c71dea18ab8becc3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 24 Feb 2026 12:30:35 -0600
Subject: [PATCH 0396/1000] Added a pending_requests boolean varaible.

---
 libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
index f6049d508c..133fe8dc75 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
@@ -21,6 +21,7 @@ struct memory_actor_state {
 	std::vector<std::queue<memory_request_token>> requests;
 	std::vector<device_ptr> devices;
 	int num_devices;
+	bool pending_requests = false;
 
 };
 

From c1d4bd81db7aa7b959661cd6d90f626242655441 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 24 Feb 2026 12:54:40 -0600
Subject: [PATCH 0397/1000] Implemented initial version.

---
 .../memory_actor/memory_actor.cpp             | 191 +++++++++---------
 1 file changed, 100 insertions(+), 91 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
index 2c156b0491..6d60292352 100644
--- a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
+++ b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
@@ -1,4 +1,6 @@
 #include "caf/cuda/control-layer/all-control-layer.hpp"
+#include "caf/cuda/manager.hpp"
+#include "caf/cuda/control-layer/return_payloads/ack.hpp"
 #include <string>
 #include <iostream>
 
@@ -10,98 +12,105 @@ namespace caf::cuda {
 
 caf::behavior scheduler_actor(caf::stateful_actor<memory_actor_state>* self, int num_devices) {
 
-   // set device number
-    state.device_number = device_number;
-
-    //check if multiple gpus
-    state.multiple_gpus = multi_gpu;
-
-   // default behavior
-    state.table = std::make_unique<behavior_table>(state);
-    state.current_behavior = state.table -> get(behavior_token("single_usage"));
-    state.current_behavior->on_enter();
-
-
-
-    return {
-        [&](const token_ptr& tok) {
-            // std::cout << "Received token\n";
-            state.current_behavior->receive(tok);
-        },
-        
-   [&state](const caf::cuda::behavior_token_ptr& tok) -> bool {
-    auto* next = state.table -> get(*tok);
-    if (next) {
-        if (next != state.current_behavior) {
-            state.current_behavior->on_exit();   // cleanup current behavior
-            state.current_behavior = next;       // swap behavior
-            state.current_behavior->on_enter();  // init new behavior
-            std::cout << "[INFO] Behavior changed to: " << state.current_behavior->name() << "\n";
-	    return true; // behavior changed
-        } else {
-            std::cout << "[INFO] Behavior already active: " << state.current_behavior->name() << "\n";
-            return false; // behavior was already current
-        }
-    } else {
-        std::cout << "[WARN] No behavior found for token: " << tok->name() << "\n";
-        return false; // no change
-    }
-	}
-	,
-        [&](std::vector<token_ptr> tokens) {
-            for (size_t i = 0; i < tokens.size(); ++i) {
-                state.current_behavior->receive(tokens[i]);
-            }
-        },
-
-	//can send the scheduler a message if you want 
-	//it is more than happy to print it out for you 
-        [=](std::string word) {
-             std::cout << "Received message " << word << "\n";
-        },
-	
-	//message handler for reclaim
-	[&](int value, int memory,int runtime,int dependency) {	
-
-                //std::cout << "Received reclaim request\n";
-		state.current_behavior->reclaim(value,memory,runtime,dependency);
-	},
-
-
-	//message handler for reclaim
-	[&](ack payload) {	
-		state.current_behavior->reclaim(payload);
-	},
-
-
-
-
-	//handler sent to set the scheduler actors 
-	//do not send a message more than once
-	//or else undefined behavior
-	[&](std::vector<caf::actor> s) {
-		state.schedulers = s;
-	},
-
-
-
-	//message handler for a request for work from another scheduler actor
-	[&](int device_number) {
-		state.current_behavior -> handle_load_balance_request(device_number);
-	},
-
-	//message handler for work being transfered over from another scheduler actor
-	[&](std::vector<kernel_graph> work_graphs) {
-		state.current_behavior -> receive_work(work_graphs);
-	},
-	
-	//TEMPORARY FIX SINCE CAF TYPE ID IS STATIC SO POLYMORPHISM WONT WORK HERE
-	//TODO FIGURE OUT A WAY FOR ACK AND ITS CHILDREN TO BE 1 SINGLE CLASS AND
-	//DOWNCASTED EASILY
-	[&](transfer_ack payload) {
-		state.current_behavior->reclaim(static_cast<ack&>(payload));
+	self -> state().num_devices = num_devices;
+
+	//initialize data structures 
+
+	for (int i = 0; i < num_devices; i++) {
+		std::queue<memory_request_token> r;
+		self->state().requests.emplace_back(std::move(r));
+		self->state().devices.emplace_back(manager::get().find_device(i));
 	}
-    };
+
+
+	return {
+		[&](const memory_request_token& token) {
+			int device_number = token.getDeviceNumber();
+			int free_memory =
+				static_cast<int>(
+						self->state().devices[device_number]
+						->available_memory_bytes());
+			//if we have enough memory reply immediately
+			if (free_memory > token.getSize()) {
+				ack msg;
+				self->mail(std::move(msg))
+					.send(token.getReplyActor());
+
+			}
+			else {
+				self->state().requests[device_number].push(std::move(token));
+				if (!self->state().pending_requests) {
+					//will begin a timer to check if memory is free
+					anon_mail(ack(TIMER)).delay(std::chrono::seconds(1)).send(self);	
+					self -> state().pending_requests = true;
+				}
+				//otherwise it will awake and check at a later time
+			}
+
+
+		},
+
+			//typically here to peridocially check if memory is free
+			[&](ack message) {
+
+				if (message.getType() != TIMER)
+					return;
+
+				bool still_pending = false;
+
+				for (int i = 0; i < self->state().num_devices; i++) {
+
+					auto& q = self->state().requests[i];
+
+					if (q.empty())
+						continue;
+
+					int device_number = i;
+					int free_memory =
+						static_cast<int>(self->state().devices[device_number]
+								->available_memory_bytes());
+
+					// Traverse queue until:
+					//  - empty
+					//  - or first unsatisfied request
+					while (!q.empty()) {
+
+						auto token = std::move(q.front());
+
+						if (free_memory > token.getSize()) {
+
+							q.pop();
+
+							ack msg;
+							self->mail(std::move(msg))
+								.send(token.getReplyActor());
+
+							// optionally update free_memory if allocations are immediate
+							// free_memory -= token.getSize();
+
+						} else {
+							// cannot satisfy head → stop processing this queue
+							break;
+						}
+					}
+
+					if (!q.empty())
+						still_pending = true;
+				}
+
+				if (still_pending) {
+					anon_mail(ack(TIMER))
+						.delay(std::chrono::seconds(1))
+						.send(self);
+					self->state().pending_requests = true;
+				} else {
+					self->state().pending_requests = false;
+				}
+			} //end of lambda
+
+
+
+	};
 }
 
 } // namespace caf::cuda

From 7e101e883741d167cdfd6bd2ffbf24d7e41f0c62 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 24 Feb 2026 12:59:48 -0600
Subject: [PATCH 0398/1000] Updated config to indicate usage for the memory
 manager actor.

---
 libcaf_cuda/caf/cuda/manager_config.hpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/manager_config.hpp b/libcaf_cuda/caf/cuda/manager_config.hpp
index 06fa0b8282..bc630d95f3 100644
--- a/libcaf_cuda/caf/cuda/manager_config.hpp
+++ b/libcaf_cuda/caf/cuda/manager_config.hpp
@@ -10,11 +10,14 @@ class manager_config {
 public:
     manager_config() : scheduler_on(false) {}      // initialize the bool
     manager_config(bool scheduler) : scheduler_on(scheduler) {}
-
-    bool getSchedulerOn() const { return scheduler_on; } // should be const
+    manager_config(bool scheduler, bool memory_manager) : scheduler_on(scheduler), memory_manager_on(memory_manager) {}
+    
+    bool getSchedulerOn() const { return scheduler_on; }
+    bool getMemoryManagerOn() const { return memory_manager_on; }
 
 private:
     bool scheduler_on;
+    bool memory_manager_on = false;
 };
 
 } // namespace caf::cuda

From 3ddec427dcda12e4525070ad7e10a2844f95b901 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 24 Feb 2026 13:32:30 -0600
Subject: [PATCH 0399/1000] Implemented get_memory_actor, init_memory_actor and
 destroy_memory_actor methods as well as updated some other code to accomodate
 the possibility of memory_actor existing in the system.

---
 libcaf_cuda/caf/cuda/manager.hpp | 15 +++++++--
 libcaf_cuda/src/manager.cpp      | 55 ++++++++++++++++++++++++++------
 2 files changed, 58 insertions(+), 12 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index 44f15e7b3c..7fba0006f8 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -58,8 +58,6 @@ class CAF_CUDA_EXPORT manager {
   /// Deletes the singleton if needed (optional).
   //deletes the scheduler actor as well if it exists
   static void shutdown();
-  caf::actor get_scheduler_actor();
-  
 
   // Prevent copy/assignment
   manager(const manager&) = delete;
@@ -179,12 +177,19 @@ class CAF_CUDA_EXPORT manager {
 
   int get_num_devices();
 
+
+
+  caf::actor get_scheduler_actor();
+
+
+
   //methods used to send scheduler actors messages
   void send_scheduler_actor_message(token_ptr token,int device_number = -1);
   void send_scheduler_actor_message(std::vector<token_ptr> tokens,int device_number = -1);
   void send_scheduler_actor_message(behavior_token_ptr token,int device_number);
   void send_scheduler_actor_message(std::string behavior,int device_number);
 
+  caf::actor get_memory_actor();
 
 private:
   explicit manager(caf::actor_system& sys)
@@ -199,11 +204,17 @@ class CAF_CUDA_EXPORT manager {
   bool compile_nvrtc_program(const char* source, CUdevice device, std::vector<char>& ptx_out);
 
   void init_scheduler_actors(caf::actor_system&);
+  
+  //methods to create and destroy memory_actor
+  void init_memory_actor(caf::actor_system&);
+  void destroy_memory_actor();
 
   static manager* instance_;
   static std::mutex mutex_;
   bool scheduler_on = false;
+  bool memory_manager_on = false;
   caf::actor scheduler_actor_handle;
+  caf::actor memory_actor_handle;
   std::vector<caf::actor> scheduler_actors;
 };
 
diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index cc356f9b2f..28e33d25b3 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -56,14 +56,17 @@ void manager::init(caf::actor_system& sys, manager_config config) {
     caf::init_global_meta_objects<caf::id_block::cuda_control>();
 
     instance_->scheduler_on = config.getSchedulerOn();
+    instance_->memory_manager_on = config.getMemoryManagerOn();
 
-    if (instance_->scheduler_on) {
-        
-	    
+    if (instance_->scheduler_on) { 
 	    instance_ -> init_scheduler_actors(sys);
-	    //instance_->scheduler_actor_handle =
-          //  sys.spawn(scheduler_actor,0);
     }
+    if (instance_->memory_manager_on) {
+	    instance_->init_memory_actor(sys);
+    }
+
+
+
 }
 
 int manager::get_num_devices() {return platform_ -> get_num_devices();}
@@ -123,11 +126,6 @@ void manager::shutdown() {
         
 	for (int i = 0; i < instance_ ->  platform_ -> get_num_devices(); i++) {
 	    
-//	anon_send_exit(
-  //          instance_->scheduler_actor_handle,
-    //        caf::exit_reason::user_shutdown
-      //  );
-     
 	anon_send_exit(
             instance_->scheduler_actors[i],
             caf::exit_reason::user_shutdown
@@ -139,6 +137,12 @@ void manager::shutdown() {
 
     }
 
+
+    if (instance_->memory_manager_on) {
+	    instance_->destroy_memory_actor();
+    }
+
+
     delete instance_;
     instance_ = nullptr;
 }
@@ -368,4 +372,35 @@ void manager::send_scheduler_actor_message(std::string behavior, int device_numb
 }
 
 
+void manager::init_memory_actor(caf::actor_system& sys) {
+    if (memory_actor_handle)
+        return; // already initialized
+
+    int num_devices = platform_->get_num_devices();
+
+    memory_actor_handle =
+        sys.spawn(memory_actor, num_devices);
+}
+
+void manager::destroy_memory_actor() {
+    if (!memory_actor_handle)
+        return;
+
+    anon_send_exit(
+        memory_actor_handle,
+        caf::exit_reason::user_shutdown
+    );
+
+    memory_actor_handle = caf::actor{};
+}
+
+caf::actor manager::get_memory_actor() {
+    if (!instance_ || !instance_->memory_actor_handle) {
+        throw std::runtime_error("Memory actor not initialized");
+    }
+
+    return instance_->memory_actor_handle;
+}
+
+
 } // namespace caf::cuda

From 6e7d3da7a73e480571859298d1a03dcf3ab31909 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 24 Feb 2026 13:38:24 -0600
Subject: [PATCH 0400/1000] Initial commit, copied over from
 command-runner-integration-tests

---
 .../memory_actor_test/CMakeLists.txt          |  44 ++
 .../memory_actor_test/compile_kernels.sh      |  18 +
 .../memory_actor_test/main.test.cpp           | 418 ++++++++++++++++++
 .../memory_actor_test/mmul.cu                 |  16 +
 4 files changed, 496 insertions(+)
 create mode 100644 libcaf_cuda/tests/control-layer-tests/memory_actor_test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/control-layer-tests/memory_actor_test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
 create mode 100644 libcaf_cuda/tests/control-layer-tests/memory_actor_test/mmul.cu

diff --git a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/CMakeLists.txt b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/compile_kernels.sh b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/compile_kernels.sh
new file mode 100755
index 0000000000..586196454e
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/compile_kernels.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+
+# Compile genMatrix.cu to fatbin in current directory
+#nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
+#echo "Generated generate_random_matrix.fatbin"
+
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
new file mode 100644
index 0000000000..f5de194eb7
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
@@ -0,0 +1,418 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include <unistd.h>
+#include "caf/actor_registry.hpp"
+#include <chrono>
+#include <iostream>
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+struct exit_actor_state {
+	int completed = 0;
+};
+
+
+caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
+
+
+	return {
+		[=](int num_completed) {
+			self->state().completed += num_completed;
+			
+			std::cout << "Actors finished is " << self->state().completed << "\n";
+			if (self->state().completed >= limit) {
+			
+				caf::cuda::manager::shutdown();
+				self->quit();
+			}
+		}
+	};
+
+
+}
+
+
+
+
+
+
+// Define a custom type ID block for custom actors
+CAF_ADD_ATOM(cuda,shared_mem)
+
+
+
+
+
+// Extend your actor state to keep the start time
+struct mmul_actor_state {
+  static inline const char* name = "my_actor";
+  int N = 1024; // example state variable
+  int id = rand(); // an actor id
+  // per-actor timing start
+  std::chrono::high_resolution_clock::time_point start_time;
+  int times = 0;
+  caf::cuda::program_ptr program = caf::cuda::manager::get().create_program_from_cubin("../mmul.cubin","matrixMul");
+  int THREADS = 32;
+  int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS,BLOCKS,1,THREADS,THREADS,THREADS);
+};
+
+
+
+
+//commands classes used to launch kernels 
+using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
+using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
+
+using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
+
+mmulCommand mmul;
+matrixGenCommand randomMatrix;
+mmulAsyncCommand mmulAsync;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
+ 
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+	
+  	caf::actor scheduler = mgr.get_scheduler_actor();
+	
+	//send a launch token
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
+			self -> state().dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+	  [=] (caf::cuda::response_token_ptr res_token) {
+	
+		 if (res_token -> getType() == LAUNCH_RESPONSE) {
+		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+		  //assume N = 1024
+		  int N = self -> state().N;
+		  std::vector<int> matrix1(N*N);
+		  matrix1.reserve(N);
+		  std::vector<int> matrix2(N*N);
+		  matrix2.reserve(N);
+
+		  //std::cout << "GPU ACTOR sending data to compute\n";
+		  self -> mail(matrix1,matrix2,res_token,N).send(self);
+	 
+		 }
+		 else {
+			 std::cout << "Got a memory response token\n";
+		 }
+		 //token should drop out of scope now, triggering a response 
+	  },
+
+    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+	const caf::cuda::response_token_ptr& res_token, int N) {
+ 
+
+	    //caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
+
+	    //caf::cuda::launch_response_token& kt =
+	//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
+
+		  //std::cout << "GPU ACTOR  computing\n";
+	    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //create program and dims   
+  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+  const int THREADS = 32;
+  const int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    //create args
+    auto arg1 = caf::cuda::create_in_arg(matrixA);
+    auto arg2 = caf::cuda::create_in_arg(matrixB);
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+		  //std::cout << "GPU ACTOR done  computing\n";
+    //verify its own result 
+    self -> mail(matrixA,matrixB,matrixC,N).send(self);
+
+    },
+
+    // 3rd handler: CPU atom + matrices + N
+    [=](const std::vector<int>& matrixA,
+    const std::vector<int>& matrixB,
+    const std::vector<int>& matrixC,
+    int N) {
+
+  using clock = std::chrono::high_resolution_clock;
+
+  auto start = clock::now();
+
+  //std::cout << "GPU ACTOR verifying\n";
+
+  std::vector<int> result(N * N);
+
+  serial_matrix_multiply(matrixA, matrixB, result, N);
+
+  if (result == matrixC) {
+    std::cout << "actor with id " << self->state().id
+              << " references match\n";
+  } else {
+    std::cout << "actor with id " << self->state().id
+              << " references did not match\n";
+  }
+
+  auto end = clock::now();
+
+  auto ms =
+    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[TIMING] verification took "
+            << ms << " ms (actor id "
+            << self->state().id << ")\n";
+
+  // signal exit actor and quit
+  self->mail(1).send(exit_actor);
+  self->quit();
+
+    }
+  };
+}
+
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  int limit = 1;
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+
+  for (int i = 0; i < limit; i++) {
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
+  }
+
+
+ // std::cout << actors.size() << "\n";
+
+  }
+
+   sys.await_all_actors_done();
+}
+
+// Stateful actor behavior
+caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
+ 
+
+	//set the value of N correctly to overide the base option.	
+	self->state().N = N;
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+	
+  	caf::actor scheduler = mgr.get_scheduler_actor();
+	
+	//send a launch token
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
+			self -> state().dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+	  [=] (caf::cuda::response_token_ptr res_token) {
+	
+		 if (res_token -> getType() == LAUNCH_RESPONSE) {
+		  int N = self -> state().N;
+
+		  caf::cuda::command_runner<in_out<int>> mem_transfer_command;
+
+
+		  std::vector<int> matrix1(N*N);
+		  std::vector<int> matrix2(N*N);
+
+		  caf::cuda::mem_ptr<int> matrixA = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix1});
+		  caf::cuda::mem_ptr<int> matrixB = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix2});
+
+		  //std::cout << "GPU ACTOR sending data to compute\n";
+		  self -> mail(matrixA,matrixB,res_token,N).send(self);
+	 
+		 }
+		 else {
+			 //std::cout << "Got a memory response token\n";
+		 }
+		 //token should drop out of scope now, triggering a response 
+	  },
+
+    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+    [=](const caf::cuda::mem_ptr<int>& matrixA,
+        const caf::cuda::mem_ptr<int>& matrixB,
+	const caf::cuda::response_token_ptr& res_token, int N) {
+ 
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+	//create program and dims   
+ 	auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+  	const int THREADS = 32;
+  	const int BLOCKS = (N + THREADS - 1) / THREADS;
+  	caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    //create args
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    auto temp = mmulAsync.run_async(program,dims,res_token,matrixA,matrixB,arg3,arg4);
+    caf::cuda::mem_ptr<int> matrixC = std::get<2>(temp);
+
+    std::vector<int> matrix1 = matrixA -> copy_to_host();
+    std::vector<int> matrix2 = matrixB -> copy_to_host();
+    std::vector<int> matrix3 = matrixC -> copy_to_host();
+
+		  //std::cout << "GPU ACTOR done  computing\n";
+    //verify its own result 
+    self -> mail(matrix1,matrix2,matrix3,N).send(self);
+
+    },
+
+    // 3rd handler: CPU atom + matrices + N
+    [=](const std::vector<int>& matrixA,
+    const std::vector<int>& matrixB,
+    const std::vector<int>& matrixC,
+    int N) {
+
+  using clock = std::chrono::high_resolution_clock;
+
+  auto start = clock::now();
+
+  //std::cout << "GPU ACTOR verifying\n";
+
+  std::vector<int> result(N * N);
+
+  serial_matrix_multiply(matrixA, matrixB, result, N);
+
+  if (result == matrixC) {
+    std::cout << "actor with id " << self->state().id
+              << " references match\n";
+  } else {
+    std::cout << "actor with id " << self->state().id
+              << " references did not match\n";
+  }
+
+  auto end = clock::now();
+
+  auto ms =
+    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[TIMING] verification took "
+            << ms << " ms (actor id "
+            << self->state().id << ")\n";
+
+  // signal exit actor and quit
+  self->mail(1).send(exit_actor);
+  self->quit();
+
+    }
+  };
+}
+
+
+
+void run_async_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  int limit = 1;
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+
+  for (int i = 0; i < limit; i++) {
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_async_actor_fun,exit_actor,matrix_size));
+  }
+
+
+ // std::cout << actors.size() << "\n";
+
+  }
+
+   sys.await_all_actors_done();
+}
+
+
+
+void caf_main(caf::actor_system& sys) {
+  
+	
+
+	caf::cuda::manager_config man_config(true); //turns the scheduler on
+	caf::cuda::manager::init(sys,man_config);
+       	run_async_mmul_test(sys,10,500);
+	
+	//tests will delete the old manager so will have to reinit if you do this 
+	//in conjunction with each other	
+	//caf::cuda::manager::init(sys,man_config);
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/mmul.cu b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From 6f3994436ecc7bdea981793fdd4b9b7b4bd4be4a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 24 Feb 2026 13:50:32 -0600
Subject: [PATCH 0401/1000] Removed redunant code.

---
 .../memory_actor_test/main.test.cpp           | 149 ------------------
 1 file changed, 149 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
index f5de194eb7..b9c9c8de05 100644
--- a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
@@ -50,10 +50,6 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int lim
 
 
 
-// Define a custom type ID block for custom actors
-CAF_ADD_ATOM(cuda,shared_mem)
-
-
 
 
 
@@ -252,151 +248,6 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
    sys.await_all_actors_done();
 }
 
-// Stateful actor behavior
-caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
- 
-
-	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-	
-  	caf::actor scheduler = mgr.get_scheduler_actor();
-	
-	//send a launch token
-	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
-			self -> state().dims,
-			0,
-			"hello",
-			self
-			);
-	self -> mail(launch_token).send(scheduler);
-
-	return {
-
-	  [=] (caf::cuda::response_token_ptr res_token) {
-	
-		 if (res_token -> getType() == LAUNCH_RESPONSE) {
-		  int N = self -> state().N;
-
-		  caf::cuda::command_runner<in_out<int>> mem_transfer_command;
-
-
-		  std::vector<int> matrix1(N*N);
-		  std::vector<int> matrix2(N*N);
-
-		  caf::cuda::mem_ptr<int> matrixA = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix1});
-		  caf::cuda::mem_ptr<int> matrixB = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix2});
-
-		  //std::cout << "GPU ACTOR sending data to compute\n";
-		  self -> mail(matrixA,matrixB,res_token,N).send(self);
-	 
-		 }
-		 else {
-			 //std::cout << "Got a memory response token\n";
-		 }
-		 //token should drop out of scope now, triggering a response 
-	  },
-
-    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const caf::cuda::mem_ptr<int>& matrixA,
-        const caf::cuda::mem_ptr<int>& matrixB,
-	const caf::cuda::response_token_ptr& res_token, int N) {
- 
-	caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-	//create program and dims   
- 	auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-  	const int THREADS = 32;
-  	const int BLOCKS = (N + THREADS - 1) / THREADS;
-  	caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    //create args
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
-
-    auto temp = mmulAsync.run_async(program,dims,res_token,matrixA,matrixB,arg3,arg4);
-    caf::cuda::mem_ptr<int> matrixC = std::get<2>(temp);
-
-    std::vector<int> matrix1 = matrixA -> copy_to_host();
-    std::vector<int> matrix2 = matrixB -> copy_to_host();
-    std::vector<int> matrix3 = matrixC -> copy_to_host();
-
-		  //std::cout << "GPU ACTOR done  computing\n";
-    //verify its own result 
-    self -> mail(matrix1,matrix2,matrix3,N).send(self);
-
-    },
-
-    // 3rd handler: CPU atom + matrices + N
-    [=](const std::vector<int>& matrixA,
-    const std::vector<int>& matrixB,
-    const std::vector<int>& matrixC,
-    int N) {
-
-  using clock = std::chrono::high_resolution_clock;
-
-  auto start = clock::now();
-
-  //std::cout << "GPU ACTOR verifying\n";
-
-  std::vector<int> result(N * N);
-
-  serial_matrix_multiply(matrixA, matrixB, result, N);
-
-  if (result == matrixC) {
-    std::cout << "actor with id " << self->state().id
-              << " references match\n";
-  } else {
-    std::cout << "actor with id " << self->state().id
-              << " references did not match\n";
-  }
-
-  auto end = clock::now();
-
-  auto ms =
-    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-  std::cout << "[TIMING] verification took "
-            << ms << " ms (actor id "
-            << self->state().id << ")\n";
-
-  // signal exit actor and quit
-  self->mail(1).send(exit_actor);
-  self->quit();
-
-    }
-  };
-}
-
-
-
-void run_async_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  int limit = 1;
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
-
-  for (int i = 0; i < limit; i++) {
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_async_actor_fun,exit_actor,matrix_size));
-  }
-
-
- // std::cout << actors.size() << "\n";
-
-  }
-
-   sys.await_all_actors_done();
-}
-
 
 
 void caf_main(caf::actor_system& sys) {

From dfdc92219bc8cedfd264ef27e230d67b0602d4e1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 25 Feb 2026 11:05:45 -0600
Subject: [PATCH 0402/1000] Added avaible memory to memory actor state so it
 can track, available memory

---
 libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
index 133fe8dc75..866feda489 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
@@ -20,6 +20,7 @@ struct memory_actor_state {
 
 	std::vector<std::queue<memory_request_token>> requests;
 	std::vector<device_ptr> devices;
+	std::vector<std::size_t> available_memory;
 	int num_devices;
 	bool pending_requests = false;
 

From ad7ab79f53b3b71aee06b876a55ac4ea8a569fcc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 25 Feb 2026 11:18:57 -0600
Subject: [PATCH 0403/1000] Updated memory actor to have a count of free memory
 and perodically synchronizes its count with the available memory.

---
 .../memory_actor/memory_actor.cpp             | 47 ++++++++++++-------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
index 6d60292352..175d1a56cb 100644
--- a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
+++ b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
@@ -20,22 +20,23 @@ caf::behavior scheduler_actor(caf::stateful_actor<memory_actor_state>* self, int
 		std::queue<memory_request_token> r;
 		self->state().requests.emplace_back(std::move(r));
 		self->state().devices.emplace_back(manager::get().find_device(i));
+		// Initialize internal memory counter
+		self->state().available_memory.push_back(
+				dev->available_memory_bytes()
+				);
 	}
 
 
 	return {
 		[&](const memory_request_token& token) {
 			int device_number = token.getDeviceNumber();
-			int free_memory =
-				static_cast<int>(
-						self->state().devices[device_number]
-						->available_memory_bytes());
-			//if we have enough memory reply immediately
-			if (free_memory > token.getSize()) {
-				ack msg;
-				self->mail(std::move(msg))
-					.send(token.getReplyActor());
+			auto& free_memory =
+				self->state().available_memory[device_number];
 
+			if (free_memory >= static_cast<std::size_t>(token.getSize())) {
+				free_memory -= token.getSize();
+				self->send(token.getReplyActor(),
+						ack(CAF_CUDA_ACK_MEMORY));
 			}
 			else {
 				self->state().requests[device_number].push(std::move(token));
@@ -58,6 +59,18 @@ caf::behavior scheduler_actor(caf::stateful_actor<memory_actor_state>* self, int
 
 				bool still_pending = false;
 
+
+
+				//  First: synchronize internal counters with real device
+				for (int i = 0; i < self->state().num_devices; i++) {
+
+					if (!self->state().requests[i].empty()) {
+						self->state().available_memory[i] =
+							self->state().devices[i]->available_memory_bytes();
+					}
+				}
+
+				// Then process queues
 				for (int i = 0; i < self->state().num_devices; i++) {
 
 					auto& q = self->state().requests[i];
@@ -77,18 +90,16 @@ caf::behavior scheduler_actor(caf::stateful_actor<memory_actor_state>* self, int
 
 						auto token = std::move(q.front());
 
-						if (free_memory > token.getSize()) {
-
-							q.pop();
+						if (free_memory >= token.getSize()) {
 
-							ack msg;
-							self->mail(std::move(msg))
-								.send(token.getReplyActor());
+							free_memory -= token.getSize();
 
-							// optionally update free_memory if allocations are immediate
-							// free_memory -= token.getSize();
+							self->send(token.getReplyActor(),
+									ack(CAF_CUDA_ACK_MEMORY));
+							q.pop();	
+				 		}
 
-						} else {
+						else {
 							// cannot satisfy head → stop processing this queue
 							break;
 						}

From a1b2d5ee95629aeb78871732a0adef8c8ced085f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 25 Feb 2026 14:20:53 -0600
Subject: [PATCH 0404/1000] Fixed compiler issues.

---
 .../control-layer/memory_actor/memory_actor.cpp   | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
index 175d1a56cb..9ce4c72608 100644
--- a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
+++ b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
@@ -22,7 +22,7 @@ caf::behavior scheduler_actor(caf::stateful_actor<memory_actor_state>* self, int
 		self->state().devices.emplace_back(manager::get().find_device(i));
 		// Initialize internal memory counter
 		self->state().available_memory.push_back(
-				dev->available_memory_bytes()
+				manager::get().find_device(i)->available_memory_bytes()
 				);
 	}
 
@@ -35,8 +35,7 @@ caf::behavior scheduler_actor(caf::stateful_actor<memory_actor_state>* self, int
 
 			if (free_memory >= static_cast<std::size_t>(token.getSize())) {
 				free_memory -= token.getSize();
-				self->send(token.getReplyActor(),
-						ack(CAF_CUDA_ACK_MEMORY));
+				self->mail(ack(CAF_CUDA_ACK_MEMORY)).send(token.getReplyActor());
 			}
 			else {
 				self->state().requests[device_number].push(std::move(token));
@@ -87,18 +86,12 @@ caf::behavior scheduler_actor(caf::stateful_actor<memory_actor_state>* self, int
 					//  - empty
 					//  - or first unsatisfied request
 					while (!q.empty()) {
-
 						auto token = std::move(q.front());
-
 						if (free_memory >= token.getSize()) {
-
-							free_memory -= token.getSize();
-
-							self->send(token.getReplyActor(),
-									ack(CAF_CUDA_ACK_MEMORY));
+							free_memory -= token.getSize();	
+							self->mail(ack(CAF_CUDA_ACK_MEMORY)).send(token.getReplyActor());
 							q.pop();	
 				 		}
-
 						else {
 							// cannot satisfy head → stop processing this queue
 							break;

From 4a6b9d739c7cece36afc13fa6896587837164e0f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 25 Feb 2026 14:29:45 -0600
Subject: [PATCH 0405/1000] Created template for memory hog actor.

---
 .../memory_actor_test/main.test.cpp           | 152 ++----------------
 1 file changed, 12 insertions(+), 140 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
index b9c9c8de05..9d326941bd 100644
--- a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
@@ -54,17 +54,8 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int lim
 
 
 // Extend your actor state to keep the start time
-struct mmul_actor_state {
-  static inline const char* name = "my_actor";
-  int N = 1024; // example state variable
-  int id = rand(); // an actor id
-  // per-actor timing start
-  std::chrono::high_resolution_clock::time_point start_time;
-  int times = 0;
-  caf::cuda::program_ptr program = caf::cuda::manager::get().create_program_from_cubin("../mmul.cubin","matrixMul");
-  int THREADS = 32;
-  int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS,BLOCKS,1,THREADS,THREADS,THREADS);
+struct memory_hog_actor_state {
+	int bytes;
 };
 
 
@@ -81,142 +72,23 @@ matrixGenCommand randomMatrix;
 mmulAsyncCommand mmulAsync;
 
 
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-  
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
-    }
-  }
-}
-
-
-
-
-// Stateful actor behavior
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
+caf::behavior memory_hog_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int bytes) {
  
 
-	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-	
-  	caf::actor scheduler = mgr.get_scheduler_actor();
-	
-	//send a launch token
-	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
-			self -> state().dims,
-			0,
-			"hello",
-			self
-			);
-	self -> mail(launch_token).send(scheduler);
+	self->state().bytes = bytes; 
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
+  	caf::actor memory_actor = mgr.get_memory_actor();
+	//send a memory request token
+	caf::cuda::memory_request_token request(bytes,0,self);
+	self ->mail(request).send(memory_actor);
 
 	return {
 
-	  [=] (caf::cuda::response_token_ptr res_token) {
-	
-		 if (res_token -> getType() == LAUNCH_RESPONSE) {
-		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-		  //assume N = 1024
-		  int N = self -> state().N;
-		  std::vector<int> matrix1(N*N);
-		  matrix1.reserve(N);
-		  std::vector<int> matrix2(N*N);
-		  matrix2.reserve(N);
-
-		  //std::cout << "GPU ACTOR sending data to compute\n";
-		  self -> mail(matrix1,matrix2,res_token,N).send(self);
-	 
-		 }
-		 else {
-			 std::cout << "Got a memory response token\n";
-		 }
-		 //token should drop out of scope now, triggering a response 
+	  [=] (caf::cuda::ack msg ) {
+		  self->mail(1).send(exit_actor);
+		  self->quit();
 	  },
 
-    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB,
-	const caf::cuda::response_token_ptr& res_token, int N) {
- 
-
-	    //caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
-
-	    //caf::cuda::launch_response_token& kt =
-	//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
-
-		  //std::cout << "GPU ACTOR  computing\n";
-	    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //create program and dims   
-  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    //create args
-    auto arg1 = caf::cuda::create_in_arg(matrixA);
-    auto arg2 = caf::cuda::create_in_arg(matrixB);
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
-
-    auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-		  //std::cout << "GPU ACTOR done  computing\n";
-    //verify its own result 
-    self -> mail(matrixA,matrixB,matrixC,N).send(self);
-
-    },
-
-    // 3rd handler: CPU atom + matrices + N
-    [=](const std::vector<int>& matrixA,
-    const std::vector<int>& matrixB,
-    const std::vector<int>& matrixC,
-    int N) {
-
-  using clock = std::chrono::high_resolution_clock;
-
-  auto start = clock::now();
-
-  //std::cout << "GPU ACTOR verifying\n";
-
-  std::vector<int> result(N * N);
-
-  serial_matrix_multiply(matrixA, matrixB, result, N);
-
-  if (result == matrixC) {
-    std::cout << "actor with id " << self->state().id
-              << " references match\n";
-  } else {
-    std::cout << "actor with id " << self->state().id
-              << " references did not match\n";
-  }
-
-  auto end = clock::now();
-
-  auto ms =
-    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-  std::cout << "[TIMING] verification took "
-            << ms << " ms (actor id "
-            << self->state().id << ")\n";
-
-  // signal exit actor and quit
-  self->mail(1).send(exit_actor);
-  self->quit();
-
-    }
   };
 }
 

From b4dcd9183ca0ed188f8ade713feaf153dbd6e470 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 25 Feb 2026 14:56:18 -0600
Subject: [PATCH 0406/1000] Changed size to be of size_t type instead of int
 Change was made to prevent integer overflow bugs.

---
 .../memory_actor/memory_request_token.hpp              |  4 ++--
 .../memory_actor_test/main.test.cpp                    | 10 ++++++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_request_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_request_token.hpp
index f83aa0ae6e..294345377d 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_request_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_request_token.hpp
@@ -10,7 +10,7 @@ namespace caf::cuda {
 
 class CAF_CUDA_EXPORT memory_request_token {
 public:
-  memory_request_token(int size,
+  memory_request_token(std::size_t size,
                        int device_number,
                        caf::actor replyActor)
     : 
@@ -28,7 +28,7 @@ class CAF_CUDA_EXPORT memory_request_token {
   caf::actor getReplyActor() const { return replyActor_; }
 
 private:
-  int size_;
+  std::size_t size_;
   int device_number_;
   caf::actor replyActor_;
 };
diff --git a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
index 9d326941bd..7a829ad4ce 100644
--- a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
@@ -85,6 +85,16 @@ caf::behavior memory_hog_actor_fun(caf::stateful_actor<mmul_actor_state>* self,c
 	return {
 
 	  [=] (caf::cuda::ack msg ) {
+		  caf::cuda::command_runner mem_transfer_command<>;
+		  std::vector<int> big_buffer(bytes/sizeof(int),0);
+
+		  caf::cuda_mem_ptr<int> temp = mem_transfer_command.transfer_memory(0,1,in_out{big_buffer});
+
+		  //hold onto memory for a few seconds
+		  std::cout << "Memory hog holding onto memory for 5 seconds\n";
+		  std::this_thread::sleep_for(std::chrono::seconds(5));
+		  std::cout << "Memory hog releasing memory\n";
+		  
 		  self->mail(1).send(exit_actor);
 		  self->quit();
 	  },

From be708022397a0240f0113f612e1bdc99d3e1bc43 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 25 Feb 2026 14:58:04 -0600
Subject: [PATCH 0407/1000] Changed getter to be of size_t to comply with
 previous changes.

---
 .../cuda/control-layer/memory_actor/memory_request_token.hpp    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_request_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_request_token.hpp
index 294345377d..4fd4546c53 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_request_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_request_token.hpp
@@ -23,7 +23,7 @@ class CAF_CUDA_EXPORT memory_request_token {
 
   int getType() const { return MEMORY; }
 
-  int getSize() const { return size_; }
+  std::size_t getSize() const { return size_; }
   int getDeviceNumber() const { return device_number_; }
   caf::actor getReplyActor() const { return replyActor_; }
 

From 2f846fe17904e6fd83cf673587eaf66269bc9520 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 25 Feb 2026 14:59:41 -0600
Subject: [PATCH 0408/1000] Updated memory_actor to corrspond with size_t
 changes Changes are being made to prevent integer overflow.

---
 .../src/control-layer/memory_actor/memory_actor.cpp       | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
index 9ce4c72608..b43eb08649 100644
--- a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
+++ b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
@@ -33,7 +33,7 @@ caf::behavior scheduler_actor(caf::stateful_actor<memory_actor_state>* self, int
 			auto& free_memory =
 				self->state().available_memory[device_number];
 
-			if (free_memory >= static_cast<std::size_t>(token.getSize())) {
+			if (free_memory >= token.getSize()) {
 				free_memory -= token.getSize();
 				self->mail(ack(CAF_CUDA_ACK_MEMORY)).send(token.getReplyActor());
 			}
@@ -78,10 +78,8 @@ caf::behavior scheduler_actor(caf::stateful_actor<memory_actor_state>* self, int
 						continue;
 
 					int device_number = i;
-					int free_memory =
-						static_cast<int>(self->state().devices[device_number]
-								->available_memory_bytes());
-
+					std::size_t free_memory = self->state().devices[device_number]
+								->available_memory_bytes();
 					// Traverse queue until:
 					//  - empty
 					//  - or first unsatisfied request

From fceff34eb30678003e83eb8dcd61a070afbc3931 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 25 Feb 2026 15:10:03 -0600
Subject: [PATCH 0409/1000] implemented memory_test.

---
 .../memory_actor_test/main.test.cpp           | 42 ++++---------------
 1 file changed, 9 insertions(+), 33 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
index 7a829ad4ce..2e6613d79c 100644
--- a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
@@ -46,21 +46,10 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int lim
 }
 
 
-
-
-
-
-
-
-
-// Extend your actor state to keep the start time
 struct memory_hog_actor_state {
-	int bytes;
+	std::size_t bytes;
 };
 
-
-
-
 //commands classes used to launch kernels 
 using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
 using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
@@ -72,9 +61,8 @@ matrixGenCommand randomMatrix;
 mmulAsyncCommand mmulAsync;
 
 
-caf::behavior memory_hog_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int bytes) {
+caf::behavior memory_hog_actor_fun(caf::stateful_actor<memory_hog_actor_state>* self,caf::actor exit_actor,std::size_t bytes) {
  
-
 	self->state().bytes = bytes; 
 	caf::cuda::manager& mgr = caf::cuda::manager::get();
   	caf::actor memory_actor = mgr.get_memory_actor();
@@ -85,10 +73,10 @@ caf::behavior memory_hog_actor_fun(caf::stateful_actor<mmul_actor_state>* self,c
 	return {
 
 	  [=] (caf::cuda::ack msg ) {
-		  caf::cuda::command_runner mem_transfer_command<>;
+		  caf::cuda::command_runner<> mem_transfer_command;
 		  std::vector<int> big_buffer(bytes/sizeof(int),0);
 
-		  caf::cuda_mem_ptr<int> temp = mem_transfer_command.transfer_memory(0,1,in_out{big_buffer});
+		  caf::cuda::mem_ptr<int> temp = mem_transfer_command.transfer_memory(0,1,in_out{big_buffer});
 
 		  //hold onto memory for a few seconds
 		  std::cout << "Memory hog holding onto memory for 5 seconds\n";
@@ -104,27 +92,19 @@ caf::behavior memory_hog_actor_fun(caf::stateful_actor<mmul_actor_state>* self,c
 
 
 
-void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+void run_memory_hog_test(caf::actor_system& sys, std::size_t bytes, int num_actors) {
   if (num_actors < 1) {
     std::cerr << "[ERROR] Number of actors must be >= 1\n";
     return;
   }
 
-  int limit = 1;
 
   caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
-
-  for (int i = 0; i < limit; i++) {
   // Spawn num_actors actors running the mmul behavior
   std::vector<caf::actor> actors;
   actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
-  }
-
-
- // std::cout << actors.size() << "\n";
-
+  for (int i = 0; i < num_actors; i++){
+    actors.push_back(sys.spawn(memory_hog_actor_fun,exit_actor,bytes));
   }
 
    sys.await_all_actors_done();
@@ -136,13 +116,9 @@ void caf_main(caf::actor_system& sys) {
   
 	
 
-	caf::cuda::manager_config man_config(true); //turns the scheduler on
+	caf::cuda::manager_config man_config(false,true); //turns the memory actor on
 	caf::cuda::manager::init(sys,man_config);
-       	run_async_mmul_test(sys,10,500);
-	
-	//tests will delete the old manager so will have to reinit if you do this 
-	//in conjunction with each other	
-	//caf::cuda::manager::init(sys,man_config);
+       	run_memory_hog_test(sys,3221225472,8);
 }
 
 

From b27edb5f89c2c7959442963e265b5c1a280bf3ad Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 25 Feb 2026 16:18:12 -0600
Subject: [PATCH 0410/1000] Changed signature from scheduler_actor to
 memory_actor. This change was made to fix an unintended linker issue where it
 could not find the memory_actor since its signature was wrong.

---
 libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
index b43eb08649..f3eb0147e7 100644
--- a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
+++ b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
@@ -10,7 +10,7 @@
  */
 namespace caf::cuda {
 
-caf::behavior scheduler_actor(caf::stateful_actor<memory_actor_state>* self, int num_devices) {
+caf::behavior memory_actor(caf::stateful_actor<memory_actor_state>* self, int num_devices) {
 
 	self -> state().num_devices = num_devices;
 

From b18415340833837eabe90bee4577c13892f969a8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 26 Feb 2026 13:30:21 -0600
Subject: [PATCH 0411/1000] Changed lambda captures to capture self by value
 instead of by reference. This change was made to prevent a memory issue where
 self would be captured but then deleted, causing a segfault.

---
 libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
index f3eb0147e7..6dfff3f7b5 100644
--- a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
+++ b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
@@ -28,8 +28,9 @@ caf::behavior memory_actor(caf::stateful_actor<memory_actor_state>* self, int nu
 
 
 	return {
-		[&](const memory_request_token& token) {
+		[self](const memory_request_token& token) {
 			int device_number = token.getDeviceNumber();
+			
 			auto& free_memory =
 				self->state().available_memory[device_number];
 
@@ -51,7 +52,7 @@ caf::behavior memory_actor(caf::stateful_actor<memory_actor_state>* self, int nu
 		},
 
 			//typically here to peridocially check if memory is free
-			[&](ack message) {
+			[self](ack message) {
 
 				if (message.getType() != TIMER)
 					return;

From 89dce94e78b0d67ad20542f241097097049a93cf Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 26 Feb 2026 14:18:35 -0600
Subject: [PATCH 0412/1000] Added rvalue method overloads for create_in and
 create_in_out args to reduce unecesarry copying.

---
 libcaf_cuda/caf/cuda/helpers.hpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/helpers.hpp b/libcaf_cuda/caf/cuda/helpers.hpp
index 1b45a8081a..b151625795 100644
--- a/libcaf_cuda/caf/cuda/helpers.hpp
+++ b/libcaf_cuda/caf/cuda/helpers.hpp
@@ -63,6 +63,11 @@ in<T> create_in_arg(const std::vector<T>& buffer) {
   return in<T>{buffer};
 }
 
+template <typename T>
+in<T> create_in_out_arg(std::vector<T>&& buffer) {
+    return in<T>{std::move(buffer)}; // moves into variant, no copy
+}
+
 
 //creates an output buffer/ write only buffer on the gpu
 
@@ -91,6 +96,12 @@ in_out<T> create_in_out_arg(const T& val) {
   return in_out<T>{val};
 }
 
+template <typename T>
+in_out<T> create_in_out_arg(std::vector<T>&& buffer) {
+    return in_out<T>{std::move(buffer)}; // moves into variant, no copy
+}
+
+
 template <typename T>
 in_out<T> create_in_out_arg(const std::vector<T>& buffer) {
   return in_out<T>{buffer};

From ac1b60d24e1d1efdc0585767e6dbfe1974f98ff7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 26 Feb 2026 14:31:04 -0600
Subject: [PATCH 0413/1000] Added std::move to in_out factories in actors to
 reduce uneccesary copies in the test to ensure that admals law is not abused
 to skew test results.

---
 .../core_usage_behavior_tests/main.test.cpp       | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
index 7ee728e8a3..eec64bab2e 100644
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
@@ -282,8 +282,8 @@ caf::behavior mmul_actor_fun_no_verify(
 
 				//create program and dims   
 				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrix1);
-				auto arg2 = caf::cuda::create_in_arg(matrix2);
+				auto arg1 = caf::cuda::create_in_arg(std::move(matrix1));
+				auto arg2 = caf::cuda::create_in_arg(std::move(matrix2));
 				auto arg3 = caf::cuda::create_out_arg(N*N);
 				auto arg4 = caf::cuda::create_in_arg(N);
 
@@ -325,8 +325,8 @@ caf::behavior mmul_actor_fun_no_verify(
 
 				//create program and dims   
 				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrixA);
-				auto arg2 = caf::cuda::create_in_arg(matrixB);
+				auto arg1 = caf::cuda::create_in_arg(std::move(matrixA));
+				auto arg2 = caf::cuda::create_in_arg(std::move(matrixB));
 				auto arg3 = caf::cuda::create_out_arg(N*N);
 				auto arg4 = caf::cuda::create_in_arg(N);
 
@@ -377,8 +377,8 @@ caf::behavior mmul_actor_fun_no_schedule(
     	    //std::cout << "Hello\n";
 	    caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-            auto arg1 = caf::cuda::create_in_arg(matrixA);
-            auto arg2 = caf::cuda::create_in_arg(matrixB);
+            auto arg1 = caf::cuda::create_in_arg(std::move(matrixA));
+            auto arg2 = caf::cuda::create_in_arg(std::move(matrixB));
             auto arg3 = caf::cuda::create_out_arg(N_local * N_local);
             auto arg4 = caf::cuda::create_in_arg(N_local);
 
@@ -528,9 +528,6 @@ void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, i
 
     caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-    mgr.send_scheduler_actor_message("green",0);
-
-    // CREATE ONCE
     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
     const int THREADS = 32;

From 07124dd8e471f33fa8ddb04b052828af69e74b23 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 26 Feb 2026 14:48:32 -0600
Subject: [PATCH 0414/1000] made memory_hog actor have big_buffer as a
 parameter. This change was made to prevent putting too much stress on the
 stack, while still being able to ensure that there is enough memory to take
 all of the gpu memory.

---
 .../memory_actor_test/main.test.cpp                    | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
index 2e6613d79c..3109469351 100644
--- a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
@@ -61,8 +61,7 @@ matrixGenCommand randomMatrix;
 mmulAsyncCommand mmulAsync;
 
 
-caf::behavior memory_hog_actor_fun(caf::stateful_actor<memory_hog_actor_state>* self,caf::actor exit_actor,std::size_t bytes) {
- 
+caf::behavior memory_hog_actor_fun(caf::stateful_actor<memory_hog_actor_state>* self,caf::actor exit_actor,std::size_t bytes,std::vector<int> big_buffer) { 
 	self->state().bytes = bytes; 
 	caf::cuda::manager& mgr = caf::cuda::manager::get();
   	caf::actor memory_actor = mgr.get_memory_actor();
@@ -74,9 +73,9 @@ caf::behavior memory_hog_actor_fun(caf::stateful_actor<memory_hog_actor_state>*
 
 	  [=] (caf::cuda::ack msg ) {
 		  caf::cuda::command_runner<> mem_transfer_command;
-		  std::vector<int> big_buffer(bytes/sizeof(int),0);
 
-		  caf::cuda::mem_ptr<int> temp = mem_transfer_command.transfer_memory(0,1,in_out{big_buffer});
+
+		  caf::cuda::mem_ptr<int> temp = mem_transfer_command.transfer_memory(0,1,in_out{std::move(big_buffer)});
 
 		  //hold onto memory for a few seconds
 		  std::cout << "Memory hog holding onto memory for 5 seconds\n";
@@ -102,9 +101,10 @@ void run_memory_hog_test(caf::actor_system& sys, std::size_t bytes, int num_acto
   caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
   // Spawn num_actors actors running the mmul behavior
   std::vector<caf::actor> actors;
+  std::vector<int> big_buffer(bytes/sizeof(int));
   actors.reserve(num_actors);
   for (int i = 0; i < num_actors; i++){
-    actors.push_back(sys.spawn(memory_hog_actor_fun,exit_actor,bytes));
+    actors.push_back(sys.spawn(memory_hog_actor_fun,exit_actor,bytes,big_buffer));
   }
 
    sys.await_all_actors_done();

From dfa90a24450650352111dbaf9a50b44af32c4010 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 26 Feb 2026 14:55:54 -0600
Subject: [PATCH 0415/1000] Made big_buffer global memory to reduce strain on
 the CPU.

---
 .../memory_actor_test/main.test.cpp                   | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
index 3109469351..e853fed4ce 100644
--- a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
@@ -16,6 +16,9 @@
 //#include <caf/atoms.hpp>
 
 
+// outside any function
+std::vector<int> big_buffer;
+
 
 using namespace caf;
 using namespace std::chrono_literals;
@@ -61,7 +64,7 @@ matrixGenCommand randomMatrix;
 mmulAsyncCommand mmulAsync;
 
 
-caf::behavior memory_hog_actor_fun(caf::stateful_actor<memory_hog_actor_state>* self,caf::actor exit_actor,std::size_t bytes,std::vector<int> big_buffer) { 
+caf::behavior memory_hog_actor_fun(caf::stateful_actor<memory_hog_actor_state>* self,caf::actor exit_actor,std::size_t bytes) { 
 	self->state().bytes = bytes; 
 	caf::cuda::manager& mgr = caf::cuda::manager::get();
   	caf::actor memory_actor = mgr.get_memory_actor();
@@ -101,10 +104,10 @@ void run_memory_hog_test(caf::actor_system& sys, std::size_t bytes, int num_acto
   caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
   // Spawn num_actors actors running the mmul behavior
   std::vector<caf::actor> actors;
-  std::vector<int> big_buffer(bytes/sizeof(int));
+  big_buffer.resize(bytes / sizeof(int));
   actors.reserve(num_actors);
   for (int i = 0; i < num_actors; i++){
-    actors.push_back(sys.spawn(memory_hog_actor_fun,exit_actor,bytes,big_buffer));
+    actors.push_back(sys.spawn(memory_hog_actor_fun,exit_actor,bytes));
   }
 
    sys.await_all_actors_done();
@@ -118,7 +121,7 @@ void caf_main(caf::actor_system& sys) {
 
 	caf::cuda::manager_config man_config(false,true); //turns the memory actor on
 	caf::cuda::manager::init(sys,man_config);
-       	run_memory_hog_test(sys,3221225472,8);
+       	run_memory_hog_test(sys,6221225472,8);
 }
 
 

From 754d4eb43a05fb981fe942d303f909e4f5c76507 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 26 Feb 2026 15:33:20 -0600
Subject: [PATCH 0416/1000] initial commit.

---
 .../memory_actor/memory_response_token.hpp    | 59 +++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_response_token.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_response_token.hpp
new file mode 100644
index 0000000000..405bcdedd2
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_response_token.hpp
@@ -0,0 +1,59 @@
+#pragma once
+#include <caf/actor.hpp>
+#include "caf/cuda/control-layer/return_payloads/ack.hpp"
+#include "caf/cuda/global_export.hpp"
+#include <utility>
+
+namespace caf::cuda {
+
+/// RAII-style memory tracker that returns memory to the memory actor
+/// when it goes out of scope.
+class CAF_CUDA_EXPORT memory_response_token {
+public:
+    memory_response_token(std::size_t memory_in_bytes,
+                          int device_number,
+                          caf::actor memory_actor_handle) 
+        : memory_bytes_(memory_in_bytes),
+          device_number_(device_number),
+          memory_actor_(std::move(memory_actor_handle)) {}
+
+    // Move constructor (allow moving, but disable copying)
+    memory_response_token(memory_response_token&& other) noexcept
+        : memory_bytes_(other.memory_bytes_),
+          device_number_(other.device_number_),
+          memory_actor_(std::move(other.memory_actor_)) 
+    {
+        other.memory_bytes_ = 0;  // prevent double return
+    }
+
+    memory_response_token& operator=(memory_response_token&& other) noexcept {
+        if (this != &other) {
+            memory_bytes_ = other.memory_bytes_;
+            device_number_ = other.device_number_;
+            memory_actor_ = std::move(other.memory_actor_);
+            other.memory_bytes_ = 0;  // prevent double return
+        }
+        return *this;
+    }
+
+    // Delete copy constructor/assignment to avoid double returns
+    memory_response_token(const memory_response_token&) = delete;
+    memory_response_token& operator=(const memory_response_token&) = delete;
+
+    ~memory_response_token() {
+        if (memory_bytes_ > 0 && memory_actor_) {
+            // RAII return: tell memory actor we freed memory
+            anon_mail(ack(CAF_CUDA_ACK_MEMORY))
+                .arg(device_number_)
+                .arg(memory_bytes_)
+                .send(memory_actor_);
+        }
+    }
+
+private:
+    std::size_t memory_bytes_;
+    int device_number_;
+    caf::actor memory_actor_;
+};
+
+} // namespace caf::cuda

From d18281137b0886437f809c0756aad87d3a4776bc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 26 Feb 2026 15:41:02 -0600
Subject: [PATCH 0417/1000] Renamed memory_response_token to mem_token since
 name was already taken.

---
 .../control-layer/memory_actor/mem_token.hpp  | 57 +++++++++++++++++++
 .../memory_actor/memory_response_token.hpp    |  2 +-
 2 files changed, 58 insertions(+), 1 deletion(-)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp
new file mode 100644
index 0000000000..e4cca0a2f1
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp
@@ -0,0 +1,57 @@
+#pragma once
+#include <caf/actor.hpp>
+#include "caf/cuda/control-layer/return_payloads/ack.hpp"
+#include "caf/cuda/global_export.hpp"
+#include <utility>
+
+namespace caf::cuda {
+
+/// RAII-style memory tracker that returns memory to the memory actor
+/// when it goes out of scope.
+class CAF_CUDA_EXPORT mem_token {
+public:
+    mem_token(std::size_t memory_in_bytes,
+              int device_number,
+              caf::actor memory_actor_handle) 
+        : memory_bytes_(memory_in_bytes),
+          device_number_(device_number),
+          memory_actor_(std::move(memory_actor_handle)) {}
+
+    // Move constructor (allow moving, but disable copying)
+    mem_token(mem_token&& other) noexcept
+        : memory_bytes_(other.memory_bytes_),
+          device_number_(other.device_number_),
+          memory_actor_(std::move(other.memory_actor_)) 
+    {
+        other.memory_bytes_ = 0;  // prevent double return
+    }
+
+    mem_token& operator=(mem_token&& other) noexcept {
+        if (this != &other) {
+            memory_bytes_ = other.memory_bytes_;
+            device_number_ = other.device_number_;
+            memory_actor_ = std::move(other.memory_actor_);
+            other.memory_bytes_ = 0;  // prevent double return
+        }
+        return *this;
+    }
+
+    // Delete copy constructor/assignment to avoid double returns
+    mem_token(const mem_token&) = delete;
+    mem_token& operator=(const mem_token&) = delete;
+
+    ~mem_token() {
+        if (memory_bytes_ > 0 && memory_actor_) {
+            // RAII return: tell memory actor we freed memory
+            anon_mail(device_number_,memory_bytes_)
+                .send(memory_actor_);
+        }
+    }
+
+private:
+    std::size_t memory_bytes_;
+    int device_number_;
+    caf::actor memory_actor_;
+};
+
+} // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_response_token.hpp
index 405bcdedd2..66f5806530 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_response_token.hpp
@@ -10,7 +10,7 @@ namespace caf::cuda {
 /// when it goes out of scope.
 class CAF_CUDA_EXPORT memory_response_token {
 public:
-    memory_response_token(std::size_t memory_in_bytes,
+    mem_response_token(std::size_t memory_in_bytes,
                           int device_number,
                           caf::actor memory_actor_handle) 
         : memory_bytes_(memory_in_bytes),

From 2b830b5531b5b121c6c0fc91361e7df89d1d44dd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 26 Feb 2026 15:42:06 -0600
Subject: [PATCH 0418/1000] Included mem_token.

---
 .../caf/cuda/control-layer/memory_actor/memory_actor.hpp        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
index 866feda489..4112688882 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_actor.hpp
@@ -5,7 +5,7 @@
 #include "caf/cuda/device.hpp"
 #include "caf/cuda/global_export.hpp"
 #include "caf/cuda/control-layer/memory_actor/memory_request_token.hpp"
-
+#include "caf/cuda/control-layer/memory_actor/mem_token.hpp"
 /*
  * The memory actor 
  * meant to help prevent out of memory errors by employing s/r/r IPC 

From 55ae7a9f1ee0d09b81efdedc96d21241f7029f97 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 26 Feb 2026 17:38:31 -0600
Subject: [PATCH 0419/1000] Moved to mem_token.

---
 .../memory_actor/memory_response_token.hpp    | 59 -------------------
 1 file changed, 59 deletions(-)
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_response_token.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_response_token.hpp
deleted file mode 100644
index 66f5806530..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/memory_actor/memory_response_token.hpp
+++ /dev/null
@@ -1,59 +0,0 @@
-#pragma once
-#include <caf/actor.hpp>
-#include "caf/cuda/control-layer/return_payloads/ack.hpp"
-#include "caf/cuda/global_export.hpp"
-#include <utility>
-
-namespace caf::cuda {
-
-/// RAII-style memory tracker that returns memory to the memory actor
-/// when it goes out of scope.
-class CAF_CUDA_EXPORT memory_response_token {
-public:
-    mem_response_token(std::size_t memory_in_bytes,
-                          int device_number,
-                          caf::actor memory_actor_handle) 
-        : memory_bytes_(memory_in_bytes),
-          device_number_(device_number),
-          memory_actor_(std::move(memory_actor_handle)) {}
-
-    // Move constructor (allow moving, but disable copying)
-    memory_response_token(memory_response_token&& other) noexcept
-        : memory_bytes_(other.memory_bytes_),
-          device_number_(other.device_number_),
-          memory_actor_(std::move(other.memory_actor_)) 
-    {
-        other.memory_bytes_ = 0;  // prevent double return
-    }
-
-    memory_response_token& operator=(memory_response_token&& other) noexcept {
-        if (this != &other) {
-            memory_bytes_ = other.memory_bytes_;
-            device_number_ = other.device_number_;
-            memory_actor_ = std::move(other.memory_actor_);
-            other.memory_bytes_ = 0;  // prevent double return
-        }
-        return *this;
-    }
-
-    // Delete copy constructor/assignment to avoid double returns
-    memory_response_token(const memory_response_token&) = delete;
-    memory_response_token& operator=(const memory_response_token&) = delete;
-
-    ~memory_response_token() {
-        if (memory_bytes_ > 0 && memory_actor_) {
-            // RAII return: tell memory actor we freed memory
-            anon_mail(ack(CAF_CUDA_ACK_MEMORY))
-                .arg(device_number_)
-                .arg(memory_bytes_)
-                .send(memory_actor_);
-        }
-    }
-
-private:
-    std::size_t memory_bytes_;
-    int device_number_;
-    caf::actor memory_actor_;
-};
-
-} // namespace caf::cuda

From 3b54019d5190b8d361b0f54401371ca8c0e43478 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 27 Feb 2026 17:18:09 -0600
Subject: [PATCH 0420/1000] Updated to include mem_token.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 70dd94f9ff..8a9529728f 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -26,6 +26,7 @@
 //memory actor includes
 #include "caf/cuda/control-layer/memory_actor/memory_actor.hpp"
 #include "caf/cuda/control-layer/memory_actor/memory_request_token.hpp"
+#include "caf/cuda/control-layer/memory_actor/mem_token.hpp"
 
 
 // -----------------------------------------------------------------------------
@@ -42,6 +43,7 @@ CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::response_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_transfer_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::memory_response_token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::memory_request_token))
+CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::mem_token))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::transfer_token>))
 CAF_ADD_TYPE_ID(cuda_control, (std::vector<caf::intrusive_ptr<caf::cuda::token>>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::kernel_graph))
@@ -61,6 +63,7 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::launch_response_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_transfer_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_request_token)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::memory_response_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::response_token)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::transfer_token)

From f52fdd0e45e9199e6ee8f418f49d6d2ab5619add Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 27 Feb 2026 17:22:28 -0600
Subject: [PATCH 0421/1000] Updated constructors to be caf compliant.

---
 .../control-layer/memory_actor/mem_token.hpp  | 27 +++++--------------
 1 file changed, 7 insertions(+), 20 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp
index e4cca0a2f1..b44d1d7301 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp
@@ -10,35 +10,22 @@ namespace caf::cuda {
 /// when it goes out of scope.
 class CAF_CUDA_EXPORT mem_token {
 public:
-    mem_token(std::size_t memory_in_bytes,
+
+   //for caf id system
+    mem_token() = default;
+    
+	mem_token(std::size_t memory_in_bytes,
               int device_number,
               caf::actor memory_actor_handle) 
         : memory_bytes_(memory_in_bytes),
           device_number_(device_number),
           memory_actor_(std::move(memory_actor_handle)) {}
 
-    // Move constructor (allow moving, but disable copying)
-    mem_token(mem_token&& other) noexcept
-        : memory_bytes_(other.memory_bytes_),
-          device_number_(other.device_number_),
-          memory_actor_(std::move(other.memory_actor_)) 
-    {
-        other.memory_bytes_ = 0;  // prevent double return
-    }
 
-    mem_token& operator=(mem_token&& other) noexcept {
-        if (this != &other) {
-            memory_bytes_ = other.memory_bytes_;
-            device_number_ = other.device_number_;
-            memory_actor_ = std::move(other.memory_actor_);
-            other.memory_bytes_ = 0;  // prevent double return
-        }
-        return *this;
-    }
 
     // Delete copy constructor/assignment to avoid double returns
-    mem_token(const mem_token&) = delete;
-    mem_token& operator=(const mem_token&) = delete;
+    //mem_token(const mem_token&) = delete;
+    //mem_token& operator=(const mem_token&) = delete;
 
     ~mem_token() {
         if (memory_bytes_ > 0 && memory_actor_) {

From f840b9910387cc6279a8fa7e29783f75aa749edc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 27 Feb 2026 17:27:07 -0600
Subject: [PATCH 0422/1000] Updated memory_actor to be no longer be on a timer
 but wait for replies since this is a better solution.

---
 .../memory_actor/memory_actor.cpp             | 188 ++++++++----------
 1 file changed, 78 insertions(+), 110 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
index 6dfff3f7b5..11c897bda7 100644
--- a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
+++ b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
@@ -1,119 +1,87 @@
 #include "caf/cuda/control-layer/all-control-layer.hpp"
 #include "caf/cuda/manager.hpp"
 #include "caf/cuda/control-layer/return_payloads/ack.hpp"
-#include <string>
+#include "caf/cuda/control-layer/memory_actor/memory_request_token.hpp"
+#include "caf/cuda/control-layer/memory_actor/mem_token.hpp"
 #include <iostream>
 
-/*
- * This class is meant to handle memory managment/coordination  via s/r/r IPC
- * it has nothing to do with the mem_ptr<T> (yet), that is kernel layer
- */
 namespace caf::cuda {
 
-caf::behavior memory_actor(caf::stateful_actor<memory_actor_state>* self, int num_devices) {
-
-	self -> state().num_devices = num_devices;
-
-	//initialize data structures 
-
-	for (int i = 0; i < num_devices; i++) {
-		std::queue<memory_request_token> r;
-		self->state().requests.emplace_back(std::move(r));
-		self->state().devices.emplace_back(manager::get().find_device(i));
-		// Initialize internal memory counter
-		self->state().available_memory.push_back(
-				manager::get().find_device(i)->available_memory_bytes()
-				);
-	}
-
-
-	return {
-		[self](const memory_request_token& token) {
-			int device_number = token.getDeviceNumber();
-			
-			auto& free_memory =
-				self->state().available_memory[device_number];
-
-			if (free_memory >= token.getSize()) {
-				free_memory -= token.getSize();
-				self->mail(ack(CAF_CUDA_ACK_MEMORY)).send(token.getReplyActor());
-			}
-			else {
-				self->state().requests[device_number].push(std::move(token));
-				if (!self->state().pending_requests) {
-					//will begin a timer to check if memory is free
-					anon_mail(ack(TIMER)).delay(std::chrono::seconds(1)).send(self);	
-					self -> state().pending_requests = true;
-				}
-				//otherwise it will awake and check at a later time
-			}
-
-
-		},
-
-			//typically here to peridocially check if memory is free
-			[self](ack message) {
-
-				if (message.getType() != TIMER)
-					return;
-
-				bool still_pending = false;
-
-
-
-				//  First: synchronize internal counters with real device
-				for (int i = 0; i < self->state().num_devices; i++) {
-
-					if (!self->state().requests[i].empty()) {
-						self->state().available_memory[i] =
-							self->state().devices[i]->available_memory_bytes();
-					}
-				}
-
-				// Then process queues
-				for (int i = 0; i < self->state().num_devices; i++) {
-
-					auto& q = self->state().requests[i];
-
-					if (q.empty())
-						continue;
-
-					int device_number = i;
-					std::size_t free_memory = self->state().devices[device_number]
-								->available_memory_bytes();
-					// Traverse queue until:
-					//  - empty
-					//  - or first unsatisfied request
-					while (!q.empty()) {
-						auto token = std::move(q.front());
-						if (free_memory >= token.getSize()) {
-							free_memory -= token.getSize();	
-							self->mail(ack(CAF_CUDA_ACK_MEMORY)).send(token.getReplyActor());
-							q.pop();	
-				 		}
-						else {
-							// cannot satisfy head → stop processing this queue
-							break;
-						}
-					}
-
-					if (!q.empty())
-						still_pending = true;
-				}
-
-				if (still_pending) {
-					anon_mail(ack(TIMER))
-						.delay(std::chrono::seconds(1))
-						.send(self);
-					self->state().pending_requests = true;
-				} else {
-					self->state().pending_requests = false;
-				}
-			} //end of lambda
-
-
-
-	};
+caf::behavior memory_actor(caf::stateful_actor<memory_actor_state>* self,
+                           int num_devices) {
+
+    // initialize
+    self->state().num_devices = num_devices;
+    self->state().requests.clear();
+    self->state().devices.clear();
+    self->state().available_memory.clear();
+    self->state().requests.resize(num_devices);
+    self->state().devices.resize(num_devices);
+    self->state().available_memory.resize(num_devices);
+
+    for (int i = 0; i < num_devices; ++i) {
+        auto dev = manager::get().find_device(i);
+        self->state().devices[i] = dev;
+        // initialize internal counter to what device reports at startup
+        self->state().available_memory[i] =
+            dev ? dev->available_memory_bytes() : 0;
+    }
+
+    // Handlers:
+    return {
+        // 1) memory request handler
+        [self](const memory_request_token& token) {
+            int device_number = token.getDeviceNumber();
+            std::size_t req_bytes = token.getSize();
+
+            // basic sanity checks
+            if (device_number < 0 || device_number >= self->state().num_devices) {
+                std::cerr << "[memory_actor] invalid device_number " << device_number << "\n";
+                return;
+            }
+
+            auto &avail = self->state().available_memory[device_number];
+
+            if (avail >= req_bytes) {
+                // grant immediately: reserve in internal counter and notify requester
+                avail -= req_bytes;
+                // keep behavior: send ack to the reply actor so it can construct mem_token
+                self->mail(std::move(mem_token(req_bytes,device_number,self))).send(token.getReplyActor());
+            } else {
+                // queue request (no polling). It will be reconsidered when a release arrives.
+                self->state().requests[device_number].push(std::move(token));
+            }
+        },
+
+        // 2) release handler: anonymous message (int device, std::size_t bytes)
+        //    This is the message produced by mem_token's destructor via anon_send.
+        [self](int device_number, std::size_t bytes_released) {
+            if (device_number < 0 || device_number >= self->state().num_devices) {
+                std::cerr << "[memory_actor] release: invalid device_number " << device_number << "\n";
+                return;
+            }
+
+            // add released memory back to internal counter
+            self->state().available_memory[device_number] += bytes_released;
+
+            // try to satisfy queued requests for this device
+            auto &q = self->state().requests[device_number];
+            while (!q.empty()) {
+                auto &front = q.front();
+                std::size_t need = front.getSize();
+
+                if (self->state().available_memory[device_number] >= need) {
+                    // reserve and grant
+                    self->state().available_memory[device_number] -= need;
+                    self->mail(ack(CAF_CUDA_ACK_MEMORY)).send(front.getReplyActor());
+                    q.pop();
+                } else {
+                    // still can't satisfy head request
+                    break;
+                }
+            }
+        }
+    };
 }
 
-} // namespace caf::cuda
+} // namespace caf::cuda // namespace caf::cuda

From 4e81b7783e2d5721f848700d012cd9e33abe2b9c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 27 Feb 2026 17:29:47 -0600
Subject: [PATCH 0423/1000] Updated memory_hog_actor handle to be of mem_token
 to corrospond to recent changes made to memory_actor.

---
 .../memory_actor_test/main.test.cpp           | 74 ++++++++++++-------
 1 file changed, 47 insertions(+), 27 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
index e853fed4ce..ebba3e6fdc 100644
--- a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
@@ -64,36 +64,56 @@ matrixGenCommand randomMatrix;
 mmulAsyncCommand mmulAsync;
 
 
-caf::behavior memory_hog_actor_fun(caf::stateful_actor<memory_hog_actor_state>* self,caf::actor exit_actor,std::size_t bytes) { 
-	self->state().bytes = bytes; 
-	caf::cuda::manager& mgr = caf::cuda::manager::get();
-  	caf::actor memory_actor = mgr.get_memory_actor();
-	//send a memory request token
-	caf::cuda::memory_request_token request(bytes,0,self);
-	self ->mail(request).send(memory_actor);
 
-	return {
-
-	  [=] (caf::cuda::ack msg ) {
-		  caf::cuda::command_runner<> mem_transfer_command;
-
-
-		  caf::cuda::mem_ptr<int> temp = mem_transfer_command.transfer_memory(0,1,in_out{std::move(big_buffer)});
-
-		  //hold onto memory for a few seconds
-		  std::cout << "Memory hog holding onto memory for 5 seconds\n";
-		  std::this_thread::sleep_for(std::chrono::seconds(5));
-		  std::cout << "Memory hog releasing memory\n";
-		  
-		  self->mail(1).send(exit_actor);
-		  self->quit();
-	  },
-
-  };
+caf::behavior memory_hog_actor_fun(caf::stateful_actor<memory_hog_actor_state>* self,
+                                   caf::actor exit_actor,
+                                   std::size_t bytes) { 
+    self->state().bytes = bytes; 
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+    caf::actor memory_actor = mgr.get_memory_actor();
+
+    // send a memory request token
+    caf::cuda::memory_request_token request(bytes, 0, self);
+    self->mail(request).send(memory_actor);
+
+    std::cout << "Booting\n";
+    return {
+        [=](caf::cuda::ack mem_token) {
+            caf::cuda::command_runner<> mem_transfer_command;
+
+            try {
+                std::cout << "Grabbing onto memory\n";
+
+                // use global big_buffer directly
+                caf::cuda::mem_ptr<int> temp = 
+                    mem_transfer_command.transfer_memory(0, 1, in_out<int>{big_buffer});
+
+                std::cout << "Memory transfer successful!\n";
+            }
+            catch (const std::runtime_error& e) {
+                std::cerr << "[ERROR] Runtime exception during memory transfer: " << e.what() << "\n";
+            }
+            catch (const std::bad_alloc& e) {
+                std::cerr << "[ERROR] Allocation failed during memory transfer: " << e.what() << "\n";
+            }
+            catch (const std::exception& e) {
+                std::cerr << "[ERROR] Exception during memory transfer: " << e.what() << "\n";
+            }
+            catch (...) {
+                std::cerr << "[ERROR] Unknown exception during memory transfer\n";
+            }
+
+            // hold onto memory for a few seconds
+            std::cout << "Memory hog holding onto memory for 5 seconds\n";
+            std::this_thread::sleep_for(std::chrono::seconds(5));
+            std::cout << "Memory hog releasing memory\n";
+
+            self->mail(1).send(exit_actor);
+            self->quit();
+        },
+    };
 }
 
-
-
 void run_memory_hog_test(caf::actor_system& sys, std::size_t bytes, int num_actors) {
   if (num_actors < 1) {
     std::cerr << "[ERROR] Number of actors must be >= 1\n";

From 35cb472ea8710a986aeae190d007cfaa532e1395 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 27 Feb 2026 17:30:41 -0600
Subject: [PATCH 0424/1000] Updated memory_hog_actor handle to be of mem_token
 to corrospond to recent changes made to memory_actor.

---
 .../tests/control-layer-tests/memory_actor_test/main.test.cpp   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
index ebba3e6fdc..ff7a173418 100644
--- a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
@@ -78,7 +78,7 @@ caf::behavior memory_hog_actor_fun(caf::stateful_actor<memory_hog_actor_state>*
 
     std::cout << "Booting\n";
     return {
-        [=](caf::cuda::ack mem_token) {
+        [=](caf::cuda::mem_token msg) {
             caf::cuda::command_runner<> mem_transfer_command;
 
             try {

From f1f953c02a475b5bf6782efc6bf7940300388656 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 27 Feb 2026 17:38:55 -0600
Subject: [PATCH 0425/1000] Updated copy and move constructors to ensure
 duplicate return messages  are not being sent.

---
 .../control-layer/memory_actor/mem_token.hpp  | 66 ++++++++++++++-----
 1 file changed, 51 insertions(+), 15 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp
index b44d1d7301..694a0b1322 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp
@@ -10,34 +10,70 @@ namespace caf::cuda {
 /// when it goes out of scope.
 class CAF_CUDA_EXPORT mem_token {
 public:
-
-   //for caf id system
     mem_token() = default;
-    
-	mem_token(std::size_t memory_in_bytes,
+
+    mem_token(std::size_t memory_in_bytes,
               int device_number,
-              caf::actor memory_actor_handle) 
+              caf::actor memory_actor_handle)
         : memory_bytes_(memory_in_bytes),
           device_number_(device_number),
           memory_actor_(std::move(memory_actor_handle)) {}
 
+    // Copy constructor: transfer ownership
+    mem_token(const mem_token& other)
+        : memory_bytes_(other.memory_bytes_),
+          device_number_(other.device_number_),
+          memory_actor_(other.memory_actor_) 
+    {
+        const_cast<mem_token&>(other).memory_bytes_ = 0;
+    }
+
+    // Copy assignment: transfer ownership
+    mem_token& operator=(const mem_token& other) {
+        if (this != &other) {
+            // First, return any existing memory
+            release();
+            memory_bytes_ = other.memory_bytes_;
+            device_number_ = other.device_number_;
+            memory_actor_ = other.memory_actor_;
+            const_cast<mem_token&>(other).memory_bytes_ = 0;
+        }
+        return *this;
+    }
+
+    // Move constructor
+    mem_token(mem_token&& other) noexcept
+        : memory_bytes_(other.memory_bytes_),
+          device_number_(other.device_number_),
+          memory_actor_(std::move(other.memory_actor_))
+    {
+        other.memory_bytes_ = 0;
+    }
 
+    // Move assignment
+    mem_token& operator=(mem_token&& other) noexcept {
+        if (this != &other) {
+            release();
+            memory_bytes_ = other.memory_bytes_;
+            device_number_ = other.device_number_;
+            memory_actor_ = std::move(other.memory_actor_);
+            other.memory_bytes_ = 0;
+        }
+        return *this;
+    }
 
-    // Delete copy constructor/assignment to avoid double returns
-    //mem_token(const mem_token&) = delete;
-    //mem_token& operator=(const mem_token&) = delete;
+    ~mem_token() { release(); }
 
-    ~mem_token() {
+private:
+    void release() {
         if (memory_bytes_ > 0 && memory_actor_) {
-            // RAII return: tell memory actor we freed memory
-            anon_mail(device_number_,memory_bytes_)
-                .send(memory_actor_);
+            anon_mail(device_number_, memory_bytes_).send(memory_actor_);
+            memory_bytes_ = 0;
         }
     }
 
-private:
-    std::size_t memory_bytes_;
-    int device_number_;
+    std::size_t memory_bytes_ = 0;
+    int device_number_ = -1;
     caf::actor memory_actor_;
 };
 

From 85758802e56a6445d0a3d2850eb9606a6610cbb9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 27 Feb 2026 17:42:28 -0600
Subject: [PATCH 0426/1000] Changed message type being sent on line 76 from ack
 to mem_token. This change was made to fix an error where a message other than
 the message passing interface was being sent out, causing undefined behavior.

---
 libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
index 11c897bda7..bdd7ff80c3 100644
--- a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
+++ b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
@@ -73,7 +73,7 @@ caf::behavior memory_actor(caf::stateful_actor<memory_actor_state>* self,
                 if (self->state().available_memory[device_number] >= need) {
                     // reserve and grant
                     self->state().available_memory[device_number] -= need;
-                    self->mail(ack(CAF_CUDA_ACK_MEMORY)).send(front.getReplyActor());
+                    self->mail(std::move(mem_token(need,device_number,self))).send(front.getReplyActor());
                     q.pop();
                 } else {
                     // still can't satisfy head request

From 97514c6bc2a3ca58beb24b41268cc4a3f8f0574c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 27 Feb 2026 17:48:41 -0600
Subject: [PATCH 0427/1000] initial commit.

---
 .../caf/cuda/control-layer/exit_actor/exit_actor.hpp      | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/exit_actor/exit_actor.hpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/exit_actor/exit_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/exit_actor/exit_actor.hpp
new file mode 100644
index 0000000000..2ec0692eee
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/exit_actor/exit_actor.hpp
@@ -0,0 +1,8 @@
+#pragma once
+#include <caf/all.hpp>
+
+struct exit_actor_state {
+        int completed = 0;
+};
+
+caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit); 

From 5b05c06f3ffaf84e54e222c8a5ef2d5fd7da028a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 27 Feb 2026 17:50:12 -0600
Subject: [PATCH 0428/1000] put exit_actor in correct namespace.

---
 libcaf_cuda/caf/cuda/control-layer/exit_actor/exit_actor.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/exit_actor/exit_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/exit_actor/exit_actor.hpp
index 2ec0692eee..15ff353a38 100644
--- a/libcaf_cuda/caf/cuda/control-layer/exit_actor/exit_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/exit_actor/exit_actor.hpp
@@ -1,8 +1,10 @@
 #pragma once
 #include <caf/all.hpp>
 
+namespace caf::cuda {
 struct exit_actor_state {
         int completed = 0;
 };
 
 caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit); 
+} //namespace caf::cuda

From fcb7505881b095ae973068ac9ed7cd5c9dd5db6d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 27 Feb 2026 17:53:32 -0600
Subject: [PATCH 0429/1000] Updated to include exit actor include.

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 8a9529728f..41977ebf0f 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -28,6 +28,9 @@
 #include "caf/cuda/control-layer/memory_actor/memory_request_token.hpp"
 #include "caf/cuda/control-layer/memory_actor/mem_token.hpp"
 
+//exit actor includes
+#include "caf/cuda/control-layer/exit_actor/exit_actor.hpp"
+
 
 // -----------------------------------------------------------------------------
 // Type IDs (required for typed behaviors)

From beb9e08661110f98b3df7f50fb6c29c893a692ce Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 27 Feb 2026 17:53:44 -0600
Subject: [PATCH 0430/1000] Initial commit.

---
 .../control-layer/exit_actor/exit_actor.cpp   | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 libcaf_cuda/src/control-layer/exit_actor/exit_actor.cpp

diff --git a/libcaf_cuda/src/control-layer/exit_actor/exit_actor.cpp b/libcaf_cuda/src/control-layer/exit_actor/exit_actor.cpp
new file mode 100644
index 0000000000..6c3f49db7b
--- /dev/null
+++ b/libcaf_cuda/src/control-layer/exit_actor/exit_actor.cpp
@@ -0,0 +1,24 @@
+#include "caf/cuda/control-layer/all-control-layer.hpp"
+#include "caf/cuda/manager.hpp"
+
+namespace caf::cuda {
+caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
+
+
+        return {
+                [=](int num_completed) {
+                        self->state().completed += num_completed;
+
+                        std::cout << "Actors finished is " << self->state().completed << "\n";
+                        if (self->state().completed >= limit) {
+
+                                caf::cuda::manager::shutdown();
+                                self->quit();
+                        }
+                }
+        };
+
+
+}
+} //namespace caf::cuda
+

From af2889867744907824a2a83a8dfcf635f6441cc4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 27 Feb 2026 18:00:55 -0600
Subject: [PATCH 0431/1000] Implemented spawn_exit_actor method.

---
 libcaf_cuda/caf/cuda/manager.hpp | 2 ++
 libcaf_cuda/src/manager.cpp      | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index 7fba0006f8..4256bb7334 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -191,6 +191,8 @@ class CAF_CUDA_EXPORT manager {
 
   caf::actor get_memory_actor();
 
+  caf::actor spawn_exit_actor(int num_actors);
+
 private:
   explicit manager(caf::actor_system& sys)
     : system_(sys), platform_(platform::create()) {
diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 28e33d25b3..d258be8677 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -402,5 +402,11 @@ caf::actor manager::get_memory_actor() {
     return instance_->memory_actor_handle;
 }
 
+caf::actor manager::spawn_exit_actor(int num_actors) {
+
+	return system_.spawn(exit_actor_fun,num_actors);
+
+}
+
 
 } // namespace caf::cuda

From 3318735fe97c3d0373915f8ec8d206fa5cc4f2fb Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 27 Feb 2026 18:05:20 -0600
Subject: [PATCH 0432/1000] Integrated caf::cuda exit actor and replaced it
 with exit_actor in the test. Change is being made that since exit_actor is so
 frequent it makes sense to integrate it into the framework.

---
 .../memory_actor_test/main.test.cpp           | 26 ++-----------------
 1 file changed, 2 insertions(+), 24 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
index ff7a173418..bce3d93aa4 100644
--- a/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/memory_actor_test/main.test.cpp
@@ -24,29 +24,6 @@ using namespace caf;
 using namespace std::chrono_literals;
 
 
-struct exit_actor_state {
-	int completed = 0;
-};
-
-
-caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
-
-
-	return {
-		[=](int num_completed) {
-			self->state().completed += num_completed;
-			
-			std::cout << "Actors finished is " << self->state().completed << "\n";
-			if (self->state().completed >= limit) {
-			
-				caf::cuda::manager::shutdown();
-				self->quit();
-			}
-		}
-	};
-
-
-}
 
 
 struct memory_hog_actor_state {
@@ -120,8 +97,9 @@ void run_memory_hog_test(caf::actor_system& sys, std::size_t bytes, int num_acto
     return;
   }
 
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
+  caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
   // Spawn num_actors actors running the mmul behavior
   std::vector<caf::actor> actors;
   big_buffer.resize(bytes / sizeof(int));

From 272ed444c649d90413edd699bbc848c246a70ff0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 09:41:44 -0600
Subject: [PATCH 0433/1000] iniital commit.

---
 libcaf_cuda/caf/component-actors/all-component-actors.hpp | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 libcaf_cuda/caf/component-actors/all-component-actors.hpp

diff --git a/libcaf_cuda/caf/component-actors/all-component-actors.hpp b/libcaf_cuda/caf/component-actors/all-component-actors.hpp
new file mode 100644
index 0000000000..6f70f09bee
--- /dev/null
+++ b/libcaf_cuda/caf/component-actors/all-component-actors.hpp
@@ -0,0 +1 @@
+#pragma once

From 59652d9b5845bed58a08a0d070933a141fb6ddac Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 10:16:05 -0600
Subject: [PATCH 0434/1000] Initial commit.

---
 .../mmul_actor/mmul_actor.hpp                 | 85 +++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp

diff --git a/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp b/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp
new file mode 100644
index 0000000000..4c30c9df35
--- /dev/null
+++ b/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp
@@ -0,0 +1,85 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include "caf/cuda/all.hpp"
+
+/*
+ * An actor meant to represent matrix multiply 
+ * kernel for now must be supplied since we do not yet support kernel compilation
+ * auto integration, it is assumed you will supply a matrix multiple kernel to this
+ * actor
+ * it takes in 2 mem_ptrs to the representing matrix A and matrix B,matrix size, 
+ * device number, and stream id  
+ * and replys with the result 
+ */
+
+namespace caf::cuda {
+
+struct mmul_actor_state {
+  static inline const char* name = "mmul_actor";
+  program_ptr mmul_kernel;
+};
+
+template <class T>
+using mmul_async_command =
+  command_runner<
+    mem_ptr<T>,
+    mem_ptr<T>,
+    out<T>,
+    in<T>
+  >;
+
+template <class T>
+caf::behavior mmul_async_actor_fun(
+    caf::stateful_actor<mmul_actor_state>* self,
+    program_ptr mmul_kernel)
+{
+  using mem_t = mem_ptr<T>;
+  using runner_t = mmul_async_command<T>;
+
+  self->state().mmul_kernel = mmul_kernel;
+
+  runner_t mmul;
+
+  return {
+
+    [=](mem_t matrixA,
+        mem_t matrixB,
+        int N,
+        int device_number,
+        int stream_id)
+        -> mem_t
+    {
+      program_ptr kernel = self->state().mmul_kernel;
+
+      const int THREADS = 32;
+      const int BLOCKS = (N + THREADS - 1) / THREADS;
+
+      nd_range dims(
+        BLOCKS, BLOCKS, 1,
+        THREADS, THREADS, 1);
+
+      out<T> arg3 = create_out_arg<T>(N * N);
+      in<T>  arg4 = create_in_arg<T>(static_cast<T>(N));
+
+      std::tuple<mem_t, mem_t, mem_t, mem_t> result_tuple =
+        mmul.run_async(
+          kernel,
+          dims,
+          stream_id,
+          0,
+          device_number,
+          matrixA,
+          matrixB,
+          arg3,
+          arg4);
+
+      mem_t output_device_buffer = std::get<2>(result_tuple);
+
+      return output_device_buffer;
+    }
+
+  };
+}
+
+} // namespace caf::cuda

From ecc5c900ac46f2587df27c1ff0cada82c6f15295 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 10:36:45 -0600
Subject: [PATCH 0435/1000] Changed name from mmul_async_actor_fun to
 mmul_actor_fun

---
 libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp b/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp
index 4c30c9df35..06c8527462 100644
--- a/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp
+++ b/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp
@@ -30,7 +30,7 @@ using mmul_async_command =
   >;
 
 template <class T>
-caf::behavior mmul_async_actor_fun(
+caf::behavior mmul_actor_fun(
     caf::stateful_actor<mmul_actor_state>* self,
     program_ptr mmul_kernel)
 {

From aeba6ec0619331a0a7df21b8e96bed88bf13b2bc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 10:41:54 -0600
Subject: [PATCH 0436/1000] Initial commit.

---
 .../mmul-actor-test/main.test.cpp             | 173 ++++++++++++++++++
 1 file changed, 173 insertions(+)
 create mode 100644 libcaf_cuda/tests/component-actors-test/mmul-actor-test/main.test.cpp

diff --git a/libcaf_cuda/tests/component-actors-test/mmul-actor-test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/main.test.cpp
new file mode 100644
index 0000000000..4c45af130f
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/main.test.cpp
@@ -0,0 +1,173 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/cuda/component-actors/mmul_actor/mmul_actor.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
+  return {
+
+	
+	  // 1st handler matrices +  N, launches a kenrel and sends its result to itself for verification
+	  [=](const std::vector<int> matrixA,
+			  const std::vector<int> matrixB, int N) {
+
+
+		  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+		  //create program and dims   
+		  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+		  //create args
+		  auto arg1 = caf::cuda::create_in_arg(matrixA);
+		  auto arg2 = caf::cuda::create_in_arg(matrixB);
+
+		  caf::actor mmul_actor = self -> spawn(caf::cuda::mmul_actor_fun<int>,program);
+
+		  int device = 0;
+		  int stream = 1;
+
+		  self->request(mmul_actor,
+				  std::chrono::seconds(10),
+				  arg1,
+				  arg2,
+				  N,
+				  device,
+				  stream)
+			  .then(
+					  [&](caf::cuda::mem_ptr<int> dC) {
+
+					  std::vector<int> matrixC =
+					  dC->copy_to_host();
+					  std::vector<int> result(N*N);
+
+					  serial_matrix_multiply(matrixA,matrixB,result,N);
+
+					  if (result == matrixC) {
+
+					  std::cout << "actor matrix references match\n";
+
+					  }
+
+					  else {
+					  std::cout << "actor matrix  references did not match\n";
+					  }
+
+					  self-> quit();
+
+					  }
+	  });
+
+
+}
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  // ------------------------------------
+  // Start timing
+  // ------------------------------------
+  auto start = std::chrono::steady_clock::now();
+
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+ 
+ using clock = std::chrono::steady_clock;
+
+auto t_start = clock::now();
+
+for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_actor_fun));
+}
+
+auto t_end = clock::now();
+
+auto elapsed_ms =
+    std::chrono::duration_cast<std::chrono::milliseconds>(
+        t_end - t_start).count();
+
+std::cout << "[SPAWN] spawned "
+          << num_actors
+          << " actors in "
+          << elapsed_ms
+          << " ms\n";
+ 
+ 
+ 
+  // Actor 0 generates matrices and broadcasts to others
+  caf::anon_mail(matrix_size, actors).send(actors[0]);
+
+  // Wait for all actors to finish
+  sys.await_all_actors_done();
+
+  // ------------------------------------
+  // Stop timing
+  // ------------------------------------
+  auto end = std::chrono::steady_clock::now();
+  auto duration_ms =
+      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
+            << ", actors=" << num_actors
+            << ", time=" << duration_ms << " ms\n";
+}
+
+void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager::init(sys);
+
+  run_mmul_test(sys,512,512);
+  //run_async_mmul_test(sys,100,1);
+  //run_async_mmul_perf_test(sys,1024,200);
+
+  // run the async (no-shared) suite:
+  //benchmark_async_perf_all(sys);
+
+  // run the shared-memory suite:
+  //benchmark_shared_perf_all(sys);
+
+  //run_mmul_scaling_tests(sys);
+}
+
+
+
+
+CAF_MAIN()

From 57f5c512802cf4e461e2007cbb2c97f72e4f522e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 10:43:04 -0600
Subject: [PATCH 0437/1000] Initial commit.

---
 .../mmul-actor-test/CMakeLists.txt            |  44 ++++++
 .../mmul-actor-test/compile_kernels.sh        |  13 ++
 .../mmul-actor-test/mmul-actors.cpp           | 125 ++++++++++++++++++
 .../mmul-actor-test/mmul.cu                   |  16 +++
 4 files changed, 198 insertions(+)
 create mode 100644 libcaf_cuda/tests/component-actors-test/mmul-actor-test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/component-actors-test/mmul-actor-test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/component-actors-test/mmul-actor-test/mmul-actors.cpp
 create mode 100644 libcaf_cuda/tests/component-actors-test/mmul-actor-test/mmul.cu

diff --git a/libcaf_cuda/tests/component-actors-test/mmul-actor-test/CMakeLists.txt b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..12886f5340
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/component-actors-test/mmul-actor-test/compile_kernels.sh b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/compile_kernels.sh
new file mode 100755
index 0000000000..f32480e5cb
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/compile_kernels.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/component-actors-test/mmul-actor-test/mmul-actors.cpp b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/mmul-actors.cpp
new file mode 100644
index 0000000000..9865fb5da4
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/mmul-actors.cpp
@@ -0,0 +1,125 @@
+//file not in use since I am not wasting time messing around with cmake
+
+
+#include "main.test.hpp"
+
+const char* matrixMulKernel = R"(
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+)";
+
+
+
+// Check result on the CPU
+void verify_result(vector<int> &a, vector<int> &b, vector<int> &c, int N) {
+  // For every row...
+  for (int i = 0; i < N; i++) {
+    // For every column...
+    for (int j = 0; j < N; j++) {
+      // For every element in the row-column pair
+      int tmp = 0;
+      for (int k = 0; k < N; k++) {
+        // Accumulate the partial results
+        tmp += a[i * N + k] * b[k * N + j];
+      }
+
+      // Check against the CPU result
+      assert(tmp == c[i * N + j]);
+    }
+  }
+}
+void test_mmul(caf::actor_system& sys) {
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  // Matrix dimension (N x N)
+  int N = 1024;
+  int THREADS = 32;
+  int BLOCKS = N / THREADS;
+
+  // Setup kernel launch configuration
+  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+  // Spawn CUDA actor for matrix multiplication kernel
+  auto gpuActor = mgr.spawn(matrixMulKernel, "matrixMul", dim,
+                            in<int>{}, in<int>{}, out<int>{}, in<int>{});
+
+  // Allocate and initialize host matrices
+  std::vector<int> h_a(N * N);
+  std::vector<int> h_b(N * N);
+  std::vector<int> h_c(N * N, 0);  // initialized to 0
+  std::vector<int> h_n(1, N);
+
+  std::generate(h_a.begin(), h_a.end(), []() { return rand() % 10; });
+  std::generate(h_b.begin(), h_b.end(), []() { return rand() % 10; });
+
+  // Compose device arguments
+  auto arg1 = caf::cuda::create_in_arg(h_a);
+  auto arg2 = caf::cuda::create_in_arg(h_b);
+  auto arg3 = caf::cuda::create_out_arg(h_c);
+  auto arg4 = caf::cuda::create_in_arg(h_n);
+
+  // Spawn an actor to send the message and receive the result
+  sys.spawn([=](caf::event_based_actor* self_actor) {
+    self_actor->mail(gpuActor, arg1, arg2, arg3, arg4)
+      .request(gpuActor, 30s).then(
+        [=](const std::vector<output_buffer>& outputs) {
+          std::vector<int> result;
+          bool got_output = false;
+
+          // Extract the result from outputs
+          for (const auto& out : outputs) {
+            std::visit([&](const auto& vec) {
+              if constexpr (std::is_same_v<std::decay_t<decltype(vec)>, std::vector<int>>) {
+                result = vec;
+                got_output = true;
+              }
+            }, out.data);
+          }
+
+          if (!got_output) {
+            aout(self_actor) << "No output data received!\n";
+          } else {
+            aout(self_actor) << "Verifying result..." << std::endl;
+
+            // Verify GPU result against CPU computation
+            std::vector<int> expected(N * N);
+            for (int i = 0; i < N; ++i) {
+              for (int j = 0; j < N; ++j) {
+                int tmp = 0;
+                for (int k = 0; k < N; ++k) {
+                  tmp += h_a[i * N + k] * h_b[k * N + j];
+                }
+                expected[i * N + j] = tmp;
+              }
+            }
+
+            bool success = std::equal(result.begin(), result.end(), expected.begin());
+            if (success) {
+              aout(self_actor) << "Matrix multiplication result verified successfully!\n";
+            } else {
+              aout(self_actor) << "Mismatch found in matrix multiplication results!\n";
+            }
+          }
+
+          self_actor->quit();
+        }
+      );
+  });
+
+  std::this_thread::sleep_for(std::chrono::seconds(5)); // Wait for actor to complete
+}
+
+
+
+
diff --git a/libcaf_cuda/tests/component-actors-test/mmul-actor-test/mmul.cu b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From 50d5fda17367d3089fb0f1a92c7d99a16ae1158c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 10:45:29 -0600
Subject: [PATCH 0438/1000] Updated cmakelists to point to the correct path.

---
 .../tests/component-actors-test/mmul-actor-test/CMakeLists.txt  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/component-actors-test/mmul-actor-test/CMakeLists.txt b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/CMakeLists.txt
index 12886f5340..89bcf5ba7c 100644
--- a/libcaf_cuda/tests/component-actors-test/mmul-actor-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/CMakeLists.txt
@@ -6,7 +6,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS      OFF)
 
 # 2) Set CAF source and build directories
-set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
 set(CAF_BUILD "${CAF_SRC}/build")
 
 

From 1ccb3f2de068d16b34c863a6bf6a4c65c834c8a8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 10:49:48 -0600
Subject: [PATCH 0439/1000] Fixed include path to mmul_actor.

---
 .../component-actors-test/mmul-actor-test/main.test.cpp     | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/component-actors-test/mmul-actor-test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/main.test.cpp
index 4c45af130f..590d4a4f09 100644
--- a/libcaf_cuda/tests/component-actors-test/mmul-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/main.test.cpp
@@ -1,6 +1,6 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
-#include <caf/cuda/component-actors/mmul_actor/mmul_actor.hpp>
+#include <caf/component-actors/mmul_actor/mmul_actor.hpp>
 #include <iostream>
 #include <iomanip>
 #include <vector>
@@ -37,9 +37,11 @@ void serial_matrix_multiply(const std::vector<int>& a,
 
 
 
+struct mmul_state {
+};
 
 // Stateful actor behavior
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
   return {
 
 	

From 14d09f0681af1b6f70bb67c682ba431d0a501145 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 10:51:13 -0600
Subject: [PATCH 0440/1000] added mutable to response lambda to fix c++
 compiler errors.

---
 libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp b/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp
index 06c8527462..e6b57b85a1 100644
--- a/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp
+++ b/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp
@@ -31,7 +31,7 @@ using mmul_async_command =
 
 template <class T>
 caf::behavior mmul_actor_fun(
-    caf::stateful_actor<mmul_actor_state>* self,
+    caf::stateful_actor<caf::cuda::mmul_actor_state>* self,
     program_ptr mmul_kernel)
 {
   using mem_t = mem_ptr<T>;
@@ -48,7 +48,7 @@ caf::behavior mmul_actor_fun(
         int N,
         int device_number,
         int stream_id)
-        -> mem_t
+        mutable -> mem_t
     {
       program_ptr kernel = self->state().mmul_kernel;
 

From 07b389715a49f789ca90e5d036236f6f36389aac Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 11:02:36 -0600
Subject: [PATCH 0441/1000] Update tests so it now passes succesfully.

---
 .../mmul-actor-test/main.test.cpp             | 121 +++++++-----------
 1 file changed, 45 insertions(+), 76 deletions(-)

diff --git a/libcaf_cuda/tests/component-actors-test/mmul-actor-test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/main.test.cpp
index 590d4a4f09..63d0cd8e7c 100644
--- a/libcaf_cuda/tests/component-actors-test/mmul-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/component-actors-test/mmul-actor-test/main.test.cpp
@@ -35,74 +35,67 @@ void serial_matrix_multiply(const std::vector<int>& a,
   }
 }
 
+using command =
+  caf::cuda::command_runner<>;
 
+command mmul_command;
 
 struct mmul_state {
 };
 
-// Stateful actor behavior
 caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
   return {
 
-	
-	  // 1st handler matrices +  N, launches a kenrel and sends its result to itself for verification
-	  [=](const std::vector<int> matrixA,
-			  const std::vector<int> matrixB, int N) {
+    // 1st handler matrices + N, launches a kernel and verifies result
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        int N) {
 
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-		  caf::cuda::manager& mgr = caf::cuda::manager::get();
+      int device = 0;
+      int stream = 1;
+      
+      // Create program and dims
+      auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
-		  //create program and dims   
-		  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-		  //create args
-		  auto arg1 = caf::cuda::create_in_arg(matrixA);
-		  auto arg2 = caf::cuda::create_in_arg(matrixB);
+      auto arg1 = mmul_command.transfer_memory(device,
+		      stream,
+		      caf::cuda::create_in_arg(matrixA));
+      auto arg2 = mmul_command.transfer_memory(device,
+		      stream,
+		      caf::cuda::create_in_arg(matrixB));
 
-		  caf::actor mmul_actor = self -> spawn(caf::cuda::mmul_actor_fun<int>,program);
+      caf::actor mmul_actor =
+        self->spawn(caf::cuda::mmul_actor_fun<int>, program);
 
-		  int device = 0;
-		  int stream = 1;
 
-		  self->request(mmul_actor,
-				  std::chrono::seconds(10),
-				  arg1,
-				  arg2,
-				  N,
-				  device,
-				  stream)
-			  .then(
-					  [&](caf::cuda::mem_ptr<int> dC) {
+      self->mail(arg1, arg2, N, device, stream)
+        .request(mmul_actor, std::chrono::seconds(10))
+        .then(
+          [=](caf::cuda::mem_ptr<int> dC) {
 
-					  std::vector<int> matrixC =
-					  dC->copy_to_host();
-					  std::vector<int> result(N*N);
+            std::vector<int> matrixC = dC->copy_to_host();
+            std::vector<int> result(N * N);
 
-					  serial_matrix_multiply(matrixA,matrixB,result,N);
+            serial_matrix_multiply(matrixA, matrixB, result, N);
 
-					  if (result == matrixC) {
-
-					  std::cout << "actor matrix references match\n";
-
-					  }
-
-					  else {
-					  std::cout << "actor matrix  references did not match\n";
-					  }
-
-					  self-> quit();
-
-					  }
-	  });
+            if (result == matrixC)
+              std::cout << "actor matrix references match\n";
+            else
+              std::cout << "actor matrix references did not match\n";
 
+            self->quit();
+          }
+        );
+    }
 
+  };
 }
 
 
-void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
+void run_mmul_test(caf::actor_system& sys, int matrix_size) {
 
   // ------------------------------------
   // Start timing
@@ -110,33 +103,20 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
   auto start = std::chrono::steady_clock::now();
 
   // Spawn num_actors actors running the mmul behavior
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
+  std::vector<int> matrixA(matrix_size * matrix_size,2);
+  std::vector<int> matrixB(matrix_size * matrix_size,3);
  
  using clock = std::chrono::steady_clock;
 
 auto t_start = clock::now();
 
-for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_actor_fun));
-}
+caf::actor a =sys.spawn(mmul_actor_fun);
+
+anon_mail(matrixA,matrixB,matrix_size).send(a);
 
 auto t_end = clock::now();
 
-auto elapsed_ms =
-    std::chrono::duration_cast<std::chrono::milliseconds>(
-        t_end - t_start).count();
 
-std::cout << "[SPAWN] spawned "
-          << num_actors
-          << " actors in "
-          << elapsed_ms
-          << " ms\n";
- 
- 
- 
-  // Actor 0 generates matrices and broadcasts to others
-  caf::anon_mail(matrix_size, actors).send(actors[0]);
 
   // Wait for all actors to finish
   sys.await_all_actors_done();
@@ -149,24 +129,13 @@ std::cout << "[SPAWN] spawned "
       std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
 
   std::cout << "[MMUL TEST] matrix_size=" << matrix_size
-            << ", actors=" << num_actors
             << ", time=" << duration_ms << " ms\n";
 }
 
 void caf_main(caf::actor_system& sys) {
   caf::cuda::manager::init(sys);
+  run_mmul_test(sys,10);
 
-  run_mmul_test(sys,512,512);
-  //run_async_mmul_test(sys,100,1);
-  //run_async_mmul_perf_test(sys,1024,200);
-
-  // run the async (no-shared) suite:
-  //benchmark_async_perf_all(sys);
-
-  // run the shared-memory suite:
-  //benchmark_shared_perf_all(sys);
-
-  //run_mmul_scaling_tests(sys);
 }
 
 

From 73e46591e1ae2ba5f8a893f607ea2caf33a0a0e0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 11:10:56 -0600
Subject: [PATCH 0442/1000] Initial commit, copied from mmul actor test.

---
 .../blocking-actor-test/CMakeLists.txt        |  44 ++++++
 .../blocking-actor-test/compile_kernels.sh    |  13 ++
 .../blocking-actor-test/main.test.cpp         | 144 ++++++++++++++++++
 .../blocking-actor-test/mmul-actors.cpp       | 125 +++++++++++++++
 .../blocking-actor-test/mmul.cu               |  16 ++
 5 files changed, 342 insertions(+)
 create mode 100644 libcaf_cuda/tests/component-actors-test/blocking-actor-test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/component-actors-test/blocking-actor-test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/component-actors-test/blocking-actor-test/main.test.cpp
 create mode 100644 libcaf_cuda/tests/component-actors-test/blocking-actor-test/mmul-actors.cpp
 create mode 100644 libcaf_cuda/tests/component-actors-test/blocking-actor-test/mmul.cu

diff --git a/libcaf_cuda/tests/component-actors-test/blocking-actor-test/CMakeLists.txt b/libcaf_cuda/tests/component-actors-test/blocking-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/blocking-actor-test/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/component-actors-test/blocking-actor-test/compile_kernels.sh b/libcaf_cuda/tests/component-actors-test/blocking-actor-test/compile_kernels.sh
new file mode 100755
index 0000000000..f32480e5cb
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/blocking-actor-test/compile_kernels.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/component-actors-test/blocking-actor-test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/blocking-actor-test/main.test.cpp
new file mode 100644
index 0000000000..63d0cd8e7c
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/blocking-actor-test/main.test.cpp
@@ -0,0 +1,144 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/component-actors/mmul_actor/mmul_actor.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+using command =
+  caf::cuda::command_runner<>;
+
+command mmul_command;
+
+struct mmul_state {
+};
+
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
+  return {
+
+    // 1st handler matrices + N, launches a kernel and verifies result
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        int N) {
+
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+      int device = 0;
+      int stream = 1;
+      
+      // Create program and dims
+      auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+      auto arg1 = mmul_command.transfer_memory(device,
+		      stream,
+		      caf::cuda::create_in_arg(matrixA));
+      auto arg2 = mmul_command.transfer_memory(device,
+		      stream,
+		      caf::cuda::create_in_arg(matrixB));
+
+      caf::actor mmul_actor =
+        self->spawn(caf::cuda::mmul_actor_fun<int>, program);
+
+
+      self->mail(arg1, arg2, N, device, stream)
+        .request(mmul_actor, std::chrono::seconds(10))
+        .then(
+          [=](caf::cuda::mem_ptr<int> dC) {
+
+            std::vector<int> matrixC = dC->copy_to_host();
+            std::vector<int> result(N * N);
+
+            serial_matrix_multiply(matrixA, matrixB, result, N);
+
+            if (result == matrixC)
+              std::cout << "actor matrix references match\n";
+            else
+              std::cout << "actor matrix references did not match\n";
+
+            self->quit();
+          }
+        );
+    }
+
+  };
+}
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size) {
+
+  // ------------------------------------
+  // Start timing
+  // ------------------------------------
+  auto start = std::chrono::steady_clock::now();
+
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<int> matrixA(matrix_size * matrix_size,2);
+  std::vector<int> matrixB(matrix_size * matrix_size,3);
+ 
+ using clock = std::chrono::steady_clock;
+
+auto t_start = clock::now();
+
+caf::actor a =sys.spawn(mmul_actor_fun);
+
+anon_mail(matrixA,matrixB,matrix_size).send(a);
+
+auto t_end = clock::now();
+
+
+
+  // Wait for all actors to finish
+  sys.await_all_actors_done();
+
+  // ------------------------------------
+  // Stop timing
+  // ------------------------------------
+  auto end = std::chrono::steady_clock::now();
+  auto duration_ms =
+      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
+            << ", time=" << duration_ms << " ms\n";
+}
+
+void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager::init(sys);
+  run_mmul_test(sys,10);
+
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/component-actors-test/blocking-actor-test/mmul-actors.cpp b/libcaf_cuda/tests/component-actors-test/blocking-actor-test/mmul-actors.cpp
new file mode 100644
index 0000000000..9865fb5da4
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/blocking-actor-test/mmul-actors.cpp
@@ -0,0 +1,125 @@
+//file not in use since I am not wasting time messing around with cmake
+
+
+#include "main.test.hpp"
+
+const char* matrixMulKernel = R"(
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+)";
+
+
+
+// Check result on the CPU
+void verify_result(vector<int> &a, vector<int> &b, vector<int> &c, int N) {
+  // For every row...
+  for (int i = 0; i < N; i++) {
+    // For every column...
+    for (int j = 0; j < N; j++) {
+      // For every element in the row-column pair
+      int tmp = 0;
+      for (int k = 0; k < N; k++) {
+        // Accumulate the partial results
+        tmp += a[i * N + k] * b[k * N + j];
+      }
+
+      // Check against the CPU result
+      assert(tmp == c[i * N + j]);
+    }
+  }
+}
+void test_mmul(caf::actor_system& sys) {
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  // Matrix dimension (N x N)
+  int N = 1024;
+  int THREADS = 32;
+  int BLOCKS = N / THREADS;
+
+  // Setup kernel launch configuration
+  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+  // Spawn CUDA actor for matrix multiplication kernel
+  auto gpuActor = mgr.spawn(matrixMulKernel, "matrixMul", dim,
+                            in<int>{}, in<int>{}, out<int>{}, in<int>{});
+
+  // Allocate and initialize host matrices
+  std::vector<int> h_a(N * N);
+  std::vector<int> h_b(N * N);
+  std::vector<int> h_c(N * N, 0);  // initialized to 0
+  std::vector<int> h_n(1, N);
+
+  std::generate(h_a.begin(), h_a.end(), []() { return rand() % 10; });
+  std::generate(h_b.begin(), h_b.end(), []() { return rand() % 10; });
+
+  // Compose device arguments
+  auto arg1 = caf::cuda::create_in_arg(h_a);
+  auto arg2 = caf::cuda::create_in_arg(h_b);
+  auto arg3 = caf::cuda::create_out_arg(h_c);
+  auto arg4 = caf::cuda::create_in_arg(h_n);
+
+  // Spawn an actor to send the message and receive the result
+  sys.spawn([=](caf::event_based_actor* self_actor) {
+    self_actor->mail(gpuActor, arg1, arg2, arg3, arg4)
+      .request(gpuActor, 30s).then(
+        [=](const std::vector<output_buffer>& outputs) {
+          std::vector<int> result;
+          bool got_output = false;
+
+          // Extract the result from outputs
+          for (const auto& out : outputs) {
+            std::visit([&](const auto& vec) {
+              if constexpr (std::is_same_v<std::decay_t<decltype(vec)>, std::vector<int>>) {
+                result = vec;
+                got_output = true;
+              }
+            }, out.data);
+          }
+
+          if (!got_output) {
+            aout(self_actor) << "No output data received!\n";
+          } else {
+            aout(self_actor) << "Verifying result..." << std::endl;
+
+            // Verify GPU result against CPU computation
+            std::vector<int> expected(N * N);
+            for (int i = 0; i < N; ++i) {
+              for (int j = 0; j < N; ++j) {
+                int tmp = 0;
+                for (int k = 0; k < N; ++k) {
+                  tmp += h_a[i * N + k] * h_b[k * N + j];
+                }
+                expected[i * N + j] = tmp;
+              }
+            }
+
+            bool success = std::equal(result.begin(), result.end(), expected.begin());
+            if (success) {
+              aout(self_actor) << "Matrix multiplication result verified successfully!\n";
+            } else {
+              aout(self_actor) << "Mismatch found in matrix multiplication results!\n";
+            }
+          }
+
+          self_actor->quit();
+        }
+      );
+  });
+
+  std::this_thread::sleep_for(std::chrono::seconds(5)); // Wait for actor to complete
+}
+
+
+
+
diff --git a/libcaf_cuda/tests/component-actors-test/blocking-actor-test/mmul.cu b/libcaf_cuda/tests/component-actors-test/blocking-actor-test/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/blocking-actor-test/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From b4ef3f8a99d3c2dbb47c454bc8e0579d6a5e60ff Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 11:13:42 -0600
Subject: [PATCH 0443/1000] Initial commit.

---
 .../mem_transfer_actor/mem_transfer_actor.hpp | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 libcaf_cuda/caf/component-actors/mem_transfer_actor/mem_transfer_actor.hpp

diff --git a/libcaf_cuda/caf/component-actors/mem_transfer_actor/mem_transfer_actor.hpp b/libcaf_cuda/caf/component-actors/mem_transfer_actor/mem_transfer_actor.hpp
new file mode 100644
index 0000000000..bdd9b651ec
--- /dev/null
+++ b/libcaf_cuda/caf/component-actors/mem_transfer_actor/mem_transfer_actor.hpp
@@ -0,0 +1,28 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include "caf/cuda/mem_ref.hpp"
+
+/*
+ * This actor is meant to handle memory transfer requests 
+ * since copy_to_host is blocking we need this actor to 
+ * block while not blocking the scheduler thread 
+ */
+
+
+namespace caf::cuda {
+
+template <class T>
+void mem_transfer_actor_fun(caf::blocking_actor* self) {
+  self->receive_while([&] { return true; })(
+    [&](mem_ptr<T> d_mem) {
+
+      // Blocking call happens safely here
+      std::vector<T> host_data = d_mem->copy_to_host();
+
+      self->reply(std::move(host_data));
+    }
+  );
+}
+
+} // namespace caf::cuda

From e9c1d48479c3b3052ea4bfbcc1e2d06497c6fe10 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 11:33:40 -0600
Subject: [PATCH 0444/1000] Fixed compiler errors.

---
 .../mem_transfer_actor/mem_transfer_actor.hpp            | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/caf/component-actors/mem_transfer_actor/mem_transfer_actor.hpp b/libcaf_cuda/caf/component-actors/mem_transfer_actor/mem_transfer_actor.hpp
index bdd9b651ec..e109725fed 100644
--- a/libcaf_cuda/caf/component-actors/mem_transfer_actor/mem_transfer_actor.hpp
+++ b/libcaf_cuda/caf/component-actors/mem_transfer_actor/mem_transfer_actor.hpp
@@ -14,13 +14,12 @@ namespace caf::cuda {
 
 template <class T>
 void mem_transfer_actor_fun(caf::blocking_actor* self) {
-  self->receive_while([&] { return true; })(
-    [&](mem_ptr<T> d_mem) {
 
-      // Blocking call happens safely here
-      std::vector<T> host_data = d_mem->copy_to_host();
+  self->receive(
+    [&](mem_ptr<T> d_mem) -> std::vector<T> {
 
-      self->reply(std::move(host_data));
+      // blocking call is safe here
+      return d_mem->copy_to_host();
     }
   );
 }

From 4cfa288ca1ed4515827ab2edaac11b91fc037733 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 11:39:53 -0600
Subject: [PATCH 0445/1000] mem_transfer_actor passes test.

---
 .../blocking-actor-test/main.test.cpp         | 82 +++++++++++--------
 1 file changed, 46 insertions(+), 36 deletions(-)

diff --git a/libcaf_cuda/tests/component-actors-test/blocking-actor-test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/blocking-actor-test/main.test.cpp
index 63d0cd8e7c..18ea8e4e52 100644
--- a/libcaf_cuda/tests/component-actors-test/blocking-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/component-actors-test/blocking-actor-test/main.test.cpp
@@ -1,6 +1,7 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
 #include <caf/component-actors/mmul_actor/mmul_actor.hpp>
+#include <caf/component-actors/mem_transfer_actor/mem_transfer_actor.hpp>
 #include <iostream>
 #include <iomanip>
 #include <vector>
@@ -44,54 +45,63 @@ struct mmul_state {
 };
 
 caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
-  return {
+	return {
 
-    // 1st handler matrices + N, launches a kernel and verifies result
-    [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB,
-        int N) {
+		// 1st handler matrices + N, launches a kernel and verifies result
+		[=](const std::vector<int>& matrixA,
+				const std::vector<int>& matrixB,
+				int N) {
 
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
+			caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-      int device = 0;
-      int stream = 1;
-      
-      // Create program and dims
-      auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+			int device = 0;
+			int stream = 1;
 
-      auto arg1 = mmul_command.transfer_memory(device,
-		      stream,
-		      caf::cuda::create_in_arg(matrixA));
-      auto arg2 = mmul_command.transfer_memory(device,
-		      stream,
-		      caf::cuda::create_in_arg(matrixB));
+			// Create program and dims
+			auto program =
+				mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
-      caf::actor mmul_actor =
-        self->spawn(caf::cuda::mmul_actor_fun<int>, program);
+			auto arg1 = mmul_command.transfer_memory(device,
+					stream,
+					caf::cuda::create_in_arg(matrixA));
+			auto arg2 = mmul_command.transfer_memory(device,
+					stream,
+					caf::cuda::create_in_arg(matrixB));
 
+			caf::actor mmul_actor =
+				self->spawn(caf::cuda::mmul_actor_fun<int>, program);
 
-      self->mail(arg1, arg2, N, device, stream)
-        .request(mmul_actor, std::chrono::seconds(10))
-        .then(
-          [=](caf::cuda::mem_ptr<int> dC) {
+			caf::actor mem_transfer_actor =
+				self->spawn(caf::cuda::mem_transfer_actor_fun<int>);
 
-            std::vector<int> matrixC = dC->copy_to_host();
-            std::vector<int> result(N * N);
 
-            serial_matrix_multiply(matrixA, matrixB, result, N);
 
-            if (result == matrixC)
-              std::cout << "actor matrix references match\n";
-            else
-              std::cout << "actor matrix references did not match\n";
+			self->mail(arg1, arg2, N, device, stream)
+				.request(mmul_actor, std::chrono::seconds(10))
+				.then(
+						[=](caf::cuda::mem_ptr<int> dC) {
 
-            self->quit();
-          }
-        );
-    }
+						self ->mail(dC).request(mem_transfer_actor,std::chrono::seconds(10))
+						.then(
+								[=](const std::vector<int>& matrixC) {
+
+
+								std::vector<int> result(N * N);
+
+								serial_matrix_multiply(matrixA, matrixB, result, N);
+
+								if (result == matrixC)
+								std::cout << "actor matrix references match\n";
+								else
+								std::cout << "actor matrix references did not match\n";
+
+								self->quit();
+								});
+								}
+						     );
+		}
 
-  };
+	};
 }
 
 

From 82b93b586cd1c815b600070a0473e20db2e6cb3e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 11:44:57 -0600
Subject: [PATCH 0446/1000] Initial commit.

---
 .../random_mmul_gen_multiply_actor.hpp                           | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 libcaf_cuda/caf/component-actors/random_mmul_gen_multiply_actor/random_mmul_gen_multiply_actor.hpp

diff --git a/libcaf_cuda/caf/component-actors/random_mmul_gen_multiply_actor/random_mmul_gen_multiply_actor.hpp b/libcaf_cuda/caf/component-actors/random_mmul_gen_multiply_actor/random_mmul_gen_multiply_actor.hpp
new file mode 100644
index 0000000000..6f70f09bee
--- /dev/null
+++ b/libcaf_cuda/caf/component-actors/random_mmul_gen_multiply_actor/random_mmul_gen_multiply_actor.hpp
@@ -0,0 +1 @@
+#pragma once

From 174338f131682c876a9d6af583ac2a7d3b2cd760 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 11:50:47 -0600
Subject: [PATCH 0447/1000] Initial commit.

---
 .../vector-add-actor-test/CMakeLists.txt      |  44 ++++++
 .../vector-add-actor-test/compile_kernels.sh  |  13 ++
 .../vector-add-actor-test/main.test.cpp       | 144 ++++++++++++++++++
 .../vector-add-actor-test/mmul-actors.cpp     | 125 +++++++++++++++
 .../vector-add-actor-test/vector_add.cu       |   6 +
 5 files changed, 332 insertions(+)
 create mode 100644 libcaf_cuda/tests/component-actors-test/vector-add-actor-test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/component-actors-test/vector-add-actor-test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp
 create mode 100644 libcaf_cuda/tests/component-actors-test/vector-add-actor-test/mmul-actors.cpp
 create mode 100644 libcaf_cuda/tests/component-actors-test/vector-add-actor-test/vector_add.cu

diff --git a/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/CMakeLists.txt b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/compile_kernels.sh b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/compile_kernels.sh
new file mode 100755
index 0000000000..04e08e301a
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/compile_kernels.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin vector_add.cu -o vector_add.cubin
+echo "Generated mmul.cubin"
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp
new file mode 100644
index 0000000000..63d0cd8e7c
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp
@@ -0,0 +1,144 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/component-actors/mmul_actor/mmul_actor.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+using command =
+  caf::cuda::command_runner<>;
+
+command mmul_command;
+
+struct mmul_state {
+};
+
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
+  return {
+
+    // 1st handler matrices + N, launches a kernel and verifies result
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        int N) {
+
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+      int device = 0;
+      int stream = 1;
+      
+      // Create program and dims
+      auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+      auto arg1 = mmul_command.transfer_memory(device,
+		      stream,
+		      caf::cuda::create_in_arg(matrixA));
+      auto arg2 = mmul_command.transfer_memory(device,
+		      stream,
+		      caf::cuda::create_in_arg(matrixB));
+
+      caf::actor mmul_actor =
+        self->spawn(caf::cuda::mmul_actor_fun<int>, program);
+
+
+      self->mail(arg1, arg2, N, device, stream)
+        .request(mmul_actor, std::chrono::seconds(10))
+        .then(
+          [=](caf::cuda::mem_ptr<int> dC) {
+
+            std::vector<int> matrixC = dC->copy_to_host();
+            std::vector<int> result(N * N);
+
+            serial_matrix_multiply(matrixA, matrixB, result, N);
+
+            if (result == matrixC)
+              std::cout << "actor matrix references match\n";
+            else
+              std::cout << "actor matrix references did not match\n";
+
+            self->quit();
+          }
+        );
+    }
+
+  };
+}
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size) {
+
+  // ------------------------------------
+  // Start timing
+  // ------------------------------------
+  auto start = std::chrono::steady_clock::now();
+
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<int> matrixA(matrix_size * matrix_size,2);
+  std::vector<int> matrixB(matrix_size * matrix_size,3);
+ 
+ using clock = std::chrono::steady_clock;
+
+auto t_start = clock::now();
+
+caf::actor a =sys.spawn(mmul_actor_fun);
+
+anon_mail(matrixA,matrixB,matrix_size).send(a);
+
+auto t_end = clock::now();
+
+
+
+  // Wait for all actors to finish
+  sys.await_all_actors_done();
+
+  // ------------------------------------
+  // Stop timing
+  // ------------------------------------
+  auto end = std::chrono::steady_clock::now();
+  auto duration_ms =
+      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
+            << ", time=" << duration_ms << " ms\n";
+}
+
+void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager::init(sys);
+  run_mmul_test(sys,10);
+
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/mmul-actors.cpp b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/mmul-actors.cpp
new file mode 100644
index 0000000000..9865fb5da4
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/mmul-actors.cpp
@@ -0,0 +1,125 @@
+//file not in use since I am not wasting time messing around with cmake
+
+
+#include "main.test.hpp"
+
+const char* matrixMulKernel = R"(
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+)";
+
+
+
+// Check result on the CPU
+void verify_result(vector<int> &a, vector<int> &b, vector<int> &c, int N) {
+  // For every row...
+  for (int i = 0; i < N; i++) {
+    // For every column...
+    for (int j = 0; j < N; j++) {
+      // For every element in the row-column pair
+      int tmp = 0;
+      for (int k = 0; k < N; k++) {
+        // Accumulate the partial results
+        tmp += a[i * N + k] * b[k * N + j];
+      }
+
+      // Check against the CPU result
+      assert(tmp == c[i * N + j]);
+    }
+  }
+}
+void test_mmul(caf::actor_system& sys) {
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  // Matrix dimension (N x N)
+  int N = 1024;
+  int THREADS = 32;
+  int BLOCKS = N / THREADS;
+
+  // Setup kernel launch configuration
+  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+  // Spawn CUDA actor for matrix multiplication kernel
+  auto gpuActor = mgr.spawn(matrixMulKernel, "matrixMul", dim,
+                            in<int>{}, in<int>{}, out<int>{}, in<int>{});
+
+  // Allocate and initialize host matrices
+  std::vector<int> h_a(N * N);
+  std::vector<int> h_b(N * N);
+  std::vector<int> h_c(N * N, 0);  // initialized to 0
+  std::vector<int> h_n(1, N);
+
+  std::generate(h_a.begin(), h_a.end(), []() { return rand() % 10; });
+  std::generate(h_b.begin(), h_b.end(), []() { return rand() % 10; });
+
+  // Compose device arguments
+  auto arg1 = caf::cuda::create_in_arg(h_a);
+  auto arg2 = caf::cuda::create_in_arg(h_b);
+  auto arg3 = caf::cuda::create_out_arg(h_c);
+  auto arg4 = caf::cuda::create_in_arg(h_n);
+
+  // Spawn an actor to send the message and receive the result
+  sys.spawn([=](caf::event_based_actor* self_actor) {
+    self_actor->mail(gpuActor, arg1, arg2, arg3, arg4)
+      .request(gpuActor, 30s).then(
+        [=](const std::vector<output_buffer>& outputs) {
+          std::vector<int> result;
+          bool got_output = false;
+
+          // Extract the result from outputs
+          for (const auto& out : outputs) {
+            std::visit([&](const auto& vec) {
+              if constexpr (std::is_same_v<std::decay_t<decltype(vec)>, std::vector<int>>) {
+                result = vec;
+                got_output = true;
+              }
+            }, out.data);
+          }
+
+          if (!got_output) {
+            aout(self_actor) << "No output data received!\n";
+          } else {
+            aout(self_actor) << "Verifying result..." << std::endl;
+
+            // Verify GPU result against CPU computation
+            std::vector<int> expected(N * N);
+            for (int i = 0; i < N; ++i) {
+              for (int j = 0; j < N; ++j) {
+                int tmp = 0;
+                for (int k = 0; k < N; ++k) {
+                  tmp += h_a[i * N + k] * h_b[k * N + j];
+                }
+                expected[i * N + j] = tmp;
+              }
+            }
+
+            bool success = std::equal(result.begin(), result.end(), expected.begin());
+            if (success) {
+              aout(self_actor) << "Matrix multiplication result verified successfully!\n";
+            } else {
+              aout(self_actor) << "Mismatch found in matrix multiplication results!\n";
+            }
+          }
+
+          self_actor->quit();
+        }
+      );
+  });
+
+  std::this_thread::sleep_for(std::chrono::seconds(5)); // Wait for actor to complete
+}
+
+
+
+
diff --git a/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/vector_add.cu b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/vector_add.cu
new file mode 100644
index 0000000000..5cdda286f9
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/vector_add.cu
@@ -0,0 +1,6 @@
+extern "C" __global__ void vectorAdd(const int* A, const int* B, int* C, int N) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx < N) {
+        C[idx] = A[idx] + B[idx];
+    }
+}

From 097bc61b7bb7becab3efd2e4d2cb123e379c01cf Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 11:52:21 -0600
Subject: [PATCH 0448/1000] Initial commit.

---
 .../vector_add_actor/vector_add_actor.hpp     | 65 +++++++++++++++++++
 1 file changed, 65 insertions(+)
 create mode 100644 libcaf_cuda/caf/component-actors/vector_add_actor/vector_add_actor.hpp

diff --git a/libcaf_cuda/caf/component-actors/vector_add_actor/vector_add_actor.hpp b/libcaf_cuda/caf/component-actors/vector_add_actor/vector_add_actor.hpp
new file mode 100644
index 0000000000..ea8fa496b6
--- /dev/null
+++ b/libcaf_cuda/caf/component-actors/vector_add_actor/vector_add_actor.hpp
@@ -0,0 +1,65 @@
+#pragma once
+#include <caf/all.hpp>
+#include "caf/cuda/all.hpp"
+
+namespace caf::cuda {
+
+// State for the vector add actor
+struct vector_add_actor_state {
+    static inline const char* name = "vector_add_actor";
+    program_ptr vector_add_kernel;
+};
+
+// Command runner type for vector add
+template <class T>
+using vector_add_command = command_runner<
+    mem_ptr<T>,  // input A
+    mem_ptr<T>,  // input B
+    out<T>,      // output C
+    in<T>        // length N
+>;
+
+// Actor behavior
+template <class T>
+caf::behavior vector_add_actor_fun(
+    caf::stateful_actor<vector_add_actor_state>* self,
+    program_ptr vector_add_kernel)
+{
+    using mem_t = mem_ptr<T>;
+
+    vector_add_command<T> runner;  // non-static, per actor
+    self->state().vector_add_kernel = vector_add_kernel;
+
+    return {
+        [=](mem_t vecA,
+            mem_t vecB,
+            int N,
+            int device_number,
+            int stream_id)
+        -> mem_ptr<T>
+        {
+            nd_range dims((N + 255) / 256, 1, 1, 256, 1, 1);
+
+            auto arg3 = create_out_arg<T>(N);
+            auto arg4 = create_in_arg<T>(N);
+
+            auto result_tuple = runner.run_async(
+                vector_add_kernel,
+                dims,
+                stream_id,
+                0,
+                device_number,
+                vecA,
+                vecB,
+                arg3,
+                arg4
+            );
+
+            // Extract mem_ptr<T> result (2nd index of result tuple)
+            return extract_mem_ptr<T>(result_tuple, 2);
+        }
+    };
+}
+
+} // namespace caf::cuda
+

From 5492745fb369aab06893cea226c4d51c65cbcc32 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 12:18:52 -0600
Subject: [PATCH 0449/1000] Fixed return of result and added mutable to lambda
 to fix compiler issues.

---
 .../component-actors/vector_add_actor/vector_add_actor.hpp    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/component-actors/vector_add_actor/vector_add_actor.hpp b/libcaf_cuda/caf/component-actors/vector_add_actor/vector_add_actor.hpp
index ea8fa496b6..7593bfeab6 100644
--- a/libcaf_cuda/caf/component-actors/vector_add_actor/vector_add_actor.hpp
+++ b/libcaf_cuda/caf/component-actors/vector_add_actor/vector_add_actor.hpp
@@ -36,7 +36,7 @@ caf::behavior vector_add_actor_fun(
             int N,
             int device_number,
             int stream_id)
-        -> mem_ptr<T>
+         mutable ->  mem_ptr<T>
         {
             nd_range dims((N + 255) / 256, 1, 1, 256, 1, 1);
 
@@ -56,7 +56,7 @@ caf::behavior vector_add_actor_fun(
             );
 
             // Extract mem_ptr<T> result (2nd index of result tuple)
-            return extract_mem_ptr<T>(result_tuple, 2);
+            return std::get<2>(result_tuple);
         }
     };
 }

From 0674ea477db96f75418f1006019df8944bf1010b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 12:19:11 -0600
Subject: [PATCH 0450/1000] Initial commit.

---
 .../vector-add-actor-test/main.test.cpp       | 182 ++++++------------
 1 file changed, 62 insertions(+), 120 deletions(-)

diff --git a/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp
index 63d0cd8e7c..598f06b00f 100644
--- a/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp
@@ -1,144 +1,86 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
-#include <caf/component-actors/mmul_actor/mmul_actor.hpp>
+#include <caf/component-actors/vector_add_actor/vector_add_actor.hpp>
 #include <iostream>
-#include <iomanip>
 #include <vector>
 #include <chrono>
-#include <thread>
-#include <algorithm>
-#include <numeric>
-#include <random>
-#include "caf/actor_registry.hpp"
-//#include <caf/atoms.hpp>
-
-
 
 using namespace caf;
 using namespace std::chrono_literals;
 
-
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-  
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
+// Simple serial vector addition for verification
+template <typename T>
+void serial_vector_add(const std::vector<T>& a,
+                       const std::vector<T>& b,
+                       std::vector<T>& c) {
+    for (size_t i = 0; i < a.size(); ++i) {
+        c[i] = a[i] + b[i];
     }
-  }
 }
 
-using command =
-  caf::cuda::command_runner<>;
-
-command mmul_command;
-
-struct mmul_state {
-};
-
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
-  return {
-
-    // 1st handler matrices + N, launches a kernel and verifies result
-    [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB,
-        int N) {
-
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-      int device = 0;
-      int stream = 1;
-      
-      // Create program and dims
-      auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-      auto arg1 = mmul_command.transfer_memory(device,
-		      stream,
-		      caf::cuda::create_in_arg(matrixA));
-      auto arg2 = mmul_command.transfer_memory(device,
-		      stream,
-		      caf::cuda::create_in_arg(matrixB));
-
-      caf::actor mmul_actor =
-        self->spawn(caf::cuda::mmul_actor_fun<int>, program);
-
-
-      self->mail(arg1, arg2, N, device, stream)
-        .request(mmul_actor, std::chrono::seconds(10))
-        .then(
-          [=](caf::cuda::mem_ptr<int> dC) {
-
-            std::vector<int> matrixC = dC->copy_to_host();
-            std::vector<int> result(N * N);
-
-            serial_matrix_multiply(matrixA, matrixB, result, N);
-
-            if (result == matrixC)
-              std::cout << "actor matrix references match\n";
-            else
-              std::cout << "actor matrix references did not match\n";
-
-            self->quit();
-          }
-        );
-    }
-
-  };
+struct supervisor_state {};
+
+// Supervisor actor behavior
+caf::behavior vector_add_supervisor(caf::stateful_actor<supervisor_state> * self,
+                                    const std::vector<int>& vecA,
+                                    const std::vector<int>& vecB,
+                                    size_t vec_size,
+                                    caf::cuda::program_ptr program) {
+    return {
+        [&](const unit_t&) mutable {
+
+            // Spawn the vector add worker actor
+            caf::actor worker = self ->spawn(caf::cuda::vector_add_actor_fun<int>, program);
+
+            // Transfer memory to device
+            auto dA = caf::cuda::create_in_arg(vecA);
+            auto dB = caf::cuda::create_in_arg(vecB);
+
+            // Send to worker actor and request result
+            self->mail(dA, dB, vec_size, 0, 1) // device=0, stream=1
+                .request(worker, std::chrono::seconds(10))
+                .then([&](caf::cuda::mem_ptr<int> dC) {
+                    std::vector<int> result(vec_size);
+                    std::vector<int> vecC = dC->copy_to_host();
+
+                    serial_vector_add(vecA, vecB, result);
+
+                    if (result == vecC)
+                        std::cout << "Vector addition result matches!\n";
+                    else
+                        std::cout << "Vector addition result mismatch!\n";
+
+                    // Quit the supervisor
+                    self->quit();
+                });
+        }
+    };
 }
 
+void run_vector_add_test(actor_system& sys, size_t vec_size) {
+    caf::cuda::manager::init(sys);
 
-void run_mmul_test(caf::actor_system& sys, int matrix_size) {
-
-  // ------------------------------------
-  // Start timing
-  // ------------------------------------
-  auto start = std::chrono::steady_clock::now();
-
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<int> matrixA(matrix_size * matrix_size,2);
-  std::vector<int> matrixB(matrix_size * matrix_size,3);
- 
- using clock = std::chrono::steady_clock;
-
-auto t_start = clock::now();
-
-caf::actor a =sys.spawn(mmul_actor_fun);
+    // Create program pointer for the vector add kernel
+    auto program = caf::cuda::manager::get()
+                       .create_program_from_cubin("../vector_add.cubin", "vectorAdd");
 
-anon_mail(matrixA,matrixB,matrix_size).send(a);
+    // Generate test data
+    std::vector<int> vecA(vec_size, 2);
+    std::vector<int> vecB(vec_size, 3);
 
-auto t_end = clock::now();
+    // Spawn the supervisor actor
+    caf::actor supervisor = sys.spawn(vector_add_supervisor,
+                                       vecA, vecB, vec_size, program);
 
+    // Trigger the supervisor
+    anon_mail(unit).send(supervisor);
 
-
-  // Wait for all actors to finish
-  sys.await_all_actors_done();
-
-  // ------------------------------------
-  // Stop timing
-  // ------------------------------------
-  auto end = std::chrono::steady_clock::now();
-  auto duration_ms =
-      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
-            << ", time=" << duration_ms << " ms\n";
+    // Wait for all actors to finish
+    sys.await_all_actors_done();
 }
 
-void caf_main(caf::actor_system& sys) {
-  caf::cuda::manager::init(sys);
-  run_mmul_test(sys,10);
-
+void caf_main(actor_system& sys) {
+    run_vector_add_test(sys, 1024);
 }
 
-
-
-
 CAF_MAIN()

From 563d135869d7f53090faa49bfb9794032c592287 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 12:35:09 -0600
Subject: [PATCH 0451/1000] Fixed template issue with in<int> was in<T>
 mistakenly causing issues with kernels launching other than type int.

---
 libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp b/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp
index e6b57b85a1..e839b4d7f6 100644
--- a/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp
+++ b/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp
@@ -26,7 +26,7 @@ using mmul_async_command =
     mem_ptr<T>,
     mem_ptr<T>,
     out<T>,
-    in<T>
+    in<int>
   >;
 
 template <class T>
@@ -60,7 +60,7 @@ caf::behavior mmul_actor_fun(
         THREADS, THREADS, 1);
 
       out<T> arg3 = create_out_arg<T>(N * N);
-      in<T>  arg4 = create_in_arg<T>(static_cast<T>(N));
+      in<int>  arg4 = create_in_arg<int>(N));
 
       std::tuple<mem_t, mem_t, mem_t, mem_t> result_tuple =
         mmul.run_async(

From ecc9b29e58f04dbc586b0ed50c2920cc13730561 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 12:35:59 -0600
Subject: [PATCH 0452/1000] Fixed missing closing bracket syntax error.

---
 libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp b/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp
index e839b4d7f6..8cef4a6d4c 100644
--- a/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp
+++ b/libcaf_cuda/caf/component-actors/mmul_actor/mmul_actor.hpp
@@ -60,7 +60,7 @@ caf::behavior mmul_actor_fun(
         THREADS, THREADS, 1);
 
       out<T> arg3 = create_out_arg<T>(N * N);
-      in<int>  arg4 = create_in_arg<int>(N));
+      in<int>  arg4 = create_in_arg<int>(N);
 
       std::tuple<mem_t, mem_t, mem_t, mem_t> result_tuple =
         mmul.run_async(

From 5bae609b9d3164f0ca9dce77fcd7f13ffba6ff67 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 12:46:15 -0600
Subject: [PATCH 0453/1000] Updated actor to fix issues reguarding length of
 the kernel being templated with type T when it should have been type int.

---
 .../vector_add_actor/vector_add_actor.hpp     | 38 +++++++++++--------
 1 file changed, 23 insertions(+), 15 deletions(-)

diff --git a/libcaf_cuda/caf/component-actors/vector_add_actor/vector_add_actor.hpp b/libcaf_cuda/caf/component-actors/vector_add_actor/vector_add_actor.hpp
index 7593bfeab6..ae58d38a3b 100644
--- a/libcaf_cuda/caf/component-actors/vector_add_actor/vector_add_actor.hpp
+++ b/libcaf_cuda/caf/component-actors/vector_add_actor/vector_add_actor.hpp
@@ -1,4 +1,5 @@
 #pragma once
+
 #include <caf/all.hpp>
 #include "caf/cuda/all.hpp"
 
@@ -16,7 +17,7 @@ using vector_add_command = command_runner<
     mem_ptr<T>,  // input A
     mem_ptr<T>,  // input B
     out<T>,      // output C
-    in<T>        // length N
+    in<int>      // length N (always an int)
 >;
 
 // Actor behavior
@@ -26,40 +27,47 @@ caf::behavior vector_add_actor_fun(
     program_ptr vector_add_kernel)
 {
     using mem_t = mem_ptr<T>;
+    using runner_t = vector_add_command<T>;
 
-    vector_add_command<T> runner;  // non-static, per actor
     self->state().vector_add_kernel = vector_add_kernel;
+    runner_t runner;  // per-actor runner instance
 
     return {
         [=](mem_t vecA,
             mem_t vecB,
             int N,
             int device_number,
-            int stream_id)
-         mutable ->  mem_ptr<T>
+            int stream_id) mutable -> mem_t
         {
-            nd_range dims((N + 255) / 256, 1, 1, 256, 1, 1);
+            program_ptr kernel = self->state().vector_add_kernel;
+
+            const int THREADS = 256;
+            const int BLOCKS = (N + THREADS - 1) / THREADS;
+
+            nd_range dims(BLOCKS, 1, 1, THREADS, 1, 1);
 
-            auto arg3 = create_out_arg<T>(N);
-            auto arg4 = create_in_arg<T>(N);
+            out<T> arg_out  = create_out_arg<T>(N);
+            in<int> arg_len = create_in_arg<int>(N);  // length as int
 
+            // Run the kernel asynchronously
             auto result_tuple = runner.run_async(
-                vector_add_kernel,
+                kernel,
                 dims,
                 stream_id,
-                0,
-                device_number,
+                0,             // shared_memory
+                device_number, // device
                 vecA,
                 vecB,
-                arg3,
-                arg4
+                arg_out,
+                arg_len
             );
 
-            // Extract mem_ptr<T> result (2nd index of result tuple)
-            return std::get<2>(result_tuple);
+            // Extract the output buffer (3rd element of tuple)
+            mem_t output_device_buffer = std::get<2>(result_tuple);
+
+            return output_device_buffer;
         }
     };
 }
 
 } // namespace caf::cuda
-

From 0af3fcbc215116733975ab909b84cbbfec61debd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 12:49:04 -0600
Subject: [PATCH 0454/1000] fixed supervisor actor handle miscapture issue
 leading to segfaults.

---
 .../vector-add-actor-test/main.test.cpp                  | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp
index 598f06b00f..d475cde1f4 100644
--- a/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp
@@ -26,11 +26,14 @@ caf::behavior vector_add_supervisor(caf::stateful_actor<supervisor_state> * self
                                     const std::vector<int>& vecB,
                                     size_t vec_size,
                                     caf::cuda::program_ptr program) {
-    return {
-        [&](const unit_t&) mutable {
+   
+            caf::actor worker = self ->spawn(caf::cuda::vector_add_actor_fun<int>, program);
+       
+	return {
+        [&,self,worker](const unit_t&) mutable {
 
             // Spawn the vector add worker actor
-            caf::actor worker = self ->spawn(caf::cuda::vector_add_actor_fun<int>, program);
+            //caf::actor worker = self ->spawn(caf::cuda::vector_add_actor_fun<int>, program);
 
             // Transfer memory to device
             auto dA = caf::cuda::create_in_arg(vecA);

From ae090b3043b32a3cc20f474f64166fdbaff4d00f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 28 Feb 2026 13:03:43 -0600
Subject: [PATCH 0455/1000] FIxed many lambda capture issues with the
 supervisor actor.

---
 .../vector-add-actor-test/main.test.cpp             | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp
index d475cde1f4..aa601711d1 100644
--- a/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/component-actors-test/vector-add-actor-test/main.test.cpp
@@ -18,26 +18,29 @@ void serial_vector_add(const std::vector<T>& a,
     }
 }
 
+
+caf::cuda::command_runner<> runner;
+
 struct supervisor_state {};
 
 // Supervisor actor behavior
 caf::behavior vector_add_supervisor(caf::stateful_actor<supervisor_state> * self,
                                     const std::vector<int>& vecA,
                                     const std::vector<int>& vecB,
-                                    size_t vec_size,
+                                    int vec_size,
                                     caf::cuda::program_ptr program) {
    
             caf::actor worker = self ->spawn(caf::cuda::vector_add_actor_fun<int>, program);
        
 	return {
-        [&,self,worker](const unit_t&) mutable {
+        [&,self,worker,vecA,vecB,vec_size](const unit_t&) mutable {
 
             // Spawn the vector add worker actor
             //caf::actor worker = self ->spawn(caf::cuda::vector_add_actor_fun<int>, program);
 
             // Transfer memory to device
-            auto dA = caf::cuda::create_in_arg(vecA);
-            auto dB = caf::cuda::create_in_arg(vecB);
+            auto dA = runner.transfer_memory(0,1,caf::cuda::create_in_arg(vecA));
+            auto dB = runner.transfer_memory(0,1,caf::cuda::create_in_arg(vecB));
 
             // Send to worker actor and request result
             self->mail(dA, dB, vec_size, 0, 1) // device=0, stream=1
@@ -60,7 +63,7 @@ caf::behavior vector_add_supervisor(caf::stateful_actor<supervisor_state> * self
     };
 }
 
-void run_vector_add_test(actor_system& sys, size_t vec_size) {
+void run_vector_add_test(actor_system& sys, int vec_size) {
     caf::cuda::manager::init(sys);
 
     // Create program pointer for the vector add kernel

From 3f3aa1f645954cff2b8177878ba10303277397fc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 1 Mar 2026 10:18:55 -0600
Subject: [PATCH 0456/1000] Initial commit.

---
 .../mmul_actor_not_square.hpp                 | 160 ++++++++++++++++++
 1 file changed, 160 insertions(+)
 create mode 100644 libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp

diff --git a/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp b/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp
new file mode 100644
index 0000000000..3a5bae1082
--- /dev/null
+++ b/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp
@@ -0,0 +1,160 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include "caf/cuda/all.hpp"
+#include <string>
+#include <type_traits>
+
+
+/*
+ * An actor meant to represent matrix multiply 
+ * kernel for now must be supplied since we do support kernel compilation
+ * auto integration but it is via strings so you can supply own kernel to reduce needless recompilation, it is assumed you will supply a matrix multiple kernel to this
+ * actor
+ * it takes in 2 mem_ptrs to the representing matrix A and matrix B,matrix size, 
+ * device number, and stream id  
+ * and replys with the result 
+ */
+
+
+namespace caf::cuda {
+
+// ------------------- Float version -------------------
+inline std::string mmulNS_kernel_source_float = R"(
+extern "C" __global__
+void matrixMulNS(const float* A, const float* B, float* C,
+                 int M, int K, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (row < M && col < N) {
+        float temp = 0.0f;
+        for (int k = 0; k < K; ++k) {
+            temp += A[row * K + k] * B[k * N + col];
+        }
+        C[row * N + col] = temp;
+    }
+}
+)";
+
+// ------------------- Double version -------------------
+inline std::string mmulNS_kernel_source_double = R"(
+extern "C" __global__
+void matrixMulNS(const double* A, const double* B, double* C,
+                  int M, int K, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (row < M && col < N) {
+        double temp = 0.0;
+        for (int k = 0; k < K; ++k) {
+            temp += A[row * K + k] * B[k * N + col];
+        }
+        C[row * N + col] = temp;
+    }
+}
+)";
+
+// ------------------- Integer version -------------------
+inline std::string mmulNS_kernel_source_int = R"(
+extern "C" __global__
+void matrixMulNS(const int* A, const int* B, int* C,
+                 int M, int K, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (row < M && col < N) {
+        int temp = 0;
+        for (int k = 0; k < K; ++k) {
+            temp += A[row * K + k] * B[k * N + col];
+        }
+        C[row * N + col] = temp;
+    }
+}
+)";
+
+// ------------------- Template selector -------------------
+template <typename T>
+inline std::string get_mmulNS_kernel_source() {
+    if constexpr (std::is_same_v<T, float>) {
+        return mmulNS_kernel_source_float;
+    } else if constexpr (std::is_same_v<T, double>) {
+        return mmulNS_kernel_source_double;
+    } else if constexpr (std::is_same_v<T, int>) {
+        return mmulNS_kernel_source_int;
+    } else {
+        static_assert(!sizeof(T*), "Unsupported type for mmulNS kernel");
+    }
+}
+
+
+
+
+struct mmul_actor_not_square_state {
+  static inline const char* name = "mmul_actor";
+  program_ptr mmul_kernel;
+};
+
+template <class T>
+using mmul_async_command =
+  command_runner<
+    mem_ptr<T>,
+    mem_ptr<T>,
+    out<T>,
+    in<int>
+  >;
+
+template <class T>
+caf::behavior mmul_actor_NS_fun(
+    caf::stateful_actor<caf::cuda::mmul_actor_not_square_state>* self,
+    program_ptr mmul_kernel)
+{
+  using mem_t = mem_ptr<T>;
+  using runner_t = mmul_async_command<T>;
+
+  self->state().mmul_kernel = mmul_kernel;
+
+  runner_t mmul;
+
+  return {
+
+    [=](mem_t matrixA,
+        mem_t matrixB,
+        int N,
+        int device_number,
+        int stream_id)
+        mutable -> mem_t
+    {
+      program_ptr kernel = self->state().mmul_kernel;
+
+      const int THREADS = 32;
+      const int BLOCKS = (N + THREADS - 1) / THREADS;
+
+      nd_range dims(
+        BLOCKS, BLOCKS, 1,
+        THREADS, THREADS, 1);
+
+      out<T> arg3 = create_out_arg<T>(N * N);
+      in<int>  arg4 = create_in_arg<int>(N);
+
+      std::tuple<mem_t, mem_t, mem_t, mem_t> result_tuple =
+        mmul.run_async(
+          kernel,
+          dims,
+          stream_id,
+          0,
+          device_number,
+          matrixA,
+          matrixB,
+          arg3,
+          arg4);
+
+      mem_t output_device_buffer = std::get<2>(result_tuple);
+
+      return output_device_buffer;
+    }
+
+  };
+}
+
+} // namespace caf::cuda

From 4df07eb32c1cf77d0cd83ee3a738d764ef6a7c85 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 1 Mar 2026 10:39:15 -0600
Subject: [PATCH 0457/1000] Inital commit.

---
 .../double_mmul_non_square.cu                 |  17 ++
 .../float_mmul_non_square.cu                  |  14 ++
 .../int_mmul_non_square.cu                    |  17 ++
 .../mmul_actor_not_square.hpp                 | 146 +++++-------------
 4 files changed, 85 insertions(+), 109 deletions(-)
 create mode 100644 libcaf_cuda/caf/component-actors/mmul_actor_not_square/double_mmul_non_square.cu
 create mode 100644 libcaf_cuda/caf/component-actors/mmul_actor_not_square/float_mmul_non_square.cu
 create mode 100644 libcaf_cuda/caf/component-actors/mmul_actor_not_square/int_mmul_non_square.cu

diff --git a/libcaf_cuda/caf/component-actors/mmul_actor_not_square/double_mmul_non_square.cu b/libcaf_cuda/caf/component-actors/mmul_actor_not_square/double_mmul_non_square.cu
new file mode 100644
index 0000000000..b6cff01eb7
--- /dev/null
+++ b/libcaf_cuda/caf/component-actors/mmul_actor_not_square/double_mmul_non_square.cu
@@ -0,0 +1,17 @@
+extern "C" __global__
+void mmul_non_square_double(const double* A,
+                            const double* B,
+                            double* C,
+                            int M, int K, int N)
+{
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (row < M && col < N) {
+        double sum = 0.0;
+        for (int k = 0; k < K; ++k) {
+            sum += A[row * K + k] * B[k * N + col];
+        }
+        C[row * N + col] = sum;
+    }
+}
diff --git a/libcaf_cuda/caf/component-actors/mmul_actor_not_square/float_mmul_non_square.cu b/libcaf_cuda/caf/component-actors/mmul_actor_not_square/float_mmul_non_square.cu
new file mode 100644
index 0000000000..983d7208fb
--- /dev/null
+++ b/libcaf_cuda/caf/component-actors/mmul_actor_not_square/float_mmul_non_square.cu
@@ -0,0 +1,14 @@
+extern "C" __global__
+void mmul_non_square(const float* A, const float* B, float* C,
+                     int M, int K, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y; // row in C
+    int col = blockIdx.x * blockDim.x + threadIdx.x; // col in C
+
+    if (row < M && col < N) {
+        float sum = 0.0f;
+        for (int k = 0; k < K; ++k) {
+            sum += A[row * K + k] * B[k * N + col];
+        }
+        C[row * N + col] = sum;
+    }
+}
diff --git a/libcaf_cuda/caf/component-actors/mmul_actor_not_square/int_mmul_non_square.cu b/libcaf_cuda/caf/component-actors/mmul_actor_not_square/int_mmul_non_square.cu
new file mode 100644
index 0000000000..afccac59ee
--- /dev/null
+++ b/libcaf_cuda/caf/component-actors/mmul_actor_not_square/int_mmul_non_square.cu
@@ -0,0 +1,17 @@
+extern "C" __global__
+void mmul_non_square_int(const int* A,
+                         const int* B,
+                         int* C,
+                         int M, int K, int N)
+{
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (row < M && col < N) {
+        int sum = 0;
+        for (int k = 0; k < K; ++k) {
+            sum += A[row * K + k] * B[k * N + col];
+        }
+        C[row * N + col] = sum;
+    }
+}
diff --git a/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp b/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp
index 3a5bae1082..fc2d24a99f 100644
--- a/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp
+++ b/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp
@@ -6,89 +6,8 @@
 #include <type_traits>
 
 
-/*
- * An actor meant to represent matrix multiply 
- * kernel for now must be supplied since we do support kernel compilation
- * auto integration but it is via strings so you can supply own kernel to reduce needless recompilation, it is assumed you will supply a matrix multiple kernel to this
- * actor
- * it takes in 2 mem_ptrs to the representing matrix A and matrix B,matrix size, 
- * device number, and stream id  
- * and replys with the result 
- */
-
-
 namespace caf::cuda {
 
-// ------------------- Float version -------------------
-inline std::string mmulNS_kernel_source_float = R"(
-extern "C" __global__
-void matrixMulNS(const float* A, const float* B, float* C,
-                 int M, int K, int N) {
-    int row = blockIdx.y * blockDim.y + threadIdx.y;
-    int col = blockIdx.x * blockDim.x + threadIdx.x;
-
-    if (row < M && col < N) {
-        float temp = 0.0f;
-        for (int k = 0; k < K; ++k) {
-            temp += A[row * K + k] * B[k * N + col];
-        }
-        C[row * N + col] = temp;
-    }
-}
-)";
-
-// ------------------- Double version -------------------
-inline std::string mmulNS_kernel_source_double = R"(
-extern "C" __global__
-void matrixMulNS(const double* A, const double* B, double* C,
-                  int M, int K, int N) {
-    int row = blockIdx.y * blockDim.y + threadIdx.y;
-    int col = blockIdx.x * blockDim.x + threadIdx.x;
-
-    if (row < M && col < N) {
-        double temp = 0.0;
-        for (int k = 0; k < K; ++k) {
-            temp += A[row * K + k] * B[k * N + col];
-        }
-        C[row * N + col] = temp;
-    }
-}
-)";
-
-// ------------------- Integer version -------------------
-inline std::string mmulNS_kernel_source_int = R"(
-extern "C" __global__
-void matrixMulNS(const int* A, const int* B, int* C,
-                 int M, int K, int N) {
-    int row = blockIdx.y * blockDim.y + threadIdx.y;
-    int col = blockIdx.x * blockDim.x + threadIdx.x;
-
-    if (row < M && col < N) {
-        int temp = 0;
-        for (int k = 0; k < K; ++k) {
-            temp += A[row * K + k] * B[k * N + col];
-        }
-        C[row * N + col] = temp;
-    }
-}
-)";
-
-// ------------------- Template selector -------------------
-template <typename T>
-inline std::string get_mmulNS_kernel_source() {
-    if constexpr (std::is_same_v<T, float>) {
-        return mmulNS_kernel_source_float;
-    } else if constexpr (std::is_same_v<T, double>) {
-        return mmulNS_kernel_source_double;
-    } else if constexpr (std::is_same_v<T, int>) {
-        return mmulNS_kernel_source_int;
-    } else {
-        static_assert(!sizeof(T*), "Unsupported type for mmulNS kernel");
-    }
-}
-
-
-
 
 struct mmul_actor_not_square_state {
   static inline const char* name = "mmul_actor";
@@ -101,6 +20,8 @@ using mmul_async_command =
     mem_ptr<T>,
     mem_ptr<T>,
     out<T>,
+    in<int>,
+    in<int>,
     in<int>
   >;
 
@@ -120,38 +41,45 @@ caf::behavior mmul_actor_NS_fun(
 
     [=](mem_t matrixA,
         mem_t matrixB,
-        int N,
+        int M,
+	int K,
+	int N,
         int device_number,
         int stream_id)
         mutable -> mem_t
     {
-      program_ptr kernel = self->state().mmul_kernel;
-
-      const int THREADS = 32;
-      const int BLOCKS = (N + THREADS - 1) / THREADS;
-
-      nd_range dims(
-        BLOCKS, BLOCKS, 1,
-        THREADS, THREADS, 1);
-
-      out<T> arg3 = create_out_arg<T>(N * N);
-      in<int>  arg4 = create_in_arg<int>(N);
-
-      std::tuple<mem_t, mem_t, mem_t, mem_t> result_tuple =
-        mmul.run_async(
-          kernel,
-          dims,
-          stream_id,
-          0,
-          device_number,
-          matrixA,
-          matrixB,
-          arg3,
-          arg4);
-
-      mem_t output_device_buffer = std::get<2>(result_tuple);
-
-      return output_device_buffer;
+	    program_ptr kernel = self->state().mmul_kernel;
+	    const int THREADS_X = 32;
+	    const int THREADS_Y = 32;
+
+	    const int BLOCKS_X = (N + THREADS_X - 1) / THREADS_X;
+	    const int BLOCKS_Y = (M + THREADS_Y - 1) / THREADS_Y;
+
+	    nd_range dims(
+			    BLOCKS_X, BLOCKS_Y, 1,
+			    THREADS_X, THREADS_Y, 1);
+	    out<T> argC = create_out_arg<T>(M * N);
+	    in<int>  argN = create_in_arg<int>(N);
+	    in<int> argK = create_in_arg<int>(K);
+	    in<int> argM = create_in_arg<int>(M);
+
+	    std::tuple<mem_t, mem_t, mem_t, mem_t,mem_t,mem_t> result_tuple =
+		    mmul.run_async(
+				    kernel,
+				    dims,
+				    stream_id,
+				    0,
+				    device_number,
+				    matrixA,
+				    matrixB,
+				    argC,
+				    argM,
+				    argK,
+				    argN);
+
+	    mem_t output_device_buffer = std::get<2>(result_tuple);
+
+	    return output_device_buffer;
     }
 
   };

From 66fef483914742d51e4af5660003880be700d325 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 1 Mar 2026 10:41:15 -0600
Subject: [PATCH 0458/1000] Putting kernel next to mmul_actor.

---
 .../caf/component-actors/mmul_actor/mmul.cu      | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 libcaf_cuda/caf/component-actors/mmul_actor/mmul.cu

diff --git a/libcaf_cuda/caf/component-actors/mmul_actor/mmul.cu b/libcaf_cuda/caf/component-actors/mmul_actor/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/caf/component-actors/mmul_actor/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From 43eba3d1265f82ce1f378ab988920ffebf76cd84 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 1 Mar 2026 10:42:12 -0600
Subject: [PATCH 0459/1000] Initial commit.

---
 .../caf/component-actors/vector_add_actor/vector_add.cu     | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 libcaf_cuda/caf/component-actors/vector_add_actor/vector_add.cu

diff --git a/libcaf_cuda/caf/component-actors/vector_add_actor/vector_add.cu b/libcaf_cuda/caf/component-actors/vector_add_actor/vector_add.cu
new file mode 100644
index 0000000000..5cdda286f9
--- /dev/null
+++ b/libcaf_cuda/caf/component-actors/vector_add_actor/vector_add.cu
@@ -0,0 +1,6 @@
+extern "C" __global__ void vectorAdd(const int* A, const int* B, int* C, int N) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx < N) {
+        C[idx] = A[idx] + B[idx];
+    }
+}

From cfeae4351d95c06a24de615d6e275ff3bd16ff39 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 1 Mar 2026 11:06:38 -0600
Subject: [PATCH 0460/1000] Initial commit.

---
 .../rotation_actor/rotation_actor.hpp         | 51 +++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 libcaf_cuda/caf/component-actors/rotation_actor/rotation_actor.hpp

diff --git a/libcaf_cuda/caf/component-actors/rotation_actor/rotation_actor.hpp b/libcaf_cuda/caf/component-actors/rotation_actor/rotation_actor.hpp
new file mode 100644
index 0000000000..e767db60b8
--- /dev/null
+++ b/libcaf_cuda/caf/component-actors/rotation_actor/rotation_actor.hpp
@@ -0,0 +1,51 @@
+#pragma once
+#include <caf/all.hpp>
+#include "caf/cuda/all.hpp"
+#include "caf/component-actors/mmul_actor_no_square/mmul_actor_no_square.hpp"
+
+/*
+ * rotates a list of 2d vectors accoridng to a rotation matrix
+ */
+
+namespace caf::cuda {
+
+
+
+	struct rotation_actor_state {
+		caf::actor mmul_actor;
+	};
+
+	template <class T>
+	caf::behavior rotation_actor_fun(caf::actor_state<rotation_actor_state> * self,
+			caf::cuda:program_ptr mmul_kernel) {	
+		using mem_t = mem_ptr<T>;
+		self ->state().mmul_actor = self -> spawn(mmul_actor_NS_fun<T>,mmul_kernel);
+
+		return {
+		
+			[=](mem_t rotation_matrix, //2x2 rotation matrix
+			    mem_t points,
+			    int num_points,
+			    int device_number,
+			    int stream_id) {
+
+				int M = 2;
+				int K = 2;
+				int N = num_points;
+				return self->mail(rotation_matrix,
+						points,
+						M,
+						K,
+						N,
+						device_number,
+						stream_id).request(mmul_actor,std::chrono::seconds(100)).delgate(self->state().mmul_actor);
+				
+			}
+		
+		};
+	}
+
+
+
+
+} //namespace caf::cuda

From 26a17e25b6bd43bf1ded39c25a3003c89e048da5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 1 Mar 2026 11:08:44 -0600
Subject: [PATCH 0461/1000] Initial commit.

---
 .../rotation_actor_test/CMakeLists.txt        | 44 +++++++++
 .../rotation_actor_test/compile_kernels.sh    | 13 +++
 .../rotation_actor_test/main.test.cpp         | 92 +++++++++++++++++++
 .../rotation_actor_test/mmul.cu               | 14 +++
 4 files changed, 163 insertions(+)
 create mode 100644 libcaf_cuda/tests/component-actors-test/rotation_actor_test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/component-actors-test/rotation_actor_test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/component-actors-test/rotation_actor_test/main.test.cpp
 create mode 100644 libcaf_cuda/tests/component-actors-test/rotation_actor_test/mmul.cu

diff --git a/libcaf_cuda/tests/component-actors-test/rotation_actor_test/CMakeLists.txt b/libcaf_cuda/tests/component-actors-test/rotation_actor_test/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/rotation_actor_test/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/component-actors-test/rotation_actor_test/compile_kernels.sh b/libcaf_cuda/tests/component-actors-test/rotation_actor_test/compile_kernels.sh
new file mode 100755
index 0000000000..6af779abdd
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/rotation_actor_test/compile_kernels.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o vector_add.cubin
+echo "Generated mmul.cubin"
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/component-actors-test/rotation_actor_test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/rotation_actor_test/main.test.cpp
new file mode 100644
index 0000000000..aa601711d1
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/rotation_actor_test/main.test.cpp
@@ -0,0 +1,92 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/component-actors/vector_add_actor/vector_add_actor.hpp>
+#include <iostream>
+#include <vector>
+#include <chrono>
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+// Simple serial vector addition for verification
+template <typename T>
+void serial_vector_add(const std::vector<T>& a,
+                       const std::vector<T>& b,
+                       std::vector<T>& c) {
+    for (size_t i = 0; i < a.size(); ++i) {
+        c[i] = a[i] + b[i];
+    }
+}
+
+
+caf::cuda::command_runner<> runner;
+
+struct supervisor_state {};
+
+// Supervisor actor behavior
+caf::behavior vector_add_supervisor(caf::stateful_actor<supervisor_state> * self,
+                                    const std::vector<int>& vecA,
+                                    const std::vector<int>& vecB,
+                                    int vec_size,
+                                    caf::cuda::program_ptr program) {
+   
+            caf::actor worker = self ->spawn(caf::cuda::vector_add_actor_fun<int>, program);
+       
+	return {
+        [&,self,worker,vecA,vecB,vec_size](const unit_t&) mutable {
+
+            // Spawn the vector add worker actor
+            //caf::actor worker = self ->spawn(caf::cuda::vector_add_actor_fun<int>, program);
+
+            // Transfer memory to device
+            auto dA = runner.transfer_memory(0,1,caf::cuda::create_in_arg(vecA));
+            auto dB = runner.transfer_memory(0,1,caf::cuda::create_in_arg(vecB));
+
+            // Send to worker actor and request result
+            self->mail(dA, dB, vec_size, 0, 1) // device=0, stream=1
+                .request(worker, std::chrono::seconds(10))
+                .then([&](caf::cuda::mem_ptr<int> dC) {
+                    std::vector<int> result(vec_size);
+                    std::vector<int> vecC = dC->copy_to_host();
+
+                    serial_vector_add(vecA, vecB, result);
+
+                    if (result == vecC)
+                        std::cout << "Vector addition result matches!\n";
+                    else
+                        std::cout << "Vector addition result mismatch!\n";
+
+                    // Quit the supervisor
+                    self->quit();
+                });
+        }
+    };
+}
+
+void run_vector_add_test(actor_system& sys, int vec_size) {
+    caf::cuda::manager::init(sys);
+
+    // Create program pointer for the vector add kernel
+    auto program = caf::cuda::manager::get()
+                       .create_program_from_cubin("../vector_add.cubin", "vectorAdd");
+
+    // Generate test data
+    std::vector<int> vecA(vec_size, 2);
+    std::vector<int> vecB(vec_size, 3);
+
+    // Spawn the supervisor actor
+    caf::actor supervisor = sys.spawn(vector_add_supervisor,
+                                       vecA, vecB, vec_size, program);
+
+    // Trigger the supervisor
+    anon_mail(unit).send(supervisor);
+
+    // Wait for all actors to finish
+    sys.await_all_actors_done();
+}
+
+void caf_main(actor_system& sys) {
+    run_vector_add_test(sys, 1024);
+}
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/component-actors-test/rotation_actor_test/mmul.cu b/libcaf_cuda/tests/component-actors-test/rotation_actor_test/mmul.cu
new file mode 100644
index 0000000000..983d7208fb
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/rotation_actor_test/mmul.cu
@@ -0,0 +1,14 @@
+extern "C" __global__
+void mmul_non_square(const float* A, const float* B, float* C,
+                     int M, int K, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y; // row in C
+    int col = blockIdx.x * blockDim.x + threadIdx.x; // col in C
+
+    if (row < M && col < N) {
+        float sum = 0.0f;
+        for (int k = 0; k < K; ++k) {
+            sum += A[row * K + k] * B[k * N + col];
+        }
+        C[row * N + col] = sum;
+    }
+}

From 201f83b76998d03e68f2ee65844381dfb58aaac6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 1 Mar 2026 11:30:04 -0600
Subject: [PATCH 0462/1000] Wrote the initial test.

---
 .../rotation_actor_test/main.test.cpp         | 77 ++++++++++++-------
 1 file changed, 49 insertions(+), 28 deletions(-)

diff --git a/libcaf_cuda/tests/component-actors-test/rotation_actor_test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/rotation_actor_test/main.test.cpp
index aa601711d1..3ece22d4f7 100644
--- a/libcaf_cuda/tests/component-actors-test/rotation_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/component-actors-test/rotation_actor_test/main.test.cpp
@@ -1,6 +1,6 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
-#include <caf/component-actors/vector_add_actor/vector_add_actor.hpp>
+#include <caf/component-actors/rotation_actor/rotation_actor.hpp>
 #include <iostream>
 #include <vector>
 #include <chrono>
@@ -8,53 +8,63 @@
 using namespace caf;
 using namespace std::chrono_literals;
 
-// Simple serial vector addition for verification
 template <typename T>
-void serial_vector_add(const std::vector<T>& a,
-                       const std::vector<T>& b,
-                       std::vector<T>& c) {
-    for (size_t i = 0; i < a.size(); ++i) {
-        c[i] = a[i] + b[i];
+void rotate_points_cpu(const std::vector<T>& rotation_matrix,
+                       const std::vector<T>& points,
+                       std::vector<T>& result) {
+    // points: [x0, x1, ..., y0, y1, ...] row-major 2xM
+    int M = points.size() / 2;
+    for (int i = 0; i < M; ++i) {
+        T x = points[i];
+        T y = points[M + i];
+
+        // Apply rotation: [x', y'] = R * [x, y]
+        result[i] = rotation_matrix[0] * x + rotation_matrix[1] * y;       // x'
+        result[M + i] = rotation_matrix[2] * x + rotation_matrix[3] * y;   // y'
     }
 }
 
 
+
 caf::cuda::command_runner<> runner;
 
 struct supervisor_state {};
 
 // Supervisor actor behavior
-caf::behavior vector_add_supervisor(caf::stateful_actor<supervisor_state> * self,
-                                    const std::vector<int>& vecA,
-                                    const std::vector<int>& vecB,
-                                    int vec_size,
+caf::behavior rotation_supervisor(caf::stateful_actor<supervisor_state> * self,
                                     caf::cuda::program_ptr program) {
    
-            caf::actor worker = self ->spawn(caf::cuda::vector_add_actor_fun<int>, program);
+            caf::actor worker = self ->spawn(caf::cuda::rotation_actor_fun<float>, program);
        
 	return {
-        [&,self,worker,vecA,vecB,vec_size](const unit_t&) mutable {
+        [=](std::vector<float> rotation_matrix,
+		std::vector<float> points) mutable {
 
             // Spawn the vector add worker actor
             //caf::actor worker = self ->spawn(caf::cuda::vector_add_actor_fun<int>, program);
 
             // Transfer memory to device
-            auto dA = runner.transfer_memory(0,1,caf::cuda::create_in_arg(vecA));
-            auto dB = runner.transfer_memory(0,1,caf::cuda::create_in_arg(vecB));
+            auto rotation_matrix_memory = runner.transfer_memory(0,1,caf::cuda::create_in_arg(vecA));
+            auto  points_memory = runner.transfer_memory(0,1,caf::cuda::create_in_arg(vecB));
 
+	    int num_points = points.size() / 2;
             // Send to worker actor and request result
-            self->mail(dA, dB, vec_size, 0, 1) // device=0, stream=1
+            self->mail(rotation_matrix_memory,
+                                 points_memory,
+                                 num_points,
+                                 device_number,
+                                 stream_id) // device=0, stream=1
                 .request(worker, std::chrono::seconds(10))
-                .then([&](caf::cuda::mem_ptr<int> dC) {
-                    std::vector<int> result(vec_size);
-                    std::vector<int> vecC = dC->copy_to_host();
+                .then([&](caf::cuda::mem_ptr<float> rotated_points) {
+                    std::vector<float> result(points.size());
+                    std::vector<float> vecC = rotated_points->copy_to_host();
 
-                    serial_vector_add(vecA, vecB, result);
+                    rotate_points_cpu(rotation_matrix, points, result);
 
                     if (result == vecC)
-                        std::cout << "Vector addition result matches!\n";
+                        std::cout << "rotation result matches!\n";
                     else
-                        std::cout << "Vector addition result mismatch!\n";
+                        std::cout << "rotation result mismatch!\n";
 
                     // Quit the supervisor
                     self->quit();
@@ -68,18 +78,29 @@ void run_vector_add_test(actor_system& sys, int vec_size) {
 
     // Create program pointer for the vector add kernel
     auto program = caf::cuda::manager::get()
-                       .create_program_from_cubin("../vector_add.cubin", "vectorAdd");
+                       .create_program_from_cubin("../mmul.cubin", "mmul_non_square");
+
 
+  
     // Generate test data
-    std::vector<int> vecA(vec_size, 2);
-    std::vector<int> vecB(vec_size, 3);
 
+    std::vector<float> rotation_matrix = {
+	    0.0f, -1.0f,  // first row
+	    1.0f,  0.0f   // second row
+    };
+
+
+    std::vector<float> points = {
+    1.0f, 2.0f, 4.0f, 8.0f,   // x-coordinates
+    1.0f, 2.0f, 4.0f, 8.0f    // y-coordinates
+};
+
+    
     // Spawn the supervisor actor
-    caf::actor supervisor = sys.spawn(vector_add_supervisor,
-                                       vecA, vecB, vec_size, program);
+    caf::actor supervisor = sys.spawn(vector_add_supervisor,program);
 
     // Trigger the supervisor
-    anon_mail(unit).send(supervisor);
+    anon_mail(rotation_matrix,points).send(supervisor);
 
     // Wait for all actors to finish
     sys.await_all_actors_done();

From cf98d1afda0cfe6d131fdcfdd8074e83d06e3b2f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 1 Mar 2026 11:35:23 -0600
Subject: [PATCH 0463/1000] Fixed syntax errors.

---
 .../component-actors/rotation_actor/rotation_actor.hpp    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/caf/component-actors/rotation_actor/rotation_actor.hpp b/libcaf_cuda/caf/component-actors/rotation_actor/rotation_actor.hpp
index e767db60b8..a9c29f2b35 100644
--- a/libcaf_cuda/caf/component-actors/rotation_actor/rotation_actor.hpp
+++ b/libcaf_cuda/caf/component-actors/rotation_actor/rotation_actor.hpp
@@ -1,7 +1,7 @@
 #pragma once
 #include <caf/all.hpp>
 #include "caf/cuda/all.hpp"
-#include "caf/component-actors/mmul_actor_no_square/mmul_actor_no_square.hpp"
+#include "caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp"
 
 /*
  * rotates a list of 2d vectors accoridng to a rotation matrix
@@ -16,8 +16,8 @@ namespace caf::cuda {
 	};
 
 	template <class T>
-	caf::behavior rotation_actor_fun(caf::actor_state<rotation_actor_state> * self,
-			caf::cuda:program_ptr mmul_kernel) {	
+	caf::behavior rotation_actor_fun(caf::stateful_actor<rotation_actor_state> * self,
+			caf::cuda::program_ptr mmul_kernel) {	
 		using mem_t = mem_ptr<T>;
 		self ->state().mmul_actor = self -> spawn(mmul_actor_NS_fun<T>,mmul_kernel);
 
@@ -38,7 +38,7 @@ namespace caf::cuda {
 						K,
 						N,
 						device_number,
-						stream_id).request(mmul_actor,std::chrono::seconds(100)).delgate(self->state().mmul_actor);
+						stream_id).delgate(self->state().mmul_actor);
 				
 			}
 		

From d0293a2c22f5864283c7e4a9ebcb8ade99090202 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 1 Mar 2026 11:42:42 -0600
Subject: [PATCH 0464/1000] Fixed compiler errors.

---
 .../mmul_actor_not_square/mmul_actor_not_square.hpp           | 4 ++--
 .../caf/component-actors/rotation_actor/rotation_actor.hpp    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp b/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp
index fc2d24a99f..d892e39231 100644
--- a/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp
+++ b/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp
@@ -58,12 +58,12 @@ caf::behavior mmul_actor_NS_fun(
 	    nd_range dims(
 			    BLOCKS_X, BLOCKS_Y, 1,
 			    THREADS_X, THREADS_Y, 1);
-	    out<T> argC = create_out_arg<T>(M * N);
+	    out<T> argC = create_out_arg_with_size<T>(M * N);
 	    in<int>  argN = create_in_arg<int>(N);
 	    in<int> argK = create_in_arg<int>(K);
 	    in<int> argM = create_in_arg<int>(M);
 
-	    std::tuple<mem_t, mem_t, mem_t, mem_t,mem_t,mem_t> result_tuple =
+	    auto result_tuple =
 		    mmul.run_async(
 				    kernel,
 				    dims,
diff --git a/libcaf_cuda/caf/component-actors/rotation_actor/rotation_actor.hpp b/libcaf_cuda/caf/component-actors/rotation_actor/rotation_actor.hpp
index a9c29f2b35..9cc6405c1f 100644
--- a/libcaf_cuda/caf/component-actors/rotation_actor/rotation_actor.hpp
+++ b/libcaf_cuda/caf/component-actors/rotation_actor/rotation_actor.hpp
@@ -38,7 +38,7 @@ namespace caf::cuda {
 						K,
 						N,
 						device_number,
-						stream_id).delgate(self->state().mmul_actor);
+						stream_id).delegate(self->state().mmul_actor);
 				
 			}
 		

From b90978f65465710db3513c7ea20c6aeb618cf9fe Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 1 Mar 2026 11:43:19 -0600
Subject: [PATCH 0465/1000] Fixed naming issue of compiled kernel.

---
 .../rotation_actor_test/compile_kernels.sh                      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/component-actors-test/rotation_actor_test/compile_kernels.sh b/libcaf_cuda/tests/component-actors-test/rotation_actor_test/compile_kernels.sh
index 6af779abdd..f32480e5cb 100755
--- a/libcaf_cuda/tests/component-actors-test/rotation_actor_test/compile_kernels.sh
+++ b/libcaf_cuda/tests/component-actors-test/rotation_actor_test/compile_kernels.sh
@@ -7,7 +7,7 @@ SM_ARCH="sm_${ARCH/./}"
 echo "Using NVCC arch flag: $SM_ARCH"
 
 # Compile mmul.cu to cubin in current directory
-nvcc -arch=$SM_ARCH -cubin mmul.cu -o vector_add.cubin
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
 echo "Generated mmul.cubin"
 echo "All kernels compiled successfully!"
 

From 2ffd8518df66f27ed76f357a43aed9a49212773d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 1 Mar 2026 11:47:29 -0600
Subject: [PATCH 0466/1000] Fixed issue on line 58 by changing lambda to
 capture by value rather than refence to fix issue caused by refences
 outliving there values.

---
 .../rotation_actor_test/main.test.cpp                | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/tests/component-actors-test/rotation_actor_test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/rotation_actor_test/main.test.cpp
index 3ece22d4f7..9031b19ece 100644
--- a/libcaf_cuda/tests/component-actors-test/rotation_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/component-actors-test/rotation_actor_test/main.test.cpp
@@ -44,18 +44,18 @@ caf::behavior rotation_supervisor(caf::stateful_actor<supervisor_state> * self,
             //caf::actor worker = self ->spawn(caf::cuda::vector_add_actor_fun<int>, program);
 
             // Transfer memory to device
-            auto rotation_matrix_memory = runner.transfer_memory(0,1,caf::cuda::create_in_arg(vecA));
-            auto  points_memory = runner.transfer_memory(0,1,caf::cuda::create_in_arg(vecB));
+		caf::cuda::mem_ptr<float> rotation_matrix_memory = runner.transfer_memory(0,1,caf::cuda::create_in_arg(rotation_matrix));
+		caf::cuda::mem_ptr<float>  points_memory = runner.transfer_memory(0,1,caf::cuda::create_in_arg(points));
 
 	    int num_points = points.size() / 2;
             // Send to worker actor and request result
             self->mail(rotation_matrix_memory,
                                  points_memory,
                                  num_points,
-                                 device_number,
-                                 stream_id) // device=0, stream=1
+                                 0,
+                                 1) // device=0, stream=1
                 .request(worker, std::chrono::seconds(10))
-                .then([&](caf::cuda::mem_ptr<float> rotated_points) {
+                .then([=](caf::cuda::mem_ptr<float> rotated_points) {
                     std::vector<float> result(points.size());
                     std::vector<float> vecC = rotated_points->copy_to_host();
 
@@ -97,7 +97,7 @@ void run_vector_add_test(actor_system& sys, int vec_size) {
 
     
     // Spawn the supervisor actor
-    caf::actor supervisor = sys.spawn(vector_add_supervisor,program);
+    caf::actor supervisor = sys.spawn(rotation_supervisor,program);
 
     // Trigger the supervisor
     anon_mail(rotation_matrix,points).send(supervisor);

From 8633e3ef8dfd97c8f9a792ed4c4c7643f3dbf4c1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Mar 2026 09:50:59 -0600
Subject: [PATCH 0467/1000] Initial commit.

---
 .../sync_actor/sync_actor.hpp                 | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 libcaf_cuda/caf/component-actors/sync_actor/sync_actor.hpp

diff --git a/libcaf_cuda/caf/component-actors/sync_actor/sync_actor.hpp b/libcaf_cuda/caf/component-actors/sync_actor/sync_actor.hpp
new file mode 100644
index 0000000000..342806644b
--- /dev/null
+++ b/libcaf_cuda/caf/component-actors/sync_actor/sync_actor.hpp
@@ -0,0 +1,38 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include "caf/cuda/mem_ref.hpp"
+#include "caf/cuda/control-layer/all-control-layer.hpp"
+
+/*
+ * This actor is meant to act as a callback actor that will synchronize with a 
+ * stream bound to a mem_ref and reply with when it is awake 
+ * it can also synchronize and release the response token when it awakes for the 
+ * scheduler actor 
+ * giving the kernels as events illusion for the scheduler actor 
+ */
+
+
+namespace caf::cuda {
+
+template <class T>
+void synch_actor_fun(caf::blocking_actor* self) {
+
+  self->receive(
+    [&](mem_ptr<T> d_mem) -> mem_ptr<T> {
+
+      // blocking call is safe here
+      d_mem -> synchronize();  
+    return d_mem;
+    },
+    [&](mem_ptr<T> d_mem, response_token res) -> mem_ptr<T> {
+      d_mem->synchronize();
+
+      res ->release();
+       
+      return d_mem;
+    }
+  );
+}
+
+} // namespace caf::cuda

From 6675edc2d4af1db2450436eefd715626dade4ad8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Mar 2026 09:54:30 -0600
Subject: [PATCH 0468/1000] Updated header file to include all component
 actors.

---
 .../caf/component-actors/all-component-actors.hpp  | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/libcaf_cuda/caf/component-actors/all-component-actors.hpp b/libcaf_cuda/caf/component-actors/all-component-actors.hpp
index 6f70f09bee..c2222f5364 100644
--- a/libcaf_cuda/caf/component-actors/all-component-actors.hpp
+++ b/libcaf_cuda/caf/component-actors/all-component-actors.hpp
@@ -1 +1,15 @@
 #pragma once
+
+#include "caf/component-actors/mem_transfer_actor/mem_transfer_actor.hpp"
+
+#include "caf/component-actors/mmul_actor/mmul_actor.hpp"
+
+#include "caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp"
+
+#include "caf/component-actors/random_mmul_gen_multiply_actor/random_mmul_gen_multiply_actor.hpp"
+
+#include "caf/component-actors/rotation_actor/rotation_actor.hpp"
+
+#include "caf/component-actors/sync_actor/sync_actor.hpp"
+
+#include "caf/component-actors/vector_add_actor/vector_add_actor.hpp"

From 5ba2e32465e08d110b452083671058d18427a81e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Mar 2026 10:01:10 -0600
Subject: [PATCH 0469/1000] Initial commit.

---
 .../synch_actor_test/CMakeLists.txt           |   44 +
 .../synch_actor_test/compile_kernels.sh       |   21 +
 .../synch_actor_test/fault.cu                 |   33 +
 .../synch_actor_test/genMatrix.cu             |   16 +
 .../synch_actor_test/main.test.cpp            | 1064 +++++++++++++++++
 .../synch_actor_test/mmul.cu                  |   16 +
 .../synch_actor_test/shared_mmul.cu           |   51 +
 7 files changed, 1245 insertions(+)
 create mode 100644 libcaf_cuda/tests/component-actors-test/synch_actor_test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/component-actors-test/synch_actor_test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/component-actors-test/synch_actor_test/fault.cu
 create mode 100644 libcaf_cuda/tests/component-actors-test/synch_actor_test/genMatrix.cu
 create mode 100644 libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
 create mode 100644 libcaf_cuda/tests/component-actors-test/synch_actor_test/mmul.cu
 create mode 100644 libcaf_cuda/tests/component-actors-test/synch_actor_test/shared_mmul.cu

diff --git a/libcaf_cuda/tests/component-actors-test/synch_actor_test/CMakeLists.txt b/libcaf_cuda/tests/component-actors-test/synch_actor_test/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/synch_actor_test/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/component-actors-test/synch_actor_test/compile_kernels.sh b/libcaf_cuda/tests/component-actors-test/synch_actor_test/compile_kernels.sh
new file mode 100755
index 0000000000..1bcba9f066
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/synch_actor_test/compile_kernels.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+#compile fault.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin fault.cu -o fault.cubin
+echo "Generated fault.cubin"
+
+# Compile genMatrix.cu to fatbin in current directory
+nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
+echo "Generated generate_random_matrix.fatbin"
+
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/component-actors-test/synch_actor_test/fault.cu b/libcaf_cuda/tests/component-actors-test/synch_actor_test/fault.cu
new file mode 100644
index 0000000000..06ec34c594
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/synch_actor_test/fault.cu
@@ -0,0 +1,33 @@
+#include <cuda_runtime.h>
+#include <curand.h>
+#include <curand_kernel.h>
+#include <iostream>
+#include <cstdlib>
+
+// Step 1: Initialize denominators with ~50% zeros using cuRAND
+extern "C" __global__ void init_denominators(float* denominators, int n, unsigned long long seed) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+
+    curandState state;
+    curand_init(seed, idx, 0, &state);
+    float rand_val = curand_uniform(&state);
+    denominators[idx] = (rand_val < 0.5f) ? 0.0f : 1.0f;  // ~50% chance of zero
+}
+
+// Step 2: Perform division (potential div by zero -> Inf)
+extern "C" __global__ void perform_division(float* numerators, float* denominators, float* results, int n) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    results[idx] = numerators[idx] / denominators[idx];  // Triggers Inf if denominator == 0
+}
+
+// Step 3: Simple reduction to sum results (propagates Inf if present)
+extern "C" __global__ void sum_results(float* results, float* final_sum, int n) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (idx < n) {
+        // atomic add each element directly to the final sum
+        atomicAdd(final_sum, results[idx]);
+    }
+}
diff --git a/libcaf_cuda/tests/component-actors-test/synch_actor_test/genMatrix.cu b/libcaf_cuda/tests/component-actors-test/synch_actor_test/genMatrix.cu
new file mode 100644
index 0000000000..98189eb4d0
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/synch_actor_test/genMatrix.cu
@@ -0,0 +1,16 @@
+
+#include <curand_kernel.h>
+//generate_random_matrix
+extern "C" __global__
+void generate_random_matrix(int* matrix, int total_elements, int seed, int max_val) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= total_elements) return;
+
+    curandState state;
+    curand_init((unsigned long long)seed, idx, 0, &state);
+
+    unsigned int r = curand(&state);
+    matrix[idx] = r % max_val;
+}
+
+
diff --git a/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
new file mode 100644
index 0000000000..3dafb5e6c6
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
@@ -0,0 +1,1064 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include <unistd.h>
+#include "caf/actor_registry.hpp"
+#include <chrono>
+#include <iostream>
+//#include <caf/atoms.hpp>
+
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+
+
+
+struct mmul_actor_state {
+  static inline const char* name = "my_actor";
+  int last_N = 0; // example state variable
+  int id = rand(); // an actor id
+  int times = 0;
+};
+
+
+
+
+//commands classes used to launch kernels
+using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
+using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
+
+using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
+
+mmulCommand mmul;
+matrixGenCommand randomMatrix;
+mmulAsyncCommand mmulAsync;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+
+
+
+
+
+struct exit_actor_state {
+	int completed = 0;
+};
+
+
+caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
+
+
+	return {
+		[=](int num_completed) {
+			self->state().completed += num_completed;
+			
+//			std::cout << "Actors finished is " << self->state().completed << "\n";
+			if (self->state().completed >= limit) {
+			
+				caf::cuda::manager::shutdown();
+				self->quit();
+			}
+		}
+	};
+
+
+}
+
+
+
+
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun(
+    caf::stateful_actor<mmul_actor_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims)
+{
+
+
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+	caf::actor scheduler = mgr.get_scheduler_actor();
+
+	//send a launch token
+	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+			program,
+			dims,
+			0,
+			"hello",
+			self
+			);
+	self -> mail(launch_token).send(scheduler);
+
+	return {
+
+		[=] (caf::cuda::response_token_ptr res_token) {
+
+			if (res_token -> getType() == LAUNCH_RESPONSE) {
+				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+				//assume N = 1024
+				std::vector<int> matrix1(N*N);
+				matrix1.reserve(N);
+				std::vector<int> matrix2(N*N);
+				matrix2.reserve(N);
+
+				//std::cout << "GPU ACTOR sending data to compute\n";
+				self -> mail(matrix1,matrix2,res_token,N).send(self);
+
+			}
+			else {
+				std::cout << "Got a memory response token\n";
+			}
+			//token should drop out of scope now, triggering a response 
+		},
+
+			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+			[=](const std::vector<int>& matrixA,
+					const std::vector<int>& matrixB,
+					const caf::cuda::response_token_ptr& res_token, int N) {
+
+
+				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
+
+				//caf::cuda::launch_response_token& kt =
+				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
+
+				//std::cout << "GPU ACTOR  computing\n";
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+				//create program and dims   
+				auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+				const int THREADS = 32;
+				const int BLOCKS = (N + THREADS - 1) / THREADS;
+				caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrixA);
+				auto arg2 = caf::cuda::create_in_arg(matrixB);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
+
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				//std::cout << "GPU ACTOR done  computing\n";
+				//verify its own result 
+				self -> mail(matrixA,matrixB,matrixC,N).send(self);
+
+			},
+
+					// 3rd handler: CPU atom + matrices + N
+					[=](const std::vector<int>& matrixA,
+							const std::vector<int>& matrixB,
+							const std::vector<int>& matrixC,
+							int N) {
+
+						using clock = std::chrono::high_resolution_clock;
+
+						auto start = clock::now();
+
+						//std::cout << "GPU ACTOR verifying\n";
+
+						std::vector<int> result(N * N);
+
+						serial_matrix_multiply(matrixA, matrixB, result, N);
+
+						if (result == matrixC) {
+							std::cout << "actor with id " << self->state().id
+								<< " references match\n";
+						} else {
+							std::cout << "actor with id " << self->state().id
+								<< " references did not match\n";
+						}
+
+						auto end = clock::now();
+
+						auto ms =
+							std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+						std::cout << "[TIMING] verification took "
+							<< ms << " ms (actor id "
+							<< self->state().id << ")\n";
+
+						// signal exit actor and quit
+						self->mail(1).send(exit_actor);
+						self->quit();
+
+					}
+	};
+}
+
+
+
+
+
+// this actor will not verify its results
+// great for performance analysis 
+caf::behavior mmul_actor_fun_no_verify(
+    caf::stateful_actor<mmul_actor_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims,
+    bool request
+    )
+{
+
+	//set the value of N correctly to overide the base option.	
+
+	caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+	caf::actor scheduler = mgr.get_scheduler_actor();
+
+	if (request) {
+		//send a launch token
+		caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+				program,
+				dims,
+				0,
+				"hello",
+				self
+				);
+		self -> mail(launch_token).send(scheduler);
+	}
+	return {
+
+		[=] (caf::cuda::response_token_ptr res_token) {
+
+			if (res_token -> getType() == LAUNCH_RESPONSE) {
+				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
+				//assume N = 1024
+				std::vector<int> matrix1(N*N);
+				matrix1.reserve(N);
+				std::vector<int> matrix2(N*N);
+				matrix2.reserve(N);
+
+
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+				//create program and dims   
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrix1);
+				auto arg2 = caf::cuda::create_in_arg(matrix2);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
+
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+			
+			
+				//mask the transfer back to the cpu for scheduler
+				res_token -> release();
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				//std::cout << "GPU ACTOR done  computing\n";
+				// signal exit actor and quit
+				self->mail(1).send(exit_actor);
+				self->quit();
+
+				//std::cout << "GPU ACTOR sending data to compute\n";
+			//	self -> mail(matrix1,matrix2,res_token,N).send(self);
+
+			}
+			else {
+				std::cout << "Got a memory response token\n";
+			}
+			//token should drop out of scope now, triggering a response 
+		},
+
+			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+			[=](const std::vector<int>& matrixA,
+					const std::vector<int>& matrixB,
+					const caf::cuda::response_token_ptr& res_token, int N) {
+
+
+				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
+
+				//caf::cuda::launch_response_token& kt =
+				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
+
+				//std::cout << "GPU ACTOR  computing\n";
+				caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+				//create program and dims   
+				//create args
+				auto arg1 = caf::cuda::create_in_arg(matrixA);
+				auto arg2 = caf::cuda::create_in_arg(matrixB);
+				auto arg3 = caf::cuda::create_out_arg(N*N);
+				auto arg4 = caf::cuda::create_in_arg(N);
+
+				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
+			
+			
+				//mask the transfer back to the cpu for scheduler
+				res_token -> release();
+				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				//std::cout << "GPU ACTOR done  computing\n";
+				// signal exit actor and quit
+				self->mail(1).send(exit_actor);
+				self->quit();
+
+			}
+
+				
+	};
+}
+
+
+
+
+
+template <class Fn>
+double time_run(Fn&& fn) {
+  auto start = std::chrono::steady_clock::now();
+  fn();
+  auto end = std::chrono::steady_clock::now();
+  std::chrono::duration<double> elapsed = end - start;
+  return elapsed.count();
+}
+
+
+//this test is meant to demonstrate the fact that scheduler actors can 
+//migrate work to correct load imbalance
+//the sizes should be large enough such that the tests exceed 4-5 seconds in total
+//otherwise the schedulers wont care to do this fast enough
+void run_load_balance_test(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get(); 
+
+
+    //set the behaviors of each scheduler actor
+    for (int i = 0; i < mgr.get_num_devices();i++) {
+	    mgr.send_scheduler_actor_message("multilevel",i);
+    }
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    std::vector<caf::cuda::token_ptr> tokens(num_actors);
+
+    std::mt19937 rng(123456);
+    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+
+    const int THREADS = 32;
+
+    auto t_start = std::chrono::steady_clock::now();
+
+for (int i = 0; i < num_actors; ++i) {
+    int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
+    int BLOCKS = (N + THREADS - 1) / THREADS;
+
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1,
+                             THREADS, THREADS, 1);
+
+    caf::actor a = sys.spawn(
+        mmul_actor_fun_no_verify,
+        exit_actor,
+        N,
+        program,
+        dims,
+        false
+    );
+
+    tokens[i] = caf::cuda::make_launch_token(
+        program,
+        dims,
+        0, // memory usage is zero for now, we still do not track it at all
+        "hello",
+        a
+    );
+}
+
+auto t_end = std::chrono::steady_clock::now();
+
+auto us = std::chrono::duration_cast<std::chrono::microseconds>(
+              t_end - t_start
+          ).count();
+
+std::cout << "Actor spawn + token creation loop took "
+          << us << " us\n";
+
+   
+	  //send the tokens to only 1 GPU and let them
+	  //figure out that there is a load imbalance 
+	  mgr.send_scheduler_actor_message(tokens);
+          sys.await_all_actors_done();
+}
+
+
+//-------------------------------------try load balancing with actors with dependencies
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+// --- command runner types (put near top of file) -------------------------
+using initCommand =
+  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, in<int>, in<unsigned long long>>;
+
+using divCommand  =
+  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, in<int>>;
+
+using sumCommand  =
+  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, in<int>>;
+
+// single instances (can be file-global)
+static initCommand init_cmd;
+static divCommand  div_cmd;
+static sumCommand  sum_cmd;
+
+// --- pipeline actor state (device buffers persist here) ------------------
+struct pipeline_actor_state {
+    int id = rand();
+
+    int finished_stage = 0;
+
+    // device-side buffers that must persist across stages:
+    caf::cuda::mem_ptr<float> d_denoms;
+    caf::cuda::mem_ptr<float> d_results;
+    caf::cuda::mem_ptr<float> d_sum;
+    
+};
+
+// --- corrected pipeline_actor -------------------------------------------
+behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
+                        actor supervisor,
+                        caf::cuda::program_ptr p1,
+                        caf::cuda::program_ptr p2,
+                        caf::cuda::program_ptr p3,
+                        int n)
+{
+    // host-side scratch (only used for post-stage2 NaN/Inf detection)
+    std::vector<float> h_results;
+
+
+    // nd_range used for all stages (adapt to your kernels as needed)
+    caf::cuda::nd_range range{
+        {(n + 255) / 256, 1, 1},
+        {256, 1, 1}
+    };
+
+    // helper to create and send a launch token
+    auto launch = [&](caf::cuda::program_ptr prog, const std::string& stage) {
+        auto tok = make_launch_token(
+            prog,
+            range,
+            /*memory_usage=*/static_cast<int>(sizeof(float) * n),
+            stage,
+            self,
+            self->state().id  // dependency/demo id
+        );
+
+	//do not specifiy a device number to send it to, let it figure it out
+	//caf::cuda::manager::get().send_scheduler_actor_message(tok);
+    
+	//forcefully send to the first scheduler actor
+	caf::actor scheduler = caf::cuda::manager::get().get_scheduler_actor();
+	anon_mail(tok).send(scheduler);
+
+    };
+
+    // fire all three tokens (scheduler will reply with response_token on grants)
+    launch(p1, "stage1");
+    launch(p2, "stage2");
+    launch(p3, "stage3");
+
+    return {
+
+        // handle response tokens by name — opaque to reclaim payload
+        [=](caf::cuda::response_token_ptr res_token) mutable {
+
+            const auto& stage = res_token->name();
+
+	    if (res_token->getType() == LAUNCH_RESPONSE) {
+
+		    // --------------------- Stage 1: init_denominators ---------------------
+		    if (stage == "stage1") {
+			    // allocate device buffer for denominators (persist in state)
+
+			    //std::cout << "Starting stage 1\n";
+			    unsigned long long seed = static_cast<unsigned long long>(
+					    std::chrono::high_resolution_clock::now().time_since_epoch().count()
+					    );
+
+
+
+			    out<float> buffer = caf::cuda::create_out_arg_with_size<float>(n);
+			    self->state().d_denoms = init_cmd.transfer_memory(res_token,buffer);
+
+			    // run kernel on the stream/device from res_token
+			    // kernel signature: (float* denominators, int n, unsigned long long seed)
+			    init_cmd.run_async(
+					    p1,
+					    range,
+					    res_token,                            // uses token's stream/device
+					    self->state().d_denoms,               // device buffer
+					    caf::cuda::create_in_arg(n),          // n
+					    caf::cuda::create_in_arg(seed)     // seed
+					    );
+
+			    //std::cout << "Finished stage 1\n";
+			    // stage1 intentionally no checks — data may contain zeros
+			    
+			    self->state().finished_stage++;
+			    return;
+		    }
+
+		    // --------------------- Stage 2: perform_division ---------------------
+		    if (stage == "stage2") {
+			    // allocate device buffer for results (persist in state)
+			    std::vector<float> buffer1(n);
+
+			    //std::cout << "Starting stage 2\n";
+			    self->state().d_results = div_cmd.transfer_memory(res_token,out<float>{buffer1});
+
+			    // create a host numerators vector (all ones)
+			    std::vector<float> h_nums(n, 1.0f);
+
+			    // transfer numerators to device on the token's stream/device
+			    // transfer_memory returns a caf::cuda::mem_ptr<float>
+			    auto d_nums = div_cmd.transfer_memory(res_token, in_out<float>{h_nums});
+
+			    // run division kernel on the token's stream/device:
+			    // kernel signature: (float* numerators, float* denominators, float* results, int n)
+			    
+			    if (self->state().d_denoms == nullptr) {
+			    
+				    std::cout << "Error with pipeline actor d_denoms is nullptr\n";
+			    
+			    }
+
+
+			    if (d_nums == nullptr) {
+			    
+				    std::cout << "Error with pipeline actor d_denoms is nullptr\n";
+			    
+			    }
+
+			    
+			    div_cmd.run(
+					    p2,
+					    range,
+					    res_token,
+					    d_nums,
+					    self->state().d_denoms,
+					    self->state().d_results,
+					    caf::cuda::create_in_arg(n)
+				       );
+
+
+			    //there could be a division by zero in here 
+			    //but this is a load balancing test
+			    //not a fault test,
+			    //go see the fault tolerance test to see how thats handled	
+
+			    /*
+			    // extract the device results back to host for verification.
+			    // extract_vector will synchronize as needed.
+			    h_results = self->state().d_results -> copy_to_host();
+
+			    // check for NaN/Inf AFTER the kernel finished
+			    bool fault = false;
+			    for (float v : h_results) {
+				    if (!std::isfinite(v)) {
+					    fault = true;
+					    break;
+				    }
+			    }
+
+			    if (fault) {
+				    // inform supervisor and exit;
+				    anon_mail(std::string("crash")).send(supervisor);
+				    self->quit();
+				    return;
+			    }
+
+			    // stage2 passed — keep d_results in state for stage3
+			    */
+
+			    self->state().finished_stage++;
+			    return;
+		    }
+
+		    // --------------------- Stage 3: sum_results --------------------------
+		    if (stage == "stage3") {
+			    // allocate device scalar for sum result
+
+			    //std::cout << "Starting stage 3\n";
+			    std::vector<float> buffer1(1);
+
+			    self->state().d_sum = div_cmd.transfer_memory(res_token,out<float>{buffer1});
+
+			    // run reduction on the token's stream/device:
+			    // kernel signature: (float* results, float* final_sum, int n)
+			    sum_cmd.run(
+					    p3,
+					    range,
+					    res_token,
+					    self->state().d_results,
+					    self->state().d_sum,
+					    caf::cuda::create_in_arg(n)
+				       );
+
+			    // extract final scalar
+
+			    std::vector<float> buf = self->state().d_sum -> copy_to_host();
+			    float final_sum = buf[0];
+			    //std::cout << "[pipeline] completed, sum = " << final_sum << "\n";
+
+			    anon_mail(1).send(supervisor);
+
+			    // quit the pipeline actor
+			    self->state().finished_stage++;
+			    self->quit();
+			    return;
+		    }
+	    }
+
+	    else if (res_token->getType() == TRANSFER) {
+	   
+		    std::cout << "Got a transfer token\n";
+
+		   if (stage == "stage1") {
+			   res_token->release(); // no dependencies at this point clear to continue
+		   	   return;
+		   } 
+
+		   else if (stage == "stage2") {
+		  
+			   //at this point the d_results needs to be transfer over to the other device
+			   
+		           std::cout << "Transfering at stage 2\n";
+			      if (self->state().d_denoms == nullptr) {
+			    
+				    std::cout << "Error with pipeline actor d_denoms is nullptr during transfer\n";
+				    std::cout << "Completed stage is " << self->state().finished_stage << "\n";
+			    
+			    }
+			   
+			   
+			   
+			   //TODO FIX SEGFAULT TRIGGERED BY THIS LINE 
+			    in_out<float> temp_buffer{self->state().d_denoms -> copy_to_host()};
+
+			    self->state().d_denoms = div_cmd.transfer_memory(res_token,temp_buffer);
+			    //all done
+			    res_token->release();
+		   	    return;
+		   }
+
+		   else if (stage == "stage3") {
+		   
+			   //at this point d_results needs to be copied over to the new GPU   
+			    self->state().d_results = div_cmd.transfer_memory(res_token,in_out<float>{self->state().d_results->copy_to_host()});
+			    //all done
+			    res_token->release();
+			    return;
+		   }
+
+		   else { 
+			   std::cout << "Error unrecognized transfer token\n";
+		   }
+		    
+	    
+	    }
+
+            // unknown stage: ignore or log
+            std::cerr << "[pipeline] received unknown response token: " << stage << "\n";
+	}
+    };
+}
+
+
+
+
+
+
+
+
+
+//this test is meant to demonstrate the fact that scheduler actors can 
+//migrate work to correct load imbalance
+//the sizes should be large enough such that the tests exceed 4-5 seconds in total
+//otherwise the schedulers wont care to do this fast enough
+//As it turns out pipeline actor does not do enough work in order to convince the GPUs
+//that it should even attempt to migrate it
+void run_load_balance_test_with_dependencies(
+    caf::actor_system& sys,
+    const int n,
+    int num_actors,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get(); 
+
+
+    //set the behaviors of each scheduler actor
+    for (int i = 0; i < mgr.get_num_devices();i++) {
+	    mgr.send_scheduler_actor_message("multilevel",i);
+    }
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+    caf::cuda::program_ptr p1 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","init_denominators");
+    caf::cuda::program_ptr p2 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","perform_division");
+    caf::cuda::program_ptr p3 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","sum_results");
+
+
+
+
+
+
+    auto t_start = std::chrono::steady_clock::now();
+
+for (int i = 0; i < num_actors; ++i) {
+	sys.spawn(pipeline_actor, exit_actor, p1, p2, p3, n);	
+}
+  
+	  //this time the gpu actors can figure out how to send tokens to the correct GPU scheduler
+          sys.await_all_actors_done();
+}
+
+
+
+struct mmul_async_actor_state {
+  static inline const char* name = "mmul_actor";
+
+  int N = 0;
+  int id = rand();
+
+  // timing / bookkeeping only
+  std::chrono::high_resolution_clock::time_point start_time;
+  int times = 0;
+
+  // --- mmul_async state (added) -------------------------------------------
+  caf::cuda::mem_ptr<int> d_genA;                // device buffer for generated A
+  caf::cuda::mem_ptr<int> d_genB;                // device buffer for generated B
+  bool have_genA = false;
+  bool have_genB = false;
+};
+
+
+
+//we intentionally send to only 1 actor to force load balancing and also
+//see what happens if an actor gets a request that it is not responsible for
+caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_async_actor_state>* self,
+		caf::actor exit_actor) {
+  return {
+
+    // ------------------------------------------------------------------
+    // 1) Initial request: generate two matrices
+    // ------------------------------------------------------------------
+    [=](int N) {
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+      self->state().N = N;
+      self->state().have_genA = false;
+      self->state().have_genB = false;
+      self->state().d_genA = nullptr;
+      self->state().d_genB = nullptr;
+
+      // Explicit generator launch configuration
+      const int THREADS = 256;
+      const int BLOCKS  = (N * N + THREADS - 1) / THREADS;
+      caf::cuda::nd_range gen_range(BLOCKS, 1, 1,
+                                    THREADS, 1, 1);
+
+      auto gen_program =
+        mgr.create_program_from_fatbin(
+          "../generate_random_matrix.fatbin",
+          "generate_random_matrix");
+
+      auto send_launch = [&](const std::string& name) {
+        auto tok = caf::cuda::make_launch_token(
+                     gen_program,
+                     gen_range,
+                     sizeof(int) * N * N,
+                     name,
+                     self,
+		     self->state().id //use this for dependency number
+				      //required for scheduler actor internal bookeeping
+                   );
+        anon_mail(tok).send(mgr.get_scheduler_actor());
+      };
+
+      send_launch("genA");
+      send_launch("genB");
+    },
+
+    // ------------------------------------------------------------------
+    // 2) Handle scheduler response tokens
+    // ------------------------------------------------------------------
+[=](caf::cuda::response_token_ptr res_token) mutable {
+  try {
+    const auto type = res_token->getType();
+    const auto name = res_token->name();
+    int N = self->state().N;
+
+    // ----------------------------
+    // TRANSFER handling
+    // ----------------------------
+if (type == TRANSFER) {
+
+  //std::cout << "TRANSFER for " << name << std::endl;
+
+  // If mmul is moving, move BOTH matrices
+  if (name == "mmul") {
+
+    if (self->state().d_genA) {
+      auto host_copyA = self->state().d_genA->copy_to_host();
+      self->state().d_genA =
+        randomMatrix.transfer_memory(res_token, in_out<int>{host_copyA});
+    }
+
+    if (self->state().d_genB) {
+      auto host_copyB = self->state().d_genB->copy_to_host();
+      self->state().d_genB =
+        randomMatrix.transfer_memory(res_token, in_out<int>{host_copyB});
+    }
+
+    if (res_token->getDeviceNumber() == 1) {
+    //std::cout << "Moved genA and genB for mmul\n";
+  } 
+  }
+
+  res_token->release();
+  return;
+}
+
+
+    // ----------------------------
+    // LAUNCH_RESPONSE handling
+    // ----------------------------
+    if (type != LAUNCH_RESPONSE)
+      return;
+
+    // Generator completion (genA / genB)
+    if (name == "genA" || name == "genB") {
+      auto out_arg  = caf::cuda::create_out_arg(N * N);
+      auto size_arg = caf::cuda::create_in_arg(N * N);
+      auto seed_arg = caf::cuda::create_in_arg(rand());
+      auto maxval_arg = caf::cuda::create_in_arg(9999);
+
+      const int THREADS = 256;
+      const int BLOCKS  = (N * N + THREADS - 1) / THREADS;
+
+      caf::cuda::nd_range gen_range(BLOCKS,1,1, THREADS,1,1);
+
+      auto gen_program =
+        caf::cuda::manager::get().create_program_from_fatbin(
+          "../generate_random_matrix.fatbin",
+          "generate_random_matrix");
+
+      auto result = randomMatrix.run_async(
+        gen_program, gen_range, res_token,
+        out_arg, size_arg, seed_arg, maxval_arg);
+
+      auto device_buffer = std::get<0>(result);
+
+      if (name == "genA") {
+	      self->state().d_genA = device_buffer;
+      	      self->state().have_genA = true;
+      
+      }
+      else {
+	      self->state().d_genB = device_buffer;
+      	      self->state().have_genB = true;
+      }
+
+      // After handling genA / genB completion
+if (self->state().have_genA && self->state().have_genB) {
+    const int THREADS_M = 32;
+    int BLOCKS_M = (N + THREADS_M - 1) / THREADS_M;
+
+    caf::cuda::nd_range mmul_range(
+      BLOCKS_M, BLOCKS_M, 1,
+      THREADS_M, THREADS_M, 1
+    );
+
+    auto mmul_program =
+      caf::cuda::manager::get().create_program_from_cubin(
+        "../mmul.cubin",
+        "matrixMul"
+      );
+
+    // Create a launch token for mmul
+    auto mmul_token = caf::cuda::make_launch_token(
+      mmul_program,
+      mmul_range,
+      sizeof(int) * N * N,
+      "mmul",
+      self,
+      self ->state().id //needed to help track depedencies
+    );
+
+    // Send the launch token to the scheduler actor
+    anon_mail(mmul_token)
+      .send(caf::cuda::manager::get().get_scheduler_actor());
+  }
+
+
+      res_token->release();
+      return;
+    }
+
+    // ----------------------------
+    // mmul completion / kernel launch
+    // ----------------------------
+    if (name == "mmul") {
+      const int THREADS_M = 32;
+      int BLOCKS_M = (N + THREADS_M - 1) / THREADS_M;
+
+      caf::cuda::nd_range mmul_range(BLOCKS_M, BLOCKS_M, 1,
+                                    THREADS_M, THREADS_M, 1);
+
+      auto mmul_program =
+        caf::cuda::manager::get().create_program_from_cubin(
+          "../mmul.cubin",
+          "matrixMul");
+
+      auto outC = caf::cuda::create_out_arg(N * N);
+      auto inN  = caf::cuda::create_in_arg(N);
+
+      auto result = mmulAsync.run(
+        mmul_program,
+        mmul_range,
+        res_token,
+        self->state().d_genA,
+        self->state().d_genB,
+        outC,
+        inN);
+
+      std::vector<int> matrixC = caf::cuda::extract_vector<int>(result, 2);
+
+      self->state().have_genA = false;
+      self->state().have_genB = false;
+      self->state().d_genA.reset();
+      self->state().d_genB.reset();
+
+      res_token->release();
+      self->mail(1).send(exit_actor);
+    }
+
+  } catch (std::exception& e) {
+    std::cerr << "*** Caught exception: " << e.what() << "\n";
+
+    if (self->state().d_genA)
+      std::cerr << "d_genA deviceID: " << self->state().d_genA->deviceID() << "\n";
+    if (self->state().d_genB)
+      std::cerr << "d_genB deviceID: " << self->state().d_genB->deviceID() << "\n";
+    if (res_token) {
+      std::cerr << "res_token deviceID: " << res_token->getDeviceNumber() << "\n";
+      std::cerr << "res_token name: " << res_token->name() << "\n";
+    
+    }
+    
+    }
+
+
+}
+
+  };
+}
+
+
+
+
+//this test is meant to demonstrate the fact that scheduler actors can 
+//migrate work to correct load imbalance
+//the sizes should be large enough such that the tests exceed 4-5 seconds in total
+//otherwise the schedulers wont care to do this fast enough
+void run_load_balance_test_with_large_dependencies(
+    caf::actor_system& sys,
+    const int n,
+    int num_actors,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get(); 
+
+
+    //set the behaviors of each scheduler actor
+    for (int i = 0; i < mgr.get_num_devices();i++) {
+	    mgr.send_scheduler_actor_message("multilevel",i);
+    }
+
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+
+
+
+
+
+    auto t_start = std::chrono::steady_clock::now();
+
+for (int i = 0; i < num_actors; ++i) {
+	caf::actor a = sys.spawn(mmul_async_actor_fun, exit_actor);
+	anon_mail(n).send(a);	
+}
+  
+	  //this time the gpu actors can figure out how to send tokens to the correct GPU scheduler
+          sys.await_all_actors_done();
+}
+
+
+
+void caf_main(caf::actor_system& sys) {
+  
+	caf::cuda::manager_config man_config(true); //turns the scheduler on
+	caf::cuda::manager::init(sys,man_config);
+	
+	
+	//no dependencies
+//	std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048,4096};
+//	const int num_actors = 2000;
+//	run_load_balance_test(sys,sizes,num_actors);
+
+
+	//dependencies
+	run_load_balance_test_with_large_dependencies(sys,1024,2000);
+
+
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/component-actors-test/synch_actor_test/mmul.cu b/libcaf_cuda/tests/component-actors-test/synch_actor_test/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/synch_actor_test/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+
diff --git a/libcaf_cuda/tests/component-actors-test/synch_actor_test/shared_mmul.cu b/libcaf_cuda/tests/component-actors-test/synch_actor_test/shared_mmul.cu
new file mode 100644
index 0000000000..85c361ed93
--- /dev/null
+++ b/libcaf_cuda/tests/component-actors-test/synch_actor_test/shared_mmul.cu
@@ -0,0 +1,51 @@
+extern "C" __global__
+void matrixMul(const int* __restrict__ a,
+               const int* __restrict__ b,
+               int* __restrict__ c,
+               int N)
+{
+    const int TILE = 32;
+    int row = blockIdx.y * blockDim.y + threadIdx.y; // global row in C
+    int col = blockIdx.x * blockDim.x + threadIdx.x; // global col in C
+
+    __shared__ int s_a[TILE * TILE];
+    __shared__ int s_b[TILE * TILE];
+
+    int acc = 0;
+
+    // Sweep tiles across the K dimension
+    for (int i = 0; i < N; i += TILE) {
+
+        // Each thread loads one element into shared memory (with bounds checks)
+        int aCol = i + threadIdx.x;
+        int bRow = i + threadIdx.y;
+
+        // s_a[y, x] = a[row, aCol] if in range, else 0
+        if (row < N && aCol < N)
+            s_a[threadIdx.y * TILE + threadIdx.x] = a[row * N + aCol];
+        else
+            s_a[threadIdx.y * TILE + threadIdx.x] = 0;
+
+        // s_b[y, x] = b[bRow, col] if in range, else 0
+        if (bRow < N && col < N)
+            s_b[threadIdx.y * TILE + threadIdx.x] = b[bRow * N + col];
+        else
+            s_b[threadIdx.y * TILE + threadIdx.x] = 0;
+
+        __syncthreads();
+
+        // Compute partial dot product for this tile
+        #pragma unroll
+        for (int k = 0; k < TILE; ++k) {
+            acc += s_a[threadIdx.y * TILE + k] *
+                   s_b[k * TILE + threadIdx.x];
+        }
+
+        __syncthreads();
+    }
+
+    // Final write (guarded)
+    if (row < N && col < N)
+        c[row * N + col] = acc;
+}
+

From 17dbdda7368d4694f4b64fef9de44f91492f41e5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Mar 2026 10:41:09 -0600
Subject: [PATCH 0470/1000] Wrote inital verison of test for sync_actor.

---
 .../synch_actor_test/main.test.cpp            | 901 ++----------------
 1 file changed, 72 insertions(+), 829 deletions(-)

diff --git a/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
index 3dafb5e6c6..53f745203d 100644
--- a/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
@@ -1,6 +1,7 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
 #include <caf/cuda/control-layer/all-control-layer.hpp>
+#include <caf/component-actors/all-component-actors.hpp>
 #include <iostream>
 #include <iomanip>
 #include <vector>
@@ -30,6 +31,10 @@ struct mmul_actor_state {
   int last_N = 0; // example state variable
   int id = rand(); // an actor id
   int times = 0;
+  caf::actor sync_actor;
+  caf::actor_mem_transfer_actor;
+  caf::cuda::response_token r; //holding onto a response token is forbidden, as this can cause scheduler actor to never reply to itself
+			       //however we are doing it to see if the sync_actor works DO NOT TRY IN production environment 
 };
 
 
@@ -68,35 +73,6 @@ void serial_matrix_multiply(const std::vector<int>& a,
 
 
 
-struct exit_actor_state {
-	int completed = 0;
-};
-
-
-caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
-
-
-	return {
-		[=](int num_completed) {
-			self->state().completed += num_completed;
-			
-//			std::cout << "Actors finished is " << self->state().completed << "\n";
-			if (self->state().completed >= limit) {
-			
-				caf::cuda::manager::shutdown();
-				self->quit();
-			}
-		}
-	};
-
-
-}
-
-
-
-
-
-
 // Stateful actor behavior
 caf::behavior mmul_actor_fun(
     caf::stateful_actor<mmul_actor_state>* self,
@@ -121,40 +97,34 @@ caf::behavior mmul_actor_fun(
 			);
 	self -> mail(launch_token).send(scheduler);
 
+	self->state().sync_actor = self -> spawn(caf::cuda::sync_actor_fun<int>);
+	self->state().sync_actor = self -> spawn(caf::cuda::mem_transfer_actor_fun<int>);
+
 	return {
 
 		[=] (caf::cuda::response_token_ptr res_token) {
 
 			if (res_token -> getType() == LAUNCH_RESPONSE) {
-				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-				//assume N = 1024
 				std::vector<int> matrix1(N*N);
 				matrix1.reserve(N);
 				std::vector<int> matrix2(N*N);
 				matrix2.reserve(N);
 
-				//std::cout << "GPU ACTOR sending data to compute\n";
 				self -> mail(matrix1,matrix2,res_token,N).send(self);
 
 			}
 			else {
 				std::cout << "Got a memory response token\n";
 			}
-			//token should drop out of scope now, triggering a response 
 		},
 
-			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
 			[=](const std::vector<int>& matrixA,
 					const std::vector<int>& matrixB,
 					const caf::cuda::response_token_ptr& res_token, int N) {
 
 
-				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
 
-				//caf::cuda::launch_response_token& kt =
-				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
 
-				//std::cout << "GPU ACTOR  computing\n";
 				caf::cuda::manager& mgr = caf::cuda::manager::get();
 
 				//create program and dims   
@@ -163,19 +133,54 @@ caf::behavior mmul_actor_fun(
 				const int BLOCKS = (N + THREADS - 1) / THREADS;
 				caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrixA);
-				auto arg2 = caf::cuda::create_in_arg(matrixB);
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
+				auto arg1 = mmul.transfer_memory(res_token,caf::cuda::create_in_arg(matrixA));
+				auto arg2 = mmul.transfer_memory(res_token,caf::cuda::create_in_arg(matrixB));
+				auto arg3 = mmul.transfer_memory(res_token,caf::cuda::create_out_arg(N*N));
+				auto arg4 = mmul.transfer_memory(res_token,caf::cuda::create_in_arg(N));
+
+				auto tempC = mmul.run_async(program,dims,res_token,arg1,arg2,arg3,arg4);
+				//std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+
+				mem_ptr<int> bufferA = std::get<0>(tempC);
+
+
+				//hold onto the res_token to try and 
+				//trigger a deadlock
+				self->state().r = res_token;
+
+
+
+				self->mail(bufferA, res_token)
+					.request(self->state().sync_actor, std::chrono::seconds(10))
+					.then(
+							[=](mem_ptr<int> syncedA) {
+
+							self->mail(syncedA)
+							.request(self->state().mem_transfer_actor, std::chrono::seconds(10))
+							.then(
+									[=](std::vector<int> matrixA) {
 
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+									auto bufferB = std::get<1>(tempC);
 
-				//std::cout << "GPU ACTOR done  computing\n";
-				//verify its own result 
-				self -> mail(matrixA,matrixB,matrixC,N).send(self);
+									self->mail(bufferB)
+									.request(self->state().mem_transfer_actor, std::chrono::seconds(10))
+									.then(
+											[=](std::vector<int> matrixB) {
 
+											auto bufferC = std::get<2>(tempC);
+
+											self->mail(bufferC)
+											.request(self->state().mem_transfer_actor, std::chrono::seconds(10))
+											.then(
+													[=](std::vector<int> matrixC) {
+
+													// Now everything is ready
+													self->mail(matrixA, matrixB, matrixC, N)
+													.send(self);
+													});
+											});
+									});
+							});			
 			},
 
 					// 3rd handler: CPU atom + matrices + N
@@ -223,119 +228,6 @@ caf::behavior mmul_actor_fun(
 
 
 
-// this actor will not verify its results
-// great for performance analysis 
-caf::behavior mmul_actor_fun_no_verify(
-    caf::stateful_actor<mmul_actor_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims,
-    bool request
-    )
-{
-
-	//set the value of N correctly to overide the base option.	
-
-	caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-	caf::actor scheduler = mgr.get_scheduler_actor();
-
-	if (request) {
-		//send a launch token
-		caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-				program,
-				dims,
-				0,
-				"hello",
-				self
-				);
-		self -> mail(launch_token).send(scheduler);
-	}
-	return {
-
-		[=] (caf::cuda::response_token_ptr res_token) {
-
-			if (res_token -> getType() == LAUNCH_RESPONSE) {
-				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-				//assume N = 1024
-				std::vector<int> matrix1(N*N);
-				matrix1.reserve(N);
-				std::vector<int> matrix2(N*N);
-				matrix2.reserve(N);
-
-
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrix1);
-				auto arg2 = caf::cuda::create_in_arg(matrix2);
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
-
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-			
-			
-				//mask the transfer back to the cpu for scheduler
-				res_token -> release();
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				//std::cout << "GPU ACTOR done  computing\n";
-				// signal exit actor and quit
-				self->mail(1).send(exit_actor);
-				self->quit();
-
-				//std::cout << "GPU ACTOR sending data to compute\n";
-			//	self -> mail(matrix1,matrix2,res_token,N).send(self);
-
-			}
-			else {
-				std::cout << "Got a memory response token\n";
-			}
-			//token should drop out of scope now, triggering a response 
-		},
-
-			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-			[=](const std::vector<int>& matrixA,
-					const std::vector<int>& matrixB,
-					const caf::cuda::response_token_ptr& res_token, int N) {
-
-
-				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
-
-				//caf::cuda::launch_response_token& kt =
-				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
-
-				//std::cout << "GPU ACTOR  computing\n";
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrixA);
-				auto arg2 = caf::cuda::create_in_arg(matrixB);
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
-
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-			
-			
-				//mask the transfer back to the cpu for scheduler
-				res_token -> release();
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				//std::cout << "GPU ACTOR done  computing\n";
-				// signal exit actor and quit
-				self->mail(1).send(exit_actor);
-				self->quit();
-
-			}
-
-				
-	};
-}
-
-
 
 
 
@@ -349,11 +241,10 @@ double time_run(Fn&& fn) {
 }
 
 
-//this test is meant to demonstrate the fact that scheduler actors can 
-//migrate work to correct load imbalance
-//the sizes should be large enough such that the tests exceed 4-5 seconds in total
-//otherwise the schedulers wont care to do this fast enough
-void run_load_balance_test(
+//this test is meant to demonstrate the fact that sync_actors work by selecting the
+//single_usage_behavior. If nothing bad goes wrong we take that as a win 
+//maybe a performance analysis in the future 
+void run_sync_actor_test(
     caf::actor_system& sys,
     const std::vector<int>& sizes,
     int num_actors,
@@ -364,682 +255,36 @@ void run_load_balance_test(
 
     //set the behaviors of each scheduler actor
     for (int i = 0; i < mgr.get_num_devices();i++) {
-	    mgr.send_scheduler_actor_message("multilevel",i);
+	    mgr.send_scheduler_actor_message("single_usage",i);
     }
 
     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    std::vector<caf::cuda::token_ptr> tokens(num_actors);
-
-    std::mt19937 rng(123456);
-    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
-
-    const int THREADS = 32;
+    caf::actor exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_actors);
 
     auto t_start = std::chrono::steady_clock::now();
 
-for (int i = 0; i < num_actors; ++i) {
-    int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
-    int BLOCKS = (N + THREADS - 1) / THREADS;
-
-    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1,
-                             THREADS, THREADS, 1);
-
-    caf::actor a = sys.spawn(
-        mmul_actor_fun_no_verify,
-        exit_actor,
-        N,
-        program,
-        dims,
-        false
-    );
-
-    tokens[i] = caf::cuda::make_launch_token(
-        program,
-        dims,
-        0, // memory usage is zero for now, we still do not track it at all
-        "hello",
-        a
-    );
-}
-
-auto t_end = std::chrono::steady_clock::now();
-
-auto us = std::chrono::duration_cast<std::chrono::microseconds>(
-              t_end - t_start
-          ).count();
-
-std::cout << "Actor spawn + token creation loop took "
-          << us << " us\n";
+    for (int i = 0; i < num_actors; ++i) {
+	    caf::actor a = sys.spawn(
+			    mmul_actor_fun,
+			    exit_actor,
+			    N,
+			    program,
+			    dims
+			    );
 
-   
-	  //send the tokens to only 1 GPU and let them
-	  //figure out that there is a load imbalance 
-	  mgr.send_scheduler_actor_message(tokens);
-          sys.await_all_actors_done();
-}
-
-
-//-------------------------------------try load balancing with actors with dependencies
-
-
-using namespace caf;
-using namespace std::chrono_literals;
-
-
-// --- command runner types (put near top of file) -------------------------
-using initCommand =
-  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, in<int>, in<unsigned long long>>;
-
-using divCommand  =
-  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, in<int>>;
-
-using sumCommand  =
-  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, in<int>>;
-
-// single instances (can be file-global)
-static initCommand init_cmd;
-static divCommand  div_cmd;
-static sumCommand  sum_cmd;
-
-// --- pipeline actor state (device buffers persist here) ------------------
-struct pipeline_actor_state {
-    int id = rand();
-
-    int finished_stage = 0;
-
-    // device-side buffers that must persist across stages:
-    caf::cuda::mem_ptr<float> d_denoms;
-    caf::cuda::mem_ptr<float> d_results;
-    caf::cuda::mem_ptr<float> d_sum;
-    
-};
-
-// --- corrected pipeline_actor -------------------------------------------
-behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
-                        actor supervisor,
-                        caf::cuda::program_ptr p1,
-                        caf::cuda::program_ptr p2,
-                        caf::cuda::program_ptr p3,
-                        int n)
-{
-    // host-side scratch (only used for post-stage2 NaN/Inf detection)
-    std::vector<float> h_results;
-
-
-    // nd_range used for all stages (adapt to your kernels as needed)
-    caf::cuda::nd_range range{
-        {(n + 255) / 256, 1, 1},
-        {256, 1, 1}
-    };
-
-    // helper to create and send a launch token
-    auto launch = [&](caf::cuda::program_ptr prog, const std::string& stage) {
-        auto tok = make_launch_token(
-            prog,
-            range,
-            /*memory_usage=*/static_cast<int>(sizeof(float) * n),
-            stage,
-            self,
-            self->state().id  // dependency/demo id
-        );
-
-	//do not specifiy a device number to send it to, let it figure it out
-	//caf::cuda::manager::get().send_scheduler_actor_message(tok);
-    
-	//forcefully send to the first scheduler actor
-	caf::actor scheduler = caf::cuda::manager::get().get_scheduler_actor();
-	anon_mail(tok).send(scheduler);
-
-    };
-
-    // fire all three tokens (scheduler will reply with response_token on grants)
-    launch(p1, "stage1");
-    launch(p2, "stage2");
-    launch(p3, "stage3");
-
-    return {
-
-        // handle response tokens by name — opaque to reclaim payload
-        [=](caf::cuda::response_token_ptr res_token) mutable {
-
-            const auto& stage = res_token->name();
-
-	    if (res_token->getType() == LAUNCH_RESPONSE) {
-
-		    // --------------------- Stage 1: init_denominators ---------------------
-		    if (stage == "stage1") {
-			    // allocate device buffer for denominators (persist in state)
-
-			    //std::cout << "Starting stage 1\n";
-			    unsigned long long seed = static_cast<unsigned long long>(
-					    std::chrono::high_resolution_clock::now().time_since_epoch().count()
-					    );
-
-
-
-			    out<float> buffer = caf::cuda::create_out_arg_with_size<float>(n);
-			    self->state().d_denoms = init_cmd.transfer_memory(res_token,buffer);
-
-			    // run kernel on the stream/device from res_token
-			    // kernel signature: (float* denominators, int n, unsigned long long seed)
-			    init_cmd.run_async(
-					    p1,
-					    range,
-					    res_token,                            // uses token's stream/device
-					    self->state().d_denoms,               // device buffer
-					    caf::cuda::create_in_arg(n),          // n
-					    caf::cuda::create_in_arg(seed)     // seed
-					    );
-
-			    //std::cout << "Finished stage 1\n";
-			    // stage1 intentionally no checks — data may contain zeros
-			    
-			    self->state().finished_stage++;
-			    return;
-		    }
-
-		    // --------------------- Stage 2: perform_division ---------------------
-		    if (stage == "stage2") {
-			    // allocate device buffer for results (persist in state)
-			    std::vector<float> buffer1(n);
-
-			    //std::cout << "Starting stage 2\n";
-			    self->state().d_results = div_cmd.transfer_memory(res_token,out<float>{buffer1});
-
-			    // create a host numerators vector (all ones)
-			    std::vector<float> h_nums(n, 1.0f);
-
-			    // transfer numerators to device on the token's stream/device
-			    // transfer_memory returns a caf::cuda::mem_ptr<float>
-			    auto d_nums = div_cmd.transfer_memory(res_token, in_out<float>{h_nums});
-
-			    // run division kernel on the token's stream/device:
-			    // kernel signature: (float* numerators, float* denominators, float* results, int n)
-			    
-			    if (self->state().d_denoms == nullptr) {
-			    
-				    std::cout << "Error with pipeline actor d_denoms is nullptr\n";
-			    
-			    }
-
-
-			    if (d_nums == nullptr) {
-			    
-				    std::cout << "Error with pipeline actor d_denoms is nullptr\n";
-			    
-			    }
-
-			    
-			    div_cmd.run(
-					    p2,
-					    range,
-					    res_token,
-					    d_nums,
-					    self->state().d_denoms,
-					    self->state().d_results,
-					    caf::cuda::create_in_arg(n)
-				       );
-
-
-			    //there could be a division by zero in here 
-			    //but this is a load balancing test
-			    //not a fault test,
-			    //go see the fault tolerance test to see how thats handled	
-
-			    /*
-			    // extract the device results back to host for verification.
-			    // extract_vector will synchronize as needed.
-			    h_results = self->state().d_results -> copy_to_host();
-
-			    // check for NaN/Inf AFTER the kernel finished
-			    bool fault = false;
-			    for (float v : h_results) {
-				    if (!std::isfinite(v)) {
-					    fault = true;
-					    break;
-				    }
-			    }
-
-			    if (fault) {
-				    // inform supervisor and exit;
-				    anon_mail(std::string("crash")).send(supervisor);
-				    self->quit();
-				    return;
-			    }
-
-			    // stage2 passed — keep d_results in state for stage3
-			    */
-
-			    self->state().finished_stage++;
-			    return;
-		    }
-
-		    // --------------------- Stage 3: sum_results --------------------------
-		    if (stage == "stage3") {
-			    // allocate device scalar for sum result
-
-			    //std::cout << "Starting stage 3\n";
-			    std::vector<float> buffer1(1);
-
-			    self->state().d_sum = div_cmd.transfer_memory(res_token,out<float>{buffer1});
-
-			    // run reduction on the token's stream/device:
-			    // kernel signature: (float* results, float* final_sum, int n)
-			    sum_cmd.run(
-					    p3,
-					    range,
-					    res_token,
-					    self->state().d_results,
-					    self->state().d_sum,
-					    caf::cuda::create_in_arg(n)
-				       );
-
-			    // extract final scalar
-
-			    std::vector<float> buf = self->state().d_sum -> copy_to_host();
-			    float final_sum = buf[0];
-			    //std::cout << "[pipeline] completed, sum = " << final_sum << "\n";
-
-			    anon_mail(1).send(supervisor);
-
-			    // quit the pipeline actor
-			    self->state().finished_stage++;
-			    self->quit();
-			    return;
-		    }
-	    }
-
-	    else if (res_token->getType() == TRANSFER) {
-	   
-		    std::cout << "Got a transfer token\n";
-
-		   if (stage == "stage1") {
-			   res_token->release(); // no dependencies at this point clear to continue
-		   	   return;
-		   } 
-
-		   else if (stage == "stage2") {
-		  
-			   //at this point the d_results needs to be transfer over to the other device
-			   
-		           std::cout << "Transfering at stage 2\n";
-			      if (self->state().d_denoms == nullptr) {
-			    
-				    std::cout << "Error with pipeline actor d_denoms is nullptr during transfer\n";
-				    std::cout << "Completed stage is " << self->state().finished_stage << "\n";
-			    
-			    }
-			   
-			   
-			   
-			   //TODO FIX SEGFAULT TRIGGERED BY THIS LINE 
-			    in_out<float> temp_buffer{self->state().d_denoms -> copy_to_host()};
-
-			    self->state().d_denoms = div_cmd.transfer_memory(res_token,temp_buffer);
-			    //all done
-			    res_token->release();
-		   	    return;
-		   }
-
-		   else if (stage == "stage3") {
-		   
-			   //at this point d_results needs to be copied over to the new GPU   
-			    self->state().d_results = div_cmd.transfer_memory(res_token,in_out<float>{self->state().d_results->copy_to_host()});
-			    //all done
-			    res_token->release();
-			    return;
-		   }
-
-		   else { 
-			   std::cout << "Error unrecognized transfer token\n";
-		   }
-		    
-	    
-	    }
-
-            // unknown stage: ignore or log
-            std::cerr << "[pipeline] received unknown response token: " << stage << "\n";
-	}
-    };
-}
-
-
-
-
-
-
-
-
-
-//this test is meant to demonstrate the fact that scheduler actors can 
-//migrate work to correct load imbalance
-//the sizes should be large enough such that the tests exceed 4-5 seconds in total
-//otherwise the schedulers wont care to do this fast enough
-//As it turns out pipeline actor does not do enough work in order to convince the GPUs
-//that it should even attempt to migrate it
-void run_load_balance_test_with_dependencies(
-    caf::actor_system& sys,
-    const int n,
-    int num_actors,
-    bool randomize = false)
-{
-    caf::cuda::manager& mgr = caf::cuda::manager::get(); 
-
-
-    //set the behaviors of each scheduler actor
-    for (int i = 0; i < mgr.get_num_devices();i++) {
-	    mgr.send_scheduler_actor_message("multilevel",i);
     }
 
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+    auto t_end = std::chrono::steady_clock::now();
 
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-    caf::cuda::program_ptr p1 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","init_denominators");
-    caf::cuda::program_ptr p2 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","perform_division");
-    caf::cuda::program_ptr p3 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","sum_results");
+    auto us = std::chrono::duration_cast<std::chrono::microseconds>(
+		    t_end - t_start
+		    ).count();
 
-
-
-
-
-
-    auto t_start = std::chrono::steady_clock::now();
-
-for (int i = 0; i < num_actors; ++i) {
-	sys.spawn(pipeline_actor, exit_actor, p1, p2, p3, n);	
-}
-  
-	  //this time the gpu actors can figure out how to send tokens to the correct GPU scheduler
-          sys.await_all_actors_done();
+    sys.await_all_actors_done();
 }
 
 
-
-struct mmul_async_actor_state {
-  static inline const char* name = "mmul_actor";
-
-  int N = 0;
-  int id = rand();
-
-  // timing / bookkeeping only
-  std::chrono::high_resolution_clock::time_point start_time;
-  int times = 0;
-
-  // --- mmul_async state (added) -------------------------------------------
-  caf::cuda::mem_ptr<int> d_genA;                // device buffer for generated A
-  caf::cuda::mem_ptr<int> d_genB;                // device buffer for generated B
-  bool have_genA = false;
-  bool have_genB = false;
-};
-
-
-
-//we intentionally send to only 1 actor to force load balancing and also
-//see what happens if an actor gets a request that it is not responsible for
-caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_async_actor_state>* self,
-		caf::actor exit_actor) {
-  return {
-
-    // ------------------------------------------------------------------
-    // 1) Initial request: generate two matrices
-    // ------------------------------------------------------------------
-    [=](int N) {
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-      self->state().N = N;
-      self->state().have_genA = false;
-      self->state().have_genB = false;
-      self->state().d_genA = nullptr;
-      self->state().d_genB = nullptr;
-
-      // Explicit generator launch configuration
-      const int THREADS = 256;
-      const int BLOCKS  = (N * N + THREADS - 1) / THREADS;
-      caf::cuda::nd_range gen_range(BLOCKS, 1, 1,
-                                    THREADS, 1, 1);
-
-      auto gen_program =
-        mgr.create_program_from_fatbin(
-          "../generate_random_matrix.fatbin",
-          "generate_random_matrix");
-
-      auto send_launch = [&](const std::string& name) {
-        auto tok = caf::cuda::make_launch_token(
-                     gen_program,
-                     gen_range,
-                     sizeof(int) * N * N,
-                     name,
-                     self,
-		     self->state().id //use this for dependency number
-				      //required for scheduler actor internal bookeeping
-                   );
-        anon_mail(tok).send(mgr.get_scheduler_actor());
-      };
-
-      send_launch("genA");
-      send_launch("genB");
-    },
-
-    // ------------------------------------------------------------------
-    // 2) Handle scheduler response tokens
-    // ------------------------------------------------------------------
-[=](caf::cuda::response_token_ptr res_token) mutable {
-  try {
-    const auto type = res_token->getType();
-    const auto name = res_token->name();
-    int N = self->state().N;
-
-    // ----------------------------
-    // TRANSFER handling
-    // ----------------------------
-if (type == TRANSFER) {
-
-  //std::cout << "TRANSFER for " << name << std::endl;
-
-  // If mmul is moving, move BOTH matrices
-  if (name == "mmul") {
-
-    if (self->state().d_genA) {
-      auto host_copyA = self->state().d_genA->copy_to_host();
-      self->state().d_genA =
-        randomMatrix.transfer_memory(res_token, in_out<int>{host_copyA});
-    }
-
-    if (self->state().d_genB) {
-      auto host_copyB = self->state().d_genB->copy_to_host();
-      self->state().d_genB =
-        randomMatrix.transfer_memory(res_token, in_out<int>{host_copyB});
-    }
-
-    if (res_token->getDeviceNumber() == 1) {
-    //std::cout << "Moved genA and genB for mmul\n";
-  } 
-  }
-
-  res_token->release();
-  return;
-}
-
-
-    // ----------------------------
-    // LAUNCH_RESPONSE handling
-    // ----------------------------
-    if (type != LAUNCH_RESPONSE)
-      return;
-
-    // Generator completion (genA / genB)
-    if (name == "genA" || name == "genB") {
-      auto out_arg  = caf::cuda::create_out_arg(N * N);
-      auto size_arg = caf::cuda::create_in_arg(N * N);
-      auto seed_arg = caf::cuda::create_in_arg(rand());
-      auto maxval_arg = caf::cuda::create_in_arg(9999);
-
-      const int THREADS = 256;
-      const int BLOCKS  = (N * N + THREADS - 1) / THREADS;
-
-      caf::cuda::nd_range gen_range(BLOCKS,1,1, THREADS,1,1);
-
-      auto gen_program =
-        caf::cuda::manager::get().create_program_from_fatbin(
-          "../generate_random_matrix.fatbin",
-          "generate_random_matrix");
-
-      auto result = randomMatrix.run_async(
-        gen_program, gen_range, res_token,
-        out_arg, size_arg, seed_arg, maxval_arg);
-
-      auto device_buffer = std::get<0>(result);
-
-      if (name == "genA") {
-	      self->state().d_genA = device_buffer;
-      	      self->state().have_genA = true;
-      
-      }
-      else {
-	      self->state().d_genB = device_buffer;
-      	      self->state().have_genB = true;
-      }
-
-      // After handling genA / genB completion
-if (self->state().have_genA && self->state().have_genB) {
-    const int THREADS_M = 32;
-    int BLOCKS_M = (N + THREADS_M - 1) / THREADS_M;
-
-    caf::cuda::nd_range mmul_range(
-      BLOCKS_M, BLOCKS_M, 1,
-      THREADS_M, THREADS_M, 1
-    );
-
-    auto mmul_program =
-      caf::cuda::manager::get().create_program_from_cubin(
-        "../mmul.cubin",
-        "matrixMul"
-      );
-
-    // Create a launch token for mmul
-    auto mmul_token = caf::cuda::make_launch_token(
-      mmul_program,
-      mmul_range,
-      sizeof(int) * N * N,
-      "mmul",
-      self,
-      self ->state().id //needed to help track depedencies
-    );
-
-    // Send the launch token to the scheduler actor
-    anon_mail(mmul_token)
-      .send(caf::cuda::manager::get().get_scheduler_actor());
-  }
-
-
-      res_token->release();
-      return;
-    }
-
-    // ----------------------------
-    // mmul completion / kernel launch
-    // ----------------------------
-    if (name == "mmul") {
-      const int THREADS_M = 32;
-      int BLOCKS_M = (N + THREADS_M - 1) / THREADS_M;
-
-      caf::cuda::nd_range mmul_range(BLOCKS_M, BLOCKS_M, 1,
-                                    THREADS_M, THREADS_M, 1);
-
-      auto mmul_program =
-        caf::cuda::manager::get().create_program_from_cubin(
-          "../mmul.cubin",
-          "matrixMul");
-
-      auto outC = caf::cuda::create_out_arg(N * N);
-      auto inN  = caf::cuda::create_in_arg(N);
-
-      auto result = mmulAsync.run(
-        mmul_program,
-        mmul_range,
-        res_token,
-        self->state().d_genA,
-        self->state().d_genB,
-        outC,
-        inN);
-
-      std::vector<int> matrixC = caf::cuda::extract_vector<int>(result, 2);
-
-      self->state().have_genA = false;
-      self->state().have_genB = false;
-      self->state().d_genA.reset();
-      self->state().d_genB.reset();
-
-      res_token->release();
-      self->mail(1).send(exit_actor);
-    }
-
-  } catch (std::exception& e) {
-    std::cerr << "*** Caught exception: " << e.what() << "\n";
-
-    if (self->state().d_genA)
-      std::cerr << "d_genA deviceID: " << self->state().d_genA->deviceID() << "\n";
-    if (self->state().d_genB)
-      std::cerr << "d_genB deviceID: " << self->state().d_genB->deviceID() << "\n";
-    if (res_token) {
-      std::cerr << "res_token deviceID: " << res_token->getDeviceNumber() << "\n";
-      std::cerr << "res_token name: " << res_token->name() << "\n";
-    
-    }
-    
-    }
-
-
-}
-
-  };
-}
-
-
-
-
-//this test is meant to demonstrate the fact that scheduler actors can 
-//migrate work to correct load imbalance
-//the sizes should be large enough such that the tests exceed 4-5 seconds in total
-//otherwise the schedulers wont care to do this fast enough
-void run_load_balance_test_with_large_dependencies(
-    caf::actor_system& sys,
-    const int n,
-    int num_actors,
-    bool randomize = false)
-{
-    caf::cuda::manager& mgr = caf::cuda::manager::get(); 
-
-
-    //set the behaviors of each scheduler actor
-    for (int i = 0; i < mgr.get_num_devices();i++) {
-	    mgr.send_scheduler_actor_message("multilevel",i);
-    }
-
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-
-
-
-
-
-    auto t_start = std::chrono::steady_clock::now();
-
-for (int i = 0; i < num_actors; ++i) {
-	caf::actor a = sys.spawn(mmul_async_actor_fun, exit_actor);
-	anon_mail(n).send(a);	
-}
-  
-	  //this time the gpu actors can figure out how to send tokens to the correct GPU scheduler
-          sys.await_all_actors_done();
-}
-
-
-
 void caf_main(caf::actor_system& sys) {
   
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
@@ -1052,8 +297,6 @@ void caf_main(caf::actor_system& sys) {
 //	run_load_balance_test(sys,sizes,num_actors);
 
 
-	//dependencies
-	run_load_balance_test_with_large_dependencies(sys,1024,2000);
 
 
 }

From 73350dbf68e82a12145bf49cbab877d275c20a25 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Mar 2026 10:45:23 -0600
Subject: [PATCH 0471/1000] Changed message_handler from response_token to
 response_token_ptr type to fix compiler error reguarding using abstract
 classes.

---
 libcaf_cuda/caf/component-actors/sync_actor/sync_actor.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/component-actors/sync_actor/sync_actor.hpp b/libcaf_cuda/caf/component-actors/sync_actor/sync_actor.hpp
index 342806644b..ca9a15018b 100644
--- a/libcaf_cuda/caf/component-actors/sync_actor/sync_actor.hpp
+++ b/libcaf_cuda/caf/component-actors/sync_actor/sync_actor.hpp
@@ -25,7 +25,7 @@ void synch_actor_fun(caf::blocking_actor* self) {
       d_mem -> synchronize();  
     return d_mem;
     },
-    [&](mem_ptr<T> d_mem, response_token res) -> mem_ptr<T> {
+    [&](mem_ptr<T> d_mem, response_token_ptr res) -> mem_ptr<T> {
       d_mem->synchronize();
 
       res ->release();

From ecb736697cfb6f0444b42fb6f05e353ffa3d0092 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Mar 2026 10:47:42 -0600
Subject: [PATCH 0472/1000] Changed typedef declaration of mmul_async_command
 to mmul_async_not_square_command to fix conflicts with mmul_actor.hpp

---
 .../mmul_actor_not_square/mmul_actor_not_square.hpp           | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp b/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp
index d892e39231..de9bf8449b 100644
--- a/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp
+++ b/libcaf_cuda/caf/component-actors/mmul_actor_not_square/mmul_actor_not_square.hpp
@@ -15,7 +15,7 @@ struct mmul_actor_not_square_state {
 };
 
 template <class T>
-using mmul_async_command =
+using mmul_async_not_square_command =
   command_runner<
     mem_ptr<T>,
     mem_ptr<T>,
@@ -31,7 +31,7 @@ caf::behavior mmul_actor_NS_fun(
     program_ptr mmul_kernel)
 {
   using mem_t = mem_ptr<T>;
-  using runner_t = mmul_async_command<T>;
+  using runner_t = mmul_async_not_square_command<T>;
 
   self->state().mmul_kernel = mmul_kernel;
 

From 953c330b1c4d28da99cd50141990f51324a91e5b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Mar 2026 10:49:49 -0600
Subject: [PATCH 0473/1000] Fixed naming spelling mistake on the signature of
 the behavior.

---
 libcaf_cuda/caf/component-actors/sync_actor/sync_actor.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/component-actors/sync_actor/sync_actor.hpp b/libcaf_cuda/caf/component-actors/sync_actor/sync_actor.hpp
index ca9a15018b..e2370ccd72 100644
--- a/libcaf_cuda/caf/component-actors/sync_actor/sync_actor.hpp
+++ b/libcaf_cuda/caf/component-actors/sync_actor/sync_actor.hpp
@@ -16,7 +16,7 @@
 namespace caf::cuda {
 
 template <class T>
-void synch_actor_fun(caf::blocking_actor* self) {
+void sync_actor_fun(caf::blocking_actor* self) {
 
   self->receive(
     [&](mem_ptr<T> d_mem) -> mem_ptr<T> {

From aefaac9f6915f9c151d6b098296744ada40ca2be Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Mar 2026 10:56:37 -0600
Subject: [PATCH 0474/1000] Fixed compiler errors.

---
 .../synch_actor_test/main.test.cpp            | 22 +++++++++++++------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
index 53f745203d..ccc3beff69 100644
--- a/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
@@ -32,8 +32,8 @@ struct mmul_actor_state {
   int id = rand(); // an actor id
   int times = 0;
   caf::actor sync_actor;
-  caf::actor_mem_transfer_actor;
-  caf::cuda::response_token r; //holding onto a response token is forbidden, as this can cause scheduler actor to never reply to itself
+  caf::actor mem_transfer_actor;
+  caf::cuda::response_token_ptr r; //holding onto a response token is forbidden, as this can cause scheduler actor to never reply to itself
 			       //however we are doing it to see if the sync_actor works DO NOT TRY IN production environment 
 };
 
@@ -44,7 +44,7 @@ struct mmul_actor_state {
 using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
 using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
 
-using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
+using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>>;
 
 mmulCommand mmul;
 matrixGenCommand randomMatrix;
@@ -138,10 +138,11 @@ caf::behavior mmul_actor_fun(
 				auto arg3 = mmul.transfer_memory(res_token,caf::cuda::create_out_arg(N*N));
 				auto arg4 = mmul.transfer_memory(res_token,caf::cuda::create_in_arg(N));
 
-				auto tempC = mmul.run_async(program,dims,res_token,arg1,arg2,arg3,arg4);
+				auto tempC = mmulAsync.run_async(program,dims,res_token,arg1,arg2,arg3,arg4);
 				//std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
 
-				mem_ptr<int> bufferA = std::get<0>(tempC);
+				caf::cuda::mem_ptr<int> bufferA = std::get<0>(tempC);
+				
 
 
 				//hold onto the res_token to try and 
@@ -153,7 +154,7 @@ caf::behavior mmul_actor_fun(
 				self->mail(bufferA, res_token)
 					.request(self->state().sync_actor, std::chrono::seconds(10))
 					.then(
-							[=](mem_ptr<int> syncedA) {
+							[=](caf::cuda::mem_ptr<int> syncedA) {
 
 							self->mail(syncedA)
 							.request(self->state().mem_transfer_actor, std::chrono::seconds(10))
@@ -246,7 +247,7 @@ double time_run(Fn&& fn) {
 //maybe a performance analysis in the future 
 void run_sync_actor_test(
     caf::actor_system& sys,
-    const std::vector<int>& sizes,
+    int N,
     int num_actors,
     bool randomize = false)
 {
@@ -260,6 +261,13 @@ void run_sync_actor_test(
 
     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
+     const int THREADS = 32;
+      const int BLOCKS = (N + THREADS - 1) / THREADS;
+
+      caf::cuda::nd_range dims(
+        BLOCKS, BLOCKS, 1,
+        THREADS, THREADS, 1);
+
     caf::actor exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_actors);
 
     auto t_start = std::chrono::steady_clock::now();

From 5824e3cd2006ed284f6e745b7dd84416d9ccafe3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Mar 2026 10:58:33 -0600
Subject: [PATCH 0475/1000] Removed uneccesary print statements.

---
 libcaf_cuda/src/control-layer/exit_actor/exit_actor.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/exit_actor/exit_actor.cpp b/libcaf_cuda/src/control-layer/exit_actor/exit_actor.cpp
index 6c3f49db7b..7f1405af2e 100644
--- a/libcaf_cuda/src/control-layer/exit_actor/exit_actor.cpp
+++ b/libcaf_cuda/src/control-layer/exit_actor/exit_actor.cpp
@@ -9,7 +9,6 @@ caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int lim
                 [=](int num_completed) {
                         self->state().completed += num_completed;
 
-                        std::cout << "Actors finished is " << self->state().completed << "\n";
                         if (self->state().completed >= limit) {
 
                                 caf::cuda::manager::shutdown();

From a19a686e8251d8178987f66ac5c59d9232a682b9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Mar 2026 11:01:37 -0600
Subject: [PATCH 0476/1000] Updated sys.spawn(exit_actor) to
 mgr.spawn_exit_actor since the former leads to linker errors while the latter
 does not.

---
 .../tests/component-actors-test/synch_actor_test/main.test.cpp  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
index ccc3beff69..380da992c6 100644
--- a/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
@@ -268,7 +268,7 @@ void run_sync_actor_test(
         BLOCKS, BLOCKS, 1,
         THREADS, THREADS, 1);
 
-    caf::actor exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_actors);
+    caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
 
     auto t_start = std::chrono::steady_clock::now();
 

From b4fc42a74f48e81f1df35ada1eb09bccb83b66a4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Mar 2026 11:03:28 -0600
Subject: [PATCH 0477/1000] Implemented main.

---
 .../component-actors-test/synch_actor_test/main.test.cpp      | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
index 380da992c6..86d5403b2e 100644
--- a/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
@@ -298,7 +298,9 @@ void caf_main(caf::actor_system& sys) {
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	caf::cuda::manager::init(sys,man_config);
 	
-	
+
+	run_sync_actor_test(sys,1024,20);
+
 	//no dependencies
 //	std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048,4096};
 //	const int num_actors = 2000;

From 010882b4a04266ae1324385689096612802f6653 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Mar 2026 11:22:33 -0600
Subject: [PATCH 0478/1000] Fixed spelling error on line 101

---
 .../synch_actor_test/main.test.cpp            | 95 +++++++++++++------
 1 file changed, 67 insertions(+), 28 deletions(-)

diff --git a/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
index 86d5403b2e..a52c3f0805 100644
--- a/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
@@ -98,12 +98,14 @@ caf::behavior mmul_actor_fun(
 	self -> mail(launch_token).send(scheduler);
 
 	self->state().sync_actor = self -> spawn(caf::cuda::sync_actor_fun<int>);
-	self->state().sync_actor = self -> spawn(caf::cuda::mem_transfer_actor_fun<int>);
+	self->state().mem_transfer_actor = self -> spawn(caf::cuda::mem_transfer_actor_fun<int>);
 
 	return {
 
 		[=] (caf::cuda::response_token_ptr res_token) {
 
+			std::cout << "Got response\n";
+
 			if (res_token -> getType() == LAUNCH_RESPONSE) {
 				std::vector<int> matrix1(N*N);
 				matrix1.reserve(N);
@@ -125,6 +127,7 @@ caf::behavior mmul_actor_fun(
 
 
 
+				std::cout << "Working\n";
 				caf::cuda::manager& mgr = caf::cuda::manager::get();
 
 				//create program and dims   
@@ -151,38 +154,74 @@ caf::behavior mmul_actor_fun(
 
 
 
-				self->mail(bufferA, res_token)
-					.request(self->state().sync_actor, std::chrono::seconds(10))
-					.then(
-							[=](caf::cuda::mem_ptr<int> syncedA) {
+			using namespace std::chrono_literals;
+
+self->mail(bufferA, res_token)
+    .request(self->state().sync_actor, 10s)
+    .then(
+
+        // ===== SUCCESS 1 =====
+        [=](caf::cuda::mem_ptr<int> syncedA) {
+
+            self->mail(syncedA)
+                .request(self->state().mem_transfer_actor, 10s)
+                .then(
+
+                    // ===== SUCCESS 2 =====
+                    [=](std::vector<int> matrixA) {
+
+                        auto bufferB = std::get<1>(tempC);
+
+                        self->mail(bufferB)
+                            .request(self->state().mem_transfer_actor, 10s)
+                            .then(
+
+                                // ===== SUCCESS 3 =====
+                                [=](std::vector<int> matrixB) {
+
+                                    auto bufferC = std::get<2>(tempC);
+
+                                    self->mail(bufferC)
+                                        .request(self->state().mem_transfer_actor, 10s)
+                                        .then(
 
-							self->mail(syncedA)
-							.request(self->state().mem_transfer_actor, std::chrono::seconds(10))
-							.then(
-									[=](std::vector<int> matrixA) {
+                                            // ===== SUCCESS 4 =====
+                                            [=](std::vector<int> matrixC) {
 
-									auto bufferB = std::get<1>(tempC);
+                                                self->mail(matrixA, matrixB, matrixC, N)
+                                                    .send(self);
+                                            },
 
-									self->mail(bufferB)
-									.request(self->state().mem_transfer_actor, std::chrono::seconds(10))
-									.then(
-											[=](std::vector<int> matrixB) {
+                                            // ===== ERROR 4 =====
+                                            [=](caf::error& err) {
+                                                std::cout << "Transfer C failed: "
+                                                           << to_string(err) << "\n";
+                                                self->quit(err);
+                                            });
+                                },
 
-											auto bufferC = std::get<2>(tempC);
+                                // ===== ERROR 3 =====
+                                [=](caf::error& err) {
+                                    std::cout << "Transfer B failed: "
+                                               << to_string(err) << "\n";
+                                    self->quit(err);
+                                });
+                    },
 
-											self->mail(bufferC)
-											.request(self->state().mem_transfer_actor, std::chrono::seconds(10))
-											.then(
-													[=](std::vector<int> matrixC) {
+                    // ===== ERROR 2 =====
+                    [=](caf::error& err) {
+                        std::cout << "Transfer A failed: "
+                                   << to_string(err) << "\n";
+                        self->quit(err);
+                    });
+        },
 
-													// Now everything is ready
-													self->mail(matrixA, matrixB, matrixC, N)
-													.send(self);
-													});
-											});
-									});
-							});			
-			},
+        // ===== ERROR 1 =====
+        [=](caf::error& err) {
+            std::cout << "Sync failed: "
+                       << to_string(err) << "\n";
+            self->quit(err);
+        });			},
 
 					// 3rd handler: CPU atom + matrices + N
 					[=](const std::vector<int>& matrixA,
@@ -299,7 +338,7 @@ void caf_main(caf::actor_system& sys) {
 	caf::cuda::manager::init(sys,man_config);
 	
 
-	run_sync_actor_test(sys,1024,20);
+	run_sync_actor_test(sys,1024,10);
 
 	//no dependencies
 //	std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048,4096};

From 2eae23dc02f85d37bbdd0998a836969eb36e1dc4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Mar 2026 11:30:44 -0600
Subject: [PATCH 0479/1000] simplifying message on mmul actor, also test
 passes.

---
 .../synch_actor_test/main.test.cpp            | 245 ++++++------------
 1 file changed, 83 insertions(+), 162 deletions(-)

diff --git a/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp b/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
index a52c3f0805..a667386bfc 100644
--- a/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
+++ b/libcaf_cuda/tests/component-actors-test/synch_actor_test/main.test.cpp
@@ -100,168 +100,89 @@ caf::behavior mmul_actor_fun(
 	self->state().sync_actor = self -> spawn(caf::cuda::sync_actor_fun<int>);
 	self->state().mem_transfer_actor = self -> spawn(caf::cuda::mem_transfer_actor_fun<int>);
 
-	return {
-
-		[=] (caf::cuda::response_token_ptr res_token) {
-
-			std::cout << "Got response\n";
-
-			if (res_token -> getType() == LAUNCH_RESPONSE) {
-				std::vector<int> matrix1(N*N);
-				matrix1.reserve(N);
-				std::vector<int> matrix2(N*N);
-				matrix2.reserve(N);
-
-				self -> mail(matrix1,matrix2,res_token,N).send(self);
-
-			}
-			else {
-				std::cout << "Got a memory response token\n";
-			}
-		},
-
-			[=](const std::vector<int>& matrixA,
-					const std::vector<int>& matrixB,
-					const caf::cuda::response_token_ptr& res_token, int N) {
-
-
-
-
-				std::cout << "Working\n";
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-				const int THREADS = 32;
-				const int BLOCKS = (N + THREADS - 1) / THREADS;
-				caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-				auto arg1 = mmul.transfer_memory(res_token,caf::cuda::create_in_arg(matrixA));
-				auto arg2 = mmul.transfer_memory(res_token,caf::cuda::create_in_arg(matrixB));
-				auto arg3 = mmul.transfer_memory(res_token,caf::cuda::create_out_arg(N*N));
-				auto arg4 = mmul.transfer_memory(res_token,caf::cuda::create_in_arg(N));
-
-				auto tempC = mmulAsync.run_async(program,dims,res_token,arg1,arg2,arg3,arg4);
-				//std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				caf::cuda::mem_ptr<int> bufferA = std::get<0>(tempC);
-				
-
-
-				//hold onto the res_token to try and 
-				//trigger a deadlock
-				self->state().r = res_token;
-
-
-
-			using namespace std::chrono_literals;
-
-self->mail(bufferA, res_token)
-    .request(self->state().sync_actor, 10s)
-    .then(
-
-        // ===== SUCCESS 1 =====
-        [=](caf::cuda::mem_ptr<int> syncedA) {
-
-            self->mail(syncedA)
-                .request(self->state().mem_transfer_actor, 10s)
-                .then(
-
-                    // ===== SUCCESS 2 =====
-                    [=](std::vector<int> matrixA) {
-
-                        auto bufferB = std::get<1>(tempC);
-
-                        self->mail(bufferB)
-                            .request(self->state().mem_transfer_actor, 10s)
-                            .then(
-
-                                // ===== SUCCESS 3 =====
-                                [=](std::vector<int> matrixB) {
-
-                                    auto bufferC = std::get<2>(tempC);
-
-                                    self->mail(bufferC)
-                                        .request(self->state().mem_transfer_actor, 10s)
-                                        .then(
-
-                                            // ===== SUCCESS 4 =====
-                                            [=](std::vector<int> matrixC) {
-
-                                                self->mail(matrixA, matrixB, matrixC, N)
-                                                    .send(self);
-                                            },
-
-                                            // ===== ERROR 4 =====
-                                            [=](caf::error& err) {
-                                                std::cout << "Transfer C failed: "
-                                                           << to_string(err) << "\n";
-                                                self->quit(err);
-                                            });
-                                },
-
-                                // ===== ERROR 3 =====
-                                [=](caf::error& err) {
-                                    std::cout << "Transfer B failed: "
-                                               << to_string(err) << "\n";
-                                    self->quit(err);
-                                });
-                    },
-
-                    // ===== ERROR 2 =====
-                    [=](caf::error& err) {
-                        std::cout << "Transfer A failed: "
-                                   << to_string(err) << "\n";
-                        self->quit(err);
-                    });
-        },
-
-        // ===== ERROR 1 =====
-        [=](caf::error& err) {
-            std::cout << "Sync failed: "
-                       << to_string(err) << "\n";
-            self->quit(err);
-        });			},
-
-					// 3rd handler: CPU atom + matrices + N
-					[=](const std::vector<int>& matrixA,
-							const std::vector<int>& matrixB,
-							const std::vector<int>& matrixC,
-							int N) {
-
-						using clock = std::chrono::high_resolution_clock;
-
-						auto start = clock::now();
-
-						//std::cout << "GPU ACTOR verifying\n";
-
-						std::vector<int> result(N * N);
-
-						serial_matrix_multiply(matrixA, matrixB, result, N);
-
-						if (result == matrixC) {
-							std::cout << "actor with id " << self->state().id
-								<< " references match\n";
-						} else {
-							std::cout << "actor with id " << self->state().id
-								<< " references did not match\n";
-						}
-
-						auto end = clock::now();
-
-						auto ms =
-							std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-						std::cout << "[TIMING] verification took "
-							<< ms << " ms (actor id "
-							<< self->state().id << ")\n";
-
-						// signal exit actor and quit
-						self->mail(1).send(exit_actor);
-						self->quit();
-
-					}
-	};
+       return {
+
+    // 1. Handle response token
+    [=](caf::cuda::response_token_ptr res_token) {
+        std::cout << "Got response\n";
+
+        if (res_token->getType() == LAUNCH_RESPONSE) {
+            std::vector<int> matrix1(N*N);
+            std::vector<int> matrix2(N*N);
+
+            self->mail(matrix1, matrix2, res_token, N).send(self);
+
+        } else {
+            std::cout << "Got a memory response token\n";
+        }
+    },
+
+    // 2. Handle memory buffers -> GPU
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        const caf::cuda::response_token_ptr& res_token,
+        int N) {
+
+        std::cout << "Working\n";
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        const int THREADS = 32;
+        const int BLOCKS = (N + THREADS - 1) / THREADS;
+        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+        auto arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixA));
+        auto arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixB));
+        auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
+        auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
+
+        auto tempC = mmulAsync.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
+        caf::cuda::mem_ptr<int> bufferA = std::get<0>(tempC);
+
+        self->state().r = res_token; // hold token
+
+        using namespace std::chrono_literals;
+        self->mail(bufferA, res_token)
+            .request(self->state().sync_actor, 10s)
+            .then(
+                [=](caf::cuda::mem_ptr<int> /*syncedA*/) {
+                    auto bufferC = std::get<2>(tempC);
+                    self->mail(bufferC)
+                        .request(self->state().mem_transfer_actor, 10s)
+                        .then(
+                            [=](std::vector<int> matrixC) {
+                                self->mail(matrixA,matrixB,matrixC, N).send(self);
+                            },
+                            [=](caf::error& err) {
+                                std::cout << "Transfer C failed: " << to_string(err) << "\n";
+                                self->quit(err);
+                            });
+                },
+                [=](caf::error& err) {
+                    std::cout << "Sync failed: " << to_string(err) << "\n";
+                    self->quit(err);
+                });
+    },
+
+    // 3. Final CPU verification
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        const std::vector<int>& matrixC,
+        int N) {
+
+        std::vector<int> result(N * N);
+        serial_matrix_multiply(matrixA, matrixB, result, N);
+
+        if (result == matrixC) {
+            std::cout << "actor with id " << self->state().id << " references match\n";
+        } else {
+            std::cout << "actor with id " << self->state().id << " references did not match\n";
+        }
+
+        self->mail(1).send(exit_actor);
+        self->quit();
+    }
+};
 }
 
 

From a6daa01f3d523a4dd5b2197558406ebc1624f9b2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 2 Mar 2026 11:42:38 -0600
Subject: [PATCH 0480/1000] Added a bind method that enables actors to bind
 mem_token life time to the mem_ref.

---
 libcaf_cuda/caf/cuda/mem_ref.hpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/mem_ref.hpp b/libcaf_cuda/caf/cuda/mem_ref.hpp
index cacb003820..e495410817 100644
--- a/libcaf_cuda/caf/cuda/mem_ref.hpp
+++ b/libcaf_cuda/caf/cuda/mem_ref.hpp
@@ -11,6 +11,7 @@
 //#include "caf/cuda/utility.hpp"
 #include <cuda.h>
 #include <atomic>
+#include "caf/cuda/control-layer/memory_actor/mem_token.hpp"
 
 namespace caf::cuda {
 
@@ -138,6 +139,12 @@ class mem_ref : public caf::ref_counted {
     }
 
 
+    // manual binding of the resource object that denotes memory
+    // to the pointer that contains it 
+    void bind_token(caf::cuda::mem_token m_token) {
+        token = std::move(m_token);
+    }
+
 
 private:
   size_t      num_elements_{0};
@@ -149,6 +156,8 @@ class mem_ref : public caf::ref_counted {
   CUcontext ctx;
   mutable std::atomic<size_t> ref_count_{0};
 
+  mem_token token;
+
   bool is_scalar_{false};
   T    host_scalar_{};
 };

From 08decbe1f76ad36e52e38e52736c9f503ac1271a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Mar 2026 08:56:09 -0600
Subject: [PATCH 0481/1000] Initial commit.

---
 .../mmul-actor-benchmarking/CMakeLists.txt    |  44 ++++++
 .../compile_kernels.sh                        |  13 ++
 .../mmul-actor-benchmarking/main.test.cpp     | 144 ++++++++++++++++++
 .../mmul-actor-benchmarking/mmul.cu           |  16 ++
 4 files changed, 217 insertions(+)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
 create mode 100644 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/mmul.cu

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/compile_kernels.sh b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/compile_kernels.sh
new file mode 100755
index 0000000000..f32480e5cb
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/compile_kernels.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
new file mode 100644
index 0000000000..63d0cd8e7c
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
@@ -0,0 +1,144 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/component-actors/mmul_actor/mmul_actor.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+using command =
+  caf::cuda::command_runner<>;
+
+command mmul_command;
+
+struct mmul_state {
+};
+
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
+  return {
+
+    // 1st handler matrices + N, launches a kernel and verifies result
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        int N) {
+
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+      int device = 0;
+      int stream = 1;
+      
+      // Create program and dims
+      auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+      auto arg1 = mmul_command.transfer_memory(device,
+		      stream,
+		      caf::cuda::create_in_arg(matrixA));
+      auto arg2 = mmul_command.transfer_memory(device,
+		      stream,
+		      caf::cuda::create_in_arg(matrixB));
+
+      caf::actor mmul_actor =
+        self->spawn(caf::cuda::mmul_actor_fun<int>, program);
+
+
+      self->mail(arg1, arg2, N, device, stream)
+        .request(mmul_actor, std::chrono::seconds(10))
+        .then(
+          [=](caf::cuda::mem_ptr<int> dC) {
+
+            std::vector<int> matrixC = dC->copy_to_host();
+            std::vector<int> result(N * N);
+
+            serial_matrix_multiply(matrixA, matrixB, result, N);
+
+            if (result == matrixC)
+              std::cout << "actor matrix references match\n";
+            else
+              std::cout << "actor matrix references did not match\n";
+
+            self->quit();
+          }
+        );
+    }
+
+  };
+}
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size) {
+
+  // ------------------------------------
+  // Start timing
+  // ------------------------------------
+  auto start = std::chrono::steady_clock::now();
+
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<int> matrixA(matrix_size * matrix_size,2);
+  std::vector<int> matrixB(matrix_size * matrix_size,3);
+ 
+ using clock = std::chrono::steady_clock;
+
+auto t_start = clock::now();
+
+caf::actor a =sys.spawn(mmul_actor_fun);
+
+anon_mail(matrixA,matrixB,matrix_size).send(a);
+
+auto t_end = clock::now();
+
+
+
+  // Wait for all actors to finish
+  sys.await_all_actors_done();
+
+  // ------------------------------------
+  // Stop timing
+  // ------------------------------------
+  auto end = std::chrono::steady_clock::now();
+  auto duration_ms =
+      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
+            << ", time=" << duration_ms << " ms\n";
+}
+
+void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager::init(sys);
+  run_mmul_test(sys,10);
+
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/mmul.cu b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From cd4bd476091d5e98547d2755136da2a40ea33ac8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Mar 2026 08:58:19 -0600
Subject: [PATCH 0482/1000] Updated test for benchmarking, by removing
 verification.

---
 .../mmul-actor-benchmarking/main.test.cpp              | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
index 63d0cd8e7c..656ea6ab6b 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
@@ -77,8 +77,10 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
           [=](caf::cuda::mem_ptr<int> dC) {
 
             std::vector<int> matrixC = dC->copy_to_host();
-            std::vector<int> result(N * N);
+            
+	    //std::vector<int> result(N * N);
 
+	    /*
             serial_matrix_multiply(matrixA, matrixB, result, N);
 
             if (result == matrixC)
@@ -86,6 +88,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
             else
               std::cout << "actor matrix references did not match\n";
 
+	      */
             self->quit();
           }
         );
@@ -134,7 +137,10 @@ auto t_end = clock::now();
 
 void caf_main(caf::actor_system& sys) {
   caf::cuda::manager::init(sys);
-  run_mmul_test(sys,10);
+  run_mmul_test(sys,1000);
+  run_mmul_test(sys,4000);
+  run_mmul_test(sys,8000);
+  run_mmul_test(sys,12000);
 
 }
 

From b172bc58020db5398bf1ddc5bb1faed1f81557f8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Mar 2026 09:09:32 -0600
Subject: [PATCH 0483/1000] Initial commit.

---
 .../cuda-program-benchmark/compile_kernels.sh    | 13 +++++++++++++
 .../cuda-program-benchmark/mmul.cu               | 16 ++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100755 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/mmul.cu

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/compile_kernels.sh b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/compile_kernels.sh
new file mode 100755
index 0000000000..f32480e5cb
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/compile_kernels.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/mmul.cu b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From 83122edeba203b5e43865f2bbdbc26a5c4e3f957 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Mar 2026 09:13:20 -0600
Subject: [PATCH 0484/1000] Initial commit.

---
 .../matrix_mmul_driver.cpp                    | 173 ++++++++++++++++++
 1 file changed, 173 insertions(+)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mmul_driver.cpp

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mmul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mmul_driver.cpp
new file mode 100644
index 0000000000..4e5467bd4d
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mmul_driver.cpp
@@ -0,0 +1,173 @@
+// matrix_mul_driver.cpp
+#include <cuda.h>
+#include <iostream>
+#include <vector>
+#include <chrono>
+#include <fstream>
+#include <sstream>
+#include <stdexcept>
+#include <cmath>
+
+static void checkCU(CUresult r, const char* where) {
+    if (r != CUDA_SUCCESS) {
+        const char *str = nullptr;
+        cuGetErrorString(r, &str);
+        std::cerr << "CUDA Driver API error at " << where << " -> "
+                  << (str ? str : "unknown") << " (" << (int)r << ")\n";
+        std::exit(EXIT_FAILURE);
+    }
+}
+
+std::string readFile(const std::string &path) {
+    std::ifstream in(path, std::ios::in | std::ios::binary);
+    if (!in) throw std::runtime_error("Failed to open " + path);
+    std::ostringstream ss;
+    ss << in.rdbuf();
+    return ss.str();
+}
+
+void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
+    std::cout << "Running N = " << N << " ...\n";
+    size_t elements = static_cast<size_t>(N) * static_cast<size_t>(N);
+    size_t bytes = elements * sizeof(int);
+
+    // Check for overflow (extremely large N)
+    if (elements / N != static_cast<size_t>(N)) {
+        std::cerr << "Integer overflow for N = " << N << "\n";
+        return;
+    }
+
+    // Host buffers
+    std::vector<int> h_a, h_b, h_c;
+    try {
+        h_a.resize(elements);
+        h_b.resize(elements);
+        h_c.resize(elements);
+    } catch (const std::bad_alloc&) {
+        std::cerr << "Host allocation failed for N = " << N << " (need "
+                  << bytes << " bytes per matrix)\n";
+        return;
+    }
+
+    // Fill test data (simple pattern)
+    for (size_t i = 0; i < elements; ++i) {
+        h_a[i] = 1; // simple values to make it predictable
+        h_b[i] = 1;
+    }
+
+    CUdeviceptr d_a = 0, d_b = 0, d_c = 0;
+    CUresult r;
+
+    // Try allocating device memory (may fail on small GPUs for large N).
+    r = cuMemAlloc(&d_a, bytes);
+    if (r != CUDA_SUCCESS) {
+        std::cerr << "cuMemAlloc d_a failed for N=" << N << " ("
+                  << bytes << " bytes). Skipping.\n";
+        return;
+    }
+    checkCU(cuMemAlloc(&d_b, bytes), "cuMemAlloc d_b");
+    checkCU(cuMemAlloc(&d_c, bytes), "cuMemAlloc d_c");
+
+    // Grid / block
+    const unsigned int blockX = 16;
+    const unsigned int blockY = 16;
+    unsigned int gridX = static_cast<unsigned int>((N + blockX - 1) / blockX);
+    unsigned int gridY = static_cast<unsigned int>((N + blockY - 1) / blockY);
+
+    // Prepare kernel parameter values as described by Driver API:
+    // pointers to the argument values
+    // Note: pass address-of CUdeviceptr, and address-of N (int).
+    void* kernelParams[] = { &d_a, &d_b, &d_c, &N };
+
+    // Time everything from H->D copy to D->H copy and sync
+    auto t0 = std::chrono::steady_clock::now();
+
+    checkCU(cuMemcpyHtoD(d_a, h_a.data(), bytes), "cuMemcpyHtoD d_a");
+    checkCU(cuMemcpyHtoD(d_b, h_b.data(), bytes), "cuMemcpyHtoD d_b");
+
+    // Launch kernel
+    checkCU(cuLaunchKernel(kernel,
+                           gridX, gridY, 1,         // grid
+                           blockX, blockY, 1,       // block
+                           0,                       // shared mem
+                           nullptr,                 // stream
+                           kernelParams,            // kernel params
+                           nullptr), "cuLaunchKernel");
+
+    // Wait for completion
+    checkCU(cuCtxSynchronize(), "cuCtxSynchronize");
+
+    // Copy result back
+    checkCU(cuMemcpyDtoH(h_c.data(), d_c, bytes), "cuMemcpyDtoH d_c");
+
+    auto t1 = std::chrono::steady_clock::now();
+    std::chrono::duration<double> dur = t1 - t0;
+    std::cout << "N=" << N << " time (alloc+H2D+kernel+DtoH): " << dur.count() << " s\n";
+
+    // Quick spot-check for correctness on a few entries (since we used all-ones, result should be N)
+    bool ok = true;
+    if (elements > 0) {
+        // sample first, middle, last
+        std::vector<size_t> samples = {0, elements / 2, elements - 1};
+        for (size_t s : samples) {
+            if (h_c[s] != N) { ok = false; break; }
+        }
+    }
+    std::cout << "Spot-check: " << (ok ? "PASS" : "FAIL (sample mismatch)") << "\n";
+
+    // Free device memory
+    checkCU(cuMemFree(d_a), "cuMemFree d_a");
+    checkCU(cuMemFree(d_b), "cuMemFree d_b");
+    checkCU(cuMemFree(d_c), "cuMemFree d_c");
+}
+
+int main(int argc, char** argv) {
+    // optional: accept single N from command line
+    std::vector<int> sizes = {1000, 4000, 8000, 12000};
+    if (argc > 1) {
+        sizes.clear();
+        for (int i = 1; i < argc; ++i) sizes.push_back(std::stoi(argv[i]));
+    }
+
+    // Initialize driver
+    checkCU(cuInit(0), "cuInit");
+
+    // pick device 0
+    CUdevice dev;
+    checkCU(cuDeviceGet(&dev, 0), "cuDeviceGet(0)");
+    CUcontext ctx;
+    checkCU(cuCtxCreate(&ctx, 0, dev), "cuCtxCreate");
+
+    // find the PTX file next to the executable or in current dir - we assume "mmul.ptx" is present
+    const std::string ptxPath = "mmul.ptx";
+    std::string ptx;
+    try {
+        ptx = readFile(ptxPath);
+    } catch (const std::exception &e) {
+        std::cerr << "Failed to read PTX file '" << ptxPath << "': " << e.what() << "\n";
+        return EXIT_FAILURE;
+    }
+
+    // load module
+    CUmodule module;
+    checkCU(cuModuleLoadDataEx(&module, ptx.c_str(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
+
+    // get function handle
+    CUfunction kernel;
+    checkCU(cuModuleGetFunction(&kernel, module, "matrixMul"), "cuModuleGetFunction matrixMul");
+
+    // Run for each size
+    for (int N : sizes) {
+        try {
+            runMatrixMul(module, kernel, N);
+        } catch (const std::exception &e) {
+            std::cerr << "Exception while running N=" << N << ": " << e.what() << "\n";
+        }
+        std::cout << "----------------------------------------\n";
+    }
+
+    // cleanup
+    checkCU(cuModuleUnload(module), "cuModuleUnload");
+    checkCU(cuCtxDestroy(ctx), "cuCtxDestroy");
+    return 0;
+}

From 227f089562c7b23ef61610c783650491c2402b05 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Mar 2026 09:15:38 -0600
Subject: [PATCH 0485/1000] Renamed file.

---
 .../cuda-program-benchmark/CMakeLists.txt     | 52 +++++++++++++++++++
 ..._mmul_driver.cpp => matrix_mul_driver.cpp} |  0
 2 files changed, 52 insertions(+)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/CMakeLists.txt
 rename libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/{matrix_mmul_driver.cpp => matrix_mul_driver.cpp} (100%)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/CMakeLists.txt
new file mode 100644
index 0000000000..e26c6c956c
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/CMakeLists.txt
@@ -0,0 +1,52 @@
+cmake_minimum_required(VERSION 3.10)
+project(cuda_driver_mmul LANGUAGES CXX)
+
+# Find nvcc
+find_program(NVCC_EXECUTABLE nvcc)
+if(NOT NVCC_EXECUTABLE)
+    message(FATAL_ERROR "nvcc not found. Make sure CUDA toolkit is installed and nvcc is in PATH.")
+endif()
+
+# Try to locate CUDA driver library (libcuda)
+find_library(CUDA_DRIVER_LIB NAMES cuda HINTS ENV CUDA_HOME /usr/local/cuda/lib64 /usr/local/cuda/lib)
+if(NOT CUDA_DRIVER_LIB)
+    message(FATAL_ERROR "Could not find CUDA driver library (libcuda). Ensure CUDA is installed.")
+endif()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# Ensure we build PTX from mmul.cu into build directory as mmul.ptx
+set(MMUL_CU "${CMAKE_SOURCE_DIR}/mmul.cu")
+set(MMUL_PTX "${CMAKE_BINARY_DIR}/mmul.ptx")
+
+add_custom_command(
+    OUTPUT "${MMUL_PTX}"
+    COMMAND ${NVCC_EXECUTABLE} -ptx ${MMUL_CU} -o "${MMUL_PTX}"
+    DEPENDS "${MMUL_CU}"
+    COMMENT "Compiling mmul.cu -> mmul.ptx"
+    VERBATIM
+)
+
+add_custom_target(kernel_ptx ALL DEPENDS "${MMUL_PTX}")
+
+# Host executable
+add_executable(matrix_mul_driver matrix_mul_driver.cpp)
+add_dependencies(matrix_mul_driver kernel_ptx)
+
+# Make sure PTX is copied to the executable directory so the program can find it at runtime
+add_custom_command(TARGET matrix_mul_driver POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+        "${MMUL_PTX}"
+        $<TARGET_FILE_DIR:matrix_mul_driver>
+)
+
+# Link to the CUDA driver library (libcuda)
+target_link_libraries(matrix_mul_driver PRIVATE ${CUDA_DRIVER_LIB})
+
+# Include path - try to find cuda headers (optional)
+find_path(CUDA_INCLUDE_DIR cuda.h HINTS ENV CUDA_HOME /usr/local/cuda/include)
+if(CUDA_INCLUDE_DIR)
+    target_include_directories(matrix_mul_driver PRIVATE ${CUDA_INCLUDE_DIR})
+endif()
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mmul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
similarity index 100%
rename from libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mmul_driver.cpp
rename to libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp

From c19e1e28788fb3ded825761d67a10e40053fb97e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Mar 2026 09:20:05 -0600
Subject: [PATCH 0486/1000] Changed units from seconds to ms.

---
 .../cuda-program-benchmark/matrix_mul_driver.cpp          | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
index 4e5467bd4d..cf392a139d 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
@@ -101,8 +101,12 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     checkCU(cuMemcpyDtoH(h_c.data(), d_c, bytes), "cuMemcpyDtoH d_c");
 
     auto t1 = std::chrono::steady_clock::now();
-    std::chrono::duration<double> dur = t1 - t0;
-    std::cout << "N=" << N << " time (alloc+H2D+kernel+DtoH): " << dur.count() << " s\n";
+   auto dur = std::chrono::duration<double, std::milli>(t1 - t0);
+
+std::cout << "N=" << N
+          << " time (alloc+H2D+kernel+DtoH): "
+          << dur.count() << " ms\n"; 
+
 
     // Quick spot-check for correctness on a few entries (since we used all-ones, result should be N)
     bool ok = true;

From 3b498bcdd9e572748d048a0ef315c8ee4c6d10d2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Mar 2026 10:42:54 -0600
Subject: [PATCH 0487/1000] Using std::move to reduce uneccesary copies.

---
 .../mmul-actor-benchmarking/main.test.cpp     | 107 ++++++++++++++----
 1 file changed, 84 insertions(+), 23 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
index 656ea6ab6b..0ece10d702 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
@@ -46,49 +46,110 @@ struct mmul_state {
 caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
   return {
 
-    // 1st handler matrices + N, launches a kernel and verifies result
     [=](const std::vector<int>& matrixA,
         const std::vector<int>& matrixB,
         int N) {
 
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
+      using clock = std::chrono::steady_clock;
+      using ms = std::chrono::duration<double, std::milli>;
+
+      auto t_total_start = clock::now();
 
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
       int device = 0;
       int stream = 1;
-      
-      // Create program and dims
+
       auto program =
         mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
-      auto arg1 = mmul_command.transfer_memory(device,
-		      stream,
-		      caf::cuda::create_in_arg(matrixA));
-      auto arg2 = mmul_command.transfer_memory(device,
-		      stream,
-		      caf::cuda::create_in_arg(matrixB));
+      // -------------------------
+      // transfer A
+      // -------------------------
+      auto t_a_start = clock::now();
+
+      auto arg1 = mmul_command.transfer_memory(
+          device,
+          stream,
+          caf::cuda::create_in_arg(std::move(matrixA)));
+
+      auto t_a_end = clock::now();
+
+      // -------------------------
+      // transfer B
+      // -------------------------
+      auto t_b_start = clock::now();
+
+      auto arg2 = mmul_command.transfer_memory(
+          device,
+          stream,
+          caf::cuda::create_in_arg(std::move(matrixB)));
+
+      auto t_b_end = clock::now();
+
+      // -------------------------
+      // spawn
+      // -------------------------
+      auto t_spawn_start = clock::now();
 
       caf::actor mmul_actor =
         self->spawn(caf::cuda::mmul_actor_fun<int>, program);
 
+      auto t_spawn_end = clock::now();
+
+      // -------------------------
+      // request
+      // -------------------------
+      auto t_request_start = clock::now();
 
       self->mail(arg1, arg2, N, device, stream)
-        .request(mmul_actor, std::chrono::seconds(10))
+        .request(mmul_actor, std::chrono::seconds(30))
         .then(
           [=](caf::cuda::mem_ptr<int> dC) {
 
+            auto t_response_received = clock::now();
+
+            // -------------------------
+            // copy to host
+            // -------------------------
+            auto t_copy_start = clock::now();
+
             std::vector<int> matrixC = dC->copy_to_host();
-            
-	    //std::vector<int> result(N * N);
 
-	    /*
-            serial_matrix_multiply(matrixA, matrixB, result, N);
+            auto t_copy_end = clock::now();
+            auto t_total_end = clock::now();
+
+            // -------------------------
+            // Print timings
+            // -------------------------
+
+            std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
+
+            std::cout << "transfer A: "
+                      << ms(t_a_end - t_a_start).count()
+                      << " ms\n";
+
+            std::cout << "transfer B: "
+                      << ms(t_b_end - t_b_start).count()
+                      << " ms\n";
+
+            std::cout << "spawn actor: "
+                      << ms(t_spawn_end - t_spawn_start).count()
+                      << " ms\n";
+
+            std::cout << "request → response latency: "
+                      << ms(t_response_received - t_request_start).count()
+                      << " ms\n";
+
+            std::cout << "copy_to_host: "
+                      << ms(t_copy_end - t_copy_start).count()
+                      << " ms\n";
+
+            std::cout << "TOTAL end-to-end: "
+                      << ms(t_total_end - t_total_start).count()
+                      << " ms\n";
 
-            if (result == matrixC)
-              std::cout << "actor matrix references match\n";
-            else
-              std::cout << "actor matrix references did not match\n";
+            std::cout << "=============================================\n";
 
-	      */
             self->quit();
           }
         );
@@ -138,9 +199,9 @@ auto t_end = clock::now();
 void caf_main(caf::actor_system& sys) {
   caf::cuda::manager::init(sys);
   run_mmul_test(sys,1000);
-  run_mmul_test(sys,4000);
-  run_mmul_test(sys,8000);
-  run_mmul_test(sys,12000);
+  //run_mmul_test(sys,4000);
+  //run_mmul_test(sys,8000);
+  //run_mmul_test(sys,12000);
 
 }
 

From 05ac38050bf702daf107599ba6650ec7f6bf48d7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 4 Mar 2026 10:55:45 -0600
Subject: [PATCH 0488/1000] Changed make_arg to take args by refence rather
 than value to reduce uneccesary copying.

---
 libcaf_cuda/caf/cuda/device.hpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 135bacf667..a6333af4f2 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -112,17 +112,17 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
 
   // Overloads for make_arg using actor_id
   template <typename T>
-  mem_ptr<T> make_arg(in<T> arg, int actor_id) {
+  mem_ptr<T> make_arg(const in<T>& arg, int actor_id) {
     return global_argument(arg, actor_id, IN);
   }
 
   template <typename T>
-  mem_ptr<T> make_arg(in_out<T> arg, int actor_id) {
+  mem_ptr<T> make_arg(const in_out<T>& arg, int actor_id) {
     return global_argument(arg, actor_id, IN_OUT);
   }
 
   template <typename T>
-  mem_ptr<T> make_arg(out<T> arg, int actor_id) {
+  mem_ptr<T> make_arg(const out<T>& arg, int actor_id) {
     return scratch_argument(arg, actor_id, OUT);
   }
 
@@ -130,18 +130,18 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   // Overloads for make_arg using CUstream directly
 
   template <typename T>
-  mem_ptr<T> make_arg(in<T> arg, CUstream stream) {
+  mem_ptr<T> make_arg(const in<T>& arg, CUstream stream) {
      return global_argument(arg, stream, IN);
    }
 
 
   template <typename T>
-  mem_ptr<T> make_arg(in_out<T> arg, CUstream stream) {
+  mem_ptr<T> make_arg(const in_out<T>& arg, CUstream stream) {
     return global_argument(arg, stream, IN_OUT);
    }
 
   template <typename T>
-  mem_ptr<T> make_arg(out<T> arg, CUstream stream) {
+  mem_ptr<T> make_arg(const out<T>& arg, CUstream stream) {
   return scratch_argument(arg, stream, OUT);
   }
 

From 14fcc3a558dc7d209fcc1f298d711fe7b84d112e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 5 Mar 2026 06:37:53 -0600
Subject: [PATCH 0489/1000] Added a move constructor to in and inout to reduce
 copy overhead.

---
 libcaf_cuda/caf/cuda/types.hpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/types.hpp b/libcaf_cuda/caf/cuda/types.hpp
index 92dcb21bc1..d17f169e0f 100644
--- a/libcaf_cuda/caf/cuda/types.hpp
+++ b/libcaf_cuda/caf/cuda/types.hpp
@@ -91,6 +91,9 @@ class in_impl {
   in_impl(const in_impl&) = default;
   in_impl& operator=(const in_impl&) = default;
 
+  explicit in_impl(std::vector<T>&& buf)
+  : data_(std::move(buf)) {}
+
   // Move constructor
   in_impl(in_impl&& other) noexcept
     : data_(std::move(other.data_)) {
@@ -223,6 +226,8 @@ class in_out_impl {
   in_out_impl() : data_(T{}) {}
   explicit in_out_impl(const T& val) : data_(val) {}
   explicit in_out_impl(const std::vector<T>& buf) : data_(buf) {}
+ explicit in_out_impl(std::vector<T>&& buf)
+  : data_(std::move(buf)) {}
 
   in_out_impl(const in_out_impl&) = default;
   in_out_impl& operator=(const in_out_impl&) = default;

From 07ce70d89189dd83ac0f111f19cde47435d7a920 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 5 Mar 2026 06:48:55 -0600
Subject: [PATCH 0490/1000] Updated in_impl to reduce uncessary copies.

---
 libcaf_cuda/caf/cuda/types.hpp | 62 ++++++++++++++--------------------
 1 file changed, 25 insertions(+), 37 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/types.hpp b/libcaf_cuda/caf/cuda/types.hpp
index d17f169e0f..1611dbb044 100644
--- a/libcaf_cuda/caf/cuda/types.hpp
+++ b/libcaf_cuda/caf/cuda/types.hpp
@@ -71,75 +71,63 @@ struct output_buffer {
 template <typename T>
 class in_impl {
 private:
-  std::variant<T, std::vector<T>> data_;
+  T scalar_;
+  const T* ptr_;
+  size_t size_;
+  bool is_scalar_;
   bool moved_from_ = false;
 
   void check_valid() const {
     if (moved_from_)
-      throw std::runtime_error(std::string("Use-after-move detected in ") + typeid(*this).name());
+      throw std::runtime_error("Use-after-move detected in in_impl");
   }
 
 public:
   using value_type = T;
 
-  // Constructors
-  in_impl() : data_(T{}) {}
-  explicit in_impl(const T& val) : data_(val) {}
-  explicit in_impl(const std::vector<T>& buf) : data_(buf) {}
+  // scalar ctor
+  in_impl()
+    : scalar_{}, ptr_{&scalar_}, size_{1}, is_scalar_{true} {}
 
-  // Copy constructor/assignment
-  in_impl(const in_impl&) = default;
-  in_impl& operator=(const in_impl&) = default;
+  explicit in_impl(const T& val)
+    : scalar_{val}, ptr_{&scalar_}, size_{1}, is_scalar_{true} {}
 
-  explicit in_impl(std::vector<T>&& buf)
-  : data_(std::move(buf)) {}
-
-  // Move constructor
-  in_impl(in_impl&& other) noexcept
-    : data_(std::move(other.data_)) {
-    other.moved_from_ = true;
-  }
+  // zero-copy vector view
+  explicit in_impl(const std::vector<T>& buf)
+    : scalar_{}, ptr_{buf.data()}, size_{buf.size()}, is_scalar_{false} {}
 
-  // Move assignment
-  in_impl& operator=(in_impl&& other) noexcept {
-    if (this != &other) {
-      data_ = std::move(other.data_);
-      other.moved_from_ = true;
-    }
-    return *this;
-  }
+  explicit in_impl(std::vector<T>&& buf)
+    : scalar_{}, ptr_{buf.data()}, size_{buf.size()}, is_scalar_{false} {}
 
   bool is_scalar() const {
     check_valid();
-    return std::holds_alternative<T>(data_);
+    return is_scalar_;
   }
 
   const T& getscalar() const {
     check_valid();
-    if (!is_scalar())
+    if (!is_scalar_)
       throw std::runtime_error("in_impl does not hold scalar");
-    return std::get<T>(data_);
+    return scalar_;
   }
 
-  const std::vector<T>& get_buffer() const {
+  const T* data() const {
     check_valid();
-    if (is_scalar())
-      throw std::runtime_error("in_impl does not hold buffer");
-    return std::get<std::vector<T>>(data_);
+    return ptr_;
   }
 
-  const T* data() const {
+  size_t size() const {
     check_valid();
-    return is_scalar() ? &std::get<T>(data_) : std::get<std::vector<T>>(data_).data();
+    return size_;
   }
 
-  std::size_t size() const {
+  // Required for CAF serialization compatibility
+  std::vector<T> get_buffer() const {
     check_valid();
-    return is_scalar() ? 1 : std::get<std::vector<T>>(data_).size();
+    return std::vector<T>(ptr_, ptr_ + size_);
   }
 };
 
-
 //represents a write only buffer on the gpu
 template <typename T>
 class out_impl {

From 48507c4ce5f0aa6ff22c401adf83e55e9c804f41 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 5 Mar 2026 06:57:25 -0600
Subject: [PATCH 0491/1000] Updated in_out and out types to reduce uncessary
 copying.

---
 libcaf_cuda/caf/cuda/types.hpp | 104 +++++++++++++++------------------
 1 file changed, 48 insertions(+), 56 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/types.hpp b/libcaf_cuda/caf/cuda/types.hpp
index 1611dbb044..f31aa0c1f4 100644
--- a/libcaf_cuda/caf/cuda/types.hpp
+++ b/libcaf_cuda/caf/cuda/types.hpp
@@ -132,39 +132,43 @@ class in_impl {
 template <typename T>
 class out_impl {
 private:
-  std::variant<T, std::vector<T>> data_;
+  T scalar_;
+  int size_ = 1;
+  bool is_scalar_ = true;
   bool moved_from_ = false;
-  int size_ = 0; // Always store as int
 
   void check_valid() const {
     if (moved_from_)
-      throw std::runtime_error(std::string("Use-after-move detected in ") + typeid(*this).name());
+      throw std::runtime_error("Use-after-move detected in out_impl");
   }
 
 public:
   using value_type = T;
 
-  out_impl() : data_(T{}), size_(1) {}
-  
-  // Replaces old scalar constructor: just set size
-  explicit out_impl(int size) 
-    : data_(std::vector<T>(size)), size_(size) {}
-  
-  explicit out_impl(const std::vector<T>& buf) 
-    : data_(buf), size_(static_cast<int>(buf.size())) {}
+  out_impl() : scalar_{}, size_(1), is_scalar_(true) {}
+
+  // allocate GPU buffer of given size
+  explicit out_impl(int size)
+    : scalar_{}, size_(size), is_scalar_(false) {}
+
+  explicit out_impl(const std::vector<T>& buf)
+    : scalar_{}, size_(static_cast<int>(buf.size())), is_scalar_(false) {}
 
   out_impl(const out_impl&) = default;
   out_impl& operator=(const out_impl&) = default;
 
-  out_impl(out_impl&& other) noexcept
-    : data_(std::move(other.data_)), size_(other.size_) {
+  out_impl(out_impl&& other) noexcept {
+    scalar_ = other.scalar_;
+    size_ = other.size_;
+    is_scalar_ = other.is_scalar_;
     other.moved_from_ = true;
   }
 
   out_impl& operator=(out_impl&& other) noexcept {
     if (this != &other) {
-      data_ = std::move(other.data_);
+      scalar_ = other.scalar_;
       size_ = other.size_;
+      is_scalar_ = other.is_scalar_;
       other.moved_from_ = true;
     }
     return *this;
@@ -172,20 +176,12 @@ class out_impl {
 
   bool is_scalar() const {
     check_valid();
-    return std::holds_alternative<T>(data_);
-  }
-
-   const std::vector<T>& get_buffer() const {
-    check_valid();
-    if (is_scalar())
-      throw std::runtime_error("out_impl does not hold buffer");
-    return std::get<std::vector<T>>(data_);
+    return is_scalar_;
   }
 
   const T* data() const {
     check_valid();
-    return is_scalar() ? reinterpret_cast<const T*>(&size_)
-                       : std::get<std::vector<T>>(data_).data();
+    return &scalar_;
   }
 
   size_t size() const {
@@ -193,73 +189,69 @@ class out_impl {
     return static_cast<size_t>(size_);
   }
 
+  // compatibility with CAF serialization
+  std::vector<T> get_buffer() const {
+    check_valid();
+    return std::vector<T>(size_);
+  }
 };
 
 
-//creates a read-write buffer on the gpu
 template <typename T>
 class in_out_impl {
 private:
-  std::variant<T, std::vector<T>> data_;
+  T scalar_;
+  const T* ptr_;
+  size_t size_;
+  bool is_scalar_;
   bool moved_from_ = false;
 
   void check_valid() const {
     if (moved_from_)
-      throw std::runtime_error(std::string("Use-after-move detected in ") + typeid(*this).name());
+      throw std::runtime_error("Use-after-move detected in in_out_impl");
   }
 
 public:
   using value_type = T;
 
-  in_out_impl() : data_(T{}) {}
-  explicit in_out_impl(const T& val) : data_(val) {}
-  explicit in_out_impl(const std::vector<T>& buf) : data_(buf) {}
- explicit in_out_impl(std::vector<T>&& buf)
-  : data_(std::move(buf)) {}
+  in_out_impl()
+    : scalar_{}, ptr_{&scalar_}, size_{1}, is_scalar_{true} {}
 
-  in_out_impl(const in_out_impl&) = default;
-  in_out_impl& operator=(const in_out_impl&) = default;
+  explicit in_out_impl(const T& val)
+    : scalar_{val}, ptr_{&scalar_}, size_{1}, is_scalar_{true} {}
 
-  in_out_impl(in_out_impl&& other) noexcept
-    : data_(std::move(other.data_)) {
-    other.moved_from_ = true;
-  }
+  explicit in_out_impl(const std::vector<T>& buf)
+    : scalar_{}, ptr_{buf.data()}, size_{buf.size()}, is_scalar_{false} {}
 
-  in_out_impl& operator=(in_out_impl&& other) noexcept {
-    if (this != &other) {
-      data_ = std::move(other.data_);
-      other.moved_from_ = true;
-    }
-    return *this;
-  }
+  explicit in_out_impl(std::vector<T>&& buf)
+    : scalar_{}, ptr_{buf.data()}, size_{buf.size()}, is_scalar_{false} {}
 
   bool is_scalar() const {
     check_valid();
-    return std::holds_alternative<T>(data_);
+    return is_scalar_;
   }
 
   const T& getscalar() const {
     check_valid();
-    if (!is_scalar())
+    if (!is_scalar_)
       throw std::runtime_error("in_out_impl does not hold scalar");
-    return std::get<T>(data_);
+    return scalar_;
   }
 
-  const std::vector<T>& get_buffer() const {
+  const T* data() const {
     check_valid();
-    if (is_scalar())
-      throw std::runtime_error("in_out_impl does not hold buffer");
-    return std::get<std::vector<T>>(data_);
+    return ptr_;
   }
 
-  const T* data() const {
+  std::size_t size() const {
     check_valid();
-    return is_scalar() ? &std::get<T>(data_) : std::get<std::vector<T>>(data_).data();
+    return size_;
   }
 
-  std::size_t size() const {
+  // required for CAF serialization
+  std::vector<T> get_buffer() const {
     check_valid();
-    return is_scalar() ? 1 : std::get<std::vector<T>>(data_).size();
+    return std::vector<T>(ptr_, ptr_ + size_);
   }
 };
 

From 45c5160de6ec780368047a384d69a7a51d5a9053 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 5 Mar 2026 13:34:36 -0600
Subject: [PATCH 0492/1000] Implemented a copy to host overload that enables
 users to supply their own buffer, to help users remove overhead of creating
 an std::vector on the stack each time.

---
 libcaf_cuda/caf/cuda/mem_ref.hpp | 59 +++++++++++++++++++++++---------
 1 file changed, 43 insertions(+), 16 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/mem_ref.hpp b/libcaf_cuda/caf/cuda/mem_ref.hpp
index e495410817..fb7b0f85d9 100644
--- a/libcaf_cuda/caf/cuda/mem_ref.hpp
+++ b/libcaf_cuda/caf/cuda/mem_ref.hpp
@@ -108,22 +108,49 @@ class mem_ref : public caf::ref_counted {
 
   //copies gpu memory back to cpu memory in the form of an std::vector
   std::vector<T> copy_to_host() const {
-    if (access_ == IN)
-    {
-	    throw std::runtime_error("Cannt copy a read only buffer back to device\n");
-    } 
-    if (is_scalar_) {
-      return std::vector<T>{host_scalar_};
-    }
-    std::vector<T> host_data(num_elements_);
-    size_t bytes = num_elements_ * sizeof(T);
-    CHECK_CUDA(cuCtxPushCurrent(ctx));
-    CUstream s = stream_ ? stream_ : nullptr;
-    CHECK_CUDA(cuMemcpyDtoHAsync(host_data.data(), memory_, bytes, s));
-    if (s) CHECK_CUDA(cuStreamSynchronize(s));
-    else  CHECK_CUDA(cuCtxSynchronize());
-    CHECK_CUDA(cuCtxPopCurrent(nullptr));
-    return host_data;
+	  if (access_ == IN)
+	  {
+		  throw std::runtime_error("Cannt copy a read only buffer back to device\n");
+	  } 
+	  if (is_scalar_) {
+		  return std::vector<T>{host_scalar_};
+	  }
+	  std::vector<T> host_data(num_elements_);
+	  size_t bytes = num_elements_ * sizeof(T);
+	  CHECK_CUDA(cuCtxPushCurrent(ctx));
+	  CUstream s = stream_ ? stream_ : nullptr;
+	  CHECK_CUDA(cuMemcpyDtoHAsync(host_data.data(), memory_, bytes, s));
+	  if (s) CHECK_CUDA(cuStreamSynchronize(s));
+	  else  CHECK_CUDA(cuCtxSynchronize());
+	  CHECK_CUDA(cuCtxPopCurrent(nullptr));
+	  return host_data;
+  }
+
+
+  //copies buffer back to dst buffer supplied by the user
+  //count is number of elements the buffer has
+  void copy_to_host(T* dst, size_t count) const {
+	  if (access_ == IN)
+		  throw std::runtime_error("Cannot copy a read only buffer back to host");
+
+	  if (is_scalar_) {
+		  dst[0] = host_scalar_;
+		  return;
+	  }
+
+	  size_t bytes = count * sizeof(T);
+
+	  CHECK_CUDA(cuCtxPushCurrent(ctx));
+	  CUstream s = stream_ ? stream_ : nullptr;
+
+	  CHECK_CUDA(cuMemcpyDtoHAsync(dst, memory_, bytes, s));
+
+	  if (s)
+		  CHECK_CUDA(cuStreamSynchronize(s));
+	  else
+		  CHECK_CUDA(cuCtxSynchronize());
+
+	  CHECK_CUDA(cuCtxPopCurrent(nullptr));
   }
 
 

From d126e5c8def9bb0b4a4276d404c2490c33e286cf Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 5 Mar 2026 13:38:31 -0600
Subject: [PATCH 0493/1000] Moved matrixC allocation out of the actor. THis
 change is being made specifically since the cuda-benchmark does not include
 its memory allocations on the stack, so to keep everything fair this
 benchmark will not either.

---
 .../mmul-actor-benchmarking/main.test.cpp     | 60 +++++++++++++++----
 1 file changed, 48 insertions(+), 12 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
index 0ece10d702..0ee4c74afa 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
@@ -43,6 +43,12 @@ command mmul_command;
 struct mmul_state {
 };
 
+//global output buffer meant to disclude it from timing
+//the other benchmark test do not include its memory allocations in it 
+//so its only fair that we do not either 
+std::vector<int> matrixC;
+
+
 caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
   return {
 
@@ -62,32 +68,50 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
       auto program =
         mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
+      // -------------------------
+      // create_in_arg A
+      // -------------------------
+      auto t_a_inarg_start = clock::now();
+
+      auto inA = caf::cuda::create_in_arg(std::move(matrixA));
+
+      auto t_a_inarg_end = clock::now();
+
       // -------------------------
       // transfer A
       // -------------------------
-      auto t_a_start = clock::now();
+      auto t_a_transfer_start = clock::now();
 
       auto arg1 = mmul_command.transfer_memory(
           device,
           stream,
-          caf::cuda::create_in_arg(std::move(matrixA)));
+          std::move(inA));
+
+      auto t_a_transfer_end = clock::now();
+
+      // -------------------------
+      // create_in_arg B
+      // -------------------------
+      auto t_b_inarg_start = clock::now();
+
+      auto inB = caf::cuda::create_in_arg(std::move(matrixB));
 
-      auto t_a_end = clock::now();
+      auto t_b_inarg_end = clock::now();
 
       // -------------------------
       // transfer B
       // -------------------------
-      auto t_b_start = clock::now();
+      auto t_b_transfer_start = clock::now();
 
       auto arg2 = mmul_command.transfer_memory(
           device,
           stream,
-          caf::cuda::create_in_arg(std::move(matrixB)));
+          std::move(inB));
 
-      auto t_b_end = clock::now();
+      auto t_b_transfer_end = clock::now();
 
       // -------------------------
-      // spawn
+      // spawn actor
       // -------------------------
       auto t_spawn_start = clock::now();
 
@@ -108,12 +132,15 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
 
             auto t_response_received = clock::now();
 
+            //std::vector<int> matrixC(N*N);
             // -------------------------
             // copy to host
             // -------------------------
             auto t_copy_start = clock::now();
 
-            std::vector<int> matrixC = dC->copy_to_host();
+            //std::vector<int> matrixC = dC->copy_to_host();
+
+	    dC->copy_to_host(matrixC.data(),N*N);
 
             auto t_copy_end = clock::now();
             auto t_total_end = clock::now();
@@ -124,12 +151,20 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
 
             std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
 
+            std::cout << "create_in_arg A: "
+                      << ms(t_a_inarg_end - t_a_inarg_start).count()
+                      << " ms\n";
+
             std::cout << "transfer A: "
-                      << ms(t_a_end - t_a_start).count()
+                      << ms(t_a_transfer_end - t_a_transfer_start).count()
+                      << " ms\n";
+
+            std::cout << "create_in_arg B: "
+                      << ms(t_b_inarg_end - t_b_inarg_start).count()
                       << " ms\n";
 
             std::cout << "transfer B: "
-                      << ms(t_b_end - t_b_start).count()
+                      << ms(t_b_transfer_end - t_b_transfer_start).count()
                       << " ms\n";
 
             std::cout << "spawn actor: "
@@ -158,7 +193,6 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
   };
 }
 
-
 void run_mmul_test(caf::actor_system& sys, int matrix_size) {
 
   // ------------------------------------
@@ -169,7 +203,9 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size) {
   // Spawn num_actors actors running the mmul behavior
   std::vector<int> matrixA(matrix_size * matrix_size,2);
   std::vector<int> matrixB(matrix_size * matrix_size,3);
- 
+
+  matrixC.resize(matrix_size*matrix_size);
+
  using clock = std::chrono::steady_clock;
 
 auto t_start = clock::now();

From 49f0c150e61d518e14b8724fcd010d693ffba734 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 5 Mar 2026 14:06:44 -0600
Subject: [PATCH 0494/1000] Updated test to align with other benchmakrs .

---
 .../benchmark-tests/mmul-actor-benchmarking/main.test.cpp   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
index 0ee4c74afa..07a05b2aaa 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
@@ -235,9 +235,9 @@ auto t_end = clock::now();
 void caf_main(caf::actor_system& sys) {
   caf::cuda::manager::init(sys);
   run_mmul_test(sys,1000);
-  //run_mmul_test(sys,4000);
-  //run_mmul_test(sys,8000);
-  //run_mmul_test(sys,12000);
+  run_mmul_test(sys,4000);
+  run_mmul_test(sys,8000);
+  run_mmul_test(sys,12000);
 
 }
 

From ba350ec6b79cf8aa4b2d13a2c8203b396a8047c2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Mar 2026 11:59:32 -0600
Subject: [PATCH 0495/1000] Initial commit.

---
 .../scheduler-benchmark-tests/CMakeLists.txt  |  44 ++++
 .../compile_kernels.sh                        |  13 +
 .../scheduler-benchmark-tests/main.test.cpp   | 247 ++++++++++++++++++
 .../scheduler-benchmark-tests/mmul.cu         |  16 ++
 4 files changed, 320 insertions(+)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul.cu

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/compile_kernels.sh b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/compile_kernels.sh
new file mode 100755
index 0000000000..f32480e5cb
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/compile_kernels.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
new file mode 100644
index 0000000000..07a05b2aaa
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
@@ -0,0 +1,247 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/component-actors/mmul_actor/mmul_actor.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+using command =
+  caf::cuda::command_runner<>;
+
+command mmul_command;
+
+struct mmul_state {
+};
+
+//global output buffer meant to disclude it from timing
+//the other benchmark test do not include its memory allocations in it 
+//so its only fair that we do not either 
+std::vector<int> matrixC;
+
+
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
+  return {
+
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        int N) {
+
+      using clock = std::chrono::steady_clock;
+      using ms = std::chrono::duration<double, std::milli>;
+
+      auto t_total_start = clock::now();
+
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+      int device = 0;
+      int stream = 1;
+
+      auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+      // -------------------------
+      // create_in_arg A
+      // -------------------------
+      auto t_a_inarg_start = clock::now();
+
+      auto inA = caf::cuda::create_in_arg(std::move(matrixA));
+
+      auto t_a_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer A
+      // -------------------------
+      auto t_a_transfer_start = clock::now();
+
+      auto arg1 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inA));
+
+      auto t_a_transfer_end = clock::now();
+
+      // -------------------------
+      // create_in_arg B
+      // -------------------------
+      auto t_b_inarg_start = clock::now();
+
+      auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+
+      auto t_b_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer B
+      // -------------------------
+      auto t_b_transfer_start = clock::now();
+
+      auto arg2 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inB));
+
+      auto t_b_transfer_end = clock::now();
+
+      // -------------------------
+      // spawn actor
+      // -------------------------
+      auto t_spawn_start = clock::now();
+
+      caf::actor mmul_actor =
+        self->spawn(caf::cuda::mmul_actor_fun<int>, program);
+
+      auto t_spawn_end = clock::now();
+
+      // -------------------------
+      // request
+      // -------------------------
+      auto t_request_start = clock::now();
+
+      self->mail(arg1, arg2, N, device, stream)
+        .request(mmul_actor, std::chrono::seconds(30))
+        .then(
+          [=](caf::cuda::mem_ptr<int> dC) {
+
+            auto t_response_received = clock::now();
+
+            //std::vector<int> matrixC(N*N);
+            // -------------------------
+            // copy to host
+            // -------------------------
+            auto t_copy_start = clock::now();
+
+            //std::vector<int> matrixC = dC->copy_to_host();
+
+	    dC->copy_to_host(matrixC.data(),N*N);
+
+            auto t_copy_end = clock::now();
+            auto t_total_end = clock::now();
+
+            // -------------------------
+            // Print timings
+            // -------------------------
+
+            std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
+
+            std::cout << "create_in_arg A: "
+                      << ms(t_a_inarg_end - t_a_inarg_start).count()
+                      << " ms\n";
+
+            std::cout << "transfer A: "
+                      << ms(t_a_transfer_end - t_a_transfer_start).count()
+                      << " ms\n";
+
+            std::cout << "create_in_arg B: "
+                      << ms(t_b_inarg_end - t_b_inarg_start).count()
+                      << " ms\n";
+
+            std::cout << "transfer B: "
+                      << ms(t_b_transfer_end - t_b_transfer_start).count()
+                      << " ms\n";
+
+            std::cout << "spawn actor: "
+                      << ms(t_spawn_end - t_spawn_start).count()
+                      << " ms\n";
+
+            std::cout << "request → response latency: "
+                      << ms(t_response_received - t_request_start).count()
+                      << " ms\n";
+
+            std::cout << "copy_to_host: "
+                      << ms(t_copy_end - t_copy_start).count()
+                      << " ms\n";
+
+            std::cout << "TOTAL end-to-end: "
+                      << ms(t_total_end - t_total_start).count()
+                      << " ms\n";
+
+            std::cout << "=============================================\n";
+
+            self->quit();
+          }
+        );
+    }
+
+  };
+}
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size) {
+
+  // ------------------------------------
+  // Start timing
+  // ------------------------------------
+  auto start = std::chrono::steady_clock::now();
+
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<int> matrixA(matrix_size * matrix_size,2);
+  std::vector<int> matrixB(matrix_size * matrix_size,3);
+
+  matrixC.resize(matrix_size*matrix_size);
+
+ using clock = std::chrono::steady_clock;
+
+auto t_start = clock::now();
+
+caf::actor a =sys.spawn(mmul_actor_fun);
+
+anon_mail(matrixA,matrixB,matrix_size).send(a);
+
+auto t_end = clock::now();
+
+
+
+  // Wait for all actors to finish
+  sys.await_all_actors_done();
+
+  // ------------------------------------
+  // Stop timing
+  // ------------------------------------
+  auto end = std::chrono::steady_clock::now();
+  auto duration_ms =
+      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
+            << ", time=" << duration_ms << " ms\n";
+}
+
+void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager::init(sys);
+  run_mmul_test(sys,1000);
+  run_mmul_test(sys,4000);
+  run_mmul_test(sys,8000);
+  run_mmul_test(sys,12000);
+
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul.cu b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From 588df3ee05b891c97907da924f04b5d9d9297531 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Mar 2026 12:12:34 -0600
Subject: [PATCH 0496/1000] Implemented a actor with and without scheduler.

---
 .../scheduler-benchmark-tests/main.test.cpp   | 167 +++++++++++++-----
 1 file changed, 119 insertions(+), 48 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
index 07a05b2aaa..d8156bd578 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
@@ -1,6 +1,6 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
-#include <caf/component-actors/mmul_actor/mmul_actor.hpp>
+#include <caf/component-actors/all-component-actors>
 #include <iostream>
 #include <iomanip>
 #include <vector>
@@ -41,6 +41,9 @@ using command =
 command mmul_command;
 
 struct mmul_state {
+
+	caf::cuda::program_ptr mmul_kernel;
+
 };
 
 //global output buffer meant to disclude it from timing
@@ -49,9 +52,11 @@ struct mmul_state {
 std::vector<int> matrixC;
 
 
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::program_ptr mmul_kernel) {
   return {
 
+	  self->state().mmul_kernel = mmul_kernel;
+
     [=](const std::vector<int>& matrixA,
         const std::vector<int>& matrixB,
         int N) {
@@ -65,8 +70,10 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
       int device = 0;
       int stream = 1;
 
-      auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+      //auto program =
+        //mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+      auto program = self->state().mmul_kernel
 
       // -------------------------
       // create_in_arg A
@@ -123,76 +130,140 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
       // -------------------------
       // request
       // -------------------------
-      auto t_request_start = clock::now();
 
       self->mail(arg1, arg2, N, device, stream)
         .request(mmul_actor, std::chrono::seconds(30))
         .then(
           [=](caf::cuda::mem_ptr<int> dC) {
 
-            auto t_response_received = clock::now();
 
-            //std::vector<int> matrixC(N*N);
-            // -------------------------
-            // copy to host
-            // -------------------------
-            auto t_copy_start = clock::now();
+           auto t_copy_start = clock::now();
 
             //std::vector<int> matrixC = dC->copy_to_host();
-
 	    dC->copy_to_host(matrixC.data(),N*N);
 
-            auto t_copy_end = clock::now();
-            auto t_total_end = clock::now();
+	    caf::actor 
 
-            // -------------------------
-            // Print timings
-            // -------------------------
+                        self->quit();
+          }
+        );
+    }
 
-            std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
+  };
+}
 
-            std::cout << "create_in_arg A: "
-                      << ms(t_a_inarg_end - t_a_inarg_start).count()
-                      << " ms\n";
 
-            std::cout << "transfer A: "
-                      << ms(t_a_transfer_end - t_a_transfer_start).count()
-                      << " ms\n";
 
-            std::cout << "create_in_arg B: "
-                      << ms(t_b_inarg_end - t_b_inarg_start).count()
-                      << " ms\n";
 
-            std::cout << "transfer B: "
-                      << ms(t_b_transfer_end - t_b_transfer_start).count()
-                      << " ms\n";
+struct mmul_actor_with_scheduler_state {
+  static inline const char* name = "my_actor";
+  caf::actor sync_actor;
+  caf::actor mem_transfer_actor;
+};
 
-            std::cout << "spawn actor: "
-                      << ms(t_spawn_end - t_spawn_start).count()
-                      << " ms\n";
 
-            std::cout << "request → response latency: "
-                      << ms(t_response_received - t_request_start).count()
-                      << " ms\n";
+// Stateful actor behavior
+caf::behavior mmul_actor_fun_scheduler(
+    caf::stateful_actor<mmul_actor_with_scheduler_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims)
+{
 
-            std::cout << "copy_to_host: "
-                      << ms(t_copy_end - t_copy_start).count()
-                      << " ms\n";
 
-            std::cout << "TOTAL end-to-end: "
-                      << ms(t_total_end - t_total_start).count()
-                      << " ms\n";
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-            std::cout << "=============================================\n";
+        caf::actor scheduler = mgr.get_scheduler_actor();
 
-            self->quit();
-          }
-        );
-    }
+        //send a launch token
+        caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+                        program,
+                        dims,
+                        0,
+                        "hello",
+                        self
+                        );
+        self -> mail(launch_token).send(scheduler);
+
+        self->state().sync_actor = self -> spawn(caf::cuda::sync_actor_fun<int>);
+        self->state().mem_transfer_actor = self -> spawn(caf::cuda::mem_transfer_actor_fun<int>);
+
+       return {
+
+    // 1. Handle response token
+    [=](caf::cuda::response_token_ptr res_token) {
+        std::cout << "Got response\n";
+
+        if (res_token->getType() == LAUNCH_RESPONSE) {
+            std::vector<int> matrix1(N*N);
+            std::vector<int> matrix2(N*N);
+
+            self->mail(matrix1, matrix2, res_token, N).send(self);
+
+        } else {
+            std::cout << "Got a memory response token\n";
+        }
+    },
+
+    // 2. Handle memory buffers -> GPU
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        const caf::cuda::response_token_ptr& res_token,
+        int N) {
+
+        std::cout << "Working\n";
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        const int THREADS = 32;
+        const int BLOCKS = (N + THREADS - 1) / THREADS;
+        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+        auto arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixA));
+        auto arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixB));
+        auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
+        auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
+
+        auto tempC = mmulAsync.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
+        caf::cuda::mem_ptr<int> bufferA = std::get<0>(tempC);
+
+        self->state().r = res_token; // hold token
+
+        using namespace std::chrono_literals;
+        self->mail(bufferA, res_token)
+            .request(self->state().sync_actor, 10s)
+            .then(
+                [=](caf::cuda::mem_ptr<int> /*syncedA*/) {
+                    auto bufferC = std::get<2>(tempC);
+                    self->mail(bufferC)
+                        .request(self->state().mem_transfer_actor, 10s)
+                        .then(
+                            [=](std::vector<int> matrixC) {
+                                //self->mail(matrixA,matrixB,matrixC, N).send(self);
+                            
+			     self->mail(1).send(exit_actor);
+        		     self->quit();
+			    
+			    },
+                            [=](caf::error& err) {
+                                std::cout << "Transfer C failed: " << to_string(err) << "\n";
+                                self->quit(err);
+                            });
+                },
+                [=](caf::error& err) {
+                    std::cout << "Sync failed: " << to_string(err) << "\n";
+                    self->quit(err);
+                });
+    },
+       };
 
-  };
 }
 
+
+
+
+
 void run_mmul_test(caf::actor_system& sys, int matrix_size) {
 
   // ------------------------------------

From ead678568f9ac6120c455f1a249865ec3f0573a8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Mar 2026 12:16:43 -0600
Subject: [PATCH 0497/1000] Began implementing benchmark tests.

---
 .../scheduler-benchmark-tests/main.test.cpp   | 224 ++++++++++++++++--
 1 file changed, 202 insertions(+), 22 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
index d8156bd578..f62426b52c 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
@@ -264,45 +264,225 @@ caf::behavior mmul_actor_fun_scheduler(
 
 
 
-void run_mmul_test(caf::actor_system& sys, int matrix_size) {
 
-  // ------------------------------------
-  // Start timing
-  // ------------------------------------
-  auto start = std::chrono::steady_clock::now();
 
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<int> matrixA(matrix_size * matrix_size,2);
-  std::vector<int> matrixB(matrix_size * matrix_size,3);
 
-  matrixC.resize(matrix_size*matrix_size);
 
- using clock = std::chrono::steady_clock;
 
-auto t_start = clock::now();
 
-caf::actor a =sys.spawn(mmul_actor_fun);
 
-anon_mail(matrixA,matrixB,matrix_size).send(a);
+void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //change the scheduler to mulitlevle_usage
+  anon_mail(
+        caf::cuda::make_behavior_token("multilevel")
+        ).send(mgr.get_scheduler_actor());
+
+  // CREATE ONCE
+  auto program =
+      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+  const int THREADS = 32;
+  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1);
+
+  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+  for (int i = 0; i < num_actors; ++i) {
+    
+      sys.spawn(
+        mmul_actor_fun_scheduler,
+        exit_actor,
+        matrix_size,
+        program,
+        dims);
+    
+  }
+
+  sys.await_all_actors_done();
+}
+
+
+void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  /*
+  //change the scheduler to core_usage
+  anon_mail(
+        caf::cuda::make_behavior_token("core_usage")
+        ).send(mgr.get_scheduler_actor());
+
+  */
+
+    mgr.send_scheduler_actor_message("green",0);
+
+  // CREATE ONCE
+  auto program =
+      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
-auto t_end = clock::now();
+  const int THREADS = 32;
+  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1);
 
+  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
 
+  for (int i = 0; i < num_actors; ++i) {
+    
+      sys.spawn(
+        mmul_actor_fun_scheduler,
+        exit_actor,
+        matrix_size,
+        program,
+        dims);
+    
+  }
+
+ 
+  }
 
-  // Wait for all actors to finish
   sys.await_all_actors_done();
+}
+
+void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, int num_actors) {
+    if (num_actors < 1) {
+        std::cerr << "[ERROR] Number of actors must be >= 1\n";
+        return;
+    }
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    const int THREADS = 32;
+    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+
+    for (int i = 0; i < num_actors; ++i) {
+        sys.spawn(
+            mmul_actor_fun_no_schedule,
+            exit_actor,
+            matrix_size,
+            program,
+            dims
+        );
+    }
+
+    sys.await_all_actors_done();
+}
 
-  // ------------------------------------
-  // Stop timing
-  // ------------------------------------
+
+
+
+template <class Fn>
+double time_run(Fn&& fn) {
+  auto start = std::chrono::steady_clock::now();
+  fn();
   auto end = std::chrono::steady_clock::now();
-  auto duration_ms =
-      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+  std::chrono::duration<double> elapsed = end - start;
+  return elapsed.count();
+}
+void run_mmul_scaling_tests(caf::actor_system& sys,
+                            caf::cuda::manager_config man_config) {
+    const int max_size   = 2048;
+    const int min_actors = 1;
+    const int max_actors = 1024;
+
+    std::vector<int> matrix_sizes = {10};
+    for (int s = 32; s <= max_size; s *= 2)
+        matrix_sizes.push_back(s);
+
+    std::vector<int> actor_counts;
+    for (int a = min_actors; a <= max_actors; a *= 2)
+        actor_counts.push_back(a);
+
+    std::cout << "=== MMUL Scaling Tests ===\n";
+    std::cout << "Format:\n";
+    std::cout << "scheduler matrix_size actors time_seconds\n";
+
+    for (int size : matrix_sizes) {
+        for (int actors : actor_counts) {
+
+            /* ================= Scheduler-enabled (core_usage) ================= */
+            caf::cuda::manager::init(sys, man_config); // scheduler enabled
+            std::cout << "\n[RUN] scheduler=multilevel_usage "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double core_usage_time = time_run([&] {
+                run_mmul_test(sys, size, actors); // uses mmul_actor_fun_no_verify
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT core_usage "
+                      << size << " "
+                      << actors << " "
+                      << core_usage_time << "\n";
+
+            caf::cuda::manager::shutdown(); // make sure manager is cleaned up
+
+            /* ================= Scheduler-disabled actor ( uses green-light) ================= */
+
+            caf::cuda::manager::init(sys, man_config); // init with scheduler
+            std::cout << "\n[RUN] scheduler=green_light_only "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double green_light_time = time_run([&] {
+                run_mmul_test_no_scheduler(sys, size, actors); // your previous "no scheduler" actor
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT green_light_only "
+                      << size << " "
+                      << actors << " "
+                      << green_light_time << "\n";
+
+            caf::cuda::manager::shutdown();
+
+            /* ================= No scheduler at all actor ================= */
+            caf::cuda::manager_config no_sched_config(false); // disable scheduler
+            caf::cuda::manager::init(sys, no_sched_config);
+            std::cout << "\n[RUN] scheduler=none "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double no_scheduler_time = time_run([&] {
+                run_mmul_test_no_scheduler_actor(sys, size, actors); // mmul_actor_fun_no_schedule
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT none "
+                      << size << " "
+                      << actors << " "
+                      << no_scheduler_time << "\n";
+
+            caf::cuda::manager::shutdown();
+        }
+    }
 
-  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
-            << ", time=" << duration_ms << " ms\n";
+    std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
 }
 
+
+
+
 void caf_main(caf::actor_system& sys) {
   caf::cuda::manager::init(sys);
   run_mmul_test(sys,1000);

From 5827b6beec0101eef3a43ef5ca6723d621b7bdbf Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Mar 2026 14:20:50 -0600
Subject: [PATCH 0498/1000] Wrote initial version for actor using scheduler
 with no blocking and callback actors.

---
 .../scheduler-benchmark-tests/main.test.cpp   | 69 ++++++++-----------
 1 file changed, 27 insertions(+), 42 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
index f62426b52c..388db50c2c 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
@@ -46,10 +46,6 @@ struct mmul_state {
 
 };
 
-//global output buffer meant to disclude it from timing
-//the other benchmark test do not include its memory allocations in it 
-//so its only fair that we do not either 
-std::vector<int> matrixC;
 
 
 caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::program_ptr mmul_kernel) {
@@ -61,11 +57,6 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
         const std::vector<int>& matrixB,
         int N) {
 
-      using clock = std::chrono::steady_clock;
-      using ms = std::chrono::duration<double, std::milli>;
-
-      auto t_total_start = clock::now();
-
       caf::cuda::manager& mgr = caf::cuda::manager::get();
       int device = 0;
       int stream = 1;
@@ -94,20 +85,16 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
           stream,
           std::move(inA));
 
-      auto t_a_transfer_end = clock::now();
-
-      // -------------------------
-      // create_in_arg B
-      // -------------------------
+     
       auto t_b_inarg_start = clock::now();
 
+
+
       auto inB = caf::cuda::create_in_arg(std::move(matrixB));
 
       auto t_b_inarg_end = clock::now();
 
-      // -------------------------
-      // transfer B
-      // -------------------------
+     
       auto t_b_transfer_start = clock::now();
 
       auto arg2 = mmul_command.transfer_memory(
@@ -115,36 +102,22 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
           stream,
           std::move(inB));
 
-      auto t_b_transfer_end = clock::now();
 
-      // -------------------------
-      // spawn actor
-      // -------------------------
-      auto t_spawn_start = clock::now();
 
       caf::actor mmul_actor =
         self->spawn(caf::cuda::mmul_actor_fun<int>, program);
 
-      auto t_spawn_end = clock::now();
 
-      // -------------------------
-      // request
-      // -------------------------
 
       self->mail(arg1, arg2, N, device, stream)
         .request(mmul_actor, std::chrono::seconds(30))
         .then(
           [=](caf::cuda::mem_ptr<int> dC) {
 
-
-           auto t_copy_start = clock::now();
-
             //std::vector<int> matrixC = dC->copy_to_host();
 	    dC->copy_to_host(matrixC.data(),N*N);
+            self->quit();
 
-	    caf::actor 
-
-                        self->quit();
           }
         );
     }
@@ -159,6 +132,7 @@ struct mmul_actor_with_scheduler_state {
   static inline const char* name = "my_actor";
   caf::actor sync_actor;
   caf::actor mem_transfer_actor;
+  caf::actor mmul_actor;
 };
 
 
@@ -188,6 +162,7 @@ caf::behavior mmul_actor_fun_scheduler(
 
         self->state().sync_actor = self -> spawn(caf::cuda::sync_actor_fun<int>);
         self->state().mem_transfer_actor = self -> spawn(caf::cuda::mem_transfer_actor_fun<int>);
+	self->state().mmul_actor = self -> spawn(caf::cuda::mmul_actor_fun<int>,program);
 
        return {
 
@@ -215,8 +190,9 @@ caf::behavior mmul_actor_fun_scheduler(
         std::cout << "Working\n";
         caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-        const int THREADS = 32;
+       
+	/*
+	const int THREADS = 32;
         const int BLOCKS = (N + THREADS - 1) / THREADS;
         caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
@@ -225,19 +201,26 @@ caf::behavior mmul_actor_fun_scheduler(
         auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
         auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
 
+
         auto tempC = mmulAsync.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
         caf::cuda::mem_ptr<int> bufferA = std::get<0>(tempC);
+	*/
+
+	caf::cuda::mem_ptr<int> arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixA));
+	caf::cuda::mem_ptr<int> arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixB));
 
-        self->state().r = res_token; // hold token
+	self->mail(arg1,arg2,N,res_token -> getDeviceNumber(), res_token -> getStreamId())
+		.request(self->state().mmul_actor,1000s)
+		.then(
+		 [=](caf::cuda::mem_ptr<int> dC) {
 
-        using namespace std::chrono_literals;
-        self->mail(bufferA, res_token)
-            .request(self->state().sync_actor, 10s)
+
+        self->mail(dC, res_token)
+            .request(self->state().sync_actor, 1000s)
             .then(
-                [=](caf::cuda::mem_ptr<int> /*syncedA*/) {
-                    auto bufferC = std::get<2>(tempC);
-                    self->mail(bufferC)
-                        .request(self->state().mem_transfer_actor, 10s)
+                [=](caf::cuda::mem_ptr<int> /*syncedC*/) {
+                    self->mail(dC)
+                        .request(self->state().mem_transfer_actor, 1000s)
                         .then(
                             [=](std::vector<int> matrixC) {
                                 //self->mail(matrixA,matrixB,matrixC, N).send(self);
@@ -255,6 +238,7 @@ caf::behavior mmul_actor_fun_scheduler(
                     std::cout << "Sync failed: " << to_string(err) << "\n";
                     self->quit(err);
                 });
+		 });
     },
        };
 
@@ -364,6 +348,7 @@ void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, i
         return;
     }
 
+    
     caf::cuda::manager& mgr = caf::cuda::manager::get();
 
     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");

From 663ee65fc5024ccab6fd38a0e9691103cb1bc53b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Mar 2026 14:25:09 -0600
Subject: [PATCH 0499/1000] Implemented actor that does not use any scheduler.

---
 .../scheduler-benchmark-tests/main.test.cpp       | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
index 388db50c2c..5056d0479d 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
@@ -48,11 +48,14 @@ struct mmul_state {
 
 
 
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::program_ptr mmul_kernel) {
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::program_ptr mmul_kernel,int N) {
   return {
 
 	  self->state().mmul_kernel = mmul_kernel;
-
+	    std::vector<int> matrix1(N*N);
+            std::vector<int> matrix2(N*N);
+            self->mail(matrix1, matrix2, N).send(self);
+	    
     [=](const std::vector<int>& matrixA,
         const std::vector<int>& matrixB,
         int N) {
@@ -107,6 +110,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
       caf::actor mmul_actor =
         self->spawn(caf::cuda::mmul_actor_fun<int>, program);
 
+      caf::actor mem_transfer_actor = self->spawn(caf::cuda::mem_transfer_actor_fun<int>);
 
 
       self->mail(arg1, arg2, N, device, stream)
@@ -114,12 +118,13 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
         .then(
           [=](caf::cuda::mem_ptr<int> dC) {
 
+	  self->mail(dC).request(mem_transfer_actor,std::chrono::seconds(4000))
+	 .then([=] (std::vector<int>& matrixC) {
             //std::vector<int> matrixC = dC->copy_to_host();
-	    dC->copy_to_host(matrixC.data(),N*N);
             self->quit();
 
-          }
-        );
+          });
+  	});
     }
 
   };

From 064f580523332a2a7706df25bacbb8119437647d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Mar 2026 14:40:22 -0600
Subject: [PATCH 0500/1000] Fixed compiler errors.

---
 .../scheduler-benchmark-tests/main.test.cpp   | 125 ++++++++----------
 1 file changed, 54 insertions(+), 71 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
index 5056d0479d..e00cea8740 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
@@ -1,6 +1,6 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
-#include <caf/component-actors/all-component-actors>
+#include <caf/component-actors/all-component-actors.hpp>
 #include <iostream>
 #include <iomanip>
 #include <vector>
@@ -39,6 +39,7 @@ using command =
   caf::cuda::command_runner<>;
 
 command mmul_command;
+command mmul;
 
 struct mmul_state {
 
@@ -49,85 +50,70 @@ struct mmul_state {
 
 
 caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::program_ptr mmul_kernel,int N) {
-  return {
 
-	  self->state().mmul_kernel = mmul_kernel;
-	    std::vector<int> matrix1(N*N);
-            std::vector<int> matrix2(N*N);
-            self->mail(matrix1, matrix2, N).send(self);
-	    
-    [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB,
-        int N) {
-
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
-      int device = 0;
-      int stream = 1;
-
-      //auto program =
-        //mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+	self->state().mmul_kernel = mmul_kernel;
+	std::vector<int> matrix1(N*N);
+	std::vector<int> matrix2(N*N);
+	self->mail(matrix1, matrix2, N).send(self);
+	return {
+		[=](const std::vector<int>& matrixA,
+				const std::vector<int>& matrixB,
+				int N) {
 
-      auto program = self->state().mmul_kernel
+			caf::cuda::manager& mgr = caf::cuda::manager::get();
+			int device = 0;
+			int stream = 1;
 
-      // -------------------------
-      // create_in_arg A
-      // -------------------------
-      auto t_a_inarg_start = clock::now();
+			//auto program =
+			//mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
-      auto inA = caf::cuda::create_in_arg(std::move(matrixA));
+			auto program = self->state().mmul_kernel;
 
-      auto t_a_inarg_end = clock::now();
+				// -------------------------
+				// create_in_arg A
+				// -------------------------
 
-      // -------------------------
-      // transfer A
-      // -------------------------
-      auto t_a_transfer_start = clock::now();
+			auto inA = caf::cuda::create_in_arg(std::move(matrixA));
 
-      auto arg1 = mmul_command.transfer_memory(
-          device,
-          stream,
-          std::move(inA));
 
-     
-      auto t_b_inarg_start = clock::now();
+			auto arg1 = mmul_command.transfer_memory(
+					device,
+					stream,
+					std::move(inA));
 
 
 
-      auto inB = caf::cuda::create_in_arg(std::move(matrixB));
 
-      auto t_b_inarg_end = clock::now();
 
-     
-      auto t_b_transfer_start = clock::now();
+			auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+			auto arg2 = mmul_command.transfer_memory(
+					device,
+					stream,
+					std::move(inB));
 
-      auto arg2 = mmul_command.transfer_memory(
-          device,
-          stream,
-          std::move(inB));
 
 
+			caf::actor mmul_actor =
+				self->spawn(caf::cuda::mmul_actor_fun<int>, program);
 
-      caf::actor mmul_actor =
-        self->spawn(caf::cuda::mmul_actor_fun<int>, program);
+			caf::actor mem_transfer_actor = self->spawn(caf::cuda::mem_transfer_actor_fun<int>);
 
-      caf::actor mem_transfer_actor = self->spawn(caf::cuda::mem_transfer_actor_fun<int>);
 
+			self->mail(arg1, arg2, N, device, stream)
+				.request(mmul_actor, std::chrono::seconds(30))
+				.then(
+						[=](caf::cuda::mem_ptr<int> dC) {
 
-      self->mail(arg1, arg2, N, device, stream)
-        .request(mmul_actor, std::chrono::seconds(30))
-        .then(
-          [=](caf::cuda::mem_ptr<int> dC) {
+						self->mail(dC).request(mem_transfer_actor,std::chrono::seconds(4000))
+						.then([=] (std::vector<int>& matrixC) {
+								//std::vector<int> matrixC = dC->copy_to_host();
+								self->quit();
 
-	  self->mail(dC).request(mem_transfer_actor,std::chrono::seconds(4000))
-	 .then([=] (std::vector<int>& matrixC) {
-            //std::vector<int> matrixC = dC->copy_to_host();
-            self->quit();
+								});
+						});
+		}
 
-          });
-  	});
-    }
-
-  };
+	};
 }
 
 
@@ -283,7 +269,7 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
       BLOCKS, BLOCKS, 1,
       THREADS, THREADS, 1);
 
-  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+  caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
 
   for (int i = 0; i < num_actors; ++i) {
     
@@ -328,7 +314,7 @@ void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num
       BLOCKS, BLOCKS, 1,
       THREADS, THREADS, 1);
 
-  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+  caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
 
   for (int i = 0; i < num_actors; ++i) {
     
@@ -342,7 +328,6 @@ void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num
   }
 
  
-  }
 
   sys.await_all_actors_done();
 }
@@ -362,15 +347,13 @@ void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, i
     const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
     caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
+  caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
 
     for (int i = 0; i < num_actors; ++i) {
         sys.spawn(
-            mmul_actor_fun_no_schedule,
-            exit_actor,
-            matrix_size,
+            mmul_actor_fun,
             program,
-            dims
+            matrix_size
         );
     }
 
@@ -474,11 +457,11 @@ void run_mmul_scaling_tests(caf::actor_system& sys,
 
 
 void caf_main(caf::actor_system& sys) {
-  caf::cuda::manager::init(sys);
-  run_mmul_test(sys,1000);
-  run_mmul_test(sys,4000);
-  run_mmul_test(sys,8000);
-  run_mmul_test(sys,12000);
+  
+
+  caf::cuda::manager_config man_config(true);
+  caf::cuda::manager::init(sys,man_config);
+  run_mmul_scaling_tests(sys,man_config);
 
 }
 

From 1977cedb1167e4f9789cd0e4ba53a06b8a493672 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Mar 2026 14:47:31 -0600
Subject: [PATCH 0501/1000] Removed exit actor from tests with no scheduler
 actor since shutdown cordination is not required.

---
 .../scheduler-benchmark-tests/main.test.cpp        | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
index e00cea8740..2fa77efd02 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/main.test.cpp
@@ -62,7 +62,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
 
 			caf::cuda::manager& mgr = caf::cuda::manager::get();
 			int device = 0;
-			int stream = 1;
+			int stream = rand();
 
 			//auto program =
 			//mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
@@ -101,8 +101,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
 
 			self->mail(arg1, arg2, N, device, stream)
 				.request(mmul_actor, std::chrono::seconds(30))
-				.then(
-						[=](caf::cuda::mem_ptr<int> dC) {
+				.then([=](caf::cuda::mem_ptr<int> dC) {
 
 						self->mail(dC).request(mem_transfer_actor,std::chrono::seconds(4000))
 						.then([=] (std::vector<int>& matrixC) {
@@ -159,7 +158,7 @@ caf::behavior mmul_actor_fun_scheduler(
 
     // 1. Handle response token
     [=](caf::cuda::response_token_ptr res_token) {
-        std::cout << "Got response\n";
+//        std::cout << "Got response\n";
 
         if (res_token->getType() == LAUNCH_RESPONSE) {
             std::vector<int> matrix1(N*N);
@@ -168,7 +167,7 @@ caf::behavior mmul_actor_fun_scheduler(
             self->mail(matrix1, matrix2, res_token, N).send(self);
 
         } else {
-            std::cout << "Got a memory response token\n";
+  //          std::cout << "Got a memory response token\n";
         }
     },
 
@@ -178,7 +177,7 @@ caf::behavior mmul_actor_fun_scheduler(
         const caf::cuda::response_token_ptr& res_token,
         int N) {
 
-        std::cout << "Working\n";
+    //    std::cout << "Working\n";
         caf::cuda::manager& mgr = caf::cuda::manager::get();
 
        
@@ -347,7 +346,6 @@ void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, i
     const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
     caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-  caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
 
     for (int i = 0; i < num_actors; ++i) {
         sys.spawn(
@@ -460,7 +458,7 @@ void caf_main(caf::actor_system& sys) {
   
 
   caf::cuda::manager_config man_config(true);
-  caf::cuda::manager::init(sys,man_config);
+  //caf::cuda::manager::init(sys,man_config);
   run_mmul_scaling_tests(sys,man_config);
 
 }

From a27f3dc3a5dda1b501ef4b5d903d85984361e7b0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Mar 2026 15:40:52 -0600
Subject: [PATCH 0502/1000] Made memory transfers async, so that fair
 comparision between the non actor and actor approach can be made since actor
 approach using async memory transfers.

---
 .../matrix_mul_driver.cpp                     | 167 ++++++++++--------
 1 file changed, 94 insertions(+), 73 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
index cf392a139d..1c4f5361cd 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
@@ -27,102 +27,123 @@ std::string readFile(const std::string &path) {
 }
 
 void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
-    std::cout << "Running N = " << N << " ...\n";
-    size_t elements = static_cast<size_t>(N) * static_cast<size_t>(N);
+    using clock = std::chrono::steady_clock;
+    using ms = std::chrono::duration<double, std::milli>;
+
+    std::cout << "\n===== DRIVER BENCHMARK (N=" << N << ") =====\n";
+
+    size_t elements = static_cast<size_t>(N) * N;
     size_t bytes = elements * sizeof(int);
 
-    // Check for overflow (extremely large N)
-    if (elements / N != static_cast<size_t>(N)) {
-        std::cerr << "Integer overflow for N = " << N << "\n";
-        return;
-    }
+    std::vector<int> h_a(elements, 1);
+    std::vector<int> h_b(elements, 1);
+    std::vector<int> h_c(elements);
 
-    // Host buffers
-    std::vector<int> h_a, h_b, h_c;
-    try {
-        h_a.resize(elements);
-        h_b.resize(elements);
-        h_c.resize(elements);
-    } catch (const std::bad_alloc&) {
-        std::cerr << "Host allocation failed for N = " << N << " (need "
-                  << bytes << " bytes per matrix)\n";
-        return;
-    }
+    CUdeviceptr d_a, d_b, d_c;
 
-    // Fill test data (simple pattern)
-    for (size_t i = 0; i < elements; ++i) {
-        h_a[i] = 1; // simple values to make it predictable
-        h_b[i] = 1;
-    }
+    auto t_total_start = clock::now();
 
-    CUdeviceptr d_a = 0, d_b = 0, d_c = 0;
-    CUresult r;
+    // ----------------------------------
+    // Device Allocation
+    // ----------------------------------
+    auto t_alloc_start = clock::now();
 
-    // Try allocating device memory (may fail on small GPUs for large N).
-    r = cuMemAlloc(&d_a, bytes);
-    if (r != CUDA_SUCCESS) {
-        std::cerr << "cuMemAlloc d_a failed for N=" << N << " ("
-                  << bytes << " bytes). Skipping.\n";
-        return;
-    }
+    checkCU(cuMemAlloc(&d_a, bytes), "cuMemAlloc d_a");
     checkCU(cuMemAlloc(&d_b, bytes), "cuMemAlloc d_b");
     checkCU(cuMemAlloc(&d_c, bytes), "cuMemAlloc d_c");
 
-    // Grid / block
+    auto t_alloc_end = clock::now();
+
+    // ----------------------------------
+    // H2D copy A
+    // ----------------------------------
+    auto t_h2d_a_start = clock::now();
+    checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), bytes), "cuMemcpyHtoD A");
+    auto t_h2d_a_end = clock::now();
+
+    // ----------------------------------
+    // H2D copy B
+    // ----------------------------------
+    auto t_h2d_b_start = clock::now();
+    checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), bytes), "cuMemcpyHtoD B");
+    auto t_h2d_b_end = clock::now();
+
+    // ----------------------------------
+    // Kernel launch + execution
+    // ----------------------------------
     const unsigned int blockX = 16;
     const unsigned int blockY = 16;
-    unsigned int gridX = static_cast<unsigned int>((N + blockX - 1) / blockX);
-    unsigned int gridY = static_cast<unsigned int>((N + blockY - 1) / blockY);
+    unsigned int gridX = (N + blockX - 1) / blockX;
+    unsigned int gridY = (N + blockY - 1) / blockY;
 
-    // Prepare kernel parameter values as described by Driver API:
-    // pointers to the argument values
-    // Note: pass address-of CUdeviceptr, and address-of N (int).
     void* kernelParams[] = { &d_a, &d_b, &d_c, &N };
 
-    // Time everything from H->D copy to D->H copy and sync
-    auto t0 = std::chrono::steady_clock::now();
-
-    checkCU(cuMemcpyHtoD(d_a, h_a.data(), bytes), "cuMemcpyHtoD d_a");
-    checkCU(cuMemcpyHtoD(d_b, h_b.data(), bytes), "cuMemcpyHtoD d_b");
+    auto t_kernel_start = clock::now();
 
-    // Launch kernel
     checkCU(cuLaunchKernel(kernel,
-                           gridX, gridY, 1,         // grid
-                           blockX, blockY, 1,       // block
-                           0,                       // shared mem
-                           nullptr,                 // stream
-                           kernelParams,            // kernel params
-                           nullptr), "cuLaunchKernel");
-
-    // Wait for completion
+                           gridX, gridY, 1,
+                           blockX, blockY, 1,
+                           0,
+                           nullptr,
+                           kernelParams,
+                           nullptr),
+            "cuLaunchKernel");
+
     checkCU(cuCtxSynchronize(), "cuCtxSynchronize");
 
-    // Copy result back
-    checkCU(cuMemcpyDtoH(h_c.data(), d_c, bytes), "cuMemcpyDtoH d_c");
+    auto t_kernel_end = clock::now();
 
-    auto t1 = std::chrono::steady_clock::now();
-   auto dur = std::chrono::duration<double, std::milli>(t1 - t0);
+    // ----------------------------------
+    // D2H copy
+    // ----------------------------------
+    auto t_d2h_start = clock::now();
+    checkCU(cuMemcpyDtoH(h_c.data(), d_c, bytes), "cuMemcpyDtoH");
+    auto t_d2h_end = clock::now();
 
-std::cout << "N=" << N
-          << " time (alloc+H2D+kernel+DtoH): "
-          << dur.count() << " ms\n"; 
+    // ----------------------------------
+    // Free device memory
+    // ----------------------------------
+    auto t_free_start = clock::now();
+    checkCU(cuMemFree(d_a), "cuMemFree A");
+    checkCU(cuMemFree(d_b), "cuMemFree B");
+    checkCU(cuMemFree(d_c), "cuMemFree C");
+    auto t_free_end = clock::now();
 
+    auto t_total_end = clock::now();
 
-    // Quick spot-check for correctness on a few entries (since we used all-ones, result should be N)
-    bool ok = true;
-    if (elements > 0) {
-        // sample first, middle, last
-        std::vector<size_t> samples = {0, elements / 2, elements - 1};
-        for (size_t s : samples) {
-            if (h_c[s] != N) { ok = false; break; }
-        }
-    }
-    std::cout << "Spot-check: " << (ok ? "PASS" : "FAIL (sample mismatch)") << "\n";
+    // ----------------------------------
+    // Print Results
+    // ----------------------------------
 
-    // Free device memory
-    checkCU(cuMemFree(d_a), "cuMemFree d_a");
-    checkCU(cuMemFree(d_b), "cuMemFree d_b");
-    checkCU(cuMemFree(d_c), "cuMemFree d_c");
+    std::cout << "Device allocation: "
+              << ms(t_alloc_end - t_alloc_start).count()
+              << " ms\n";
+
+    std::cout << "H2D copy A: "
+              << ms(t_h2d_a_end - t_h2d_a_start).count()
+              << " ms\n";
+
+    std::cout << "H2D copy B: "
+              << ms(t_h2d_b_end - t_h2d_b_start).count()
+              << " ms\n";
+
+    std::cout << "Kernel execution: "
+              << ms(t_kernel_end - t_kernel_start).count()
+              << " ms\n";
+
+    std::cout << "D2H copy: "
+              << ms(t_d2h_end - t_d2h_start).count()
+              << " ms\n";
+
+    std::cout << "Device free: "
+              << ms(t_free_end - t_free_start).count()
+              << " ms\n";
+
+    std::cout << "TOTAL: "
+              << ms(t_total_end - t_total_start).count()
+              << " ms\n";
+
+    std::cout << "=============================================\n";
 }
 
 int main(int argc, char** argv) {

From bbc2cda767578f5b889d377ffcd3001efd3b195e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Mar 2026 15:49:37 -0600
Subject: [PATCH 0503/1000] Made memory transfers async, so that fair
 comparision between the non actor and actor approach can be made since actor
 approach using async memory transfers.

---
 .../matrix_mul_driver.cpp                     | 42 +++++++++++++------
 1 file changed, 29 insertions(+), 13 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
index 1c4f5361cd..d67cacdce3 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
@@ -32,7 +32,7 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
 
     std::cout << "\n===== DRIVER BENCHMARK (N=" << N << ") =====\n";
 
-    size_t elements = static_cast<size_t>(N) * N;
+    size_t elements = (size_t)N * (size_t)N;
     size_t bytes = elements * sizeof(int);
 
     std::vector<int> h_a(elements, 1);
@@ -40,9 +40,15 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     std::vector<int> h_c(elements);
 
     CUdeviceptr d_a, d_b, d_c;
+    CUstream stream;
 
     auto t_total_start = clock::now();
 
+    // ----------------------------------
+    // Create Stream
+    // ----------------------------------
+    checkCU(cuStreamCreate(&stream, CU_STREAM_DEFAULT), "cuStreamCreate");
+
     // ----------------------------------
     // Device Allocation
     // ----------------------------------
@@ -58,14 +64,20 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     // H2D copy A
     // ----------------------------------
     auto t_h2d_a_start = clock::now();
-    checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), bytes), "cuMemcpyHtoD A");
+
+    checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), bytes, stream), "cuMemcpyHtoDAsync A");
+    checkCU(cuStreamSynchronize(stream), "sync A");
+
     auto t_h2d_a_end = clock::now();
 
     // ----------------------------------
     // H2D copy B
     // ----------------------------------
     auto t_h2d_b_start = clock::now();
-    checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), bytes), "cuMemcpyHtoD B");
+
+    checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), bytes, stream), "cuMemcpyHtoDAsync B");
+    checkCU(cuStreamSynchronize(stream), "sync B");
+
     auto t_h2d_b_end = clock::now();
 
     // ----------------------------------
@@ -84,12 +96,12 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
                            gridX, gridY, 1,
                            blockX, blockY, 1,
                            0,
-                           nullptr,
+                           stream,
                            kernelParams,
                            nullptr),
             "cuLaunchKernel");
 
-    checkCU(cuCtxSynchronize(), "cuCtxSynchronize");
+    checkCU(cuStreamSynchronize(stream), "kernel sync");
 
     auto t_kernel_end = clock::now();
 
@@ -97,18 +109,27 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     // D2H copy
     // ----------------------------------
     auto t_d2h_start = clock::now();
+
     checkCU(cuMemcpyDtoH(h_c.data(), d_c, bytes), "cuMemcpyDtoH");
+
     auto t_d2h_end = clock::now();
 
     // ----------------------------------
     // Free device memory
     // ----------------------------------
     auto t_free_start = clock::now();
+
     checkCU(cuMemFree(d_a), "cuMemFree A");
     checkCU(cuMemFree(d_b), "cuMemFree B");
     checkCU(cuMemFree(d_c), "cuMemFree C");
+
     auto t_free_end = clock::now();
 
+    // ----------------------------------
+    // Destroy stream
+    // ----------------------------------
+    checkCU(cuStreamDestroy(stream), "cuStreamDestroy");
+
     auto t_total_end = clock::now();
 
     // ----------------------------------
@@ -147,25 +168,23 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
 }
 
 int main(int argc, char** argv) {
-    // optional: accept single N from command line
     std::vector<int> sizes = {1000, 4000, 8000, 12000};
     if (argc > 1) {
         sizes.clear();
         for (int i = 1; i < argc; ++i) sizes.push_back(std::stoi(argv[i]));
     }
 
-    // Initialize driver
     checkCU(cuInit(0), "cuInit");
 
-    // pick device 0
     CUdevice dev;
     checkCU(cuDeviceGet(&dev, 0), "cuDeviceGet(0)");
+
     CUcontext ctx;
     checkCU(cuCtxCreate(&ctx, 0, dev), "cuCtxCreate");
 
-    // find the PTX file next to the executable or in current dir - we assume "mmul.ptx" is present
     const std::string ptxPath = "mmul.ptx";
     std::string ptx;
+
     try {
         ptx = readFile(ptxPath);
     } catch (const std::exception &e) {
@@ -173,15 +192,12 @@ int main(int argc, char** argv) {
         return EXIT_FAILURE;
     }
 
-    // load module
     CUmodule module;
     checkCU(cuModuleLoadDataEx(&module, ptx.c_str(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
 
-    // get function handle
     CUfunction kernel;
     checkCU(cuModuleGetFunction(&kernel, module, "matrixMul"), "cuModuleGetFunction matrixMul");
 
-    // Run for each size
     for (int N : sizes) {
         try {
             runMatrixMul(module, kernel, N);
@@ -191,8 +207,8 @@ int main(int argc, char** argv) {
         std::cout << "----------------------------------------\n";
     }
 
-    // cleanup
     checkCU(cuModuleUnload(module), "cuModuleUnload");
     checkCU(cuCtxDestroy(ctx), "cuCtxDestroy");
+
     return 0;
 }

From b76f411802d8aad48ea4e4150dcf2f807aa20a34 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 9 Mar 2026 16:16:38 -0600
Subject: [PATCH 0504/1000] Added another variant of mmul actor to compare more
 results.

---
 .../mmul-actor-benchmarking/main.test.cpp     | 150 +++++++++++++++++-
 1 file changed, 147 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
index 07a05b2aaa..b1d98ef52a 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
@@ -59,7 +59,6 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
       using clock = std::chrono::steady_clock;
       using ms = std::chrono::duration<double, std::milli>;
 
-      auto t_total_start = clock::now();
 
       caf::cuda::manager& mgr = caf::cuda::manager::get();
       int device = 0;
@@ -68,6 +67,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
       auto program =
         mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
+      auto t_total_start = clock::now();
       // -------------------------
       // create_in_arg A
       // -------------------------
@@ -193,8 +193,149 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
   };
 }
 
+
+caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
+  return {
+
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        int N) {
+
+      using clock = std::chrono::steady_clock;
+      using ms = std::chrono::duration<double, std::milli>;
+
+
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+      int device = 0;
+      int stream = 1;
+
+      auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+      auto t_total_start = clock::now();
+      // -------------------------
+      // create_in_arg A
+      // -------------------------
+      auto t_a_inarg_start = clock::now();
+
+      auto inA = caf::cuda::create_in_arg(std::move(matrixA));
+
+      auto t_a_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer A
+      // -------------------------
+      auto t_a_transfer_start = clock::now();
+
+      auto arg1 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inA));
+
+      auto t_a_transfer_end = clock::now();
+
+      // -------------------------
+      // create_in_arg B
+      // -------------------------
+      auto t_b_inarg_start = clock::now();
+
+      auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+
+      auto t_b_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer B
+      // -------------------------
+      auto t_b_transfer_start = clock::now();
+
+      auto arg2 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inB));
+
+      auto t_b_transfer_end = clock::now();
+  const int THREADS = 32;
+      const int BLOCKS = (N + THREADS - 1) / THREADS;
+
+      caf::cuda::nd_range dims(
+        BLOCKS, BLOCKS, 1,
+        THREADS, THREADS, 1);
+
+      // -------------------------
+      // request
+      // -------------------------
+      auto t_request_start = clock::now();
+
+      caf::cuda::mmul_async_command<int> command;
+      auto output = command.run_async(
+		      program,dims,
+		      1,
+		      arg1,arg2,out<int>{N*N},in<int>{N});
+
+       auto t_response_received = clock::now();
+
+            //std::vector<int> matrixC(N*N);
+            // -------------------------
+            // copy to host
+            // -------------------------
+            auto t_copy_start = clock::now();
+
+	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);
+
+            //std::vector<int> matrixC = dC->copy_to_host();
+
+	    dC->copy_to_host(matrixC.data(),N*N);
+
+            auto t_copy_end = clock::now();
+            auto t_total_end = clock::now();
+
+            // -------------------------
+            // Print timings
+            // -------------------------
+
+            std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
+
+            std::cout << "create_in_arg A: "
+                      << ms(t_a_inarg_end - t_a_inarg_start).count()
+                      << " ms\n";
+
+            std::cout << "transfer A: "
+                      << ms(t_a_transfer_end - t_a_transfer_start).count()
+                      << " ms\n";
+
+            std::cout << "create_in_arg B: "
+                      << ms(t_b_inarg_end - t_b_inarg_start).count()
+                      << " ms\n";
+
+            std::cout << "transfer B: "
+                      << ms(t_b_transfer_end - t_b_transfer_start).count()
+                      << " ms\n";
+
+                       std::cout << "request → response latency: "
+                      << ms(t_response_received - t_request_start).count()
+                      << " ms\n";
+
+            std::cout << "copy_to_host: "
+                      << ms(t_copy_end - t_copy_start).count()
+                      << " ms\n";
+
+            std::cout << "TOTAL end-to-end: "
+                      << ms(t_total_end - t_total_start).count()
+                      << " ms\n";
+
+            std::cout << "=============================================\n";
+
+            self->quit();
+    }
+
+  };
+}
+
+
 void run_mmul_test(caf::actor_system& sys, int matrix_size) {
 
+
+  caf::cuda::manager::init(sys);
   // ------------------------------------
   // Start timing
   // ------------------------------------
@@ -210,7 +351,7 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size) {
 
 auto t_start = clock::now();
 
-caf::actor a =sys.spawn(mmul_actor_fun);
+caf::actor a =sys.spawn(mmul_actor_fun_2);
 
 anon_mail(matrixA,matrixB,matrix_size).send(a);
 
@@ -230,10 +371,13 @@ auto t_end = clock::now();
 
   std::cout << "[MMUL TEST] matrix_size=" << matrix_size
             << ", time=" << duration_ms << " ms\n";
+
+  caf::cuda::manager::shutdown();
+
 }
 
+
 void caf_main(caf::actor_system& sys) {
-  caf::cuda::manager::init(sys);
   run_mmul_test(sys,1000);
   run_mmul_test(sys,4000);
   run_mmul_test(sys,8000);

From 39c4caba67e9924f7bde2972343251502cd29435 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Mar 2026 11:04:32 -0600
Subject: [PATCH 0505/1000] Changed num_streams to 500 to coinside with max
 number of streams possible.

---
 libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
index cf3dc19a3b..fff453484a 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
@@ -17,7 +17,7 @@ struct scheduler_actor_state {
     std::queue<token_ptr> queue; // here for legacy prototype schedulers
     std::vector<kernel_graph> operations; //more modern dependency based data structure 
     int device_number;
-    int num_streams = 150; // number of streams that can be used by the scheduler
+    int num_streams = 500; // number of streams that can be used by the scheduler
     std::vector<caf::actor> schedulers; //the other scheduler actors in the system, an actor for a 
 			     //specific GPU can be accessed via there corrosponding 
 			     //device number in the std::vector

From d10a2ead5c47ae4588737273b85e212138a31fcc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Mar 2026 12:31:28 -0600
Subject: [PATCH 0506/1000] Initial commit.

---
 .../CMakeLists.txt                            |  44 ++
 .../compile_kernels.sh                        |  13 +
 .../main.test.cpp                             | 469 ++++++++++++++++++
 .../mmul-uniform-batch-benchmark/mmul.cu      |  16 +
 4 files changed, 542 insertions(+)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/main.test.cpp
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/mmul.cu

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/compile_kernels.sh b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/compile_kernels.sh
new file mode 100755
index 0000000000..f32480e5cb
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/compile_kernels.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/main.test.cpp
new file mode 100644
index 0000000000..2fa77efd02
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/main.test.cpp
@@ -0,0 +1,469 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/component-actors/all-component-actors.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+using command =
+  caf::cuda::command_runner<>;
+
+command mmul_command;
+command mmul;
+
+struct mmul_state {
+
+	caf::cuda::program_ptr mmul_kernel;
+
+};
+
+
+
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::program_ptr mmul_kernel,int N) {
+
+	self->state().mmul_kernel = mmul_kernel;
+	std::vector<int> matrix1(N*N);
+	std::vector<int> matrix2(N*N);
+	self->mail(matrix1, matrix2, N).send(self);
+	return {
+		[=](const std::vector<int>& matrixA,
+				const std::vector<int>& matrixB,
+				int N) {
+
+			caf::cuda::manager& mgr = caf::cuda::manager::get();
+			int device = 0;
+			int stream = rand();
+
+			//auto program =
+			//mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+			auto program = self->state().mmul_kernel;
+
+				// -------------------------
+				// create_in_arg A
+				// -------------------------
+
+			auto inA = caf::cuda::create_in_arg(std::move(matrixA));
+
+
+			auto arg1 = mmul_command.transfer_memory(
+					device,
+					stream,
+					std::move(inA));
+
+
+
+
+
+			auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+			auto arg2 = mmul_command.transfer_memory(
+					device,
+					stream,
+					std::move(inB));
+
+
+
+			caf::actor mmul_actor =
+				self->spawn(caf::cuda::mmul_actor_fun<int>, program);
+
+			caf::actor mem_transfer_actor = self->spawn(caf::cuda::mem_transfer_actor_fun<int>);
+
+
+			self->mail(arg1, arg2, N, device, stream)
+				.request(mmul_actor, std::chrono::seconds(30))
+				.then([=](caf::cuda::mem_ptr<int> dC) {
+
+						self->mail(dC).request(mem_transfer_actor,std::chrono::seconds(4000))
+						.then([=] (std::vector<int>& matrixC) {
+								//std::vector<int> matrixC = dC->copy_to_host();
+								self->quit();
+
+								});
+						});
+		}
+
+	};
+}
+
+
+
+
+struct mmul_actor_with_scheduler_state {
+  static inline const char* name = "my_actor";
+  caf::actor sync_actor;
+  caf::actor mem_transfer_actor;
+  caf::actor mmul_actor;
+};
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun_scheduler(
+    caf::stateful_actor<mmul_actor_with_scheduler_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims)
+{
+
+
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+        caf::actor scheduler = mgr.get_scheduler_actor();
+
+        //send a launch token
+        caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+                        program,
+                        dims,
+                        0,
+                        "hello",
+                        self
+                        );
+        self -> mail(launch_token).send(scheduler);
+
+        self->state().sync_actor = self -> spawn(caf::cuda::sync_actor_fun<int>);
+        self->state().mem_transfer_actor = self -> spawn(caf::cuda::mem_transfer_actor_fun<int>);
+	self->state().mmul_actor = self -> spawn(caf::cuda::mmul_actor_fun<int>,program);
+
+       return {
+
+    // 1. Handle response token
+    [=](caf::cuda::response_token_ptr res_token) {
+//        std::cout << "Got response\n";
+
+        if (res_token->getType() == LAUNCH_RESPONSE) {
+            std::vector<int> matrix1(N*N);
+            std::vector<int> matrix2(N*N);
+
+            self->mail(matrix1, matrix2, res_token, N).send(self);
+
+        } else {
+  //          std::cout << "Got a memory response token\n";
+        }
+    },
+
+    // 2. Handle memory buffers -> GPU
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        const caf::cuda::response_token_ptr& res_token,
+        int N) {
+
+    //    std::cout << "Working\n";
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+       
+	/*
+	const int THREADS = 32;
+        const int BLOCKS = (N + THREADS - 1) / THREADS;
+        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+        auto arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixA));
+        auto arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixB));
+        auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
+        auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
+
+
+        auto tempC = mmulAsync.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
+        caf::cuda::mem_ptr<int> bufferA = std::get<0>(tempC);
+	*/
+
+	caf::cuda::mem_ptr<int> arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixA));
+	caf::cuda::mem_ptr<int> arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixB));
+
+	self->mail(arg1,arg2,N,res_token -> getDeviceNumber(), res_token -> getStreamId())
+		.request(self->state().mmul_actor,1000s)
+		.then(
+		 [=](caf::cuda::mem_ptr<int> dC) {
+
+
+        self->mail(dC, res_token)
+            .request(self->state().sync_actor, 1000s)
+            .then(
+                [=](caf::cuda::mem_ptr<int> /*syncedC*/) {
+                    self->mail(dC)
+                        .request(self->state().mem_transfer_actor, 1000s)
+                        .then(
+                            [=](std::vector<int> matrixC) {
+                                //self->mail(matrixA,matrixB,matrixC, N).send(self);
+                            
+			     self->mail(1).send(exit_actor);
+        		     self->quit();
+			    
+			    },
+                            [=](caf::error& err) {
+                                std::cout << "Transfer C failed: " << to_string(err) << "\n";
+                                self->quit(err);
+                            });
+                },
+                [=](caf::error& err) {
+                    std::cout << "Sync failed: " << to_string(err) << "\n";
+                    self->quit(err);
+                });
+		 });
+    },
+       };
+
+}
+
+
+
+
+
+
+
+
+
+
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //change the scheduler to mulitlevle_usage
+  anon_mail(
+        caf::cuda::make_behavior_token("multilevel")
+        ).send(mgr.get_scheduler_actor());
+
+  // CREATE ONCE
+  auto program =
+      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+  const int THREADS = 32;
+  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1);
+
+  caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
+
+  for (int i = 0; i < num_actors; ++i) {
+    
+      sys.spawn(
+        mmul_actor_fun_scheduler,
+        exit_actor,
+        matrix_size,
+        program,
+        dims);
+    
+  }
+
+  sys.await_all_actors_done();
+}
+
+
+void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  /*
+  //change the scheduler to core_usage
+  anon_mail(
+        caf::cuda::make_behavior_token("core_usage")
+        ).send(mgr.get_scheduler_actor());
+
+  */
+
+    mgr.send_scheduler_actor_message("green",0);
+
+  // CREATE ONCE
+  auto program =
+      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+  const int THREADS = 32;
+  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1);
+
+  caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
+
+  for (int i = 0; i < num_actors; ++i) {
+    
+      sys.spawn(
+        mmul_actor_fun_scheduler,
+        exit_actor,
+        matrix_size,
+        program,
+        dims);
+    
+  }
+
+ 
+
+  sys.await_all_actors_done();
+}
+
+void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, int num_actors) {
+    if (num_actors < 1) {
+        std::cerr << "[ERROR] Number of actors must be >= 1\n";
+        return;
+    }
+
+    
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    const int THREADS = 32;
+    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+
+    for (int i = 0; i < num_actors; ++i) {
+        sys.spawn(
+            mmul_actor_fun,
+            program,
+            matrix_size
+        );
+    }
+
+    sys.await_all_actors_done();
+}
+
+
+
+
+template <class Fn>
+double time_run(Fn&& fn) {
+  auto start = std::chrono::steady_clock::now();
+  fn();
+  auto end = std::chrono::steady_clock::now();
+  std::chrono::duration<double> elapsed = end - start;
+  return elapsed.count();
+}
+void run_mmul_scaling_tests(caf::actor_system& sys,
+                            caf::cuda::manager_config man_config) {
+    const int max_size   = 2048;
+    const int min_actors = 1;
+    const int max_actors = 1024;
+
+    std::vector<int> matrix_sizes = {10};
+    for (int s = 32; s <= max_size; s *= 2)
+        matrix_sizes.push_back(s);
+
+    std::vector<int> actor_counts;
+    for (int a = min_actors; a <= max_actors; a *= 2)
+        actor_counts.push_back(a);
+
+    std::cout << "=== MMUL Scaling Tests ===\n";
+    std::cout << "Format:\n";
+    std::cout << "scheduler matrix_size actors time_seconds\n";
+
+    for (int size : matrix_sizes) {
+        for (int actors : actor_counts) {
+
+            /* ================= Scheduler-enabled (core_usage) ================= */
+            caf::cuda::manager::init(sys, man_config); // scheduler enabled
+            std::cout << "\n[RUN] scheduler=multilevel_usage "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double core_usage_time = time_run([&] {
+                run_mmul_test(sys, size, actors); // uses mmul_actor_fun_no_verify
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT core_usage "
+                      << size << " "
+                      << actors << " "
+                      << core_usage_time << "\n";
+
+            caf::cuda::manager::shutdown(); // make sure manager is cleaned up
+
+            /* ================= Scheduler-disabled actor ( uses green-light) ================= */
+
+            caf::cuda::manager::init(sys, man_config); // init with scheduler
+            std::cout << "\n[RUN] scheduler=green_light_only "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double green_light_time = time_run([&] {
+                run_mmul_test_no_scheduler(sys, size, actors); // your previous "no scheduler" actor
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT green_light_only "
+                      << size << " "
+                      << actors << " "
+                      << green_light_time << "\n";
+
+            caf::cuda::manager::shutdown();
+
+            /* ================= No scheduler at all actor ================= */
+            caf::cuda::manager_config no_sched_config(false); // disable scheduler
+            caf::cuda::manager::init(sys, no_sched_config);
+            std::cout << "\n[RUN] scheduler=none "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double no_scheduler_time = time_run([&] {
+                run_mmul_test_no_scheduler_actor(sys, size, actors); // mmul_actor_fun_no_schedule
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT none "
+                      << size << " "
+                      << actors << " "
+                      << no_scheduler_time << "\n";
+
+            caf::cuda::manager::shutdown();
+        }
+    }
+
+    std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
+}
+
+
+
+
+void caf_main(caf::actor_system& sys) {
+  
+
+  caf::cuda::manager_config man_config(true);
+  //caf::cuda::manager::init(sys,man_config);
+  run_mmul_scaling_tests(sys,man_config);
+
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/mmul.cu b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From 67ef96e563aee435de31bdbbdaba27ea15629c1f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Mar 2026 12:48:08 -0600
Subject: [PATCH 0507/1000] Implemented mmul_no_scheduler vs mmul actor.

---
 .../main.test.cpp                             | 168 +++++++-----------
 1 file changed, 67 insertions(+), 101 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/main.test.cpp
index 2fa77efd02..3d60943eeb 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/main.test.cpp
@@ -39,7 +39,10 @@ using command =
   caf::cuda::command_runner<>;
 
 command mmul_command;
-command mmul;
+caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int >> mmul;
+using async_command = caf::cuda::mmul_async_command<int>;
+async_command async_mmul;
+
 
 struct mmul_state {
 
@@ -67,51 +70,44 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
 			//auto program =
 			//mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
-			auto program = self->state().mmul_kernel;
-
-				// -------------------------
-				// create_in_arg A
-				// -------------------------
+		    const int THREADS = 32;
+		    const int BLOCKS = (N + THREADS - 1) / THREADS;
+		    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-			auto inA = caf::cuda::create_in_arg(std::move(matrixA));
 
+			auto program = self->state().mmul_kernel;
 
+			auto inA = caf::cuda::create_in_arg(std::move(matrixA));
 			auto arg1 = mmul_command.transfer_memory(
 					device,
 					stream,
 					std::move(inA));
 
-
-
-
-
 			auto inB = caf::cuda::create_in_arg(std::move(matrixB));
 			auto arg2 = mmul_command.transfer_memory(
 					device,
 					stream,
 					std::move(inB));
 
+			out<T> arg3 = caf::cuda::create_out_arg<int>(N * N);
+			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
 
+			auto result = 
+				async_mmul.run_async(
+						program,
+						dims,
+						stream,
+						0,
+						device_number,
+						arg1,
+						arg2,
+						arg3,
+						arg4);
 
-			caf::actor mmul_actor =
-				self->spawn(caf::cuda::mmul_actor_fun<int>, program);
-
-			caf::actor mem_transfer_actor = self->spawn(caf::cuda::mem_transfer_actor_fun<int>);
-
-
-			self->mail(arg1, arg2, N, device, stream)
-				.request(mmul_actor, std::chrono::seconds(30))
-				.then([=](caf::cuda::mem_ptr<int> dC) {
+			std::get<2>(result) -> copy_to_host();
+			self -> quit();
 
-						self->mail(dC).request(mem_transfer_actor,std::chrono::seconds(4000))
-						.then([=] (std::vector<int>& matrixC) {
-								//std::vector<int> matrixC = dC->copy_to_host();
-								self->quit();
-
-								});
-						});
 		}
-
 	};
 }
 
@@ -146,90 +142,60 @@ caf::behavior mmul_actor_fun_scheduler(
                         dims,
                         0,
                         "hello",
-                        self
+                        self,
+			rand() //dependency number, can declare indepedent but want to see what happens when you do not 
                         );
         self -> mail(launch_token).send(scheduler);
 
-        self->state().sync_actor = self -> spawn(caf::cuda::sync_actor_fun<int>);
-        self->state().mem_transfer_actor = self -> spawn(caf::cuda::mem_transfer_actor_fun<int>);
-	self->state().mmul_actor = self -> spawn(caf::cuda::mmul_actor_fun<int>,program);
+	return {
+
+		// 1. Handle response token
+		[=](caf::cuda::response_token_ptr res_token) {
+			//        std::cout << "Got response\n";
 
-       return {
+			if (res_token->getType() == LAUNCH_RESPONSE) {
+				std::vector<int> matrix1(N*N);
+				std::vector<int> matrix2(N*N);
 
-    // 1. Handle response token
-    [=](caf::cuda::response_token_ptr res_token) {
-//        std::cout << "Got response\n";
+				self->mail(matrix1, matrix2, res_token, N).send(self);
 
-        if (res_token->getType() == LAUNCH_RESPONSE) {
-            std::vector<int> matrix1(N*N);
-            std::vector<int> matrix2(N*N);
+			} else {
+				//          std::cout << "Got a memory response token\n";
+			}
+		},
 
-            self->mail(matrix1, matrix2, res_token, N).send(self);
+	    // 2. Handle memory buffers -> GPU
+	    [=](const std::vector<int>& matrixA,
+			    const std::vector<int>& matrixB,
+			    const caf::cuda::response_token_ptr& res_token,
+			    int N) {
 
-        } else {
-  //          std::cout << "Got a memory response token\n";
-        }
-    },
+		    //    std::cout << "Working\n";
+		    caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-    // 2. Handle memory buffers -> GPU
-    [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB,
-        const caf::cuda::response_token_ptr& res_token,
-        int N) {
 
-    //    std::cout << "Working\n";
-        caf::cuda::manager& mgr = caf::cuda::manager::get();
+		    const int THREADS = 32;
+		    const int BLOCKS = (N + THREADS - 1) / THREADS;
+		    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+		    auto arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixA));
+		    auto arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixB));
+		    auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
+		    auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
+
+
+		    auto tempC = mmulAsync.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
+		    caf::cuda::mem_ptr<int> bufferC = std::get<2>(tempC);
+
+		    bufferC -> synchronize();
+		    res_token->release();
+		    bufferC->copy_to_host();
+
+		    self->mail(1).send(exit_actor);
+		    self->quit();
+         }
+
 
-       
-	/*
-	const int THREADS = 32;
-        const int BLOCKS = (N + THREADS - 1) / THREADS;
-        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-        auto arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixA));
-        auto arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixB));
-        auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
-        auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
-
-
-        auto tempC = mmulAsync.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
-        caf::cuda::mem_ptr<int> bufferA = std::get<0>(tempC);
-	*/
-
-	caf::cuda::mem_ptr<int> arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixA));
-	caf::cuda::mem_ptr<int> arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixB));
-
-	self->mail(arg1,arg2,N,res_token -> getDeviceNumber(), res_token -> getStreamId())
-		.request(self->state().mmul_actor,1000s)
-		.then(
-		 [=](caf::cuda::mem_ptr<int> dC) {
-
-
-        self->mail(dC, res_token)
-            .request(self->state().sync_actor, 1000s)
-            .then(
-                [=](caf::cuda::mem_ptr<int> /*syncedC*/) {
-                    self->mail(dC)
-                        .request(self->state().mem_transfer_actor, 1000s)
-                        .then(
-                            [=](std::vector<int> matrixC) {
-                                //self->mail(matrixA,matrixB,matrixC, N).send(self);
-                            
-			     self->mail(1).send(exit_actor);
-        		     self->quit();
-			    
-			    },
-                            [=](caf::error& err) {
-                                std::cout << "Transfer C failed: " << to_string(err) << "\n";
-                                self->quit(err);
-                            });
-                },
-                [=](caf::error& err) {
-                    std::cout << "Sync failed: " << to_string(err) << "\n";
-                    self->quit(err);
-                });
-		 });
-    },
        };
 
 }

From 774746bf06b611bb36bbfdf36253d0f42e3d773f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Mar 2026 13:25:30 -0600
Subject: [PATCH 0508/1000] Saving.

---
 .../mmul-uniform-batch-benchmark/main.test.cpp              | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/main.test.cpp
index 3d60943eeb..1ff4f212e0 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/main.test.cpp
@@ -89,7 +89,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
 					stream,
 					std::move(inB));
 
-			out<T> arg3 = caf::cuda::create_out_arg<int>(N * N);
+			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
 			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
 
 			auto result = 
@@ -98,7 +98,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
 						dims,
 						stream,
 						0,
-						device_number,
+						device,
 						arg1,
 						arg2,
 						arg3,
@@ -184,7 +184,7 @@ caf::behavior mmul_actor_fun_scheduler(
 		    auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
 
 
-		    auto tempC = mmulAsync.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
+		    auto tempC = mmul.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
 		    caf::cuda::mem_ptr<int> bufferC = std::get<2>(tempC);
 
 		    bufferC -> synchronize();

From 81f3806961803b56dd03f2dc6556c5ac12da3961 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Mar 2026 13:26:16 -0600
Subject: [PATCH 0509/1000] Iniital commit.

---
 .../CMakeLists.txt                            |  44 ++
 .../compile_kernels.sh                        |  13 +
 .../main.test.cpp                             | 435 ++++++++++++++++++
 .../mmul-non-uniform-batch-benchmark/mmul.cu  |  16 +
 4 files changed, 508 insertions(+)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/mmul.cu

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/CMakeLists.txt
new file mode 100644
index 0000000000..e2259ce5fe
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/compile_kernels.sh b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/compile_kernels.sh
new file mode 100755
index 0000000000..f32480e5cb
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/compile_kernels.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
new file mode 100644
index 0000000000..1ff4f212e0
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
@@ -0,0 +1,435 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/component-actors/all-component-actors.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+using command =
+  caf::cuda::command_runner<>;
+
+command mmul_command;
+caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int >> mmul;
+using async_command = caf::cuda::mmul_async_command<int>;
+async_command async_mmul;
+
+
+struct mmul_state {
+
+	caf::cuda::program_ptr mmul_kernel;
+
+};
+
+
+
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::program_ptr mmul_kernel,int N) {
+
+	self->state().mmul_kernel = mmul_kernel;
+	std::vector<int> matrix1(N*N);
+	std::vector<int> matrix2(N*N);
+	self->mail(matrix1, matrix2, N).send(self);
+	return {
+		[=](const std::vector<int>& matrixA,
+				const std::vector<int>& matrixB,
+				int N) {
+
+			caf::cuda::manager& mgr = caf::cuda::manager::get();
+			int device = 0;
+			int stream = rand();
+
+			//auto program =
+			//mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+		    const int THREADS = 32;
+		    const int BLOCKS = (N + THREADS - 1) / THREADS;
+		    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+
+			auto program = self->state().mmul_kernel;
+
+			auto inA = caf::cuda::create_in_arg(std::move(matrixA));
+			auto arg1 = mmul_command.transfer_memory(
+					device,
+					stream,
+					std::move(inA));
+
+			auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+			auto arg2 = mmul_command.transfer_memory(
+					device,
+					stream,
+					std::move(inB));
+
+			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
+			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
+
+			auto result = 
+				async_mmul.run_async(
+						program,
+						dims,
+						stream,
+						0,
+						device,
+						arg1,
+						arg2,
+						arg3,
+						arg4);
+
+			std::get<2>(result) -> copy_to_host();
+			self -> quit();
+
+		}
+	};
+}
+
+
+
+
+struct mmul_actor_with_scheduler_state {
+  static inline const char* name = "my_actor";
+  caf::actor sync_actor;
+  caf::actor mem_transfer_actor;
+  caf::actor mmul_actor;
+};
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun_scheduler(
+    caf::stateful_actor<mmul_actor_with_scheduler_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims)
+{
+
+
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+        caf::actor scheduler = mgr.get_scheduler_actor();
+
+        //send a launch token
+        caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+                        program,
+                        dims,
+                        0,
+                        "hello",
+                        self,
+			rand() //dependency number, can declare indepedent but want to see what happens when you do not 
+                        );
+        self -> mail(launch_token).send(scheduler);
+
+	return {
+
+		// 1. Handle response token
+		[=](caf::cuda::response_token_ptr res_token) {
+			//        std::cout << "Got response\n";
+
+			if (res_token->getType() == LAUNCH_RESPONSE) {
+				std::vector<int> matrix1(N*N);
+				std::vector<int> matrix2(N*N);
+
+				self->mail(matrix1, matrix2, res_token, N).send(self);
+
+			} else {
+				//          std::cout << "Got a memory response token\n";
+			}
+		},
+
+	    // 2. Handle memory buffers -> GPU
+	    [=](const std::vector<int>& matrixA,
+			    const std::vector<int>& matrixB,
+			    const caf::cuda::response_token_ptr& res_token,
+			    int N) {
+
+		    //    std::cout << "Working\n";
+		    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+
+		    const int THREADS = 32;
+		    const int BLOCKS = (N + THREADS - 1) / THREADS;
+		    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+		    auto arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixA));
+		    auto arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixB));
+		    auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
+		    auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
+
+
+		    auto tempC = mmul.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
+		    caf::cuda::mem_ptr<int> bufferC = std::get<2>(tempC);
+
+		    bufferC -> synchronize();
+		    res_token->release();
+		    bufferC->copy_to_host();
+
+		    self->mail(1).send(exit_actor);
+		    self->quit();
+         }
+
+
+       };
+
+}
+
+
+
+
+
+
+
+
+
+
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //change the scheduler to mulitlevle_usage
+  anon_mail(
+        caf::cuda::make_behavior_token("multilevel")
+        ).send(mgr.get_scheduler_actor());
+
+  // CREATE ONCE
+  auto program =
+      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+  const int THREADS = 32;
+  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1);
+
+  caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
+
+  for (int i = 0; i < num_actors; ++i) {
+    
+      sys.spawn(
+        mmul_actor_fun_scheduler,
+        exit_actor,
+        matrix_size,
+        program,
+        dims);
+    
+  }
+
+  sys.await_all_actors_done();
+}
+
+
+void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  /*
+  //change the scheduler to core_usage
+  anon_mail(
+        caf::cuda::make_behavior_token("core_usage")
+        ).send(mgr.get_scheduler_actor());
+
+  */
+
+    mgr.send_scheduler_actor_message("green",0);
+
+  // CREATE ONCE
+  auto program =
+      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+  const int THREADS = 32;
+  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(
+      BLOCKS, BLOCKS, 1,
+      THREADS, THREADS, 1);
+
+  caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
+
+  for (int i = 0; i < num_actors; ++i) {
+    
+      sys.spawn(
+        mmul_actor_fun_scheduler,
+        exit_actor,
+        matrix_size,
+        program,
+        dims);
+    
+  }
+
+ 
+
+  sys.await_all_actors_done();
+}
+
+void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, int num_actors) {
+    if (num_actors < 1) {
+        std::cerr << "[ERROR] Number of actors must be >= 1\n";
+        return;
+    }
+
+    
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    const int THREADS = 32;
+    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+
+    for (int i = 0; i < num_actors; ++i) {
+        sys.spawn(
+            mmul_actor_fun,
+            program,
+            matrix_size
+        );
+    }
+
+    sys.await_all_actors_done();
+}
+
+
+
+
+template <class Fn>
+double time_run(Fn&& fn) {
+  auto start = std::chrono::steady_clock::now();
+  fn();
+  auto end = std::chrono::steady_clock::now();
+  std::chrono::duration<double> elapsed = end - start;
+  return elapsed.count();
+}
+void run_mmul_scaling_tests(caf::actor_system& sys,
+                            caf::cuda::manager_config man_config) {
+    const int max_size   = 2048;
+    const int min_actors = 1;
+    const int max_actors = 1024;
+
+    std::vector<int> matrix_sizes = {10};
+    for (int s = 32; s <= max_size; s *= 2)
+        matrix_sizes.push_back(s);
+
+    std::vector<int> actor_counts;
+    for (int a = min_actors; a <= max_actors; a *= 2)
+        actor_counts.push_back(a);
+
+    std::cout << "=== MMUL Scaling Tests ===\n";
+    std::cout << "Format:\n";
+    std::cout << "scheduler matrix_size actors time_seconds\n";
+
+    for (int size : matrix_sizes) {
+        for (int actors : actor_counts) {
+
+            /* ================= Scheduler-enabled (core_usage) ================= */
+            caf::cuda::manager::init(sys, man_config); // scheduler enabled
+            std::cout << "\n[RUN] scheduler=multilevel_usage "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double core_usage_time = time_run([&] {
+                run_mmul_test(sys, size, actors); // uses mmul_actor_fun_no_verify
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT core_usage "
+                      << size << " "
+                      << actors << " "
+                      << core_usage_time << "\n";
+
+            caf::cuda::manager::shutdown(); // make sure manager is cleaned up
+
+            /* ================= Scheduler-disabled actor ( uses green-light) ================= */
+
+            caf::cuda::manager::init(sys, man_config); // init with scheduler
+            std::cout << "\n[RUN] scheduler=green_light_only "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double green_light_time = time_run([&] {
+                run_mmul_test_no_scheduler(sys, size, actors); // your previous "no scheduler" actor
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT green_light_only "
+                      << size << " "
+                      << actors << " "
+                      << green_light_time << "\n";
+
+            caf::cuda::manager::shutdown();
+
+            /* ================= No scheduler at all actor ================= */
+            caf::cuda::manager_config no_sched_config(false); // disable scheduler
+            caf::cuda::manager::init(sys, no_sched_config);
+            std::cout << "\n[RUN] scheduler=none "
+                      << "matrix_size=" << size
+                      << " actors=" << actors << "\n";
+
+            double no_scheduler_time = time_run([&] {
+                run_mmul_test_no_scheduler_actor(sys, size, actors); // mmul_actor_fun_no_schedule
+            });
+
+            std::cout << std::fixed << std::setprecision(6)
+                      << "RESULT none "
+                      << size << " "
+                      << actors << " "
+                      << no_scheduler_time << "\n";
+
+            caf::cuda::manager::shutdown();
+        }
+    }
+
+    std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
+}
+
+
+
+
+void caf_main(caf::actor_system& sys) {
+  
+
+  caf::cuda::manager_config man_config(true);
+  //caf::cuda::manager::init(sys,man_config);
+  run_mmul_scaling_tests(sys,man_config);
+
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/mmul.cu b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From b0a97245d53fc1247d2547358c222fdf7fc24e30 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Mar 2026 14:01:25 -0600
Subject: [PATCH 0510/1000] saving>

---
 .../main.test.cpp                             | 48 ++++++++-----------
 1 file changed, 21 insertions(+), 27 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
index 1ff4f212e0..5bdfb958d9 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
@@ -52,11 +52,12 @@ struct mmul_state {
 
 
 
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::program_ptr mmul_kernel,int N) {
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::program_ptr mmul_kernel,
+		const std::vector<int>& matrix1,
+		const std::vector<int>& matrix2,
+		int N) {
 
 	self->state().mmul_kernel = mmul_kernel;
-	std::vector<int> matrix1(N*N);
-	std::vector<int> matrix2(N*N);
 	self->mail(matrix1, matrix2, N).send(self);
 	return {
 		[=](const std::vector<int>& matrixA,
@@ -116,9 +117,6 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
 
 struct mmul_actor_with_scheduler_state {
   static inline const char* name = "my_actor";
-  caf::actor sync_actor;
-  caf::actor mem_transfer_actor;
-  caf::actor mmul_actor;
 };
 
 
@@ -128,13 +126,15 @@ caf::behavior mmul_actor_fun_scheduler(
     caf::actor exit_actor,
     int N,
     caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims)
+    caf::cuda::nd_range dims
+    const std::vector<int>& matrix1,
+    const std::vector<int> & matrix2)
 {
 
 
         caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-        caf::actor scheduler = mgr.get_scheduler_actor();
+        //caf::actor scheduler = mgr.get_scheduler_actor();
 
         //send a launch token
         caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
@@ -145,7 +145,7 @@ caf::behavior mmul_actor_fun_scheduler(
                         self,
 			rand() //dependency number, can declare indepedent but want to see what happens when you do not 
                         );
-        self -> mail(launch_token).send(scheduler);
+        mgr.send_scheduler_message(launch_token);
 
 	return {
 
@@ -154,9 +154,6 @@ caf::behavior mmul_actor_fun_scheduler(
 			//        std::cout << "Got response\n";
 
 			if (res_token->getType() == LAUNCH_RESPONSE) {
-				std::vector<int> matrix1(N*N);
-				std::vector<int> matrix2(N*N);
-
 				self->mail(matrix1, matrix2, res_token, N).send(self);
 
 			} else {
@@ -219,10 +216,11 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
 
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-  //change the scheduler to mulitlevle_usage
-  anon_mail(
-        caf::cuda::make_behavior_token("multilevel")
-        ).send(mgr.get_scheduler_actor());
+  //set the behaviors of each scheduler actor
+    for (int i = 0; i < mgr.get_num_devices();i++) {
+            mgr.send_scheduler_actor_message("multilevel",i);
+    }
+
 
   // CREATE ONCE
   auto program =
@@ -259,17 +257,13 @@ void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num
 
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-  /*
-  //change the scheduler to core_usage
-  anon_mail(
-        caf::cuda::make_behavior_token("core_usage")
-        ).send(mgr.get_scheduler_actor());
-
-  */
-
-    mgr.send_scheduler_actor_message("green",0);
-
-  // CREATE ONCE
+ 
+  //set the behaviors of each scheduler actor
+    for (int i = 0; i < mgr.get_num_devices();i++) {
+            mgr.send_scheduler_actor_message("green",i);
+    }  
+ 
+   // CREATE ONCE
   auto program =
       mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 

From dc270d3408c3638549c6325e9f3c051e934059a2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Mar 2026 14:39:01 -0600
Subject: [PATCH 0511/1000] Saving.

---
 .../main.test.cpp                             | 724 +++++++++---------
 1 file changed, 377 insertions(+), 347 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
index 5bdfb958d9..32c50dc301 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
@@ -9,317 +9,308 @@
 #include <algorithm>
 #include <numeric>
 #include <random>
+#include <map>
+#include <functional>
+#include <memory>
+#include <mutex>
 #include "caf/actor_registry.hpp"
-//#include <caf/atoms.hpp>
-
-
 
 using namespace caf;
 using namespace std::chrono_literals;
 
-
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-  
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
-    }
-  }
-}
-
-using command =
-  caf::cuda::command_runner<>;
-
+// --------------------------- Types / globals ---------------------------
+using command = caf::cuda::command_runner<>;
 command mmul_command;
-caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int >> mmul;
 using async_command = caf::cuda::mmul_async_command<int>;
 async_command async_mmul;
+caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int >> mmul;
 
 
 struct mmul_state {
-
-	caf::cuda::program_ptr mmul_kernel;
-
+  caf::cuda::program_ptr mmul_kernel;
 };
 
-
-
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::program_ptr mmul_kernel,
-		const std::vector<int>& matrix1,
-		const std::vector<int>& matrix2,
-		int N) {
-
-	self->state().mmul_kernel = mmul_kernel;
-	self->mail(matrix1, matrix2, N).send(self);
-	return {
-		[=](const std::vector<int>& matrixA,
-				const std::vector<int>& matrixB,
-				int N) {
-
-			caf::cuda::manager& mgr = caf::cuda::manager::get();
-			int device = 0;
-			int stream = rand();
-
-			//auto program =
-			//mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-		    const int THREADS = 32;
-		    const int BLOCKS = (N + THREADS - 1) / THREADS;
-		    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-
-			auto program = self->state().mmul_kernel;
-
-			auto inA = caf::cuda::create_in_arg(std::move(matrixA));
-			auto arg1 = mmul_command.transfer_memory(
-					device,
-					stream,
-					std::move(inA));
-
-			auto inB = caf::cuda::create_in_arg(std::move(matrixB));
-			auto arg2 = mmul_command.transfer_memory(
-					device,
-					stream,
-					std::move(inB));
-
-			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
-			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
-
-			auto result = 
-				async_mmul.run_async(
-						program,
-						dims,
-						stream,
-						0,
-						device,
-						arg1,
-						arg2,
-						arg3,
-						arg4);
-
-			std::get<2>(result) -> copy_to_host();
-			self -> quit();
-
-		}
-	};
-}
-
-
-
-
 struct mmul_actor_with_scheduler_state {
   static inline const char* name = "my_actor";
 };
 
+// --------------------------- Deterministic matrix generation & pools (shared pool, pass indices) ---------------------------
+static std::vector<int> generate_matrix(int N, uint64_t seed) {
+  std::mt19937_64 rng(seed);
+  std::uniform_int_distribution<int> dist(0, 9);
+  std::vector<int> mat;
+  mat.resize((size_t)N * N);
+  for (size_t i = 0; i < mat.size(); ++i)
+    mat[i] = dist(rng);
+  return mat;
+}
 
-// Stateful actor behavior
-caf::behavior mmul_actor_fun_scheduler(
-    caf::stateful_actor<mmul_actor_with_scheduler_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims
-    const std::vector<int>& matrix1,
-    const std::vector<int> & matrix2)
-{
-
-
-        caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-        //caf::actor scheduler = mgr.get_scheduler_actor();
-
-        //send a launch token
-        caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-                        program,
-                        dims,
-                        0,
-                        "hello",
-                        self,
-			rand() //dependency number, can declare indepedent but want to see what happens when you do not 
-                        );
-        mgr.send_scheduler_message(launch_token);
-
-	return {
-
-		// 1. Handle response token
-		[=](caf::cuda::response_token_ptr res_token) {
-			//        std::cout << "Got response\n";
-
-			if (res_token->getType() == LAUNCH_RESPONSE) {
-				self->mail(matrix1, matrix2, res_token, N).send(self);
-
-			} else {
-				//          std::cout << "Got a memory response token\n";
-			}
-		},
-
-	    // 2. Handle memory buffers -> GPU
-	    [=](const std::vector<int>& matrixA,
-			    const std::vector<int>& matrixB,
-			    const caf::cuda::response_token_ptr& res_token,
-			    int N) {
-
-		    //    std::cout << "Working\n";
-		    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-
-		    const int THREADS = 32;
-		    const int BLOCKS = (N + THREADS - 1) / THREADS;
-		    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-		    auto arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixA));
-		    auto arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixB));
-		    auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
-		    auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
-
-
-		    auto tempC = mmul.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
-		    caf::cuda::mem_ptr<int> bufferC = std::get<2>(tempC);
-
-		    bufferC -> synchronize();
-		    res_token->release();
-		    bufferC->copy_to_host();
-
-		    self->mail(1).send(exit_actor);
-		    self->quit();
-         }
-
-
-       };
-
+// returns map<size, vector<shared_ptr<vector<int>>>>
+static std::map<int, std::vector<std::shared_ptr<std::vector<int>>>> prepare_matrix_pools(
+    const std::vector<int>& sizes,
+    int pool_size_per_size,
+    uint64_t master_seed) {
+
+  std::map<int, std::vector<std::shared_ptr<std::vector<int>>>> pools;
+  for (int N : sizes) {
+    int pool_sz = pool_size_per_size;
+    if (N >= 1024) pool_sz = std::min(pool_sz, 8);
+    if (N >= 2048) pool_sz = std::min(pool_sz, 4);
+
+    pools[N] = {};
+    for (int i = 0; i < pool_sz; ++i) {
+      uint64_t seed = master_seed ^ (uint64_t(N) << 32) ^ (uint64_t(i) * 0x9e3779b97f4a7c15ULL);
+      auto v = std::make_shared<std::vector<int>>(generate_matrix(N, seed));
+      pools[N].push_back(std::move(v));
+    }
+  }
+  return pools;
 }
 
+// deterministic spawn schedule generator (waves model)
+static std::vector<double> generate_spawn_schedule(
+    int actors,
+    double total_duration_seconds,
+    int max_waves,
+    uint64_t master_seed,
+    int size /* used to vary seeds per trial */) {
+
+  std::mt19937_64 rng(master_seed ^ uint64_t(size));
+  std::uniform_real_distribution<double> time_dist(0.0, total_duration_seconds);
+  std::uniform_int_distribution<int> waves_dist(1, std::max(1, max_waves));
+  std::uniform_real_distribution<double> jitter_dist(0.0, 0.25 * total_duration_seconds);
+
+  int num_waves = waves_dist(rng);
+  std::vector<std::pair<double,int>> waves;
+  std::vector<int> wave_sizes(num_waves, 0);
+  int remaining = actors;
+  for (int w = 0; w < num_waves; ++w) {
+    if (w == num_waves - 1) {
+      wave_sizes[w] = remaining;
+    } else {
+      int max_allowed = std::max(1, remaining - (num_waves - w - 1));
+      std::uniform_int_distribution<int> size_dist(1, max_allowed);
+      int chosen = size_dist(rng);
+      wave_sizes[w] = chosen;
+      remaining -= chosen;
+    }
+  }
+  for (int w = 0; w < num_waves; ++w) {
+    double t = time_dist(rng);
+    waves.emplace_back(t, wave_sizes[w]);
+  }
 
+  std::vector<double> spawn_times;
+  for (int w = 0; w < num_waves; ++w) {
+    double wt = waves[w].first;
+    int wsize = waves[w].second;
+    for (int i = 0; i < wsize; ++i) {
+      double jitter = jitter_dist(rng) * ((i % 2 == 0) ? 1.0 : -1.0);
+      double st = wt + jitter;
+      if (st < 0.0) st = 0.0;
+      if (st > total_duration_seconds) st = total_duration_seconds;
+      spawn_times.push_back(st);
+    }
+  }
+  if ((int)spawn_times.size() > actors) {
+    spawn_times.resize(actors);
+  } else if ((int)spawn_times.size() < actors) {
+    std::uniform_real_distribution<double> extra_dist(0.0, total_duration_seconds);
+    while ((int)spawn_times.size() < actors)
+      spawn_times.push_back(extra_dist(rng));
+  }
+  std::sort(spawn_times.begin(), spawn_times.end());
+  return spawn_times;
+}
 
+// utility to pick indices deterministically for each actor
+static std::pair<int,int> choose_matrix_indices_for_actor(int actor_index, int pool_size, uint64_t master_seed, int size) {
+  std::mt19937_64 rng(master_seed ^ uint64_t(actor_index) ^ uint64_t(size << 16));
+  std::uniform_int_distribution<int> idx_dist(0, std::max(0, pool_size - 1));
+  int a = idx_dist(rng);
+  int b = idx_dist(rng);
+  return {a, b};
+}
 
+// spawn actors according to schedule (sleeps on caller thread)
+static void spawn_actors_with_schedule(caf::actor_system& sys,
+                                       const std::vector<double>& spawn_times,
+                                       std::function<void(int)> spawn_cb) {
+  auto t0 = std::chrono::steady_clock::now();
+  for (size_t i = 0; i < spawn_times.size(); ++i) {
+    double target = spawn_times[i];
+    auto now = std::chrono::steady_clock::now();
+    double elapsed = std::chrono::duration<double>(now - t0).count();
+    if (target > elapsed) {
+      std::this_thread::sleep_for(std::chrono::duration<double>(target - elapsed));
+    }
+    spawn_cb(int(i));
+  }
+}
 
+// --------------------------- Actor implementations (indexed pool) ---------------------------
 
+caf::behavior mmul_actor_indexed(caf::stateful_actor<mmul_state>* self,
+                                 caf::cuda::program_ptr mmul_kernel,
+                                 int indexA,
+                                 int indexB,
+                                 int N,
+                                 std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool_ptr) {
+  // store program ptr
+  self->state().mmul_kernel = mmul_kernel;
+  // enqueue work to self to keep consistency with message-driven design
+  self->mail(indexA, indexB, N, pool_ptr).send(self);
 
+  return {
+    [=](int idxA, int idxB, int N, std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool) {
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+      int device = 0;
+      int stream = 0;
 
+      const int THREADS = 32;
+      const int BLOCKS = (N + THREADS - 1) / THREADS;
+      caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
+      auto program = self->state().mmul_kernel;
 
+      auto matA_ptr = (*pool)[idxA];
+      auto matB_ptr = (*pool)[idxB];
 
+      auto inA = caf::cuda::create_in_arg(*matA_ptr);
+      auto arg1 = mmul_command.transfer_memory(device, stream, std::move(inA));
 
-void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
+      auto inB = caf::cuda::create_in_arg(*matB_ptr);
+      auto arg2 = mmul_command.transfer_memory(device, stream, std::move(inB));
 
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
+      auto arg3 = caf::cuda::create_out_arg<int>(N * N);
+      auto arg4 = caf::cuda::create_in_arg<int>(N);
 
-  //set the behaviors of each scheduler actor
-    for (int i = 0; i < mgr.get_num_devices();i++) {
-            mgr.send_scheduler_actor_message("multilevel",i);
+      auto result = async_mmul.run_async(program, dims, stream, 0, device, arg1, arg2, arg3, arg4);
+      std::get<2>(result)->copy_to_host();
+      self->quit();
     }
-
-
-  // CREATE ONCE
-  auto program =
-      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-  const int THREADS = 32;
-  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(
-      BLOCKS, BLOCKS, 1,
-      THREADS, THREADS, 1);
-
-  caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
-
-  for (int i = 0; i < num_actors; ++i) {
-    
-      sys.spawn(
-        mmul_actor_fun_scheduler,
-        exit_actor,
-        matrix_size,
-        program,
-        dims);
-    
-  }
-
-  sys.await_all_actors_done();
+  };
 }
 
-
-void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
+caf::behavior mmul_actor_scheduler_indexed(
+    caf::stateful_actor<mmul_actor_with_scheduler_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims,
+    int indexA,
+    int indexB,
+    std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool_ptr) {
 
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 
- 
-  //set the behaviors of each scheduler actor
-    for (int i = 0; i < mgr.get_num_devices();i++) {
-            mgr.send_scheduler_actor_message("green",i);
-    }  
- 
-   // CREATE ONCE
-  auto program =
-      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-  const int THREADS = 32;
-  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(
-      BLOCKS, BLOCKS, 1,
-      THREADS, THREADS, 1);
-
-  caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
-
-  for (int i = 0; i < num_actors; ++i) {
-    
-      sys.spawn(
-        mmul_actor_fun_scheduler,
-        exit_actor,
-        matrix_size,
-        program,
-        dims);
-    
-  }
-
- 
-
-  sys.await_all_actors_done();
-}
+  // request a launch token from scheduler
+  caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+                    program,
+                    dims,
+                    0,
+                    "mmul",
+                    self,
+                    rand());
+  mgr.send_scheduler_actor_message(launch_token);
+
+  // when we receive a launch response, scheduler will send a response_token which we handle below
+  return {
+    [=](caf::cuda::response_token_ptr res_token) {
+      if (res_token->getType() == LAUNCH_RESPONSE) {
+        // send the actual work to self
+        self->mail(indexA, indexB, res_token, N, pool_ptr).send(self);
+      }
+    },
 
-void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, int num_actors) {
-    if (num_actors < 1) {
-        std::cerr << "[ERROR] Number of actors must be >= 1\n";
-        return;
-    }
+    [=](int idxA,
+        int idxB,
+        const caf::cuda::response_token_ptr& res_token,
+        int N,
+        std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool) {
 
-    
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
+      auto matA_ptr = (*pool)[idxA];
+      auto matB_ptr = (*pool)[idxB];
 
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+      auto arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(*matA_ptr));
+      auto arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(*matB_ptr));
+      auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
+      auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
 
-    const int THREADS = 32;
-    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+      auto tempC = mmul.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
+      caf::cuda::mem_ptr<int> bufferC = std::get<2>(tempC);
 
+      bufferC->synchronize();
+      res_token->release();
+      bufferC->copy_to_host();
 
-    for (int i = 0; i < num_actors; ++i) {
-        sys.spawn(
-            mmul_actor_fun,
-            program,
-            matrix_size
-        );
+      self->mail(1).send(exit_actor);
+      self->quit();
     }
-
-    sys.await_all_actors_done();
+  };
 }
 
+// --------------------------- Spawn helpers (take spawn_times and shared pool_ptr) ---------------------------
+
+void spawn_mmul_actors_with_schedule_scheduler(
+    caf::actor_system& sys,
+    const std::vector<double>& spawn_times,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims,
+    std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool_ptr,
+    uint64_t master_seed) {
+
+  int pool_sz = (int)pool_ptr->size();
+  auto spawn_cb = [&](int actor_idx) {
+    auto inds = choose_matrix_indices_for_actor(actor_idx, pool_sz, master_seed, N);
+    int idxA = inds.first;
+    int idxB = inds.second;
+
+    sys.spawn(
+      mmul_actor_scheduler_indexed,
+      exit_actor,
+      N,
+      program,
+      dims,
+      idxA,
+      idxB,
+      pool_ptr
+    );
+  };
+
+  spawn_actors_with_schedule(sys, spawn_times, spawn_cb);
+}
 
+void spawn_mmul_actors_with_schedule_no_scheduler_actor(
+    caf::actor_system& sys,
+    const std::vector<double>& spawn_times,
+    int N,
+    caf::cuda::program_ptr program,
+    std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool_ptr,
+    uint64_t master_seed) {
+
+  int pool_sz = (int)pool_ptr->size();
+  auto spawn_cb = [&](int actor_idx) {
+    auto inds = choose_matrix_indices_for_actor(actor_idx, pool_sz, master_seed, N);
+    int idxA = inds.first;
+    int idxB = inds.second;
+
+    sys.spawn(
+      mmul_actor_indexed,
+      program,
+      idxA,
+      idxB,
+      N,
+      pool_ptr
+    );
+  };
+
+  spawn_actors_with_schedule(sys, spawn_times, spawn_cb);
+}
 
+// --------------------------- Test driver (integrates everything) ---------------------------
 
 template <class Fn>
 double time_run(Fn&& fn) {
@@ -329,101 +320,140 @@ double time_run(Fn&& fn) {
   std::chrono::duration<double> elapsed = end - start;
   return elapsed.count();
 }
-void run_mmul_scaling_tests(caf::actor_system& sys,
-                            caf::cuda::manager_config man_config) {
-    const int max_size   = 2048;
-    const int min_actors = 1;
-    const int max_actors = 1024;
-
-    std::vector<int> matrix_sizes = {10};
-    for (int s = 32; s <= max_size; s *= 2)
-        matrix_sizes.push_back(s);
-
-    std::vector<int> actor_counts;
-    for (int a = min_actors; a <= max_actors; a *= 2)
-        actor_counts.push_back(a);
-
-    std::cout << "=== MMUL Scaling Tests ===\n";
-    std::cout << "Format:\n";
-    std::cout << "scheduler matrix_size actors time_seconds\n";
-
-    for (int size : matrix_sizes) {
-        for (int actors : actor_counts) {
-
-            /* ================= Scheduler-enabled (core_usage) ================= */
-            caf::cuda::manager::init(sys, man_config); // scheduler enabled
-            std::cout << "\n[RUN] scheduler=multilevel_usage "
-                      << "matrix_size=" << size
-                      << " actors=" << actors << "\n";
-
-            double core_usage_time = time_run([&] {
-                run_mmul_test(sys, size, actors); // uses mmul_actor_fun_no_verify
-            });
-
-            std::cout << std::fixed << std::setprecision(6)
-                      << "RESULT core_usage "
-                      << size << " "
-                      << actors << " "
-                      << core_usage_time << "\n";
-
-            caf::cuda::manager::shutdown(); // make sure manager is cleaned up
-
-            /* ================= Scheduler-disabled actor ( uses green-light) ================= */
-
-            caf::cuda::manager::init(sys, man_config); // init with scheduler
-            std::cout << "\n[RUN] scheduler=green_light_only "
-                      << "matrix_size=" << size
-                      << " actors=" << actors << "\n";
-
-            double green_light_time = time_run([&] {
-                run_mmul_test_no_scheduler(sys, size, actors); // your previous "no scheduler" actor
-            });
-
-            std::cout << std::fixed << std::setprecision(6)
-                      << "RESULT green_light_only "
-                      << size << " "
-                      << actors << " "
-                      << green_light_time << "\n";
-
-            caf::cuda::manager::shutdown();
-
-            /* ================= No scheduler at all actor ================= */
-            caf::cuda::manager_config no_sched_config(false); // disable scheduler
-            caf::cuda::manager::init(sys, no_sched_config);
-            std::cout << "\n[RUN] scheduler=none "
-                      << "matrix_size=" << size
-                      << " actors=" << actors << "\n";
-
-            double no_scheduler_time = time_run([&] {
-                run_mmul_test_no_scheduler_actor(sys, size, actors); // mmul_actor_fun_no_schedule
-            });
-
-            std::cout << std::fixed << std::setprecision(6)
-                      << "RESULT none "
-                      << size << " "
-                      << actors << " "
-                      << no_scheduler_time << "\n";
-
-            caf::cuda::manager::shutdown();
-        }
-    }
 
-    std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
-}
+void run_mmul_scaling_tests(caf::actor_system& sys, caf::cuda::manager_config man_config) {
+  const int max_size   = 2048;
+  const int min_actors = 1;
+  const int max_actors = 1024;
 
+  std::vector<int> matrix_sizes = {10};
+  for (int s = 32; s <= max_size; s *= 2)
+    matrix_sizes.push_back(s);
 
+  std::vector<int> actor_counts;
+  for (int a = min_actors; a <= max_actors; a *= 2)
+    actor_counts.push_back(a);
 
+  std::cout << "=== MMUL Scaling Tests ===";
+  std::cout << "Format:";
+  std::cout << "scheduler matrix_size actors time_seconds";
 
-void caf_main(caf::actor_system& sys) {
-  
+  uint64_t master_seed = 0xDEADBEEF1234ULL;
+  auto pools_map = prepare_matrix_pools(matrix_sizes, /*pool_size_per_size=*/32, master_seed);
 
-  caf::cuda::manager_config man_config(true);
-  //caf::cuda::manager::init(sys,man_config);
-  run_mmul_scaling_tests(sys,man_config);
+  for (int size : matrix_sizes) {
+    for (int actors : actor_counts) {
 
-}
+      // Prepare deterministic spawn_times once per (size, actors)
+      double total_duration = 5.0; // seconds (tunable)
+      int max_waves = 6;
+      auto spawn_times = generate_spawn_schedule(actors, total_duration, max_waves, master_seed, size);
 
+      // make a shared_ptr to the pool for this size so we can cheaply pass it to actors
+      auto pool_vec = pools_map[size];
+      auto pool_shared = std::make_shared<std::vector<std::shared_ptr<std::vector<int>>>>(std::move(pool_vec));
 
+      // ================= Scheduler-enabled (core_usage) =================
+      caf::cuda::manager::init(sys, man_config);
+      {
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
+        // set scheduler behavior per device
+        for (int i = 0; i < mgr.get_num_devices(); i++)
+          mgr.send_scheduler_actor_message("multilevel", i);
+
+        std::cout << "[RUN] scheduler=multilevel_usage \n"
+                  << "matrix_size=" << size
+                  << " actors=" << actors << "";
+
+        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        const int THREADS = 32;
+        const int BLOCKS = (size + THREADS - 1) / THREADS;
+        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+        caf::actor exit_actor = mgr.spawn_exit_actor(actors);
+
+        double core_usage_time = time_run([&] {
+          spawn_mmul_actors_with_schedule_scheduler(sys, spawn_times, exit_actor, size, program, dims, pool_shared, master_seed);
+          sys.await_all_actors_done();
+        });
+
+        std::cout << std::fixed << std::setprecision(6)
+                  << "RESULT core_usage \n"
+                  << size << " \n"
+                  << actors << " \n"
+                  << core_usage_time << "\n";
+      }
+      caf::cuda::manager::shutdown();
 
+      // ================= Scheduler-disabled actor (green-light) =================
+      caf::cuda::manager::init(sys, man_config);
+      {
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
+        for (int i = 0; i < mgr.get_num_devices(); i++)
+          mgr.send_scheduler_actor_message("green", i);
+
+        std::cout << "[RUN] scheduler=green_light_only \n"
+                  << "matrix_size=" << size
+                  << " actors=" << actors << "\n";
+
+        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        const int THREADS = 32;
+        const int BLOCKS = (size + THREADS - 1) / THREADS;
+        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+        caf::actor exit_actor = mgr.spawn_exit_actor(actors);
+
+        double green_light_time = time_run([&] {
+          // reuse the same spawn_times and pool_shared
+          spawn_mmul_actors_with_schedule_scheduler(sys, spawn_times, exit_actor, size, program, dims, pool_shared, master_seed);
+          sys.await_all_actors_done();
+        });
+
+        std::cout << std::fixed << std::setprecision(6)
+                  << "RESULT green_light_only \n"
+                  << size << " \n"
+                  << actors << " \n"
+                  << green_light_time << "\n";
+      }
+      caf::cuda::manager::shutdown();
+
+      // ================= No scheduler at all actor =================
+      caf::cuda::manager_config no_sched_config(false);
+      caf::cuda::manager::init(sys, no_sched_config);
+      {
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+        std::cout <<"[RUN] scheduler=none \n"
+                  << "matrix_size=" << size
+                  << " actors=" << actors << "\n";
+
+        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        const int THREADS = 32;
+        const int BLOCKS = (size + THREADS - 1) / THREADS;
+        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+        double no_scheduler_time = time_run([&] {
+          spawn_mmul_actors_with_schedule_no_scheduler_actor(sys, spawn_times, size, program, pool_shared, master_seed);
+          sys.await_all_actors_done();
+        });
+
+        std::cout << std::fixed << std::setprecision(6)
+                  << "RESULT none \n"
+                  << size << " \n"
+                  << actors << " \n"
+                  << no_scheduler_time << "";
+      }
+      caf::cuda::manager::shutdown();
+    }
+  }
+
+  std::cout << "=== MMUL Scaling Tests Complete ===";
+}
+
+void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager_config man_config(true);
+  run_mmul_scaling_tests(sys, man_config);
+}
 
 CAF_MAIN()
+

From bf12764674b53472660ef178c1b2c526f5d8fa4e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Mar 2026 14:43:24 -0600
Subject: [PATCH 0512/1000] Saving.

---
 .../main.test.cpp                             | 182 +++++++++++-------
 1 file changed, 111 insertions(+), 71 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
index 32c50dc301..a554245632 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
@@ -10,6 +10,7 @@
 #include <numeric>
 #include <random>
 #include <map>
+#include <unordered_map>
 #include <functional>
 #include <memory>
 #include <mutex>
@@ -23,8 +24,11 @@ using command = caf::cuda::command_runner<>;
 command mmul_command;
 using async_command = caf::cuda::mmul_async_command<int>;
 async_command async_mmul;
-caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int >> mmul;
-
+// specialized command runner used for scheduler-mode memory operations:
+caf::cuda::command_runner<caf::cuda::mem_ptr<int>,
+                         caf::cuda::mem_ptr<int>,
+                         caf::cuda::mem_ptr<int>,
+                         caf::cuda::mem_ptr<int>> mmul;
 
 struct mmul_state {
   caf::cuda::program_ptr mmul_kernel;
@@ -34,7 +38,13 @@ struct mmul_actor_with_scheduler_state {
   static inline const char* name = "my_actor";
 };
 
-// --------------------------- Deterministic matrix generation & pools (shared pool, pass indices) ---------------------------
+// --------------------------- Global matrix pools (indexed access) ---------------------------
+// Key: matrix size N -> pool of shared_ptr<vector<int>> for that N.
+// Actors will look up matrices via g_pools[N] and receive only small integer indices in messages.
+static std::unordered_map<int, std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>>> g_pools;
+static std::mutex g_pools_mutex;
+
+// --------------------------- Deterministic matrix generation & pools ---------------------------
 static std::vector<int> generate_matrix(int N, uint64_t seed) {
   std::mt19937_64 rng(seed);
   std::uniform_int_distribution<int> dist(0, 9);
@@ -148,21 +158,20 @@ static void spawn_actors_with_schedule(caf::actor_system& sys,
   }
 }
 
-// --------------------------- Actor implementations (indexed pool) ---------------------------
+// --------------------------- Actor implementations (indexed pool, pool looked up from global) ---------------------------
 
 caf::behavior mmul_actor_indexed(caf::stateful_actor<mmul_state>* self,
                                  caf::cuda::program_ptr mmul_kernel,
                                  int indexA,
                                  int indexB,
-                                 int N,
-                                 std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool_ptr) {
+                                 int N) {
   // store program ptr
   self->state().mmul_kernel = mmul_kernel;
-  // enqueue work to self to keep consistency with message-driven design
-  self->mail(indexA, indexB, N, pool_ptr).send(self);
+  // enqueue work to self (do not send pool pointer — actors will look up global pool by N)
+  self->mail(indexA, indexB, N).send(self);
 
   return {
-    [=](int idxA, int idxB, int N, std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool) {
+    [=](int idxA, int idxB, int N) {
       caf::cuda::manager& mgr = caf::cuda::manager::get();
       int device = 0;
       int stream = 0;
@@ -173,6 +182,19 @@ caf::behavior mmul_actor_indexed(caf::stateful_actor<mmul_state>* self,
 
       auto program = self->state().mmul_kernel;
 
+      // lookup pool globally
+      std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool;
+      {
+        std::lock_guard<std::mutex> lk(g_pools_mutex);
+        auto it = g_pools.find(N);
+        if (it == g_pools.end()) {
+          std::cout  << "ERROR: no pool for N=" << N << "\n";
+          self->quit();
+          return;
+        }
+        pool = it->second;
+      }
+
       auto matA_ptr = (*pool)[idxA];
       auto matB_ptr = (*pool)[idxB];
 
@@ -199,8 +221,7 @@ caf::behavior mmul_actor_scheduler_indexed(
     caf::cuda::program_ptr program,
     caf::cuda::nd_range dims,
     int indexA,
-    int indexB,
-    std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool_ptr) {
+    int indexB) {
 
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 
@@ -218,16 +239,28 @@ caf::behavior mmul_actor_scheduler_indexed(
   return {
     [=](caf::cuda::response_token_ptr res_token) {
       if (res_token->getType() == LAUNCH_RESPONSE) {
-        // send the actual work to self
-        self->mail(indexA, indexB, res_token, N, pool_ptr).send(self);
+        // send the actual work to self (only indices and token, no pool pointer)
+        self->mail(indexA, indexB, res_token, N).send(self);
       }
     },
 
     [=](int idxA,
         int idxB,
         const caf::cuda::response_token_ptr& res_token,
-        int N,
-        std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool) {
+        int N) {
+
+      // lookup pool globally
+      std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool;
+      {
+        std::lock_guard<std::mutex> lk(g_pools_mutex);
+        auto it = g_pools.find(N);
+        if (it == g_pools.end()) {
+	  std::cout << "ERROR: no pool for N=" << N << "\n";
+          self->quit();
+          return;
+        }
+        pool = it->second;
+      }
 
       auto matA_ptr = (*pool)[idxA];
       auto matB_ptr = (*pool)[idxB];
@@ -250,7 +283,7 @@ caf::behavior mmul_actor_scheduler_indexed(
   };
 }
 
-// --------------------------- Spawn helpers (take spawn_times and shared pool_ptr) ---------------------------
+// --------------------------- Spawn helpers (take spawn_times) ---------------------------
 
 void spawn_mmul_actors_with_schedule_scheduler(
     caf::actor_system& sys,
@@ -259,25 +292,25 @@ void spawn_mmul_actors_with_schedule_scheduler(
     int N,
     caf::cuda::program_ptr program,
     caf::cuda::nd_range dims,
-    std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool_ptr,
     uint64_t master_seed) {
 
-  int pool_sz = (int)pool_ptr->size();
+  // pool must already exist in g_pools[N]
+  int pool_sz = 0;
+  {
+    std::lock_guard<std::mutex> lk(g_pools_mutex);
+    auto it = g_pools.find(N);
+    if (it == g_pools.end()) return;
+    pool_sz = (int)it->second->size();
+  }
+
   auto spawn_cb = [&](int actor_idx) {
     auto inds = choose_matrix_indices_for_actor(actor_idx, pool_sz, master_seed, N);
     int idxA = inds.first;
     int idxB = inds.second;
-
-    sys.spawn(
-      mmul_actor_scheduler_indexed,
-      exit_actor,
-      N,
-      program,
-      dims,
-      idxA,
-      idxB,
-      pool_ptr
-    );
+    // To avoid ambiguity with CAF spawn overloads we call the spawn via a lambda:
+    sys.spawn([=](caf::stateful_actor<mmul_actor_with_scheduler_state>* s) -> caf::behavior {
+        return mmul_actor_scheduler_indexed(s, exit_actor, N, program, dims, idxA, idxB);
+    });
   };
 
   spawn_actors_with_schedule(sys, spawn_times, spawn_cb);
@@ -288,23 +321,25 @@ void spawn_mmul_actors_with_schedule_no_scheduler_actor(
     const std::vector<double>& spawn_times,
     int N,
     caf::cuda::program_ptr program,
-    std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool_ptr,
     uint64_t master_seed) {
 
-  int pool_sz = (int)pool_ptr->size();
+  int pool_sz = 0;
+  {
+    std::lock_guard<std::mutex> lk(g_pools_mutex);
+    auto it = g_pools.find(N);
+    if (it == g_pools.end()) return;
+    pool_sz = (int)it->second->size();
+  }
+
   auto spawn_cb = [&](int actor_idx) {
     auto inds = choose_matrix_indices_for_actor(actor_idx, pool_sz, master_seed, N);
     int idxA = inds.first;
     int idxB = inds.second;
 
-    sys.spawn(
-      mmul_actor_indexed,
-      program,
-      idxA,
-      idxB,
-      N,
-      pool_ptr
-    );
+    // spawn mmul_actor_indexed using a lambda to bind arguments (avoids CAF type registration issues)
+    sys.spawn([=](caf::stateful_actor<mmul_state>* s) -> caf::behavior {
+      return mmul_actor_indexed(s, program, idxA, idxB, N);
+    });
   };
 
   spawn_actors_with_schedule(sys, spawn_times, spawn_cb);
@@ -334,13 +369,23 @@ void run_mmul_scaling_tests(caf::actor_system& sys, caf::cuda::manager_config ma
   for (int a = min_actors; a <= max_actors; a *= 2)
     actor_counts.push_back(a);
 
-  std::cout << "=== MMUL Scaling Tests ===";
-  std::cout << "Format:";
-  std::cout << "scheduler matrix_size actors time_seconds";
+  std::cout << "=== MMUL Scaling Tests ===\n";
+  std::cout << "Format:\n";
+  std::cout << "scheduler matrix_size actors time_seconds\n";
 
   uint64_t master_seed = 0xDEADBEEF1234ULL;
   auto pools_map = prepare_matrix_pools(matrix_sizes, /*pool_size_per_size=*/32, master_seed);
 
+  // move pools into global map so actors can read them by N without sending them in messages
+  {
+    std::lock_guard<std::mutex> lk(g_pools_mutex);
+    for (auto& kv : pools_map) {
+      int N = kv.first;
+      auto vec = std::move(kv.second);
+      g_pools[N] = std::make_shared<std::vector<std::shared_ptr<std::vector<int>>>>(std::move(vec));
+    }
+  }
+
   for (int size : matrix_sizes) {
     for (int actors : actor_counts) {
 
@@ -349,10 +394,6 @@ void run_mmul_scaling_tests(caf::actor_system& sys, caf::cuda::manager_config ma
       int max_waves = 6;
       auto spawn_times = generate_spawn_schedule(actors, total_duration, max_waves, master_seed, size);
 
-      // make a shared_ptr to the pool for this size so we can cheaply pass it to actors
-      auto pool_vec = pools_map[size];
-      auto pool_shared = std::make_shared<std::vector<std::shared_ptr<std::vector<int>>>>(std::move(pool_vec));
-
       // ================= Scheduler-enabled (core_usage) =================
       caf::cuda::manager::init(sys, man_config);
       {
@@ -361,9 +402,9 @@ void run_mmul_scaling_tests(caf::actor_system& sys, caf::cuda::manager_config ma
         for (int i = 0; i < mgr.get_num_devices(); i++)
           mgr.send_scheduler_actor_message("multilevel", i);
 
-        std::cout << "[RUN] scheduler=multilevel_usage \n"
-                  << "matrix_size=" << size
-                  << " actors=" << actors << "";
+        std::cout << "[RUN] scheduler=multilevel_usage "
+                  << "matrix_size=" << size << " "
+                  << "actors=" << actors << "\n";
 
         auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
         const int THREADS = 32;
@@ -373,14 +414,14 @@ void run_mmul_scaling_tests(caf::actor_system& sys, caf::cuda::manager_config ma
         caf::actor exit_actor = mgr.spawn_exit_actor(actors);
 
         double core_usage_time = time_run([&] {
-          spawn_mmul_actors_with_schedule_scheduler(sys, spawn_times, exit_actor, size, program, dims, pool_shared, master_seed);
+          spawn_mmul_actors_with_schedule_scheduler(sys, spawn_times, exit_actor, size, program, dims, master_seed);
           sys.await_all_actors_done();
         });
 
         std::cout << std::fixed << std::setprecision(6)
-                  << "RESULT core_usage \n"
-                  << size << " \n"
-                  << actors << " \n"
+                  << "RESULT core_usage "
+                  << size << " "
+                  << actors << " "
                   << core_usage_time << "\n";
       }
       caf::cuda::manager::shutdown();
@@ -392,9 +433,9 @@ void run_mmul_scaling_tests(caf::actor_system& sys, caf::cuda::manager_config ma
         for (int i = 0; i < mgr.get_num_devices(); i++)
           mgr.send_scheduler_actor_message("green", i);
 
-        std::cout << "[RUN] scheduler=green_light_only \n"
-                  << "matrix_size=" << size
-                  << " actors=" << actors << "\n";
+        std::cout << "[RUN] scheduler=green_light_only "
+                  << "matrix_size=" << size << " "
+                  << "actors=" << actors << "\n";
 
         auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
         const int THREADS = 32;
@@ -404,15 +445,15 @@ void run_mmul_scaling_tests(caf::actor_system& sys, caf::cuda::manager_config ma
         caf::actor exit_actor = mgr.spawn_exit_actor(actors);
 
         double green_light_time = time_run([&] {
-          // reuse the same spawn_times and pool_shared
-          spawn_mmul_actors_with_schedule_scheduler(sys, spawn_times, exit_actor, size, program, dims, pool_shared, master_seed);
+          // reuse the same spawn_times and global pools
+          spawn_mmul_actors_with_schedule_scheduler(sys, spawn_times, exit_actor, size, program, dims, master_seed);
           sys.await_all_actors_done();
         });
 
         std::cout << std::fixed << std::setprecision(6)
-                  << "RESULT green_light_only \n"
-                  << size << " \n"
-                  << actors << " \n"
+                  << "RESULT green_light_only "
+                  << size << " "
+                  << actors << " "
                   << green_light_time << "\n";
       }
       caf::cuda::manager::shutdown();
@@ -423,9 +464,9 @@ void run_mmul_scaling_tests(caf::actor_system& sys, caf::cuda::manager_config ma
       {
         caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-        std::cout <<"[RUN] scheduler=none \n"
-                  << "matrix_size=" << size
-                  << " actors=" << actors << "\n";
+        std::cout << "[RUN] scheduler=none "
+                  << "matrix_size=" << size << " "
+                  << "actors=" << actors << "\n";
 
         auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
         const int THREADS = 32;
@@ -433,21 +474,21 @@ void run_mmul_scaling_tests(caf::actor_system& sys, caf::cuda::manager_config ma
         caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
         double no_scheduler_time = time_run([&] {
-          spawn_mmul_actors_with_schedule_no_scheduler_actor(sys, spawn_times, size, program, pool_shared, master_seed);
+          spawn_mmul_actors_with_schedule_no_scheduler_actor(sys, spawn_times, size, program, master_seed);
           sys.await_all_actors_done();
         });
 
         std::cout << std::fixed << std::setprecision(6)
-                  << "RESULT none \n"
-                  << size << " \n"
-                  << actors << " \n"
-                  << no_scheduler_time << "";
+                  << "RESULT none "
+                  << size << " "
+                  << actors << " "
+                  << no_scheduler_time << "\n";
       }
       caf::cuda::manager::shutdown();
     }
   }
 
-  std::cout << "=== MMUL Scaling Tests Complete ===";
+  std::cout << "=== MMUL Scaling Tests Complete ===\n";
 }
 
 void caf_main(caf::actor_system& sys) {
@@ -456,4 +497,3 @@ void caf_main(caf::actor_system& sys) {
 }
 
 CAF_MAIN()
-

From 1c893b3b02b93541f4a545554f20e6ffde71a6df Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Mar 2026 15:41:05 -0600
Subject: [PATCH 0513/1000] Wrote a static varying batch size of 10000 actors.
 Change was made to not over-enginer the problem.

---
 .../main.test.cpp                             | 712 ++++++++----------
 1 file changed, 309 insertions(+), 403 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
index a554245632..e33f4fae65 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
@@ -9,343 +9,221 @@
 #include <algorithm>
 #include <numeric>
 #include <random>
-#include <map>
-#include <unordered_map>
-#include <functional>
-#include <memory>
-#include <mutex>
 #include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
 
 using namespace caf;
 using namespace std::chrono_literals;
 
-// --------------------------- Types / globals ---------------------------
-using command = caf::cuda::command_runner<>;
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+using command =
+  caf::cuda::command_runner<>;
+
 command mmul_command;
+caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int >> mmul;
 using async_command = caf::cuda::mmul_async_command<int>;
 async_command async_mmul;
-// specialized command runner used for scheduler-mode memory operations:
-caf::cuda::command_runner<caf::cuda::mem_ptr<int>,
-                         caf::cuda::mem_ptr<int>,
-                         caf::cuda::mem_ptr<int>,
-                         caf::cuda::mem_ptr<int>> mmul;
 
-struct mmul_state {
-  caf::cuda::program_ptr mmul_kernel;
-};
 
-struct mmul_actor_with_scheduler_state {
-  static inline const char* name = "my_actor";
+
+
+
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
 };
 
-// --------------------------- Global matrix pools (indexed access) ---------------------------
-// Key: matrix size N -> pool of shared_ptr<vector<int>> for that N.
-// Actors will look up matrices via g_pools[N] and receive only small integer indices in messages.
-static std::unordered_map<int, std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>>> g_pools;
-static std::mutex g_pools_mutex;
-
-// --------------------------- Deterministic matrix generation & pools ---------------------------
-static std::vector<int> generate_matrix(int N, uint64_t seed) {
-  std::mt19937_64 rng(seed);
-  std::uniform_int_distribution<int> dist(0, 9);
-  std::vector<int> mat;
-  mat.resize((size_t)N * N);
-  for (size_t i = 0; i < mat.size(); ++i)
-    mat[i] = dist(rng);
-  return mat;
-}
+MatrixPool create_matrix_pool(const std::vector<int>& sizes) {
+    MatrixPool pool;
 
-// returns map<size, vector<shared_ptr<vector<int>>>>
-static std::map<int, std::vector<std::shared_ptr<std::vector<int>>>> prepare_matrix_pools(
-    const std::vector<int>& sizes,
-    int pool_size_per_size,
-    uint64_t master_seed) {
-
-  std::map<int, std::vector<std::shared_ptr<std::vector<int>>>> pools;
-  for (int N : sizes) {
-    int pool_sz = pool_size_per_size;
-    if (N >= 1024) pool_sz = std::min(pool_sz, 8);
-    if (N >= 2048) pool_sz = std::min(pool_sz, 4);
-
-    pools[N] = {};
-    for (int i = 0; i < pool_sz; ++i) {
-      uint64_t seed = master_seed ^ (uint64_t(N) << 32) ^ (uint64_t(i) * 0x9e3779b97f4a7c15ULL);
-      auto v = std::make_shared<std::vector<int>>(generate_matrix(N, seed));
-      pools[N].push_back(std::move(v));
+    for (int N : sizes) {
+        pool.A[N] = std::vector<int>(N*N, 1);
+        pool.B[N] = std::vector<int>(N*N, 1);
     }
-  }
-  return pools;
+
+    return pool;
 }
 
-// deterministic spawn schedule generator (waves model)
-static std::vector<double> generate_spawn_schedule(
-    int actors,
-    double total_duration_seconds,
-    int max_waves,
-    uint64_t master_seed,
-    int size /* used to vary seeds per trial */) {
-
-  std::mt19937_64 rng(master_seed ^ uint64_t(size));
-  std::uniform_real_distribution<double> time_dist(0.0, total_duration_seconds);
-  std::uniform_int_distribution<int> waves_dist(1, std::max(1, max_waves));
-  std::uniform_real_distribution<double> jitter_dist(0.0, 0.25 * total_duration_seconds);
-
-  int num_waves = waves_dist(rng);
-  std::vector<std::pair<double,int>> waves;
-  std::vector<int> wave_sizes(num_waves, 0);
-  int remaining = actors;
-  for (int w = 0; w < num_waves; ++w) {
-    if (w == num_waves - 1) {
-      wave_sizes[w] = remaining;
-    } else {
-      int max_allowed = std::max(1, remaining - (num_waves - w - 1));
-      std::uniform_int_distribution<int> size_dist(1, max_allowed);
-      int chosen = size_dist(rng);
-      wave_sizes[w] = chosen;
-      remaining -= chosen;
-    }
-  }
-  for (int w = 0; w < num_waves; ++w) {
-    double t = time_dist(rng);
-    waves.emplace_back(t, wave_sizes[w]);
-  }
 
-  std::vector<double> spawn_times;
-  for (int w = 0; w < num_waves; ++w) {
-    double wt = waves[w].first;
-    int wsize = waves[w].second;
-    for (int i = 0; i < wsize; ++i) {
-      double jitter = jitter_dist(rng) * ((i % 2 == 0) ? 1.0 : -1.0);
-      double st = wt + jitter;
-      if (st < 0.0) st = 0.0;
-      if (st > total_duration_seconds) st = total_duration_seconds;
-      spawn_times.push_back(st);
-    }
-  }
-  if ((int)spawn_times.size() > actors) {
-    spawn_times.resize(actors);
-  } else if ((int)spawn_times.size() < actors) {
-    std::uniform_real_distribution<double> extra_dist(0.0, total_duration_seconds);
-    while ((int)spawn_times.size() < actors)
-      spawn_times.push_back(extra_dist(rng));
-  }
-  std::sort(spawn_times.begin(), spawn_times.end());
-  return spawn_times;
-}
 
-// utility to pick indices deterministically for each actor
-static std::pair<int,int> choose_matrix_indices_for_actor(int actor_index, int pool_size, uint64_t master_seed, int size) {
-  std::mt19937_64 rng(master_seed ^ uint64_t(actor_index) ^ uint64_t(size << 16));
-  std::uniform_int_distribution<int> idx_dist(0, std::max(0, pool_size - 1));
-  int a = idx_dist(rng);
-  int b = idx_dist(rng);
-  return {a, b};
-}
 
-// spawn actors according to schedule (sleeps on caller thread)
-static void spawn_actors_with_schedule(caf::actor_system& sys,
-                                       const std::vector<double>& spawn_times,
-                                       std::function<void(int)> spawn_cb) {
-  auto t0 = std::chrono::steady_clock::now();
-  for (size_t i = 0; i < spawn_times.size(); ++i) {
-    double target = spawn_times[i];
-    auto now = std::chrono::steady_clock::now();
-    double elapsed = std::chrono::duration<double>(now - t0).count();
-    if (target > elapsed) {
-      std::this_thread::sleep_for(std::chrono::duration<double>(target - elapsed));
-    }
-    spawn_cb(int(i));
-  }
-}
 
-// --------------------------- Actor implementations (indexed pool, pool looked up from global) ---------------------------
-
-caf::behavior mmul_actor_indexed(caf::stateful_actor<mmul_state>* self,
-                                 caf::cuda::program_ptr mmul_kernel,
-                                 int indexA,
-                                 int indexB,
-                                 int N) {
-  // store program ptr
-  self->state().mmul_kernel = mmul_kernel;
-  // enqueue work to self (do not send pool pointer — actors will look up global pool by N)
-  self->mail(indexA, indexB, N).send(self);
-
-  return {
-    [=](int idxA, int idxB, int N) {
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
-      int device = 0;
-      int stream = 0;
-
-      const int THREADS = 32;
-      const int BLOCKS = (N + THREADS - 1) / THREADS;
-      caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-      auto program = self->state().mmul_kernel;
-
-      // lookup pool globally
-      std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool;
-      {
-        std::lock_guard<std::mutex> lk(g_pools_mutex);
-        auto it = g_pools.find(N);
-        if (it == g_pools.end()) {
-          std::cout  << "ERROR: no pool for N=" << N << "\n";
-          self->quit();
-          return;
-        }
-        pool = it->second;
-      }
 
-      auto matA_ptr = (*pool)[idxA];
-      auto matB_ptr = (*pool)[idxB];
 
-      auto inA = caf::cuda::create_in_arg(*matA_ptr);
-      auto arg1 = mmul_command.transfer_memory(device, stream, std::move(inA));
 
-      auto inB = caf::cuda::create_in_arg(*matB_ptr);
-      auto arg2 = mmul_command.transfer_memory(device, stream, std::move(inB));
 
-      auto arg3 = caf::cuda::create_out_arg<int>(N * N);
-      auto arg4 = caf::cuda::create_in_arg<int>(N);
+struct mmul_state {
 
-      auto result = async_mmul.run_async(program, dims, stream, 0, device, arg1, arg2, arg3, arg4);
-      std::get<2>(result)->copy_to_host();
-      self->quit();
-    }
-  };
+	caf::cuda::program_ptr mmul_kernel;
+
+};
+
+
+
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::program_ptr mmul_kernel,
+		const std::vector<int>& matrix1,
+		const std::vector<int>& matrix2,
+		int N) {
+
+	self->state().mmul_kernel = mmul_kernel;
+	self->mail(matrix1, matrix2, N).send(self);
+	return {
+		[=](const std::vector<int>& matrixA,
+				const std::vector<int>& matrixB,
+				int N) {
+
+			caf::cuda::manager& mgr = caf::cuda::manager::get();
+			int device = 0;
+			int stream = rand();
+
+			//auto program =
+			//mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+		    const int THREADS = 32;
+		    const int BLOCKS = (N + THREADS - 1) / THREADS;
+		    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+
+			auto program = self->state().mmul_kernel;
+
+			auto inA = caf::cuda::create_in_arg(std::move(matrixA));
+			auto arg1 = mmul_command.transfer_memory(
+					device,
+					stream,
+					std::move(inA));
+
+			auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+			auto arg2 = mmul_command.transfer_memory(
+					device,
+					stream,
+					std::move(inB));
+
+			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
+			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
+
+			auto result = 
+				async_mmul.run_async(
+						program,
+						dims,
+						stream,
+						0,
+						device,
+						arg1,
+						arg2,
+						arg3,
+						arg4);
+
+			std::get<2>(result) -> copy_to_host();
+			self -> quit();
+
+		}
+	};
 }
 
-caf::behavior mmul_actor_scheduler_indexed(
+
+
+
+struct mmul_actor_with_scheduler_state {
+  static inline const char* name = "my_actor";
+};
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun_scheduler(
     caf::stateful_actor<mmul_actor_with_scheduler_state>* self,
     caf::actor exit_actor,
     int N,
     caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims,
-    int indexA,
-    int indexB) {
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  // request a launch token from scheduler
-  caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-                    program,
-                    dims,
-                    0,
-                    "mmul",
-                    self,
-                    rand());
-  mgr.send_scheduler_actor_message(launch_token);
-
-  // when we receive a launch response, scheduler will send a response_token which we handle below
-  return {
-    [=](caf::cuda::response_token_ptr res_token) {
-      if (res_token->getType() == LAUNCH_RESPONSE) {
-        // send the actual work to self (only indices and token, no pool pointer)
-        self->mail(indexA, indexB, res_token, N).send(self);
-      }
-    },
-
-    [=](int idxA,
-        int idxB,
-        const caf::cuda::response_token_ptr& res_token,
-        int N) {
-
-      // lookup pool globally
-      std::shared_ptr<std::vector<std::shared_ptr<std::vector<int>>>> pool;
-      {
-        std::lock_guard<std::mutex> lk(g_pools_mutex);
-        auto it = g_pools.find(N);
-        if (it == g_pools.end()) {
-	  std::cout << "ERROR: no pool for N=" << N << "\n";
-          self->quit();
-          return;
-        }
-        pool = it->second;
-      }
+    caf::cuda::nd_range dims
+    const std::vector<int>& matrix1,
+    const std::vector<int> & matrix2)
+{
 
-      auto matA_ptr = (*pool)[idxA];
-      auto matB_ptr = (*pool)[idxB];
 
-      auto arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(*matA_ptr));
-      auto arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(*matB_ptr));
-      auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
-      auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-      auto tempC = mmul.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
-      caf::cuda::mem_ptr<int> bufferC = std::get<2>(tempC);
+        //caf::actor scheduler = mgr.get_scheduler_actor();
 
-      bufferC->synchronize();
-      res_token->release();
-      bufferC->copy_to_host();
+        //send a launch token
+        caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+                        program,
+                        dims,
+                        0,
+                        "hello",
+                        self,
+			rand() //dependency number, can declare indepedent but want to see what happens when you do not 
+                        );
+        mgr.send_scheduler_message(launch_token);
 
-      self->mail(1).send(exit_actor);
-      self->quit();
-    }
-  };
-}
+	return {
 
-// --------------------------- Spawn helpers (take spawn_times) ---------------------------
+		// 1. Handle response token
+		[=](caf::cuda::response_token_ptr res_token) {
+			//        std::cout << "Got response\n";
 
-void spawn_mmul_actors_with_schedule_scheduler(
-    caf::actor_system& sys,
-    const std::vector<double>& spawn_times,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims,
-    uint64_t master_seed) {
-
-  // pool must already exist in g_pools[N]
-  int pool_sz = 0;
-  {
-    std::lock_guard<std::mutex> lk(g_pools_mutex);
-    auto it = g_pools.find(N);
-    if (it == g_pools.end()) return;
-    pool_sz = (int)it->second->size();
-  }
+			if (res_token->getType() == LAUNCH_RESPONSE) {
+				self->mail(matrix1, matrix2, res_token, N).send(self);
 
-  auto spawn_cb = [&](int actor_idx) {
-    auto inds = choose_matrix_indices_for_actor(actor_idx, pool_sz, master_seed, N);
-    int idxA = inds.first;
-    int idxB = inds.second;
-    // To avoid ambiguity with CAF spawn overloads we call the spawn via a lambda:
-    sys.spawn([=](caf::stateful_actor<mmul_actor_with_scheduler_state>* s) -> caf::behavior {
-        return mmul_actor_scheduler_indexed(s, exit_actor, N, program, dims, idxA, idxB);
-    });
-  };
-
-  spawn_actors_with_schedule(sys, spawn_times, spawn_cb);
-}
+			} else {
+				//          std::cout << "Got a memory response token\n";
+			}
+		},
 
-void spawn_mmul_actors_with_schedule_no_scheduler_actor(
-    caf::actor_system& sys,
-    const std::vector<double>& spawn_times,
-    int N,
-    caf::cuda::program_ptr program,
-    uint64_t master_seed) {
-
-  int pool_sz = 0;
-  {
-    std::lock_guard<std::mutex> lk(g_pools_mutex);
-    auto it = g_pools.find(N);
-    if (it == g_pools.end()) return;
-    pool_sz = (int)it->second->size();
-  }
+	    // 2. Handle memory buffers -> GPU
+	    [=](const std::vector<int>& matrixA,
+			    const std::vector<int>& matrixB,
+			    const caf::cuda::response_token_ptr& res_token,
+			    int N) {
+
+		    //    std::cout << "Working\n";
+		    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+
+		    const int THREADS = 32;
+		    const int BLOCKS = (N + THREADS - 1) / THREADS;
+		    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+		    auto arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixA));
+		    auto arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixB));
+		    auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
+		    auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
 
-  auto spawn_cb = [&](int actor_idx) {
-    auto inds = choose_matrix_indices_for_actor(actor_idx, pool_sz, master_seed, N);
-    int idxA = inds.first;
-    int idxB = inds.second;
 
-    // spawn mmul_actor_indexed using a lambda to bind arguments (avoids CAF type registration issues)
-    sys.spawn([=](caf::stateful_actor<mmul_state>* s) -> caf::behavior {
-      return mmul_actor_indexed(s, program, idxA, idxB, N);
-    });
-  };
+		    auto tempC = mmul.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
+		    caf::cuda::mem_ptr<int> bufferC = std::get<2>(tempC);
+
+		    bufferC -> synchronize();
+		    res_token->release();
+		    bufferC->copy_to_host();
+
+		    self->mail(1).send(exit_actor);
+		    self->quit();
+         }
+
+
+       };
 
-  spawn_actors_with_schedule(sys, spawn_times, spawn_cb);
 }
 
-// --------------------------- Test driver (integrates everything) ---------------------------
 
 template <class Fn>
 double time_run(Fn&& fn) {
@@ -356,144 +234,172 @@ double time_run(Fn&& fn) {
   return elapsed.count();
 }
 
-void run_mmul_scaling_tests(caf::actor_system& sys, caf::cuda::manager_config man_config) {
-  const int max_size   = 2048;
-  const int min_actors = 1;
-  const int max_actors = 1024;
-
-  std::vector<int> matrix_sizes = {10};
-  for (int s = 32; s <= max_size; s *= 2)
-    matrix_sizes.push_back(s);
-
-  std::vector<int> actor_counts;
-  for (int a = min_actors; a <= max_actors; a *= 2)
-    actor_counts.push_back(a);
-
-  std::cout << "=== MMUL Scaling Tests ===\n";
-  std::cout << "Format:\n";
-  std::cout << "scheduler matrix_size actors time_seconds\n";
-
-  uint64_t master_seed = 0xDEADBEEF1234ULL;
-  auto pools_map = prepare_matrix_pools(matrix_sizes, /*pool_size_per_size=*/32, master_seed);
-
-  // move pools into global map so actors can read them by N without sending them in messages
-  {
-    std::lock_guard<std::mutex> lk(g_pools_mutex);
-    for (auto& kv : pools_map) {
-      int N = kv.first;
-      auto vec = std::move(kv.second);
-      g_pools[N] = std::make_shared<std::vector<std::shared_ptr<std::vector<int>>>>(std::move(vec));
+
+
+void run_mmul_mixed_batch_cuda_scheduler(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    MatrixPool pool,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    std::mt19937 rng(123456);
+    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+
+    for (int i = 0; i < num_actors; ++i) {
+
+        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
+
+	const auto& A = pool.A[N];
+	const auto& B = pool.B[N];
+
+        sys.spawn(
+            mmul_actor_fun,
+            program,
+            A,
+	    B,
+	    N);
     }
-  }
 
-  for (int size : matrix_sizes) {
-    for (int actors : actor_counts) {
+    sys.await_all_actors_done();
+}
 
-      // Prepare deterministic spawn_times once per (size, actors)
-      double total_duration = 5.0; // seconds (tunable)
-      int max_waves = 6;
-      auto spawn_times = generate_spawn_schedule(actors, total_duration, max_waves, master_seed, size);
 
-      // ================= Scheduler-enabled (core_usage) =================
-      caf::cuda::manager::init(sys, man_config);
-      {
-        caf::cuda::manager& mgr = caf::cuda::manager::get();
-        // set scheduler behavior per device
-        for (int i = 0; i < mgr.get_num_devices(); i++)
-          mgr.send_scheduler_actor_message("multilevel", i);
+void run_mmul_mixed_batch_caf_cuda_scheduler(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    MatrixPool pool,
+    bool FCFS,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-        std::cout << "[RUN] scheduler=multilevel_usage "
-                  << "matrix_size=" << size << " "
-                  << "actors=" << actors << "\n";
+    //set the scheduler actor behavior
+    if (FCFS) {
+	    for (int i = 0; i < mgr.get_num_devices(); i++) {
 
-        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-        const int THREADS = 32;
-        const int BLOCKS = (size + THREADS - 1) / THREADS;
-        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+		    mgr.send_scheduler_actor_message("green",i);
 
-        caf::actor exit_actor = mgr.spawn_exit_actor(actors);
+	    }
 
-        double core_usage_time = time_run([&] {
-          spawn_mmul_actors_with_schedule_scheduler(sys, spawn_times, exit_actor, size, program, dims, master_seed);
-          sys.await_all_actors_done();
-        });
+    }
+    else { 
+	    for (int i = 0; i < mgr.get_num_devices(); i++) {
 
-        std::cout << std::fixed << std::setprecision(6)
-                  << "RESULT core_usage "
-                  << size << " "
-                  << actors << " "
-                  << core_usage_time << "\n";
-      }
-      caf::cuda::manager::shutdown();
+		    mgr.send_scheduler_actor_message("multilevel",i);
 
-      // ================= Scheduler-disabled actor (green-light) =================
-      caf::cuda::manager::init(sys, man_config);
-      {
-        caf::cuda::manager& mgr = caf::cuda::manager::get();
-        for (int i = 0; i < mgr.get_num_devices(); i++)
-          mgr.send_scheduler_actor_message("green", i);
+	    }
+    }
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
+
+    std::mt19937 rng(123456);
+    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
 
-        std::cout << "[RUN] scheduler=green_light_only "
-                  << "matrix_size=" << size << " "
-                  << "actors=" << actors << "\n";
+    const int THREADS = 32;
+
+    for (int i = 0; i < num_actors; ++i) {
+
+        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
+        int BLOCKS = (N + THREADS - 1) / THREADS;
 
-        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-        const int THREADS = 32;
-        const int BLOCKS = (size + THREADS - 1) / THREADS;
         caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-        caf::actor exit_actor = mgr.spawn_exit_actor(actors);
+	const auto& A = pool.A[N];
+	const auto& B = pool.B[N];
 
-        double green_light_time = time_run([&] {
-          // reuse the same spawn_times and global pools
-          spawn_mmul_actors_with_schedule_scheduler(sys, spawn_times, exit_actor, size, program, dims, master_seed);
-          sys.await_all_actors_done();
-        });
 
-        std::cout << std::fixed << std::setprecision(6)
-                  << "RESULT green_light_only "
-                  << size << " "
-                  << actors << " "
-                  << green_light_time << "\n";
-      }
-      caf::cuda::manager::shutdown();
+        caf::actor a = sys.spawn(
+            mmul_actor_fun_scheduler,
+            exit_actor,
+            N,
+            program,
+            dims,
+	    A,
+	    B
+        );
+    }
+    sys.await_all_actors_done();
+}
 
-      // ================= No scheduler at all actor =================
-      caf::cuda::manager_config no_sched_config(false);
-      caf::cuda::manager::init(sys, no_sched_config);
-      {
-        caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-        std::cout << "[RUN] scheduler=none "
-                  << "matrix_size=" << size << " "
-                  << "actors=" << actors << "\n";
 
-        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-        const int THREADS = 32;
-        const int BLOCKS = (size + THREADS - 1) / THREADS;
-        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-        double no_scheduler_time = time_run([&] {
-          spawn_mmul_actors_with_schedule_no_scheduler_actor(sys, spawn_times, size, program, master_seed);
-          sys.await_all_actors_done();
-        });
 
-        std::cout << std::fixed << std::setprecision(6)
-                  << "RESULT none "
-                  << size << " "
-                  << actors << " "
-                  << no_scheduler_time << "\n";
-      }
-      caf::cuda::manager::shutdown();
+
+
+
+void run_mmul_mixed_batch_comparison(
+    caf::actor_system& sys)
+{
+    std::vector<int> sizes = {32,64,128,256,512,1024,2048,4096};
+    std::vector<int> actor_counts = {10000};
+    MatrixPool pool = create_matrix_pool(sizes);
+
+
+
+    std::cout << "\n=== MMUL Mixed Batch Comparison ===\n";
+    std::cout << "scheduler actors time_seconds\n";
+
+    for (auto actors : actor_counts) {
+
+        /* ========= core_usage BULK ========= */
+
+        {
+            caf::cuda::manager_config cfg(true);
+            caf::cuda::manager::init(sys, cfg);
+
+            double t = time_run([&] {
+                run_mmul_mixed_batch_one_mode_bulk(sys, sizes, actors, true);
+            });
+
+            std::cout << "RESULT core_usage "
+                      << actors << " "
+                      << t << "\n";
+
+            caf::cuda::manager::shutdown();
+        }
+
+        /* ========= no scheduler ========= */
+
+        {
+            caf::cuda::manager_config cfg(false);
+            caf::cuda::manager::init(sys, cfg);
+
+            double t = time_run([&] {
+                run_mmul_mixed_batch_one_mode(sys, sizes, actors, true);
+            });
+
+            std::cout << "RESULT none "
+                      << actors << " "
+                      << t << "\n";
+
+            caf::cuda::manager::shutdown();
+        }
     }
-  }
 
-  std::cout << "=== MMUL Scaling Tests Complete ===\n";
+    std::cout << "\n=== Mixed Batch Comparison Complete ===\n";
 }
 
+
+
 void caf_main(caf::actor_system& sys) {
+  
+
   caf::cuda::manager_config man_config(true);
-  run_mmul_scaling_tests(sys, man_config);
+  //caf::cuda::manager::init(sys,man_config);
+  run_mmul_scaling_tests(sys,man_config);
+
 }
 
+
+
+
 CAF_MAIN()

From 8927c8d124d0f9a3531fa8390df18bf4b8a4d057 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 12 Mar 2026 15:48:47 -0600
Subject: [PATCH 0514/1000] Fixed errors.

---
 .../main.test.cpp                             | 32 +++++++++++++++----
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
index e33f4fae65..43753af860 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
@@ -153,7 +153,7 @@ caf::behavior mmul_actor_fun_scheduler(
     caf::actor exit_actor,
     int N,
     caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims
+    caf::cuda::nd_range dims,
     const std::vector<int>& matrix1,
     const std::vector<int> & matrix2)
 {
@@ -172,7 +172,7 @@ caf::behavior mmul_actor_fun_scheduler(
                         self,
 			rand() //dependency number, can declare indepedent but want to see what happens when you do not 
                         );
-        mgr.send_scheduler_message(launch_token);
+        mgr.send_scheduler_actor_message(launch_token);
 
 	return {
 
@@ -357,15 +357,32 @@ void run_mmul_mixed_batch_comparison(
             caf::cuda::manager::init(sys, cfg);
 
             double t = time_run([&] {
-                run_mmul_mixed_batch_one_mode_bulk(sys, sizes, actors, true);
+                run_mmul_mixed_batch_caf_cuda_scheduler(sys, sizes, actors,pool,false);
             });
 
-            std::cout << "RESULT core_usage "
+            std::cout << "RESULT CAF CUDA DEFAULT SCHEDULER "
                       << actors << " "
                       << t << "\n";
 
             caf::cuda::manager::shutdown();
         }
+        /* ========= green light BULK  ========= */
+
+        {
+            caf::cuda::manager_config cfg(true);
+            caf::cuda::manager::init(sys, cfg);
+
+            double t = time_run([&] {
+                run_mmul_mixed_batch_caf_cuda_scheduler(sys, sizes, actors,pool,true);
+            });
+
+            std::cout << "RESULT CAF CUDA FCFS SCHEDULER "
+                      << actors << " "
+                      << t << "\n";
+
+            caf::cuda::manager::shutdown();
+        }
+
 
         /* ========= no scheduler ========= */
 
@@ -374,10 +391,10 @@ void run_mmul_mixed_batch_comparison(
             caf::cuda::manager::init(sys, cfg);
 
             double t = time_run([&] {
-                run_mmul_mixed_batch_one_mode(sys, sizes, actors, true);
+                run_mmul_mixed_batch_cuda_scheduler(sys, sizes, actors,pool);
             });
 
-            std::cout << "RESULT none "
+            std::cout << "RESULT CUDA SCHEDULER "
                       << actors << " "
                       << t << "\n";
 
@@ -395,7 +412,8 @@ void caf_main(caf::actor_system& sys) {
 
   caf::cuda::manager_config man_config(true);
   //caf::cuda::manager::init(sys,man_config);
-  run_mmul_scaling_tests(sys,man_config);
+
+  run_mmul_mixed_batch_comparison(sys);
 
 }
 

From 5c2bc0c9f05f022af88705c80aea484f2b793ff9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 14 Mar 2026 10:20:11 -0600
Subject: [PATCH 0515/1000] Fixed grid dimensions to fix an issue where kernel
 would be launching slower than it should have been relative to had you done
 it with CAF CUDA.

---
 .../matrix_mul_driver.cpp                     | 33 ++++++++++++-------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
index d67cacdce3..e8ebb5d83d 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
@@ -38,17 +38,18 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     std::vector<int> h_a(elements, 1);
     std::vector<int> h_b(elements, 1);
     std::vector<int> h_c(elements);
+    //int h_c[elements];
 
     CUdeviceptr d_a, d_b, d_c;
     CUstream stream;
 
-    auto t_total_start = clock::now();
-
     // ----------------------------------
     // Create Stream
     // ----------------------------------
     checkCU(cuStreamCreate(&stream, CU_STREAM_DEFAULT), "cuStreamCreate");
 
+    auto t_total_start = clock::now();
+
     // ----------------------------------
     // Device Allocation
     // ----------------------------------
@@ -66,7 +67,7 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     auto t_h2d_a_start = clock::now();
 
     checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), bytes, stream), "cuMemcpyHtoDAsync A");
-    checkCU(cuStreamSynchronize(stream), "sync A");
+    //checkCU(cuStreamSynchronize(stream), "sync A");
 
     auto t_h2d_a_end = clock::now();
 
@@ -76,15 +77,15 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     auto t_h2d_b_start = clock::now();
 
     checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), bytes, stream), "cuMemcpyHtoDAsync B");
-    checkCU(cuStreamSynchronize(stream), "sync B");
+    //checkCU(cuStreamSynchronize(stream), "sync B");
 
     auto t_h2d_b_end = clock::now();
 
     // ----------------------------------
     // Kernel launch + execution
     // ----------------------------------
-    const unsigned int blockX = 16;
-    const unsigned int blockY = 16;
+    const unsigned int blockX = 32;
+    const unsigned int blockY = 32;
     unsigned int gridX = (N + blockX - 1) / blockX;
     unsigned int gridY = (N + blockY - 1) / blockY;
 
@@ -101,7 +102,7 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
                            nullptr),
             "cuLaunchKernel");
 
-    checkCU(cuStreamSynchronize(stream), "kernel sync");
+   // checkCU(cuStreamSynchronize(stream), "kernel sync");
 
     auto t_kernel_end = clock::now();
 
@@ -110,10 +111,14 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     // ----------------------------------
     auto t_d2h_start = clock::now();
 
-    checkCU(cuMemcpyDtoH(h_c.data(), d_c, bytes), "cuMemcpyDtoH");
-
+    cuMemcpyDtoHAsync(h_c.data(), d_c, bytes, stream);
+   // cuMemcpyDtoHAsync(h_c, d_c, bytes, stream);
+    cuStreamSynchronize(stream);
     auto t_d2h_end = clock::now();
 
+    auto t_total_end = clock::now();
+    
+    
     // ----------------------------------
     // Free device memory
     // ----------------------------------
@@ -125,12 +130,14 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
 
     auto t_free_end = clock::now();
 
+
+
+
     // ----------------------------------
     // Destroy stream
     // ----------------------------------
     checkCU(cuStreamDestroy(stream), "cuStreamDestroy");
 
-    auto t_total_end = clock::now();
 
     // ----------------------------------
     // Print Results
@@ -168,8 +175,10 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
 }
 
 int main(int argc, char** argv) {
-    std::vector<int> sizes = {1000, 4000, 8000, 12000};
-    if (argc > 1) {
+   // std::vector<int> sizes = {1000, 4000, 8000, 12000};
+    
+   std::vector<int> sizes = {8000};	
+   if (argc > 1) {
         sizes.clear();
         for (int i = 1; i < argc; ++i) sizes.push_back(std::stoi(argv[i]));
     }

From 64b6d9393f017b54d15e4f6b44865733ec6bda20 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 14 Mar 2026 10:43:48 -0600
Subject: [PATCH 0516/1000] Initial commit.

---
 .../baseline-comparison/actors/CMakeLists.txt |  44 ++
 .../actors/compile_kernels.sh                 |  13 +
 .../baseline-comparison/actors/main.test.cpp  | 392 ++++++++++++++++++
 .../baseline-comparison/actors/mmul.cu        |  16 +
 4 files changed, 465 insertions(+)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
 create mode 100644 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/mmul.cu

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/CMakeLists.txt
new file mode 100644
index 0000000000..d79166c1fb
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/compile_kernels.sh b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/compile_kernels.sh
new file mode 100755
index 0000000000..f32480e5cb
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/compile_kernels.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
new file mode 100644
index 0000000000..114786c986
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
@@ -0,0 +1,392 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/component-actors/mmul_actor/mmul_actor.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+using command =
+  caf::cuda::command_runner<>;
+
+command mmul_command;
+
+struct mmul_state {
+};
+
+//global output buffer meant to disclude it from timing
+//the other benchmark test do not include its memory allocations in it 
+//so its only fair that we do not either 
+std::vector<int> matrixC;
+
+
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
+  return {
+
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        int N) {
+
+      using clock = std::chrono::steady_clock;
+      using ms = std::chrono::duration<double, std::milli>;
+
+
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+      int device = 0;
+      int stream = 1;
+
+      auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+      auto t_total_start = clock::now();
+      // -------------------------
+      // create_in_arg A
+      // -------------------------
+      auto t_a_inarg_start = clock::now();
+
+      auto inA = caf::cuda::create_in_arg(std::move(matrixA));
+
+      auto t_a_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer A
+      // -------------------------
+      auto t_a_transfer_start = clock::now();
+
+      auto arg1 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inA));
+
+      auto t_a_transfer_end = clock::now();
+
+      // -------------------------
+      // create_in_arg B
+      // -------------------------
+      auto t_b_inarg_start = clock::now();
+
+      auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+
+      auto t_b_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer B
+      // -------------------------
+      auto t_b_transfer_start = clock::now();
+
+      auto arg2 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inB));
+
+      auto t_b_transfer_end = clock::now();
+
+      // -------------------------
+      // spawn actor
+      // -------------------------
+      auto t_spawn_start = clock::now();
+
+      caf::actor mmul_actor =
+        self->spawn(caf::cuda::mmul_actor_fun<int>, program);
+
+      auto t_spawn_end = clock::now();
+
+      // -------------------------
+      // request
+      // -------------------------
+      auto t_request_start = clock::now();
+
+      self->mail(arg1, arg2, N, device, stream)
+        .request(mmul_actor, std::chrono::seconds(30))
+        .then(
+          [=](caf::cuda::mem_ptr<int> dC) {
+
+            auto t_response_received = clock::now();
+
+            //std::vector<int> matrixC(N*N);
+            // -------------------------
+            // copy to host
+            // -------------------------
+            auto t_copy_start = clock::now();
+
+            //std::vector<int> matrixC = dC->copy_to_host();
+
+	    dC->copy_to_host(matrixC.data(),N*N);
+
+            auto t_copy_end = clock::now();
+            auto t_total_end = clock::now();
+
+            // -------------------------
+            // Print timings
+            // -------------------------
+
+            std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
+
+            std::cout << "create_in_arg A: "
+                      << ms(t_a_inarg_end - t_a_inarg_start).count()
+                      << " ms\n";
+
+            std::cout << "transfer A: "
+                      << ms(t_a_transfer_end - t_a_transfer_start).count()
+                      << " ms\n";
+
+            std::cout << "create_in_arg B: "
+                      << ms(t_b_inarg_end - t_b_inarg_start).count()
+                      << " ms\n";
+
+            std::cout << "transfer B: "
+                      << ms(t_b_transfer_end - t_b_transfer_start).count()
+                      << " ms\n";
+
+            std::cout << "spawn actor: "
+                      << ms(t_spawn_end - t_spawn_start).count()
+                      << " ms\n";
+
+            std::cout << "request → response latency: "
+                      << ms(t_response_received - t_request_start).count()
+                      << " ms\n";
+
+            std::cout << "copy_to_host: "
+                      << ms(t_copy_end - t_copy_start).count()
+                      << " ms\n";
+
+            std::cout << "TOTAL end-to-end: "
+                      << ms(t_total_end - t_total_start).count()
+                      << " ms\n";
+
+            std::cout << "=============================================\n";
+
+            self->quit();
+          }
+        );
+    }
+
+  };
+}
+
+
+caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
+  return {
+
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        int N) {
+
+      using clock = std::chrono::steady_clock;
+      using ms = std::chrono::duration<double, std::milli>;
+
+
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+      int device = 0;
+      int stream = 1;
+
+      auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+      auto t_total_start = clock::now();
+      // -------------------------
+      // create_in_arg A
+      // -------------------------
+      auto t_a_inarg_start = clock::now();
+
+      auto inA = caf::cuda::create_in_arg(std::move(matrixA));
+
+      auto t_a_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer A
+      // -------------------------
+      auto t_a_transfer_start = clock::now();
+
+      auto arg1 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inA));
+
+      auto t_a_transfer_end = clock::now();
+
+      // -------------------------
+      // create_in_arg B
+      // -------------------------
+      auto t_b_inarg_start = clock::now();
+
+      auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+
+      auto t_b_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer B
+      // -------------------------
+      auto t_b_transfer_start = clock::now();
+
+      auto arg2 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inB));
+
+      auto t_b_transfer_end = clock::now();
+  
+      const int THREADS = 32;
+      const int BLOCKS = (N + THREADS - 1) / THREADS;
+
+      caf::cuda::nd_range dims(
+        BLOCKS, BLOCKS, 1,
+        THREADS, THREADS, 1);
+
+      // -------------------------
+      // request
+      // -------------------------
+      auto t_request_start = clock::now();
+
+      caf::cuda::mmul_async_command<int> command;
+      auto output = command.run_async(
+		      program,dims,
+		      1,
+		      arg1,arg2,out<int>{N*N},in<int>{N});
+
+       auto t_response_received = clock::now();
+
+            //std::vector<int> matrixC(N*N);
+            // -------------------------
+            // copy to host
+            // -------------------------
+            auto t_copy_start = clock::now();
+
+	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);
+
+            //std::vector<int> matrixC = dC->copy_to_host();
+
+	    dC->copy_to_host(matrixC.data(),N*N);
+
+            auto t_copy_end = clock::now();
+            auto t_total_end = clock::now();
+
+            // -------------------------
+            // Print timings
+            // -------------------------
+
+            std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
+
+            std::cout << "create_in_arg A: "
+                      << ms(t_a_inarg_end - t_a_inarg_start).count()
+                      << " ms\n";
+
+            std::cout << "transfer A: "
+                      << ms(t_a_transfer_end - t_a_transfer_start).count()
+                      << " ms\n";
+
+            std::cout << "create_in_arg B: "
+                      << ms(t_b_inarg_end - t_b_inarg_start).count()
+                      << " ms\n";
+
+            std::cout << "transfer B: "
+                      << ms(t_b_transfer_end - t_b_transfer_start).count()
+                      << " ms\n";
+
+                       std::cout << "request → response latency: "
+                      << ms(t_response_received - t_request_start).count()
+                      << " ms\n";
+
+            std::cout << "copy_to_host: "
+                      << ms(t_copy_end - t_copy_start).count()
+                      << " ms\n";
+
+            std::cout << "TOTAL end-to-end: "
+                      << ms(t_total_end - t_total_start).count()
+                      << " ms\n";
+
+            std::cout << "=============================================\n";
+
+           self->quit();
+    }
+
+  };
+}
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size) {
+
+
+  caf::cuda::manager::init(sys);
+  // ------------------------------------
+  // Start timing
+  // ------------------------------------
+  auto start = std::chrono::steady_clock::now();
+
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<int> matrixA(matrix_size * matrix_size,2);
+  std::vector<int> matrixB(matrix_size * matrix_size,3);
+
+  matrixC.resize(matrix_size*matrix_size);
+
+ using clock = std::chrono::steady_clock;
+
+auto t_start = clock::now();
+
+caf::actor a =sys.spawn(mmul_actor_fun_2);
+
+anon_mail(matrixA,matrixB,matrix_size).send(a);
+
+auto t_end = clock::now();
+
+
+
+  // Wait for all actors to finish
+  sys.await_all_actors_done();
+
+  // ------------------------------------
+  // Stop timing
+  // ------------------------------------
+  auto end = std::chrono::steady_clock::now();
+  auto duration_ms =
+      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
+            << ", time=" << duration_ms << " ms\n";
+
+  caf::cuda::manager::shutdown();
+
+}
+
+
+void caf_main(caf::actor_system& sys) {
+ run_mmul_test(sys,1000);
+  run_mmul_test(sys,4000);
+  run_mmul_test(sys,8000);
+  run_mmul_test(sys,12000);
+
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/mmul.cu b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From a3fafc2807af118d50eb5608d8cca17f17f7fd41 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 14 Mar 2026 10:54:23 -0600
Subject: [PATCH 0517/1000] Wrote inital test.

---
 .../baseline-comparison/actors/main.test.cpp  | 236 ++----------------
 1 file changed, 24 insertions(+), 212 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
index 114786c986..0a9431870d 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
@@ -41,6 +41,7 @@ using command =
 command mmul_command;
 
 struct mmul_state {
+	caf::cuda::program_ptr program;
 };
 
 //global output buffer meant to disclude it from timing
@@ -49,153 +50,15 @@ struct mmul_state {
 std::vector<int> matrixC;
 
 
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
-  return {
 
-    [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB,
-        int N) {
-
-      using clock = std::chrono::steady_clock;
-      using ms = std::chrono::duration<double, std::milli>;
-
-
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
-      int device = 0;
-      int stream = 1;
-
-      auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-      auto t_total_start = clock::now();
-      // -------------------------
-      // create_in_arg A
-      // -------------------------
-      auto t_a_inarg_start = clock::now();
-
-      auto inA = caf::cuda::create_in_arg(std::move(matrixA));
-
-      auto t_a_inarg_end = clock::now();
-
-      // -------------------------
-      // transfer A
-      // -------------------------
-      auto t_a_transfer_start = clock::now();
-
-      auto arg1 = mmul_command.transfer_memory(
-          device,
-          stream,
-          std::move(inA));
-
-      auto t_a_transfer_end = clock::now();
-
-      // -------------------------
-      // create_in_arg B
-      // -------------------------
-      auto t_b_inarg_start = clock::now();
-
-      auto inB = caf::cuda::create_in_arg(std::move(matrixB));
-
-      auto t_b_inarg_end = clock::now();
-
-      // -------------------------
-      // transfer B
-      // -------------------------
-      auto t_b_transfer_start = clock::now();
-
-      auto arg2 = mmul_command.transfer_memory(
-          device,
-          stream,
-          std::move(inB));
-
-      auto t_b_transfer_end = clock::now();
-
-      // -------------------------
-      // spawn actor
-      // -------------------------
-      auto t_spawn_start = clock::now();
-
-      caf::actor mmul_actor =
-        self->spawn(caf::cuda::mmul_actor_fun<int>, program);
-
-      auto t_spawn_end = clock::now();
-
-      // -------------------------
-      // request
-      // -------------------------
-      auto t_request_start = clock::now();
-
-      self->mail(arg1, arg2, N, device, stream)
-        .request(mmul_actor, std::chrono::seconds(30))
-        .then(
-          [=](caf::cuda::mem_ptr<int> dC) {
-
-            auto t_response_received = clock::now();
-
-            //std::vector<int> matrixC(N*N);
-            // -------------------------
-            // copy to host
-            // -------------------------
-            auto t_copy_start = clock::now();
-
-            //std::vector<int> matrixC = dC->copy_to_host();
-
-	    dC->copy_to_host(matrixC.data(),N*N);
-
-            auto t_copy_end = clock::now();
-            auto t_total_end = clock::now();
-
-            // -------------------------
-            // Print timings
-            // -------------------------
-
-            std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
-
-            std::cout << "create_in_arg A: "
-                      << ms(t_a_inarg_end - t_a_inarg_start).count()
-                      << " ms\n";
-
-            std::cout << "transfer A: "
-                      << ms(t_a_transfer_end - t_a_transfer_start).count()
-                      << " ms\n";
-
-            std::cout << "create_in_arg B: "
-                      << ms(t_b_inarg_end - t_b_inarg_start).count()
-                      << " ms\n";
-
-            std::cout << "transfer B: "
-                      << ms(t_b_transfer_end - t_b_transfer_start).count()
-                      << " ms\n";
-
-            std::cout << "spawn actor: "
-                      << ms(t_spawn_end - t_spawn_start).count()
-                      << " ms\n";
-
-            std::cout << "request → response latency: "
-                      << ms(t_response_received - t_request_start).count()
-                      << " ms\n";
-
-            std::cout << "copy_to_host: "
-                      << ms(t_copy_end - t_copy_start).count()
-                      << " ms\n";
-
-            std::cout << "TOTAL end-to-end: "
-                      << ms(t_total_end - t_total_start).count()
-                      << " ms\n";
-
-            std::cout << "=============================================\n";
-
-            self->quit();
-          }
-        );
-    }
 
-  };
-}
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
+		caf::cuda::program_ptr mmul_kernel) {
+  
 
+	self ->state().program = mmul_kernel;
 
-caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
-  return {
+return {
 
     [=](const std::vector<int>& matrixA,
         const std::vector<int>& matrixB,
@@ -209,10 +72,7 @@ caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
       int device = 0;
       int stream = 1;
 
-      auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-      auto t_total_start = clock::now();
+           auto t_total_start = clock::now();
       // -------------------------
       // create_in_arg A
       // -------------------------
@@ -262,104 +122,56 @@ caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
         BLOCKS, BLOCKS, 1,
         THREADS, THREADS, 1);
 
-      // -------------------------
-      // request
-      // -------------------------
-      auto t_request_start = clock::now();
-
       caf::cuda::mmul_async_command<int> command;
       auto output = command.run_async(
 		      program,dims,
 		      1,
 		      arg1,arg2,out<int>{N*N},in<int>{N});
 
-       auto t_response_received = clock::now();
-
-            //std::vector<int> matrixC(N*N);
-            // -------------------------
-            // copy to host
-            // -------------------------
-            auto t_copy_start = clock::now();
-
 	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);
 
-            //std::vector<int> matrixC = dC->copy_to_host();
 
 	    dC->copy_to_host(matrixC.data(),N*N);
 
             auto t_copy_end = clock::now();
             auto t_total_end = clock::now();
 
-            // -------------------------
-            // Print timings
-            // -------------------------
-
-            std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
-
-            std::cout << "create_in_arg A: "
-                      << ms(t_a_inarg_end - t_a_inarg_start).count()
-                      << " ms\n";
-
-            std::cout << "transfer A: "
-                      << ms(t_a_transfer_end - t_a_transfer_start).count()
-                      << " ms\n";
-
-            std::cout << "create_in_arg B: "
-                      << ms(t_b_inarg_end - t_b_inarg_start).count()
-                      << " ms\n";
-
-            std::cout << "transfer B: "
-                      << ms(t_b_transfer_end - t_b_transfer_start).count()
-                      << " ms\n";
-
-                       std::cout << "request → response latency: "
-                      << ms(t_response_received - t_request_start).count()
-                      << " ms\n";
-
-            std::cout << "copy_to_host: "
-                      << ms(t_copy_end - t_copy_start).count()
-                      << " ms\n";
-
-            std::cout << "TOTAL end-to-end: "
-                      << ms(t_total_end - t_total_start).count()
-                      << " ms\n";
-
-            std::cout << "=============================================\n";
-
-           self->quit();
     }
 
   };
 }
 
 
-void run_mmul_test(caf::actor_system& sys, int matrix_size) {
+void run_mmul_test(caf::actor_system& sys, int matrix_size,int iterations) {
 
 
   caf::cuda::manager::init(sys);
   // ------------------------------------
   // Start timing
   // ------------------------------------
-  auto start = std::chrono::steady_clock::now();
 
   // Spawn num_actors actors running the mmul behavior
   std::vector<int> matrixA(matrix_size * matrix_size,2);
   std::vector<int> matrixB(matrix_size * matrix_size,3);
 
   matrixC.resize(matrix_size*matrix_size);
+ 
+  auto mgr = caf::cuda::manager::get();
+  
+  auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
- using clock = std::chrono::steady_clock;
-
-auto t_start = clock::now();
-
-caf::actor a =sys.spawn(mmul_actor_fun_2);
 
-anon_mail(matrixA,matrixB,matrix_size).send(a);
+  using clock = std::chrono::steady_clock;
 
-auto t_end = clock::now();
+  auto start = std::chrono::steady_clock::now();
 
+  caf::actor a =sys.spawn(mmul_actor_fun_2,program);
 
+  for (int i = 0; i < iterations; i++) 
+	  anon_mail(matrixA,matrixB,matrix_size).send(a);
 
+  anon_send_exit(a);
   // Wait for all actors to finish
   sys.await_all_actors_done();
 
@@ -371,7 +183,8 @@ auto t_end = clock::now();
       std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
 
   std::cout << "[MMUL TEST] matrix_size=" << matrix_size
-            << ", time=" << duration_ms << " ms\n";
+            << "iterations = " << iterations << 
+	    ", time=" << duration_ms << " ms\n";
 
   caf::cuda::manager::shutdown();
 
@@ -379,10 +192,9 @@ auto t_end = clock::now();
 
 
 void caf_main(caf::actor_system& sys) {
- run_mmul_test(sys,1000);
-  run_mmul_test(sys,4000);
-  run_mmul_test(sys,8000);
-  run_mmul_test(sys,12000);
+
+	for (int i = 1000; i < 11000; i+=1000)
+		run_mmul_test(sys,1000,i);
 
 }
 

From 68b64c6de8680eb3257320733a481428bb322dba Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 14 Mar 2026 10:58:43 -0600
Subject: [PATCH 0518/1000] Fixed compiler errors.

---
 .../baseline-comparison/actors/main.test.cpp             | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
index 0a9431870d..c5a485896b 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
@@ -124,7 +124,8 @@ return {
 
       caf::cuda::mmul_async_command<int> command;
       auto output = command.run_async(
-		      program,dims,
+		      self->state().program,
+		      dims,
 		      1,
 		      arg1,arg2,out<int>{N*N},in<int>{N});
 
@@ -156,7 +157,7 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size,int iterations) {
 
   matrixC.resize(matrix_size*matrix_size);
  
-  auto mgr = caf::cuda::manager::get();
+  auto& mgr = caf::cuda::manager::get();
   
   auto program =
         mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
@@ -166,12 +167,12 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size,int iterations) {
 
   auto start = std::chrono::steady_clock::now();
 
-  caf::actor a =sys.spawn(mmul_actor_fun_2,program);
+  caf::actor a =sys.spawn(mmul_actor_fun,program);
 
   for (int i = 0; i < iterations; i++) 
 	  anon_mail(matrixA,matrixB,matrix_size).send(a);
 
-  anon_send_exit(a);
+  anon_send_exit(a,caf::exit_reason::kill);
   // Wait for all actors to finish
   sys.await_all_actors_done();
 

From 0623a32ae94bf3851daf10860d5b33c782bbbd94 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 14 Mar 2026 11:00:05 -0600
Subject: [PATCH 0519/1000] Initial commit.

---
 .../baseline-comparison/cuda/CMakeLists.txt   |  52 ++++
 .../cuda/compile_kernels.sh                   |  13 +
 .../cuda/matrix_mul_driver.cpp                | 223 ++++++++++++++++++
 .../baseline-comparison/cuda/mmul.cu          |  16 ++
 4 files changed, 304 insertions(+)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
 create mode 100644 libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/mmul.cu

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/CMakeLists.txt
new file mode 100644
index 0000000000..e26c6c956c
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/CMakeLists.txt
@@ -0,0 +1,52 @@
+cmake_minimum_required(VERSION 3.10)
+project(cuda_driver_mmul LANGUAGES CXX)
+
+# Find nvcc
+find_program(NVCC_EXECUTABLE nvcc)
+if(NOT NVCC_EXECUTABLE)
+    message(FATAL_ERROR "nvcc not found. Make sure CUDA toolkit is installed and nvcc is in PATH.")
+endif()
+
+# Try to locate CUDA driver library (libcuda)
+find_library(CUDA_DRIVER_LIB NAMES cuda HINTS ENV CUDA_HOME /usr/local/cuda/lib64 /usr/local/cuda/lib)
+if(NOT CUDA_DRIVER_LIB)
+    message(FATAL_ERROR "Could not find CUDA driver library (libcuda). Ensure CUDA is installed.")
+endif()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# Ensure we build PTX from mmul.cu into build directory as mmul.ptx
+set(MMUL_CU "${CMAKE_SOURCE_DIR}/mmul.cu")
+set(MMUL_PTX "${CMAKE_BINARY_DIR}/mmul.ptx")
+
+add_custom_command(
+    OUTPUT "${MMUL_PTX}"
+    COMMAND ${NVCC_EXECUTABLE} -ptx ${MMUL_CU} -o "${MMUL_PTX}"
+    DEPENDS "${MMUL_CU}"
+    COMMENT "Compiling mmul.cu -> mmul.ptx"
+    VERBATIM
+)
+
+add_custom_target(kernel_ptx ALL DEPENDS "${MMUL_PTX}")
+
+# Host executable
+add_executable(matrix_mul_driver matrix_mul_driver.cpp)
+add_dependencies(matrix_mul_driver kernel_ptx)
+
+# Make sure PTX is copied to the executable directory so the program can find it at runtime
+add_custom_command(TARGET matrix_mul_driver POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+        "${MMUL_PTX}"
+        $<TARGET_FILE_DIR:matrix_mul_driver>
+)
+
+# Link to the CUDA driver library (libcuda)
+target_link_libraries(matrix_mul_driver PRIVATE ${CUDA_DRIVER_LIB})
+
+# Include path - try to find cuda headers (optional)
+find_path(CUDA_INCLUDE_DIR cuda.h HINTS ENV CUDA_HOME /usr/local/cuda/include)
+if(CUDA_INCLUDE_DIR)
+    target_include_directories(matrix_mul_driver PRIVATE ${CUDA_INCLUDE_DIR})
+endif()
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/compile_kernels.sh b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/compile_kernels.sh
new file mode 100755
index 0000000000..f32480e5cb
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/compile_kernels.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
new file mode 100644
index 0000000000..78c868d05d
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
@@ -0,0 +1,223 @@
+// matrix_mul_driver.cpp
+#include <cuda.h>
+#include <iostream>
+#include <vector>
+#include <chrono>
+#include <fstream>
+#include <sstream>
+#include <stdexcept>
+#include <cmath>
+
+static void checkCU(CUresult r, const char* where) {
+    if (r != CUDA_SUCCESS) {
+        const char *str = nullptr;
+        cuGetErrorString(r, &str);
+        std::cerr << "CUDA Driver API error at " << where << " -> "
+                  << (str ? str : "unknown") << " (" << (int)r << ")\n";
+        std::exit(EXIT_FAILURE);
+    }
+}
+
+std::string readFile(const std::string &path) {
+    std::ifstream in(path, std::ios::in | std::ios::binary);
+    if (!in) throw std::runtime_error("Failed to open " + path);
+    std::ostringstream ss;
+    ss << in.rdbuf();
+    return ss.str();
+}
+
+void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
+    using clock = std::chrono::steady_clock;
+    using ms = std::chrono::duration<double, std::milli>;
+
+    std::cout << "\n===== DRIVER BENCHMARK (N=" << N << ") =====\n";
+
+    size_t elements = (size_t)N * (size_t)N;
+    size_t bytes = elements * sizeof(int);
+
+    std::vector<int> h_a(elements, 1);
+    std::vector<int> h_b(elements, 1);
+    std::vector<int> h_c(elements);
+    //int h_c[elements];
+
+    CUdeviceptr d_a, d_b, d_c;
+    CUstream stream;
+
+    // ----------------------------------
+    // Create Stream
+    // ----------------------------------
+    checkCU(cuStreamCreate(&stream, CU_STREAM_DEFAULT), "cuStreamCreate");
+
+    auto t_total_start = clock::now();
+
+    // ----------------------------------
+    // Device Allocation
+    // ----------------------------------
+    auto t_alloc_start = clock::now();
+
+    checkCU(cuMemAlloc(&d_a, bytes), "cuMemAlloc d_a");
+    checkCU(cuMemAlloc(&d_b, bytes), "cuMemAlloc d_b");
+    checkCU(cuMemAlloc(&d_c, bytes), "cuMemAlloc d_c");
+
+    auto t_alloc_end = clock::now();
+
+    // ----------------------------------
+    // H2D copy A
+    // ----------------------------------
+    auto t_h2d_a_start = clock::now();
+
+    checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), bytes, stream), "cuMemcpyHtoDAsync A");
+    //checkCU(cuStreamSynchronize(stream), "sync A");
+
+    auto t_h2d_a_end = clock::now();
+
+    // ----------------------------------
+    // H2D copy B
+    // ----------------------------------
+    auto t_h2d_b_start = clock::now();
+
+    checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), bytes, stream), "cuMemcpyHtoDAsync B");
+    //checkCU(cuStreamSynchronize(stream), "sync B");
+
+    auto t_h2d_b_end = clock::now();
+
+    // ----------------------------------
+    // Kernel launch + execution
+    // ----------------------------------
+    const unsigned int blockX = 32;
+    const unsigned int blockY = 32;
+    unsigned int gridX = (N + blockX - 1) / blockX;
+    unsigned int gridY = (N + blockY - 1) / blockY;
+
+    void* kernelParams[] = { &d_a, &d_b, &d_c, &N };
+
+    auto t_kernel_start = clock::now();
+
+    checkCU(cuLaunchKernel(kernel,
+                           gridX, gridY, 1,
+                           blockX, blockY, 1,
+                           0,
+                           stream,
+                           kernelParams,
+                           nullptr),
+            "cuLaunchKernel");
+
+   // checkCU(cuStreamSynchronize(stream), "kernel sync");
+
+    auto t_kernel_end = clock::now();
+
+    // ----------------------------------
+    // D2H copy
+    // ----------------------------------
+    auto t_d2h_start = clock::now();
+
+    cuMemcpyDtoHAsync(h_c.data(), d_c, bytes, stream);
+   // cuMemcpyDtoHAsync(h_c, d_c, bytes, stream);
+    cuStreamSynchronize(stream);
+    auto t_d2h_end = clock::now();
+
+    auto t_total_end = clock::now();
+    
+    
+    // ----------------------------------
+    // Free device memory
+    // ----------------------------------
+    auto t_free_start = clock::now();
+
+    checkCU(cuMemFree(d_a), "cuMemFree A");
+    checkCU(cuMemFree(d_b), "cuMemFree B");
+    checkCU(cuMemFree(d_c), "cuMemFree C");
+
+    auto t_free_end = clock::now();
+
+
+
+
+    // ----------------------------------
+    // Destroy stream
+    // ----------------------------------
+    checkCU(cuStreamDestroy(stream), "cuStreamDestroy");
+
+
+    // ----------------------------------
+    // Print Results
+    // ----------------------------------
+
+    std::cout << "Device allocation: "
+              << ms(t_alloc_end - t_alloc_start).count()
+              << " ms\n";
+
+    std::cout << "H2D copy A: "
+              << ms(t_h2d_a_end - t_h2d_a_start).count()
+              << " ms\n";
+
+    std::cout << "H2D copy B: "
+              << ms(t_h2d_b_end - t_h2d_b_start).count()
+              << " ms\n";
+
+    std::cout << "Kernel execution: "
+              << ms(t_kernel_end - t_kernel_start).count()
+              << " ms\n";
+
+    std::cout << "D2H copy: "
+              << ms(t_d2h_end - t_d2h_start).count()
+              << " ms\n";
+
+    std::cout << "Device free: "
+              << ms(t_free_end - t_free_start).count()
+              << " ms\n";
+
+    std::cout << "TOTAL: "
+              << ms(t_total_end - t_total_start).count()
+              << " ms\n";
+
+    std::cout << "=============================================\n";
+}
+
+int main(int argc, char** argv) {
+    std::vector<int> sizes = {1000, 4000, 8000, 12000};
+    
+   //std::vector<int> sizes = {8000};	
+   if (argc > 1) {
+        sizes.clear();
+        for (int i = 1; i < argc; ++i) sizes.push_back(std::stoi(argv[i]));
+    }
+
+    checkCU(cuInit(0), "cuInit");
+
+    CUdevice dev;
+    checkCU(cuDeviceGet(&dev, 0), "cuDeviceGet(0)");
+
+    CUcontext ctx;
+    checkCU(cuCtxCreate(&ctx, 0, dev), "cuCtxCreate");
+
+    const std::string ptxPath = "mmul.ptx";
+    std::string ptx;
+
+    try {
+        ptx = readFile(ptxPath);
+    } catch (const std::exception &e) {
+        std::cerr << "Failed to read PTX file '" << ptxPath << "': " << e.what() << "\n";
+        return EXIT_FAILURE;
+    }
+
+    CUmodule module;
+    checkCU(cuModuleLoadDataEx(&module, ptx.c_str(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
+
+    CUfunction kernel;
+    checkCU(cuModuleGetFunction(&kernel, module, "matrixMul"), "cuModuleGetFunction matrixMul");
+
+    for (int N : sizes) {
+        try {
+            runMatrixMul(module, kernel, N);
+        } catch (const std::exception &e) {
+            std::cerr << "Exception while running N=" << N << ": " << e.what() << "\n";
+        }
+        std::cout << "----------------------------------------\n";
+    }
+
+    checkCU(cuModuleUnload(module), "cuModuleUnload");
+    checkCU(cuCtxDestroy(ctx), "cuCtxDestroy");
+
+    return 0;
+}
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/mmul.cu b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From 00f04dfdc0eeaba1e1bcde02460fc0e931be45f4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 14 Mar 2026 11:00:37 -0600
Subject: [PATCH 0520/1000] Saving no major changes.

---
 .../control-layer/green_light_behavior.cpp    |  4 +-
 .../matrix_mul_driver.cpp                     |  4 +-
 .../mmul-actor-benchmarking/main.test.cpp     |  7 ++-
 .../main.test.cpp                             | 57 ++++++++++++-------
 .../CMakeLists.txt                            |  2 +-
 .../main.test.cpp                             |  2 +-
 6 files changed, 47 insertions(+), 29 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index f52644a660..1a595cd73c 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -31,7 +31,7 @@ void green_light_behavior::receive(const token_ptr& tok) {
      if (tok->getType() == LAUNCH) {
         //use 0 as stream id for now, eventually will have to figure out
         //stream load balancing
-        process_launch_token(tok, 0);
+        process_launch_token(tok, rand()% state_.num_streams);
     }
     else if (tok->getType() == MEMORY) {
         //use 0 as stream id for now, eventually will have to figure out
@@ -41,6 +41,7 @@ void green_light_behavior::receive(const token_ptr& tok) {
     //this may cause an issue if a message is never received then
     //we may never end up dequeueing certain requests
     //may lead to a deadlock scenario?
+   /*
     while (!state_.queue.empty()) {
         token_ptr queued = state_.queue.front();
         state_.queue.pop();
@@ -52,6 +53,7 @@ void green_light_behavior::receive(const token_ptr& tok) {
             process_memory_transfer_token(queued, 0);
         }
     }
+    */
 }
 
 
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
index e8ebb5d83d..78c868d05d 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
@@ -175,9 +175,9 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
 }
 
 int main(int argc, char** argv) {
-   // std::vector<int> sizes = {1000, 4000, 8000, 12000};
+    std::vector<int> sizes = {1000, 4000, 8000, 12000};
     
-   std::vector<int> sizes = {8000};	
+   //std::vector<int> sizes = {8000};	
    if (argc > 1) {
         sizes.clear();
         for (int i = 1; i < argc; ++i) sizes.push_back(std::stoi(argv[i]));
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
index b1d98ef52a..114786c986 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
@@ -254,7 +254,8 @@ caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
           std::move(inB));
 
       auto t_b_transfer_end = clock::now();
-  const int THREADS = 32;
+  
+      const int THREADS = 32;
       const int BLOCKS = (N + THREADS - 1) / THREADS;
 
       caf::cuda::nd_range dims(
@@ -325,7 +326,7 @@ caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
 
             std::cout << "=============================================\n";
 
-            self->quit();
+           self->quit();
     }
 
   };
@@ -378,7 +379,7 @@ auto t_end = clock::now();
 
 
 void caf_main(caf::actor_system& sys) {
-  run_mmul_test(sys,1000);
+ run_mmul_test(sys,1000);
   run_mmul_test(sys,4000);
   run_mmul_test(sys,8000);
   run_mmul_test(sys,12000);
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
index 43753af860..dbf06ff3e7 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
@@ -304,28 +304,43 @@ void run_mmul_mixed_batch_caf_cuda_scheduler(
     std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
 
     const int THREADS = 32;
+// timestamp before actor creation
+auto start_time = std::chrono::high_resolution_clock::now();
 
-    for (int i = 0; i < num_actors; ++i) {
+for (int i = 0; i < num_actors; ++i) {
 
-        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
-        int BLOCKS = (N + THREADS - 1) / THREADS;
+    int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
+    int BLOCKS = (N + THREADS - 1) / THREADS;
 
-        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-	const auto& A = pool.A[N];
-	const auto& B = pool.B[N];
+    const auto& A = pool.A[N];
+    const auto& B = pool.B[N];
+  // time the single actor creation
+    //auto start = std::chrono::high_resolution_clock::now();
 
+    caf::actor a = sys.spawn(
+        mmul_actor_fun_scheduler,
+        exit_actor,
+        N,
+        program,
+        dims,
+        A,
+        B
+    );
 
-        caf::actor a = sys.spawn(
-            mmul_actor_fun_scheduler,
-            exit_actor,
-            N,
-            program,
-            dims,
-	    A,
-	    B
-        );
-    }
+    //auto end = std::chrono::high_resolution_clock::now();
+    //auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+    //std::cout << "[INFO] Actor " << i << " creation time: " << ms << " ms\n";
+}
+
+// timestamp after actor creation
+auto end_time = std::chrono::high_resolution_clock::now();
+auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();
+
+std::cout << "[INFO] Actor creation time for " << num_actors
+          << " actors: " << ms << " ms\n";
     sys.await_all_actors_done();
 }
 
@@ -339,8 +354,8 @@ void run_mmul_mixed_batch_caf_cuda_scheduler(
 void run_mmul_mixed_batch_comparison(
     caf::actor_system& sys)
 {
-    std::vector<int> sizes = {32,64,128,256,512,1024,2048,4096};
-    std::vector<int> actor_counts = {10000};
+    std::vector<int> sizes = {32,64,128,256,512,1024,2048};
+    std::vector<int> actor_counts = {5000};
     MatrixPool pool = create_matrix_pool(sizes);
 
 
@@ -362,7 +377,7 @@ void run_mmul_mixed_batch_comparison(
 
             std::cout << "RESULT CAF CUDA DEFAULT SCHEDULER "
                       << actors << " "
-                      << t << "\n";
+                      << t << "seconds\n";
 
             caf::cuda::manager::shutdown();
         }
@@ -378,7 +393,7 @@ void run_mmul_mixed_batch_comparison(
 
             std::cout << "RESULT CAF CUDA FCFS SCHEDULER "
                       << actors << " "
-                      << t << "\n";
+                      << t << "seconds\n";
 
             caf::cuda::manager::shutdown();
         }
@@ -396,7 +411,7 @@ void run_mmul_mixed_batch_comparison(
 
             std::cout << "RESULT CUDA SCHEDULER "
                       << actors << " "
-                      << t << "\n";
+                      << t << "seconds\n";
 
             caf::cuda::manager::shutdown();
         }
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/CMakeLists.txt
index 89bcf5ba7c..e2259ce5fe 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/CMakeLists.txt
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-uniform-batch-benchmark/CMakeLists.txt
@@ -6,7 +6,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS      OFF)
 
 # 2) Set CAF source and build directories
-set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../actor-framework")
 set(CAF_BUILD "${CAF_SRC}/build")
 
 
diff --git a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
index 333b66d30f..f5de194eb7 100644
--- a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
@@ -405,7 +405,7 @@ void caf_main(caf::actor_system& sys) {
 
 	caf::cuda::manager_config man_config(true); //turns the scheduler on
 	caf::cuda::manager::init(sys,man_config);
-       	run_async_mmul_test(sys,10,1000);
+       	run_async_mmul_test(sys,10,500);
 	
 	//tests will delete the old manager so will have to reinit if you do this 
 	//in conjunction with each other	

From 00a6f2ae6b4f75482fbbe1bb68a1bcc45c17b382 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sat, 14 Mar 2026 11:08:29 -0600
Subject: [PATCH 0521/1000] Wrote initial test.

---
 .../cuda/matrix_mul_driver.cpp                | 225 ++++++------------
 1 file changed, 69 insertions(+), 156 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
index 78c868d05d..30f5c2d1bb 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
@@ -1,4 +1,3 @@
-// matrix_mul_driver.cpp
 #include <cuda.h>
 #include <iostream>
 #include <vector>
@@ -6,7 +5,6 @@
 #include <fstream>
 #include <sstream>
 #include <stdexcept>
-#include <cmath>
 
 static void checkCU(CUresult r, const char* where) {
     if (r != CUDA_SUCCESS) {
@@ -26,64 +24,9 @@ std::string readFile(const std::string &path) {
     return ss.str();
 }
 
-void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
-    using clock = std::chrono::steady_clock;
-    using ms = std::chrono::duration<double, std::milli>;
-
-    std::cout << "\n===== DRIVER BENCHMARK (N=" << N << ") =====\n";
-
-    size_t elements = (size_t)N * (size_t)N;
-    size_t bytes = elements * sizeof(int);
-
-    std::vector<int> h_a(elements, 1);
-    std::vector<int> h_b(elements, 1);
-    std::vector<int> h_c(elements);
-    //int h_c[elements];
-
-    CUdeviceptr d_a, d_b, d_c;
-    CUstream stream;
-
-    // ----------------------------------
-    // Create Stream
-    // ----------------------------------
-    checkCU(cuStreamCreate(&stream, CU_STREAM_DEFAULT), "cuStreamCreate");
-
-    auto t_total_start = clock::now();
-
-    // ----------------------------------
-    // Device Allocation
-    // ----------------------------------
-    auto t_alloc_start = clock::now();
-
-    checkCU(cuMemAlloc(&d_a, bytes), "cuMemAlloc d_a");
-    checkCU(cuMemAlloc(&d_b, bytes), "cuMemAlloc d_b");
-    checkCU(cuMemAlloc(&d_c, bytes), "cuMemAlloc d_c");
-
-    auto t_alloc_end = clock::now();
-
-    // ----------------------------------
-    // H2D copy A
-    // ----------------------------------
-    auto t_h2d_a_start = clock::now();
-
-    checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), bytes, stream), "cuMemcpyHtoDAsync A");
-    //checkCU(cuStreamSynchronize(stream), "sync A");
-
-    auto t_h2d_a_end = clock::now();
-
-    // ----------------------------------
-    // H2D copy B
-    // ----------------------------------
-    auto t_h2d_b_start = clock::now();
-
-    checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), bytes, stream), "cuMemcpyHtoDAsync B");
-    //checkCU(cuStreamSynchronize(stream), "sync B");
-
-    auto t_h2d_b_end = clock::now();
-
-    // ----------------------------------
-    // Kernel launch + execution
-    // ----------------------------------
+// Launch kernel once
+void launchKernel(CUfunction kernel, CUstream stream,
+                  CUdeviceptr d_a, CUdeviceptr d_b, CUdeviceptr d_c, int N) {
     const unsigned int blockX = 32;
     const unsigned int blockY = 32;
     unsigned int gridX = (N + blockX - 1) / blockX;
@@ -91,8 +34,6 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
 
     void* kernelParams[] = { &d_a, &d_b, &d_c, &N };
 
-    auto t_kernel_start = clock::now();
-
     checkCU(cuLaunchKernel(kernel,
                            gridX, gridY, 1,
                            blockX, blockY, 1,
@@ -101,87 +42,12 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
                            kernelParams,
                            nullptr),
             "cuLaunchKernel");
-
-   // checkCU(cuStreamSynchronize(stream), "kernel sync");
-
-    auto t_kernel_end = clock::now();
-
-    // ----------------------------------
-    // D2H copy
-    // ----------------------------------
-    auto t_d2h_start = clock::now();
-
-    cuMemcpyDtoHAsync(h_c.data(), d_c, bytes, stream);
-   // cuMemcpyDtoHAsync(h_c, d_c, bytes, stream);
-    cuStreamSynchronize(stream);
-    auto t_d2h_end = clock::now();
-
-    auto t_total_end = clock::now();
-    
-    
-    // ----------------------------------
-    // Free device memory
-    // ----------------------------------
-    auto t_free_start = clock::now();
-
-    checkCU(cuMemFree(d_a), "cuMemFree A");
-    checkCU(cuMemFree(d_b), "cuMemFree B");
-    checkCU(cuMemFree(d_c), "cuMemFree C");
-
-    auto t_free_end = clock::now();
-
-
-
-
-    // ----------------------------------
-    // Destroy stream
-    // ----------------------------------
-    checkCU(cuStreamDestroy(stream), "cuStreamDestroy");
-
-
-    // ----------------------------------
-    // Print Results
-    // ----------------------------------
-
-    std::cout << "Device allocation: "
-              << ms(t_alloc_end - t_alloc_start).count()
-              << " ms\n";
-
-    std::cout << "H2D copy A: "
-              << ms(t_h2d_a_end - t_h2d_a_start).count()
-              << " ms\n";
-
-    std::cout << "H2D copy B: "
-              << ms(t_h2d_b_end - t_h2d_b_start).count()
-              << " ms\n";
-
-    std::cout << "Kernel execution: "
-              << ms(t_kernel_end - t_kernel_start).count()
-              << " ms\n";
-
-    std::cout << "D2H copy: "
-              << ms(t_d2h_end - t_d2h_start).count()
-              << " ms\n";
-
-    std::cout << "Device free: "
-              << ms(t_free_end - t_free_start).count()
-              << " ms\n";
-
-    std::cout << "TOTAL: "
-              << ms(t_total_end - t_total_start).count()
-              << " ms\n";
-
-    std::cout << "=============================================\n";
 }
 
-int main(int argc, char** argv) {
-    std::vector<int> sizes = {1000, 4000, 8000, 12000};
-    
-   //std::vector<int> sizes = {8000};	
-   if (argc > 1) {
-        sizes.clear();
-        for (int i = 1; i < argc; ++i) sizes.push_back(std::stoi(argv[i]));
-    }
+int main() {
+    const int N = 1000;
+    std::vector<int> iteration_series = {1000, 2000, 3000, 4000, 5000,
+                                         6000, 7000, 8000, 9000, 10000};
 
     checkCU(cuInit(0), "cuInit");
 
@@ -191,15 +57,7 @@ int main(int argc, char** argv) {
     CUcontext ctx;
     checkCU(cuCtxCreate(&ctx, 0, dev), "cuCtxCreate");
 
-    const std::string ptxPath = "mmul.ptx";
-    std::string ptx;
-
-    try {
-        ptx = readFile(ptxPath);
-    } catch (const std::exception &e) {
-        std::cerr << "Failed to read PTX file '" << ptxPath << "': " << e.what() << "\n";
-        return EXIT_FAILURE;
-    }
+    std::string ptx = readFile("mmul.ptx");
 
     CUmodule module;
     checkCU(cuModuleLoadDataEx(&module, ptx.c_str(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
@@ -207,15 +65,70 @@ int main(int argc, char** argv) {
     CUfunction kernel;
     checkCU(cuModuleGetFunction(&kernel, module, "matrixMul"), "cuModuleGetFunction matrixMul");
 
-    for (int N : sizes) {
-        try {
-            runMatrixMul(module, kernel, N);
-        } catch (const std::exception &e) {
-            std::cerr << "Exception while running N=" << N << ": " << e.what() << "\n";
+    // ----------------------------------
+    // Persistent host buffers
+    // ----------------------------------
+    size_t elements = (size_t)N * N;
+    std::vector<int> h_a(elements, 1);
+    std::vector<int> h_b(elements, 1);
+    std::vector<int> h_c(elements, 0);
+
+    CUstream stream;
+    checkCU(cuStreamCreate(&stream, CU_STREAM_DEFAULT), "cuStreamCreate");
+
+    using clock = std::chrono::steady_clock;
+
+    for (int iterations : iteration_series) {
+        auto start = clock::now();
+
+        for (int i = 0; i < iterations; ++i) {
+            // ----------------------------------
+            // Allocate device memory each iteration
+            // ----------------------------------
+            CUdeviceptr d_a, d_b, d_c;
+            checkCU(cuMemAlloc(&d_a, elements * sizeof(int)), "cuMemAlloc d_a");
+            checkCU(cuMemAlloc(&d_b, elements * sizeof(int)), "cuMemAlloc d_b");
+            checkCU(cuMemAlloc(&d_c, elements * sizeof(int)), "cuMemAlloc d_c");
+
+            // ----------------------------------
+            // Copy persistent host buffers to device
+            // ----------------------------------
+            checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), elements * sizeof(int), stream), "H2D d_a");
+            checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), elements * sizeof(int), stream), "H2D d_b");
+
+            // ----------------------------------
+            // Launch kernel
+            // ----------------------------------
+            launchKernel(kernel, stream, d_a, d_b, d_c, N);
+
+            // ----------------------------------
+            // Copy result back
+            // ----------------------------------
+            checkCU(cuMemcpyDtoHAsync(h_c.data(), d_c, elements * sizeof(int), stream), "D2H d_c");
+
+            // ----------------------------------
+            // Free device memory
+            // ----------------------------------
+            checkCU(cuMemFree(d_a), "cuMemFree d_a");
+            checkCU(cuMemFree(d_b), "cuMemFree d_b");
+            checkCU(cuMemFree(d_c), "cuMemFree d_c");
         }
-        std::cout << "----------------------------------------\n";
+
+        // Synchronize stream after series
+        checkCU(cuStreamSynchronize(stream), "stream sync after series");
+
+        auto end = clock::now();
+        double total_ms = std::chrono::duration<double, std::milli>(end - start).count();
+
+        std::cout << "[SERIES RESULT] Matrix " << N << "x" << N
+                  << ", iterations = " << iterations
+                  << ", total GPU time = " << total_ms << " ms\n";
     }
 
+    // ----------------------------------
+    // Cleanup
+    // ----------------------------------
+    checkCU(cuStreamDestroy(stream), "cuStreamDestroy");
     checkCU(cuModuleUnload(module), "cuModuleUnload");
     checkCU(cuCtxDestroy(ctx), "cuCtxDestroy");
 

From dd4b3f0f14f59c993e24c66450bbbff4b7df7b25 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Sun, 15 Mar 2026 14:59:53 -0600
Subject: [PATCH 0522/1000] Implemented a prototype.

---
 .../mmul_batch_scheduler_behavior.hpp         |  85 +++++
 .../mmul_batch_scheduler_behavior.cpp         | 301 ++++++++++++++++++
 2 files changed, 386 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp
 create mode 100644 libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp
new file mode 100644
index 0000000000..1bf2894a86
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp
@@ -0,0 +1,85 @@
+// ========================= mmul_batch_scheduler_behavior.hpp =========================
+#pragma once
+
+#include "caf/cuda/control-layer/behavior.hpp"
+#include "caf/cuda/control-layer/kernel_graph.hpp"
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/manager.hpp"
+#include "caf/cuda/control-layer/launch_token.hpp"
+
+#include <deque>
+#include <unordered_map>
+#include <vector>
+#include <atomic>
+
+namespace caf::cuda {
+
+class mmul_batch_scheduler_behavior : public scheduler_actor_behavior {
+public:
+    explicit mmul_batch_scheduler_behavior(scheduler_actor_state& state);
+    ~mmul_batch_scheduler_behavior() override;
+
+    void on_enter() override;
+    void schedule() override;
+    void receive(const token_ptr& tok) override;
+
+    void reclaim(int blocks_consumed, int memory_returned, int time, int dependency_number) override;
+    void reclaim(ack& return_msg) override;
+
+    std::string name() const override { return "mmul_batch_scheduler"; }
+
+protected:
+    void process_launch_token(const token_ptr& tok, int stream_id, int assigned_queue);
+
+private:
+    enum queue_type { LOW = 0, MED = 1, HIGH = 2 };
+
+    int small_block_threshold = 64;
+    int medium_block_threshold = 1024;
+
+    int max_concurrent_low = 8;
+    int max_concurrent_med = 4;
+    int max_concurrent_high = 1;
+
+    int low_stream_begin = 0;
+    int low_stream_end = 0;
+    int med_stream_begin = 0;
+    int med_stream_end = 0;
+    int high_stream_begin = 0;
+    int high_stream_end = 0;
+
+    std::atomic<int> low_stream_counter{0};
+    std::atomic<int> med_stream_counter{0};
+    std::atomic<int> high_stream_counter{0};
+
+    std::atomic<int> active_low{0};
+    std::atomic<int> active_med{0};
+    std::atomic<int> active_high{0};
+
+    std::unordered_map<int, queue_type> dispatched_dependency_queue;
+
+    std::deque<graph_ref> low_queue;
+    std::deque<graph_ref> med_queue;
+    std::deque<graph_ref> high_queue;
+
+    device_ptr device_;
+    int num_streams = 0;
+
+    std::unordered_map<int,kernel_graph> graphs;
+    std::vector<kernel_graph> independent_graphs;
+
+    std::unordered_map<int,int> dependency_device_map;
+
+    void init_state();
+    void enqueue_graph_by_blocks(const graph_ref& ref, queue_type forced_qt = LOW);
+    void try_dispatch_queue(std::deque<graph_ref>& q, queue_type qt);
+    kernel_graph* resolve(const graph_ref& ref);
+
+    int get_stream_for_queue(queue_type qt);
+    queue_type classify_blocks(int blocks) const;
+};
+
+} // namespace caf::cuda
+
+
+
diff --git a/libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp b/libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp
new file mode 100644
index 0000000000..0a975c8df3
--- /dev/null
+++ b/libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp
@@ -0,0 +1,301 @@
+
+// ========================= mmul_batch_scheduler_behavior.cpp =========================
+
+#include "caf/cuda/control-layer/behavior.hpp"
+#include "caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp"
+#include "caf/cuda/control-layer/kernel_graph.hpp"
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/manager.hpp"
+#include "caf/cuda/control-layer/launch_token.hpp"
+#include "caf/cuda/control-layer/all-control-layer.hpp"
+
+#include <deque>
+#include <optional>
+#include <algorithm>
+#include <unordered_map>
+#include <vector>
+#include <atomic>
+
+namespace caf::cuda {
+
+// A scheduler tuned for batched matrix-multiply kernels (mmul).
+// Classification is based on the kernel grid block count instead of SM occupancy.
+// Streams are partitioned per-class and each class has a configurable maximum
+// concurrent kernels.
+
+// -------------------- implementation --------------------
+
+mmul_batch_scheduler_behavior::mmul_batch_scheduler_behavior(scheduler_actor_state& state)
+    : scheduler_actor_behavior(state) {
+    init_state();
+}
+
+mmul_batch_scheduler_behavior::~mmul_batch_scheduler_behavior() {}
+
+void mmul_batch_scheduler_behavior::init_state() {
+    device_ = manager::get().find_device(state_.device_number);
+    num_streams = state_.num_streams;
+
+    // default partitioning: LOW:50% of streams, MED:25%, HIGH:25%
+    low_stream_begin = 0;
+    low_stream_end = std::max(1, num_streams / 2);
+
+    med_stream_begin = low_stream_end;
+    med_stream_end = med_stream_begin + std::max(1, num_streams / 4);
+
+    high_stream_begin = med_stream_end;
+    high_stream_end = std::max(high_stream_begin + 1, num_streams);
+
+    // clamp ends to num_streams
+    if (low_stream_end > num_streams) low_stream_end = num_streams;
+    if (med_stream_end > num_streams) med_stream_end = num_streams;
+    if (high_stream_end > num_streams) high_stream_end = num_streams;
+
+    // thresholds and max concurrents can be tuned by editing these members
+    small_block_threshold = 64;
+    medium_block_threshold = 1024;
+
+    max_concurrent_low = 8;
+    max_concurrent_med = 4;
+    max_concurrent_high = 1;
+}
+
+void mmul_batch_scheduler_behavior::on_enter() {
+    // nothing special for now
+}
+
+void mmul_batch_scheduler_behavior::receive(const token_ptr& tok) {
+    if (!tok) return;
+
+    if (tok->getType() == LAUNCH) {
+        // Inspect block count now to choose queue + stream before creating the graph.
+        const auto& launch = static_cast<const launch_token&>(*tok);
+        int blocks = launch.getBlocks();
+        queue_type qt = classify_blocks(blocks);
+        int assigned_stream = get_stream_for_queue(qt);
+
+        if (tok->isIndependent()) {
+            // create independent graph with preselected stream
+            kernel_graph new_graph(state_.device_number, assigned_stream);
+            new_graph.add_operation(tok);
+            independent_graphs.push_back(std::move(new_graph));
+            graph_ref ref{graph_ref::kind_t::independent, -1, static_cast<int>(independent_graphs.size() - 1)};
+            enqueue_graph_by_blocks(ref, qt);
+            schedule();
+            return;
+        }
+
+        int dep = tok->getDependency();
+
+        // If multiple GPUs are in play, forward to the owning device if found
+        if (state_.multiple_gpus) {
+            int dev_num = -1;
+            auto it = dependency_device_map.find(dep);
+            if (it != dependency_device_map.end()) dev_num = it->second;
+
+            if (dev_num != state_.device_number && dev_num != -1) {
+                // preserve stream choice on target side by forwarding token (target will reclassify)
+                anon_mail(tok).send(state_.schedulers[dev_num]);
+                return;
+            }
+        }
+
+        if (graphs.contains(dep)) {
+            // append operation to existing graph; keep the stream already assigned to that graph
+            graphs[dep].add_operation(tok);
+            graph_ref ref{graph_ref::kind_t::dependent, dep};
+            enqueue_graph_by_blocks(ref, qt);
+        } else {
+            // create a new dependent graph, assign the stream we selected earlier
+            kernel_graph new_graph(state_.device_number, assigned_stream);
+            new_graph.add_operation(tok);
+            graphs[dep] = std::move(new_graph);
+            graph_ref ref{graph_ref::kind_t::dependent, dep};
+            // record that we own this dependency on this device
+            dependency_device_map[dep] = state_.device_number;
+            enqueue_graph_by_blocks(ref, qt);
+        }
+
+        schedule();
+
+    } else if (tok->getType() == MEMORY) {
+        process_memory_transfer_token(tok, 0);
+    }
+}
+
+mmul_batch_scheduler_behavior::queue_type mmul_batch_scheduler_behavior::classify_blocks(int blocks) const {
+    if (blocks <= small_block_threshold) return LOW;
+    if (blocks <= medium_block_threshold) return MED;
+    return HIGH;
+}
+
+void mmul_batch_scheduler_behavior::enqueue_graph_by_blocks(const graph_ref& ref, queue_type forced_qt) {
+    kernel_graph* g = resolve(ref);
+    if (!g || g->empty()) return;
+
+    // If caller provided forced_qt, prefer it. Otherwise classify by peeking the front op.
+    queue_type qt = forced_qt;
+
+    // push into the appropriate queue
+    switch (qt) {
+        case LOW:  low_queue.push_back(ref); break;
+        case MED:  med_queue.push_back(ref); break;
+        case HIGH: high_queue.push_back(ref); break;
+    }
+}
+
+int mmul_batch_scheduler_behavior::get_stream_for_queue(queue_type qt) {
+    if (qt == LOW) {
+        int range = std::max(1, low_stream_end - low_stream_begin);
+        int idx = low_stream_counter++ % range;
+        return low_stream_begin + idx;
+    }
+    if (qt == MED) {
+        int range = std::max(1, med_stream_end - med_stream_begin);
+        int idx = med_stream_counter++ % range;
+        return med_stream_begin + idx;
+    }
+    // HIGH
+    int range = std::max(1, high_stream_end - high_stream_begin);
+    int idx = high_stream_counter++ % range;
+    return high_stream_begin + idx;
+}
+
+void mmul_batch_scheduler_behavior::try_dispatch_queue(std::deque<graph_ref>& q, queue_type qt) {
+    while (!q.empty()) {
+        // check concurrency cap for this queue
+        if (qt == LOW && active_low.load() >= max_concurrent_low) break;
+        if (qt == MED && active_med.load() >= max_concurrent_med) break;
+        if (qt == HIGH && active_high.load() >= max_concurrent_high) break;
+
+        graph_ref ref = q.front();
+        kernel_graph* g = resolve(ref);
+        if (!g || g->empty()) {
+            q.pop_front();
+            continue;
+        }
+
+        token_ptr tok = g->peek();
+        if (!tok || tok->getType() != LAUNCH) {
+            q.pop_front();
+            continue;
+        }
+
+        // Use the stream pre-assigned to the graph at creation time
+        int stream = g->stream_id();
+
+        // pop from queue before launching
+        q.pop_front();
+
+        // take the operation from the graph
+        token_ptr op = g->getOperation();
+        if (!op) continue;
+
+        int dep = op->getDependency();
+
+        // increment active counter and record mapping by dependency (if dependent)
+        switch (qt) {
+            case LOW:  active_low.fetch_add(1); break;
+            case MED:  active_med.fetch_add(1); break;
+            case HIGH: active_high.fetch_add(1); break;
+        }
+
+        if (dep != INDEPENDENT) {
+            dispatched_dependency_queue[dep] = qt;
+        } else {
+            // Independent graphs: we don't have a unique dependency id to map on reclaim.
+            // Best-effort: nothing to record. If independent graphs are common, consider
+            // generating a unique id per-independent-graph and setting it in the
+            // launch_response_token's reclaim_dependency_ so reclaim(...) can map it back.
+        }
+
+        process_launch_token(op, stream, static_cast<int>(qt));
+    }
+}
+
+void mmul_batch_scheduler_behavior::schedule() {
+    // Favor small kernels first to keep latency low and allow concurrency
+    try_dispatch_queue(low_queue, LOW);
+    try_dispatch_queue(med_queue, MED);
+    try_dispatch_queue(high_queue, HIGH);
+}
+
+void mmul_batch_scheduler_behavior::process_launch_token(const token_ptr& tok, int stream_id, int assigned_queue) {
+    // Create a launch response token and send it (same pattern as multilevel)
+    const auto& launch = static_cast<const launch_token&>(*tok);
+    auto response = make_launch_response_token(state_.self, launch, state_.device_number, stream_id, /*reclaim_value*/ 0, /*reclaim_runtime*/ 0);
+    anon_mail(response).send(launch.getReplyActor());
+
+    // Note: actual reclaim will arrive via anon_mail(reclaim_value, reclaim_memory, reclaim_runtime, reclaim_dependency)
+    // when launch_response_token::release() runs on the device side. That triggers reclaim(...) in this actor.
+}
+
+void mmul_batch_scheduler_behavior::reclaim(int blocks_consumed,
+                                            int memory_returned,
+                                            int time,
+                                            int dependency_number) {
+    // This reclaim() is called when the device (or the launch_response_token destructor)
+    // sends the 4-tuple (reclaim_value, reclaim_memory, reclaim_runtime, reclaim_dependency).
+
+    // resource tracking (if other components use available_SM/available_memory)
+    //available_SM += blocks_consumed;
+    //available_memory += memory_returned;
+
+    // If reclaim references a dependent graph, decrement the active counter we recorded
+    if (dependency_number != INDEPENDENT) {
+        auto it = dispatched_dependency_queue.find(dependency_number);
+        if (it != dispatched_dependency_queue.end()) {
+            queue_type qt = it->second;
+            dispatched_dependency_queue.erase(it);
+            switch (qt) {
+                case LOW:  active_low.fetch_sub(1); break;
+                case MED:  active_med.fetch_sub(1); break;
+                case HIGH: active_high.fetch_sub(1); break;
+            }
+        }
+
+        // re-enqueue the graph if there are queued operations on that dependency
+        if (graphs.contains(dependency_number)) {
+            graph_ref ref{graph_ref::kind_t::dependent, dependency_number};
+            enqueue_graph_by_blocks(ref);
+        }
+    } else {
+        // Independent graphs
+        // The reclaim reports INDEPENDENT as the dependency. Without a unique ID we can't map
+        // this reclaim back to a specific independent graph's active counter. If independent
+        // graphs are common, add a unique graph id into the reclaim fields in
+        // launch_response_token so we can correctly decrement the active counter here.
+    }
+
+    // attempt to schedule immediately after resources returned
+    schedule();
+}
+
+void mmul_batch_scheduler_behavior::reclaim(ack& return_msg) {
+    // The actor-level ack path: if the device or other parts send specialized acks
+    // (e.g. TIMER, CAF_CUDA_ACK_TRANSFER), handle them here.
+    if (return_msg.getType() == TIMER) {
+        // no-op for now
+        return;
+    } else if (return_msg.getType() == CAF_CUDA_ACK_TRANSFER) {
+        //process_transfer_ack(return_msg);
+    }
+}
+
+kernel_graph* mmul_batch_scheduler_behavior::resolve(const graph_ref& ref) {
+    switch (ref.kind) {
+        case graph_ref::kind_t::dependent: {
+            auto it = graphs.find(ref.dependency);
+            if (it == graphs.end()) return nullptr;
+            return &it->second;
+        }
+        case graph_ref::kind_t::independent: {
+            if (ref.index >= independent_graphs.size()) return nullptr;
+            return &independent_graphs[ref.index];
+        }
+    }
+    return nullptr;
+}
+
+} // namespace caf::cuda
+

From 0682e1c0f36bbd0048f30963c7b83edba7152cd7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 16 Mar 2026 10:20:03 -0600
Subject: [PATCH 0523/1000] Added more CUDA benchmarks.

---
 .../main.test.cpp                             | 102 +++++++++++++++++-
 1 file changed, 101 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
index dbf06ff3e7..45e7938277 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
@@ -421,6 +421,104 @@ void run_mmul_mixed_batch_comparison(
 }
 
 
+// Spawn actors memory-efficiently using counters
+void run_mmul_spawn_counter(
+    actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    const MatrixPool& pool,
+    bool largest_first = false,
+    bool smallest_first = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    const int THREADS = 32;
+
+    // Determine spawn order
+    std::vector<int> spawn_order = sizes;
+    if (largest_first) std::sort(spawn_order.rbegin(), spawn_order.rend());
+    if (smallest_first) std::sort(spawn_order.begin(), spawn_order.end());
+
+    // Initialize counters
+    std::unordered_map<int,int> spawned_count;
+    for (auto N : spawn_order) spawned_count[N] = 0;
+
+    int total_spawned = 0;
+    int num_sizes = spawn_order.size();
+    int base_quota = num_actors / num_sizes;
+    int remainder = num_actors % num_sizes;
+
+    for (size_t idx = 0; idx < spawn_order.size(); ++idx) {
+        int N = spawn_order[idx];
+        int limit = base_quota + (idx == spawn_order.size() - 1 ? remainder : 0);
+
+        while (spawned_count[N] < limit && total_spawned < num_actors) {
+            const auto& A = pool.A.at(N);
+            const auto& B = pool.B.at(N);
+            caf::cuda::nd_range dims((N+THREADS-1)/THREADS, (N+THREADS-1)/THREADS, 1, THREADS, THREADS, 1);
+
+sys.spawn(mmul_actor_fun, program, A, B, N);
+            spawned_count[N]++;
+            total_spawned++;
+        }
+    }
+
+    sys.await_all_actors_done();
+}
+
+
+void run_actor_spawn_order_comparison(actor_system& sys) {
+    std::vector<int> sizes = {32,64,128,256,512,1024,2048};
+    int num_actors = 5000;
+    MatrixPool pool = create_matrix_pool(sizes);
+
+    std::cout << "\n=== Actor Spawn Order Comparison ===\n";
+    std::cout << "order num_actors time_seconds\n";
+
+    // Round-robin
+    {
+        caf::cuda::manager_config cfg(false);
+        caf::cuda::manager::init(sys, cfg);
+         double t = time_run([&] {
+                run_mmul_mixed_batch_cuda_scheduler(sys, sizes, num_actors , pool);
+            });
+        std::cout << "round_robin " << num_actors << " " << t << "\n";
+        caf::cuda::manager::shutdown();
+    }
+
+    // Largest-first
+    {
+        caf::cuda::manager_config cfg(false);
+        caf::cuda::manager::init(sys, cfg);
+        double t = time_run([&] {
+            run_mmul_spawn_counter(sys, sizes, num_actors, pool, true, false);
+        });
+        std::cout << "largest_first " << num_actors << " " << t << "\n";
+        caf::cuda::manager::shutdown();
+    }
+
+    // Smallest-first
+    {
+        caf::cuda::manager_config cfg(false);
+        caf::cuda::manager::init(sys, cfg);
+        double t = time_run([&] {
+            run_mmul_spawn_counter(sys, sizes, num_actors, pool, false, true);
+        });
+        std::cout << "smallest_first " << num_actors << " " << t << "\n";
+        caf::cuda::manager::shutdown();
+    }
+
+    std::cout << "=== Spawn Order Comparison Complete ===\n";
+}
+
+
+
+
+
+
+
+
 
 void caf_main(caf::actor_system& sys) {
   
@@ -428,7 +526,9 @@ void caf_main(caf::actor_system& sys) {
   caf::cuda::manager_config man_config(true);
   //caf::cuda::manager::init(sys,man_config);
 
-  run_mmul_mixed_batch_comparison(sys);
+ // run_mmul_mixed_batch_comparison(sys);
+
+  run_actor_spawn_order_comparison(sys)
 
 }
 

From 1e77d8d8c9dac2b17d96972d4e64455d348f5869 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 16 Mar 2026 10:20:27 -0600
Subject: [PATCH 0524/1000] FIxed compiler error.

---
 .../mmul-non-uniform-batch-benchmark/main.test.cpp              | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
index 45e7938277..5ffe36aabc 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
@@ -528,7 +528,7 @@ void caf_main(caf::actor_system& sys) {
 
  // run_mmul_mixed_batch_comparison(sys);
 
-  run_actor_spawn_order_comparison(sys)
+  run_actor_spawn_order_comparison(sys);
 
 }
 

From ee7d426b184fe4bc1107c9604bbf70d4a4657798 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 16 Mar 2026 15:53:21 -0600
Subject: [PATCH 0525/1000] Saving.

---
 libcaf_cuda/src/control-layer/behavior_table.cpp     |  2 ++
 .../control-layer/mmul_batch_scheduler_behavior.cpp  | 12 ++++++------
 .../baseline-comparison/actors/main.test.cpp         |  2 +-
 .../mmul-non-uniform-batch-benchmark/main.test.cpp   | 10 +++++-----
 4 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/behavior_table.cpp b/libcaf_cuda/src/control-layer/behavior_table.cpp
index 2503ea91dc..9866bda5d3 100644
--- a/libcaf_cuda/src/control-layer/behavior_table.cpp
+++ b/libcaf_cuda/src/control-layer/behavior_table.cpp
@@ -9,6 +9,7 @@
 #include "caf/cuda/control-layer/core_usage_behavior.hpp"
 #include "caf/cuda/control-layer/single_usage_behavior.hpp"
 #include "caf/cuda/control-layer/multilevel_usage_behavior.hpp"
+#include "caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp"
 #include "caf/cuda/control-layer/pressure_scheduler.hpp"
 #include "caf/cuda/control-layer/all-control-layer.hpp"
 
@@ -23,6 +24,7 @@ namespace caf::cuda {
         add("single_usage", new single_usage_behavior(state));
         add("multilevel",   new multilevel_usage_behavior(state));
         add("pressure",     new pressure_scheduler(state));
+        add("mmul",     new mmul_batch_scheduler_behavior(state));
     }
 
 
diff --git a/libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp b/libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp
index 0a975c8df3..0a685c05e7 100644
--- a/libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp
@@ -34,7 +34,7 @@ mmul_batch_scheduler_behavior::~mmul_batch_scheduler_behavior() {}
 
 void mmul_batch_scheduler_behavior::init_state() {
     device_ = manager::get().find_device(state_.device_number);
-    num_streams = state_.num_streams;
+    num_streams = 128;
 
     // default partitioning: LOW:50% of streams, MED:25%, HIGH:25%
     low_stream_begin = 0;
@@ -52,12 +52,12 @@ void mmul_batch_scheduler_behavior::init_state() {
     if (high_stream_end > num_streams) high_stream_end = num_streams;
 
     // thresholds and max concurrents can be tuned by editing these members
-    small_block_threshold = 64;
-    medium_block_threshold = 1024;
+    small_block_threshold = 128;
+    medium_block_threshold = 512;
 
-    max_concurrent_low = 8;
-    max_concurrent_med = 4;
-    max_concurrent_high = 1;
+    max_concurrent_low = 20000;
+    max_concurrent_med = 16000;
+    max_concurrent_high = 60000;
 }
 
 void mmul_batch_scheduler_behavior::on_enter() {
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
index c5a485896b..c52d09612b 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
@@ -184,7 +184,7 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size,int iterations) {
       std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
 
   std::cout << "[MMUL TEST] matrix_size=" << matrix_size
-            << "iterations = " << iterations << 
+            << " iterations = " << iterations << 
 	    ", time=" << duration_ms << " ms\n";
 
   caf::cuda::manager::shutdown();
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
index 5ffe36aabc..6996503d48 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-non-uniform-batch-benchmark/main.test.cpp
@@ -458,10 +458,10 @@ void run_mmul_spawn_counter(
             const auto& B = pool.B.at(N);
             caf::cuda::nd_range dims((N+THREADS-1)/THREADS, (N+THREADS-1)/THREADS, 1, THREADS, THREADS, 1);
 
-sys.spawn(mmul_actor_fun, program, A, B, N);
-            spawned_count[N]++;
-            total_spawned++;
-        }
+	    sys.spawn(mmul_actor_fun, program, A, B, N);
+	    spawned_count[N]++;
+	    total_spawned++;
+	}
     }
 
     sys.await_all_actors_done();
@@ -469,7 +469,7 @@ sys.spawn(mmul_actor_fun, program, A, B, N);
 
 
 void run_actor_spawn_order_comparison(actor_system& sys) {
-    std::vector<int> sizes = {32,64,128,256,512,1024,2048};
+    std::vector<int> sizes = {10,32,64,128,256,512,1024,2048};
     int num_actors = 5000;
     MatrixPool pool = create_matrix_pool(sizes);
 

From c366deabf28da2580e0b0dfda0e9c5820f022ae1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 16 Mar 2026 16:14:18 -0600
Subject: [PATCH 0526/1000] Implemented a feature that toggles off return
 messages, since we could possibly flood the scheduler actor with too many
 return messages causing a slowdown.

---
 .../control-layer/launch_response_token.hpp   | 41 +++++++++++--------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
index ccad9be3b5..8554c941be 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -28,7 +28,8 @@ class CAF_CUDA_EXPORT launch_response_token : public response_token {
                           int reclaim_value = 0,
                           int reclaim_memory_returned = 0,
                           int reclaim_runtime = 0,
-                          int reclaim_dependency = 0)
+                          int reclaim_dependency = 0,
+                          bool send_mail = true)
         : response_token(std::move(receiver), device_num, stream_id, memory_usage),
           range_(std::move(range)),
           id_(std::move(id)),
@@ -36,7 +37,8 @@ class CAF_CUDA_EXPORT launch_response_token : public response_token {
           reclaim_value_(reclaim_value),
           reclaim_memory_returned_(reclaim_memory_returned),
           reclaim_runtime_(reclaim_runtime),
-          reclaim_dependency_(reclaim_dependency) {}
+          reclaim_dependency_(reclaim_dependency),
+          send_mail_(send_mail) {}
 
     // Construct from a launch_token
     // Memory returned and dependency are copied from launch_token
@@ -45,16 +47,17 @@ class CAF_CUDA_EXPORT launch_response_token : public response_token {
                           int device_num,
                           int stream_id,
                           int reclaim_value = 0,
-                          int reclaim_runtime = 0)
+                          int reclaim_runtime = 0,
+                          bool send_mail = true)
         : response_token(std::move(receiver), device_num, stream_id, token.getMemoryUsage()),
           range_(token.getRange()),
           id_(token.getId()),
           released_(false),
           reclaim_value_(reclaim_value),
-          reclaim_memory_returned_(token.getMemoryUsage()), // copy from launch_token
+          reclaim_memory_returned_(token.getMemoryUsage()),
           reclaim_runtime_(reclaim_runtime),
-          reclaim_dependency_(token.getDependency())        // copy from launch_token
-    {}
+          reclaim_dependency_(token.getDependency()),
+          send_mail_(send_mail) {}
 
     ~launch_response_token() {
         release();
@@ -65,8 +68,9 @@ class CAF_CUDA_EXPORT launch_response_token : public response_token {
     const nd_range& getRange() const { return range_; }
 
     const std::string& getId() const { return id_; }
-    const std::string& name() const override {return id_;}
-    
+
+    const std::string& name() const override { return id_; }
+
     // Return requested number of CUDA blocks
     int getBlocks() const {
         return static_cast<int>(
@@ -84,13 +88,17 @@ class CAF_CUDA_EXPORT launch_response_token : public response_token {
             return; // already released
         }
 
+        if (!send_mail_) {
+            return; // mail sending disabled
+        }
+
         try {
-            // Send a message containing all four reclaim fields
             caf::anon_mail(
-                           reclaim_value_,
-                           reclaim_memory_returned_,
-                           reclaim_runtime_,
-                           reclaim_dependency_).urgent().send(receiver_);
+                reclaim_value_,
+                reclaim_memory_returned_,
+                reclaim_runtime_,
+                reclaim_dependency_
+            ).urgent().send(receiver_);
         } catch (...) {
             // swallow exceptions — destructor safe
         }
@@ -102,20 +110,21 @@ class CAF_CUDA_EXPORT launch_response_token : public response_token {
     int reclaim_runtime() const { return reclaim_runtime_; }
     int reclaim_dependency() const { return reclaim_dependency_; }
 
-
 private:
     nd_range range_;
     std::string id_;
-    std::atomic<bool> released_;
+    std::atomic<bool> released_{false};
 
     // Reclaim fields
     int reclaim_value_ = 0;
     int reclaim_memory_returned_ = 0;
     int reclaim_runtime_ = 0;
     int reclaim_dependency_ = 0;
+
+    // Toggle for sending anon_mail
+    bool send_mail_ = true;
 };
 
 using kernel_launch_token = caf::intrusive_ptr<launch_response_token>;
 
 } // namespace caf::cuda
-

From f01ebaa5a25f278ff20460ef6ffad850dcc05f1a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 16 Mar 2026 16:14:52 -0600
Subject: [PATCH 0527/1000] Toggled off return messages from green_light
 behavior since it does not need them.

---
 .../control-layer/green_light_behavior.cpp    | 47 +++++++++----------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
index 1a595cd73c..4d2e3d0ae8 100644
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
@@ -24,36 +24,35 @@ void green_light_behavior::schedule() {
 }
 
 void green_light_behavior::receive(const token_ptr& tok) {
-   
 
-//	std::cout << "Green light receive\n";
+//  std::cout << "Green light receive\n";
 
-     if (tok->getType() == LAUNCH) {
-        //use 0 as stream id for now, eventually will have to figure out
-        //stream load balancing
-        process_launch_token(tok, rand()% state_.num_streams);
+    if (tok->getType() == LAUNCH) {
+
+        const auto& launch = static_cast<const launch_token&>(*tok);
+
+        int stream_id = rand() % state_.num_streams;
+
+        // Manually construct launch_response_token with anon_mail disabled
+        response_token_ptr response(
+            new launch_response_token(
+                state_.self,
+                launch,
+                state_.device_number,
+                stream_id,
+                0,      // reclaim_value
+                0,      // reclaim_runtime
+                false   // send_mail disabled
+            )
+        );
+
+        anon_mail(response).send(launch.getReplyActor());
+
+        //(void)response; // suppress unused warning
     }
     else if (tok->getType() == MEMORY) {
-        //use 0 as stream id for now, eventually will have to figure out
-        //stream load balancing
         process_memory_transfer_token(tok, 0);
     }
-    //this may cause an issue if a message is never received then
-    //we may never end up dequeueing certain requests
-    //may lead to a deadlock scenario?
-   /*
-    while (!state_.queue.empty()) {
-        token_ptr queued = state_.queue.front();
-        state_.queue.pop();
-        if (queued->getType() == LAUNCH) {
-            // safe: we've checked the runtime type
-            process_launch_token(queued, 0);
-        }
-        else if (queued->getType() == MEMORY) {
-            process_memory_transfer_token(queued, 0);
-        }
-    }
-    */
 }
 
 

From c637a999529e782823062039be5c59490cc6d392 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 17 Mar 2026 15:33:24 -0600
Subject: [PATCH 0528/1000] Initial commit.

---
 .../CMakeLists.txt                            |  44 ++
 .../compile_kernels.sh                        |  13 +
 .../main.test.cpp                             | 538 ++++++++++++++++++
 .../mmul-randonom-batch-benchmark/mmul.cu     |  16 +
 4 files changed, 611 insertions(+)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/mmul.cu

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
new file mode 100644
index 0000000000..e2259ce5fe
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/compile_kernels.sh b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/compile_kernels.sh
new file mode 100755
index 0000000000..f32480e5cb
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/compile_kernels.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
new file mode 100644
index 0000000000..dd0dd2f7b6
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -0,0 +1,538 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/component-actors/all-component-actors.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+using command =
+  caf::cuda::command_runner<>;
+
+command mmul_command;
+caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int >> mmul;
+using async_command = caf::cuda::mmul_async_command<int>;
+async_command async_mmul;
+
+
+
+
+
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
+};
+
+MatrixPool create_matrix_pool(const std::vector<int>& sizes) {
+    MatrixPool pool;
+
+    for (int N : sizes) {
+        pool.A[N] = std::vector<int>(N*N, 1);
+        pool.B[N] = std::vector<int>(N*N, 1);
+    }
+
+    return pool;
+}
+
+
+
+
+
+
+
+
+
+struct mmul_state {
+
+	caf::cuda::program_ptr mmul_kernel;
+
+};
+
+
+
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::program_ptr mmul_kernel,
+		const std::vector<int>& matrix1,
+		const std::vector<int>& matrix2,
+		int N) {
+
+	self->state().mmul_kernel = mmul_kernel;
+	self->mail(matrix1, matrix2, N).send(self);
+	return {
+		[=](const std::vector<int>& matrixA,
+				const std::vector<int>& matrixB,
+				int N) {
+
+			caf::cuda::manager& mgr = caf::cuda::manager::get();
+			int device = 0;
+			int stream = rand();
+
+			//auto program =
+			//mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+		    const int THREADS = 32;
+		    const int BLOCKS = (N + THREADS - 1) / THREADS;
+		    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+
+			auto program = self->state().mmul_kernel;
+
+			auto inA = caf::cuda::create_in_arg(std::move(matrixA));
+			auto arg1 = mmul_command.transfer_memory(
+					device,
+					stream,
+					std::move(inA));
+
+			auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+			auto arg2 = mmul_command.transfer_memory(
+					device,
+					stream,
+					std::move(inB));
+
+			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
+			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
+
+			auto result = 
+				async_mmul.run_async(
+						program,
+						dims,
+						stream,
+						0,
+						device,
+						arg1,
+						arg2,
+						arg3,
+						arg4);
+
+			std::get<2>(result) -> copy_to_host();
+			self -> quit();
+
+		}
+	};
+}
+
+
+
+
+struct mmul_actor_with_scheduler_state {
+  static inline const char* name = "my_actor";
+};
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun_scheduler(
+    caf::stateful_actor<mmul_actor_with_scheduler_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims,
+    const std::vector<int>& matrix1,
+    const std::vector<int> & matrix2)
+{
+
+
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+        //caf::actor scheduler = mgr.get_scheduler_actor();
+
+        //send a launch token
+        caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+                        program,
+                        dims,
+                        0,
+                        "hello",
+                        self,
+			rand() //dependency number, can declare indepedent but want to see what happens when you do not 
+                        );
+        mgr.send_scheduler_actor_message(launch_token);
+
+	return {
+
+		// 1. Handle response token
+		[=](caf::cuda::response_token_ptr res_token) {
+			//        std::cout << "Got response\n";
+
+			if (res_token->getType() == LAUNCH_RESPONSE) {
+				self->mail(matrix1, matrix2, res_token, N).send(self);
+
+			} else {
+				//          std::cout << "Got a memory response token\n";
+			}
+		},
+
+	    // 2. Handle memory buffers -> GPU
+	    [=](const std::vector<int>& matrixA,
+			    const std::vector<int>& matrixB,
+			    const caf::cuda::response_token_ptr& res_token,
+			    int N) {
+
+		    //    std::cout << "Working\n";
+		    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+
+		    const int THREADS = 32;
+		    const int BLOCKS = (N + THREADS - 1) / THREADS;
+		    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+		    auto arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixA));
+		    auto arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixB));
+		    auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
+		    auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
+
+
+		    auto tempC = mmul.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
+		    caf::cuda::mem_ptr<int> bufferC = std::get<2>(tempC);
+
+		    bufferC -> synchronize();
+		    res_token->release();
+		    bufferC->copy_to_host();
+
+		    self->mail(1).send(exit_actor);
+		    self->quit();
+         }
+
+
+       };
+
+}
+
+
+template <class Fn>
+double time_run(Fn&& fn) {
+  auto start = std::chrono::steady_clock::now();
+  fn();
+  auto end = std::chrono::steady_clock::now();
+  std::chrono::duration<double> elapsed = end - start;
+  return elapsed.count();
+}
+
+
+
+void run_mmul_mixed_batch_cuda_scheduler(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    MatrixPool pool,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    std::mt19937 rng(123456);
+    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+
+    for (int i = 0; i < num_actors; ++i) {
+
+        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
+
+	const auto& A = pool.A[N];
+	const auto& B = pool.B[N];
+
+        sys.spawn(
+            mmul_actor_fun,
+            program,
+            A,
+	    B,
+	    N);
+    }
+
+    sys.await_all_actors_done();
+}
+
+
+void run_mmul_mixed_batch_caf_cuda_scheduler(
+    caf::actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    MatrixPool pool,
+    bool FCFS,
+    bool randomize = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+    //set the scheduler actor behavior
+    if (FCFS) {
+	    for (int i = 0; i < mgr.get_num_devices(); i++) {
+
+		    mgr.send_scheduler_actor_message("green",i);
+
+	    }
+
+    }
+    else { 
+	    for (int i = 0; i < mgr.get_num_devices(); i++) {
+
+		    mgr.send_scheduler_actor_message("multilevel",i);
+
+	    }
+    }
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
+
+    std::mt19937 rng(123456);
+    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+
+    const int THREADS = 32;
+// timestamp before actor creation
+auto start_time = std::chrono::high_resolution_clock::now();
+
+for (int i = 0; i < num_actors; ++i) {
+
+    int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
+    int BLOCKS = (N + THREADS - 1) / THREADS;
+
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    const auto& A = pool.A[N];
+    const auto& B = pool.B[N];
+  //time the single actor creation
+   //auto start = std::chrono::high_resolution_clock::now();
+
+    caf::actor a = sys.spawn(
+        mmul_actor_fun_scheduler,
+        exit_actor,
+        N,
+        program,
+        dims,
+        A,
+        B
+    );
+
+    //auto end = std::chrono::high_resolution_clock::now();
+    //auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+   ///std::cout << "[INFO] Actor " << i << " creation time: " << ms << " ms\n";
+}
+
+// timestamp after actor creation
+auto end_time = std::chrono::high_resolution_clock::now();
+auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();
+
+std::cout << "[INFO] Actor creation time for " << num_actors
+          << " actors: " << ms << " ms\n";
+    sys.await_all_actors_done();
+}
+
+
+
+
+
+
+
+
+void run_mmul_mixed_batch_comparison(
+    caf::actor_system& sys)
+{
+    std::vector<int> sizes = {32,64,128,256,512,1024};
+    std::vector<int> actor_counts = {5000};
+    MatrixPool pool = create_matrix_pool(sizes);
+
+
+
+    std::cout << "\n=== MMUL Mixed Batch Comparison ===\n";
+    std::cout << "scheduler actors time_seconds\n";
+
+    for (auto actors : actor_counts) {
+
+        /* ========= core_usage BULK ========= */
+
+        {
+            caf::cuda::manager_config cfg(true);
+            caf::cuda::manager::init(sys, cfg);
+
+            double t = time_run([&] {
+                run_mmul_mixed_batch_caf_cuda_scheduler(sys, sizes, actors,pool,false);
+            });
+
+            std::cout << "RESULT CAF CUDA DEFAULT SCHEDULER "
+                      << actors << " "
+                      << t << "seconds\n";
+
+            caf::cuda::manager::shutdown();
+        }
+        /* ========= green light BULK  ========= */
+
+        {
+            caf::cuda::manager_config cfg(true);
+            caf::cuda::manager::init(sys, cfg);
+
+            double t = time_run([&] {
+                run_mmul_mixed_batch_caf_cuda_scheduler(sys, sizes, actors,pool,true);
+            });
+
+            std::cout << "RESULT CAF CUDA FCFS SCHEDULER "
+                      << actors << " "
+                      << t << "seconds\n";
+
+            caf::cuda::manager::shutdown();
+        }
+
+
+        /* ========= no scheduler ========= */
+
+        {
+            caf::cuda::manager_config cfg(false);
+            caf::cuda::manager::init(sys, cfg);
+
+            double t = time_run([&] {
+                run_mmul_mixed_batch_cuda_scheduler(sys, sizes, actors,pool);
+            });
+
+            std::cout << "RESULT CUDA SCHEDULER "
+                      << actors << " "
+                      << t << "seconds\n";
+
+            caf::cuda::manager::shutdown();
+        }
+    }
+
+    std::cout << "\n=== Mixed Batch Comparison Complete ===\n";
+}
+
+
+// Spawn actors memory-efficiently using counters
+void run_mmul_spawn_counter(
+    actor_system& sys,
+    const std::vector<int>& sizes,
+    int num_actors,
+    const MatrixPool& pool,
+    bool largest_first = false,
+    bool smallest_first = false)
+{
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    const int THREADS = 32;
+
+    // Determine spawn order
+    std::vector<int> spawn_order = sizes;
+    if (largest_first) std::sort(spawn_order.rbegin(), spawn_order.rend());
+    if (smallest_first) std::sort(spawn_order.begin(), spawn_order.end());
+
+    // Initialize counters
+    std::unordered_map<int,int> spawned_count;
+    for (auto N : spawn_order) spawned_count[N] = 0;
+
+    int total_spawned = 0;
+    int num_sizes = spawn_order.size();
+    int base_quota = num_actors / num_sizes;
+    int remainder = num_actors % num_sizes;
+
+    for (size_t idx = 0; idx < spawn_order.size(); ++idx) {
+        int N = spawn_order[idx];
+        int limit = base_quota + (idx == spawn_order.size() - 1 ? remainder : 0);
+
+        while (spawned_count[N] < limit && total_spawned < num_actors) {
+            const auto& A = pool.A.at(N);
+            const auto& B = pool.B.at(N);
+            caf::cuda::nd_range dims((N+THREADS-1)/THREADS, (N+THREADS-1)/THREADS, 1, THREADS, THREADS, 1);
+
+	    sys.spawn(mmul_actor_fun, program, A, B, N);
+	    spawned_count[N]++;
+	    total_spawned++;
+	}
+    }
+
+    sys.await_all_actors_done();
+}
+
+
+void run_actor_spawn_order_comparison(actor_system& sys) {
+    std::vector<int> sizes = {10,32,64,128,256,512,1024,2048};
+    int num_actors = 5000;
+    MatrixPool pool = create_matrix_pool(sizes);
+
+    std::cout << "\n=== Actor Spawn Order Comparison ===\n";
+    std::cout << "order num_actors time_seconds\n";
+
+    // Round-robin
+    {
+        caf::cuda::manager_config cfg(false);
+        caf::cuda::manager::init(sys, cfg);
+         double t = time_run([&] {
+                run_mmul_mixed_batch_cuda_scheduler(sys, sizes, num_actors , pool);
+            });
+        std::cout << "round_robin " << num_actors << " " << t << "\n";
+        caf::cuda::manager::shutdown();
+    }
+
+    // Largest-first
+    {
+        caf::cuda::manager_config cfg(false);
+        caf::cuda::manager::init(sys, cfg);
+        double t = time_run([&] {
+            run_mmul_spawn_counter(sys, sizes, num_actors, pool, true, false);
+        });
+        std::cout << "largest_first " << num_actors << " " << t << "\n";
+        caf::cuda::manager::shutdown();
+    }
+
+    // Smallest-first
+    {
+        caf::cuda::manager_config cfg(false);
+        caf::cuda::manager::init(sys, cfg);
+        double t = time_run([&] {
+            run_mmul_spawn_counter(sys, sizes, num_actors, pool, false, true);
+        });
+        std::cout << "smallest_first " << num_actors << " " << t << "\n";
+        caf::cuda::manager::shutdown();
+    }
+
+    std::cout << "=== Spawn Order Comparison Complete ===\n";
+}
+
+
+
+
+
+
+
+
+
+void caf_main(caf::actor_system& sys) {
+  
+
+  caf::cuda::manager_config man_config(true);
+  //caf::cuda::manager::init(sys,man_config);
+
+ run_mmul_mixed_batch_comparison(sys);
+
+  //run_actor_spawn_order_comparison(sys);
+
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/mmul.cu b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From f12f06643d4cf4143333280014eed44ada7cdb45 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Mar 2026 14:45:00 -0600
Subject: [PATCH 0529/1000] Made the matrix pool generation random.

---
 .../main.test.cpp                             | 20 +++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index dd0dd2f7b6..2ca68a3ab2 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -10,6 +10,7 @@
 #include <numeric>
 #include <random>
 #include "caf/actor_registry.hpp"
+#include <random>
 //#include <caf/atoms.hpp>
 
 
@@ -52,12 +53,23 @@ struct MatrixPool {
     std::unordered_map<int, std::vector<int>> B;
 };
 
-MatrixPool create_matrix_pool(const std::vector<int>& sizes) {
+
+MatrixPool create_matrix_pool_random(
+    int num_sizes,
+    int min_N,
+    int max_N,
+    unsigned int seed
+) {
     MatrixPool pool;
 
-    for (int N : sizes) {
-        pool.A[N] = std::vector<int>(N*N, 1);
-        pool.B[N] = std::vector<int>(N*N, 1);
+    std::mt19937 rng(seed);  // deterministic RNG
+    std::uniform_int_distribution<int> dist(min_N, max_N);
+
+    for (int i = 0; i < num_sizes; ++i) {
+        int N = dist(rng);
+
+        pool.A[N] = std::vector<int>(N * N, 1);
+        pool.B[N] = std::vector<int>(N * N, 1);
     }
 
     return pool;

From bd7930a503cdd0c6085069991cc71d2296321726 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Mar 2026 14:46:11 -0600
Subject: [PATCH 0530/1000] Made the matrix pool generation random.

---
 .../mmul-randonom-batch-benchmark/main.test.cpp   | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 2ca68a3ab2..10905ae50e 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -54,6 +54,8 @@ struct MatrixPool {
 };
 
 
+#include <unordered_set>
+
 MatrixPool create_matrix_pool_random(
     int num_sizes,
     int min_N,
@@ -62,14 +64,17 @@ MatrixPool create_matrix_pool_random(
 ) {
     MatrixPool pool;
 
-    std::mt19937 rng(seed);  // deterministic RNG
+    std::mt19937 rng(seed);
     std::uniform_int_distribution<int> dist(min_N, max_N);
 
-    for (int i = 0; i < num_sizes; ++i) {
-        int N = dist(rng);
+    std::unordered_set<int> used;
 
-        pool.A[N] = std::vector<int>(N * N, 1);
-        pool.B[N] = std::vector<int>(N * N, 1);
+    while (used.size() < static_cast<size_t>(num_sizes)) {
+        int N = dist(rng);
+        if (used.insert(N).second) {
+            pool.A[N] = std::vector<int>(N * N, 1);
+            pool.B[N] = std::vector<int>(N * N, 1);
+        }
     }
 
     return pool;

From 7f22953d7ad2557fe59170d882b6eb76856088a6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Mar 2026 14:59:45 -0600
Subject: [PATCH 0531/1000] Updated actor signatures.

---
 .../main.test.cpp                             | 29 +++++++++----------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 10905ae50e..25c079e815 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -11,6 +11,7 @@
 #include <random>
 #include "caf/actor_registry.hpp"
 #include <random>
+#include <unordered_set>
 //#include <caf/atoms.hpp>
 
 
@@ -54,7 +55,6 @@ struct MatrixPool {
 };
 
 
-#include <unordered_set>
 
 MatrixPool create_matrix_pool_random(
     int num_sizes,
@@ -97,16 +97,15 @@ struct mmul_state {
 
 
 caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::program_ptr mmul_kernel,
-		const std::vector<int>& matrix1,
-		const std::vector<int>& matrix2,
+		const in<int> matrixA,
+		const in<int> matrixB,
 		int N) {
 
 	self->state().mmul_kernel = mmul_kernel;
 	self->mail(matrix1, matrix2, N).send(self);
 	return {
-		[=](const std::vector<int>& matrixA,
-				const std::vector<int>& matrixB,
-				int N) {
+		
+		[=](int N) {
 
 			caf::cuda::manager& mgr = caf::cuda::manager::get();
 			int device = 0;
@@ -122,13 +121,13 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
 
 			auto program = self->state().mmul_kernel;
 
-			auto inA = caf::cuda::create_in_arg(std::move(matrixA));
+			auto inA = std::move(matrixA);
 			auto arg1 = mmul_command.transfer_memory(
 					device,
 					stream,
 					std::move(inA));
 
-			auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+			auto inB = std::move(matrixB);
 			auto arg2 = mmul_command.transfer_memory(
 					device,
 					stream,
@@ -171,8 +170,8 @@ caf::behavior mmul_actor_fun_scheduler(
     int N,
     caf::cuda::program_ptr program,
     caf::cuda::nd_range dims,
-    const std::vector<int>& matrix1,
-    const std::vector<int> & matrix2)
+    const in<int> matrixA,
+    const in<int> matrixB)
 {
 
 
@@ -198,7 +197,7 @@ caf::behavior mmul_actor_fun_scheduler(
 			//        std::cout << "Got response\n";
 
 			if (res_token->getType() == LAUNCH_RESPONSE) {
-				self->mail(matrix1, matrix2, res_token, N).send(self);
+				self->mail(res_token, N).send(self);
 
 			} else {
 				//          std::cout << "Got a memory response token\n";
@@ -206,9 +205,7 @@ caf::behavior mmul_actor_fun_scheduler(
 		},
 
 	    // 2. Handle memory buffers -> GPU
-	    [=](const std::vector<int>& matrixA,
-			    const std::vector<int>& matrixB,
-			    const caf::cuda::response_token_ptr& res_token,
+	    [=] (const caf::cuda::response_token_ptr& res_token,
 			    int N) {
 
 		    //    std::cout << "Working\n";
@@ -219,8 +216,8 @@ caf::behavior mmul_actor_fun_scheduler(
 		    const int BLOCKS = (N + THREADS - 1) / THREADS;
 		    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-		    auto arg1 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixA));
-		    auto arg2 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(matrixB));
+		    auto arg1 = mmul.transfer_memory(res_token, matrixA);
+		    auto arg2 = mmul.transfer_memory(res_token, matrixB);
 		    auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
 		    auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
 

From 0ccc5f67f986b1e782731c17ab70653188d2fd66 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Mar 2026 15:20:23 -0600
Subject: [PATCH 0532/1000] Wrote some supervisor code.

---
 .../main.test.cpp                             | 69 ++++++++++++++++++-
 1 file changed, 68 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 25c079e815..1506a4038f 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -83,6 +83,73 @@ MatrixPool create_matrix_pool_random(
 
 
 
+struct supervisor_actor_state {
+	int num_actors;
+	int num_waves;
+	int completed;
+	int max_waves;
+	MatrixPool pool;
+};
+
+
+
+//runs for FCFS behavior
+caf::behavior supervisor_actor_fun(caf::stateful_actor<supervisor_actor_state>* self, int num_actors,
+		int max_waves,
+		MatrixPool pool) {
+
+	self -> state().num_actors = num_actors;
+	self -> state().completed = 0;
+	self->state().max_waves = max_waves;
+	self->state().num_waves = 0;
+	self -> state().pool = pool;
+
+	self->mail("spawn").send(self);
+
+	return {
+
+		[=](std::string command) {
+		
+			if (command == "spawn") {
+		
+
+
+
+
+			
+			}
+		
+		
+		},
+		[=](int completed) {
+		
+			self->state().completed+=completed;
+
+			if (self->state().completed >= self->state().limit) {
+			
+				self->state().num_waves +=1;
+
+				if (self->state().num_waves >= self->state().max_waves) {
+				
+					caf::cuda::manager::shutdown();
+					self -> quit();
+				
+				}
+				else {
+				
+					self->mail("spawn").send(self);
+				
+				}
+			
+			}
+		
+		}
+
+
+	};
+
+
+}
 
 
 
@@ -102,7 +169,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
 		int N) {
 
 	self->state().mmul_kernel = mmul_kernel;
-	self->mail(matrix1, matrix2, N).send(self);
+	self->mail(N).send(self);
 	return {
 		
 		[=](int N) {

From 9e840ee15ee46c2713e7e241fd5b6aacdcf4a822 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Mar 2026 15:29:03 -0600
Subject: [PATCH 0533/1000] Updated supervisor to correcty spawn
 mmul_scheduler_actors.

---
 .../main.test.cpp                             | 417 ++++--------------
 1 file changed, 89 insertions(+), 328 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 1506a4038f..47661b6ad4 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -82,73 +82,113 @@ MatrixPool create_matrix_pool_random(
 
 
 
-
 struct supervisor_actor_state {
-	int num_actors;
-	int num_waves;
-	int completed;
-	int max_waves;
-	MatrixPool pool;
+    int num_actors;
+    int num_waves;
+    int completed;
+    int max_waves;
+
+    MatrixPool pool;
+
+    std::vector<int> sizes;   // cached keys
+    std::mt19937 rng;         // RNG
 };
 
 
 
 //runs for FCFS behavior
-caf::behavior supervisor_actor_fun(caf::stateful_actor<supervisor_actor_state>* self, int num_actors,
-		int max_waves,
-		MatrixPool pool) {
+caf::behavior supervisor_actor_fun(
+    caf::stateful_actor<supervisor_actor_state>* self,
+    int num_actors,
+    int max_waves,
+    MatrixPool pool,
+    caf::cuda::program_ptr program
+) {
+    // --- Initialize state ---
+    self->state().num_actors = num_actors;
+    self->state().completed = 0;
+    self->state().max_waves = max_waves;
+    self->state().num_waves = 0;
+    self->state().pool = std::move(pool);
+
+    self->state().rng = std::mt19937(42); // reproducible
+
+    // Extract sizes once
+    for (const auto& [N, _] : self->state().pool.A) {
+        self->state().sizes.push_back(N);
+    }
 
-	self -> state().num_actors = num_actors;
-	self -> state().completed = 0;
-	self->state().max_waves = max_waves;
-	self->state().num_waves = 0;
-	self -> state().pool = pool;
+     caf::cuda::manager& mgr = caf::cuda::manager::get();
+     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
-	self->mail("spawn").send(self);
+    // Kick off first wave
+    self->mail("spawn").send(self);
 
-	return {
+    return {
 
-		[=](std::string command) {
-		
-			if (command == "spawn") {
-		
+        // =========================
+        // SPAWN WAVE
+        // =========================
+        [=](std::string cmd) {
+            if (cmd != "spawn")
+                return;
 
+            self->state().completed = 0;
 
+            std::uniform_int_distribution<size_t> dist(
+                0, self->state().sizes.size() - 1);
 
+            for (int i = 0; i < self->state().num_actors; ++i) {
 
-			
-			}
-		
-		
-		},
-		[=](int completed) {
-		
-			self->state().completed+=completed;
-
-			if (self->state().completed >= self->state().limit) {
-			
-				self->state().num_waves +=1;
-
-				if (self->state().num_waves >= self->state().max_waves) {
-				
-					caf::cuda::manager::shutdown();
-					self -> quit();
-				
-				}
-				else {
-				
-					self->mail("spawn").send(self);
-				
-				}
-			
-			}
-		
-		}
+                int N = self->state().sizes[dist(self->state().rng)];
 
+                const auto& A = self->state().pool.A[N];
+                const auto& B = self->state().pool.B[N];
 
-	};
+                const int THREADS = 32;
+                const int BLOCKS = (N + THREADS - 1) / THREADS;
+
+                caf::cuda::nd_range dims(
+                    BLOCKS, BLOCKS, 1,
+                    THREADS, THREADS, 1
+                );
+
+                self->spawn(
+                    mmul_actor_fun_scheduler,
+                    self, // exit actor
+                    N,
+                    program,
+                    dims,
+                    caf::cuda::create_in_arg(A),
+                    caf::cuda::create_in_arg(B)
+                );
+            }
+        },
+
+        // =========================
+        // COMPLETION TRACKING
+        // =========================
+        [=](int done) {
+            self->state().completed += done;
 
+            if (self->state().completed >= self->state().num_actors) {
 
+                self->state().num_waves++;
+
+                std::cout << "Completed wave "
+                          << self->state().num_waves
+                          << "/" << self->state().max_waves
+                          << std::endl;
+
+                if (self->state().num_waves >= self->state().max_waves) {
+                    caf::cuda::manager::shutdown();
+                    self->quit();
+                } else {
+                    self->mail("spawn").send(self);
+                }
+            }
+        }
+    };
 }
 
 
@@ -317,283 +357,6 @@ double time_run(Fn&& fn) {
 
 
 
-void run_mmul_mixed_batch_cuda_scheduler(
-    caf::actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors,
-    MatrixPool pool,
-    bool randomize = false)
-{
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    std::mt19937 rng(123456);
-    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
-
-    for (int i = 0; i < num_actors; ++i) {
-
-        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
-
-	const auto& A = pool.A[N];
-	const auto& B = pool.B[N];
-
-        sys.spawn(
-            mmul_actor_fun,
-            program,
-            A,
-	    B,
-	    N);
-    }
-
-    sys.await_all_actors_done();
-}
-
-
-void run_mmul_mixed_batch_caf_cuda_scheduler(
-    caf::actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors,
-    MatrixPool pool,
-    bool FCFS,
-    bool randomize = false)
-{
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-    //set the scheduler actor behavior
-    if (FCFS) {
-	    for (int i = 0; i < mgr.get_num_devices(); i++) {
-
-		    mgr.send_scheduler_actor_message("green",i);
-
-	    }
-
-    }
-    else { 
-	    for (int i = 0; i < mgr.get_num_devices(); i++) {
-
-		    mgr.send_scheduler_actor_message("multilevel",i);
-
-	    }
-    }
-
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    caf::actor exit_actor = mgr.spawn_exit_actor(num_actors);
-
-    std::mt19937 rng(123456);
-    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
-
-    const int THREADS = 32;
-// timestamp before actor creation
-auto start_time = std::chrono::high_resolution_clock::now();
-
-for (int i = 0; i < num_actors; ++i) {
-
-    int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
-    int BLOCKS = (N + THREADS - 1) / THREADS;
-
-    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    const auto& A = pool.A[N];
-    const auto& B = pool.B[N];
-  //time the single actor creation
-   //auto start = std::chrono::high_resolution_clock::now();
-
-    caf::actor a = sys.spawn(
-        mmul_actor_fun_scheduler,
-        exit_actor,
-        N,
-        program,
-        dims,
-        A,
-        B
-    );
-
-    //auto end = std::chrono::high_resolution_clock::now();
-    //auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-   ///std::cout << "[INFO] Actor " << i << " creation time: " << ms << " ms\n";
-}
-
-// timestamp after actor creation
-auto end_time = std::chrono::high_resolution_clock::now();
-auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();
-
-std::cout << "[INFO] Actor creation time for " << num_actors
-          << " actors: " << ms << " ms\n";
-    sys.await_all_actors_done();
-}
-
-
-
-
-
-
-
-
-void run_mmul_mixed_batch_comparison(
-    caf::actor_system& sys)
-{
-    std::vector<int> sizes = {32,64,128,256,512,1024};
-    std::vector<int> actor_counts = {5000};
-    MatrixPool pool = create_matrix_pool(sizes);
-
-
-
-    std::cout << "\n=== MMUL Mixed Batch Comparison ===\n";
-    std::cout << "scheduler actors time_seconds\n";
-
-    for (auto actors : actor_counts) {
-
-        /* ========= core_usage BULK ========= */
-
-        {
-            caf::cuda::manager_config cfg(true);
-            caf::cuda::manager::init(sys, cfg);
-
-            double t = time_run([&] {
-                run_mmul_mixed_batch_caf_cuda_scheduler(sys, sizes, actors,pool,false);
-            });
-
-            std::cout << "RESULT CAF CUDA DEFAULT SCHEDULER "
-                      << actors << " "
-                      << t << "seconds\n";
-
-            caf::cuda::manager::shutdown();
-        }
-        /* ========= green light BULK  ========= */
-
-        {
-            caf::cuda::manager_config cfg(true);
-            caf::cuda::manager::init(sys, cfg);
-
-            double t = time_run([&] {
-                run_mmul_mixed_batch_caf_cuda_scheduler(sys, sizes, actors,pool,true);
-            });
-
-            std::cout << "RESULT CAF CUDA FCFS SCHEDULER "
-                      << actors << " "
-                      << t << "seconds\n";
-
-            caf::cuda::manager::shutdown();
-        }
-
-
-        /* ========= no scheduler ========= */
-
-        {
-            caf::cuda::manager_config cfg(false);
-            caf::cuda::manager::init(sys, cfg);
-
-            double t = time_run([&] {
-                run_mmul_mixed_batch_cuda_scheduler(sys, sizes, actors,pool);
-            });
-
-            std::cout << "RESULT CUDA SCHEDULER "
-                      << actors << " "
-                      << t << "seconds\n";
-
-            caf::cuda::manager::shutdown();
-        }
-    }
-
-    std::cout << "\n=== Mixed Batch Comparison Complete ===\n";
-}
-
-
-// Spawn actors memory-efficiently using counters
-void run_mmul_spawn_counter(
-    actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors,
-    const MatrixPool& pool,
-    bool largest_first = false,
-    bool smallest_first = false)
-{
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    const int THREADS = 32;
-
-    // Determine spawn order
-    std::vector<int> spawn_order = sizes;
-    if (largest_first) std::sort(spawn_order.rbegin(), spawn_order.rend());
-    if (smallest_first) std::sort(spawn_order.begin(), spawn_order.end());
-
-    // Initialize counters
-    std::unordered_map<int,int> spawned_count;
-    for (auto N : spawn_order) spawned_count[N] = 0;
-
-    int total_spawned = 0;
-    int num_sizes = spawn_order.size();
-    int base_quota = num_actors / num_sizes;
-    int remainder = num_actors % num_sizes;
-
-    for (size_t idx = 0; idx < spawn_order.size(); ++idx) {
-        int N = spawn_order[idx];
-        int limit = base_quota + (idx == spawn_order.size() - 1 ? remainder : 0);
-
-        while (spawned_count[N] < limit && total_spawned < num_actors) {
-            const auto& A = pool.A.at(N);
-            const auto& B = pool.B.at(N);
-            caf::cuda::nd_range dims((N+THREADS-1)/THREADS, (N+THREADS-1)/THREADS, 1, THREADS, THREADS, 1);
-
-	    sys.spawn(mmul_actor_fun, program, A, B, N);
-	    spawned_count[N]++;
-	    total_spawned++;
-	}
-    }
-
-    sys.await_all_actors_done();
-}
-
-
-void run_actor_spawn_order_comparison(actor_system& sys) {
-    std::vector<int> sizes = {10,32,64,128,256,512,1024,2048};
-    int num_actors = 5000;
-    MatrixPool pool = create_matrix_pool(sizes);
-
-    std::cout << "\n=== Actor Spawn Order Comparison ===\n";
-    std::cout << "order num_actors time_seconds\n";
-
-    // Round-robin
-    {
-        caf::cuda::manager_config cfg(false);
-        caf::cuda::manager::init(sys, cfg);
-         double t = time_run([&] {
-                run_mmul_mixed_batch_cuda_scheduler(sys, sizes, num_actors , pool);
-            });
-        std::cout << "round_robin " << num_actors << " " << t << "\n";
-        caf::cuda::manager::shutdown();
-    }
-
-    // Largest-first
-    {
-        caf::cuda::manager_config cfg(false);
-        caf::cuda::manager::init(sys, cfg);
-        double t = time_run([&] {
-            run_mmul_spawn_counter(sys, sizes, num_actors, pool, true, false);
-        });
-        std::cout << "largest_first " << num_actors << " " << t << "\n";
-        caf::cuda::manager::shutdown();
-    }
-
-    // Smallest-first
-    {
-        caf::cuda::manager_config cfg(false);
-        caf::cuda::manager::init(sys, cfg);
-        double t = time_run([&] {
-            run_mmul_spawn_counter(sys, sizes, num_actors, pool, false, true);
-        });
-        std::cout << "smallest_first " << num_actors << " " << t << "\n";
-        caf::cuda::manager::shutdown();
-    }
-
-    std::cout << "=== Spawn Order Comparison Complete ===\n";
-}
-
-
 
 
 
@@ -607,9 +370,7 @@ void caf_main(caf::actor_system& sys) {
   caf::cuda::manager_config man_config(true);
   //caf::cuda::manager::init(sys,man_config);
 
- run_mmul_mixed_batch_comparison(sys);
 
-  //run_actor_spawn_order_comparison(sys);
 
 }
 

From 5116b489845829e53bc88bf64487be07b275b8f7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Mar 2026 15:53:19 -0600
Subject: [PATCH 0534/1000] Fixed compiler errors.

---
 .../main.test.cpp                             | 200 +++++++++---------
 1 file changed, 102 insertions(+), 98 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 47661b6ad4..6c31059890 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -96,103 +96,6 @@ struct supervisor_actor_state {
 
 
 
-//runs for FCFS behavior
-caf::behavior supervisor_actor_fun(
-    caf::stateful_actor<supervisor_actor_state>* self,
-    int num_actors,
-    int max_waves,
-    MatrixPool pool,
-    caf::cuda::program_ptr program
-) {
-    // --- Initialize state ---
-    self->state().num_actors = num_actors;
-    self->state().completed = 0;
-    self->state().max_waves = max_waves;
-    self->state().num_waves = 0;
-    self->state().pool = std::move(pool);
-
-    self->state().rng = std::mt19937(42); // reproducible
-
-    // Extract sizes once
-    for (const auto& [N, _] : self->state().pool.A) {
-        self->state().sizes.push_back(N);
-    }
-
-     caf::cuda::manager& mgr = caf::cuda::manager::get();
-     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    // Kick off first wave
-    self->mail("spawn").send(self);
-
-    return {
-
-        // =========================
-        // SPAWN WAVE
-        // =========================
-        [=](std::string cmd) {
-            if (cmd != "spawn")
-                return;
-
-            self->state().completed = 0;
-
-            std::uniform_int_distribution<size_t> dist(
-                0, self->state().sizes.size() - 1);
-
-            for (int i = 0; i < self->state().num_actors; ++i) {
-
-                int N = self->state().sizes[dist(self->state().rng)];
-
-                const auto& A = self->state().pool.A[N];
-                const auto& B = self->state().pool.B[N];
-
-                const int THREADS = 32;
-                const int BLOCKS = (N + THREADS - 1) / THREADS;
-
-                caf::cuda::nd_range dims(
-                    BLOCKS, BLOCKS, 1,
-                    THREADS, THREADS, 1
-                );
-
-                self->spawn(
-                    mmul_actor_fun_scheduler,
-                    self, // exit actor
-                    N,
-                    program,
-                    dims,
-                    caf::cuda::create_in_arg(A),
-                    caf::cuda::create_in_arg(B)
-                );
-            }
-        },
-
-        // =========================
-        // COMPLETION TRACKING
-        // =========================
-        [=](int done) {
-            self->state().completed += done;
-
-            if (self->state().completed >= self->state().num_actors) {
-
-                self->state().num_waves++;
-
-                std::cout << "Completed wave "
-                          << self->state().num_waves
-                          << "/" << self->state().max_waves
-                          << std::endl;
-
-                if (self->state().num_waves >= self->state().max_waves) {
-                    caf::cuda::manager::shutdown();
-                    self->quit();
-                } else {
-                    self->mail("spawn").send(self);
-                }
-            }
-        }
-    };
-}
-
-
-
 
 
 struct mmul_state {
@@ -264,7 +167,6 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
 
 
 
-
 struct mmul_actor_with_scheduler_state {
   static inline const char* name = "my_actor";
 };
@@ -346,6 +248,108 @@ caf::behavior mmul_actor_fun_scheduler(
 }
 
 
+
+
+//runs for FCFS behavior
+caf::behavior supervisor_actor_fun(
+    caf::stateful_actor<supervisor_actor_state>* self,
+    int num_actors,
+    int max_waves,
+    MatrixPool pool
+) {
+    // --- Initialize state ---
+    self->state().num_actors = num_actors;
+    self->state().completed = 0;
+    self->state().max_waves = max_waves;
+    self->state().num_waves = 0;
+    self->state().pool = std::move(pool);
+
+    self->state().rng = std::mt19937(42); // reproducible
+
+    // Extract sizes once
+    for (const auto& [N, _] : self->state().pool.A) {
+        self->state().sizes.push_back(N);
+    }
+
+     caf::cuda::manager& mgr = caf::cuda::manager::get();
+     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    // Kick off first wave
+    self->mail("spawn").send(self);
+
+    return {
+
+        // =========================
+        // SPAWN WAVE
+        // =========================
+        [=](std::string cmd) {
+            if (cmd != "spawn")
+                return;
+
+            self->state().completed = 0;
+
+            std::uniform_int_distribution<size_t> dist(
+                0, self->state().sizes.size() - 1);
+
+            for (int i = 0; i < self->state().num_actors; ++i) {
+
+                int N = self->state().sizes[dist(self->state().rng)];
+
+                const auto& A = self->state().pool.A[N];
+                const auto& B = self->state().pool.B[N];
+
+                const int THREADS = 32;
+                const int BLOCKS = (N + THREADS - 1) / THREADS;
+
+                caf::cuda::nd_range dims(
+                    BLOCKS, BLOCKS, 1,
+                    THREADS, THREADS, 1
+                );
+
+                self->spawn(
+                    mmul_actor_fun_scheduler,
+                    self, // exit actor
+                    N,
+                    program,
+                    dims,
+                    caf::cuda::create_in_arg(A),
+                    caf::cuda::create_in_arg(B)
+                );
+            }
+        },
+
+        // =========================
+        // COMPLETION TRACKING
+        // =========================
+        [=](int done) {
+            self->state().completed += done;
+
+            if (self->state().completed >= self->state().num_actors) {
+
+                self->state().num_waves++;
+
+                std::cout << "Completed wave "
+                          << self->state().num_waves
+                          << "/" << self->state().max_waves
+                          << std::endl;
+
+                if (self->state().num_waves >= self->state().max_waves) {
+                    caf::cuda::manager::shutdown();
+                    self->quit();
+                } else {
+                    self->mail("spawn").send(self);
+                }
+            }
+        }
+    };
+}
+
+
+
+
+
+
+
 template <class Fn>
 double time_run(Fn&& fn) {
   auto start = std::chrono::steady_clock::now();

From 3276cdca5bd7edc693882e2abf4aa1db6bfe6b8a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Mar 2026 16:04:10 -0600
Subject: [PATCH 0535/1000] Implemented test driver.

---
 .../main.test.cpp                             | 61 +++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 6c31059890..d02bd56f1d 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -361,7 +361,65 @@ double time_run(Fn&& fn) {
 
 
 
+void run_mmul_random_scaling_tests(caf::actor_system& sys,
+                                  caf::cuda::manager_config man_config) {
 
+    const int min_N = 256;
+    const int max_N = 2048;
+    const int num_sizes = 10;
+
+    const int max_waves = 5;
+
+    const std::vector<int> actor_counts = {
+        1, 2, 4, 8, 16, 32, 64
+    };
+
+
+    for (int num_actors : actor_counts) {
+
+    
+	    // Initialize CUDA manager
+	    caf::cuda::manager::init(sys, man_config);
+	    std::cout << "=====================================\n";
+	    std::cout << "Running with " << num_actors << " actors\n";
+
+	    auto& mgr = caf::cuda::manager::get();
+	    for (int i = 0; i < mgr.get_num_devices(); i++) {
+		    mgr.send_scheduler_actor_message("green",i); 
+	    }
+
+
+        // Generate deterministic random pool
+        MatrixPool pool = create_matrix_pool_random(
+            num_sizes,
+            min_N,
+            max_N,
+            42  // fixed seed
+        );
+
+        double elapsed = time_run([&]() {
+
+            auto sup = sys.spawn(
+                supervisor_actor_fun,
+                num_actors,
+                max_waves,
+                pool
+            );
+
+            // Block until supervisor finishes
+            scoped_actor self{sys};
+            self->wait_for(sup);
+        });
+
+        std::cout << "Total time: "
+                  << std::fixed << std::setprecision(6)
+                  << elapsed << " seconds\n";
+    
+	caf::cuda::manager::shutdown();
+    }
+
+    caf::cuda::manager::shutdown();
+}
 
 
 
@@ -375,6 +433,9 @@ void caf_main(caf::actor_system& sys) {
   //caf::cuda::manager::init(sys,man_config);
 
 
+  run_mmul_random_scaling_tests(sys,man_config);
+
+
 
 }
 

From 0455e4b38ed6c850d433bb0426d172ee7f95ff8f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Mar 2026 16:11:39 -0600
Subject: [PATCH 0536/1000] Updated the test driver to take a really long time.

---
 .../mmul-randonom-batch-benchmark/main.test.cpp             | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index d02bd56f1d..a16d394879 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -366,12 +366,12 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 
     const int min_N = 256;
     const int max_N = 2048;
-    const int num_sizes = 10;
+    const int num_sizes = 200;
 
-    const int max_waves = 5;
+    const int max_waves = 5000;
 
     const std::vector<int> actor_counts = {
-        1, 2, 4, 8, 16, 32, 64
+        10000
     };
 
 

From 9765d0814535588e237e2bb6ba46157c1bf939d4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Mar 2026 16:20:13 -0600
Subject: [PATCH 0537/1000] Updated supervisor to time its lifecycle.

---
 .../main.test.cpp                             | 45 ++++++++++++++-----
 1 file changed, 35 insertions(+), 10 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index a16d394879..16c08447ce 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -90,8 +90,12 @@ struct supervisor_actor_state {
 
     MatrixPool pool;
 
-    std::vector<int> sizes;   // cached keys
-    std::mt19937 rng;         // RNG
+    std::vector<int> sizes;
+    std::mt19937 rng;
+
+    // 🔥 Timing
+    std::chrono::steady_clock::time_point start_time;
+    std::chrono::steady_clock::time_point wave_start_time;
 };
 
 
@@ -264,15 +268,18 @@ caf::behavior supervisor_actor_fun(
     self->state().num_waves = 0;
     self->state().pool = std::move(pool);
 
-    self->state().rng = std::mt19937(42); // reproducible
+    self->state().rng = std::mt19937(42);
 
-    // Extract sizes once
+    // Extract sizes
     for (const auto& [N, _] : self->state().pool.A) {
         self->state().sizes.push_back(N);
     }
 
-     caf::cuda::manager& mgr = caf::cuda::manager::get();
-     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    // 🔥 Start total timer
+    self->state().start_time = std::chrono::steady_clock::now();
 
     // Kick off first wave
     self->mail("spawn").send(self);
@@ -288,6 +295,9 @@ caf::behavior supervisor_actor_fun(
 
             self->state().completed = 0;
 
+            // 🔥 Start wave timer
+            self->state().wave_start_time = std::chrono::steady_clock::now();
+
             std::uniform_int_distribution<size_t> dist(
                 0, self->state().sizes.size() - 1);
 
@@ -308,7 +318,7 @@ caf::behavior supervisor_actor_fun(
 
                 self->spawn(
                     mmul_actor_fun_scheduler,
-                    self, // exit actor
+                    self,
                     N,
                     program,
                     dims,
@@ -326,14 +336,29 @@ caf::behavior supervisor_actor_fun(
 
             if (self->state().completed >= self->state().num_actors) {
 
+                // 🔥 End wave timing
+                auto wave_end = std::chrono::steady_clock::now();
+                std::chrono::duration<double> wave_time =
+                    wave_end - self->state().wave_start_time;
+
                 self->state().num_waves++;
 
-                std::cout << "Completed wave "
+                std::cout << "Wave "
                           << self->state().num_waves
-                          << "/" << self->state().max_waves
-                          << std::endl;
+                          << " completed in "
+                          << wave_time.count() << " s\n";
 
                 if (self->state().num_waves >= self->state().max_waves) {
+
+                    // 🔥 End total timing
+                    auto end_time = std::chrono::steady_clock::now();
+                    std::chrono::duration<double> total_time =
+                        end_time - self->state().start_time;
+
+                    std::cout << "\n===== SUPERVISOR TOTAL TIME =====\n";
+                    std::cout << "Total runtime: "
+                              << total_time.count() << " s\n";
+
                     caf::cuda::manager::shutdown();
                     self->quit();
                 } else {

From 140bd92649aaf18955b475681d30e76f3f30a659 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Mar 2026 16:49:51 -0600
Subject: [PATCH 0538/1000] Updated mmul_actor fun.

---
 .../main.test.cpp                             | 116 +++++++-----------
 1 file changed, 46 insertions(+), 70 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 16c08447ce..4f8ababcc2 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -82,23 +82,6 @@ MatrixPool create_matrix_pool_random(
 
 
 
-struct supervisor_actor_state {
-    int num_actors;
-    int num_waves;
-    int completed;
-    int max_waves;
-
-    MatrixPool pool;
-
-    std::vector<int> sizes;
-    std::mt19937 rng;
-
-    // 🔥 Timing
-    std::chrono::steady_clock::time_point start_time;
-    std::chrono::steady_clock::time_point wave_start_time;
-};
-
-
 
 
 
@@ -110,10 +93,14 @@ struct mmul_state {
 
 
 
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::program_ptr mmul_kernel,
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
+		caf::actor exit_actor,
+		int N,
+		caf::cuda::nd_range dims,
+		caf::cuda::program_ptr mmul_kernel,
 		const in<int> matrixA,
-		const in<int> matrixB,
-		int N) {
+		const in<int> matrixB
+		) {
 
 	self->state().mmul_kernel = mmul_kernel;
 	self->mail(N).send(self);
@@ -128,11 +115,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
 			//auto program =
 			//mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
-		    const int THREADS = 32;
-		    const int BLOCKS = (N + THREADS - 1) / THREADS;
-		    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-
+		    
 			auto program = self->state().mmul_kernel;
 
 			auto inA = std::move(matrixA);
@@ -163,6 +146,7 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,caf::cuda::pr
 						arg4);
 
 			std::get<2>(result) -> copy_to_host();
+			self->mail(1).send(exit_actor);
 			self -> quit();
 
 		}
@@ -254,56 +238,64 @@ caf::behavior mmul_actor_fun_scheduler(
 
 
 
-//runs for FCFS behavior
+// ---------------------------- SUPERVISOR ACTOR ----------------------------
+struct supervisor_actor_state {
+    int num_actors;
+    int num_waves;
+    int completed;
+    int max_waves;
+
+    MatrixPool pool;
+
+    // Precomputed sequence of N values
+    std::vector<int> Ns;
+    int next_task;
+
+    // Timing
+    std::chrono::steady_clock::time_point start_time;
+    std::chrono::steady_clock::time_point wave_start_time;
+};
+
 caf::behavior supervisor_actor_fun(
     caf::stateful_actor<supervisor_actor_state>* self,
     int num_actors,
     int max_waves,
-    MatrixPool pool
+    MatrixPool pool,
+    const std::vector<int>& Ns,      // deterministic task sizes
+    auto actor_fun                   // pass in mmul_actor_fun or mmul_actor_fun_scheduler
 ) {
-    // --- Initialize state ---
+    // Initialize state
     self->state().num_actors = num_actors;
     self->state().completed = 0;
     self->state().max_waves = max_waves;
     self->state().num_waves = 0;
     self->state().pool = std::move(pool);
 
-    self->state().rng = std::mt19937(42);
+    self->state().Ns = Ns;
+    self->state().next_task = 0;
 
-    // Extract sizes
-    for (const auto& [N, _] : self->state().pool.A) {
-        self->state().sizes.push_back(N);
-    }
+    // Start timing
+    self->state().start_time = std::chrono::steady_clock::now();
 
     caf::cuda::manager& mgr = caf::cuda::manager::get();
     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
-    // 🔥 Start total timer
-    self->state().start_time = std::chrono::steady_clock::now();
-
     // Kick off first wave
     self->mail("spawn").send(self);
 
     return {
-
-        // =========================
-        // SPAWN WAVE
-        // =========================
+        // -------------------- SPAWN WAVE --------------------
         [=](std::string cmd) {
-            if (cmd != "spawn")
-                return;
+            if (cmd != "spawn") return;
 
             self->state().completed = 0;
-
-            // 🔥 Start wave timer
             self->state().wave_start_time = std::chrono::steady_clock::now();
 
-            std::uniform_int_distribution<size_t> dist(
-                0, self->state().sizes.size() - 1);
-
             for (int i = 0; i < self->state().num_actors; ++i) {
+                if (self->state().next_task >= self->state().Ns.size())
+                    break;
 
-                int N = self->state().sizes[dist(self->state().rng)];
+                int N = self->state().Ns[self->state().next_task++];
 
                 const auto& A = self->state().pool.A[N];
                 const auto& B = self->state().pool.B[N];
@@ -311,46 +303,30 @@ caf::behavior supervisor_actor_fun(
                 const int THREADS = 32;
                 const int BLOCKS = (N + THREADS - 1) / THREADS;
 
-                caf::cuda::nd_range dims(
-                    BLOCKS, BLOCKS, 1,
-                    THREADS, THREADS, 1
-                );
-
-                self->spawn(
-                    mmul_actor_fun_scheduler,
-                    self,
-                    N,
-                    program,
-                    dims,
-                    caf::cuda::create_in_arg(A),
-                    caf::cuda::create_in_arg(B)
-                );
+                caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+                self->spawn(actor_fun, self, N, program, dims,
+                            caf::cuda::create_in_arg(A),
+                            caf::cuda::create_in_arg(B));
             }
         },
 
-        // =========================
-        // COMPLETION TRACKING
-        // =========================
+        // -------------------- COMPLETION TRACKING --------------------
         [=](int done) {
             self->state().completed += done;
 
             if (self->state().completed >= self->state().num_actors) {
-
-                // 🔥 End wave timing
                 auto wave_end = std::chrono::steady_clock::now();
                 std::chrono::duration<double> wave_time =
                     wave_end - self->state().wave_start_time;
 
                 self->state().num_waves++;
-
                 std::cout << "Wave "
                           << self->state().num_waves
                           << " completed in "
                           << wave_time.count() << " s\n";
 
                 if (self->state().num_waves >= self->state().max_waves) {
-
-                    // 🔥 End total timing
                     auto end_time = std::chrono::steady_clock::now();
                     std::chrono::duration<double> total_time =
                         end_time - self->state().start_time;

From 664a97e65187501c20149c3ede4027175af01930 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Mar 2026 17:06:38 -0600
Subject: [PATCH 0539/1000] Updated test to use both scheduler and no scheduler
 while ensuring they make the same choices whilist being random.

---
 .../main.test.cpp                             | 91 +++++++++++++++----
 1 file changed, 74 insertions(+), 17 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 4f8ababcc2..43721f34ab 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -95,14 +95,13 @@ struct mmul_state {
 
 caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
 		caf::actor exit_actor,
-		int N,
+		caf::cuda::program_ptr program,
 		caf::cuda::nd_range dims,
-		caf::cuda::program_ptr mmul_kernel,
+		int N,
 		const in<int> matrixA,
 		const in<int> matrixB
 		) {
 
-	self->state().mmul_kernel = mmul_kernel;
 	self->mail(N).send(self);
 	return {
 		
@@ -116,7 +115,6 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
 			//mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
 		    
-			auto program = self->state().mmul_kernel;
 
 			auto inA = std::move(matrixA);
 			auto arg1 = mmul_command.transfer_memory(
@@ -261,9 +259,9 @@ caf::behavior supervisor_actor_fun(
     int num_actors,
     int max_waves,
     MatrixPool pool,
-    const std::vector<int>& Ns,      // deterministic task sizes
-    auto actor_fun                   // pass in mmul_actor_fun or mmul_actor_fun_scheduler
-) {
+    const std::vector<int>& Ns,      // deterministic task sizes 
+    bool use_scheduler
+    ) {
     // Initialize state
     self->state().num_actors = num_actors;
     self->state().completed = 0;
@@ -305,9 +303,16 @@ caf::behavior supervisor_actor_fun(
 
                 caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-                self->spawn(actor_fun, self, N, program, dims,
+		if (use_scheduler) {
+			self->spawn(mmul_actor_fun_scheduler, self, N, program, dims,
                             caf::cuda::create_in_arg(A),
                             caf::cuda::create_in_arg(B));
+		}
+		else {
+			self->spawn(mmul_actor_fun, self, program, dims,N,
+                            caf::cuda::create_in_arg(A),
+                            caf::cuda::create_in_arg(B));
+		}
             }
         },
 
@@ -372,10 +377,35 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     const int max_waves = 5000;
 
     const std::vector<int> actor_counts = {
-        10000
+	    10000
     };
 
 
+    int num_actors = actor_counts[0];
+
+        // Generate deterministic random pool
+        MatrixPool pool = create_matrix_pool_random(
+            num_sizes,
+            min_N,
+            max_N,
+            42  // fixed seed
+        );
+
+    // Precompute all task Ns (total_tasks = num_actors * max_waves)
+    std::vector<int> sizes;
+    for (const auto& [N, _] : pool.A) sizes.push_back(N);
+
+    int total_tasks = num_actors * max_waves;
+    std::vector<int> Ns;
+    Ns.reserve(total_tasks);
+
+    std::mt19937 rng(42);
+    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+    for (int i = 0; i < total_tasks; ++i)
+	    Ns.push_back(sizes[dist(rng)]);
+
+
+    //scheduler
     for (int num_actors : actor_counts) {
 
     
@@ -390,13 +420,6 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 	    }
 
 
-        // Generate deterministic random pool
-        MatrixPool pool = create_matrix_pool_random(
-            num_sizes,
-            min_N,
-            max_N,
-            42  // fixed seed
-        );
 
         double elapsed = time_run([&]() {
 
@@ -404,7 +427,9 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
                 supervisor_actor_fun,
                 num_actors,
                 max_waves,
-                pool
+                pool,
+		Ns,
+		true
             );
 
             // Block until supervisor finishes
@@ -418,7 +443,39 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     
 	caf::cuda::manager::shutdown();
     }
+    
+    //no scheduler
+    for (int num_actors : actor_counts) {
+
+    
+	    // Initialize CUDA manager
+	    caf::cuda::manager::init(sys);
+	    std::cout << "=====================================\n";
+	    std::cout << "Running no scheduler with " << num_actors << " actors\n";
+
+      
+	    double elapsed = time_run([&]() {
+
+            auto sup = sys.spawn(
+                supervisor_actor_fun,
+                num_actors,
+                max_waves,
+                pool,
+		Ns,
+		false
+            );
+
+            // Block until supervisor finishes
+            scoped_actor self{sys};
+            self->wait_for(sup);
+        });
 
+        std::cout << "Total time: "
+                  << std::fixed << std::setprecision(6)
+                  << elapsed << " seconds\n";
+    
+	caf::cuda::manager::shutdown();
+    }
     caf::cuda::manager::shutdown();
 }
 

From c5ca98473e14978cc01c7e374fd00db7986eaffa Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Mar 2026 17:13:11 -0600
Subject: [PATCH 0540/1000] Fixed issues.

---
 .../main.test.cpp                             | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 43721f34ab..e744fe9474 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -295,6 +295,7 @@ caf::behavior supervisor_actor_fun(
 
                 int N = self->state().Ns[self->state().next_task++];
 
+
                 const auto& A = self->state().pool.A[N];
                 const auto& B = self->state().pool.B[N];
 
@@ -374,10 +375,10 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     const int max_N = 2048;
     const int num_sizes = 200;
 
-    const int max_waves = 5000;
+    const int max_waves = 1;
 
     const std::vector<int> actor_counts = {
-	    10000
+	    1000
     };
 
 
@@ -412,7 +413,7 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 	    // Initialize CUDA manager
 	    caf::cuda::manager::init(sys, man_config);
 	    std::cout << "=====================================\n";
-	    std::cout << "Running with " << num_actors << " actors\n";
+	    std::cout << "Running with Scheduler with " << num_actors << " actors\n";
 
 	    auto& mgr = caf::cuda::manager::get();
 	    for (int i = 0; i < mgr.get_num_devices(); i++) {
@@ -432,10 +433,10 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 		true
             );
 
-            // Block until supervisor finishes
-            scoped_actor self{sys};
-            self->wait_for(sup);
-        });
+        
+	      sys.await_all_actors_done();
+	    
+	    });
 
         std::cout << "Total time: "
                   << std::fixed << std::setprecision(6)
@@ -465,10 +466,9 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 		false
             );
 
-            // Block until supervisor finishes
-            scoped_actor self{sys};
-            self->wait_for(sup);
-        });
+	      sys.await_all_actors_done();
+	    
+	    });
 
         std::cout << "Total time: "
                   << std::fixed << std::setprecision(6)

From d5ecb71408029483b67b9902110ea29dd41d8406 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 18 Mar 2026 17:23:02 -0600
Subject: [PATCH 0541/1000] Saving.

---
 .../main.test.cpp                              | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index e744fe9474..55e15c9be5 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -327,10 +327,10 @@ caf::behavior supervisor_actor_fun(
                     wave_end - self->state().wave_start_time;
 
                 self->state().num_waves++;
-                std::cout << "Wave "
-                          << self->state().num_waves
-                          << " completed in "
-                          << wave_time.count() << " s\n";
+                //std::cout << "Wave "
+                  //        << self->state().num_waves
+                    //      << " completed in "
+                      //    << wave_time.count() << " s\n";
 
                 if (self->state().num_waves >= self->state().max_waves) {
                     auto end_time = std::chrono::steady_clock::now();
@@ -372,13 +372,13 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
                                   caf::cuda::manager_config man_config) {
 
     const int min_N = 256;
-    const int max_N = 2048;
+    const int max_N = 1024;
     const int num_sizes = 200;
 
     const int max_waves = 1;
 
     const std::vector<int> actor_counts = {
-	    1000
+	    10000
     };
 
 
@@ -438,9 +438,6 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 	    
 	    });
 
-        std::cout << "Total time: "
-                  << std::fixed << std::setprecision(6)
-                  << elapsed << " seconds\n";
     
 	caf::cuda::manager::shutdown();
     }
@@ -470,9 +467,6 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 	    
 	    });
 
-        std::cout << "Total time: "
-                  << std::fixed << std::setprecision(6)
-                  << elapsed << " seconds\n";
     
 	caf::cuda::manager::shutdown();
     }

From 5579d35bef6451a7baaee0b19f7f9e37e8702650 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 19 Mar 2026 14:01:23 -0600
Subject: [PATCH 0542/1000] Updated operations on both actors to ensure the
 only difference between the two was multiplying matrices, as there was some
 additional overhead incurred by the synchronize command in mmul with
 scheduler actor that was not present in mmul actor

---
 .../main.test.cpp                             | 160 +++++++++++-------
 1 file changed, 102 insertions(+), 58 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 55e15c9be5..660feb8555 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -102,51 +102,68 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
 		const in<int> matrixB
 		) {
 
+
+
 	self->mail(N).send(self);
+
 	return {
-		
+
 		[=](int N) {
 
-			caf::cuda::manager& mgr = caf::cuda::manager::get();
+			auto total_start = std::chrono::steady_clock::now();
+
 			int device = 0;
 			int stream = rand();
 
-			//auto program =
-			//mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-		    
+			// ---------------- H2D ----------------
+			auto h2d_start = std::chrono::steady_clock::now();
 
-			auto inA = std::move(matrixA);
-			auto arg1 = mmul_command.transfer_memory(
-					device,
-					stream,
-					std::move(inA));
+			auto arg1 = mmul_command.transfer_memory(device, stream, std::move(matrixA));
+			auto arg2 = mmul_command.transfer_memory(device, stream, std::move(matrixB));
 
-			auto inB = std::move(matrixB);
-			auto arg2 = mmul_command.transfer_memory(
-					device,
-					stream,
-					std::move(inB));
+			auto h2d_end = std::chrono::steady_clock::now();
 
+			// ---------------- Kernel ----------------
 			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
 			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
 
-			auto result = 
-				async_mmul.run_async(
-						program,
-						dims,
-						stream,
-						0,
-						device,
-						arg1,
-						arg2,
-						arg3,
-						arg4);
-
-			std::get<2>(result) -> copy_to_host();
-			self->mail(1).send(exit_actor);
-			self -> quit();
+			auto kernel_start = std::chrono::steady_clock::now();
+
+			auto result = async_mmul.run_async(
+					program, dims, stream, 0, device,
+					arg1, arg2, arg3, arg4);
+
+			//std::get<2>(result)->synchronize();
+
+			auto kernel_end = std::chrono::steady_clock::now();
+
+			// ---------------- D2H ----------------
+			auto d2h_start = std::chrono::steady_clock::now();
+
+			std::get<2>(result)->copy_to_host();
+
+			auto d2h_end = std::chrono::steady_clock::now();
+
+			auto total_end = std::chrono::steady_clock::now();
+
+			// ---------------- COMPUTE ----------------
+			auto h2d = std::chrono::duration<double>(h2d_end - h2d_start).count();
+			auto kernel = std::chrono::duration<double>(kernel_end - kernel_start).count();
+			auto d2h = std::chrono::duration<double>(d2h_end - d2h_start).count();
+			auto total = std::chrono::duration<double>(total_end - total_start).count();
+
+			/*
+			// ---------------- PRINT ----------------
+			std::cout << "\n[NO SCHEDULER] N=" << N << "\n";
+			std::cout << "H2D:    " << h2d * 1000 << " ms\n";
+			std::cout << "Kernel: " << kernel * 1000 << " ms\n";
+			std::cout << "D2H:    " << d2h * 1000 << " ms\n";
+			std::cout << "TOTAL:  " << total * 1000 << " ms\n";
+			std::cout << "SUM:    " << (h2d + kernel + d2h) * 1000 << " ms\n";
 
+			*/
+			self->mail(1).send(exit_actor);
+			self->quit();
 		}
 	};
 }
@@ -170,9 +187,10 @@ caf::behavior mmul_actor_fun_scheduler(
 {
 
 
+
         caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-        //caf::actor scheduler = mgr.get_scheduler_actor();
+        caf::actor scheduler = mgr.get_scheduler_actor();
 
         //send a launch token
         caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
@@ -199,34 +217,60 @@ caf::behavior mmul_actor_fun_scheduler(
 			}
 		},
 
-	    // 2. Handle memory buffers -> GPU
-	    [=] (const caf::cuda::response_token_ptr& res_token,
-			    int N) {
+			// 2. Handle memory buffers -> GPU
+			[=](const caf::cuda::response_token_ptr& res_token, int N) {
+
+				auto total_start = std::chrono::steady_clock::now();
+
+				// ---------------- H2D ----------------
+				auto h2d_start = std::chrono::steady_clock::now();
+
+				auto arg1 = mmul.transfer_memory(res_token, std::move(matrixA));
+				auto arg2 = mmul.transfer_memory(res_token, std::move(matrixB));
+				auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
+				auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
+
+				auto h2d_end = std::chrono::steady_clock::now();
+
+				// ---------------- Kernel ----------------
+				auto kernel_start = std::chrono::steady_clock::now();
+
+				auto tempC = mmul.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
+				auto bufferC = std::get<2>(tempC);
 
-		    //    std::cout << "Working\n";
-		    caf::cuda::manager& mgr = caf::cuda::manager::get();
+				//bufferC->synchronize();
 
+				auto kernel_end = std::chrono::steady_clock::now();
 
-		    const int THREADS = 32;
-		    const int BLOCKS = (N + THREADS - 1) / THREADS;
-		    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+				// ---------------- D2H ----------------
+				auto d2h_start = std::chrono::steady_clock::now();
 
-		    auto arg1 = mmul.transfer_memory(res_token, matrixA);
-		    auto arg2 = mmul.transfer_memory(res_token, matrixB);
-		    auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
-		    auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
+				bufferC->copy_to_host();
 
+				auto d2h_end = std::chrono::steady_clock::now();
 
-		    auto tempC = mmul.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
-		    caf::cuda::mem_ptr<int> bufferC = std::get<2>(tempC);
+				res_token->release();
 
-		    bufferC -> synchronize();
-		    res_token->release();
-		    bufferC->copy_to_host();
+				auto total_end = std::chrono::steady_clock::now();
 
-		    self->mail(1).send(exit_actor);
-		    self->quit();
-         }
+				// ---------------- COMPUTE ----------------
+				auto h2d = std::chrono::duration<double>(h2d_end - h2d_start).count();
+				auto kernel = std::chrono::duration<double>(kernel_end - kernel_start).count();
+				auto d2h = std::chrono::duration<double>(d2h_end - d2h_start).count();
+				auto total = std::chrono::duration<double>(total_end - total_start).count();
+
+				/*
+				std::cout << "\n[SCHEDULER] N=" << N << "\n";
+				std::cout << "H2D:    " << h2d * 1000 << " ms\n";
+				std::cout << "Kernel: " << kernel * 1000 << " ms\n";
+				std::cout << "D2H:    " << d2h * 1000 << " ms\n";
+				std::cout << "TOTAL:  " << total * 1000 << " ms\n";
+				std::cout << "SUM:    " << (h2d + kernel + d2h) * 1000 << " ms\n";
+
+				*/
+				self->mail(1).send(exit_actor);
+				self->quit();
+			}
 
 
        };
@@ -272,8 +316,6 @@ caf::behavior supervisor_actor_fun(
     self->state().Ns = Ns;
     self->state().next_task = 0;
 
-    // Start timing
-    self->state().start_time = std::chrono::steady_clock::now();
 
     caf::cuda::manager& mgr = caf::cuda::manager::get();
     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
@@ -281,6 +323,8 @@ caf::behavior supervisor_actor_fun(
     // Kick off first wave
     self->mail("spawn").send(self);
 
+    // Start timing
+    self->state().start_time = std::chrono::steady_clock::now();
     return {
         // -------------------- SPAWN WAVE --------------------
         [=](std::string cmd) {
@@ -371,14 +415,14 @@ double time_run(Fn&& fn) {
 void run_mmul_random_scaling_tests(caf::actor_system& sys,
                                   caf::cuda::manager_config man_config) {
 
-    const int min_N = 256;
-    const int max_N = 1024;
-    const int num_sizes = 200;
+    const int min_N = 32;
+    const int max_N = 2048;
+    const int num_sizes = 100;
 
     const int max_waves = 1;
 
     const std::vector<int> actor_counts = {
-	    10000
+	    1000
     };
 
 

From 81df560c62f410b775360ff630caef7e12f91bdd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 19 Mar 2026 14:41:46 -0600
Subject: [PATCH 0543/1000] Updated test to be 2 and a half minutes .

---
 .../mmul-randonom-batch-benchmark/main.test.cpp               | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 660feb8555..815a172def 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -417,12 +417,12 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 
     const int min_N = 32;
     const int max_N = 2048;
-    const int num_sizes = 100;
+    const int num_sizes = 1000;
 
     const int max_waves = 1;
 
     const std::vector<int> actor_counts = {
-	    1000
+	    30000
     };
 
 

From 3bd0c9cd9fe006c25a0e39c4cba5579980dcdca7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 20 Mar 2026 10:45:27 -0600
Subject: [PATCH 0544/1000] Made some changes but probably will revert back to
 the commit before this .

---
 .../main.test.cpp                             | 42 +++++++++++--------
 1 file changed, 25 insertions(+), 17 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 815a172def..566ae26604 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -112,21 +112,23 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
 
 			auto total_start = std::chrono::steady_clock::now();
 
-			int device = 0;
+			int device = rand() % caf::cuda::manager::get().get_num_devices();
 			int stream = rand();
 
+			std::cout << "device=" << device <<  "\n";
+			std::cout << "N=" << N <<  "\n";
 			// ---------------- H2D ----------------
 			auto h2d_start = std::chrono::steady_clock::now();
 
 			auto arg1 = mmul_command.transfer_memory(device, stream, std::move(matrixA));
 			auto arg2 = mmul_command.transfer_memory(device, stream, std::move(matrixB));
 
-			auto h2d_end = std::chrono::steady_clock::now();
 
 			// ---------------- Kernel ----------------
 			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
 			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
 
+			auto h2d_end = std::chrono::steady_clock::now();
 			auto kernel_start = std::chrono::steady_clock::now();
 
 			auto result = async_mmul.run_async(
@@ -146,14 +148,15 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
 
 			auto total_end = std::chrono::steady_clock::now();
 
+			/*
 			// ---------------- COMPUTE ----------------
 			auto h2d = std::chrono::duration<double>(h2d_end - h2d_start).count();
 			auto kernel = std::chrono::duration<double>(kernel_end - kernel_start).count();
 			auto d2h = std::chrono::duration<double>(d2h_end - d2h_start).count();
 			auto total = std::chrono::duration<double>(total_end - total_start).count();
 
-			/*
 			// ---------------- PRINT ----------------
+			
 			std::cout << "\n[NO SCHEDULER] N=" << N << "\n";
 			std::cout << "H2D:    " << h2d * 1000 << " ms\n";
 			std::cout << "Kernel: " << kernel * 1000 << " ms\n";
@@ -198,8 +201,7 @@ caf::behavior mmul_actor_fun_scheduler(
                         dims,
                         0,
                         "hello",
-                        self,
-			rand() //dependency number, can declare indepedent but want to see what happens when you do not 
+                        self
                         );
         mgr.send_scheduler_actor_message(launch_token);
 
@@ -225,17 +227,24 @@ caf::behavior mmul_actor_fun_scheduler(
 				// ---------------- H2D ----------------
 				auto h2d_start = std::chrono::steady_clock::now();
 
-				auto arg1 = mmul.transfer_memory(res_token, std::move(matrixA));
-				auto arg2 = mmul.transfer_memory(res_token, std::move(matrixB));
-				auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
-				auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
+				auto arg1 = mmul.transfer_memory(res_token -> getDeviceNumber(),res_token -> getStreamId(), std::move(matrixA));
+				auto arg2 = mmul.transfer_memory(res_token -> getDeviceNumber(), res_token -> getStreamId(), std::move(matrixB));
+				//auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
+				//auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
+
+				std::cout << "res_token did = " << res_token -> getDeviceNumber() << "\n";
+				std::cout << "N = " << N << "\n";
+				 out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
+	 			 in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
 
 				auto h2d_end = std::chrono::steady_clock::now();
 
 				// ---------------- Kernel ----------------
 				auto kernel_start = std::chrono::steady_clock::now();
 
-				auto tempC = mmul.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
+
+
+				auto tempC = async_mmul.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
 				auto bufferC = std::get<2>(tempC);
 
 				//bufferC->synchronize();
@@ -254,20 +263,19 @@ caf::behavior mmul_actor_fun_scheduler(
 				auto total_end = std::chrono::steady_clock::now();
 
 				// ---------------- COMPUTE ----------------
+				/*
 				auto h2d = std::chrono::duration<double>(h2d_end - h2d_start).count();
 				auto kernel = std::chrono::duration<double>(kernel_end - kernel_start).count();
 				auto d2h = std::chrono::duration<double>(d2h_end - d2h_start).count();
 				auto total = std::chrono::duration<double>(total_end - total_start).count();
 
-				/*
-				std::cout << "\n[SCHEDULER] N=" << N << "\n";
 				std::cout << "H2D:    " << h2d * 1000 << " ms\n";
 				std::cout << "Kernel: " << kernel * 1000 << " ms\n";
 				std::cout << "D2H:    " << d2h * 1000 << " ms\n";
 				std::cout << "TOTAL:  " << total * 1000 << " ms\n";
-				std::cout << "SUM:    " << (h2d + kernel + d2h) * 1000 << " ms\n";
-
+				std::cout << "SUM:    " << (h2d + kernel + d2h) * 1000 << " ms\n\n";
 				*/
+
 				self->mail(1).send(exit_actor);
 				self->quit();
 			}
@@ -415,14 +423,14 @@ double time_run(Fn&& fn) {
 void run_mmul_random_scaling_tests(caf::actor_system& sys,
                                   caf::cuda::manager_config man_config) {
 
-    const int min_N = 32;
+    const int min_N = 2048;
     const int max_N = 2048;
-    const int num_sizes = 1000;
+    const int num_sizes = 1;
 
     const int max_waves = 1;
 
     const std::vector<int> actor_counts = {
-	    30000
+	    10
     };
 
 

From 16f88c036cdde2acef0bc24f2d3f0bfe27920afb Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 23 Mar 2026 14:28:16 -0600
Subject: [PATCH 0545/1000] Wrote a test version for a new scheduler actor.
 Change is being made in an attempt to bring down the overhead cost of the
 scheduler actor.

---
 .../main.test.cpp                             | 75 ++++++++++++++++++-
 1 file changed, 73 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 566ae26604..035dbdc729 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -404,6 +404,79 @@ caf::behavior supervisor_actor_fun(
 }
 
 
+struct scheduler_actor_state {
+
+	std::vector<caf::actor> subscribers;
+	int num_devices;
+	std::vector<int> costs;
+};
+
+
+
+caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self) {
+
+	self->state().num_devices = caf::cuda::manager::get().get_num_devices();
+	self->state().costs.resize(self->state().num_devices);
+
+	self->mail("publish").urgent().delay(std::chrono::milliseconds(5)).send(self);
+
+	return {
+
+		[=](std::string command,int device, int cost) {
+
+			if (command == "add") {
+
+				self->state().costs[device] +=cost;
+
+			}
+			else if (command == "subtract") {
+
+				int value = std::min(self->state().costs[device] - cost,0);
+				self->state().costs[device] = value;	
+			}
+
+
+		},
+			[=](std::string command, caf::actor actor) {
+
+				auto& subs = self->state().subscribers;
+
+				if (command == "subscribe") {
+					// avoid duplicates
+					if (std::find(subs.begin(), subs.end(), actor) == subs.end()) {
+						subs.push_back(actor);
+						//self->monitor(actor); // track lifecycle
+					}
+				}
+
+				else if (command == "unsubscribe") {
+					subs.erase(
+							std::remove(subs.begin(), subs.end(), actor),
+							subs.end()
+						  );
+					//self->demonitor(actor);
+				}
+			},
+
+			[=](std::string command) {
+				if (command == "publish") {
+					for (caf::actor a : self->state().subscribers) {
+
+						self -> mail(self->state().costs).urgent().send(a);
+
+					}
+					self->mail("publish").urgent().delay(std::chrono::milliseconds(5)).send(self);
+				}
+
+			}	
+
+	};
+}
+
+
+
+
+
 
 
 
@@ -528,8 +601,6 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 
 
 
-
-
 void caf_main(caf::actor_system& sys) {
   
 

From 2dae4c73488a4059afda7b0486b4e964913f761c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 23 Mar 2026 14:44:39 -0600
Subject: [PATCH 0546/1000] Wrote the initial verison for
 mmul_scheduler_actor_fun_2.

---
 .../main.test.cpp                             | 106 ++++++++++++++++++
 1 file changed, 106 insertions(+)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 035dbdc729..2b4c900abf 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -288,6 +288,112 @@ caf::behavior mmul_actor_fun_scheduler(
 
 
 
+
+caf::behavior mmul_actor_fun_scheduler2(caf::stateful_actor<mmul_state>* self,
+		caf::actor exit_actor,
+		caf::actor scheduler_actor,
+		caf::cuda::program_ptr program,
+		caf::cuda::nd_range dims,
+		int stream,
+		int N,
+		const in<int> matrixA,
+		const in<int> matrixB
+		) {
+
+
+	int device = stream % caf::cuda::manager::get().get_num_devices();
+
+	self->mail(N).send(self);
+
+	return {
+
+		//message from the new scheduler actor 
+		[=](std::vector<int> costs) {
+			//do nothing this is an overhead test 
+		
+		
+		},
+		[=](int N) {
+
+			auto total_start = std::chrono::steady_clock::now();
+
+			//int device = rand() % caf::cuda::manager::get().get_num_devices();
+			//int stream = rand();
+
+			//std::cout << "device=" << device <<  "\n";
+			//std::cout << "N=" << N <<  "\n";
+
+
+
+			//declare the cost of doing work to the scheduler actor, for now we can impose
+			//a heuristic of just N, the size of the matrix
+			self->mail("add",device,N).send(scheduler_actor);
+
+
+			// ---------------- H2D ----------------
+			auto h2d_start = std::chrono::steady_clock::now();
+
+
+			auto arg1 = mmul_command.transfer_memory(device, stream, std::move(matrixA));
+			auto arg2 = mmul_command.transfer_memory(device, stream, std::move(matrixB));
+
+
+			// ---------------- Kernel ----------------
+			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
+			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
+
+			auto h2d_end = std::chrono::steady_clock::now();
+			auto kernel_start = std::chrono::steady_clock::now();
+
+			auto result = async_mmul.run_async(
+					program, dims, stream, 0, device,
+					arg1, arg2, arg3, arg4);
+
+			//std::get<2>(result)->synchronize();
+
+			auto kernel_end = std::chrono::steady_clock::now();
+
+			// ---------------- D2H ----------------
+			auto d2h_start = std::chrono::steady_clock::now();
+
+			std::get<2>(result)->copy_to_host();
+
+
+
+			//likewise tell the scheduler we are done doing work
+			self->mail("subtract",device,N).send(scheduler_actor);
+
+			auto d2h_end = std::chrono::steady_clock::now();
+
+			auto total_end = std::chrono::steady_clock::now();
+
+			/*
+			// ---------------- COMPUTE ----------------
+			auto h2d = std::chrono::duration<double>(h2d_end - h2d_start).count();
+			auto kernel = std::chrono::duration<double>(kernel_end - kernel_start).count();
+			auto d2h = std::chrono::duration<double>(d2h_end - d2h_start).count();
+			auto total = std::chrono::duration<double>(total_end - total_start).count();
+
+			// ---------------- PRINT ----------------
+			
+			std::cout << "\n[NO SCHEDULER] N=" << N << "\n";
+			std::cout << "H2D:    " << h2d * 1000 << " ms\n";
+			std::cout << "Kernel: " << kernel * 1000 << " ms\n";
+			std::cout << "D2H:    " << d2h * 1000 << " ms\n";
+			std::cout << "TOTAL:  " << total * 1000 << " ms\n";
+			std::cout << "SUM:    " << (h2d + kernel + d2h) * 1000 << " ms\n";
+
+			*/
+			self->mail(1).send(exit_actor);
+			self->quit();
+		}
+	};
+}
+
+
+
+
+
 // ---------------------------- SUPERVISOR ACTOR ----------------------------
 struct supervisor_actor_state {
     int num_actors;

From ae51aa6c2c23a1b687496f16df6d79541daa295d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 23 Mar 2026 14:47:23 -0600
Subject: [PATCH 0547/1000] Made device and stream deterministic, so that
 results of experiment are easier to interpret.

---
 .../mmul-randonom-batch-benchmark/main.test.cpp          | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 2b4c900abf..708ecd7472 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -98,12 +98,13 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
 		caf::cuda::program_ptr program,
 		caf::cuda::nd_range dims,
 		int N,
+		int stream,
 		const in<int> matrixA,
 		const in<int> matrixB
 		) {
 
 
-
+	int device = stream % caf::cuda::manager::get().get_num_devices();
 	self->mail(N).send(self);
 
 	return {
@@ -112,11 +113,9 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
 
 			auto total_start = std::chrono::steady_clock::now();
 
-			int device = rand() % caf::cuda::manager::get().get_num_devices();
-			int stream = rand();
 
-			std::cout << "device=" << device <<  "\n";
-			std::cout << "N=" << N <<  "\n";
+//			std::cout << "device=" << device <<  "\n";
+//			std::cout << "N=" << N <<  "\n";
 			// ---------------- H2D ----------------
 			auto h2d_start = std::chrono::steady_clock::now();
 

From 0953b1211ad2697c765a76d8f3613a0eea81d7d0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 23 Mar 2026 14:54:07 -0600
Subject: [PATCH 0548/1000] Changed the use scheduler variable in the
 supervisor to call mmul_actor_fun_supervisor2 Change is being made to
 overhead test the new scheduler.

---
 .../main.test.cpp                             | 156 +++++++++---------
 1 file changed, 82 insertions(+), 74 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 708ecd7472..3af47dc3c8 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -393,6 +393,78 @@ caf::behavior mmul_actor_fun_scheduler2(caf::stateful_actor<mmul_state>* self,
 
 
 
+
+struct scheduler_actor_state {
+
+	std::vector<caf::actor> subscribers;
+	int num_devices;
+	std::vector<int> costs;
+};
+
+
+
+caf::behavior scheduler_actor_fun(caf::stateful_actor<scheduler_actor_state>* self) {
+
+	self->state().num_devices = caf::cuda::manager::get().get_num_devices();
+	self->state().costs.resize(self->state().num_devices);
+
+	self->mail("publish").urgent().delay(std::chrono::milliseconds(5)).send(self);
+
+	return {
+
+		[=](std::string command,int device, int cost) {
+
+			if (command == "add") {
+
+				self->state().costs[device] +=cost;
+
+			}
+			else if (command == "subtract") {
+
+				int value = std::min(self->state().costs[device] - cost,0);
+				self->state().costs[device] = value;	
+			}
+
+
+		},
+			[=](std::string command, caf::actor actor) {
+
+				auto& subs = self->state().subscribers;
+
+				if (command == "subscribe") {
+					// avoid duplicates
+					if (std::find(subs.begin(), subs.end(), actor) == subs.end()) {
+						subs.push_back(actor);
+						//self->monitor(actor); // track lifecycle
+					}
+				}
+
+				else if (command == "unsubscribe") {
+					subs.erase(
+							std::remove(subs.begin(), subs.end(), actor),
+							subs.end()
+						  );
+					//self->demonitor(actor);
+				}
+			},
+
+			[=](std::string command) {
+				if (command == "publish") {
+					for (caf::actor a : self->state().subscribers) {
+
+						self -> mail(self->state().costs).urgent().send(a);
+
+					}
+					self->mail("publish").urgent().delay(std::chrono::milliseconds(5)).send(self);
+				}
+
+			}	
+
+	};
+}
+
+
+
 // ---------------------------- SUPERVISOR ACTOR ----------------------------
 struct supervisor_actor_state {
     int num_actors;
@@ -436,6 +508,8 @@ caf::behavior supervisor_actor_fun(
     // Kick off first wave
     self->mail("spawn").send(self);
 
+    caf::actor scheduler_actor = self->spawn(scheduler_actor_fun);
+
     // Start timing
     self->state().start_time = std::chrono::steady_clock::now();
     return {
@@ -462,12 +536,18 @@ caf::behavior supervisor_actor_fun(
                 caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
 		if (use_scheduler) {
-			self->spawn(mmul_actor_fun_scheduler, self, N, program, dims,
+			self->spawn(mmul_actor_fun_scheduler2, 
+					self,
+					scheduler_actor,
+				       	program, 
+					dims,
+					i,
+					N,
                             caf::cuda::create_in_arg(A),
                             caf::cuda::create_in_arg(B));
 		}
 		else {
-			self->spawn(mmul_actor_fun, self, program, dims,N,
+			self->spawn(mmul_actor_fun, self, program, dims,i,N,
                             caf::cuda::create_in_arg(A),
                             caf::cuda::create_in_arg(B));
 		}
@@ -509,78 +589,6 @@ caf::behavior supervisor_actor_fun(
 }
 
 
-struct scheduler_actor_state {
-
-	std::vector<caf::actor> subscribers;
-	int num_devices;
-	std::vector<int> costs;
-};
-
-
-
-caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self) {
-
-	self->state().num_devices = caf::cuda::manager::get().get_num_devices();
-	self->state().costs.resize(self->state().num_devices);
-
-	self->mail("publish").urgent().delay(std::chrono::milliseconds(5)).send(self);
-
-	return {
-
-		[=](std::string command,int device, int cost) {
-
-			if (command == "add") {
-
-				self->state().costs[device] +=cost;
-
-			}
-			else if (command == "subtract") {
-
-				int value = std::min(self->state().costs[device] - cost,0);
-				self->state().costs[device] = value;	
-			}
-
-
-		},
-			[=](std::string command, caf::actor actor) {
-
-				auto& subs = self->state().subscribers;
-
-				if (command == "subscribe") {
-					// avoid duplicates
-					if (std::find(subs.begin(), subs.end(), actor) == subs.end()) {
-						subs.push_back(actor);
-						//self->monitor(actor); // track lifecycle
-					}
-				}
-
-				else if (command == "unsubscribe") {
-					subs.erase(
-							std::remove(subs.begin(), subs.end(), actor),
-							subs.end()
-						  );
-					//self->demonitor(actor);
-				}
-			},
-
-			[=](std::string command) {
-				if (command == "publish") {
-					for (caf::actor a : self->state().subscribers) {
-
-						self -> mail(self->state().costs).urgent().send(a);
-
-					}
-					self->mail("publish").urgent().delay(std::chrono::milliseconds(5)).send(self);
-				}
-
-			}	
-
-	};
-}
-
-
-
-
 
 
 

From 9806dbe2ec190044d9e22985c495adc955e75d43 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 23 Mar 2026 14:58:05 -0600
Subject: [PATCH 0549/1000] Fixed no shutdown issue with the scheduler actor.

---
 .../mmul-randonom-batch-benchmark/main.test.cpp          | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 3af47dc3c8..1169048225 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -578,7 +578,12 @@ caf::behavior supervisor_actor_fun(
                     std::cout << "Total runtime: "
                               << total_time.count() << " s\n";
 
-                    caf::cuda::manager::shutdown();
+
+		    anon_send_exit(
+				    scheduler_actor,
+				    caf::exit_reason::user_shutdown
+				  );
+		    caf::cuda::manager::shutdown();
                     self->quit();
                 } else {
                     self->mail("spawn").send(self);
@@ -616,7 +621,7 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     const int max_waves = 1;
 
     const std::vector<int> actor_counts = {
-	    10
+	    1024
     };
 
 

From e20850ddf9260b648eef3ecae30944d29055529d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 23 Mar 2026 14:59:55 -0600
Subject: [PATCH 0550/1000] Added subscribe and unsubscribe to the scheduler
 actor.

---
 .../mmul-randonom-batch-benchmark/main.test.cpp                | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 1169048225..3434eb075c 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -302,6 +302,8 @@ caf::behavior mmul_actor_fun_scheduler2(caf::stateful_actor<mmul_state>* self,
 
 	int device = stream % caf::cuda::manager::get().get_num_devices();
 
+	self->mail("subscribe",self).send(scheduler_actor);
+
 	self->mail(N).send(self);
 
 	return {
@@ -384,6 +386,7 @@ caf::behavior mmul_actor_fun_scheduler2(caf::stateful_actor<mmul_state>* self,
 
 			*/
 			self->mail(1).send(exit_actor);
+			self->mail("unsubscribe",self).send(scheduler_actor);
 			self->quit();
 		}
 	};

From 85ed8d33b1fc4232bf8cc6c4d6226e6a470d1c99 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 23 Mar 2026 15:02:17 -0600
Subject: [PATCH 0551/1000] Fixed parameter mismatch bug with mmul_actor_fun
 where stream and N were mismatched causing incorrect behavior.

---
 .../mmul-randonom-batch-benchmark/main.test.cpp                 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 3434eb075c..5af98e6b1d 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -97,8 +97,8 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
 		caf::actor exit_actor,
 		caf::cuda::program_ptr program,
 		caf::cuda::nd_range dims,
-		int N,
 		int stream,
+		int N,
 		const in<int> matrixA,
 		const in<int> matrixB
 		) {

From 6d69afb8a29e78c32e225bff8198cc401ab3cb35 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 23 Mar 2026 16:49:28 -0600
Subject: [PATCH 0552/1000] Fixed issue in the test where the number of memory
 sizes was actually the prime difference between scheduler and no scheduler
 (at least for the prototype). It is difficult to tell if this is sufficient
 since we may still be hiding behind ahmdal's law, but neverless for now,
 using 10 sizes will work.

---
 .../mmul-randonom-batch-benchmark/main.test.cpp | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 5af98e6b1d..607340fddd 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -617,14 +617,14 @@ double time_run(Fn&& fn) {
 void run_mmul_random_scaling_tests(caf::actor_system& sys,
                                   caf::cuda::manager_config man_config) {
 
-    const int min_N = 2048;
+    const int min_N = 32;
     const int max_N = 2048;
-    const int num_sizes = 1;
+    const int num_sizes = 10;
 
     const int max_waves = 1;
 
     const std::vector<int> actor_counts = {
-	    1024
+	    30000
     };
 
 
@@ -652,19 +652,16 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 	    Ns.push_back(sizes[dist(rng)]);
 
 
+   
     //scheduler
     for (int num_actors : actor_counts) {
 
     
 	    // Initialize CUDA manager
-	    caf::cuda::manager::init(sys, man_config);
+	    caf::cuda::manager::init(sys);
 	    std::cout << "=====================================\n";
 	    std::cout << "Running with Scheduler with " << num_actors << " actors\n";
 
-	    auto& mgr = caf::cuda::manager::get();
-	    for (int i = 0; i < mgr.get_num_devices(); i++) {
-		    mgr.send_scheduler_actor_message("green",i); 
-	    }
 
 
 
@@ -716,6 +713,10 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     
 	caf::cuda::manager::shutdown();
     }
+
+
+
+
     caf::cuda::manager::shutdown();
 }
 

From d9bff9f22186f0989fb5bb01befc075e52cc83d8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 24 Mar 2026 12:49:44 -0600
Subject: [PATCH 0553/1000] Modified tests to include a uniform scaling test,
 as well as added more actor counts to the random test. THese changes were
 made to put all scheduler prototype benchmark tests in a single cpp file
 making it easier to run scripts.

---
 .../main.test.cpp                             | 95 +++++++++++++++++--
 1 file changed, 87 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 607340fddd..44d5829d62 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -624,7 +624,7 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     const int max_waves = 1;
 
     const std::vector<int> actor_counts = {
-	    30000
+	   30000,40000,50000
     };
 
 
@@ -659,10 +659,8 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     
 	    // Initialize CUDA manager
 	    caf::cuda::manager::init(sys);
-	    std::cout << "=====================================\n";
-	    std::cout << "Running with Scheduler with " << num_actors << " actors\n";
-
-
+  std::cout << "=====================================\n";
+        std::cout << "Random Scaling WITH scheduler | actors=" << num_actors << "\n";
 
 
         double elapsed = time_run([&]() {
@@ -692,9 +690,7 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 	    // Initialize CUDA manager
 	    caf::cuda::manager::init(sys);
 	    std::cout << "=====================================\n";
-	    std::cout << "Running no scheduler with " << num_actors << " actors\n";
-
-      
+	    std::cout << "Random Scaling NO scheduler | actors=" << num_actors << "\n";      
 	    double elapsed = time_run([&]() {
 
             auto sup = sys.spawn(
@@ -720,6 +716,89 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     caf::cuda::manager::shutdown();
 }
 
+void run_mmul_uniform_scaling_tests(caf::actor_system& sys,
+                                   caf::cuda::manager_config man_config) {
+
+    const int max_waves = 1;
+
+    // Matrix sizes: 1,2,4,...,2048
+    std::vector<int> matrix_sizes;
+    for (int n = 1; n <= 2048; n *= 2)
+        matrix_sizes.push_back(n);
+
+    // Actor counts: 10 → 1000
+    std::vector<int> actor_counts;
+    for (int a = 10; a <= 1000; a += 10)
+        actor_counts.push_back(a);
+
+    for (int N : matrix_sizes) {
+        for (int num_actors : actor_counts) {
+
+            // Create uniform pool (single size)
+            MatrixPool pool = create_matrix_pool_random(
+                1,
+                N,
+                N,
+                42
+            );
+
+            int total_tasks = num_actors * max_waves;
+
+            std::vector<int> Ns(total_tasks, N);
+
+            // ========================
+            // WITH SCHEDULER
+            // ========================
+            caf::cuda::manager::init(sys);
+            std::cout << "=====================================\n";
+            std::cout << "Uniform WITH scheduler | N=" << N
+                      << " actors=" << num_actors << "\n";
+
+            time_run([&]() {
+                auto sup = sys.spawn(
+                    supervisor_actor_fun,
+                    num_actors,
+                    max_waves,
+                    pool,
+                    Ns,
+                    true
+                );
+                sys.await_all_actors_done();
+            });
+
+            caf::cuda::manager::shutdown();
+
+            // ========================
+            // WITHOUT SCHEDULER
+            // ========================
+            caf::cuda::manager::init(sys);
+            std::cout << "=====================================\n";
+            std::cout << "Uniform NO scheduler | N=" << N
+                      << " actors=" << num_actors << "\n";
+
+            time_run([&]() {
+                auto sup = sys.spawn(
+                    supervisor_actor_fun,
+                    num_actors,
+                    max_waves,
+                    pool,
+                    Ns,
+                    false
+                );
+                sys.await_all_actors_done();
+            });
+
+            caf::cuda::manager::shutdown();
+        }
+    }
+}
+
+
+
+
+
+
+
 
 
 

From 1035df1b77d6f42ba7764b907c3d75b12d90ac04 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 24 Mar 2026 13:23:33 -0600
Subject: [PATCH 0554/1000] FIxed issue with actor counts size being too small
 when running the tests causing an exit deadlock.

---
 .../mmul-randonom-batch-benchmark/main.test.cpp                 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 44d5829d62..207397a9a7 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -628,7 +628,7 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     };
 
 
-    int num_actors = actor_counts[0];
+    int num_actors = actor_counts[actor_counts.size()];
 
         // Generate deterministic random pool
         MatrixPool pool = create_matrix_pool_random(

From 42ab10c40356efc565da63b557842a8bc6973e2b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 24 Mar 2026 13:31:52 -0600
Subject: [PATCH 0555/1000] Fixed the issue.

---
 .../mmul-randonom-batch-benchmark/main.test.cpp               | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 207397a9a7..731b08f4ab 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -624,11 +624,11 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     const int max_waves = 1;
 
     const std::vector<int> actor_counts = {
-	   30000,40000,50000
+	   1,30000,40000,50000
     };
 
 
-    int num_actors = actor_counts[actor_counts.size()];
+    int num_actors = actor_counts[actor_counts.size()-1];
 
         // Generate deterministic random pool
         MatrixPool pool = create_matrix_pool_random(

From 10fc66087ca5ba07efeb231893ac42a91112137a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 25 Mar 2026 10:06:26 -0600
Subject: [PATCH 0556/1000] Added a commented out cuctxcreate for compatability
 with newer versions of cuda.

---
 libcaf_cuda/src/platform.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libcaf_cuda/src/platform.cpp b/libcaf_cuda/src/platform.cpp
index 175dfe1808..c1477a84e8 100644
--- a/libcaf_cuda/src/platform.cpp
+++ b/libcaf_cuda/src/platform.cpp
@@ -29,6 +29,9 @@ platform::platform() {
     check(cuDeviceGetName(name, sizeof(name), cuda_device), "cuDeviceGetName");
     device_names[i] = name;
 
+//Use this if cuCtxCreate throws a compiler error
+//    check(cuCtxCreate(&contexts_[i],nullptr ,CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, cuda_device), "cuCtxCreate");
+
     check(cuCtxCreate(&contexts_[i], CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, cuda_device), "cuCtxCreate");
     devices_[i] = make_counted<device>(cuda_device, contexts_[i], name, i);
   }

From 33022cbebe56d9f925c10bbbfc88a122f8b1e661 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 25 Mar 2026 11:05:28 -0600
Subject: [PATCH 0557/1000] Saving before making changes.

---
 .../mmul-randonom-batch-benchmark/main.test.cpp                  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 731b08f4ab..6e9e9df19a 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -810,6 +810,7 @@ void caf_main(caf::actor_system& sys) {
 
 
   run_mmul_random_scaling_tests(sys,man_config);
+  run_mmul_uniform_scaling_tests(sys,man_config);
 
 
 

From 7b780da09738274a3caba90b436b744aec524251 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 26 Mar 2026 13:38:15 -0600
Subject: [PATCH 0558/1000] Updated supervisor so that when it creates mmul
 actors with scheduler it will auto subscribe them also scheduler actor timer
 is 50ms now. This change was made to make scheduler benchmark tests more
 worst case scenario.

---
 .../main.test.cpp                             | 37 ++++++++++++++++---
 1 file changed, 31 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 6e9e9df19a..cbb361ede4 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -302,7 +302,7 @@ caf::behavior mmul_actor_fun_scheduler2(caf::stateful_actor<mmul_state>* self,
 
 	int device = stream % caf::cuda::manager::get().get_num_devices();
 
-	self->mail("subscribe",self).send(scheduler_actor);
+	//self->mail("subscribe",self).send(scheduler_actor);
 
 	self->mail(N).send(self);
 
@@ -312,6 +312,7 @@ caf::behavior mmul_actor_fun_scheduler2(caf::stateful_actor<mmul_state>* self,
 		[=](std::vector<int> costs) {
 			//do nothing this is an overhead test 
 		
+			//std::cout << "N=" << N <<  "\n";
 		
 		},
 		[=](int N) {
@@ -411,7 +412,9 @@ caf::behavior scheduler_actor_fun(caf::stateful_actor<scheduler_actor_state>* se
 	self->state().num_devices = caf::cuda::manager::get().get_num_devices();
 	self->state().costs.resize(self->state().num_devices);
 
-	self->mail("publish").urgent().delay(std::chrono::milliseconds(5)).send(self);
+	int time = 50;
+
+	self->mail("publish").urgent().delay(std::chrono::milliseconds(time)).send(self);
 
 	return {
 
@@ -435,6 +438,7 @@ caf::behavior scheduler_actor_fun(caf::stateful_actor<scheduler_actor_state>* se
 				auto& subs = self->state().subscribers;
 
 				if (command == "subscribe") {
+					//std::cout << "Thank you for subscribing\n";
 					// avoid duplicates
 					if (std::find(subs.begin(), subs.end(), actor) == subs.end()) {
 						subs.push_back(actor);
@@ -453,12 +457,13 @@ caf::behavior scheduler_actor_fun(caf::stateful_actor<scheduler_actor_state>* se
 
 			[=](std::string command) {
 				if (command == "publish") {
+					//std::cout << "size = " << self->state().subscribers.size() << "\n";
 					for (caf::actor a : self->state().subscribers) {
 
 						self -> mail(self->state().costs).urgent().send(a);
 
 					}
-					self->mail("publish").urgent().delay(std::chrono::milliseconds(5)).send(self);
+					self->mail("publish").urgent().delay(std::chrono::milliseconds(time)).send(self);
 				}
 
 			}	
@@ -520,6 +525,9 @@ caf::behavior supervisor_actor_fun(
         [=](std::string cmd) {
             if (cmd != "spawn") return;
 
+	     std::chrono::steady_clock::time_point cmd_start_time = std::chrono::steady_clock::now();
+	    
+
             self->state().completed = 0;
             self->state().wave_start_time = std::chrono::steady_clock::now();
 
@@ -539,7 +547,7 @@ caf::behavior supervisor_actor_fun(
                 caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
 		if (use_scheduler) {
-			self->spawn(mmul_actor_fun_scheduler2, 
+			caf::actor a = 	self->spawn(mmul_actor_fun_scheduler2, 
 					self,
 					scheduler_actor,
 				       	program, 
@@ -548,6 +556,9 @@ caf::behavior supervisor_actor_fun(
 					N,
                             caf::cuda::create_in_arg(A),
                             caf::cuda::create_in_arg(B));
+
+
+		 self->mail("subscribe",a).send(scheduler_actor);
 		}
 		else {
 			self->spawn(mmul_actor_fun, self, program, dims,i,N,
@@ -555,7 +566,21 @@ caf::behavior supervisor_actor_fun(
                             caf::cuda::create_in_arg(B));
 		}
             }
-        },
+        
+	
+
+    std::chrono::steady_clock::time_point cmd_end_time = std::chrono::steady_clock::now();
+
+      std::chrono::duration<double> total_time =
+                        cmd_end_time - cmd_start_time;
+
+                    std::cout << "\n===== SUPERVISOR TOTAL TIME spawn =====\n";
+                    std::cout << "Total runtime: "
+                              << total_time.count() << " s\n";
+
+
+	
+	},
 
         // -------------------- COMPLETION TRACKING --------------------
         [=](int done) {
@@ -624,7 +649,7 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     const int max_waves = 1;
 
     const std::vector<int> actor_counts = {
-	   1,30000,40000,50000
+	  30000,40000,50000
     };
 
 

From f2f5aa082a5f199632f0936048a944f362e2d40e Mon Sep 17 00:00:00 2001
From: KyleKlenk <kyle.c.klenk@gmail.com>
Date: Mon, 6 Apr 2026 10:00:36 -0600
Subject: [PATCH 0559/1000] Making new test directory

---
 libcaf_cuda/sc26/README.md | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 libcaf_cuda/sc26/README.md

diff --git a/libcaf_cuda/sc26/README.md b/libcaf_cuda/sc26/README.md
new file mode 100644
index 0000000000..e69de29bb2

From 1fe848a5578febead1381f5bd35c4d28d12d8920 Mon Sep 17 00:00:00 2001
From: KyleKlenk <kyle.c.klenk@gmail.com>
Date: Mon, 6 Apr 2026 10:33:32 -0600
Subject: [PATCH 0560/1000] Initial tests for Runtime-Overhead and
 Sequence-Independent-Tests

---
 .gitignore                                    |   5 +
 .../sc26/Runtime-Overhead/actor_facade.cpp    |  59 +++
 .../sc26/Runtime-Overhead/command_runner.cpp  | 393 ++++++++++++++++++
 .../sc26/Runtime-Overhead/cuda_native.cpp     | 225 ++++++++++
 libcaf_cuda/sc26/Runtime-Overhead/mmul.cu     |  16 +
 .../actor_facade.cpp                          |   0
 .../command_runner.cpp                        | 205 +++++++++
 .../cuda_native.cpp                           | 156 +++++++
 8 files changed, 1059 insertions(+)
 create mode 100644 libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
 create mode 100644 libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
 create mode 100644 libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
 create mode 100644 libcaf_cuda/sc26/Runtime-Overhead/mmul.cu
 create mode 100644 libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
 create mode 100644 libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
 create mode 100644 libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp

diff --git a/.gitignore b/.gitignore
index ff9dc549a5..5f1ffabf2a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,8 @@ manual/libcaf_core
 manual/libcaf_io
 manual/libcaf_net
 manual/libcaf_openssl
+libcaf_cuda/sc26/Runtime-Overhead/compile.sh
+libcaf_cuda/sc26/Runtime-Overhead/cuda_native
+libcaf_cuda/sc26/Runtime-Overhead/mmul.cubin
+libcaf_cuda/sc26/Sequence-Independent-Tasks/compile.sh
+libcaf_cuda/sc26/Runtime-Overhead/command_runner
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp b/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
new file mode 100644
index 0000000000..a1aa4e537a
--- /dev/null
+++ b/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
@@ -0,0 +1,59 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include "../common/kernel_paths.hpp"
+#include "vector"
+
+class MatMult {
+  caf::event_based_actor* self_;
+  std::vector<int> A_ = {1, 2,
+                         3, 4};
+  std::vector<int> B_ = {5, 6,
+                         7, 8};
+
+
+  public:
+    MatMult(caf::event_based_actor* self) : self_(self) {};
+
+    caf::behavior make_behavior() {
+      return {
+        [this](int N) {
+
+          int THREADS = 32;
+          int BLOCKS = (N + THREADS - 1) / THREADS;
+
+          // Launch dims: 2x2 threads is enough for a 2x2 product
+          caf::cuda::nd_range dim(
+              BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+          auto gpuActor = self_->system().cuda_manager().spawnFromCUBIN(
+              actor_tests::paths::matmul_verbose_cubin, "matrixMul", dim,
+              in<int>{}, in<int>{}, out<int>{}, in<int>{});
+
+          self_->mail(
+              caf::cuda::create_in_arg(A_), 
+              caf::cuda::create_in_arg(B_), 
+              caf::cuda::create_out_arg_with_size<int>(N * N), // Seems there is no need to create a host buffer, or maybe that is what this does?
+              caf::cuda::create_in_arg(N))
+              .send(gpuActor); 
+        },
+
+        [this](const std::vector<output_buffer>& result) {
+          std::vector<int> output = caf::cuda::extract_vector<int>(result);
+          self_->println("Received result from GPU actor {}, {}, {}, {} ", 
+                             output[0], output[1], output[2], output[3]);
+          self_->quit();
+        }
+      };
+    }
+};
+
+
+void caf_main(caf::actor_system& sys) {
+    caf::scoped_actor self{sys};
+    self->println("Hello, CAF!");
+    auto test_actor = self->spawn(caf::actor_from_state<MatMult>);
+    self->mail(2).send(test_actor);
+    self->await_all_other_actors_done();
+}
+
+CAF_MAIN()
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
new file mode 100644
index 0000000000..3abf48a2e5
--- /dev/null
+++ b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
@@ -0,0 +1,393 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/component-actors/mmul_actor/mmul_actor.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+using command =
+  caf::cuda::command_runner<>;
+
+command mmul_command;
+
+struct mmul_state {
+};
+
+//global output buffer meant to disclude it from timing
+//the other benchmark test do not include its memory allocations in it 
+//so its only fair that we do not either 
+std::vector<int> matrixC;
+
+
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
+  return {
+
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        int N) {
+      self->println("fun 1");
+      using clock = std::chrono::steady_clock;
+      using ms = std::chrono::duration<double, std::milli>;
+
+
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+      int device = 0;
+      int stream = 1;
+
+      auto program =
+        mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
+
+      auto t_total_start = clock::now();
+      // -------------------------
+      // create_in_arg A
+      // -------------------------
+      auto t_a_inarg_start = clock::now();
+
+      auto inA = caf::cuda::create_in_arg(std::move(matrixA));
+
+      auto t_a_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer A
+      // -------------------------
+      auto t_a_transfer_start = clock::now();
+
+      auto arg1 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inA));
+
+      auto t_a_transfer_end = clock::now();
+
+      // -------------------------
+      // create_in_arg B
+      // -------------------------
+      auto t_b_inarg_start = clock::now();
+
+      auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+
+      auto t_b_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer B
+      // -------------------------
+      auto t_b_transfer_start = clock::now();
+
+      auto arg2 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inB));
+
+      auto t_b_transfer_end = clock::now();
+
+      // -------------------------
+      // spawn actor
+      // -------------------------
+      auto t_spawn_start = clock::now();
+
+      caf::actor mmul_actor =
+        self->spawn(caf::cuda::mmul_actor_fun<int>, program);
+
+      auto t_spawn_end = clock::now();
+
+      // -------------------------
+      // request
+      // -------------------------
+      auto t_request_start = clock::now();
+
+      self->mail(arg1, arg2, N, device, stream)
+        .request(mmul_actor, std::chrono::seconds(30))
+        .then(
+          [=](caf::cuda::mem_ptr<int> dC) {
+
+            auto t_response_received = clock::now();
+
+            //std::vector<int> matrixC(N*N);
+            // -------------------------
+            // copy to host
+            // -------------------------
+            auto t_copy_start = clock::now();
+
+            //std::vector<int> matrixC = dC->copy_to_host();
+
+	    dC->copy_to_host(matrixC.data(),N*N);
+
+            auto t_copy_end = clock::now();
+            auto t_total_end = clock::now();
+
+            // -------------------------
+            // Print timings
+            // -------------------------
+
+            std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
+
+            std::cout << "create_in_arg A: "
+                      << ms(t_a_inarg_end - t_a_inarg_start).count()
+                      << " ms\n";
+
+            std::cout << "transfer A: "
+                      << ms(t_a_transfer_end - t_a_transfer_start).count()
+                      << " ms\n";
+
+            std::cout << "create_in_arg B: "
+                      << ms(t_b_inarg_end - t_b_inarg_start).count()
+                      << " ms\n";
+
+            std::cout << "transfer B: "
+                      << ms(t_b_transfer_end - t_b_transfer_start).count()
+                      << " ms\n";
+
+            std::cout << "spawn actor: "
+                      << ms(t_spawn_end - t_spawn_start).count()
+                      << " ms\n";
+
+            std::cout << "request → response latency: "
+                      << ms(t_response_received - t_request_start).count()
+                      << " ms\n";
+
+            std::cout << "copy_to_host: "
+                      << ms(t_copy_end - t_copy_start).count()
+                      << " ms\n";
+
+            std::cout << "TOTAL end-to-end: "
+                      << ms(t_total_end - t_total_start).count()
+                      << " ms\n";
+
+            std::cout << "=============================================\n";
+
+            self->quit();
+          }
+        );
+    }
+
+  };
+}
+
+
+caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
+  return {
+
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        int N) {
+      
+      self->println("fun 2");
+      using clock = std::chrono::steady_clock;
+      using ms = std::chrono::duration<double, std::milli>;
+
+
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+      int device = 0;
+      int stream = 1;
+
+      auto program =
+        mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
+
+      auto t_total_start = clock::now();
+      // -------------------------
+      // create_in_arg A
+      // -------------------------
+      auto t_a_inarg_start = clock::now();
+
+      auto inA = caf::cuda::create_in_arg(std::move(matrixA));
+
+      auto t_a_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer A
+      // -------------------------
+      auto t_a_transfer_start = clock::now();
+
+      auto arg1 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inA));
+
+      auto t_a_transfer_end = clock::now();
+
+      // -------------------------
+      // create_in_arg B
+      // -------------------------
+      auto t_b_inarg_start = clock::now();
+
+      auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+
+      auto t_b_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer B
+      // -------------------------
+      auto t_b_transfer_start = clock::now();
+
+      auto arg2 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inB));
+
+      auto t_b_transfer_end = clock::now();
+  
+      const int THREADS = 32;
+      const int BLOCKS = (N + THREADS - 1) / THREADS;
+
+      caf::cuda::nd_range dims(
+        BLOCKS, BLOCKS, 1,
+        THREADS, THREADS, 1);
+
+      // -------------------------
+      // request
+      // -------------------------
+      auto t_request_start = clock::now();
+
+      caf::cuda::mmul_async_command<int> command;
+      auto output = command.run_async(
+		      program,dims,
+		      1,
+		      arg1,arg2,out<int>{N*N},in<int>{N});
+
+       auto t_response_received = clock::now();
+
+            //std::vector<int> matrixC(N*N);
+            // -------------------------
+            // copy to host
+            // -------------------------
+            auto t_copy_start = clock::now();
+
+	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);
+
+            //std::vector<int> matrixC = dC->copy_to_host();
+
+	    dC->copy_to_host(matrixC.data(),N*N);
+
+            auto t_copy_end = clock::now();
+            auto t_total_end = clock::now();
+
+            // -------------------------
+            // Print timings
+            // -------------------------
+
+            std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
+
+            std::cout << "create_in_arg A: "
+                      << ms(t_a_inarg_end - t_a_inarg_start).count()
+                      << " ms\n";
+
+            std::cout << "transfer A: "
+                      << ms(t_a_transfer_end - t_a_transfer_start).count()
+                      << " ms\n";
+
+            std::cout << "create_in_arg B: "
+                      << ms(t_b_inarg_end - t_b_inarg_start).count()
+                      << " ms\n";
+
+            std::cout << "transfer B: "
+                      << ms(t_b_transfer_end - t_b_transfer_start).count()
+                      << " ms\n";
+
+                       std::cout << "request → response latency: "
+                      << ms(t_response_received - t_request_start).count()
+                      << " ms\n";
+
+            std::cout << "copy_to_host: "
+                      << ms(t_copy_end - t_copy_start).count()
+                      << " ms\n";
+
+            std::cout << "TOTAL end-to-end: "
+                      << ms(t_total_end - t_total_start).count()
+                      << " ms\n";
+
+            std::cout << "=============================================\n";
+
+           self->quit();
+    }
+
+  };
+}
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size) {
+
+
+  caf::cuda::manager::init(sys);
+  // ------------------------------------
+  // Start timing
+  // ------------------------------------
+  auto start = std::chrono::steady_clock::now();
+
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<int> matrixA(matrix_size * matrix_size,2);
+  std::vector<int> matrixB(matrix_size * matrix_size,3);
+
+  matrixC.resize(matrix_size*matrix_size);
+
+ using clock = std::chrono::steady_clock;
+
+auto t_start = clock::now();
+
+caf::actor a =sys.spawn(mmul_actor_fun_2);
+
+anon_mail(matrixA,matrixB,matrix_size).send(a);
+
+auto t_end = clock::now();
+
+
+
+  // Wait for all actors to finish
+  sys.await_all_actors_done();
+
+  // ------------------------------------
+  // Stop timing
+  // ------------------------------------
+  auto end = std::chrono::steady_clock::now();
+  auto duration_ms =
+      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
+            << ", time=" << duration_ms << " ms\n";
+
+  caf::cuda::manager::shutdown();
+
+}
+
+
+void caf_main(caf::actor_system& sys) {
+ run_mmul_test(sys,1000);
+  run_mmul_test(sys,4000);
+  run_mmul_test(sys,8000);
+  run_mmul_test(sys,12000);
+
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp b/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
new file mode 100644
index 0000000000..32f2faeaf3
--- /dev/null
+++ b/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
@@ -0,0 +1,225 @@
+// matrix_mul_driver.cpp
+#include <cuda.h>
+#include <iostream>
+#include <vector>
+#include <chrono>
+#include <stdexcept>
+#include <random>
+
+static const unsigned int RANDOM_SEED = 42;
+
+static void checkCU(CUresult r, const char* where) {
+    if (r != CUDA_SUCCESS) {
+        const char *str = nullptr;
+        cuGetErrorString(r, &str);
+        std::cerr << "CUDA Driver API error at " << where << " -> "
+                  << (str ? str : "unknown") << " (" << (int)r << ")\n";
+        std::exit(EXIT_FAILURE);
+    }
+}
+
+// runMatrixMul: executes the kernel and returns the total duration in milliseconds
+double runMatrixMul(CUmodule module, CUfunction kernel, int N) {
+    using clock = std::chrono::steady_clock;
+    using ms = std::chrono::duration<double, std::milli>;
+
+    std::cout << "\n===== DRIVER BENCHMARK (N=" << N << ") =====\n";
+
+    size_t elements = (size_t)N * (size_t)N;
+    size_t bytes = elements * sizeof(int);
+
+    std::mt19937 rng(RANDOM_SEED);
+    std::uniform_int_distribution<int> dist(1, 10);
+
+    std::vector<int> h_a(elements);
+    std::vector<int> h_b(elements);
+    std::vector<int> h_c(elements);
+
+    for (auto& v : h_a) v = dist(rng);
+    for (auto& v : h_b) v = dist(rng);
+
+    CUdeviceptr d_a, d_b, d_c;
+    CUstream stream;
+
+    // ----------------------------------
+    // Create Stream
+    // ----------------------------------
+    checkCU(cuStreamCreate(&stream, CU_STREAM_DEFAULT), "cuStreamCreate");
+
+    auto t_total_start = clock::now();
+
+    // ----------------------------------
+    // Device Allocation
+    // ----------------------------------
+    auto t_alloc_start = clock::now();
+
+    checkCU(cuMemAlloc(&d_a, bytes), "cuMemAlloc d_a");
+    checkCU(cuMemAlloc(&d_b, bytes), "cuMemAlloc d_b");
+    checkCU(cuMemAlloc(&d_c, bytes), "cuMemAlloc d_c");
+
+    auto t_alloc_end = clock::now();
+
+    // ----------------------------------
+    // H2D copy A
+    // ----------------------------------
+    auto t_h2d_a_start = clock::now();
+
+    checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), bytes, stream), "cuMemcpyHtoDAsync A");
+    //checkCU(cuStreamSynchronize(stream), "sync A");
+
+    auto t_h2d_a_end = clock::now();
+
+    // ----------------------------------
+    // H2D copy B
+    // ----------------------------------
+    auto t_h2d_b_start = clock::now();
+
+    checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), bytes, stream), "cuMemcpyHtoDAsync B");
+    //checkCU(cuStreamSynchronize(stream), "sync B");
+
+    auto t_h2d_b_end = clock::now();
+
+    // ----------------------------------
+    // Kernel launch + execution
+    // ----------------------------------
+    const unsigned int blockX = 32;
+    const unsigned int blockY = 32;
+    unsigned int gridX = (N + blockX - 1) / blockX;
+    unsigned int gridY = (N + blockY - 1) / blockY;
+
+    void* kernelParams[] = { &d_a, &d_b, &d_c, &N };
+
+    auto t_kernel_start = clock::now();
+
+    checkCU(cuLaunchKernel(kernel,
+                           gridX, gridY, 1,
+                           blockX, blockY, 1,
+                           0,
+                           stream,
+                           kernelParams,
+                           nullptr),
+            "cuLaunchKernel");
+
+   // checkCU(cuStreamSynchronize(stream), "kernel sync");
+
+    auto t_kernel_end = clock::now();
+
+    // ----------------------------------
+    // D2H copy
+    // ----------------------------------
+    auto t_d2h_start = clock::now();
+
+    cuMemcpyDtoHAsync(h_c.data(), d_c, bytes, stream);
+   // cuMemcpyDtoHAsync(h_c, d_c, bytes, stream);
+    cuStreamSynchronize(stream);
+    auto t_d2h_end = clock::now();
+
+    // ----------------------------------
+    // Free device memory
+    // (included in the total to match the actor benchmark which frees
+    //  device memory inside runner.run_into())
+    // ----------------------------------
+    auto t_free_start = clock::now();
+
+    checkCU(cuMemFree(d_a), "cuMemFree A");
+    checkCU(cuMemFree(d_b), "cuMemFree B");
+    checkCU(cuMemFree(d_c), "cuMemFree C");
+
+    auto t_free_end = clock::now();
+
+    auto t_total_end = clock::now();
+
+
+
+
+    // ----------------------------------
+    // Destroy stream
+    // ----------------------------------
+    checkCU(cuStreamDestroy(stream), "cuStreamDestroy");
+
+
+    // ----------------------------------
+    // Print Results
+    // ----------------------------------
+
+    std::cout << "Device allocation: "
+              << ms(t_alloc_end - t_alloc_start).count()
+              << " ms\n";
+
+    std::cout << "H2D copy A: "
+              << ms(t_h2d_a_end - t_h2d_a_start).count()
+              << " ms\n";
+
+    std::cout << "H2D copy B: "
+              << ms(t_h2d_b_end - t_h2d_b_start).count()
+              << " ms\n";
+
+    std::cout << "Kernel execution: "
+              << ms(t_kernel_end - t_kernel_start).count()
+              << " ms\n";
+
+    std::cout << "D2H copy: "
+              << ms(t_d2h_end - t_d2h_start).count()
+              << " ms\n";
+
+    std::cout << "Device free: "
+              << ms(t_free_end - t_free_start).count()
+              << " ms\n";
+
+    double total_ms = ms(t_total_end - t_total_start).count();
+
+    std::cout << "TOTAL: " << total_ms << " ms\n";
+    std::cout << "=============================================\n";
+
+    return total_ms;
+}
+
+int main(int argc, char** argv) {
+    std::vector<int> sizes = {1000, 2000, 4000, 8000, 16000};
+    
+   if (argc > 1) {
+        sizes.clear();
+        for (int i = 1; i < argc; ++i) sizes.push_back(std::stoi(argv[i]));
+    }
+
+    checkCU(cuInit(0), "cuInit");
+
+    CUdevice dev;
+    checkCU(cuDeviceGet(&dev, 0), "cuDeviceGet(0)");
+
+    CUcontext ctx;
+    checkCU(cuCtxCreate(&ctx, 0, dev), "cuCtxCreate");
+
+    CUmodule module;
+    checkCU(cuModuleLoad(&module, "mmul.cubin"), "cuModuleLoad mmul.cubin");
+
+    CUfunction kernel;
+    checkCU(cuModuleGetFunction(&kernel, module, "matrixMul"), "cuModuleGetFunction matrixMul");
+
+    // Warmup: small run to prime CUDA lazy-init / cubin load before timed tests
+    runMatrixMul(module, kernel, 64);
+    std::cout << "--- warmup complete ---\n";
+
+    std::vector<std::pair<int,double>> results;
+
+    for (int N : sizes) {
+        try {
+            double t = runMatrixMul(module, kernel, N);
+            results.emplace_back(N, t);
+        } catch (const std::exception &e) {
+            std::cerr << "Exception while running N=" << N << ": " << e.what() << "\n";
+        }
+        std::cout << "----------------------------------------\n";
+    }
+
+    // Print summary of results
+    std::cout << "\nMatrix size : time (ms)\n";
+    for (auto &p : results) {
+        std::cout << p.first << " : " << p.second << " ms\n";
+    }
+
+    checkCU(cuModuleUnload(module), "cuModuleUnload");
+    checkCU(cuCtxDestroy(ctx), "cuCtxDestroy");
+
+    return 0;
+}
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/mmul.cu b/libcaf_cuda/sc26/Runtime-Overhead/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/sc26/Runtime-Overhead/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
new file mode 100644
index 0000000000..c52d09612b
--- /dev/null
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
@@ -0,0 +1,205 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/component-actors/mmul_actor/mmul_actor.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+using command =
+  caf::cuda::command_runner<>;
+
+command mmul_command;
+
+struct mmul_state {
+	caf::cuda::program_ptr program;
+};
+
+//global output buffer meant to disclude it from timing
+//the other benchmark test do not include its memory allocations in it 
+//so its only fair that we do not either 
+std::vector<int> matrixC;
+
+
+
+
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
+		caf::cuda::program_ptr mmul_kernel) {
+  
+
+	self ->state().program = mmul_kernel;
+
+return {
+
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        int N) {
+
+      using clock = std::chrono::steady_clock;
+      using ms = std::chrono::duration<double, std::milli>;
+
+
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+      int device = 0;
+      int stream = 1;
+
+           auto t_total_start = clock::now();
+      // -------------------------
+      // create_in_arg A
+      // -------------------------
+      auto t_a_inarg_start = clock::now();
+
+      auto inA = caf::cuda::create_in_arg(std::move(matrixA));
+
+      auto t_a_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer A
+      // -------------------------
+      auto t_a_transfer_start = clock::now();
+
+      auto arg1 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inA));
+
+      auto t_a_transfer_end = clock::now();
+
+      // -------------------------
+      // create_in_arg B
+      // -------------------------
+      auto t_b_inarg_start = clock::now();
+
+      auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+
+      auto t_b_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer B
+      // -------------------------
+      auto t_b_transfer_start = clock::now();
+
+      auto arg2 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inB));
+
+      auto t_b_transfer_end = clock::now();
+  
+      const int THREADS = 32;
+      const int BLOCKS = (N + THREADS - 1) / THREADS;
+
+      caf::cuda::nd_range dims(
+        BLOCKS, BLOCKS, 1,
+        THREADS, THREADS, 1);
+
+      caf::cuda::mmul_async_command<int> command;
+      auto output = command.run_async(
+		      self->state().program,
+		      dims,
+		      1,
+		      arg1,arg2,out<int>{N*N},in<int>{N});
+
+	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);
+
+
+	    dC->copy_to_host(matrixC.data(),N*N);
+
+            auto t_copy_end = clock::now();
+            auto t_total_end = clock::now();
+
+    }
+
+  };
+}
+
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size,int iterations) {
+
+
+  caf::cuda::manager::init(sys);
+  // ------------------------------------
+  // Start timing
+  // ------------------------------------
+
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<int> matrixA(matrix_size * matrix_size,2);
+  std::vector<int> matrixB(matrix_size * matrix_size,3);
+
+  matrixC.resize(matrix_size*matrix_size);
+ 
+  auto& mgr = caf::cuda::manager::get();
+  
+  auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+
+  using clock = std::chrono::steady_clock;
+
+  auto start = std::chrono::steady_clock::now();
+
+  caf::actor a =sys.spawn(mmul_actor_fun,program);
+
+  for (int i = 0; i < iterations; i++) 
+	  anon_mail(matrixA,matrixB,matrix_size).send(a);
+
+  anon_send_exit(a,caf::exit_reason::kill);
+  // Wait for all actors to finish
+  sys.await_all_actors_done();
+
+  // ------------------------------------
+  // Stop timing
+  // ------------------------------------
+  auto end = std::chrono::steady_clock::now();
+  auto duration_ms =
+      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
+            << " iterations = " << iterations << 
+	    ", time=" << duration_ms << " ms\n";
+
+  caf::cuda::manager::shutdown();
+
+}
+
+
+void caf_main(caf::actor_system& sys) {
+
+	for (int i = 1000; i < 11000; i+=1000)
+		run_mmul_test(sys,1000,i);
+
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp
new file mode 100644
index 0000000000..1046f774fd
--- /dev/null
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp
@@ -0,0 +1,156 @@
+#include <cuda.h>
+#include <iostream>
+#include <vector>
+#include <chrono>
+#include <fstream>
+#include <sstream>
+#include <stdexcept>
+
+static void checkCU(CUresult r, const char* where) {
+    if (r != CUDA_SUCCESS) {
+        const char *str = nullptr;
+        cuGetErrorString(r, &str);
+        std::cerr << "CUDA Driver API error at " << where << " -> "
+                  << (str ? str : "unknown") << " (" << (int)r << ")\n";
+        std::exit(EXIT_FAILURE);
+    }
+}
+
+std::string readFile(const std::string &path) {
+    std::ifstream in(path, std::ios::in | std::ios::binary);
+    if (!in) throw std::runtime_error("Failed to open " + path);
+    std::ostringstream ss;
+    ss << in.rdbuf();
+    return ss.str();
+}
+
+// Launch kernel once
+void launchKernel(CUfunction kernel, CUstream stream,
+                  CUdeviceptr d_a, CUdeviceptr d_b, CUdeviceptr d_c, int N) {
+    const unsigned int blockX = 32;
+    const unsigned int blockY = 32;
+    unsigned int gridX = (N + blockX - 1) / blockX;
+    unsigned int gridY = (N + blockY - 1) / blockY;
+
+    void* kernelParams[] = { &d_a, &d_b, &d_c, &N };
+
+    checkCU(cuLaunchKernel(kernel,
+                           gridX, gridY, 1,
+                           blockX, blockY, 1,
+                           0,
+                           stream,
+                           kernelParams,
+                           nullptr),
+            "cuLaunchKernel");
+}
+
+int main() {
+    const int N = 1000;
+    std::vector<int> iteration_series = {1000, 2000, 3000, 4000, 5000,
+                                         6000, 7000, 8000, 9000, 10000};
+
+    checkCU(cuInit(0), "cuInit");
+
+    CUdevice dev;
+    checkCU(cuDeviceGet(&dev, 0), "cuDeviceGet(0)");
+
+    CUcontext ctx;
+    checkCU(cuCtxCreate(&ctx, 0, dev), "cuCtxCreate");
+
+    std::string cubin = readFile("mmul.cubin");
+
+    CUmodule module;
+    checkCU(cuModuleLoadDataEx(&module, cubin.data(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
+
+    CUfunction kernel;
+    checkCU(cuModuleGetFunction(&kernel, module, "matrixMul"), "cuModuleGetFunction matrixMul");
+
+    // ----------------------------------
+    // Persistent host buffers
+    // ----------------------------------
+    size_t elements = (size_t)N * N;
+    std::vector<int> h_a(elements, 1);
+    std::vector<int> h_b(elements, 1);
+    std::vector<int> h_c(elements, 0);
+
+    CUstream stream;
+    checkCU(cuStreamCreate(&stream, CU_STREAM_DEFAULT), "cuStreamCreate");
+
+    using clock = std::chrono::steady_clock;
+
+    // Warmup: prime CUDA context and allocator before timed series
+    {
+        const int warmup_iters = 10;
+        for (int i = 0; i < warmup_iters; ++i) {
+            CUdeviceptr d_a, d_b, d_c;
+            checkCU(cuMemAlloc(&d_a, elements * sizeof(int)), "warmup alloc d_a");
+            checkCU(cuMemAlloc(&d_b, elements * sizeof(int)), "warmup alloc d_b");
+            checkCU(cuMemAlloc(&d_c, elements * sizeof(int)), "warmup alloc d_c");
+            checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), elements * sizeof(int), stream), "warmup H2D d_a");
+            checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), elements * sizeof(int), stream), "warmup H2D d_b");
+            launchKernel(kernel, stream, d_a, d_b, d_c, N);
+            checkCU(cuMemcpyDtoHAsync(h_c.data(), d_c, elements * sizeof(int), stream), "warmup D2H d_c");
+            checkCU(cuMemFree(d_a), "warmup free d_a");
+            checkCU(cuMemFree(d_b), "warmup free d_b");
+            checkCU(cuMemFree(d_c), "warmup free d_c");
+        }
+        checkCU(cuStreamSynchronize(stream), "warmup sync");
+        std::cout << "--- warmup complete ---\n";
+    }
+
+    for (int iterations : iteration_series) {
+        auto start = clock::now();
+
+        for (int i = 0; i < iterations; ++i) {
+            // ----------------------------------
+            // Allocate device memory each iteration
+            // ----------------------------------
+            CUdeviceptr d_a, d_b, d_c;
+            checkCU(cuMemAlloc(&d_a, elements * sizeof(int)), "cuMemAlloc d_a");
+            checkCU(cuMemAlloc(&d_b, elements * sizeof(int)), "cuMemAlloc d_b");
+            checkCU(cuMemAlloc(&d_c, elements * sizeof(int)), "cuMemAlloc d_c");
+
+            // ----------------------------------
+            // Copy persistent host buffers to device
+            // ----------------------------------
+            checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), elements * sizeof(int), stream), "H2D d_a");
+            checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), elements * sizeof(int), stream), "H2D d_b");
+
+            // ----------------------------------
+            // Launch kernel
+            // ----------------------------------
+            launchKernel(kernel, stream, d_a, d_b, d_c, N);
+
+            // ----------------------------------
+            // Copy result back
+            // ----------------------------------
+            checkCU(cuMemcpyDtoHAsync(h_c.data(), d_c, elements * sizeof(int), stream), "D2H d_c");
+
+            // ----------------------------------
+            // Free device memory
+            // ----------------------------------
+            checkCU(cuMemFree(d_a), "cuMemFree d_a");
+            checkCU(cuMemFree(d_b), "cuMemFree d_b");
+            checkCU(cuMemFree(d_c), "cuMemFree d_c");
+        }
+
+        // Synchronize stream after series
+        checkCU(cuStreamSynchronize(stream), "stream sync after series");
+
+        auto end = clock::now();
+        double total_ms = std::chrono::duration<double, std::milli>(end - start).count();
+
+        std::cout << "[SERIES RESULT] Matrix " << N << "x" << N
+                  << ", iterations = " << iterations
+                  << ", total GPU time = " << total_ms << " ms\n";
+    }
+
+    // ----------------------------------
+    // Cleanup
+    // ----------------------------------
+    checkCU(cuStreamDestroy(stream), "cuStreamDestroy");
+    checkCU(cuModuleUnload(module), "cuModuleUnload");
+    checkCU(cuCtxDestroy(ctx), "cuCtxDestroy");
+
+    return 0;
+}

From 922cf62d0662f5b4928b3d3ef87a48b74bf3b7d6 Mon Sep 17 00:00:00 2001
From: KyleKlenk <kyle.c.klenk@gmail.com>
Date: Mon, 6 Apr 2026 10:39:54 -0600
Subject: [PATCH 0561/1000] Add foundation for fault-tolerance test

---
 .gitignore                                    |   3 +-
 libcaf_cuda/sc26/Fault-Tolerance/Makefile     |  39 ++
 libcaf_cuda/sc26/Fault-Tolerance/main.cpp     | 372 ++++++++++++++++++
 .../sc26/Fault-Tolerance/monte_carlo.cu       |  25 ++
 libcaf_cuda/sc26/Runtime-Overhead/Makefile    |  46 +++
 .../sc26/Sequence-Independent-Tasks/Makefile  |  44 +++
 6 files changed, 528 insertions(+), 1 deletion(-)
 create mode 100644 libcaf_cuda/sc26/Fault-Tolerance/Makefile
 create mode 100644 libcaf_cuda/sc26/Fault-Tolerance/main.cpp
 create mode 100644 libcaf_cuda/sc26/Fault-Tolerance/monte_carlo.cu
 create mode 100644 libcaf_cuda/sc26/Runtime-Overhead/Makefile
 create mode 100644 libcaf_cuda/sc26/Sequence-Independent-Tasks/Makefile

diff --git a/.gitignore b/.gitignore
index 5f1ffabf2a..eb938ddb25 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,6 @@
 .make-release-steps.bash
 .push-doxygen-steps.bash
-Makefile
+# Makefile
 bin/*
 build/*
 doxygen-log.txt
@@ -16,3 +16,4 @@ libcaf_cuda/sc26/Runtime-Overhead/cuda_native
 libcaf_cuda/sc26/Runtime-Overhead/mmul.cubin
 libcaf_cuda/sc26/Sequence-Independent-Tasks/compile.sh
 libcaf_cuda/sc26/Runtime-Overhead/command_runner
+libcaf_cuda/sc26/Fault-Tolerance/compile.sh
diff --git a/libcaf_cuda/sc26/Fault-Tolerance/Makefile b/libcaf_cuda/sc26/Fault-Tolerance/Makefile
new file mode 100644
index 0000000000..aef4ca6a08
--- /dev/null
+++ b/libcaf_cuda/sc26/Fault-Tolerance/Makefile
@@ -0,0 +1,39 @@
+NVCC ?= nvcc
+CXX ?= g++
+CXXFLAGS = -std=c++20 -include cstddef -include cerrno -include climits
+
+CUDA_INCLUDE_DIR ?=
+CUDA_LIB_DIR ?= 
+
+CAF_INCLUDE_DIR ?=
+CAF_LIB_DIR ?=
+
+LIBS = -L$(CAF_LIB_DIR) \
+ 			-lcaf_core \
+			-lcaf_io \
+			-lcaf_net \
+			-lcaf_cuda \
+			-L$(CUDA_LIB_DIR) \
+			-lcuda
+
+INCLUDES = -I$(CAF_INCLUDE_DIR) \
+		   -I$(CUDA_INCLUDE_DIR)
+
+LDFLAGS=-Wl,-rpath,$(CAF_LIB_DIR) \
+		 		-Wl,-rpath,$(CUDA_LIB_DIR)
+NVCC_ARCH ?= sm_$(shell nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.')
+
+all: monte_carlo main
+
+monte_carlo: monte_carlo.cu
+	$(NVCC) -g -arch=$(NVCC_ARCH) --cubin monte_carlo.cu -o monte_carlo.cubin
+
+main: main.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o main main.cpp $(LIBS) $(LDFLAGS)
+
+
+clean:
+	rm -f monte_carlo.cubin
+	rm -f main
+
+.PHONY: all clean
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Fault-Tolerance/main.cpp b/libcaf_cuda/sc26/Fault-Tolerance/main.cpp
new file mode 100644
index 0000000000..895671bc89
--- /dev/null
+++ b/libcaf_cuda/sc26/Fault-Tolerance/main.cpp
@@ -0,0 +1,372 @@
+// example_14: Fault-tolerant Monte Carlo π estimation with GPU workers.
+//
+// Demonstrates:
+//  1. Multiple GPU worker actors each owning a separate CUDA stream, running
+//     Monte Carlo batches concurrently on the same physical device.
+//  2. A supervisor actor that distributes batches, monitors workers, and
+//     recovers automatically when a worker is killed mid-computation.
+//  3. Fault injection: halfway through the run the supervisor force-kills
+//     Worker 0, then detects the down_msg via the monitor callback, reschedules
+//     the lost batch to a surviving worker, and respawns a replacement.
+//  4. Final π estimate converges to the correct answer despite the disruption.
+//
+// Architecture:
+//   main → Supervisor : start_atom
+//   Supervisor spawns N WorkerActors, monitors each with callback.
+//   Supervisor → Worker[k] : (seed, num_samples)   [request().then()]
+//   Worker[k] → GPU via run_async_notify() on its private stream.
+//   GPU → Worker[k] : gpu_done_atom  (from cuLaunchHostFunc on CUDA thread)
+//   Worker[k] → Supervisor : int M   (number of in-circle hits)
+//   [halfway] Supervisor kills Worker[0] → monitor callback fires immediately
+//   Supervisor reschedules in-flight batch, respawns Worker[0], continues.
+//
+// Note on single-GPU machines: all workers share one physical device but each
+// gets a distinct CUDA stream (keyed by actor_id inside get_stream_for_actor).
+// Stream-level concurrency is real; for multi-GPU, pin worker N to device N
+// by using the find_device(N) overload of create_program_from_cubin.
+
+#include <caf/all.hpp>
+#include <caf/actor_cast.hpp>
+#include <caf/cuda/all.hpp>
+#include "../common/kernel_paths.hpp"
+
+#include <deque>
+#include <map>
+#include <vector>
+#include <iostream>
+#include <cmath>
+#include <chrono>
+
+using namespace caf;
+using namespace caf::cuda;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Atoms
+// ─────────────────────────────────────────────────────────────────────────────
+CAF_BEGIN_TYPE_ID_BLOCK(monte_carlo_app, caf::id_block::cuda::end)
+    CAF_ADD_ATOM(monte_carlo_app, start_atom)
+    CAF_ADD_ATOM(monte_carlo_app, kill_atom)
+    CAF_ADD_ATOM(monte_carlo_app, done_atom)
+CAF_END_TYPE_ID_BLOCK(monte_carlo_app)
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Work batch description
+// ─────────────────────────────────────────────────────────────────────────────
+struct WorkBatch {
+    int batch_id;
+    int seed;         // deterministic: batch_id * 1000003
+    int num_samples;  // constant per run
+};
+
+// ─────────────────────────────────────────────────────────────────────────────
+// command_runner type alias for monteCarloKernel
+//   arg 0: in<int>  seed
+//   arg 1: in<int>  num_samples
+//   arg 2: out<int> hit_count
+// ─────────────────────────────────────────────────────────────────────────────
+// Use in_out<int> (not out<int>) for the hit counter so the framework
+// copies value 0 to device memory before each kernel launch.  out<int> uses
+// cuMemAlloc without zero-init, which would corrupt the atomicAdd result.
+// Use in_out<int> for the hit counter so 0 is uploaded to device before
+// each launch — out<int> (scratch_argument) uses bare cuMemAlloc with no
+// zero-init, which would corrupt the atomicAdd result.
+using mc_runner = command_runner<in<int>, in<int>, in_out<int>>;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// WorkerActor
+//
+// Owns one mc_runner (and therefore one private CUDA stream).
+// Handles a single kernel request at a time:
+//   receive (seed, num_samples) → launch GPU → reply int M to sender
+// ─────────────────────────────────────────────────────────────────────────────
+class WorkerActor {
+    event_based_actor* self_;
+    int worker_index_;  // for logging
+
+    mc_runner runner_;
+
+    // Keep all mem_ptrs alive until the stream is idle (gpu_done_atom).
+    // Using a tuple matching the runner return: <mem_ptr<int>, mem_ptr<int>, mem_ptr<int>>
+    std::tuple<mem_ptr<int>, mem_ptr<int>, mem_ptr<int>> pending_refs_;
+
+    // The out<int> mem_ptr (alias into pending_refs_ for convenience).
+    mem_ptr<int> result_ptr_ = nullptr;
+
+    // Response promise to deliver the hit count back to the supervisor.
+    typed_response_promise<int> pending_rp_;
+
+public:
+    WorkerActor(event_based_actor* self, int index)
+        : self_(self), worker_index_(index) {}
+
+    behavior make_behavior() {
+        return {
+            // ── Kernel launch request from the supervisor ─────────────────
+            [this](int seed, int num_samples) -> result<int> {
+                pending_rp_ = self_->make_response_promise<int>();
+
+                auto& mgr = self_->system().cuda_manager();
+                // All workers use device 0 on a single-GPU machine.
+                // On multi-GPU hardware, replace 0 with worker_index_.
+                auto program = mgr.create_program_from_cubin(
+                    actor_tests::paths::monte_carlo_cubin, "monteCarloKernel");
+
+                nd_range dims(/*grid*/64, 1, 1, /*block*/256, 1, 1);
+
+                auto arg_seed    = create_in_arg(seed);
+                auto arg_samples = create_in_arg(num_samples);
+                // Zero-init the hit counter (in_out so value 0 is copied to
+                // device before the kernel runs; out<int> is NOT zero-inited).
+                const std::vector<int> zero_buf{0};
+                auto arg_out = create_in_out_arg(zero_buf);
+
+                // run_async_notify: kernel is launched non-blocking; when the
+                // CUDA stream goes idle the runtime calls gpu_done_atom on this
+                // actor from its internal thread.
+                pending_refs_ = runner_.run_async_notify(
+                    program, dims,
+                    actor_cast<actor>(self_),
+                    arg_seed, arg_samples, arg_out);
+
+                // The in_out<int> result is the third (last) element.
+                result_ptr_ = std::get<2>(pending_refs_);
+
+                self_->println("[Worker {:2d}] Launched GPU kernel: "
+                               "seed={}, samples={}", worker_index_, seed, num_samples);
+                return pending_rp_;
+            },
+
+            // ── GPU stream idle callback ───────────────────────────────────
+            [this](gpu_done_atom) {
+                std::vector<int> host = result_ptr_->copy_to_host();
+                int M = host.empty() ? 0 : host[0];
+
+                self_->println("[Worker {:2d}] gpu_done_atom: M={}", worker_index_, M);
+
+                // Safe to release device memory now (stream confirmed idle).
+                result_ptr_  = nullptr;
+                pending_refs_ = {};
+
+                pending_rp_.deliver(M);
+            }
+        };
+    }
+};
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Supervisor
+//
+// Manages the work queue, dispatches batches to workers, handles fault
+// injection at mid-run, recovers lost batches, and accumulates the final
+// π estimate.
+// ─────────────────────────────────────────────────────────────────────────────
+class Supervisor {
+    event_based_actor* self_;
+
+    // Configuration
+    int num_workers_;
+    int total_batches_;
+    int samples_per_batch_;
+
+    // Worker handles (index-stable; replaced on respawn)
+    std::vector<actor> workers_;
+
+    // Pending work
+    std::deque<WorkBatch> work_queue_;
+
+    // In-flight tracking: actor address → batch currently running in that worker
+    std::map<actor_addr, WorkBatch> in_flight_;
+
+    // Accumulation
+    long long total_hits_     = 0;
+    long long total_expected_ = 0; // total samples we PLAN to complete
+    int       batches_done_   = 0;
+
+    // Fault control
+    bool fault_injected_ = false;
+
+    // The scoped_actor that main() uses to wait for us.
+    actor parent_;
+
+public:
+    Supervisor(event_based_actor* self,
+               int num_workers, int total_batches, int samples_per_batch,
+               actor parent)
+        : self_(self)
+        , num_workers_(num_workers)
+        , total_batches_(total_batches)
+        , samples_per_batch_(samples_per_batch)
+        , parent_(std::move(parent))
+    {}
+
+    behavior make_behavior() {
+        return {
+            // ── Boot ──────────────────────────────────────────────────────
+            [this](start_atom) {
+                self_->println("[Supervisor] Starting. Workers={}, Batches={}, "
+                               "Samples/batch={}", num_workers_, total_batches_, samples_per_batch_);
+
+                // Pre-load work queue.
+                for (int b = 0; b < total_batches_; ++b) {
+                    work_queue_.push_back({b, b * 1000003, samples_per_batch_});
+                }
+                total_expected_ = (long long)total_batches_ * samples_per_batch_;
+
+                // Spawn workers and immediately dispatch the first batch each.
+                workers_.resize(num_workers_);
+                for (int k = 0; k < num_workers_; ++k) {
+                    workers_[k] = spawn_worker(k);
+                    if (!work_queue_.empty())
+                        dispatch_to(k);
+                }
+            },
+
+            // ── Externally-triggered fault injection (e.g., from main) ───
+            // Also fired internally when half the batches complete.
+            [this](kill_atom) {
+                if (!fault_injected_ && !workers_.empty()) {
+                    fault_injected_ = true;
+                    self_->println("[Supervisor] *** Injecting fault: killing Worker 0 ***");
+                    anon_send_exit(workers_[0], exit_reason::kill);
+                }
+            }
+        };
+    }
+
+private:
+    // ── Helpers ──────────────────────────────────────────────────────────────
+
+    actor spawn_worker(int idx) {
+        actor w = self_->spawn(actor_from_state<WorkerActor>, idx);
+
+        // Monitor with callback — non-deprecated, preferred API.
+        self_->monitor(w, [this, idx](const error& reason) {
+            self_->println("[Supervisor] Worker {} died: {}", idx, to_string(reason));
+
+            actor_addr dead_addr = workers_[idx].address();
+
+            auto it = in_flight_.find(dead_addr);
+            if (it != in_flight_.end()) {
+                WorkBatch lost = it->second;
+                self_->println("[Supervisor] Re-queuing lost batch {} (seed={})",
+                               lost.batch_id, lost.seed);
+                work_queue_.push_front(lost);  // high-priority: run next
+                in_flight_.erase(it);
+            }
+
+            // Respawn and immediately give it work if available.
+            workers_[idx] = spawn_worker(idx);
+            self_->println("[Supervisor] Worker {} respawned.", idx);
+            if (!work_queue_.empty())
+                dispatch_to(idx);
+        });
+
+        return w;
+    }
+
+    void dispatch_to(int idx) {
+        WorkBatch batch = work_queue_.front();
+        work_queue_.pop_front();
+
+        self_->println("[Supervisor] Dispatch batch {} → Worker {} (seed={})",
+                       batch.batch_id, idx, batch.seed);
+
+        // Snapshot the address NOW so the in-flight record and the then()
+        // cleanup both refer to the same worker instance, even if workers_[idx]
+        // is replaced by the time the response arrives.
+        actor_addr src = workers_[idx].address();
+        in_flight_.emplace(src, batch);
+
+        self_->mail(batch.seed, batch.num_samples)
+            .request(workers_[idx], infinite)
+            .then(
+                [this, idx, src](int M) {
+                    total_hits_ += M;
+                    batches_done_++;
+                    in_flight_.erase(src);
+
+                    double pi_est = 4.0 * total_hits_
+                                    / ((double)batches_done_ * samples_per_batch_);
+                    self_->println("[Supervisor] Batch complete ({}/{}). π ≈ {:.6f}",
+                                   batches_done_, total_batches_, pi_est);
+
+                    // Inject fault exactly at the halfway point.
+                    if (batches_done_ == total_batches_ / 2) {
+                        self_->mail(kill_atom_v).send(self_);
+                    }
+
+                    if (batches_done_ == total_batches_) {
+                        finish();
+                        return;
+                    }
+                    if (!work_queue_.empty())
+                        dispatch_to(idx);
+                },
+                [this, idx](const error& err) {
+                    // Worker died while we were waiting for its result.
+                    // The monitor callback already rescheduled the in-flight batch.
+                    self_->println("[Supervisor] Request to Worker {} failed: {} "
+                                   "(batch rescheduled via monitor callback)", idx, to_string(err));
+                });
+    }
+
+    void finish() {
+        long long total_samples_completed =
+            (long long)batches_done_ * samples_per_batch_;
+        double pi_final = 4.0 * total_hits_ / (double)total_samples_completed;
+        double error_pct = std::abs(pi_final - M_PI) / M_PI * 100.0;
+
+        self_->println("\n=== RESULT ===");
+        self_->println("Batches completed : {}", batches_done_);
+        self_->println("Total samples     : {}", total_samples_completed);
+        self_->println("Total hits        : {}", total_hits_);
+        self_->println("π estimate        : {:.8f}", pi_final);
+        self_->println("Error vs π        : {:.4f}%\n", error_pct);
+
+        // Shut down all workers.
+        for (auto& w : workers_)
+            anon_send_exit(w, exit_reason::user_shutdown);
+
+        // Signal completion to main.
+        self_->mail(done_atom_v).send(parent_);
+        self_->quit();
+    }
+};
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Config & entry point
+// ─────────────────────────────────────────────────────────────────────────────
+class config : public actor_system_config {
+public:
+    config() {
+        // Deliberately use more than 1 thread so concurrent workers are real.
+        // Uncomment to pin to 1 thread and confirm everything still works:
+        // set("caf.scheduler.max-threads", 1u);
+    }
+};
+
+void caf_main(actor_system& sys, const config&) {
+    constexpr int NUM_WORKERS      = 2;
+    constexpr int TOTAL_BATCHES    = 20;
+    constexpr int SAMPLES_PER_BATCH = 10'000'000;
+
+    sys.println("=== Monte Carlo π Estimation (Fault-Tolerant) ===");
+    sys.println("Workers: {}   Batches: {}   Samples/batch: {}",
+                NUM_WORKERS, TOTAL_BATCHES, SAMPLES_PER_BATCH);
+    sys.println("Total samples planned: {}\n",
+                (long long)TOTAL_BATCHES * SAMPLES_PER_BATCH);
+
+    scoped_actor self{sys};
+
+    actor supervisor = self->spawn(
+        actor_from_state<Supervisor>,
+        NUM_WORKERS, TOTAL_BATCHES, SAMPLES_PER_BATCH,
+        actor_cast<actor>(self));
+
+    self->mail(start_atom_v).send(supervisor);
+
+    // Block until the supervisor signals done (or terminates cleanly).
+    self->receive([](done_atom) {});
+}
+
+CAF_MAIN(caf::cuda::manager, id_block::monte_carlo_app)
diff --git a/libcaf_cuda/sc26/Fault-Tolerance/monte_carlo.cu b/libcaf_cuda/sc26/Fault-Tolerance/monte_carlo.cu
new file mode 100644
index 0000000000..f77f71b647
--- /dev/null
+++ b/libcaf_cuda/sc26/Fault-Tolerance/monte_carlo.cu
@@ -0,0 +1,25 @@
+#include <cuda.h>
+#include <curand_kernel.h>
+
+extern "C" __global__
+void monteCarloKernel(int seed, int total_samples, int* out_count) {
+    int tid = blockIdx.x * blockDim.x + threadIdx.x;
+    int n_threads = gridDim.x * blockDim.x;
+
+    curandState_t rng;
+    curand_init(seed, tid, 0, &rng);
+
+    long long local_count = 0;
+    for (int i = tid; i < total_samples; i += n_threads) {
+        float x = curand_uniform(&rng);
+        float y = curand_uniform(&rng);
+        if (x * x + y * y <= 1.0f)
+            local_count++;
+    }
+
+    for (int offset = 16; offset > 0; offset >>= 1)
+        local_count += __shfl_down_sync(0xffffffff, local_count, offset);
+
+    if ((threadIdx.x & 31) == 0)
+        atomicAdd(out_count, static_cast<int>(local_count));
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/Makefile b/libcaf_cuda/sc26/Runtime-Overhead/Makefile
new file mode 100644
index 0000000000..9ce456d713
--- /dev/null
+++ b/libcaf_cuda/sc26/Runtime-Overhead/Makefile
@@ -0,0 +1,46 @@
+NVCC ?= nvcc
+CXX ?= g++
+CXXFLAGS = -std=c++20 -include cstddef -include cerrno -include climits
+
+CUDA_INCLUDE_DIR ?=
+CUDA_LIB_DIR ?= 
+
+CAF_INCLUDE_DIR ?=
+CAF_LIB_DIR ?=
+
+LIBS = -L$(CAF_LIB_DIR) \
+ 			-lcaf_core \
+			-lcaf_io \
+			-lcaf_net \
+			-lcaf_cuda \
+			-L$(CUDA_LIB_DIR) \
+			-lcuda
+
+INCLUDES = -I$(CAF_INCLUDE_DIR) \
+		   -I$(CUDA_INCLUDE_DIR)
+
+LDFLAGS=-Wl,-rpath,$(CAF_LIB_DIR) \
+		 		-Wl,-rpath,$(CUDA_LIB_DIR)
+NVCC_ARCH ?= sm_$(shell nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.')
+
+all: mmul_kernel cuda_native command_runner actor_facade
+
+mmul_kernel: mmul.cu
+	$(NVCC) -g -arch=$(NVCC_ARCH) --cubin mmul.cu -o mmul.cubin
+
+cuda_native: cuda_native.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o cuda_native cuda_native.cpp $(LIBS) $(LDFLAGS)
+
+actors_facade: actor_facade.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o actors_facade actor_facade.cpp $(LIBS) $(LDFLAGS)
+
+command_runner: command_runner.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o command_runner command_runner.cpp $(LIBS) $(LDFLAGS)
+
+clean:
+	rm -f mmul.cubin
+	rm -f cuda_native
+	rm -f main_actor_facade
+	rm -f command_runner
+
+.PHONY: all clean
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/Makefile b/libcaf_cuda/sc26/Sequence-Independent-Tasks/Makefile
new file mode 100644
index 0000000000..7bf02dff95
--- /dev/null
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/Makefile
@@ -0,0 +1,44 @@
+NVCC ?= nvcc
+CXX ?= g++
+CXXFLAGS = -std=c++20 -include cstddef -include cerrno -include climits
+
+CUDA_INCLUDE_DIR ?=
+CUDA_LIB_DIR ?= 
+
+CAF_INCLUDE_DIR ?=
+CAF_LIB_DIR ?=
+
+LIBS = -L$(CAF_LIB_DIR) \
+ 			-lcaf_core \
+			-lcaf_io \
+			-lcaf_net \
+			-lcaf_cuda \
+			-L$(CUDA_LIB_DIR) \
+			-lcuda
+
+INCLUDES = -I$(CAF_INCLUDE_DIR) \
+		  	 	 -I$(CUDA_INCLUDE_DIR)
+
+LDFLAGS=-Wl,-rpath,$(CAF_LIB_DIR) \
+		 		-Wl,-rpath,$(CUDA_LIB_DIR)
+NVCC_ARCH ?= sm_$(shell nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.')
+
+all: mmul_kernel base_cuda_program actor_facade actor_command_runner
+
+mmul_kernel: mmul.cu
+	$(NVCC) -g -arch=$(NVCC_ARCH) --cubin mmul.cu -o mmul.cubin
+
+base_cuda_program: main_cuda_native.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o main_cuda_native main_cuda_native.cpp $(LIBS) $(LDFLAGS)
+
+actor_facade: main_actor_facade.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o main_actor_facade main_actor_facade.cpp $(LIBS) $(LDFLAGS)
+
+actor_command_runner: main_command_runner.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o main_command_runner main_command_runner.cpp $(LIBS) $(LDFLAGS)
+
+clean:
+	rm -f mmul.cubin
+	rm -f main_cuda_native
+	rm -f main_actor_facade
+	rm -f main_command_runner
\ No newline at end of file

From 607cc2f4399f03e3f0eccfa4ee35698e356d1226 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 6 Apr 2026 10:50:18 -0600
Subject: [PATCH 0562/1000] Moved test from scheduler benchmark over to this
 directory, still have not cut out the scheduler stuff yet.

---
 .../Batched-Matrix-Multiply/CMakeLists.txt    |  44 +
 .../compile_kernels.sh                        |  13 +
 .../Batched-Matrix-Multiply/main.test.cpp     | 847 ++++++++++++++++++
 .../sc26/Batched-Matrix-Multiply/mmul.cu      |  16 +
 4 files changed, 920 insertions(+)
 create mode 100644 libcaf_cuda/sc26/Batched-Matrix-Multiply/CMakeLists.txt
 create mode 100755 libcaf_cuda/sc26/Batched-Matrix-Multiply/compile_kernels.sh
 create mode 100644 libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp
 create mode 100644 libcaf_cuda/sc26/Batched-Matrix-Multiply/mmul.cu

diff --git a/libcaf_cuda/sc26/Batched-Matrix-Multiply/CMakeLists.txt b/libcaf_cuda/sc26/Batched-Matrix-Multiply/CMakeLists.txt
new file mode 100644
index 0000000000..12886f5340
--- /dev/null
+++ b/libcaf_cuda/sc26/Batched-Matrix-Multiply/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/sc26/Batched-Matrix-Multiply/compile_kernels.sh b/libcaf_cuda/sc26/Batched-Matrix-Multiply/compile_kernels.sh
new file mode 100755
index 0000000000..f32480e5cb
--- /dev/null
+++ b/libcaf_cuda/sc26/Batched-Matrix-Multiply/compile_kernels.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp b/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp
new file mode 100644
index 0000000000..663434ab63
--- /dev/null
+++ b/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp
@@ -0,0 +1,847 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/component-actors/all-component-actors.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+#include <random>
+#include <unordered_set>
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+using command =
+  caf::cuda::command_runner<>;
+
+command mmul_command;
+caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int >> mmul;
+using async_command = caf::cuda::mmul_async_command<int>;
+async_command async_mmul;
+
+
+
+
+
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
+};
+
+
+
+MatrixPool create_matrix_pool_random(
+    int num_sizes,
+    int min_N,
+    int max_N,
+    unsigned int seed
+) {
+    MatrixPool pool;
+
+    std::mt19937 rng(seed);
+    std::uniform_int_distribution<int> dist(min_N, max_N);
+
+    std::unordered_set<int> used;
+
+    while (used.size() < static_cast<size_t>(num_sizes)) {
+        int N = dist(rng);
+        if (used.insert(N).second) {
+            pool.A[N] = std::vector<int>(N * N, 1);
+            pool.B[N] = std::vector<int>(N * N, 1);
+        }
+    }
+
+    return pool;
+}
+
+
+
+
+
+
+struct mmul_state {
+
+	caf::cuda::program_ptr mmul_kernel;
+
+};
+
+
+
+caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
+		caf::actor exit_actor,
+		caf::cuda::program_ptr program,
+		caf::cuda::nd_range dims,
+		int stream,
+		int N,
+		const in<int> matrixA,
+		const in<int> matrixB
+		) {
+
+
+	int device = stream % caf::cuda::manager::get().get_num_devices();
+	self->mail(N).send(self);
+
+	return {
+
+		[=](int N) {
+
+			auto total_start = std::chrono::steady_clock::now();
+
+
+//			std::cout << "device=" << device <<  "\n";
+//			std::cout << "N=" << N <<  "\n";
+			// ---------------- H2D ----------------
+			auto h2d_start = std::chrono::steady_clock::now();
+
+			auto arg1 = mmul_command.transfer_memory(device, stream, std::move(matrixA));
+			auto arg2 = mmul_command.transfer_memory(device, stream, std::move(matrixB));
+
+
+			// ---------------- Kernel ----------------
+			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
+			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
+
+			auto h2d_end = std::chrono::steady_clock::now();
+			auto kernel_start = std::chrono::steady_clock::now();
+
+			auto result = async_mmul.run_async(
+					program, dims, stream, 0, device,
+					arg1, arg2, arg3, arg4);
+
+			//std::get<2>(result)->synchronize();
+
+			auto kernel_end = std::chrono::steady_clock::now();
+
+			// ---------------- D2H ----------------
+			auto d2h_start = std::chrono::steady_clock::now();
+
+			std::get<2>(result)->copy_to_host();
+
+			auto d2h_end = std::chrono::steady_clock::now();
+
+			auto total_end = std::chrono::steady_clock::now();
+
+			/*
+			// ---------------- COMPUTE ----------------
+			auto h2d = std::chrono::duration<double>(h2d_end - h2d_start).count();
+			auto kernel = std::chrono::duration<double>(kernel_end - kernel_start).count();
+			auto d2h = std::chrono::duration<double>(d2h_end - d2h_start).count();
+			auto total = std::chrono::duration<double>(total_end - total_start).count();
+
+			// ---------------- PRINT ----------------
+			
+			std::cout << "\n[NO SCHEDULER] N=" << N << "\n";
+			std::cout << "H2D:    " << h2d * 1000 << " ms\n";
+			std::cout << "Kernel: " << kernel * 1000 << " ms\n";
+			std::cout << "D2H:    " << d2h * 1000 << " ms\n";
+			std::cout << "TOTAL:  " << total * 1000 << " ms\n";
+			std::cout << "SUM:    " << (h2d + kernel + d2h) * 1000 << " ms\n";
+
+			*/
+			self->mail(1).send(exit_actor);
+			self->quit();
+		}
+	};
+}
+
+
+
+struct mmul_actor_with_scheduler_state {
+  static inline const char* name = "my_actor";
+};
+
+
+// Stateful actor behavior
+caf::behavior mmul_actor_fun_scheduler(
+    caf::stateful_actor<mmul_actor_with_scheduler_state>* self,
+    caf::actor exit_actor,
+    int N,
+    caf::cuda::program_ptr program,
+    caf::cuda::nd_range dims,
+    const in<int> matrixA,
+    const in<int> matrixB)
+{
+
+
+
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+        caf::actor scheduler = mgr.get_scheduler_actor();
+
+        //send a launch token
+        caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
+                        program,
+                        dims,
+                        0,
+                        "hello",
+                        self
+                        );
+        mgr.send_scheduler_actor_message(launch_token);
+
+	return {
+
+		// 1. Handle response token
+		[=](caf::cuda::response_token_ptr res_token) {
+			//        std::cout << "Got response\n";
+
+			if (res_token->getType() == LAUNCH_RESPONSE) {
+				self->mail(res_token, N).send(self);
+
+			} else {
+				//          std::cout << "Got a memory response token\n";
+			}
+		},
+
+			// 2. Handle memory buffers -> GPU
+			[=](const caf::cuda::response_token_ptr& res_token, int N) {
+
+				auto total_start = std::chrono::steady_clock::now();
+
+				// ---------------- H2D ----------------
+				auto h2d_start = std::chrono::steady_clock::now();
+
+				auto arg1 = mmul.transfer_memory(res_token -> getDeviceNumber(),res_token -> getStreamId(), std::move(matrixA));
+				auto arg2 = mmul.transfer_memory(res_token -> getDeviceNumber(), res_token -> getStreamId(), std::move(matrixB));
+				//auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
+				//auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
+
+				std::cout << "res_token did = " << res_token -> getDeviceNumber() << "\n";
+				std::cout << "N = " << N << "\n";
+				 out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
+	 			 in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
+
+				auto h2d_end = std::chrono::steady_clock::now();
+
+				// ---------------- Kernel ----------------
+				auto kernel_start = std::chrono::steady_clock::now();
+
+
+
+				auto tempC = async_mmul.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
+				auto bufferC = std::get<2>(tempC);
+
+				//bufferC->synchronize();
+
+				auto kernel_end = std::chrono::steady_clock::now();
+
+				// ---------------- D2H ----------------
+				auto d2h_start = std::chrono::steady_clock::now();
+
+				bufferC->copy_to_host();
+
+				auto d2h_end = std::chrono::steady_clock::now();
+
+				res_token->release();
+
+				auto total_end = std::chrono::steady_clock::now();
+
+				// ---------------- COMPUTE ----------------
+				/*
+				auto h2d = std::chrono::duration<double>(h2d_end - h2d_start).count();
+				auto kernel = std::chrono::duration<double>(kernel_end - kernel_start).count();
+				auto d2h = std::chrono::duration<double>(d2h_end - d2h_start).count();
+				auto total = std::chrono::duration<double>(total_end - total_start).count();
+
+				std::cout << "H2D:    " << h2d * 1000 << " ms\n";
+				std::cout << "Kernel: " << kernel * 1000 << " ms\n";
+				std::cout << "D2H:    " << d2h * 1000 << " ms\n";
+				std::cout << "TOTAL:  " << total * 1000 << " ms\n";
+				std::cout << "SUM:    " << (h2d + kernel + d2h) * 1000 << " ms\n\n";
+				*/
+
+				self->mail(1).send(exit_actor);
+				self->quit();
+			}
+
+
+       };
+
+}
+
+
+
+
+
+caf::behavior mmul_actor_fun_scheduler2(caf::stateful_actor<mmul_state>* self,
+		caf::actor exit_actor,
+		caf::actor scheduler_actor,
+		caf::cuda::program_ptr program,
+		caf::cuda::nd_range dims,
+		int stream,
+		int N,
+		const in<int> matrixA,
+		const in<int> matrixB
+		) {
+
+
+	int device = stream % caf::cuda::manager::get().get_num_devices();
+
+	//self->mail("subscribe",self).send(scheduler_actor);
+
+	self->mail(N).send(self);
+
+	return {
+
+		//message from the new scheduler actor 
+		[=](std::vector<int> costs) {
+			//do nothing this is an overhead test 
+		
+			//std::cout << "N=" << N <<  "\n";
+		
+		},
+		[=](int N) {
+
+			auto total_start = std::chrono::steady_clock::now();
+
+			//int device = rand() % caf::cuda::manager::get().get_num_devices();
+			//int stream = rand();
+
+			//std::cout << "device=" << device <<  "\n";
+			//std::cout << "N=" << N <<  "\n";
+
+
+
+			//declare the cost of doing work to the scheduler actor, for now we can impose
+			//a heuristic of just N, the size of the matrix
+			self->mail("add",device,N).send(scheduler_actor);
+
+
+			// ---------------- H2D ----------------
+			auto h2d_start = std::chrono::steady_clock::now();
+
+
+			auto arg1 = mmul_command.transfer_memory(device, stream, std::move(matrixA));
+			auto arg2 = mmul_command.transfer_memory(device, stream, std::move(matrixB));
+
+
+			// ---------------- Kernel ----------------
+			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
+			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
+
+			auto h2d_end = std::chrono::steady_clock::now();
+			auto kernel_start = std::chrono::steady_clock::now();
+
+			auto result = async_mmul.run_async(
+					program, dims, stream, 0, device,
+					arg1, arg2, arg3, arg4);
+
+			//std::get<2>(result)->synchronize();
+
+			auto kernel_end = std::chrono::steady_clock::now();
+
+			// ---------------- D2H ----------------
+			auto d2h_start = std::chrono::steady_clock::now();
+
+			std::get<2>(result)->copy_to_host();
+
+
+
+			//likewise tell the scheduler we are done doing work
+			self->mail("subtract",device,N).send(scheduler_actor);
+
+			auto d2h_end = std::chrono::steady_clock::now();
+
+			auto total_end = std::chrono::steady_clock::now();
+
+			/*
+			// ---------------- COMPUTE ----------------
+			auto h2d = std::chrono::duration<double>(h2d_end - h2d_start).count();
+			auto kernel = std::chrono::duration<double>(kernel_end - kernel_start).count();
+			auto d2h = std::chrono::duration<double>(d2h_end - d2h_start).count();
+			auto total = std::chrono::duration<double>(total_end - total_start).count();
+
+			// ---------------- PRINT ----------------
+			
+			std::cout << "\n[NO SCHEDULER] N=" << N << "\n";
+			std::cout << "H2D:    " << h2d * 1000 << " ms\n";
+			std::cout << "Kernel: " << kernel * 1000 << " ms\n";
+			std::cout << "D2H:    " << d2h * 1000 << " ms\n";
+			std::cout << "TOTAL:  " << total * 1000 << " ms\n";
+			std::cout << "SUM:    " << (h2d + kernel + d2h) * 1000 << " ms\n";
+
+			*/
+			self->mail(1).send(exit_actor);
+			self->mail("unsubscribe",self).send(scheduler_actor);
+			self->quit();
+		}
+	};
+}
+
+
+
+
+
+
+struct scheduler_actor_state {
+
+	std::vector<caf::actor> subscribers;
+	int num_devices;
+	std::vector<int> costs;
+};
+
+
+
+caf::behavior scheduler_actor_fun(caf::stateful_actor<scheduler_actor_state>* self) {
+
+	self->state().num_devices = caf::cuda::manager::get().get_num_devices();
+	self->state().costs.resize(self->state().num_devices);
+
+	int time = 50;
+
+	self->mail("publish").urgent().delay(std::chrono::milliseconds(time)).send(self);
+
+	return {
+
+		[=](std::string command,int device, int cost) {
+
+			if (command == "add") {
+
+				self->state().costs[device] +=cost;
+
+			}
+			else if (command == "subtract") {
+
+				int value = std::min(self->state().costs[device] - cost,0);
+				self->state().costs[device] = value;	
+			}
+
+
+		},
+			[=](std::string command, caf::actor actor) {
+
+				auto& subs = self->state().subscribers;
+
+				if (command == "subscribe") {
+					//std::cout << "Thank you for subscribing\n";
+					// avoid duplicates
+					if (std::find(subs.begin(), subs.end(), actor) == subs.end()) {
+						subs.push_back(actor);
+						//self->monitor(actor); // track lifecycle
+					}
+				}
+
+				else if (command == "unsubscribe") {
+					subs.erase(
+							std::remove(subs.begin(), subs.end(), actor),
+							subs.end()
+						  );
+					//self->demonitor(actor);
+				}
+			},
+
+			[=](std::string command) {
+				if (command == "publish") {
+					//std::cout << "size = " << self->state().subscribers.size() << "\n";
+					for (caf::actor a : self->state().subscribers) {
+
+						self -> mail(self->state().costs).urgent().send(a);
+
+					}
+					self->mail("publish").urgent().delay(std::chrono::milliseconds(time)).send(self);
+				}
+
+			}	
+
+	};
+}
+
+
+
+// ---------------------------- SUPERVISOR ACTOR ----------------------------
+struct supervisor_actor_state {
+    int num_actors;
+    int num_waves;
+    int completed;
+    int max_waves;
+
+    MatrixPool pool;
+
+    // Precomputed sequence of N values
+    std::vector<int> Ns;
+    int next_task;
+
+    // Timing
+    std::chrono::steady_clock::time_point start_time;
+    std::chrono::steady_clock::time_point wave_start_time;
+};
+
+caf::behavior supervisor_actor_fun(
+    caf::stateful_actor<supervisor_actor_state>* self,
+    int num_actors,
+    int max_waves,
+    MatrixPool pool,
+    const std::vector<int>& Ns,      // deterministic task sizes 
+    bool use_scheduler
+    ) {
+    // Initialize state
+    self->state().num_actors = num_actors;
+    self->state().completed = 0;
+    self->state().max_waves = max_waves;
+    self->state().num_waves = 0;
+    self->state().pool = std::move(pool);
+
+    self->state().Ns = Ns;
+    self->state().next_task = 0;
+
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    // Kick off first wave
+    self->mail("spawn").send(self);
+
+    caf::actor scheduler_actor = self->spawn(scheduler_actor_fun);
+
+    // Start timing
+    self->state().start_time = std::chrono::steady_clock::now();
+    return {
+        // -------------------- SPAWN WAVE --------------------
+        [=](std::string cmd) {
+            if (cmd != "spawn") return;
+
+	     std::chrono::steady_clock::time_point cmd_start_time = std::chrono::steady_clock::now();
+	    
+
+            self->state().completed = 0;
+            self->state().wave_start_time = std::chrono::steady_clock::now();
+
+            for (int i = 0; i < self->state().num_actors; ++i) {
+                if (self->state().next_task >= self->state().Ns.size())
+                    break;
+
+                int N = self->state().Ns[self->state().next_task++];
+
+
+                const auto& A = self->state().pool.A[N];
+                const auto& B = self->state().pool.B[N];
+
+                const int THREADS = 32;
+                const int BLOCKS = (N + THREADS - 1) / THREADS;
+
+                caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+		if (use_scheduler) {
+			caf::actor a = 	self->spawn(mmul_actor_fun_scheduler2, 
+					self,
+					scheduler_actor,
+				       	program, 
+					dims,
+					i,
+					N,
+                            caf::cuda::create_in_arg(A),
+                            caf::cuda::create_in_arg(B));
+
+
+		 self->mail("subscribe",a).send(scheduler_actor);
+		}
+		else {
+			self->spawn(mmul_actor_fun, self, program, dims,i,N,
+                            caf::cuda::create_in_arg(A),
+                            caf::cuda::create_in_arg(B));
+		}
+            }
+        
+	
+
+    std::chrono::steady_clock::time_point cmd_end_time = std::chrono::steady_clock::now();
+
+      std::chrono::duration<double> total_time =
+                        cmd_end_time - cmd_start_time;
+
+                    std::cout << "\n===== SUPERVISOR TOTAL TIME spawn =====\n";
+                    std::cout << "Total runtime: "
+                              << total_time.count() << " s\n";
+
+
+	
+	},
+
+        // -------------------- COMPLETION TRACKING --------------------
+        [=](int done) {
+            self->state().completed += done;
+
+            if (self->state().completed >= self->state().num_actors) {
+                auto wave_end = std::chrono::steady_clock::now();
+                std::chrono::duration<double> wave_time =
+                    wave_end - self->state().wave_start_time;
+
+                self->state().num_waves++;
+                //std::cout << "Wave "
+                  //        << self->state().num_waves
+                    //      << " completed in "
+                      //    << wave_time.count() << " s\n";
+
+                if (self->state().num_waves >= self->state().max_waves) {
+                    auto end_time = std::chrono::steady_clock::now();
+                    std::chrono::duration<double> total_time =
+                        end_time - self->state().start_time;
+
+                    std::cout << "\n===== SUPERVISOR TOTAL TIME =====\n";
+                    std::cout << "Total runtime: "
+                              << total_time.count() << " s\n";
+
+
+		    anon_send_exit(
+				    scheduler_actor,
+				    caf::exit_reason::user_shutdown
+				  );
+		    caf::cuda::manager::shutdown();
+                    self->quit();
+                } else {
+                    self->mail("spawn").send(self);
+                }
+            }
+        }
+    };
+}
+
+
+
+
+
+
+
+
+template <class Fn>
+double time_run(Fn&& fn) {
+  auto start = std::chrono::steady_clock::now();
+  fn();
+  auto end = std::chrono::steady_clock::now();
+  std::chrono::duration<double> elapsed = end - start;
+  return elapsed.count();
+}
+
+
+
+void run_mmul_random_scaling_tests(caf::actor_system& sys,
+                                  caf::cuda::manager_config man_config) {
+
+    const int min_N = 32;
+    const int max_N = 2048;
+    const int num_sizes = 10;
+
+    const int max_waves = 1;
+
+    const std::vector<int> actor_counts = {
+	  30000,40000,50000
+    };
+
+
+    int num_actors = actor_counts[actor_counts.size()-1];
+
+        // Generate deterministic random pool
+        MatrixPool pool = create_matrix_pool_random(
+            num_sizes,
+            min_N,
+            max_N,
+            42  // fixed seed
+        );
+
+    // Precompute all task Ns (total_tasks = num_actors * max_waves)
+    std::vector<int> sizes;
+    for (const auto& [N, _] : pool.A) sizes.push_back(N);
+
+    int total_tasks = num_actors * max_waves;
+    std::vector<int> Ns;
+    Ns.reserve(total_tasks);
+
+    std::mt19937 rng(42);
+    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+    for (int i = 0; i < total_tasks; ++i)
+	    Ns.push_back(sizes[dist(rng)]);
+
+
+   
+    //scheduler
+    for (int num_actors : actor_counts) {
+
+    
+	    // Initialize CUDA manager
+	    caf::cuda::manager::init(sys);
+  std::cout << "=====================================\n";
+        std::cout << "Random Scaling WITH scheduler | actors=" << num_actors << "\n";
+
+
+        double elapsed = time_run([&]() {
+
+            auto sup = sys.spawn(
+                supervisor_actor_fun,
+                num_actors,
+                max_waves,
+                pool,
+		Ns,
+		true
+            );
+
+        
+	      sys.await_all_actors_done();
+	    
+	    });
+
+    
+	caf::cuda::manager::shutdown();
+    }
+    
+    //no scheduler
+    for (int num_actors : actor_counts) {
+
+    
+	    // Initialize CUDA manager
+	    caf::cuda::manager::init(sys);
+	    std::cout << "=====================================\n";
+	    std::cout << "Random Scaling NO scheduler | actors=" << num_actors << "\n";      
+	    double elapsed = time_run([&]() {
+
+            auto sup = sys.spawn(
+                supervisor_actor_fun,
+                num_actors,
+                max_waves,
+                pool,
+		Ns,
+		false
+            );
+
+	      sys.await_all_actors_done();
+	    
+	    });
+
+    
+	caf::cuda::manager::shutdown();
+    }
+
+
+
+
+    caf::cuda::manager::shutdown();
+}
+
+void run_mmul_uniform_scaling_tests(caf::actor_system& sys,
+                                   caf::cuda::manager_config man_config) {
+
+    const int max_waves = 1;
+
+    // Matrix sizes: 1,2,4,...,2048
+    std::vector<int> matrix_sizes;
+    for (int n = 1; n <= 2048; n *= 2)
+        matrix_sizes.push_back(n);
+
+    // Actor counts: 10 → 1000
+    std::vector<int> actor_counts;
+    for (int a = 10; a <= 1000; a += 10)
+        actor_counts.push_back(a);
+
+    for (int N : matrix_sizes) {
+        for (int num_actors : actor_counts) {
+
+            // Create uniform pool (single size)
+            MatrixPool pool = create_matrix_pool_random(
+                1,
+                N,
+                N,
+                42
+            );
+
+            int total_tasks = num_actors * max_waves;
+
+            std::vector<int> Ns(total_tasks, N);
+
+            // ========================
+            // WITH SCHEDULER
+            // ========================
+            caf::cuda::manager::init(sys);
+            std::cout << "=====================================\n";
+            std::cout << "Uniform WITH scheduler | N=" << N
+                      << " actors=" << num_actors << "\n";
+
+            time_run([&]() {
+                auto sup = sys.spawn(
+                    supervisor_actor_fun,
+                    num_actors,
+                    max_waves,
+                    pool,
+                    Ns,
+                    true
+                );
+                sys.await_all_actors_done();
+            });
+
+            caf::cuda::manager::shutdown();
+
+            // ========================
+            // WITHOUT SCHEDULER
+            // ========================
+            caf::cuda::manager::init(sys);
+            std::cout << "=====================================\n";
+            std::cout << "Uniform NO scheduler | N=" << N
+                      << " actors=" << num_actors << "\n";
+
+            time_run([&]() {
+                auto sup = sys.spawn(
+                    supervisor_actor_fun,
+                    num_actors,
+                    max_waves,
+                    pool,
+                    Ns,
+                    false
+                );
+                sys.await_all_actors_done();
+            });
+
+            caf::cuda::manager::shutdown();
+        }
+    }
+}
+
+
+
+
+
+
+
+
+
+
+void caf_main(caf::actor_system& sys) {
+  
+
+  caf::cuda::manager_config man_config(true);
+  //caf::cuda::manager::init(sys,man_config);
+
+
+  run_mmul_random_scaling_tests(sys,man_config);
+  //run_mmul_uniform_scaling_tests(sys,man_config);
+
+
+
+}
+
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/sc26/Batched-Matrix-Multiply/mmul.cu b/libcaf_cuda/sc26/Batched-Matrix-Multiply/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/sc26/Batched-Matrix-Multiply/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From aa08538b4cfde6eeed1bdfa675bead9b08892ff9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 6 Apr 2026 10:55:32 -0600
Subject: [PATCH 0563/1000] Saving scripts used for data generation.

---
 .../scripts/Batched-Matrix-Multiply/plot.py   | 194 ++++++++++++++++++
 .../analyze_mmul_benchmarks.py                |  92 +++++++++
 .../generate_graphs.py                        |  67 ++++++
 3 files changed, 353 insertions(+)
 create mode 100644 libcaf_cuda/sc26/scripts/Batched-Matrix-Multiply/plot.py
 create mode 100644 libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
 create mode 100644 libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py

diff --git a/libcaf_cuda/sc26/scripts/Batched-Matrix-Multiply/plot.py b/libcaf_cuda/sc26/scripts/Batched-Matrix-Multiply/plot.py
new file mode 100644
index 0000000000..05ba000e6f
--- /dev/null
+++ b/libcaf_cuda/sc26/scripts/Batched-Matrix-Multiply/plot.py
@@ -0,0 +1,194 @@
+import os
+import re
+import numpy as np
+import matplotlib.pyplot as plt
+from collections import defaultdict
+
+def parse_file(filename, random_data, uniform_data):
+    if not os.path.exists(filename):
+        print(f"⚠️  File not found: {filename}")
+        return
+    with open(filename, 'r', encoding='utf-8') as f:
+        lines = [line.strip() for line in f if line.strip()]
+    
+    i = 0
+    while i < len(lines) - 5:
+        line = lines[i]
+        if ("Random Scaling" in line or "Uniform" in line) and \
+           ("WITH scheduler" in line or "NO scheduler" in line) and \
+           "actors=" in line:
+            
+            is_random = "Random Scaling" in line
+            scheduler = "WITH scheduler" in line
+            
+            actors_m = re.search(r'actors=(\d+)', line)
+            if not actors_m:
+                i += 1
+                continue
+            actors = int(actors_m.group(1))
+            
+            N_val = None
+            if not is_random:
+                n_m = re.search(r'N=(\d+)', line)
+                if not n_m:
+                    i += 1
+                    continue
+                N_val = int(n_m.group(1))
+            
+            # === FIXED: take the SECOND block (real total runtime) ===
+            # i+1 : spawn header
+            # i+2 : spawn runtime
+            # i+3 : total header (no "spawn")
+            # i+4 : total runtime  ← this is what we want
+            if ("SUPERVISOR TOTAL TIME" in lines[i + 3] and
+                "spawn" not in lines[i + 3] and
+                "Total runtime:" in lines[i + 4]):
+                
+                runtime_m = re.search(r'Total runtime:\s*([\d.]+)\s*s', lines[i + 4])
+                if runtime_m:
+                    runtime = float(runtime_m.group(1))
+                    if is_random:
+                        random_data[(scheduler, actors)].append(runtime)
+                    else:
+                        uniform_data[(N_val, scheduler, actors)].append(runtime)
+            
+            i += 5   # skip entire config block
+            continue
+        i += 1
+
+
+# ====================== MAIN ======================
+random_data = defaultdict(list)
+uniform_data = defaultdict(list)
+
+print("📂 Parsing 10 output files...")
+for i in range(1, 11):
+    fname = f"output{i}.txt"
+    parse_file(fname, random_data, uniform_data)
+
+print(f"✅ Parsed {len(random_data)} Random Scaling configs and {len(uniform_data)} Uniform configs.")
+
+# --------------------- Random Scaling Graph (actors ≥ 30 000) ---------------------
+if random_data:
+    all_actors = sorted({act for (_, act) in random_data.keys() if act > 1})
+    
+    with_r, no_r = [], []
+    for act in all_actors:
+        key_w = (True, act)
+        key_n = (False, act)
+        avg_w = np.mean(random_data[key_w]) if key_w in random_data and random_data[key_w] else np.nan
+        avg_n = np.mean(random_data[key_n]) if key_n in random_data and random_data[key_n] else np.nan
+        with_r.append(avg_w)
+        no_r.append(avg_n)
+    
+    plt.figure(figsize=(11, 7))
+    plt.plot(all_actors, with_r, 'o-', linewidth=2.5, label='With Scheduler')
+    plt.plot(all_actors, no_r, 's-', linewidth=2.5, label='No Scheduler')
+    plt.xlabel('Number of Actors')
+    plt.ylabel('Average Runtime (seconds)')
+    plt.title('Heterogeneous Scaling On gpufarm5 — With Scheduler vs No Scheduler\n(actors ≥ 30 000, averaged over 10 runs)')
+    plt.legend(fontsize=12)
+    plt.grid(True, alpha=0.3)
+    plt.tight_layout()
+    plt.savefig('random_scaling_comparison.png', dpi=300)
+    plt.close()
+    print("📊 Saved: random_scaling_comparison.png")
+
+# --------------------- Uniform Graphs ---------------------
+if uniform_data:
+    unique_ns = sorted({n for (n, _, _) in uniform_data.keys()})
+    print(f"📈 Generating {len(unique_ns)} Uniform graphs...")
+    for N in unique_ns:
+        actors_for_n = sorted({act for (nn, _, act) in uniform_data if nn == N})
+        with_r, no_r = [], []
+        for act in actors_for_n:
+            key_w = (N, True, act)
+            key_n = (N, False, act)
+            avg_w = np.mean(uniform_data[key_w]) if key_w in uniform_data and uniform_data[key_w] else np.nan
+            avg_n = np.mean(uniform_data[key_n]) if key_n in uniform_data and uniform_data[key_n] else np.nan
+            with_r.append(avg_w)
+            no_r.append(avg_n)
+        
+        plt.figure(figsize=(12, 8))
+        plt.plot(actors_for_n, with_r, 'o-', linewidth=2.5, label='With Scheduler')
+        plt.plot(actors_for_n, no_r, 's-', linewidth=2.5, label='No Scheduler')
+        plt.xlabel('Number of Actors')
+        plt.ylabel('Average Runtime (seconds)')
+        plt.title(f'Uniform Test On gpufarm5 — N = {N}\nWith Scheduler vs No Scheduler (10-run average)')
+        plt.legend(fontsize=12)
+        plt.grid(True, alpha=0.3)
+        plt.tight_layout()
+        plt.savefig(f'uniform_N_{N}_comparison.png', dpi=300)
+        plt.close()
+        print(f"   → uniform_N_{N}_comparison.png")
+
+# ====================== NEW: MEAN DIFFERENCE STATISTICS ======================
+print("\n" + "="*80)
+print("📊 MEAN DIFFERENCE (With Scheduler vs No Scheduler)")
+print("="*80)
+
+# --- Random Scaling ---
+if random_data:
+    print("\n🔹 Random Scaling (Heterogeneous, actors ≥ 30 000)")
+    all_actors = sorted({act for (_, act) in random_data.keys() if act > 1})
+    
+    total_with, total_no = [], []
+    
+    print(f"{'Actors':>8} | {'With Scheduler':>15} | {'No Scheduler':>15} | {'Diff (s)':>12} | {'Diff (%)':>10}")
+    print("-" * 78)
+    
+    for act in all_actors:
+        key_w = (True, act)
+        key_n = (False, act)
+        avg_w = np.mean(random_data.get(key_w, [])) if random_data.get(key_w) else np.nan
+        avg_n = np.mean(random_data.get(key_n, [])) if random_data.get(key_n) else np.nan
+        diff_s = avg_w - avg_n
+        diff_pct = (diff_s / avg_n * 100) if avg_n and not np.isnan(avg_n) else np.nan
+        
+        print(f"{act:8,} | {avg_w:15.3f} | {avg_n:15.3f} | {diff_s:12.3f} | {diff_pct:9.2f}%")
+        
+        total_with.extend(random_data.get(key_w, []))
+        total_no.extend(random_data.get(key_n, []))
+    
+    # Overall mean for Random Scaling
+    if total_with and total_no:
+        mean_w = np.mean(total_with)
+        mean_n = np.mean(total_no)
+        overall_diff_s = mean_w - mean_n
+        overall_diff_pct = (overall_diff_s / mean_n * 100) if mean_n else 0.0
+        
+        print("-" * 78)
+        print(f"OVERALL (all {len(total_with)} runs):")
+        print(f"  Mean With Scheduler : {mean_w:8.3f} s")
+        print(f"  Mean No Scheduler   : {mean_n:8.3f} s")
+        print(f"  Absolute difference : {overall_diff_s:8.3f} s")
+        print(f"  Percentage difference : {overall_diff_pct:6.2f}%")
+
+# --- Uniform (all N combined) ---
+if uniform_data:
+    print("\n🔹 Uniform Tests (ALL N combined)")
+    total_with_u, total_no_u = [], []
+    
+    unique_ns = sorted({n for (n, _, _) in uniform_data.keys()})
+    for N in unique_ns:
+        actors_for_n = sorted({act for (nn, _, act) in uniform_data if nn == N})
+        for act in actors_for_n:
+            key_w = (N, True, act)
+            key_n = (N, False, act)
+            total_with_u.extend(uniform_data.get(key_w, []))
+            total_no_u.extend(uniform_data.get(key_n, []))
+    
+    if total_with_u and total_no_u:
+        mean_w_u = np.mean(total_with_u)
+        mean_n_u = np.mean(total_no_u)
+        overall_diff_s_u = mean_w_u - mean_n_u
+        overall_diff_pct_u = (overall_diff_s_u / mean_n_u * 100) if mean_n_u else 0.0
+        
+        print(f"  Mean With Scheduler : {mean_w_u:8.3f} s")
+        print(f"  Mean No Scheduler   : {mean_n_u:8.3f} s")
+        print(f"  Absolute difference : {overall_diff_s_u:8.3f} s")
+        print(f"  Percentage difference : {overall_diff_pct_u:6.2f}%")
+    else:
+        print("  (No uniform data found)")
+
+print("\n🎉 Done! Graphs saved + mean differences printed above.")
diff --git a/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py b/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
new file mode 100644
index 0000000000..8c65efd1d4
--- /dev/null
+++ b/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+import glob
+import re
+import numpy as np
+import matplotlib.pyplot as plt
+from collections import defaultdict
+
+# Directory containing benchmark outputs
+DATA_DIR = "/student/nqr159/data/mmul-actor-test/benchmark-results"
+
+# Patterns
+driver_files = glob.glob(f"{DATA_DIR}/matrix_mul_driver_run*.txt")
+actor_files = glob.glob(f"{DATA_DIR}/test_run*.txt")
+
+# Regex
+size_pattern = re.compile(r"N=(\d+)")
+driver_total_pattern = re.compile(r"TOTAL:\s+([\d.]+)")
+actor_total_pattern = re.compile(r"TOTAL end-to-end:\s+([\d.]+)")
+
+driver_data = defaultdict(list)
+actor_data = defaultdict(list)
+
+
+def parse_driver(file):
+    with open(file) as f:
+        current_size = None
+        for line in f:
+            size_match = size_pattern.search(line)
+            if size_match:
+                current_size = int(size_match.group(1))
+
+            total_match = driver_total_pattern.search(line)
+            if total_match and current_size:
+                driver_data[current_size].append(float(total_match.group(1)))
+
+
+def parse_actor(file):
+    with open(file) as f:
+        current_size = None
+        for line in f:
+            size_match = size_pattern.search(line)
+            if size_match:
+                current_size = int(size_match.group(1))
+
+            total_match = actor_total_pattern.search(line)
+            if total_match and current_size:
+                actor_data[current_size].append(float(total_match.group(1)))
+
+
+# Parse files
+for f in driver_files:
+    parse_driver(f)
+
+for f in actor_files:
+    parse_actor(f)
+
+
+# Compute means
+sizes = sorted(driver_data.keys())
+
+driver_means = [np.mean(driver_data[s]) for s in sizes]
+actor_means = [np.mean(actor_data[s]) for s in sizes]
+
+# Compute differences
+abs_diff = [a - d for d, a in zip(driver_means, actor_means)]
+speedup = [d / a if a != 0 else float('inf') for d, a in zip(driver_means, actor_means)]
+percent_diff = [((a - d) / d) * 100 if d != 0 else 0 for d, a in zip(driver_means, actor_means)]
+
+print("===== MEAN RESULTS =====")
+print("N     | CUDA (ms) | Actors (ms) | Diff (ms) | % Diff | Speedup (CUDA/Actors)")
+print("-" * 80)
+
+for s, d, a, diff, pct, sp in zip(sizes, driver_means, actor_means, abs_diff, percent_diff, speedup):
+    print(f"N={s:5d} | {d:10.3f} | {a:11.3f} | {diff:9.3f} | {pct:7.2f}% | {sp:8.3f}")
+
+
+# Plot
+plt.figure()
+
+plt.plot(sizes, driver_means, marker='o', label="CUDA (Driver)")
+plt.plot(sizes, actor_means, marker='s', label="CUDA Actors")
+
+plt.xlabel("Matrix Size (N)")
+plt.ylabel("Mean Execution Time (ms)")
+plt.title("CUDA vs CUDA Actor Matrix Multiplication Performance")
+plt.legend()
+plt.grid(True)
+
+plt.savefig(f"{DATA_DIR}/mmul_benchmark_plot.png", dpi=300)
+
+plt.show()
diff --git a/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py b/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py
new file mode 100644
index 0000000000..13e2cefda4
--- /dev/null
+++ b/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+import re
+import numpy as np
+import matplotlib.pyplot as plt
+from collections import defaultdict
+
+cuda_data = defaultdict(list)
+caf_data = defaultdict(list)
+
+# -----------------------------
+# Parse CUDA baseline logs
+# -----------------------------
+for i in range(1, 11):
+    filename = f"matrix_mul_driver_run{i}.txt"
+    with open(filename) as f:
+        for line in f:
+            m = re.search(r"iterations = (\d+), total GPU time = ([0-9.]+)", line)
+            if m:
+                it = int(m.group(1))
+                time = float(m.group(2))
+                cuda_data[it].append(time)
+
+# -----------------------------
+# Parse CAF CUDA logs
+# -----------------------------
+for i in range(1, 11):
+    filename = f"test_run{i}.txt"
+    with open(filename) as f:
+        for line in f:
+            m = re.search(r"iterations = (\d+), time=([0-9.]+)", line)
+            if m:
+                it = int(m.group(1))
+                time = float(m.group(2))
+                caf_data[it].append(time)
+
+iterations = sorted(cuda_data.keys())
+
+cuda_mean = [np.mean(cuda_data[it]) for it in iterations]
+caf_mean = [np.mean(caf_data[it]) for it in iterations]
+
+# -----------------------------
+# CLI Output
+# -----------------------------
+print("\nMean Performance Comparison\n")
+print(f"{'Iterations':>10} {'CUDA(ms)':>12} {'CAF CUDA(ms)':>15} {'Diff(ms)':>12} {'Overhead %':>12}")
+
+for i, it in enumerate(iterations):
+    diff = caf_mean[i] - cuda_mean[i]
+    pct = (diff / cuda_mean[i]) * 100
+    print(f"{it:>10} {cuda_mean[i]:>12.2f} {caf_mean[i]:>15.2f} {diff:>12.2f} {pct:>11.2f}%")
+
+# -----------------------------
+# Plot
+# -----------------------------
+plt.figure(figsize=(8,6))
+plt.plot(iterations, cuda_mean, marker='o', label="CUDA")
+plt.plot(iterations, caf_mean, marker='o', label="CAF CUDA")
+
+plt.xlabel("Iterations")
+plt.ylabel("Time (ms)")
+plt.title("Matrix Multiplication Performance")
+plt.legend()
+plt.grid(True)
+
+plt.savefig("mmul_comparison.png")
+plt.show()

From 302ee39c2875807f9d565e1632920ce0cb757a80 Mon Sep 17 00:00:00 2001
From: KyleKlenk <kyle.c.klenk@gmail.com>
Date: Mon, 6 Apr 2026 11:38:44 -0600
Subject: [PATCH 0564/1000] Modify Tests

---
 .gitignore                                    |   2 +
 libcaf_cuda/sc26/Fault-Tolerance/main.cpp     |  56 ++----
 libcaf_cuda/sc26/Runtime-Overhead/Makefile    |   4 +-
 .../sc26/Runtime-Overhead/actor_facade.cpp    | 127 +++++++-----
 .../sc26/Runtime-Overhead/command_runner.cpp  | 187 +-----------------
 5 files changed, 107 insertions(+), 269 deletions(-)

diff --git a/.gitignore b/.gitignore
index eb938ddb25..bbca618c21 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,5 @@ libcaf_cuda/sc26/Runtime-Overhead/mmul.cubin
 libcaf_cuda/sc26/Sequence-Independent-Tasks/compile.sh
 libcaf_cuda/sc26/Runtime-Overhead/command_runner
 libcaf_cuda/sc26/Fault-Tolerance/compile.sh
+libcaf_cuda/sc26/Fault-Tolerance/main
+libcaf_cuda/sc26/Runtime-Overhead/actor_facade
diff --git a/libcaf_cuda/sc26/Fault-Tolerance/main.cpp b/libcaf_cuda/sc26/Fault-Tolerance/main.cpp
index 895671bc89..f254538b98 100644
--- a/libcaf_cuda/sc26/Fault-Tolerance/main.cpp
+++ b/libcaf_cuda/sc26/Fault-Tolerance/main.cpp
@@ -28,7 +28,6 @@
 #include <caf/all.hpp>
 #include <caf/actor_cast.hpp>
 #include <caf/cuda/all.hpp>
-#include "../common/kernel_paths.hpp"
 
 #include <deque>
 #include <map>
@@ -85,16 +84,6 @@ class WorkerActor {
 
     mc_runner runner_;
 
-    // Keep all mem_ptrs alive until the stream is idle (gpu_done_atom).
-    // Using a tuple matching the runner return: <mem_ptr<int>, mem_ptr<int>, mem_ptr<int>>
-    std::tuple<mem_ptr<int>, mem_ptr<int>, mem_ptr<int>> pending_refs_;
-
-    // The out<int> mem_ptr (alias into pending_refs_ for convenience).
-    mem_ptr<int> result_ptr_ = nullptr;
-
-    // Response promise to deliver the hit count back to the supervisor.
-    typed_response_promise<int> pending_rp_;
-
 public:
     WorkerActor(event_based_actor* self, int index)
         : self_(self), worker_index_(index) {}
@@ -102,14 +91,13 @@ class WorkerActor {
     behavior make_behavior() {
         return {
             // ── Kernel launch request from the supervisor ─────────────────
-            [this](int seed, int num_samples) -> result<int> {
-                pending_rp_ = self_->make_response_promise<int>();
+            [this](int seed, int num_samples) -> int {
 
-                auto& mgr = self_->system().cuda_manager();
+                auto& mgr = caf::cuda::manager::get();
                 // All workers use device 0 on a single-GPU machine.
                 // On multi-GPU hardware, replace 0 with worker_index_.
                 auto program = mgr.create_program_from_cubin(
-                    actor_tests::paths::monte_carlo_cubin, "monteCarloKernel");
+                    "monte_carlo.cubin", "monteCarloKernel");
 
                 nd_range dims(/*grid*/64, 1, 1, /*block*/256, 1, 1);
 
@@ -120,34 +108,27 @@ class WorkerActor {
                 const std::vector<int> zero_buf{0};
                 auto arg_out = create_in_out_arg(zero_buf);
 
-                // run_async_notify: kernel is launched non-blocking; when the
-                // CUDA stream goes idle the runtime calls gpu_done_atom on this
-                // actor from its internal thread.
-                pending_refs_ = runner_.run_async_notify(
+                self_->println("[Worker {:2d}] Launched GPU kernel: "
+                               "seed={}, samples={}", worker_index_, seed, num_samples);
+
+                // For command_runner: actor_id = worker_index_ + 1 to keep streams distinct
+                auto output = runner_.run_async(
                     program, dims,
-                    actor_cast<actor>(self_),
+                    worker_index_ + 1, /*shared_memory=*/0, /*device_number=*/0,
                     arg_seed, arg_samples, arg_out);
 
                 // The in_out<int> result is the third (last) element.
-                result_ptr_ = std::get<2>(pending_refs_);
+                auto result_ptr = std::get<2>(output);
 
-                self_->println("[Worker {:2d}] Launched GPU kernel: "
-                               "seed={}, samples={}", worker_index_, seed, num_samples);
-                return pending_rp_;
-            },
-
-            // ── GPU stream idle callback ───────────────────────────────────
-            [this](gpu_done_atom) {
-                std::vector<int> host = result_ptr_->copy_to_host();
+                // Block thread to wait for GPU and copy result to host.
+                std::vector<int> host = result_ptr->copy_to_host();
                 int M = host.empty() ? 0 : host[0];
 
-                self_->println("[Worker {:2d}] gpu_done_atom: M={}", worker_index_, M);
-
-                // Safe to release device memory now (stream confirmed idle).
-                result_ptr_  = nullptr;
-                pending_refs_ = {};
+                self_->println("[Worker {:2d}] GPU complete: M={}", worker_index_, M);
 
-                pending_rp_.deliver(M);
+                // Safe to release device memory now implicitly as result_ptr goes out of scope.
+                
+                return M;
             }
         };
     }
@@ -346,6 +327,7 @@ class config : public actor_system_config {
 };
 
 void caf_main(actor_system& sys, const config&) {
+    caf::cuda::manager::init(sys);
     constexpr int NUM_WORKERS      = 2;
     constexpr int TOTAL_BATCHES    = 20;
     constexpr int SAMPLES_PER_BATCH = 10'000'000;
@@ -367,6 +349,8 @@ void caf_main(actor_system& sys, const config&) {
 
     // Block until the supervisor signals done (or terminates cleanly).
     self->receive([](done_atom) {});
+    
+    caf::cuda::manager::shutdown();
 }
 
-CAF_MAIN(caf::cuda::manager, id_block::monte_carlo_app)
+CAF_MAIN(id_block::monte_carlo_app)
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/Makefile b/libcaf_cuda/sc26/Runtime-Overhead/Makefile
index 9ce456d713..a894e9b48c 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/Makefile
+++ b/libcaf_cuda/sc26/Runtime-Overhead/Makefile
@@ -31,8 +31,8 @@ mmul_kernel: mmul.cu
 cuda_native: cuda_native.cpp
 	$(CXX) $(CXXFLAGS) $(INCLUDES) -o cuda_native cuda_native.cpp $(LIBS) $(LDFLAGS)
 
-actors_facade: actor_facade.cpp
-	$(CXX) $(CXXFLAGS) $(INCLUDES) -o actors_facade actor_facade.cpp $(LIBS) $(LDFLAGS)
+actor_facade: actor_facade.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o actor_facade actor_facade.cpp $(LIBS) $(LDFLAGS)
 
 command_runner: command_runner.cpp
 	$(CXX) $(CXXFLAGS) $(INCLUDES) -o command_runner command_runner.cpp $(LIBS) $(LDFLAGS)
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp b/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
index a1aa4e537a..0f25c01d3a 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
@@ -1,59 +1,82 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
-#include "../common/kernel_paths.hpp"
-#include "vector"
-
-class MatMult {
-  caf::event_based_actor* self_;
-  std::vector<int> A_ = {1, 2,
-                         3, 4};
-  std::vector<int> B_ = {5, 6,
-                         7, 8};
-
-
-  public:
-    MatMult(caf::event_based_actor* self) : self_(self) {};
-
-    caf::behavior make_behavior() {
-      return {
-        [this](int N) {
-
-          int THREADS = 32;
-          int BLOCKS = (N + THREADS - 1) / THREADS;
-
-          // Launch dims: 2x2 threads is enough for a 2x2 product
-          caf::cuda::nd_range dim(
-              BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-          auto gpuActor = self_->system().cuda_manager().spawnFromCUBIN(
-              actor_tests::paths::matmul_verbose_cubin, "matrixMul", dim,
-              in<int>{}, in<int>{}, out<int>{}, in<int>{});
-
-          self_->mail(
-              caf::cuda::create_in_arg(A_), 
-              caf::cuda::create_in_arg(B_), 
-              caf::cuda::create_out_arg_with_size<int>(N * N), // Seems there is no need to create a host buffer, or maybe that is what this does?
-              caf::cuda::create_in_arg(N))
-              .send(gpuActor); 
-        },
-
-        [this](const std::vector<output_buffer>& result) {
-          std::vector<int> output = caf::cuda::extract_vector<int>(result);
-          self_->println("Received result from GPU actor {}, {}, {}, {} ", 
-                             output[0], output[1], output[2], output[3]);
-          self_->quit();
-        }
-      };
-    }
-};
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
 
+using namespace caf;
+using namespace std::chrono_literals;
+
+void run_mmul_test(caf::actor_system& sys, int matrix_size) {
+  caf::cuda::manager::init(sys);
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  auto t_total_start = std::chrono::steady_clock::now();
+
+  int THREADS = 32;
+  int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+  std::vector<int> h_a(matrix_size * matrix_size, 2);
+  std::vector<int> h_b(matrix_size * matrix_size, 3);
+  std::vector<int> h_c(matrix_size * matrix_size, 0);
+
+  auto gpuActor = mgr.spawnFromCUBIN("mmul.cubin", "matrixMul", dim,
+                                     in<int>{}, in<int>{}, out<int>{}, in<int>{});
+
+  auto t_spawn_end = std::chrono::steady_clock::now();
+
+  using clock = std::chrono::steady_clock;
+  auto t_a_inarg_start = clock::now();
+  auto arg1 = caf::cuda::create_in_arg(h_a);
+  auto t_a_inarg_end = clock::now();
+
+  auto t_b_inarg_start = clock::now();
+  auto arg2 = caf::cuda::create_in_arg(h_b);
+  auto t_b_inarg_end = clock::now();
+
+  auto arg3 = caf::cuda::create_out_arg(h_c);
+  auto arg4 = caf::cuda::create_in_arg(matrix_size);
+
+  sys.spawn([=](event_based_actor* self_actor) {
+    auto t_request_start = clock::now();
+    self_actor->mail(gpuActor, arg1, arg2, arg3, arg4)
+      .request(gpuActor, 100s).then(
+        [=](const std::vector<output_buffer>& outputs) {
+          auto t_response_received = clock::now();
+          auto t_total_end = clock::now();
+
+          using ms = std::chrono::duration<double, std::milli>;
+          std::cout << "\n===== ACTOR FACADE BENCHMARK RESULTS (N=" << matrix_size << ") =====\n";
+          std::cout << "spawn actor: " << ms(t_spawn_end - t_total_start).count() << " ms\n";
+          std::cout << "create_in_arg A: " << ms(t_a_inarg_end - t_a_inarg_start).count() << " ms\n";
+          std::cout << "create_in_arg B: " << ms(t_b_inarg_end - t_b_inarg_start).count() << " ms\n";
+          std::cout << "request \xE2\x86\x92 response latency (includes transfers & exec): "
+                    << ms(t_response_received - t_request_start).count() << " ms\n";
+          std::cout << "TOTAL end-to-end: " << ms(t_total_end - t_total_start).count() << " ms\n";
+          std::cout << "========================================================\n";
+
+          self_actor->send_exit(gpuActor, exit_reason::user_shutdown);
+          self_actor->quit();
+        });
+  });
+
+  sys.await_all_actors_done();
+
+  auto end = std::chrono::steady_clock::now();
+  auto duration_ms = std::chrono::duration_cast<std::chrono::milliseconds>(end - t_total_start).count();
+
+  std::cout << "[MMUL FACADE TEST] matrix_size=" << matrix_size << ", time=" << duration_ms << " ms\n";
+
+  caf::cuda::manager::shutdown();
+}
 
 void caf_main(caf::actor_system& sys) {
-    caf::scoped_actor self{sys};
-    self->println("Hello, CAF!");
-    auto test_actor = self->spawn(caf::actor_from_state<MatMult>);
-    self->mail(2).send(test_actor);
-    self->await_all_other_actors_done();
+  run_mmul_test(sys, 1000);
+  run_mmul_test(sys, 4000);
+  run_mmul_test(sys, 8000);
+  run_mmul_test(sys, 12000);
 }
 
-CAF_MAIN()
\ No newline at end of file
+CAF_MAIN()
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
index 3abf48a2e5..1523087a18 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
@@ -10,33 +10,11 @@
 #include <numeric>
 #include <random>
 #include "caf/actor_registry.hpp"
-//#include <caf/atoms.hpp>
-
-
 
 using namespace caf;
 using namespace std::chrono_literals;
 
-
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-  
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
-    }
-  }
-}
-
-using command =
-  caf::cuda::command_runner<>;
+using command = caf::cuda::command_runner<>;
 
 command mmul_command;
 
@@ -48,160 +26,13 @@ struct mmul_state {
 //so its only fair that we do not either 
 std::vector<int> matrixC;
 
-
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
-  return {
-
-    [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB,
-        int N) {
-      self->println("fun 1");
-      using clock = std::chrono::steady_clock;
-      using ms = std::chrono::duration<double, std::milli>;
-
-
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
-      int device = 0;
-      int stream = 1;
-
-      auto program =
-        mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
-
-      auto t_total_start = clock::now();
-      // -------------------------
-      // create_in_arg A
-      // -------------------------
-      auto t_a_inarg_start = clock::now();
-
-      auto inA = caf::cuda::create_in_arg(std::move(matrixA));
-
-      auto t_a_inarg_end = clock::now();
-
-      // -------------------------
-      // transfer A
-      // -------------------------
-      auto t_a_transfer_start = clock::now();
-
-      auto arg1 = mmul_command.transfer_memory(
-          device,
-          stream,
-          std::move(inA));
-
-      auto t_a_transfer_end = clock::now();
-
-      // -------------------------
-      // create_in_arg B
-      // -------------------------
-      auto t_b_inarg_start = clock::now();
-
-      auto inB = caf::cuda::create_in_arg(std::move(matrixB));
-
-      auto t_b_inarg_end = clock::now();
-
-      // -------------------------
-      // transfer B
-      // -------------------------
-      auto t_b_transfer_start = clock::now();
-
-      auto arg2 = mmul_command.transfer_memory(
-          device,
-          stream,
-          std::move(inB));
-
-      auto t_b_transfer_end = clock::now();
-
-      // -------------------------
-      // spawn actor
-      // -------------------------
-      auto t_spawn_start = clock::now();
-
-      caf::actor mmul_actor =
-        self->spawn(caf::cuda::mmul_actor_fun<int>, program);
-
-      auto t_spawn_end = clock::now();
-
-      // -------------------------
-      // request
-      // -------------------------
-      auto t_request_start = clock::now();
-
-      self->mail(arg1, arg2, N, device, stream)
-        .request(mmul_actor, std::chrono::seconds(30))
-        .then(
-          [=](caf::cuda::mem_ptr<int> dC) {
-
-            auto t_response_received = clock::now();
-
-            //std::vector<int> matrixC(N*N);
-            // -------------------------
-            // copy to host
-            // -------------------------
-            auto t_copy_start = clock::now();
-
-            //std::vector<int> matrixC = dC->copy_to_host();
-
-	    dC->copy_to_host(matrixC.data(),N*N);
-
-            auto t_copy_end = clock::now();
-            auto t_total_end = clock::now();
-
-            // -------------------------
-            // Print timings
-            // -------------------------
-
-            std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
-
-            std::cout << "create_in_arg A: "
-                      << ms(t_a_inarg_end - t_a_inarg_start).count()
-                      << " ms\n";
-
-            std::cout << "transfer A: "
-                      << ms(t_a_transfer_end - t_a_transfer_start).count()
-                      << " ms\n";
-
-            std::cout << "create_in_arg B: "
-                      << ms(t_b_inarg_end - t_b_inarg_start).count()
-                      << " ms\n";
-
-            std::cout << "transfer B: "
-                      << ms(t_b_transfer_end - t_b_transfer_start).count()
-                      << " ms\n";
-
-            std::cout << "spawn actor: "
-                      << ms(t_spawn_end - t_spawn_start).count()
-                      << " ms\n";
-
-            std::cout << "request → response latency: "
-                      << ms(t_response_received - t_request_start).count()
-                      << " ms\n";
-
-            std::cout << "copy_to_host: "
-                      << ms(t_copy_end - t_copy_start).count()
-                      << " ms\n";
-
-            std::cout << "TOTAL end-to-end: "
-                      << ms(t_total_end - t_total_start).count()
-                      << " ms\n";
-
-            std::cout << "=============================================\n";
-
-            self->quit();
-          }
-        );
-    }
-
-  };
-}
-
-
-caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
+caf::behavior mmul_actor(caf::stateful_actor<mmul_state>* self) {
   return {
 
     [=](const std::vector<int>& matrixA,
         const std::vector<int>& matrixB,
         int N) {
       
-      self->println("fun 2");
       using clock = std::chrono::steady_clock;
       using ms = std::chrono::duration<double, std::milli>;
 
@@ -349,17 +180,15 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size) {
 
   matrixC.resize(matrix_size*matrix_size);
 
- using clock = std::chrono::steady_clock;
-
-auto t_start = clock::now();
-
-caf::actor a =sys.spawn(mmul_actor_fun_2);
+  using clock = std::chrono::steady_clock;
 
-anon_mail(matrixA,matrixB,matrix_size).send(a);
+  auto t_start = clock::now();
 
-auto t_end = clock::now();
+  caf::actor a =sys.spawn(mmul_actor);
 
+  anon_mail(matrixA,matrixB,matrix_size).send(a);
 
+  auto t_end = clock::now();
 
   // Wait for all actors to finish
   sys.await_all_actors_done();
@@ -380,7 +209,7 @@ auto t_end = clock::now();
 
 
 void caf_main(caf::actor_system& sys) {
- run_mmul_test(sys,1000);
+  run_mmul_test(sys,1000);
   run_mmul_test(sys,4000);
   run_mmul_test(sys,8000);
   run_mmul_test(sys,12000);

From 30773629908b60f54564c703d75ba615570aae3a Mon Sep 17 00:00:00 2001
From: KyleKlenk <kyle.c.klenk@gmail.com>
Date: Mon, 6 Apr 2026 11:58:37 -0600
Subject: [PATCH 0565/1000] Created actor_facade test for the
 sequence-of-independed-tasks

---
 .../sc26/Sequence-Independent-Tasks/Makefile  |  12 +-
 .../actor_facade.cpp                          | 108 ++++++++++++++++++
 .../sc26/Sequence-Independent-Tasks/mmul.cu   |  16 +++
 3 files changed, 130 insertions(+), 6 deletions(-)
 create mode 100644 libcaf_cuda/sc26/Sequence-Independent-Tasks/mmul.cu

diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/Makefile b/libcaf_cuda/sc26/Sequence-Independent-Tasks/Makefile
index 7bf02dff95..a43dcbdc26 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/Makefile
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/Makefile
@@ -28,14 +28,14 @@ all: mmul_kernel base_cuda_program actor_facade actor_command_runner
 mmul_kernel: mmul.cu
 	$(NVCC) -g -arch=$(NVCC_ARCH) --cubin mmul.cu -o mmul.cubin
 
-base_cuda_program: main_cuda_native.cpp
-	$(CXX) $(CXXFLAGS) $(INCLUDES) -o main_cuda_native main_cuda_native.cpp $(LIBS) $(LDFLAGS)
+base_cuda_program: cuda_native.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o main_cuda_native cuda_native.cpp $(LIBS) $(LDFLAGS)
 
-actor_facade: main_actor_facade.cpp
-	$(CXX) $(CXXFLAGS) $(INCLUDES) -o main_actor_facade main_actor_facade.cpp $(LIBS) $(LDFLAGS)
+actor_facade: actor_facade.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o main_actor_facade actor_facade.cpp $(LIBS) $(LDFLAGS)
 
-actor_command_runner: main_command_runner.cpp
-	$(CXX) $(CXXFLAGS) $(INCLUDES) -o main_command_runner main_command_runner.cpp $(LIBS) $(LDFLAGS)
+actor_command_runner: command_runner.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o main_command_runner command_runner.cpp $(LIBS) $(LDFLAGS)
 
 clean:
 	rm -f mmul.cubin
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
index e69de29bb2..9af09a96e2 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
@@ -0,0 +1,108 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+struct bench_state {
+  int completed = 0;
+  int total = 0;
+  std::chrono::steady_clock::time_point start_time;
+  caf::actor gpuActor;
+  int matrix_size = 0;
+  std::vector<int> h_a;
+  std::vector<int> h_b;
+  std::vector<int> h_c;
+};
+
+caf::behavior bench_coordinator(caf::stateful_actor<bench_state>* self, caf::actor gpuActor, int matrix_size, int iterations) {
+  self->state().completed = 0;
+  self->state().total = iterations;
+  self->state().gpuActor = gpuActor;
+  self->state().matrix_size = matrix_size;
+  
+  self->monitor(gpuActor, [self](const error& err) {
+    std::cout << "GPU Actor died unexpectedly! " << to_string(err) << std::endl;
+    self->quit();
+  });
+
+  return {
+    [=](int) {
+      self->state().start_time = std::chrono::steady_clock::now();
+      
+      // Persistent host vectors in state
+      self->state().h_a.assign(matrix_size * matrix_size, 2);
+      self->state().h_b.assign(matrix_size * matrix_size, 3);
+      self->state().h_c.assign(matrix_size * matrix_size, 0);
+      
+      for (int i = 0; i < iterations; ++i) {
+        auto arg1 = caf::cuda::create_in_arg(self->state().h_a);
+        auto arg2 = caf::cuda::create_in_arg(self->state().h_b);
+        auto arg3 = caf::cuda::create_out_arg(self->state().h_c);
+        auto arg4 = caf::cuda::create_in_arg(matrix_size);
+        
+        self->mail(gpuActor, arg1, arg2, arg3, arg4)
+          .request(gpuActor, infinite)
+          .then(
+            [=](const std::vector<output_buffer>& /*outputs*/) {
+              self->state().completed++;
+              if (self->state().completed == self->state().total) {
+                auto end_time = std::chrono::steady_clock::now();
+                using ms = std::chrono::duration<double, std::milli>;
+                double duration_ms = ms(end_time - self->state().start_time).count();
+                std::cout << "[SERIES RESULT] Matrix " << self->state().matrix_size << "x" << self->state().matrix_size
+                          << ", iterations = " << self->state().total
+                          << ", total CPU/Actor time = " << duration_ms << " ms\n";
+                self->send_exit(self->state().gpuActor, exit_reason::user_shutdown);
+                self->quit();
+              }
+            },
+            [=](const error& err) {
+              std::cout << "Error in iteration: " << to_string(err) << std::endl;
+              self->state().completed++;
+              if (self->state().completed == self->state().total) {
+                self->send_exit(self->state().gpuActor, exit_reason::user_shutdown);
+                self->quit();
+              }
+            }
+          );
+      }
+    }
+  };
+}
+
+void run_series(caf::actor_system& sys, int matrix_size, int iterations) {
+  caf::cuda::manager::init(sys);
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  int THREADS = 32;
+  int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+  // Spawn the GPU actor
+  auto gpuActor = mgr.spawnFromCUBIN("mmul.cubin", "matrixMul", dim,
+                                     in<int>{}, in<int>{}, out<int>{}, in<int>{});
+
+  // Spawn coordinator
+  auto coordinator = sys.spawn(bench_coordinator, gpuActor, matrix_size, iterations);
+  
+  // Start benchmark
+  anon_mail(1).send(coordinator);
+
+  // Wait for it to finish and kill GPU actor internally
+  sys.await_all_actors_done();
+
+  caf::cuda::manager::shutdown();
+}
+
+void caf_main(caf::actor_system& sys) {
+  for (int i = 1000; i <= 10000; i += 1000) {
+    run_series(sys, 1000, i);
+  }
+}
+
+CAF_MAIN()
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/mmul.cu b/libcaf_cuda/sc26/Sequence-Independent-Tasks/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From e218d06fc2cdf0bcf11f2a93fe898e51c41d624d Mon Sep 17 00:00:00 2001
From: KyleKlenk <kyle.c.klenk@gmail.com>
Date: Mon, 6 Apr 2026 12:56:32 -0600
Subject: [PATCH 0566/1000] Clean up runtime overhead tests

---
 .../sc26/Runtime-Overhead/actor_facade.cpp    | 54 ++++++++------
 .../sc26/Runtime-Overhead/command_runner.cpp  | 30 +++++---
 .../sc26/Runtime-Overhead/cuda_native.cpp     | 71 ++-----------------
 .../command_runner.cpp                        |  2 +-
 4 files changed, 61 insertions(+), 96 deletions(-)

diff --git a/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp b/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
index 0f25c01d3a..51701c9ed3 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
@@ -4,30 +4,42 @@
 #include <iomanip>
 #include <vector>
 #include <chrono>
+#include <random>
 
 using namespace caf;
 using namespace std::chrono_literals;
 
+static const unsigned int RANDOM_SEED = 42;
+
 void run_mmul_test(caf::actor_system& sys, int matrix_size) {
-  caf::cuda::manager::init(sys);
+  // F5: manager::init/shutdown moved to caf_main — called once for all sizes
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 
-  auto t_total_start = std::chrono::steady_clock::now();
-
+  // F3: spawn GPU actor (loads cubin) BEFORE timing begins, so module-load
+  //     overhead is excluded from the per-size measurement — matching
+  //     cuda_native which loads the module once outside all per-size timing.
   int THREADS = 32;
   int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
   caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-  std::vector<int> h_a(matrix_size * matrix_size, 2);
-  std::vector<int> h_b(matrix_size * matrix_size, 3);
-  std::vector<int> h_c(matrix_size * matrix_size, 0);
-
   auto gpuActor = mgr.spawnFromCUBIN("mmul.cubin", "matrixMul", dim,
                                      in<int>{}, in<int>{}, out<int>{}, in<int>{});
 
-  auto t_spawn_end = std::chrono::steady_clock::now();
-
   using clock = std::chrono::steady_clock;
+  using ms = std::chrono::duration<double, std::milli>;
+
+  // F4: use mt19937(42) to match cuda_native data initialisation
+  std::mt19937 rng(RANDOM_SEED);
+  std::uniform_int_distribution<int> dist(1, 10);
+  std::vector<int> h_a(matrix_size * matrix_size);
+  std::vector<int> h_b(matrix_size * matrix_size);
+  std::vector<int> h_c(matrix_size * matrix_size, 0);
+  for (auto& v : h_a) v = dist(rng);
+  for (auto& v : h_b) v = dist(rng);
+
+  // TIMING STARTS — after spawn/module-load, matching cuda_native's exclusion
+  auto t_total_start = clock::now();
+
   auto t_a_inarg_start = clock::now();
   auto arg1 = caf::cuda::create_in_arg(h_a);
   auto t_a_inarg_end = clock::now();
@@ -47,15 +59,13 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size) {
           auto t_response_received = clock::now();
           auto t_total_end = clock::now();
 
-          using ms = std::chrono::duration<double, std::milli>;
           std::cout << "\n===== ACTOR FACADE BENCHMARK RESULTS (N=" << matrix_size << ") =====\n";
-          std::cout << "spawn actor: " << ms(t_spawn_end - t_total_start).count() << " ms\n";
           std::cout << "create_in_arg A: " << ms(t_a_inarg_end - t_a_inarg_start).count() << " ms\n";
           std::cout << "create_in_arg B: " << ms(t_b_inarg_end - t_b_inarg_start).count() << " ms\n";
           std::cout << "request \xE2\x86\x92 response latency (includes transfers & exec): "
                     << ms(t_response_received - t_request_start).count() << " ms\n";
           std::cout << "TOTAL end-to-end: " << ms(t_total_end - t_total_start).count() << " ms\n";
-          std::cout << "========================================================\n";
+          std::cout << "=============================================\n";
 
           self_actor->send_exit(gpuActor, exit_reason::user_shutdown);
           self_actor->quit();
@@ -63,20 +73,24 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size) {
   });
 
   sys.await_all_actors_done();
-
-  auto end = std::chrono::steady_clock::now();
-  auto duration_ms = std::chrono::duration_cast<std::chrono::milliseconds>(end - t_total_start).count();
-
-  std::cout << "[MMUL FACADE TEST] matrix_size=" << matrix_size << ", time=" << duration_ms << " ms\n";
-
-  caf::cuda::manager::shutdown();
 }
 
 void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager::init(sys);  // F5: init once before all sizes
+
+  // F2: warmup run to prime CUDA context, JIT, and CAF infrastructure
+  std::cout << "--- warmup starting ---\n";
+  run_mmul_test(sys, 64);
+  std::cout << "--- warmup complete ---\n";
+
+  // F1: unified sizes matching cuda_native: {1000, 2000, 4000, 8000, 12000}
   run_mmul_test(sys, 1000);
+  run_mmul_test(sys, 2000);
   run_mmul_test(sys, 4000);
   run_mmul_test(sys, 8000);
-  run_mmul_test(sys, 12000);
+  run_mmul_test(sys, 16000);
+
+  caf::cuda::manager::shutdown();  // F5: shutdown once after all sizes
 }
 
 CAF_MAIN()
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
index 1523087a18..4f40e312c2 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
@@ -26,6 +26,8 @@ struct mmul_state {
 //so its only fair that we do not either 
 std::vector<int> matrixC;
 
+static const unsigned int RANDOM_SEED = 42;
+
 caf::behavior mmul_actor(caf::stateful_actor<mmul_state>* self) {
   return {
 
@@ -166,17 +168,20 @@ caf::behavior mmul_actor(caf::stateful_actor<mmul_state>* self) {
 
 
 void run_mmul_test(caf::actor_system& sys, int matrix_size) {
+  // F5: manager::init/shutdown moved to caf_main — called once for all sizes
 
-
-  caf::cuda::manager::init(sys);
   // ------------------------------------
   // Start timing
   // ------------------------------------
   auto start = std::chrono::steady_clock::now();
 
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<int> matrixA(matrix_size * matrix_size,2);
-  std::vector<int> matrixB(matrix_size * matrix_size,3);
+  // F4: use mt19937(42) to match cuda_native data initialisation
+  std::mt19937 rng(RANDOM_SEED);
+  std::uniform_int_distribution<int> dist(1, 10);
+  std::vector<int> matrixA(matrix_size * matrix_size);
+  std::vector<int> matrixB(matrix_size * matrix_size);
+  for (auto& v : matrixA) v = dist(rng);
+  for (auto& v : matrixB) v = dist(rng);
 
   matrixC.resize(matrix_size*matrix_size);
 
@@ -202,18 +207,25 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size) {
 
   std::cout << "[MMUL TEST] matrix_size=" << matrix_size
             << ", time=" << duration_ms << " ms\n";
-
-  caf::cuda::manager::shutdown();
-
 }
 
 
 void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager::init(sys);  // F5: init once before all sizes
+
+  // F2: warmup run to prime CUDA context, JIT, and CAF infrastructure
+  std::cout << "--- warmup starting ---\n";
+  run_mmul_test(sys, 64);
+  std::cout << "--- warmup complete ---\n";
+
+  // F1: unified sizes matching cuda_native: {1000, 2000, 4000, 8000, 12000}
   run_mmul_test(sys,1000);
+  run_mmul_test(sys,2000);
   run_mmul_test(sys,4000);
   run_mmul_test(sys,8000);
-  run_mmul_test(sys,12000);
+  run_mmul_test(sys,16000);
 
+  caf::cuda::manager::shutdown();  // F5: shutdown once after all sizes
 }
 
 
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp b/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
index 32f2faeaf3..507e4e480b 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
@@ -51,36 +51,18 @@ double runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     // ----------------------------------
     // Device Allocation
     // ----------------------------------
-    auto t_alloc_start = clock::now();
-
     checkCU(cuMemAlloc(&d_a, bytes), "cuMemAlloc d_a");
     checkCU(cuMemAlloc(&d_b, bytes), "cuMemAlloc d_b");
     checkCU(cuMemAlloc(&d_c, bytes), "cuMemAlloc d_c");
 
-    auto t_alloc_end = clock::now();
-
     // ----------------------------------
-    // H2D copy A
+    // H2D copies (async, pipelined with kernel)
     // ----------------------------------
-    auto t_h2d_a_start = clock::now();
-
     checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), bytes, stream), "cuMemcpyHtoDAsync A");
-    //checkCU(cuStreamSynchronize(stream), "sync A");
-
-    auto t_h2d_a_end = clock::now();
-
-    // ----------------------------------
-    // H2D copy B
-    // ----------------------------------
-    auto t_h2d_b_start = clock::now();
-
     checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), bytes, stream), "cuMemcpyHtoDAsync B");
-    //checkCU(cuStreamSynchronize(stream), "sync B");
-
-    auto t_h2d_b_end = clock::now();
 
     // ----------------------------------
-    // Kernel launch + execution
+    // Kernel launch
     // ----------------------------------
     const unsigned int blockX = 32;
     const unsigned int blockY = 32;
@@ -89,8 +71,6 @@ double runMatrixMul(CUmodule module, CUfunction kernel, int N) {
 
     void* kernelParams[] = { &d_a, &d_b, &d_c, &N };
 
-    auto t_kernel_start = clock::now();
-
     checkCU(cuLaunchKernel(kernel,
                            gridX, gridY, 1,
                            blockX, blockY, 1,
@@ -100,72 +80,31 @@ double runMatrixMul(CUmodule module, CUfunction kernel, int N) {
                            nullptr),
             "cuLaunchKernel");
 
-   // checkCU(cuStreamSynchronize(stream), "kernel sync");
-
-    auto t_kernel_end = clock::now();
-
     // ----------------------------------
-    // D2H copy
+    // D2H copy + synchronize all pending GPU work
     // ----------------------------------
-    auto t_d2h_start = clock::now();
-
     cuMemcpyDtoHAsync(h_c.data(), d_c, bytes, stream);
-   // cuMemcpyDtoHAsync(h_c, d_c, bytes, stream);
     cuStreamSynchronize(stream);
-    auto t_d2h_end = clock::now();
 
     // ----------------------------------
     // Free device memory
-    // (included in the total to match the actor benchmark which frees
-    //  device memory inside runner.run_into())
     // ----------------------------------
-    auto t_free_start = clock::now();
-
     checkCU(cuMemFree(d_a), "cuMemFree A");
     checkCU(cuMemFree(d_b), "cuMemFree B");
     checkCU(cuMemFree(d_c), "cuMemFree C");
 
-    auto t_free_end = clock::now();
-
     auto t_total_end = clock::now();
 
-
-
-
     // ----------------------------------
     // Destroy stream
     // ----------------------------------
     checkCU(cuStreamDestroy(stream), "cuStreamDestroy");
 
-
     // ----------------------------------
     // Print Results
+    // (Sub-timings omitted: intermediate async ops have no sync point
+    //  so CPU-side timestamps do not reflect actual GPU phase durations.)
     // ----------------------------------
-
-    std::cout << "Device allocation: "
-              << ms(t_alloc_end - t_alloc_start).count()
-              << " ms\n";
-
-    std::cout << "H2D copy A: "
-              << ms(t_h2d_a_end - t_h2d_a_start).count()
-              << " ms\n";
-
-    std::cout << "H2D copy B: "
-              << ms(t_h2d_b_end - t_h2d_b_start).count()
-              << " ms\n";
-
-    std::cout << "Kernel execution: "
-              << ms(t_kernel_end - t_kernel_start).count()
-              << " ms\n";
-
-    std::cout << "D2H copy: "
-              << ms(t_d2h_end - t_d2h_start).count()
-              << " ms\n";
-
-    std::cout << "Device free: "
-              << ms(t_free_end - t_free_start).count()
-              << " ms\n";
-
     double total_ms = ms(t_total_end - t_total_start).count();
 
     std::cout << "TOTAL: " << total_ms << " ms\n";
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
index c52d09612b..39f65bb24b 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
@@ -160,7 +160,7 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size,int iterations) {
   auto& mgr = caf::cuda::manager::get();
   
   auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
 
 
   using clock = std::chrono::steady_clock;

From ba7f126e9468626539659e5c495599275004e4c6 Mon Sep 17 00:00:00 2001
From: KyleKlenk <kyle.c.klenk@gmail.com>
Date: Mon, 6 Apr 2026 13:42:07 -0600
Subject: [PATCH 0567/1000] Cleanup of indpendent tasks

---
 .../actor_facade.cpp                          | 147 +++++++-----
 .../command_runner.cpp                        | 212 ++++++++----------
 2 files changed, 185 insertions(+), 174 deletions(-)

diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
index 9af09a96e2..d711ff752b 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
@@ -11,6 +11,7 @@ using namespace std::chrono_literals;
 struct bench_state {
   int completed = 0;
   int total = 0;
+  bool is_warmup = false;
   std::chrono::steady_clock::time_point start_time;
   caf::actor gpuActor;
   int matrix_size = 0;
@@ -19,90 +20,124 @@ struct bench_state {
   std::vector<int> h_c;
 };
 
-caf::behavior bench_coordinator(caf::stateful_actor<bench_state>* self, caf::actor gpuActor, int matrix_size, int iterations) {
-  self->state().completed = 0;
-  self->state().total = iterations;
-  self->state().gpuActor = gpuActor;
-  self->state().matrix_size = matrix_size;
-  
+// Forward declaration
+void send_next_request(caf::stateful_actor<bench_state>* self);
+
+// S3 fix: back-pressure — sends exactly one request and re-chains on response,
+// bounding in-flight memory to a single 8 MB pair of in-args at any time.
+void send_next_request(caf::stateful_actor<bench_state>* self) {
+  auto& st = self->state();
+  auto arg1 = caf::cuda::create_in_arg(st.h_a);
+  auto arg2 = caf::cuda::create_in_arg(st.h_b);
+  auto arg3 = caf::cuda::create_out_arg(st.h_c);
+  auto arg4 = caf::cuda::create_in_arg(st.matrix_size);
+
+  self->mail(st.gpuActor, arg1, arg2, arg3, arg4)
+    .request(st.gpuActor, infinite)
+    .then(
+      [=](const std::vector<output_buffer>& /*outputs*/) {
+        auto& st2 = self->state();
+        st2.completed++;
+
+        // S6 fix: milestone reporting every 1000 completions
+        if (!st2.is_warmup && st2.completed % 1000 == 0) {
+          auto now = std::chrono::steady_clock::now();
+          using ms = std::chrono::duration<double, std::milli>;
+          double elapsed = ms(now - st2.start_time).count();
+          std::cout << "[MILESTONE] " << st2.completed << " / " << st2.total
+                    << " iterations, elapsed = " << elapsed << " ms\n";
+        }
+
+        if (st2.completed == st2.total) {
+          if (!st2.is_warmup) {
+            auto end_time = std::chrono::steady_clock::now();
+            using ms = std::chrono::duration<double, std::milli>;
+            double duration_ms = ms(end_time - st2.start_time).count();
+            std::cout << "[SERIES RESULT] Matrix " << st2.matrix_size << "x" << st2.matrix_size
+                      << ", iterations = " << st2.total
+                      << ", total CPU/Actor time = " << duration_ms << " ms\n";
+          }
+          self->send_exit(st2.gpuActor, exit_reason::user_shutdown);
+          self->quit();
+        } else {
+          send_next_request(self);
+        }
+      },
+      [=](const error& err) {
+        if (!self->state().is_warmup)
+          std::cout << "Error in iteration: " << to_string(err) << std::endl;
+        auto& st2 = self->state();
+        st2.completed++;
+        if (st2.completed == st2.total) {
+          self->send_exit(st2.gpuActor, exit_reason::user_shutdown);
+          self->quit();
+        } else {
+          send_next_request(self);
+        }
+      }
+    );
+}
+
+caf::behavior bench_coordinator(caf::stateful_actor<bench_state>* self,
+                                 caf::actor gpuActor,
+                                 int matrix_size,
+                                 int iterations,
+                                 bool is_warmup) {
+  auto& st = self->state();
+  st.completed = 0;
+  st.total = iterations;
+  st.is_warmup = is_warmup;
+  st.gpuActor = gpuActor;
+  st.matrix_size = matrix_size;
+  st.h_a.assign(matrix_size * matrix_size, 2);
+  st.h_b.assign(matrix_size * matrix_size, 3);
+  st.h_c.assign(matrix_size * matrix_size, 0);
+
   self->monitor(gpuActor, [self](const error& err) {
-    std::cout << "GPU Actor died unexpectedly! " << to_string(err) << std::endl;
+    if (!self->state().is_warmup)
+      std::cout << "GPU Actor died unexpectedly! " << to_string(err) << std::endl;
     self->quit();
   });
 
   return {
     [=](int) {
       self->state().start_time = std::chrono::steady_clock::now();
-      
-      // Persistent host vectors in state
-      self->state().h_a.assign(matrix_size * matrix_size, 2);
-      self->state().h_b.assign(matrix_size * matrix_size, 3);
-      self->state().h_c.assign(matrix_size * matrix_size, 0);
-      
-      for (int i = 0; i < iterations; ++i) {
-        auto arg1 = caf::cuda::create_in_arg(self->state().h_a);
-        auto arg2 = caf::cuda::create_in_arg(self->state().h_b);
-        auto arg3 = caf::cuda::create_out_arg(self->state().h_c);
-        auto arg4 = caf::cuda::create_in_arg(matrix_size);
-        
-        self->mail(gpuActor, arg1, arg2, arg3, arg4)
-          .request(gpuActor, infinite)
-          .then(
-            [=](const std::vector<output_buffer>& /*outputs*/) {
-              self->state().completed++;
-              if (self->state().completed == self->state().total) {
-                auto end_time = std::chrono::steady_clock::now();
-                using ms = std::chrono::duration<double, std::milli>;
-                double duration_ms = ms(end_time - self->state().start_time).count();
-                std::cout << "[SERIES RESULT] Matrix " << self->state().matrix_size << "x" << self->state().matrix_size
-                          << ", iterations = " << self->state().total
-                          << ", total CPU/Actor time = " << duration_ms << " ms\n";
-                self->send_exit(self->state().gpuActor, exit_reason::user_shutdown);
-                self->quit();
-              }
-            },
-            [=](const error& err) {
-              std::cout << "Error in iteration: " << to_string(err) << std::endl;
-              self->state().completed++;
-              if (self->state().completed == self->state().total) {
-                self->send_exit(self->state().gpuActor, exit_reason::user_shutdown);
-                self->quit();
-              }
-            }
-          );
-      }
+      // S3 fix: fire one request; subsequent requests are chained via back-pressure
+      send_next_request(self);
     }
   };
 }
 
-void run_series(caf::actor_system& sys, int matrix_size, int iterations) {
-  caf::cuda::manager::init(sys);
+// S4 fix: manager::init/shutdown moved to caf_main — called once for all series
+void run_series(caf::actor_system& sys, int matrix_size, int iterations,
+                bool is_warmup = false) {
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 
   int THREADS = 32;
   int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
   caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-  // Spawn the GPU actor
   auto gpuActor = mgr.spawnFromCUBIN("mmul.cubin", "matrixMul", dim,
                                      in<int>{}, in<int>{}, out<int>{}, in<int>{});
 
-  // Spawn coordinator
-  auto coordinator = sys.spawn(bench_coordinator, gpuActor, matrix_size, iterations);
-  
-  // Start benchmark
+  auto coordinator = sys.spawn(bench_coordinator, gpuActor, matrix_size, iterations, is_warmup);
   anon_mail(1).send(coordinator);
-
-  // Wait for it to finish and kill GPU actor internally
   sys.await_all_actors_done();
-
-  caf::cuda::manager::shutdown();
 }
 
 void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager::init(sys);  // S4 fix: init once before all series
+
+  // S2 fix: warmup run to prime CUDA context and CAF infrastructure
+  std::cout << "--- warmup starting ---\n";
+  run_series(sys, 1000, 10, /*is_warmup=*/true);
+  std::cout << "--- warmup complete ---\n";
+
   for (int i = 1000; i <= 10000; i += 1000) {
     run_series(sys, 1000, i);
   }
+
+  caf::cuda::manager::shutdown();  // S4 fix: shutdown once after all series
 }
 
 CAF_MAIN()
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
index 39f65bb24b..fb31676bbf 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
@@ -18,23 +18,6 @@ using namespace caf;
 using namespace std::chrono_literals;
 
 
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-  
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
-    }
-  }
-}
-
 using command =
   caf::cuda::command_runner<>;
 
@@ -42,79 +25,67 @@ command mmul_command;
 
 struct mmul_state {
 	caf::cuda::program_ptr program;
+	// S3 fix: matrices stored in actor state; messages carry only an int trigger,
+	// eliminating the ~8 MB per-message vector copy (80 GB total for 10 k iters).
+	std::vector<int> matrixA;
+	std::vector<int> matrixB;
+	int N = 0;
+	int completed = 0;
+	int total = 0;
+	bool is_warmup = false;
+	std::chrono::steady_clock::time_point start_time;
 };
 
-//global output buffer meant to disclude it from timing
-//the other benchmark test do not include its memory allocations in it 
-//so its only fair that we do not either 
+// S3 fix: global output buffer excludes its allocation from timing,
+// consistent with the other benchmarks.
 std::vector<int> matrixC;
 
 
 
 
 caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
-		caf::cuda::program_ptr mmul_kernel) {
-  
-
-	self ->state().program = mmul_kernel;
+		caf::cuda::program_ptr mmul_kernel,
+		std::vector<int> matrixA_,
+		std::vector<int> matrixB_,
+		int N_,
+		int total_,
+		bool is_warmup_) {
+
+	auto& st = self->state();
+	st.program = mmul_kernel;
+	st.matrixA = std::move(matrixA_);
+	st.matrixB = std::move(matrixB_);
+	st.N = N_;
+	st.total = total_;
+	st.completed = 0;
+	st.is_warmup = is_warmup_;
 
 return {
 
-    [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB,
-        int N) {
+    // S1 fix: actor receives a trigger (int) instead of full matrix vectors.
+    // It self-quits when all iterations are processed — no external kill needed.
+    [=](int /*trigger*/) {
+      auto& st = self->state();
+
+      // Record start time on the first iteration
+      if (st.completed == 0) {
+        st.start_time = std::chrono::steady_clock::now();
+      }
 
       using clock = std::chrono::steady_clock;
       using ms = std::chrono::duration<double, std::milli>;
 
-
       caf::cuda::manager& mgr = caf::cuda::manager::get();
       int device = 0;
       int stream = 1;
+      const int N = st.N;
 
-           auto t_total_start = clock::now();
-      // -------------------------
-      // create_in_arg A
-      // -------------------------
-      auto t_a_inarg_start = clock::now();
-
-      auto inA = caf::cuda::create_in_arg(std::move(matrixA));
-
-      auto t_a_inarg_end = clock::now();
-
-      // -------------------------
-      // transfer A
-      // -------------------------
-      auto t_a_transfer_start = clock::now();
-
-      auto arg1 = mmul_command.transfer_memory(
-          device,
-          stream,
-          std::move(inA));
-
-      auto t_a_transfer_end = clock::now();
-
-      // -------------------------
-      // create_in_arg B
-      // -------------------------
-      auto t_b_inarg_start = clock::now();
-
-      auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+      auto inA = caf::cuda::create_in_arg(st.matrixA);
+      auto arg1 = mmul_command.transfer_memory(device, stream, std::move(inA));
 
-      auto t_b_inarg_end = clock::now();
+      auto inB = caf::cuda::create_in_arg(st.matrixB);
+      auto arg2 = mmul_command.transfer_memory(device, stream, std::move(inB));
 
-      // -------------------------
-      // transfer B
-      // -------------------------
-      auto t_b_transfer_start = clock::now();
-
-      auto arg2 = mmul_command.transfer_memory(
-          device,
-          stream,
-          std::move(inB));
-
-      auto t_b_transfer_end = clock::now();
-  
       const int THREADS = 32;
       const int BLOCKS = (N + THREADS - 1) / THREADS;
 
@@ -122,80 +93,85 @@ return {
         BLOCKS, BLOCKS, 1,
         THREADS, THREADS, 1);
 
-      caf::cuda::mmul_async_command<int> command;
-      auto output = command.run_async(
-		      self->state().program,
+      caf::cuda::mmul_async_command<int> cmd;
+      auto output = cmd.run_async(
+		      st.program,
 		      dims,
 		      1,
-		      arg1,arg2,out<int>{N*N},in<int>{N});
+		      arg1, arg2, out<int>{N * N}, in<int>{N});
 
 	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);
+	    dC->copy_to_host(matrixC.data(), N * N);
 
+      st.completed++;
 
-	    dC->copy_to_host(matrixC.data(),N*N);
-
-            auto t_copy_end = clock::now();
-            auto t_total_end = clock::now();
+      // S6 fix: milestone reporting every 1000 completions
+      if (!st.is_warmup && st.completed % 1000 == 0) {
+        auto now = clock::now();
+        double elapsed = ms(now - st.start_time).count();
+        std::cout << "[MILESTONE] " << st.completed << " / " << st.total
+                  << " iterations, elapsed = " << elapsed << " ms\n";
+      }
 
+      if (st.completed == st.total) {
+        if (!st.is_warmup) {
+          auto end = clock::now();
+          double total_ms = ms(end - st.start_time).count();
+          // S5 fix: standardised tag matches cuda_native and actor_facade output
+          std::cout << "[SERIES RESULT] Matrix " << N << "x" << N
+                    << ", iterations = " << st.total
+                    << ", total time = " << total_ms << " ms\n";
+        }
+        // S1 fix: graceful self-quit — all prior mailbox messages already processed
+        self->quit();
+      }
     }
 
   };
 }
 
 
-void run_mmul_test(caf::actor_system& sys, int matrix_size,int iterations) {
-
-
-  caf::cuda::manager::init(sys);
-  // ------------------------------------
-  // Start timing
-  // ------------------------------------
-
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<int> matrixA(matrix_size * matrix_size,2);
-  std::vector<int> matrixB(matrix_size * matrix_size,3);
+// S4 fix: program loaded once in caf_main and passed in; manager::init/shutdown
+// called once rather than once-per-series.
+void run_mmul_test(caf::actor_system& sys,
+                   caf::cuda::program_ptr program,
+                   int matrix_size, int iterations,
+                   bool is_warmup = false) {
 
-  matrixC.resize(matrix_size*matrix_size);
- 
-  auto& mgr = caf::cuda::manager::get();
-  
-  auto program =
-        mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
+  std::vector<int> matA(matrix_size * matrix_size, 2);
+  std::vector<int> matB(matrix_size * matrix_size, 3);
+  matrixC.resize(matrix_size * matrix_size);
 
+  // S3 fix: spawn actor with matrices in state; send trigger-only int messages
+  caf::actor a = sys.spawn(mmul_actor_fun, program,
+                            std::move(matA), std::move(matB),
+                            matrix_size, iterations, is_warmup);
 
-  using clock = std::chrono::steady_clock;
+  // S3 fix: int triggers carry no matrix data — no OOM risk regardless of iteration count
+  // S1 fix: actor self-quits on completion; no anon_send_exit(kill) needed
+  for (int i = 0; i < iterations; i++)
+	  anon_mail(i).send(a);
 
-  auto start = std::chrono::steady_clock::now();
-
-  caf::actor a =sys.spawn(mmul_actor_fun,program);
-
-  for (int i = 0; i < iterations; i++) 
-	  anon_mail(matrixA,matrixB,matrix_size).send(a);
-
-  anon_send_exit(a,caf::exit_reason::kill);
-  // Wait for all actors to finish
   sys.await_all_actors_done();
+}
 
-  // ------------------------------------
-  // Stop timing
-  // ------------------------------------
-  auto end = std::chrono::steady_clock::now();
-  auto duration_ms =
-      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
 
-  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
-            << " iterations = " << iterations << 
-	    ", time=" << duration_ms << " ms\n";
+void caf_main(caf::actor_system& sys) {
 
-  caf::cuda::manager::shutdown();
+  caf::cuda::manager::init(sys);  // S4 fix: init once before all series
 
-}
+  auto program = caf::cuda::manager::get()
+                     .create_program_from_cubin("mmul.cubin", "matrixMul");
 
+  // S2 fix: warmup run to prime CUDA context before timed series
+  std::cout << "--- warmup starting ---\n";
+  run_mmul_test(sys, program, 1000, 10, /*is_warmup=*/true);
+  std::cout << "--- warmup complete ---\n";
 
-void caf_main(caf::actor_system& sys) {
+  for (int i = 1000; i < 11000; i += 1000)
+	  run_mmul_test(sys, program, 1000, i);
 
-	for (int i = 1000; i < 11000; i+=1000)
-		run_mmul_test(sys,1000,i);
+  caf::cuda::manager::shutdown();  // S4 fix: shutdown once after all series
 
 }
 

From 743400d558009d7cb66064ea5564c1445f2d6317 Mon Sep 17 00:00:00 2001
From: KyleKlenk <kyle.c.klenk@gmail.com>
Date: Mon, 6 Apr 2026 15:20:00 -0600
Subject: [PATCH 0568/1000] Fix N=16000 in runtime-overhead and add benchmark
 applications

---
 .gitignore                                    |   5 +
 .../sc26/Runtime-Overhead/actor_facade.cpp    |   2 +-
 .../sc26/Runtime-Overhead/command_runner.cpp  |   2 +-
 .../sc26/benchmarks/run_runtime_overhead.py   | 312 +++++++++++++
 .../benchmarks/run_sequence_independent.py    | 429 ++++++++++++++++++
 5 files changed, 748 insertions(+), 2 deletions(-)
 create mode 100755 libcaf_cuda/sc26/benchmarks/run_runtime_overhead.py
 create mode 100755 libcaf_cuda/sc26/benchmarks/run_sequence_independent.py

diff --git a/.gitignore b/.gitignore
index bbca618c21..3be245b338 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,8 @@ libcaf_cuda/sc26/Runtime-Overhead/command_runner
 libcaf_cuda/sc26/Fault-Tolerance/compile.sh
 libcaf_cuda/sc26/Fault-Tolerance/main
 libcaf_cuda/sc26/Runtime-Overhead/actor_facade
+libcaf_cuda/sc26/Fault-Tolerance/monte_carlo.cubin
+libcaf_cuda/sc26/Sequence-Independent-Tasks/main_actor_facade
+libcaf_cuda/sc26/Sequence-Independent-Tasks/main_command_runner
+libcaf_cuda/sc26/Sequence-Independent-Tasks/main_cuda_native
+libcaf_cuda/sc26/Sequence-Independent-Tasks/mmul.cubin
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp b/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
index 51701c9ed3..bbe09c9bdf 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
@@ -83,7 +83,7 @@ void caf_main(caf::actor_system& sys) {
   run_mmul_test(sys, 64);
   std::cout << "--- warmup complete ---\n";
 
-  // F1: unified sizes matching cuda_native: {1000, 2000, 4000, 8000, 12000}
+  // F1: unified sizes matching cuda_native: {1000, 2000, 4000, 8000, 16000}
   run_mmul_test(sys, 1000);
   run_mmul_test(sys, 2000);
   run_mmul_test(sys, 4000);
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
index 4f40e312c2..95879df2b8 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
@@ -218,7 +218,7 @@ void caf_main(caf::actor_system& sys) {
   run_mmul_test(sys, 64);
   std::cout << "--- warmup complete ---\n";
 
-  // F1: unified sizes matching cuda_native: {1000, 2000, 4000, 8000, 12000}
+  // F1: unified sizes matching cuda_native: {1000, 2000, 4000, 8000, 16000}
   run_mmul_test(sys,1000);
   run_mmul_test(sys,2000);
   run_mmul_test(sys,4000);
diff --git a/libcaf_cuda/sc26/benchmarks/run_runtime_overhead.py b/libcaf_cuda/sc26/benchmarks/run_runtime_overhead.py
new file mode 100755
index 0000000000..65a03bafd2
--- /dev/null
+++ b/libcaf_cuda/sc26/benchmarks/run_runtime_overhead.py
@@ -0,0 +1,312 @@
+#!/usr/bin/env python3
+"""
+Runtime-Overhead Benchmark Harness
+====================================
+Runs cuda_native, actor_facade, and command_runner each NUM_RUNS times.
+Each run captures the total time for every matrix size, writes all raw data
+to an output file, then appends mean / min / max / stddev statistics and a
+cross-implementation comparison table at the bottom.
+
+Usage:
+    cd sc26/
+    python3 benchmarks/run_runtime_overhead.py [--runs N] [--output PATH]
+
+Output:
+    Runtime-Overhead/results/benchmark_results.txt  (default)
+"""
+
+import argparse
+import os
+import re
+import subprocess
+import statistics
+import sys
+from collections import defaultdict
+from datetime import datetime
+from pathlib import Path
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+BENCH_DIR = Path(__file__).resolve().parent.parent / "Runtime-Overhead"
+
+BINARIES = {
+    "cuda_native":    BENCH_DIR / "cuda_native",
+    "actor_facade":   BENCH_DIR / "actor_facade",
+    "command_runner": BENCH_DIR / "command_runner",
+}
+
+# Patterns shared across both implementations.
+# cuda_native prints:  N=1000 ... TOTAL: 42.37 ms
+# actor/command print: N=1000 ... TOTAL end-to-end: 42.37 ms
+RE_SIZE  = re.compile(r"N=(\d+)")
+RE_TOTAL_NATIVE = re.compile(r"^TOTAL:\s+([\d.]+)", re.MULTILINE)
+RE_TOTAL_ACTOR  = re.compile(r"^TOTAL end-to-end:\s+([\d.]+)", re.MULTILINE)
+
+# Warmup size — excluded from parsed results
+WARMUP_SIZE = 64
+
+ENV = {**os.environ, "CUDA_VISIBLE_DEVICES": "0"}
+
+# ---------------------------------------------------------------------------
+# Running
+# ---------------------------------------------------------------------------
+
+def run_binary(binary: Path, run_index: int, timeout: int = 600) -> str:
+    """Run a single binary and return its stdout as a string."""
+    print(f"    run {run_index} ...", flush=True)
+    result = subprocess.run(
+        [str(binary)],
+        capture_output=True,
+        text=True,
+        cwd=str(BENCH_DIR),
+        env=ENV,
+        timeout=timeout,
+    )
+    if result.returncode != 0:
+        print(f"      [WARNING] exit code {result.returncode}", file=sys.stderr)
+        if result.stderr:
+            print(result.stderr[:400], file=sys.stderr)
+    return result.stdout
+
+
+# ---------------------------------------------------------------------------
+# Parsing
+# ---------------------------------------------------------------------------
+
+def parse_output(name: str, stdout: str) -> dict[int, float]:
+    """
+    Returns {matrix_size: total_ms} for one run output.
+    Tracks the most-recently seen N= header so each TOTAL is paired correctly.
+    """
+    results: dict[int, float] = {}
+    current_size: int | None = None
+    total_re = RE_TOTAL_NATIVE if name == "cuda_native" else RE_TOTAL_ACTOR
+
+    for line in stdout.splitlines():
+        m_size = RE_SIZE.search(line)
+        if m_size:
+            current_size = int(m_size.group(1))
+        m_total = total_re.search(line)
+        if m_total and current_size is not None:
+            if current_size != WARMUP_SIZE:
+                results[current_size] = float(m_total.group(1))
+            current_size = None  # reset; next hit belongs to the next block
+
+    return results
+
+
+# ---------------------------------------------------------------------------
+# Reporting helpers
+# ---------------------------------------------------------------------------
+
+def header_line(char: str = "=", width: int = 72) -> str:
+    return char * width
+
+
+def section(title: str, width: int = 72) -> str:
+    pad = (width - len(title) - 2) // 2
+    return "=" * pad + f" {title} " + "=" * (width - pad - len(title) - 2)
+
+
+def stats_table(data: dict[int, list[float]]) -> str:
+    """Produce a per-size statistics table from {size: [times]}."""
+    sizes = sorted(data.keys())
+    lines = []
+    col = 12
+    lines.append(
+        f"{'N':>6}  {'Mean (ms)':>{col}}  {'Min (ms)':>{col}}  "
+        f"{'Max (ms)':>{col}}  {'StdDev (ms)':>{col}}  {'Runs':>5}"
+    )
+    lines.append("-" * 66)
+    for s in sizes:
+        vals = data[s]
+        if not vals:
+            continue
+        mean   = statistics.mean(vals)
+        lo     = min(vals)
+        hi     = max(vals)
+        stddev = statistics.stdev(vals) if len(vals) > 1 else 0.0
+        lines.append(
+            f"{s:>6}  {mean:>{col}.3f}  {lo:>{col}.3f}  "
+            f"{hi:>{col}.3f}  {stddev:>{col}.3f}  {len(vals):>5}"
+        )
+    return "\n".join(lines)
+
+
+def comparison_table(all_data: dict[str, dict[int, list[float]]]) -> str:
+    """
+    Cross-implementation comparison using mean times.
+    Overhead columns show actor overhead vs cuda_native baseline.
+    """
+    names  = list(all_data.keys())
+    # Collect sizes present in ALL implementations
+    size_sets = [set(all_data[n].keys()) for n in names]
+    sizes = sorted(size_sets[0].intersection(*size_sets[1:]))
+    if not sizes:
+        return "(no common matrix sizes found across all implementations)"
+
+    native_means: dict[int, float] = {}
+    impl_means:   dict[str, dict[int, float]] = {n: {} for n in names}
+    for s in sizes:
+        for n in names:
+            vals = all_data[n].get(s, [])
+            if vals:
+                impl_means[n][s] = statistics.mean(vals)
+        if "cuda_native" in impl_means and s in impl_means["cuda_native"]:
+            native_means[s] = impl_means["cuda_native"][s]
+
+    lines = []
+    col = 14
+    header = f"{'N':>6}"
+    for n in names:
+        header += f"  {n[:col]:>{col}}"
+    if "cuda_native" in names:
+        others = [n for n in names if n != "cuda_native"]
+        for n in others:
+            header += f"  {'vs_native_%':>12}"
+    lines.append(header)
+    lines.append("-" * (8 + col * len(names) + 14 * max(0, len(names) - 1)))
+
+    for s in sizes:
+        row = f"{s:>6}"
+        for n in names:
+            val = impl_means[n].get(s, float("nan"))
+            row += f"  {val:>{col}.3f}"
+        if "cuda_native" in names:
+            baseline = native_means.get(s)
+            others = [n for n in names if n != "cuda_native"]
+            for n in others:
+                val = impl_means[n].get(s)
+                if val is not None and baseline and baseline > 0:
+                    pct = (val - baseline) / baseline * 100
+                    sign = "+" if pct >= 0 else ""
+                    row += f"  {sign}{pct:>11.2f}%"
+                else:
+                    row += f"  {'N/A':>12}"
+        lines.append(row)
+
+    return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Runtime-Overhead benchmark harness")
+    parser.add_argument("--runs",   type=int,  default=10, help="Number of times to run each binary (default: 10)")
+    parser.add_argument("--output", type=str,  default=str(BENCH_DIR / "results" / "benchmark_results.txt"),
+                        help="Path to the output file")
+    args = parser.parse_args()
+
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    num_runs = args.runs
+
+    # {impl_name: {matrix_size: [time_ms, ...]}}
+    all_data: dict[str, dict[int, list[float]]] = {}
+    # {impl_name: list of raw stdout strings (one per run)}
+    raw_outputs: dict[str, list[str]] = {}
+
+    for name, binary in BINARIES.items():
+        print(f"\n[{name}]", flush=True)
+        if not binary.exists():
+            print(f"  SKIP — binary not found: {binary}", file=sys.stderr)
+            continue
+
+        all_data[name]   = defaultdict(list)
+        raw_outputs[name] = []
+
+        for i in range(1, num_runs + 1):
+            stdout = run_binary(binary, i)
+            raw_outputs[name].append(stdout)
+            parsed = parse_output(name, stdout)
+            for size, t in parsed.items():
+                all_data[name][size].append(t)
+
+    # ------------------------------------------------------------------
+    # Write output file
+    # ------------------------------------------------------------------
+    lines: list[str] = []
+
+    lines.append(header_line())
+    lines.append(section("RUNTIME OVERHEAD BENCHMARK"))
+    lines.append(header_line())
+    lines.append(f"Date       : {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    lines.append(f"Runs       : {num_runs}")
+    lines.append(f"GPU        : CUDA_VISIBLE_DEVICES=0")
+    lines.append(f"Directory  : {BENCH_DIR}")
+    lines.append("")
+
+    for name in BINARIES:
+        if name not in all_data:
+            continue
+
+        lines.append(header_line("-"))
+        lines.append(f"  Implementation: {name}")
+        lines.append(header_line("-"))
+
+        lines.append("\n--- Raw Run Data ---")
+        sizes = sorted(all_data[name].keys())
+        # Per-run table
+        col = 14
+        header = f"{'Run':>5}"
+        for s in sizes:
+            header += f"  {'N='+str(s):>{col}}"
+        lines.append(header)
+        lines.append("-" * (7 + col * len(sizes)))
+
+        for i, run_vals in enumerate(zip(*[all_data[name][s] for s in sizes]), start=1):
+            row = f"{i:>5}"
+            for val in run_vals:
+                row += f"  {val:>{col}.3f}"
+            lines.append(row)
+
+        lines.append("")
+
+        # Also include any sizes that might not appear in all runs
+        # (some runs may have failed mid-way)
+        uneven = {s for s in sizes if len(all_data[name][s]) != num_runs}
+        if uneven:
+            lines.append(
+                f"  [NOTE] The following sizes had fewer than {num_runs} successful "
+                f"data points: {sorted(uneven)}"
+            )
+            lines.append("")
+
+        lines.append("--- Statistics ---")
+        lines.append(stats_table(all_data[name]))
+        lines.append("")
+
+    # Cross-implementation comparison (only if we have at least 2 implementations)
+    if len(all_data) >= 2:
+        lines.append(header_line())
+        lines.append(section("CROSS-IMPLEMENTATION COMPARISON (mean times, ms)"))
+        lines.append(header_line())
+        lines.append(comparison_table(all_data))
+        lines.append("")
+        lines.append(
+            "vs_native_%: positive = actor implementation is slower than cuda_native"
+        )
+        lines.append("")
+
+    # Full raw stdout dumps
+    lines.append(header_line())
+    lines.append(section("FULL RAW OUTPUT (all runs)"))
+    lines.append(header_line())
+    for name in BINARIES:
+        if name not in raw_outputs:
+            continue
+        lines.append(f"\n{'='*10} {name} {'='*10}")
+        for i, stdout in enumerate(raw_outputs[name], start=1):
+            lines.append(f"\n--- {name} | run {i} ---")
+            lines.append(stdout.strip())
+
+    output_path.write_text("\n".join(lines) + "\n")
+    print(f"\nResults written to: {output_path}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/libcaf_cuda/sc26/benchmarks/run_sequence_independent.py b/libcaf_cuda/sc26/benchmarks/run_sequence_independent.py
new file mode 100755
index 0000000000..8fb32142af
--- /dev/null
+++ b/libcaf_cuda/sc26/benchmarks/run_sequence_independent.py
@@ -0,0 +1,429 @@
+#!/usr/bin/env python3
+"""
+Sequence-Independent-Tasks Benchmark Harness
+=============================================
+Runs main_cuda_native, main_actor_facade, and main_command_runner each
+NUM_RUNS times.  For each run the script captures the [MILESTONE] lines
+emitted every 1 000 iterations and records them.  The output file contains:
+
+  1. All raw [MILESTONE] and [SERIES RESULT] lines from every run.
+  2. Per-series statistics (mean / min / max / stddev of the cumulative
+     elapsed time at each 1 000-iteration milestone across all runs).
+  3. Incremental time per 1 000-iteration step (derived from adjacent
+     milestones) — this shows how long each individual 1 000-iteration
+     "slice" took on average.
+  4. A cross-implementation comparison table.
+
+Usage:
+    cd sc26/
+    python3 benchmarks/run_sequence_independent.py [--runs N] [--output PATH]
+
+Output:
+    Sequence-Independent-Tasks/results/benchmark_results.txt  (default)
+"""
+
+import argparse
+import os
+import re
+import subprocess
+import statistics
+import sys
+from collections import defaultdict
+from datetime import datetime
+from pathlib import Path
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+BENCH_DIR = Path(__file__).resolve().parent.parent / "Sequence-Independent-Tasks"
+
+BINARIES = {
+    "main_cuda_native":    BENCH_DIR / "main_cuda_native",
+    "main_actor_facade":   BENCH_DIR / "main_actor_facade",
+    "main_command_runner": BENCH_DIR / "main_command_runner",
+}
+
+# [MILESTONE] 3000 / 10000 iterations, elapsed = 1564.72 ms
+RE_MILESTONE = re.compile(
+    r"\[MILESTONE\]\s+(\d+)\s*/\s*(\d+)\s+iterations,\s+elapsed\s*=\s*([\d.]+)"
+)
+# [SERIES RESULT] Matrix 1000x1000, iterations = 5000, total ... time = 2601.3 ms
+RE_SERIES = re.compile(
+    r"\[SERIES RESULT\].*iterations\s*=\s*(\d+).*?=\s*([\d.]+)\s*ms"
+)
+
+ENV = {**os.environ, "CUDA_VISIBLE_DEVICES": "0"}
+
+# ---------------------------------------------------------------------------
+# Running
+# ---------------------------------------------------------------------------
+
+def run_binary(binary: Path, run_index: int, timeout: int = 3600) -> str:
+    """Run a single binary and return its stdout."""
+    print(f"    run {run_index} ...", end=" ", flush=True)
+    result = subprocess.run(
+        [str(binary)],
+        capture_output=True,
+        text=True,
+        cwd=str(BENCH_DIR),
+        env=ENV,
+        timeout=timeout,
+    )
+    if result.returncode != 0:
+        print(f"exit={result.returncode}", flush=True)
+        if result.stderr:
+            print(result.stderr[:400], file=sys.stderr)
+    else:
+        print("ok", flush=True)
+    return result.stdout
+
+
+# ---------------------------------------------------------------------------
+# Parsing
+# ---------------------------------------------------------------------------
+
+# Data structure produced by parse_output:
+#   milestones[series_total][milestone_iter] = elapsed_ms
+#   series[series_total] = total_ms   (from [SERIES RESULT] line; may be absent)
+RunData = tuple[dict[int, dict[int, float]], dict[int, float]]
+
+
+def parse_output(stdout: str) -> RunData:
+    """
+    Parse one run's stdout.
+    Returns:
+        milestones: {series_total: {milestone_iter: elapsed_ms}}
+        series_totals: {series_total: elapsed_ms}  (from [SERIES RESULT] lines)
+    The 'series_total' key is the total iterations for that series (e.g. 3000).
+    """
+    milestones: dict[int, dict[int, float]] = {}
+    series_totals: dict[int, float] = {}
+
+    # As we scan lines we need to track which series we are in.
+    # A milestone line carries both completed and total, so we use
+    # the 'total' field as the series key directly.
+    for line in stdout.splitlines():
+        m = RE_MILESTONE.search(line)
+        if m:
+            completed   = int(m.group(1))
+            series_total = int(m.group(2))
+            elapsed_ms   = float(m.group(3))
+            milestones.setdefault(series_total, {})[completed] = elapsed_ms
+
+        s = RE_SERIES.search(line)
+        if s:
+            total_iters = int(s.group(1))
+            total_ms    = float(s.group(2))
+            series_totals[total_iters] = total_ms
+
+    return milestones, series_totals
+
+
+# ---------------------------------------------------------------------------
+# Statistics helpers
+# ---------------------------------------------------------------------------
+
+def stat_row(vals: list[float], width: int = 12) -> str:
+    if not vals:
+        return f"{'N/A':>{width}}  {'N/A':>{width}}  {'N/A':>{width}}  {'N/A':>{width}}"
+    mean   = statistics.mean(vals)
+    lo     = min(vals)
+    hi     = max(vals)
+    stddev = statistics.stdev(vals) if len(vals) > 1 else 0.0
+    return (f"{mean:>{width}.2f}  {lo:>{width}.2f}  "
+            f"{hi:>{width}.2f}  {stddev:>{width}.2f}")
+
+
+def header_line(char: str = "=", width: int = 80) -> str:
+    return char * width
+
+
+def section(title: str, width: int = 80) -> str:
+    pad = (width - len(title) - 2) // 2
+    return char_pad(char="=", n=pad) + f" {title} " + char_pad(char="=", n=width - pad - len(title) - 2)
+
+
+def char_pad(char: str, n: int) -> str:
+    return char * max(0, n)
+
+
+# ---------------------------------------------------------------------------
+# Per-implementation report builder
+# ---------------------------------------------------------------------------
+
+def build_impl_report(name: str, all_runs: list[RunData], num_runs: int) -> list[str]:
+    """Return lines for one implementation's section of the output file."""
+    lines: list[str] = []
+    lines.append(header_line("-"))
+    lines.append(f"  Implementation: {name}")
+    lines.append(header_line("-"))
+
+    # Collect all series counts seen
+    all_series: set[int] = set()
+    for milestones, series_totals in all_runs:
+        all_series.update(milestones.keys())
+        all_series.update(series_totals.keys())
+    sorted_series = sorted(all_series)
+
+    # ----------------------------------------------------------------
+    # 1. Raw run data  (one mini-table per series)
+    # ----------------------------------------------------------------
+    lines.append("\n--- Raw Milestone Data (cumulative elapsed ms) ---")
+
+    for series_total in sorted_series:
+        # Collect all milestone checkpoints seen for this series
+        all_checkpoints: set[int] = set()
+        for milestones, _ in all_runs:
+            if series_total in milestones:
+                all_checkpoints.update(milestones[series_total].keys())
+        checkpoints = sorted(all_checkpoints)
+
+        if not checkpoints:
+            continue
+
+        col = 12
+        lines.append(f"\n  Series: {series_total} iterations")
+        header = f"    {'Run':>5}"
+        for cp in checkpoints:
+            header += f"  {'@'+str(cp):>{col}}"
+        header += f"  {'SERIES_END':>{col}}"
+        lines.append(header)
+        lines.append("    " + "-" * (7 + col * (len(checkpoints) + 1)))
+
+        for i, (milestones, series_totals) in enumerate(all_runs, start=1):
+            row = f"    {i:>5}"
+            for cp in checkpoints:
+                val = milestones.get(series_total, {}).get(cp)
+                row += f"  {val:>{col}.2f}" if val is not None else f"  {'N/A':>{col}}"
+            # Series-end total from [SERIES RESULT] line (may differ slightly from
+            # last milestone if the last milestone IS the series-end)
+            end_val = series_totals.get(series_total)
+            row += f"  {end_val:>{col}.2f}" if end_val is not None else f"  {'N/A':>{col}}"
+            lines.append(row)
+
+    # ----------------------------------------------------------------
+    # 2. Statistics per series per milestone checkpoint
+    # ----------------------------------------------------------------
+    lines.append("\n--- Statistics per Milestone (cumulative elapsed ms) ---")
+    lines.append(
+        f"  {'Series':>8}  {'Milestone':>10}  "
+        f"{'Mean':>12}  {'Min':>12}  {'Max':>12}  {'StdDev':>12}  {'n':>4}"
+    )
+    lines.append("  " + "-" * 72)
+
+    for series_total in sorted_series:
+        all_checkpoints = set()
+        for milestones, _ in all_runs:
+            if series_total in milestones:
+                all_checkpoints.update(milestones[series_total].keys())
+        checkpoints = sorted(all_checkpoints)
+
+        for cp in checkpoints:
+            vals = [
+                milestones.get(series_total, {}).get(cp)
+                for milestones, _ in all_runs
+                if milestones.get(series_total, {}).get(cp) is not None
+            ]
+            n = len(vals)
+            lines.append(
+                f"  {series_total:>8}  {cp:>10}  {stat_row(vals)}  {n:>4}"
+            )
+
+    # ----------------------------------------------------------------
+    # 3. Average time per 1000-iteration increment  (incremental, not cumulative)
+    # ----------------------------------------------------------------
+    lines.append("\n--- Average Incremental Time per 1000 Iterations ---")
+    lines.append(
+        "  (derived from consecutive milestone differences; shows per-slice throughput)"
+    )
+    lines.append(
+        f"  {'Series':>8}  {'Increment':>14}  "
+        f"{'Mean (ms)':>12}  {'Min':>10}  {'Max':>10}  {'StdDev':>10}"
+    )
+    lines.append("  " + "-" * 72)
+
+    for series_total in sorted_series:
+        all_checkpoints = set()
+        for milestones, _ in all_runs:
+            if series_total in milestones:
+                all_checkpoints.update(milestones[series_total].keys())
+        checkpoints = sorted(all_checkpoints)
+        if not checkpoints:
+            continue
+
+        # Build increments: [checkpoint[0] - 0, cp[1] - cp[0], ...]
+        prev_cps = [0] + checkpoints[:-1]
+        for prev_cp, cp in zip(prev_cps, checkpoints):
+            incremental_vals = []
+            for milestones, _ in all_runs:
+                m = milestones.get(series_total, {})
+                curr = m.get(cp)
+                prev = m.get(prev_cp) if prev_cp != 0 else 0.0
+                if curr is not None and prev is not None:
+                    incremental_vals.append(curr - prev)
+
+            label = f"{prev_cp+1}-{cp}"
+            lines.append(
+                f"  {series_total:>8}  {label:>14}  {stat_row(incremental_vals, width=10)}"
+            )
+
+    lines.append("")
+    return lines
+
+
+# ---------------------------------------------------------------------------
+# Cross-implementation comparison
+# ---------------------------------------------------------------------------
+
+def build_comparison(all_impl_data: dict[str, list[RunData]]) -> list[str]:
+    """Compare mean elapsed time at each common milestone across implementations."""
+    lines: list[str] = []
+
+    # Find common (series_total, checkpoint) pairs
+    impl_means: dict[str, dict[tuple[int, int], float]] = {}
+
+    for name, runs in all_impl_data.items():
+        impl_means[name] = {}
+        by_key: dict[tuple[int, int], list[float]] = defaultdict(list)
+        for milestones, _ in runs:
+            for series_total, checkpoints in milestones.items():
+                for cp, elapsed in checkpoints.items():
+                    by_key[(series_total, cp)].append(elapsed)
+        for key, vals in by_key.items():
+            impl_means[name][key] = statistics.mean(vals)
+
+    # Intersect all keys
+    if not impl_means:
+        return ["(no data)"]
+
+    all_keys = set.intersection(*[set(d.keys()) for d in impl_means.values()])
+    if not all_keys:
+        return ["(no common milestone keys across implementations)"]
+
+    sorted_keys = sorted(all_keys)
+    names = list(all_impl_data.keys())
+    col = 16
+
+    header = f"  {'series':>8}  {'milestone':>10}"
+    for n in names:
+        header += f"  {n[:col]:>{col}}"
+    # Overhead vs first implementation
+    if len(names) > 1:
+        base = names[0]
+        for n in names[1:]:
+            header += f"  {'vs_'+base[:8]+' %':>14}"
+    lines.append(header)
+    lines.append("  " + "-" * (22 + col * len(names) + 16 * max(0, len(names) - 1)))
+
+    for series_total, cp in sorted_keys:
+        row = f"  {series_total:>8}  {cp:>10}"
+        base_val: float | None = None
+        for i, n in enumerate(names):
+            val = impl_means[n].get((series_total, cp))
+            row += f"  {val:>{col}.2f}" if val is not None else f"  {'N/A':>{col}}"
+            if i == 0:
+                base_val = val
+        if len(names) > 1 and base_val and base_val > 0:
+            for n in names[1:]:
+                val = impl_means[n].get((series_total, cp))
+                if val is not None:
+                    pct = (val - base_val) / base_val * 100
+                    sign = "+" if pct >= 0 else ""
+                    row += f"  {sign}{pct:>13.2f}%"
+                else:
+                    row += f"  {'N/A':>14}"
+        lines.append(row)
+
+    return lines
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Sequence-Independent-Tasks benchmark harness"
+    )
+    parser.add_argument("--runs",   type=int,  default=10,
+                        help="Number of times to run each binary (default: 10)")
+    parser.add_argument("--output", type=str,
+                        default=str(BENCH_DIR / "results" / "benchmark_results.txt"),
+                        help="Path to the output file")
+    args = parser.parse_args()
+
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    num_runs = args.runs
+
+    # {impl_name: [RunData, ...]}
+    all_impl_data: dict[str, list[RunData]] = {}
+    raw_outputs:   dict[str, list[str]]     = {}
+
+    for name, binary in BINARIES.items():
+        print(f"\n[{name}]", flush=True)
+        if not binary.exists():
+            print(f"  SKIP — binary not found: {binary}", file=sys.stderr)
+            continue
+
+        all_impl_data[name] = []
+        raw_outputs[name]   = []
+
+        for i in range(1, num_runs + 1):
+            stdout = run_binary(binary, i)
+            raw_outputs[name].append(stdout)
+            all_impl_data[name].append(parse_output(stdout))
+
+    # ------------------------------------------------------------------
+    # Build output file
+    # ------------------------------------------------------------------
+    lines: list[str] = []
+
+    lines.append(header_line())
+    lines.append(section("SEQUENCE-INDEPENDENT TASKS BENCHMARK"))
+    lines.append(header_line())
+    lines.append(f"Date       : {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    lines.append(f"Runs       : {num_runs}")
+    lines.append(f"GPU        : CUDA_VISIBLE_DEVICES=0")
+    lines.append(f"Directory  : {BENCH_DIR}")
+    lines.append(
+        "Milestones : cumulative elapsed time (ms) since series start, "
+        "printed every 1 000 iterations by the binary"
+    )
+    lines.append("")
+
+    for name in BINARIES:
+        if name not in all_impl_data:
+            continue
+        lines.extend(build_impl_report(name, all_impl_data[name], num_runs))
+
+    if len(all_impl_data) >= 2:
+        lines.append(header_line())
+        lines.append(section("CROSS-IMPLEMENTATION COMPARISON (mean cumulative elapsed ms)"))
+        lines.append(header_line())
+        lines.append(
+            "  Positive % = the implementation is slower than the baseline (first column)"
+        )
+        lines.append("")
+        lines.extend(build_comparison(all_impl_data))
+        lines.append("")
+
+    # Full raw stdout dumps
+    lines.append(header_line())
+    lines.append(section("FULL RAW OUTPUT (all runs)"))
+    lines.append(header_line())
+    for name in BINARIES:
+        if name not in raw_outputs:
+            continue
+        lines.append(f"\n{'='*10} {name} {'='*10}")
+        for i, stdout in enumerate(raw_outputs[name], start=1):
+            lines.append(f"\n--- {name} | run {i} ---")
+            lines.append(stdout.strip())
+
+    output_path.write_text("\n".join(lines) + "\n")
+    print(f"\nResults written to: {output_path}")
+
+
+if __name__ == "__main__":
+    main()

From 54506e5b65494bea1ec5faad76abdc0b7a607a57 Mon Sep 17 00:00:00 2001
From: KyleKlenk <kyle.c.klenk@gmail.com>
Date: Tue, 7 Apr 2026 09:54:16 -0600
Subject: [PATCH 0569/1000] Edits to fault tolerance test

---
 .gitignore                                    |   9 +
 libcaf_cuda/sc26/Fault-Tolerance/main.cpp     | 443 ++++++----
 libcaf_cuda/sc26/Runtime-Overhead/Makefile    |   2 +-
 .../sc26/Runtime-Overhead/actor_facade.cpp    |   9 +-
 .../sc26/Runtime-Overhead/command_runner.cpp  |   9 +-
 .../actor_facade.cpp                          |   9 +-
 .../command_runner.cpp                        |   7 +-
 .../cuda_native.cpp                           |  93 ++-
 .../benchmarks/run_sequence_independent.py    |  15 +-
 .../sc26/scripts/Fault-Tolerance/plot.py      | 781 ++++++++++++++++++
 10 files changed, 1169 insertions(+), 208 deletions(-)
 create mode 100644 libcaf_cuda/sc26/scripts/Fault-Tolerance/plot.py

diff --git a/.gitignore b/.gitignore
index 3be245b338..4b19f428a5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,3 +24,12 @@ libcaf_cuda/sc26/Sequence-Independent-Tasks/main_actor_facade
 libcaf_cuda/sc26/Sequence-Independent-Tasks/main_command_runner
 libcaf_cuda/sc26/Sequence-Independent-Tasks/main_cuda_native
 libcaf_cuda/sc26/Sequence-Independent-Tasks/mmul.cubin
+libcaf_cuda/sc26/Fault-Tolerance/run.log
+libcaf_cuda/sc26/Fault-Tolerance/baseline.log
+libcaf_cuda/sc26/benchmarks/__pycache__/
+libcaf_cuda/sc26/scripts/Fault-Tolerance/.venv/
+libcaf_cuda/sc26/scripts/Fault-Tolerance/plots
+libcaf_cuda/sc26/Runtime-Overhead/results/benchmark_results.txt
+libcaf_cuda/sc26/Runtime-Overhead/results/benchmark_results_1.txt
+libcaf_cuda/sc26/Sequence-Independent-Tasks/results/cleaned.txt
+libcaf_cuda/sc26/Sequence-Independent-Tasks/results/benchmark_results.txt
diff --git a/libcaf_cuda/sc26/Fault-Tolerance/main.cpp b/libcaf_cuda/sc26/Fault-Tolerance/main.cpp
index f254538b98..dc0f724958 100644
--- a/libcaf_cuda/sc26/Fault-Tolerance/main.cpp
+++ b/libcaf_cuda/sc26/Fault-Tolerance/main.cpp
@@ -1,88 +1,98 @@
-// example_14: Fault-tolerant Monte Carlo π estimation with GPU workers.
+// Fault-Tolerant Monte Carlo π estimation with GPU workers.
+// SC26-hardened — implements all Opus review recommendations:
 //
-// Demonstrates:
-//  1. Multiple GPU worker actors each owning a separate CUDA stream, running
-//     Monte Carlo batches concurrently on the same physical device.
-//  2. A supervisor actor that distributes batches, monitors workers, and
-//     recovers automatically when a worker is killed mid-computation.
-//  3. Fault injection: halfway through the run the supervisor force-kills
-//     Worker 0, then detects the down_msg via the monitor callback, reschedules
-//     the lost batch to a surviving worker, and respawns a replacement.
-//  4. Final π estimate converges to the correct answer despite the disruption.
+//  1. Parameterized fault count (--num-faults) with evenly-spaced injection
+//     across different worker slots, proving recovery works at arbitrary points
+//     and repeatedly.
+//  2. Scalable defaults: 4 workers, 200 batches, 50M samples/batch.
+//  3. Per-recovery latency timing:
+//       t_fault_detected  (monitor callback fires)
+//       t_recovery_dispatched (recovery batch sent to new worker)
+//       t_recovery_completed  (recovery batch reply received)
+//  4. No-fault baseline mode (--no-fault) for direct throughput comparison.
+//  5. Structured output lines for benchmark script parsing:
+//       [FAULT_DETECTED]  [FAULT_DISPATCH]  [FAULT_RECOVERED]
+//       [PROGRESS]        [RESULT]
 //
-// Architecture:
-//   main → Supervisor : start_atom
-//   Supervisor spawns N WorkerActors, monitors each with callback.
-//   Supervisor → Worker[k] : (seed, num_samples)   [request().then()]
-//   Worker[k] → GPU via run_async_notify() on its private stream.
-//   GPU → Worker[k] : gpu_done_atom  (from cuLaunchHostFunc on CUDA thread)
-//   Worker[k] → Supervisor : int M   (number of in-circle hits)
-//   [halfway] Supervisor kills Worker[0] → monitor callback fires immediately
-//   Supervisor reschedules in-flight batch, respawns Worker[0], continues.
-//
-// Note on single-GPU machines: all workers share one physical device but each
-// gets a distinct CUDA stream (keyed by actor_id inside get_stream_for_actor).
-// Stream-level concurrency is real; for multi-GPU, pin worker N to device N
-// by using the find_device(N) overload of create_program_from_cubin.
+// Architecture: main → Supervisor → WorkerActor → command_runner → GPU.
+// Workers share device 0 via distinct CUDA streams (one per worker index).
 
 #include <caf/all.hpp>
 #include <caf/actor_cast.hpp>
 #include <caf/cuda/all.hpp>
 
+#include <algorithm>
+#include <chrono>
+#include <cmath>
 #include <deque>
 #include <map>
 #include <vector>
 #include <iostream>
-#include <cmath>
-#include <chrono>
 
 using namespace caf;
 using namespace caf::cuda;
 
+using sc = std::chrono::steady_clock;
+using tp = std::chrono::time_point<sc>;
+
+static double ms_between(tp a, tp b) {
+    return std::chrono::duration<double, std::milli>(b - a).count();
+}
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Atoms
 // ─────────────────────────────────────────────────────────────────────────────
 CAF_BEGIN_TYPE_ID_BLOCK(monte_carlo_app, caf::id_block::cuda::end)
     CAF_ADD_ATOM(monte_carlo_app, start_atom)
-    CAF_ADD_ATOM(monte_carlo_app, kill_atom)
     CAF_ADD_ATOM(monte_carlo_app, done_atom)
 CAF_END_TYPE_ID_BLOCK(monte_carlo_app)
 
 // ─────────────────────────────────────────────────────────────────────────────
-// Work batch description
+// WorkBatch: one unit of Monte Carlo work.
+//   fault_index >= 0 marks a re-dispatched recovery batch and links it to the
+//   corresponding FaultRecord so latency timestamps can be filled in.
 // ─────────────────────────────────────────────────────────────────────────────
 struct WorkBatch {
     int batch_id;
-    int seed;         // deterministic: batch_id * 1000003
-    int num_samples;  // constant per run
+    int seed;           // deterministic: batch_id * 1000003
+    int num_samples;
+    int fault_index = -1; // -1 = normal; >= 0 = recovery for fault_records_[i]
+};
+
+// ─────────────────────────────────────────────────────────────────────────────
+// FaultRecord: per-fault timing and recovery metadata
+// ─────────────────────────────────────────────────────────────────────────────
+struct FaultRecord {
+    int  fault_index;
+    int  worker_slot;
+    int  batch_id    = -1;   // batch that was in-flight when the fault fired
+    tp   t_detected;          // when monitor callback fired
+    tp   t_dispatched;        // when the recovery batch was re-sent
+    tp   t_completed;         // when the recovery batch reply arrived
+    bool dispatched  = false;
+    bool completed   = false;
 };
 
 // ─────────────────────────────────────────────────────────────────────────────
-// command_runner type alias for monteCarloKernel
-//   arg 0: in<int>  seed
-//   arg 1: in<int>  num_samples
-//   arg 2: out<int> hit_count
+// mc_runner type alias for monteCarloKernel
+//   arg 0: in<int>      seed
+//   arg 1: in<int>      num_samples
+//   arg 2: in_out<int>  hit_count — zero-inited before each launch so the
+//                                    atomicAdd accumulation starts clean.
 // ─────────────────────────────────────────────────────────────────────────────
-// Use in_out<int> (not out<int>) for the hit counter so the framework
-// copies value 0 to device memory before each kernel launch.  out<int> uses
-// cuMemAlloc without zero-init, which would corrupt the atomicAdd result.
-// Use in_out<int> for the hit counter so 0 is uploaded to device before
-// each launch — out<int> (scratch_argument) uses bare cuMemAlloc with no
-// zero-init, which would corrupt the atomicAdd result.
 using mc_runner = command_runner<in<int>, in<int>, in_out<int>>;
 
 // ─────────────────────────────────────────────────────────────────────────────
 // WorkerActor
 //
 // Owns one mc_runner (and therefore one private CUDA stream).
-// Handles a single kernel request at a time:
-//   receive (seed, num_samples) → launch GPU → reply int M to sender
+// Handles one kernel request at a time:
+//   receive (seed, num_samples) → launch GPU → reply int M (in-circle hits)
 // ─────────────────────────────────────────────────────────────────────────────
 class WorkerActor {
     event_based_actor* self_;
-    int worker_index_;  // for logging
-
-    mc_runner runner_;
+    int                worker_index_;
+    mc_runner          runner_;
 
 public:
     WorkerActor(event_based_actor* self, int index)
@@ -90,12 +100,8 @@ class WorkerActor {
 
     behavior make_behavior() {
         return {
-            // ── Kernel launch request from the supervisor ─────────────────
             [this](int seed, int num_samples) -> int {
-
                 auto& mgr = caf::cuda::manager::get();
-                // All workers use device 0 on a single-GPU machine.
-                // On multi-GPU hardware, replace 0 with worker_index_.
                 auto program = mgr.create_program_from_cubin(
                     "monte_carlo.cubin", "monteCarloKernel");
 
@@ -103,32 +109,19 @@ class WorkerActor {
 
                 auto arg_seed    = create_in_arg(seed);
                 auto arg_samples = create_in_arg(num_samples);
-                // Zero-init the hit counter (in_out so value 0 is copied to
-                // device before the kernel runs; out<int> is NOT zero-inited).
+                // Zero-init the hit counter (in_out copies 0 to device first).
                 const std::vector<int> zero_buf{0};
                 auto arg_out = create_in_out_arg(zero_buf);
 
-                self_->println("[Worker {:2d}] Launched GPU kernel: "
-                               "seed={}, samples={}", worker_index_, seed, num_samples);
-
-                // For command_runner: actor_id = worker_index_ + 1 to keep streams distinct
+                // stream key = worker_index_ + 1 to keep streams distinct per slot.
                 auto output = runner_.run_async(
                     program, dims,
                     worker_index_ + 1, /*shared_memory=*/0, /*device_number=*/0,
                     arg_seed, arg_samples, arg_out);
 
-                // The in_out<int> result is the third (last) element.
                 auto result_ptr = std::get<2>(output);
-
-                // Block thread to wait for GPU and copy result to host.
                 std::vector<int> host = result_ptr->copy_to_host();
-                int M = host.empty() ? 0 : host[0];
-
-                self_->println("[Worker {:2d}] GPU complete: M={}", worker_index_, M);
-
-                // Safe to release device memory now implicitly as result_ptr goes out of scope.
-                
-                return M;
+                return host.empty() ? 0 : host[0];
             }
         };
     }
@@ -137,91 +130,101 @@ class WorkerActor {
 // ─────────────────────────────────────────────────────────────────────────────
 // Supervisor
 //
-// Manages the work queue, dispatches batches to workers, handles fault
-// injection at mid-run, recovers lost batches, and accumulates the final
-// π estimate.
+// Distributes batches across workers, tracks in-flight work, injects faults
+// at evenly-spaced completion milestones (cycling through worker slots), and
+// measures per-fault recovery latency from detection through completion.
 // ─────────────────────────────────────────────────────────────────────────────
 class Supervisor {
     event_based_actor* self_;
 
-    // Configuration
-    int num_workers_;
-    int total_batches_;
-    int samples_per_batch_;
+    // ── configuration ────────────────────────────────────────────────────────
+    int  num_workers_;
+    int  total_batches_;
+    int  samples_per_batch_;
+    int  num_faults_;
+    bool no_fault_;
 
-    // Worker handles (index-stable; replaced on respawn)
+    // ── worker handles (index-stable; replaced on respawn) ───────────────────
     std::vector<actor> workers_;
 
-    // Pending work
-    std::deque<WorkBatch> work_queue_;
+    // ── work tracking ─────────────────────────────────────────────────────────
+    std::deque<WorkBatch>            work_queue_;
+    std::map<actor_addr, WorkBatch>  in_flight_;  // addr → batch in that worker
 
-    // In-flight tracking: actor address → batch currently running in that worker
-    std::map<actor_addr, WorkBatch> in_flight_;
+    // ── accumulation ──────────────────────────────────────────────────────────
+    long long total_hits_   = 0;
+    int       batches_done_ = 0;
 
-    // Accumulation
-    long long total_hits_     = 0;
-    long long total_expected_ = 0; // total samples we PLAN to complete
-    int       batches_done_   = 0;
+    // ── fault injection ───────────────────────────────────────────────────────
+    int              faults_injected_ = 0;
+    std::vector<int> fault_triggers_; // batches_done_ values at which to fire each fault
+    std::vector<FaultRecord> fault_records_;
 
-    // Fault control
-    bool fault_injected_ = false;
+    // ── timing ────────────────────────────────────────────────────────────────
+    tp t_start_;
 
-    // The scoped_actor that main() uses to wait for us.
+    // ── parent (main's scoped_actor) ──────────────────────────────────────────
     actor parent_;
 
 public:
     Supervisor(event_based_actor* self,
                int num_workers, int total_batches, int samples_per_batch,
+               int num_faults, bool no_fault,
                actor parent)
         : self_(self)
         , num_workers_(num_workers)
         , total_batches_(total_batches)
         , samples_per_batch_(samples_per_batch)
+        , num_faults_(no_fault ? 0 : num_faults)
+        , no_fault_(no_fault)
         , parent_(std::move(parent))
     {}
 
     behavior make_behavior() {
         return {
-            // ── Boot ──────────────────────────────────────────────────────
             [this](start_atom) {
-                self_->println("[Supervisor] Starting. Workers={}, Batches={}, "
-                               "Samples/batch={}", num_workers_, total_batches_, samples_per_batch_);
-
-                // Pre-load work queue.
-                for (int b = 0; b < total_batches_; ++b) {
-                    work_queue_.push_back({b, b * 1000003, samples_per_batch_});
+                // Evenly-spaced triggers: fault i fires after batch completion count
+                // (i+1) * total_batches / (num_faults + 1), clamped to [1, total-1].
+                fault_triggers_.resize(num_faults_);
+                for (int i = 0; i < num_faults_; ++i) {
+                    int t = (i + 1) * total_batches_ / (num_faults_ + 1);
+                    fault_triggers_[i] = std::max(1, std::min(total_batches_ - 1, t));
                 }
-                total_expected_ = (long long)total_batches_ * samples_per_batch_;
 
-                // Spawn workers and immediately dispatch the first batch each.
+                self_->println("[Supervisor] Starting: workers={} batches={} "
+                               "samples/batch={} faults={} mode={}",
+                               num_workers_, total_batches_, samples_per_batch_,
+                               num_faults_, no_fault_ ? "baseline" : "fault-injection");
+                for (int i = 0; i < num_faults_; ++i)
+                    self_->println("[Supervisor] Fault {} target: batches_done={}",
+                                   i, fault_triggers_[i]);
+
+                // Populate work queue.
+                for (int b = 0; b < total_batches_; ++b)
+                    work_queue_.push_back({b, b * 1000003, samples_per_batch_, -1});
+
+                t_start_ = sc::now();
+
+                // Spawn workers and immediately dispatch the first batch to each.
                 workers_.resize(num_workers_);
                 for (int k = 0; k < num_workers_; ++k) {
                     workers_[k] = spawn_worker(k);
                     if (!work_queue_.empty())
                         dispatch_to(k);
                 }
-            },
-
-            // ── Externally-triggered fault injection (e.g., from main) ───
-            // Also fired internally when half the batches complete.
-            [this](kill_atom) {
-                if (!fault_injected_ && !workers_.empty()) {
-                    fault_injected_ = true;
-                    self_->println("[Supervisor] *** Injecting fault: killing Worker 0 ***");
-                    anon_send_exit(workers_[0], exit_reason::kill);
-                }
             }
         };
     }
 
 private:
-    // ── Helpers ──────────────────────────────────────────────────────────────
+    // ── spawn_worker ── creates a WorkerActor with a monitor recovery callback
 
     actor spawn_worker(int idx) {
         actor w = self_->spawn(actor_from_state<WorkerActor>, idx);
 
-        // Monitor with callback — non-deprecated, preferred API.
+        // Monitor with callback — preferred non-deprecated API.
         self_->monitor(w, [this, idx](const error& reason) {
+            tp t_detected = sc::now();
             self_->println("[Supervisor] Worker {} died: {}", idx, to_string(reason));
 
             actor_addr dead_addr = workers_[idx].address();
@@ -229,15 +232,37 @@ class Supervisor {
             auto it = in_flight_.find(dead_addr);
             if (it != in_flight_.end()) {
                 WorkBatch lost = it->second;
-                self_->println("[Supervisor] Re-queuing lost batch {} (seed={})",
-                               lost.batch_id, lost.seed);
-                work_queue_.push_front(lost);  // high-priority: run next
                 in_flight_.erase(it);
+
+                // Locate the most recently created fault record for this slot
+                // that has not yet been dispatched (it was just injected).
+                int fi = -1;
+                for (int i = (int)fault_records_.size() - 1; i >= 0; --i) {
+                    if (fault_records_[i].worker_slot == idx
+                            && !fault_records_[i].dispatched) {
+                        fi = i;
+                        fault_records_[fi].batch_id   = lost.batch_id;
+                        fault_records_[fi].t_detected = t_detected;
+                        break;
+                    }
+                }
+
+                double elapsed_ms = ms_between(t_start_, t_detected);
+                self_->println("[FAULT_DETECTED] fault={} worker={} batch={} time_ms={:.3f}",
+                               fi, idx, lost.batch_id, elapsed_ms);
+
+                // Tag the re-queued batch for recovery tracking.
+                lost.fault_index = fi;
+                work_queue_.push_front(lost);  // high-priority: run next
+            } else {
+                self_->println("[Supervisor] Worker {} was idle when it died.", idx);
             }
 
-            // Respawn and immediately give it work if available.
+            // Respawn and immediately give it work (recovery batch or next).
             workers_[idx] = spawn_worker(idx);
             self_->println("[Supervisor] Worker {} respawned.", idx);
+            self_->println("[WORKER_RESPAWN] worker={} time_s={:.6f}",
+                           idx, ms_between(t_start_, sc::now()) / 1000.0);
             if (!work_queue_.empty())
                 dispatch_to(idx);
         });
@@ -245,37 +270,84 @@ class Supervisor {
         return w;
     }
 
+    // ── dispatch_to ── pulls the next batch from work_queue_ and sends it to
+    //   workers_[idx].  Recovery batches (fault_index >= 0) get their dispatch
+    //   timestamp recorded here.
+
     void dispatch_to(int idx) {
         WorkBatch batch = work_queue_.front();
         work_queue_.pop_front();
 
-        self_->println("[Supervisor] Dispatch batch {} → Worker {} (seed={})",
-                       batch.batch_id, idx, batch.seed);
+        // Recovery dispatch: stamp the dispatch timestamp.
+        if (batch.fault_index >= 0) {
+            int fi = batch.fault_index;
+            if (fi < (int)fault_records_.size()) {
+                fault_records_[fi].t_dispatched = sc::now();
+                fault_records_[fi].dispatched   = true;
+                double d2d = ms_between(fault_records_[fi].t_detected,
+                                        fault_records_[fi].t_dispatched);
+                self_->println("[FAULT_DISPATCH] fault={} worker={} batch={} "
+                               "detect_to_dispatch_ms={:.3f}",
+                               fi, idx, batch.batch_id, d2d);
+            }
+        }
+
+        self_->println("[Supervisor] Dispatch batch {:3d}/{} → Worker {}{}",
+                       batch.batch_id, total_batches_ - 1, idx,
+                       batch.fault_index >= 0 ? " [RECOVERY]" : "");
 
-        // Snapshot the address NOW so the in-flight record and the then()
-        // cleanup both refer to the same worker instance, even if workers_[idx]
-        // is replaced by the time the response arrives.
+        // Snapshot address now so in-flight record survives a potential worker replace.
         actor_addr src = workers_[idx].address();
         in_flight_.emplace(src, batch);
+        self_->println("[BATCH_START] batch={} worker={} time_s={:.6f}",
+                       batch.batch_id, idx, ms_between(t_start_, sc::now()) / 1000.0);
 
         self_->mail(batch.seed, batch.num_samples)
             .request(workers_[idx], infinite)
             .then(
-                [this, idx, src](int M) {
+                [this, idx, src, batch](int M) {
                     total_hits_ += M;
                     batches_done_++;
                     in_flight_.erase(src);
+                    self_->println("[BATCH_END] batch={} worker={} time_s={:.6f}",
+                                   batch.batch_id, idx, ms_between(t_start_, sc::now()) / 1000.0);
+
+                    // Recovery completion: stamp t_completed and print latency.
+                    if (batch.fault_index >= 0) {
+                        int fi = batch.fault_index;
+                        if (fi < (int)fault_records_.size()) {
+                            fault_records_[fi].t_completed = sc::now();
+                            fault_records_[fi].completed   = true;
+                            double d2d   = ms_between(fault_records_[fi].t_detected,
+                                                      fault_records_[fi].t_dispatched);
+                            double d2c   = ms_between(fault_records_[fi].t_dispatched,
+                                                      fault_records_[fi].t_completed);
+                            double total = ms_between(fault_records_[fi].t_detected,
+                                                      fault_records_[fi].t_completed);
+                            self_->println("[FAULT_RECOVERED] fault={} worker={} batch={} "
+                                           "detect_to_dispatch_ms={:.3f} "
+                                           "dispatch_to_complete_ms={:.3f} "
+                                           "total_recovery_ms={:.3f}",
+                                           fi, fault_records_[fi].worker_slot,
+                                           batch.batch_id, d2d, d2c, total);
+                        }
+                    }
 
-                    double pi_est = 4.0 * total_hits_
-                                    / ((double)batches_done_ * samples_per_batch_);
-                    self_->println("[Supervisor] Batch complete ({}/{}). π ≈ {:.6f}",
-                                   batches_done_, total_batches_, pi_est);
-
-                    // Inject fault exactly at the halfway point.
-                    if (batches_done_ == total_batches_ / 2) {
-                        self_->mail(kill_atom_v).send(self_);
+                    // Periodic progress reporting (~every 5% of total batches).
+                    int interval = std::max(1, total_batches_ / 20);
+                    if (batches_done_ % interval == 0 || batches_done_ == total_batches_) {
+                        double elapsed_s = ms_between(t_start_, sc::now()) / 1000.0;
+                        double rate      = batches_done_ / elapsed_s;
+                        double pi_est    = 4.0 * total_hits_
+                                           / ((double)batches_done_ * samples_per_batch_);
+                        self_->println("[PROGRESS] batch={}/{} pi={:.6f} "
+                                       "time_s={:.3f} rate={:.2f} batches/s",
+                                       batches_done_, total_batches_, pi_est, elapsed_s, rate);
                     }
 
+                    // Check whether any fault trigger point has been reached.
+                    check_fault_injection();
+
                     if (batches_done_ == total_batches_) {
                         finish();
                         return;
@@ -284,31 +356,93 @@ class Supervisor {
                         dispatch_to(idx);
                 },
                 [this, idx](const error& err) {
-                    // Worker died while we were waiting for its result.
-                    // The monitor callback already rescheduled the in-flight batch.
+                    // Worker died mid-request; monitor callback already re-queued the batch.
                     self_->println("[Supervisor] Request to Worker {} failed: {} "
-                                   "(batch rescheduled via monitor callback)", idx, to_string(err));
+                                   "(batch rescheduled via monitor callback)",
+                                   idx, to_string(err));
                 });
     }
 
+    // ── check_fault_injection ── called on every batch completion.
+    //   Injects the next scheduled fault when batches_done_ hits the trigger.
+    //   Cycles through worker slots so different workers are targeted.
+
+    void check_fault_injection() {
+        while (faults_injected_ < num_faults_
+               && batches_done_ >= fault_triggers_[faults_injected_]) {
+            int slot = faults_injected_ % num_workers_;
+            self_->println("[Supervisor] *** Injecting fault {}/{}: killing Worker {} "
+                           "at batches_done={}/{} ***",
+                           faults_injected_ + 1, num_faults_,
+                           slot, batches_done_, total_batches_);
+
+            // Pre-register record; t_detected is set inside the monitor callback.
+            FaultRecord fr;
+            fr.fault_index = faults_injected_;
+            fr.worker_slot = slot;
+            fault_records_.push_back(fr);
+
+            anon_send_exit(workers_[slot], exit_reason::kill);
+            faults_injected_++;
+        }
+    }
+
+    // ── finish ── prints final statistics and signals the main thread.
+
     void finish() {
-        long long total_samples_completed =
-            (long long)batches_done_ * samples_per_batch_;
-        double pi_final = 4.0 * total_hits_ / (double)total_samples_completed;
-        double error_pct = std::abs(pi_final - M_PI) / M_PI * 100.0;
+        tp     t_end      = sc::now();
+        double elapsed_s  = ms_between(t_start_, t_end) / 1000.0;
+        long long total_s = (long long)batches_done_ * samples_per_batch_;
+        double pi_final   = 4.0 * total_hits_ / (double)total_s;
+        double error_pct  = std::abs(pi_final - M_PI) / M_PI * 100.0;
+        double throughput = batches_done_ / elapsed_s;
 
         self_->println("\n=== RESULT ===");
+        self_->println("Mode              : {}",
+                       no_fault_ ? "no-fault baseline" : "fault-injection");
+        self_->println("Workers           : {}", num_workers_);
         self_->println("Batches completed : {}", batches_done_);
-        self_->println("Total samples     : {}", total_samples_completed);
+        self_->println("Total samples     : {}", total_s);
         self_->println("Total hits        : {}", total_hits_);
         self_->println("π estimate        : {:.8f}", pi_final);
-        self_->println("Error vs π        : {:.4f}%\n", error_pct);
+        self_->println("Error vs π        : {:.4f}%", error_pct);
+        self_->println("Elapsed time      : {:.3f} s", elapsed_s);
+        self_->println("Throughput        : {:.2f} batches/s", throughput);
+        self_->println("Faults injected   : {}", faults_injected_);
+
+        if (!fault_records_.empty()) {
+            self_->println("\n=== RECOVERY LATENCIES ===");
+            for (auto& fr : fault_records_) {
+                if (fr.completed) {
+                    double d2d   = ms_between(fr.t_detected, fr.t_dispatched);
+                    double d2c   = ms_between(fr.t_dispatched, fr.t_completed);
+                    double total = ms_between(fr.t_detected, fr.t_completed);
+                    self_->println("  Fault {:2d}: worker={} batch={:3d}  "
+                                   "detect→dispatch={:.3f}ms  "
+                                   "dispatch→complete={:.3f}ms  "
+                                   "total_recovery={:.3f}ms",
+                                   fr.fault_index, fr.worker_slot, fr.batch_id,
+                                   d2d, d2c, total);
+                } else {
+                    self_->println("  Fault {:2d}: worker={} "
+                                   "(incomplete — batch never recovered)",
+                                   fr.fault_index, fr.worker_slot);
+                }
+            }
+        }
+
+        // Machine-readable summary line for benchmark script parsing.
+        self_->println("\n[RESULT] mode={} workers={} batches={} total_samples={} "
+                       "pi={:.8f} error_pct={:.4f} elapsed_s={:.3f} "
+                       "throughput={:.2f} faults={}",
+                       no_fault_ ? "baseline" : "fault",
+                       num_workers_, batches_done_, total_s,
+                       pi_final, error_pct, elapsed_s, throughput, faults_injected_);
 
-        // Shut down all workers.
+        // Shut down all workers gracefully.
         for (auto& w : workers_)
             anon_send_exit(w, exit_reason::user_shutdown);
 
-        // Signal completion to main.
         self_->mail(done_atom_v).send(parent_);
         self_->quit();
     }
@@ -319,37 +453,50 @@ class Supervisor {
 // ─────────────────────────────────────────────────────────────────────────────
 class config : public actor_system_config {
 public:
+    int  num_workers       = 4;
+    int  total_batches     = 200;
+    int  samples_per_batch = 50'000'000;
+    int  num_faults        = 1;
+    bool no_fault          = false;
+
     config() {
-        // Deliberately use more than 1 thread so concurrent workers are real.
-        // Uncomment to pin to 1 thread and confirm everything still works:
-        // set("caf.scheduler.max-threads", 1u);
+        opt_group{custom_options_, "global"}
+            .add(num_workers,       "num-workers,W",
+                 "number of GPU worker actors (default: 4)")
+            .add(total_batches,     "num-batches,B",
+                 "total Monte Carlo batches (default: 200)")
+            .add(samples_per_batch, "samples-per-batch,S",
+                 "RNG samples per batch (default: 50 000 000)")
+            .add(num_faults,        "num-faults,F",
+                 "number of faults to inject, evenly spaced (default: 1)")
+            .add(no_fault,          "no-fault,n",
+                 "disable fault injection — baseline throughput run");
     }
 };
 
-void caf_main(actor_system& sys, const config&) {
+void caf_main(actor_system& sys, const config& cfg) {
     caf::cuda::manager::init(sys);
-    constexpr int NUM_WORKERS      = 2;
-    constexpr int TOTAL_BATCHES    = 20;
-    constexpr int SAMPLES_PER_BATCH = 10'000'000;
 
-    sys.println("=== Monte Carlo π Estimation (Fault-Tolerant) ===");
-    sys.println("Workers: {}   Batches: {}   Samples/batch: {}",
-                NUM_WORKERS, TOTAL_BATCHES, SAMPLES_PER_BATCH);
+    sys.println("=== Monte Carlo π Estimation (Fault-Tolerant, SC26) ===");
+    sys.println("Workers: {}  Batches: {}  Samples/batch: {}  Faults: {}  Mode: {}",
+                cfg.num_workers, cfg.total_batches, cfg.samples_per_batch,
+                cfg.num_faults, cfg.no_fault ? "baseline" : "fault-injection");
     sys.println("Total samples planned: {}\n",
-                (long long)TOTAL_BATCHES * SAMPLES_PER_BATCH);
+                (long long)cfg.total_batches * cfg.samples_per_batch);
 
     scoped_actor self{sys};
 
     actor supervisor = self->spawn(
         actor_from_state<Supervisor>,
-        NUM_WORKERS, TOTAL_BATCHES, SAMPLES_PER_BATCH,
+        cfg.num_workers, cfg.total_batches, cfg.samples_per_batch,
+        cfg.num_faults, cfg.no_fault,
         actor_cast<actor>(self));
 
     self->mail(start_atom_v).send(supervisor);
 
-    // Block until the supervisor signals done (or terminates cleanly).
+    // Block until the supervisor signals done.
     self->receive([](done_atom) {});
-    
+
     caf::cuda::manager::shutdown();
 }
 
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/Makefile b/libcaf_cuda/sc26/Runtime-Overhead/Makefile
index a894e9b48c..3b096cd8f6 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/Makefile
+++ b/libcaf_cuda/sc26/Runtime-Overhead/Makefile
@@ -26,7 +26,7 @@ NVCC_ARCH ?= sm_$(shell nvidia-smi --query-gpu=compute_cap --format=csv,noheader
 all: mmul_kernel cuda_native command_runner actor_facade
 
 mmul_kernel: mmul.cu
-	$(NVCC) -g -arch=$(NVCC_ARCH) --cubin mmul.cu -o mmul.cubin
+	$(NVCC) -arch=$(NVCC_ARCH) --cubin mmul.cu -o mmul.cubin
 
 cuda_native: cuda_native.cpp
 	$(CXX) $(CXXFLAGS) $(INCLUDES) -o cuda_native cuda_native.cpp $(LIBS) $(LDFLAGS)
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp b/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
index bbe09c9bdf..8b411bf2c3 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
@@ -75,7 +75,14 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size) {
   sys.await_all_actors_done();
 }
 
-void caf_main(caf::actor_system& sys) {
+class config : public actor_system_config {
+public:
+  config() {
+      set("caf.scheduler.max-threads", 1u);
+  }
+};
+
+void caf_main(caf::actor_system& sys, const config& cfg) {
   caf::cuda::manager::init(sys);  // F5: init once before all sizes
 
   // F2: warmup run to prime CUDA context, JIT, and CAF infrastructure
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
index 95879df2b8..216fefb423 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
@@ -209,8 +209,15 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size) {
             << ", time=" << duration_ms << " ms\n";
 }
 
+class config : public actor_system_config {
+public:
+  config() {
+      set("caf.scheduler.max-threads", 1u);
+  }
+};
+
 
-void caf_main(caf::actor_system& sys) {
+void caf_main(caf::actor_system& sys, const config& cfg) {
   caf::cuda::manager::init(sys);  // F5: init once before all sizes
 
   // F2: warmup run to prime CUDA context, JIT, and CAF infrastructure
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
index d711ff752b..3f40cc7dce 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
@@ -126,16 +126,17 @@ void run_series(caf::actor_system& sys, int matrix_size, int iterations,
 }
 
 void caf_main(caf::actor_system& sys) {
+  constexpr int matrix_size = 1000;
+  constexpr int total_iterations = 10000;
+
   caf::cuda::manager::init(sys);  // S4 fix: init once before all series
 
   // S2 fix: warmup run to prime CUDA context and CAF infrastructure
   std::cout << "--- warmup starting ---\n";
-  run_series(sys, 1000, 10, /*is_warmup=*/true);
+  run_series(sys, matrix_size, 10, /*is_warmup=*/true);
   std::cout << "--- warmup complete ---\n";
 
-  for (int i = 1000; i <= 10000; i += 1000) {
-    run_series(sys, 1000, i);
-  }
+  run_series(sys, matrix_size, total_iterations);
 
   caf::cuda::manager::shutdown();  // S4 fix: shutdown once after all series
 }
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
index fb31676bbf..bfe615f729 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
@@ -157,6 +157,8 @@ void run_mmul_test(caf::actor_system& sys,
 
 
 void caf_main(caf::actor_system& sys) {
+  constexpr int matrix_size = 1000;
+  constexpr int total_iterations = 10000;
 
   caf::cuda::manager::init(sys);  // S4 fix: init once before all series
 
@@ -165,11 +167,10 @@ void caf_main(caf::actor_system& sys) {
 
   // S2 fix: warmup run to prime CUDA context before timed series
   std::cout << "--- warmup starting ---\n";
-  run_mmul_test(sys, program, 1000, 10, /*is_warmup=*/true);
+  run_mmul_test(sys, program, matrix_size, 10, /*is_warmup=*/true);
   std::cout << "--- warmup complete ---\n";
 
-  for (int i = 1000; i < 11000; i += 1000)
-	  run_mmul_test(sys, program, 1000, i);
+  run_mmul_test(sys, program, matrix_size, total_iterations);
 
   caf::cuda::manager::shutdown();  // S4 fix: shutdown once after all series
 
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp
index 1046f774fd..fdbd3e5218 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp
@@ -46,8 +46,8 @@ void launchKernel(CUfunction kernel, CUstream stream,
 
 int main() {
     const int N = 1000;
-    std::vector<int> iteration_series = {1000, 2000, 3000, 4000, 5000,
-                                         6000, 7000, 8000, 9000, 10000};
+    const int total_iterations = 10000;
+    const int milestone_interval = 1000;
 
     checkCU(cuInit(0), "cuInit");
 
@@ -98,52 +98,57 @@ int main() {
         std::cout << "--- warmup complete ---\n";
     }
 
-    for (int iterations : iteration_series) {
-        auto start = clock::now();
-
-        for (int i = 0; i < iterations; ++i) {
-            // ----------------------------------
-            // Allocate device memory each iteration
-            // ----------------------------------
-            CUdeviceptr d_a, d_b, d_c;
-            checkCU(cuMemAlloc(&d_a, elements * sizeof(int)), "cuMemAlloc d_a");
-            checkCU(cuMemAlloc(&d_b, elements * sizeof(int)), "cuMemAlloc d_b");
-            checkCU(cuMemAlloc(&d_c, elements * sizeof(int)), "cuMemAlloc d_c");
-
-            // ----------------------------------
-            // Copy persistent host buffers to device
-            // ----------------------------------
-            checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), elements * sizeof(int), stream), "H2D d_a");
-            checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), elements * sizeof(int), stream), "H2D d_b");
-
-            // ----------------------------------
-            // Launch kernel
-            // ----------------------------------
-            launchKernel(kernel, stream, d_a, d_b, d_c, N);
-
-            // ----------------------------------
-            // Copy result back
-            // ----------------------------------
-            checkCU(cuMemcpyDtoHAsync(h_c.data(), d_c, elements * sizeof(int), stream), "D2H d_c");
-
-            // ----------------------------------
-            // Free device memory
-            // ----------------------------------
-            checkCU(cuMemFree(d_a), "cuMemFree d_a");
-            checkCU(cuMemFree(d_b), "cuMemFree d_b");
-            checkCU(cuMemFree(d_c), "cuMemFree d_c");
+    auto start = clock::now();
+
+    for (int i = 0; i < total_iterations; ++i) {
+        // ----------------------------------
+        // Allocate device memory each iteration
+        // ----------------------------------
+        CUdeviceptr d_a, d_b, d_c;
+        checkCU(cuMemAlloc(&d_a, elements * sizeof(int)), "cuMemAlloc d_a");
+        checkCU(cuMemAlloc(&d_b, elements * sizeof(int)), "cuMemAlloc d_b");
+        checkCU(cuMemAlloc(&d_c, elements * sizeof(int)), "cuMemAlloc d_c");
+
+        // ----------------------------------
+        // Copy persistent host buffers to device
+        // ----------------------------------
+        checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), elements * sizeof(int), stream), "H2D d_a");
+        checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), elements * sizeof(int), stream), "H2D d_b");
+
+        // ----------------------------------
+        // Launch kernel
+        // ----------------------------------
+        launchKernel(kernel, stream, d_a, d_b, d_c, N);
+
+        // ----------------------------------
+        // Copy result back
+        // ----------------------------------
+        checkCU(cuMemcpyDtoHAsync(h_c.data(), d_c, elements * sizeof(int), stream), "D2H d_c");
+
+        // ----------------------------------
+        // Free device memory
+        // ----------------------------------
+        checkCU(cuMemFree(d_a), "cuMemFree d_a");
+        checkCU(cuMemFree(d_b), "cuMemFree d_b");
+        checkCU(cuMemFree(d_c), "cuMemFree d_c");
+
+        if ((i + 1) % milestone_interval == 0) {
+            checkCU(cuStreamSynchronize(stream), "stream sync at milestone");
+            auto now = clock::now();
+            double elapsed_ms = std::chrono::duration<double, std::milli>(now - start).count();
+            std::cout << "[MILESTONE] " << (i + 1) << " / " << total_iterations
+                      << " iterations, elapsed = " << elapsed_ms << " ms\n";
         }
+    }
 
-        // Synchronize stream after series
-        checkCU(cuStreamSynchronize(stream), "stream sync after series");
+    checkCU(cuStreamSynchronize(stream), "stream sync after series");
 
-        auto end = clock::now();
-        double total_ms = std::chrono::duration<double, std::milli>(end - start).count();
+    auto end = clock::now();
+    double total_ms = std::chrono::duration<double, std::milli>(end - start).count();
 
-        std::cout << "[SERIES RESULT] Matrix " << N << "x" << N
-                  << ", iterations = " << iterations
-                  << ", total GPU time = " << total_ms << " ms\n";
-    }
+    std::cout << "[SERIES RESULT] Matrix " << N << "x" << N
+              << ", iterations = " << total_iterations
+              << ", total GPU time = " << total_ms << " ms\n";
 
     // ----------------------------------
     // Cleanup
diff --git a/libcaf_cuda/sc26/benchmarks/run_sequence_independent.py b/libcaf_cuda/sc26/benchmarks/run_sequence_independent.py
index 8fb32142af..9f630ba7f1 100755
--- a/libcaf_cuda/sc26/benchmarks/run_sequence_independent.py
+++ b/libcaf_cuda/sc26/benchmarks/run_sequence_independent.py
@@ -3,12 +3,13 @@
 Sequence-Independent-Tasks Benchmark Harness
 =============================================
 Runs main_cuda_native, main_actor_facade, and main_command_runner each
-NUM_RUNS times.  For each run the script captures the [MILESTONE] lines
-emitted every 1 000 iterations and records them.  The output file contains:
+NUM_RUNS times. Each run is expected to execute one timed 10 000-iteration
+series, emit cumulative [MILESTONE] lines every 1 000 iterations, and print
+one final [SERIES RESULT] line. The output file contains:
 
   1. All raw [MILESTONE] and [SERIES RESULT] lines from every run.
-  2. Per-series statistics (mean / min / max / stddev of the cumulative
-     elapsed time at each 1 000-iteration milestone across all runs).
+  2. Statistics across runs for the 10 000-iteration series at each
+      1 000-iteration milestone (mean / min / max / stddev).
   3. Incremental time per 1 000-iteration step (derived from adjacent
      milestones) — this shows how long each individual 1 000-iteration
      "slice" took on average.
@@ -48,12 +49,13 @@
 RE_MILESTONE = re.compile(
     r"\[MILESTONE\]\s+(\d+)\s*/\s*(\d+)\s+iterations,\s+elapsed\s*=\s*([\d.]+)"
 )
-# [SERIES RESULT] Matrix 1000x1000, iterations = 5000, total ... time = 2601.3 ms
+# [SERIES RESULT] Matrix 1000x1000, iterations = 10000, total ... time = 2601.3 ms
 RE_SERIES = re.compile(
     r"\[SERIES RESULT\].*iterations\s*=\s*(\d+).*?=\s*([\d.]+)\s*ms"
 )
 
 ENV = {**os.environ, "CUDA_VISIBLE_DEVICES": "0"}
+TARGET_ITERATIONS = 10000
 
 # ---------------------------------------------------------------------------
 # Running
@@ -95,7 +97,8 @@ def parse_output(stdout: str) -> RunData:
     Returns:
         milestones: {series_total: {milestone_iter: elapsed_ms}}
         series_totals: {series_total: elapsed_ms}  (from [SERIES RESULT] lines)
-    The 'series_total' key is the total iterations for that series (e.g. 3000).
+    The 'series_total' key is the total iterations for that series.
+    For this benchmark, each run should normally only contain total=10000.
     """
     milestones: dict[int, dict[int, float]] = {}
     series_totals: dict[int, float] = {}
diff --git a/libcaf_cuda/sc26/scripts/Fault-Tolerance/plot.py b/libcaf_cuda/sc26/scripts/Fault-Tolerance/plot.py
new file mode 100644
index 0000000000..cbf4f1ce94
--- /dev/null
+++ b/libcaf_cuda/sc26/scripts/Fault-Tolerance/plot.py
@@ -0,0 +1,781 @@
+#!/usr/bin/env python3
+"""SC26 Fault-Tolerance Benchmark — Paper Plot Generator
+
+Parses structured output from the Monte Carlo π fault-tolerance binary and
+generates the 5 visualizations recommended by the SC26 review:
+
+  gantt.pdf           Recovery timeline  (Gantt chart per worker lane)
+  throughput.pdf      Throughput under stress with fault injection markers
+  pi_convergence.pdf  π-estimate convergence with fault annotation
+  recovery_cdf.pdf    CDF of per-fault recovery latency (multi-run)
+  degradation.pdf     Throughput degradation bar chart (baseline vs N faults)
+
+The binary must be compiled with the SC26-hardened main.cpp that emits
+[BATCH_START], [BATCH_END], [WORKER_RESPAWN], [FAULT_DETECTED],
+[FAULT_DISPATCH], [FAULT_RECOVERED], [PROGRESS], and [RESULT] lines.
+
+Usage examples
+--------------
+# Single fault-injection run (generates plots 1–3 and whatever CDF data exists):
+  python3 plot.py --log run.log
+
+# Add a no-fault baseline for throughput / degradation comparison:
+  python3 plot.py --log fault.log --baseline baseline.log
+
+# Pass multiple fault-injection logs (for CDF with many data points):
+  python3 plot.py --log-dir logs/fault/ --baseline baseline.log
+
+# Run the binary directly (captures output automatically):
+  python3 plot.py --run ./main
+  python3 plot.py --run ./main --args "--num-faults 3 --num-workers 4 --num-batches 200"
+
+# Full degradation chart: provide logs for each fault count explicitly:
+  python3 plot.py --baseline b.log --fault1 f1.log --fault2 f2.log --fault3 f3.log
+
+All plots are written to --out-dir (default: ./plots/).
+Use --no-pdf to save as PNG instead of PDF.
+"""
+
+import argparse
+import math
+import re
+import statistics
+import subprocess
+import sys
+from collections import defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+import matplotlib
+import matplotlib.ticker
+matplotlib.use("Agg")
+import matplotlib.patches as mpatches
+import matplotlib.pyplot as plt
+from matplotlib.lines import Line2D
+import numpy as np
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Plot styling
+# ─────────────────────────────────────────────────────────────────────────────
+
+STYLE = {
+    "font.size": 11,
+    "axes.titlesize": 12,
+    "axes.labelsize": 11,
+    "xtick.labelsize": 10,
+    "ytick.labelsize": 10,
+    "legend.fontsize": 9,
+    "figure.dpi": 150,
+    "savefig.bbox": "tight",
+    "pdf.fonttype": 42,  # embed TrueType fonts in PDF (required by most venues)
+    "axes.spines.top": False,
+    "axes.spines.right": False,
+    "axes.grid": True,
+    "grid.alpha": 0.3,
+}
+
+COLORS = {
+    "normal":     "#4C8BBF",  # steel blue  — normal computation
+    "recovery":   "#E8781A",  # orange      — recovery batch
+    "baseline":   "#555555",  # dark grey   — baseline reference
+    "fault_line": "#CC3333",  # red         — fault injection marker
+    "pi_ref":     "#2CA02C",  # green       — π reference / respawn marker
+    "worker_bg":  "#F4F7FB",  # very light blue — worker lane background
+}
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Data model
+# ─────────────────────────────────────────────────────────────────────────────
+
+@dataclass
+class BatchEvent:
+    batch_id:    int
+    worker_id:   int
+    start_s:     float
+    end_s:       float = None
+    is_recovery: bool  = False
+
+
+@dataclass
+class FaultEvent:
+    fault_index:            int
+    worker_id:              int
+    batch_id:               int
+    detected_s:             float
+    dispatch_s:             float = None
+    recover_s:              float = None
+    detect_to_dispatch_ms:  float = None
+    dispatch_to_complete_ms: float = None
+    total_recovery_ms:      float = None
+
+
+@dataclass
+class ProgressPoint:
+    batch_done:    int
+    total_batches: int
+    pi:            float
+    time_s:        float
+    rate:          float   # cumulative batches/s at this point
+
+
+@dataclass
+class WorkerRespawn:
+    worker_id: int
+    time_s:    float
+
+
+@dataclass
+class RunSummary:
+    mode:             str   = "unknown"
+    num_workers:      int   = 0
+    total_batches:    int   = 0
+    total_samples:    int   = 0
+    samples_per_batch: int  = 0
+    pi:               float = 0.0
+    error_pct:        float = 0.0
+    elapsed_s:        float = 0.0
+    throughput:       float = 0.0   # batches/s
+    faults_injected:  int   = 0
+
+
+@dataclass
+class RunData:
+    batches:  list = field(default_factory=list)   # list[BatchEvent]
+    faults:   list = field(default_factory=list)   # list[FaultEvent]
+    progress: list = field(default_factory=list)   # list[ProgressPoint]
+    respawns: list = field(default_factory=list)   # list[WorkerRespawn]
+    summary:  RunSummary = field(default_factory=RunSummary)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Parser
+# ─────────────────────────────────────────────────────────────────────────────
+
+def parse_log(text: str) -> RunData:
+    """Parse all structured lines emitted by the SC26 fault-tolerance binary."""
+    data = RunData()
+    fault_map: dict[int, FaultEvent] = {}
+    batch_seen: dict[int, bool] = {}   # batch_id -> True once we've seen first BATCH_START
+
+    for line in text.splitlines():
+        line = line.strip()
+
+        # [BATCH_START] batch=X worker=Y time_s=Z
+        m = re.match(r"\[BATCH_START\] batch=(\d+) worker=(\d+) time_s=([\d.]+)", line)
+        if m:
+            bid, wid, t = int(m.group(1)), int(m.group(2)), float(m.group(3))
+            is_rec = bid in batch_seen
+            batch_seen[bid] = True
+            data.batches.append(BatchEvent(bid, wid, t, is_recovery=is_rec))
+            continue
+
+        # [BATCH_END] batch=X worker=Y time_s=Z
+        m = re.match(r"\[BATCH_END\] batch=(\d+) worker=(\d+) time_s=([\d.]+)", line)
+        if m:
+            bid, wid, t = int(m.group(1)), int(m.group(2)), float(m.group(3))
+            # Match to latest unfinished event for this (batch_id, worker_id) pair
+            for ev in reversed(data.batches):
+                if ev.batch_id == bid and ev.worker_id == wid and ev.end_s is None:
+                    ev.end_s = t
+                    break
+            continue
+
+        # [WORKER_RESPAWN] worker=X time_s=Z
+        m = re.match(r"\[WORKER_RESPAWN\] worker=(\d+) time_s=([\d.]+)", line)
+        if m:
+            data.respawns.append(WorkerRespawn(int(m.group(1)), float(m.group(2))))
+            continue
+
+        # [FAULT_DETECTED] fault=F worker=W batch=B time_ms=T
+        m = re.match(
+            r"\[FAULT_DETECTED\] fault=(-?\d+) worker=(\d+) batch=(\d+) time_ms=([\d.]+)",
+            line,
+        )
+        if m:
+            fi, wid, bid, t_ms = int(m.group(1)), int(m.group(2)), int(m.group(3)), float(m.group(4))
+            fe = FaultEvent(fi, wid, bid, t_ms / 1000.0)
+            fault_map[fi] = fe
+            data.faults.append(fe)
+            continue
+
+        # [FAULT_DISPATCH] fault=F worker=W batch=B detect_to_dispatch_ms=D
+        m = re.match(
+            r"\[FAULT_DISPATCH\] fault=(\d+) worker=\d+ batch=\d+ detect_to_dispatch_ms=([\d.]+)",
+            line,
+        )
+        if m:
+            fi, d2d = int(m.group(1)), float(m.group(2))
+            if fi in fault_map:
+                fault_map[fi].detect_to_dispatch_ms = d2d
+                fault_map[fi].dispatch_s = fault_map[fi].detected_s + d2d / 1000.0
+            continue
+
+        # [FAULT_RECOVERED] fault=F worker=W batch=B detect_to_dispatch_ms=A
+        #                   dispatch_to_complete_ms=B total_recovery_ms=C
+        m = re.match(
+            r"\[FAULT_RECOVERED\] fault=(\d+) worker=\d+ batch=\d+ "
+            r"detect_to_dispatch_ms=([\d.]+) dispatch_to_complete_ms=([\d.]+) "
+            r"total_recovery_ms=([\d.]+)",
+            line,
+        )
+        if m:
+            fi = int(m.group(1))
+            d2d, d2c, tot = float(m.group(2)), float(m.group(3)), float(m.group(4))
+            if fi in fault_map:
+                fault_map[fi].detect_to_dispatch_ms  = d2d
+                fault_map[fi].dispatch_to_complete_ms = d2c
+                fault_map[fi].total_recovery_ms       = tot
+                base_s = fault_map[fi].dispatch_s or fault_map[fi].detected_s
+                fault_map[fi].recover_s = base_s + d2c / 1000.0
+            continue
+
+        # [PROGRESS] batch=X/Y pi=Z time_s=W rate=R batches/s
+        m = re.match(
+            r"\[PROGRESS\] batch=(\d+)/(\d+) pi=([\d.]+) time_s=([\d.]+) rate=([\d.]+) batches/s",
+            line,
+        )
+        if m:
+            data.progress.append(ProgressPoint(
+                int(m.group(1)), int(m.group(2)),
+                float(m.group(3)), float(m.group(4)), float(m.group(5)),
+            ))
+            continue
+
+        # [RESULT] mode=X workers=Y batches=Z total_samples=W pi=V error_pct=U
+        #          elapsed_s=T throughput=S faults=R
+        m = re.match(
+            r"\[RESULT\] mode=(\S+) workers=(\d+) batches=(\d+) total_samples=(\d+) "
+            r"pi=([\d.]+) error_pct=([\d.]+) elapsed_s=([\d.]+) "
+            r"throughput=([\d.]+) faults=(\d+)",
+            line,
+        )
+        if m:
+            s = data.summary
+            s.mode             = m.group(1)
+            s.num_workers      = int(m.group(2))
+            s.total_batches    = int(m.group(3))
+            s.total_samples    = int(m.group(4))
+            s.pi               = float(m.group(5))
+            s.error_pct        = float(m.group(6))
+            s.elapsed_s        = float(m.group(7))
+            s.throughput       = float(m.group(8))
+            s.faults_injected  = int(m.group(9))
+            if s.total_batches > 0:
+                s.samples_per_batch = s.total_samples // s.total_batches
+            continue
+
+        # Header line: "Workers: X  Batches: Y  Samples/batch: Z  ..."
+        m = re.match(r"Workers:\s*(\d+)\s+Batches:\s*(\d+)\s+Samples/batch:\s*(\d+)", line)
+        if m and data.summary.samples_per_batch == 0:
+            data.summary.samples_per_batch = int(m.group(3))
+            if data.summary.num_workers == 0:
+                data.summary.num_workers = int(m.group(1))
+
+    return data
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Plot 1: Recovery Timeline (Gantt chart)
+# ─────────────────────────────────────────────────────────────────────────────
+
+def plot_gantt(run: RunData, path: Path) -> None:
+    """Horizontal Gantt chart: one lane per worker, bars for each batch.
+
+    Normal batches: steel blue.
+    Recovery (re-dispatched) batches: orange with hatching.
+    Worker-killed marker: red ×.
+    Worker-respawned marker: green ▲.
+    Dashed red vertical line: fault detected instant.
+    """
+    if not run.batches:
+        print("  [gantt]      SKIP — no [BATCH_START]/[BATCH_END] data.\n"
+              "               Recompile Fault-Tolerance/main.cpp and re-run.")
+        return
+
+    s        = run.summary
+    workers  = sorted({b.worker_id for b in run.batches})
+    y_map    = {w: i for i, w in enumerate(workers)}
+    bar_h    = 0.55
+    fig_h    = max(3.0, len(workers) * 0.9 + 1.8)
+    fig, ax  = plt.subplots(figsize=(12, fig_h))
+
+    # Worker lane backgrounds
+    for w in workers:
+        ax.axhspan(y_map[w] - 0.5, y_map[w] + 0.5,
+                   color=COLORS["worker_bg"], zorder=0, linewidth=0)
+
+    # Batch bars
+    for b in run.batches:
+        y  = y_map[b.worker_id]
+        x0 = b.start_s
+        x1 = b.end_s
+        if x1 is None:
+            # Worker was killed before this batch completed — clip bar to fault time
+            clipped = [f.detected_s for f in run.faults
+                       if f.worker_id == b.worker_id and f.detected_s >= x0]
+            x1 = min(clipped) if clipped else x0 + 1e-4
+        color = COLORS["recovery"] if b.is_recovery else COLORS["normal"]
+        hatch = "///" if b.is_recovery else None
+        ax.barh(y, x1 - x0, left=x0, height=bar_h,
+                color=color, hatch=hatch, edgecolor="white",
+                linewidth=0.4, zorder=2, alpha=0.85)
+
+    # Fault: red × at detection time and dashed vertical line
+    for f in run.faults:
+        y = y_map.get(f.worker_id, -1)
+        if y < 0:
+            continue
+        ax.scatter(f.detected_s, y, marker="x", color=COLORS["fault_line"],
+                   s=140, zorder=6, linewidths=2.5)
+        ax.axvline(f.detected_s, color=COLORS["fault_line"],
+                   linestyle="--", linewidth=0.9, alpha=0.5, zorder=1)
+
+    # Respawn: green ▲ just below the lane centre
+    for r in run.respawns:
+        y = y_map.get(r.worker_id, -1)
+        if y < 0:
+            continue
+        ax.scatter(r.time_s, y - bar_h * 0.35, marker="^",
+                   color=COLORS["pi_ref"], s=90, zorder=6)
+
+    # Axis labels
+    ax.set_xlabel("Wall-clock time (s)")
+    ax.set_ylabel("Worker")
+    ax.set_yticks(list(y_map.values()))
+    ax.set_yticklabels([f"Worker {w}" for w in workers])
+    ax.set_xlim(left=0)
+
+    # Legend
+    handles = [
+        mpatches.Patch(color=COLORS["normal"], label="Active computation"),
+        mpatches.Patch(color=COLORS["recovery"], hatch="///", label="Recovery batch"),
+        Line2D([0], [0], marker="x", color="w",
+               markeredgecolor=COLORS["fault_line"], markeredgewidth=2.5,
+               markersize=10, linewidth=0, label="Worker killed"),
+        Line2D([0], [0], marker="^", color="w",
+               markerfacecolor=COLORS["pi_ref"], markersize=8,
+               linewidth=0, label="Worker respawned"),
+    ]
+    ax.legend(handles=handles, loc="lower right", framealpha=0.92)
+
+    n_faults = s.faults_injected
+    ax.set_title(
+        f"Recovery Timeline — {s.num_workers} workers, "
+        f"{n_faults} fault{'s' if n_faults != 1 else ''}, "
+        f"{s.total_batches} batches"
+    )
+    fig.tight_layout()
+    fig.savefig(path)
+    plt.close(fig)
+    print(f"  [gantt]      → {path}")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Plot 2: Throughput Under Stress
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _rolling_throughput(
+    completion_times: list[float], window: int = 5
+) -> tuple[list[float], list[float]]:
+    """Sliding-window throughput from sorted batch completion timestamps.
+
+    Returns (time_points, batches_per_second) with len = len(times) - window + 1.
+    """
+    times = sorted(completion_times)
+    ts, vs = [], []
+    for i in range(window - 1, len(times)):
+        dt = times[i] - times[i - (window - 1)]
+        if dt > 1e-9:
+            ts.append(times[i])
+            vs.append(window / dt)
+    return ts, vs
+
+
+def plot_throughput(
+    run: RunData,
+    baseline: Optional[RunData],
+    path: Path,
+    window: int = 5,
+) -> None:
+    """Throughput over time for a fault-injection run, with optional baseline overlay.
+
+    If BATCH_END data is present uses rolling-window throughput (precise).
+    Falls back to the cumulative-average `rate` field from [PROGRESS] lines.
+    Vertical dashed red lines mark each fault injection.
+    """
+    fig, ax = plt.subplots(figsize=(10, 5))
+    s = run.summary
+
+    def _plot_run(rd: RunData, label: str, color: str, ls: str = "-") -> None:
+        end_times = [b.end_s for b in rd.batches if b.end_s is not None]
+        if len(end_times) >= window:
+            ts, vs = _rolling_throughput(end_times, window)
+            ax.plot(ts, vs, color=color, linewidth=1.6, linestyle=ls,
+                    label=f"{label} (rolling w={window})")
+        elif rd.progress:
+            pts = rd.progress
+            ax.plot([p.time_s for p in pts], [p.rate for p in pts],
+                    color=color, linewidth=1.6, linestyle=ls,
+                    label=f"{label} (cumul. avg)")
+
+    _plot_run(run, "Fault run", COLORS["normal"])
+
+    if baseline:
+        # Try rolling throughput from baseline; fall back to flat reference line
+        bt = [b.end_s for b in baseline.batches if b.end_s is not None]
+        if len(bt) >= window:
+            _plot_run(baseline, "Baseline", COLORS["baseline"], ls="--")
+        elif baseline.summary.throughput > 0:
+            ax.axhline(baseline.summary.throughput, color=COLORS["baseline"],
+                       linestyle="--", linewidth=1.5,
+                       label=f"Baseline: {baseline.summary.throughput:.1f} batches/s",
+                       alpha=0.75)
+
+    # Fault injection vertical lines
+    ymax = ax.get_ylim()[1] if ax.get_ylim()[1] > 0 else 1.0
+    for i, f in enumerate(run.faults):
+        lbl = "Fault injection" if i == 0 else "_nolegend_"
+        ax.axvline(f.detected_s, color=COLORS["fault_line"],
+                   linestyle="--", linewidth=1.2, alpha=0.85, label=lbl, zorder=3)
+
+    ax.set_xlabel("Wall-clock time (s)")
+    ax.set_ylabel("Throughput (batches / s)")
+    ax.set_xlim(left=0)
+    ax.set_ylim(bottom=0)
+    ax.legend(framealpha=0.92)
+    n_faults = s.faults_injected
+    ax.set_title(
+        f"Throughput Under Stress — {s.num_workers} workers, "
+        f"{n_faults} fault{'s' if n_faults != 1 else ''}"
+    )
+    fig.tight_layout()
+    fig.savefig(path)
+    plt.close(fig)
+    print(f"  [throughput] → {path}")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Plot 3: π Convergence
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _fmt_samples(x: float, _pos) -> str:
+    if x >= 1e9:
+        return f"{x/1e9:.1f}B"
+    if x >= 1e6:
+        return f"{x/1e6:.0f}M"
+    return f"{x/1e3:.0f}K"
+
+
+def plot_pi_convergence(run: RunData, path: Path) -> None:
+    """π estimate vs cumulative samples.
+
+    The curve should plateau briefly after each fault injection (batch
+    stalled), then resume converging. Vertical dashed lines show where faults
+    were detected (mapped to the nearest [PROGRESS] sample count).
+    """
+    if not run.progress:
+        print("  [pi_conv]    SKIP — no [PROGRESS] data.")
+        return
+
+    s   = run.summary
+    spb = s.samples_per_batch or 1
+
+    xs = [p.batch_done * spb for p in run.progress]   # cumulative samples
+    ys = [p.pi for p in run.progress]
+
+    fig, ax = plt.subplots(figsize=(10, 5))
+    ax.plot(xs, ys, color=COLORS["normal"], linewidth=1.6,
+            label="Estimated π", zorder=3)
+    ax.axhline(math.pi, color=COLORS["pi_ref"], linewidth=1.3, linestyle="-",
+               label=f"π = {math.pi:.6f}", zorder=2)
+
+    # Map each fault detection time to the nearest cumulative sample count
+    prog_times   = [p.time_s   for p in run.progress]
+    prog_samples = [p.batch_done * spb for p in run.progress]
+
+    for i, f in enumerate(run.faults):
+        sample_at_fault = None
+        for j, t in enumerate(prog_times):
+            if t >= f.detected_s:
+                sample_at_fault = prog_samples[j]
+                break
+        if sample_at_fault is None and prog_samples:
+            sample_at_fault = prog_samples[-1]
+        if sample_at_fault is not None:
+            lbl = "Fault injection" if i == 0 else "_nolegend_"
+            ax.axvline(sample_at_fault, color=COLORS["fault_line"],
+                       linestyle="--", linewidth=1.2, alpha=0.85, label=lbl)
+
+    ax.xaxis.set_major_formatter(matplotlib.ticker.FuncFormatter(_fmt_samples))
+    ax.set_xlabel("Cumulative Monte Carlo samples")
+    ax.set_ylabel("π estimate")
+    ax.legend(framealpha=0.92)
+    ax.set_title(
+        f"π Convergence — {s.num_workers} workers, "
+        f"{s.total_batches} batches × {_fmt_samples(spb, None)} samples"
+    )
+    fig.tight_layout()
+    fig.savefig(path)
+    plt.close(fig)
+    print(f"  [pi_conv]    → {path}")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Plot 4: Recovery Latency CDF
+# ─────────────────────────────────────────────────────────────────────────────
+
+def plot_recovery_cdf(all_runs: list[RunData], path: Path) -> None:
+    """Empirical CDF of total recovery latency across all FAULT_RECOVERED events.
+
+    Annotates the median and 95th-percentile with vertical dashed lines.
+    Useful for claiming bounded recovery in the paper.
+    """
+    latencies = [
+        f.total_recovery_ms
+        for run in all_runs
+        for f in run.faults
+        if f.total_recovery_ms is not None
+    ]
+
+    if not latencies:
+        print("  [cdf]        SKIP — no [FAULT_RECOVERED] data.")
+        return
+
+    lat = sorted(latencies)
+    n   = len(lat)
+    cdf = [(i + 1) / n for i in range(n)]
+
+    fig, ax = plt.subplots(figsize=(8, 5))
+    ax.step(lat, cdf, where="post", color=COLORS["normal"], linewidth=2,
+            label=f"Empirical CDF (n={n})")
+    ax.scatter(lat, cdf, color=COLORS["normal"], s=30, zorder=5)
+
+    p50 = float(np.percentile(lat, 50))
+    p95 = float(np.percentile(lat, 95))
+    ax.axvline(p50, color=COLORS["pi_ref"], linestyle="--", linewidth=1.3,
+               label=f"Median: {p50:.1f} ms")
+    ax.axvline(p95, color=COLORS["fault_line"], linestyle="--", linewidth=1.3,
+               label=f"P95: {p95:.1f} ms")
+
+    if n > 1:
+        mean_ms = statistics.mean(lat)
+        std_ms  = statistics.stdev(lat)
+        ax.text(0.97, 0.07,
+                f"mean = {mean_ms:.1f} ms\nσ = {std_ms:.1f} ms",
+                transform=ax.transAxes, ha="right", va="bottom",
+                fontsize=9, bbox=dict(boxstyle="round,pad=0.3", fc="white", alpha=0.85))
+
+    ax.set_xlabel("Recovery latency (ms)")
+    ax.set_ylabel("CDF   P(recovery ≤ t)")
+    ax.set_xlim(left=0)
+    ax.set_ylim(0, 1.07)
+    ax.legend(framealpha=0.92)
+    ax.set_title(f"Recovery Latency CDF  ({n} fault events, {len(all_runs)} run{'s' if len(all_runs)>1 else ''})")
+    fig.tight_layout()
+    fig.savefig(path)
+    plt.close(fig)
+    print(f"  [cdf]        → {path}")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Plot 5: Throughput Degradation
+# ─────────────────────────────────────────────────────────────────────────────
+
+def plot_degradation(runs_by_faults: dict[int, list[RunData]], path: Path) -> None:
+    """Grouped bar chart of mean throughput (batches/s) per fault count.
+
+    Shows percentage of baseline above each bar.  Error bars show ±1σ when
+    multiple trials are provided per configuration.
+    """
+    fault_counts = sorted(runs_by_faults.keys())
+    means, stds, counts = [], [], []
+    baseline_mean = None
+
+    for nf in fault_counts:
+        tp_vals = [r.summary.throughput for r in runs_by_faults[nf]
+                   if r.summary.throughput > 0]
+        if not tp_vals:
+            means.append(0.0); stds.append(0.0); counts.append(0)
+            continue
+        m = statistics.mean(tp_vals)
+        σ = statistics.stdev(tp_vals) if len(tp_vals) > 1 else 0.0
+        means.append(m); stds.append(σ); counts.append(len(tp_vals))
+        if nf == 0:
+            baseline_mean = m
+
+    if not any(m > 0 for m in means):
+        print("  [degradation] SKIP — no throughput data found.")
+        return
+
+    x      = np.arange(len(fault_counts))
+    colors = [COLORS["baseline"] if fc == 0 else COLORS["normal"] for fc in fault_counts]
+
+    fig, ax = plt.subplots(figsize=(max(6, len(fault_counts) * 1.5 + 2), 5))
+    bars = ax.bar(x, means, yerr=stds, capsize=5, color=colors, alpha=0.85,
+                  error_kw={"elinewidth": 1.5, "ecolor": "black", "capthick": 1.5})
+
+    # Annotate with percentage of baseline
+    if baseline_mean and baseline_mean > 0:
+        for i, (m, σ) in enumerate(zip(means, stds)):
+            if m <= 0:
+                continue
+            pct = 100.0 * m / baseline_mean
+            offset = σ + max(baseline_mean * 0.02, 0.5)
+            ax.text(i, m + offset, f"{pct:.1f}%",
+                    ha="center", va="bottom", fontsize=9, fontweight="bold")
+        ax.axhline(baseline_mean, color=COLORS["baseline"],
+                   linestyle="--", linewidth=1.2, alpha=0.65,
+                   label=f"Baseline: {baseline_mean:.1f} batches/s")
+        ax.legend(framealpha=0.92)
+
+    ax.set_xticks(x)
+    ax.set_xticklabels([f"{fc} fault{'s' if fc != 1 else ''}" for fc in fault_counts])
+    ax.set_xlabel("Faults injected per run")
+    ax.set_ylabel("Throughput (batches / s)")
+    ax.set_ylim(bottom=0)
+    ax.set_title("Throughput Degradation vs. Fault Count")
+
+    if any(c > 1 for c in counts):
+        fig.text(0.99, 0.01, "Error bars: ±1σ across trials",
+                 ha="right", va="bottom", fontsize=8, color="grey")
+
+    fig.tight_layout()
+    fig.savefig(path)
+    plt.close(fig)
+    print(f"  [degradation] → {path}")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Helpers
+# ─────────────────────────────────────────────────────────────────────────────
+
+def load_log(p: Path) -> RunData:
+    data = parse_log(p.read_text())
+    s = data.summary
+    print(f"  Loaded {p.name}: mode={s.mode} workers={s.num_workers} "
+          f"batches={s.total_batches} faults={s.faults_injected} "
+          f"throughput={s.throughput:.1f} batches/s")
+    return data
+
+
+def run_binary(binary: Path, extra_args: list, cwd: Path) -> RunData:
+    cmd = [str(binary.resolve())] + extra_args
+    print(f"  Running: {' '.join(cmd)}", flush=True)
+    result = subprocess.run(cmd, capture_output=True, text=True,
+                             cwd=str(cwd.resolve()), timeout=7200)
+    if result.returncode != 0:
+        print(f"  WARNING: binary exited {result.returncode}", file=sys.stderr)
+        print(result.stderr[:500], file=sys.stderr)
+    return parse_log(result.stdout)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Entry point
+# ─────────────────────────────────────────────────────────────────────────────
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Generate SC26 fault-tolerance paper plots.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+    parser.add_argument("--log",      type=Path, metavar="FILE",
+                        help="Fault-injection run log file")
+    parser.add_argument("--baseline", type=Path, metavar="FILE",
+                        help="No-fault baseline run log file")
+    parser.add_argument("--log-dir",  type=Path, metavar="DIR",
+                        help="Directory of fault-injection .log/.txt files (for CDF)")
+    parser.add_argument("--run",      type=Path, metavar="BINARY",
+                        help="Path to the fault-tolerance binary (run it directly)")
+    parser.add_argument("--args",     type=str,  default="", metavar="ARGS",
+                        help="Extra CLI arguments forwarded to --run binary")
+    parser.add_argument("--fault1",   type=Path, metavar="FILE",
+                        help="Log for 1-fault run  (degradation plot)")
+    parser.add_argument("--fault2",   type=Path, metavar="FILE",
+                        help="Log for 2-fault run  (degradation plot)")
+    parser.add_argument("--fault3",   type=Path, metavar="FILE",
+                        help="Log for 3-fault run  (degradation plot)")
+    parser.add_argument("--out-dir",  type=Path, default=Path("plots"),
+                        help="Output directory (default: ./plots/)")
+    parser.add_argument("--no-pdf",   action="store_true",
+                        help="Save as PNG instead of PDF (useful for quick previews)")
+    parser.add_argument("--window",   type=int, default=5, metavar="W",
+                        help="Rolling window size for throughput plot (default: 5)")
+    args = parser.parse_args()
+
+    if not any([args.log, args.baseline, args.log_dir, args.run,
+                args.fault1, args.fault2, args.fault3]):
+        parser.error(
+            "No input provided. Pass --log, --baseline, --run, --log-dir, "
+            "or --fault1/2/3."
+        )
+
+    out_dir = args.out_dir
+    out_dir.mkdir(parents=True, exist_ok=True)
+    ext = ".png" if args.no_pdf else ".pdf"
+
+    plt.rcParams.update(STYLE)
+
+    # ── Collect fault-injection run(s) ───────────────────────────────────────
+    fault_runs: list[RunData] = []
+
+    if args.run:
+        cwd   = args.run.parent
+        extra = args.args.split() if args.args else []
+        fault_runs.append(run_binary(args.run, extra, cwd))
+
+    if args.log:
+        fault_runs.append(load_log(args.log))
+
+    if args.log_dir:
+        log_files = sorted(args.log_dir.glob("*.log")) + sorted(args.log_dir.glob("*.txt"))
+        if not log_files:
+            print(f"  WARNING: no .log/.txt files found in {args.log_dir}", file=sys.stderr)
+        for lf in log_files:
+            fault_runs.append(load_log(lf))
+
+    # ── Baseline ─────────────────────────────────────────────────────────────
+    baseline: Optional[RunData] = None
+    if args.baseline:
+        baseline = load_log(args.baseline)
+
+    # ── Primary run ──────────────────────────────────────────────────────────
+    primary = fault_runs[0] if fault_runs else baseline
+    if primary is None:
+        parser.error("No primary run available. Pass at least one of --log / --run / --baseline.")
+
+    s = primary.summary
+    print(f"\nPrimary: mode={s.mode} workers={s.num_workers} "
+          f"batches={s.total_batches} faults={s.faults_injected} "
+          f"throughput={s.throughput:.1f} batches/s\n")
+
+    # ── Plots ─────────────────────────────────────────────────────────────────
+    print(f"Generating plots → {out_dir}/\n")
+
+    plot_gantt(primary, out_dir / f"gantt{ext}")
+    plot_throughput(primary, baseline, out_dir / f"throughput{ext}", window=args.window)
+    plot_pi_convergence(primary, out_dir / f"pi_convergence{ext}")
+    plot_recovery_cdf(fault_runs if fault_runs else [primary], out_dir / f"recovery_cdf{ext}")
+
+    # Degradation: assemble {num_faults → [RunData]}
+    runs_by_faults: dict[int, list[RunData]] = defaultdict(list)
+    if baseline:
+        runs_by_faults[0].append(baseline)
+    for r in fault_runs:
+        runs_by_faults[r.summary.faults_injected].append(r)
+    for nf, path_arg in [(1, args.fault1), (2, args.fault2), (3, args.fault3)]:
+        if path_arg:
+            runs_by_faults[nf].append(load_log(path_arg))
+
+    if len(runs_by_faults) >= 2:
+        plot_degradation(dict(runs_by_faults), out_dir / f"degradation{ext}")
+    else:
+        print("  [degradation] SKIP — need logs for ≥2 fault counts "
+              "(pass --baseline and --log, or explicit --fault1/2/3).")
+
+    print("\nDone.")
+
+
+if __name__ == "__main__":
+    main()

From 603c0200d4a6c99d995624e1dd0c8d405a9b1fc3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 7 Apr 2026 18:39:38 -0600
Subject: [PATCH 0570/1000] Comminting scripts.

---
 .../parse_gpu_scaling.py                      | 128 ++++++++++++++++++
 .../Multi-GPU-Scaling-test/run_test.sh        |  31 +++++
 2 files changed, 159 insertions(+)
 create mode 100644 libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/parse_gpu_scaling.py
 create mode 100755 libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/run_test.sh

diff --git a/libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/parse_gpu_scaling.py b/libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/parse_gpu_scaling.py
new file mode 100644
index 0000000000..077beec867
--- /dev/null
+++ b/libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/parse_gpu_scaling.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+import re
+import argparse
+from pathlib import Path
+from collections import defaultdict
+from statistics import mean
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+
+# Only match NO scheduler sections
+NO_SCHED_PATTERN = re.compile(
+    r"Random Scaling NO scheduler \| actors=(\d+)\s+"
+    r"===== SUPERVISOR TOTAL TIME spawn =====.*?"
+    r"===== SUPERVISOR TOTAL TIME =====\s*"
+    r"Total runtime:\s*([0-9]*\.?[0-9]+)\s*s",
+    re.DOTALL,
+)
+
+
+def parse_file(path: Path):
+    text = path.read_text(errors="replace")
+    results = []
+    for actors_str, runtime_str in NO_SCHED_PATTERN.findall(text):
+        results.append((int(actors_str), float(runtime_str)))
+    return results
+
+
+def collect_data(base_dir: Path):
+    """
+    data[actors][gpu_count] = [runtime1, runtime2, ...]
+    """
+    data = defaultdict(lambda: defaultdict(list))
+
+    for gpu_dir in sorted(base_dir.glob("gpus_*")):
+        if not gpu_dir.is_dir():
+            continue
+
+        m = re.match(r"gpus_(\d+)$", gpu_dir.name)
+        if not m:
+            continue
+        gpu_count = int(m.group(1))
+
+        for txt_file in sorted(gpu_dir.glob("*.txt")):
+            for actors, runtime in parse_file(txt_file):
+                data[actors][gpu_count].append(runtime)
+
+    return data
+
+
+def make_plots(data, out_dir: Path):
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    for actors in sorted(data.keys()):
+        gpu_map = data[actors]
+        gpu_counts = sorted(gpu_map.keys())
+        if not gpu_counts:
+            continue
+
+        means = [mean(gpu_map[g]) for g in gpu_counts]
+
+        # 🔥 FIX: use categorical spacing instead of numeric spacing
+        x_pos = list(range(len(gpu_counts)))
+
+        fig, ax = plt.subplots(figsize=(8, 5))
+        bars = ax.bar(x_pos, means)
+
+        # Add value labels on top of bars
+        for bar, value in zip(bars, means):
+            ax.text(
+                bar.get_x() + bar.get_width() / 2,
+                bar.get_height(),
+                f"{value:.2f}",
+                ha="center",
+                va="bottom",
+                fontsize=9,
+            )
+
+        fig.suptitle("multi-gpu scaling test on gpufarm7")
+        ax.set_title(f"actors = {actors}")
+        ax.set_xlabel("Number of GPUs")
+        ax.set_ylabel("Runtime (s)")
+
+        # Show actual GPU counts as labels (1,2,4,7) but evenly spaced
+        ax.set_xticks(x_pos)
+        ax.set_xticklabels(gpu_counts)
+
+        ax.grid(True, axis="y", alpha=0.3)
+
+        fig.tight_layout(rect=[0, 0, 1, 0.95])
+
+        out_file = out_dir / f"multi_gpu_scaling_actors_{actors}.png"
+        fig.savefig(out_file, dpi=200)
+        plt.close(fig)
+
+        print(f"Saved {out_file}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate GPU scaling bar charts (NO scheduler only)"
+    )
+    parser.add_argument(
+        "--base-dir",
+        type=Path,
+        default=Path("."),
+        help="Directory containing gpus_* folders",
+    )
+    parser.add_argument(
+        "--out-dir",
+        type=Path,
+        default=Path("gpu_scaling_plots"),
+        help="Output directory for plots",
+    )
+    args = parser.parse_args()
+
+    data = collect_data(args.base_dir)
+
+    if not data:
+        raise SystemExit("No data found.")
+
+    make_plots(data, args.out_dir)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/run_test.sh b/libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/run_test.sh
new file mode 100755
index 0000000000..9cc3b17b2a
--- /dev/null
+++ b/libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/run_test.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+BASE_DIR=/student/nqr159/data/scheduler-test/hetergenous-workloads/gpu_scaling_tests
+BIN=/student/nqr159/gpufarm5/actor-framework/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/build/test
+
+mkdir -p "$BASE_DIR"
+
+# Define GPU configurations
+declare -A GPU_CONFIGS
+GPU_CONFIGS[1]="0"
+GPU_CONFIGS[2]="0,1"
+GPU_CONFIGS[4]="0,1,2,3"
+GPU_CONFIGS[7]="0,1,2,3,4,5,6"
+
+for gpus in 1 2 4 7; do
+  echo "=============================="
+  echo "Testing with $gpus GPU(s)"
+  echo "=============================="
+
+  OUT_DIR="$BASE_DIR/gpus_${gpus}"
+  mkdir -p "$OUT_DIR"
+
+  CUDA_DEVICES=${GPU_CONFIGS[$gpus]}
+
+  for i in {1..10}; do
+    echo "Running iteration $i with $gpus GPU(s)..."
+
+    CUDA_VISIBLE_DEVICES=$CUDA_DEVICES \
+    "$BIN" > "$OUT_DIR/output_${i}.txt"
+  done
+done

From 163e9b91b0958868a0232a75938f0b802eefb971 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 4 May 2026 13:08:33 -0600
Subject: [PATCH 0571/1000] Updated code to compile correctly based on cuda
 version.

---
 libcaf_cuda/src/platform.cpp | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/src/platform.cpp b/libcaf_cuda/src/platform.cpp
index c1477a84e8..561b49887d 100644
--- a/libcaf_cuda/src/platform.cpp
+++ b/libcaf_cuda/src/platform.cpp
@@ -29,11 +29,22 @@ platform::platform() {
     check(cuDeviceGetName(name, sizeof(name), cuda_device), "cuDeviceGetName");
     device_names[i] = name;
 
-//Use this if cuCtxCreate throws a compiler error
-//    check(cuCtxCreate(&contexts_[i],nullptr ,CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, cuda_device), "cuCtxCreate");
 
-    check(cuCtxCreate(&contexts_[i], CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, cuda_device), "cuCtxCreate");
+#if CUDA_VERSION >= 13000
+    {
+      CUctxCreateParams ctx_params = {};
+      check(cuCtxCreate(&contexts_[i], 
+                        &ctx_params, 
+                        CU_CTX_SCHED_BLOCKING_SYNC | CU_CTX_MAP_HOST, 
+                        cuda_device),"creating context");
+    }
+#else
+    check(cuCtxCreate(&contexts_[i], 
+                      CU_CTX_SCHED_BLOCKING_SYNC | CU_CTX_MAP_HOST, 
+                      cuda_device),"creating context");
+#endif
     devices_[i] = make_counted<device>(cuda_device, contexts_[i], name, i);
+ 
   }
 
   // Check if all devices are the same by comparing their names

From b9e605d6fdef8517592c29d6f0bfce640cd0ba8d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 5 May 2026 08:32:14 -0600
Subject: [PATCH 0572/1000] Updated script to be compatable with the
 run_runtime_overhead.py script.

---
 .../analyze_mmul_benchmarks.py                | 182 +++++++++---------
 1 file changed, 94 insertions(+), 88 deletions(-)

diff --git a/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py b/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
index 8c65efd1d4..773b058a6a 100644
--- a/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
+++ b/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
@@ -1,92 +1,98 @@
 #!/usr/bin/env python3
 
-import glob
 import re
-import numpy as np
 import matplotlib.pyplot as plt
-from collections import defaultdict
-
-# Directory containing benchmark outputs
-DATA_DIR = "/student/nqr159/data/mmul-actor-test/benchmark-results"
-
-# Patterns
-driver_files = glob.glob(f"{DATA_DIR}/matrix_mul_driver_run*.txt")
-actor_files = glob.glob(f"{DATA_DIR}/test_run*.txt")
-
-# Regex
-size_pattern = re.compile(r"N=(\d+)")
-driver_total_pattern = re.compile(r"TOTAL:\s+([\d.]+)")
-actor_total_pattern = re.compile(r"TOTAL end-to-end:\s+([\d.]+)")
-
-driver_data = defaultdict(list)
-actor_data = defaultdict(list)
-
-
-def parse_driver(file):
-    with open(file) as f:
-        current_size = None
-        for line in f:
-            size_match = size_pattern.search(line)
-            if size_match:
-                current_size = int(size_match.group(1))
-
-            total_match = driver_total_pattern.search(line)
-            if total_match and current_size:
-                driver_data[current_size].append(float(total_match.group(1)))
-
-
-def parse_actor(file):
-    with open(file) as f:
-        current_size = None
-        for line in f:
-            size_match = size_pattern.search(line)
-            if size_match:
-                current_size = int(size_match.group(1))
-
-            total_match = actor_total_pattern.search(line)
-            if total_match and current_size:
-                actor_data[current_size].append(float(total_match.group(1)))
-
-
-# Parse files
-for f in driver_files:
-    parse_driver(f)
-
-for f in actor_files:
-    parse_actor(f)
-
-
-# Compute means
-sizes = sorted(driver_data.keys())
-
-driver_means = [np.mean(driver_data[s]) for s in sizes]
-actor_means = [np.mean(actor_data[s]) for s in sizes]
-
-# Compute differences
-abs_diff = [a - d for d, a in zip(driver_means, actor_means)]
-speedup = [d / a if a != 0 else float('inf') for d, a in zip(driver_means, actor_means)]
-percent_diff = [((a - d) / d) * 100 if d != 0 else 0 for d, a in zip(driver_means, actor_means)]
-
-print("===== MEAN RESULTS =====")
-print("N     | CUDA (ms) | Actors (ms) | Diff (ms) | % Diff | Speedup (CUDA/Actors)")
-print("-" * 80)
-
-for s, d, a, diff, pct, sp in zip(sizes, driver_means, actor_means, abs_diff, percent_diff, speedup):
-    print(f"N={s:5d} | {d:10.3f} | {a:11.3f} | {diff:9.3f} | {pct:7.2f}% | {sp:8.3f}")
-
-
-# Plot
-plt.figure()
-
-plt.plot(sizes, driver_means, marker='o', label="CUDA (Driver)")
-plt.plot(sizes, actor_means, marker='s', label="CUDA Actors")
-
-plt.xlabel("Matrix Size (N)")
-plt.ylabel("Mean Execution Time (ms)")
-plt.title("CUDA vs CUDA Actor Matrix Multiplication Performance")
-plt.legend()
-plt.grid(True)
-
-plt.savefig(f"{DATA_DIR}/mmul_benchmark_plot.png", dpi=300)
-
-plt.show()
+from matplotlib.ticker import ScalarFormatter
+from pathlib import Path
+
+# Path Configuration
+# This script is located in sc26/scripts/Runtime-Overhead/
+# We expect results in sc26/Runtime-Overhead/results/
+SCRIPT_DIR = Path(__file__).resolve().parent
+RESULTS_FILE = SCRIPT_DIR.parent.parent / "Runtime-Overhead" / "results" / "benchmark_results.txt"
+OUTPUT_PLOT = SCRIPT_DIR.parent.parent / "Runtime-Overhead" / "results" / "runtime_overhead_comparison.png"
+
+def parse_comparison_table():
+    """Parses the 'CROSS-IMPLEMENTATION COMPARISON' table from the results file."""
+    if not RESULTS_FILE.exists():
+        print(f"Error: Results file not found at {RESULTS_FILE}")
+        return None
+
+    with open(RESULTS_FILE, 'r', encoding='utf-8') as f:
+        content = f.read()
+
+    # Locate the comparison section. It uses a specific header format.
+    # We capture the data rows between the dashes and the next double newline.
+    table_pattern = re.compile(
+        r"CROSS-IMPLEMENTATION COMPARISON.*?N\s+cuda_native\s+actor_facade\s+command_runner.*?\n-+\n(.*?)(?=\n\n|\Z)",
+        re.DOTALL
+    )
+    
+    match = table_pattern.search(content)
+    if not match:
+        print("Error: Could not find the comparison table in the benchmark results.")
+        return None
+
+    data = {
+        "N": [],
+        "cuda_native": [],
+        "actor_facade": [],
+        "command_runner": []
+    }
+
+    rows = match.group(1).strip().splitlines()
+    for row in rows:
+        parts = row.split()
+        # Expected format: N, cuda_native_mean, actor_facade_mean, command_runner_mean, ...
+        if len(parts) >= 4:
+            try:
+                data["N"].append(int(parts[0]))
+                data["cuda_native"].append(float(parts[1]))
+                data["actor_facade"].append(float(parts[2]))
+                data["command_runner"].append(float(parts[3]))
+            except ValueError:
+                continue # Skip header/footer noise if any
+
+    return data
+
+def generate_plot(data):
+    """Generates a PNG graph comparing mean execution times."""
+    if not data or not data["N"]:
+        return
+
+    plt.figure(figsize=(11, 7))
+
+    # Plotting each implementation
+    plt.plot(data["N"], data["cuda_native"], marker='o', linestyle='-', label='CUDA Native (Baseline)')
+    plt.plot(data["N"], data["actor_facade"], marker='s', linestyle='--', label='CAF Actor Facade')
+    plt.plot(data["N"], data["command_runner"], marker='^', linestyle=':', label='CAF Command Runner')
+
+    plt.title('Matrix Multiplication Performance by Size', fontsize=14, fontweight='bold')
+    plt.xlabel('Matrix Size N', fontsize=12)
+    plt.ylabel('Mean Execution Time (ms)', fontsize=12)
+    
+    plt.grid(True, which="both", linestyle='--', alpha=0.6)
+    plt.legend(fontsize=10)
+    
+    # Apply log scale for better visualization if the N range is large
+    if max(data["N"]) / min(data["N"]) > 10:
+        plt.xscale('log')
+        plt.yscale('log')
+
+    # Set X-axis ticks to exactly the N values tested to avoid abbreviation
+    plt.xticks(data["N"], data["N"])
+
+    # Force scalar formatting for Y axis to avoid scientific notation (e.g., 10^x)
+    ax = plt.gca()
+    formatter = ScalarFormatter()
+    formatter.set_scientific(False)
+    ax.yaxis.set_major_formatter(formatter)
+
+    plt.tight_layout()
+    plt.savefig(OUTPUT_PLOT, dpi=300)
+    print(f"Successfully generated comparison graph: {OUTPUT_PLOT}")
+
+if __name__ == "__main__":
+    results = parse_comparison_table()
+    if results:
+        generate_plot(results)

From 756317638f7ce71f64fc0c0abbccd10202864b72 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 6 May 2026 08:46:22 -0600
Subject: [PATCH 0573/1000] update plot script to charge a log file if
 specified one does not already existi

---
 .../sc26/scripts/Fault-Tolerance/plot.py      | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/libcaf_cuda/sc26/scripts/Fault-Tolerance/plot.py b/libcaf_cuda/sc26/scripts/Fault-Tolerance/plot.py
index cbf4f1ce94..c586f0ffed 100644
--- a/libcaf_cuda/sc26/scripts/Fault-Tolerance/plot.py
+++ b/libcaf_cuda/sc26/scripts/Fault-Tolerance/plot.py
@@ -738,6 +738,42 @@ def main() -> None:
     # ── Baseline ─────────────────────────────────────────────────────────────
     baseline: Optional[RunData] = None
     if args.baseline:
+        if not args.baseline.exists():
+            if args.run:
+                print(f"  Baseline log {args.baseline} not found. Generating it now...")
+                cwd = args.run.parent
+                base_extra = args.args.split() if args.args else []
+                # Remove any existing fault injection flags and force baseline mode.
+                filtered = []
+                skip = False
+                for a in base_extra:
+                    if skip:
+                        skip = False
+                        continue
+                    if a in ("--num-faults", "-F"):
+                        skip = True
+                        continue
+                    if a in ("--no-fault", "-n"):
+                        continue
+                    filtered.append(a)
+                filtered.append("--no-fault")
+
+                cmd = [str(args.run.resolve())] + filtered
+                print(f"  Executing baseline run: {' '.join(cmd)}", flush=True)
+                res = subprocess.run(cmd, capture_output=True, text=True,
+                                     cwd=str(cwd.resolve()), timeout=7200)
+                if res.returncode == 0:
+                    args.baseline.parent.mkdir(parents=True, exist_ok=True)
+                    args.baseline.write_text(res.stdout)
+                    print(f"  Baseline log saved to {args.baseline}")
+                else:
+                    print(f"  FATAL: Baseline generation failed (exit {res.returncode})", file=sys.stderr)
+                    sys.exit(res.returncode)
+            else:
+                print(f"  ERROR: Baseline file {args.baseline} not found and no --run binary provided to create it.",
+                      file=sys.stderr)
+                sys.exit(1)
+
         baseline = load_log(args.baseline)
 
     # ── Primary run ──────────────────────────────────────────────────────────

From e4abbaa0d0244b1cff22cf6adac7d78acee31d0f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 6 May 2026 08:47:24 -0600
Subject: [PATCH 0574/1000] Update script. '

---
 .../generate_graphs.py                        | 84 +++++++++++--------
 1 file changed, 48 insertions(+), 36 deletions(-)

diff --git a/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py b/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py
index 13e2cefda4..6a27697b17 100644
--- a/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py
+++ b/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py
@@ -3,65 +3,77 @@
 import re
 import numpy as np
 import matplotlib.pyplot as plt
-from collections import defaultdict
-
-cuda_data = defaultdict(list)
-caf_data = defaultdict(list)
+from pathlib import Path
 
 # -----------------------------
-# Parse CUDA baseline logs
+# Path Configuration
 # -----------------------------
-for i in range(1, 11):
-    filename = f"matrix_mul_driver_run{i}.txt"
-    with open(filename) as f:
-        for line in f:
-            m = re.search(r"iterations = (\d+), total GPU time = ([0-9.]+)", line)
-            if m:
-                it = int(m.group(1))
-                time = float(m.group(2))
-                cuda_data[it].append(time)
+SCRIPT_DIR = Path(__file__).resolve().parent
+RESULTS_DIR = SCRIPT_DIR.parent.parent / "Sequence-Independent-Tasks" / "results"
+RESULTS_FILE = RESULTS_DIR / "benchmark_results.txt"
+OUTPUT_PLOT = RESULTS_DIR / "mmul_comparison.png"
 
 # -----------------------------
-# Parse CAF CUDA logs
+# Parse Benchmark Results
 # -----------------------------
-for i in range(1, 11):
-    filename = f"test_run{i}.txt"
-    with open(filename) as f:
-        for line in f:
-            m = re.search(r"iterations = (\d+), time=([0-9.]+)", line)
-            if m:
-                it = int(m.group(1))
-                time = float(m.group(2))
-                caf_data[it].append(time)
+def parse_results():
+    if not RESULTS_FILE.exists():
+        print(f"Error: Results file not found at {RESULTS_FILE}")
+        return None
+
+    with open(RESULTS_FILE, 'r') as f:
+        content = f.read()
+
+    # Extract the comparison table data
+    # Row format: series milestone main_cuda_native main_actor_facade main_command_runner ...
+    table_pattern = re.compile(
+        r"CROSS-IMPLEMENTATION COMPARISON.*?series\s+milestone.*?\n\s*-+\n(.*?)(?=\n\n|\Z)",
+        re.DOTALL
+    )
+    
+    match = table_pattern.search(content)
+    if not match:
+        print("Error: Could not find comparison table in results.")
+        return None
 
-iterations = sorted(cuda_data.keys())
+    data = {"it": [], "cuda": [], "facade": [], "runner": []}
+    for line in match.group(1).strip().splitlines():
+        parts = line.split()
+        if len(parts) >= 5:
+            data["it"].append(int(parts[1]))      # milestone
+            data["cuda"].append(float(parts[2]))    # main_cuda_native
+            data["facade"].append(float(parts[3]))  # main_actor_facade
+            data["runner"].append(float(parts[4]))  # main_command_runner
+    return data
 
-cuda_mean = [np.mean(cuda_data[it]) for it in iterations]
-caf_mean = [np.mean(caf_data[it]) for it in iterations]
+data = parse_results()
+if not data:
+    exit(1)
 
 # -----------------------------
 # CLI Output
 # -----------------------------
 print("\nMean Performance Comparison\n")
-print(f"{'Iterations':>10} {'CUDA(ms)':>12} {'CAF CUDA(ms)':>15} {'Diff(ms)':>12} {'Overhead %':>12}")
+print(f"{'Iterations':>10} {'CUDA(ms)':>12} {'Facade(ms)':>12} {'Runner(ms)':>12} {'Facade Ovhd %':>15}")
 
-for i, it in enumerate(iterations):
-    diff = caf_mean[i] - cuda_mean[i]
-    pct = (diff / cuda_mean[i]) * 100
-    print(f"{it:>10} {cuda_mean[i]:>12.2f} {caf_mean[i]:>15.2f} {diff:>12.2f} {pct:>11.2f}%")
+for i in range(len(data["it"])):
+    pct = ((data["facade"][i] - data["cuda"][i]) / data["cuda"][i]) * 100
+    print(f"{data['it'][i]:>10} {data['cuda'][i]:>12.2f} {data['facade'][i]:>12.2f} {data['runner'][i]:>12.2f} {pct:>14.2f}%")
 
 # -----------------------------
 # Plot
 # -----------------------------
 plt.figure(figsize=(8,6))
-plt.plot(iterations, cuda_mean, marker='o', label="CUDA")
-plt.plot(iterations, caf_mean, marker='o', label="CAF CUDA")
+plt.plot(data["it"], data["cuda"], marker='o', label="CUDA Native")
+plt.plot(data["it"], data["facade"], marker='s', label="CAF Actor Facade")
+plt.plot(data["it"], data["runner"], marker='^', label="CAF Command Runner")
 
-plt.xlabel("Iterations")
+plt.xlabel("Iterations (Milestone)")
 plt.ylabel("Time (ms)")
 plt.title("Matrix Multiplication Performance")
 plt.legend()
 plt.grid(True)
+plt.tight_layout()
 
-plt.savefig("mmul_comparison.png")
+plt.savefig(OUTPUT_PLOT)
 plt.show()

From 77d6e90bfb22072c8ab1444edc0b616deafc96fd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 6 May 2026 09:50:05 -0600
Subject: [PATCH 0575/1000] Modified run async to properly use device numbers.
 Change was made to fix an error where when using multiple GPUs, since no
 device was selected it would use a lotery scheduler  causing memory device
 mismatch errors

---
 libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp           | 2 +-
 libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
index 216fefb423..26cf1ebc1e 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
@@ -104,7 +104,7 @@ caf::behavior mmul_actor(caf::stateful_actor<mmul_state>* self) {
       caf::cuda::mmul_async_command<int> command;
       auto output = command.run_async(
 		      program,dims,
-		      1,
+		      1, 0, device,
 		      arg1,arg2,out<int>{N*N},in<int>{N});
 
        auto t_response_received = clock::now();
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
index bfe615f729..3f7472854b 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
@@ -97,7 +97,7 @@ return {
       auto output = cmd.run_async(
 		      st.program,
 		      dims,
-		      1,
+		      1, 0, device,
 		      arg1, arg2, out<int>{N * N}, in<int>{N});
 
 	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);

From 202d2e721847e06e0e715e9c403bc80dcbd0dab0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 7 May 2026 11:38:15 -0600
Subject: [PATCH 0576/1000] changed thread scheduling from CU_CTX_BLOCKING to
 CU_CTX_AUTO, this change is being made since it does utilmately affect
 performance and it threw some perfomance tests way out since CUDA was usiing
 ctx auto and CAF CUDA was not using ctx auto but ctx block sync

---
 libcaf_cuda/src/platform.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/src/platform.cpp b/libcaf_cuda/src/platform.cpp
index 561b49887d..ff63e42420 100644
--- a/libcaf_cuda/src/platform.cpp
+++ b/libcaf_cuda/src/platform.cpp
@@ -35,12 +35,12 @@ platform::platform() {
       CUctxCreateParams ctx_params = {};
       check(cuCtxCreate(&contexts_[i], 
                         &ctx_params, 
-                        CU_CTX_SCHED_BLOCKING_SYNC | CU_CTX_MAP_HOST, 
+                        CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, 
                         cuda_device),"creating context");
     }
 #else
     check(cuCtxCreate(&contexts_[i], 
-                      CU_CTX_SCHED_BLOCKING_SYNC | CU_CTX_MAP_HOST, 
+                      CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, 
                       cuda_device),"creating context");
 #endif
     devices_[i] = make_counted<device>(cuda_device, contexts_[i], name, i);
@@ -121,4 +121,3 @@ void platform::release_streams_for_actor(int actor_id) {
 }
 
 } // namespace caf::cuda
-

From 6398cd7a1caa0858fe38ea4d48d2649613d1ced9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 7 May 2026 12:44:27 -0600
Subject: [PATCH 0577/1000] initial commit.

---
 .../actor-tests/callback-test/CMakeLists.txt  |  44 +++
 .../callback-test/compile_kernels.sh          |  14 +
 .../actor-tests/callback-test/main.test.cpp   | 252 ++++++++++++++++++
 .../tests/actor-tests/callback-test/mmul.cu   |  16 ++
 4 files changed, 326 insertions(+)
 create mode 100644 libcaf_cuda/tests/actor-tests/callback-test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/actor-tests/callback-test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/actor-tests/callback-test/main.test.cpp
 create mode 100644 libcaf_cuda/tests/actor-tests/callback-test/mmul.cu

diff --git a/libcaf_cuda/tests/actor-tests/callback-test/CMakeLists.txt b/libcaf_cuda/tests/actor-tests/callback-test/CMakeLists.txt
new file mode 100644
index 0000000000..89bcf5ba7c
--- /dev/null
+++ b/libcaf_cuda/tests/actor-tests/callback-test/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
diff --git a/libcaf_cuda/tests/actor-tests/callback-test/compile_kernels.sh b/libcaf_cuda/tests/actor-tests/callback-test/compile_kernels.sh
new file mode 100755
index 0000000000..bf9c447a49
--- /dev/null
+++ b/libcaf_cuda/tests/actor-tests/callback-test/compile_kernels.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile mmul.cu to cubin in current directory
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+echo "Generated mmul.cubin"
+
+echo "All kernels compiled successfully!"
+
diff --git a/libcaf_cuda/tests/actor-tests/callback-test/main.test.cpp b/libcaf_cuda/tests/actor-tests/callback-test/main.test.cpp
new file mode 100644
index 0000000000..83a44f3249
--- /dev/null
+++ b/libcaf_cuda/tests/actor-tests/callback-test/main.test.cpp
@@ -0,0 +1,252 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+// Define a custom type ID block for custom actors
+CAF_ADD_ATOM(cuda,shared_mem)
+
+
+
+
+
+
+#include <chrono>
+#include <iostream>
+
+// Extend your actor state to keep the start time
+struct mmul_actor_state {
+  static inline const char* name = "my_actor";
+  int last_N = 0; // example state variable
+  int id = rand(); 
+  // per-actor timing start
+  std::chrono::high_resolution_clock::time_point start_time;
+  int times = 0;
+};
+
+
+
+
+//commands classes used to launch kernels 
+using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
+using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
+
+using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
+
+mmulCommand mmul;
+matrixGenCommand randomMatrix;
+mmulAsyncCommand mmulAsync;
+
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+
+// Stateful actor behavior
+caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
+  return {
+    // 1st handler: Just int N, and who to send the matrices to
+    [=](int N, std::vector<caf::actor> receivers) {
+
+        caf::cuda::manager& mgr = caf::cuda::manager::get();
+        //create the program and configure the dimesnions of the kernel
+        auto program = mgr.create_program_from_fatbin("../generate_random_matrix.fatbin","generate_random_matrix");
+	int THREADS = 256;
+	int BLOCKS = (N*N + THREADS - 1) / THREADS;
+  	caf::cuda::nd_range dim(BLOCKS,1, 1, THREADS,1, 1);
+
+	//tag the arguments so that caf::cuda knows what to do with them	
+         auto arg1 = caf::cuda::create_out_arg(N*N); //output buffer indicate its size, caf::cuda will handle the rest
+          auto arg2 = caf::cuda::create_in_arg(N*N); //matrix size
+          auto arg3 = caf::cuda::create_in_arg(rand()); //seed
+	  auto arg4 = caf::cuda::create_in_arg(9999); //max valux
+	  
+          auto arg3B = caf::cuda::create_in_arg(rand()); //seed
+	  int device_number= 74; //arbitary number to show that 
+				 //can give illusion of selecting gpus that are
+				 //not there
+
+
+	  //launch kernels and collect their outputs
+	  auto tempA = randomMatrix.run_async(program,dim, self -> state().id,0,device_number,arg1,arg2,arg3,arg4);
+	  auto tempB = randomMatrix.run_async(program,dim, self -> state().id,0,device_number,arg1,arg2,arg3B,arg4);
+	  caf::cuda::mem_ptr<int> matrixA =  std::get<0>(tempA);
+	  caf::cuda::mem_ptr<int> matrixB = std::get<0>(tempB);
+
+	  //ensure the data is actually done being worked on
+	  matrixA -> synchronize();
+	  matrixB -> synchronize();
+
+	  std::cout << "Broadcasting\n";
+	  //broadcast the result out to receviers.
+	  for (auto actor: receivers) {
+	  
+		  self->mail(3,matrixA,matrixB,N,device_number).send(actor);
+	  }
+
+    },
+
+    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
+    [=](const caf::cuda::mem_ptr<int> matrixA,
+        const caf::cuda::mem_ptr<int> matrixB, int N,int device_number) {
+ 
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //create program and dims   
+  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
+  const int THREADS = 32;
+  const int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    //create args
+    auto arg1 = matrixA;
+    auto arg2 = matrixB;
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+
+    auto tempC = mmulAsync.run(program,dims,self -> state().id,0,device_number,arg1,arg2,arg3,arg4);
+
+    std::vector<int> matrix1 = matrixA -> copy_to_host();
+    std::vector<int> matrix2 = matrixB -> copy_to_host();    
+    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC,2);
+
+    //verify its own result 
+    self -> mail(matrix1,matrix2,matrixC,N).send(self);
+
+    },
+
+ // 3nd handler: GPU atom + matrices + N, launches a kenrel using shared memory and sends its result to itself for verification
+    [=](int x,const caf::cuda::mem_ptr<int> matrixA,
+        const caf::cuda::mem_ptr<int> matrixB, int N,int device_number) {
+ 
+
+  caf::cuda::manager& mgr = caf::cuda::manager::get();
+
+  //create program and dims   
+  auto program = mgr.create_program_from_cubin("../shared_mmul.cubin","matrixMul");
+  const int THREADS = 32;
+  const int BLOCKS = (N + THREADS - 1) / THREADS;
+  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+  int shared_mem = 8192; //we need 8KB of shared memory here
+    //create args
+    auto arg1 = matrixA;
+    auto arg2 = matrixB;
+    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+
+    auto tempC = mmulAsync.run(program,dims,self -> state().id,shared_mem,device_number,arg1,arg2,arg3,arg4);
+
+    std::vector<int> matrix1 = matrixA -> copy_to_host();
+    std::vector<int> matrix2 = matrixB -> copy_to_host();    
+    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC,2);
+
+    //verify its own result 
+    self -> mail(matrix1,matrix2,matrixC,N).send(self);
+
+    },
+
+
+
+    // 3rd handler: CPU atom + matrices + N
+    [=](const std::vector<int>& matrixA,
+    const std::vector<int> &matrixB,
+    const std::vector<int> &matrixC, int N) {
+
+    std::vector<int> result(N * N);
+
+    serial_matrix_multiply(matrixA, matrixB, result, N);
+
+    if (result == matrixC) {
+        std::cout << "actor with id " << self->state().id << " references match\n";
+    }
+    else {
+        std::cout << "actor with id " << self->state().id << " references did not match\n";
+
+    }
+
+
+    /*
+    auto print_matrix = [N](const std::vector<int>& mat, const std::string& name) {
+            std::cout << name << ":\n";
+            for (int i = 0; i < N; ++i) {
+                for (int j = 0; j < N; ++j) {
+                    std::cout << mat[i * N + j] << " ";
+                }
+                std::cout << "\n";
+            }
+            std::cout << std::endl;
+        };
+
+        print_matrix(matrixA, "Matrix A");
+        print_matrix(matrixB, "Matrix B");
+        print_matrix(result, "Result Matrix");
+        print_matrix(matrixC, "GPU Result Matrix");
+	*/
+    self->quit();
+    }
+  };
+}
+
+
+void run_async_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
+  if (num_actors < 1) {
+    std::cerr << "[ERROR] Number of actors must be >= 1\n";
+    return;
+  }
+
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<caf::actor> actors;
+  actors.reserve(num_actors);
+  for (int i = 0; i < num_actors; ++i) {
+    actors.push_back(sys.spawn(mmul_async_actor_fun));
+  }
+
+  // Actor 0 generates matrices and broadcasts to others 
+  caf::anon_mail(matrix_size, actors).send(actors[0]);
+
+   sys.await_all_actors_done();
+}
+
+
+void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager::init(sys);
+
+  //run_async_mmul_test(sys,100,1);
+
+ 
+}
+
+
+
+CAF_MAIN()
diff --git a/libcaf_cuda/tests/actor-tests/callback-test/mmul.cu b/libcaf_cuda/tests/actor-tests/callback-test/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/actor-tests/callback-test/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From a4903beef4b7d4f79fe1a55e8303add43c33a727 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 7 May 2026 12:55:40 -0600
Subject: [PATCH 0578/1000] implement copy_to_host_async methods. Change was
 made to give a more actor like way of synchronizing rather then explicitly
 blocking.

---
 libcaf_cuda/caf/cuda/mem_ref.hpp | 87 +++++++++++++++++++++++++++++++-
 1 file changed, 86 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/mem_ref.hpp b/libcaf_cuda/caf/cuda/mem_ref.hpp
index fb7b0f85d9..21f95969f1 100644
--- a/libcaf_cuda/caf/cuda/mem_ref.hpp
+++ b/libcaf_cuda/caf/cuda/mem_ref.hpp
@@ -8,6 +8,7 @@
 #include <chrono>
 #include <iostream>
 #include "caf/cuda/types.hpp"
+#include <caf/actor.hpp>
 //#include "caf/cuda/utility.hpp"
 #include <cuda.h>
 #include <atomic>
@@ -153,6 +154,91 @@ class mem_ref : public caf::ref_counted {
 	  CHECK_CUDA(cuCtxPopCurrent(nullptr));
   }
 
+  /**
+   * @brief Asynchronously copies GPU memory to a host buffer and triggers a user-defined callback.
+   * 
+   * This version allows the user to provide their own destination buffer.
+   * 
+   * @param dst Pointer to the destination host memory. Must remain valid until the callback executes.
+   * @param count Number of elements to copy.
+   * @param callback A function/lambda with the signature void(T* data, size_t size).
+   */
+  template <typename F>
+  void copy_to_host_async(T* dst, size_t count, F callback) const {
+    if (access_ == IN)
+      throw std::runtime_error("Cannot copy a read-only buffer back to host");
+
+    struct State {
+      caf::intrusive_ptr<const mem_ref<T>> self;
+      T* dst;
+      size_t count;
+      F user_callback;
+    };
+
+    auto* state = new State{this, dst, count, std::move(callback)};
+
+    CHECK_CUDA(cuCtxPushCurrent(ctx));
+    CUstream s = stream_ ? stream_ : nullptr;
+
+    if (!is_scalar_) {
+      size_t bytes = count * sizeof(T);
+      CHECK_CUDA(cuMemcpyDtoHAsync(dst, memory_, bytes, s));
+    }
+
+    auto host_fn = [](void* userData) {
+      auto* s_ptr = static_cast<State*>(userData);
+      if (s_ptr->self->is_scalar_)
+        s_ptr->dst[0] = s_ptr->self->host_scalar_;
+      
+      s_ptr->user_callback(s_ptr->dst, s_ptr->count);
+      delete s_ptr;
+    };
+
+    CHECK_CUDA(cuLaunchHostFunc(s, host_fn, state));
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+  }
+
+  /**
+   * @brief Asynchronously copies GPU memory and provides an std::vector<T> to the callback.
+   * 
+   * This version manages the host buffer for you.
+   * 
+   * @param callback A function/lambda with the signature void(std::vector<T>&& data).
+   */
+  template <typename F>
+  void copy_to_host_async(F callback) const {
+    if (access_ == IN)
+      throw std::runtime_error("Cannot copy a read-only buffer back to host");
+
+    struct State {
+      caf::intrusive_ptr<const mem_ref<T>> self;
+      std::vector<T> buffer;
+      F user_callback;
+    };
+
+    auto* state = new State{this, std::vector<T>(num_elements_), std::move(callback)};
+
+    CHECK_CUDA(cuCtxPushCurrent(ctx));
+    CUstream s = stream_ ? stream_ : nullptr;
+
+    if (!is_scalar_) {
+      size_t bytes = num_elements_ * sizeof(T);
+      CHECK_CUDA(cuMemcpyDtoHAsync(state->buffer.data(), memory_, bytes, s));
+    }
+
+    auto host_fn = [](void* userData) {
+      auto* s_ptr = static_cast<State*>(userData);
+      if (s_ptr->self->is_scalar_)
+        s_ptr->buffer[0] = s_ptr->self->host_scalar_;
+
+      s_ptr->user_callback(std::move(s_ptr->buffer));
+      delete s_ptr;
+    };
+
+    CHECK_CUDA(cuLaunchHostFunc(s, host_fn, state));
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+  }
+
 
 
     //reference counting for auto garabage collection
@@ -193,4 +279,3 @@ template <class T>
 using mem_ptr = caf::intrusive_ptr<mem_ref<T>>;
 
 } // namespace caf::cuda
-

From bc7bd268626db52a32afda4d8d9b8218a0698476 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 7 May 2026 12:56:22 -0600
Subject: [PATCH 0579/1000] updated test to test the correctness of the new
 copy_to_host_async method

---
 .../actor-tests/callback-test/main.test.cpp   | 195 ++++--------------
 1 file changed, 44 insertions(+), 151 deletions(-)

diff --git a/libcaf_cuda/tests/actor-tests/callback-test/main.test.cpp b/libcaf_cuda/tests/actor-tests/callback-test/main.test.cpp
index 83a44f3249..9c8517d583 100644
--- a/libcaf_cuda/tests/actor-tests/callback-test/main.test.cpp
+++ b/libcaf_cuda/tests/actor-tests/callback-test/main.test.cpp
@@ -72,147 +72,59 @@ void serial_matrix_multiply(const std::vector<int>& a,
 // Stateful actor behavior
 caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
   return {
-    // 1st handler: Just int N, and who to send the matrices to
-    [=](int N, std::vector<caf::actor> receivers) {
-
-        caf::cuda::manager& mgr = caf::cuda::manager::get();
-        //create the program and configure the dimesnions of the kernel
-        auto program = mgr.create_program_from_fatbin("../generate_random_matrix.fatbin","generate_random_matrix");
-	int THREADS = 256;
-	int BLOCKS = (N*N + THREADS - 1) / THREADS;
-  	caf::cuda::nd_range dim(BLOCKS,1, 1, THREADS,1, 1);
-
-	//tag the arguments so that caf::cuda knows what to do with them	
-         auto arg1 = caf::cuda::create_out_arg(N*N); //output buffer indicate its size, caf::cuda will handle the rest
-          auto arg2 = caf::cuda::create_in_arg(N*N); //matrix size
-          auto arg3 = caf::cuda::create_in_arg(rand()); //seed
-	  auto arg4 = caf::cuda::create_in_arg(9999); //max valux
-	  
-          auto arg3B = caf::cuda::create_in_arg(rand()); //seed
-	  int device_number= 74; //arbitary number to show that 
-				 //can give illusion of selecting gpus that are
-				 //not there
-
-
-	  //launch kernels and collect their outputs
-	  auto tempA = randomMatrix.run_async(program,dim, self -> state().id,0,device_number,arg1,arg2,arg3,arg4);
-	  auto tempB = randomMatrix.run_async(program,dim, self -> state().id,0,device_number,arg1,arg2,arg3B,arg4);
-	  caf::cuda::mem_ptr<int> matrixA =  std::get<0>(tempA);
-	  caf::cuda::mem_ptr<int> matrixB = std::get<0>(tempB);
-
-	  //ensure the data is actually done being worked on
-	  matrixA -> synchronize();
-	  matrixB -> synchronize();
-
-	  std::cout << "Broadcasting\n";
-	  //broadcast the result out to receviers.
-	  for (auto actor: receivers) {
-	  
-		  self->mail(3,matrixA,matrixB,N,device_number).send(actor);
-	  }
-
+    // 1. Initial trigger: Setup data, launch kernel, and register the async callback.
+    [=](int N) {
+      std::cout << "Starting Async Callback Test with N=" << N << std::endl;
+      
+      // Initialize test data on host
+      std::vector<int> h_a(N * N);
+      std::vector<int> h_b(N * N);
+      std::iota(h_a.begin(), h_a.end(), 1);
+      std::iota(h_b.begin(), h_b.end(), 1);
+
+      caf::cuda::manager& mgr = caf::cuda::manager::get();
+      auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+      const int THREADS = 32;
+      const int BLOCKS = (N + THREADS - 1) / THREADS;
+      caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+      // Launch the kernel asynchronously using host vectors.
+      // run_async returns a tuple of mem_ptrs for each argument.
+      auto results = mmul.run_async(program, dims, self->state().id, 
+                                    caf::cuda::create_in_arg(h_a),
+                                    caf::cuda::create_in_arg(h_b),
+                                    caf::cuda::create_out_arg(N * N),
+                                    caf::cuda::create_in_arg(N));
+
+      // The output matrix 'c' is the 3rd argument (index 2).
+      auto matrixC_ptr = std::get<2>(results);
+      auto self_hdl = caf::actor_cast<caf::actor>(self);
+
+      // Invoke the new async copy with a callback. 
+      // When the GPU is done, this lambda runs on a driver thread.
+      matrixC_ptr->copy_to_host_async([self_hdl, h_a = std::move(h_a), h_b = std::move(h_b), N](std::vector<int> h_c) mutable {
+        std::cout << "GPU Work Complete. Callback triggered. Notifying actor..." << std::endl;
+        // Use anon_mail to safely send the data back to the actor system.
+        caf::anon_mail(std::move(h_a), std::move(h_b), std::move(h_c), N).send(self_hdl);
+      });
     },
 
-    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const caf::cuda::mem_ptr<int> matrixA,
-        const caf::cuda::mem_ptr<int> matrixB, int N,int device_number) {
- 
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //create program and dims   
-  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    //create args
-    auto arg1 = matrixA;
-    auto arg2 = matrixB;
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
-
-
-    auto tempC = mmulAsync.run(program,dims,self -> state().id,0,device_number,arg1,arg2,arg3,arg4);
-
-    std::vector<int> matrix1 = matrixA -> copy_to_host();
-    std::vector<int> matrix2 = matrixB -> copy_to_host();    
-    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC,2);
-
-    //verify its own result 
-    self -> mail(matrix1,matrix2,matrixC,N).send(self);
-
-    },
-
- // 3nd handler: GPU atom + matrices + N, launches a kenrel using shared memory and sends its result to itself for verification
-    [=](int x,const caf::cuda::mem_ptr<int> matrixA,
-        const caf::cuda::mem_ptr<int> matrixB, int N,int device_number) {
- 
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //create program and dims   
-  auto program = mgr.create_program_from_cubin("../shared_mmul.cubin","matrixMul");
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  int shared_mem = 8192; //we need 8KB of shared memory here
-    //create args
-    auto arg1 = matrixA;
-    auto arg2 = matrixB;
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
-
-
-    auto tempC = mmulAsync.run(program,dims,self -> state().id,shared_mem,device_number,arg1,arg2,arg3,arg4);
-
-    std::vector<int> matrix1 = matrixA -> copy_to_host();
-    std::vector<int> matrix2 = matrixB -> copy_to_host();    
-    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC,2);
-
-    //verify its own result 
-    self -> mail(matrix1,matrix2,matrixC,N).send(self);
-
-    },
-
-
-
-    // 3rd handler: CPU atom + matrices + N
+    // 2. Verification handler: Called when the async callback sends the data.
     [=](const std::vector<int>& matrixA,
     const std::vector<int> &matrixB,
     const std::vector<int> &matrixC, int N) {
 
+    std::cout << "Actor received results. Verifying correctness..." << std::endl;
     std::vector<int> result(N * N);
-
     serial_matrix_multiply(matrixA, matrixB, result, N);
 
     if (result == matrixC) {
-        std::cout << "actor with id " << self->state().id << " references match\n";
+        std::cout << "SUCCESS: Actor id=" << self->state().id << " results match!" << std::endl;
     }
     else {
-        std::cout << "actor with id " << self->state().id << " references did not match\n";
-
+        std::cout << "FAILURE: Actor id=" << self->state().id << " results do NOT match!" << std::endl;
     }
-
-
-    /*
-    auto print_matrix = [N](const std::vector<int>& mat, const std::string& name) {
-            std::cout << name << ":\n";
-            for (int i = 0; i < N; ++i) {
-                for (int j = 0; j < N; ++j) {
-                    std::cout << mat[i * N + j] << " ";
-                }
-                std::cout << "\n";
-            }
-            std::cout << std::endl;
-        };
-
-        print_matrix(matrixA, "Matrix A");
-        print_matrix(matrixB, "Matrix B");
-        print_matrix(result, "Result Matrix");
-        print_matrix(matrixC, "GPU Result Matrix");
-	*/
     self->quit();
     }
   };
@@ -220,33 +132,14 @@ caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self)
 
 
 void run_async_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_async_actor_fun));
-  }
-
-  // Actor 0 generates matrices and broadcasts to others 
-  caf::anon_mail(matrix_size, actors).send(actors[0]);
-
-   sys.await_all_actors_done();
+  auto worker = sys.spawn(mmul_async_actor_fun);
+  caf::anon_mail(matrix_size).send(worker);
+  sys.await_all_actors_done();
 }
 
-
 void caf_main(caf::actor_system& sys) {
   caf::cuda::manager::init(sys);
-
-  //run_async_mmul_test(sys,100,1);
-
- 
+  run_async_mmul_test(sys, 1024, 1);
 }
 
-
-
 CAF_MAIN()

From 3e40086d7fa6ae85a5bfdf3649172fc8aa8727f8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 7 May 2026 13:34:32 -0600
Subject: [PATCH 0580/1000] changed cuMemFree to cuMemFreeAsync in reset. This
 change is being made to fix issues where memory would be deleted before it
 was supposed ton be due to smart pointer dropping out of scope as well as be
 more asynchronous programming friendly

---
 libcaf_cuda/caf/cuda/mem_ref.hpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/mem_ref.hpp b/libcaf_cuda/caf/cuda/mem_ref.hpp
index 21f95969f1..f1cc198afe 100644
--- a/libcaf_cuda/caf/cuda/mem_ref.hpp
+++ b/libcaf_cuda/caf/cuda/mem_ref.hpp
@@ -98,7 +98,9 @@ class mem_ref : public caf::ref_counted {
   //sets all its attributes to null or -1
   void reset() {
     if (!is_scalar_ && memory_) {
-      CHECK_CUDA(cuMemFree(memory_));
+      CHECK_CUDA(cuCtxPushCurrent(ctx));
+      CHECK_CUDA(cuMemFreeAsync(memory_, stream_));
+      CHECK_CUDA(cuCtxPopCurrent(nullptr));
       memory_ = 0;
     }
     num_elements_ = 0;

From dca787d8bc8e41a4ae936d438c7459910c13defd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 7 May 2026 13:58:24 -0600
Subject: [PATCH 0581/1000] Saving.

---
 .../main.test.cpp                             | 608 +-----------------
 1 file changed, 27 insertions(+), 581 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index cbb361ede4..bbf1ebe24f 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -10,7 +10,6 @@
 #include <numeric>
 #include <random>
 #include "caf/actor_registry.hpp"
-#include <random>
 #include <unordered_set>
 //#include <caf/atoms.hpp>
 
@@ -19,35 +18,10 @@
 using namespace caf;
 using namespace std::chrono_literals;
 
-
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-  
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
-    }
-  }
-}
-
-using command =
-  caf::cuda::command_runner<>;
-
-command mmul_command;
-caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int >> mmul;
-using async_command = caf::cuda::mmul_async_command<int>;
-async_command async_mmul;
-
-
-
-
+// Command runners for GPU operations
+caf::cuda::command_runner<> mmul_command;
+using mmul_async_t = caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>, out<int>, in<int>>;
+mmul_async_t async_mmul;
 
 struct MatrixPool {
     std::unordered_map<int, std::vector<int>> A;
@@ -80,14 +54,9 @@ MatrixPool create_matrix_pool_random(
     return pool;
 }
 
-
-
-
-
-
 struct mmul_state {
-
 	caf::cuda::program_ptr mmul_kernel;
+    int N_val = 0;
 
 };
 
@@ -106,373 +75,34 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
 
 	int device = stream % caf::cuda::manager::get().get_num_devices();
 	self->mail(N).send(self);
+    self->state().N_val = N;
 
 	return {
-
 		[=](int N) {
-
-			auto total_start = std::chrono::steady_clock::now();
-
-
-//			std::cout << "device=" << device <<  "\n";
-//			std::cout << "N=" << N <<  "\n";
-			// ---------------- H2D ----------------
-			auto h2d_start = std::chrono::steady_clock::now();
-
 			auto arg1 = mmul_command.transfer_memory(device, stream, std::move(matrixA));
 			auto arg2 = mmul_command.transfer_memory(device, stream, std::move(matrixB));
 
-
-			// ---------------- Kernel ----------------
 			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
 			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
 
-			auto h2d_end = std::chrono::steady_clock::now();
-			auto kernel_start = std::chrono::steady_clock::now();
-
 			auto result = async_mmul.run_async(
 					program, dims, stream, 0, device,
 					arg1, arg2, arg3, arg4);
 
-			//std::get<2>(result)->synchronize();
-
-			auto kernel_end = std::chrono::steady_clock::now();
-
-			// ---------------- D2H ----------------
-			auto d2h_start = std::chrono::steady_clock::now();
-
-			std::get<2>(result)->copy_to_host();
-
-			auto d2h_end = std::chrono::steady_clock::now();
-
-			auto total_end = std::chrono::steady_clock::now();
-
-			/*
-			// ---------------- COMPUTE ----------------
-			auto h2d = std::chrono::duration<double>(h2d_end - h2d_start).count();
-			auto kernel = std::chrono::duration<double>(kernel_end - kernel_start).count();
-			auto d2h = std::chrono::duration<double>(d2h_end - d2h_start).count();
-			auto total = std::chrono::duration<double>(total_end - total_start).count();
-
-			// ---------------- PRINT ----------------
-			
-			std::cout << "\n[NO SCHEDULER] N=" << N << "\n";
-			std::cout << "H2D:    " << h2d * 1000 << " ms\n";
-			std::cout << "Kernel: " << kernel * 1000 << " ms\n";
-			std::cout << "D2H:    " << d2h * 1000 << " ms\n";
-			std::cout << "TOTAL:  " << total * 1000 << " ms\n";
-			std::cout << "SUM:    " << (h2d + kernel + d2h) * 1000 << " ms\n";
-
-			*/
-			self->mail(1).send(exit_actor);
-			self->quit();
-		}
-	};
-}
-
-
-
-struct mmul_actor_with_scheduler_state {
-  static inline const char* name = "my_actor";
-};
-
-
-// Stateful actor behavior
-caf::behavior mmul_actor_fun_scheduler(
-    caf::stateful_actor<mmul_actor_with_scheduler_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims,
-    const in<int> matrixA,
-    const in<int> matrixB)
-{
-
-
-
-        caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-        caf::actor scheduler = mgr.get_scheduler_actor();
-
-        //send a launch token
-        caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-                        program,
-                        dims,
-                        0,
-                        "hello",
-                        self
-                        );
-        mgr.send_scheduler_actor_message(launch_token);
-
-	return {
-
-		// 1. Handle response token
-		[=](caf::cuda::response_token_ptr res_token) {
-			//        std::cout << "Got response\n";
-
-			if (res_token->getType() == LAUNCH_RESPONSE) {
-				self->mail(res_token, N).send(self);
-
-			} else {
-				//          std::cout << "Got a memory response token\n";
-			}
-		},
-
-			// 2. Handle memory buffers -> GPU
-			[=](const caf::cuda::response_token_ptr& res_token, int N) {
-
-				auto total_start = std::chrono::steady_clock::now();
-
-				// ---------------- H2D ----------------
-				auto h2d_start = std::chrono::steady_clock::now();
-
-				auto arg1 = mmul.transfer_memory(res_token -> getDeviceNumber(),res_token -> getStreamId(), std::move(matrixA));
-				auto arg2 = mmul.transfer_memory(res_token -> getDeviceNumber(), res_token -> getStreamId(), std::move(matrixB));
-				//auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
-				//auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
-
-				std::cout << "res_token did = " << res_token -> getDeviceNumber() << "\n";
-				std::cout << "N = " << N << "\n";
-				 out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
-	 			 in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
-
-				auto h2d_end = std::chrono::steady_clock::now();
-
-				// ---------------- Kernel ----------------
-				auto kernel_start = std::chrono::steady_clock::now();
-
-
-
-				auto tempC = async_mmul.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
-				auto bufferC = std::get<2>(tempC);
-
-				//bufferC->synchronize();
+			auto bufferC = std::get<2>(result);
 
-				auto kernel_end = std::chrono::steady_clock::now();
-
-				// ---------------- D2H ----------------
-				auto d2h_start = std::chrono::steady_clock::now();
-
-				bufferC->copy_to_host();
-
-				auto d2h_end = std::chrono::steady_clock::now();
-
-				res_token->release();
-
-				auto total_end = std::chrono::steady_clock::now();
-
-				// ---------------- COMPUTE ----------------
-				/*
-				auto h2d = std::chrono::duration<double>(h2d_end - h2d_start).count();
-				auto kernel = std::chrono::duration<double>(kernel_end - kernel_start).count();
-				auto d2h = std::chrono::duration<double>(d2h_end - d2h_start).count();
-				auto total = std::chrono::duration<double>(total_end - total_start).count();
-
-				std::cout << "H2D:    " << h2d * 1000 << " ms\n";
-				std::cout << "Kernel: " << kernel * 1000 << " ms\n";
-				std::cout << "D2H:    " << d2h * 1000 << " ms\n";
-				std::cout << "TOTAL:  " << total * 1000 << " ms\n";
-				std::cout << "SUM:    " << (h2d + kernel + d2h) * 1000 << " ms\n\n";
-				*/
-
-				self->mail(1).send(exit_actor);
-				self->quit();
-			}
-
-
-       };
-
-}
-
-
-
-
-
-caf::behavior mmul_actor_fun_scheduler2(caf::stateful_actor<mmul_state>* self,
-		caf::actor exit_actor,
-		caf::actor scheduler_actor,
-		caf::cuda::program_ptr program,
-		caf::cuda::nd_range dims,
-		int stream,
-		int N,
-		const in<int> matrixA,
-		const in<int> matrixB
-		) {
-
-
-	int device = stream % caf::cuda::manager::get().get_num_devices();
-
-	//self->mail("subscribe",self).send(scheduler_actor);
-
-	self->mail(N).send(self);
-
-	return {
-
-		//message from the new scheduler actor 
-		[=](std::vector<int> costs) {
-			//do nothing this is an overhead test 
-		
-			//std::cout << "N=" << N <<  "\n";
-		
+            // Asynchronously copy data back to host. This triggers a message to 'self' 
+            // when the transfer is finished, instead of blocking the actor thread.
+            bufferC->copy_to_host_async([self](std::vector<int>&& /*data*/) {
+                caf::anon_mail(uint64_t{1}).send(self);
+            });
 		},
-		[=](int N) {
-
-			auto total_start = std::chrono::steady_clock::now();
-
-			//int device = rand() % caf::cuda::manager::get().get_num_devices();
-			//int stream = rand();
-
-			//std::cout << "device=" << device <<  "\n";
-			//std::cout << "N=" << N <<  "\n";
-
-
-
-			//declare the cost of doing work to the scheduler actor, for now we can impose
-			//a heuristic of just N, the size of the matrix
-			self->mail("add",device,N).send(scheduler_actor);
-
-
-			// ---------------- H2D ----------------
-			auto h2d_start = std::chrono::steady_clock::now();
-
-
-			auto arg1 = mmul_command.transfer_memory(device, stream, std::move(matrixA));
-			auto arg2 = mmul_command.transfer_memory(device, stream, std::move(matrixB));
-
-
-			// ---------------- Kernel ----------------
-			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
-			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
-
-			auto h2d_end = std::chrono::steady_clock::now();
-			auto kernel_start = std::chrono::steady_clock::now();
-
-			auto result = async_mmul.run_async(
-					program, dims, stream, 0, device,
-					arg1, arg2, arg3, arg4);
-
-			//std::get<2>(result)->synchronize();
-
-			auto kernel_end = std::chrono::steady_clock::now();
-
-			// ---------------- D2H ----------------
-			auto d2h_start = std::chrono::steady_clock::now();
-
-			std::get<2>(result)->copy_to_host();
-
-
-
-			//likewise tell the scheduler we are done doing work
-			self->mail("subtract",device,N).send(scheduler_actor);
-
-			auto d2h_end = std::chrono::steady_clock::now();
-
-			auto total_end = std::chrono::steady_clock::now();
-
-			/*
-			// ---------------- COMPUTE ----------------
-			auto h2d = std::chrono::duration<double>(h2d_end - h2d_start).count();
-			auto kernel = std::chrono::duration<double>(kernel_end - kernel_start).count();
-			auto d2h = std::chrono::duration<double>(d2h_end - d2h_start).count();
-			auto total = std::chrono::duration<double>(total_end - total_start).count();
-
-			// ---------------- PRINT ----------------
-			
-			std::cout << "\n[NO SCHEDULER] N=" << N << "\n";
-			std::cout << "H2D:    " << h2d * 1000 << " ms\n";
-			std::cout << "Kernel: " << kernel * 1000 << " ms\n";
-			std::cout << "D2H:    " << d2h * 1000 << " ms\n";
-			std::cout << "TOTAL:  " << total * 1000 << " ms\n";
-			std::cout << "SUM:    " << (h2d + kernel + d2h) * 1000 << " ms\n";
-
-			*/
+        [=](uint64_t /* completion_token */) {
 			self->mail(1).send(exit_actor);
-			self->mail("unsubscribe",self).send(scheduler_actor);
 			self->quit();
-		}
-	};
-}
-
-
-
-
-
-
-struct scheduler_actor_state {
-
-	std::vector<caf::actor> subscribers;
-	int num_devices;
-	std::vector<int> costs;
-};
-
-
-
-caf::behavior scheduler_actor_fun(caf::stateful_actor<scheduler_actor_state>* self) {
-
-	self->state().num_devices = caf::cuda::manager::get().get_num_devices();
-	self->state().costs.resize(self->state().num_devices);
-
-	int time = 50;
-
-	self->mail("publish").urgent().delay(std::chrono::milliseconds(time)).send(self);
-
-	return {
-
-		[=](std::string command,int device, int cost) {
-
-			if (command == "add") {
-
-				self->state().costs[device] +=cost;
-
-			}
-			else if (command == "subtract") {
-
-				int value = std::min(self->state().costs[device] - cost,0);
-				self->state().costs[device] = value;	
-			}
-
-
-		},
-			[=](std::string command, caf::actor actor) {
-
-				auto& subs = self->state().subscribers;
-
-				if (command == "subscribe") {
-					//std::cout << "Thank you for subscribing\n";
-					// avoid duplicates
-					if (std::find(subs.begin(), subs.end(), actor) == subs.end()) {
-						subs.push_back(actor);
-						//self->monitor(actor); // track lifecycle
-					}
-				}
-
-				else if (command == "unsubscribe") {
-					subs.erase(
-							std::remove(subs.begin(), subs.end(), actor),
-							subs.end()
-						  );
-					//self->demonitor(actor);
-				}
-			},
-
-			[=](std::string command) {
-				if (command == "publish") {
-					//std::cout << "size = " << self->state().subscribers.size() << "\n";
-					for (caf::actor a : self->state().subscribers) {
-
-						self -> mail(self->state().costs).urgent().send(a);
-
-					}
-					self->mail("publish").urgent().delay(std::chrono::milliseconds(time)).send(self);
-				}
-
-			}	
-
+        }
 	};
 }
-
-
-
 // ---------------------------- SUPERVISOR ACTOR ----------------------------
 struct supervisor_actor_state {
     int num_actors;
@@ -496,8 +126,7 @@ caf::behavior supervisor_actor_fun(
     int num_actors,
     int max_waves,
     MatrixPool pool,
-    const std::vector<int>& Ns,      // deterministic task sizes 
-    bool use_scheduler
+    const std::vector<int>& Ns      // deterministic task sizes 
     ) {
     // Initialize state
     self->state().num_actors = num_actors;
@@ -516,8 +145,6 @@ caf::behavior supervisor_actor_fun(
     // Kick off first wave
     self->mail("spawn").send(self);
 
-    caf::actor scheduler_actor = self->spawn(scheduler_actor_fun);
-
     // Start timing
     self->state().start_time = std::chrono::steady_clock::now();
     return {
@@ -536,35 +163,14 @@ caf::behavior supervisor_actor_fun(
                     break;
 
                 int N = self->state().Ns[self->state().next_task++];
-
-
                 const auto& A = self->state().pool.A[N];
                 const auto& B = self->state().pool.B[N];
-
                 const int THREADS = 32;
                 const int BLOCKS = (N + THREADS - 1) / THREADS;
-
                 caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-		if (use_scheduler) {
-			caf::actor a = 	self->spawn(mmul_actor_fun_scheduler2, 
-					self,
-					scheduler_actor,
-				       	program, 
-					dims,
-					i,
-					N,
-                            caf::cuda::create_in_arg(A),
-                            caf::cuda::create_in_arg(B));
-
-
-		 self->mail("subscribe",a).send(scheduler_actor);
-		}
-		else {
-			self->spawn(mmul_actor_fun, self, program, dims,i,N,
+                self->spawn(mmul_actor_fun, self, program, dims,i,N,
                             caf::cuda::create_in_arg(A),
                             caf::cuda::create_in_arg(B));
-		}
             }
         
 	
@@ -577,10 +183,7 @@ caf::behavior supervisor_actor_fun(
                     std::cout << "\n===== SUPERVISOR TOTAL TIME spawn =====\n";
                     std::cout << "Total runtime: "
                               << total_time.count() << " s\n";
-
-
-	
-	},
+        },
 
         // -------------------- COMPLETION TRACKING --------------------
         [=](int done) {
@@ -592,11 +195,6 @@ caf::behavior supervisor_actor_fun(
                     wave_end - self->state().wave_start_time;
 
                 self->state().num_waves++;
-                //std::cout << "Wave "
-                  //        << self->state().num_waves
-                    //      << " completed in "
-                      //    << wave_time.count() << " s\n";
-
                 if (self->state().num_waves >= self->state().max_waves) {
                     auto end_time = std::chrono::steady_clock::now();
                     std::chrono::duration<double> total_time =
@@ -605,12 +203,7 @@ caf::behavior supervisor_actor_fun(
                     std::cout << "\n===== SUPERVISOR TOTAL TIME =====\n";
                     std::cout << "Total runtime: "
                               << total_time.count() << " s\n";
-
-
-		    anon_send_exit(
-				    scheduler_actor,
-				    caf::exit_reason::user_shutdown
-				  );
+                    
 		    caf::cuda::manager::shutdown();
                     self->quit();
                 } else {
@@ -620,14 +213,6 @@ caf::behavior supervisor_actor_fun(
         }
     };
 }
-
-
-
-
-
-
-
-
 template <class Fn>
 double time_run(Fn&& fn) {
   auto start = std::chrono::steady_clock::now();
@@ -637,8 +222,6 @@ double time_run(Fn&& fn) {
   return elapsed.count();
 }
 
-
-
 void run_mmul_random_scaling_tests(caf::actor_system& sys,
                                   caf::cuda::manager_config man_config) {
 
@@ -679,15 +262,14 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 
    
     //scheduler
+    caf::cuda::manager_config scheduler_off(false);
     for (int num_actors : actor_counts) {
-
-    
 	    // Initialize CUDA manager
-	    caf::cuda::manager::init(sys);
-  std::cout << "=====================================\n";
-        std::cout << "Random Scaling WITH scheduler | actors=" << num_actors << "\n";
-
+	    caf::cuda::manager::init(sys, scheduler_off);
+        std::cout << "=====================================\n";
+        std::cout << "Random Scaling | actors=" << num_actors << "\n";
 
+        // Execute the supervisor which manages the asynchronous workload
         double elapsed = time_run([&]() {
 
             auto sup = sys.spawn(
@@ -695,153 +277,17 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
                 num_actors,
                 max_waves,
                 pool,
-		Ns,
-		true
-            );
-
-        
-	      sys.await_all_actors_done();
-	    
-	    });
-
-    
-	caf::cuda::manager::shutdown();
-    }
-    
-    //no scheduler
-    for (int num_actors : actor_counts) {
-
-    
-	    // Initialize CUDA manager
-	    caf::cuda::manager::init(sys);
-	    std::cout << "=====================================\n";
-	    std::cout << "Random Scaling NO scheduler | actors=" << num_actors << "\n";      
-	    double elapsed = time_run([&]() {
-
-            auto sup = sys.spawn(
-                supervisor_actor_fun,
-                num_actors,
-                max_waves,
-                pool,
-		Ns,
-		false
+		Ns
             );
 
 	      sys.await_all_actors_done();
-	    
 	    });
 
-    
-	caf::cuda::manager::shutdown();
+        caf::cuda::manager::shutdown();
     }
-
-
-
-
-    caf::cuda::manager::shutdown();
 }
-
-void run_mmul_uniform_scaling_tests(caf::actor_system& sys,
-                                   caf::cuda::manager_config man_config) {
-
-    const int max_waves = 1;
-
-    // Matrix sizes: 1,2,4,...,2048
-    std::vector<int> matrix_sizes;
-    for (int n = 1; n <= 2048; n *= 2)
-        matrix_sizes.push_back(n);
-
-    // Actor counts: 10 → 1000
-    std::vector<int> actor_counts;
-    for (int a = 10; a <= 1000; a += 10)
-        actor_counts.push_back(a);
-
-    for (int N : matrix_sizes) {
-        for (int num_actors : actor_counts) {
-
-            // Create uniform pool (single size)
-            MatrixPool pool = create_matrix_pool_random(
-                1,
-                N,
-                N,
-                42
-            );
-
-            int total_tasks = num_actors * max_waves;
-
-            std::vector<int> Ns(total_tasks, N);
-
-            // ========================
-            // WITH SCHEDULER
-            // ========================
-            caf::cuda::manager::init(sys);
-            std::cout << "=====================================\n";
-            std::cout << "Uniform WITH scheduler | N=" << N
-                      << " actors=" << num_actors << "\n";
-
-            time_run([&]() {
-                auto sup = sys.spawn(
-                    supervisor_actor_fun,
-                    num_actors,
-                    max_waves,
-                    pool,
-                    Ns,
-                    true
-                );
-                sys.await_all_actors_done();
-            });
-
-            caf::cuda::manager::shutdown();
-
-            // ========================
-            // WITHOUT SCHEDULER
-            // ========================
-            caf::cuda::manager::init(sys);
-            std::cout << "=====================================\n";
-            std::cout << "Uniform NO scheduler | N=" << N
-                      << " actors=" << num_actors << "\n";
-
-            time_run([&]() {
-                auto sup = sys.spawn(
-                    supervisor_actor_fun,
-                    num_actors,
-                    max_waves,
-                    pool,
-                    Ns,
-                    false
-                );
-                sys.await_all_actors_done();
-            });
-
-            caf::cuda::manager::shutdown();
-        }
-    }
-}
-
-
-
-
-
-
-
-
-
-
 void caf_main(caf::actor_system& sys) {
-  
-
-  caf::cuda::manager_config man_config(true);
-  //caf::cuda::manager::init(sys,man_config);
-
-
-  run_mmul_random_scaling_tests(sys,man_config);
-  run_mmul_uniform_scaling_tests(sys,man_config);
-
-
-
+  caf::cuda::manager_config man_config(false);
+  run_mmul_random_scaling_tests(sys, man_config);
 }
-
-
-
-
 CAF_MAIN()

From 044d78206e307034c23cfbc6beafd90c0daf5d6d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 7 May 2026 14:41:08 -0600
Subject: [PATCH 0582/1000] Moved callback functionality from mem_ref to
 command runner and modified tests accordingly. This change was made to fix an
 issue where mem_ptr deconstructor would be called inside the callback
 triggering cuda operations, which cuda forbids generating a forbidden
 operation erorr.

---
 libcaf_cuda/caf/cuda/command_runner.hpp       | 82 ++++++++++++++++-
 libcaf_cuda/caf/cuda/mem_ref.hpp              | 89 +------------------
 .../actor-tests/callback-test/main.test.cpp   |  2 +-
 .../main.test.cpp                             |  7 +-
 4 files changed, 88 insertions(+), 92 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index ca5bd03a1f..4ff43e77ff 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -2,6 +2,7 @@
 
 #include "caf/cuda/command.hpp"
 #include "caf/cuda/memory_command.hpp"
+#include <thread>
 #include "caf/cuda/program.hpp"
 #include "caf/cuda/nd_range.hpp"
 #include "caf/cuda/platform.hpp"
@@ -213,6 +214,86 @@ class command_runner {
     }
 
 
+    /**
+     * @brief Asynchronously copies GPU memory and provides an std::vector<T> to the callback.
+     * 
+     * The mem_ptr is kept alive until the callback completes.
+     */
+    template <typename T, typename F>
+    void copy_to_host_async(mem_ptr<T> ptr, F callback) {
+      if (ptr->access() == IN)
+        throw std::runtime_error("Cannot copy a read-only buffer back to host");
+
+      struct State {
+        std::vector<T> buffer;
+        F user_callback;
+        bool is_scalar;
+        T host_scalar;
+      };
+
+      auto* state = new State{std::vector<T>(ptr->size()), std::move(callback), ptr->is_scalar(), *ptr->host_scalar_ptr()};
+
+      CHECK_CUDA(cuCtxPushCurrent(ptr->get_ctx()));
+      CUstream s = ptr->stream();
+
+      if (!ptr->is_scalar()) {
+        size_t bytes = ptr->size() * sizeof(T);
+        CHECK_CUDA(cuMemcpyDtoHAsync(state->buffer.data(), ptr->mem(), bytes, s));
+      }
+
+      auto host_fn = [](void* userData) {
+        auto* s_ptr = static_cast<State*>(userData);
+        if (s_ptr->is_scalar)
+          s_ptr->buffer[0] = s_ptr->host_scalar;
+
+        s_ptr->user_callback(std::move(s_ptr->buffer));
+        
+        delete s_ptr;
+      };
+
+      CHECK_CUDA(cuLaunchHostFunc(s, host_fn, state));
+      CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    }
+
+    /**
+     * @brief Asynchronously copies GPU memory to a user-provided host buffer.
+     */
+    template <typename T, typename F>
+    void copy_to_host_async(mem_ptr<T> ptr, T* dst, size_t count, F callback) {
+      if (ptr->access() == IN)
+        throw std::runtime_error("Cannot copy a read-only buffer back to host");
+
+      struct State {
+        T* dst;
+        size_t count;
+        F user_callback;
+        bool is_scalar;
+        T host_scalar;
+      };
+
+      auto* state = new State{dst, count, std::move(callback), ptr->is_scalar(), *ptr->host_scalar_ptr()};
+
+      CHECK_CUDA(cuCtxPushCurrent(ptr->get_ctx()));
+      CUstream s = ptr->stream();
+
+      if (!ptr->is_scalar()) {
+        size_t bytes = count * sizeof(T);
+        CHECK_CUDA(cuMemcpyDtoHAsync(dst, ptr->mem(), bytes, s));
+      }
+
+      auto host_fn = [](void* userData) {
+        auto* s_ptr = static_cast<State*>(userData);
+        if (s_ptr->is_scalar)
+          s_ptr->dst[0] = s_ptr->host_scalar;
+
+        s_ptr->user_callback(s_ptr->dst, s_ptr->count);
+        delete s_ptr;
+      };
+
+      CHECK_CUDA(cuLaunchHostFunc(s, host_fn, state));
+      CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    }
+
 
   // -------------------------------
   // Destroy streams for a given actor ID
@@ -230,4 +311,3 @@ class command_runner {
 };
 
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/mem_ref.hpp b/libcaf_cuda/caf/cuda/mem_ref.hpp
index f1cc198afe..c7ef5fe453 100644
--- a/libcaf_cuda/caf/cuda/mem_ref.hpp
+++ b/libcaf_cuda/caf/cuda/mem_ref.hpp
@@ -6,6 +6,7 @@
 #include <vector>
 #include <stdexcept>
 #include <chrono>
+#include <thread>
 #include <iostream>
 #include "caf/cuda/types.hpp"
 #include <caf/actor.hpp>
@@ -81,6 +82,7 @@ class mem_ref : public caf::ref_counted {
   CUstream stream() const noexcept { return stream_; }
   int deviceID() const noexcept { return device_id;}
   int deviceNumber() const noexcept { return device_id;}
+  CUcontext get_ctx() const noexcept { return ctx; }
 
  //if it is ever needed, you can force synchronization on a mem_ptr
  //to ensure data on the device that the mem_ptr points to
@@ -156,93 +158,6 @@ class mem_ref : public caf::ref_counted {
 	  CHECK_CUDA(cuCtxPopCurrent(nullptr));
   }
 
-  /**
-   * @brief Asynchronously copies GPU memory to a host buffer and triggers a user-defined callback.
-   * 
-   * This version allows the user to provide their own destination buffer.
-   * 
-   * @param dst Pointer to the destination host memory. Must remain valid until the callback executes.
-   * @param count Number of elements to copy.
-   * @param callback A function/lambda with the signature void(T* data, size_t size).
-   */
-  template <typename F>
-  void copy_to_host_async(T* dst, size_t count, F callback) const {
-    if (access_ == IN)
-      throw std::runtime_error("Cannot copy a read-only buffer back to host");
-
-    struct State {
-      caf::intrusive_ptr<const mem_ref<T>> self;
-      T* dst;
-      size_t count;
-      F user_callback;
-    };
-
-    auto* state = new State{this, dst, count, std::move(callback)};
-
-    CHECK_CUDA(cuCtxPushCurrent(ctx));
-    CUstream s = stream_ ? stream_ : nullptr;
-
-    if (!is_scalar_) {
-      size_t bytes = count * sizeof(T);
-      CHECK_CUDA(cuMemcpyDtoHAsync(dst, memory_, bytes, s));
-    }
-
-    auto host_fn = [](void* userData) {
-      auto* s_ptr = static_cast<State*>(userData);
-      if (s_ptr->self->is_scalar_)
-        s_ptr->dst[0] = s_ptr->self->host_scalar_;
-      
-      s_ptr->user_callback(s_ptr->dst, s_ptr->count);
-      delete s_ptr;
-    };
-
-    CHECK_CUDA(cuLaunchHostFunc(s, host_fn, state));
-    CHECK_CUDA(cuCtxPopCurrent(nullptr));
-  }
-
-  /**
-   * @brief Asynchronously copies GPU memory and provides an std::vector<T> to the callback.
-   * 
-   * This version manages the host buffer for you.
-   * 
-   * @param callback A function/lambda with the signature void(std::vector<T>&& data).
-   */
-  template <typename F>
-  void copy_to_host_async(F callback) const {
-    if (access_ == IN)
-      throw std::runtime_error("Cannot copy a read-only buffer back to host");
-
-    struct State {
-      caf::intrusive_ptr<const mem_ref<T>> self;
-      std::vector<T> buffer;
-      F user_callback;
-    };
-
-    auto* state = new State{this, std::vector<T>(num_elements_), std::move(callback)};
-
-    CHECK_CUDA(cuCtxPushCurrent(ctx));
-    CUstream s = stream_ ? stream_ : nullptr;
-
-    if (!is_scalar_) {
-      size_t bytes = num_elements_ * sizeof(T);
-      CHECK_CUDA(cuMemcpyDtoHAsync(state->buffer.data(), memory_, bytes, s));
-    }
-
-    auto host_fn = [](void* userData) {
-      auto* s_ptr = static_cast<State*>(userData);
-      if (s_ptr->self->is_scalar_)
-        s_ptr->buffer[0] = s_ptr->self->host_scalar_;
-
-      s_ptr->user_callback(std::move(s_ptr->buffer));
-      delete s_ptr;
-    };
-
-    CHECK_CUDA(cuLaunchHostFunc(s, host_fn, state));
-    CHECK_CUDA(cuCtxPopCurrent(nullptr));
-  }
-
-
-
     //reference counting for auto garabage collection
     friend void intrusive_ptr_add_ref(const mem_ref<T>* p) noexcept {
         p->ref_count_.fetch_add(1, std::memory_order_relaxed);
diff --git a/libcaf_cuda/tests/actor-tests/callback-test/main.test.cpp b/libcaf_cuda/tests/actor-tests/callback-test/main.test.cpp
index 9c8517d583..ac8a355338 100644
--- a/libcaf_cuda/tests/actor-tests/callback-test/main.test.cpp
+++ b/libcaf_cuda/tests/actor-tests/callback-test/main.test.cpp
@@ -103,7 +103,7 @@ caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self)
 
       // Invoke the new async copy with a callback. 
       // When the GPU is done, this lambda runs on a driver thread.
-      matrixC_ptr->copy_to_host_async([self_hdl, h_a = std::move(h_a), h_b = std::move(h_b), N](std::vector<int> h_c) mutable {
+      mmul.copy_to_host_async(matrixC_ptr, [self_hdl, h_a = std::move(h_a), h_b = std::move(h_b), N](std::vector<int> h_c) mutable {
         std::cout << "GPU Work Complete. Callback triggered. Notifying actor..." << std::endl;
         // Use anon_mail to safely send the data back to the actor system.
         caf::anon_mail(std::move(h_a), std::move(h_b), std::move(h_c), N).send(self_hdl);
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index bbf1ebe24f..8069858144 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -90,11 +90,12 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
 					arg1, arg2, arg3, arg4);
 
 			auto bufferC = std::get<2>(result);
+            auto self_hdl = caf::actor_cast<caf::actor>(self);
 
             // Asynchronously copy data back to host. This triggers a message to 'self' 
             // when the transfer is finished, instead of blocking the actor thread.
-            bufferC->copy_to_host_async([self](std::vector<int>&& /*data*/) {
-                caf::anon_mail(uint64_t{1}).send(self);
+            mmul_command.copy_to_host_async(bufferC, [self_hdl](std::vector<int>&& /*data*/) {
+                caf::anon_mail(uint64_t{1}).send(self_hdl);
             });
 		},
         [=](uint64_t /* completion_token */) {
@@ -232,7 +233,7 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     const int max_waves = 1;
 
     const std::vector<int> actor_counts = {
-	  30000,40000,50000
+	  1,30000,40000,50000
     };
 
 

From 3e8c0e987becd1a3b4afb3e437cc5cbc76442174 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 7 May 2026 15:14:43 -0600
Subject: [PATCH 0583/1000] In process of overhauling test.

---
 .../main.test.cpp                             | 379 ++++++++++--------
 1 file changed, 220 insertions(+), 159 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 8069858144..066c957237 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -13,15 +13,24 @@
 #include <unordered_set>
 //#include <caf/atoms.hpp>
 
-
-
 using namespace caf;
 using namespace std::chrono_literals;
 
+// ─────────────────────────────────────────────────────────────────────────────
+// Atoms
+// ─────────────────────────────────────────────────────────────────────────────
+CAF_BEGIN_TYPE_ID_BLOCK(mmul_benchmark, caf::id_block::cuda::end)
+    CAF_ADD_ATOM(mmul_benchmark, get_work_atom)
+    CAF_ADD_ATOM(mmul_benchmark, task_done_atom)
+    CAF_ADD_ATOM(mmul_benchmark, release_memory_atom)
+    CAF_ADD_ATOM(mmul_benchmark, request_work_atom)
+    CAF_ADD_ATOM(mmul_benchmark, worker_done_atom)
+CAF_END_TYPE_ID_BLOCK(mmul_benchmark)
+
 // Command runners for GPU operations
 caf::cuda::command_runner<> mmul_command;
-using mmul_async_t = caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>, out<int>, in<int>>;
-mmul_async_t async_mmul;
+using mmul_kernel_t = caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>, out<int>, in<int>>;
+mmul_kernel_t mmul_kernel;
 
 struct MatrixPool {
     std::unordered_map<int, std::vector<int>> A;
@@ -54,166 +63,222 @@ MatrixPool create_matrix_pool_random(
     return pool;
 }
 
-struct mmul_state {
-	caf::cuda::program_ptr mmul_kernel;
-    int N_val = 0;
-
+// ---------------------------- DEVICE/GPU ACTOR ----------------------------
+// Holds the data partitions and dispatches tasks to workers.
+struct device_actor_state {
+    MatrixPool pool;
+    std::vector<int> tasks;
+    size_t next_task_idx = 0;
+    size_t total_device_memory_bytes = 0;
+    size_t current_allocated_memory_bytes = 0;
+    int active_workers = 0;
+    int device_id = -1;
 };
 
+caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
+                               MatrixPool pool, std::vector<int> tasks, int num_workers, int dev_id) {
+    self->state().pool = std::move(pool);
+    self->state().tasks = std::move(tasks);
+    self->state().device_id = dev_id;
+    self->state().active_workers = num_workers;
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+    caf::cuda::device_ptr dev_obj = mgr.find_device(dev_id);
+    if (dev_obj) {
+        self->state().total_device_memory_bytes = dev_obj->total_memory_bytes();
+    }
 
+    return {
+        [=](get_work_atom) -> result<int, in<int>, in<int>> {
+            auto& st = self->state();
+            if (st.next_task_idx >= st.tasks.size())
+                return sec::end_of_stream; // Explicit signal: work is done
 
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
-		caf::actor exit_actor,
-		caf::cuda::program_ptr program,
-		caf::cuda::nd_range dims,
-		int stream,
-		int N,
-		const in<int> matrixA,
-		const in<int> matrixB
-		) {
+            int N = st.tasks[st.next_task_idx]; // Peek, don't increment yet
 
+            // Calculate memory needed for A, B, C
+            size_t memory_needed = (size_t)N * N * sizeof(int) * 3; // A, B, C
 
-	int device = stream % caf::cuda::manager::get().get_num_devices();
-	self->mail(N).send(self);
-    self->state().N_val = N;
+            if (st.current_allocated_memory_bytes + memory_needed > st.total_device_memory_bytes) {
+                return make_error(sec::runtime_error, "Device Actor: Not enough memory");
+            }
 
-	return {
-		[=](int N) {
-			auto arg1 = mmul_command.transfer_memory(device, stream, std::move(matrixA));
-			auto arg2 = mmul_command.transfer_memory(device, stream, std::move(matrixB));
+            // If memory is available, reserve it and dispatch
+            st.current_allocated_memory_bytes += memory_needed;
+            N = st.tasks[st.next_task_idx++]; // Now actually take the task
+            return {N, caf::cuda::create_in_arg(st.pool.A[N]), caf::cuda::create_in_arg(st.pool.B[N])};
+        },
+        [=](release_memory_atom, int N_completed) {
+            auto& st = self->state();
+            size_t memory_released = (size_t)N_completed * N_completed * sizeof(int) * 3;
+            st.current_allocated_memory_bytes -= memory_released;
+        },
+        [=](worker_done_atom) {
+            auto& st = self->state();
+            if (--st.active_workers <= 0) {
+                self->quit();
+            }
+        }
+    };
+}
 
-			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
-			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
+// ---------------------------- WORKER ACTOR ----------------------------
+// Manages 1 stream and pulls work from the Device Actor.
+struct worker_state {
+    int device_id;
+    int stream_id;
+    caf::cuda::program_ptr program;
+    caf::actor device_actor;
+    caf::actor supervisor;
+    int max_in_flight_tasks;
+    int in_flight_tasks_count = 0;
+    bool draining = false;
+};
 
-			auto result = async_mmul.run_async(
-					program, dims, stream, 0, device,
-					arg1, arg2, arg3, arg4);
+caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
+                              caf::actor supervisor, caf::actor device_actor, caf::cuda::program_ptr program,
+                              int dev_id, int stream_id, int max_in_flight_tasks) {
+    self->state().supervisor = supervisor;
+    self->state().device_actor = device_actor;
+    self->state().program = program;
+    self->state().device_id = dev_id;
+    self->state().stream_id = stream_id;
+    self->state().max_in_flight_tasks = max_in_flight_tasks;
+
+    // Trigger initial work requests up to max_in_flight_tasks
+    for (int i = 0; i < max_in_flight_tasks; ++i) {
+        self->mail(request_work_atom_v).send(self);
+    }
 
-			auto bufferC = std::get<2>(result);
-            auto self_hdl = caf::actor_cast<caf::actor>(self);
+    return {
+        [=](request_work_atom) {
+            if (self->state().in_flight_tasks_count >= self->state().max_in_flight_tasks) {
+                return; // Already at max capacity, don't request more yet
+            }
 
-            // Asynchronously copy data back to host. This triggers a message to 'self' 
-            // when the transfer is finished, instead of blocking the actor thread.
-            mmul_command.copy_to_host_async(bufferC, [self_hdl](std::vector<int>&& /*data*/) {
-                caf::anon_mail(uint64_t{1}).send(self_hdl);
-            });
-		},
-        [=](uint64_t /* completion_token */) {
-			self->mail(1).send(exit_actor);
-			self->quit();
+            self->mail(get_work_atom_v).request(self->state().device_actor, infinite).then(
+                [=](int N, in<int> matrixA, in<int> matrixB) {
+                    auto& st = self->state();
+                    st.in_flight_tasks_count++; 
+
+                    // GPU Pipeline: Transfer -> Kernel -> Copyback
+                    auto arg1 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixA));
+                    auto arg2 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixB));
+
+                    const int THREADS = 32;
+                    const int BLOCKS = (N + THREADS - 1) / THREADS;
+                    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+                    auto result = mmul_kernel.run_async(st.program, dims, st.stream_id, 0, st.device_id,
+                                                       arg1, arg2,
+                                                       caf::cuda::create_out_arg<int>(N * N),
+                                                       caf::cuda::create_in_arg<int>(N));
+
+                    auto bufferC = std::get<2>(result);
+                    auto self_hdl = caf::actor_cast<caf::actor>(self);
+
+                    mmul_command.copy_to_host_async(bufferC, [self_hdl, N_task = N](std::vector<int>&&) {
+                        caf::anon_mail(task_done_atom_v, N_task).send(self_hdl); // Pass N back to self
+                    });
+                },
+                [=](error& err) {
+                    auto& st = self->state();
+                    if (err == sec::runtime_error) {
+                        // Not enough memory, retry after a delay
+                        self->println("Worker {}: Not enough memory, retrying for work...", st.stream_id);
+                        self->delayed_anon_send(self, 100ms, request_work_atom_v);
+                    } else if (err == sec::end_of_stream) {
+                        st.draining = true; // Mark as draining, let in-flight finish
+                        if (st.in_flight_tasks_count == 0) {
+                            self->mail(worker_done_atom_v).send(st.device_actor);
+                            mmul_command.release_stream_for_actor(st.stream_id);
+                            self->quit();
+                        }
+                    }
+                }
+            );
+        },
+        [=](task_done_atom, int N_completed) {
+            auto& st = self->state();
+            st.in_flight_tasks_count--; // Decrement count
+            self->mail(1).send(st.supervisor); // Notify supervisor
+            self->mail(release_memory_atom_v, N_completed).send(st.device_actor); // Release memory
+            self->mail(request_work_atom_v).send(self); // Request next task if capacity allows
+            
+            if (st.draining && st.in_flight_tasks_count == 0) {
+                self->mail(worker_done_atom_v).send(st.device_actor);
+                mmul_command.release_stream_for_actor(st.stream_id);
+                self->quit();
+            } else if (!st.draining) {
+                self->mail(request_work_atom_v).send(self); // Request next task if capacity allows
+            }
         }
-	};
+    };
 }
+
 // ---------------------------- SUPERVISOR ACTOR ----------------------------
 struct supervisor_actor_state {
-    int num_actors;
-    int num_waves;
-    int completed;
-    int max_waves;
-
-    MatrixPool pool;
-
-    // Precomputed sequence of N values
-    std::vector<int> Ns;
-    int next_task;
-
-    // Timing
+    int total_tasks;
+    int completed = 0;
     std::chrono::steady_clock::time_point start_time;
-    std::chrono::steady_clock::time_point wave_start_time;
 };
 
 caf::behavior supervisor_actor_fun(
     caf::stateful_actor<supervisor_actor_state>* self,
-    int num_actors,
-    int max_waves,
+    int total_tasks,
+    int workers_per_gpu,
+    int max_in_flight_tasks_per_worker,
     MatrixPool pool,
-    const std::vector<int>& Ns      // deterministic task sizes 
+    const std::vector<int>& Ns
     ) {
-    // Initialize state
-    self->state().num_actors = num_actors;
-    self->state().completed = 0;
-    self->state().max_waves = max_waves;
-    self->state().num_waves = 0;
-    self->state().pool = std::move(pool);
-
-    self->state().Ns = Ns;
-    self->state().next_task = 0;
-
+    self->state().total_tasks = total_tasks;
+    self->state().start_time = std::chrono::steady_clock::now();
 
     caf::cuda::manager& mgr = caf::cuda::manager::get();
+    int num_gpus = mgr.get_num_devices();
     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
-    // Kick off first wave
-    self->mail("spawn").send(self);
+    // Partition tasks across GPUs and spawn one Device Actor per GPU
+    size_t tasks_per_gpu_base = Ns.size() / num_gpus;
+    size_t remainder_tasks = Ns.size() % num_gpus;
 
-    // Start timing
-    self->state().start_time = std::chrono::steady_clock::now();
-    return {
-        // -------------------- SPAWN WAVE --------------------
-        [=](std::string cmd) {
-            if (cmd != "spawn") return;
-
-	     std::chrono::steady_clock::time_point cmd_start_time = std::chrono::steady_clock::now();
-	    
-
-            self->state().completed = 0;
-            self->state().wave_start_time = std::chrono::steady_clock::now();
-
-            for (int i = 0; i < self->state().num_actors; ++i) {
-                if (self->state().next_task >= self->state().Ns.size())
-                    break;
-
-                int N = self->state().Ns[self->state().next_task++];
-                const auto& A = self->state().pool.A[N];
-                const auto& B = self->state().pool.B[N];
-                const int THREADS = 32;
-                const int BLOCKS = (N + THREADS - 1) / THREADS;
-                caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-                self->spawn(mmul_actor_fun, self, program, dims,i,N,
-                            caf::cuda::create_in_arg(A),
-                            caf::cuda::create_in_arg(B));
-            }
+    for (int i = 0; i < num_gpus; ++i) {
+        size_t current_gpu_tasks_count = tasks_per_gpu_base + (i < remainder_tasks ? 1 : 0);
+        auto start_it = Ns.begin();
+        std::advance(start_it, i * tasks_per_gpu_base + std::min((size_t)i, remainder_tasks));
         
-	
-
-    std::chrono::steady_clock::time_point cmd_end_time = std::chrono::steady_clock::now();
+        auto end_it = start_it;
+        std::advance(end_it, current_gpu_tasks_count);
 
-      std::chrono::duration<double> total_time =
-                        cmd_end_time - cmd_start_time;
+        std::vector<int> gpu_tasks(start_it, end_it);
 
-                    std::cout << "\n===== SUPERVISOR TOTAL TIME spawn =====\n";
-                    std::cout << "Total runtime: "
-                              << total_time.count() << " s\n";
-        },
+        auto dev_actor = self->spawn(gpu_device_actor, pool, std::move(gpu_tasks), workers_per_gpu, i);
+        
+        // Spawn workers for this GPU
+        for (int j = 0; j < workers_per_gpu; ++j) {
+            int stream_id = (i * 1000) + j; // Unique stream per worker
+            self->spawn(mmul_worker_fun, self, dev_actor, program, i, stream_id, max_in_flight_tasks_per_worker);
+        }
+    }
 
-        // -------------------- COMPLETION TRACKING --------------------
+    return {
         [=](int done) {
             self->state().completed += done;
-
-            if (self->state().completed >= self->state().num_actors) {
-                auto wave_end = std::chrono::steady_clock::now();
-                std::chrono::duration<double> wave_time =
-                    wave_end - self->state().wave_start_time;
-
-                self->state().num_waves++;
-                if (self->state().num_waves >= self->state().max_waves) {
-                    auto end_time = std::chrono::steady_clock::now();
-                    std::chrono::duration<double> total_time =
-                        end_time - self->state().start_time;
-
-                    std::cout << "\n===== SUPERVISOR TOTAL TIME =====\n";
-                    std::cout << "Total runtime: "
-                              << total_time.count() << " s\n";
-                    
-		    caf::cuda::manager::shutdown();
-                    self->quit();
-                } else {
-                    self->mail("spawn").send(self);
-                }
+            if (self->state().completed >= self->state().total_tasks) {
+                auto end_time = std::chrono::steady_clock::now();
+                std::chrono::duration<double> total_time = end_time - self->state().start_time;
+
+                std::cout << "\n===== BENCHMARK COMPLETE =====\n";
+                std::cout << "Tasks: " << self->state().total_tasks << "\n";
+                std::cout << "Runtime: " << total_time.count() << " s\n";
+                
+                caf::cuda::manager::shutdown();
+                self->quit();
             }
         }
     };
 }
+
 template <class Fn>
 double time_run(Fn&& fn) {
   auto start = std::chrono::steady_clock::now();
@@ -230,55 +295,51 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     const int max_N = 2048;
     const int num_sizes = 10;
 
-    const int max_waves = 1;
+    const int workers_per_gpu = 16; // Admission control: only 16 concurrent tasks per GPU
+    const int max_in_flight_tasks_per_worker = 2; // Each worker keeps 2 tasks in flight
 
     const std::vector<int> actor_counts = {
 	  1,30000,40000,50000
     };
 
+    // Generate deterministic random pool once
+    MatrixPool pool = create_matrix_pool_random(
+        num_sizes,
+        min_N,
+        max_N,
+        42  // fixed seed
+    );
 
-    int num_actors = actor_counts[actor_counts.size()-1];
-
-        // Generate deterministic random pool
-        MatrixPool pool = create_matrix_pool_random(
-            num_sizes,
-            min_N,
-            max_N,
-            42  // fixed seed
-        );
-
-    // Precompute all task Ns (total_tasks = num_actors * max_waves)
-    std::vector<int> sizes;
-    for (const auto& [N, _] : pool.A) sizes.push_back(N);
-
-    int total_tasks = num_actors * max_waves;
-    std::vector<int> Ns;
-    Ns.reserve(total_tasks);
-
-    std::mt19937 rng(42);
-    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
-    for (int i = 0; i < total_tasks; ++i)
-	    Ns.push_back(sizes[dist(rng)]);
-
-
-   
     //scheduler
     caf::cuda::manager_config scheduler_off(false);
-    for (int num_actors : actor_counts) {
+    for (int num_tasks_for_this_run : actor_counts) {
 	    // Initialize CUDA manager
 	    caf::cuda::manager::init(sys, scheduler_off);
         std::cout << "=====================================\n";
-        std::cout << "Random Scaling | actors=" << num_actors << "\n";
+        std::cout << "Random Scaling | actors=" << num_tasks_for_this_run << "\n";
+
+        // Precompute all task Ns for this run
+        std::vector<int> sizes;
+        for (const auto& [N, _] : pool.A) sizes.push_back(N);
+
+        std::vector<int> Ns_for_this_run;
+        Ns_for_this_run.reserve(num_tasks_for_this_run);
+
+        std::mt19937 rng(42);
+        std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+        for (int i = 0; i < num_tasks_for_this_run; ++i)
+	        Ns_for_this_run.push_back(sizes[dist(rng)]);
 
         // Execute the supervisor which manages the asynchronous workload
         double elapsed = time_run([&]() {
 
             auto sup = sys.spawn(
                 supervisor_actor_fun,
-                num_actors,
-                max_waves,
+                (int)Ns_for_this_run.size(), // total_tasks
+                workers_per_gpu,
+                max_in_flight_tasks_per_worker,
                 pool,
-		Ns
+		Ns_for_this_run
             );
 
 	      sys.await_all_actors_done();
@@ -291,4 +352,4 @@ void caf_main(caf::actor_system& sys) {
   caf::cuda::manager_config man_config(false);
   run_mmul_random_scaling_tests(sys, man_config);
 }
-CAF_MAIN()
+CAF_MAIN(id_block::mmul_benchmark)

From 80ec8f01610155c9e7449c6fc8a6a279e9acc3ea Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 8 May 2026 08:36:09 -0600
Subject: [PATCH 0584/1000] added a helper method to give the memory for a
 specifc device

---
 libcaf_cuda/caf/cuda/manager.hpp | 3 +--
 libcaf_cuda/src/manager.cpp      | 5 +++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index 4256bb7334..3883f67efa 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -177,7 +177,7 @@ class CAF_CUDA_EXPORT manager {
 
   int get_num_devices();
 
-
+  double available_memory_mb(int id = 0);
 
   caf::actor get_scheduler_actor();
 
@@ -221,4 +221,3 @@ class CAF_CUDA_EXPORT manager {
 };
 
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index d258be8677..d9f7403c0e 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -179,6 +179,11 @@ device_ptr manager::find_device(int id) {
 
 }
 
+double manager::available_memory_mb(int id) {
+  auto dev = find_device(id);
+  return dev ? dev->available_memory_mb() : 0.0;
+}
+
 //creates a program ptr given a kernel and a string 
 program_ptr manager::create_program(const char * kernel,
                                     const std::string& name,

From 07748ca8acf125b03cabdeda91ca125c7970b870 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 8 May 2026 08:44:35 -0600
Subject: [PATCH 0585/1000] Added a new test to check if memory allocation and
 dellalocation is being done correctly.

---
 .../actor-tests/callback-test/CMakeLists.txt  |  10 +-
 .../callback-test/memory_stress.test.cpp      | 126 ++++++++++++++++++
 2 files changed, 135 insertions(+), 1 deletion(-)
 create mode 100644 libcaf_cuda/tests/actor-tests/callback-test/memory_stress.test.cpp

diff --git a/libcaf_cuda/tests/actor-tests/callback-test/CMakeLists.txt b/libcaf_cuda/tests/actor-tests/callback-test/CMakeLists.txt
index 89bcf5ba7c..2c6e09e13c 100644
--- a/libcaf_cuda/tests/actor-tests/callback-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/actor-tests/callback-test/CMakeLists.txt
@@ -41,4 +41,12 @@ target_link_libraries(test
     CUDA::nvrtc
 )
 
-
+add_executable(memory_stress_test memory_stress.test.cpp)
+target_compile_definitions(memory_stress_test PRIVATE CAF_ENABLE_LOGGING)
+target_link_libraries(memory_stress_test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
diff --git a/libcaf_cuda/tests/actor-tests/callback-test/memory_stress.test.cpp b/libcaf_cuda/tests/actor-tests/callback-test/memory_stress.test.cpp
new file mode 100644
index 0000000000..93523f3cd3
--- /dev/null
+++ b/libcaf_cuda/tests/actor-tests/callback-test/memory_stress.test.cpp
@@ -0,0 +1,126 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include <numeric>
+
+using namespace caf;
+
+CAF_BEGIN_TYPE_ID_BLOCK(memory_stress, caf::id_block::cuda::end)
+  CAF_ADD_ATOM(memory_stress, start_atom)
+  CAF_ADD_ATOM(memory_stress, task_done_atom)
+CAF_END_TYPE_ID_BLOCK(memory_stress)
+
+using mmulCommand = caf::cuda::command_runner<in<int>, 
+                                              in<int>, 
+                                              out<int>, 
+                                              in<int>>;
+mmulCommand mmul;
+
+// Pre-allocated static data to avoid CPU overhead during the test
+static std::vector<int> GLOBAL_A;
+static std::vector<int> GLOBAL_B;
+
+struct worker_state {
+  static inline const char* name = "stress_worker";
+  int actor_id = rand();
+  caf::cuda::program_ptr program;
+};
+
+caf::behavior stress_worker_fun(caf::stateful_actor<worker_state>* self, caf::actor supervisor) {
+  // Load the program once during actor initialization
+  auto& mgr = caf::cuda::manager::get();
+  self->state().program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+  return {
+    [=](int N) {
+      const int THREADS = 32;
+      const int BLOCKS = (N + THREADS - 1) / THREADS;
+      caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+      // We use the pre-allocated GLOBAL vectors. 
+      auto results = mmul.run_async(self->state().program, dims, self->state().actor_id,
+                                    caf::cuda::create_in_arg(GLOBAL_A),
+                                    caf::cuda::create_in_arg(GLOBAL_B),
+                                    caf::cuda::create_out_arg(N * N),
+                                    caf::cuda::create_in_arg(N));
+
+      auto matrixC_ptr = std::get<2>(results);
+
+      // USER REQUEST: Do NOT capture matrixC_ptr in the lambda.
+      // This will allow the mem_ptr to be destroyed immediately after this call.
+      mmul.copy_to_host_async(matrixC_ptr, [supervisor](std::vector<int> h_c) mutable {
+        caf::anon_mail(task_done_atom_v).send(supervisor);
+      });
+    }
+  };
+}
+
+struct supervisor_state {
+  int total_tasks = 2000;
+  int tasks_submitted = 0;
+  int tasks_completed = 0;
+  int max_in_flight = 20; 
+  caf::actor worker;
+  size_t initial_mem = 0;
+};
+
+caf::behavior supervisor_fun(caf::stateful_actor<supervisor_state>* self) {
+  auto submit_work = [=] {
+    while (self->state().tasks_submitted < self->state().total_tasks &&
+           (self->state().tasks_submitted - self->state().tasks_completed) < self->state().max_in_flight) {
+      self->mail(1024).send(self->state().worker);
+      self->state().tasks_submitted++;
+    }
+  };
+
+  return {
+    [=](start_atom) {
+      auto& mgr_ref = caf::cuda::manager::get();
+      self->state().initial_mem = mgr_ref.available_memory_mb();
+      
+      std::cout << "--- GPU Memory Stress Test Initialized ---" << std::endl;
+      std::cout << "Initial Free Memory: " << self->state().initial_mem << " MB" << std::endl;
+      std::cout << "Target Workload: " << self->state().total_tasks << " tasks" << std::endl;
+
+      self->state().worker = self->spawn(stress_worker_fun, caf::actor_cast<caf::actor>(self));
+      submit_work();
+    },
+    [=](task_done_atom) {
+      self->state().tasks_completed++;
+
+      if (self->state().tasks_completed % 100 == 0) {
+        auto current_free = caf::cuda::manager::get().available_memory_mb();
+        std::cout << "Progress: " << self->state().tasks_completed << "/" << self->state().total_tasks
+                  << " | Current Free GPU Memory: " << current_free << " MB" << std::endl;
+      }
+
+      if (self->state().tasks_completed == self->state().total_tasks) {
+        std::cout << "Test Finished. Final Free Memory: " << caf::cuda::manager::get().available_memory_mb() << " MB" << std::endl;
+        self->send_exit(self->state().worker, exit_reason::user_shutdown);
+        self->quit();
+      } else {
+        submit_work();
+      }
+    }
+  };
+}
+
+void setup_global_data(int N) {
+  GLOBAL_A.resize(N * N);
+  GLOBAL_B.resize(N * N);
+  std::iota(GLOBAL_A.begin(), GLOBAL_A.end(), 1);
+  std::iota(GLOBAL_B.begin(), GLOBAL_B.end(), 1);
+}
+
+void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager::init(sys);
+  setup_global_data(1024);
+  
+  auto sv = sys.spawn(supervisor_fun);
+  caf::anon_mail(start_atom_v).send(sv);
+  
+  sys.await_all_actors_done();
+}
+
+CAF_MAIN(id_block::memory_stress)

From 5f9fd3a8054c2c38ffdddd1747fd964ea371a92d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 8 May 2026 09:45:51 -0600
Subject: [PATCH 0586/1000] changed calls that require allocating memory from
 cuMemAlloc to cuMemAllocAsync. This change was made to fix an issue
 introduced by making mem_ref deallocation asynchronous as the stream order
 allocator in CUDA and cumemalloc would be competeing for memory causing
 unecessary OOM issues

---
 libcaf_cuda/caf/cuda/device.hpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index a6333af4f2..bc41ecf2af 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -383,7 +383,7 @@ mem_ptr<T> global_argument(const in<T>& arg, CUstream stream, int access) {
   size_t bytes = arg.size() * sizeof(T);
   CUdeviceptr dev_ptr;
   CHECK_CUDA(cuCtxPushCurrent(getContext()));
-  CHECK_CUDA(cuMemAlloc(&dev_ptr, bytes));
+  CHECK_CUDA(cuMemAllocAsync(&dev_ptr, bytes, stream));
   CHECK_CUDA(cuMemcpyHtoDAsync(dev_ptr, arg.data(), bytes, stream));
   CHECK_CUDA(cuCtxPopCurrent(nullptr));
   return caf::intrusive_ptr<mem_ref<T>>(
@@ -400,7 +400,7 @@ mem_ptr<T> global_argument(const in_out<T>& arg, CUstream stream, int access) {
   size_t bytes = arg.size() * sizeof(T);
   CUdeviceptr dev_ptr;
   CHECK_CUDA(cuCtxPushCurrent(getContext()));
-  CHECK_CUDA(cuMemAlloc(&dev_ptr, bytes));
+  CHECK_CUDA(cuMemAllocAsync(&dev_ptr, bytes, stream));
   CHECK_CUDA(cuMemcpyHtoDAsync(dev_ptr, arg.data(), bytes, stream));
   CHECK_CUDA(cuCtxPopCurrent(nullptr));
   return caf::intrusive_ptr<mem_ref<T>>(
@@ -413,7 +413,7 @@ mem_ptr<T> scratch_argument(const out<T>& arg, CUstream stream, int access) {
   size_t size =  arg.size();
   CUdeviceptr dev_ptr;
   CHECK_CUDA(cuCtxPushCurrent(getContext()));
-  CHECK_CUDA(cuMemAlloc(&dev_ptr, size * sizeof(T)));
+  CHECK_CUDA(cuMemAllocAsync(&dev_ptr, size * sizeof(T), stream));
   CHECK_CUDA(cuCtxPopCurrent(nullptr));
   return caf::intrusive_ptr<mem_ref<T>>(
     new mem_ref<T>(size, dev_ptr, access, id_, 0, getContext(), stream));
@@ -456,4 +456,3 @@ mem_ptr<T> scratch_argument(const out<T>& arg, CUstream stream, int access) {
 };
 
 } // namespace caf::cuda
-

From 803842efcfa553aa3f78ad44560c353e8e515d34 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 8 May 2026 09:48:13 -0600
Subject: [PATCH 0587/1000] changed error checking on reset

---
 libcaf_cuda/caf/cuda/mem_ref.hpp | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/mem_ref.hpp b/libcaf_cuda/caf/cuda/mem_ref.hpp
index c7ef5fe453..751ce70083 100644
--- a/libcaf_cuda/caf/cuda/mem_ref.hpp
+++ b/libcaf_cuda/caf/cuda/mem_ref.hpp
@@ -100,9 +100,24 @@ class mem_ref : public caf::ref_counted {
   //sets all its attributes to null or -1
   void reset() {
     if (!is_scalar_ && memory_) {
-      CHECK_CUDA(cuCtxPushCurrent(ctx));
-      CHECK_CUDA(cuMemFreeAsync(memory_, stream_));
-      CHECK_CUDA(cuCtxPopCurrent(nullptr));
+      if (ctx) {
+        CUresult res = cuCtxPushCurrent(ctx);
+        if (res == CUDA_SUCCESS) {
+          CUresult free_res = cuMemFreeAsync(memory_, stream_);
+          if (free_res != CUDA_SUCCESS) {
+            const char* err_str = nullptr;
+            cuGetErrorString(free_res, &err_str);
+            std::cerr << "[ERROR] cuMemFreeAsync failed in mem_ref::reset: " 
+                      << (err_str ? err_str : "unknown error") << std::endl;
+          }
+          cuCtxPopCurrent(nullptr);
+        } else {
+          const char* err_str = nullptr;
+          cuGetErrorString(res, &err_str);
+          std::cerr << "[ERROR] cuCtxPushCurrent failed in mem_ref::reset: " 
+                    << (err_str ? err_str : "unknown error") << std::endl;
+        }
+      }
       memory_ = 0;
     }
     num_elements_ = 0;

From b07ca8316a7d420714990c28936b8e4f1fea8379 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 8 May 2026 10:15:17 -0600
Subject: [PATCH 0588/1000] fixed interaction between supervisor and worker
 actor so that it would not think it was OOM.

---
 .../main.test.cpp                             | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 066c957237..8bedda0f6f 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -99,6 +99,10 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
             // Calculate memory needed for A, B, C
             size_t memory_needed = (size_t)N * N * sizeof(int) * 3; // A, B, C
 
+            // std::cout << "total memory bytes is " << st.total_device_memory_bytes << "\n";
+            // std::cout << "current allocated memory bytes is " << st.current_allocated_memory_bytes << "\n";
+            // std::cout << "memory needed is " << memory_needed << "\n";
+
             if (st.current_allocated_memory_bytes + memory_needed > st.total_device_memory_bytes) {
                 return make_error(sec::runtime_error, "Device Actor: Not enough memory");
             }
@@ -152,15 +156,14 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
 
     return {
         [=](request_work_atom) {
-            if (self->state().in_flight_tasks_count >= self->state().max_in_flight_tasks) {
+            auto& st = self->state();
+            if (st.in_flight_tasks_count >= st.max_in_flight_tasks || st.draining) {
                 return; // Already at max capacity, don't request more yet
             }
 
-            self->mail(get_work_atom_v).request(self->state().device_actor, infinite).then(
+            st.in_flight_tasks_count++; // Mark as pending immediately
+            self->mail(get_work_atom_v).request(st.device_actor, infinite).then(
                 [=](int N, in<int> matrixA, in<int> matrixB) {
-                    auto& st = self->state();
-                    st.in_flight_tasks_count++; 
-
                     // GPU Pipeline: Transfer -> Kernel -> Copyback
                     auto arg1 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixA));
                     auto arg2 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixB));
@@ -183,6 +186,7 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
                 },
                 [=](error& err) {
                     auto& st = self->state();
+                    st.in_flight_tasks_count--; // Revert pending status on failure
                     if (err == sec::runtime_error) {
                         // Not enough memory, retry after a delay
                         self->println("Worker {}: Not enough memory, retrying for work...", st.stream_id);
@@ -203,7 +207,6 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
             st.in_flight_tasks_count--; // Decrement count
             self->mail(1).send(st.supervisor); // Notify supervisor
             self->mail(release_memory_atom_v, N_completed).send(st.device_actor); // Release memory
-            self->mail(request_work_atom_v).send(self); // Request next task if capacity allows
             
             if (st.draining && st.in_flight_tasks_count == 0) {
                 self->mail(worker_done_atom_v).send(st.device_actor);
@@ -295,8 +298,8 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     const int max_N = 2048;
     const int num_sizes = 10;
 
-    const int workers_per_gpu = 16; // Admission control: only 16 concurrent tasks per GPU
-    const int max_in_flight_tasks_per_worker = 2; // Each worker keeps 2 tasks in flight
+    const int workers_per_gpu = 8; // Admission control: only 16 concurrent tasks per GPU
+    const int max_in_flight_tasks_per_worker = 3; // Each worker keeps 2 tasks in flight
 
     const std::vector<int> actor_counts = {
 	  1,30000,40000,50000

From e10ae2486cea18e3f158011f77f331c42c96425d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 08:42:08 -0600
Subject: [PATCH 0589/1000] Initial commit.

---
 .../work-stealing.cpp                         | 295 ++++++++++++++++++
 1 file changed, 295 insertions(+)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
new file mode 100644
index 0000000000..a0cc63eda3
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -0,0 +1,295 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include <chrono>
+#include <random>
+#include <unordered_map>
+#include <deque>
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Atoms
+// ─────────────────────────────────────────────────────────────────────────────
+CAF_BEGIN_TYPE_ID_BLOCK(dynamic_work_stealing, caf::id_block::cuda::end)
+    CAF_ADD_ATOM(dynamic_work_stealing, get_work_atom)
+    CAF_ADD_ATOM(dynamic_work_stealing, task_done_atom)
+    CAF_ADD_ATOM(dynamic_work_stealing, release_memory_atom)
+    CAF_ADD_ATOM(dynamic_work_stealing, request_work_atom)
+    CAF_ADD_ATOM(dynamic_work_stealing, worker_done_atom)
+    CAF_ADD_ATOM(dynamic_work_stealing, refill_buffer_atom)
+CAF_END_TYPE_ID_BLOCK(dynamic_work_stealing)
+
+// Command runners for GPU operations
+using mmul_kernel_t = caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>, out<int>, in<int>>;
+mmul_kernel_t mmul_kernel;
+caf::cuda::command_runner<> mmul_command;
+
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
+};
+
+// ---------------------------- GLOBAL TASK POOL ----------------------------
+// The central source of truth for work. Implements a pull-based model.
+struct task_pool_state {
+    std::vector<int> tasks;
+    size_t next_task_idx = 0;
+};
+
+caf::behavior global_task_pool(caf::stateful_actor<task_pool_state>* self, std::vector<int> tasks) {
+    self->state().tasks = std::move(tasks);
+    return {
+        [=](get_work_atom, size_t batch_size) -> result<std::vector<int>> {
+            auto& st = self->state();
+            if (st.next_task_idx >= st.tasks.size())
+                return sec::end_of_stream;
+            size_t count = std::min(batch_size, st.tasks.size() - st.next_task_idx);
+            std::vector<int> batch(st.tasks.begin() + st.next_task_idx, 
+                                   st.tasks.begin() + st.next_task_idx + count);
+            st.next_task_idx += count;
+            return batch;
+        }
+    };
+}
+
+// ---------------------------- DEVICE BROKER ----------------------------
+// Manages memory for a specific GPU and steals (pulls) work from the Global Pool.
+struct device_actor_state {
+    MatrixPool pool;
+    caf::actor global_pool;
+    std::deque<int> local_tasks; // Local buffer to keep GPU busy
+    size_t total_mem = 0;
+    size_t current_mem = 0;
+    int device_id = -1;
+    int active_workers = 0;
+    size_t batch_size = 0;
+    size_t low_water_mark = 0;
+    bool fetching = false;
+};
+
+caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
+                               MatrixPool pool, caf::actor global_pool, int num_workers, int dev_id, int max_in_flight) {
+    self->state().pool = std::move(pool);
+    self->state().global_pool = global_pool;
+    self->state().device_id = dev_id;
+    self->state().active_workers = num_workers;
+
+    // Dynamically calculate prefetch markers based on the total pipeline capacity
+    self->state().low_water_mark = static_cast<size_t>(num_workers * max_in_flight);
+    self->state().batch_size = self->state().low_water_mark * 2;
+
+    auto dev_obj = caf::cuda::manager::get().find_device(dev_id);
+    if (dev_obj)
+        self->state().total_mem = dev_obj->total_memory_bytes();
+
+    // Helper to refill the local task buffer from the global pool
+    auto refill = [=]() {
+        auto& st = self->state();
+        if (st.fetching || st.local_tasks.size() >= st.low_water_mark + st.batch_size)
+            return;
+
+        st.fetching = true;
+        self->mail(get_work_atom_v, (size_t)st.batch_size).request(st.global_pool, infinite).then(
+            [=](std::vector<int>& batch) {
+                auto& st_inner = self->state();
+                for (int N : batch)
+                    st_inner.local_tasks.push_back(N);
+                st_inner.fetching = false;
+                if (st_inner.local_tasks.size() < st_inner.low_water_mark)
+                    self->mail(refill_buffer_atom_v).send(self);
+            },
+            [=](error& err) {
+                self->state().fetching = false;
+            }
+        );
+    };
+
+    return {
+        [=](refill_buffer_atom) {
+            refill();
+        },
+        [=](get_work_atom) -> caf::result<int, in<int>, in<int>> {
+            auto& st = self->state();
+
+            // If we have tasks locally, satisfy the request immediately
+            if (!st.local_tasks.empty()) {
+                int N = st.local_tasks.front();
+                size_t needed = (size_t)N * N * sizeof(int) * 3;
+                if (st.current_mem + needed > st.total_mem)
+                    return make_error(sec::runtime_error, "Out of GPU Memory");
+                
+                st.local_tasks.pop_front();
+                st.current_mem += needed;
+                
+                // Proactively steal more work if the buffer is getting low
+                if (st.local_tasks.size() < st.low_water_mark)
+                    refill();
+
+                return {N, caf::cuda::create_in_arg(st.pool.A[N]), 
+                           caf::cuda::create_in_arg(st.pool.B[N])};
+            }
+
+            // Buffer empty: must fetch from global pool reactively
+            auto promise = self->make_response_promise<int, in<int>, in<int>>();
+            self->mail(get_work_atom_v, (size_t)st.batch_size).request(st.global_pool, infinite).then(
+                [=](std::vector<int>& batch) mutable {
+                    auto& st_inner = self->state();
+                    int N = batch.front();
+                    for(size_t i = 1; i < batch.size(); ++i) st_inner.local_tasks.push_back(batch[i]);
+                    
+                    st_inner.current_mem += (size_t)N * N * sizeof(int) * 3;
+                    promise.deliver(N, caf::cuda::create_in_arg(st_inner.pool.A[N]), 
+                                       caf::cuda::create_in_arg(st_inner.pool.B[N]));
+                },
+                [=](error& err) mutable { promise.deliver(err); }
+            );
+            return promise;
+        },
+        [=](release_memory_atom, int N) {
+            self->state().current_mem -= (size_t)N * N * sizeof(int) * 3;
+            refill(); // Try to get more work now that memory is free
+        },
+        [=](worker_done_atom) {
+            if (--self->state().active_workers <= 0)
+                self->quit();
+        }
+    };
+}
+
+// ---------------------------- WORKER ----------------------------
+struct worker_state {
+    caf::actor device_actor;
+    caf::actor supervisor;
+    caf::cuda::program_ptr program;
+    int dev_id;
+    int stream_id;
+    int in_flight = 0;
+    int max_in_flight = 2; // Keep the pipeline full
+    bool draining = false;
+};
+
+caf::behavior mmul_worker(caf::stateful_actor<worker_state>* self, 
+                          caf::actor supervisor, caf::actor device_actor, 
+                          caf::cuda::program_ptr prog, int dev, int stream, int max_in_flight) {
+    self->state().supervisor = supervisor;
+    self->state().device_actor = device_actor;
+    self->state().program = prog;
+    self->state().dev_id = dev;
+    self->state().stream_id = stream;
+    self->state().max_in_flight = max_in_flight;
+
+    for (int i = 0; i < self->state().max_in_flight; ++i)
+        self->mail(request_work_atom_v).send(self);
+
+    return {
+        [=](request_work_atom) {
+            auto& st = self->state();
+            if (st.in_flight >= st.max_in_flight || st.draining) return;
+            st.in_flight++;
+            self->mail(get_work_atom_v).request(st.device_actor, infinite).then(
+                [=](int N, in<int> A, in<int> B) {
+                    auto a1 = mmul_command.transfer_memory(st.dev_id, st.stream_id, std::move(A));
+                    auto a2 = mmul_command.transfer_memory(st.dev_id, st.stream_id, std::move(B));
+                    caf::cuda::nd_range dims((N + 31) / 32, (N + 31) / 32, 1, 32, 32, 1);
+                    auto res = mmul_kernel.run_async(st.program, dims, st.stream_id, 0, st.dev_id,
+                                                     a1, a2, caf::cuda::create_out_arg<int>(N * N),
+                                                     caf::cuda::create_in_arg<int>(N));
+                    auto self_ptr = caf::actor_cast<caf::actor>(self);
+                    mmul_command.copy_to_host_async(std::get<2>(res), [self_ptr, N](std::vector<int>&&) {
+                        caf::anon_mail(task_done_atom_v, N).send(self_ptr);
+                    });
+                },
+                [=](error& err) {
+                    st.in_flight--;
+                    if (err == sec::end_of_stream) {
+                        st.draining = true;
+                        if (st.in_flight == 0) {
+                            self->mail(worker_done_atom_v).send(st.device_actor);
+                            self->quit();
+                        }
+                    } else {
+                        self->delayed_anon_send(self, 100ms, request_work_atom_v);
+                    }
+                }
+            );
+        },
+        [=](task_done_atom, int N) {
+            self->state().in_flight--;
+            self->mail(1).send(self->state().supervisor);
+            self->mail(release_memory_atom_v, N).send(self->state().device_actor);
+            self->mail(request_work_atom_v).send(self);
+        }
+    };
+}
+
+// ---------------------------- SUPERVISOR ----------------------------
+struct supervisor_state {
+    int total;
+    int done = 0;
+    std::chrono::steady_clock::time_point start;
+};
+
+caf::behavior supervisor_actor(caf::stateful_actor<supervisor_state>* self, 
+                               int total, 
+                               std::vector<int> Ns,
+                               int workers_per_gpu,
+                               int max_in_flight) {
+    self->state().total = total;
+    self->state().start = std::chrono::steady_clock::now();
+    auto pool = self->spawn(global_task_pool, std::move(Ns));
+    
+    MatrixPool m_pool;
+    for (int n : {256, 2048}) {
+        m_pool.A[n] = std::vector<int>(n * n, 1);
+        m_pool.B[n] = std::vector<int>(n * n, 1);
+    }
+
+    auto& mgr = caf::cuda::manager::get();
+    auto prog = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    for (int i = 0; i < mgr.get_num_devices(); ++i) {
+        auto broker = self->spawn(gpu_device_actor, m_pool, pool, workers_per_gpu, i, max_in_flight);
+        for (int j = 0; j < workers_per_gpu; ++j)
+            self->spawn(mmul_worker, self, broker, prog, i, (i * 100) + j, max_in_flight);
+    }
+
+    return {
+        [=](int count) {
+            self->state().done += count;
+            if (self->state().done >= self->state().total) {
+                auto elapsed = std::chrono::steady_clock::now() - self->state().start;
+                std::cout << "Dynamic Work-Stealing Complete.\n"
+                          << "Makespan: " << std::chrono::duration<double>(elapsed).count() << "s\n";
+                caf::cuda::manager::shutdown();
+                self->quit();
+            }
+        }
+    };
+}
+
+void caf_main(caf::actor_system& sys) {
+    caf::cuda::manager_config cfg(false);
+    caf::cuda::manager::init(sys, cfg);
+
+    int total_tasks = 2000;
+    int workers_per_gpu = 8;
+    int max_in_flight = 3;
+
+    std::vector<int> Ns;
+    std::mt19937 rng(42);
+    std::uniform_real_distribution<double> dist(0.0, 1.0); // Power Law setup
+
+    // Power Law simulation: 10% Heavy (2048), 90% Light (256)
+    for (int i = 0; i < total_tasks; ++i) {
+        if (dist(rng) < 0.1) Ns.push_back(2048);
+        else Ns.push_back(256);
+    }
+
+    sys.spawn(supervisor_actor, total_tasks, std::move(Ns), workers_per_gpu, max_in_flight);
+    sys.await_all_actors_done();
+}
+
+CAF_MAIN(id_block::dynamic_work_stealing)

From 86fd1b2e4b9d0a0d13561866d5c188314026136c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 08:45:22 -0600
Subject: [PATCH 0590/1000] Implementing a workstealing test.

---
 .../mmul-randonom-batch-benchmark/CMakeLists.txt    | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
index e2259ce5fe..b65a44ffe0 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
@@ -29,10 +29,13 @@ include_directories(
 )
 
 
-# 5) Declare your executable
+# 5) Declare your executables
 add_executable(test main.test.cpp)
 target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
 
+add_executable(work-stealing work-stealing.cpp)
+target_compile_definitions(work-stealing PRIVATE CAF_ENABLE_LOGGING)
+
 target_link_libraries(test
   PRIVATE
     "${CAF_BUILD}/libcaf_core/libcaf_core.so"
@@ -41,4 +44,10 @@ target_link_libraries(test
     CUDA::nvrtc
 )
 
-
+target_link_libraries(work-stealing
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)

From f2fb350ba1d5d13fde7269434802ed23765ea7ed Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 08:50:24 -0600
Subject: [PATCH 0591/1000] .Fixed syntax errors.

---
 .../work-stealing.cpp                         | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index a0cc63eda3..8f414d332b 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -190,11 +190,12 @@ caf::behavior mmul_worker(caf::stateful_actor<worker_state>* self,
             if (st.in_flight >= st.max_in_flight || st.draining) return;
             st.in_flight++;
             self->mail(get_work_atom_v).request(st.device_actor, infinite).then(
-                [=](int N, in<int> A, in<int> B) {
-                    auto a1 = mmul_command.transfer_memory(st.dev_id, st.stream_id, std::move(A));
-                    auto a2 = mmul_command.transfer_memory(st.dev_id, st.stream_id, std::move(B));
+                [=](int N, in<int> A, in<int> B) mutable {
+                    auto& st_inner = self->state();
+                    auto a1 = mmul_command.transfer_memory(st_inner.dev_id, st_inner.stream_id, std::move(A));
+                    auto a2 = mmul_command.transfer_memory(st_inner.dev_id, st_inner.stream_id, std::move(B));
                     caf::cuda::nd_range dims((N + 31) / 32, (N + 31) / 32, 1, 32, 32, 1);
-                    auto res = mmul_kernel.run_async(st.program, dims, st.stream_id, 0, st.dev_id,
+                    auto res = mmul_kernel.run_async(st_inner.program, dims, st_inner.stream_id, 0, st_inner.dev_id,
                                                      a1, a2, caf::cuda::create_out_arg<int>(N * N),
                                                      caf::cuda::create_in_arg<int>(N));
                     auto self_ptr = caf::actor_cast<caf::actor>(self);
@@ -202,12 +203,13 @@ caf::behavior mmul_worker(caf::stateful_actor<worker_state>* self,
                         caf::anon_mail(task_done_atom_v, N).send(self_ptr);
                     });
                 },
-                [=](error& err) {
-                    st.in_flight--;
+                [=](error& err) mutable {
+                    auto& st_inner = self->state();
+                    st_inner.in_flight--;
                     if (err == sec::end_of_stream) {
-                        st.draining = true;
-                        if (st.in_flight == 0) {
-                            self->mail(worker_done_atom_v).send(st.device_actor);
+                        st_inner.draining = true;
+                        if (st_inner.in_flight == 0) {
+                            self->mail(worker_done_atom_v).send(st_inner.device_actor);
                             self->quit();
                         }
                     } else {

From a1190f4668e5b601c9e5551b6960df6b6689936f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 08:52:43 -0600
Subject: [PATCH 0592/1000] Updated task size to be larger.

---
 .../mmul-randonom-batch-benchmark/work-stealing.cpp             | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index 8f414d332b..23f8b09602 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -276,7 +276,7 @@ void caf_main(caf::actor_system& sys) {
     caf::cuda::manager_config cfg(false);
     caf::cuda::manager::init(sys, cfg);
 
-    int total_tasks = 2000;
+    int total_tasks = 30000;
     int workers_per_gpu = 8;
     int max_in_flight = 3;
 

From 17f48b16291e0ee3fb396f1b55aa896454eeed20 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 09:08:38 -0600
Subject: [PATCH 0593/1000] Implemented a cuda equavalent of work-stealing.cpp.

---
 .../CMakeLists.txt                            |   3 +
 .../cuda-baseline.cpp                         | 187 ++++++++++++++++++
 2 files changed, 190 insertions(+)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
index b65a44ffe0..e180d2c860 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
@@ -36,6 +36,9 @@ target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
 add_executable(work-stealing work-stealing.cpp)
 target_compile_definitions(work-stealing PRIVATE CAF_ENABLE_LOGGING)
 
+add_executable(cuda-baseline cuda-baseline.cpp)
+target_link_libraries(cuda-baseline PRIVATE CUDA::cuda_driver)
+
 target_link_libraries(test
   PRIVATE
     "${CAF_BUILD}/libcaf_core/libcaf_core.so"
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
new file mode 100644
index 0000000000..5f3f497856
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -0,0 +1,187 @@
+#include <cuda.h> // CUDA Driver API
+#include <iostream>
+#include <vector>
+#include <chrono>
+#include <random>
+#include <thread>
+#include <algorithm>
+#include <stdexcept> // For runtime_error
+
+struct Task {
+    int N;
+};
+
+// Per-GPU execution logic
+void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_gpu, 
+                CUcontext ctx, CUfunction kernel_func, const std::vector<int>& h_256, const std::vector<int>& h_2048) {
+    // Set the CUDA context for this thread
+    CUresult err = cuCtxSetCurrent(ctx);
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error setting context for device " << device_id << ": " << err_str << std::endl;
+        return;
+    }
+
+    // Prepare Streams
+    std::vector<CUstream> streams(streams_per_gpu);
+    for (int i = 0; i < streams_per_gpu; ++i) {
+        cuStreamCreate(&streams[i], CU_STREAM_NON_BLOCKING);
+    }
+
+    // Use the first stream for initial allocations and cleanup
+    CUstream default_stream = streams[0];
+
+    // Process assigned tasks
+    for (size_t i = 0; i < tasks.size(); ++i) {
+        int N = tasks[i].N;
+        CUstream stream = streams[i % streams_per_gpu];
+        size_t bytes = static_cast<size_t>(N) * N * sizeof(int);
+
+        CUdeviceptr d_a, d_b, d_c;
+
+        // Allocate GPU memory for this specific task
+        cuMemAllocAsync(&d_a, bytes, stream);
+        cuMemAllocAsync(&d_b, bytes, stream);
+        cuMemAllocAsync(&d_c, bytes, stream);
+
+        // Perform Host-to-Device transfer
+        const int* h_src = (N == 256) ? h_256.data() : h_2048.data();
+        cuMemcpyHtoDAsync(d_a, h_src, bytes, stream);
+        cuMemcpyHtoDAsync(d_b, h_src, bytes, stream);
+
+        // Kernel arguments for cuLaunchKernel
+        void *kernel_args[] = { &d_a, &d_b, &d_c, &N };
+
+        // Kernel dimensions
+        dim3 threads(32, 32);
+        dim3 blocks((N + 31) / 32, (N + 31) / 32);
+
+        cuLaunchKernel(kernel_func, blocks.x, blocks.y, blocks.z,
+                       threads.x, threads.y, threads.z,
+                       0, stream, kernel_args, nullptr);
+        
+        // Simulating the result retrieval (Copy back)
+        std::vector<int> h_res(N * N);
+        cuMemcpyDtoHAsync(reinterpret_cast<CUdeviceptr>(h_res.data()), d_c, bytes, stream);
+
+        // Free GPU memory for this task
+        cuMemFreeAsync(d_a, stream);
+        cuMemFreeAsync(d_b, stream);
+        cuMemFreeAsync(d_c, stream);
+    }
+
+    // Synchronize this GPU context
+    cuCtxSynchronize();
+
+    // Cleanup
+    for (auto s : streams) {
+        cuStreamDestroy(s);
+    }
+}
+
+int main() {
+    CUresult err;
+
+    // Initialize the CUDA Driver API
+    err = cuInit(0);
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error initializing CUDA Driver API: " << err_str << std::endl;
+        return 1;
+    }
+
+    int num_gpus;
+    cuDeviceGetCount(&num_gpus);
+    
+    if (num_gpus == 0) {
+        std::cerr << "No CUDA devices found." << std::endl;
+        return 1;
+    }
+
+    const int total_tasks = 30000;
+    const int streams_per_gpu = 8;
+
+    std::vector<Task> all_tasks;
+    std::mt19937 rng(42);
+    std::uniform_real_distribution<double> dist(0.0, 1.0);
+
+    std::cout << "Generating " << total_tasks << " tasks (10% Heavy, 90% Light)..." << std::endl;
+    for (int i = 0; i < total_tasks; ++i) {
+        if (dist(rng) < 0.1) all_tasks.push_back({2048});
+        else all_tasks.push_back({256});
+    }
+
+    // Prepare Host-side MatrixPool (on CPU)
+    std::vector<int> h_256(256 * 256, 1);
+    std::vector<int> h_2048(2048 * 2048, 1);
+
+    std::vector<CUcontext> contexts(num_gpus);
+    std::vector<CUfunction> kernel_funcs(num_gpus);
+    CUmodule module;
+
+    // Load the cubin module (assuming mmul.cu is compiled to mmul.cubin)
+    err = cuModuleLoad(&module, "../mmul.cubin");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error loading module ../mmul.cubin: " << err_str << std::endl;
+        return 1;
+    }
+
+    // Get function handle for matrixMul
+    err = cuModuleGetFunction(&kernel_funcs[0], module, "matrixMul");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error getting function matrixMul: " << err_str << std::endl;
+        return 1;
+    }
+    // Assuming all GPUs can use the same function handle from the same module.
+    for (int i = 1; i < num_gpus; ++i) {
+        kernel_funcs[i] = kernel_funcs[0];
+    }
+
+    // Create a CUDA context for each device
+    for (int i = 0; i < num_gpus; ++i) {
+        CUdevice dev;
+        cuDeviceGet(&dev, i);
+        cuCtxCreate(&contexts[i], 0, dev); // Flag 0 for default context creation
+    }
+
+    // ─────────────────────────────────────────────────────────────────────────
+    // Static Round-Robin Partitioning
+    // ─────────────────────────────────────────────────────────────────────────
+    std::vector<std::vector<Task>> partitions(num_gpus);
+    for (int i = 0; i < total_tasks; ++i) {
+        partitions[i % num_gpus].push_back(all_tasks[i]);
+    }
+
+    std::cout << "Starting Static Partitioning Benchmark on " << num_gpus << " GPUs..." << std::endl;
+    auto start = std::chrono::steady_clock::now();
+
+    std::vector<std::thread> threads;
+    for (int i = 0; i < num_gpus; ++i) { // Pass context and kernel function to each worker
+        threads.emplace_back(gpu_worker, i, std::ref(partitions[i]), streams_per_gpu, 
+                            contexts[i], kernel_funcs[i], std::ref(h_256), std::ref(h_2048));
+    }
+
+    for (auto& t : threads) {
+        t.join();
+    }
+
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+
+    std::cout << "Static CUDA Complete." << std::endl;
+    std::cout << "Makespan: " << elapsed.count() << "s" << std::endl;
+
+    // Cleanup contexts and module
+    for (int i = 0; i < num_gpus; ++i) {
+        cuCtxDestroy(contexts[i]);
+    }
+    cuModuleUnload(module);
+
+    return 0;
+}

From e8005bff2cbac85a5afbc9755a941b82531f4f75 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 09:12:01 -0600
Subject: [PATCH 0594/1000] Fixed syntax errors.

---
 .../mmul-randonom-batch-benchmark/cuda-baseline.cpp   | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index 5f3f497856..f948ebd3cc 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -5,6 +5,7 @@
 #include <random>
 #include <thread>
 #include <algorithm>
+#include <functional>
 #include <stdexcept> // For runtime_error
 
 struct Task {
@@ -54,16 +55,16 @@ void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_g
         void *kernel_args[] = { &d_a, &d_b, &d_c, &N };
 
         // Kernel dimensions
-        dim3 threads(32, 32);
-        dim3 blocks((N + 31) / 32, (N + 31) / 32);
+        unsigned int block_dim = 32;
+        unsigned int grid_dim = (N + block_dim - 1) / block_dim;
 
-        cuLaunchKernel(kernel_func, blocks.x, blocks.y, blocks.z,
-                       threads.x, threads.y, threads.z,
+        cuLaunchKernel(kernel_func, grid_dim, grid_dim, 1,
+                       block_dim, block_dim, 1,
                        0, stream, kernel_args, nullptr);
         
         // Simulating the result retrieval (Copy back)
         std::vector<int> h_res(N * N);
-        cuMemcpyDtoHAsync(reinterpret_cast<CUdeviceptr>(h_res.data()), d_c, bytes, stream);
+        cuMemcpyDtoHAsync(h_res.data(), d_c, bytes, stream);
 
         // Free GPU memory for this task
         cuMemFreeAsync(d_a, stream);

From 1fc71e377609ed4bd323788d3bf199f9d86c74d5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 09:15:57 -0600
Subject: [PATCH 0595/1000] Fixed invalid context error.

---
 .../cuda-baseline.cpp                         | 49 ++++++++++++++++---
 1 file changed, 42 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index f948ebd3cc..88e3d1b073 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -122,12 +122,39 @@ int main() {
     std::vector<CUfunction> kernel_funcs(num_gpus);
     CUmodule module;
 
+    // Create a CUDA context for each device
+    for (int i = 0; i < num_gpus; ++i) {
+        CUdevice dev;
+        cuDeviceGet(&dev, i);
+        err = cuCtxCreate(&contexts[i], 0, dev); // Flag 0 for default context creation
+        if (err != CUDA_SUCCESS) {
+            const char* err_str;
+            cuGetErrorString(err, &err_str);
+            std::cerr << "Error creating context for device " << i << ": " << err_str << std::endl;
+            // Clean up already created contexts
+            for (int j = 0; j < i; ++j) cuCtxDestroy(contexts[j]);
+            return 1;
+        }
+    }
+
+    // Make the context for device 0 current on the main thread before loading the module
+    err = cuCtxPushCurrent(contexts[0]);
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error pushing context for device 0: " << err_str << std::endl;
+        for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
+        return 1;
+    }
+
     // Load the cubin module (assuming mmul.cu is compiled to mmul.cubin)
     err = cuModuleLoad(&module, "../mmul.cubin");
     if (err != CUDA_SUCCESS) {
         const char* err_str;
         cuGetErrorString(err, &err_str);
         std::cerr << "Error loading module ../mmul.cubin: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr); // Pop context on error
+        for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
         return 1;
     }
 
@@ -137,20 +164,28 @@ int main() {
         const char* err_str;
         cuGetErrorString(err, &err_str);
         std::cerr << "Error getting function matrixMul: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr); // Pop context on error
+        cuModuleUnload(module);
+        for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
         return 1;
     }
+
+    // Pop the context from the main thread
+    err = cuCtxPopCurrent(nullptr);
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error popping context from main thread: " << err_str << std::endl;
+        cuModuleUnload(module);
+        for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
+        return 1;
+    }
+
     // Assuming all GPUs can use the same function handle from the same module.
     for (int i = 1; i < num_gpus; ++i) {
         kernel_funcs[i] = kernel_funcs[0];
     }
 
-    // Create a CUDA context for each device
-    for (int i = 0; i < num_gpus; ++i) {
-        CUdevice dev;
-        cuDeviceGet(&dev, i);
-        cuCtxCreate(&contexts[i], 0, dev); // Flag 0 for default context creation
-    }
-
     // ─────────────────────────────────────────────────────────────────────────
     // Static Round-Robin Partitioning
     // ─────────────────────────────────────────────────────────────────────────

From 6d987e1ddce853e9dac17b1296cac58b80f81cb4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 11:56:58 -0600
Subject: [PATCH 0596/1000] added work stealing from multiply GPUs from a
 global queue to increase efficiency over static partitioning

---
 .../main.test.cpp                             | 144 +++++++++++++-----
 1 file changed, 102 insertions(+), 42 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 8bedda0f6f..271464aff7 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -10,6 +10,8 @@
 #include <numeric>
 #include <random>
 #include "caf/actor_registry.hpp"
+#include <deque>
+#include <unordered_map>
 #include <unordered_set>
 //#include <caf/atoms.hpp>
 
@@ -25,6 +27,7 @@ CAF_BEGIN_TYPE_ID_BLOCK(mmul_benchmark, caf::id_block::cuda::end)
     CAF_ADD_ATOM(mmul_benchmark, release_memory_atom)
     CAF_ADD_ATOM(mmul_benchmark, request_work_atom)
     CAF_ADD_ATOM(mmul_benchmark, worker_done_atom)
+    CAF_ADD_ATOM(mmul_benchmark, refill_buffer_atom)
 CAF_END_TYPE_ID_BLOCK(mmul_benchmark)
 
 // Command runners for GPU operations
@@ -63,59 +66,130 @@ MatrixPool create_matrix_pool_random(
     return pool;
 }
 
+// ---------------------------- GLOBAL TASK POOL ----------------------------
+// The central source of truth for work. Implements a pull-based model.
+struct task_pool_state {
+    std::vector<int> tasks;
+    size_t next_task_idx = 0;
+};
+
+caf::behavior global_task_pool(caf::stateful_actor<task_pool_state>* self, std::vector<int> tasks) {
+    self->state().tasks = std::move(tasks);
+    return {
+        [=](get_work_atom, size_t batch_size) -> result<std::vector<int>> {
+            auto& st = self->state();
+            if (st.next_task_idx >= st.tasks.size())
+                return sec::end_of_stream;
+            size_t count = std::min(batch_size, st.tasks.size() - st.next_task_idx);
+            std::vector<int> batch(st.tasks.begin() + st.next_task_idx, 
+                                   st.tasks.begin() + st.next_task_idx + count);
+            st.next_task_idx += count;
+            return batch;
+        }
+    };
+}
+
 // ---------------------------- DEVICE/GPU ACTOR ----------------------------
-// Holds the data partitions and dispatches tasks to workers.
+// Manages memory for a specific GPU and steals (pulls) work from the Global Pool.
 struct device_actor_state {
     MatrixPool pool;
-    std::vector<int> tasks;
-    size_t next_task_idx = 0;
+    caf::actor global_pool;
+    std::deque<int> local_tasks; // Local buffer to keep GPU busy
     size_t total_device_memory_bytes = 0;
     size_t current_allocated_memory_bytes = 0;
     int active_workers = 0;
     int device_id = -1;
+    size_t batch_size = 0;
+    size_t low_water_mark = 0;
+    bool fetching = false;
 };
 
 caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
-                               MatrixPool pool, std::vector<int> tasks, int num_workers, int dev_id) {
+                               MatrixPool pool, caf::actor global_pool, int num_workers, int dev_id, int max_in_flight) {
     self->state().pool = std::move(pool);
-    self->state().tasks = std::move(tasks);
+    self->state().global_pool = global_pool;
     self->state().device_id = dev_id;
     self->state().active_workers = num_workers;
 
+    // Dynamically calculate prefetch markers based on the total pipeline capacity
+    self->state().low_water_mark = static_cast<size_t>(num_workers * max_in_flight);
+    self->state().batch_size = self->state().low_water_mark * 2;
+
     caf::cuda::manager& mgr = caf::cuda::manager::get();
     caf::cuda::device_ptr dev_obj = mgr.find_device(dev_id);
     if (dev_obj) {
         self->state().total_device_memory_bytes = dev_obj->total_memory_bytes();
     }
 
+    // Helper to refill the local task buffer from the global pool
+    auto refill = [=]() {
+        auto& st = self->state();
+        if (st.fetching || st.local_tasks.size() >= st.low_water_mark + st.batch_size)
+            return;
+
+        st.fetching = true;
+        self->mail(get_work_atom_v, (size_t)st.batch_size).request(st.global_pool, infinite).then(
+            [=](std::vector<int>& batch) {
+                auto& st_inner = self->state();
+                for (int N : batch)
+                    st_inner.local_tasks.push_back(N);
+                st_inner.fetching = false;
+                if (st_inner.local_tasks.size() < st_inner.low_water_mark)
+                    self->mail(refill_buffer_atom_v).send(self);
+            },
+            [=](error& err) {
+                self->state().fetching = false;
+            }
+        );
+    };
+
     return {
-        [=](get_work_atom) -> result<int, in<int>, in<int>> {
+        [=](refill_buffer_atom) {
+            refill();
+        },
+        [=](get_work_atom) -> caf::result<int, in<int>, in<int>> {
             auto& st = self->state();
-            if (st.next_task_idx >= st.tasks.size())
-                return sec::end_of_stream; // Explicit signal: work is done
 
-            int N = st.tasks[st.next_task_idx]; // Peek, don't increment yet
-
-            // Calculate memory needed for A, B, C
-            size_t memory_needed = (size_t)N * N * sizeof(int) * 3; // A, B, C
-
-            // std::cout << "total memory bytes is " << st.total_device_memory_bytes << "\n";
-            // std::cout << "current allocated memory bytes is " << st.current_allocated_memory_bytes << "\n";
-            // std::cout << "memory needed is " << memory_needed << "\n";
+            // If we have tasks locally, satisfy the request immediately
+            if (!st.local_tasks.empty()) {
+                int N = st.local_tasks.front();
+                size_t memory_needed = (size_t)N * N * sizeof(int) * 3;
+                if (st.current_allocated_memory_bytes + memory_needed > st.total_device_memory_bytes)
+                    return make_error(sec::runtime_error, "Device Actor: Not enough memory");
+                
+                st.local_tasks.pop_front();
+                st.current_allocated_memory_bytes += memory_needed;
+                
+                // Proactively steal more work if the buffer is getting low
+                if (st.local_tasks.size() < st.low_water_mark)
+                    refill();
 
-            if (st.current_allocated_memory_bytes + memory_needed > st.total_device_memory_bytes) {
-                return make_error(sec::runtime_error, "Device Actor: Not enough memory");
+                return {N, caf::cuda::create_in_arg(st.pool.A[N]), 
+                           caf::cuda::create_in_arg(st.pool.B[N])};
             }
 
-            // If memory is available, reserve it and dispatch
-            st.current_allocated_memory_bytes += memory_needed;
-            N = st.tasks[st.next_task_idx++]; // Now actually take the task
-            return {N, caf::cuda::create_in_arg(st.pool.A[N]), caf::cuda::create_in_arg(st.pool.B[N])};
+            // Buffer empty: must fetch from global pool reactively
+            auto promise = self->make_response_promise<int, in<int>, in<int>>();
+            self->mail(get_work_atom_v, (size_t)st.batch_size).request(st.global_pool, infinite).then(
+                [=](std::vector<int>& batch) mutable {
+                    auto& st_inner = self->state();
+                    int N = batch.front();
+                    for(size_t i = 1; i < batch.size(); ++i) st_inner.local_tasks.push_back(batch[i]);
+                    
+                    size_t needed = (size_t)N * N * sizeof(int) * 3;
+                    st_inner.current_allocated_memory_bytes += needed;
+                    promise.deliver(N, caf::cuda::create_in_arg(st_inner.pool.A[N]), 
+                                       caf::cuda::create_in_arg(st_inner.pool.B[N]));
+                },
+                [=](error& err) mutable { promise.deliver(err); }
+            );
+            return promise;
         },
         [=](release_memory_atom, int N_completed) {
             auto& st = self->state();
             size_t memory_released = (size_t)N_completed * N_completed * sizeof(int) * 3;
             st.current_allocated_memory_bytes -= memory_released;
+            refill(); // Try to get more work now that memory is free
         },
         [=](worker_done_atom) {
             auto& st = self->state();
@@ -232,36 +306,22 @@ caf::behavior supervisor_actor_fun(
     int workers_per_gpu,
     int max_in_flight_tasks_per_worker,
     MatrixPool pool,
-    const std::vector<int>& Ns
+    std::vector<int> Ns
     ) {
     self->state().total_tasks = total_tasks;
     self->state().start_time = std::chrono::steady_clock::now();
 
+    auto pool_actor = self->spawn(global_task_pool, std::move(Ns));
+
     caf::cuda::manager& mgr = caf::cuda::manager::get();
     int num_gpus = mgr.get_num_devices();
     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
-    // Partition tasks across GPUs and spawn one Device Actor per GPU
-    size_t tasks_per_gpu_base = Ns.size() / num_gpus;
-    size_t remainder_tasks = Ns.size() % num_gpus;
-
     for (int i = 0; i < num_gpus; ++i) {
-        size_t current_gpu_tasks_count = tasks_per_gpu_base + (i < remainder_tasks ? 1 : 0);
-        auto start_it = Ns.begin();
-        std::advance(start_it, i * tasks_per_gpu_base + std::min((size_t)i, remainder_tasks));
+        auto broker = self->spawn(gpu_device_actor, pool, pool_actor, workers_per_gpu, i, max_in_flight_tasks_per_worker);
         
-        auto end_it = start_it;
-        std::advance(end_it, current_gpu_tasks_count);
-
-        std::vector<int> gpu_tasks(start_it, end_it);
-
-        auto dev_actor = self->spawn(gpu_device_actor, pool, std::move(gpu_tasks), workers_per_gpu, i);
-        
-        // Spawn workers for this GPU
-        for (int j = 0; j < workers_per_gpu; ++j) {
-            int stream_id = (i * 1000) + j; // Unique stream per worker
-            self->spawn(mmul_worker_fun, self, dev_actor, program, i, stream_id, max_in_flight_tasks_per_worker);
-        }
+        for (int j = 0; j < workers_per_gpu; ++j)
+            self->spawn(mmul_worker_fun, self, broker, program, i, (i * 1000) + j, max_in_flight_tasks_per_worker);
     }
 
     return {

From 9d4bb4708048b1bf7f14ae7b1b26402bda4c66b8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 12:20:38 -0600
Subject: [PATCH 0597/1000] moved copy to host logic in command runner into 
 copy_back_command object. This change was made to ensure better conform the
 design to the command pattern

---
 libcaf_cuda/caf/cuda/command_runner.hpp | 89 +++++-----------------
 libcaf_cuda/caf/cuda/memory_command.hpp | 99 ++++++++++++++++++++++++-
 2 files changed, 115 insertions(+), 73 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index 4ff43e77ff..d16c428e92 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -214,84 +214,29 @@ class command_runner {
     }
 
 
-    /**
-     * @brief Asynchronously copies GPU memory and provides an std::vector<T> to the callback.
-     * 
-     * The mem_ptr is kept alive until the callback completes.
-     */
+    // -------------------------------------------------------------------------
+    // COPY BACK
+    // -------------------------------------------------------------------------
+
+    // Synchronous copy back
+    template <typename T>
+    std::vector<T> copy_to_host(mem_ptr<T> ptr) {
+        copy_back_command<T> cmd(std::move(ptr));
+        return cmd.run();
+    }
+
+    // Asynchronous copy back (default)
     template <typename T, typename F>
     void copy_to_host_async(mem_ptr<T> ptr, F callback) {
-      if (ptr->access() == IN)
-        throw std::runtime_error("Cannot copy a read-only buffer back to host");
-
-      struct State {
-        std::vector<T> buffer;
-        F user_callback;
-        bool is_scalar;
-        T host_scalar;
-      };
-
-      auto* state = new State{std::vector<T>(ptr->size()), std::move(callback), ptr->is_scalar(), *ptr->host_scalar_ptr()};
-
-      CHECK_CUDA(cuCtxPushCurrent(ptr->get_ctx()));
-      CUstream s = ptr->stream();
-
-      if (!ptr->is_scalar()) {
-        size_t bytes = ptr->size() * sizeof(T);
-        CHECK_CUDA(cuMemcpyDtoHAsync(state->buffer.data(), ptr->mem(), bytes, s));
-      }
-
-      auto host_fn = [](void* userData) {
-        auto* s_ptr = static_cast<State*>(userData);
-        if (s_ptr->is_scalar)
-          s_ptr->buffer[0] = s_ptr->host_scalar;
-
-        s_ptr->user_callback(std::move(s_ptr->buffer));
-        
-        delete s_ptr;
-      };
-
-      CHECK_CUDA(cuLaunchHostFunc(s, host_fn, state));
-      CHECK_CUDA(cuCtxPopCurrent(nullptr));
+        auto cmd = caf::make_counted<copy_back_command<T>>(std::move(ptr));
+        cmd->run_async(std::move(callback));
     }
 
-    /**
-     * @brief Asynchronously copies GPU memory to a user-provided host buffer.
-     */
+    // Asynchronous copy back to user-provided buffer
     template <typename T, typename F>
     void copy_to_host_async(mem_ptr<T> ptr, T* dst, size_t count, F callback) {
-      if (ptr->access() == IN)
-        throw std::runtime_error("Cannot copy a read-only buffer back to host");
-
-      struct State {
-        T* dst;
-        size_t count;
-        F user_callback;
-        bool is_scalar;
-        T host_scalar;
-      };
-
-      auto* state = new State{dst, count, std::move(callback), ptr->is_scalar(), *ptr->host_scalar_ptr()};
-
-      CHECK_CUDA(cuCtxPushCurrent(ptr->get_ctx()));
-      CUstream s = ptr->stream();
-
-      if (!ptr->is_scalar()) {
-        size_t bytes = count * sizeof(T);
-        CHECK_CUDA(cuMemcpyDtoHAsync(dst, ptr->mem(), bytes, s));
-      }
-
-      auto host_fn = [](void* userData) {
-        auto* s_ptr = static_cast<State*>(userData);
-        if (s_ptr->is_scalar)
-          s_ptr->dst[0] = s_ptr->host_scalar;
-
-        s_ptr->user_callback(s_ptr->dst, s_ptr->count);
-        delete s_ptr;
-      };
-
-      CHECK_CUDA(cuLaunchHostFunc(s, host_fn, state));
-      CHECK_CUDA(cuCtxPopCurrent(nullptr));
+        auto cmd = caf::make_counted<copy_back_command<T>>(std::move(ptr));
+        cmd->run_async(dst, count, std::move(callback));
     }
 
 
diff --git a/libcaf_cuda/caf/cuda/memory_command.hpp b/libcaf_cuda/caf/cuda/memory_command.hpp
index 29b115b129..e793ab1c36 100644
--- a/libcaf_cuda/caf/cuda/memory_command.hpp
+++ b/libcaf_cuda/caf/cuda/memory_command.hpp
@@ -46,5 +46,102 @@ class memory_command : public caf::ref_counted {
   T arg_;
 };
 
-} // namespace caf::cuda
+// ===========================================================================
+// COPY BACK COMMAND
+// Handles transferring memory from device to host.
+// ===========================================================================
+template <typename T>
+class copy_back_command : public caf::ref_counted {
+public:
+  copy_back_command(mem_ptr<T> ptr) : ptr_(std::move(ptr)) {
+    if (!ptr_)
+      throw std::runtime_error("copy_back_command: null mem_ptr");
+  }
+
+  // Synchronous execution
+  std::vector<T> run() {
+    if (ptr_->access() == IN)
+      throw std::runtime_error("Cannot copy a read-only buffer back to host");
+    return ptr_->copy_to_host();
+  }
+
+  // Asynchronous execution with internal buffer allocation
+  template <typename F>
+  void run_async(F callback) {
+    if (ptr_->access() == IN)
+      throw std::runtime_error("Cannot copy a read-only buffer back to host");
+
+    struct State {
+      std::vector<T> buffer;
+      F user_callback;
+      bool is_scalar;
+      T host_scalar;
+    };
+
+    auto* state = new State{std::vector<T>(ptr_->size()), std::move(callback),
+                            ptr_->is_scalar(), *ptr_->host_scalar_ptr()};
+
+    CHECK_CUDA(cuCtxPushCurrent(ptr_->get_ctx()));
+    CUstream s = ptr_->stream();
+
+    if (!ptr_->is_scalar()) {
+      size_t bytes = ptr_->size() * sizeof(T);
+      CHECK_CUDA(cuMemcpyDtoHAsync(state->buffer.data(), ptr_->mem(), bytes, s));
+    }
+
+    auto host_fn = [](void* userData) {
+      auto* s_ptr = static_cast<State*>(userData);
+      if (s_ptr->is_scalar)
+        s_ptr->buffer[0] = s_ptr->host_scalar;
+
+      s_ptr->user_callback(std::move(s_ptr->buffer));
+      delete s_ptr;
+    };
 
+    CHECK_CUDA(cuLaunchHostFunc(s, host_fn, state));
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+  }
+
+  // Asynchronous execution with user-provided buffer
+  template <typename F>
+  void run_async(T* dst, size_t count, F callback) {
+    if (ptr_->access() == IN)
+      throw std::runtime_error("Cannot copy a read-only buffer back to host");
+
+    struct State {
+      T* dst;
+      size_t count;
+      F user_callback;
+      bool is_scalar;
+      T host_scalar;
+    };
+
+    auto* state = new State{dst, count, std::move(callback),
+                            ptr_->is_scalar(), *ptr_->host_scalar_ptr()};
+
+    CHECK_CUDA(cuCtxPushCurrent(ptr_->get_ctx()));
+    CUstream s = ptr_->stream();
+
+    if (!ptr_->is_scalar()) {
+      size_t bytes = count * sizeof(T);
+      CHECK_CUDA(cuMemcpyDtoHAsync(dst, ptr_->mem(), bytes, s));
+    }
+
+    auto host_fn = [](void* userData) {
+      auto* s_ptr = static_cast<State*>(userData);
+      if (s_ptr->is_scalar)
+        s_ptr->dst[0] = s_ptr->host_scalar;
+
+      s_ptr->user_callback(s_ptr->dst, s_ptr->count);
+      delete s_ptr;
+    };
+
+    CHECK_CUDA(cuLaunchHostFunc(s, host_fn, state));
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+  }
+
+private:
+  mem_ptr<T> ptr_;
+};
+
+} // namespace caf::cuda

From e44c507dc524f6f988257a4108643f47cec1ff7a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 12:30:49 -0600
Subject: [PATCH 0598/1000] addded synchronous memoyr host to device optiions,
 just in case someone wanted them

---
 libcaf_cuda/caf/cuda/command_runner.hpp | 26 +++++++++++++--
 libcaf_cuda/caf/cuda/memory_command.hpp | 43 +++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index d16c428e92..7b2040a183 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -186,6 +186,10 @@ class command_runner {
 
 
 
+    // -------------------------------------------------------------------------
+    // MEMORY TRANSFER
+    // Single transfer per command, returns device buffer
+    // -------------------------------------------------------------------------
     // -------------------------------------------------------------------------
     // MEMORY TRANSFER
     // Single transfer per command, returns device buffer
@@ -195,11 +199,21 @@ class command_runner {
                                       int stream_id,
                                       T arg)
     {
-        // stack-allocate memory_command and execute transfer
+        // default: asynchronous transfer
         memory_command<T> cmd(device_number, stream_id, std::move(arg));
         return cmd.enqueue();
     }
 
+    // Synchronous transfer for testing
+    template <typename T>
+    mem_ptr<raw_t<T>> transfer_memory_sync(int device_number,
+                                           int stream_id,
+                                           T arg)
+    {
+        memory_command<T> cmd(device_number, stream_id, std::move(arg));
+        return cmd.run_sync();
+    }
+
    // -------------------------------------------------------------------------
     // MEMORY TRANSFER
     // Single transfer per command, returns device buffer
@@ -209,8 +223,14 @@ class command_runner {
     mem_ptr<raw_t<T>> transfer_memory(const response_token_ptr& token,
                                       T arg)
     {
-        // stack-allocate memory_command and execute transfer
-        return transfer_memory(token -> getDeviceNumber(),token -> getStreamId(),arg); 
+        return transfer_memory(token->getDeviceNumber(), token->getStreamId(), std::move(arg));
+    }
+
+    template <typename T>
+    mem_ptr<raw_t<T>> transfer_memory_sync(const response_token_ptr& token,
+                                           T arg)
+    {
+        return transfer_memory_sync(token->getDeviceNumber(), token->getStreamId(), std::move(arg));
     }
 
 
diff --git a/libcaf_cuda/caf/cuda/memory_command.hpp b/libcaf_cuda/caf/cuda/memory_command.hpp
index e793ab1c36..828e1bbf1d 100644
--- a/libcaf_cuda/caf/cuda/memory_command.hpp
+++ b/libcaf_cuda/caf/cuda/memory_command.hpp
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <utility>
+#include <type_traits>
 
 #include <caf/ref_counted.hpp>
 #include <caf/intrusive_ptr.hpp>
@@ -40,6 +41,48 @@ class memory_command : public caf::ref_counted {
     return dev_->make_arg(arg_, stream);
   }
 
+  // -------------------------------------------------------------------------
+  // Execute memory transfer synchronously (blocking)
+  // -------------------------------------------------------------------------
+  result_type run_sync() {
+    CHECK_CUDA(cuCtxPushCurrent(dev_->getContext()));
+    CUstream stream = dev_->get_stream_for_actor(stream_id_);
+
+    int access = NOT_IN_USE;
+    if constexpr (std::is_same_v<T, in<raw_t<T>>>)
+      access = IN;
+    else if constexpr (std::is_same_v<T, out<raw_t<T>>>)
+      access = OUT;
+    else if constexpr (std::is_same_v<T, in_out<raw_t<T>>>)
+      access = IN_OUT;
+
+    if (arg_.is_scalar()) {
+      raw_t<T> val{};
+      if constexpr (!std::is_same_v<T, out<raw_t<T>>>) {
+        val = arg_.getscalar();
+      }
+      auto res = caf::make_counted<mem_ref<raw_t<T>>>(val, access, dev_->getId(), 0, dev_->getContext(), stream);
+      CHECK_CUDA(cuCtxPopCurrent(nullptr));
+      return res;
+    }
+
+    size_t size = arg_.size();
+    size_t bytes = size * sizeof(raw_t<T>);
+    CUdeviceptr mem;
+
+    // Synchronous allocation
+    CHECK_CUDA(cuMemAlloc(&mem, bytes));
+
+    if constexpr (!std::is_same_v<T, out<raw_t<T>>>) {
+      // Synchronous copy from host to device
+      CHECK_CUDA(cuMemcpyHtoD(mem, arg_.data(), bytes));
+    }
+
+    auto res = caf::make_counted<mem_ref<raw_t<T>>>(size, mem, access, dev_->getId(), 0, dev_->getContext(), stream);
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    return res;
+  }
+
 private:
   int stream_id_;
   device_ptr dev_;

From f15f4ad38f0f67b098f5251e79bad03a10c30a91 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 12:54:01 -0600
Subject: [PATCH 0599/1000] updated test to be in more align with asynchronous
 changes made to GPU actors

---
 .../baseline-comparison/actors/main.test.cpp  | 26 ++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
index c52d09612b..3c9c07186a 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
@@ -42,6 +42,8 @@ command mmul_command;
 
 struct mmul_state {
 	caf::cuda::program_ptr program;
+	int total_expected = 0;
+	int results_received = 0;
 };
 
 //global output buffer meant to disclude it from timing
@@ -53,13 +55,12 @@ std::vector<int> matrixC;
 
 
 caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
-		caf::cuda::program_ptr mmul_kernel) {
-  
+		caf::cuda::program_ptr mmul_kernel, int iterations) {
 
 	self ->state().program = mmul_kernel;
+	self ->state().total_expected = iterations;
 
 return {
-
     [=](const std::vector<int>& matrixA,
         const std::vector<int>& matrixB,
         int N) {
@@ -131,14 +132,16 @@ return {
 
 	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);
 
-
-	    dC->copy_to_host(matrixC.data(),N*N);
-
-            auto t_copy_end = clock::now();
-            auto t_total_end = clock::now();
-
+            auto self_hdl = caf::actor_cast<caf::actor>(self);
+            mmul_command.copy_to_host_async(dC, matrixC.data(), N*N, [self_hdl](int*, size_t) {
+                caf::anon_mail(kernel_done_atom_v).send(self_hdl);
+            });
+    },
+    [=](kernel_done_atom) {
+        if (++self->state().results_received == self->state().total_expected) {
+            self->quit();
+        }
     }
-
   };
 }
 
@@ -167,12 +170,11 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size,int iterations) {
 
   auto start = std::chrono::steady_clock::now();
 
-  caf::actor a =sys.spawn(mmul_actor_fun,program);
+  caf::actor a = sys.spawn(mmul_actor_fun, program, iterations);
 
   for (int i = 0; i < iterations; i++) 
 	  anon_mail(matrixA,matrixB,matrix_size).send(a);
 
-  anon_send_exit(a,caf::exit_reason::kill);
   // Wait for all actors to finish
   sys.await_all_actors_done();
 

From 8d8a84b4471417ed5d7007f85fb6ab55eddc511a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 12:56:46 -0600
Subject: [PATCH 0600/1000] made driver more asynchronous

---
 .../baseline-comparison/cuda/matrix_mul_driver.cpp   | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
index 30f5c2d1bb..629577c08c 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
@@ -86,9 +86,9 @@ int main() {
             // Allocate device memory each iteration
             // ----------------------------------
             CUdeviceptr d_a, d_b, d_c;
-            checkCU(cuMemAlloc(&d_a, elements * sizeof(int)), "cuMemAlloc d_a");
-            checkCU(cuMemAlloc(&d_b, elements * sizeof(int)), "cuMemAlloc d_b");
-            checkCU(cuMemAlloc(&d_c, elements * sizeof(int)), "cuMemAlloc d_c");
+            checkCU(cuMemAllocAsync(&d_a, elements * sizeof(int), stream), "cuMemAllocAsync d_a");
+            checkCU(cuMemAllocAsync(&d_b, elements * sizeof(int), stream), "cuMemAllocAsync d_b");
+            checkCU(cuMemAllocAsync(&d_c, elements * sizeof(int), stream), "cuMemAllocAsync d_c");
 
             // ----------------------------------
             // Copy persistent host buffers to device
@@ -109,9 +109,9 @@ int main() {
             // ----------------------------------
             // Free device memory
             // ----------------------------------
-            checkCU(cuMemFree(d_a), "cuMemFree d_a");
-            checkCU(cuMemFree(d_b), "cuMemFree d_b");
-            checkCU(cuMemFree(d_c), "cuMemFree d_c");
+            checkCU(cuMemFreeAsync(d_a, stream), "cuMemFreeAsync d_a");
+            checkCU(cuMemFreeAsync(d_b, stream), "cuMemFreeAsync d_b");
+            checkCU(cuMemFreeAsync(d_c, stream), "cuMemFreeAsync d_c");
         }
 
         // Synchronize stream after series

From 6e9841d235838a4420698dfc1573b849fec89c40 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 13:09:48 -0600
Subject: [PATCH 0601/1000] added a per async memory transfer

---
 libcaf_cuda/caf/cuda/command_runner.hpp         |  7 +++++++
 .../baseline-comparison/actors/main.test.cpp    | 17 ++++++++++-------
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index 7b2040a183..e350106adf 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -245,6 +245,13 @@ class command_runner {
         return cmd.run();
     }
 
+    // Enqueue copy back without any host-side synchronization or callback
+    template <typename T>
+    void copy_to_host_async(mem_ptr<T> ptr, T* dst, size_t count) {
+        copy_back_command<T> cmd(std::move(ptr));
+        cmd.enqueue(dst, count);
+    }
+
     // Asynchronous copy back (default)
     template <typename T, typename F>
     void copy_to_host_async(mem_ptr<T> ptr, F callback) {
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
index 3c9c07186a..4090dd7298 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
@@ -132,15 +132,18 @@ return {
 
 	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);
 
-            auto self_hdl = caf::actor_cast<caf::actor>(self);
-            mmul_command.copy_to_host_async(dC, matrixC.data(), N*N, [self_hdl](int*, size_t) {
-                caf::anon_mail(kernel_done_atom_v).send(self_hdl);
-            });
+            if (++self->state().results_received == self->state().total_expected) {
+                auto self_hdl = caf::actor_cast<caf::actor>(self);
+                mmul_command.copy_to_host_async(dC, matrixC.data(), N*N, [self_hdl](int*, size_t) {
+                    caf::anon_mail(kernel_done_atom_v).send(self_hdl);
+                });
+            } else {
+                // Fire-and-forget: No host callback, no synchronization
+                mmul_command.copy_to_host_async(dC, matrixC.data(), N*N);
+            }
     },
     [=](kernel_done_atom) {
-        if (++self->state().results_received == self->state().total_expected) {
-            self->quit();
-        }
+        self->quit();
     }
   };
 }

From 8b533a10cd650200e3511730170d25512a1dc57f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 13:09:59 -0600
Subject: [PATCH 0602/1000] added a per async memory transfer

---
 libcaf_cuda/caf/cuda/memory_command.hpp | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/memory_command.hpp b/libcaf_cuda/caf/cuda/memory_command.hpp
index 828e1bbf1d..264085e516 100644
--- a/libcaf_cuda/caf/cuda/memory_command.hpp
+++ b/libcaf_cuda/caf/cuda/memory_command.hpp
@@ -108,6 +108,28 @@ class copy_back_command : public caf::ref_counted {
     return ptr_->copy_to_host();
   }
 
+  // Enqueue the transfer on the stream without any host callback or synchronization
+  void enqueue(T* dst, size_t count) {
+    if (ptr_->access() == IN)
+      throw std::runtime_error("Cannot copy a read-only buffer back to host");
+
+    CHECK_CUDA(cuCtxPushCurrent(ptr_->get_ctx()));
+    CUstream s = ptr_->stream();
+
+    if (ptr_->is_scalar()) {
+      // For scalars, the value is already on the host.
+      // This assignment happens immediately on the CPU and is NOT stream-ordered.
+      // If stream ordering is required for scalars, use run_async with a callback.
+      dst[0] = *ptr_->host_scalar_ptr();
+    } else {
+      size_t bytes = count * sizeof(T);
+      // Pure asynchronous copy with no host-side tracking.
+      CHECK_CUDA(cuMemcpyDtoHAsync(dst, ptr_->mem(), bytes, s));
+    }
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+  }
+
   // Asynchronous execution with internal buffer allocation
   template <typename F>
   void run_async(F callback) {

From ec5243558d368d9015a15d46d782ab94f5f1f1d0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 11 May 2026 13:35:31 -0600
Subject: [PATCH 0603/1000] Made tests more asynchronous to be in line with
 current changes.

---
 .../cuda-program-benchmark/matrix_mul_driver.cpp     | 12 ++++++------
 .../mmul-actor-benchmarking/main.test.cpp            | 12 +++++-------
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
index 78c868d05d..df432935e3 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
@@ -55,9 +55,9 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     // ----------------------------------
     auto t_alloc_start = clock::now();
 
-    checkCU(cuMemAlloc(&d_a, bytes), "cuMemAlloc d_a");
-    checkCU(cuMemAlloc(&d_b, bytes), "cuMemAlloc d_b");
-    checkCU(cuMemAlloc(&d_c, bytes), "cuMemAlloc d_c");
+    checkCU(cuMemAllocAsync(&d_a, bytes, stream), "cuMemAllocAsync d_a");
+    checkCU(cuMemAllocAsync(&d_b, bytes, stream), "cuMemAllocAsync d_b");
+    checkCU(cuMemAllocAsync(&d_c, bytes, stream), "cuMemAllocAsync d_c");
 
     auto t_alloc_end = clock::now();
 
@@ -124,9 +124,9 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     // ----------------------------------
     auto t_free_start = clock::now();
 
-    checkCU(cuMemFree(d_a), "cuMemFree A");
-    checkCU(cuMemFree(d_b), "cuMemFree B");
-    checkCU(cuMemFree(d_c), "cuMemFree C");
+    checkCU(cuMemFreeAsync(d_a, stream), "cuMemFreeAsync A");
+    checkCU(cuMemFreeAsync(d_b, stream), "cuMemFreeAsync B");
+    checkCU(cuMemFreeAsync(d_c, stream), "cuMemFreeAsync C");
 
     auto t_free_end = clock::now();
 
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
index 114786c986..36e46f7a99 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
@@ -137,10 +137,9 @@ caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
             // copy to host
             // -------------------------
             auto t_copy_start = clock::now();
-
-            //std::vector<int> matrixC = dC->copy_to_host();
-
-	    dC->copy_to_host(matrixC.data(),N*N);
+            
+            mmul_command.copy_to_host_async(dC, matrixC.data(), N * N);
+            dC->synchronize();
 
             auto t_copy_end = clock::now();
             auto t_total_end = clock::now();
@@ -283,9 +282,8 @@ caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
 
 	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);
 
-            //std::vector<int> matrixC = dC->copy_to_host();
-
-	    dC->copy_to_host(matrixC.data(),N*N);
+            mmul_command.copy_to_host_async(dC, matrixC.data(), N * N);
+            dC->synchronize();
 
             auto t_copy_end = clock::now();
             auto t_total_end = clock::now();

From 81e10fa6587bdbcbe16a04cdc00eb296c341d6d5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 12 May 2026 09:01:39 -0600
Subject: [PATCH 0604/1000] Updated tests to use multiple task types.

---
 .../cuda-baseline.cpp                         | 17 ++++-----
 .../work-stealing.cpp                         | 36 ++++++++++---------
 2 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index 88e3d1b073..bc6a571328 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -93,30 +93,32 @@ int main() {
         return 1;
     }
 
+    const int streams_per_gpu = 8;
+    std::vector<int> task_counts = {30000, 40000, 50000};
+
     int num_gpus;
     cuDeviceGetCount(&num_gpus);
-    
     if (num_gpus == 0) {
         std::cerr << "No CUDA devices found." << std::endl;
         return 1;
     }
 
-    const int total_tasks = 30000;
-    const int streams_per_gpu = 8;
+    for (int total_tasks : task_counts) {
+    std::cout << "=====================================" << std::endl;
+    std::cout << "Task count: " << total_tasks << " (10% Heavy, 90% Light)" << std::endl;
 
     std::vector<Task> all_tasks;
     std::mt19937 rng(42);
     std::uniform_real_distribution<double> dist(0.0, 1.0);
 
-    std::cout << "Generating " << total_tasks << " tasks (10% Heavy, 90% Light)..." << std::endl;
     for (int i = 0; i < total_tasks; ++i) {
         if (dist(rng) < 0.1) all_tasks.push_back({2048});
         else all_tasks.push_back({256});
     }
 
     // Prepare Host-side MatrixPool (on CPU)
-    std::vector<int> h_256(256 * 256, 1);
-    std::vector<int> h_2048(2048 * 2048, 1);
+    static std::vector<int> h_256(256 * 256, 1);
+    static std::vector<int> h_2048(2048 * 2048, 1);
 
     std::vector<CUcontext> contexts(num_gpus);
     std::vector<CUfunction> kernel_funcs(num_gpus);
@@ -194,7 +196,6 @@ int main() {
         partitions[i % num_gpus].push_back(all_tasks[i]);
     }
 
-    std::cout << "Starting Static Partitioning Benchmark on " << num_gpus << " GPUs..." << std::endl;
     auto start = std::chrono::steady_clock::now();
 
     std::vector<std::thread> threads;
@@ -210,7 +211,6 @@ int main() {
     auto end = std::chrono::steady_clock::now();
     std::chrono::duration<double> elapsed = end - start;
 
-    std::cout << "Static CUDA Complete." << std::endl;
     std::cout << "Makespan: " << elapsed.count() << "s" << std::endl;
 
     // Cleanup contexts and module
@@ -218,6 +218,7 @@ int main() {
         cuCtxDestroy(contexts[i]);
     }
     cuModuleUnload(module);
+    }
 
     return 0;
 }
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index 23f8b09602..43bfcf8aab 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -264,8 +264,7 @@ caf::behavior supervisor_actor(caf::stateful_actor<supervisor_state>* self,
             if (self->state().done >= self->state().total) {
                 auto elapsed = std::chrono::steady_clock::now() - self->state().start;
                 std::cout << "Dynamic Work-Stealing Complete.\n"
-                          << "Makespan: " << std::chrono::duration<double>(elapsed).count() << "s\n";
-                caf::cuda::manager::shutdown();
+                          << "Task Count: " << self->state().total << " | Makespan: " << std::chrono::duration<double>(elapsed).count() << "s\n";
                 self->quit();
             }
         }
@@ -273,25 +272,30 @@ caf::behavior supervisor_actor(caf::stateful_actor<supervisor_state>* self,
 }
 
 void caf_main(caf::actor_system& sys) {
-    caf::cuda::manager_config cfg(false);
-    caf::cuda::manager::init(sys, cfg);
-
-    int total_tasks = 30000;
     int workers_per_gpu = 8;
     int max_in_flight = 3;
+    std::vector<int> task_counts = {30000, 40000, 50000};
 
-    std::vector<int> Ns;
-    std::mt19937 rng(42);
-    std::uniform_real_distribution<double> dist(0.0, 1.0); // Power Law setup
+    for (int total_tasks : task_counts) {
+        caf::cuda::manager_config cfg(false);
+        caf::cuda::manager::init(sys, cfg);
+        std::cout << "=====================================\n";
+        std::cout << "Task count: " << total_tasks << "\n";
 
-    // Power Law simulation: 10% Heavy (2048), 90% Light (256)
-    for (int i = 0; i < total_tasks; ++i) {
-        if (dist(rng) < 0.1) Ns.push_back(2048);
-        else Ns.push_back(256);
-    }
+        std::vector<int> Ns;
+        std::mt19937 rng(42);
+        std::uniform_real_distribution<double> dist(0.0, 1.0); // Power Law setup
 
-    sys.spawn(supervisor_actor, total_tasks, std::move(Ns), workers_per_gpu, max_in_flight);
-    sys.await_all_actors_done();
+        // Power Law simulation: 10% Heavy (2048), 90% Light (256)
+        for (int i = 0; i < total_tasks; ++i) {
+            if (dist(rng) < 0.1) Ns.push_back(2048);
+            else Ns.push_back(256);
+        }
+
+        sys.spawn(supervisor_actor, total_tasks, std::move(Ns), workers_per_gpu, max_in_flight);
+        sys.await_all_actors_done();
+        caf::cuda::manager::shutdown();
+    }
 }
 
 CAF_MAIN(id_block::dynamic_work_stealing)

From edc12f20b7a5a492b51f9573928e83120d3da5eb Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 12 May 2026 09:52:52 -0600
Subject: [PATCH 0605/1000] modified test to use only 1 thread

---
 .../mmul-actor-benchmarking/main.test.cpp     | 39 ++++++++++++++++---
 1 file changed, 33 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
index 36e46f7a99..b087e35733 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
@@ -377,14 +377,41 @@ auto t_end = clock::now();
 
 
 void caf_main(caf::actor_system& sys) {
- run_mmul_test(sys,1000);
-  run_mmul_test(sys,4000);
-  run_mmul_test(sys,8000);
-  run_mmul_test(sys,12000);
-
+    run_mmul_test(sys, 1000);
+    run_mmul_test(sys, 4000);
+    run_mmul_test(sys, 8000);
+    run_mmul_test(sys, 12000);
 }
 
+int main(int argc, char** argv) {
+  // Initialize user defined types and messages if needed.
+  //init_global_meta_objects<custom_types_1>();
+  
+  // Initialize the global type information.
+  core::init_global_meta_objects();
+
+  // Create the config.
+  actor_system_config cfg;
 
+  // --- SINGLE THREAD CONFIGURATION ---
+  cfg.set("caf.scheduler.max-threads", 1);
+  cfg.set("caf.scheduler.policy", "sharing"); 
+  // ------------------------------------
 
+  // Read CLI options. (Note: CLI flags like --caf.scheduler.max-threads=4 
+  // will override the hardcoded '1' above if provided by the user).
+  auto err = cfg.parse(argc, argv);
+  if (err)
+    return EXIT_FAILURE;
 
-CAF_MAIN()
+  if (cfg.helptext_printed())
+    return 0;
+
+  // Create the actor system (the scheduler starts here).
+  actor_system sys{cfg};
+
+  // Run user-defined code.
+  caf_main(sys);
+  
+  return 0;
+}
\ No newline at end of file

From de32cb75aaeb7eeaacec5800aacfecc7bc02917e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 12 May 2026 09:56:28 -0600
Subject: [PATCH 0606/1000] made test single core

---
 .../cuda-program-benchmark/matrix_mul_driver.cpp              | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
index df432935e3..c27370d98b 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
@@ -175,9 +175,9 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
 }
 
 int main(int argc, char** argv) {
-    std::vector<int> sizes = {1000, 4000, 8000, 12000};
+  std::vector<int> sizes = {1000, 4000, 8000, 12000};
     
-   //std::vector<int> sizes = {8000};	
+//   std::vector<int> sizes = {12000};	
    if (argc > 1) {
         sizes.clear();
         for (int i = 1; i < argc; ++i) sizes.push_back(std::stoi(argv[i]));

From d1bd324006afd1b625fa414c60d23cb0e5ede5e1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 12 May 2026 10:14:41 -0600
Subject: [PATCH 0607/1000] made single thread

---
 .../baseline-comparison/actors/main.test.cpp  | 33 ++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
index 4090dd7298..856213023c 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
@@ -205,6 +205,37 @@ void caf_main(caf::actor_system& sys) {
 }
 
 
+int main(int argc, char** argv) {
+  // Initialize user defined types and messages if needed.
+  //init_global_meta_objects<custom_types_1>();
+  
+  // Initialize the global type information.
+  core::init_global_meta_objects();
+
+  // Create the config.
+  actor_system_config cfg;
+
+  // --- SINGLE THREAD CONFIGURATION ---
+  cfg.set("caf.scheduler.max-threads", 1);
+  cfg.set("caf.scheduler.policy", "sharing"); 
+  // ------------------------------------
+
+  // Read CLI options. (Note: CLI flags like --caf.scheduler.max-threads=4 
+  // will override the hardcoded '1' above if provided by the user).
+  auto err = cfg.parse(argc, argv);
+  if (err)
+    return EXIT_FAILURE;
 
+  if (cfg.helptext_printed())
+    return 0;
+
+  // Create the actor system (the scheduler starts here).
+  actor_system sys{cfg};
+
+  // Run user-defined code.
+  caf_main(sys);
+  
+  return 0;
+}
 
-CAF_MAIN()
+// CAF_MAIN()

From fa3c3aef34c7839e71faf54302bab3739b725bde Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 12 May 2026 10:19:20 -0600
Subject: [PATCH 0608/1000] fixed random device selection issue

---
 .../baseline-comparison/actors/main.test.cpp                    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
index 856213023c..79d34ad42a 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
@@ -127,7 +127,7 @@ return {
       auto output = command.run_async(
 		      self->state().program,
 		      dims,
-		      1,
+		      1, 0, device,
 		      arg1,arg2,out<int>{N*N},in<int>{N});
 
 	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);

From 4538befc77d1fbf4a971313236e747c09b8ecdb5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 12 May 2026 10:21:04 -0600
Subject: [PATCH 0609/1000] fixed random device issue

---
 .../tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
index b087e35733..9ecd2257ce 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
@@ -269,7 +269,7 @@ caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
       caf::cuda::mmul_async_command<int> command;
       auto output = command.run_async(
 		      program,dims,
-		      1,
+		      1, 0, device,
 		      arg1,arg2,out<int>{N*N},in<int>{N});
 
        auto t_response_received = clock::now();

From 34f57cf6c0f41a60c45ab383aaed5e09beb1d4fe Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 12 May 2026 10:50:07 -0600
Subject: [PATCH 0610/1000] Initial commit.

---
 .../workflow-tests/mcts/CMakeLists.txt        |  52 +++++
 .../workflow-tests/mcts/main.cpp              | 179 ++++++++++++++++++
 .../workflow-tests/mcts/simulation_kernel.cu  |  73 +++++++
 3 files changed, 304 insertions(+)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/main.cpp
 create mode 100644 libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/simulation_kernel.cu

diff --git a/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/CMakeLists.txt
new file mode 100644
index 0000000000..cda77250ef
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/CMakeLists.txt
@@ -0,0 +1,52 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executables
+add_executable(test main.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+target_link_libraries(work-stealing
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
diff --git a/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/main.cpp b/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/main.cpp
new file mode 100644
index 0000000000..fe75205a8b
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/main.cpp
@@ -0,0 +1,179 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include <cmath>
+#include <algorithm>
+#include <memory>
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+// Atoms for MCTS operations
+using search_atom = atom_constant<uint32_t, 1>;
+using simulate_atom = atom_constant<uint32_t, 2>;
+using update_atom = atom_constant<uint32_t, 3>;
+using expand_atom = atom_constant<uint32_t, 4>;
+
+// Mock Game State
+struct game_state {
+    int board[9]; // e.g., Tic-Tac-Toe
+    int turn;     // 1 or 2
+
+    template <class Inspector>
+    friend bool inspect(Inspector& f, game_state& x) {
+        return f.object(x).fields(f.field("board", x.board), f.field("turn", x.turn));
+    }
+};
+
+// Command runner for the GPU Simulation kernel
+// The kernel takes the current game state and returns a win probability (float)
+using simulation_command = caf::cuda::command_runner<
+    in<game_state>, // Input game state
+    out<float>,     // Output score
+    in<int>         // Seed offset
+>;
+
+struct mcts_node_state {
+    game_state state;
+    caf::actor parent;
+    std::vector<caf::actor> children;
+    
+    int visits = 0;
+    float total_value = 0.0f;
+    bool is_expanded = false;
+
+    caf::cuda::program_ptr sim_program;
+
+    float ucb1(int parent_visits) const {
+        if (visits == 0) return 1e9f; // Priority for unvisited nodes
+        return (total_value / visits) + 1.41f * std::sqrt(std::log(parent_visits) / visits);
+    }
+};
+
+// The MCTS Node Actor
+caf::behavior mcts_node_fun(caf::stateful_actor<mcts_node_state>* self, 
+                            caf::actor parent, 
+                            game_state state,
+                            caf::cuda::program_ptr sim_prog) {
+    self->state().parent = parent;
+    self->state().state = state;
+    self->state().sim_program = sim_prog;
+
+    return {
+        // Traversal / Selection Phase
+        [=](search_atom) {
+            if (!self->state().is_expanded) {
+                // Leaf node reached: Trigger Simulation on GPU
+                self->mail(simulate_atom_v).send(self);
+            } else if (self->state().children.empty()) {
+                // Terminal node or no moves possible
+                self->mail(update_atom_v, 0.5f).send(self);
+            } else {
+                // Select best child using UCB1
+                auto it = std::max_element(self->state().children.begin(), 
+                                         self->state().children.end(),
+                                         [=](const caf::actor& a, const caf::actor& b) {
+                                             // Note: In a real app, you'd request stats from children 
+                                             // or cache them in the parent state for speed.
+                                             return true; 
+                                         });
+                self->mail(search_atom_v).send(*it);
+            }
+        },
+
+        // Simulation Phase: Launching CUDA Kernel
+        [=](simulate_atom) {
+            simulation_command cmd;
+            int device = 0;
+            int actor_id = static_cast<int>(self->id());
+
+            // Prepare GPU arguments
+            auto in_state = caf::cuda::create_in_arg(self->state().state);
+            auto out_score = caf::cuda::create_out_arg_with_size<float>(1);
+            auto seed_arg = caf::cuda::create_in_arg(actor_id);
+
+            // Configure Kernel Dims (1 block, 1 thread for a single simulation rollout)
+            // In a real scenario, you'd run many rollouts in parallel on the GPU.
+            caf::cuda::nd_range dims(1, 1, 1, 1, 1, 1);
+
+            auto results = cmd.run(self->state().sim_program, dims, actor_id, in_state, out_score, seed_arg);
+            float win_rate = caf::cuda::extract_vector<float>(results)[0];
+
+            // After GPU returns result, trigger Expansion and Backpropagation
+            self->mail(expand_atom_v).send(self);
+            self->mail(update_atom_v, win_rate).send(self);
+        },
+
+        // Expansion Phase: Spawning child actors for new moves
+        [=](expand_atom) {
+            if (self->state().is_expanded) return;
+
+            // Mock expansion: spawn 3 child actors representing possible moves
+            for (int i = 0; i < 3; ++i) {
+                game_state next_state = self->state().state;
+                next_state.turn = (next_state.turn == 1) ? 2 : 1;
+                
+                auto child = self->spawn(mcts_node_fun, 
+                                       caf::actor_cast<caf::actor>(self), 
+                                       next_state, 
+                                       self->state().sim_program);
+                self->state().children.push_back(child);
+            }
+            self->state().is_expanded = true;
+        },
+
+        // Backpropagation Phase
+        [=](update_atom, float result) {
+            self->state().visits++;
+            self->state().total_value += result;
+
+            if (self->state().parent) {
+                // Pass result up the tree
+                self->mail(update_atom_v, result).send(self->state().parent);
+            } else {
+                std::cout << "[Root] Search iteration complete. Root visits: " 
+                          << self->state().visits << std::endl;
+            }
+        }
+    };
+}
+
+void run_mcts_demo(caf::actor_system& sys) {
+    caf::cuda::manager::init(sys);
+    auto& mgr = caf::cuda::manager::get();
+
+    // Load the simulation kernel
+    // This kernel would perform random rollouts from the given state
+    auto program = mgr.create_program_from_cubin("simulation_kernel.cubin", "evaluate_state");
+
+    game_state initial_state;
+    initial_state.turn = 1;
+    for(int& i : initial_state.board) i = 0;
+
+    // Spawn the Root actor
+    auto root = sys.spawn(mcts_node_fun, nullptr, initial_state, program);
+
+    std::cout << "Starting MCTS iterations..." << std::endl;
+
+    // Run 100 search iterations
+    for (int i = 0; i < 100; ++i) {
+        caf::anon_mail(search_atom_v).send(root);
+    }
+
+    // Wait for some results
+    std::this_thread::sleep_for(2s);
+
+    // Demonstration of pruning: kill the root to stop the whole tree
+    caf::anon_mail(exit_msg{root, exit_reason::user_shutdown}).send(root);
+    
+    sys.await_all_actors_done();
+    caf::cuda::manager::shutdown();
+    std::cout << "MCTS Demo finished." << std::endl;
+}
+
+void caf_main(caf::actor_system& sys) {
+    run_mcts_demo(sys);
+}
+
+CAF_MAIN(caf::cuda::id_block)
diff --git a/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/simulation_kernel.cu b/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/simulation_kernel.cu
new file mode 100644
index 0000000000..1130865f4f
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/simulation_kernel.cu
@@ -0,0 +1,73 @@
+#include <cuda_runtime.h>
+#include <curand_kernel.h>
+
+// Must match the C++ definition exactly
+struct game_state {
+    int board[9]; 
+    int turn;     
+};
+
+__device__ int check_winner(const int* board) {
+    // Rows
+    for (int i = 0; i < 9; i += 3)
+        if (board[i] != 0 && board[i] == board[i+1] && board[i] == board[i+2]) return board[i];
+    // Cols
+    for (int i = 0; i < 3; ++i)
+        if (board[i] != 0 && board[i] == board[i+3] && board[i] == board[i+6]) return board[i];
+    // Diagonals
+    if (board[0] != 0 && board[0] == board[4] && board[8]) return board[0];
+    if (board[2] != 0 && board[2] == board[4] && board[6]) return board[2];
+
+    // Check for draw
+    bool full = true;
+    for (int i = 0; i < 9; ++i) if (board[i] == 0) full = false;
+    if (full) return 3; // 3 represents Draw
+
+    return 0; // Ongoing
+}
+
+__device__ float perform_rollout(game_state state, curandState* local_state) {
+    int current_turn = state.turn;
+    int winner = 0;
+
+    // Play randomly until terminal state
+    for (int move = 0; move < 9; ++move) {
+        winner = check_winner(state.board);
+        if (winner != 0) break;
+
+        // Find available moves
+        int available[9];
+        int count = 0;
+        for (int i = 0; i < 9; ++i) {
+            if (state.board[i] == 0) available[count++] = i;
+        }
+
+        if (count == 0) break;
+
+        // Pick a random move
+        int pick = curand(local_state) % count;
+        state.board[available[pick]] = current_turn;
+        current_turn = (current_turn == 1) ? 2 : 1;
+    }
+
+    if (winner == 3) return 0.5f; // Draw
+    if (winner == 1) return 1.0f; // Player 1 wins
+    return 0.0f;                 // Player 2 wins (or loss for P1)
+}
+
+extern "C" {
+
+__global__ void evaluate_state(const game_state* initial_state, float* score_out, int seed_offset) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    
+    // Initialize random number generator
+    curandState local_state;
+    curand_init(1234ULL, idx + seed_offset, 0, &local_state);
+
+    // In this simple example, one thread does one rollout.
+    // In a high-perf MCTS, one thread block might cooperate 
+    // to do hundreds of rollouts for the same state.
+    score_out[idx] = perform_rollout(*initial_state, &local_state);
+}
+
+} // extern "C"

From f6167efd60a39bdcde81106d4b1bc727fede86bd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 12 May 2026 10:56:55 -0600
Subject: [PATCH 0611/1000] Fixed syntax and logic errors.

---
 .../workflow-tests/mcts/main.cpp              | 39 +++++++++++--------
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/main.cpp b/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/main.cpp
index fe75205a8b..80699a5865 100644
--- a/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/main.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/main.cpp
@@ -29,15 +29,16 @@ struct game_state {
 // Command runner for the GPU Simulation kernel
 // The kernel takes the current game state and returns a win probability (float)
 using simulation_command = caf::cuda::command_runner<
-    in<game_state>, // Input game state
-    out<float>,     // Output score
-    in<int>         // Seed offset
+    caf::cuda::in<game_state>, // Input game state
+    caf::cuda::out<float>,     // Output score
+    caf::cuda::in<int>         // Seed offset
 >;
 
 struct mcts_node_state {
     game_state state;
     caf::actor parent;
     std::vector<caf::actor> children;
+    int stream_id = -1;
     
     int visits = 0;
     float total_value = 0.0f;
@@ -55,10 +56,12 @@ struct mcts_node_state {
 caf::behavior mcts_node_fun(caf::stateful_actor<mcts_node_state>* self, 
                             caf::actor parent, 
                             game_state state,
-                            caf::cuda::program_ptr sim_prog) {
+                            caf::cuda::program_ptr sim_prog,
+                            int stream_id) {
     self->state().parent = parent;
     self->state().state = state;
     self->state().sim_program = sim_prog;
+    self->state().stream_id = (stream_id == -1) ? static_cast<int>(self->id()) : stream_id;
 
     return {
         // Traversal / Selection Phase
@@ -86,23 +89,26 @@ caf::behavior mcts_node_fun(caf::stateful_actor<mcts_node_state>* self,
         [=](simulate_atom) {
             simulation_command cmd;
             int device = 0;
-            int actor_id = static_cast<int>(self->id());
+            int stream_id = self->state().stream_id;
+            int node_id = static_cast<int>(self->id()); // Unique ID for RNG seed
 
             // Prepare GPU arguments
             auto in_state = caf::cuda::create_in_arg(self->state().state);
             auto out_score = caf::cuda::create_out_arg_with_size<float>(1);
-            auto seed_arg = caf::cuda::create_in_arg(actor_id);
+            auto seed_arg = caf::cuda::create_in_arg(node_id);
 
             // Configure Kernel Dims (1 block, 1 thread for a single simulation rollout)
             // In a real scenario, you'd run many rollouts in parallel on the GPU.
             caf::cuda::nd_range dims(1, 1, 1, 1, 1, 1);
 
-            auto results = cmd.run(self->state().sim_program, dims, actor_id, in_state, out_score, seed_arg);
-            float win_rate = caf::cuda::extract_vector<float>(results)[0];
-
-            // After GPU returns result, trigger Expansion and Backpropagation
-            self->mail(expand_atom_v).send(self);
-            self->mail(update_atom_v, win_rate).send(self);
+            // Use run_async to avoid blocking the actor thread
+            auto results = cmd.run_async(self->state().sim_program, dims, stream_id, in_state, out_score, seed_arg);
+            auto score_ptr = std::get<1>(results);
+            auto self_hdl = caf::actor_cast<caf::actor>(self);
+            cmd.copy_to_host_async(score_ptr, [self_hdl](std::vector<float> win_rates) {
+                caf::anon_mail(expand_atom_v).send(self_hdl);
+                caf::anon_mail(update_atom_v, win_rates[0]).send(self_hdl);
+            });
         },
 
         // Expansion Phase: Spawning child actors for new moves
@@ -116,8 +122,9 @@ caf::behavior mcts_node_fun(caf::stateful_actor<mcts_node_state>* self,
                 
                 auto child = self->spawn(mcts_node_fun, 
                                        caf::actor_cast<caf::actor>(self), 
-                                       next_state, 
-                                       self->state().sim_program);
+                                       next_state,
+                                       self->state().sim_program,
+                                       self->state().stream_id);
                 self->state().children.push_back(child);
             }
             self->state().is_expanded = true;
@@ -152,7 +159,7 @@ void run_mcts_demo(caf::actor_system& sys) {
     for(int& i : initial_state.board) i = 0;
 
     // Spawn the Root actor
-    auto root = sys.spawn(mcts_node_fun, nullptr, initial_state, program);
+    auto root = sys.spawn(mcts_node_fun, nullptr, initial_state, program, -1);
 
     std::cout << "Starting MCTS iterations..." << std::endl;
 
@@ -176,4 +183,4 @@ void caf_main(caf::actor_system& sys) {
     run_mcts_demo(sys);
 }
 
-CAF_MAIN(caf::cuda::id_block)
+CAF_MAIN(caf::cuda::id_block)
\ No newline at end of file

From 959cdc8d2faf9ae0dc349d1fc21a22627996e7ff Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 13 May 2026 12:42:06 -0600
Subject: [PATCH 0612/1000] added a add callback method to insert callback
 functions on streams

---
 libcaf_cuda/caf/cuda/command_runner.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index e350106adf..d919a22e48 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -2,6 +2,7 @@
 
 #include "caf/cuda/command.hpp"
 #include "caf/cuda/memory_command.hpp"
+#include <cuda.h>
 #include <thread>
 #include "caf/cuda/program.hpp"
 #include "caf/cuda/nd_range.hpp"

From 18d799b037b2643ceda250beee9f8c885e90ded9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 13 May 2026 13:02:27 -0600
Subject: [PATCH 0613/1000] added add callback on command runner

---
 libcaf_cuda/caf/cuda/command_runner.hpp | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index d919a22e48..d25599334d 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -276,6 +276,22 @@ class command_runner {
       plat->release_streams_for_actor(actor_id);
   }
 
+  // -------------------------------
+  // Register a callback on the actor's stream
+  // -------------------------------
+  template <typename F>
+  void add_callback(int stream_id, int device_number, F callback) {
+      auto plat = platform::create();
+      auto dev = plat->schedule(stream_id, device_number);
+      auto stream = dev->get_stream_for_actor(stream_id);
+      auto* f_ptr = new F(std::move(callback));
+      auto res = cuLaunchHostFunc(stream, [](void* data) {
+          auto* f = static_cast<F*>(data);
+          (*f)();
+          delete f;
+      }, f_ptr);
+      if (res != CUDA_SUCCESS) { delete f_ptr; check(res, "cuLaunchHostFunc"); }
+  }
 
 
 

From 859528db91409b0d9d5799affa8942c73f734f05 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 13 May 2026 13:06:35 -0600
Subject: [PATCH 0614/1000] added more atoms for actor facade

---
 libcaf_cuda/caf/cuda/global.hpp | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index 28c056502a..9b32b4a77d 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -185,9 +185,12 @@ CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id)
   
   //atoms 
   CAF_ADD_ATOM(cuda, kernel_done_atom)
-  //CAF_ADD_ATOM(cuda, become)
-  //CAF_ADD_ATOM(cuda, launch_behavior)
-  //CAF_ADD_ATOM(cuda, update_behavior)
+  CAF_ADD_ATOM(cuda, htod_atom)
+  CAF_ADD_ATOM(cuda, kernel_atom)
+  CAF_ADD_ATOM(cuda, dtoh_atom)
+  CAF_ADD_ATOM(cuda, htod_done_atom)
+  CAF_ADD_ATOM(cuda, dtoh_done_atom)
+  CAF_ADD_ATOM(cuda, gpu_done_atom)
 
 CAF_END_TYPE_ID_BLOCK(cuda)
 

From 94599bb304a0b4ccbf5ebffba8a3706b41c4990d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 13 May 2026 13:35:03 -0600
Subject: [PATCH 0615/1000] saving updates made to the actor facade

---
 libcaf_cuda/caf/cuda/actor_facade.hpp | 337 ++++++--------------------
 1 file changed, 78 insertions(+), 259 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/actor_facade.hpp b/libcaf_cuda/caf/cuda/actor_facade.hpp
index 620c596fa9..625a1054f3 100644
--- a/libcaf_cuda/caf/cuda/actor_facade.hpp
+++ b/libcaf_cuda/caf/cuda/actor_facade.hpp
@@ -1,283 +1,102 @@
 #pragma once
 
-#include <stdexcept>
 #include <functional>
-#include <tuple>
 #include <queue>
-#include <utility>
-#include <type_traits>
 
-#include <caf/local_actor.hpp>
 #include <caf/actor.hpp>
+#include <caf/actor_cast.hpp>
+#include <caf/actor_config.hpp>
+#include <caf/actor_system.hpp>
+#include <caf/anon_mail.hpp>
+#include <caf/event_based_actor.hpp>
 #include <caf/response_promise.hpp>
-#include <caf/scheduler.hpp>
-#include <caf/resumable.hpp>
 
-#include "caf/cuda/nd_range.hpp"
-#include "caf/cuda/global.hpp"
-#include "caf/cuda/program.hpp"
+#include <cuda.h>
+
 #include "caf/cuda/command.hpp"
+#include "caf/cuda/global.hpp"
+#include "caf/cuda/helpers.hpp"
 #include "caf/cuda/platform.hpp"
-#include <random>
-#include <climits>
-#include <thread>
+#include "caf/cuda/program.hpp"
+
 
 namespace caf::cuda {
 
-//An actor that acts as a gateway to the gpu 
-//you can send it messages that is of the parameters of the kernel
-//you wish to launch
-//and it will reply with an output_buffer
+// ---------------------------------------------------------------------------
+// actor_facade: Stateless wrapper for GPU kernels. Launches async work via 
+// command_runner and notifies the requester directly from CUDA callbacks.
+// ---------------------------------------------------------------------------
 template <bool PassConfig, class... Ts>
-class actor_facade : public caf::local_actor, public caf::resumable {
+class actor_facade : public event_based_actor {
 public:
-
-  //Factory methods to create the actor
-  static caf::actor create(
-    caf::actor_system& sys,
-    caf::actor_config&& actor_conf,
-    program_ptr program,
-    nd_range dims,
-    Ts&&... xs
-  ) {
-    return caf::make_actor<actor_facade<PassConfig, std::decay_t<Ts>...>, caf::actor>(
-      sys.next_actor_id(),
-      sys.node(),
-      &sys,
-      std::move(actor_conf),
-      std::move(program),
-      std::move(dims),
-      std::forward<Ts>(xs)...);
-  }
-
-  static caf::actor create(
-    caf::actor_system* sys,
-    caf::actor_config&& actor_conf,
-    program_ptr program,
-    nd_range dims,
-    Ts&&... xs
-  ) {
-    return caf::make_actor<actor_facade<PassConfig, std::decay_t<Ts>...>, caf::actor>(
-      sys->next_actor_id(),
-      sys->node(),
-      sys,
-      std::move(actor_conf),
-      std::move(program),
-      std::move(dims),
-      std::forward<Ts>(xs)...);
-  }
-
-  //constructor
-  actor_facade(caf::actor_config&& cfg, program_ptr prog, nd_range nd, Ts&&... xs)
-    : local_actor(cfg),
-      config_(std::move(cfg)),
-      program_(std::move(prog)),
-      dims_(nd) {
-  }
-
-  //deconstructor
-  ~actor_facade() {
-    auto plat = platform::create();
-    plat->release_streams_for_actor(actor_id);
-  }
-
-  //creates a command and enqueues the kernel to be launched
-  void create_command(program_ptr program, Ts&&... xs) {
-    using command_t = command<caf::actor, Ts...>;
-    auto rp = make_response_promise();
-    auto cmd = make_counted<command_t>(
-      program,
-      dims_,
-      actor_id,
-      std::forward<Ts>(xs)...);
-    rp.deliver(cmd->enqueue());
-    //anon_mail(kernel_done_atom_v).send(caf::actor_cast<caf::actor>(this));
-  }
-
-  //does the same thing as create_command
-  void run_kernel(Ts&... xs) {
-    create_command(program_, std::forward<Ts>(xs)...);
-  }
-
-private:
-  caf::actor_config config_;
-  program_ptr program_;
-  nd_range dims_;
-  std::queue<mailbox_element_ptr> mailbox_;
-  std::atomic<int> pending_promises_ = 0;
-  std::atomic<bool> shutdown_requested_ = false;
-  int actor_id = generate_id();
-  std::atomic_flag resuming_flag_ = ATOMIC_FLAG_INIT;
-  caf::actor self_ =   caf::actor_cast<caf::actor>(this);
-
-  //creates an id for the actor facade, used for stream allocation and 
-  //deallocation
-  int generate_id() {
-      return random_number();	  
-  }
-
-  //helper method that is used to handle an incoming message
-  bool handle_message(const message& msg) {
-    if (!msg.types().empty() && msg.types()[0] == caf::type_id_v<caf::actor>) {
-      auto sender = msg.get_as<caf::actor>(0);
-      if (msg.match_elements<caf::actor, Ts...>()) {
-        return unpack_and_run_wrapped(sender, msg, std::index_sequence_for<Ts...>{});
-      }
-      if (msg.match_elements<caf::actor, raw_t<Ts>...>()) {
-        return unpack_and_run(sender, msg, std::index_sequence_for<Ts...>{});
+  using mem_tuple = std::tuple<mem_ptr<raw_t<Ts>>...>;
+
+  static caf::actor create(caf::actor_system& sys,
+                           program_ptr program,
+                           nd_range dims,
+                           atom_value reply_atom = 0) {
+    return caf::actor_cast<caf::actor>(
+      sys.spawn<actor_facade>(std::move(program), std::move(dims), reply_atom));
+  }
+
+  actor_facade(caf::actor_config& cfg, program_ptr program, nd_range dims, atom_value reply_atom)
+    : caf::event_based_actor(cfg),
+      program_(std::move(program)),
+      dims_(std::move(dims)),
+      reply_atom_(reply_atom) {
+    actor_id_ = this->id();
+  }
+
+  ~actor_facade() override {
+      command_runner<Ts...> runner;
+      runner.release_stream_for_actor(actor_id_);
+  }
+
+  caf::behavior make_behavior() override {
+    return {
+      [this](atom_value stage, int device_num, Ts... args) {
+        enqueue_impl(device_num, stage, std::move(args)...);
+      },
+      [this](atom_value stage, Ts... args) {
+        enqueue_impl(-1, stage, std::move(args)...);
+      },
+      [this](Ts... args) {
+        enqueue_impl(-1, kernel_atom_v, std::move(args)...);
       }
-    }
-
-    if (!msg.types().empty()) { 
-	    return unpack_and_run_wrapped_async(msg, std::index_sequence_for<Ts...>{});
-    }
-    std::cout << "[WARNING], message format not recognized by actor facade, dropping message\n";
-    
-     return false;
-  }
-
-  //unpacks a message and launches the kernel
-  template <std::size_t... Is>
-  bool unpack_and_run_wrapped(caf::actor sender, const message& msg, std::index_sequence<Is...>) {
-    auto wrapped = std::make_tuple(msg.get_as<Ts>(Is + 1)...);
-    run_kernel(std::get<Is>(wrapped)...);
-    return true;
-  }
-
-  //unpacks a message and launches a kernel
-  template <std::size_t... Is>
-  bool unpack_and_run(caf::actor sender, const message& msg, std::index_sequence<Is...>) {
-    auto unpacked = std::make_tuple(msg.get_as<raw_t<Ts>>(Is + 1)...);
-    auto wrapped = std::make_tuple(Ts(std::get<Is>(unpacked))...);
-    run_kernel(std::get<Is>(wrapped)...);
-    return true;
-  }
-
-
-  //unpacks a message and launches a kernel
-  template <std::size_t... Is>
-  bool unpack_and_run_wrapped_async(const message& msg, std::index_sequence<Is...>) {
-    auto wrapped = std::make_tuple(msg.get_as<Ts>(Is)...);
-    run_kernel(std::get<Is>(wrapped)...);
-    return true;
+    };
   }
 
-
-
-
-  subtype_t subtype() const noexcept override {
-    return subtype_t(0);
-  }
-
-  //handles scheduling for caf, will return based on what work needs to be done
- resumable::resume_result resume(::caf::scheduler* sched, size_t max_throughput) override {
-  if (resuming_flag_.test_and_set(std::memory_order_acquire)) {
-    return resumable::resume_later;
-  }
-
-  //ensure the lock is released on exit of this method 
-  auto clear_flag = caf::detail::scope_guard([this] noexcept {
-    resuming_flag_.clear(std::memory_order_release);
-  });
-
-  size_t processed = 0;
-
-  while (!mailbox_.empty() && processed < max_throughput) {
-    auto msg = std::move(mailbox_.front());
-    mailbox_.pop();
-
-    if (!msg || !msg->content().ptr()) {
-      std::cout << "[Thread " << std::this_thread::get_id()
-                << "] Dropping message with no content\n";
-      continue;
-    }
-
-    pending_promises_++;
-    current_mailbox_element(msg.get());
-
-    if (msg->content().match_elements<kernel_done_atom>()) {
-      if (--pending_promises_ == 0 && shutdown_requested_) {
-        quit(exit_reason::user_shutdown);
-        return resumable::done;
-      }
-      current_mailbox_element(nullptr);
-      ++processed;
-      continue;
-    }
-
-    //check for exit message if yes begin the shutdown process
-    if (msg->content().match_elements<exit_msg>()) {
-      auto exit = msg->content().get_as<exit_msg>(0);
-      shutdown_requested_ = true;
-      if (--pending_promises_ == 0) {
-        quit(static_cast<exit_reason>(exit.reason.code()));
-        return resumable::done;
+private:
+  template <class... Us>
+  void enqueue_impl(int device_num, atom_value stage, Us&&... xs) {
+    command_runner<Ts...> runner;
+    // Pass actor_id_ as stream_id, and the received device_num
+    auto results = runner.run_async(program_, dims_, actor_id_, 0, device_num, std::forward<Us>(xs)...);
+
+    auto sender = this->current_sender();
+    auto r_atom = reply_atom_;
+    auto stream_id = actor_id_;
+
+    atom_value stage_done = gpu_done_atom_v;
+    if (stage == htod_atom_v) stage_done = htod_done_atom_v;
+    else if (stage == kernel_atom_v) stage_done = kernel_done_atom_v;
+    else if (stage == dtoh_atom_v) stage_done = dtoh_done_atom_v;
+
+    // Pass actor_id_ as stream_id, and the received device_num to add_callback
+    runner.add_callback(stream_id, device_num, [sender, r_atom, stage_done, results]() mutable {
+      auto msg = caf::make_message(stage_done, results);
+      if (r_atom != 0) {
+        caf::anon_mail(r_atom, std::move(msg)).send(sender);
       } else {
-        current_mailbox_element(nullptr);
-        return resumable::resume_later;
+        caf::anon_mail(std::move(msg)).send(sender);
       }
-    }
-
-    //process the message and begin launching the kernel
-    handle_message(msg->content());
-    //pending_promises_--;
-    current_mailbox_element(nullptr);
-    ++processed;
+    });
   }
 
-  // If there's still more work, return resume_later
-  if (!mailbox_.empty())
-    return resumable::resume_later;
-
-  return shutdown_requested_ ? resumable::resume_later : resumable::done;
-}
-
-  void ref_resumable() const noexcept override  {}
-
-  void deref_resumable() const noexcept override  {}
-
-  //add a message to its mailbox and enqueue the actor on the scheduler
-  bool enqueue(mailbox_element_ptr what, ::caf::scheduler* sched) override {
-    if (!what || shutdown_requested_)
-      return false;
-
-    bool was_empty = mailbox_.empty();
-    mailbox_.push(std::move(what));
-    if (was_empty && sched) {
-      sched->schedule(this);
-      return true;
-    }
-
-    return false;
-  }
-
-  //schedule the actor on startup
-  void launch(::caf::scheduler* sched, bool lazy, [[maybe_unused]] bool interruptible) override {
-    if (!lazy && sched) {
-      sched->schedule(this);
-    }
-  }
-
-  void do_unstash(mailbox_element_ptr what) override {
-    if (what) {
-      mailbox_.push(std::move(what));
-    }
-  }
-
-  //close the mailbox when done
-  void force_close_mailbox() override  {
-    while (!mailbox_.empty()) {
-      mailbox_.pop();
-    }
-  }
-
-
-  //helper method to be executed when an exit message is received 
-  void quit(exit_reason reason) {
-   self_ = nullptr; 
-   force_close_mailbox();
-    current_mailbox_element(nullptr);
-  }
+  program_ptr  program_;
+  nd_range     dims_;
+  caf::actor_id actor_id_;
+  atom_value   reply_atom_;
 };
 
-} // namespace caf::cuda
+} // namespace caf::cuda
\ No newline at end of file

From e32950152a99e1539308f298904a1de40c3d60c8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 13 May 2026 14:02:46 -0600
Subject: [PATCH 0616/1000] added bulk transfer HTOD commands

---
 libcaf_cuda/caf/cuda/command_runner.hpp |  27 ++---
 libcaf_cuda/caf/cuda/memory_command.hpp | 133 ++++++++++++++++++------
 2 files changed, 114 insertions(+), 46 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index d25599334d..cba310ad0c 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -205,14 +205,15 @@ class command_runner {
         return cmd.enqueue();
     }
 
-    // Synchronous transfer for testing
-    template <typename T>
-    mem_ptr<raw_t<T>> transfer_memory_sync(int device_number,
-                                           int stream_id,
-                                           T arg)
+    // Bulk asynchronous transfer: returns std::tuple<mem_ptr<raw_t<Us>>...>
+    template <class... Us>
+    auto transfer_memory(int device_number,
+                         int stream_id,
+                         Us&&... args)
     {
-        memory_command<T> cmd(device_number, stream_id, std::move(arg));
-        return cmd.run_sync();
+        auto cmd = caf::make_counted<bulk_memory_command<std::decay_t<Us>...>>(
+          device_number, stream_id, std::forward<Us>(args)...);
+        return cmd->enqueue();
     }
 
    // -------------------------------------------------------------------------
@@ -227,14 +228,16 @@ class command_runner {
         return transfer_memory(token->getDeviceNumber(), token->getStreamId(), std::move(arg));
     }
 
-    template <typename T>
-    mem_ptr<raw_t<T>> transfer_memory_sync(const response_token_ptr& token,
-                                           T arg)
+    // Bulk asynchronous transfer using a response token
+    template <class... Us>
+    auto transfer_memory(const response_token_ptr& token,
+                         Us&&... args)
     {
-        return transfer_memory_sync(token->getDeviceNumber(), token->getStreamId(), std::move(arg));
+        return transfer_memory(token->getDeviceNumber(), 
+                               token->getStreamId(), 
+                               std::forward<Us>(args)...);
     }
 
-
     // -------------------------------------------------------------------------
     // COPY BACK
     // -------------------------------------------------------------------------
diff --git a/libcaf_cuda/caf/cuda/memory_command.hpp b/libcaf_cuda/caf/cuda/memory_command.hpp
index 264085e516..182c61a9ac 100644
--- a/libcaf_cuda/caf/cuda/memory_command.hpp
+++ b/libcaf_cuda/caf/cuda/memory_command.hpp
@@ -2,6 +2,7 @@
 
 #include <utility>
 #include <type_traits>
+#include <tuple>
 
 #include <caf/ref_counted.hpp>
 #include <caf/intrusive_ptr.hpp>
@@ -41,52 +42,116 @@ class memory_command : public caf::ref_counted {
     return dev_->make_arg(arg_, stream);
   }
 
-  // -------------------------------------------------------------------------
-  // Execute memory transfer synchronously (blocking)
-  // -------------------------------------------------------------------------
-  result_type run_sync() {
-    CHECK_CUDA(cuCtxPushCurrent(dev_->getContext()));
+private:
+  int stream_id_;
+  device_ptr dev_;
+  T arg_;
+};
+
+// ===========================================================================
+// BULK MEMORY COMMAND
+// Handles multiple transfers at once and returns a tuple of mem_ptrs.
+// ===========================================================================
+template <typename... Ts>
+class bulk_memory_command : public caf::ref_counted {
+public:
+  using result_type = std::tuple<mem_ptr<raw_t<Ts>>...>;
+
+  bulk_memory_command(int device_number,
+                      int stream_id,
+                      Ts... args)
+      : stream_id_(stream_id),
+        args_(std::move(args)...) {
+    dev_ = platform::create()->schedule(stream_id_, device_number);
+  }
+
+  // Asynchronous execution
+  result_type enqueue() {
     CUstream stream = dev_->get_stream_for_actor(stream_id_);
+    return std::apply([&](auto&&... arg) {
+      return std::make_tuple(dev_->make_arg(arg, stream)...);
+    }, args_);
+  }
 
-    int access = NOT_IN_USE;
-    if constexpr (std::is_same_v<T, in<raw_t<T>>>)
-      access = IN;
-    else if constexpr (std::is_same_v<T, out<raw_t<T>>>)
-      access = OUT;
-    else if constexpr (std::is_same_v<T, in_out<raw_t<T>>>)
-      access = IN_OUT;
-
-    if (arg_.is_scalar()) {
-      raw_t<T> val{};
-      if constexpr (!std::is_same_v<T, out<raw_t<T>>>) {
-        val = arg_.getscalar();
-      }
-      auto res = caf::make_counted<mem_ref<raw_t<T>>>(val, access, dev_->getId(), 0, dev_->getContext(), stream);
-      CHECK_CUDA(cuCtxPopCurrent(nullptr));
-      return res;
-    }
+private:
+  int stream_id_;
+  device_ptr dev_;
+  std::tuple<Ts...> args_;
+};
+
+// ===========================================================================
+// BULK MEMORY COMMAND
+// Handles multiple transfers at once.
+// ===========================================================================
+template <typename... Ts>
+class bulk_memory_command : public caf::ref_counted {
+public:
+  using result_type = std::tuple<mem_ptr<raw_t<Ts>>...>;
+
+  bulk_memory_command(int device_number,
+                      int stream_id,
+                      Ts... args)
+      : stream_id_(stream_id),
+        args_(std::move(args)...) {
+    dev_ = platform::create()->schedule(stream_id_, device_number);
+  }
 
-    size_t size = arg_.size();
-    size_t bytes = size * sizeof(raw_t<T>);
-    CUdeviceptr mem;
+  // Asynchronous execution
+  result_type enqueue() {
+    CUstream stream = dev_->get_stream_for_actor(stream_id_);
+    return std::apply([&](auto&&... arg) {
+      return std::make_tuple(dev_->make_arg(arg, stream)...);
+    }, args_);
+  }
 
-    // Synchronous allocation
-    CHECK_CUDA(cuMemAlloc(&mem, bytes));
+  // Synchronous execution (blocking)
+  result_type run_sync() {
+    CHECK_CUDA(cuCtxPushCurrent(dev_->getContext()));
+    CUstream stream = dev_->get_stream_for_actor(stream_id_);
 
-    if constexpr (!std::is_same_v<T, out<raw_t<T>>>) {
-      // Synchronous copy from host to device
-      CHECK_CUDA(cuMemcpyHtoD(mem, arg_.data(), bytes));
-    }
+    auto result = std::apply([&](auto&&... arg) {
+      return std::make_tuple(([&]() {
+        using arg_t = std::decay_t<decltype(arg)>;
+        int access = NOT_IN_USE;
+        if constexpr (std::is_same_v<arg_t, in<raw_t<arg_t>>>)
+          access = IN;
+        else if constexpr (std::is_same_v<arg_t, out<raw_t<arg_t>>>)
+          access = OUT;
+        else if constexpr (std::is_same_v<arg_t, in_out<raw_t<arg_t>>>)
+          access = IN_OUT;
+
+        if (arg.is_scalar()) {
+          raw_t<arg_t> val{};
+          if constexpr (!std::is_same_v<arg_t, out<raw_t<arg_t>>>) {
+            val = arg.getscalar();
+          }
+          return caf::make_counted<mem_ref<raw_t<arg_t>>>(val, access, dev_->getId(), 0, dev_->getContext(), stream);
+        }
+
+        size_t size = arg.size();
+        size_t bytes = size * sizeof(raw_t<arg_t>);
+        CUdeviceptr mem;
+
+        // Synchronous allocation
+        CHECK_CUDA(cuMemAlloc(&mem, bytes));
+
+        if constexpr (!std::is_same_v<arg_t, out<raw_t<arg_t>>>) {
+          // Synchronous copy from host to device
+          CHECK_CUDA(cuMemcpyHtoD(mem, arg.data(), bytes));
+        }
+
+        return caf::make_counted<mem_ref<raw_t<arg_t>>>(size, mem, access, dev_->getId(), 0, dev_->getContext(), stream);
+      }())...);
+    }, args_);
 
-    auto res = caf::make_counted<mem_ref<raw_t<T>>>(size, mem, access, dev_->getId(), 0, dev_->getContext(), stream);
     CHECK_CUDA(cuCtxPopCurrent(nullptr));
-    return res;
+    return result;
   }
 
 private:
   int stream_id_;
   device_ptr dev_;
-  T arg_;
+  std::tuple<Ts...> args_;
 };
 
 // ===========================================================================

From 1bab9a2b1a9be36a2d319455961ba806c1be0c1f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 13 May 2026 14:13:42 -0600
Subject: [PATCH 0617/1000] fixed compiler errors

---
 libcaf_cuda/caf/cuda/memory_command.hpp | 74 -------------------------
 1 file changed, 74 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/memory_command.hpp b/libcaf_cuda/caf/cuda/memory_command.hpp
index 182c61a9ac..f47b2250c8 100644
--- a/libcaf_cuda/caf/cuda/memory_command.hpp
+++ b/libcaf_cuda/caf/cuda/memory_command.hpp
@@ -79,80 +79,6 @@ class bulk_memory_command : public caf::ref_counted {
   std::tuple<Ts...> args_;
 };
 
-// ===========================================================================
-// BULK MEMORY COMMAND
-// Handles multiple transfers at once.
-// ===========================================================================
-template <typename... Ts>
-class bulk_memory_command : public caf::ref_counted {
-public:
-  using result_type = std::tuple<mem_ptr<raw_t<Ts>>...>;
-
-  bulk_memory_command(int device_number,
-                      int stream_id,
-                      Ts... args)
-      : stream_id_(stream_id),
-        args_(std::move(args)...) {
-    dev_ = platform::create()->schedule(stream_id_, device_number);
-  }
-
-  // Asynchronous execution
-  result_type enqueue() {
-    CUstream stream = dev_->get_stream_for_actor(stream_id_);
-    return std::apply([&](auto&&... arg) {
-      return std::make_tuple(dev_->make_arg(arg, stream)...);
-    }, args_);
-  }
-
-  // Synchronous execution (blocking)
-  result_type run_sync() {
-    CHECK_CUDA(cuCtxPushCurrent(dev_->getContext()));
-    CUstream stream = dev_->get_stream_for_actor(stream_id_);
-
-    auto result = std::apply([&](auto&&... arg) {
-      return std::make_tuple(([&]() {
-        using arg_t = std::decay_t<decltype(arg)>;
-        int access = NOT_IN_USE;
-        if constexpr (std::is_same_v<arg_t, in<raw_t<arg_t>>>)
-          access = IN;
-        else if constexpr (std::is_same_v<arg_t, out<raw_t<arg_t>>>)
-          access = OUT;
-        else if constexpr (std::is_same_v<arg_t, in_out<raw_t<arg_t>>>)
-          access = IN_OUT;
-
-        if (arg.is_scalar()) {
-          raw_t<arg_t> val{};
-          if constexpr (!std::is_same_v<arg_t, out<raw_t<arg_t>>>) {
-            val = arg.getscalar();
-          }
-          return caf::make_counted<mem_ref<raw_t<arg_t>>>(val, access, dev_->getId(), 0, dev_->getContext(), stream);
-        }
-
-        size_t size = arg.size();
-        size_t bytes = size * sizeof(raw_t<arg_t>);
-        CUdeviceptr mem;
-
-        // Synchronous allocation
-        CHECK_CUDA(cuMemAlloc(&mem, bytes));
-
-        if constexpr (!std::is_same_v<arg_t, out<raw_t<arg_t>>>) {
-          // Synchronous copy from host to device
-          CHECK_CUDA(cuMemcpyHtoD(mem, arg.data(), bytes));
-        }
-
-        return caf::make_counted<mem_ref<raw_t<arg_t>>>(size, mem, access, dev_->getId(), 0, dev_->getContext(), stream);
-      }())...);
-    }, args_);
-
-    CHECK_CUDA(cuCtxPopCurrent(nullptr));
-    return result;
-  }
-
-private:
-  int stream_id_;
-  device_ptr dev_;
-  std::tuple<Ts...> args_;
-};
 
 // ===========================================================================
 // COPY BACK COMMAND

From 25cadb8fac736bef8a0ea18b0ab3eca50ce1f90e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 13 May 2026 14:14:12 -0600
Subject: [PATCH 0618/1000] commented out actor facade for now since breaks
 build

---
 libcaf_cuda/caf/cuda/actor_facade.hpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/actor_facade.hpp b/libcaf_cuda/caf/cuda/actor_facade.hpp
index 625a1054f3..abe84638f4 100644
--- a/libcaf_cuda/caf/cuda/actor_facade.hpp
+++ b/libcaf_cuda/caf/cuda/actor_facade.hpp
@@ -19,6 +19,7 @@
 #include "caf/cuda/platform.hpp"
 #include "caf/cuda/program.hpp"
 
+/*
 
 namespace caf::cuda {
 
@@ -99,4 +100,5 @@ class actor_facade : public event_based_actor {
   atom_value   reply_atom_;
 };
 
-} // namespace caf::cuda
\ No newline at end of file
+} // namespace caf::cuda
+ */
\ No newline at end of file

From 070fb02845feb8587cd6b6d89eae0e9bffd19fdf Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 13 May 2026 15:19:19 -0600
Subject: [PATCH 0619/1000] made workloads more irregular

---
 .../cuda-baseline.cpp                         | 100 ++++++++++++++----
 .../work-stealing.cpp                         |  81 ++++++++++----
 2 files changed, 144 insertions(+), 37 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index bc6a571328..da1e95ca64 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -7,6 +7,8 @@
 #include <algorithm>
 #include <functional>
 #include <stdexcept> // For runtime_error
+#include <unordered_map> // For MatrixPool
+#include <unordered_set> // For create_matrix_pool_random
 
 struct Task {
     int N;
@@ -14,7 +16,30 @@ struct Task {
 
 // Per-GPU execution logic
 void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_gpu, 
-                CUcontext ctx, CUfunction kernel_func, const std::vector<int>& h_256, const std::vector<int>& h_2048) {
+                CUcontext ctx, CUfunction kernel_func, const std::unordered_map<int, std::vector<int>>& host_matrix_A, const std::unordered_map<int, std::vector<int>>& host_matrix_B) {
+    // Set the CUDA context for this thread
+    CUresult err = cuCtxSetCurrent(ctx);
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error setting context for device " << device_id << ": " << err_str << std::endl;
+        return;
+    }
+
+    // Prepare Streams
+    std::vector<CUstream> streams(streams_per_gpu);
+    for (int i = 0; i < streams_per_gpu; ++i) {
+        cuStreamCreate(&streams[i], CU_STREAM_NON_BLOCKING);
+    }
+
+    // Use the first stream for initial allocations and cleanup
+    CUstream default_stream = streams[0];
+
+    // Process assigned tasks
+    for (size_t i = 0; i < tasks.size(); ++i) {
+        int N = tasks[i].N;
+        CUstream stream = streams[i % streams_per_gpu];
+        size_t bytes = static_cast<size_t>(N) * N * sizeof(int);
     // Set the CUDA context for this thread
     CUresult err = cuCtxSetCurrent(ctx);
     if (err != CUDA_SUCCESS) {
@@ -47,9 +72,8 @@ void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_g
         cuMemAllocAsync(&d_c, bytes, stream);
 
         // Perform Host-to-Device transfer
-        const int* h_src = (N == 256) ? h_256.data() : h_2048.data();
-        cuMemcpyHtoDAsync(d_a, h_src, bytes, stream);
-        cuMemcpyHtoDAsync(d_b, h_src, bytes, stream);
+        cuMemcpyHtoDAsync(d_a, host_matrix_A.at(N).data(), bytes, stream);
+        cuMemcpyHtoDAsync(d_b, host_matrix_B.at(N).data(), bytes, stream);
 
         // Kernel arguments for cuLaunchKernel
         void *kernel_args[] = { &d_a, &d_b, &d_c, &N };
@@ -81,6 +105,32 @@ void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_g
     }
 }
 
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
+};
+
+MatrixPool create_matrix_pool_random(
+    int num_sizes,
+    int min_N,
+    int max_N,
+    unsigned int seed
+) {
+    MatrixPool pool;
+    std::mt19937 rng(seed);
+    std::uniform_int_distribution<int> dist_N(min_N / 32, max_N / 32);
+    std::unordered_set<int> used_Ns;
+    while (used_Ns.size() < static_cast<size_t>(num_sizes)) {
+        int N_val = dist_N(rng) * 32;
+        if (N_val == 0) continue;
+        if (used_Ns.insert(N_val).second) {
+            pool.A[N_val] = std::vector<int>(N_val * N_val, 1);
+            pool.B[N_val] = std::vector<int>(N_val * N_val, 1);
+        }
+    }
+    return pool;
+}
+
 int main() {
     CUresult err;
 
@@ -103,22 +153,32 @@ int main() {
         return 1;
     }
 
-    for (int total_tasks : task_counts) {
-    std::cout << "=====================================" << std::endl;
-    std::cout << "Task count: " << total_tasks << " (10% Heavy, 90% Light)" << std::endl;
+    // Define parameters for irregular workload
+    const int num_matrix_sizes = 1000// Number of distinct N values
+    const int min_N_val = 32;
+    const int max_N_val = 2048;
+    const unsigned int pool_seed = 42; // Fixed seed for deterministic pool generation
 
-    std::vector<Task> all_tasks;
-    std::mt19937 rng(42);
-    std::uniform_real_distribution<double> dist(0.0, 1.0);
+    // Create the host-side matrix pool once
+    MatrixPool global_host_matrix_pool = create_matrix_pool_random(num_matrix_sizes, min_N_val, max_N_val, pool_seed);
 
-    for (int i = 0; i < total_tasks; ++i) {
-        if (dist(rng) < 0.1) all_tasks.push_back({2048});
-        else all_tasks.push_back({256});
+    // Extract available N values from the pool for task generation
+    std::vector<int> available_Ns;
+    for (const auto& pair : global_host_matrix_pool.A) {
+        available_Ns.push_back(pair.first);
+    }
+    if (available_Ns.empty()) {
+        std::cerr << "Error: No matrix sizes generated in the pool." << std::endl;
+        return 1;
     }
 
-    // Prepare Host-side MatrixPool (on CPU)
-    static std::vector<int> h_256(256 * 256, 1);
-    static std::vector<int> h_2048(2048 * 2048, 1);
+    for (int total_tasks : task_counts) {
+    std::cout << "=====================================" << std::endl;
+    std::cout << "Task count: " << total_tasks << " (Irregular Workload)" << std::endl;
+
+    std::vector<Task> all_tasks;
+    std::mt19937 rng_tasks(42); // Fixed seed for task distribution
+    std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
 
     std::vector<CUcontext> contexts(num_gpus);
     std::vector<CUfunction> kernel_funcs(num_gpus);
@@ -188,6 +248,11 @@ int main() {
         kernel_funcs[i] = kernel_funcs[0];
     }
 
+    for (int i = 0; i < total_tasks; ++i) {
+        int N_for_task = available_Ns[dist_N_idx(rng_tasks)];
+        all_tasks.push_back({N_for_task});
+    }
+
     // ─────────────────────────────────────────────────────────────────────────
     // Static Round-Robin Partitioning
     // ─────────────────────────────────────────────────────────────────────────
@@ -200,8 +265,7 @@ int main() {
 
     std::vector<std::thread> threads;
     for (int i = 0; i < num_gpus; ++i) { // Pass context and kernel function to each worker
-        threads.emplace_back(gpu_worker, i, std::ref(partitions[i]), streams_per_gpu, 
-                            contexts[i], kernel_funcs[i], std::ref(h_256), std::ref(h_2048));
+        threads.emplace_back(gpu_worker, i, std::ref(partitions[i]), streams_per_gpu, contexts[i], kernel_funcs[i], std::ref(global_host_matrix_pool.A), std::ref(global_host_matrix_pool.B));
     }
 
     for (auto& t : threads) {
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index 43bfcf8aab..379cd60e26 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -6,6 +6,7 @@
 #include <random>
 #include <unordered_map>
 #include <deque>
+#include <unordered_set> // For create_matrix_pool_random
 
 using namespace caf;
 using namespace std::chrono_literals;
@@ -32,6 +33,30 @@ struct MatrixPool {
     std::unordered_map<int, std::vector<int>> B;
 };
 
+MatrixPool create_matrix_pool_random(
+    int num_sizes,
+    int min_N,
+    int max_N,
+    unsigned int seed
+) {
+    MatrixPool pool;
+    std::mt19937 rng(seed);
+    // Ensure N values are multiples of 32 for typical matrix sizes
+    std::uniform_int_distribution<int> dist_N(min_N / 32, max_N / 32);
+
+    std::unordered_set<int> used_Ns;
+
+    while (used_Ns.size() < static_cast<size_t>(num_sizes)) {
+        int N_val = dist_N(rng) * 32;
+        if (N_val == 0) continue; // Avoid N=0
+        if (used_Ns.insert(N_val).second) {
+            pool.A[N_val] = std::vector<int>(N_val * N_val, 1);
+            pool.B[N_val] = std::vector<int>(N_val * N_val, 1);
+        }
+    }
+    return pool;
+}
+
 // ---------------------------- GLOBAL TASK POOL ----------------------------
 // The central source of truth for work. Implements a pull-based model.
 struct task_pool_state {
@@ -70,7 +95,7 @@ struct device_actor_state {
     bool fetching = false;
 };
 
-caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
+caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self, // Changed signature
                                MatrixPool pool, caf::actor global_pool, int num_workers, int dev_id, int max_in_flight) {
     self->state().pool = std::move(pool);
     self->state().global_pool = global_pool;
@@ -235,25 +260,28 @@ struct supervisor_state {
 };
 
 caf::behavior supervisor_actor(caf::stateful_actor<supervisor_state>* self, 
-                               int total, 
-                               std::vector<int> Ns,
+                               int total,
+                               std::vector<int> Ns_for_tasks, // Renamed for clarity
+                               MatrixPool host_matrix_pool, // Pass the pre-generated pool
                                int workers_per_gpu,
                                int max_in_flight) {
     self->state().total = total;
     self->state().start = std::chrono::steady_clock::now();
-    auto pool = self->spawn(global_task_pool, std::move(Ns));
+    auto pool = self->spawn(global_task_pool, std::move(Ns_for_tasks));
     
-    MatrixPool m_pool;
-    for (int n : {256, 2048}) {
-        m_pool.A[n] = std::vector<int>(n * n, 1);
-        m_pool.B[n] = std::vector<int>(n * n, 1);
-    }
+    // The MatrixPool is now passed as an argument, no need to create it here
+    // MatrixPool m_pool;
+    // for (int n : {256, 2048}) {
+    //     m_pool.A[n] = std::vector<int>(n * n, 1);
+    //     m_pool.B[n] = std::vector<int>(n * n, 1);
+    // }
 
     auto& mgr = caf::cuda::manager::get();
     auto prog = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
     for (int i = 0; i < mgr.get_num_devices(); ++i) {
-        auto broker = self->spawn(gpu_device_actor, m_pool, pool, workers_per_gpu, i, max_in_flight);
+        // Pass the host_matrix_pool to the gpu_device_actor
+        auto broker = self->spawn(gpu_device_actor, host_matrix_pool, pool, workers_per_gpu, i, max_in_flight);
         for (int j = 0; j < workers_per_gpu; ++j)
             self->spawn(mmul_worker, self, broker, prog, i, (i * 100) + j, max_in_flight);
     }
@@ -276,23 +304,38 @@ void caf_main(caf::actor_system& sys) {
     int max_in_flight = 3;
     std::vector<int> task_counts = {30000, 40000, 50000};
 
+    // Define parameters for irregular workload
+    const int num_matrix_sizes = 1000// Number of distinct N values
+    const int min_N_val = 32;
+    const int max_N_val = 2048;
+    const unsigned int pool_seed = 42; // Fixed seed for deterministic pool generation
+
+    // Create the host-side matrix pool once
+    MatrixPool global_host_matrix_pool = create_matrix_pool_random(num_matrix_sizes, min_N_val, max_N_val, pool_seed);
+
+    // Extract available N values from the pool for task generation
+    std::vector<int> available_Ns;
+    for (const auto& pair : global_host_matrix_pool.A) {
+        available_Ns.push_back(pair.first);
+    }
+    if (available_Ns.empty()) {
+        std::cerr << "Error: No matrix sizes generated in the pool." << std::endl;
+        return;
+    }
+
     for (int total_tasks : task_counts) {
         caf::cuda::manager_config cfg(false);
         caf::cuda::manager::init(sys, cfg);
         std::cout << "=====================================\n";
         std::cout << "Task count: " << total_tasks << "\n";
 
-        std::vector<int> Ns;
-        std::mt19937 rng(42);
-        std::uniform_real_distribution<double> dist(0.0, 1.0); // Power Law setup
-
-        // Power Law simulation: 10% Heavy (2048), 90% Light (256)
+        std::vector<int> Ns_for_this_run;
+        std::mt19937 rng_tasks(42); // Fixed seed for task distribution
+        std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
         for (int i = 0; i < total_tasks; ++i) {
-            if (dist(rng) < 0.1) Ns.push_back(2048);
-            else Ns.push_back(256);
+            Ns_for_this_run.push_back(available_Ns[dist_N_idx(rng_tasks)]);
         }
-
-        sys.spawn(supervisor_actor, total_tasks, std::move(Ns), workers_per_gpu, max_in_flight);
+        sys.spawn(supervisor_actor, total_tasks, std::move(Ns_for_this_run), global_host_matrix_pool, workers_per_gpu, max_in_flight);
         sys.await_all_actors_done();
         caf::cuda::manager::shutdown();
     }

From 9fa34897890ab8491081a4e1eff724121a7e66f7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 13 May 2026 15:22:38 -0600
Subject: [PATCH 0620/1000] fixed syntax errors

---
 .../cuda-baseline.cpp                         | 25 +------------------
 .../work-stealing.cpp                         |  2 +-
 2 files changed, 2 insertions(+), 25 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index da1e95ca64..c276344db1 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -35,29 +35,6 @@ void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_g
     // Use the first stream for initial allocations and cleanup
     CUstream default_stream = streams[0];
 
-    // Process assigned tasks
-    for (size_t i = 0; i < tasks.size(); ++i) {
-        int N = tasks[i].N;
-        CUstream stream = streams[i % streams_per_gpu];
-        size_t bytes = static_cast<size_t>(N) * N * sizeof(int);
-    // Set the CUDA context for this thread
-    CUresult err = cuCtxSetCurrent(ctx);
-    if (err != CUDA_SUCCESS) {
-        const char* err_str;
-        cuGetErrorString(err, &err_str);
-        std::cerr << "Error setting context for device " << device_id << ": " << err_str << std::endl;
-        return;
-    }
-
-    // Prepare Streams
-    std::vector<CUstream> streams(streams_per_gpu);
-    for (int i = 0; i < streams_per_gpu; ++i) {
-        cuStreamCreate(&streams[i], CU_STREAM_NON_BLOCKING);
-    }
-
-    // Use the first stream for initial allocations and cleanup
-    CUstream default_stream = streams[0];
-
     // Process assigned tasks
     for (size_t i = 0; i < tasks.size(); ++i) {
         int N = tasks[i].N;
@@ -154,7 +131,7 @@ int main() {
     }
 
     // Define parameters for irregular workload
-    const int num_matrix_sizes = 1000// Number of distinct N values
+    const int num_matrix_sizes = 1000; // Number of distinct N values
     const int min_N_val = 32;
     const int max_N_val = 2048;
     const unsigned int pool_seed = 42; // Fixed seed for deterministic pool generation
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index 379cd60e26..b8a03e8b1a 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -305,7 +305,7 @@ void caf_main(caf::actor_system& sys) {
     std::vector<int> task_counts = {30000, 40000, 50000};
 
     // Define parameters for irregular workload
-    const int num_matrix_sizes = 1000// Number of distinct N values
+    const int num_matrix_sizes = 1000; // Number of distinct N values
     const int min_N_val = 32;
     const int max_N_val = 2048;
     const unsigned int pool_seed = 42; // Fixed seed for deterministic pool generation

From 945f1445cfb0fb43c1ff23691d1121cf093c92e8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 14 May 2026 08:32:06 -0600
Subject: [PATCH 0621/1000] added include to command to fix compiler warning

---
 libcaf_cuda/caf/cuda/command.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/command.hpp b/libcaf_cuda/caf/cuda/command.hpp
index 601554d433..3e81a44a7a 100644
--- a/libcaf_cuda/caf/cuda/command.hpp
+++ b/libcaf_cuda/caf/cuda/command.hpp
@@ -13,6 +13,7 @@
 #include "caf/cuda/platform.hpp"
 #include "caf/cuda/mem_ref.hpp"
 #include "caf/cuda/device.hpp"
+#include "caf/cuda/program.hpp"
 
 
 
@@ -158,4 +159,3 @@ class command : public base_command<Actor, Ts...> {
 };
 
 } // namespace caf::cuda
-

From d1ef7a4cded2260a06f2747c75af6d61b9199a11 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 14 May 2026 08:52:37 -0600
Subject: [PATCH 0622/1000] updated task size

---
 .../mmul-randonom-batch-benchmark/cuda-baseline.cpp  |  2 +-
 .../mmul-randonom-batch-benchmark/work-stealing.cpp  | 12 +++++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index c276344db1..8bd07e6981 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -121,7 +121,7 @@ int main() {
     }
 
     const int streams_per_gpu = 8;
-    std::vector<int> task_counts = {30000, 40000, 50000};
+    std::vector<int> task_counts = {50000,100000};
 
     int num_gpus;
     cuDeviceGetCount(&num_gpus);
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index b8a03e8b1a..5de67861b1 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -247,7 +247,13 @@ caf::behavior mmul_worker(caf::stateful_actor<worker_state>* self,
             self->state().in_flight--;
             self->mail(1).send(self->state().supervisor);
             self->mail(release_memory_atom_v, N).send(self->state().device_actor);
-            self->mail(request_work_atom_v).send(self);
+            
+            if (self->state().draining && self->state().in_flight == 0) {
+                self->mail(worker_done_atom_v).send(self->state().device_actor);
+                self->quit();
+            } else if (!self->state().draining) {
+                self->mail(request_work_atom_v).send(self);
+            }
         }
     };
 }
@@ -302,10 +308,10 @@ caf::behavior supervisor_actor(caf::stateful_actor<supervisor_state>* self,
 void caf_main(caf::actor_system& sys) {
     int workers_per_gpu = 8;
     int max_in_flight = 3;
-    std::vector<int> task_counts = {30000, 40000, 50000};
+    std::vector<int> task_counts = {50000,100000};
 
     // Define parameters for irregular workload
-    const int num_matrix_sizes = 1000; // Number of distinct N values
+    const int num_matrix_sizes = 60; // Number of distinct N values
     const int min_N_val = 32;
     const int max_N_val = 2048;
     const unsigned int pool_seed = 42; // Fixed seed for deterministic pool generation

From 0164d7f14fa618c1ce37e89f92c90da1858ec273 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 14 May 2026 09:04:07 -0600
Subject: [PATCH 0623/1000] updated number of tasks

---
 .../mmul-randonom-batch-benchmark/cuda-baseline.cpp             | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index 8bd07e6981..f7ab4a0c7b 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -131,7 +131,7 @@ int main() {
     }
 
     // Define parameters for irregular workload
-    const int num_matrix_sizes = 1000; // Number of distinct N values
+    const int num_matrix_sizes = 60; // Number of distinct N values
     const int min_N_val = 32;
     const int max_N_val = 2048;
     const unsigned int pool_seed = 42; // Fixed seed for deterministic pool generation

From 7f94cc190d118a90857d41c0700f14ab019f669e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 14 May 2026 09:43:04 -0600
Subject: [PATCH 0624/1000] Updated work stealing test to include more kernels
 to increase the irregularity of the workload.

---
 .../compile_kernels.sh                        |  17 +-
 .../mmul-randonom-batch-benchmark/conv1d.cu   |  15 +
 .../cuda-baseline.cpp                         | 146 ++++--
 .../vector_add.cu                             |   6 +
 .../work-stealing.cpp                         | 463 +++++++++++-------
 5 files changed, 424 insertions(+), 223 deletions(-)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/conv1d.cu
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/vector_add.cu

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/compile_kernels.sh b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/compile_kernels.sh
index f32480e5cb..d504d43a43 100755
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/compile_kernels.sh
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/compile_kernels.sh
@@ -1,13 +1,6 @@
 #!/bin/bash
-set -e
-
-# Detect first GPU compute capability
-ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
-SM_ARCH="sm_${ARCH/./}"
-echo "Using NVCC arch flag: $SM_ARCH"
-
-# Compile mmul.cu to cubin in current directory
-nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
-echo "Generated mmul.cubin"
-echo "All kernels compiled successfully!"
-
+# Compile kernels to cubin for Driver API loading
+nvcc -cubin mmul.cu -o ../mmul.cubin
+nvcc -cubin vector_add.cu -o ../vector_add.cubin
+nvcc -cubin conv1d.cu -o ../conv1d.cubin
+echo "Kernels compiled successfully."
\ No newline at end of file
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/conv1d.cu b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/conv1d.cu
new file mode 100644
index 0000000000..aa897ede25
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/conv1d.cu
@@ -0,0 +1,15 @@
+extern "C" __global__ void conv1d(const int* A, const int* K, int* C, int N) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int W = 5; // Fixed filter width for benchmark simplicity
+    if (idx < N) {
+        int sum = 0;
+        int halfW = W / 2;
+        for (int i = 0; i < W; ++i) {
+            int col = idx + i - halfW;
+            if (col >= 0 && col < N) {
+                sum += A[col] * K[i];
+            }
+        }
+        C[idx] = sum;
+    }
+}
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index f7ab4a0c7b..17bd75e52a 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -10,13 +10,24 @@
 #include <unordered_map> // For MatrixPool
 #include <unordered_set> // For create_matrix_pool_random
 
+enum TaskType { MMUL = 0, VADD = 1, CONV = 2 };
+
 struct Task {
     int N;
+    TaskType type;
+};
+
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
+    std::unordered_map<int, std::vector<int>> vec_A;
+    std::unordered_map<int, std::vector<int>> vec_B;
+    std::unordered_map<int, std::vector<int>> conv_A;
+    std::unordered_map<int, std::vector<int>> conv_K;
 };
 
 // Per-GPU execution logic
-void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_gpu, 
-                CUcontext ctx, CUfunction kernel_func, const std::unordered_map<int, std::vector<int>>& host_matrix_A, const std::unordered_map<int, std::vector<int>>& host_matrix_B) {
+void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_gpu, CUcontext ctx, CUfunction mmul_func, CUfunction vadd_func, CUfunction conv_func, const MatrixPool& pool) {
     // Set the CUDA context for this thread
     CUresult err = cuCtxSetCurrent(ctx);
     if (err != CUDA_SUCCESS) {
@@ -39,33 +50,51 @@ void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_g
     for (size_t i = 0; i < tasks.size(); ++i) {
         int N = tasks[i].N;
         CUstream stream = streams[i % streams_per_gpu];
-        size_t bytes = static_cast<size_t>(N) * N * sizeof(int);
+        TaskType type = tasks[i].type;
+        size_t bytes_a = (type == MMUL) ? (size_t)N * N * sizeof(int) : (size_t)N * sizeof(int);
+        size_t bytes_b = (type == CONV) ? 5 * sizeof(int) : bytes_a;
+        size_t bytes_out = (type == MMUL) ? (size_t)N * N * sizeof(int) : (size_t)N * sizeof(int);
 
         CUdeviceptr d_a, d_b, d_c;
 
-        // Allocate GPU memory for this specific task
-        cuMemAllocAsync(&d_a, bytes, stream);
-        cuMemAllocAsync(&d_b, bytes, stream);
-        cuMemAllocAsync(&d_c, bytes, stream);
+        cuMemAllocAsync(&d_a, bytes_a, stream);
+        cuMemAllocAsync(&d_b, bytes_b, stream);
+        cuMemAllocAsync(&d_c, bytes_out, stream);
 
         // Perform Host-to-Device transfer
-        cuMemcpyHtoDAsync(d_a, host_matrix_A.at(N).data(), bytes, stream);
-        cuMemcpyHtoDAsync(d_b, host_matrix_B.at(N).data(), bytes, stream);
+        const std::vector<int>& h_a = (type == MMUL) ? pool.A.at(N) : (type == VADD ? pool.vec_A.at(N) : pool.conv_A.at(N));
+        const std::vector<int>& h_b = (type == MMUL) ? pool.B.at(N) : (type == VADD ? pool.vec_B.at(N) : pool.conv_K.at(N));
+
+        cuMemcpyHtoDAsync(d_a, h_a.data(), bytes_a, stream);
+        cuMemcpyHtoDAsync(d_b, h_b.data(), bytes_b, stream);
 
         // Kernel arguments for cuLaunchKernel
         void *kernel_args[] = { &d_a, &d_b, &d_c, &N };
 
-        // Kernel dimensions
-        unsigned int block_dim = 32;
-        unsigned int grid_dim = (N + block_dim - 1) / block_dim;
-
-        cuLaunchKernel(kernel_func, grid_dim, grid_dim, 1,
-                       block_dim, block_dim, 1,
-                       0, stream, kernel_args, nullptr);
+        if (type == MMUL) {
+            unsigned int block_dim = 32;
+            unsigned int grid_dim = (N + block_dim - 1) / block_dim;
+            cuLaunchKernel(mmul_func, grid_dim, grid_dim, 1,
+                           block_dim, block_dim, 1,
+                           0, stream, kernel_args, nullptr);
+        } else if (type == VADD) {
+            unsigned int block_dim = 256;
+            unsigned int grid_dim = (N + block_dim - 1) / block_dim;
+            cuLaunchKernel(vadd_func, grid_dim, 1, 1,
+                           block_dim, 1, 1,
+                           0, stream, kernel_args, nullptr);
+        } else {
+            unsigned int block_dim = 256;
+            unsigned int grid_dim = (N + block_dim - 1) / block_dim;
+            cuLaunchKernel(conv_func, grid_dim, 1, 1,
+                           block_dim, 1, 1,
+                           0, stream, kernel_args, nullptr);
+        }
         
         // Simulating the result retrieval (Copy back)
-        std::vector<int> h_res(N * N);
-        cuMemcpyDtoHAsync(h_res.data(), d_c, bytes, stream);
+        size_t res_count = (type == MMUL) ? (size_t)N * N : (size_t)N;
+        std::vector<int> h_res(res_count);
+        cuMemcpyDtoHAsync(h_res.data(), d_c, bytes_out, stream);
 
         // Free GPU memory for this task
         cuMemFreeAsync(d_a, stream);
@@ -82,11 +111,6 @@ void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_g
     }
 }
 
-struct MatrixPool {
-    std::unordered_map<int, std::vector<int>> A;
-    std::unordered_map<int, std::vector<int>> B;
-};
-
 MatrixPool create_matrix_pool_random(
     int num_sizes,
     int min_N,
@@ -103,6 +127,10 @@ MatrixPool create_matrix_pool_random(
         if (used_Ns.insert(N_val).second) {
             pool.A[N_val] = std::vector<int>(N_val * N_val, 1);
             pool.B[N_val] = std::vector<int>(N_val * N_val, 1);
+            pool.vec_A[N_val] = std::vector<int>(N_val, 1);
+            pool.vec_B[N_val] = std::vector<int>(N_val, 1);
+            pool.conv_A[N_val] = std::vector<int>(N_val, 1);
+            pool.conv_K[N_val] = std::vector<int>(5, 1);
         }
     }
     return pool;
@@ -158,8 +186,12 @@ int main() {
     std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
 
     std::vector<CUcontext> contexts(num_gpus);
-    std::vector<CUfunction> kernel_funcs(num_gpus);
-    CUmodule module;
+    std::vector<CUfunction> mmul_funcs(num_gpus);
+    std::vector<CUfunction> vadd_funcs(num_gpus);
+    std::vector<CUfunction> conv_funcs(num_gpus);
+    CUmodule mmul_mod;
+    CUmodule vadd_mod;
+    CUmodule conv_mod;
 
     // Create a CUDA context for each device
     for (int i = 0; i < num_gpus; ++i) {
@@ -187,47 +219,89 @@ int main() {
     }
 
     // Load the cubin module (assuming mmul.cu is compiled to mmul.cubin)
-    err = cuModuleLoad(&module, "../mmul.cubin");
+    err = cuModuleLoad(&mmul_mod, "../mmul.cubin");
     if (err != CUDA_SUCCESS) {
         const char* err_str;
         cuGetErrorString(err, &err_str);
         std::cerr << "Error loading module ../mmul.cubin: " << err_str << std::endl;
         cuCtxPopCurrent(nullptr); // Pop context on error
-        for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
         return 1;
     }
 
-    // Get function handle for matrixMul
-    err = cuModuleGetFunction(&kernel_funcs[0], module, "matrixMul");
+    err = cuModuleLoad(&vadd_mod, "../vector_add.cubin");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error loading module ../vector_add.cubin: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr);
+        return 1;
+    }
+
+    err = cuModuleLoad(&conv_mod, "../conv1d.cubin");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error loading module ../conv1d.cubin: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr);
+        return 1;
+    }
+
+    // Get function handles
+    err = cuModuleGetFunction(&mmul_funcs[0], mmul_mod, "matrixMul");
     if (err != CUDA_SUCCESS) {
         const char* err_str;
         cuGetErrorString(err, &err_str);
         std::cerr << "Error getting function matrixMul: " << err_str << std::endl;
         cuCtxPopCurrent(nullptr); // Pop context on error
-        cuModuleUnload(module);
+        cuModuleUnload(mmul_mod);
+        cuModuleUnload(vadd_mod);
         for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
         return 1;
     }
 
+    err = cuModuleGetFunction(&vadd_funcs[0], vadd_mod, "vectorAdd");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error getting function vectorAdd: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr);
+        return 1;
+    }
+
+    err = cuModuleGetFunction(&conv_funcs[0], conv_mod, "conv1d");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error getting function conv1d: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr);
+        return 1;
+    }
+
     // Pop the context from the main thread
     err = cuCtxPopCurrent(nullptr);
     if (err != CUDA_SUCCESS) {
         const char* err_str;
         cuGetErrorString(err, &err_str);
         std::cerr << "Error popping context from main thread: " << err_str << std::endl;
-        cuModuleUnload(module);
+        cuModuleUnload(mmul_mod);
+        cuModuleUnload(vadd_mod);
+        cuModuleUnload(conv_mod);
         for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
         return 1;
     }
 
     // Assuming all GPUs can use the same function handle from the same module.
     for (int i = 1; i < num_gpus; ++i) {
-        kernel_funcs[i] = kernel_funcs[0];
+        mmul_funcs[i] = mmul_funcs[0];
+        vadd_funcs[i] = vadd_funcs[0];
+        conv_funcs[i] = conv_funcs[0];
     }
 
+    std::uniform_int_distribution<int> dist_type(0, 2);
     for (int i = 0; i < total_tasks; ++i) {
         int N_for_task = available_Ns[dist_N_idx(rng_tasks)];
-        all_tasks.push_back({N_for_task});
+        TaskType t_type = static_cast<TaskType>(dist_type(rng_tasks));
+        all_tasks.push_back({N_for_task, t_type});
     }
 
     // ─────────────────────────────────────────────────────────────────────────
@@ -242,7 +316,7 @@ int main() {
 
     std::vector<std::thread> threads;
     for (int i = 0; i < num_gpus; ++i) { // Pass context and kernel function to each worker
-        threads.emplace_back(gpu_worker, i, std::ref(partitions[i]), streams_per_gpu, contexts[i], kernel_funcs[i], std::ref(global_host_matrix_pool.A), std::ref(global_host_matrix_pool.B));
+        threads.emplace_back(gpu_worker, i, std::ref(partitions[i]), streams_per_gpu, contexts[i], mmul_funcs[i], vadd_funcs[i], conv_funcs[i], std::ref(global_host_matrix_pool));
     }
 
     for (auto& t : threads) {
@@ -258,7 +332,9 @@ int main() {
     for (int i = 0; i < num_gpus; ++i) {
         cuCtxDestroy(contexts[i]);
     }
-    cuModuleUnload(module);
+    cuModuleUnload(mmul_mod);
+    cuModuleUnload(vadd_mod);
+    cuModuleUnload(conv_mod);
     }
 
     return 0;
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/vector_add.cu b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/vector_add.cu
new file mode 100644
index 0000000000..4bf501d5d1
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/vector_add.cu
@@ -0,0 +1,6 @@
+extern "C" __global__ void vectorAdd(const int* A, const int* B, int* C, int N) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx < N) {
+        C[idx] = A[idx] + B[idx];
+    }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index 5de67861b1..61b81e2bab 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -1,12 +1,19 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
+#include <caf/component-actors/all-component-actors.hpp>
 #include <iostream>
+#include <iomanip>
 #include <vector>
 #include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
 #include <random>
-#include <unordered_map>
+#include "caf/actor_registry.hpp"
 #include <deque>
-#include <unordered_set> // For create_matrix_pool_random
+#include <unordered_map>
+#include <unordered_set>
+//#include <caf/atoms.hpp>
 
 using namespace caf;
 using namespace std::chrono_literals;
@@ -14,23 +21,34 @@ using namespace std::chrono_literals;
 // ─────────────────────────────────────────────────────────────────────────────
 // Atoms
 // ─────────────────────────────────────────────────────────────────────────────
-CAF_BEGIN_TYPE_ID_BLOCK(dynamic_work_stealing, caf::id_block::cuda::end)
-    CAF_ADD_ATOM(dynamic_work_stealing, get_work_atom)
-    CAF_ADD_ATOM(dynamic_work_stealing, task_done_atom)
-    CAF_ADD_ATOM(dynamic_work_stealing, release_memory_atom)
-    CAF_ADD_ATOM(dynamic_work_stealing, request_work_atom)
-    CAF_ADD_ATOM(dynamic_work_stealing, worker_done_atom)
-    CAF_ADD_ATOM(dynamic_work_stealing, refill_buffer_atom)
-CAF_END_TYPE_ID_BLOCK(dynamic_work_stealing)
+CAF_BEGIN_TYPE_ID_BLOCK(mmul_benchmark, caf::id_block::cuda::end)
+    CAF_ADD_ATOM(mmul_benchmark, get_work_atom)
+    CAF_ADD_ATOM(mmul_benchmark, task_done_atom)
+    CAF_ADD_ATOM(mmul_benchmark, release_memory_atom)
+    CAF_ADD_ATOM(mmul_benchmark, request_work_atom)
+    CAF_ADD_ATOM(mmul_benchmark, worker_done_atom)
+    CAF_ADD_ATOM(mmul_benchmark, refill_buffer_atom)
+CAF_END_TYPE_ID_BLOCK(mmul_benchmark)
+
+enum TaskType { MMUL = 0, VADD = 1, CONV = 2 };
+
+struct Task {
+    int N;
+    TaskType type;
+};
 
 // Command runners for GPU operations
-using mmul_kernel_t = caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>, out<int>, in<int>>;
-mmul_kernel_t mmul_kernel;
 caf::cuda::command_runner<> mmul_command;
+using kernel_runner_t = caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>, out<int>, in<int>>;
+kernel_runner_t kernel_runner;
 
 struct MatrixPool {
     std::unordered_map<int, std::vector<int>> A;
     std::unordered_map<int, std::vector<int>> B;
+    std::unordered_map<int, std::vector<int>> vec_A;
+    std::unordered_map<int, std::vector<int>> vec_B;
+    std::unordered_map<int, std::vector<int>> conv_A;
+    std::unordered_map<int, std::vector<int>> conv_K;
 };
 
 MatrixPool create_matrix_pool_random(
@@ -40,62 +58,66 @@ MatrixPool create_matrix_pool_random(
     unsigned int seed
 ) {
     MatrixPool pool;
-    std::mt19937 rng(seed);
-    // Ensure N values are multiples of 32 for typical matrix sizes
-    std::uniform_int_distribution<int> dist_N(min_N / 32, max_N / 32);
 
-    std::unordered_set<int> used_Ns;
-
-    while (used_Ns.size() < static_cast<size_t>(num_sizes)) {
-        int N_val = dist_N(rng) * 32;
-        if (N_val == 0) continue; // Avoid N=0
-        if (used_Ns.insert(N_val).second) {
-            pool.A[N_val] = std::vector<int>(N_val * N_val, 1);
-            pool.B[N_val] = std::vector<int>(N_val * N_val, 1);
+    std::mt19937 rng(seed);
+    std::uniform_int_distribution<int> dist(min_N, max_N);
+
+    std::unordered_set<int> used;
+
+    while (used.size() < static_cast<size_t>(num_sizes)) {
+        int N = dist(rng);
+        if (used.insert(N).second) {
+            pool.A[N] = std::vector<int>(N * N, 1);
+            pool.B[N] = std::vector<int>(N * N, 1);
+            pool.vec_A[N] = std::vector<int>(N, 1);
+            pool.vec_B[N] = std::vector<int>(N, 1);
+            pool.conv_A[N] = std::vector<int>(N, 1);
+            pool.conv_K[N] = std::vector<int>(5, 1);
         }
     }
+
     return pool;
 }
 
 // ---------------------------- GLOBAL TASK POOL ----------------------------
 // The central source of truth for work. Implements a pull-based model.
 struct task_pool_state {
-    std::vector<int> tasks;
+    std::vector<Task> tasks;
     size_t next_task_idx = 0;
 };
 
-caf::behavior global_task_pool(caf::stateful_actor<task_pool_state>* self, std::vector<int> tasks) {
+caf::behavior global_task_pool(caf::stateful_actor<task_pool_state>* self, std::vector<Task> tasks) {
     self->state().tasks = std::move(tasks);
     return {
-        [=](get_work_atom, size_t batch_size) -> result<std::vector<int>> {
+        [=](get_work_atom, size_t batch_size) -> result<std::vector<Task>> {
             auto& st = self->state();
             if (st.next_task_idx >= st.tasks.size())
                 return sec::end_of_stream;
             size_t count = std::min(batch_size, st.tasks.size() - st.next_task_idx);
-            std::vector<int> batch(st.tasks.begin() + st.next_task_idx, 
-                                   st.tasks.begin() + st.next_task_idx + count);
+            std::vector<Task> batch(st.tasks.begin() + st.next_task_idx, 
+                                    st.tasks.begin() + st.next_task_idx + count);
             st.next_task_idx += count;
             return batch;
         }
     };
 }
 
-// ---------------------------- DEVICE BROKER ----------------------------
+// ---------------------------- DEVICE/GPU ACTOR ----------------------------
 // Manages memory for a specific GPU and steals (pulls) work from the Global Pool.
 struct device_actor_state {
     MatrixPool pool;
     caf::actor global_pool;
-    std::deque<int> local_tasks; // Local buffer to keep GPU busy
-    size_t total_mem = 0;
-    size_t current_mem = 0;
-    int device_id = -1;
+    std::deque<Task> local_tasks; // Local buffer to keep GPU busy
+    size_t total_device_memory_bytes = 0;
+    size_t current_allocated_memory_bytes = 0;
     int active_workers = 0;
+    int device_id = -1;
     size_t batch_size = 0;
     size_t low_water_mark = 0;
     bool fetching = false;
 };
 
-caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self, // Changed signature
+caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
                                MatrixPool pool, caf::actor global_pool, int num_workers, int dev_id, int max_in_flight) {
     self->state().pool = std::move(pool);
     self->state().global_pool = global_pool;
@@ -106,9 +128,11 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self, //
     self->state().low_water_mark = static_cast<size_t>(num_workers * max_in_flight);
     self->state().batch_size = self->state().low_water_mark * 2;
 
-    auto dev_obj = caf::cuda::manager::get().find_device(dev_id);
-    if (dev_obj)
-        self->state().total_mem = dev_obj->total_memory_bytes();
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+    caf::cuda::device_ptr dev_obj = mgr.find_device(dev_id);
+    if (dev_obj) {
+        self->state().total_device_memory_bytes = dev_obj->total_memory_bytes();
+    }
 
     // Helper to refill the local task buffer from the global pool
     auto refill = [=]() {
@@ -117,11 +141,11 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self, //
             return;
 
         st.fetching = true;
-        self->mail(get_work_atom_v, (size_t)st.batch_size).request(st.global_pool, infinite).then(
-            [=](std::vector<int>& batch) {
+        self->mail(get_work_atom_v, st.batch_size).request(st.global_pool, infinite).then(
+            [=](std::vector<Task>& batch) {
                 auto& st_inner = self->state();
-                for (int N : batch)
-                    st_inner.local_tasks.push_back(N);
+                for (auto& task : batch)
+                    st_inner.local_tasks.push_back(task);
                 st_inner.fetching = false;
                 if (st_inner.local_tasks.size() < st_inner.low_water_mark)
                     self->mail(refill_buffer_atom_v).send(self);
@@ -136,215 +160,302 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self, //
         [=](refill_buffer_atom) {
             refill();
         },
-        [=](get_work_atom) -> caf::result<int, in<int>, in<int>> {
+        [=](get_work_atom) -> caf::result<int, int, in<int>, in<int>> {
             auto& st = self->state();
 
             // If we have tasks locally, satisfy the request immediately
             if (!st.local_tasks.empty()) {
-                int N = st.local_tasks.front();
-                size_t needed = (size_t)N * N * sizeof(int) * 3;
-                if (st.current_mem + needed > st.total_mem)
-                    return make_error(sec::runtime_error, "Out of GPU Memory");
+                Task t = st.local_tasks.front();
+                int N = t.N;
+                size_t memory_needed = (t.type == MMUL) ? (size_t)N * N * sizeof(int) * 3 : (size_t)N * sizeof(int) * 3; // Approx
+                if (st.current_allocated_memory_bytes + memory_needed > st.total_device_memory_bytes)
+                    return make_error(sec::runtime_error, "Device Actor: Not enough memory");
                 
                 st.local_tasks.pop_front();
-                st.current_mem += needed;
+                st.current_allocated_memory_bytes += memory_needed;
                 
-                // Proactively steal more work if the buffer is getting low
                 if (st.local_tasks.size() < st.low_water_mark)
                     refill();
 
-                return {N, caf::cuda::create_in_arg(st.pool.A[N]), 
-                           caf::cuda::create_in_arg(st.pool.B[N])};
+                auto& h_a = (t.type == MMUL) ? st.pool.A[N] : (t.type == VADD ? st.pool.vec_A[N] : st.pool.conv_A[N]);
+                auto& h_b = (t.type == MMUL) ? st.pool.B[N] : (t.type == VADD ? st.pool.vec_B[N] : st.pool.conv_K[N]);
+
+                return {N, static_cast<int>(t.type), caf::cuda::create_in_arg(h_a), 
+                           caf::cuda::create_in_arg(h_b)};
             }
 
             // Buffer empty: must fetch from global pool reactively
-            auto promise = self->make_response_promise<int, in<int>, in<int>>();
-            self->mail(get_work_atom_v, (size_t)st.batch_size).request(st.global_pool, infinite).then(
-                [=](std::vector<int>& batch) mutable {
+            auto promise = self->make_response_promise<int, int, in<int>, in<int>>();
+            self->mail(get_work_atom_v, st.batch_size).request(st.global_pool, infinite).then(
+                [=](std::vector<Task>& batch) mutable {
                     auto& st_inner = self->state();
-                    int N = batch.front();
+                    Task t = batch.front();
+                    int N = t.N;
                     for(size_t i = 1; i < batch.size(); ++i) st_inner.local_tasks.push_back(batch[i]);
                     
-                    st_inner.current_mem += (size_t)N * N * sizeof(int) * 3;
-                    promise.deliver(N, caf::cuda::create_in_arg(st_inner.pool.A[N]), 
-                                       caf::cuda::create_in_arg(st_inner.pool.B[N]));
+                    size_t needed = (t.type == MMUL) ? (size_t)N * N * sizeof(int) * 3 : (size_t)N * sizeof(int) * 3; // Approx
+                    st_inner.current_allocated_memory_bytes += needed;
+
+                    auto& h_a = (t.type == MMUL) ? st_inner.pool.A[N] : (t.type == VADD ? st_inner.pool.vec_A[N] : st_inner.pool.conv_A[N]);
+                    auto& h_b = (t.type == MMUL) ? st_inner.pool.B[N] : (t.type == VADD ? st_inner.pool.vec_B[N] : st_inner.pool.conv_K[N]);
+                    promise.deliver(N, static_cast<int>(t.type), caf::cuda::create_in_arg(h_a), 
+                                       caf::cuda::create_in_arg(h_b));
                 },
                 [=](error& err) mutable { promise.deliver(err); }
             );
             return promise;
         },
-        [=](release_memory_atom, int N) {
-            self->state().current_mem -= (size_t)N * N * sizeof(int) * 3;
+        [=](release_memory_atom, int N_completed, int type) {
+            auto& st = self->state();
+            TaskType t_type = static_cast<TaskType>(type);
+            size_t memory_released = (t_type == MMUL) ? (size_t)N_completed * N_completed * sizeof(int) * 3 : (size_t)N_completed * sizeof(int) * 3; // Approx
+            st.current_allocated_memory_bytes -= memory_released;
             refill(); // Try to get more work now that memory is free
         },
         [=](worker_done_atom) {
-            if (--self->state().active_workers <= 0)
+            auto& st = self->state();
+            if (--st.active_workers <= 0) {
                 self->quit();
+            }
         }
     };
 }
 
-// ---------------------------- WORKER ----------------------------
+// ---------------------------- WORKER ACTOR ----------------------------
+// Manages 1 stream and pulls work from the Device Actor.
 struct worker_state {
+    int device_id;
+    int stream_id;
+    caf::cuda::program_ptr mmul_prog;
+    caf::cuda::program_ptr vadd_prog;
+    caf::cuda::program_ptr conv_prog;
     caf::actor device_actor;
     caf::actor supervisor;
-    caf::cuda::program_ptr program;
-    int dev_id;
-    int stream_id;
-    int in_flight = 0;
-    int max_in_flight = 2; // Keep the pipeline full
+    int max_in_flight_tasks;
+    int in_flight_tasks_count = 0;
     bool draining = false;
 };
 
-caf::behavior mmul_worker(caf::stateful_actor<worker_state>* self, 
-                          caf::actor supervisor, caf::actor device_actor, 
-                          caf::cuda::program_ptr prog, int dev, int stream, int max_in_flight) {
+caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
+                              caf::actor supervisor, caf::actor device_actor, caf::cuda::program_ptr mmul_p, caf::cuda::program_ptr vadd_p, caf::cuda::program_ptr conv_p,
+                              int dev_id, int stream_id, int max_in_flight_tasks) {
     self->state().supervisor = supervisor;
     self->state().device_actor = device_actor;
-    self->state().program = prog;
-    self->state().dev_id = dev;
-    self->state().stream_id = stream;
-    self->state().max_in_flight = max_in_flight;
+    self->state().mmul_prog = mmul_p;
+    self->state().vadd_prog = vadd_p;
+    self->state().conv_prog = conv_p;
+    self->state().device_id = dev_id;
+    self->state().stream_id = stream_id;
+    self->state().max_in_flight_tasks = max_in_flight_tasks;
 
-    for (int i = 0; i < self->state().max_in_flight; ++i)
+    // Trigger initial work requests up to max_in_flight_tasks
+    for (int i = 0; i < max_in_flight_tasks; ++i) {
         self->mail(request_work_atom_v).send(self);
+    }
 
     return {
         [=](request_work_atom) {
             auto& st = self->state();
-            if (st.in_flight >= st.max_in_flight || st.draining) return;
-            st.in_flight++;
+            if (st.in_flight_tasks_count >= st.max_in_flight_tasks || st.draining) {
+                return; // Already at max capacity, don't request more yet
+            }
+
+            st.in_flight_tasks_count++; // Mark as pending immediately
             self->mail(get_work_atom_v).request(st.device_actor, infinite).then(
-                [=](int N, in<int> A, in<int> B) mutable {
-                    auto& st_inner = self->state();
-                    auto a1 = mmul_command.transfer_memory(st_inner.dev_id, st_inner.stream_id, std::move(A));
-                    auto a2 = mmul_command.transfer_memory(st_inner.dev_id, st_inner.stream_id, std::move(B));
-                    caf::cuda::nd_range dims((N + 31) / 32, (N + 31) / 32, 1, 32, 32, 1);
-                    auto res = mmul_kernel.run_async(st_inner.program, dims, st_inner.stream_id, 0, st_inner.dev_id,
-                                                     a1, a2, caf::cuda::create_out_arg<int>(N * N),
-                                                     caf::cuda::create_in_arg<int>(N));
-                    auto self_ptr = caf::actor_cast<caf::actor>(self);
-                    mmul_command.copy_to_host_async(std::get<2>(res), [self_ptr, N](std::vector<int>&&) {
-                        caf::anon_mail(task_done_atom_v, N).send(self_ptr);
+                [=](int N, int type, in<int> matrixA, in<int> matrixB) {
+                    TaskType t_type = static_cast<TaskType>(type);
+                    // GPU Pipeline: Transfer -> Kernel -> Copyback
+                    auto arg1 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixA));
+                    auto arg2 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixB));
+
+                    caf::cuda::nd_range dims;
+                    caf::cuda::program_ptr prog;
+                    int out_size;
+                    if (t_type == MMUL) {
+                        dims = caf::cuda::nd_range((N+31)/32, (N+31)/32, 1, 32, 32, 1);
+                        prog = st.mmul_prog;
+                        out_size = N * N;
+                    } else if (t_type == VADD) {
+                        dims = caf::cuda::nd_range((N+255)/256, 1, 1, 256, 1, 1);
+                        prog = st.vadd_prog;
+                        out_size = N;
+                    } else {
+                        dims = caf::cuda::nd_range((N+255)/256, 1, 1, 256, 1, 1);
+                        prog = st.conv_prog;
+                        out_size = N;
+                    }
+
+                    auto result = kernel_runner.run_async(prog, dims, st.stream_id, 0, st.device_id,
+                                                         arg1, arg2, caf::cuda::create_out_arg<int>(out_size), caf::cuda::create_in_arg<int>(N));
+
+                    auto bufferC = std::get<2>(result);
+                    auto self_hdl = caf::actor_cast<caf::actor>(self);
+
+                    mmul_command.copy_to_host_async(bufferC, [self_hdl, N_task = N, type](std::vector<int>&&) {
+                        caf::anon_mail(task_done_atom_v, N_task, type).send(self_hdl);
                     });
                 },
-                [=](error& err) mutable {
-                    auto& st_inner = self->state();
-                    st_inner.in_flight--;
-                    if (err == sec::end_of_stream) {
-                        st_inner.draining = true;
-                        if (st_inner.in_flight == 0) {
-                            self->mail(worker_done_atom_v).send(st_inner.device_actor);
+                [=](error& err) {
+                    auto& st = self->state();
+                    st.in_flight_tasks_count--; // Revert pending status on failure
+                    if (err == sec::runtime_error) {
+                        // Not enough memory, retry after a delay
+                        self->println("Worker {}: Not enough memory, retrying for work...", st.stream_id);
+                        self->delayed_anon_send(self, 100ms, request_work_atom_v);
+                    } else if (err == sec::end_of_stream) {
+                        st.draining = true; // Mark as draining, let in-flight finish
+                        if (st.in_flight_tasks_count == 0) {
+                            self->mail(worker_done_atom_v).send(st.device_actor);
+                            mmul_command.release_stream_for_actor(st.stream_id);
                             self->quit();
                         }
-                    } else {
-                        self->delayed_anon_send(self, 100ms, request_work_atom_v);
                     }
                 }
             );
         },
-        [=](task_done_atom, int N) {
-            self->state().in_flight--;
-            self->mail(1).send(self->state().supervisor);
-            self->mail(release_memory_atom_v, N).send(self->state().device_actor);
+        [=](task_done_atom, int N_completed, int type) {
+            auto& st = self->state();
+            st.in_flight_tasks_count--; // Decrement count
+            self->mail(1).send(st.supervisor); // Notify supervisor
+            self->mail(release_memory_atom_v, N_completed, type).send(st.device_actor); // Release memory
             
-            if (self->state().draining && self->state().in_flight == 0) {
-                self->mail(worker_done_atom_v).send(self->state().device_actor);
+            if (st.draining && st.in_flight_tasks_count == 0) {
+                self->mail(worker_done_atom_v).send(st.device_actor);
+                mmul_command.release_stream_for_actor(st.stream_id);
                 self->quit();
-            } else if (!self->state().draining) {
-                self->mail(request_work_atom_v).send(self);
+            } else if (!st.draining) {
+                self->mail(request_work_atom_v).send(self); // Request next task if capacity allows
             }
         }
     };
 }
 
-// ---------------------------- SUPERVISOR ----------------------------
-struct supervisor_state {
-    int total;
-    int done = 0;
-    std::chrono::steady_clock::time_point start;
+// ---------------------------- SUPERVISOR ACTOR ----------------------------
+struct supervisor_actor_state {
+    int total_tasks;
+    int completed = 0;
+    std::chrono::steady_clock::time_point start_time;
 };
 
-caf::behavior supervisor_actor(caf::stateful_actor<supervisor_state>* self, 
-                               int total,
-                               std::vector<int> Ns_for_tasks, // Renamed for clarity
-                               MatrixPool host_matrix_pool, // Pass the pre-generated pool
-                               int workers_per_gpu,
-                               int max_in_flight) {
-    self->state().total = total;
-    self->state().start = std::chrono::steady_clock::now();
-    auto pool = self->spawn(global_task_pool, std::move(Ns_for_tasks));
-    
-    // The MatrixPool is now passed as an argument, no need to create it here
-    // MatrixPool m_pool;
-    // for (int n : {256, 2048}) {
-    //     m_pool.A[n] = std::vector<int>(n * n, 1);
-    //     m_pool.B[n] = std::vector<int>(n * n, 1);
-    // }
-
-    auto& mgr = caf::cuda::manager::get();
-    auto prog = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    for (int i = 0; i < mgr.get_num_devices(); ++i) {
-        // Pass the host_matrix_pool to the gpu_device_actor
-        auto broker = self->spawn(gpu_device_actor, host_matrix_pool, pool, workers_per_gpu, i, max_in_flight);
+caf::behavior supervisor_actor_fun(
+    caf::stateful_actor<supervisor_actor_state>* self,
+    int total_tasks,
+    int workers_per_gpu,
+    int max_in_flight_tasks_per_worker,
+    MatrixPool pool,
+    std::vector<Task> tasks
+    ) {
+    self->state().total_tasks = total_tasks;
+    self->state().start_time = std::chrono::steady_clock::now();
+
+    auto pool_actor = self->spawn(global_task_pool, std::move(tasks));
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+    int num_gpus = mgr.get_num_devices();
+    auto mmul_p = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+    auto vadd_p = mgr.create_program_from_cubin("../vector_add.cubin", "vectorAdd");
+    auto conv_p = mgr.create_program_from_cubin("../conv1d.cubin", "conv1d");
+
+    for (int i = 0; i < num_gpus; ++i) {
+        auto broker = self->spawn(gpu_device_actor, pool, pool_actor, workers_per_gpu, i, max_in_flight_tasks_per_worker);
+        
         for (int j = 0; j < workers_per_gpu; ++j)
-            self->spawn(mmul_worker, self, broker, prog, i, (i * 100) + j, max_in_flight);
+            self->spawn(mmul_worker_fun, self, broker, mmul_p, vadd_p, conv_p, i, (i * 1000) + j, max_in_flight_tasks_per_worker);
     }
 
     return {
-        [=](int count) {
-            self->state().done += count;
-            if (self->state().done >= self->state().total) {
-                auto elapsed = std::chrono::steady_clock::now() - self->state().start;
-                std::cout << "Dynamic Work-Stealing Complete.\n"
-                          << "Task Count: " << self->state().total << " | Makespan: " << std::chrono::duration<double>(elapsed).count() << "s\n";
+        [=](int done) {
+            self->state().completed += done;
+            if (self->state().completed >= self->state().total_tasks) {
+                auto end_time = std::chrono::steady_clock::now();
+                std::chrono::duration<double> total_time = end_time - self->state().start_time;
+
+                std::cout << "\n===== BENCHMARK COMPLETE =====\n";
+                std::cout << "Tasks: " << self->state().total_tasks << "\n";
+                std::cout << "Runtime: " << total_time.count() << " s\n";
+                
+                caf::cuda::manager::shutdown();
                 self->quit();
             }
         }
     };
 }
 
-void caf_main(caf::actor_system& sys) {
-    int workers_per_gpu = 8;
-    int max_in_flight = 3;
-    std::vector<int> task_counts = {50000,100000};
-
-    // Define parameters for irregular workload
-    const int num_matrix_sizes = 60; // Number of distinct N values
-    const int min_N_val = 32;
-    const int max_N_val = 2048;
-    const unsigned int pool_seed = 42; // Fixed seed for deterministic pool generation
-
-    // Create the host-side matrix pool once
-    MatrixPool global_host_matrix_pool = create_matrix_pool_random(num_matrix_sizes, min_N_val, max_N_val, pool_seed);
-
-    // Extract available N values from the pool for task generation
-    std::vector<int> available_Ns;
-    for (const auto& pair : global_host_matrix_pool.A) {
-        available_Ns.push_back(pair.first);
-    }
-    if (available_Ns.empty()) {
-        std::cerr << "Error: No matrix sizes generated in the pool." << std::endl;
-        return;
-    }
+template <class Fn>
+double time_run(Fn&& fn) {
+  auto start = std::chrono::steady_clock::now();
+  fn();
+  auto end = std::chrono::steady_clock::now();
+  std::chrono::duration<double> elapsed = end - start;
+  return elapsed.count();
+}
 
-    for (int total_tasks : task_counts) {
-        caf::cuda::manager_config cfg(false);
-        caf::cuda::manager::init(sys, cfg);
-        std::cout << "=====================================\n";
-        std::cout << "Task count: " << total_tasks << "\n";
+void run_mmul_random_scaling_tests(caf::actor_system& sys,
+                                  caf::cuda::manager_config man_config) {
+
+    const int min_N = 32;
+    const int max_N = 2048;
+    const int num_sizes = 10;
 
-        std::vector<int> Ns_for_this_run;
-        std::mt19937 rng_tasks(42); // Fixed seed for task distribution
-        std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
-        for (int i = 0; i < total_tasks; ++i) {
-            Ns_for_this_run.push_back(available_Ns[dist_N_idx(rng_tasks)]);
+    const int workers_per_gpu = 8; // Admission control: only 16 concurrent tasks per GPU
+    const int max_in_flight_tasks_per_worker = 3; // Each worker keeps 2 tasks in flight
+
+    const std::vector<int> actor_counts = {
+	  1,30000,40000,50000
+    };
+
+    // Generate deterministic random pool once
+    MatrixPool pool = create_matrix_pool_random(
+        num_sizes,
+        min_N,
+        max_N,
+        42  // fixed seed
+    );
+
+    //scheduler
+    caf::cuda::manager_config scheduler_off(false);
+    for (int num_tasks_for_this_run : actor_counts) {
+	    // Initialize CUDA manager
+	    caf::cuda::manager::init(sys, scheduler_off);
+        std::cout << "=====================================\n";
+        std::cout << "Random Scaling | actors=" << num_tasks_for_this_run << "\n";
+
+        // Precompute all task Ns for this run
+        std::vector<int> sizes;
+        for (const auto& [N, _] : pool.A) sizes.push_back(N);
+
+        std::vector<Task> tasks_for_this_run;
+        tasks_for_this_run.reserve(num_tasks_for_this_run);
+
+        std::mt19937 rng(42);
+        std::uniform_int_distribution<size_t> dist_size(0, sizes.size() - 1);
+        std::uniform_int_distribution<int> dist_type(0, 2);
+        for (int i = 0; i < num_tasks_for_this_run; ++i) {
+            int N = sizes[dist_size(rng)];
+            TaskType type = static_cast<TaskType>(dist_type(rng));
+            tasks_for_this_run.push_back({N, type});
         }
-        sys.spawn(supervisor_actor, total_tasks, std::move(Ns_for_this_run), global_host_matrix_pool, workers_per_gpu, max_in_flight);
-        sys.await_all_actors_done();
+
+        // Execute the supervisor which manages the asynchronous workload
+        double elapsed = time_run([&]() {
+
+            auto sup = sys.spawn(
+                supervisor_actor_fun,
+                (int)Ns_for_this_run.size(), // total_tasks
+                workers_per_gpu,
+                max_in_flight_tasks_per_worker,
+                pool,
+                tasks_for_this_run
+            );
+
+	      sys.await_all_actors_done();
+	    });
+
         caf::cuda::manager::shutdown();
     }
 }
-
-CAF_MAIN(id_block::dynamic_work_stealing)
+void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager_config man_config(false);
+  run_mmul_random_scaling_tests(sys, man_config);
+}
+CAF_MAIN(id_block::mmul_benchmark)

From 76ffcc83f3f1d7a808512b5dbb7de114eb4d0b6d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 14 May 2026 09:48:18 -0600
Subject: [PATCH 0625/1000] Fixed compiler errors.

---
 .../work-stealing.cpp                         | 39 +++++++++++++++----
 1 file changed, 32 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index 61b81e2bab..4724b46216 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -18,6 +18,34 @@
 using namespace caf;
 using namespace std::chrono_literals;
 
+
+enum TaskType { MMUL = 0, VADD = 1, CONV = 2 };
+
+struct Task {
+    int N;
+    TaskType type;
+};
+
+// Inspect function for TaskType enum to enable CAF serialization
+template <class Inspector>
+bool inspect(Inspector& f, TaskType& x) {
+  auto val = static_cast<int>(x);
+  if (f.apply(val)) {
+    if constexpr (Inspector::is_loading)
+      x = static_cast<TaskType>(val);
+    return true;
+  }
+  return false;
+}
+
+// Inspect function for Task struct to enable CAF serialization
+template <class Inspector>
+bool inspect(Inspector& f, Task& x) {
+  return f.object(x).fields(f.field("N", x.N), f.field("type", x.type));
+};
+
+
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Atoms
 // ─────────────────────────────────────────────────────────────────────────────
@@ -27,15 +55,12 @@ CAF_BEGIN_TYPE_ID_BLOCK(mmul_benchmark, caf::id_block::cuda::end)
     CAF_ADD_ATOM(mmul_benchmark, release_memory_atom)
     CAF_ADD_ATOM(mmul_benchmark, request_work_atom)
     CAF_ADD_ATOM(mmul_benchmark, worker_done_atom)
+    CAF_ADD_TYPE_ID(mmul_benchmark, (TaskType))
+    CAF_ADD_TYPE_ID(mmul_benchmark, (Task))
+    CAF_ADD_TYPE_ID(mmul_benchmark, (std::vector<Task>))
     CAF_ADD_ATOM(mmul_benchmark, refill_buffer_atom)
 CAF_END_TYPE_ID_BLOCK(mmul_benchmark)
 
-enum TaskType { MMUL = 0, VADD = 1, CONV = 2 };
-
-struct Task {
-    int N;
-    TaskType type;
-};
 
 // Command runners for GPU operations
 caf::cuda::command_runner<> mmul_command;
@@ -441,7 +466,7 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 
             auto sup = sys.spawn(
                 supervisor_actor_fun,
-                (int)Ns_for_this_run.size(), // total_tasks
+                (int)tasks_for_this_run.size(), // total_tasks
                 workers_per_gpu,
                 max_in_flight_tasks_per_worker,
                 pool,

From ddd1fa0af4e39f9f6bf6d947414ab8064e367af9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 14 May 2026 09:52:44 -0600
Subject: [PATCH 0626/1000] updated task sizes

---
 .../mmul-randonom-batch-benchmark/work-stealing.cpp           | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index 4724b46216..bbf3c9745a 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -420,13 +420,13 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 
     const int min_N = 32;
     const int max_N = 2048;
-    const int num_sizes = 10;
+    const int num_sizes = 60;
 
     const int workers_per_gpu = 8; // Admission control: only 16 concurrent tasks per GPU
     const int max_in_flight_tasks_per_worker = 3; // Each worker keeps 2 tasks in flight
 
     const std::vector<int> actor_counts = {
-	  1,30000,40000,50000
+	  50000,100000
     };
 
     // Generate deterministic random pool once

From 10d85a89cd476edd114502dc3903d4ee02462b4b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 14 May 2026 10:42:39 -0600
Subject: [PATCH 0627/1000] Saving major changes.

---
 .../mmul-randonom-batch-benchmark/cuda-baseline.cpp         | 1 +
 .../mmul-randonom-batch-benchmark/work-stealing.cpp         | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index 17bd75e52a..3060955cca 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -172,6 +172,7 @@ int main() {
     for (const auto& pair : global_host_matrix_pool.A) {
         available_Ns.push_back(pair.first);
     }
+    std::sort(available_Ns.begin(), available_Ns.end());
     if (available_Ns.empty()) {
         std::cerr << "Error: No matrix sizes generated in the pool." << std::endl;
         return 1;
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index bbf3c9745a..485fb65050 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -85,12 +85,13 @@ MatrixPool create_matrix_pool_random(
     MatrixPool pool;
 
     std::mt19937 rng(seed);
-    std::uniform_int_distribution<int> dist(min_N, max_N);
+    std::uniform_int_distribution<int> dist(min_N / 32, max_N / 32);
 
     std::unordered_set<int> used;
 
     while (used.size() < static_cast<size_t>(num_sizes)) {
-        int N = dist(rng);
+        int N = dist(rng) * 32;
+        if (N == 0) continue;
         if (used.insert(N).second) {
             pool.A[N] = std::vector<int>(N * N, 1);
             pool.B[N] = std::vector<int>(N * N, 1);
@@ -448,6 +449,7 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
         // Precompute all task Ns for this run
         std::vector<int> sizes;
         for (const auto& [N, _] : pool.A) sizes.push_back(N);
+        std::sort(sizes.begin(), sizes.end());
 
         std::vector<Task> tasks_for_this_run;
         tasks_for_this_run.reserve(num_tasks_for_this_run);

From 5eb97de0f3cb0c01ecf740ac6cb70bd5146712c7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 14 May 2026 11:50:53 -0600
Subject: [PATCH 0628/1000] added a global output buffer to potientally reduce
 allocation bottlenecks

---
 .../mmul-randonom-batch-benchmark/cuda-baseline.cpp  | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index 3060955cca..0d1c6fe1af 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -27,7 +27,8 @@ struct MatrixPool {
 };
 
 // Per-GPU execution logic
-void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_gpu, CUcontext ctx, CUfunction mmul_func, CUfunction vadd_func, CUfunction conv_func, const MatrixPool& pool) {
+void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_gpu, CUcontext ctx, CUfunction mmul_func,
+     CUfunction vadd_func, CUfunction conv_func, const MatrixPool& pool, int* shared_dtoh_buffer) {
     // Set the CUDA context for this thread
     CUresult err = cuCtxSetCurrent(ctx);
     if (err != CUDA_SUCCESS) {
@@ -93,8 +94,7 @@ void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_g
         
         // Simulating the result retrieval (Copy back)
         size_t res_count = (type == MMUL) ? (size_t)N * N : (size_t)N;
-        std::vector<int> h_res(res_count);
-        cuMemcpyDtoHAsync(h_res.data(), d_c, bytes_out, stream);
+        cuMemcpyDtoHAsync(shared_dtoh_buffer, d_c, bytes_out, stream);
 
         // Free GPU memory for this task
         cuMemFreeAsync(d_a, stream);
@@ -313,11 +313,15 @@ int main() {
         partitions[i % num_gpus].push_back(all_tasks[i]);
     }
 
+    // Preallocate a single large host buffer for DTOH transfers to save RAM and keep things fair.
+    std::vector<int> shared_dtoh_buffer((size_t)max_N_val * max_N_val);
+
     auto start = std::chrono::steady_clock::now();
 
     std::vector<std::thread> threads;
     for (int i = 0; i < num_gpus; ++i) { // Pass context and kernel function to each worker
-        threads.emplace_back(gpu_worker, i, std::ref(partitions[i]), streams_per_gpu, contexts[i], mmul_funcs[i], vadd_funcs[i], conv_funcs[i], std::ref(global_host_matrix_pool));
+        threads.emplace_back(gpu_worker, i, std::ref(partitions[i]), streams_per_gpu, contexts[i], mmul_funcs[i], vadd_funcs[i], conv_funcs[i],
+         std::ref(global_host_matrix_pool), shared_dtoh_buffer.data());
     }
 
     for (auto& t : threads) {

From db6c7359355b9c4e6c5299c26353477e9d5442d6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 14 May 2026 12:04:40 -0600
Subject: [PATCH 0629/1000] Addded global DTOH buffer to reduce potienal CPU
 bottlenecks.

---
 .../work-stealing.cpp                         | 24 ++++++++++++-------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index 485fb65050..f630a7814b 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -259,12 +259,13 @@ struct worker_state {
     caf::actor supervisor;
     int max_in_flight_tasks;
     int in_flight_tasks_count = 0;
+    int* dtoh_buffer_ptr = nullptr;
     bool draining = false;
 };
 
 caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
                               caf::actor supervisor, caf::actor device_actor, caf::cuda::program_ptr mmul_p, caf::cuda::program_ptr vadd_p, caf::cuda::program_ptr conv_p,
-                              int dev_id, int stream_id, int max_in_flight_tasks) {
+                              int dev_id, int stream_id, int max_in_flight_tasks, int* d_buf) {
     self->state().supervisor = supervisor;
     self->state().device_actor = device_actor;
     self->state().mmul_prog = mmul_p;
@@ -272,6 +273,7 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
     self->state().conv_prog = conv_p;
     self->state().device_id = dev_id;
     self->state().stream_id = stream_id;
+    self->state().dtoh_buffer_ptr = d_buf;
     self->state().max_in_flight_tasks = max_in_flight_tasks;
 
     // Trigger initial work requests up to max_in_flight_tasks
@@ -317,7 +319,7 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
                     auto bufferC = std::get<2>(result);
                     auto self_hdl = caf::actor_cast<caf::actor>(self);
 
-                    mmul_command.copy_to_host_async(bufferC, [self_hdl, N_task = N, type](std::vector<int>&&) {
+                    mmul_command.copy_to_host_async(bufferC, st.dtoh_buffer_ptr, (size_t)out_size, [self_hdl, N_task = N, type](int*, size_t) {
                         caf::anon_mail(task_done_atom_v, N_task, type).send(self_hdl);
                     });
                 },
@@ -369,7 +371,8 @@ caf::behavior supervisor_actor_fun(
     int workers_per_gpu,
     int max_in_flight_tasks_per_worker,
     MatrixPool pool,
-    std::vector<Task> tasks
+    std::vector<Task> tasks,
+    int* shared_dtoh_ptr
     ) {
     self->state().total_tasks = total_tasks;
     self->state().start_time = std::chrono::steady_clock::now();
@@ -385,8 +388,9 @@ caf::behavior supervisor_actor_fun(
     for (int i = 0; i < num_gpus; ++i) {
         auto broker = self->spawn(gpu_device_actor, pool, pool_actor, workers_per_gpu, i, max_in_flight_tasks_per_worker);
         
-        for (int j = 0; j < workers_per_gpu; ++j)
-            self->spawn(mmul_worker_fun, self, broker, mmul_p, vadd_p, conv_p, i, (i * 1000) + j, max_in_flight_tasks_per_worker);
+        for (int j = 0; j < workers_per_gpu; ++j) {
+            self->spawn(mmul_worker_fun, self, broker, mmul_p, vadd_p, conv_p, i, (i * 1000) + j, max_in_flight_tasks_per_worker, shared_dtoh_ptr);
+        }
     }
 
     return {
@@ -423,8 +427,8 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     const int max_N = 2048;
     const int num_sizes = 60;
 
-    const int workers_per_gpu = 8; // Admission control: only 16 concurrent tasks per GPU
-    const int max_in_flight_tasks_per_worker = 3; // Each worker keeps 2 tasks in flight
+    const int workers_per_gpu = 4; // Admission control: only 16 concurrent tasks per GPU
+    const int max_in_flight_tasks_per_worker = 5; // Each worker keeps 2 tasks in flight
 
     const std::vector<int> actor_counts = {
 	  50000,100000
@@ -463,6 +467,9 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
             tasks_for_this_run.push_back({N, type});
         }
 
+        // Preallocate a single large host buffer for DTOH transfers to save RAM and keep things fair.
+        std::vector<int> shared_dtoh_buffer((size_t)max_N * max_N);
+
         // Execute the supervisor which manages the asynchronous workload
         double elapsed = time_run([&]() {
 
@@ -472,7 +479,8 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
                 workers_per_gpu,
                 max_in_flight_tasks_per_worker,
                 pool,
-                tasks_for_this_run
+                tasks_for_this_run,
+                    shared_dtoh_buffer.data()
             );
 
 	      sys.await_all_actors_done();

From c871f5d9738171716da24e7d9111f2acb96f6141 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 14 May 2026 12:07:37 -0600
Subject: [PATCH 0630/1000] compiles more kernels

---
 .../compile_kernels.sh                           | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/compile_kernels.sh b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/compile_kernels.sh
index d504d43a43..c0bebf8833 100755
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/compile_kernels.sh
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/compile_kernels.sh
@@ -1,6 +1,14 @@
 #!/bin/bash
+set -e
+
+# Detect first GPU compute capability to ensure binary compatibility
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
 # Compile kernels to cubin for Driver API loading
-nvcc -cubin mmul.cu -o ../mmul.cubin
-nvcc -cubin vector_add.cu -o ../vector_add.cubin
-nvcc -cubin conv1d.cu -o ../conv1d.cubin
-echo "Kernels compiled successfully."
\ No newline at end of file
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+nvcc -arch=$SM_ARCH -cubin vector_add.cu -o vector_add.cubin
+nvcc -arch=$SM_ARCH -cubin conv1d.cu -o conv1d.cubin
+
+echo "Kernels compiled successfully for $SM_ARCH."
\ No newline at end of file

From f5cdd3f2414ef28cd5297603d5c8083adcdff5b5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 09:52:48 -0600
Subject: [PATCH 0631/1000] updated actor facade to asynchronousl;y copy memory
 back to host  to improve performance

---
 libcaf_cuda/caf/cuda/actor_facade.hpp | 112 ++++++++++++++++++++------
 1 file changed, 86 insertions(+), 26 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/actor_facade.hpp b/libcaf_cuda/caf/cuda/actor_facade.hpp
index abe84638f4..fffcdddd48 100644
--- a/libcaf_cuda/caf/cuda/actor_facade.hpp
+++ b/libcaf_cuda/caf/cuda/actor_facade.hpp
@@ -10,8 +10,8 @@
 #include <caf/anon_mail.hpp>
 #include <caf/event_based_actor.hpp>
 #include <caf/response_promise.hpp>
-
 #include <cuda.h>
+#include <utility> // For std::index_sequence, std::integral_constant
 
 #include "caf/cuda/command.hpp"
 #include "caf/cuda/global.hpp"
@@ -19,7 +19,6 @@
 #include "caf/cuda/platform.hpp"
 #include "caf/cuda/program.hpp"
 
-/*
 
 namespace caf::cuda {
 
@@ -55,41 +54,103 @@ class actor_facade : public event_based_actor {
 
   caf::behavior make_behavior() override {
     return {
-      [this](atom_value stage, int device_num, Ts... args) {
-        enqueue_impl(device_num, stage, std::move(args)...);
+      [this](int device_num, int stream_id, std::vector<int> output_indices, Ts... args) {
+        enqueue_impl(device_num, stream_id, std::move(output_indices), std::forward<Ts>(args)...);
+      },
+      [this](int device_num, int stream_id, Ts... args) {
+        // Copy everything back if indices are omitted
+        enqueue_impl(device_num, stream_id, {}, std::forward<Ts>(args)...);
+      },
+      [this](int device_num, std::vector<int> output_indices, Ts... args) {
+        enqueue_impl(device_num, static_cast<int>(actor_id_), std::move(output_indices),
+                     std::forward<Ts>(args)...);
       },
-      [this](atom_value stage, Ts... args) {
-        enqueue_impl(-1, stage, std::move(args)...);
+      [this](std::vector<int> output_indices, Ts... args) {
+        enqueue_impl(-1, static_cast<int>(actor_id_), std::move(output_indices),
+                     std::forward<Ts>(args)...);
+      },
+      [this](int device_num, Ts... args) {
+        // Copy everything back if indices are omitted
+        enqueue_impl(device_num, static_cast<int>(actor_id_), {}, std::forward<Ts>(args)...);
       },
       [this](Ts... args) {
-        enqueue_impl(-1, kernel_atom_v, std::move(args)...);
+        // Copy everything back if indices are omitted
+        enqueue_impl(-1, static_cast<int>(actor_id_), {}, std::forward<Ts>(args)...);
       }
     };
   }
 
 private:
   template <class... Us>
-  void enqueue_impl(int device_num, atom_value stage, Us&&... xs) {
+  void enqueue_impl(int device_num, int stream_id, std::vector<int> output_indices, Us&&... xs) {
     command_runner<Ts...> runner;
-    // Pass actor_id_ as stream_id, and the received device_num
-    auto results = runner.run_async(program_, dims_, actor_id_, 0, device_num, std::forward<Us>(xs)...);
+    // Launch kernel asynchronously
+    auto results = runner.run_async(program_, dims_, stream_id, 0, 
+                                    device_num, std::forward<Us>(xs)...);
 
     auto sender = this->current_sender();
     auto r_atom = reply_atom_;
-    auto stream_id = actor_id_;
-
-    atom_value stage_done = gpu_done_atom_v;
-    if (stage == htod_atom_v) stage_done = htod_done_atom_v;
-    else if (stage == kernel_atom_v) stage_done = kernel_done_atom_v;
-    else if (stage == dtoh_atom_v) stage_done = dtoh_done_atom_v;
-
-    // Pass actor_id_ as stream_id, and the received device_num to add_callback
-    runner.add_callback(stream_id, device_num, [sender, r_atom, stage_done, results]() mutable {
-      auto msg = caf::make_message(stage_done, results);
-      if (r_atom != 0) {
-        caf::anon_mail(r_atom, std::move(msg)).send(sender);
-      } else {
-        caf::anon_mail(std::move(msg)).send(sender);
+
+    // If no indices specified, default to copying all arguments back
+    if (output_indices.empty()) {
+      for (int i = 0; i < static_cast<int>(sizeof...(Ts)); ++i) {
+        output_indices.push_back(i);
+      }
+    }
+
+    // Track if we actually queued any transfers to host
+    // Even if we don't, we will send a final 'done' signal via callback
+    // to ensure the stream is drained.
+
+    if (!output_indices.empty()) {
+        // Helper to dispatch a lambda based on a runtime index
+        auto switch_on_index = [&](int runtime_idx, auto&& func, auto... Is) {
+            ([&] {
+                if (runtime_idx == Is) {
+                    func(std::integral_constant<int, Is>{});
+                }
+            }(), ...);
+        };
+
+        for (int idx : output_indices) {
+            if (idx >= 0 && idx < sizeof...(Ts)) {
+                switch_on_index(idx, [&](auto current_idx_constant) {
+                    // Compile-time index extraction
+                    constexpr std::size_t Index = current_idx_constant; // Compile-time index
+                    using MemPtrType = std::tuple_element_t<Index, mem_tuple>;
+                    using ValueType = typename MemPtrType::element_type::value_type;
+
+                    MemPtrType mem_ptr = std::get<Index>(results);
+
+                    // Only copy if the mem_ptr is valid and has OUT or IN_OUT access
+                    if (mem_ptr && (mem_ptr->access() == OUT || mem_ptr->access() == IN_OUT)) {
+                        runner.copy_to_host_async(mem_ptr, 
+                          [sender, r_atom, Index](std::vector<ValueType>&& data) {
+                            // Send back the data along with the original index
+                            if (sender) { // Ensure sender is still valid
+                                if (r_atom != 0) {
+                                    caf::anon_mail(r_atom, Index, std::move(data)).send(sender);
+                                } else {
+                                    caf::anon_mail(Index, std::move(data)).send(sender);
+                                }
+                            }
+                        });
+                    }
+                }, std::make_index_sequence<sizeof...(Ts)>{});
+            } else {
+                this->println("Warning: Output index {} is out of bounds (0-{})", idx, sizeof...(Ts) - 1);
+            }
+        }
+    }
+
+    // Finally, alert the sender that everything is finished.
+    // This callback is queued on the stream after all kernel and transfer commands.
+    runner.add_callback(stream_id, device_num, [sender, r_atom]() mutable {
+      if (sender) {
+        if (r_atom != 0)
+          caf::anon_mail(r_atom, gpu_done_atom_v).send(sender);
+        else
+          caf::anon_mail(gpu_done_atom_v).send(sender);
       }
     });
   }
@@ -100,5 +161,4 @@ class actor_facade : public event_based_actor {
   atom_value   reply_atom_;
 };
 
-} // namespace caf::cuda
- */
\ No newline at end of file
+} // namespace caf::cuda
\ No newline at end of file

From e2b7d8d106f383e287bc4bb968049b3c6f397aa3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 10:17:21 -0600
Subject: [PATCH 0632/1000] fixed compiler errors.

---
 libcaf_cuda/caf/cuda/actor_facade.hpp | 108 +++++++++++---------------
 1 file changed, 45 insertions(+), 63 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/actor_facade.hpp b/libcaf_cuda/caf/cuda/actor_facade.hpp
index fffcdddd48..9f20f3ba85 100644
--- a/libcaf_cuda/caf/cuda/actor_facade.hpp
+++ b/libcaf_cuda/caf/cuda/actor_facade.hpp
@@ -9,15 +9,14 @@
 #include <caf/actor_system.hpp>
 #include <caf/anon_mail.hpp>
 #include <caf/event_based_actor.hpp>
-#include <caf/response_promise.hpp>
 #include <cuda.h>
-#include <utility> // For std::index_sequence, std::integral_constant
 
 #include "caf/cuda/command.hpp"
 #include "caf/cuda/global.hpp"
 #include "caf/cuda/helpers.hpp"
 #include "caf/cuda/platform.hpp"
 #include "caf/cuda/program.hpp"
+#include "caf/cuda/command_runner.hpp"
 
 
 namespace caf::cuda {
@@ -34,22 +33,22 @@ class actor_facade : public event_based_actor {
   static caf::actor create(caf::actor_system& sys,
                            program_ptr program,
                            nd_range dims,
-                           atom_value reply_atom = 0) {
+                           int reply_id = 0) {
     return caf::actor_cast<caf::actor>(
-      sys.spawn<actor_facade>(std::move(program), std::move(dims), reply_atom));
+      sys.spawn<actor_facade>(std::move(program), std::move(dims), reply_id));
   }
 
-  actor_facade(caf::actor_config& cfg, program_ptr program, nd_range dims, atom_value reply_atom)
+  actor_facade(caf::actor_config& cfg, program_ptr program, nd_range dims, int reply_id)
     : caf::event_based_actor(cfg),
       program_(std::move(program)),
       dims_(std::move(dims)),
-      reply_atom_(reply_atom) {
+      reply_id_(reply_id) {
     actor_id_ = this->id();
   }
 
   ~actor_facade() override {
-      command_runner<Ts...> runner;
-      runner.release_stream_for_actor(actor_id_);
+    command_runner<> runner;
+    runner.release_stream_for_actor(actor_id_);
   }
 
   caf::behavior make_behavior() override {
@@ -58,7 +57,6 @@ class actor_facade : public event_based_actor {
         enqueue_impl(device_num, stream_id, std::move(output_indices), std::forward<Ts>(args)...);
       },
       [this](int device_num, int stream_id, Ts... args) {
-        // Copy everything back if indices are omitted
         enqueue_impl(device_num, stream_id, {}, std::forward<Ts>(args)...);
       },
       [this](int device_num, std::vector<int> output_indices, Ts... args) {
@@ -84,81 +82,65 @@ class actor_facade : public event_based_actor {
   template <class... Us>
   void enqueue_impl(int device_num, int stream_id, std::vector<int> output_indices, Us&&... xs) {
     command_runner<Ts...> runner;
-    // Launch kernel asynchronously
     auto results = runner.run_async(program_, dims_, stream_id, 0, 
                                     device_num, std::forward<Us>(xs)...);
 
     auto sender = this->current_sender();
-    auto r_atom = reply_atom_;
+    auto r_id = reply_id_;
 
-    // If no indices specified, default to copying all arguments back
     if (output_indices.empty()) {
       for (int i = 0; i < static_cast<int>(sizeof...(Ts)); ++i) {
         output_indices.push_back(i);
       }
     }
 
-    // Track if we actually queued any transfers to host
-    // Even if we don't, we will send a final 'done' signal via callback
-    // to ensure the stream is drained.
-
-    if (!output_indices.empty()) {
-        // Helper to dispatch a lambda based on a runtime index
-        auto switch_on_index = [&](int runtime_idx, auto&& func, auto... Is) {
-            ([&] {
-                if (runtime_idx == Is) {
-                    func(std::integral_constant<int, Is>{});
-                }
-            }(), ...);
-        };
-
-        for (int idx : output_indices) {
-            if (idx >= 0 && idx < sizeof...(Ts)) {
-                switch_on_index(idx, [&](auto current_idx_constant) {
-                    // Compile-time index extraction
-                    constexpr std::size_t Index = current_idx_constant; // Compile-time index
-                    using MemPtrType = std::tuple_element_t<Index, mem_tuple>;
-                    using ValueType = typename MemPtrType::element_type::value_type;
-
-                    MemPtrType mem_ptr = std::get<Index>(results);
-
-                    // Only copy if the mem_ptr is valid and has OUT or IN_OUT access
-                    if (mem_ptr && (mem_ptr->access() == OUT || mem_ptr->access() == IN_OUT)) {
-                        runner.copy_to_host_async(mem_ptr, 
-                          [sender, r_atom, Index](std::vector<ValueType>&& data) {
-                            // Send back the data along with the original index
-                            if (sender) { // Ensure sender is still valid
-                                if (r_atom != 0) {
-                                    caf::anon_mail(r_atom, Index, std::move(data)).send(sender);
-                                } else {
-                                    caf::anon_mail(Index, std::move(data)).send(sender);
-                                }
-                            }
-                        });
-                    }
-                }, std::make_index_sequence<sizeof...(Ts)>{});
-            } else {
-                this->println("Warning: Output index {} is out of bounds (0-{})", idx, sizeof...(Ts) - 1);
-            }
-        }
+    for (int idx : output_indices) {
+      if (idx >= 0 && idx < static_cast<int>(sizeof...(Ts))) {
+        // Dispatch runtime index to compile-time sequence
+        dispatch_index(idx, [&](auto current_idx_constant) {
+          constexpr std::size_t Index = current_idx_constant;
+          using MemPtrType = std::tuple_element_t<Index, mem_tuple>;
+          using ValueType = typename MemPtrType::element_type::value_type;
+
+          MemPtrType mem_ptr = std::get<Index>(results);
+
+          if (mem_ptr && (mem_ptr->access() == OUT || mem_ptr->access() == IN_OUT)) {
+            runner.copy_to_host_async(mem_ptr, [sender, r_id, Index](std::vector<ValueType>&& data) {
+              if (sender) {
+                // Send: Correlation ID, Argument Index, Data Vector
+                caf::anon_mail(r_id, static_cast<int>(Index), std::move(data)).send(sender);
+              }
+            });
+          }
+        });
+      } else {
+        this->println("Warning: Output index {} is out of bounds", idx);
+      }
     }
 
-    // Finally, alert the sender that everything is finished.
-    // This callback is queued on the stream after all kernel and transfer commands.
-    runner.add_callback(stream_id, device_num, [sender, r_atom]() mutable {
+    // Final completion alert (Correlation ID, -1 to signal "all finished")
+    runner.add_callback(stream_id, device_num, [sender, r_id]() mutable {
       if (sender) {
-        if (r_atom != 0)
-          caf::anon_mail(r_atom, gpu_done_atom_v).send(sender);
-        else
-          caf::anon_mail(gpu_done_atom_v).send(sender);
+        caf::anon_mail(r_id, -1).send(sender);
       }
     });
   }
 
+  // Helper to map runtime index to compile-time index for tuple access
+  template <class F>
+  void dispatch_index(int idx, F&& f) {
+    dispatch_index_helper(std::make_index_sequence<sizeof...(Ts)>{}, idx, std::forward<F>(f));
+  }
+
+  template <std::size_t... Is, class F>
+  void dispatch_index_helper(std::index_sequence<Is...>, int idx, F&& f) {
+    (..., (static_cast<int>(Is) == idx ? f(std::integral_constant<std::size_t, Is>{}) : (void)0));
+  }
+
   program_ptr  program_;
   nd_range     dims_;
   caf::actor_id actor_id_;
-  atom_value   reply_atom_;
+  int          reply_id_;
 };
 
 } // namespace caf::cuda
\ No newline at end of file

From 72331f66c2b575e6d4f7ded6ca104b9e6085274c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 10:29:33 -0600
Subject: [PATCH 0633/1000] Updated file with new actor facade tests.

---
 .../tests/actor-facade-test/main.test.cpp     | 1818 +----------------
 .../tests/actor-facade-test/main.test.hpp     |   14 -
 .../tests/actor-facade-test/mmul-actors.cpp   |  125 --
 3 files changed, 86 insertions(+), 1871 deletions(-)
 delete mode 100644 libcaf_cuda/tests/actor-facade-test/main.test.hpp
 delete mode 100644 libcaf_cuda/tests/actor-facade-test/mmul-actors.cpp

diff --git a/libcaf_cuda/tests/actor-facade-test/main.test.cpp b/libcaf_cuda/tests/actor-facade-test/main.test.cpp
index 20f2288f4c..3496631138 100644
--- a/libcaf_cuda/tests/actor-facade-test/main.test.cpp
+++ b/libcaf_cuda/tests/actor-facade-test/main.test.cpp
@@ -1,6 +1,3 @@
-
-#include "main.test.hpp"
-
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
 #include <iostream>
@@ -11,1758 +8,115 @@
 #include <algorithm>
 #include <numeric>
 #include "caf/actor_registry.hpp"
-
-
-
 using namespace caf;
 using namespace std::chrono_literals;
 
-const char* kernel_code = R"(
-extern "C" __global__
-void compare_strings(const char* a, const char* b, int* result, int * length) {
-    int idx = blockIdx.x * blockDim.x + threadIdx.x;
-    if (idx < * length) {
-        result[idx] = (a[idx] == b[idx]) ? 1 : 0;
-    }
-}
-)";
-
-const char* matrixMulKernel2 = R"(
-extern "C" __global__
-void matrixMul(const int* a, const int* b, int* c, int *N_val) {
-    int N = *N_val;
-    int row = blockIdx.y * blockDim.y + threadIdx.y;
-    int col = blockIdx.x * blockDim.x + threadIdx.x;
-    if (row < N && col < N) {
-        int temp = 0;
+// CPU Matrix Multiplication for correctness verification
+void verify_mmul(const std::vector<int>& a, const std::vector<int>& b, 
+                 const std::vector<int>& c, int N) {
+    std::vector<int> expected(N * N, 0);
+    for (int i = 0; i < N; ++i) {
         for (int k = 0; k < N; ++k) {
-            temp += a[row * N + k] * b[k * N + col];
-        }
-        c[row * N + col] = temp;
-    }
-}
-)";
-
-
-const char* matrixMulKernel = R"(
-extern "C" __global__
-void matrixMul(const int* a, const int* b, int* c, int N) {
-    //printf("%d\n",N);
-    int row = blockIdx.y * blockDim.y + threadIdx.y;
-    int col = blockIdx.x * blockDim.x + threadIdx.x;
-    if (row < N && col < N) {
-        int temp = 0;
-        for (int k = 0; k < N; ++k) {
-            temp += a[row * N + k] * b[k * N + col];
-        }
-        c[row * N + col] = temp;
-    }
-}
-)";
-
-
-
-
-void actor_facade_launch_kernel_test(actor_system& sys) {
-  std::cout << "[TEST] Starting actor_facade_launch_kernel_test\n";
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  int length = 10;
-  std::vector<char> str1(length, 'A');
-  std::vector<char> str2(length, 'A');
-  std::vector<int> result(length);
-  std::vector<int> len(1, length);
-
-  caf::cuda::nd_range dim(10, 1, 1, 1, 1, 1);
-
-  auto gpuActor = mgr.spawn(kernel_code, "compare_strings", dim,
-                            in<char>{}, in<char>{}, out<int>{}, in<int>{});
-
-  auto arg1 = caf::cuda::create_in_arg(str1);
-  auto arg2 = caf::cuda::create_in_arg(str2);
-  auto arg3 = caf::cuda::create_out_arg(result);
-  auto arg4 = caf::cuda::create_in_arg(len);
-
-  sys.spawn([=](event_based_actor* self_actor) {
-    auto start = std::chrono::high_resolution_clock::now();
-    self_actor->mail(gpuActor, arg1, arg2, arg3, arg4)
-      .request(gpuActor, 10s).then(
-        [=](const std::vector<output_buffer>& outputs) {
-          auto end = std::chrono::high_resolution_clock::now();
-          std::chrono::duration<double> elapsed = end - start;
-          std::cout << "[INFO] Kernel round-trip time: " << elapsed.count() << " seconds\n";
-
-          for (size_t i = 0; i < outputs.size(); ++i) {
-            std::visit([&](const auto& vec) {
-              std::cout << "Output[" << i << "]: ";
-              for (const auto& val : vec) {
-                std::cout << val << " ";
-              }
-              std::cout << "\n";
-            }, outputs[i].data);
-          }
-
-          self_actor->send_exit(gpuActor, exit_reason::user_shutdown);
-          self_actor->quit();
-        });
-  });
-
-  sys.await_all_actors_done();
-}
-
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-  
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
-    }
-  }
-}
-
-void serial_matrix_multiply_test() {
-  std::cout << "[TEST] Starting serial_matrix_multiply_test\n";
-
-  int N = 32000;
-  std::vector<int> h_a(N * N);
-  std::vector<int> h_b(N * N);
-  std::vector<int> h_c(N * N, 0);
-
-  std::generate(h_a.begin(), h_a.end(), []() { return rand() % 10; });
-  std::generate(h_b.begin(), h_b.end(), []() { return rand() % 10; });
-
-  auto start = std::chrono::high_resolution_clock::now();
-
-  serial_matrix_multiply(h_a, h_b, h_c, N);
-
-  auto end = std::chrono::high_resolution_clock::now();
-  std::chrono::duration<double> duration = end - start;
-
-  std::cout << std::fixed << std::setprecision(6);
-  std::cout << "[INFO] Serial matrix multiplication time: "
-            << duration.count() << " seconds\n";
-}
-
-void test_mmul(caf::actor_system& sys,int N) {
-  std::cout << "[TEST] Starting test_mmul\n";
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //int N = 32000;
-  int THREADS = 32;
-  int BLOCKS = (N + THREADS - 1) / THREADS;
-
-  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  auto gpuActor = mgr.spawn(matrixMulKernel, "matrixMul", dim,
-                            in<int>{}, in<int>{}, out<int>{}, in<int>{});
-
-  std::vector<int> h_a(N * N);
-  std::vector<int> h_b(N * N);
-  std::vector<int> h_c(N * N, 0);
-  std::vector<int> h_ref(N * N, 0);
-  std::vector<int> h_n(1, N);
-
-  std::generate(h_a.begin(), h_a.end(), []() { return rand() % 10; });
-  std::generate(h_b.begin(), h_b.end(), []() { return rand() % 10; });
-
-  serial_matrix_multiply(h_a, h_b, h_ref, N);
-
-  auto arg1 = caf::cuda::create_in_arg(h_a);
-  auto arg2 = caf::cuda::create_in_arg(h_b);
-  auto arg3 = caf::cuda::create_out_arg(h_c);
-  auto arg4 = caf::cuda::create_in_arg(N);
-
-  sys.spawn([=](event_based_actor* self_actor) {
-    auto start = std::chrono::high_resolution_clock::now();
-    self_actor->mail(gpuActor, arg1, arg2, arg3, arg4)
-      .request(gpuActor, 10s).then(
-        [=](const std::vector<output_buffer>& outputs) {
-          auto end = std::chrono::high_resolution_clock::now();
-          std::chrono::duration<double> elapsed = end - start;
-          std::vector<int> result;
-          for (const auto& out : outputs) {
-            std::visit([&](const auto& vec) {
-              if constexpr (std::is_same_v<std::decay_t<decltype(vec)>, std::vector<int>>) {
-                result = vec;
-              }
-            }, out.data);
-          }
-
-          bool match = result == h_ref ;
-          std::cout << "[INFO] Kernel round-trip time: " << elapsed.count() << " seconds\n";
-          std::cout << (match ? "[PASS] GPU result matches reference\n" : "[FAIL] Mismatch in GPU result\n");
-          self_actor->send_exit(gpuActor, exit_reason::user_shutdown);
-          self_actor->quit();
-        });
-  });
-
-  sys.await_all_actors_done();
-}
-
-
-void test_mmul_plain(caf::actor_system& sys,int N) {
-  std::cout << "[TEST] Starting test_mmul\n";
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //int N = 32000;
-  int THREADS = 32;
-  int BLOCKS = (N + THREADS - 1) / THREADS;
-
-  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  auto gpuActor = mgr.spawn(matrixMulKernel2, "matrixMul", dim,
-                            in<int>{}, in<int>{}, out<int>{}, in<int>{});
-
-  std::vector<int> h_a(N * N);
-  std::vector<int> h_b(N * N);
-  std::vector<int> h_c(N * N, 0);
-  std::vector<int> h_ref(N * N, 0);
-  std::vector<int> h_n(1, N);
-
-  std::generate(h_a.begin(), h_a.end(), []() { return rand() % 10; });
-  std::generate(h_b.begin(), h_b.end(), []() { return rand() % 10; });
-
-
-  auto arg1 = caf::cuda::create_in_arg(h_a);
-  auto arg2 = caf::cuda::create_in_arg(h_b);
-  auto arg3 = caf::cuda::create_out_arg(h_c);
-  auto arg4 = caf::cuda::create_in_arg(h_n);
-
-  sys.spawn([=](event_based_actor* self_actor) {
-    auto start = std::chrono::high_resolution_clock::now();
-    self_actor->mail(gpuActor, arg1, arg2, arg3, arg4)
-      .request(gpuActor, 10s).then(
-        [=](const std::vector<output_buffer>& outputs) {
-          auto end = std::chrono::high_resolution_clock::now();
-          std::chrono::duration<double> elapsed = end - start;
-          std::vector<int> result;
-          for (const auto& out : outputs) {
-            std::visit([&](const auto& vec) {
-              if constexpr (std::is_same_v<std::decay_t<decltype(vec)>, std::vector<int>>) {
-                result = vec;
-              }
-            }, out.data);
-          }
-
-          bool match = result == h_ref ;
-          std::cout << "[INFO] Kernel round-trip time: " << elapsed.count() << " seconds\n";
-          //std::cout << (match ? "[PASS] GPU result matches reference\n" : "[FAIL] Mismatch in GPU result\n");
-          self_actor->send_exit(gpuActor, exit_reason::user_shutdown);
-          self_actor->quit();
-        });
-  });
-
-  sys.await_all_actors_done();
-}
-
-void test_mmul_large(caf::actor_system& sys) {
-  std::cout << "[TEST] Starting test_mmul\n";
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  int N = 10000;
-  int THREADS = 32;
-  int BLOCKS = (N + THREADS - 1) / THREADS;
-
-  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  auto gpuActor = mgr.spawn(matrixMulKernel, "matrixMul", dim,
-                            in<int>{}, in<int>{}, out<int>{}, in<int>{});
-
-  std::vector<int> h_a(N * N);
-  std::vector<int> h_b(N * N);
-  std::vector<int> h_c(N * N, 0);
-  std::vector<int> h_ref(N * N, 0);
-  std::vector<int> h_n(1, N);
-
-  std::generate(h_a.begin(), h_a.end(), []() { return rand() % 10; });
-  std::generate(h_b.begin(), h_b.end(), []() { return rand() % 10; });
-
-  serial_matrix_multiply(h_a, h_b, h_ref, N);
-
-  auto arg1 = caf::cuda::create_in_arg(h_a);
-  auto arg2 = caf::cuda::create_in_arg(h_b);
-  auto arg3 = caf::cuda::create_out_arg(h_c);
-  auto arg4 = caf::cuda::create_in_arg(h_n);
-
-  sys.spawn([=](event_based_actor* self_actor) {
-    auto start = std::chrono::high_resolution_clock::now();
-    self_actor->mail(gpuActor, arg1, arg2, arg3, arg4)
-      .request(gpuActor, 10s).then(
-        [=](const std::vector<output_buffer>& outputs) {
-          auto end = std::chrono::high_resolution_clock::now();
-          std::chrono::duration<double> elapsed = end - start;
-          std::vector<int> result;
-          for (const auto& out : outputs) {
-            std::visit([&](const auto& vec) {
-              if constexpr (std::is_same_v<std::decay_t<decltype(vec)>, std::vector<int>>) {
-                result = vec;
-              }
-            }, out.data);
-          }
-
-          bool match = result == h_ref;
-          std::cout << "[INFO] Kernel round-trip time: " << elapsed.count() << " seconds\n";
-          std::cout << (match ? "[PASS] GPU result matches reference\n" : "[FAIL] Mismatch in GPU result\n");
-          self_actor->send_exit(gpuActor, exit_reason::user_shutdown);
-          self_actor->quit();
-        });
-  });
-
-  sys.await_all_actors_done();
-}
-
-void test_mmul_from_cubin(caf::actor_system& sys, int N) {
-  std::cout << "[TEST] Starting test_mmul_from_cubin\n";
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  int THREADS = 32;
-  int BLOCKS = (N + THREADS - 1) / THREADS;
-
-  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  std::vector<int> h_a(N * N);
-  std::vector<int> h_b(N * N);
-  std::vector<int> h_c(N * N, 0);
-  std::vector<int> h_ref(N * N, 0);
-  std::vector<int> h_n(1, N);
-
-  std::generate(h_a.begin(), h_a.end(), []() { return rand() % 10; });
-  std::generate(h_b.begin(), h_b.end(), []() { return rand() % 10; });
-
-
-  serial_matrix_multiply(h_a, h_b, h_ref, N);
-
-  auto arg1 = caf::cuda::create_in_arg(h_a);
-  auto arg2 = caf::cuda::create_in_arg(h_b);
-  auto arg3 = caf::cuda::create_out_arg(h_c);
-  auto arg4 = caf::cuda::create_in_arg(N);
-
-  // Spawn actor from precompiled cubin file
-  auto gpuActor = mgr.spawnFromCUBIN("../mmul.cubin", "matrixMul", dim,
-                                  in<int>{}, in<int>{}, out<int>{}, in<int>{});
-
-  sys.spawn([=](caf::event_based_actor* self_actor) {
-    auto start = std::chrono::high_resolution_clock::now();
-
-    self_actor->mail(gpuActor, arg1, arg2, arg3, arg4)
-      .request(gpuActor, std::chrono::seconds(10))
-      .then([=](const std::vector<output_buffer>& outputs) {
-        auto end = std::chrono::high_resolution_clock::now();
-        std::chrono::duration<double> elapsed = end - start;
-
-        std::vector<int> result;
-        for (const auto& out : outputs) {
-          std::visit([&](const auto& vec) {
-            using T = std::decay_t<decltype(vec)>;
-            if constexpr (std::is_same_v<T, std::vector<int>>) {
-              result = vec;
+            int aik = a[i * N + k];
+            for (int j = 0; j < N; ++j) {
+                expected[i * N + j] += aik * b[k * N + j];
             }
-          }, out.data);
         }
-
-        // Compare result with reference
-        bool match = (result == h_ref);
-        std::cout << "[INFO] Kernel round-trip time: " << elapsed.count() << " seconds\n";
-        std::cout << (match ? "[PASS] GPU result matches reference\n" : "[FAIL] Mismatch in GPU result\n");
-
-        self_actor->send_exit(gpuActor, caf::exit_reason::user_shutdown);
-        self_actor->quit();
-      });
-  });
-
-  sys.await_all_actors_done();
-}
-
-
-
-
-
-void test_mmul_from_ptx(caf::actor_system& sys, int N) {
-  std::cout << "[TEST] Starting test_mmul_from_ptx\n";
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  int THREADS = 32;
-  int BLOCKS = (N + THREADS - 1) / THREADS;
-
-  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  std::vector<int> h_a(N * N);
-  std::vector<int> h_b(N * N);
-  std::vector<int> h_c(N * N, 0);
-  std::vector<int> h_ref(N * N, 0);
-  std::vector<int> h_n(1, N);
-
-  std::generate(h_a.begin(), h_a.end(), []() { return rand() % 10; });
-  std::generate(h_b.begin(), h_b.end(), []() { return rand() % 10; });
-
-
-  serial_matrix_multiply(h_a, h_b, h_ref, N);
-
-  auto arg1 = caf::cuda::create_in_arg(h_a);
-  auto arg2 = caf::cuda::create_in_arg(h_b);
-  auto arg3 = caf::cuda::create_out_arg(h_c);
-  auto arg4 = caf::cuda::create_in_arg(N);
-
-  // Spawn actor from precompiled PTX file
-  auto gpuActor = mgr.spawnFromPTX("../mmul.ptx", "matrixMul", dim,
-                                  in<int>{}, in<int>{}, out<int>{}, in<int>{});
-
-  sys.spawn([=](caf::event_based_actor* self_actor) {
-    auto start = std::chrono::high_resolution_clock::now();
-
-    self_actor->mail(gpuActor, arg1, arg2, arg3, arg4)
-      .request(gpuActor, std::chrono::seconds(10))
-      .then([=](const std::vector<output_buffer>& outputs) {
-        auto end = std::chrono::high_resolution_clock::now();
-        std::chrono::duration<double> elapsed = end - start;
-
-        std::vector<int> result;
-        for (const auto& out : outputs) {
-          std::visit([&](const auto& vec) {
-            using T = std::decay_t<decltype(vec)>;
-            if constexpr (std::is_same_v<T, std::vector<int>>) {
-              result = vec;
-            }
-          }, out.data);
-        }
-
-        // Compare result with reference
-        bool match = (result == h_ref);
-        std::cout << "[INFO] Kernel round-trip time: " << elapsed.count() << " seconds\n";
-        std::cout << (match ? "[PASS] GPU result matches reference\n" : "[FAIL] Mismatch in GPU result\n");
-
-        self_actor->send_exit(gpuActor, caf::exit_reason::user_shutdown);
-        self_actor->quit();
-      });
-  });
-
-  sys.await_all_actors_done();
-}
-
-
-
-void test_mmul_raw_data(caf::actor_system& sys) {
-  std::cout << "[TEST] Starting test_mmul_raw_data\n";
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  int N = 1024;
-  int THREADS = 32;
-  int BLOCKS = (N + THREADS - 1) / THREADS;
-
-  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  std::vector<int> h_a(N * N);
-  std::vector<int> h_b(N * N);
-  std::vector<int> h_c(N * N, 0);
-  std::vector<int> h_ref(N * N, 0);
-  std::vector<int> h_n(1, N);
-
-  std::generate(h_a.begin(), h_a.end(), []() { return rand() % 10; });
-  std::generate(h_b.begin(), h_b.end(), []() { return rand() % 10; });
-
-  auto gpuActor = mgr.spawn(matrixMulKernel, "matrixMul", dim,
-                            in<int>{}, in<int>{}, out<int>{}, in<int>{});
-
-  sys.spawn([=](caf::event_based_actor* self) {
-    auto start = std::chrono::high_resolution_clock::now();
-    self->mail(gpuActor, h_a, h_b, h_c, h_n).request(gpuActor, 10s).then(
-      [=](const std::vector<output_buffer>& outputs) {
-        auto end = std::chrono::high_resolution_clock::now();
-        std::chrono::duration<double> elapsed = end - start;
-
-        std::vector<int> result;
-        for (const auto& out : outputs) {
-          std::visit([&](const auto& vec) {
-            using T = std::decay_t<decltype(vec)>;
-            if constexpr (std::is_same_v<T, std::vector<int>>) {
-              result = vec;
-            }
-          }, out.data);
-        }
-
-        bool match = result == h_ref;
-        std::cout << "[INFO] Kernel round-trip time: " << elapsed.count() << " seconds\n";
-        std::cout << (match ? "[PASS] GPU result matches reference\n"
-                            : "[FAIL] Mismatch in GPU result\n");
-
-        self->send_exit(gpuActor, caf::exit_reason::user_shutdown);
-        self->quit();
-      });
-  });
-
-  sys.await_all_actors_done();
-}
-#include <caf/all.hpp>
-#include <caf/cuda/all.hpp>
-#include <chrono>
-#include <numeric>
-#include <vector>
-#include <random>
-#include <iostream>
-
-using Clock = std::chrono::high_resolution_clock;
-
-struct supervisor_state {
-  caf::actor gpu_actor;
-  std::vector<int> h_a;
-  std::vector<int> h_b;
-  std::vector<int> h_c;
-  std::vector<int> h_n;
-
-  std::vector<double> kernel_times;
-  std::vector<double> full_times;
-  int count = 0;
-  int id = 0;
-  int N = 0;
-};
-
-caf::behavior supervisor_fun(caf::stateful_actor<supervisor_state>* self, int id, int N) {
-  auto& st = self->state();
-  st.id = id;
-  st.N = N;
-
-  // Generate matrices once
-  st.h_a.resize(st.N * st.N);
-  st.h_b.resize(st.N * st.N);
-  st.h_c.resize(st.N * st.N, 0);
-  st.h_n = {st.N};
-
-  std::generate(st.h_a.begin(), st.h_a.end(), [] { return rand() % 10; });
-  std::generate(st.h_b.begin(), st.h_b.end(), [] { return rand() % 10; });
-
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  st.gpu_actor = caf::cuda::manager::get().spawnFromCUBIN("../mmul.cubin", "matrixMul", dims,
-                                                 in<int>{}, in<int>{}, out<int>{}, in<int>{});
-
-  auto run_iteration = [self]() {
-    auto& st_ref = self->state();
-
-    auto iteration_start = Clock::now();
-
-    auto arg1 = caf::cuda::create_in_arg(st_ref.h_a);
-    auto arg2 = caf::cuda::create_in_arg(st_ref.h_b);
-    auto arg3 = caf::cuda::create_out_arg(st_ref.h_c);
-    auto arg4 = caf::cuda::create_in_arg(st_ref.N);
-
-    auto kernel_start = Clock::now();
-
-    self->mail(st_ref.gpu_actor, arg1, arg2, arg3, arg4)
-      .request(st_ref.gpu_actor, std::chrono::seconds(100))
-      .then(
-        [self, iteration_start, kernel_start](const std::vector<output_buffer>&) {
-          auto& st_ref = self->state();
-          auto kernel_end = Clock::now();
-          auto iteration_end = Clock::now();
-
-          double kernel_time = std::chrono::duration<double>(kernel_end - kernel_start).count();
-          double full_time = std::chrono::duration<double>(iteration_end - iteration_start).count();
-
-          std::cout << "[INFO] Supervisor " << st_ref.id
-                    << " Iteration " << st_ref.count
-                    << " Kernel round-trip: " << kernel_time << " s, "
-                    << "Full iteration time: " << full_time << " s\n";
-
-          st_ref.kernel_times.push_back(kernel_time);
-          st_ref.full_times.push_back(full_time);
-          ++st_ref.count;
-
-          if (st_ref.count < 20) {
-            self->mail(std::string("start")).send(self);
-          } else {
-            double kernel_avg = std::accumulate(st_ref.kernel_times.begin(), st_ref.kernel_times.end(), 0.0) / st_ref.kernel_times.size();
-            double full_avg = std::accumulate(st_ref.full_times.begin(), st_ref.full_times.end(), 0.0) / st_ref.full_times.size();
-
-            std::cout << "[INFO] Supervisor " << st_ref.id
-                      << " Kernel average: " << kernel_avg << " s, "
-                      << "Full iteration average: " << full_avg << " s\n";
-
-            self->send_exit(st_ref.gpu_actor, caf::exit_reason::user_shutdown);
-            self->quit();
-          }
-        },
-        [self](caf::error& err) {
-          std::cerr << "[ERROR] Kernel execution failed: " << caf::to_string(err) << std::endl;
-          self->quit(err);
-        });
-  };
-
-  return {
-    [=](const std::string& msg) {
-      if (msg == "start") {
-        run_iteration();
-      }
     }
-  };
-}
-
-
-// Driver function
-inline void run_concurrent_mmul_test(caf::actor_system& sys,
-                                     int num_supervisors,
-                                     int matrix_size) {
-  auto start = Clock::now();
-
-  for (int i = 0; i < num_supervisors; ++i) {
-    auto sup = sys.spawn(supervisor_fun, i, matrix_size);
-    caf::anon_send(sup, std::string("start"));
-  }
-
-  sys.await_all_actors_done();
-
-  auto end = Clock::now();
-  std::chrono::duration<double> duration = end - start;
-  std::cout << "[TIMER] run_concurrent_mmul_test took: "
-            << duration.count() << " seconds\n";
-}
-
-
-
-
-
-caf::behavior supervisor_fun_validate(caf::stateful_actor<supervisor_state>* self, int id, int N) {
-  auto& st = self->state();
-  st.id = id;
-  st.N = N;
-
-  st.h_a.resize(st.N * st.N);
-  st.h_b.resize(st.N * st.N);
-  st.h_c.resize(st.N * st.N, 0);
-  st.h_n = {st.N};
-
-
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  st.gpu_actor = caf::cuda::manager::get().spawnFromCUBIN("../mmul.cubin", "matrixMul", dims,
-                                                 in<int>{}, in<int>{}, out<int>{}, in<int>{});
-
-  auto run_iteration = [&,self]() {
-    auto& st_ref = self->state();
-
-    auto iteration_start = Clock::now();
-
-    //generate new matrix 
-    std::generate(st.h_a.begin(), st.h_a.end(), [] { return rand() % 10; });
-    std::generate(st.h_b.begin(), st.h_b.end(), [] { return rand() % 10; });
-    auto arg1 = caf::cuda::create_in_arg(st.h_a);
-    auto arg2 = caf::cuda::create_in_arg(st.h_b);
-    auto arg3 = caf::cuda::create_out_arg(st.N * st.N);
-    auto arg4 = caf::cuda::create_in_arg(st.N); //i have no idea why this needs to be squared but it does
-
-    auto kernel_start = Clock::now();
-
-    self->mail(st_ref.gpu_actor, arg1, arg2, arg3, arg4)
-      .request(st_ref.gpu_actor, std::chrono::seconds(100))
-      .then(
-        [&,self, iteration_start, kernel_start](const std::vector<output_buffer>& outputs) {
-          auto& st_ref = self->state();
-          auto kernel_end = Clock::now();
-          auto iteration_end = Clock::now();
-
-          double kernel_time = std::chrono::duration<double>(kernel_end - kernel_start).count();
-          double full_time = std::chrono::duration<double>(iteration_end - iteration_start).count();
-
-          std::cout << "[INFO] Supervisor " << st_ref.id
-                    << " Iteration " << st_ref.count
-                    << " Kernel round-trip: " << kernel_time << " s, "
-                    << "Full iteration time: " << full_time << " s\n";
-
-          st_ref.kernel_times.push_back(kernel_time);
-          st_ref.full_times.push_back(full_time);
-          ++st_ref.count;
-        
-	  std::vector<int> result;
-        
-	  for (const auto& out : outputs) {
-          std::visit([&](const auto& vec) {
-            using T = std::decay_t<decltype(vec)>;
-            if constexpr (std::is_same_v<T, std::vector<int>>) {
-              result = vec;
-            }
-          }, out.data);
+    bool correct = true;
+    for (size_t i = 0; i < expected.size(); ++i) {
+        if (c[i] != expected[i]) {
+            correct = false;
+            break;
         }
-
-std::vector<int> h_ref(st.N * st.N, 0);
-serial_matrix_multiply(st.h_a, st.h_b, h_ref, st.N);
-bool match = result == h_ref;
-
-if (match) {
-  std::cout << "[PASS] GPU result matches reference\n";
-} else {
-  std::cout << "[FAIL] Mismatch in GPU result\n";
-
-  std::cout << "Expected (h_ref): ";
-  for (int val : h_ref)
-    std::cout << val << ' ';
-  std::cout << '\n';
-
-  std::cout << "Actual (result):  ";
-  for (int val : result)
-    std::cout << val << ' ';
-  std::cout << '\n';
-}
-
-          if (st_ref.count < 20) {
-            self->mail(std::string("start")).send(self);
-          } else {
-            double kernel_avg = std::accumulate(st_ref.kernel_times.begin(), st_ref.kernel_times.end(), 0.0) / st_ref.kernel_times.size();
-            double full_avg = std::accumulate(st_ref.full_times.begin(), st_ref.full_times.end(), 0.0) / st_ref.full_times.size();
-
-            std::cout << "[INFO] Supervisor " << st_ref.id
-                      << " Kernel average: " << kernel_avg << " s, "
-                      << "Full iteration average: " << full_avg << " s\n";
-
-            self->send_exit(st_ref.gpu_actor, caf::exit_reason::user_shutdown);
-            self->quit();
-          }
-        },
-        [self](caf::error& err) {
-          std::cerr << "[ERROR] Kernel execution failed: " << caf::to_string(err) << std::endl;
-          self->quit(err);
-        });
-  };
-
-  return {
-    [=](const std::string& msg) {
-      if (msg == "start") {
-        run_iteration();
-      }
     }
-  };
-}
-
-
-
-
-// Driver function
-inline void run_concurrent_mmul_validate_test(caf::actor_system& sys,
-                                     int num_supervisors,
-                                     int matrix_size) {
-  auto start = Clock::now();
-
-  for (int i = 0; i < num_supervisors; ++i) {
-    auto sup = sys.spawn(supervisor_fun_validate, i, matrix_size);
-    caf::anon_send(sup, std::string("start"));
-  }
-
-  sys.await_all_actors_done();
-
-  auto end = Clock::now();
-  std::chrono::duration<double> duration = end - start;
-  std::cout << "[TIMER] run_concurrent_mmul_test took: "
-            << duration.count() << " seconds\n";
-}
-
-
-
-// === Global matrices for shared use ===
-std::vector<int> global_a;
-std::vector<int> global_b;
-std::vector<std::vector<int>> global_cs;
-std::vector<int> global_c; //yes each actor gets the same output buffer
-			   //this shouldnt matter anyways due to just performance testing
-			   //and gpu actors dont share state
-
-caf::behavior supervisor_global_fun(caf::stateful_actor<supervisor_state>* self, int id, int N) {
-  auto& st = self->state();
-  st.id = id;
-  st.N = N;
-
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  st.gpu_actor = caf::cuda::manager::get().spawnFromCUBIN(
-    "../mmul.cubin", "matrixMul", dims,
-    in<int>{}, in<int>{}, out<int>{}, in<int>{}
-  );
-
-  // === Register lifecycle hooks only once ===
-
-  self->attach_functor([=](const caf::error& reason) {
-    std::cout << "[EXIT] Supervisor " << self->state().id
-              << " died with reason: " << caf::to_string(reason)
-              << ", after iteration: " << self->state().count << "\n";
-  });
-
- self->set_exit_handler([=](const caf::exit_msg& msg) {
-  std::cout << "[EXIT HANDLER] Supervisor " << self->state().id
-            << " received exit from actor: " << to_string(msg.source)
-            << ", reason: " << caf::to_string(msg.reason) << "\n";
-
-  if (msg.source == st.gpu_actor) {
-    std::cerr << "[ERROR] GPU actor crashed or terminated unexpectedly!\n";
-  }
-});
-
-
-
-self->monitor(st.gpu_actor);  // detect if GPU actor dies unexpectedly
-
-auto gpu = st.gpu_actor;
-self->attach_functor([gpu](const caf::error& reason) {
-    std::cout << "[Supervisor] GPU actor exited with reason: " << caf::to_string(reason) << "\n";
-});
-
-
-// And handle the exit message from that GPU actor:
-self->set_exit_handler([gpu](const caf::exit_msg& msg) {
-    if (msg.source == gpu) {
-        std::cerr << "[Supervisor] GPU actor terminated! Reason: " << caf::to_string(msg.reason) << "\n";
+    if (correct) {
+        std::cout << "[SUCCESS] Matrix multiplication result is correct." << std::endl;
+    } else {
+        std::cout << "[FAILURE] Matrix multiplication result mismatch!" << std::endl;
     }
-});
-
-
-
-self->system().registry().put(st.gpu_actor.id(), st.gpu_actor);
-self->attach_functor([=](const caf::error& reason) {
-  std::cout << "[GPU Actor Terminated] Reason: " << to_string(reason) << "\n";
-});
-
-
-
-
-  auto run_iteration = [self]() {
-    auto& st_ref = self->state();
-    int N_val = st_ref.N;
-
-    auto iteration_start = Clock::now();
-
-    auto arg1 = caf::cuda::create_in_arg(global_a);
-    auto arg2 = caf::cuda::create_in_arg(global_b);
-    auto arg3 = caf::cuda::create_out_arg(global_c);
-    auto arg4 = caf::cuda::create_in_arg(N_val);
-
-    auto kernel_start = Clock::now();
-
-    self->mail(arg1, arg2, arg3, arg4)
-      .request(st_ref.gpu_actor, std::chrono::seconds(1000))
-      .then(
-        [self, iteration_start, kernel_start](const std::vector<output_buffer>&) {
-          auto& st_ref = self->state();
-
-          auto kernel_end = Clock::now();
-          auto iteration_end = Clock::now();
-
-          double kernel_time = std::chrono::duration<double>(kernel_end - kernel_start).count();
-          double full_time = std::chrono::duration<double>(iteration_end - iteration_start).count();
-
-          std::cout << "[INFO] [GPU GLOBAL] Supervisor " << st_ref.id
-                    << " Iteration " << st_ref.count
-                    << " Kernel round-trip: " << kernel_time << " s, "
-                    << "Full iteration time: " << full_time << " s\n";
-
-          st_ref.kernel_times.push_back(kernel_time);
-          st_ref.full_times.push_back(full_time);
-          ++st_ref.count;
-
-          if (st_ref.count < 20) {
-            std::cout << "[DEBUG] Supervisor " << st_ref.id
-                      << " scheduling iteration " << st_ref.count << "\n";
-          self->mail(std::string("start")).send(self);
-
-	  } else {
-            double kernel_avg = std::accumulate(
-              st_ref.kernel_times.begin(), st_ref.kernel_times.end(), 0.0
-            ) / st_ref.kernel_times.size();
-
-            double full_avg = std::accumulate(
-              st_ref.full_times.begin(), st_ref.full_times.end(), 0.0
-            ) / st_ref.full_times.size();
-
-            std::cout << "[INFO] [GPU GLOBAL] Supervisor " << st_ref.id
-                      << " Kernel average: " << kernel_avg << " s, "
-                      << "Full iteration average: " << full_avg << " s\n";
-
-            std::cout << "[DEBUG] Supervisor " << st_ref.id
-                      << " quitting after iteration " << st_ref.count << "\n";
-
-            self->send_exit(st_ref.gpu_actor, caf::exit_reason::user_shutdown);
-            self->quit();
-          }
-        },
-        [self](caf::error& err) {
-          std::cerr << "[ERROR] [GPU GLOBAL] Kernel execution failed: "
-                    << caf::to_string(err) << "\n";
-          self->quit(err);
-        });
-  };
-
-  return {
-    [=](const std::string& msg) {
-      if (msg == "start") {
-        run_iteration();
-      }
-    }
-  };
-}
-
-
-// === New Test Function ===
-inline void run_concurrent_mmul_test_global(caf::actor_system& sys,
-                                            int num_supervisors,
-                                            int matrix_size) {
-  auto start = Clock::now();
-
-  int N = matrix_size;
-  size_t matrix_elements = static_cast<size_t>(N) * N;
-
-  // Global inputs
-  global_a.assign(matrix_elements, 0);
-  global_b.assign(matrix_elements, 0);
-  global_c.assign(matrix_elements,0);
-  //global_cs.resize(num_supervisors);
-  //for (int i = 0; i < num_supervisors; ++i)
-    //global_cs[i].assign(matrix_elements, 0);
-
-  // Optional: Populate input with actual data
-  //std::generate(global_a.begin(), global_a.end(), [] { return rand() % 10; });
-  //std::generate(global_b.begin(), global_b.end(), [] { return rand() % 10; });
-
-  // Spawn supervisors
-  for (int i = 0; i < num_supervisors; ++i) {
-    auto sup = sys.spawn(supervisor_global_fun, i, N);
-    caf::anon_send(sup, std::string("start"));
-  }
-
-  sys.await_all_actors_done();
-
-  auto end = Clock::now();
-  std::chrono::duration<double> duration = end - start;
-  std::cout << "[TIMER] run_concurrent_mmul_test_global took: "
-            << duration.count() << " seconds\n";
-}
-
-
-// === Global matrices for CPU serial multiply ===
-std::vector<int> cpu_global_a;
-std::vector<int> cpu_global_b;
-std::vector<int> cpu_global_c; // Shared output buffer
-
-// === Messages ===
-using matrix_msg = caf::message; // We'll send references via vector<int> const&
-
-// === Worker Actor: Does the serial multiply on request ===
-caf::behavior cpu_worker_fun(caf::event_based_actor* self) {
-  return {
-    [=](const std::vector<int>& a, const std::vector<int>& b, std::vector<int>& c, int N) {
-      serial_matrix_multiply(a, b, c, N);
-      // Reply with empty message or some confirmation (could send duration, etc.)
-      //self->send(self->current_sender());
-    }
-  };
-}
-
-// === Supervisor State ===
-struct cpu_supervisor_state {
-  caf::actor worker;
-  int id = 0;
-  int N = 0;
-  int count = 0;
-  std::vector<double> run_times;
-};
-
-// === Supervisor Actor ===
-caf::behavior cpu_supervisor_global_fun(caf::stateful_actor<cpu_supervisor_state>* self, int id, int N) {
-  auto& st = self->state();
-  st.id = id;
-  st.N = N;
-  st.count = 0;
-
-  // Spawn worker actor once
-  st.worker = self->spawn(cpu_worker_fun);
-
-  auto run_iteration = [self]() {
-    auto& st_ref = self->state();
-    auto start = Clock::now();
-
-    // Send global buffers to worker actor for multiplication
-    self->request(st_ref.worker, std::chrono::seconds(1000),
-                  cpu_global_a, cpu_global_b, cpu_global_c, st_ref.N)
-      .then(
-        [self, start]() {
-          auto end = Clock::now();
-          double duration = std::chrono::duration<double>(end - start).count();
-
-          auto& st_ref = self->state();
-          std::cout << "[INFO] [CPU GLOBAL] Supervisor " << st_ref.id
-                    << " Iteration " << st_ref.count
-                    << " Serial multiply time: " << duration << " s\n";
-
-          st_ref.run_times.push_back(duration);
-          ++st_ref.count;
-
-          if (st_ref.count < 20) {
-            self->mail(std::string("start")).send(self);
-          } else {
-            double avg = std::accumulate(st_ref.run_times.begin(), st_ref.run_times.end(), 0.0) / st_ref.run_times.size();
-            std::cout << "[INFO] [CPU GLOBAL] Supervisor " << st_ref.id
-                      << " Average serial multiply time: " << avg << " s\n";
-
-            self->send_exit(st_ref.worker, caf::exit_reason::user_shutdown);
-            self->quit();
-          }
-        },
-        [self](caf::error& err) {
-          std::cerr << "[ERROR] [CPU GLOBAL] Worker call failed: " << caf::to_string(err) << std::endl;
-          self->quit(err);
-        });
-  };
-
-  return {
-    [=](const std::string& msg) {
-      if (msg == "start") {
-        run_iteration();
-      }
-    }
-  };
-}
-
-// === CPU Global Matrix Test Function with Worker ===
-inline void run_concurrent_serial_mmul_test_global_with_worker(caf::actor_system& sys,
-                                                               int num_supervisors,
-                                                               int matrix_size) {
-  auto start = Clock::now();
-
-  int N = matrix_size;
-  size_t matrix_elements = static_cast<size_t>(N) * N;
-
-  // Initialize global matrices once
-  cpu_global_a.assign(matrix_elements, 0);
-  cpu_global_b.assign(matrix_elements, 0);
-  cpu_global_c.assign(matrix_elements, 0);
-
-  // Optional: fill inputs with some data
-  std::generate(cpu_global_a.begin(), cpu_global_a.end(), [] { return rand() % 10; });
-  std::generate(cpu_global_b.begin(), cpu_global_b.end(), [] { return rand() % 10; });
-
-  // Spawn supervisors (which spawn workers internally)
-  for (int i = 0; i < num_supervisors; ++i) {
-    auto sup = sys.spawn(cpu_supervisor_global_fun, i, N);
-    caf::anon_send(sup, std::string("start"));
-  }
-
-  sys.await_all_actors_done();
-
-  auto end = Clock::now();
-  std::chrono::duration<double> duration = end - start;
-  std::cout << "[TIMER] run_concurrent_serial_mmul_test_global_with_worker took: "
-            << duration.count() << " seconds\n";
 }
 
-
-
-
-void run_all_concurrent_tests(caf::actor_system& sys) {
-  std::vector<int> matrix_sizes = {1024, 2048, 4096};
-  std::vector<int> actor_counts = {1, 50, 200};
-
-  for (auto N : matrix_sizes) {
-    for (auto num_actors : actor_counts) {
-      std::cout << "\n=== Running tests for N = " << N
-                << ", num_actors = " << num_actors << " ===\n";
-
-      std::cout << "[RUN] GPU concurrent test (global matrices)...\n";
-      run_concurrent_mmul_test_global(sys, num_actors, N);
-
-      //std::cout << "[RUN] CPU concurrent test with worker (global matrices)...\n";
-      //run_concurrent_serial_mmul_test_global_with_worker(sys, num_actors, N);
-    }
-  }
-}
-
-
-
-struct supervisor_state_shared {
-  std::vector<int> h_a;
-  std::vector<int> h_b;
-  std::vector<int> h_c;
-  std::vector<int> h_n;
-
-  std::vector<double> kernel_times;
-  std::vector<double> full_times;
-  caf::actor gpu_actor;
-  int count = 0;
-  int id = 0;
-  int N = 0;
-};
-
-caf::behavior supervisor_shared_fun(caf::stateful_actor<supervisor_state_shared>* self,
-                                    int id, int N, caf::actor shared_gpu_actor) {
-  auto& st = self->state();
-  st.id = id;
-  st.N = N;
-  st.gpu_actor = shared_gpu_actor;
-
-  self->attach_functor([&st](const caf::error& reason) {
-    std::cout << "[EXIT] [Shared Supervisor] " << st.id
-              << " exited, reason: " << caf::to_string(reason)
-              << ", iterations: " << st.count << "\n";
-  });
-
-  auto run_iteration = [self]() {
-    auto& st_ref = self->state();
-    int N_val = st_ref.N;
-    auto iteration_start = Clock::now();
-
-    std::cout << "[DEBUG] [Supervisor " << st_ref.id << "] Preparing kernel arguments...\n";
-
-    auto arg1 = caf::cuda::create_in_arg(global_a);
-    auto arg2 = caf::cuda::create_in_arg(global_b);
-    auto arg3 = caf::cuda::create_out_arg(global_c); // Shared output buffer
-    auto arg4 = caf::cuda::create_in_arg(N_val);
-
-    std::cout << "[DEBUG] [Supervisor " << st_ref.id << "] Arguments prepared: "
-              << "A(" << global_a.size() << "), "
-              << "B(" << global_b.size() << "), "
-              << "C(" << global_c.size() << "), "
-              << "N(" << N_val << ")\n";
-
-    std::cout << "[DEBUG] [Supervisor " << st_ref.id << "] Sending message via mail and requesting response...\n";
-
-    auto kernel_start = Clock::now();
-
-    self->mail(st_ref.gpu_actor, arg1, arg2, arg3, arg4)
-      .request(st_ref.gpu_actor, std::chrono::seconds(1000))
-      .then(
-        [self, iteration_start, kernel_start](const std::vector<output_buffer>&) {
-          auto& st_ref = self->state();
-          auto kernel_end = Clock::now();
-          auto iteration_end = Clock::now();
-
-          double kernel_time = std::chrono::duration<double>(kernel_end - kernel_start).count();
-          double full_time = std::chrono::duration<double>(iteration_end - iteration_start).count();
-
-          std::cout << "[INFO] [GPU SHARED] Supervisor " << st_ref.id
-                    << " Iteration " << st_ref.count
-                    << " Kernel round-trip: " << kernel_time << " s, "
-                    << "Full iteration time: " << full_time << " s\n";
-
-          st_ref.kernel_times.push_back(kernel_time);
-          st_ref.full_times.push_back(full_time);
-          ++st_ref.count;
-
-          if (st_ref.count < 20) {
-            std::cout << "[DEBUG] [Supervisor " << st_ref.id << "] Scheduling next iteration...\n";
-            self->mail(std::string("start")).send(self);
-          } else {
-            double kernel_avg = std::accumulate(st_ref.kernel_times.begin(), st_ref.kernel_times.end(), 0.0) / st_ref.kernel_times.size();
-            double full_avg = std::accumulate(st_ref.full_times.begin(), st_ref.full_times.end(), 0.0) / st_ref.full_times.size();
-
-            std::cout << "[RESULT] [GPU SHARED] Supervisor " << st_ref.id
-                      << " Kernel average: " << kernel_avg << " s, "
-                      << "Full iteration average: " << full_avg << " s\n";
-
-            self->quit();
-          }
-        },
-        [self](caf::error& err) {
-          auto& st_ref = self->state();
-          std::cerr << "[ERROR] [GPU SHARED] Supervisor " << st_ref.id
-                    << " Kernel execution failed: "
-                    << caf::to_string(err) << "\n";
-          self->quit(err);
-        });
-  };
-
-  return {
-    [run_iteration, self](const std::string& msg) {
-      auto& st_ref = self->state();
-      if (msg == "start") {
-        std::cout << "[DEBUG] [Supervisor " << st_ref.id << "] Received start message\n";
-        std::cout << "[INFO] [GPU SHARED] Supervisor " << st_ref.id
-                  << " starting iteration " << st_ref.count << "\n";
-        run_iteration();
-      }
-    }
-  };
-}
-
-
-
-
-
-
-
-
-inline void run_concurrent_mmul_test_shared_gpu(caf::actor_system& sys,
-                                                int num_supervisors,
-                                                int matrix_size) {
-  auto start = Clock::now();
-
-  int N = matrix_size;
-  size_t matrix_elements = static_cast<size_t>(N) * N;
-
-  // Global inputs
-  global_a.assign(matrix_elements, 1);
-  global_b.assign(matrix_elements, 2);
-  global_c.assign(matrix_elements, 0);
-
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  caf::actor shared_gpu_actor = caf::cuda::manager::get().spawnFromCUBIN(
-    "../mmul.cubin", "matrixMul", dims,
-    in<int>{}, in<int>{}, out<int>{}, in<int>{}
-  );
-
-  sys.registry().put(shared_gpu_actor.id(), shared_gpu_actor);
-
-  // Launch supervisors sharing the same GPU actor
-  for (int i = 0; i < num_supervisors; ++i) {
-    auto sup = sys.spawn(supervisor_shared_fun, i, N, shared_gpu_actor);
-    caf::anon_send(sup, std::string("start"));
-  }
-
-  sys.await_all_actors_done();
-
-  auto end = Clock::now();
-  std::chrono::duration<double> duration = end - start;
-  std::cout << "[TIMER] run_concurrent_mmul_test_shared_gpu took: "
-            << duration.count() << " seconds\n";
-
-  // Send exit to shared GPU actor after all supervisors finish
-  anon_send_exit(shared_gpu_actor, caf::exit_reason::user_shutdown);
-}
-
-
-
-
-#include <vector>
-#include <chrono>
-#include <numeric>
-#include <iostream>
-#include <random>
-#include <queue>
-
-#include <caf/all.hpp>
-#include "caf/cuda/actor_facade.hpp"
-#include "caf/cuda/command.hpp"
-#include "caf/cuda/manager.hpp"
-
-using Clock = std::chrono::high_resolution_clock;
-
-// === Global matrices for shared use ===
-                           // GPU actors don't share state, so this is fine
-
-struct supervisor_sync_state {
-  int id = 0;
-  int N = 0;
-  int count = 0;
-  std::vector<double> kernel_times;
-  std::vector<double> full_times;
-  caf::actor gpu_actor;
-  // Queue to store start times for each iteration
-  std::queue<std::pair<Clock::time_point, Clock::time_point>> start_times; // {iteration_start, kernel_start}
+// State for the test manager actor
+struct test_actor_state {
+    std::vector<int> h_a;
+    std::vector<int> h_b;
+    std::vector<int> h_c;
+    int N;
+    std::chrono::steady_clock::time_point start_time;
 };
 
-caf::behavior supervisor_global_sync_fun(caf::stateful_actor<supervisor_sync_state>* self, int id, int N) {
-  auto& st = self->state();
-  st.id = id;
-  st.N = N;
-
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  st.gpu_actor = caf::cuda::manager::get().spawnFromCUBIN(
-    "../mmul.cubin", "matrixMul", dims,
-    in<int>{}, in<int>{}, out<int>{}, in<int>{}
-  );
-
-  // === Register lifecycle hooks only once ===
-  self->attach_functor([=](const caf::error& reason) {
-    std::cout << "[EXIT] Supervisor " << self->state().id
-              << " died with reason: " << caf::to_string(reason)
-              << ", after iteration: " << self->state().count << "\n";
-  });
-
-  self->set_exit_handler([=](const caf::exit_msg& msg) {
-    std::cout << "[EXIT HANDLER] Supervisor " << self->state().id
-              << " received exit from actor: " << to_string(msg.source)
-              << ", reason: " << caf::to_string(msg.reason) << "\n";
-
-    if (msg.source == st.gpu_actor) {
-      std::cerr << "[ERROR] GPU actor crashed or terminated unexpectedly!\n";
-    }
-  });
-
-  self->monitor(st.gpu_actor);  // Detect if GPU actor dies unexpectedly
-
-  auto gpu = st.gpu_actor;
-  self->attach_functor([gpu](const caf::error& reason) {
-    std::cout << "[Supervisor] GPU actor exited with reason: " << caf::to_string(reason) << "\n";
-  });
-
-  self->set_exit_handler([gpu](const caf::exit_msg& msg) {
-    if (msg.source == gpu) {
-      std::cerr << "[Supervisor] GPU actor terminated! Reason: " << caf::to_string(msg.reason) << "\n";
-    }
-  });
-
-  self->system().registry().put(st.gpu_actor.id(), st.gpu_actor);
-  self->attach_functor([=](const caf::error& reason) {
-    std::cout << "[GPU Actor Terminated] Reason: " << to_string(reason) << "\n";
-  });
-
-  auto run_iteration = [self]() {
-    auto& st_ref = self->state();
-    int N_val = st_ref.N;
-
-    auto iteration_start = Clock::now();
-    auto kernel_start = Clock::now();
-    
-    auto arg1 = caf::cuda::create_in_arg(global_a);
-    auto arg2 = caf::cuda::create_in_arg(global_b);
-    auto arg3 = caf::cuda::create_out_arg(global_c);
-    auto arg4 = caf::cuda::create_in_arg(N_val);
-
-    /*
-    auto arg1 = caf::cuda::create_in_arg(std::vector<int>(global_a));
-    auto arg2 = caf::cuda::create_in_arg(std::vector<int>(global_b));
-    auto arg3 = caf::cuda::create_out_arg(std::vector<int>(global_c));
-    auto arg4 = caf::cuda::create_in_arg(N_val);
-    */
-
-
-    // Store start times for this iteration
-    st_ref.start_times.emplace(std::make_pair(iteration_start, kernel_start));
-
-    // Send message synchronously to GPU actor
+// Actor behavior to test the actor_facade
+caf::behavior mmul_facade_test(caf::stateful_actor<test_actor_state>* self, 
+                               caf::actor facade, int N) {
+    self->state().N = N;
+    // Initialize matrices with some values
+    self->state().h_a.assign(N * N, 1);
+    self->state().h_b.assign(N * N, 2);
+    self->state().h_c.resize(N * N);
+
+    // Wrap host buffers in tagged arguments
+    auto arg1 = caf::cuda::create_in_arg(self->state().h_a);
+    auto arg2 = caf::cuda::create_in_arg(self->state().h_b);
+    auto arg3 = caf::cuda::create_out_arg_with_size<int>(N * N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    std::cout << "[INFO] Launching actor_facade test for N=" << N << "..." << std::endl;
+    self->state().start_time = std::chrono::steady_clock::now();
     
-
-    //TODO is this broken?
-     self->mail(self,arg1, arg2, arg3, arg4).send(st_ref.gpu_actor);
-
-      //self->send(st_ref.gpu_actor, self, arg1, arg2, arg3, arg4);
-     //self->send(st_ref.gpu_actor, self, global_a, global_b, global_c, N_val);
-  };
-
-  return {
-    [=](const std::string& msg) {
-      if (msg == "start") {
-        run_iteration();
-      }
-    },
-    [=](const std::vector<output_buffer>&) {
-      auto& st_ref = self->state();
-
-      auto kernel_end = Clock::now();
-      auto iteration_end = Clock::now();
-
-      // Retrieve and remove the start times for this iteration
-      if (st_ref.start_times.empty()) {
-        std::cerr << "[ERROR] [GPU GLOBAL SYNC] Supervisor " << st_ref.id
-                  << " received response but no start times available!\n";
-        self->quit(caf::make_error(caf::sec::runtime_error, "No start times for iteration"));
-        return;
-      }
-
-      auto [iteration_start, kernel_start] = st_ref.start_times.front();
-      st_ref.start_times.pop();
-
-      double kernel_time = std::chrono::duration<double>(kernel_end - kernel_start).count();
-      double full_time = std::chrono::duration<double>(iteration_end - iteration_start).count();
-
-      std::cout << "[INFO] [GPU GLOBAL SYNC] Supervisor " << st_ref.id
-                << " Iteration " << st_ref.count
-                << " Kernel round-trip: " << kernel_time << " s, "
-                << "Full iteration time: " << full_time << " s\n";
-
-      st_ref.kernel_times.push_back(kernel_time);
-      st_ref.full_times.push_back(full_time);
-      ++st_ref.count;
-
-      if (st_ref.count < 20) {
-        std::cout << "[DEBUG] Supervisor " << st_ref.id
-                  << " scheduling iteration " << st_ref.count << "\n";
-        self->mail(std::string("start")).send(self);
-      } else {
-        double kernel_avg = std::accumulate(
-          st_ref.kernel_times.begin(), st_ref.kernel_times.end(), 0.0
-        ) / st_ref.kernel_times.size();
-
-        double full_avg = std::accumulate(
-          st_ref.full_times.begin(), st_ref.full_times.end(), 0.0
-        ) / st_ref.full_times.size();
-
-        std::cout << "[INFO] [GPU GLOBAL SYNC] Supervisor " << st_ref.id
-                  << " Kernel average: " << kernel_avg << " s, "
-                  << "Full iteration average: " << full_avg << " s\n";
-
-        std::cout << "[DEBUG] Supervisor " << st_ref.id
-                  << " quitting after iteration " << st_ref.count << "\n";
-
-        self->send_exit(st_ref.gpu_actor, caf::exit_reason::user_shutdown);
-        self->quit();
-      }
-    },
-    [=](caf::error& err) {
-      std::cerr << "[ERROR] [GPU GLOBAL SYNC] Kernel execution failed: "
-                << caf::to_string(err) << "\n";
-      self->quit(err);
-    }
-  };
-}
-
-// === New Test Function ===
-inline void run_concurrent_mmul_test_global_sync(caf::actor_system& sys,
-                                                int num_supervisors,
-                                                int matrix_size) {
-  auto start = Clock::now();
-
-  int N = matrix_size;
-  size_t matrix_elements = static_cast<size_t>(N) * N;
-
-  // Global inputs
-  global_a.assign(matrix_elements, 0);
-  global_b.assign(matrix_elements, 0);
-  global_c.assign(matrix_elements, 0);
-
-  // Optional: Populate input with actual data
-  //std::generate(global_a.begin(), global_a.end(), [] { return rand() % 10; });
-  //std::generate(global_b.begin(), global_b.end(), [] { return rand() % 10; });
-
-  // Spawn supervisors
-  for (int i = 0; i < num_supervisors; ++i) {
-    auto sup = sys.spawn(supervisor_global_sync_fun, i, N);
-    caf::anon_send(sup, std::string("start"));
-  }
-
-  sys.await_all_actors_done();
-
-  auto end = Clock::now();
-  std::chrono::duration<double> duration = end - start;
-  std::cout << "[TIMER] run_concurrent_mmul_test_global_sync took: "
-            << duration.count() << " seconds\n";
-}
-
-
-// === Worker Actor: Performs serial matrix multiplication ===
-caf::behavior cpu_worker_per_actor_fun(caf::event_based_actor* self) {
-  return {
-    [=](const std::vector<int>& a, const std::vector<int>& b, int N) {
-      std::vector<int> c(N * N);
-      serial_matrix_multiply(a, b, c, N);
-      // Result is local and not returned, focusing on timing
-    }
-  };
-}
-
-// === Supervisor State ===
-struct cpu_supervisor_per_actor_state {
-  caf::actor worker;
-  int id = 0;
-  int N = 0;
-  int num_iterations = 0;
-  int count = 0;
-  std::vector<double> run_times;
-  std::vector<int> a;
-  std::vector<int> b;
-};
-
-// === Supervisor Actor ===
-caf::behavior cpu_supervisor_per_actor_fun(caf::stateful_actor<cpu_supervisor_per_actor_state>* self, int id, int N, int num_iterations) {
-  auto& st = self->state();
-  st.id = id;
-  st.N = N;
-  st.num_iterations = num_iterations;
-  st.count = 0;
-  size_t matrix_elements = static_cast<size_t>(N) * N;
-  st.a.assign(matrix_elements, 0);
-  st.b.assign(matrix_elements, 0);
-  // Initialize matrices with random data
-  std::generate(st.a.begin(), st.a.end(), [] { return rand() % 10; });
-  std::generate(st.b.begin(), st.b.end(), [] { return rand() % 10; });
-
-  // Spawn worker actor
-  st.worker = self->spawn(cpu_worker_per_actor_fun);
-
-  auto run_iteration = [self]() {
-    auto& st_ref = self->state();
-    auto start = Clock::now();
-
-    // Send matrices to worker
-    self->request(st_ref.worker, std::chrono::seconds(1000),
-                  st_ref.a, st_ref.b, st_ref.N)
-      .then(
-        [self, start]() {
-          auto end = Clock::now();
-          double duration = std::chrono::duration<double>(end - start).count();
-
-          auto& st_ref = self->state();
-          std::cout << "[INFO] [CPU PER ACTOR] Supervisor " << st_ref.id
-                    << " (Matrix Size: " << st_ref.N << "x" << st_ref.N
-                    << ", Total Iterations: " << st_ref.num_iterations << ")"
-                    << " Iteration " << st_ref.count
-                    << " Serial multiply time: " << duration << " s\n";
-
-          st_ref.run_times.push_back(duration);
-          ++st_ref.count;
-
-          if (st_ref.count < st_ref.num_iterations) {
-            self->mail(std::string("start")).send(self);
-          } else {
-            double avg = std::accumulate(st_ref.run_times.begin(), st_ref.run_times.end(), 0.0) / st_ref.run_times.size();
-            std::cout << "[INFO] [CPU PER ACTOR] Supervisor " << st_ref.id
-                      << " (Matrix Size: " << st_ref.N << "x" << st_ref.N
-                      << ", Total Iterations: " << st_ref.num_iterations << ")"
-                      << " Average serial multiply time: " << avg << " s\n";
-
-            self->send_exit(st_ref.worker, caf::exit_reason::user_shutdown);
-            self->quit();
-          }
-        },
-        [self](caf::error& err) {
-          std::cerr << "[ERROR] [CPU PER ACTOR] Worker call failed: " << caf::to_string(err) << std::endl;
-          self->quit(err);
-        });
-  };
-
-  return {
-    [=](const std::string& msg) {
-      if (msg == "start") {
-        run_iteration();
-      }
-    }
-  };
-}
-
-
-
-// === Test Function ===
-inline void run_concurrent_serial_mmul_test_per_actor(caf::actor_system& sys,
-                                                      const std::vector<int>& sizes,
-                                                      const std::vector<int>& iterations) {
-  if (sizes.size() != iterations.size()) {
-    std::cerr << "Error: sizes and iterations must have the same length\n";
-    return;
-  }
-  int num_supervisors = sizes.size();
-
-  auto start = Clock::now();
-
-  // Spawn supervisors with individual sizes and iterations
-  for (int i = 0; i < num_supervisors; ++i) {
-    int N = sizes[i];
-    int num_iter = iterations[i];
-    auto sup = sys.spawn(cpu_supervisor_per_actor_fun, i, N, num_iter);
-    caf::anon_send(sup, std::string("start"));
-  }
-
-  sys.await_all_actors_done();
-
-  auto end = Clock::now();
-  std::chrono::duration<double> duration = end - start;
-  std::cout << "[TIMER] run_concurrent_serial_mmul_test_per_actor took: "
-            << duration.count() << " seconds\n";
-}
-
-
-// New state for the GPU supervisor per actor test
-struct gpu_supervisor_per_actor_state {
-  caf::actor gpu_actor;
-  int id = 0;
-  int N = 0;
-  int num_iterations = 0;
-  int count = 0;
-  std::vector<double> kernel_times;
-  std::vector<double> full_times;
-  std::vector<int> h_a;
-  std::vector<int> h_b;
-  std::vector<int> h_c;
-  std::vector<int> h_n;
-};
-
-// New supervisor behavior function
-caf::behavior gpu_supervisor_per_actor_fun(caf::stateful_actor<gpu_supervisor_per_actor_state>* self, int id, int N, int num_iterations) {
-  auto& st = self->state();
-  st.id = id;
-  st.N = N;
-  st.num_iterations = num_iterations;
-  st.count = 0;
-
-  // Initialize matrices
-  st.h_a.resize(st.N * st.N);
-  st.h_b.resize(st.N * st.N);
-  st.h_c.resize(st.N * st.N, 0);
-  st.h_n = {st.N};
-
-  std::generate(st.h_a.begin(), st.h_a.end(), [] { return rand() % 10; });
-  std::generate(st.h_b.begin(), st.h_b.end(), [] { return rand() % 10; });
-
-  // Spawn GPU actor
-  const int THREADS = 32;
-  const int BLOCKS = (st.N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-  st.gpu_actor = caf::cuda::manager::get().spawnFromCUBIN(
-    "../mmul.cubin", "matrixMul", dims,
-    in<int>{}, in<int>{}, out<int>{}, in<int>{}
-  );
-
-  auto run_iteration = [self]() {
-    auto& st_ref = self->state();
-    auto iteration_start = Clock::now();
-
-    auto arg1 = caf::cuda::create_in_arg(st_ref.h_a);
-    auto arg2 = caf::cuda::create_in_arg(st_ref.h_b);
-    auto arg3 = caf::cuda::create_out_arg(st_ref.h_c);
-    auto arg4 = caf::cuda::create_in_arg(st_ref.N);
-
-    auto kernel_start = Clock::now();
-
-    self->mail(st_ref.gpu_actor, arg1, arg2, arg3, arg4)
-      .request(st_ref.gpu_actor, std::chrono::seconds(100))
-      .then(
-        [self, iteration_start, kernel_start](const std::vector<output_buffer>&) {
-          auto& st_ref = self->state();
-          auto kernel_end = Clock::now();
-          auto iteration_end = Clock::now();
-
-          double kernel_time = std::chrono::duration<double>(kernel_end - kernel_start).count();
-          double full_time = std::chrono::duration<double>(iteration_end - iteration_start).count();
-
-          std::cout << "[INFO] [GPU PER ACTOR] Supervisor " << st_ref.id
-                    << " (Matrix Size: " << st_ref.N << "x" << st_ref.N
-                    << ", Total Iterations: " << st_ref.num_iterations << ")"
-                    << " Iteration " << st_ref.count
-                    << " Kernel round-trip: " << kernel_time << " s, "
-                    << "Full iteration time: " << full_time << " s\n";
-
-          st_ref.kernel_times.push_back(kernel_time);
-          st_ref.full_times.push_back(full_time);
-          ++st_ref.count;
-
-          if (st_ref.count < st_ref.num_iterations) {
-            self->mail(std::string("start")).send(self);
-          } else {
-            double kernel_avg = std::accumulate(st_ref.kernel_times.begin(), st_ref.kernel_times.end(), 0.0) / st_ref.kernel_times.size();
-            double full_avg = std::accumulate(st_ref.full_times.begin(), st_ref.full_times.end(), 0.0) / st_ref.full_times.size();
-
-            std::cout << "[INFO] [GPU PER ACTOR] Supervisor " << st_ref.id
-                      << " (Matrix Size: " << st_ref.N << "x" << st_ref.N
-                      << ", Total Iterations: " << st_ref.num_iterations << ")"
-                      << " Kernel average: " << kernel_avg << " s, "
-                      << "Full iteration average: " << full_avg << " s\n";
-
-            self->send_exit(st_ref.gpu_actor, caf::exit_reason::user_shutdown);
-            self->quit();
-          }
+    // Send work to the facade.
+    // The facade will return individual buffers and a completion signal.
+    self->mail(arg1, arg2, arg3, arg4).send(facade);
+
+    return {
+        // Handler for data returned from device (corresponds to OUT/IN_OUT buffers)
+        [=](int r_id, int index, std::vector<int> data) {
+            // index 2 is matrix 'c' in matrixMul(a, b, c, N)
+            if (index == 2) {
+                self->state().h_c = std::move(data);
+            }
         },
-        [self](caf::error& err) {
-          std::cerr << "[ERROR] [GPU PER ACTOR] Kernel execution failed: " << caf::to_string(err) << std::endl;
-          self->quit(err);
-        }
-      );
-  };
-
-  return {
-    [=](const std::string& msg) {
-      if (msg == "start") {
-        run_iteration();
-      }
-    }
-  };
-}
-
-// New test function
-inline void run_concurrent_gpu_mmul_test_per_actor(caf::actor_system& sys,
-                                                   const std::vector<int>& sizes,
-                                                   const std::vector<int>& iterations) {
-  if (sizes.size() != iterations.size()) {
-    std::cerr << "Error: sizes and iterations must have the same length\n";
-    return;
-  }
-
-  int num_supervisors = sizes.size();
-  auto start = Clock::now();
-
-  for (int i = 0; i < num_supervisors; ++i) {
-    int N = sizes[i];
-    int num_iter = iterations[i];
-    auto sup = sys.spawn(gpu_supervisor_per_actor_fun, i, N, num_iter);
-    caf::anon_send(sup, std::string("start"));
-  }
-
-  sys.await_all_actors_done();
-
-  auto end = Clock::now();
-  std::chrono::duration<double> duration = end - start;
-  std::cout << "[TIMER] run_concurrent_gpu_mmul_test_per_actor took: "
-            << duration.count() << " seconds\n";
-}
-
-
-
-void run_gpu_batch_tests(actor_system& sys) {
-    std::vector<int> requested_batch_sizes = {200, 400,600,1000};
-    std::vector<int> base_sizes = {32, 64, 128, 256, 512, 1024, 2048, 4096};
-
-    for (int batch_size : requested_batch_sizes) {
-        // Round up to nearest multiple of 8
-        int adjusted_size = ((batch_size + 7) / 8) * 8;
-
-        std::vector<int> iterations(adjusted_size, 20);
-        std::vector<int> sizes;
-
-        for (int i = 0; i < adjusted_size / static_cast<int>(base_sizes.size()); ++i) {
-            sizes.insert(sizes.end(), base_sizes.begin(), base_sizes.end());
+        // Handler for the completion signal (-1)
+        [=](int r_id, int index) {
+            if (index == -1) {
+                auto end_time = std::chrono::steady_clock::now();
+                std::chrono::duration<double> elapsed = end_time - self->state().start_time;
+                
+                std::cout << "\n===== actor_facade Performance Result =====" << std::endl;
+                std::cout << "Round-trip Latency: " << std::fixed << std::setprecision(6) 
+                          << elapsed.count() << " seconds" << std::endl;
+
+                verify_mmul(self->state().h_a, self->state().h_b, self->state().h_c, self->state().N);
+                
+                self->send_exit(facade, exit_reason::user_shutdown);
+                self->quit();
+            }
         }
-
-        std::cout << "=== Running batch size: " << adjusted_size
-                  << " (original request: " << batch_size << ") ===" << std::endl;
-
-	run_concurrent_gpu_mmul_test_per_actor(sys, sizes, iterations);
-    }
+    };
 }
 
 void caf_main(caf::actor_system& sys) {
-  caf::cuda::manager::init(sys);
-  //test_main(sys);
-  //actor_facade_launch_kernel_test(sys);
-   //test_mmul(sys,1024);
-   //test_mmul_from_ptx(sys,1024);
-   test_mmul_from_cubin(sys,100);
-   //test_mmul_plain(sys,1024);
-  //test_mmul_large(sys);
-  //run_concurrent_serial_mmul_test_global_with_worker(sys,1,1024);
-  //run_concurrent_mmul_validate_test(sys,100,60);
-   //run_all_concurrent_tests(sys);
+    // Initialize the CUDA subsystem
+    caf::cuda::manager::init(sys);
+    auto& mgr = caf::cuda::manager::get();
 
-  //run_concurrent_mmul_test_shared_gpu(sys,2,50);
-  //test_mmul_sync(sys,50);
-//  run_concurrent_mmul_test_global_sync(sys,20,1024);
-  //run_concurrent_mmul_test_sync(sys,50,1024);
+    // Problem size
+    int N = 1024;
+    int THREADS = 32;
+    int BLOCKS = (N + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-   //test_mmul_from_cubin(sys,50);
-   //test_mmul_from_cubin(sys,1024);
-   //run_concurrent_mmul_test_global(sys,50,1024);
-   //run_concurrent_mmul_test(sys,4000,50);
+    // Spawn the actor_facade using the CUBIN file.
+    // Signature: matrixMul(const int* a, const int* b, int* c, int N)
+    auto facade = mgr.spawnFromCUBIN(
+        "../mmul.cubin", "matrixMul", dims,
+        in<int>{}, in<int>{}, out<int>{}, in<int>{});
 
-  //run_concurrent_mmul_validate_test(sys,100,60);
+    // Spawn the test coordinator actor
+    sys.spawn(mmul_facade_test, facade, N);
 
+    sys.await_all_actors_done();
+    caf::cuda::manager::shutdown();
 }
 
-
-
-
-CAF_MAIN()
+CAF_MAIN(cuda)
diff --git a/libcaf_cuda/tests/actor-facade-test/main.test.hpp b/libcaf_cuda/tests/actor-facade-test/main.test.hpp
deleted file mode 100644
index 8f2f2e2042..0000000000
--- a/libcaf_cuda/tests/actor-facade-test/main.test.hpp
+++ /dev/null
@@ -1,14 +0,0 @@
-#pragma once
-
-#include <caf/all.hpp>  // Includes most CAF essentials
-
-#include "caf/cuda/actor_facade.hpp"
-#include "caf/cuda/manager.hpp"
-#include "caf/cuda/nd_range.hpp"
-#include "caf/cuda/all.hpp"
-#include <caf/type_id.hpp>
-#include "caf/detail/test.hpp"
-#include <cassert>
-
-//function signatures of tests
-void test_mmul(caf::actor_system& sys);
diff --git a/libcaf_cuda/tests/actor-facade-test/mmul-actors.cpp b/libcaf_cuda/tests/actor-facade-test/mmul-actors.cpp
deleted file mode 100644
index 9865fb5da4..0000000000
--- a/libcaf_cuda/tests/actor-facade-test/mmul-actors.cpp
+++ /dev/null
@@ -1,125 +0,0 @@
-//file not in use since I am not wasting time messing around with cmake
-
-
-#include "main.test.hpp"
-
-const char* matrixMulKernel = R"(
-extern "C" __global__
-void matrixMul(const int* a, const int* b, int* c, int N) {
-    int row = blockIdx.y * blockDim.y + threadIdx.y;
-    int col = blockIdx.x * blockDim.x + threadIdx.x;
-
-    if (row < N && col < N) {
-        int temp = 0;
-        for (int k = 0; k < N; ++k) {
-            temp += a[row * N + k] * b[k * N + col];
-        }
-        c[row * N + col] = temp;
-    }
-}
-)";
-
-
-
-// Check result on the CPU
-void verify_result(vector<int> &a, vector<int> &b, vector<int> &c, int N) {
-  // For every row...
-  for (int i = 0; i < N; i++) {
-    // For every column...
-    for (int j = 0; j < N; j++) {
-      // For every element in the row-column pair
-      int tmp = 0;
-      for (int k = 0; k < N; k++) {
-        // Accumulate the partial results
-        tmp += a[i * N + k] * b[k * N + j];
-      }
-
-      // Check against the CPU result
-      assert(tmp == c[i * N + j]);
-    }
-  }
-}
-void test_mmul(caf::actor_system& sys) {
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  // Matrix dimension (N x N)
-  int N = 1024;
-  int THREADS = 32;
-  int BLOCKS = N / THREADS;
-
-  // Setup kernel launch configuration
-  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  // Spawn CUDA actor for matrix multiplication kernel
-  auto gpuActor = mgr.spawn(matrixMulKernel, "matrixMul", dim,
-                            in<int>{}, in<int>{}, out<int>{}, in<int>{});
-
-  // Allocate and initialize host matrices
-  std::vector<int> h_a(N * N);
-  std::vector<int> h_b(N * N);
-  std::vector<int> h_c(N * N, 0);  // initialized to 0
-  std::vector<int> h_n(1, N);
-
-  std::generate(h_a.begin(), h_a.end(), []() { return rand() % 10; });
-  std::generate(h_b.begin(), h_b.end(), []() { return rand() % 10; });
-
-  // Compose device arguments
-  auto arg1 = caf::cuda::create_in_arg(h_a);
-  auto arg2 = caf::cuda::create_in_arg(h_b);
-  auto arg3 = caf::cuda::create_out_arg(h_c);
-  auto arg4 = caf::cuda::create_in_arg(h_n);
-
-  // Spawn an actor to send the message and receive the result
-  sys.spawn([=](caf::event_based_actor* self_actor) {
-    self_actor->mail(gpuActor, arg1, arg2, arg3, arg4)
-      .request(gpuActor, 30s).then(
-        [=](const std::vector<output_buffer>& outputs) {
-          std::vector<int> result;
-          bool got_output = false;
-
-          // Extract the result from outputs
-          for (const auto& out : outputs) {
-            std::visit([&](const auto& vec) {
-              if constexpr (std::is_same_v<std::decay_t<decltype(vec)>, std::vector<int>>) {
-                result = vec;
-                got_output = true;
-              }
-            }, out.data);
-          }
-
-          if (!got_output) {
-            aout(self_actor) << "No output data received!\n";
-          } else {
-            aout(self_actor) << "Verifying result..." << std::endl;
-
-            // Verify GPU result against CPU computation
-            std::vector<int> expected(N * N);
-            for (int i = 0; i < N; ++i) {
-              for (int j = 0; j < N; ++j) {
-                int tmp = 0;
-                for (int k = 0; k < N; ++k) {
-                  tmp += h_a[i * N + k] * h_b[k * N + j];
-                }
-                expected[i * N + j] = tmp;
-              }
-            }
-
-            bool success = std::equal(result.begin(), result.end(), expected.begin());
-            if (success) {
-              aout(self_actor) << "Matrix multiplication result verified successfully!\n";
-            } else {
-              aout(self_actor) << "Mismatch found in matrix multiplication results!\n";
-            }
-          }
-
-          self_actor->quit();
-        }
-      );
-  });
-
-  std::this_thread::sleep_for(std::chrono::seconds(5)); // Wait for actor to complete
-}
-
-
-
-

From 09fd919dd5628dbffdb0851eaf1b65968486dc58 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 10:34:29 -0600
Subject: [PATCH 0634/1000] fixed compiler errors and updated spawn helper to
 coninside with new actor facade changes

---
 libcaf_cuda/caf/cuda/actor_facade.hpp   |  2 +-
 libcaf_cuda/caf/detail/spawn_helper.hpp | 47 +++++++++----------------
 2 files changed, 18 insertions(+), 31 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/actor_facade.hpp b/libcaf_cuda/caf/cuda/actor_facade.hpp
index 9f20f3ba85..9bf1faf579 100644
--- a/libcaf_cuda/caf/cuda/actor_facade.hpp
+++ b/libcaf_cuda/caf/cuda/actor_facade.hpp
@@ -85,7 +85,7 @@ class actor_facade : public event_based_actor {
     auto results = runner.run_async(program_, dims_, stream_id, 0, 
                                     device_num, std::forward<Us>(xs)...);
 
-    auto sender = this->current_sender();
+    auto sender = actor_cast<actor>(this->current_sender());
     auto r_id = reply_id_;
 
     if (output_indices.empty()) {
diff --git a/libcaf_cuda/caf/detail/spawn_helper.hpp b/libcaf_cuda/caf/detail/spawn_helper.hpp
index 531d34a23f..cba59a2cbe 100644
--- a/libcaf_cuda/caf/detail/spawn_helper.hpp
+++ b/libcaf_cuda/caf/detail/spawn_helper.hpp
@@ -1,43 +1,30 @@
 #pragma once
 
-#include <functional>
+#include <caf/actor.hpp>
+#include <caf/actor_cast.hpp>
+#include <caf/actor_config.hpp>
+#include <caf/actor_system.hpp>
 
-#include "caf/actor.hpp"
-#include "caf/actor_cast.hpp"
-#include "caf/actor_system.hpp"
-#include "caf/actor_system_config.hpp"
 #include "caf/cuda/actor_facade.hpp"
 #include "caf/cuda/program.hpp"
 #include "caf/cuda/nd_range.hpp"
-#include <type_traits>
 
-//file that aids manager to help spawn in the actor facade
-namespace caf {
-namespace detail {
+namespace caf::detail {
 
+// Helper to spawn actor_facade.
+// Ts... are the types of the kernel arguments (e.g., in<int>, out<int>).
 template <bool PassConfig, class... Ts>
 struct cuda_spawn_helper {
-  using impl = cuda::actor_facade<PassConfig, std::decay_t<Ts>...>;
- 
-  //this operator should spawn in a facade with a program
-  actor operator()(
-		  actor_system * sys,
-		  actor_config&& cfg,
-		   caf::cuda::program_ptr prog,
-		   caf::cuda::nd_range dims,
-                   Ts&&... xs) const {
-    return actor_cast<actor>(impl::create(
-			    sys,
-			    std::move(cfg),              
-			    prog,
-			    dims,
-			    std::forward<Ts>(xs)...));
+  // Us... are the actual values of the kernel arguments passed to spawnFromCUBIN.
+  // These are NOT passed to the actor_facade constructor/create method.
+  // They are sent as a message to the spawned actor_facade later.
+  template <class... Us>
+  caf::actor operator()(caf::actor_system* sys, caf::actor_config&& /*cfg*/,
+                        caf::cuda::program_ptr prog, caf::cuda::nd_range dims,
+                        Us&&... /*xs*/) const {
+    using impl = caf::cuda::actor_facade<PassConfig, Ts...>;
+    return impl::create(*sys, std::move(prog), std::move(dims), 0);
   }
-
-
-
 };
 
-} // namespace detail
-} // namespace caf
-
+} // namespace caf::detail
\ No newline at end of file

From 4db56beefe6659913095b598501dcd0433bc8445 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 10:41:38 -0600
Subject: [PATCH 0635/1000] updated actor facade tests to accomodate the new
 actor facade

---
 .../tests/actor-facade-test/main.test.cpp     | 74 +++++++++++++++----
 1 file changed, 59 insertions(+), 15 deletions(-)

diff --git a/libcaf_cuda/tests/actor-facade-test/main.test.cpp b/libcaf_cuda/tests/actor-facade-test/main.test.cpp
index 3496631138..f605d35917 100644
--- a/libcaf_cuda/tests/actor-facade-test/main.test.cpp
+++ b/libcaf_cuda/tests/actor-facade-test/main.test.cpp
@@ -61,31 +61,70 @@ caf::behavior mmul_facade_test(caf::stateful_actor<test_actor_state>* self,
     auto arg3 = caf::cuda::create_out_arg_with_size<int>(N * N);
     auto arg4 = caf::cuda::create_in_arg(N);
 
-    std::cout << "[INFO] Launching actor_facade test for N=" << N << "..." << std::endl;
+    std::cout << "[INFO] Launching Basic actor_facade test..." << std::endl;
     self->state().start_time = std::chrono::steady_clock::now();
     
-    // Send work to the facade.
-    // The facade will return individual buffers and a completion signal.
+    // Basic call: Copy everything back by default
     self->mail(arg1, arg2, arg3, arg4).send(facade);
 
     return {
-        // Handler for data returned from device (corresponds to OUT/IN_OUT buffers)
         [=](int r_id, int index, std::vector<int> data) {
-            // index 2 is matrix 'c' in matrixMul(a, b, c, N)
-            if (index == 2) {
-                self->state().h_c = std::move(data);
+            if (index == 2) self->state().h_c = std::move(data);
+        },
+        [=](int r_id, int index) {
+            if (index == -1) {
+                auto end_time = std::chrono::steady_clock::now();
+                std::chrono::duration<double> elapsed = end_time - self->state().start_time;
+                std::cout << "[BASIC] Latency: " << elapsed.count() << "s" << std::endl;
+                verify_mmul(self->state().h_a, self->state().h_b, self->state().h_c, self->state().N);
+                self->quit();
             }
+        }
+    };
+}
+
+// New Separate Actor for Advanced Testing (Stream, Device, Selective Index)
+caf::behavior mmul_advanced_facade_test(caf::stateful_actor<test_actor_state>* self, 
+                                        caf::actor facade, int N,
+                                        int device_num, int stream_id,
+                                        std::vector<int> output_indices) {
+    self->state().N = N;
+    self->state().h_a.assign(N * N, 3); // Use different values to ensure fresh run
+    self->state().h_b.assign(N * N, 4);
+    self->state().h_c.resize(N * N);
+
+    auto arg1 = caf::cuda::create_in_arg(self->state().h_a);
+    auto arg2 = caf::cuda::create_in_arg(self->state().h_b);
+    auto arg3 = caf::cuda::create_out_arg_with_size<int>(N * N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    std::cout << "[INFO] Launching Advanced actor_facade test:" << std::endl;
+    std::cout << "       Device: " << device_num << ", Stream: " << stream_id 
+              << ", Indices: { ";
+    for(int i : output_indices) std::cout << i << " ";
+    std::cout << "}" << std::endl;
+              
+    self->state().start_time = std::chrono::steady_clock::now();
+    
+    // Advanced call: Use specific routing and selective index copy-back
+    self->mail(device_num, stream_id, output_indices, arg1, arg2, arg3, arg4).send(facade);
+
+    return {
+        [=](int r_id, int index, std::vector<int> data) {
+            // Verify that we ONLY get index 2, as requested in output_indices
+            bool requested = std::find(output_indices.begin(), output_indices.end(), index) != output_indices.end();
+            if (!requested) {
+                std::cout << "[ERROR] Received unrequested index: " << index << std::endl;
+            }
+            if (index == 2) self->state().h_c = std::move(data);
         },
-        // Handler for the completion signal (-1)
         [=](int r_id, int index) {
             if (index == -1) {
                 auto end_time = std::chrono::steady_clock::now();
                 std::chrono::duration<double> elapsed = end_time - self->state().start_time;
                 
-                std::cout << "\n===== actor_facade Performance Result =====" << std::endl;
-                std::cout << "Round-trip Latency: " << std::fixed << std::setprecision(6) 
-                          << elapsed.count() << " seconds" << std::endl;
-
+                std::cout << "===== Advanced Performance Result =====" << std::endl;
+                std::cout << "Round-trip Latency: " << elapsed.count() << " seconds" << std::endl;
                 verify_mmul(self->state().h_a, self->state().h_b, self->state().h_c, self->state().N);
                 
                 self->send_exit(facade, exit_reason::user_shutdown);
@@ -112,11 +151,16 @@ void caf_main(caf::actor_system& sys) {
         "../mmul.cubin", "matrixMul", dims,
         in<int>{}, in<int>{}, out<int>{}, in<int>{});
 
-    // Spawn the test coordinator actor
-    sys.spawn(mmul_facade_test, facade, N);
+    // 1. Run Basic Test
+    auto basic_tester = sys.spawn(mmul_facade_test, facade, N);
+    
+    // 2. Run Advanced Test (Stream 777, Device 0, Index 2 Only)
+    // We can spawn it now; it will execute after the basic tester finishes or in parallel.
+    // Note: If you want sequential execution, use request().then() or a supervisor.
+    sys.spawn(mmul_advanced_facade_test, facade, N, 0, 777, std::vector<int>{2});
 
     sys.await_all_actors_done();
     caf::cuda::manager::shutdown();
 }
 
-CAF_MAIN(cuda)
+CAF_MAIN(id_block::cuda)

From b1f949b1a3ebb10baf821b85217b89156b3161e3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 10:48:30 -0600
Subject: [PATCH 0636/1000] Initial commit.

---
 .../actor-facade-test/latency_bench.test.cpp  |  93 ++++++++++++++++
 .../throughput_bench.test.cpp                 | 100 ++++++++++++++++++
 2 files changed, 193 insertions(+)
 create mode 100644 libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp
 create mode 100644 libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp

diff --git a/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp b/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp
new file mode 100644
index 0000000000..77e9c8bf83
--- /dev/null
+++ b/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp
@@ -0,0 +1,93 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include <chrono>
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+struct latency_test_state {
+    std::chrono::steady_clock::time_point start_time;
+    int N;
+};
+
+caf::behavior latency_manager(caf::stateful_actor<latency_test_state>* self, 
+                              caf::actor facade, int N) {
+    self->state().N = N;
+    
+    std::vector<int> h_a(N * N, 2);
+    std::vector<int> h_b(N * N, 3);
+    
+    auto arg1 = caf::cuda::create_in_arg(std::move(h_a));
+    auto arg2 = caf::cuda::create_in_arg(std::move(h_b));
+    auto arg3 = caf::cuda::create_out_arg_with_size<int>(N * N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    self->state().start_time = std::chrono::steady_clock::now();
+    
+    // Launch the work
+    self->mail(arg1, arg2, arg3, arg4).send(facade);
+
+    return {
+        [=](int r_id, int index, std::vector<int> data) {
+            // We don't need to do anything with the data for timing
+        },
+        [=](int r_id, int index) {
+            if (index == -1) { // Completion signal
+                auto end_time = std::chrono::steady_clock::now();
+                auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
+                    end_time - self->state().start_time).count();
+                
+                std::cout << "[LATENCY TEST] matrix_size=" << self->state().N 
+                          << ", time=" << elapsed << " ms" << std::endl;
+                
+                self->send_exit(facade, exit_reason::user_shutdown);
+                self->quit();
+            }
+        }
+    };
+}
+
+void run_latency_test(caf::actor_system& sys, int matrix_size) {
+    caf::cuda::manager::init(sys);
+    auto& mgr = caf::cuda::manager::get();
+
+    int THREADS = 32;
+    int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    // Spawn facade
+    auto facade = mgr.spawnFromCUBIN(
+        "../mmul.cubin", "matrixMul", dims,
+        in<int>{}, in<int>{}, out<int>{}, in<int>{});
+
+    sys.spawn(latency_manager, facade, matrix_size);
+    sys.await_all_actors_done();
+    caf::cuda::manager::shutdown();
+}
+
+void caf_main(caf::actor_system& sys) {
+    std::vector<int> sizes = {1000, 4000, 8000, 12000};
+    for (int size : sizes) {
+        run_latency_test(sys, size);
+    }
+}
+
+int main(int argc, char** argv) {
+    core::init_global_meta_objects();
+    actor_system_config cfg;
+    cfg.set("caf.scheduler.max-threads", 1);
+    cfg.set("caf.scheduler.policy", "sharing");
+    
+    auto err = cfg.parse(argc, argv);
+    if (err) return EXIT_FAILURE;
+    if (cfg.helptext_printed()) return 0;
+
+    actor_system sys{cfg};
+    caf_main(sys);
+    
+    return 0;
+}
+
+CAF_MAIN(id_block::cuda)
\ No newline at end of file
diff --git a/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp b/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp
new file mode 100644
index 0000000000..ba7732627f
--- /dev/null
+++ b/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp
@@ -0,0 +1,100 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include <chrono>
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+struct throughput_state {
+    int total_expected = 0;
+    int results_received = 0;
+    std::chrono::steady_clock::time_point start_time;
+    int N;
+};
+
+caf::behavior throughput_manager(caf::stateful_actor<throughput_state>* self, 
+                                 caf::actor facade, int N, int iterations) {
+    self->state().total_expected = iterations;
+    self->state().N = N;
+    self->state().start_time = std::chrono::steady_clock::now();
+
+    std::vector<int> h_a(N * N, 2);
+    std::vector<int> h_b(N * N, 3);
+
+    for (int i = 0; i < iterations; ++i) {
+        auto arg1 = caf::cuda::create_in_arg(h_a);
+        auto arg2 = caf::cuda::create_in_arg(h_b);
+        auto arg3 = caf::cuda::create_out_arg_with_size<int>(N * N);
+        auto arg4 = caf::cuda::create_in_arg(N);
+        self->mail(arg1, arg2, arg3, arg4).send(facade);
+    }
+
+    return {
+        [=](int r_id, int index, std::vector<int> data) {
+            // Ignore data buffers
+        },
+        [=](int r_id, int index) {
+            if (index == -1) { // Completion signal for one request
+                if (++self->state().results_received == self->state().total_expected) {
+                    auto end_time = std::chrono::steady_clock::now();
+                    auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
+                        end_time - self->state().start_time).count();
+                    
+                    std::cout << "[THROUGHPUT TEST] matrix_size=" << self->state().N
+                              << " iterations=" << self->state().total_expected
+                              << ", total_time=" << elapsed << " ms" << std::endl;
+                    
+                    self->send_exit(facade, exit_reason::user_shutdown);
+                    self->quit();
+                }
+            }
+        }
+    };
+}
+
+void run_throughput_test(caf::actor_system& sys, int matrix_size, int iterations) {
+    caf::cuda::manager::init(sys);
+    auto& mgr = caf::cuda::manager::get();
+
+    int THREADS = 32;
+    int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    // Spawn one facade to handle the stream
+    auto facade = mgr.spawnFromCUBIN(
+        "../mmul.cubin", "matrixMul", dims,
+        in<int>{}, in<int>{}, out<int>{}, in<int>{});
+
+    sys.spawn(throughput_manager, facade, matrix_size, iterations);
+    sys.await_all_actors_done();
+    caf::cuda::manager::shutdown();
+}
+
+void caf_main(caf::actor_system& sys) {
+    int fixed_size = 1000;
+    for (int i = 1000; i <= 10000; i += 1000) {
+        run_throughput_test(sys, fixed_size, i);
+    }
+}
+
+int main(int argc, char** argv) {
+    core::init_global_meta_objects();
+    actor_system_config cfg;
+    
+    // Single thread configuration per user snippet
+    cfg.set("caf.scheduler.max-threads", 1);
+    cfg.set("caf.scheduler.policy", "sharing");
+    
+    auto err = cfg.parse(argc, argv);
+    if (err) return EXIT_FAILURE;
+    if (cfg.helptext_printed()) return 0;
+
+    actor_system sys{cfg};
+    caf_main(sys);
+    
+    return 0;
+}
+
+CAF_MAIN(id_block::cuda)
\ No newline at end of file

From b5cd2c37a78266edd654fb02883cc25902fdf640 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 10:49:22 -0600
Subject: [PATCH 0637/1000] Fixed timer.

---
 libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp b/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp
index ba7732627f..7e4a84716a 100644
--- a/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp
+++ b/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp
@@ -18,10 +18,10 @@ caf::behavior throughput_manager(caf::stateful_actor<throughput_state>* self,
                                  caf::actor facade, int N, int iterations) {
     self->state().total_expected = iterations;
     self->state().N = N;
-    self->state().start_time = std::chrono::steady_clock::now();
 
     std::vector<int> h_a(N * N, 2);
     std::vector<int> h_b(N * N, 3);
+    self->state().start_time = std::chrono::steady_clock::now();
 
     for (int i = 0; i < iterations; ++i) {
         auto arg1 = caf::cuda::create_in_arg(h_a);

From 8eb67ea6afbb3814a311b60d44c7df058186223b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 10:51:19 -0600
Subject: [PATCH 0638/1000] Updated cmakelists to build new tests.

---
 .../tests/actor-facade-test/CMakeLists.txt    | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt b/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
index 36f3a20942..3c56e8d703 100644
--- a/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
@@ -31,7 +31,12 @@ include_directories(
 
 # 5) Declare your executable
 add_executable(test main.test.cpp)
+add_executable(latency_bench_test latency_bench.test.cpp)
+add_executable(throughput_bench_test throughput_bench.test.cpp)
+
 target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+target_compile_definitions(latency_bench_test PRIVATE CAF_ENABLE_LOGGING)
+target_compile_definitions(throughput_bench_test PRIVATE CAF_ENABLE_LOGGING)
 
 target_link_libraries(test
   PRIVATE
@@ -41,3 +46,17 @@ target_link_libraries(test
     CUDA::nvrtc
 )
 
+target_link_libraries(latency_bench_test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+target_link_libraries(throughput_bench_test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)

From eaab5ac7149a1163ca573672ac2625ec89fffcd0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 10:53:12 -0600
Subject: [PATCH 0639/1000] Fixed compiler errors.

---
 .../actor-facade-test/latency_bench.test.cpp   |  1 -
 .../throughput_bench.test.cpp                  | 18 ------------------
 2 files changed, 19 deletions(-)

diff --git a/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp b/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp
index 77e9c8bf83..87ee36a202 100644
--- a/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp
+++ b/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp
@@ -90,4 +90,3 @@ int main(int argc, char** argv) {
     return 0;
 }
 
-CAF_MAIN(id_block::cuda)
\ No newline at end of file
diff --git a/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp b/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp
index 7e4a84716a..e720156a2b 100644
--- a/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp
+++ b/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp
@@ -79,22 +79,4 @@ void caf_main(caf::actor_system& sys) {
     }
 }
 
-int main(int argc, char** argv) {
-    core::init_global_meta_objects();
-    actor_system_config cfg;
-    
-    // Single thread configuration per user snippet
-    cfg.set("caf.scheduler.max-threads", 1);
-    cfg.set("caf.scheduler.policy", "sharing");
-    
-    auto err = cfg.parse(argc, argv);
-    if (err) return EXIT_FAILURE;
-    if (cfg.helptext_printed()) return 0;
-
-    actor_system sys{cfg};
-    caf_main(sys);
-    
-    return 0;
-}
-
 CAF_MAIN(id_block::cuda)
\ No newline at end of file

From 15a4f8ba21f756dfd9334d8cfb368604116d7559 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 11:05:23 -0600
Subject: [PATCH 0640/1000] Fixed memory errors.

---
 .../actor-facade-test/latency_bench.test.cpp  | 32 +++++++-----
 .../throughput_bench.test.cpp                 | 49 ++++++++++++++-----
 2 files changed, 58 insertions(+), 23 deletions(-)

diff --git a/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp b/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp
index 87ee36a202..1c2248b6f1 100644
--- a/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp
+++ b/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp
@@ -10,24 +10,33 @@ using namespace std::chrono_literals;
 struct latency_test_state {
     std::chrono::steady_clock::time_point start_time;
     int N;
+    std::vector<int> h_a;
+    std::vector<int> h_b;
+    in<int> arg1;
+    in<int> arg2;
+    out<int> arg3;
+    in<int> arg4;
 };
 
 caf::behavior latency_manager(caf::stateful_actor<latency_test_state>* self, 
                               caf::actor facade, int N) {
-    self->state().N = N;
+    auto& st = self->state();
+    st.N = N;
     
-    std::vector<int> h_a(N * N, 2);
-    std::vector<int> h_b(N * N, 3);
-    
-    auto arg1 = caf::cuda::create_in_arg(std::move(h_a));
-    auto arg2 = caf::cuda::create_in_arg(std::move(h_b));
-    auto arg3 = caf::cuda::create_out_arg_with_size<int>(N * N);
-    auto arg4 = caf::cuda::create_in_arg(N);
+    // Initialize data and reuseable kernel arguments in state
+    st.h_a.assign(N * N, 2);
+    st.h_b.assign(N * N, 3);
+    st.arg1 = caf::cuda::create_in_arg(st.h_a);
+    st.arg2 = caf::cuda::create_in_arg(st.h_b);
+    st.arg3 = caf::cuda::create_out_arg_with_size<int>(N * N);
+    st.arg4 = caf::cuda::create_in_arg(N);
 
-    self->state().start_time = std::chrono::steady_clock::now();
+    // Only copy back index 2 (Matrix C)
+    std::vector<int> output_indices = {2};
+    st.start_time = std::chrono::steady_clock::now();
     
-    // Launch the work
-    self->mail(arg1, arg2, arg3, arg4).send(facade);
+    // Launch the work with selective index-based copy-back
+    self->mail(output_indices, st.arg1, st.arg2, st.arg3, st.arg4).send(facade);
 
     return {
         [=](int r_id, int index, std::vector<int> data) {
@@ -89,4 +98,3 @@ int main(int argc, char** argv) {
     
     return 0;
 }
-
diff --git a/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp b/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp
index e720156a2b..774f7289dd 100644
--- a/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp
+++ b/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp
@@ -12,23 +12,34 @@ struct throughput_state {
     int results_received = 0;
     std::chrono::steady_clock::time_point start_time;
     int N;
+    std::vector<int> h_a;
+    std::vector<int> h_b;
+    in<int> arg1;
+    in<int> arg2;
+    out<int> arg3;
+    in<int> arg4;
 };
 
 caf::behavior throughput_manager(caf::stateful_actor<throughput_state>* self, 
                                  caf::actor facade, int N, int iterations) {
-    self->state().total_expected = iterations;
-    self->state().N = N;
+    auto& st = self->state();
+    st.total_expected = iterations;
+    st.N = N;
 
-    std::vector<int> h_a(N * N, 2);
-    std::vector<int> h_b(N * N, 3);
-    self->state().start_time = std::chrono::steady_clock::now();
+    // Initialize data and reuseable kernel arguments in state
+    st.h_a.assign(N * N, 2);
+    st.h_b.assign(N * N, 3);
+    st.arg1 = caf::cuda::create_in_arg(st.h_a);
+    st.arg2 = caf::cuda::create_in_arg(st.h_b);
+    st.arg3 = caf::cuda::create_out_arg_with_size<int>(N * N);
+    st.arg4 = caf::cuda::create_in_arg(N);
+
+    // Only copy back index 2 (Matrix C)
+    std::vector<int> output_indices = {2};
+    st.start_time = std::chrono::steady_clock::now();
 
     for (int i = 0; i < iterations; ++i) {
-        auto arg1 = caf::cuda::create_in_arg(h_a);
-        auto arg2 = caf::cuda::create_in_arg(h_b);
-        auto arg3 = caf::cuda::create_out_arg_with_size<int>(N * N);
-        auto arg4 = caf::cuda::create_in_arg(N);
-        self->mail(arg1, arg2, arg3, arg4).send(facade);
+        self->mail(output_indices, st.arg1, st.arg2, st.arg3, st.arg4).send(facade);
     }
 
     return {
@@ -79,4 +90,20 @@ void caf_main(caf::actor_system& sys) {
     }
 }
 
-CAF_MAIN(id_block::cuda)
\ No newline at end of file
+int main(int argc, char** argv) {
+    core::init_global_meta_objects();
+    actor_system_config cfg;
+    
+    // Single thread configuration per user snippet
+    cfg.set("caf.scheduler.max-threads", 1);
+    cfg.set("caf.scheduler.policy", "sharing");
+    
+    auto err = cfg.parse(argc, argv);
+    if (err) return EXIT_FAILURE;
+    if (cfg.helptext_printed()) return 0;
+
+    actor_system sys{cfg};
+    caf_main(sys);
+    
+    return 0;
+}

From 78a9451a9d5f540f3054e496463d66b017a96d7e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 11:30:19 -0600
Subject: [PATCH 0641/1000] Added output mapping capabilites to the actor
 facade. Change was made so that if a user wants to copy an argument back to a
 specific buffer they can

---
 libcaf_cuda/caf/cuda/actor_facade.hpp         | 75 +++++++++++++++----
 libcaf_cuda/caf/cuda/global.hpp               | 18 +++++
 libcaf_cuda/caf/cuda/types.hpp                |  8 +-
 .../tests/actor-facade-test/main.test.cpp     | 53 +++++++++++++
 4 files changed, 136 insertions(+), 18 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/actor_facade.hpp b/libcaf_cuda/caf/cuda/actor_facade.hpp
index 9bf1faf579..f328ef6bff 100644
--- a/libcaf_cuda/caf/cuda/actor_facade.hpp
+++ b/libcaf_cuda/caf/cuda/actor_facade.hpp
@@ -2,6 +2,7 @@
 
 #include <functional>
 #include <queue>
+#include <algorithm>
 
 #include <caf/actor.hpp>
 #include <caf/actor_cast.hpp>
@@ -54,33 +55,44 @@ class actor_facade : public event_based_actor {
   caf::behavior make_behavior() override {
     return {
       [this](int device_num, int stream_id, std::vector<int> output_indices, Ts... args) {
-        enqueue_impl(device_num, stream_id, std::move(output_indices), std::forward<Ts>(args)...);
+        enqueue_impl(device_num, stream_id, std::move(output_indices), {}, std::forward<Ts>(args)...);
       },
       [this](int device_num, int stream_id, Ts... args) {
-        enqueue_impl(device_num, stream_id, {}, std::forward<Ts>(args)...);
+        enqueue_impl(device_num, stream_id, {}, {}, std::forward<Ts>(args)...);
       },
       [this](int device_num, std::vector<int> output_indices, Ts... args) {
-        enqueue_impl(device_num, static_cast<int>(actor_id_), std::move(output_indices),
+        enqueue_impl(device_num, static_cast<int>(actor_id_), std::move(output_indices), {},
                      std::forward<Ts>(args)...);
       },
       [this](std::vector<int> output_indices, Ts... args) {
-        enqueue_impl(-1, static_cast<int>(actor_id_), std::move(output_indices),
+        enqueue_impl(-1, static_cast<int>(actor_id_), std::move(output_indices), {},
                      std::forward<Ts>(args)...);
       },
       [this](int device_num, Ts... args) {
         // Copy everything back if indices are omitted
-        enqueue_impl(device_num, static_cast<int>(actor_id_), {}, std::forward<Ts>(args)...);
+        enqueue_impl(device_num, static_cast<int>(actor_id_), {}, {}, std::forward<Ts>(args)...);
       },
       [this](Ts... args) {
         // Copy everything back if indices are omitted
-        enqueue_impl(-1, static_cast<int>(actor_id_), {}, std::forward<Ts>(args)...);
+        enqueue_impl(-1, static_cast<int>(actor_id_), {}, {}, std::forward<Ts>(args)...);
+      },
+      // Mapping handlers
+      [this](int device_num, int stream_id, std::vector<output_mapping> mappings, Ts... args) {
+        enqueue_impl(device_num, stream_id, {}, std::move(mappings), std::forward<Ts>(args)...);
+      },
+      [this](int device_num, std::vector<output_mapping> mappings, Ts... args) {
+        enqueue_impl(device_num, static_cast<int>(actor_id_), {}, std::move(mappings), std::forward<Ts>(args)...);
+      },
+      [this](std::vector<output_mapping> mappings, Ts... args) {
+        enqueue_impl(-1, static_cast<int>(actor_id_), {}, std::move(mappings), std::forward<Ts>(args)...);
       }
     };
   }
 
 private:
   template <class... Us>
-  void enqueue_impl(int device_num, int stream_id, std::vector<int> output_indices, Us&&... xs) {
+  void enqueue_impl(int device_num, int stream_id, std::vector<int> output_indices, 
+                    std::vector<output_mapping> mappings, Us&&... xs) {
     command_runner<Ts...> runner;
     auto results = runner.run_async(program_, dims_, stream_id, 0, 
                                     device_num, std::forward<Us>(xs)...);
@@ -88,16 +100,24 @@ class actor_facade : public event_based_actor {
     auto sender = actor_cast<actor>(this->current_sender());
     auto r_id = reply_id_;
 
-    if (output_indices.empty()) {
+    // Determine which indices to process based on requests and mappings
+    std::vector<int> targets = output_indices;
+    for (const auto& m : mappings) {
+      if (std::find(targets.begin(), targets.end(), m.index) == targets.end()) {
+        targets.push_back(m.index);
+      }
+    }
+
+    if (targets.empty() && mappings.empty()) {
       for (int i = 0; i < static_cast<int>(sizeof...(Ts)); ++i) {
-        output_indices.push_back(i);
+        targets.push_back(i);
       }
     }
 
-    for (int idx : output_indices) {
+    for (int idx : targets) {
       if (idx >= 0 && idx < static_cast<int>(sizeof...(Ts))) {
         // Dispatch runtime index to compile-time sequence
-        dispatch_index(idx, [&](auto current_idx_constant) {
+        dispatch_index(idx, [&](auto current_idx_constant) mutable {
           constexpr std::size_t Index = current_idx_constant;
           using MemPtrType = std::tuple_element_t<Index, mem_tuple>;
           using ValueType = typename MemPtrType::element_type::value_type;
@@ -105,12 +125,35 @@ class actor_facade : public event_based_actor {
           MemPtrType mem_ptr = std::get<Index>(results);
 
           if (mem_ptr && (mem_ptr->access() == OUT || mem_ptr->access() == IN_OUT)) {
-            runner.copy_to_host_async(mem_ptr, [sender, r_id, Index](std::vector<ValueType>&& data) {
-              if (sender) {
-                // Send: Correlation ID, Argument Index, Data Vector
-                caf::anon_mail(r_id, static_cast<int>(Index), std::move(data)).send(sender);
+            // Check if a custom destination is provided for this index
+            void* custom_dst = nullptr;
+            size_t dst_count = 0;
+            for (const auto& m : mappings) {
+              if (m.index == idx) {
+                custom_dst = m.dst;
+                dst_count = m.count;
+                break;
               }
-            });
+            }
+
+            if (custom_dst) {
+              // Copy into user-provided buffer
+              runner.copy_to_host_async(mem_ptr, static_cast<ValueType*>(custom_dst), dst_count, 
+                [sender, r_id, Index](ValueType*, size_t) {
+                  if (sender) {
+                    // Notify requester that this index is ready in their buffer
+                    caf::anon_mail(r_id, static_cast<int>(Index)).send(sender);
+                  }
+                });
+            } else {
+              // Default: Copy into a new vector and send back
+              runner.copy_to_host_async(mem_ptr, [sender, r_id, Index](std::vector<ValueType>&& data) {
+                if (sender) {
+                  // Send: Correlation ID, Argument Index, Data Vector
+                  caf::anon_mail(r_id, static_cast<int>(Index), std::move(data)).send(sender);
+                }
+              });
+            }
           }
         });
       } else {
diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index 9b32b4a77d..dad370ec61 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -117,6 +117,20 @@ bool inspect(Inspector& f, std::vector<output_buffer>& x) {
   return f.object(x).fields(f.field("elements", x));
 }
 
+// Serialization support for output_mapping (global namespace)
+template <class Inspector>
+bool inspect(Inspector& f, output_mapping& x) {
+  return f.object(x).fields(f.field("index", x.index),
+                            f.field("dst", x.dst),
+                            f.field("count", x.count));
+}
+
+// Serialization support for std::vector<output_mapping> (global namespace)
+template <class Inspector>
+bool inspect(Inspector& f, std::vector<output_mapping>& x) {
+  return f.object(x).fields(f.field("elements", x));
+}
+
 // Serialization support for raw vector types
 template <class Inspector>
 bool inspect(Inspector& f, std::vector<char>& x) {
@@ -178,6 +192,8 @@ CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id)
   CAF_ADD_TYPE_ID(cuda, (buffer_variant))
   CAF_ADD_TYPE_ID(cuda, (output_buffer))
   CAF_ADD_TYPE_ID(cuda, (std::vector<output_buffer>))
+  CAF_ADD_TYPE_ID(cuda, (output_mapping))
+  CAF_ADD_TYPE_ID(cuda, (std::vector<output_mapping>))
   CAF_ADD_TYPE_ID(cuda,(caf::cuda::mem_ptr<int>))  
   CAF_ADD_TYPE_ID(cuda,(caf::cuda::mem_ptr<float>))  
   CAF_ADD_TYPE_ID(cuda,(caf::cuda::mem_ptr<double>))  
@@ -200,3 +216,5 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<double>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<char>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::nd_range)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::program_ptr)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(output_mapping)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<output_mapping>)
diff --git a/libcaf_cuda/caf/cuda/types.hpp b/libcaf_cuda/caf/cuda/types.hpp
index f31aa0c1f4..db479b4d55 100644
--- a/libcaf_cuda/caf/cuda/types.hpp
+++ b/libcaf_cuda/caf/cuda/types.hpp
@@ -55,6 +55,12 @@ class actor_facade;
 
 } // namespace caf::cuda
 
+// Structure for mapping kernel output indices to specific host memory buffers
+struct output_mapping {
+  int index;
+  void* dst; // Destination host pointer
+  size_t count; // Number of elements to copy
+};
 
 // === buffer_variant and output_buffer outside namespace or inside as needed ===
 
@@ -298,5 +304,3 @@ struct raw_type<caf::cuda::mem_ptr<T>> {
 
 template <typename T>
 using raw_t = typename raw_type<T>::type;
-
-
diff --git a/libcaf_cuda/tests/actor-facade-test/main.test.cpp b/libcaf_cuda/tests/actor-facade-test/main.test.cpp
index f605d35917..c3d78d13c5 100644
--- a/libcaf_cuda/tests/actor-facade-test/main.test.cpp
+++ b/libcaf_cuda/tests/actor-facade-test/main.test.cpp
@@ -134,6 +134,56 @@ caf::behavior mmul_advanced_facade_test(caf::stateful_actor<test_actor_state>* s
     };
 }
 
+// Test actor behavior for output_mapping
+caf::behavior mmul_mapping_test(caf::stateful_actor<test_actor_state>* self, 
+                                caf::actor facade, int N) {
+    self->state().N = N;
+    self->state().h_a.assign(N * N, 5);
+    self->state().h_b.assign(N * N, 6);
+    self->state().h_c.assign(N * N, 0); // Target buffer for direct copy
+
+    auto arg1 = caf::cuda::create_in_arg(self->state().h_a);
+    auto arg2 = caf::cuda::create_in_arg(self->state().h_b);
+    auto arg3 = caf::cuda::create_out_arg_with_size<int>(N * N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    // Define the output mapping for index 2 (Matrix C)
+    output_mapping m{2, self->state().h_c.data(), self->state().h_c.size()};
+    std::vector<output_mapping> mappings{m};
+
+    std::cout << "[INFO] Launching output_mapping test..." << std::endl;
+    self->state().start_time = std::chrono::steady_clock::now();
+    
+    // Launch using the mapping-enabled overload
+    self->mail(mappings, arg1, arg2, arg3, arg4).send(facade);
+
+    return {
+        [=](int r_id, int index, std::vector<int> data) {
+            // This should not be called for index 2 because it is mapped
+            if (index == 2) {
+                std::cout << "[ERROR] Received data vector for mapped index 2!" << std::endl;
+            }
+        },
+        [=](int r_id, int index) {
+            if (index == 2) {
+                std::cout << "[MAPPING] Received notification for mapped index 2." << std::endl;
+            } else if (index == -1) {
+                auto end_time = std::chrono::steady_clock::now();
+                std::chrono::duration<double> elapsed = end_time - self->state().start_time;
+                
+                std::cout << "===== Mapping Performance Result =====" << std::endl;
+                std::cout << "Round-trip Latency: " << elapsed.count() << " seconds" << std::endl;
+                
+                // Verify the data was copied directly into h_c
+                verify_mmul(self->state().h_a, self->state().h_b, self->state().h_c, self->state().N);
+                
+                self->send_exit(facade, exit_reason::user_shutdown);
+                self->quit();
+            }
+        }
+    };
+}
+
 void caf_main(caf::actor_system& sys) {
     // Initialize the CUDA subsystem
     caf::cuda::manager::init(sys);
@@ -159,6 +209,9 @@ void caf_main(caf::actor_system& sys) {
     // Note: If you want sequential execution, use request().then() or a supervisor.
     sys.spawn(mmul_advanced_facade_test, facade, N, 0, 777, std::vector<int>{2});
 
+    // 3. Run Mapping Test
+    sys.spawn(mmul_mapping_test, facade, N);
+
     sys.await_all_actors_done();
     caf::cuda::manager::shutdown();
 }

From 9c5fef8816960869d90d09da8a5dedf01a06f955 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 11:34:40 -0600
Subject: [PATCH 0642/1000] Implemented test where there is a global buffer so
 it can be used for comparision with other methods.

---
 .../tests/actor-facade-test/CMakeLists.txt    |  10 ++
 .../throughput_mapping_bench.test.cpp         | 105 ++++++++++++++++++
 2 files changed, 115 insertions(+)
 create mode 100644 libcaf_cuda/tests/actor-facade-test/throughput_mapping_bench.test.cpp

diff --git a/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt b/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
index 3c56e8d703..befb04c112 100644
--- a/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
@@ -33,10 +33,12 @@ include_directories(
 add_executable(test main.test.cpp)
 add_executable(latency_bench_test latency_bench.test.cpp)
 add_executable(throughput_bench_test throughput_bench.test.cpp)
+add_executable(throughput_mapping_bench_test throughput_mapping_bench.test.cpp)
 
 target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
 target_compile_definitions(latency_bench_test PRIVATE CAF_ENABLE_LOGGING)
 target_compile_definitions(throughput_bench_test PRIVATE CAF_ENABLE_LOGGING)
+target_compile_definitions(throughput_mapping_bench_test PRIVATE CAF_ENABLE_LOGGING)
 
 target_link_libraries(test
   PRIVATE
@@ -45,6 +47,14 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
 )
+target_link_libraries(throughput_mapping_bench_test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+)
 
 target_link_libraries(latency_bench_test
   PRIVATE
diff --git a/libcaf_cuda/tests/actor-facade-test/throughput_mapping_bench.test.cpp b/libcaf_cuda/tests/actor-facade-test/throughput_mapping_bench.test.cpp
new file mode 100644
index 0000000000..581b4dab25
--- /dev/null
+++ b/libcaf_cuda/tests/actor-facade-test/throughput_mapping_bench.test.cpp
@@ -0,0 +1,105 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include <chrono>
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+struct mapping_throughput_state {
+    int total_expected = 0;
+    int results_received = 0;
+    std::chrono::steady_clock::time_point start_time;
+    int N;
+    std::vector<int> h_a;
+    std::vector<int> h_b;
+    std::vector<int> h_c_global; // The persistent buffer
+    in<int> arg1;
+    in<int> arg2;
+    out<int> arg3;
+    in<int> arg4;
+};
+
+caf::behavior throughput_mapping_manager(caf::stateful_actor<mapping_throughput_state>* self, 
+                                         caf::actor facade, int N, int iterations) {
+    auto& st = self->state();
+    st.total_expected = iterations;
+    st.N = N;
+
+    // Initialize data and reuseable kernel arguments
+    st.h_a.assign(N * N, 2);
+    st.h_b.assign(N * N, 3);
+    st.h_c_global.assign(N * N, 0); // Pre-allocate the global destination
+    
+    st.arg1 = caf::cuda::create_in_arg(st.h_a);
+    st.arg2 = caf::cuda::create_in_arg(st.h_b);
+    st.arg3 = caf::cuda::create_out_arg_with_size<int>(N * N);
+    st.arg4 = caf::cuda::create_in_arg(N);
+
+    // Define the mapping once
+    output_mapping mapping{2, st.h_c_global.data(), st.h_c_global.size()};
+    std::vector<output_mapping> mappings = {mapping};
+
+    st.start_time = std::chrono::steady_clock::now();
+
+    for (int i = 0; i < iterations; ++i) {
+        // Send using the mappings overload
+        self->mail(mappings, st.arg1, st.arg2, st.arg3, st.arg4).send(facade);
+    }
+
+    return {
+        [=](int r_id, int index) {
+            if (index == -1) { // Completion signal for the whole message
+                if (++self->state().results_received == self->state().total_expected) {
+                    auto end_time = std::chrono::steady_clock::now();
+                    auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
+                        end_time - self->state().start_time).count();
+                    
+                    std::cout << "[MAPPING THROUGHPUT TEST] matrix_size=" << self->state().N
+                              << " iterations=" << self->state().total_expected
+                              << ", total_time=" << elapsed << " ms" << std::endl;
+                    
+                    self->send_exit(facade, exit_reason::user_shutdown);
+                    self->quit();
+                }
+            }
+            // Note: index == 2 is also received as a notification, but we only 
+            // care about the final -1 to signal everything (kernel + copy) is done.
+        }
+    };
+}
+
+void run_mapping_throughput_test(caf::actor_system& sys, int matrix_size, int iterations) {
+    caf::cuda::manager::init(sys);
+    auto& mgr = caf::cuda::manager::get();
+
+    int THREADS = 32;
+    int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+    auto facade = mgr.spawnFromCUBIN(
+        "../mmul.cubin", "matrixMul", dims,
+        in<int>{}, in<int>{}, out<int>{}, in<int>{});
+
+    sys.spawn(throughput_mapping_manager, facade, matrix_size, iterations);
+    sys.await_all_actors_done();
+    caf::cuda::manager::shutdown();
+}
+
+void caf_main(caf::actor_system& sys) {
+    int fixed_size = 1000;
+    for (int i = 1000; i <= 10000; i += 1000) {
+        run_mapping_throughput_test(sys, fixed_size, i);
+    }
+}
+
+int main(int argc, char** argv) {
+    core::init_global_meta_objects();
+    actor_system_config cfg;
+    cfg.set("caf.scheduler.max-threads", 1);
+    cfg.set("caf.scheduler.policy", "sharing");
+    actor_system sys{cfg.parse(argc, argv)};
+    caf_main(sys);
+    return 0;
+}
\ No newline at end of file

From cbcf54ac0ae0b19bd94d30a2b6cc692007735e34 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 11:36:54 -0600
Subject: [PATCH 0643/1000] Fixed errors.

---
 libcaf_cuda/tests/actor-facade-test/CMakeLists.txt         | 2 +-
 .../actor-facade-test/throughput_mapping_bench.test.cpp    | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt b/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
index befb04c112..f6f5dc669c 100644
--- a/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
@@ -54,7 +54,7 @@ target_link_libraries(throughput_mapping_bench_test
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
 )
-)
+
 
 target_link_libraries(latency_bench_test
   PRIVATE
diff --git a/libcaf_cuda/tests/actor-facade-test/throughput_mapping_bench.test.cpp b/libcaf_cuda/tests/actor-facade-test/throughput_mapping_bench.test.cpp
index 581b4dab25..0333126314 100644
--- a/libcaf_cuda/tests/actor-facade-test/throughput_mapping_bench.test.cpp
+++ b/libcaf_cuda/tests/actor-facade-test/throughput_mapping_bench.test.cpp
@@ -99,7 +99,12 @@ int main(int argc, char** argv) {
     actor_system_config cfg;
     cfg.set("caf.scheduler.max-threads", 1);
     cfg.set("caf.scheduler.policy", "sharing");
-    actor_system sys{cfg.parse(argc, argv)};
+
+    auto err = cfg.parse(argc, argv);
+    if (err) return EXIT_FAILURE;
+    if (cfg.helptext_printed()) return 0;
+
+    actor_system sys{cfg};
     caf_main(sys);
     return 0;
 }
\ No newline at end of file

From 891c636affe20fc1646ec23b477bda7bd25e1da8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 12:05:54 -0600
Subject: [PATCH 0644/1000] updated in and in_out types to capture raw
 pointers. This change is being implemented so that people are not bound to
 just using std::vector objects but instead can use custom data structures as
 they see fit, so long as it can be copied on the GPU

---
 libcaf_cuda/caf/cuda/helpers.hpp | 12 ++++++++++++
 libcaf_cuda/caf/cuda/types.hpp   |  8 ++++++++
 2 files changed, 20 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/helpers.hpp b/libcaf_cuda/caf/cuda/helpers.hpp
index b151625795..80be75ec4a 100644
--- a/libcaf_cuda/caf/cuda/helpers.hpp
+++ b/libcaf_cuda/caf/cuda/helpers.hpp
@@ -63,6 +63,12 @@ in<T> create_in_arg(const std::vector<T>& buffer) {
   return in<T>{buffer};
 }
 
+//creates a tag of a readonly buffer on the gpu from raw pointer
+template <typename T>
+in<T> create_in_arg(const T* ptr, size_t size) {
+  return in<T>{ptr, size};
+}
+
 template <typename T>
 in<T> create_in_out_arg(std::vector<T>&& buffer) {
     return in<T>{std::move(buffer)}; // moves into variant, no copy
@@ -107,4 +113,10 @@ in_out<T> create_in_out_arg(const std::vector<T>& buffer) {
   return in_out<T>{buffer};
 }
 
+// Create `in_out<T>` from raw pointer
+template <typename T>
+in_out<T> create_in_out_arg(const T* ptr, size_t size) {
+  return in_out<T>{ptr, size};
+}
+
 } //namespace caf cuda
diff --git a/libcaf_cuda/caf/cuda/types.hpp b/libcaf_cuda/caf/cuda/types.hpp
index db479b4d55..bd2466840d 100644
--- a/libcaf_cuda/caf/cuda/types.hpp
+++ b/libcaf_cuda/caf/cuda/types.hpp
@@ -105,6 +105,10 @@ class in_impl {
   explicit in_impl(std::vector<T>&& buf)
     : scalar_{}, ptr_{buf.data()}, size_{buf.size()}, is_scalar_{false} {}
 
+  // raw pointer constructor
+  explicit in_impl(const T* ptr, size_t size)
+    : scalar_{}, ptr_{ptr}, size_{size}, is_scalar_{false} {}
+
   bool is_scalar() const {
     check_valid();
     return is_scalar_;
@@ -232,6 +236,10 @@ class in_out_impl {
   explicit in_out_impl(std::vector<T>&& buf)
     : scalar_{}, ptr_{buf.data()}, size_{buf.size()}, is_scalar_{false} {}
 
+  // raw pointer constructor
+  explicit in_out_impl(const T* ptr, size_t size)
+    : scalar_{}, ptr_{ptr}, size_{size}, is_scalar_{false} {}
+
   bool is_scalar() const {
     check_valid();
     return is_scalar_;

From 59de3c7e9932e7774708e62b6a3748b77cd89d11 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 12:37:37 -0600
Subject: [PATCH 0645/1000] Implemented functionality such that can reset cuda
 context of a given device number

---
 libcaf_cuda/caf/cuda/command_runner.hpp |  8 ++++++++
 libcaf_cuda/caf/cuda/device.hpp         | 11 +++++++++++
 libcaf_cuda/caf/cuda/platform.hpp       | 21 ++++++++++++++++++++-
 libcaf_cuda/caf/cuda/streampool.hpp     |  7 ++++++-
 4 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index cba310ad0c..f69cff7983 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -296,6 +296,14 @@ class command_runner {
       if (res != CUDA_SUCCESS) { delete f_ptr; check(res, "cuLaunchHostFunc"); }
   }
 
+  // -------------------------------
+  // Resets the CUDA context for a given device number.
+  // This will force the device to flush its stream pool and create a new context.
+  // -------------------------------
+  void reset_context(int device_number) {
+      auto plat = platform::create();
+      plat->reset_device_context(device_number);
+  }
 
 
 
diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index bc41ecf2af..9390fcc74b 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -55,6 +55,17 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   CUcontext getContext(int) { return context_; }
 
 
+  // Resets the CUDA context for this device.
+  // Destroys the old context and reinitializes the stream pool with the new one.
+  void reset_context(CUcontext new_ctx) {
+    // Destroy the old context
+    if (context_) {
+      check(cuCtxDestroy(context_), "cuCtxDestroy in device::reset_context");
+    }
+    // Set the new context and reinitialize the stream table
+    context_ = new_ctx;
+    stream_table_ = DeviceStreamTable(context_, stream_table_.pool_size());
+  }
   // Number of streaming multiprocessors (SMs)
   int num_sms() const noexcept { return sm_count_; }
 
diff --git a/libcaf_cuda/caf/cuda/platform.hpp b/libcaf_cuda/caf/cuda/platform.hpp
index 26e61aea99..87f06449ce 100644
--- a/libcaf_cuda/caf/cuda/platform.hpp
+++ b/libcaf_cuda/caf/cuda/platform.hpp
@@ -46,6 +46,26 @@ class CAF_CUDA_EXPORT platform : public ref_counted {
   //releases a stream for an actor
   void release_streams_for_actor(int actor_id);
 
+  // Resets the CUDA context for a specific device.
+  // This will destroy the existing context for the device, create a new one,
+  // and update the device object and platform's internal records.
+  void reset_device_context(int device_id) {
+    if (device_id < 0 || device_id >= static_cast<int>(devices_.size())) {
+      throw std::out_of_range("Invalid device_id for reset_device_context");
+    }
+
+    device_ptr dev_obj = devices_[device_id];
+    CUdevice cu_dev = dev_obj->getDevice(); // Get the underlying CUdevice
+
+    // Create a new context
+    CUcontext new_ctx;
+    CHECK_CUDA(cuCtxCreate(&new_ctx, 0, cu_dev));
+
+    // Update the device object and platform's internal contexts_ vector
+    dev_obj->reset_context(new_ctx);
+    contexts_[device_id] = new_ctx;
+  }
+
   //returns how many devices are currently on the GPU
   int get_num_devices();
 
@@ -67,4 +87,3 @@ inline void intrusive_ptr_add_ref(platform* p) { p->ref(); }
 inline void intrusive_ptr_release(platform* p) { p->deref(); }
 
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/streampool.hpp b/libcaf_cuda/caf/cuda/streampool.hpp
index a0e17bf78b..1e3eef5bf5 100644
--- a/libcaf_cuda/caf/cuda/streampool.hpp
+++ b/libcaf_cuda/caf/cuda/streampool.hpp
@@ -52,6 +52,9 @@ class CAF_CUDA_EXPORT StreamPool {
   size_t num_available() const;
 
 private:
+  // Add a getter for max_size_
+  size_t max_size() const { return max_size_; }
+
   /// Create a new CUDA stream in the context `ctx_`.
   CUstream create_stream();
 
@@ -81,6 +84,9 @@ class CAF_CUDA_EXPORT DeviceStreamTable {
   /// Release the stream assigned to an actor back to the pool and erase the mapping.
   void release_stream(int actor_id);
 
+  // Add a getter for pool_size
+  size_t pool_size() const { return pool_.max_size(); }
+
 private:
   StreamPool pool_;
   std::unordered_map<int, CUstream> table_; ///< actor_id -> stream
@@ -88,4 +94,3 @@ class CAF_CUDA_EXPORT DeviceStreamTable {
 };
 
 } // namespace caf::cuda
-

From fdc4f3d46b107fafa538dd1c2cc9bab2590366f8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 12:40:35 -0600
Subject: [PATCH 0646/1000] intial commit.

---
 .../many-independent-tasks/CMakeLists.txt     |  46 ++
 .../many-independent-tasks/compile_kernels.sh |  14 +
 .../many-independent-tasks/conv1d.cu          |  15 +
 .../many-independent-tasks/cuda-baseline.cpp  | 346 ++++++++++++
 .../many-independent-tasks/mmul.cu            |  16 +
 .../many-independent-tasks/vector_add.cu      |   6 +
 .../many-independent-tasks/work-stealing.cpp  | 496 ++++++++++++++++++
 7 files changed, 939 insertions(+)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/conv1d.cu
 create mode 100644 libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/cuda-baseline.cpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/mmul.cu
 create mode 100644 libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/vector_add.cu
 create mode 100644 libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/CMakeLists.txt b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/CMakeLists.txt
new file mode 100644
index 0000000000..cad90b5763
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/CMakeLists.txt
@@ -0,0 +1,46 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executables
+
+add_executable(work-stealing work-stealing.cpp)
+target_compile_definitions(work-stealing PRIVATE CAF_ENABLE_LOGGING)
+
+add_executable(cuda-baseline cuda-baseline.cpp)
+target_link_libraries(cuda-baseline PRIVATE CUDA::cuda_driver)
+
+target_link_libraries(work-stealing
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
diff --git a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/compile_kernels.sh b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/compile_kernels.sh
new file mode 100755
index 0000000000..c0bebf8833
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/compile_kernels.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability to ensure binary compatibility
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile kernels to cubin for Driver API loading
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+nvcc -arch=$SM_ARCH -cubin vector_add.cu -o vector_add.cubin
+nvcc -arch=$SM_ARCH -cubin conv1d.cu -o conv1d.cubin
+
+echo "Kernels compiled successfully for $SM_ARCH."
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/conv1d.cu b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/conv1d.cu
new file mode 100644
index 0000000000..aa897ede25
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/conv1d.cu
@@ -0,0 +1,15 @@
+extern "C" __global__ void conv1d(const int* A, const int* K, int* C, int N) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int W = 5; // Fixed filter width for benchmark simplicity
+    if (idx < N) {
+        int sum = 0;
+        int halfW = W / 2;
+        for (int i = 0; i < W; ++i) {
+            int col = idx + i - halfW;
+            if (col >= 0 && col < N) {
+                sum += A[col] * K[i];
+            }
+        }
+        C[idx] = sum;
+    }
+}
diff --git a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/cuda-baseline.cpp b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/cuda-baseline.cpp
new file mode 100644
index 0000000000..0d1c6fe1af
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/cuda-baseline.cpp
@@ -0,0 +1,346 @@
+#include <cuda.h> // CUDA Driver API
+#include <iostream>
+#include <vector>
+#include <chrono>
+#include <random>
+#include <thread>
+#include <algorithm>
+#include <functional>
+#include <stdexcept> // For runtime_error
+#include <unordered_map> // For MatrixPool
+#include <unordered_set> // For create_matrix_pool_random
+
+enum TaskType { MMUL = 0, VADD = 1, CONV = 2 };
+
+struct Task {
+    int N;
+    TaskType type;
+};
+
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
+    std::unordered_map<int, std::vector<int>> vec_A;
+    std::unordered_map<int, std::vector<int>> vec_B;
+    std::unordered_map<int, std::vector<int>> conv_A;
+    std::unordered_map<int, std::vector<int>> conv_K;
+};
+
+// Per-GPU execution logic
+void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_gpu, CUcontext ctx, CUfunction mmul_func,
+     CUfunction vadd_func, CUfunction conv_func, const MatrixPool& pool, int* shared_dtoh_buffer) {
+    // Set the CUDA context for this thread
+    CUresult err = cuCtxSetCurrent(ctx);
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error setting context for device " << device_id << ": " << err_str << std::endl;
+        return;
+    }
+
+    // Prepare Streams
+    std::vector<CUstream> streams(streams_per_gpu);
+    for (int i = 0; i < streams_per_gpu; ++i) {
+        cuStreamCreate(&streams[i], CU_STREAM_NON_BLOCKING);
+    }
+
+    // Use the first stream for initial allocations and cleanup
+    CUstream default_stream = streams[0];
+
+    // Process assigned tasks
+    for (size_t i = 0; i < tasks.size(); ++i) {
+        int N = tasks[i].N;
+        CUstream stream = streams[i % streams_per_gpu];
+        TaskType type = tasks[i].type;
+        size_t bytes_a = (type == MMUL) ? (size_t)N * N * sizeof(int) : (size_t)N * sizeof(int);
+        size_t bytes_b = (type == CONV) ? 5 * sizeof(int) : bytes_a;
+        size_t bytes_out = (type == MMUL) ? (size_t)N * N * sizeof(int) : (size_t)N * sizeof(int);
+
+        CUdeviceptr d_a, d_b, d_c;
+
+        cuMemAllocAsync(&d_a, bytes_a, stream);
+        cuMemAllocAsync(&d_b, bytes_b, stream);
+        cuMemAllocAsync(&d_c, bytes_out, stream);
+
+        // Perform Host-to-Device transfer
+        const std::vector<int>& h_a = (type == MMUL) ? pool.A.at(N) : (type == VADD ? pool.vec_A.at(N) : pool.conv_A.at(N));
+        const std::vector<int>& h_b = (type == MMUL) ? pool.B.at(N) : (type == VADD ? pool.vec_B.at(N) : pool.conv_K.at(N));
+
+        cuMemcpyHtoDAsync(d_a, h_a.data(), bytes_a, stream);
+        cuMemcpyHtoDAsync(d_b, h_b.data(), bytes_b, stream);
+
+        // Kernel arguments for cuLaunchKernel
+        void *kernel_args[] = { &d_a, &d_b, &d_c, &N };
+
+        if (type == MMUL) {
+            unsigned int block_dim = 32;
+            unsigned int grid_dim = (N + block_dim - 1) / block_dim;
+            cuLaunchKernel(mmul_func, grid_dim, grid_dim, 1,
+                           block_dim, block_dim, 1,
+                           0, stream, kernel_args, nullptr);
+        } else if (type == VADD) {
+            unsigned int block_dim = 256;
+            unsigned int grid_dim = (N + block_dim - 1) / block_dim;
+            cuLaunchKernel(vadd_func, grid_dim, 1, 1,
+                           block_dim, 1, 1,
+                           0, stream, kernel_args, nullptr);
+        } else {
+            unsigned int block_dim = 256;
+            unsigned int grid_dim = (N + block_dim - 1) / block_dim;
+            cuLaunchKernel(conv_func, grid_dim, 1, 1,
+                           block_dim, 1, 1,
+                           0, stream, kernel_args, nullptr);
+        }
+        
+        // Simulating the result retrieval (Copy back)
+        size_t res_count = (type == MMUL) ? (size_t)N * N : (size_t)N;
+        cuMemcpyDtoHAsync(shared_dtoh_buffer, d_c, bytes_out, stream);
+
+        // Free GPU memory for this task
+        cuMemFreeAsync(d_a, stream);
+        cuMemFreeAsync(d_b, stream);
+        cuMemFreeAsync(d_c, stream);
+    }
+
+    // Synchronize this GPU context
+    cuCtxSynchronize();
+
+    // Cleanup
+    for (auto s : streams) {
+        cuStreamDestroy(s);
+    }
+}
+
+MatrixPool create_matrix_pool_random(
+    int num_sizes,
+    int min_N,
+    int max_N,
+    unsigned int seed
+) {
+    MatrixPool pool;
+    std::mt19937 rng(seed);
+    std::uniform_int_distribution<int> dist_N(min_N / 32, max_N / 32);
+    std::unordered_set<int> used_Ns;
+    while (used_Ns.size() < static_cast<size_t>(num_sizes)) {
+        int N_val = dist_N(rng) * 32;
+        if (N_val == 0) continue;
+        if (used_Ns.insert(N_val).second) {
+            pool.A[N_val] = std::vector<int>(N_val * N_val, 1);
+            pool.B[N_val] = std::vector<int>(N_val * N_val, 1);
+            pool.vec_A[N_val] = std::vector<int>(N_val, 1);
+            pool.vec_B[N_val] = std::vector<int>(N_val, 1);
+            pool.conv_A[N_val] = std::vector<int>(N_val, 1);
+            pool.conv_K[N_val] = std::vector<int>(5, 1);
+        }
+    }
+    return pool;
+}
+
+int main() {
+    CUresult err;
+
+    // Initialize the CUDA Driver API
+    err = cuInit(0);
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error initializing CUDA Driver API: " << err_str << std::endl;
+        return 1;
+    }
+
+    const int streams_per_gpu = 8;
+    std::vector<int> task_counts = {50000,100000};
+
+    int num_gpus;
+    cuDeviceGetCount(&num_gpus);
+    if (num_gpus == 0) {
+        std::cerr << "No CUDA devices found." << std::endl;
+        return 1;
+    }
+
+    // Define parameters for irregular workload
+    const int num_matrix_sizes = 60; // Number of distinct N values
+    const int min_N_val = 32;
+    const int max_N_val = 2048;
+    const unsigned int pool_seed = 42; // Fixed seed for deterministic pool generation
+
+    // Create the host-side matrix pool once
+    MatrixPool global_host_matrix_pool = create_matrix_pool_random(num_matrix_sizes, min_N_val, max_N_val, pool_seed);
+
+    // Extract available N values from the pool for task generation
+    std::vector<int> available_Ns;
+    for (const auto& pair : global_host_matrix_pool.A) {
+        available_Ns.push_back(pair.first);
+    }
+    std::sort(available_Ns.begin(), available_Ns.end());
+    if (available_Ns.empty()) {
+        std::cerr << "Error: No matrix sizes generated in the pool." << std::endl;
+        return 1;
+    }
+
+    for (int total_tasks : task_counts) {
+    std::cout << "=====================================" << std::endl;
+    std::cout << "Task count: " << total_tasks << " (Irregular Workload)" << std::endl;
+
+    std::vector<Task> all_tasks;
+    std::mt19937 rng_tasks(42); // Fixed seed for task distribution
+    std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
+
+    std::vector<CUcontext> contexts(num_gpus);
+    std::vector<CUfunction> mmul_funcs(num_gpus);
+    std::vector<CUfunction> vadd_funcs(num_gpus);
+    std::vector<CUfunction> conv_funcs(num_gpus);
+    CUmodule mmul_mod;
+    CUmodule vadd_mod;
+    CUmodule conv_mod;
+
+    // Create a CUDA context for each device
+    for (int i = 0; i < num_gpus; ++i) {
+        CUdevice dev;
+        cuDeviceGet(&dev, i);
+        err = cuCtxCreate(&contexts[i], 0, dev); // Flag 0 for default context creation
+        if (err != CUDA_SUCCESS) {
+            const char* err_str;
+            cuGetErrorString(err, &err_str);
+            std::cerr << "Error creating context for device " << i << ": " << err_str << std::endl;
+            // Clean up already created contexts
+            for (int j = 0; j < i; ++j) cuCtxDestroy(contexts[j]);
+            return 1;
+        }
+    }
+
+    // Make the context for device 0 current on the main thread before loading the module
+    err = cuCtxPushCurrent(contexts[0]);
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error pushing context for device 0: " << err_str << std::endl;
+        for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
+        return 1;
+    }
+
+    // Load the cubin module (assuming mmul.cu is compiled to mmul.cubin)
+    err = cuModuleLoad(&mmul_mod, "../mmul.cubin");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error loading module ../mmul.cubin: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr); // Pop context on error
+        return 1;
+    }
+
+    err = cuModuleLoad(&vadd_mod, "../vector_add.cubin");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error loading module ../vector_add.cubin: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr);
+        return 1;
+    }
+
+    err = cuModuleLoad(&conv_mod, "../conv1d.cubin");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error loading module ../conv1d.cubin: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr);
+        return 1;
+    }
+
+    // Get function handles
+    err = cuModuleGetFunction(&mmul_funcs[0], mmul_mod, "matrixMul");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error getting function matrixMul: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr); // Pop context on error
+        cuModuleUnload(mmul_mod);
+        cuModuleUnload(vadd_mod);
+        for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
+        return 1;
+    }
+
+    err = cuModuleGetFunction(&vadd_funcs[0], vadd_mod, "vectorAdd");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error getting function vectorAdd: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr);
+        return 1;
+    }
+
+    err = cuModuleGetFunction(&conv_funcs[0], conv_mod, "conv1d");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error getting function conv1d: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr);
+        return 1;
+    }
+
+    // Pop the context from the main thread
+    err = cuCtxPopCurrent(nullptr);
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error popping context from main thread: " << err_str << std::endl;
+        cuModuleUnload(mmul_mod);
+        cuModuleUnload(vadd_mod);
+        cuModuleUnload(conv_mod);
+        for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
+        return 1;
+    }
+
+    // Assuming all GPUs can use the same function handle from the same module.
+    for (int i = 1; i < num_gpus; ++i) {
+        mmul_funcs[i] = mmul_funcs[0];
+        vadd_funcs[i] = vadd_funcs[0];
+        conv_funcs[i] = conv_funcs[0];
+    }
+
+    std::uniform_int_distribution<int> dist_type(0, 2);
+    for (int i = 0; i < total_tasks; ++i) {
+        int N_for_task = available_Ns[dist_N_idx(rng_tasks)];
+        TaskType t_type = static_cast<TaskType>(dist_type(rng_tasks));
+        all_tasks.push_back({N_for_task, t_type});
+    }
+
+    // ─────────────────────────────────────────────────────────────────────────
+    // Static Round-Robin Partitioning
+    // ─────────────────────────────────────────────────────────────────────────
+    std::vector<std::vector<Task>> partitions(num_gpus);
+    for (int i = 0; i < total_tasks; ++i) {
+        partitions[i % num_gpus].push_back(all_tasks[i]);
+    }
+
+    // Preallocate a single large host buffer for DTOH transfers to save RAM and keep things fair.
+    std::vector<int> shared_dtoh_buffer((size_t)max_N_val * max_N_val);
+
+    auto start = std::chrono::steady_clock::now();
+
+    std::vector<std::thread> threads;
+    for (int i = 0; i < num_gpus; ++i) { // Pass context and kernel function to each worker
+        threads.emplace_back(gpu_worker, i, std::ref(partitions[i]), streams_per_gpu, contexts[i], mmul_funcs[i], vadd_funcs[i], conv_funcs[i],
+         std::ref(global_host_matrix_pool), shared_dtoh_buffer.data());
+    }
+
+    for (auto& t : threads) {
+        t.join();
+    }
+
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+
+    std::cout << "Makespan: " << elapsed.count() << "s" << std::endl;
+
+    // Cleanup contexts and module
+    for (int i = 0; i < num_gpus; ++i) {
+        cuCtxDestroy(contexts[i]);
+    }
+    cuModuleUnload(mmul_mod);
+    cuModuleUnload(vadd_mod);
+    cuModuleUnload(conv_mod);
+    }
+
+    return 0;
+}
diff --git a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/mmul.cu b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+
diff --git a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/vector_add.cu b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/vector_add.cu
new file mode 100644
index 0000000000..4bf501d5d1
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/vector_add.cu
@@ -0,0 +1,6 @@
+extern "C" __global__ void vectorAdd(const int* A, const int* B, int* C, int N) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx < N) {
+        C[idx] = A[idx] + B[idx];
+    }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp
new file mode 100644
index 0000000000..f630a7814b
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp
@@ -0,0 +1,496 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/component-actors/all-component-actors.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+#include <deque>
+#include <unordered_map>
+#include <unordered_set>
+//#include <caf/atoms.hpp>
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+enum TaskType { MMUL = 0, VADD = 1, CONV = 2 };
+
+struct Task {
+    int N;
+    TaskType type;
+};
+
+// Inspect function for TaskType enum to enable CAF serialization
+template <class Inspector>
+bool inspect(Inspector& f, TaskType& x) {
+  auto val = static_cast<int>(x);
+  if (f.apply(val)) {
+    if constexpr (Inspector::is_loading)
+      x = static_cast<TaskType>(val);
+    return true;
+  }
+  return false;
+}
+
+// Inspect function for Task struct to enable CAF serialization
+template <class Inspector>
+bool inspect(Inspector& f, Task& x) {
+  return f.object(x).fields(f.field("N", x.N), f.field("type", x.type));
+};
+
+
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Atoms
+// ─────────────────────────────────────────────────────────────────────────────
+CAF_BEGIN_TYPE_ID_BLOCK(mmul_benchmark, caf::id_block::cuda::end)
+    CAF_ADD_ATOM(mmul_benchmark, get_work_atom)
+    CAF_ADD_ATOM(mmul_benchmark, task_done_atom)
+    CAF_ADD_ATOM(mmul_benchmark, release_memory_atom)
+    CAF_ADD_ATOM(mmul_benchmark, request_work_atom)
+    CAF_ADD_ATOM(mmul_benchmark, worker_done_atom)
+    CAF_ADD_TYPE_ID(mmul_benchmark, (TaskType))
+    CAF_ADD_TYPE_ID(mmul_benchmark, (Task))
+    CAF_ADD_TYPE_ID(mmul_benchmark, (std::vector<Task>))
+    CAF_ADD_ATOM(mmul_benchmark, refill_buffer_atom)
+CAF_END_TYPE_ID_BLOCK(mmul_benchmark)
+
+
+// Command runners for GPU operations
+caf::cuda::command_runner<> mmul_command;
+using kernel_runner_t = caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>, out<int>, in<int>>;
+kernel_runner_t kernel_runner;
+
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
+    std::unordered_map<int, std::vector<int>> vec_A;
+    std::unordered_map<int, std::vector<int>> vec_B;
+    std::unordered_map<int, std::vector<int>> conv_A;
+    std::unordered_map<int, std::vector<int>> conv_K;
+};
+
+MatrixPool create_matrix_pool_random(
+    int num_sizes,
+    int min_N,
+    int max_N,
+    unsigned int seed
+) {
+    MatrixPool pool;
+
+    std::mt19937 rng(seed);
+    std::uniform_int_distribution<int> dist(min_N / 32, max_N / 32);
+
+    std::unordered_set<int> used;
+
+    while (used.size() < static_cast<size_t>(num_sizes)) {
+        int N = dist(rng) * 32;
+        if (N == 0) continue;
+        if (used.insert(N).second) {
+            pool.A[N] = std::vector<int>(N * N, 1);
+            pool.B[N] = std::vector<int>(N * N, 1);
+            pool.vec_A[N] = std::vector<int>(N, 1);
+            pool.vec_B[N] = std::vector<int>(N, 1);
+            pool.conv_A[N] = std::vector<int>(N, 1);
+            pool.conv_K[N] = std::vector<int>(5, 1);
+        }
+    }
+
+    return pool;
+}
+
+// ---------------------------- GLOBAL TASK POOL ----------------------------
+// The central source of truth for work. Implements a pull-based model.
+struct task_pool_state {
+    std::vector<Task> tasks;
+    size_t next_task_idx = 0;
+};
+
+caf::behavior global_task_pool(caf::stateful_actor<task_pool_state>* self, std::vector<Task> tasks) {
+    self->state().tasks = std::move(tasks);
+    return {
+        [=](get_work_atom, size_t batch_size) -> result<std::vector<Task>> {
+            auto& st = self->state();
+            if (st.next_task_idx >= st.tasks.size())
+                return sec::end_of_stream;
+            size_t count = std::min(batch_size, st.tasks.size() - st.next_task_idx);
+            std::vector<Task> batch(st.tasks.begin() + st.next_task_idx, 
+                                    st.tasks.begin() + st.next_task_idx + count);
+            st.next_task_idx += count;
+            return batch;
+        }
+    };
+}
+
+// ---------------------------- DEVICE/GPU ACTOR ----------------------------
+// Manages memory for a specific GPU and steals (pulls) work from the Global Pool.
+struct device_actor_state {
+    MatrixPool pool;
+    caf::actor global_pool;
+    std::deque<Task> local_tasks; // Local buffer to keep GPU busy
+    size_t total_device_memory_bytes = 0;
+    size_t current_allocated_memory_bytes = 0;
+    int active_workers = 0;
+    int device_id = -1;
+    size_t batch_size = 0;
+    size_t low_water_mark = 0;
+    bool fetching = false;
+};
+
+caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
+                               MatrixPool pool, caf::actor global_pool, int num_workers, int dev_id, int max_in_flight) {
+    self->state().pool = std::move(pool);
+    self->state().global_pool = global_pool;
+    self->state().device_id = dev_id;
+    self->state().active_workers = num_workers;
+
+    // Dynamically calculate prefetch markers based on the total pipeline capacity
+    self->state().low_water_mark = static_cast<size_t>(num_workers * max_in_flight);
+    self->state().batch_size = self->state().low_water_mark * 2;
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+    caf::cuda::device_ptr dev_obj = mgr.find_device(dev_id);
+    if (dev_obj) {
+        self->state().total_device_memory_bytes = dev_obj->total_memory_bytes();
+    }
+
+    // Helper to refill the local task buffer from the global pool
+    auto refill = [=]() {
+        auto& st = self->state();
+        if (st.fetching || st.local_tasks.size() >= st.low_water_mark + st.batch_size)
+            return;
+
+        st.fetching = true;
+        self->mail(get_work_atom_v, st.batch_size).request(st.global_pool, infinite).then(
+            [=](std::vector<Task>& batch) {
+                auto& st_inner = self->state();
+                for (auto& task : batch)
+                    st_inner.local_tasks.push_back(task);
+                st_inner.fetching = false;
+                if (st_inner.local_tasks.size() < st_inner.low_water_mark)
+                    self->mail(refill_buffer_atom_v).send(self);
+            },
+            [=](error& err) {
+                self->state().fetching = false;
+            }
+        );
+    };
+
+    return {
+        [=](refill_buffer_atom) {
+            refill();
+        },
+        [=](get_work_atom) -> caf::result<int, int, in<int>, in<int>> {
+            auto& st = self->state();
+
+            // If we have tasks locally, satisfy the request immediately
+            if (!st.local_tasks.empty()) {
+                Task t = st.local_tasks.front();
+                int N = t.N;
+                size_t memory_needed = (t.type == MMUL) ? (size_t)N * N * sizeof(int) * 3 : (size_t)N * sizeof(int) * 3; // Approx
+                if (st.current_allocated_memory_bytes + memory_needed > st.total_device_memory_bytes)
+                    return make_error(sec::runtime_error, "Device Actor: Not enough memory");
+                
+                st.local_tasks.pop_front();
+                st.current_allocated_memory_bytes += memory_needed;
+                
+                if (st.local_tasks.size() < st.low_water_mark)
+                    refill();
+
+                auto& h_a = (t.type == MMUL) ? st.pool.A[N] : (t.type == VADD ? st.pool.vec_A[N] : st.pool.conv_A[N]);
+                auto& h_b = (t.type == MMUL) ? st.pool.B[N] : (t.type == VADD ? st.pool.vec_B[N] : st.pool.conv_K[N]);
+
+                return {N, static_cast<int>(t.type), caf::cuda::create_in_arg(h_a), 
+                           caf::cuda::create_in_arg(h_b)};
+            }
+
+            // Buffer empty: must fetch from global pool reactively
+            auto promise = self->make_response_promise<int, int, in<int>, in<int>>();
+            self->mail(get_work_atom_v, st.batch_size).request(st.global_pool, infinite).then(
+                [=](std::vector<Task>& batch) mutable {
+                    auto& st_inner = self->state();
+                    Task t = batch.front();
+                    int N = t.N;
+                    for(size_t i = 1; i < batch.size(); ++i) st_inner.local_tasks.push_back(batch[i]);
+                    
+                    size_t needed = (t.type == MMUL) ? (size_t)N * N * sizeof(int) * 3 : (size_t)N * sizeof(int) * 3; // Approx
+                    st_inner.current_allocated_memory_bytes += needed;
+
+                    auto& h_a = (t.type == MMUL) ? st_inner.pool.A[N] : (t.type == VADD ? st_inner.pool.vec_A[N] : st_inner.pool.conv_A[N]);
+                    auto& h_b = (t.type == MMUL) ? st_inner.pool.B[N] : (t.type == VADD ? st_inner.pool.vec_B[N] : st_inner.pool.conv_K[N]);
+                    promise.deliver(N, static_cast<int>(t.type), caf::cuda::create_in_arg(h_a), 
+                                       caf::cuda::create_in_arg(h_b));
+                },
+                [=](error& err) mutable { promise.deliver(err); }
+            );
+            return promise;
+        },
+        [=](release_memory_atom, int N_completed, int type) {
+            auto& st = self->state();
+            TaskType t_type = static_cast<TaskType>(type);
+            size_t memory_released = (t_type == MMUL) ? (size_t)N_completed * N_completed * sizeof(int) * 3 : (size_t)N_completed * sizeof(int) * 3; // Approx
+            st.current_allocated_memory_bytes -= memory_released;
+            refill(); // Try to get more work now that memory is free
+        },
+        [=](worker_done_atom) {
+            auto& st = self->state();
+            if (--st.active_workers <= 0) {
+                self->quit();
+            }
+        }
+    };
+}
+
+// ---------------------------- WORKER ACTOR ----------------------------
+// Manages 1 stream and pulls work from the Device Actor.
+struct worker_state {
+    int device_id;
+    int stream_id;
+    caf::cuda::program_ptr mmul_prog;
+    caf::cuda::program_ptr vadd_prog;
+    caf::cuda::program_ptr conv_prog;
+    caf::actor device_actor;
+    caf::actor supervisor;
+    int max_in_flight_tasks;
+    int in_flight_tasks_count = 0;
+    int* dtoh_buffer_ptr = nullptr;
+    bool draining = false;
+};
+
+caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
+                              caf::actor supervisor, caf::actor device_actor, caf::cuda::program_ptr mmul_p, caf::cuda::program_ptr vadd_p, caf::cuda::program_ptr conv_p,
+                              int dev_id, int stream_id, int max_in_flight_tasks, int* d_buf) {
+    self->state().supervisor = supervisor;
+    self->state().device_actor = device_actor;
+    self->state().mmul_prog = mmul_p;
+    self->state().vadd_prog = vadd_p;
+    self->state().conv_prog = conv_p;
+    self->state().device_id = dev_id;
+    self->state().stream_id = stream_id;
+    self->state().dtoh_buffer_ptr = d_buf;
+    self->state().max_in_flight_tasks = max_in_flight_tasks;
+
+    // Trigger initial work requests up to max_in_flight_tasks
+    for (int i = 0; i < max_in_flight_tasks; ++i) {
+        self->mail(request_work_atom_v).send(self);
+    }
+
+    return {
+        [=](request_work_atom) {
+            auto& st = self->state();
+            if (st.in_flight_tasks_count >= st.max_in_flight_tasks || st.draining) {
+                return; // Already at max capacity, don't request more yet
+            }
+
+            st.in_flight_tasks_count++; // Mark as pending immediately
+            self->mail(get_work_atom_v).request(st.device_actor, infinite).then(
+                [=](int N, int type, in<int> matrixA, in<int> matrixB) {
+                    TaskType t_type = static_cast<TaskType>(type);
+                    // GPU Pipeline: Transfer -> Kernel -> Copyback
+                    auto arg1 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixA));
+                    auto arg2 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixB));
+
+                    caf::cuda::nd_range dims;
+                    caf::cuda::program_ptr prog;
+                    int out_size;
+                    if (t_type == MMUL) {
+                        dims = caf::cuda::nd_range((N+31)/32, (N+31)/32, 1, 32, 32, 1);
+                        prog = st.mmul_prog;
+                        out_size = N * N;
+                    } else if (t_type == VADD) {
+                        dims = caf::cuda::nd_range((N+255)/256, 1, 1, 256, 1, 1);
+                        prog = st.vadd_prog;
+                        out_size = N;
+                    } else {
+                        dims = caf::cuda::nd_range((N+255)/256, 1, 1, 256, 1, 1);
+                        prog = st.conv_prog;
+                        out_size = N;
+                    }
+
+                    auto result = kernel_runner.run_async(prog, dims, st.stream_id, 0, st.device_id,
+                                                         arg1, arg2, caf::cuda::create_out_arg<int>(out_size), caf::cuda::create_in_arg<int>(N));
+
+                    auto bufferC = std::get<2>(result);
+                    auto self_hdl = caf::actor_cast<caf::actor>(self);
+
+                    mmul_command.copy_to_host_async(bufferC, st.dtoh_buffer_ptr, (size_t)out_size, [self_hdl, N_task = N, type](int*, size_t) {
+                        caf::anon_mail(task_done_atom_v, N_task, type).send(self_hdl);
+                    });
+                },
+                [=](error& err) {
+                    auto& st = self->state();
+                    st.in_flight_tasks_count--; // Revert pending status on failure
+                    if (err == sec::runtime_error) {
+                        // Not enough memory, retry after a delay
+                        self->println("Worker {}: Not enough memory, retrying for work...", st.stream_id);
+                        self->delayed_anon_send(self, 100ms, request_work_atom_v);
+                    } else if (err == sec::end_of_stream) {
+                        st.draining = true; // Mark as draining, let in-flight finish
+                        if (st.in_flight_tasks_count == 0) {
+                            self->mail(worker_done_atom_v).send(st.device_actor);
+                            mmul_command.release_stream_for_actor(st.stream_id);
+                            self->quit();
+                        }
+                    }
+                }
+            );
+        },
+        [=](task_done_atom, int N_completed, int type) {
+            auto& st = self->state();
+            st.in_flight_tasks_count--; // Decrement count
+            self->mail(1).send(st.supervisor); // Notify supervisor
+            self->mail(release_memory_atom_v, N_completed, type).send(st.device_actor); // Release memory
+            
+            if (st.draining && st.in_flight_tasks_count == 0) {
+                self->mail(worker_done_atom_v).send(st.device_actor);
+                mmul_command.release_stream_for_actor(st.stream_id);
+                self->quit();
+            } else if (!st.draining) {
+                self->mail(request_work_atom_v).send(self); // Request next task if capacity allows
+            }
+        }
+    };
+}
+
+// ---------------------------- SUPERVISOR ACTOR ----------------------------
+struct supervisor_actor_state {
+    int total_tasks;
+    int completed = 0;
+    std::chrono::steady_clock::time_point start_time;
+};
+
+caf::behavior supervisor_actor_fun(
+    caf::stateful_actor<supervisor_actor_state>* self,
+    int total_tasks,
+    int workers_per_gpu,
+    int max_in_flight_tasks_per_worker,
+    MatrixPool pool,
+    std::vector<Task> tasks,
+    int* shared_dtoh_ptr
+    ) {
+    self->state().total_tasks = total_tasks;
+    self->state().start_time = std::chrono::steady_clock::now();
+
+    auto pool_actor = self->spawn(global_task_pool, std::move(tasks));
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+    int num_gpus = mgr.get_num_devices();
+    auto mmul_p = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+    auto vadd_p = mgr.create_program_from_cubin("../vector_add.cubin", "vectorAdd");
+    auto conv_p = mgr.create_program_from_cubin("../conv1d.cubin", "conv1d");
+
+    for (int i = 0; i < num_gpus; ++i) {
+        auto broker = self->spawn(gpu_device_actor, pool, pool_actor, workers_per_gpu, i, max_in_flight_tasks_per_worker);
+        
+        for (int j = 0; j < workers_per_gpu; ++j) {
+            self->spawn(mmul_worker_fun, self, broker, mmul_p, vadd_p, conv_p, i, (i * 1000) + j, max_in_flight_tasks_per_worker, shared_dtoh_ptr);
+        }
+    }
+
+    return {
+        [=](int done) {
+            self->state().completed += done;
+            if (self->state().completed >= self->state().total_tasks) {
+                auto end_time = std::chrono::steady_clock::now();
+                std::chrono::duration<double> total_time = end_time - self->state().start_time;
+
+                std::cout << "\n===== BENCHMARK COMPLETE =====\n";
+                std::cout << "Tasks: " << self->state().total_tasks << "\n";
+                std::cout << "Runtime: " << total_time.count() << " s\n";
+                
+                caf::cuda::manager::shutdown();
+                self->quit();
+            }
+        }
+    };
+}
+
+template <class Fn>
+double time_run(Fn&& fn) {
+  auto start = std::chrono::steady_clock::now();
+  fn();
+  auto end = std::chrono::steady_clock::now();
+  std::chrono::duration<double> elapsed = end - start;
+  return elapsed.count();
+}
+
+void run_mmul_random_scaling_tests(caf::actor_system& sys,
+                                  caf::cuda::manager_config man_config) {
+
+    const int min_N = 32;
+    const int max_N = 2048;
+    const int num_sizes = 60;
+
+    const int workers_per_gpu = 4; // Admission control: only 16 concurrent tasks per GPU
+    const int max_in_flight_tasks_per_worker = 5; // Each worker keeps 2 tasks in flight
+
+    const std::vector<int> actor_counts = {
+	  50000,100000
+    };
+
+    // Generate deterministic random pool once
+    MatrixPool pool = create_matrix_pool_random(
+        num_sizes,
+        min_N,
+        max_N,
+        42  // fixed seed
+    );
+
+    //scheduler
+    caf::cuda::manager_config scheduler_off(false);
+    for (int num_tasks_for_this_run : actor_counts) {
+	    // Initialize CUDA manager
+	    caf::cuda::manager::init(sys, scheduler_off);
+        std::cout << "=====================================\n";
+        std::cout << "Random Scaling | actors=" << num_tasks_for_this_run << "\n";
+
+        // Precompute all task Ns for this run
+        std::vector<int> sizes;
+        for (const auto& [N, _] : pool.A) sizes.push_back(N);
+        std::sort(sizes.begin(), sizes.end());
+
+        std::vector<Task> tasks_for_this_run;
+        tasks_for_this_run.reserve(num_tasks_for_this_run);
+
+        std::mt19937 rng(42);
+        std::uniform_int_distribution<size_t> dist_size(0, sizes.size() - 1);
+        std::uniform_int_distribution<int> dist_type(0, 2);
+        for (int i = 0; i < num_tasks_for_this_run; ++i) {
+            int N = sizes[dist_size(rng)];
+            TaskType type = static_cast<TaskType>(dist_type(rng));
+            tasks_for_this_run.push_back({N, type});
+        }
+
+        // Preallocate a single large host buffer for DTOH transfers to save RAM and keep things fair.
+        std::vector<int> shared_dtoh_buffer((size_t)max_N * max_N);
+
+        // Execute the supervisor which manages the asynchronous workload
+        double elapsed = time_run([&]() {
+
+            auto sup = sys.spawn(
+                supervisor_actor_fun,
+                (int)tasks_for_this_run.size(), // total_tasks
+                workers_per_gpu,
+                max_in_flight_tasks_per_worker,
+                pool,
+                tasks_for_this_run,
+                    shared_dtoh_buffer.data()
+            );
+
+	      sys.await_all_actors_done();
+	    });
+
+        caf::cuda::manager::shutdown();
+    }
+}
+void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager_config man_config(false);
+  run_mmul_random_scaling_tests(sys, man_config);
+}
+CAF_MAIN(id_block::mmul_benchmark)

From 62d636d1fb5c20e3ab8cb4a51f37bcabd9d4de79 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 12:55:37 -0600
Subject: [PATCH 0647/1000] fixed compiler errors

---
 libcaf_cuda/caf/cuda/device.hpp     | 11 ++++++-----
 libcaf_cuda/caf/cuda/streampool.hpp |  5 ++++-
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 9390fcc74b..186b5dfc4f 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -10,6 +10,7 @@
 #include <shared_mutex>
 #include <tuple>
 #include <mutex>
+#include <memory>
 
 #include <caf/intrusive_ptr.hpp>
 #include <caf/ref_counted.hpp>
@@ -32,7 +33,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       context_(context),
       id_(id),
       name_(name),
-      stream_table_(context, stream_pool_size) {
+      stream_table_(std::make_unique<DeviceStreamTable>(context, stream_pool_size)) {
  	      init_device_properties();
       }
 
@@ -64,7 +65,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     }
     // Set the new context and reinitialize the stream table
     context_ = new_ctx;
-    stream_table_ = DeviceStreamTable(context_, stream_table_.pool_size());
+    stream_table_ = std::make_unique<DeviceStreamTable>(context_, stream_table_->pool_size());
   }
   // Number of streaming multiprocessors (SMs)
   int num_sms() const noexcept { return sm_count_; }
@@ -112,12 +113,12 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
 
   //returns the CUStream associated with the actor id 
   CUstream get_stream_for_actor(int actor_id) {
-    return stream_table_.get_stream(actor_id);
+    return stream_table_->get_stream(actor_id);
   }
 
   //releases the CUStream associated with the actor id 
   void release_stream_for_actor(int actor_id) {
-    stream_table_.release_stream(actor_id);
+    stream_table_->release_stream(actor_id);
   }
 
 
@@ -308,7 +309,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   CUcontext context_;
   int id_;
   const char* name_;
-  DeviceStreamTable stream_table_;
+  std::unique_ptr<DeviceStreamTable> stream_table_;
   std::mutex stream_mutex_;
 
   // Cached GPU properties (queried once during construction)
diff --git a/libcaf_cuda/caf/cuda/streampool.hpp b/libcaf_cuda/caf/cuda/streampool.hpp
index 1e3eef5bf5..569aff7f61 100644
--- a/libcaf_cuda/caf/cuda/streampool.hpp
+++ b/libcaf_cuda/caf/cuda/streampool.hpp
@@ -51,10 +51,13 @@ class CAF_CUDA_EXPORT StreamPool {
   /// Return number of streams currently available in the pool.
   size_t num_available() const;
 
-private:
   // Add a getter for max_size_
   size_t max_size() const { return max_size_; }
 
+  
+private:
+  
+
   /// Create a new CUDA stream in the context `ctx_`.
   CUstream create_stream();
 

From c88b8e6005e5abd994ad134de0a98796bb5f81de Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 13:31:04 -0600
Subject: [PATCH 0648/1000] Updated example to be fault tolerant (I hope)>

---
 .../many-independent-tasks/work-stealing.cpp  | 379 ++++++++++++------
 1 file changed, 248 insertions(+), 131 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp
index f630a7814b..b0b8abcf13 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp
@@ -18,10 +18,10 @@
 using namespace caf;
 using namespace std::chrono_literals;
 
-
 enum TaskType { MMUL = 0, VADD = 1, CONV = 2 };
 
 struct Task {
+    int id;
     int N;
     TaskType type;
 };
@@ -41,7 +41,7 @@ bool inspect(Inspector& f, TaskType& x) {
 // Inspect function for Task struct to enable CAF serialization
 template <class Inspector>
 bool inspect(Inspector& f, Task& x) {
-  return f.object(x).fields(f.field("N", x.N), f.field("type", x.type));
+  return f.object(x).fields(f.field("id", x.id), f.field("N", x.N), f.field("type", x.type));
 };
 
 
@@ -59,6 +59,7 @@ CAF_BEGIN_TYPE_ID_BLOCK(mmul_benchmark, caf::id_block::cuda::end)
     CAF_ADD_TYPE_ID(mmul_benchmark, (Task))
     CAF_ADD_TYPE_ID(mmul_benchmark, (std::vector<Task>))
     CAF_ADD_ATOM(mmul_benchmark, refill_buffer_atom)
+    CAF_ADD_ATOM(mmul_benchmark, restart_atom)
 CAF_END_TYPE_ID_BLOCK(mmul_benchmark)
 
 
@@ -133,22 +134,198 @@ caf::behavior global_task_pool(caf::stateful_actor<task_pool_state>* self, std::
 struct device_actor_state {
     MatrixPool pool;
     caf::actor global_pool;
+    caf::actor supervisor;
     std::deque<Task> local_tasks; // Local buffer to keep GPU busy
+    std::unordered_map<int, Task> in_progress;
+    std::vector<caf::actor> workers;
     size_t total_device_memory_bytes = 0;
     size_t current_allocated_memory_bytes = 0;
+    int num_workers_target = 0;
     int active_workers = 0;
     int device_id = -1;
     size_t batch_size = 0;
     size_t low_water_mark = 0;
+    int max_in_flight_per_worker = 0;
+    int* shared_dtoh_ptr = nullptr;
+    caf::cuda::program_ptr mmul_p;
+    caf::cuda::program_ptr vadd_p;
+    caf::cuda::program_ptr conv_p;
+    caf::cuda::program_ptr poison_p;
     bool fetching = false;
+    bool resetting = false;
 };
 
+// ---------------------------- WORKER ACTOR ----------------------------
+// Manages 1 stream and pulls work from the Device Actor.
+struct worker_state {
+    int device_id;
+    int stream_id;
+    caf::cuda::program_ptr mmul_prog;
+    caf::cuda::program_ptr vadd_prog;
+    caf::cuda::program_ptr conv_prog;
+    caf::cuda::program_ptr poison_prog;
+    caf::actor device_actor;
+    caf::actor supervisor;
+    int max_in_flight_tasks;
+    int in_flight_tasks_count = 0;
+    int* dtoh_buffer_ptr = nullptr;
+    bool draining = false;
+};
+
+caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
+                              caf::actor supervisor, caf::actor device_actor,
+                              caf::cuda::program_ptr mmul_p, caf::cuda::program_ptr vadd_p, 
+                              caf::cuda::program_ptr conv_p, caf::cuda::program_ptr poison_p,
+                              int dev_id, int stream_id, int max_in_flight_tasks, int* d_buf,
+                              int poison_chance) {
+    self->state().supervisor = supervisor;
+    self->state().device_actor = device_actor;
+    self->state().mmul_prog = mmul_p;
+    self->state().vadd_prog = vadd_p;
+    self->state().conv_prog = conv_p;
+    self->state().poison_prog = poison_p;
+    self->state().device_id = dev_id;
+    self->state().stream_id = stream_id;
+    self->state().dtoh_buffer_ptr = d_buf;
+    self->state().max_in_flight_tasks = max_in_flight_tasks;
+
+    // Trigger initial work requests up to max_in_flight_tasks
+    for (int i = 0; i < max_in_flight_tasks; ++i) {
+        self->mail(request_work_atom_v).send(self);
+    }
+
+    return {
+        [=](request_work_atom) {
+            auto& st = self->state();
+            if (st.in_flight_tasks_count >= st.max_in_flight_tasks || st.draining) {
+                return; // Already at max capacity, don't request more yet
+            }
+
+            st.in_flight_tasks_count++; // Mark as pending immediately
+            self->mail(get_work_atom_v).request(st.device_actor, infinite).then(
+                [=](int task_id, int N, int type, in<int> matrixA, in<int> matrixB) {
+                    auto& st_inner = self->state();
+                    try {
+                        TaskType t_type = static_cast<TaskType>(type);
+
+                        // Randomly decide to poison the context
+                        std::random_device rd;
+                        std::mt19937 gen(rd());
+                        std::uniform_int_distribution<> distrib(0, 99); // 0-99 for percentage
+                        if (distrib(gen) < poison_chance) {
+                            self->println("Worker {}: LAUNCHING POISON KERNEL! KABOOM incoming...", st_inner.stream_id);
+                            auto arg_dummy = mmul_command.transfer_memory(st_inner.device_id, st_inner.stream_id, caf::cuda::create_in_arg(42));
+                            caf::cuda::command_runner<caf::cuda::mem_ptr<int>> poison_runner;
+                            poison_runner.run_async(st_inner.poison_prog, caf::cuda::nd_range(1,1,1,1,1,1), 
+                                                   st_inner.stream_id, 0, st_inner.device_id, arg_dummy);
+                            // The next CUDA call will trigger the error
+                            caf::cuda::command_runner<> sync_runner;
+                            auto dev_obj = caf::cuda::platform::create()->getDevice(st_inner.device_id);
+                            CHECK_CUDA(cuStreamSynchronize(dev_obj->get_stream_for_actor(st_inner.stream_id)));
+                        }
+
+                        // GPU Pipeline: Transfer -> Kernel -> Copyback
+                        auto arg1 = mmul_command.transfer_memory(st_inner.device_id, st_inner.stream_id, std::move(matrixA));
+                        auto arg2 = mmul_command.transfer_memory(st_inner.device_id, st_inner.stream_id, std::move(matrixB));
+
+                        caf::cuda::nd_range dims;
+                        caf::cuda::program_ptr prog;
+                        int out_size;
+                        if (t_type == MMUL) {
+                            dims = caf::cuda::nd_range((N+31)/32, (N+31)/32, 1, 32, 32, 1);
+                            prog = st_inner.mmul_prog;
+                            out_size = N * N;
+                        } else if (t_type == VADD) {
+                            dims = caf::cuda::nd_range((N+255)/256, 1, 1, 256, 1, 1);
+                            prog = st_inner.vadd_prog;
+                            out_size = N;
+                        } else {
+                            dims = caf::cuda::nd_range((N+255)/256, 1, 1, 256, 1, 1);
+                            prog = st_inner.conv_prog;
+                            out_size = N;
+                        }
+
+                        auto result = kernel_runner.run_async(prog, dims, st_inner.stream_id, 0, st_inner.device_id,
+                                                             arg1, arg2, caf::cuda::create_out_arg<int>(out_size), caf::cuda::create_in_arg<int>(N));
+
+                        auto bufferC = std::get<2>(result);
+                        auto self_hdl = caf::actor_cast<caf::actor>(self);
+
+                        mmul_command.copy_to_host_async(bufferC, st_inner.dtoh_buffer_ptr, (size_t)out_size, [self_hdl, task_id, N_task = N, type](int*, size_t) {
+                            caf::anon_mail(task_done_atom_v, task_id, N_task, type).send(self_hdl);
+                        });
+                    } catch (const std::exception& e) {
+                        // Catch any CUDA errors and terminate the worker, triggering device actor's down_handler
+                        self->println("Worker {}: GPU Error Detected: {}. Terminating actor.", st_inner.stream_id, e.what());
+                        self->quit(make_error(sec::runtime_error, e.what()));
+                    }
+                },
+                [=](error& err) {
+                    auto& st = self->state();
+                    st.in_flight_tasks_count--; // Revert pending status on failure
+                    if (err == sec::runtime_error) {
+                        // Not enough memory, retry after a delay
+                        self->println("Worker {}: Not enough memory, retrying for work...", st.stream_id);
+                        self->delayed_anon_send(self, 100ms, request_work_atom_v);
+                    } else if (err == sec::end_of_stream) {
+                        st.draining = true; // Mark as draining, let in-flight finish
+                        if (st.in_flight_tasks_count == 0) {
+                            self->mail(worker_done_atom_v).send(st.device_actor);
+                            mmul_command.release_stream_for_actor(st.stream_id);
+                            self->quit();
+                        }
+                    }
+                }
+            );
+        },
+        [=](task_done_atom, int task_id, int N_completed, int type) {
+            auto& st = self->state();
+            st.in_flight_tasks_count--; // Decrement count
+            self->mail(task_done_atom_v, task_id, N_completed, type).send(st.device_actor); // Notify device actor
+            
+            if (st.draining && st.in_flight_tasks_count == 0) {
+                self->mail(worker_done_atom_v).send(st.device_actor);
+                mmul_command.release_stream_for_actor(st.stream_id);
+                self->quit();
+            } else if (!st.draining) {
+                self->mail(request_work_atom_v).send(self); // Request next task if capacity allows
+            }
+        }
+    };
+}
+
 caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
-                               MatrixPool pool, caf::actor global_pool, int num_workers, int dev_id, int max_in_flight) {
+                               MatrixPool pool, caf::actor global_pool, caf::actor supervisor,
+                               int num_workers, int dev_id, int max_in_flight,
+                               caf::cuda::program_ptr mmul_p, caf::cuda::program_ptr vadd_p,
+                               caf::cuda::program_ptr conv_p, caf::cuda::program_ptr poison_p,
+                               int* d_buf, int poison_chance) {
     self->state().pool = std::move(pool);
     self->state().global_pool = global_pool;
+    self->state().supervisor = supervisor;
     self->state().device_id = dev_id;
-    self->state().active_workers = num_workers;
+    self->state().num_workers_target = num_workers;
+    self->state().max_in_flight_per_worker = max_in_flight;
+    self->state().mmul_p = mmul_p;
+    self->state().vadd_p = vadd_p;
+    self->state().conv_p = conv_p;
+    self->state().poison_p = poison_p;
+    self->state().shared_dtoh_ptr = d_buf;
+
+    auto spawn_workers = [=]() {
+        for (int j = 0; j < self->state().num_workers_target; ++j) {
+            auto w = self->spawn(mmul_worker_fun, self->state().supervisor, self, self->state().mmul_p,
+                                 self->state().vadd_p, self->state().conv_p, self->state().poison_p,
+                                 self->state().device_id, (self->state().device_id * 1000) + j,
+                                 self->state().max_in_flight_per_worker, self->state().shared_dtoh_ptr,
+                                 poison_chance);
+            self->monitor(w);
+            self->state().workers.push_back(w);
+            self->state().active_workers++;
+        }
+    };
+
+    spawn_workers();
 
     // Dynamically calculate prefetch markers based on the total pipeline capacity
     self->state().low_water_mark = static_cast<size_t>(num_workers * max_in_flight);
@@ -182,23 +359,62 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
         );
     };
 
+    self->set_down_handler([=](caf::down_msg& msg) {
+        auto& st = self->state();
+        if (st.resetting) return;
+
+        if (msg.reason != caf::exit_reason::user_shutdown && msg.reason != caf::exit_reason::normal) {
+            self->println("Device Actor {}: Worker failure detected (reason: {}). Resetting context...", st.device_id, msg.reason);
+            st.resetting = true;
+
+            // 1. Kill remaining workers
+            for (auto& w : st.workers) {
+                self->demonitor(w);
+                self->send_exit(w, caf::exit_reason::kill);
+            }
+            st.workers.clear();
+            st.active_workers = 0;
+
+            // 2. Move in-progress tasks back to local queue
+            for (auto& pair : st.in_progress) {
+                st.local_tasks.push_back(pair.second);
+            }
+            st.in_progress.clear();
+
+            // 3. Reset the actual CUDA context
+            caf::cuda::command_runner<> runner;
+            runner.reset_context(st.device_id);
+
+            // 4. Schedule restart after 1 second to let mailbox clear
+            self->delayed_anon_send(self, 1s, restart_atom_v);
+        }
+    });
+
     return {
+        [=](restart_atom) {
+            self->println("Device Actor {}: Restarting workers...", self->state().device_id);
+            self->state().resetting = false;
+            spawn_workers();
+            refill();
+        },
         [=](refill_buffer_atom) {
             refill();
         },
-        [=](get_work_atom) -> caf::result<int, int, in<int>, in<int>> {
+        [=](get_work_atom) -> caf::result<int, int, int, in<int>, in<int>> {
             auto& st = self->state();
+            if (st.resetting) return make_error(sec::runtime_error, "Device is resetting");
 
             // If we have tasks locally, satisfy the request immediately
             if (!st.local_tasks.empty()) {
                 Task t = st.local_tasks.front();
                 int N = t.N;
-                size_t memory_needed = (t.type == MMUL) ? (size_t)N * N * sizeof(int) * 3 : (size_t)N * sizeof(int) * 3; // Approx
+                size_t memory_needed = (t.type == MMUL) ? (size_t)N * N * (size_t)sizeof(int) * 3 : (size_t)N * (size_t)sizeof(int) * 3;
                 if (st.current_allocated_memory_bytes + memory_needed > st.total_device_memory_bytes)
                     return make_error(sec::runtime_error, "Device Actor: Not enough memory");
                 
                 st.local_tasks.pop_front();
                 st.current_allocated_memory_bytes += memory_needed;
+                st.in_progress[t.id] = t;
                 
                 if (st.local_tasks.size() < st.low_water_mark)
                     refill();
@@ -206,12 +422,12 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
                 auto& h_a = (t.type == MMUL) ? st.pool.A[N] : (t.type == VADD ? st.pool.vec_A[N] : st.pool.conv_A[N]);
                 auto& h_b = (t.type == MMUL) ? st.pool.B[N] : (t.type == VADD ? st.pool.vec_B[N] : st.pool.conv_K[N]);
 
-                return {N, static_cast<int>(t.type), caf::cuda::create_in_arg(h_a), 
+                return {t.id, N, static_cast<int>(t.type), caf::cuda::create_in_arg(h_a), 
                            caf::cuda::create_in_arg(h_b)};
             }
 
             // Buffer empty: must fetch from global pool reactively
-            auto promise = self->make_response_promise<int, int, in<int>, in<int>>();
+            auto promise = self->make_response_promise<int, int, int, in<int>, in<int>>();
             self->mail(get_work_atom_v, st.batch_size).request(st.global_pool, infinite).then(
                 [=](std::vector<Task>& batch) mutable {
                     auto& st_inner = self->state();
@@ -219,23 +435,33 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
                     int N = t.N;
                     for(size_t i = 1; i < batch.size(); ++i) st_inner.local_tasks.push_back(batch[i]);
                     
-                    size_t needed = (t.type == MMUL) ? (size_t)N * N * sizeof(int) * 3 : (size_t)N * sizeof(int) * 3; // Approx
+                    size_t needed = (t.type == MMUL) ? (size_t)N * N * (size_t)sizeof(int) * 3 : (size_t)N * (size_t)sizeof(int) * 3;
                     st_inner.current_allocated_memory_bytes += needed;
+                    st_inner.in_progress[t.id] = t;
 
                     auto& h_a = (t.type == MMUL) ? st_inner.pool.A[N] : (t.type == VADD ? st_inner.pool.vec_A[N] : st_inner.pool.conv_A[N]);
                     auto& h_b = (t.type == MMUL) ? st_inner.pool.B[N] : (t.type == VADD ? st_inner.pool.vec_B[N] : st_inner.pool.conv_K[N]);
-                    promise.deliver(N, static_cast<int>(t.type), caf::cuda::create_in_arg(h_a), 
+                    promise.deliver(t.id, N, static_cast<int>(t.type), caf::cuda::create_in_arg(h_a), 
                                        caf::cuda::create_in_arg(h_b));
                 },
                 [=](error& err) mutable { promise.deliver(err); }
             );
             return promise;
         },
-        [=](release_memory_atom, int N_completed, int type) {
+        [=](task_done_atom, int task_id, int N_completed, int type) {
             auto& st = self->state();
-            TaskType t_type = static_cast<TaskType>(type);
-            size_t memory_released = (t_type == MMUL) ? (size_t)N_completed * N_completed * sizeof(int) * 3 : (size_t)N_completed * sizeof(int) * 3; // Approx
-            st.current_allocated_memory_bytes -= memory_released;
+            
+            auto it = st.in_progress.find(task_id);
+            if (it != st.in_progress.end()) {
+                TaskType t_type = static_cast<TaskType>(type);
+                size_t memory_released = (t_type == MMUL) ? (size_t)N_completed * N_completed * (size_t)sizeof(int) * 3 : (size_t)N_completed * (size_t)sizeof(int) * 3;
+                st.current_allocated_memory_bytes -= memory_released;
+                st.in_progress.erase(it);
+                
+                // Notify supervisor that one task is finished
+                self->mail(1).send(st.supervisor);
+            }
+            
             refill(); // Try to get more work now that memory is free
         },
         [=](worker_done_atom) {
@@ -247,117 +473,6 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
     };
 }
 
-// ---------------------------- WORKER ACTOR ----------------------------
-// Manages 1 stream and pulls work from the Device Actor.
-struct worker_state {
-    int device_id;
-    int stream_id;
-    caf::cuda::program_ptr mmul_prog;
-    caf::cuda::program_ptr vadd_prog;
-    caf::cuda::program_ptr conv_prog;
-    caf::actor device_actor;
-    caf::actor supervisor;
-    int max_in_flight_tasks;
-    int in_flight_tasks_count = 0;
-    int* dtoh_buffer_ptr = nullptr;
-    bool draining = false;
-};
-
-caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
-                              caf::actor supervisor, caf::actor device_actor, caf::cuda::program_ptr mmul_p, caf::cuda::program_ptr vadd_p, caf::cuda::program_ptr conv_p,
-                              int dev_id, int stream_id, int max_in_flight_tasks, int* d_buf) {
-    self->state().supervisor = supervisor;
-    self->state().device_actor = device_actor;
-    self->state().mmul_prog = mmul_p;
-    self->state().vadd_prog = vadd_p;
-    self->state().conv_prog = conv_p;
-    self->state().device_id = dev_id;
-    self->state().stream_id = stream_id;
-    self->state().dtoh_buffer_ptr = d_buf;
-    self->state().max_in_flight_tasks = max_in_flight_tasks;
-
-    // Trigger initial work requests up to max_in_flight_tasks
-    for (int i = 0; i < max_in_flight_tasks; ++i) {
-        self->mail(request_work_atom_v).send(self);
-    }
-
-    return {
-        [=](request_work_atom) {
-            auto& st = self->state();
-            if (st.in_flight_tasks_count >= st.max_in_flight_tasks || st.draining) {
-                return; // Already at max capacity, don't request more yet
-            }
-
-            st.in_flight_tasks_count++; // Mark as pending immediately
-            self->mail(get_work_atom_v).request(st.device_actor, infinite).then(
-                [=](int N, int type, in<int> matrixA, in<int> matrixB) {
-                    TaskType t_type = static_cast<TaskType>(type);
-                    // GPU Pipeline: Transfer -> Kernel -> Copyback
-                    auto arg1 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixA));
-                    auto arg2 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixB));
-
-                    caf::cuda::nd_range dims;
-                    caf::cuda::program_ptr prog;
-                    int out_size;
-                    if (t_type == MMUL) {
-                        dims = caf::cuda::nd_range((N+31)/32, (N+31)/32, 1, 32, 32, 1);
-                        prog = st.mmul_prog;
-                        out_size = N * N;
-                    } else if (t_type == VADD) {
-                        dims = caf::cuda::nd_range((N+255)/256, 1, 1, 256, 1, 1);
-                        prog = st.vadd_prog;
-                        out_size = N;
-                    } else {
-                        dims = caf::cuda::nd_range((N+255)/256, 1, 1, 256, 1, 1);
-                        prog = st.conv_prog;
-                        out_size = N;
-                    }
-
-                    auto result = kernel_runner.run_async(prog, dims, st.stream_id, 0, st.device_id,
-                                                         arg1, arg2, caf::cuda::create_out_arg<int>(out_size), caf::cuda::create_in_arg<int>(N));
-
-                    auto bufferC = std::get<2>(result);
-                    auto self_hdl = caf::actor_cast<caf::actor>(self);
-
-                    mmul_command.copy_to_host_async(bufferC, st.dtoh_buffer_ptr, (size_t)out_size, [self_hdl, N_task = N, type](int*, size_t) {
-                        caf::anon_mail(task_done_atom_v, N_task, type).send(self_hdl);
-                    });
-                },
-                [=](error& err) {
-                    auto& st = self->state();
-                    st.in_flight_tasks_count--; // Revert pending status on failure
-                    if (err == sec::runtime_error) {
-                        // Not enough memory, retry after a delay
-                        self->println("Worker {}: Not enough memory, retrying for work...", st.stream_id);
-                        self->delayed_anon_send(self, 100ms, request_work_atom_v);
-                    } else if (err == sec::end_of_stream) {
-                        st.draining = true; // Mark as draining, let in-flight finish
-                        if (st.in_flight_tasks_count == 0) {
-                            self->mail(worker_done_atom_v).send(st.device_actor);
-                            mmul_command.release_stream_for_actor(st.stream_id);
-                            self->quit();
-                        }
-                    }
-                }
-            );
-        },
-        [=](task_done_atom, int N_completed, int type) {
-            auto& st = self->state();
-            st.in_flight_tasks_count--; // Decrement count
-            self->mail(1).send(st.supervisor); // Notify supervisor
-            self->mail(release_memory_atom_v, N_completed, type).send(st.device_actor); // Release memory
-            
-            if (st.draining && st.in_flight_tasks_count == 0) {
-                self->mail(worker_done_atom_v).send(st.device_actor);
-                mmul_command.release_stream_for_actor(st.stream_id);
-                self->quit();
-            } else if (!st.draining) {
-                self->mail(request_work_atom_v).send(self); // Request next task if capacity allows
-            }
-        }
-    };
-}
-
 // ---------------------------- SUPERVISOR ACTOR ----------------------------
 struct supervisor_actor_state {
     int total_tasks;
@@ -384,13 +499,15 @@ caf::behavior supervisor_actor_fun(
     auto mmul_p = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
     auto vadd_p = mgr.create_program_from_cubin("../vector_add.cubin", "vectorAdd");
     auto conv_p = mgr.create_program_from_cubin("../conv1d.cubin", "conv1d");
+    
+    auto poison_p = mgr.create_program_from_cubin("../poison.cubin", "poison_kernel");
 
     for (int i = 0; i < num_gpus; ++i) {
-        auto broker = self->spawn(gpu_device_actor, pool, pool_actor, workers_per_gpu, i, max_in_flight_tasks_per_worker);
-        
-        for (int j = 0; j < workers_per_gpu; ++j) {
-            self->spawn(mmul_worker_fun, self, broker, mmul_p, vadd_p, conv_p, i, (i * 1000) + j, max_in_flight_tasks_per_worker, shared_dtoh_ptr);
-        }
+        self->spawn(gpu_device_actor, pool, pool_actor, self, workers_per_gpu, i,
+                    max_in_flight_tasks_per_worker, 
+                    mmul_p, vadd_p, conv_p, poison_p,
+                    shared_dtoh_ptr,
+                    10); // 10% chance for a worker to poison the context
     }
 
     return {
@@ -464,7 +581,7 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
         for (int i = 0; i < num_tasks_for_this_run; ++i) {
             int N = sizes[dist_size(rng)];
             TaskType type = static_cast<TaskType>(dist_type(rng));
-            tasks_for_this_run.push_back({N, type});
+            tasks_for_this_run.push_back({i, N, type});
         }
 
         // Preallocate a single large host buffer for DTOH transfers to save RAM and keep things fair.

From 60cc21177506441f05a17b086f9eab5b4f952bc0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 13:32:02 -0600
Subject: [PATCH 0649/1000] Implemented a bad kernel.

---
 .../fault-tolerance-tests/many-independent-tasks/poison.cu | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/poison.cu

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/poison.cu b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/poison.cu
new file mode 100644
index 0000000000..42592cb357
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/poison.cu
@@ -0,0 +1,7 @@
+extern "C" __global__ void poison_kernel(int* dummy) {
+    // Perform a dummy operation
+    *dummy = 0;
+    // Intentionally trigger a memory fault (Illegal Address)
+    int* p = (int*)0;
+    *p = 42;
+}

From 1f0e3ea0865e94b4c1e609830ba66e1a8c0e9c0e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 13:33:58 -0600
Subject: [PATCH 0650/1000] Added posiion kernel.

---
 .../many-independent-tasks/compile_kernels.sh                  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/compile_kernels.sh b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/compile_kernels.sh
index c0bebf8833..2fce7f0c3b 100755
--- a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/compile_kernels.sh
+++ b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/compile_kernels.sh
@@ -10,5 +10,6 @@ echo "Using NVCC arch flag: $SM_ARCH"
 nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
 nvcc -arch=$SM_ARCH -cubin vector_add.cu -o vector_add.cubin
 nvcc -arch=$SM_ARCH -cubin conv1d.cu -o conv1d.cubin
+nvcc -arch=$SM_ARCH -cubin poison.cu -o poison.cubin
 
-echo "Kernels compiled successfully for $SM_ARCH."
\ No newline at end of file
+echo "Kernels compiled successfully for $SM_ARCH."

From 1776e1940be3d49bfea78fb8c8bd796122ee7d18 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 15 May 2026 13:36:53 -0600
Subject: [PATCH 0651/1000] Removed compiler warnings.

---
 .../many-independent-tasks/work-stealing.cpp  | 35 ++++++++++---------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp
index b0b8abcf13..d8edddf2f6 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp
@@ -57,6 +57,7 @@ CAF_BEGIN_TYPE_ID_BLOCK(mmul_benchmark, caf::id_block::cuda::end)
     CAF_ADD_ATOM(mmul_benchmark, worker_done_atom)
     CAF_ADD_TYPE_ID(mmul_benchmark, (TaskType))
     CAF_ADD_TYPE_ID(mmul_benchmark, (Task))
+    CAF_ADD_ATOM(mmul_benchmark, worker_failed_atom)
     CAF_ADD_TYPE_ID(mmul_benchmark, (std::vector<Task>))
     CAF_ADD_ATOM(mmul_benchmark, refill_buffer_atom)
     CAF_ADD_ATOM(mmul_benchmark, restart_atom)
@@ -319,7 +320,11 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
                                  self->state().device_id, (self->state().device_id * 1000) + j,
                                  self->state().max_in_flight_per_worker, self->state().shared_dtoh_ptr,
                                  poison_chance);
-            self->monitor(w);
+            self->monitor(w, [self](const error& err) {
+                if (err && err != exit_reason::normal && err != exit_reason::user_shutdown && err != exit_reason::kill) {
+                    anon_mail(worker_failed_atom_v, err).send(self);
+                }
+            });
             self->state().workers.push_back(w);
             self->state().active_workers++;
         }
@@ -359,18 +364,23 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
         );
     };
 
-    self->set_down_handler([=](caf::down_msg& msg) {
-        auto& st = self->state();
-        if (st.resetting) return;
+    return {
+        [=](restart_atom) {
+            self->println("Device Actor {}: Restarting workers...", self->state().device_id);
+            self->state().resetting = false;
+            spawn_workers();
+            refill();
+        },
+        [=](worker_failed_atom, const error& reason) {
+            auto& st = self->state();
+            if (st.resetting) return;
 
-        if (msg.reason != caf::exit_reason::user_shutdown && msg.reason != caf::exit_reason::normal) {
-            self->println("Device Actor {}: Worker failure detected (reason: {}). Resetting context...", st.device_id, msg.reason);
+            self->println("Device Actor {}: Worker failure detected (reason: {}). Resetting context...", st.device_id, reason);
             st.resetting = true;
 
             // 1. Kill remaining workers
             for (auto& w : st.workers) {
-                self->demonitor(w);
-                self->send_exit(w, caf::exit_reason::kill);
+                self->send_exit(w, exit_reason::kill);
             }
             st.workers.clear();
             st.active_workers = 0;
@@ -387,15 +397,6 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
 
             // 4. Schedule restart after 1 second to let mailbox clear
             self->delayed_anon_send(self, 1s, restart_atom_v);
-        }
-    });
-
-    return {
-        [=](restart_atom) {
-            self->println("Device Actor {}: Restarting workers...", self->state().device_id);
-            self->state().resetting = false;
-            spawn_workers();
-            refill();
         },
         [=](refill_buffer_atom) {
             refill();

From 7a9bdd4919f552169b507503bdc7dfe349847ec3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 19 May 2026 09:40:34 -0600
Subject: [PATCH 0652/1000] added supervisor actor to have handle to device
 actors. Change was made to fix an exit error wher since no one referenced the
 device actor it would immediately exit since its reference count would drop
 to zero

---
 .../many-independent-tasks/work-stealing.cpp  | 48 ++++++++++++++-----
 1 file changed, 37 insertions(+), 11 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp
index d8edddf2f6..a0dc42ad26 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/many-independent-tasks/work-stealing.cpp
@@ -116,9 +116,16 @@ struct task_pool_state {
 
 caf::behavior global_task_pool(caf::stateful_actor<task_pool_state>* self, std::vector<Task> tasks) {
     self->state().tasks = std::move(tasks);
+    self->println("Global task pool spawned with {} tasks", self->state().tasks.size());
+
+    self->attach_functor([self](const error& reason) {
+        self->println("global task pool quitting, reason: {}", reason);
+    });
+
     return {
         [=](get_work_atom, size_t batch_size) -> result<std::vector<Task>> {
             auto& st = self->state();
+            // std::cout << "refilling\n";
             if (st.next_task_idx >= st.tasks.size())
                 return sec::end_of_stream;
             size_t count = std::min(batch_size, st.tasks.size() - st.next_task_idx);
@@ -192,6 +199,7 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
 
     // Trigger initial work requests up to max_in_flight_tasks
     for (int i = 0; i < max_in_flight_tasks; ++i) {
+
         self->mail(request_work_atom_v).send(self);
     }
 
@@ -201,11 +209,14 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
             if (st.in_flight_tasks_count >= st.max_in_flight_tasks || st.draining) {
                 return; // Already at max capacity, don't request more yet
             }
+            // std::cout << "Worker requesting work\n";
 
             st.in_flight_tasks_count++; // Mark as pending immediately
             self->mail(get_work_atom_v).request(st.device_actor, infinite).then(
                 [=](int task_id, int N, int type, in<int> matrixA, in<int> matrixB) {
                     auto& st_inner = self->state();
+                        // std::cout << "Worker got work\n";
+
                     try {
                         TaskType t_type = static_cast<TaskType>(type);
 
@@ -220,9 +231,9 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
                             poison_runner.run_async(st_inner.poison_prog, caf::cuda::nd_range(1,1,1,1,1,1), 
                                                    st_inner.stream_id, 0, st_inner.device_id, arg_dummy);
                             // The next CUDA call will trigger the error
-                            caf::cuda::command_runner<> sync_runner;
-                            auto dev_obj = caf::cuda::platform::create()->getDevice(st_inner.device_id);
-                            CHECK_CUDA(cuStreamSynchronize(dev_obj->get_stream_for_actor(st_inner.stream_id)));
+                            // caf::cuda::command_runner<> sync_runner;
+                            // // auto dev_obj = caf::cuda::platform::create()->getDevice(st_inner.device_id);
+                            // // CHECK_CUDA(cuStreamSynchronize(dev_obj->get_stream_for_actor(st_inner.stream_id)));
                         }
 
                         // GPU Pipeline: Transfer -> Kernel -> Copyback
@@ -263,6 +274,7 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
                 },
                 [=](error& err) {
                     auto& st = self->state();
+                    self->println("Worker {}: encountered error: {}", st.stream_id, err);
                     st.in_flight_tasks_count--; // Revert pending status on failure
                     if (err == sec::runtime_error) {
                         // Not enough memory, retry after a delay
@@ -315,6 +327,7 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
 
     auto spawn_workers = [=]() {
         for (int j = 0; j < self->state().num_workers_target; ++j) {
+            // std::cout << "device actor creating workers\n";
             auto w = self->spawn(mmul_worker_fun, self->state().supervisor, self, self->state().mmul_p,
                                  self->state().vadd_p, self->state().conv_p, self->state().poison_p,
                                  self->state().device_id, (self->state().device_id * 1000) + j,
@@ -345,10 +358,12 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
     // Helper to refill the local task buffer from the global pool
     auto refill = [=]() {
         auto& st = self->state();
+        // std::cout << "calling refill\n";
         if (st.fetching || st.local_tasks.size() >= st.low_water_mark + st.batch_size)
             return;
 
         st.fetching = true;
+        // std::cout << "refilling buffer\n";
         self->mail(get_work_atom_v, st.batch_size).request(st.global_pool, infinite).then(
             [=](std::vector<Task>& batch) {
                 auto& st_inner = self->state();
@@ -357,13 +372,19 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
                 st_inner.fetching = false;
                 if (st_inner.local_tasks.size() < st_inner.low_water_mark)
                     self->mail(refill_buffer_atom_v).send(self);
+                // std::cout << "refilled buffer\n";
+
             },
             [=](error& err) {
+                // std::cout << "Hello\n";
                 self->state().fetching = false;
             }
         );
     };
 
+    self->attach_functor([self, dev_id](const error& reason) {
+        self->println("device actor {} quitting, reason: {}", dev_id, reason);
+    });
     return {
         [=](restart_atom) {
             self->println("Device Actor {}: Restarting workers...", self->state().device_id);
@@ -402,6 +423,7 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
             refill();
         },
         [=](get_work_atom) -> caf::result<int, int, int, in<int>, in<int>> {
+            // std::cout << "Device actor giving work\n";
             auto& st = self->state();
             if (st.resetting) return make_error(sec::runtime_error, "Device is resetting");
 
@@ -479,6 +501,7 @@ struct supervisor_actor_state {
     int total_tasks;
     int completed = 0;
     std::chrono::steady_clock::time_point start_time;
+    std::vector<caf::actor> device_actors;
 };
 
 caf::behavior supervisor_actor_fun(
@@ -503,12 +526,15 @@ caf::behavior supervisor_actor_fun(
     
     auto poison_p = mgr.create_program_from_cubin("../poison.cubin", "poison_kernel");
 
+    // std::cout << "creating workers\n";
     for (int i = 0; i < num_gpus; ++i) {
-        self->spawn(gpu_device_actor, pool, pool_actor, self, workers_per_gpu, i,
-                    max_in_flight_tasks_per_worker, 
-                    mmul_p, vadd_p, conv_p, poison_p,
-                    shared_dtoh_ptr,
-                    10); // 10% chance for a worker to poison the context
+        auto dev_actor = self->spawn(gpu_device_actor, pool, pool_actor, self,
+                                     workers_per_gpu, i,
+                                     max_in_flight_tasks_per_worker,
+                                     mmul_p, vadd_p, conv_p, poison_p,
+                                     shared_dtoh_ptr,
+                                     10); // 10% chance for a worker to poison
+        self->state().device_actors.push_back(dev_actor);
     }
 
     return {
@@ -545,11 +571,11 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     const int max_N = 2048;
     const int num_sizes = 60;
 
-    const int workers_per_gpu = 4; // Admission control: only 16 concurrent tasks per GPU
-    const int max_in_flight_tasks_per_worker = 5; // Each worker keeps 2 tasks in flight
+    const int workers_per_gpu = 8; // Admission control: only 16 concurrent tasks per GPU
+    const int max_in_flight_tasks_per_worker = 3; // Each worker keeps 2 tasks in flight
 
     const std::vector<int> actor_counts = {
-	  50000,100000
+	  10000
     };
 
     // Generate deterministic random pool once

From 865352d8cc8201d712f9c6842a9b35362be2aadf Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 19 May 2026 15:31:11 -0600
Subject: [PATCH 0653/1000] updated documentation

---
 libcaf_cuda/documentation.txt | 128 ++++++++++++++++++++++++----------
 1 file changed, 91 insertions(+), 37 deletions(-)

diff --git a/libcaf_cuda/documentation.txt b/libcaf_cuda/documentation.txt
index 28e4029387..1f3541c7d6 100644
--- a/libcaf_cuda/documentation.txt
+++ b/libcaf_cuda/documentation.txt
@@ -30,7 +30,17 @@ Lastly it is also expected that arguments that you give to GPU Actors appear in
 
 == The Actor Facade
 All GPU programming boils down to the following: write a kernel, configure its dimensions, transfer the memory to the GPU, launch the kernel and finally transfer the memory from the GPU back to the device. GPU Actors have two primary ways of configuring how you want to navigate this workflow. 
-The first way is using the custom GPU Actor called actor facade. 
+The first way is using the **Actor Facade**. 
+
+The `actor_facade` is a stateless, high-level wrapper for GPU kernels. It is designed to be fully asynchronous: when you send it a message, it immediately enqueues the GPU work and returns. Results are pushed back to the requester via individual messages once the GPU completes the work and the data is transferred back to the host.
+
+**Key Advantages:**
+- **Non-blocking:** It never stalls a CAF worker thread.
+- **Push-based:** Results are "pushed" to your actor as they become available.
+- **Selective Transfers:** You can specify exactly which buffers to copy back.
+
+### Example Kernel: mmul.cu
+For clarity, here is the CUDA kernel code (`mmul.cu`) used in the following examples. It performs standard matrix multiplication on the GPU.
 
 {{{
 // mmul.cu
@@ -46,42 +56,85 @@ void matrixMul(const int* a, const int* b, int* c, int N) {
         c[row * N + col] = temp;
     }
 }
-
 }}}
 
+### Spawning and Communicating with the Facade
+The facade is spawned using a kernel file and argument tags. Because it is asynchronous, do not use `.request().then()`. Instead, use `send()` and define handlers in your actor's behavior to receive the results.
+
 {{{
- // Spawn actor from precompiled cubin file
+  // Create a facade for matrix multiplication
   caf::actor gpuActor = mgr.spawnFromCUBIN(
-                  "../mmul.cubin", //kernel file location
-                  "matrixMul", //kernel name 
-                  dim, //kernel dimensions             
-                  in<int>{}, in<int>{}, out<int>{}, in<int>{} //kernel arg tags 
-                                                              //in order they appear in kernel
+                  "../mmul.cubin", "matrixMul", dim,
+                  in<int>{}, in<int>{}, out<int>{}, in<int>{} 
                   );
+  // Launching the kernel
+  self->mail(arg1, arg2, arg3, arg4).send(gpuActor);
+
+  // Handling results
+  return {
+    = {
+      // Handle vector data (e.g., matrix result at index 2)
+    },
+    = {
+      if (index == -1) 
+        self->println("All GPU work and transfers complete.");
+    }
+  };
+}}}
 
+### Understanding the Message Flow
+When you launch a kernel via the facade, it doesn't send a single reply. Instead, it sends a sequence of messages back to the requester:
+
+1. **Per Output Buffer:** For every buffer being copied back, you receive **one** message. 
+   - If it's a standard output, you get a **Data Result**: `(int reply_id, int arg_index, std::vector<T> data)`.
+   - If you used `output_mapping`, you get a **Mapping Notification**: `(int reply_id, int arg_index)`. This tells you the GPU is done writing to the memory address you provided.
+2. **Final Signal:** Once all kernels are finished and all data transfers for that specific launch are complete, you receive **exactly one Completion Signal**: `(int reply_id, int signal)` where the signal is `-1`.
+
+For example, if you launch a kernel with two output buffers, you will receive a total of three messages: two data/notification messages and one completion signal.
+
+### Selective Transfers and Output Mapping
+By default, the facade copies back all `out<T>` and `in_out<T>` arguments. You can optimize this by passing a list of indices or specific memory mappings.
 
-}}}
-All you need to do is tell it where to find the kernel, the dimensions for that kernel and specify the argument tags in order it appears in the kernel. From here you can send the gpuActor a message in the form of tagged kernel args and it will automatically launch the kernel for you and return an output_buffer which you can call extract_vector<T> on to retrieve the std::vector you are looking for. Results in the output buffer appear in the order they appear in the kernel arguments, minus the in<T>/readonly buffers since they are automatically garabage collected.
 {{{
-  //tag the buffers so that the actor facade knows what to do with it 
-  in<T>  arg1 = caf::cuda::create_in_arg(h_a); //matrix A readonly buffer
-  in<T> arg2 = caf::cuda::create_in_arg(h_b); //matrix B readonly buffer
-  out<T> arg3 = caf::cuda::create_out_arg_with_size<int>(N*N); //matrix size writeonly buffer
-  in<T> arg4 = caf::cuda::create_in_arg(N); // int size, readonly scalar
-
-  self_actor->mail(arg1, arg2, arg3, arg4)
-      .request(gpuActor, std::chrono::seconds(10))
-      .then([=](const std::vector<output_buffer>& outputs) {
-        std::vector<int> result = caf::cuda::extract_vector<int>(outputs); //collect the result buffer from output
-         //do something with it 
+  // Only copy back the 3rd argument (index 2)
+  std::vector<int> indices = {2};
+  self->mail(indices, arg1, arg2, arg3, arg4).send(gpuActor);
 
+  // Or, copy directly into a pre-allocated host buffer (output mapping)
+  std::vector<caf::cuda::output_mapping> mappings = { {2, my_raw_ptr, count} };
+  self->mail(mappings, arg1, arg2, arg3, arg4).send(gpuActor);
 }}}
 
 
-
 == Command Runner
-The second way of expressing this workflow is using the command runner as an entry point to the GPU to create your own custom GPU actors by mixing up the original caf actors along with the cuda api that GPU Actors has. The Command Runner class functionally does everything that the actor facade can do (minus receiving and replying to messages that's the custom actors job) as well as some additional advanced features such as controlling which GPU and stream your kernel launches on, shared memory support and GPU memory management for more complicated pipelines. These features will be discussed further down below as right now we will just show the same operation that the actor facade did above using command runner.
+The second way of expressing this workflow is using the `command_runner` as an entry point to the GPU. This allows you to create custom GPU actors by combining standard CAF actor logic with the GPU Actors API. 
+
+**Warning:** It is highly recommended to use asynchronous methods (`run_async`, `copy_to_host_async`, etc.). Synchronous methods like `run()` and `copy_to_host()` block the calling thread until the GPU operation and data transfer are complete. In a CAF-based system, this stalls the worker thread, preventing it from processing other actors and significantly reducing overall system throughput.
+
+The Command Runner class functionally does everything that the actor facade can do (minus receiving and replying to messages) as well as some additional advanced features such as controlling which GPU and stream your kernel launches on, shared memory support and GPU memory management.
 
+### Asynchronous Operations
+The `command_runner` provides several asynchronous operations, allowing for non-blocking execution and more fine-grained control over the GPU workflow.
+
+**Asynchronous Kernel Launch (`run_async`)**
+Unlike the synchronous `run()` method, `run_async()` immediately returns a tuple of `mem_ptr`s representing the output and in-out arguments on the GPU. This allows the CAF worker thread to continue execution or handle other messages while the GPU kernel is running.
+
+**Asynchronous Memory Transfers (`transfer_memory` and `copy_to_host_async`)**
+The `transfer_memory` methods are designed for asynchronous data transfer to the device. Similarly, `copy_to_host_async` allows for non-blocking transfer of data from the GPU back to the host. These methods can often take callbacks, enabling the execution of host-side code once the transfer is complete, without blocking the calling thread.
+
+**Stream Callbacks (`add_callback`)**
+The `add_callback` method allows you to register a host-side lambda to be executed on the associated CUDA stream. This callback triggers only after all preceding tasks (kernels, transfers) enqueued on that stream are finished.
+
+{{{
+    // Example: Launch and notify when done
+    auto results = mmul.run_async(program, dim, stream_id, arg1, arg2, arg3, arg4);
+    
+    runner.add_callback(stream_id, device_num, self {
+        self->println("GPU Kernel and transfers are finished!");
+    });
+}}}
+
+While you can do whatever you want in these callbacks, so long as you do not invoke other cuda calls. The best pattern is to use callbacks to send messages to other actors when an operation on the GPU is completed
 
 {{{
 
@@ -109,24 +162,23 @@ mmulCommand mmul;
     out<int> arg3 = caf::cuda::create_out_arg_with_size<int>(N*N); //matrix C (specify with size)
     in<int> arg4 = caf::cuda::create_in_arg(N); //size of the matrices
 
-    //launch kernel and collect the output 
-    auto tempC = mmul..run(
+   //launch kernel asynchronously to avoid blocking worker threads
+    auto results = mmul.run_async(
                           program,//kernel to launch
                           dim, //kernel dimensions
-                          id, //actor id 
+                          stream_id, // stream identifier
                           arg1,arg2,arg3,arg4 //kernel args in order that they appear in the kernel  
                           );
-    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
+    
+    // results is std::tuple<mem_ptr<int>...>
+    auto matrixC_ptr = std::get<2>(results);
 
+    // results is a tuple of mem_ptr<int>. 
+    // Use copy_to_host_async to get data back without blocking.
 }}}
 
 For a more in depth look on how this is done go to https://github.com/uofs-simlab/actor-framework/blob/main/libcaf_cuda/examples/custom-actor-examples/mmul.example.cpp 
 
-== Actor id's
-Actors use ids as a way to request resources on the GPU, mainly streams (command queues on the GPU). The actor facade will automatically get its own id and separate GPU resources from other actors. However if you are creating your own custom GPU actor, then you can decide if 2 or more actors or kernel launches share the same GPU resources, such as streams by using the same id per request. It is worth mentioning that at a certain limit (after 500 actor ids) GPU resources will be shared amongst actors to prevent extreme overhead costs of managing all the GPU resources or even crashing for having too much resources allocated
-
-
-
 == Multiple GPU Support and Device Numbers
 GPU Actors will attempt to use all GPUs (devices) it detects when launching kernels. The only time it will not use a GPU it detects is if it detects that the GPUs are different from one another since there is no way of knowing what program belongs on which device without querying the user, in which case it will revert to using the first device it detects. When using the actor facade or the command runner and do not specify a device number, it will use a simple lottery scheduler as a way of determining which GPU to launch your kernel on. If you are using command runner you can decide what device your kernel launches on by specifying a device number. The agreement is as follows, any set of commands that have the same device numbers will launch on the same device, while any 2 sets of commands with different device numbers may or may not launch on the same device, depending on how many devices are found on the computer. It will module arithmetic the device number with the number of devices to determine which device to put the command on. For example given device numbers 0,1,2,3,4 if the number of devices on a computer is 4. Then commands with device numbers 0,4 would appear on device 0 (since 0%4 and 4%4 = 0), commands with device numbers of 1 would appear on device 1, and device numbers 2 and 3 would appear on the corresponding device. If there are only 2 devices then commands with device even device numbers (0,2,4) would appear on device 0, while commands with odd device numbers (1,3) would appear on device 1. 
 
@@ -176,16 +228,20 @@ void generate_random_matrix(int* matrix, int total_elements, int seed, int max_v
                           );
            caf::cuda::mem_ptr<int> matrixA =  std::get<0>(tempA); //fetch the memory of the first argument of the kernel
 }}}
-
+<br>
 In the example listed above we can see that getting the zeroth element of the tuple, grabs the memory associated with the matrix parameter/buffer in the generate_random_matrix kernel.
 It is worth mentioning that if the user wishes to use the manual memory management via run_async(), they are responsible for handling the synchronization of the memory and kernels associated with it as well as ensuring that the memory does not end up on a different device than it is allocated on. mem_ptrs do have methods to help out with this. 
 
-For starters recall that GPU Actors use multiple streams and using memory between streams will cause race conditions,there are two ways of handling this. By far the simplest way to handle this is to ensure each actor or command that interacts with the mem_ptr uses the same actor id. In doing so it will guarantee that the same stream is used on the memory and will preserve in order operations. Alternatively you can call mem_ptr synchronize() method, which will synchronize all GPU operations on that memory with the CPU.
+Recall that GPU Actors use multiple streams; using memory across different streams without care will cause race conditions. The simplest fix is to ensure each command interacting with a `mem_ptr` uses the same `stream_id`. This guarantees in-order execution. Alternatively, you can call the `synchronize()` method, though this is a blocking operation.
 
 For ensuring memory does not end up on a different device than it is currently allocated on. The user needs to ensure the device number is the same for all mem_ptrs being used in run_async. mem_ptrs do track what device number they use (although it may differ from the user's device number if it is higher than the number of GPUs, it will end up being on the same device) with the deviceNumber() method.
+<br>
+**Asynchronous Memory Deallocation**
+`mem_ptr`s are smart pointers managing GPU memory. When a `mem_ptr`'s reference count drops to zero, the GPU memory is not immediately freed in a blocking fashion. Instead, the deallocation is enqueued onto the associated CUDA stream. This ensures that memory is only released after all pending GPU operations using that memory have completed, and it prevents the CPU from stalling during the `free` call. 
 
-Lastly, to transfer memory from the GPU to the CPU use the copy_to_host() method. It is a blocking operation (since it is synchronizing with the GPU) that will always return an std::vector<T>.
+The `actor_facade`'s destructor explicitly calls `release_stream_for_actor`, which ensures that the stream resources and any pending asynchronous deallocations tied to that actor's lifecycle are cleaned up properly.
 
+To transfer memory from the GPU to the CPU, always prefer `copy_to_host_async()` with a callback to keep your actor system responsive.
 
 
 
@@ -204,5 +260,3 @@ The command runner does support kernels with shared memory (actor facade does no
 
 
 For more details, see the example code found at https://github.com/uofs-simlab/actor-framework/tree/main/libcaf_cuda/examples
-
-

From 40f14a374c6c624bd55ebda47610e2571f540f09 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 10:30:28 -0600
Subject: [PATCH 0654/1000] updated actor facade to have a return_mem_ptr atom
 that will return all memory handles isntead of transfering anything back to
 host

---
 libcaf_cuda/caf/cuda/actor_facade.hpp         | 65 +++++++++++++------
 libcaf_cuda/caf/cuda/global.hpp               |  1 +
 .../tests/actor-facade-test/main.test.cpp     | 56 ++++++++++++++++
 3 files changed, 102 insertions(+), 20 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/actor_facade.hpp b/libcaf_cuda/caf/cuda/actor_facade.hpp
index f328ef6bff..a74438e9b0 100644
--- a/libcaf_cuda/caf/cuda/actor_facade.hpp
+++ b/libcaf_cuda/caf/cuda/actor_facade.hpp
@@ -54,37 +54,46 @@ class actor_facade : public event_based_actor {
 
   caf::behavior make_behavior() override {
     return {
+      [this](return_mem_ptr_atom, int device_num, int stream_id, std::vector<int> output_indices, Ts... args) {
+        enqueue_impl(device_num, stream_id, std::move(output_indices), {}, true, std::forward<Ts>(args)...);
+      },
+      [this](return_mem_ptr_atom, std::vector<int> output_indices, Ts... args) {
+        enqueue_impl(-1, static_cast<int>(actor_id_), std::move(output_indices), {}, true, std::forward<Ts>(args)...);
+      },
+      [this](return_mem_ptr_atom, Ts... args) {
+        enqueue_impl(-1, static_cast<int>(actor_id_), {}, {}, true, std::forward<Ts>(args)...);
+      },
       [this](int device_num, int stream_id, std::vector<int> output_indices, Ts... args) {
-        enqueue_impl(device_num, stream_id, std::move(output_indices), {}, std::forward<Ts>(args)...);
+        enqueue_impl(device_num, stream_id, std::move(output_indices), {}, false, std::forward<Ts>(args)...);
       },
       [this](int device_num, int stream_id, Ts... args) {
-        enqueue_impl(device_num, stream_id, {}, {}, std::forward<Ts>(args)...);
+        enqueue_impl(device_num, stream_id, {}, {}, false, std::forward<Ts>(args)...);
       },
       [this](int device_num, std::vector<int> output_indices, Ts... args) {
-        enqueue_impl(device_num, static_cast<int>(actor_id_), std::move(output_indices), {},
+        enqueue_impl(device_num, static_cast<int>(actor_id_), std::move(output_indices), {}, false,
                      std::forward<Ts>(args)...);
       },
       [this](std::vector<int> output_indices, Ts... args) {
-        enqueue_impl(-1, static_cast<int>(actor_id_), std::move(output_indices), {},
+        enqueue_impl(-1, static_cast<int>(actor_id_), std::move(output_indices), {}, false,
                      std::forward<Ts>(args)...);
       },
       [this](int device_num, Ts... args) {
         // Copy everything back if indices are omitted
-        enqueue_impl(device_num, static_cast<int>(actor_id_), {}, {}, std::forward<Ts>(args)...);
+        enqueue_impl(device_num, static_cast<int>(actor_id_), {}, {}, false, std::forward<Ts>(args)...);
       },
       [this](Ts... args) {
         // Copy everything back if indices are omitted
-        enqueue_impl(-1, static_cast<int>(actor_id_), {}, {}, std::forward<Ts>(args)...);
+        enqueue_impl(-1, static_cast<int>(actor_id_), {}, {}, false, std::forward<Ts>(args)...);
       },
       // Mapping handlers
       [this](int device_num, int stream_id, std::vector<output_mapping> mappings, Ts... args) {
-        enqueue_impl(device_num, stream_id, {}, std::move(mappings), std::forward<Ts>(args)...);
+        enqueue_impl(device_num, stream_id, {}, std::move(mappings), false, std::forward<Ts>(args)...);
       },
       [this](int device_num, std::vector<output_mapping> mappings, Ts... args) {
-        enqueue_impl(device_num, static_cast<int>(actor_id_), {}, std::move(mappings), std::forward<Ts>(args)...);
+        enqueue_impl(device_num, static_cast<int>(actor_id_), {}, std::move(mappings), false, std::forward<Ts>(args)...);
       },
       [this](std::vector<output_mapping> mappings, Ts... args) {
-        enqueue_impl(-1, static_cast<int>(actor_id_), {}, std::move(mappings), std::forward<Ts>(args)...);
+        enqueue_impl(-1, static_cast<int>(actor_id_), {}, std::move(mappings), false, std::forward<Ts>(args)...);
       }
     };
   }
@@ -92,7 +101,7 @@ class actor_facade : public event_based_actor {
 private:
   template <class... Us>
   void enqueue_impl(int device_num, int stream_id, std::vector<int> output_indices, 
-                    std::vector<output_mapping> mappings, Us&&... xs) {
+                    std::vector<output_mapping> mappings, bool return_mem_ptrs, Us&&... xs) {
     command_runner<Ts...> runner;
     auto results = runner.run_async(program_, dims_, stream_id, 0, 
                                     device_num, std::forward<Us>(xs)...);
@@ -100,8 +109,32 @@ class actor_facade : public event_based_actor {
     auto sender = actor_cast<actor>(this->current_sender());
     auto r_id = reply_id_;
 
+    if (sender) {
+      if (return_mem_ptrs) {
+        send_mem_ptr_handles(sender, r_id, results);
+      } else {
+        process_host_transfers(sender, r_id, results, std::move(output_indices), std::move(mappings));
+      }
+    }
+
+    // Final completion alert (Correlation ID, -1 to signal "all finished")
+    runner.add_callback(stream_id, device_num, [sender, r_id]() mutable {
+      if (sender) {
+        caf::anon_mail(r_id, -1).send(sender);
+      }
+    });
+  }
+
+  void send_mem_ptr_handles(const actor& sender, int r_id, const mem_tuple& results) {
+    std::apply([&](auto&&... args) {
+      caf::anon_mail(r_id, std::forward<decltype(args)>(args)...).send(sender);
+    }, results);
+  }
+
+  void process_host_transfers(const actor& sender, int r_id, const mem_tuple& results,
+                              std::vector<int> output_indices, std::vector<output_mapping> mappings) {
     // Determine which indices to process based on requests and mappings
-    std::vector<int> targets = output_indices;
+    std::vector<int> targets = std::move(output_indices);
     for (const auto& m : mappings) {
       if (std::find(targets.begin(), targets.end(), m.index) == targets.end()) {
         targets.push_back(m.index);
@@ -114,6 +147,7 @@ class actor_facade : public event_based_actor {
       }
     }
 
+    command_runner<Ts...> runner;
     for (int idx : targets) {
       if (idx >= 0 && idx < static_cast<int>(sizeof...(Ts))) {
         // Dispatch runtime index to compile-time sequence
@@ -141,7 +175,6 @@ class actor_facade : public event_based_actor {
               runner.copy_to_host_async(mem_ptr, static_cast<ValueType*>(custom_dst), dst_count, 
                 [sender, r_id, Index](ValueType*, size_t) {
                   if (sender) {
-                    // Notify requester that this index is ready in their buffer
                     caf::anon_mail(r_id, static_cast<int>(Index)).send(sender);
                   }
                 });
@@ -149,7 +182,6 @@ class actor_facade : public event_based_actor {
               // Default: Copy into a new vector and send back
               runner.copy_to_host_async(mem_ptr, [sender, r_id, Index](std::vector<ValueType>&& data) {
                 if (sender) {
-                  // Send: Correlation ID, Argument Index, Data Vector
                   caf::anon_mail(r_id, static_cast<int>(Index), std::move(data)).send(sender);
                 }
               });
@@ -160,13 +192,6 @@ class actor_facade : public event_based_actor {
         this->println("Warning: Output index {} is out of bounds", idx);
       }
     }
-
-    // Final completion alert (Correlation ID, -1 to signal "all finished")
-    runner.add_callback(stream_id, device_num, [sender, r_id]() mutable {
-      if (sender) {
-        caf::anon_mail(r_id, -1).send(sender);
-      }
-    });
   }
 
   // Helper to map runtime index to compile-time index for tuple access
diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index dad370ec61..3367d97391 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -207,6 +207,7 @@ CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id)
   CAF_ADD_ATOM(cuda, htod_done_atom)
   CAF_ADD_ATOM(cuda, dtoh_done_atom)
   CAF_ADD_ATOM(cuda, gpu_done_atom)
+  CAF_ADD_ATOM(cuda, return_mem_ptr_atom)
 
 CAF_END_TYPE_ID_BLOCK(cuda)
 
diff --git a/libcaf_cuda/tests/actor-facade-test/main.test.cpp b/libcaf_cuda/tests/actor-facade-test/main.test.cpp
index c3d78d13c5..d12c2eb7b1 100644
--- a/libcaf_cuda/tests/actor-facade-test/main.test.cpp
+++ b/libcaf_cuda/tests/actor-facade-test/main.test.cpp
@@ -184,6 +184,59 @@ caf::behavior mmul_mapping_test(caf::stateful_actor<test_actor_state>* self,
     };
 }
 
+// New Separate Actor for testing mem_ptr return
+caf::behavior mmul_mem_ptr_return_test(caf::stateful_actor<test_actor_state>* self, 
+                                       caf::actor facade, int N) {
+    self->state().N = N;
+    self->state().h_a.assign(N * N, 7); // Use different values
+    self->state().h_b.assign(N * N, 8);
+    self->state().h_c.resize(N * N); // This will be filled from mem_ptr copy
+
+    auto arg1 = caf::cuda::create_in_arg(self->state().h_a);
+    auto arg2 = caf::cuda::create_in_arg(self->state().h_b);
+    auto arg3 = caf::cuda::create_out_arg_with_size<int>(N * N);
+    auto arg4 = caf::cuda::create_in_arg(N);
+
+    std::cout << "[INFO] Launching mem_ptr return test..." << std::endl;
+    self->state().start_time = std::chrono::steady_clock::now();
+    
+    // Request mem_ptrs back directly
+    self->mail(return_mem_ptr_atom_v, arg1, arg2, arg3, arg4).send(facade);
+
+    return {
+        [=](int r_id, caf::cuda::mem_ptr<int> a_ptr, caf::cuda::mem_ptr<int> b_ptr, 
+            caf::cuda::mem_ptr<int> c_ptr, caf::cuda::mem_ptr<int> n_ptr) {
+            
+            auto end_time = std::chrono::steady_clock::now();
+            std::chrono::duration<double> elapsed = end_time - self->state().start_time;
+            std::cout << "[MEM_PTR_RETURN] Latency: " << elapsed.count() << "s" << std::endl;
+
+            // Only copy the output mem_ptr (c_ptr) to host, as IN arguments cannot be copied back.
+            std::vector<int> h_c_from_gpu = c_ptr->copy_to_host();
+            
+            // Verify the output matrix C using the original host inputs
+            verify_mmul(self->state().h_a, self->state().h_b, h_c_from_gpu, self->state().N);
+
+            // Optionally, verify mem_ptr metadata for all returned pointers
+            if (a_ptr && b_ptr && c_ptr && n_ptr) {
+                if (a_ptr->size() == self->state().N * self->state().N && a_ptr->access() == IN &&
+                    b_ptr->size() == self->state().N * self->state().N && b_ptr->access() == IN &&
+                    c_ptr->size() == self->state().N * self->state().N && c_ptr->access() == OUT &&
+                    n_ptr->size() == 1 && n_ptr->access() == IN) {
+                    std::cout << "[SUCCESS] All mem_ptrs returned are valid and metadata is correct." << std::endl;
+                } else {
+                    std::cout << "[FAILURE] Mem_ptr metadata mismatch!" << std::endl;
+                }
+            } else {
+                std::cout << "[FAILURE] One or more mem_ptrs returned are null!" << std::endl;
+            }
+
+            self->send_exit(facade, exit_reason::user_shutdown);
+            self->quit();
+        }
+    };
+}
+
 void caf_main(caf::actor_system& sys) {
     // Initialize the CUDA subsystem
     caf::cuda::manager::init(sys);
@@ -212,6 +265,9 @@ void caf_main(caf::actor_system& sys) {
     // 3. Run Mapping Test
     sys.spawn(mmul_mapping_test, facade, N);
 
+    // 4. Run mem_ptr return test
+    sys.spawn(mmul_mem_ptr_return_test, facade, N);
+
     sys.await_all_actors_done();
     caf::cuda::manager::shutdown();
 }

From 9d119cb3707ad7650538c2bd6148901d18fdd806 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 10:46:37 -0600
Subject: [PATCH 0655/1000] updated benchmakring tests to use cubin files

---
 .../matrix_mul_driver.cpp                     |  45 +++---
 .../mmul-actor-benchmarking/main.test.cpp     | 150 +-----------------
 2 files changed, 28 insertions(+), 167 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
index c27370d98b..a555bb50ce 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
@@ -26,7 +26,9 @@ std::string readFile(const std::string &path) {
     return ss.str();
 }
 
-void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
+std::vector<int> h_c; // Declared globally
+
+void runMatrixMul(CUmodule module, CUfunction kernel, int N, CUstream stream) {
     using clock = std::chrono::steady_clock;
     using ms = std::chrono::duration<double, std::milli>;
 
@@ -35,18 +37,11 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     size_t elements = (size_t)N * (size_t)N;
     size_t bytes = elements * sizeof(int);
 
-    std::vector<int> h_a(elements, 1);
-    std::vector<int> h_b(elements, 1);
-    std::vector<int> h_c(elements);
-    //int h_c[elements];
+    std::vector<int> h_a(elements, 1); // These remain local as they are initialized with N
+    std::vector<int> h_b(elements, 1); // These remain local as they are initialized with N
+    h_c.resize(elements); // Resize the global h_c for the current N
 
     CUdeviceptr d_a, d_b, d_c;
-    CUstream stream;
-
-    // ----------------------------------
-    // Create Stream
-    // ----------------------------------
-    checkCU(cuStreamCreate(&stream, CU_STREAM_DEFAULT), "cuStreamCreate");
 
     auto t_total_start = clock::now();
 
@@ -67,7 +62,7 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     auto t_h2d_a_start = clock::now();
 
     checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), bytes, stream), "cuMemcpyHtoDAsync A");
-    //checkCU(cuStreamSynchronize(stream), "sync A");
+    std::cout << "  (Transfer size: " << bytes << " bytes)\n";
 
     auto t_h2d_a_end = clock::now();
 
@@ -78,6 +73,7 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
 
     checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), bytes, stream), "cuMemcpyHtoDAsync B");
     //checkCU(cuStreamSynchronize(stream), "sync B");
+    std::cout << "  (Transfer size: " << bytes << " bytes)\n";
 
     auto t_h2d_b_end = clock::now();
 
@@ -114,6 +110,7 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     cuMemcpyDtoHAsync(h_c.data(), d_c, bytes, stream);
    // cuMemcpyDtoHAsync(h_c, d_c, bytes, stream);
     cuStreamSynchronize(stream);
+    std::cout << "  (Transfer size: " << bytes << " bytes)\n";
     auto t_d2h_end = clock::now();
 
     auto t_total_end = clock::now();
@@ -133,12 +130,6 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N) {
 
 
 
-    // ----------------------------------
-    // Destroy stream
-    // ----------------------------------
-    checkCU(cuStreamDestroy(stream), "cuStreamDestroy");
-
-
     // ----------------------------------
     // Print Results
     // ----------------------------------
@@ -189,33 +180,37 @@ int main(int argc, char** argv) {
     checkCU(cuDeviceGet(&dev, 0), "cuDeviceGet(0)");
 
     CUcontext ctx;
-    checkCU(cuCtxCreate(&ctx, 0, dev), "cuCtxCreate");
+    checkCU(cuCtxCreate(&ctx, CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, dev), "cuCtxCreate");
 
-    const std::string ptxPath = "mmul.ptx";
-    std::string ptx;
+    const std::string cubinPath = "../mmul.cubin";
+    std::string cubin;
 
     try {
-        ptx = readFile(ptxPath);
+        cubin = readFile(cubinPath);
     } catch (const std::exception &e) {
-        std::cerr << "Failed to read PTX file '" << ptxPath << "': " << e.what() << "\n";
+        std::cerr << "Failed to read CUBIN file '" << cubinPath << "': " << e.what() << "\n";
         return EXIT_FAILURE;
     }
 
     CUmodule module;
-    checkCU(cuModuleLoadDataEx(&module, ptx.c_str(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
+    checkCU(cuModuleLoadDataEx(&module, cubin.data(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
 
     CUfunction kernel;
     checkCU(cuModuleGetFunction(&kernel, module, "matrixMul"), "cuModuleGetFunction matrixMul");
 
+    CUstream stream;
+    checkCU(cuStreamCreate(&stream, CU_STREAM_DEFAULT), "cuStreamCreate");
+
     for (int N : sizes) {
         try {
-            runMatrixMul(module, kernel, N);
+            runMatrixMul(module, kernel, N, stream);
         } catch (const std::exception &e) {
             std::cerr << "Exception while running N=" << N << ": " << e.what() << "\n";
         }
         std::cout << "----------------------------------------\n";
     }
 
+    checkCU(cuStreamDestroy(stream), "cuStreamDestroy");
     checkCU(cuModuleUnload(module), "cuModuleUnload");
     checkCU(cuCtxDestroy(ctx), "cuCtxDestroy");
 
diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
index 9ecd2257ce..31e65b18ae 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/main.test.cpp
@@ -49,148 +49,6 @@ struct mmul_state {
 std::vector<int> matrixC;
 
 
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self) {
-  return {
-
-    [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB,
-        int N) {
-
-      using clock = std::chrono::steady_clock;
-      using ms = std::chrono::duration<double, std::milli>;
-
-
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
-      int device = 0;
-      int stream = 1;
-
-      auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-      auto t_total_start = clock::now();
-      // -------------------------
-      // create_in_arg A
-      // -------------------------
-      auto t_a_inarg_start = clock::now();
-
-      auto inA = caf::cuda::create_in_arg(std::move(matrixA));
-
-      auto t_a_inarg_end = clock::now();
-
-      // -------------------------
-      // transfer A
-      // -------------------------
-      auto t_a_transfer_start = clock::now();
-
-      auto arg1 = mmul_command.transfer_memory(
-          device,
-          stream,
-          std::move(inA));
-
-      auto t_a_transfer_end = clock::now();
-
-      // -------------------------
-      // create_in_arg B
-      // -------------------------
-      auto t_b_inarg_start = clock::now();
-
-      auto inB = caf::cuda::create_in_arg(std::move(matrixB));
-
-      auto t_b_inarg_end = clock::now();
-
-      // -------------------------
-      // transfer B
-      // -------------------------
-      auto t_b_transfer_start = clock::now();
-
-      auto arg2 = mmul_command.transfer_memory(
-          device,
-          stream,
-          std::move(inB));
-
-      auto t_b_transfer_end = clock::now();
-
-      // -------------------------
-      // spawn actor
-      // -------------------------
-      auto t_spawn_start = clock::now();
-
-      caf::actor mmul_actor =
-        self->spawn(caf::cuda::mmul_actor_fun<int>, program);
-
-      auto t_spawn_end = clock::now();
-
-      // -------------------------
-      // request
-      // -------------------------
-      auto t_request_start = clock::now();
-
-      self->mail(arg1, arg2, N, device, stream)
-        .request(mmul_actor, std::chrono::seconds(30))
-        .then(
-          [=](caf::cuda::mem_ptr<int> dC) {
-
-            auto t_response_received = clock::now();
-
-            //std::vector<int> matrixC(N*N);
-            // -------------------------
-            // copy to host
-            // -------------------------
-            auto t_copy_start = clock::now();
-            
-            mmul_command.copy_to_host_async(dC, matrixC.data(), N * N);
-            dC->synchronize();
-
-            auto t_copy_end = clock::now();
-            auto t_total_end = clock::now();
-
-            // -------------------------
-            // Print timings
-            // -------------------------
-
-            std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
-
-            std::cout << "create_in_arg A: "
-                      << ms(t_a_inarg_end - t_a_inarg_start).count()
-                      << " ms\n";
-
-            std::cout << "transfer A: "
-                      << ms(t_a_transfer_end - t_a_transfer_start).count()
-                      << " ms\n";
-
-            std::cout << "create_in_arg B: "
-                      << ms(t_b_inarg_end - t_b_inarg_start).count()
-                      << " ms\n";
-
-            std::cout << "transfer B: "
-                      << ms(t_b_transfer_end - t_b_transfer_start).count()
-                      << " ms\n";
-
-            std::cout << "spawn actor: "
-                      << ms(t_spawn_end - t_spawn_start).count()
-                      << " ms\n";
-
-            std::cout << "request → response latency: "
-                      << ms(t_response_received - t_request_start).count()
-                      << " ms\n";
-
-            std::cout << "copy_to_host: "
-                      << ms(t_copy_end - t_copy_start).count()
-                      << " ms\n";
-
-            std::cout << "TOTAL end-to-end: "
-                      << ms(t_total_end - t_total_start).count()
-                      << " ms\n";
-
-            std::cout << "=============================================\n";
-
-            self->quit();
-          }
-        );
-    }
-
-  };
-}
 
 
 caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
@@ -203,6 +61,10 @@ caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
       using clock = std::chrono::steady_clock;
       using ms = std::chrono::duration<double, std::milli>;
 
+      size_t bytes_a = matrixA.size() * sizeof(int);
+      size_t bytes_b = matrixB.size() * sizeof(int);
+      size_t bytes_c = matrixC.size() * sizeof(int);
+
 
       caf::cuda::manager& mgr = caf::cuda::manager::get();
       int device = 0;
@@ -231,6 +93,7 @@ caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
           stream,
           std::move(inA));
 
+
       auto t_a_transfer_end = clock::now();
 
       // -------------------------
@@ -293,6 +156,9 @@ caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
             // -------------------------
 
             std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
+            std::cout << "  Transfer size A: " << bytes_a << " bytes\n";
+            std::cout << "  Transfer size B: " << bytes_b << " bytes\n";
+            std::cout << "  Transfer size C: " << bytes_c << " bytes\n";
 
             std::cout << "create_in_arg A: "
                       << ms(t_a_inarg_end - t_a_inarg_start).count()

From bd7293a98a42f2b1302dfad3f7329bfde6aacc98 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 10:50:33 -0600
Subject: [PATCH 0656/1000] made reset device context compatable between
 versions of cuda

---
 libcaf_cuda/caf/cuda/platform.hpp | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/platform.hpp b/libcaf_cuda/caf/cuda/platform.hpp
index 87f06449ce..a29c0ecf06 100644
--- a/libcaf_cuda/caf/cuda/platform.hpp
+++ b/libcaf_cuda/caf/cuda/platform.hpp
@@ -59,7 +59,19 @@ class CAF_CUDA_EXPORT platform : public ref_counted {
 
     // Create a new context
     CUcontext new_ctx;
-    CHECK_CUDA(cuCtxCreate(&new_ctx, 0, cu_dev));
+#if CUDA_VERSION >= 13000
+    {
+      CUctxCreateParams ctx_params = {};
+      CHECK_CUDA(cuCtxCreate(&new_ctx, 
+                             &ctx_params, 
+                             CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, 
+                             cu_dev));
+    }
+#else
+    CHECK_CUDA(cuCtxCreate(&new_ctx, 
+                           CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, 
+                           cu_dev));
+#endif
 
     // Update the device object and platform's internal contexts_ vector
     dev_obj->reset_context(new_ctx);

From 450bf2fd52a49377c1ad2602907285a3fa7a6107 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 12:34:55 -0600
Subject: [PATCH 0657/1000] updated code to not use custreamsynchronize since
 that is expensive operation

---
 .../matrix_mul_driver.cpp                     | 36 ++++++++++++++++---
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
index a555bb50ce..2f0654cc50 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/cuda-program-benchmark/matrix_mul_driver.cpp
@@ -7,6 +7,19 @@
 #include <sstream>
 #include <stdexcept>
 #include <cmath>
+#include <atomic>
+#include <thread>
+
+struct TimingState {
+    std::chrono::steady_clock::time_point end_time;
+    std::atomic<bool> ready{false};
+};
+
+void completion_callback(void* userData) {
+    auto* state = static_cast<TimingState*>(userData);
+    state->end_time = std::chrono::steady_clock::now();
+    state->ready = true;
+}
 
 static void checkCU(CUresult r, const char* where) {
     if (r != CUDA_SUCCESS) {
@@ -106,15 +119,24 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N, CUstream stream) {
     // D2H copy
     // ----------------------------------
     auto t_d2h_start = clock::now();
+    TimingState t_state;
 
     cuMemcpyDtoHAsync(h_c.data(), d_c, bytes, stream);
-   // cuMemcpyDtoHAsync(h_c, d_c, bytes, stream);
-    cuStreamSynchronize(stream);
-    std::cout << "  (Transfer size: " << bytes << " bytes)\n";
-    auto t_d2h_end = clock::now();
 
-    auto t_total_end = clock::now();
+    // Enqueue the host function to capture timing when the copy finishes
+    checkCU(cuLaunchHostFunc(stream, completion_callback, &t_state), "cuLaunchHostFunc");
+
+    // In a real actor, we would not wait here. 
+    // For this benchmark driver, we wait for the callback to fire.
+    while (!t_state.ready) {
+        std::this_thread::yield();
+    }
+
+    std::cout << "  (Transfer size: " << bytes << " bytes)\n";
     
+    auto t_d2h_end = t_state.end_time;
+    auto t_total_end = t_state.end_time;
+
     
     // ----------------------------------
     // Free device memory
@@ -204,6 +226,10 @@ int main(int argc, char** argv) {
     for (int N : sizes) {
         try {
             runMatrixMul(module, kernel, N, stream);
+            
+            // Ensure stream is completely empty before starting the next size
+            checkCU(cuStreamSynchronize(stream), "cuStreamSynchronize between sizes");
+            
         } catch (const std::exception &e) {
             std::cerr << "Exception while running N=" << N << ": " << e.what() << "\n";
         }

From 5b58f4dc54a2b74d5e397e3f05ed822d563c2f60 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 14:01:57 -0600
Subject: [PATCH 0658/1000] Initial commit.

---
 libcaf_cuda/caf/actorBLAS/actorBLAS.hpp | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 libcaf_cuda/caf/actorBLAS/actorBLAS.hpp

diff --git a/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp b/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp
new file mode 100644
index 0000000000..e69de29bb2

From 7cb92e2023b522ab523c24c98bc5fdbc40545066 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 14:22:10 -0600
Subject: [PATCH 0659/1000] added support for cublas handles for these classes

---
 libcaf_cuda/caf/cuda/device.hpp     | 12 +++++++++
 libcaf_cuda/caf/cuda/streampool.hpp | 41 +++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 186b5dfc4f..26e2ca78ad 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -121,6 +121,17 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     stream_table_->release_stream(actor_id);
   }
 
+  /// Enable cuBLAS support.
+  void enable_cublas() {
+    if (!cublas_table_)
+      cublas_table_ = std::make_unique<DeviceCublasHandleTable>(context_);
+  }
+
+  /// Returns the cuBLAS handle associated with the actor id.
+  cublasHandle_t get_cublas_handle(int actor_id) {
+    if (!cublas_table_) return nullptr;
+    return cublas_table_->get_handle(actor_id, get_stream_for_actor(actor_id));
+  }
 
   // Overloads for make_arg using actor_id
   template <typename T>
@@ -310,6 +321,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   int id_;
   const char* name_;
   std::unique_ptr<DeviceStreamTable> stream_table_;
+  std::unique_ptr<DeviceCublasHandleTable> cublas_table_;
   std::mutex stream_mutex_;
 
   // Cached GPU properties (queried once during construction)
diff --git a/libcaf_cuda/caf/cuda/streampool.hpp b/libcaf_cuda/caf/cuda/streampool.hpp
index 569aff7f61..2a3edc7d90 100644
--- a/libcaf_cuda/caf/cuda/streampool.hpp
+++ b/libcaf_cuda/caf/cuda/streampool.hpp
@@ -7,6 +7,7 @@
 #include <mutex>
 #include <shared_mutex>
 #include <cuda.h>
+#include <cublas_v2.h>
 #include <stdexcept>
 
 namespace caf::cuda {
@@ -71,6 +72,46 @@ class CAF_CUDA_EXPORT StreamPool {
   mutable std::mutex pool_mutex_;          ///< Protects the pool state
 };
 
+/// Pool of cuBLAS handles. Capped at 32.
+class CAF_CUDA_EXPORT CublasHandlePool {
+public:
+  explicit CublasHandlePool(CUcontext ctx, size_t max_size = 32);
+  ~CublasHandlePool();
+
+  cublasHandle_t acquire();
+  void release(cublasHandle_t h);
+
+  size_t max_size() const { return max_size_; }
+
+private:
+  cublasHandle_t create_handle();
+
+  CUcontext ctx_;
+  std::deque<cublasHandle_t> available_handles_;
+  std::vector<cublasHandle_t> all_handles_;
+  size_t max_size_;
+  mutable std::mutex pool_mutex_;
+};
+
+/// Per-device cuBLAS handle table.
+class CAF_CUDA_EXPORT DeviceCublasHandleTable {
+public:
+  explicit DeviceCublasHandleTable(CUcontext ctx, size_t pool_size = 32);
+
+  /// Get the cuBLAS handle for an actor and bind it to a stream.
+  cublasHandle_t get_handle(int actor_id, CUstream stream);
+
+  /// Release the handle assigned to an actor.
+  void release_handle(int actor_id);
+
+  size_t pool_size() const { return pool_.max_size(); }
+
+private:
+  CublasHandlePool pool_;
+  std::unordered_map<int, cublasHandle_t> table_; ///< actor_id -> handle
+  mutable std::shared_mutex table_mutex_;
+};
+
 /// Per-device stream manager. Assigns streams to actor IDs.
 /// 
 /// `DeviceStreamTable` caches an assigned stream per `actor_id`. This makes the

From 2fd6dd46df86d1fe88fa17deafe4fcf63c21b463 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 14:23:08 -0600
Subject: [PATCH 0660/1000] added support for cublas for these classes as well
 as cut out all scheduler stuff from manager. Scheduler stuff is being removed
 from manager since it is depreciated and should no longer be used

---
 libcaf_cuda/caf/cuda/manager.hpp        |  15 ---
 libcaf_cuda/caf/cuda/manager_config.hpp |  11 +-
 libcaf_cuda/src/manager.cpp             | 147 +-----------------------
 libcaf_cuda/src/streampool.cpp          |  95 ++++++++++++++-
 4 files changed, 103 insertions(+), 165 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index 3883f67efa..8b1399a56e 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -179,16 +179,6 @@ class CAF_CUDA_EXPORT manager {
 
   double available_memory_mb(int id = 0);
 
-  caf::actor get_scheduler_actor();
-
-
-
-  //methods used to send scheduler actors messages
-  void send_scheduler_actor_message(token_ptr token,int device_number = -1);
-  void send_scheduler_actor_message(std::vector<token_ptr> tokens,int device_number = -1);
-  void send_scheduler_actor_message(behavior_token_ptr token,int device_number);
-  void send_scheduler_actor_message(std::string behavior,int device_number);
-
   caf::actor get_memory_actor();
 
   caf::actor spawn_exit_actor(int num_actors);
@@ -205,19 +195,14 @@ class CAF_CUDA_EXPORT manager {
   //helper to compile a nvrtc program
   bool compile_nvrtc_program(const char* source, CUdevice device, std::vector<char>& ptx_out);
 
-  void init_scheduler_actors(caf::actor_system&);
-  
   //methods to create and destroy memory_actor
   void init_memory_actor(caf::actor_system&);
   void destroy_memory_actor();
 
   static manager* instance_;
   static std::mutex mutex_;
-  bool scheduler_on = false;
   bool memory_manager_on = false;
-  caf::actor scheduler_actor_handle;
   caf::actor memory_actor_handle;
-  std::vector<caf::actor> scheduler_actors;
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/manager_config.hpp b/libcaf_cuda/caf/cuda/manager_config.hpp
index bc630d95f3..3cad6f9ba6 100644
--- a/libcaf_cuda/caf/cuda/manager_config.hpp
+++ b/libcaf_cuda/caf/cuda/manager_config.hpp
@@ -8,17 +8,16 @@ namespace caf::cuda {
 
 class manager_config {
 public:
-    manager_config() : scheduler_on(false) {}      // initialize the bool
-    manager_config(bool scheduler) : scheduler_on(scheduler) {}
-    manager_config(bool scheduler, bool memory_manager) : scheduler_on(scheduler), memory_manager_on(memory_manager) {}
+    manager_config() : actorBLAS(false), memory_manager_on(false) {}
+    manager_config(bool blas) : actorBLAS(blas), memory_manager_on(false) {}
+    manager_config(bool blas, bool memory_manager) : actorBLAS(blas), memory_manager_on(memory_manager) {}
     
-    bool getSchedulerOn() const { return scheduler_on; }
+    bool getActorBLAS() const { return actorBLAS; }
     bool getMemoryManagerOn() const { return memory_manager_on; }
 
 private:
-    bool scheduler_on;
+    bool actorBLAS;
     bool memory_manager_on = false;
 };
 
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index d9f7403c0e..24282683af 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -55,48 +55,19 @@ void manager::init(caf::actor_system& sys, manager_config config) {
     caf::init_global_meta_objects<caf::id_block::cuda>();
     caf::init_global_meta_objects<caf::id_block::cuda_control>();
 
-    instance_->scheduler_on = config.getSchedulerOn();
     instance_->memory_manager_on = config.getMemoryManagerOn();
 
-    if (instance_->scheduler_on) { 
-	    instance_ -> init_scheduler_actors(sys);
+    if (config.getActorBLAS()) {
+        for (auto& dev : instance_->platform_->devices())
+            dev->enable_cublas();
     }
+
     if (instance_->memory_manager_on) {
 	    instance_->init_memory_actor(sys);
     }
-
-
-
 }
 
 int manager::get_num_devices() {return platform_ -> get_num_devices();}
-
-void manager::init_scheduler_actors(caf::actor_system& sys) {
-
-	int num_devices = platform_ -> get_num_devices();
-
-	bool multi_gpu = num_devices > 1;
-	for (int i = 0; i < num_devices; i++) {
-	
-		instance_ -> scheduler_actors.push_back( sys.spawn(scheduler_actor,i,multi_gpu));
-	
-	}
-
-	//if there is multiple GPUs send every scheduler actor contact information 
-	//about the other on	
-	if (num_devices > 1) {
-	
-		for (int i = 0; i < num_devices; i++) {
-			anon_mail(scheduler_actors).send(instance_ -> scheduler_actors[i]);
-		}
-	
-	}
-
-}
-
-
-
-
 // --------------------------------
 // Static get()
 // --------------------------------
@@ -122,52 +93,14 @@ void manager::shutdown() {
     if (!instance_)
         return;
 
-    if (instance_->scheduler_on) {
-        
-	for (int i = 0; i < instance_ ->  platform_ -> get_num_devices(); i++) {
-	    
-	anon_send_exit(
-            instance_->scheduler_actors[i],
-            caf::exit_reason::user_shutdown
-        );
-
-
-
-    }
-
-    }
-
-
     if (instance_->memory_manager_on) {
 	    instance_->destroy_memory_actor();
     }
 
-
     delete instance_;
     instance_ = nullptr;
 }
 
-// --------------------------------
-// Static getter for scheduler actor
-// --------------------------------
-// this is legacy code, do not use
-// only exists to be backwards compatable with tests 
-caf::actor manager::get_scheduler_actor() {
-    	//this is a read only data no need for lock
-	//std::lock_guard<std::mutex> guard(mutex_);
-
-
-    if (!instance_) {
-        throw std::runtime_error("CUDA manager not initialized");
-    }
-
-    return instance_->scheduler_actors[0];
-}
-
-
-
-
-
 device_ptr manager::find_device(std::size_t) const {
   throw std::runtime_error("OpenCL support disabled: manager::find_device");
 }
@@ -305,78 +238,6 @@ bool manager::compile_nvrtc_program(const char* source, CUdevice device, std::ve
 	return caf::cuda::compile_nvrtc_program(source,device,ptx_out);
 }
 
-// ---------------------------------------------
-// Send single token
-// ---------------------------------------------
-void manager::send_scheduler_actor_message(token_ptr token, int device_number) {
-    if (!scheduler_on || scheduler_actors.empty())
-        return;
-
-    int num_devices = static_cast<int>(scheduler_actors.size());
-    int target = -1;
-
-    if (device_number != -1) {
-        // Explicit device
-        if (device_number >= num_devices)
-            return; // silently discard
-        target = device_number;
-    } else {
-        // No device specified
-        if (!token->isIndependent()) {
-            target = token->getDependency() % num_devices;
-            if (target < 0)
-                target += num_devices;
-        } else {
-            target = rand() % num_devices;
-        }
-    }
-
-    anon_mail(token).send(scheduler_actors[target]);
-}
-
-// ---------------------------------------------
-// Send vector of tokens
-// ---------------------------------------------
-void manager::send_scheduler_actor_message(std::vector<token_ptr> tokens,
-                                           int device_number) {
-    if (!scheduler_on || scheduler_actors.empty() || tokens.empty())
-        return;
-
-    int num_devices = static_cast<int>(scheduler_actors.size());
-    int target = -1;
-
-    if (device_number != -1) {
-        // Explicit device
-        if (device_number >= num_devices)
-            return; // silently discard
-        target = device_number;
-    } else {
-        // No device specified → random
-        target = rand() % num_devices;
-    }
-
-    anon_mail(std::move(tokens)).send(scheduler_actors[target]);
-}
-
-void manager::send_scheduler_actor_message(behavior_token_ptr token, int device_number) {
-    if (!scheduler_on || scheduler_actors.empty())
-        return;
-
-    int num_devices = static_cast<int>(scheduler_actors.size());
-
-    // Drop if device number is invalid
-    if (device_number < 0 || device_number >= num_devices)
-        return;
-
-    anon_mail(token).send(scheduler_actors[device_number]);
-}
-
-void manager::send_scheduler_actor_message(std::string behavior, int device_number) {
-    auto token = caf::cuda::make_behavior_token(std::move(behavior));
-    send_scheduler_actor_message(token, device_number);
-}
-
-
 void manager::init_memory_actor(caf::actor_system& sys) {
     if (memory_actor_handle)
         return; // already initialized
diff --git a/libcaf_cuda/src/streampool.cpp b/libcaf_cuda/src/streampool.cpp
index 13aec19d02..e569f09ad3 100644
--- a/libcaf_cuda/src/streampool.cpp
+++ b/libcaf_cuda/src/streampool.cpp
@@ -174,5 +174,98 @@ void DeviceStreamTable::release_stream(int actor_id) {
   }
 }
 
-} // namespace caf::cuda
+// ---------------------- CublasHandlePool ----------------------
+
+CublasHandlePool::CublasHandlePool(CUcontext ctx, size_t max_size)
+    : ctx_(ctx), max_size_(std::min(max_size, (size_t)32)) {}
+
+CublasHandlePool::~CublasHandlePool() {
+  for (auto h : all_handles_) {
+    cublasDestroy(h);
+  }
+}
+
+cublasHandle_t CublasHandlePool::acquire() {
+  std::lock_guard<std::mutex> guard(pool_mutex_);
+
+  if (!available_handles_.empty()) {
+    cublasHandle_t h = available_handles_.front();
+    available_handles_.pop_front();
+    return h;
+  }
 
+  if (all_handles_.size() < max_size_) {
+    cublasHandle_t h = create_handle();
+    all_handles_.push_back(h);
+    return h;
+  }
+
+  if (!all_handles_.empty()) {
+    static size_t rr_idx = 0;
+    cublasHandle_t h = all_handles_[rr_idx];
+    rr_idx = (rr_idx + 1) % all_handles_.size();
+    return h;
+  }
+
+  throw std::runtime_error("CublasHandlePool: no handles available");
+}
+
+void CublasHandlePool::release(cublasHandle_t h) {
+  std::lock_guard<std::mutex> guard(pool_mutex_);
+  available_handles_.push_back(h);
+}
+
+cublasHandle_t CublasHandlePool::create_handle() {
+  CHECK_CUDA(cuCtxPushCurrent(ctx_));
+  cublasHandle_t h;
+  cublasStatus_t status = cublasCreate(&h);
+  CHECK_CUDA(cuCtxPopCurrent(nullptr));
+
+  if (status != CUBLAS_STATUS_SUCCESS)
+    throw std::runtime_error("cublasCreate failed");
+  return h;
+}
+
+// ---------------------- DeviceCublasHandleTable ----------------------
+
+DeviceCublasHandleTable::DeviceCublasHandleTable(CUcontext ctx, size_t pool_size)
+    : pool_(ctx, pool_size) {}
+
+cublasHandle_t DeviceCublasHandleTable::get_handle(int actor_id, CUstream stream) {
+  cublasHandle_t h = nullptr;
+  {
+    std::shared_lock<std::shared_mutex> read_lock(table_mutex_);
+    auto it = table_.find(actor_id);
+    if (it != table_.end())
+      h = it->second;
+  }
+
+  if (!h) {
+    std::unique_lock<std::shared_mutex> write_lock(table_mutex_);
+    auto it = table_.find(actor_id);
+    if (it != table_.end()) {
+      h = it->second;
+    } else {
+      h = pool_.acquire();
+      table_[actor_id] = h;
+    }
+  }
+
+  // Always bind the handle to the caller's stream
+  cublasStatus_t status = cublasSetStream(h, stream);
+  if (status != CUBLAS_STATUS_SUCCESS)
+    throw std::runtime_error("cublasSetStream failed");
+
+  return h;
+}
+
+void DeviceCublasHandleTable::release_handle(int actor_id) {
+  std::unique_lock<std::shared_mutex> write_lock(table_mutex_);
+  auto it = table_.find(actor_id);
+  if (it != table_.end()) {
+    pool_.release(it->second);
+    table_.erase(it);
+  }
+}
+
+} // namespace caf::cuda

From 9d4e5fd4d8bcb29a4c04792d106b69d2e5260627 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 14:43:04 -0600
Subject: [PATCH 0661/1000] Initial commit.

---
 .../caf/actorBLAS/gemv-actor/gemv-actor.hpp   | 147 ++++++++++++++++++
 .../matrix-vector-actor/gemv-actor.hpp        | 147 ++++++++++++++++++
 2 files changed, 294 insertions(+)
 create mode 100644 libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp
 create mode 100644 libcaf_cuda/caf/actorBLAS/matrix-vector-actor/gemv-actor.hpp

diff --git a/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp b/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp
new file mode 100644
index 0000000000..27d87127d3
--- /dev/null
+++ b/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp
@@ -0,0 +1,147 @@
+#pragma once
+
+#include <caf/actor.hpp>
+#include <caf/actor_cast.hpp>
+#include <caf/actor_config.hpp>
+#include <caf/actor_system.hpp>
+#include <caf/anon_mail.hpp>
+#include <caf/event_based_actor.hpp>
+
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/mem_ref.hpp"
+#include "caf/cuda/command_runner.hpp"
+#include "caf/cuda/platform.hpp"
+#include "caf/cuda/types.hpp"
+
+namespace caf::cuda {
+
+/// GEMV Actor for single-precision matrix-vector multiplication.
+/// Message Signature: (in<float> A, in<float> x, out<float> y, int m, int n, [float alpha, float beta])
+class gemv_actor : public event_based_actor {
+public:
+  static caf::actor spawn(caf::actor_system& sys, int reply_id = 0) {
+    return sys.spawn<gemv_actor>(reply_id);
+  }
+
+  gemv_actor(caf::actor_config& cfg, int reply_id = 0) 
+    : event_based_actor(cfg), reply_id_(reply_id) {
+    actor_id_ = static_cast<int>(this->id());
+  }
+
+  ~gemv_actor() override {
+    command_runner<> runner;
+    runner.release_stream_for_actor(actor_id_);
+  }
+
+  caf::behavior make_behavior() override {
+    return {
+      // Standard host buffer based calls
+      [this](in<float> A, in<float> x, out<float> y, int m, int n) {
+        enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, false);
+      },
+      [this](in<float> A, in<float> x, out<float> y, int m, int n, float alpha, float beta) {
+        enqueue_gemv(-1, actor_id_, A, x, y, m, n, alpha, beta, false);
+      },
+      // Routing control overloads
+      [this](int device_num, int stream_id, in<float> A, in<float> x, out<float> y, int m, int n) {
+        enqueue_gemv(device_num, stream_id, A, x, y, m, n, 1.0f, 0.0f, false);
+      },
+      // mem_ptr based calls (useful for pipelines)
+      [this](mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n) {
+        enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, false);
+      },
+      [this](mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n, float alpha, float beta) {
+        enqueue_gemv(-1, actor_id_, A, x, y, m, n, alpha, beta, false);
+      },
+      [this](int device_num, int stream_id, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n) {
+        enqueue_gemv(device_num, stream_id, A, x, y, m, n, 1.0f, 0.0f, false);
+      },
+      // Mem ptr return overloads
+      [this](return_mem_ptr_atom, in<float> A, in<float> x, out<float> y, int m, int n) {
+        enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, int device_num, int stream_id, in<float> A, in<float> x, out<float> y, int m, int n) {
+        enqueue_gemv(device_num, stream_id, A, x, y, m, n, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n) {
+        enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, true);
+      }
+    };
+  }
+
+private:
+  // Overload for Host-wrapped buffers
+  void enqueue_gemv(int device_num, int stream_id, 
+                    in<float> A_arg, in<float> x_arg, out<float> y_arg, 
+                    int m, int n, float alpha, float beta, bool return_ptrs) {
+    command_runner<in<float>, in<float>, out<float>> runner;
+    
+    // Allocate/Transfer memory.
+    auto results = runner.transfer_memory(device_num, stream_id, A_arg, x_arg, y_arg);
+    execute_and_reply(device_num, stream_id, std::get<0>(results), 
+                      std::get<1>(results), std::get<2>(results), 
+                      m, n, alpha, beta, return_ptrs);
+  }
+
+  // Overload for already-existing Device buffers
+  void enqueue_gemv(int device_num, int stream_id, 
+                    mem_ptr<float> A_ptr, mem_ptr<float> x_ptr, mem_ptr<float> y_ptr, 
+                    int m, int n, float alpha, float beta, bool return_ptrs) {
+    // Pass through command_runner to ensure proper ref-counting/scheduling
+    command_runner<mem_ptr<float>, mem_ptr<float>, mem_ptr<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, A_ptr, x_ptr, y_ptr);
+    execute_and_reply(device_num, stream_id, std::get<0>(results), 
+                      std::get<1>(results), std::get<2>(results), 
+                      m, n, alpha, beta, return_ptrs);
+  }
+
+  void execute_and_reply(int device_num, int stream_id, 
+                         mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, 
+                         int m, int n, float alpha, float beta, bool return_ptrs) {
+    // Get device based on scheduling rules
+    auto plat = platform::create();
+    device_ptr dev;
+    if (device_num == -1)
+      dev = plat->schedule(stream_id);
+    else
+      dev = plat->schedule(stream_id, device_num);
+
+    // Perform cuBLAS operation
+    dev->sgemv(stream_id, m, n, alpha, A, x, beta, y);
+
+    // Handle message routing back to requester
+    handle_reply(device_num, stream_id, A, x, y, return_ptrs);
+  }
+
+  void handle_reply(int device_num, int stream_id, 
+                    mem_ptr<float> A_ptr, mem_ptr<float> x_ptr, mem_ptr<float> y_ptr, 
+                    bool return_ptrs) {
+    command_runner<mem_ptr<float>> runner;
+    auto sender = actor_cast<actor>(this->current_sender());
+    if (!sender) return;
+    
+    auto r_id = reply_id_;
+
+    if (return_ptrs) {
+      caf::anon_mail(r_id, A_ptr, x_ptr, y_ptr).send(sender);
+    } else {
+      runner.copy_to_host_async(y_ptr, [sender, r_id](std::vector<float>&& data) {
+        if (sender) {
+          caf::anon_mail(r_id, 2, std::move(data)).send(sender);
+        }
+      });
+    }
+
+    // Stream completion callback
+    runner.add_callback(stream_id, device_num, [sender, r_id]() mutable {
+      if (sender) {
+        caf::anon_mail(r_id, -1).send(sender);
+      }
+    });
+  }
+
+  int actor_id_;
+  int reply_id_;
+};
+
+} // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/caf/actorBLAS/matrix-vector-actor/gemv-actor.hpp b/libcaf_cuda/caf/actorBLAS/matrix-vector-actor/gemv-actor.hpp
new file mode 100644
index 0000000000..27d87127d3
--- /dev/null
+++ b/libcaf_cuda/caf/actorBLAS/matrix-vector-actor/gemv-actor.hpp
@@ -0,0 +1,147 @@
+#pragma once
+
+#include <caf/actor.hpp>
+#include <caf/actor_cast.hpp>
+#include <caf/actor_config.hpp>
+#include <caf/actor_system.hpp>
+#include <caf/anon_mail.hpp>
+#include <caf/event_based_actor.hpp>
+
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/mem_ref.hpp"
+#include "caf/cuda/command_runner.hpp"
+#include "caf/cuda/platform.hpp"
+#include "caf/cuda/types.hpp"
+
+namespace caf::cuda {
+
+/// GEMV Actor for single-precision matrix-vector multiplication.
+/// Message Signature: (in<float> A, in<float> x, out<float> y, int m, int n, [float alpha, float beta])
+class gemv_actor : public event_based_actor {
+public:
+  static caf::actor spawn(caf::actor_system& sys, int reply_id = 0) {
+    return sys.spawn<gemv_actor>(reply_id);
+  }
+
+  gemv_actor(caf::actor_config& cfg, int reply_id = 0) 
+    : event_based_actor(cfg), reply_id_(reply_id) {
+    actor_id_ = static_cast<int>(this->id());
+  }
+
+  ~gemv_actor() override {
+    command_runner<> runner;
+    runner.release_stream_for_actor(actor_id_);
+  }
+
+  caf::behavior make_behavior() override {
+    return {
+      // Standard host buffer based calls
+      [this](in<float> A, in<float> x, out<float> y, int m, int n) {
+        enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, false);
+      },
+      [this](in<float> A, in<float> x, out<float> y, int m, int n, float alpha, float beta) {
+        enqueue_gemv(-1, actor_id_, A, x, y, m, n, alpha, beta, false);
+      },
+      // Routing control overloads
+      [this](int device_num, int stream_id, in<float> A, in<float> x, out<float> y, int m, int n) {
+        enqueue_gemv(device_num, stream_id, A, x, y, m, n, 1.0f, 0.0f, false);
+      },
+      // mem_ptr based calls (useful for pipelines)
+      [this](mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n) {
+        enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, false);
+      },
+      [this](mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n, float alpha, float beta) {
+        enqueue_gemv(-1, actor_id_, A, x, y, m, n, alpha, beta, false);
+      },
+      [this](int device_num, int stream_id, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n) {
+        enqueue_gemv(device_num, stream_id, A, x, y, m, n, 1.0f, 0.0f, false);
+      },
+      // Mem ptr return overloads
+      [this](return_mem_ptr_atom, in<float> A, in<float> x, out<float> y, int m, int n) {
+        enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, int device_num, int stream_id, in<float> A, in<float> x, out<float> y, int m, int n) {
+        enqueue_gemv(device_num, stream_id, A, x, y, m, n, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n) {
+        enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, true);
+      }
+    };
+  }
+
+private:
+  // Overload for Host-wrapped buffers
+  void enqueue_gemv(int device_num, int stream_id, 
+                    in<float> A_arg, in<float> x_arg, out<float> y_arg, 
+                    int m, int n, float alpha, float beta, bool return_ptrs) {
+    command_runner<in<float>, in<float>, out<float>> runner;
+    
+    // Allocate/Transfer memory.
+    auto results = runner.transfer_memory(device_num, stream_id, A_arg, x_arg, y_arg);
+    execute_and_reply(device_num, stream_id, std::get<0>(results), 
+                      std::get<1>(results), std::get<2>(results), 
+                      m, n, alpha, beta, return_ptrs);
+  }
+
+  // Overload for already-existing Device buffers
+  void enqueue_gemv(int device_num, int stream_id, 
+                    mem_ptr<float> A_ptr, mem_ptr<float> x_ptr, mem_ptr<float> y_ptr, 
+                    int m, int n, float alpha, float beta, bool return_ptrs) {
+    // Pass through command_runner to ensure proper ref-counting/scheduling
+    command_runner<mem_ptr<float>, mem_ptr<float>, mem_ptr<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, A_ptr, x_ptr, y_ptr);
+    execute_and_reply(device_num, stream_id, std::get<0>(results), 
+                      std::get<1>(results), std::get<2>(results), 
+                      m, n, alpha, beta, return_ptrs);
+  }
+
+  void execute_and_reply(int device_num, int stream_id, 
+                         mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, 
+                         int m, int n, float alpha, float beta, bool return_ptrs) {
+    // Get device based on scheduling rules
+    auto plat = platform::create();
+    device_ptr dev;
+    if (device_num == -1)
+      dev = plat->schedule(stream_id);
+    else
+      dev = plat->schedule(stream_id, device_num);
+
+    // Perform cuBLAS operation
+    dev->sgemv(stream_id, m, n, alpha, A, x, beta, y);
+
+    // Handle message routing back to requester
+    handle_reply(device_num, stream_id, A, x, y, return_ptrs);
+  }
+
+  void handle_reply(int device_num, int stream_id, 
+                    mem_ptr<float> A_ptr, mem_ptr<float> x_ptr, mem_ptr<float> y_ptr, 
+                    bool return_ptrs) {
+    command_runner<mem_ptr<float>> runner;
+    auto sender = actor_cast<actor>(this->current_sender());
+    if (!sender) return;
+    
+    auto r_id = reply_id_;
+
+    if (return_ptrs) {
+      caf::anon_mail(r_id, A_ptr, x_ptr, y_ptr).send(sender);
+    } else {
+      runner.copy_to_host_async(y_ptr, [sender, r_id](std::vector<float>&& data) {
+        if (sender) {
+          caf::anon_mail(r_id, 2, std::move(data)).send(sender);
+        }
+      });
+    }
+
+    // Stream completion callback
+    runner.add_callback(stream_id, device_num, [sender, r_id]() mutable {
+      if (sender) {
+        caf::anon_mail(r_id, -1).send(sender);
+      }
+    });
+  }
+
+  int actor_id_;
+  int reply_id_;
+};
+
+} // namespace caf::cuda
\ No newline at end of file

From 26617fbbe94681ccec9136330b30fd0c18174c56 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 14:43:19 -0600
Subject: [PATCH 0662/1000] Deleted.

---
 .../matrix-vector-actor/gemv-actor.hpp        | 147 ------------------
 1 file changed, 147 deletions(-)
 delete mode 100644 libcaf_cuda/caf/actorBLAS/matrix-vector-actor/gemv-actor.hpp

diff --git a/libcaf_cuda/caf/actorBLAS/matrix-vector-actor/gemv-actor.hpp b/libcaf_cuda/caf/actorBLAS/matrix-vector-actor/gemv-actor.hpp
deleted file mode 100644
index 27d87127d3..0000000000
--- a/libcaf_cuda/caf/actorBLAS/matrix-vector-actor/gemv-actor.hpp
+++ /dev/null
@@ -1,147 +0,0 @@
-#pragma once
-
-#include <caf/actor.hpp>
-#include <caf/actor_cast.hpp>
-#include <caf/actor_config.hpp>
-#include <caf/actor_system.hpp>
-#include <caf/anon_mail.hpp>
-#include <caf/event_based_actor.hpp>
-
-#include "caf/cuda/device.hpp"
-#include "caf/cuda/mem_ref.hpp"
-#include "caf/cuda/command_runner.hpp"
-#include "caf/cuda/platform.hpp"
-#include "caf/cuda/types.hpp"
-
-namespace caf::cuda {
-
-/// GEMV Actor for single-precision matrix-vector multiplication.
-/// Message Signature: (in<float> A, in<float> x, out<float> y, int m, int n, [float alpha, float beta])
-class gemv_actor : public event_based_actor {
-public:
-  static caf::actor spawn(caf::actor_system& sys, int reply_id = 0) {
-    return sys.spawn<gemv_actor>(reply_id);
-  }
-
-  gemv_actor(caf::actor_config& cfg, int reply_id = 0) 
-    : event_based_actor(cfg), reply_id_(reply_id) {
-    actor_id_ = static_cast<int>(this->id());
-  }
-
-  ~gemv_actor() override {
-    command_runner<> runner;
-    runner.release_stream_for_actor(actor_id_);
-  }
-
-  caf::behavior make_behavior() override {
-    return {
-      // Standard host buffer based calls
-      [this](in<float> A, in<float> x, out<float> y, int m, int n) {
-        enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, false);
-      },
-      [this](in<float> A, in<float> x, out<float> y, int m, int n, float alpha, float beta) {
-        enqueue_gemv(-1, actor_id_, A, x, y, m, n, alpha, beta, false);
-      },
-      // Routing control overloads
-      [this](int device_num, int stream_id, in<float> A, in<float> x, out<float> y, int m, int n) {
-        enqueue_gemv(device_num, stream_id, A, x, y, m, n, 1.0f, 0.0f, false);
-      },
-      // mem_ptr based calls (useful for pipelines)
-      [this](mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n) {
-        enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, false);
-      },
-      [this](mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n, float alpha, float beta) {
-        enqueue_gemv(-1, actor_id_, A, x, y, m, n, alpha, beta, false);
-      },
-      [this](int device_num, int stream_id, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n) {
-        enqueue_gemv(device_num, stream_id, A, x, y, m, n, 1.0f, 0.0f, false);
-      },
-      // Mem ptr return overloads
-      [this](return_mem_ptr_atom, in<float> A, in<float> x, out<float> y, int m, int n) {
-        enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, true);
-      },
-      [this](return_mem_ptr_atom, int device_num, int stream_id, in<float> A, in<float> x, out<float> y, int m, int n) {
-        enqueue_gemv(device_num, stream_id, A, x, y, m, n, 1.0f, 0.0f, true);
-      },
-      [this](return_mem_ptr_atom, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n) {
-        enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, true);
-      }
-    };
-  }
-
-private:
-  // Overload for Host-wrapped buffers
-  void enqueue_gemv(int device_num, int stream_id, 
-                    in<float> A_arg, in<float> x_arg, out<float> y_arg, 
-                    int m, int n, float alpha, float beta, bool return_ptrs) {
-    command_runner<in<float>, in<float>, out<float>> runner;
-    
-    // Allocate/Transfer memory.
-    auto results = runner.transfer_memory(device_num, stream_id, A_arg, x_arg, y_arg);
-    execute_and_reply(device_num, stream_id, std::get<0>(results), 
-                      std::get<1>(results), std::get<2>(results), 
-                      m, n, alpha, beta, return_ptrs);
-  }
-
-  // Overload for already-existing Device buffers
-  void enqueue_gemv(int device_num, int stream_id, 
-                    mem_ptr<float> A_ptr, mem_ptr<float> x_ptr, mem_ptr<float> y_ptr, 
-                    int m, int n, float alpha, float beta, bool return_ptrs) {
-    // Pass through command_runner to ensure proper ref-counting/scheduling
-    command_runner<mem_ptr<float>, mem_ptr<float>, mem_ptr<float>> runner;
-    auto results = runner.transfer_memory(device_num, stream_id, A_ptr, x_ptr, y_ptr);
-    execute_and_reply(device_num, stream_id, std::get<0>(results), 
-                      std::get<1>(results), std::get<2>(results), 
-                      m, n, alpha, beta, return_ptrs);
-  }
-
-  void execute_and_reply(int device_num, int stream_id, 
-                         mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, 
-                         int m, int n, float alpha, float beta, bool return_ptrs) {
-    // Get device based on scheduling rules
-    auto plat = platform::create();
-    device_ptr dev;
-    if (device_num == -1)
-      dev = plat->schedule(stream_id);
-    else
-      dev = plat->schedule(stream_id, device_num);
-
-    // Perform cuBLAS operation
-    dev->sgemv(stream_id, m, n, alpha, A, x, beta, y);
-
-    // Handle message routing back to requester
-    handle_reply(device_num, stream_id, A, x, y, return_ptrs);
-  }
-
-  void handle_reply(int device_num, int stream_id, 
-                    mem_ptr<float> A_ptr, mem_ptr<float> x_ptr, mem_ptr<float> y_ptr, 
-                    bool return_ptrs) {
-    command_runner<mem_ptr<float>> runner;
-    auto sender = actor_cast<actor>(this->current_sender());
-    if (!sender) return;
-    
-    auto r_id = reply_id_;
-
-    if (return_ptrs) {
-      caf::anon_mail(r_id, A_ptr, x_ptr, y_ptr).send(sender);
-    } else {
-      runner.copy_to_host_async(y_ptr, [sender, r_id](std::vector<float>&& data) {
-        if (sender) {
-          caf::anon_mail(r_id, 2, std::move(data)).send(sender);
-        }
-      });
-    }
-
-    // Stream completion callback
-    runner.add_callback(stream_id, device_num, [sender, r_id]() mutable {
-      if (sender) {
-        caf::anon_mail(r_id, -1).send(sender);
-      }
-    });
-  }
-
-  int actor_id_;
-  int reply_id_;
-};
-
-} // namespace caf::cuda
\ No newline at end of file

From 00b0756282f12146101ee408bd883017f1fc6a64 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 14:44:18 -0600
Subject: [PATCH 0663/1000] Initial commit.

---
 libcaf_cuda/caf/actorBLAS/actorBLAS.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp b/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp
index e69de29bb2..3270532c61 100644
--- a/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp
+++ b/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp
@@ -0,0 +1,2 @@
+#pragma once
+#include "caf/actorBLAS/gemv-actor/gemv-actor.hpp"

From ab3086ef33c0c835c318a878dddd3a6971f3d872 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 14:46:03 -0600
Subject: [PATCH 0664/1000] added a sgemv function to call matrix vector
 multiplication

---
 libcaf_cuda/caf/cuda/device.hpp | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 26e2ca78ad..13c8bf2f6e 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -133,6 +133,34 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     return cublas_table_->get_handle(actor_id, get_stream_for_actor(actor_id));
   }
 
+  /// Performs single precision matrix-vector multiplication (y = alpha*A*x + beta*y).
+  /// Assumes A is in row-major order of dimensions m x n.
+  void sgemv(int actor_id, int m, int n, float alpha, mem_ptr<float> A,
+             mem_ptr<float> x, float beta, mem_ptr<float> y) {
+    cublasHandle_t handle = get_cublas_handle(actor_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    
+    // Row-major matrix A (m x n) is stored as m rows of n elements.
+    // Viewed as column-major by cuBLAS, this is a n x m matrix.
+    // To compute y = A * x:
+    // Op(Memory) * x = (n x m)^T * (n x 1) = (m x n) * (n x 1) = (m x 1).
+    // We use CUBLAS_OP_T. LDA is the 'rows' in the column-major view, which is n.
+    cublasStatus_t status = cublasSgemv(handle, CUBLAS_OP_T, 
+                                        n, m, 
+                                        &alpha,
+                                        reinterpret_cast<const float*>(A->mem()), n,
+                                        reinterpret_cast<const float*>(x->mem()), 1,
+                                        &beta,
+                                        reinterpret_cast<float*>(y->mem()), 1);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasSgemv failed on device " + std::to_string(id_));
+  }
+
   // Overloads for make_arg using actor_id
   template <typename T>
   mem_ptr<T> make_arg(const in<T>& arg, int actor_id) {

From b04e94697b2f7f5bcbbbef176e10e78cd9c5e642 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 14:52:12 -0600
Subject: [PATCH 0665/1000] Initial commit.

---
 .../gemv-actor-test/CMakeLists.txt            | 72 +++++++++++++++
 .../gemv-actor-test/main.test.cpp             | 90 +++++++++++++++++++
 2 files changed, 162 insertions(+)
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp

diff --git a/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..1eee5904c7
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/CMakeLists.txt
@@ -0,0 +1,72 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+add_executable(latency_bench_test latency_bench.test.cpp)
+add_executable(throughput_bench_test throughput_bench.test.cpp)
+add_executable(throughput_mapping_bench_test throughput_mapping_bench.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+target_compile_definitions(latency_bench_test PRIVATE CAF_ENABLE_LOGGING)
+target_compile_definitions(throughput_bench_test PRIVATE CAF_ENABLE_LOGGING)
+target_compile_definitions(throughput_mapping_bench_test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+target_link_libraries(throughput_mapping_bench_test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+
+
+target_link_libraries(latency_bench_test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
+target_link_libraries(throughput_bench_test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+)
diff --git a/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp
new file mode 100644
index 0000000000..6a5e4211a4
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp
@@ -0,0 +1,90 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include "caf/actor_registry.hpp"
+#include "caf/actorBLAS/gemv-actor/gemv-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+// Verify GEMV result: y = alpha * A * x + beta * y
+// With A (all 3.0s) and x (all 4.0s), the dot product sum is (3.0 * 4.0 * n).
+// We use alpha = 1.0 / n to normalize the output to exactly 12.0f as requested.
+void verify_gemv_correctness(int m, int n, float alpha, float beta, 
+                             const std::vector<float>& y_result) {
+    float expected_val = alpha * (3.0f * 4.0f * n); 
+    bool all_correct = true;
+    for (size_t i = 0; i < y_result.size(); ++i) {
+        if (std::abs(y_result[i] - expected_val) > 1e-4) {
+            all_correct = false;
+            std::cout << "[ERROR] Mismatch at index " << i 
+                      << ": Expected " << expected_val 
+                      << ", Got " << y_result[i] << std::endl;
+            break;
+        }
+    }
+
+    if (all_correct) {
+        std::cout << "[SUCCESS] GEMV actor produced correct results." << std::endl;
+        std::cout << "          Output vector is filled with: " << expected_val << std::endl;
+    }
+}
+
+void caf_main(actor_system& sys) {
+    // Initialize the manager with BLAS enabled to initialize cuBLAS handles
+    manager_config config(true); 
+    manager::init(sys, config);
+
+    // Matrix dimensions 64x64 (> 32x32)
+    int m = 64;
+    int n = 64;
+    
+    // Scalar alpha set to 1/n to normalize the dot product sum (3*4*n) to 12.0
+    float alpha = 1.0f / static_cast<float>(n);
+    float beta = 0.0f;
+
+    // Initialize host data
+    std::vector<float> h_A(m * n, 3.0f);
+    std::vector<float> h_x(n, 4.0f);
+    std::vector<float> h_y(m, 0.0f);
+
+    // Spawn the gemv_actor
+    auto blas_actor = sys.spawn<gemv_actor>();
+
+    // Prepare arguments using wrapper tags
+    auto A_arg = create_in_arg(h_A);
+    auto x_arg = create_in_arg(h_x);
+    auto y_arg = create_out_arg(h_y);
+
+    scoped_actor self{sys};
+
+    std::cout << "[INFO] Testing gemv_actor with " << m << "x" << n << " matrix..." << std::endl;
+    
+    // Send the GEMV request to the actor
+    self->mail(A_arg, x_arg, y_arg, m, n, alpha, beta).send(blas_actor);
+
+    // Receive results: gemv_actor sends data for 'out' buffers followed by a completion signal
+    self->receive(
+        [&](int reply_id, int arg_index, std::vector<float> data) {
+            if (arg_index == 2) { // index 2 corresponds to y_arg
+                verify_gemv_correctness(m, n, alpha, beta, data);
+            }
+        },
+        [&](int reply_id, int signal) {
+            if (signal == -1) {
+                std::cout << "[INFO] GEMV actor execution and transfer complete." << std::endl;
+            }
+        }
+    );
+
+    self->send_exit(blas_actor, exit_reason::user_shutdown);
+    manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda)

From e3119a9ffeaad77de8d55806618a9b6762c2a7ea Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 14:54:52 -0600
Subject: [PATCH 0666/1000] Fixed errors.

---
 .../gemv-actor-test/CMakeLists.txt            | 28 -------------------
 .../gemv-actor-test/main.test.cpp             |  2 +-
 2 files changed, 1 insertion(+), 29 deletions(-)

diff --git a/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/CMakeLists.txt
index 1eee5904c7..b759f25d78 100644
--- a/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/CMakeLists.txt
@@ -31,14 +31,8 @@ include_directories(
 
 # 5) Declare your executable
 add_executable(test main.test.cpp)
-add_executable(latency_bench_test latency_bench.test.cpp)
-add_executable(throughput_bench_test throughput_bench.test.cpp)
-add_executable(throughput_mapping_bench_test throughput_mapping_bench.test.cpp)
 
 target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
-target_compile_definitions(latency_bench_test PRIVATE CAF_ENABLE_LOGGING)
-target_compile_definitions(throughput_bench_test PRIVATE CAF_ENABLE_LOGGING)
-target_compile_definitions(throughput_mapping_bench_test PRIVATE CAF_ENABLE_LOGGING)
 
 target_link_libraries(test
   PRIVATE
@@ -47,26 +41,4 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
 )
-target_link_libraries(throughput_mapping_bench_test
-  PRIVATE
-    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-    CUDA::nvrtc
-)
-
 
-target_link_libraries(latency_bench_test
-  PRIVATE
-    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-    CUDA::nvrtc
-)
-target_link_libraries(throughput_bench_test
-  PRIVATE
-    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-    CUDA::nvrtc
-)
diff --git a/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp
index 6a5e4211a4..4778bbc847 100644
--- a/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp
@@ -55,7 +55,7 @@ void caf_main(actor_system& sys) {
     std::vector<float> h_y(m, 0.0f);
 
     // Spawn the gemv_actor
-    auto blas_actor = sys.spawn<gemv_actor>();
+    auto blas_actor = sys.spawn(gemv_actor);
 
     // Prepare arguments using wrapper tags
     auto A_arg = create_in_arg(h_A);

From 78a0ee1f7376b2b18ead628a1d41d7ff0fb12677 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 14:59:15 -0600
Subject: [PATCH 0667/1000] added float types to be passed as messages

---
 libcaf_cuda/caf/cuda/global.hpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index 3367d97391..6badd2745a 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -185,8 +185,11 @@ CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id)
   CAF_ADD_TYPE_ID(cuda, (std::vector<int>))
   CAF_ADD_TYPE_ID(cuda, (in<int>))
   CAF_ADD_TYPE_ID(cuda, (in<char>))
+  CAF_ADD_TYPE_ID(cuda, (in<float>))
   CAF_ADD_TYPE_ID(cuda, (out<int>))
+  CAF_ADD_TYPE_ID(cuda, (out<float>))
   CAF_ADD_TYPE_ID(cuda, (in_out<int>))
+  CAF_ADD_TYPE_ID(cuda, (in_out<float>))
   CAF_ADD_TYPE_ID(cuda, (std::vector<float>))
   CAF_ADD_TYPE_ID(cuda, (std::vector<double>))
   CAF_ADD_TYPE_ID(cuda, (buffer_variant))

From 33852ace38d1d97f5342f396078448878582d93d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 20 May 2026 15:01:02 -0600
Subject: [PATCH 0668/1000] Included cublas library.

---
 libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/CMakeLists.txt
index b759f25d78..fa9dc8e759 100644
--- a/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/CMakeLists.txt
@@ -40,5 +40,6 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
+    CUDA::cublas
 )
 

From 53e27c3ad999c4543fe41406e9787568353a735e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 08:15:19 -0600
Subject: [PATCH 0669/1000] updated gemv-actor and tests so that actor will
 pass tests

---
 .../caf/actorBLAS/gemv-actor/gemv-actor.hpp   |  19 ++-
 .../gemv-actor-test/main.test.cpp             | 108 +++++++++++++++---
 2 files changed, 107 insertions(+), 20 deletions(-)

diff --git a/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp b/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp
index 27d87127d3..5ff877c92c 100644
--- a/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp
+++ b/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp
@@ -46,6 +46,9 @@ class gemv_actor : public event_based_actor {
       [this](int device_num, int stream_id, in<float> A, in<float> x, out<float> y, int m, int n) {
         enqueue_gemv(device_num, stream_id, A, x, y, m, n, 1.0f, 0.0f, false);
       },
+      [this](int device_num, int stream_id, in<float> A, in<float> x, out<float> y, int m, int n, float alpha, float beta) {
+        enqueue_gemv(device_num, stream_id, A, x, y, m, n, alpha, beta, false);
+      },
       // mem_ptr based calls (useful for pipelines)
       [this](mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n) {
         enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, false);
@@ -56,16 +59,28 @@ class gemv_actor : public event_based_actor {
       [this](int device_num, int stream_id, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n) {
         enqueue_gemv(device_num, stream_id, A, x, y, m, n, 1.0f, 0.0f, false);
       },
+      [this](int device_num, int stream_id, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n, float alpha, float beta) {
+        enqueue_gemv(device_num, stream_id, A, x, y, m, n, alpha, beta, false);
+      },
       // Mem ptr return overloads
       [this](return_mem_ptr_atom, in<float> A, in<float> x, out<float> y, int m, int n) {
         enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, true);
       },
+      [this](return_mem_ptr_atom, in<float> A, in<float> x, out<float> y, int m, int n, float alpha, float beta) {
+        enqueue_gemv(-1, actor_id_, A, x, y, m, n, alpha, beta, true);
+      },
       [this](return_mem_ptr_atom, int device_num, int stream_id, in<float> A, in<float> x, out<float> y, int m, int n) {
         enqueue_gemv(device_num, stream_id, A, x, y, m, n, 1.0f, 0.0f, true);
       },
+      [this](return_mem_ptr_atom, int device_num, int stream_id, in<float> A, in<float> x, out<float> y, int m, int n, float alpha, float beta) {
+        enqueue_gemv(device_num, stream_id, A, x, y, m, n, alpha, beta, true);
+      },
       [this](return_mem_ptr_atom, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n) {
         enqueue_gemv(-1, actor_id_, A, x, y, m, n, 1.0f, 0.0f, true);
-      }
+      },
+      [this](return_mem_ptr_atom, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n, float alpha, float beta) {
+        enqueue_gemv(-1, actor_id_, A, x, y, m, n, alpha, beta, true);
+      },
     };
   }
 
@@ -133,7 +148,7 @@ class gemv_actor : public event_based_actor {
     }
 
     // Stream completion callback
-    runner.add_callback(stream_id, device_num, [sender, r_id]() mutable {
+    runner.add_callback(stream_id, device_num, [sender, r_id]() {
       if (sender) {
         caf::anon_mail(r_id, -1).send(sender);
       }
diff --git a/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp
index 4778bbc847..bfacc4408d 100644
--- a/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp
@@ -55,7 +55,7 @@ void caf_main(actor_system& sys) {
     std::vector<float> h_y(m, 0.0f);
 
     // Spawn the gemv_actor
-    auto blas_actor = sys.spawn(gemv_actor);
+    auto blas_actor = sys.spawn<gemv_actor>(1);
 
     // Prepare arguments using wrapper tags
     auto A_arg = create_in_arg(h_A);
@@ -64,27 +64,99 @@ void caf_main(actor_system& sys) {
 
     scoped_actor self{sys};
 
-    std::cout << "[INFO] Testing gemv_actor with " << m << "x" << n << " matrix..." << std::endl;
-    
-    // Send the GEMV request to the actor
-    self->mail(A_arg, x_arg, y_arg, m, n, alpha, beta).send(blas_actor);
-
-    // Receive results: gemv_actor sends data for 'out' buffers followed by a completion signal
-    self->receive(
-        [&](int reply_id, int arg_index, std::vector<float> data) {
-            if (arg_index == 2) { // index 2 corresponds to y_arg
-                verify_gemv_correctness(m, n, alpha, beta, data);
+    // Test 1: Standard host-buffer based call
+    {
+        std::cout << "[INFO] Test 1: Testing gemv_actor with host-buffer arguments..." << std::endl;
+        self->mail(A_arg, x_arg, y_arg, m, n, alpha, beta).send(blas_actor);
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 2) { // index 2 corresponds to y_arg
+                    verify_gemv_correctness(m, n, alpha, beta, data);
+                }
+            },
+            [&](int reply_id, int signal) {
+                if (signal == -1) {
+                    std::cout << "[INFO] Test 1 complete." << std::endl;
+                }
             }
-        },
-        [&](int reply_id, int signal) {
-            if (signal == -1) {
-                std::cout << "[INFO] GEMV actor execution and transfer complete." << std::endl;
+        );
+    }
+
+    // Test 2: mem_ptr inputs
+    {
+        std::cout << "\n[INFO] Test 2: Testing gemv_actor with mem_ptr inputs..." << std::endl;
+        command_runner<in<float>, in<float>, out<float>> setup_runner;
+        // Manually transfer data to the GPU to get mem_ptr handles
+        auto results = setup_runner.transfer_memory(0, 0, create_in_arg(h_A), create_in_arg(h_x), create_out_arg(h_y));
+        auto A_ptr = std::get<0>(results);
+        auto x_ptr = std::get<1>(results);
+        auto y_ptr = std::get<2>(results);
+
+        self->mail(A_ptr, x_ptr, y_ptr, m, n, alpha, beta).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 2) {
+                    verify_gemv_correctness(m, n, alpha, beta, data);
+                }
+            },
+            [&](int reply_id, int signal) {
+                if (signal == -1) {
+                    std::cout << "[INFO] Test 2 complete." << std::endl;
+                }
             }
-        }
-    );
+        );
+    }
+
+    // Test 3: Routing control (device/stream) + mem_ptr
+    {
+        std::cout << "\n[INFO] Test 3: Testing gemv_actor with specific device/stream and mem_ptr..." << std::endl;
+        int device_num = 0;
+        int stream_id = 42; 
+        command_runner<in<float>, in<float>, out<float>> setup_runner;
+        auto results = setup_runner.transfer_memory(device_num, stream_id, create_in_arg(h_A), create_in_arg(h_x), create_out_arg(h_y));
+        
+        self->mail(device_num, stream_id, std::get<0>(results), std::get<1>(results), std::get<2>(results), m, n, alpha, beta).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 2) {
+                    verify_gemv_correctness(m, n, alpha, beta, data);
+                }
+            },
+            [&](int reply_id, int signal) {
+                if (signal == -1) {
+                    std::cout << "[INFO] Test 3 complete." << std::endl;
+                }
+            }
+        );
+    }
+
+    // Test 4: return_mem_ptr_atom (returning device handles)
+    {
+        std::cout << "\n[INFO] Test 4: Testing gemv_actor with return_mem_ptr_atom..." << std::endl;
+        self->mail(return_mem_ptr_atom{}, A_arg, x_arg, y_arg, m, n, alpha, beta).send(blas_actor);
+
+        // We expect two messages back: the data (mem_ptrs) and the signal (-1)
+        self->receive(
+            [&](int reply_id, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y) {
+                command_runner<float> runner;
+                auto host_y = runner.copy_to_host(y);
+                verify_gemv_correctness(m, n, alpha, beta, host_y);
+            }
+        );
+
+        // Separate receive for the completion signal to ensure both are processed
+        self->receive(
+            [&](int reply_id, int signal) {
+                if (signal == -1) {
+                    std::cout << "[INFO] Test 4 complete." << std::endl;
+                }
+            }
+        );
+    }
 
     self->send_exit(blas_actor, exit_reason::user_shutdown);
     manager::shutdown();
 }
-
 CAF_MAIN(id_block::cuda)

From 7938481de7cd1c90e78eddd4033b07472a042a2d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 08:34:53 -0600
Subject: [PATCH 0670/1000] Initial commit.

---
 .../caf/actorBLAS/syrk-actor/syrk-actor.hpp   | 140 ++++++++++++++++++
 .../syrk-actor-test/CMakeLists.txt            |  45 ++++++
 .../syrk-actor-test/main.test.cpp             |  76 ++++++++++
 3 files changed, 261 insertions(+)
 create mode 100644 libcaf_cuda/caf/actorBLAS/syrk-actor/syrk-actor.hpp
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/syrk-actor-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/syrk-actor-test/main.test.cpp

diff --git a/libcaf_cuda/caf/actorBLAS/syrk-actor/syrk-actor.hpp b/libcaf_cuda/caf/actorBLAS/syrk-actor/syrk-actor.hpp
new file mode 100644
index 0000000000..72dfa9789b
--- /dev/null
+++ b/libcaf_cuda/caf/actorBLAS/syrk-actor/syrk-actor.hpp
@@ -0,0 +1,140 @@
+#pragma once
+
+#include <caf/actor.hpp>
+#include <caf/actor_cast.hpp>
+#include <caf/actor_config.hpp>
+#include <caf/actor_system.hpp>
+#include <caf/anon_mail.hpp>
+#include <caf/event_based_actor.hpp>
+
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/mem_ref.hpp"
+#include "caf/cuda/command_runner.hpp"
+#include "caf/cuda/platform.hpp"
+#include "caf/cuda/types.hpp"
+
+namespace caf::cuda {
+
+/// SYRK Actor for single-precision symmetric rank-k update.
+/// Formula: C = alpha * A * A^T + beta * C
+/// Message Signature: (in<float> A, in_out<float> C, int n, int k, [float alpha, float beta])
+class syrk_actor : public event_based_actor {
+public:
+  static caf::actor spawn(caf::actor_system& sys, int reply_id = 0) {
+    return sys.spawn<syrk_actor>(reply_id);
+  }
+
+  syrk_actor(caf::actor_config& cfg, int reply_id = 0) 
+    : event_based_actor(cfg), reply_id_(reply_id) {
+    actor_id_ = static_cast<int>(this->id());
+  }
+
+  ~syrk_actor() override {
+    command_runner<> runner;
+    runner.release_stream_for_actor(actor_id_);
+  }
+
+  caf::behavior make_behavior() override {
+    return {
+      // Standard host buffer based calls
+      [this](in<float> A, in_out<float> C, int n, int k) {
+        enqueue_syrk(-1, actor_id_, A, C, n, k, 1.0f, 0.0f, false);
+      },
+      [this](in<float> A, in_out<float> C, int n, int k, float alpha, float beta) {
+        enqueue_syrk(-1, actor_id_, A, C, n, k, alpha, beta, false);
+      },
+      // Routing control overloads
+      [this](int device_num, int stream_id, in<float> A, in_out<float> C, int n, int k) {
+        enqueue_syrk(device_num, stream_id, A, C, n, k, 1.0f, 0.0f, false);
+      },
+      [this](int device_num, int stream_id, in<float> A, in_out<float> C, int n, int k, float alpha, float beta) {
+        enqueue_syrk(device_num, stream_id, A, C, n, k, alpha, beta, false);
+      },
+      // mem_ptr based calls
+      [this](mem_ptr<float> A, mem_ptr<float> C, int n, int k) {
+        enqueue_syrk(-1, actor_id_, A, C, n, k, 1.0f, 0.0f, false);
+      },
+      [this](mem_ptr<float> A, mem_ptr<float> C, int n, int k, float alpha, float beta) {
+        enqueue_syrk(-1, actor_id_, A, C, n, k, alpha, beta, false);
+      },
+      // Mem ptr return overloads
+      [this](return_mem_ptr_atom, in<float> A, in_out<float> C, int n, int k) {
+        enqueue_syrk(-1, actor_id_, A, C, n, k, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, in<float> A, in_out<float> C, int n, int k, float alpha, float beta) {
+        enqueue_syrk(-1, actor_id_, A, C, n, k, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, mem_ptr<float> A, mem_ptr<float> C, int n, int k) {
+        enqueue_syrk(-1, actor_id_, A, C, n, k, 1.0f, 0.0f, true);
+      }
+    };
+  }
+
+private:
+  // Overload for Host-wrapped buffers
+  void enqueue_syrk(int device_num, int stream_id, 
+                    in<float> A_arg, in_out<float> C_arg, 
+                    int n, int k, float alpha, float beta, bool return_ptrs) {
+    command_runner<in<float>, in_out<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, A_arg, C_arg);
+    execute_and_reply(device_num, stream_id, std::get<0>(results), 
+                      std::get<1>(results), n, k, alpha, beta, return_ptrs);
+  }
+
+  // Overload for already-existing Device buffers
+  void enqueue_syrk(int device_num, int stream_id, 
+                    mem_ptr<float> A_ptr, mem_ptr<float> C_ptr, 
+                    int n, int k, float alpha, float beta, bool return_ptrs) {
+    command_runner<mem_ptr<float>, mem_ptr<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, A_ptr, C_ptr);
+    execute_and_reply(device_num, stream_id, std::get<0>(results), 
+                      std::get<1>(results), n, k, alpha, beta, return_ptrs);
+  }
+
+  void execute_and_reply(int device_num, int stream_id, 
+                         mem_ptr<float> A, mem_ptr<float> C, 
+                         int n, int k, float alpha, float beta, bool return_ptrs) {
+    auto plat = platform::create();
+    device_ptr dev;
+    if (device_num == -1)
+      dev = plat->schedule(stream_id);
+    else
+      dev = plat->schedule(stream_id, device_num);
+
+    dev->ssyrk(stream_id, n, k, alpha, A, beta, C);
+
+    handle_reply(device_num, stream_id, A, C, return_ptrs);
+  }
+
+  void handle_reply(int device_num, int stream_id, 
+                    mem_ptr<float> A_ptr, mem_ptr<float> C_ptr, 
+                    bool return_ptrs) {
+    command_runner<float> runner;
+    auto sender = actor_cast<actor>(this->current_sender());
+    if (!sender) return;
+    
+    auto r_id = reply_id_;
+
+    if (return_ptrs) {
+      caf::anon_mail(r_id, A_ptr, C_ptr).send(sender);
+    } else {
+      // Copy C back to host (index 1 in the original arg list)
+      runner.copy_to_host_async(C_ptr, [sender, r_id](std::vector<float>&& data) {
+        if (sender) {
+          caf::anon_mail(r_id, 1, std::move(data)).send(sender);
+        }
+      });
+    }
+
+    runner.add_callback(stream_id, device_num, [sender, r_id]() {
+      if (sender) {
+        caf::anon_mail(r_id, -1).send(sender);
+      }
+    });
+  }
+
+  int actor_id_;
+  int reply_id_;
+};
+
+} // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/tests/actorBLAS-test/syrk-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorBLAS-test/syrk-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..fa9dc8e759
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/syrk-actor-test/CMakeLists.txt
@@ -0,0 +1,45 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+)
+
diff --git a/libcaf_cuda/tests/actorBLAS-test/syrk-actor-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/syrk-actor-test/main.test.cpp
new file mode 100644
index 0000000000..fe9d1f67ec
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/syrk-actor-test/main.test.cpp
@@ -0,0 +1,76 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include "caf/actorBLAS/syrk-actor/syrk-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+void verify_syrk_correctness(int n, int k, float alpha, float beta, 
+                             const std::vector<float>& C_result) {
+    // Expected value for C = alpha * (A * A^T) + beta * C_init
+    // Since A is all 1.0s (n x k), each element of A*A^T is k.
+    // With alpha=1 and beta=0, expected is exactly k.
+    // Note: cuBLAS SYRK only updates the lower triangle by default in our implementation.
+    float expected_val = alpha * static_cast<float>(k); 
+    bool all_correct = true;
+    for (int i = 0; i < n; ++i) {
+        for (int j = 0; j <= i; ++j) { // Check lower triangle
+            float val = C_result[i * n + j];
+            if (std::abs(val - expected_val) > 1e-4) {
+                all_correct = false;
+                std::cout << "[ERROR] Mismatch at index (" << i << "," << j << "): "
+                          << "Expected " << expected_val << ", Got " << val << std::endl;
+                break;
+            }
+        }
+    }
+
+    if (all_correct) {
+        std::cout << "[SUCCESS] SYRK actor produced correct lower triangle results." << std::endl;
+    }
+}
+
+void caf_main(actor_system& sys) {
+    manager_config config(true); 
+    manager::init(sys, config);
+
+    int n = 32;
+    int k = 64;
+    float alpha = 1.0f;
+    float beta = 0.0f;
+
+    std::vector<float> h_A(n * k, 1.0f);
+    std::vector<float> h_C(n * n, 0.0f);
+
+    auto blas_actor = sys.spawn<syrk_actor>(1);
+
+    auto A_arg = create_in_arg(h_A);
+    auto C_arg = create_in_out_arg(h_C);
+
+    scoped_actor self{sys};
+
+    // Test 1: Standard host-buffer based call
+    {
+        std::cout << "[INFO] Test 1: Testing syrk_actor with host-buffer arguments..." << std::endl;
+        self->mail(A_arg, C_arg, n, k, alpha, beta).send(blas_actor);
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 1) { // index 1 corresponds to C_arg
+                    verify_syrk_correctness(n, k, alpha, beta, data);
+                }
+            },
+            [&](int reply_id, int signal) {
+                if (signal == -1) {
+                    std::cout << "[INFO] Test 1 complete." << std::endl;
+                }
+            }
+        );
+    }
+
+    self->send_exit(blas_actor, exit_reason::user_shutdown);
+    manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda)
\ No newline at end of file

From f40fc693856c46285b84512a7e1e064612868e84 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 08:36:01 -0600
Subject: [PATCH 0671/1000] added method for cublas syrk

---
 libcaf_cuda/caf/cuda/device.hpp | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 13c8bf2f6e..1618c21c1a 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -161,6 +161,32 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasSgemv failed on device " + std::to_string(id_));
   }
 
+  /// Performs symmetric rank-k update (C = alpha*A*A^T + beta*C).
+  /// Assumes A is in row-major order of dimensions n x k, and C is n x n.
+  void ssyrk(int actor_id, int n, int k, float alpha, mem_ptr<float> A,
+             float beta, mem_ptr<float> C) {
+    cublasHandle_t handle = get_cublas_handle(actor_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    
+    // Row-major matrix A (n x k) is viewed as column-major k x n.
+    // To compute C = alpha * A * A^T + beta * C:
+    // We use CUBLAS_OP_T so that op(A) is (k x n)^T = n x k.
+    // LDA is the number of rows in the column-major view, which is k.
+    cublasStatus_t status = cublasSsyrk(handle, CUBLAS_FILL_MODE_LOWER, CUBLAS_OP_T,
+                                        n, k,
+                                        &alpha,
+                                        reinterpret_cast<const float*>(A->mem()), k,
+                                        &beta,
+                                        reinterpret_cast<float*>(C->mem()), n);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasSsyrk failed on device " + std::to_string(id_));
+  }
+
   // Overloads for make_arg using actor_id
   template <typename T>
   mem_ptr<T> make_arg(const in<T>& arg, int actor_id) {

From a2845782427de2b0ad3fea1993449d9d535109a8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 08:43:17 -0600
Subject: [PATCH 0672/1000] updated syrk to be consistant with row major order

---
 libcaf_cuda/caf/cuda/device.hpp | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 1618c21c1a..61de51416e 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -171,11 +171,12 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
 
     CHECK_CUDA(cuCtxPushCurrent(context_));
     
-    // Row-major matrix A (n x k) is viewed as column-major k x n.
+    // Row-major matrix A (n x k) viewed as column-major is k x n.
     // To compute C = alpha * A * A^T + beta * C:
-    // We use CUBLAS_OP_T so that op(A) is (k x n)^T = n x k.
-    // LDA is the number of rows in the column-major view, which is k.
-    cublasStatus_t status = cublasSsyrk(handle, CUBLAS_FILL_MODE_LOWER, CUBLAS_OP_T,
+    // We use CUBLAS_OP_T so that (k x n)^T * (k x n) = (n x k) * (k x n) = n x n.
+    // Note: We use CUBLAS_FILL_MODE_UPPER because the upper triangle in 
+    // column-major maps to the lower triangle in row-major layout.
+    cublasStatus_t status = cublasSsyrk(handle, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_T,
                                         n, k,
                                         &alpha,
                                         reinterpret_cast<const float*>(A->mem()), k,

From 5a8de081ce1daa6954ba91e59adf6305d27aec62 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 09:02:35 -0600
Subject: [PATCH 0673/1000] updated actors to no longer send a signal message
 after stream completetion and  adjusted changes accordingly. Change was made
 to reduce redundant messages

---
 .../caf/actorBLAS/gemv-actor/gemv-actor.hpp   |  7 --
 .../caf/actorBLAS/syrk-actor/syrk-actor.hpp   | 22 +++--
 libcaf_cuda/caf/cuda/actor_facade.hpp         |  7 --
 .../gemv-actor-test/main.test.cpp             | 28 +-----
 .../syrk-actor-test/main.test.cpp             | 91 +++++++++++++++++--
 5 files changed, 101 insertions(+), 54 deletions(-)

diff --git a/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp b/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp
index 5ff877c92c..2f30960a46 100644
--- a/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp
+++ b/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp
@@ -146,13 +146,6 @@ class gemv_actor : public event_based_actor {
         }
       });
     }
-
-    // Stream completion callback
-    runner.add_callback(stream_id, device_num, [sender, r_id]() {
-      if (sender) {
-        caf::anon_mail(r_id, -1).send(sender);
-      }
-    });
   }
 
   int actor_id_;
diff --git a/libcaf_cuda/caf/actorBLAS/syrk-actor/syrk-actor.hpp b/libcaf_cuda/caf/actorBLAS/syrk-actor/syrk-actor.hpp
index 72dfa9789b..516745c6fa 100644
--- a/libcaf_cuda/caf/actorBLAS/syrk-actor/syrk-actor.hpp
+++ b/libcaf_cuda/caf/actorBLAS/syrk-actor/syrk-actor.hpp
@@ -50,13 +50,24 @@ class syrk_actor : public event_based_actor {
       [this](int device_num, int stream_id, in<float> A, in_out<float> C, int n, int k, float alpha, float beta) {
         enqueue_syrk(device_num, stream_id, A, C, n, k, alpha, beta, false);
       },
-      // mem_ptr based calls
+      // mem_ptr based calls (Implicit routing)
       [this](mem_ptr<float> A, mem_ptr<float> C, int n, int k) {
         enqueue_syrk(-1, actor_id_, A, C, n, k, 1.0f, 0.0f, false);
       },
       [this](mem_ptr<float> A, mem_ptr<float> C, int n, int k, float alpha, float beta) {
         enqueue_syrk(-1, actor_id_, A, C, n, k, alpha, beta, false);
       },
+      // mem_ptr based calls (Explicit routing)
+      [this](int device_num, int stream_id, mem_ptr<float> A, mem_ptr<float> C, int n, int k) {
+        enqueue_syrk(device_num, stream_id, A, C, n, k, 1.0f, 0.0f, false);
+      },
+      [this](int device_num, int stream_id, mem_ptr<float> A, mem_ptr<float> C, int n, int k, float alpha, float beta) {
+        enqueue_syrk(device_num, stream_id, A, C, n, k, alpha, beta, false);
+      },
+      // Return mem_ptr with explicit routing
+      [this](return_mem_ptr_atom, int device_num, int stream_id, mem_ptr<float> A, mem_ptr<float> C, int n, int k, float alpha, float beta) {
+        enqueue_syrk(device_num, stream_id, A, C, n, k, alpha, beta, true);
+      },
       // Mem ptr return overloads
       [this](return_mem_ptr_atom, in<float> A, in_out<float> C, int n, int k) {
         enqueue_syrk(-1, actor_id_, A, C, n, k, 1.0f, 0.0f, true);
@@ -66,6 +77,9 @@ class syrk_actor : public event_based_actor {
       },
       [this](return_mem_ptr_atom, mem_ptr<float> A, mem_ptr<float> C, int n, int k) {
         enqueue_syrk(-1, actor_id_, A, C, n, k, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, mem_ptr<float> A, mem_ptr<float> C, int n, int k, float alpha, float beta) {
+        enqueue_syrk(-1, actor_id_, A, C, n, k, alpha, beta, true);
       }
     };
   }
@@ -125,12 +139,6 @@ class syrk_actor : public event_based_actor {
         }
       });
     }
-
-    runner.add_callback(stream_id, device_num, [sender, r_id]() {
-      if (sender) {
-        caf::anon_mail(r_id, -1).send(sender);
-      }
-    });
   }
 
   int actor_id_;
diff --git a/libcaf_cuda/caf/cuda/actor_facade.hpp b/libcaf_cuda/caf/cuda/actor_facade.hpp
index a74438e9b0..ae6e1e845a 100644
--- a/libcaf_cuda/caf/cuda/actor_facade.hpp
+++ b/libcaf_cuda/caf/cuda/actor_facade.hpp
@@ -116,13 +116,6 @@ class actor_facade : public event_based_actor {
         process_host_transfers(sender, r_id, results, std::move(output_indices), std::move(mappings));
       }
     }
-
-    // Final completion alert (Correlation ID, -1 to signal "all finished")
-    runner.add_callback(stream_id, device_num, [sender, r_id]() mutable {
-      if (sender) {
-        caf::anon_mail(r_id, -1).send(sender);
-      }
-    });
   }
 
   void send_mem_ptr_handles(const actor& sender, int r_id, const mem_tuple& results) {
diff --git a/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp
index bfacc4408d..2e070afabb 100644
--- a/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorBLAS-test/gemv-actor-test/main.test.cpp
@@ -73,13 +73,9 @@ void caf_main(actor_system& sys) {
                 if (arg_index == 2) { // index 2 corresponds to y_arg
                     verify_gemv_correctness(m, n, alpha, beta, data);
                 }
-            },
-            [&](int reply_id, int signal) {
-                if (signal == -1) {
-                    std::cout << "[INFO] Test 1 complete." << std::endl;
-                }
             }
         );
+        std::cout << "[INFO] Test 1 complete." << std::endl;
     }
 
     // Test 2: mem_ptr inputs
@@ -99,13 +95,9 @@ void caf_main(actor_system& sys) {
                 if (arg_index == 2) {
                     verify_gemv_correctness(m, n, alpha, beta, data);
                 }
-            },
-            [&](int reply_id, int signal) {
-                if (signal == -1) {
-                    std::cout << "[INFO] Test 2 complete." << std::endl;
-                }
             }
         );
+        std::cout << "[INFO] Test 2 complete." << std::endl;
     }
 
     // Test 3: Routing control (device/stream) + mem_ptr
@@ -123,13 +115,9 @@ void caf_main(actor_system& sys) {
                 if (arg_index == 2) {
                     verify_gemv_correctness(m, n, alpha, beta, data);
                 }
-            },
-            [&](int reply_id, int signal) {
-                if (signal == -1) {
-                    std::cout << "[INFO] Test 3 complete." << std::endl;
-                }
             }
         );
+        std::cout << "[INFO] Test 3 complete." << std::endl;
     }
 
     // Test 4: return_mem_ptr_atom (returning device handles)
@@ -145,15 +133,7 @@ void caf_main(actor_system& sys) {
                 verify_gemv_correctness(m, n, alpha, beta, host_y);
             }
         );
-
-        // Separate receive for the completion signal to ensure both are processed
-        self->receive(
-            [&](int reply_id, int signal) {
-                if (signal == -1) {
-                    std::cout << "[INFO] Test 4 complete." << std::endl;
-                }
-            }
-        );
+        std::cout << "[INFO] Test 4 complete." << std::endl;
     }
 
     self->send_exit(blas_actor, exit_reason::user_shutdown);
diff --git a/libcaf_cuda/tests/actorBLAS-test/syrk-actor-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/syrk-actor-test/main.test.cpp
index fe9d1f67ec..9ff75cce4c 100644
--- a/libcaf_cuda/tests/actorBLAS-test/syrk-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorBLAS-test/syrk-actor-test/main.test.cpp
@@ -8,12 +8,11 @@ using namespace caf;
 using namespace caf::cuda;
 
 void verify_syrk_correctness(int n, int k, float alpha, float beta, 
-                             const std::vector<float>& C_result) {
+                             const std::vector<float>& C_result, float initial_c_val = 0.0f) {
     // Expected value for C = alpha * (A * A^T) + beta * C_init
     // Since A is all 1.0s (n x k), each element of A*A^T is k.
-    // With alpha=1 and beta=0, expected is exactly k.
-    // Note: cuBLAS SYRK only updates the lower triangle by default in our implementation.
-    float expected_val = alpha * static_cast<float>(k); 
+    // expected = alpha * (k) + beta * initial_c_val
+    float expected_val = alpha * static_cast<float>(k) + beta * initial_c_val; 
     bool all_correct = true;
     for (int i = 0; i < n; ++i) {
         for (int j = 0; j <= i; ++j) { // Check lower triangle
@@ -60,17 +59,91 @@ void caf_main(actor_system& sys) {
                 if (arg_index == 1) { // index 1 corresponds to C_arg
                     verify_syrk_correctness(n, k, alpha, beta, data);
                 }
-            },
-            [&](int reply_id, int signal) {
-                if (signal == -1) {
-                    std::cout << "[INFO] Test 1 complete." << std::endl;
+            }
+        );
+        std::cout << "[INFO] Test 1 complete." << std::endl;
+    }
+
+    // Test 2: mem_ptr inputs
+    {
+        std::cout << "\n[INFO] Test 2: Testing syrk_actor with mem_ptr inputs..." << std::endl;
+        command_runner<in<float>, in_out<float>> setup_runner;
+        // Manually transfer data to the GPU to get mem_ptr handles
+        auto results = setup_runner.transfer_memory(0, 0, create_in_arg(h_A), create_in_out_arg(h_C));
+        auto A_ptr = std::get<0>(results);
+        auto C_ptr = std::get<1>(results);
+
+        self->mail(A_ptr, C_ptr, n, k, alpha, beta).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 1) {
+                    verify_syrk_correctness(n, k, alpha, beta, data);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 2 complete." << std::endl;
+    }
+
+    // Test 3: Routing control (device/stream) + mem_ptr
+    {
+        std::cout << "\n[INFO] Test 3: Testing syrk_actor with specific device/stream and mem_ptr..." << std::endl;
+        int device_num = 0;
+        int stream_id = 42; 
+        command_runner<in<float>, in_out<float>> setup_runner;
+        auto results = setup_runner.transfer_memory(device_num, stream_id, create_in_arg(h_A), create_in_out_arg(h_C));
+        
+        self->mail(device_num, stream_id, std::get<0>(results), std::get<1>(results), n, k, alpha, beta).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 1) {
+                    verify_syrk_correctness(n, k, alpha, beta, data);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 3 complete." << std::endl;
+    }
+
+    // Test 4: return_mem_ptr_atom (returning device handles)
+    {
+        std::cout << "\n[INFO] Test 4: Testing syrk_actor with return_mem_ptr_atom..." << std::endl;
+        self->mail(return_mem_ptr_atom{}, A_arg, C_arg, n, k, alpha, beta).send(blas_actor);
+
+        // We expect two messages back: the data handles (mem_ptrs) and the signal (-1)
+        self->receive(
+            [&](int reply_id, mem_ptr<float> A, mem_ptr<float> C) {
+                command_runner<float> runner;
+                // Since syrk is async, we copy C back to host to verify
+                auto host_C = runner.copy_to_host(C);
+                verify_syrk_correctness(n, k, alpha, beta, host_C);
+            }
+        );
+        std::cout << "[INFO] Test 4 complete." << std::endl;
+    }
+
+    // Test 5: Accumulation test (beta != 0)
+    {
+        std::cout << "\n[INFO] Test 5: Testing syrk_actor with accumulation (beta=1.0)..." << std::endl;
+        float beta_accum = 1.0f;
+        std::vector<float> h_C_ones(n * n, 1.0f); // initial C is all 1.0s
+        auto C_accum_arg = create_in_out_arg(h_C_ones);
+
+        self->mail(A_arg, C_accum_arg, n, k, alpha, beta_accum).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 1) {
+                    // Expected: alpha*k + beta*1.0 = 1.0*64 + 1.0*1.0 = 65.0
+                    verify_syrk_correctness(n, k, alpha, beta_accum, data, 1.0f);
                 }
             }
         );
+        std::cout << "[INFO] Test 5 complete." << std::endl;
     }
 
     self->send_exit(blas_actor, exit_reason::user_shutdown);
     manager::shutdown();
 }
 
-CAF_MAIN(id_block::cuda)
\ No newline at end of file
+CAF_MAIN(id_block::cuda)

From c65dbbfd5677579cfb20c97fb076e1bb143e90b8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 09:12:32 -0600
Subject: [PATCH 0674/1000] Updated tests to coninside with the actor facade
 changes.

---
 .../tests/actor-facade-test/CMakeLists.txt    |  4 +++
 .../actor-facade-test/latency_bench.test.cpp  |  9 ++---
 .../tests/actor-facade-test/main.test.cpp     | 36 +++++++++----------
 .../throughput_bench.test.cpp                 |  5 +--
 .../throughput_mapping_bench.test.cpp         |  4 +--
 5 files changed, 26 insertions(+), 32 deletions(-)

diff --git a/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt b/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
index f6f5dc669c..2e6c1faf1f 100644
--- a/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
@@ -46,6 +46,7 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
+    CUDA::cublas
 )
 target_link_libraries(throughput_mapping_bench_test
   PRIVATE
@@ -53,6 +54,7 @@ target_link_libraries(throughput_mapping_bench_test
     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
+    CUDA::cublas
 )
 
 
@@ -61,6 +63,7 @@ target_link_libraries(latency_bench_test
     "${CAF_BUILD}/libcaf_core/libcaf_core.so"
     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::cublas
     CUDA::nvrtc
 )
 target_link_libraries(throughput_bench_test
@@ -69,4 +72,5 @@ target_link_libraries(throughput_bench_test
     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
+    CUDA::cublas
 )
diff --git a/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp b/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp
index 1c2248b6f1..1f0b19622d 100644
--- a/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp
+++ b/libcaf_cuda/tests/actor-facade-test/latency_bench.test.cpp
@@ -40,17 +40,14 @@ caf::behavior latency_manager(caf::stateful_actor<latency_test_state>* self,
 
     return {
         [=](int r_id, int index, std::vector<int> data) {
-            // We don't need to do anything with the data for timing
-        },
-        [=](int r_id, int index) {
-            if (index == -1) { // Completion signal
+            if (index == 2) { // Completion signal for Matrix C
                 auto end_time = std::chrono::steady_clock::now();
                 auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
                     end_time - self->state().start_time).count();
-                
+
                 std::cout << "[LATENCY TEST] matrix_size=" << self->state().N 
                           << ", time=" << elapsed << " ms" << std::endl;
-                
+
                 self->send_exit(facade, exit_reason::user_shutdown);
                 self->quit();
             }
diff --git a/libcaf_cuda/tests/actor-facade-test/main.test.cpp b/libcaf_cuda/tests/actor-facade-test/main.test.cpp
index d12c2eb7b1..fa34d40827 100644
--- a/libcaf_cuda/tests/actor-facade-test/main.test.cpp
+++ b/libcaf_cuda/tests/actor-facade-test/main.test.cpp
@@ -69,10 +69,8 @@ caf::behavior mmul_facade_test(caf::stateful_actor<test_actor_state>* self,
 
     return {
         [=](int r_id, int index, std::vector<int> data) {
-            if (index == 2) self->state().h_c = std::move(data);
-        },
-        [=](int r_id, int index) {
-            if (index == -1) {
+            if (index == 2) {
+                self->state().h_c = std::move(data);
                 auto end_time = std::chrono::steady_clock::now();
                 std::chrono::duration<double> elapsed = end_time - self->state().start_time;
                 std::cout << "[BASIC] Latency: " << elapsed.count() << "s" << std::endl;
@@ -80,6 +78,7 @@ caf::behavior mmul_facade_test(caf::stateful_actor<test_actor_state>* self,
                 self->quit();
             }
         }
+        // No longer expecting index == -1 completion signal
     };
 }
 
@@ -111,22 +110,22 @@ caf::behavior mmul_advanced_facade_test(caf::stateful_actor<test_actor_state>* s
 
     return {
         [=](int r_id, int index, std::vector<int> data) {
-            // Verify that we ONLY get index 2, as requested in output_indices
-            bool requested = std::find(output_indices.begin(), output_indices.end(), index) != output_indices.end();
-            if (!requested) {
-                std::cout << "[ERROR] Received unrequested index: " << index << std::endl;
-            }
-            if (index == 2) self->state().h_c = std::move(data);
-        },
-        [=](int r_id, int index) {
-            if (index == -1) {
+            if (index == 2) {
+                self->state().h_c = std::move(data);
+
+                // Verify that we ONLY get index 2, as requested in output_indices
+                bool requested = std::find(output_indices.begin(), output_indices.end(), index) != output_indices.end();
+                if (!requested) {
+                    std::cout << "[ERROR] Received unrequested index: " << index << std::endl;
+                }
+
                 auto end_time = std::chrono::steady_clock::now();
                 std::chrono::duration<double> elapsed = end_time - self->state().start_time;
-                
+
                 std::cout << "===== Advanced Performance Result =====" << std::endl;
                 std::cout << "Round-trip Latency: " << elapsed.count() << " seconds" << std::endl;
                 verify_mmul(self->state().h_a, self->state().h_b, self->state().h_c, self->state().N);
-                
+
                 self->send_exit(facade, exit_reason::user_shutdown);
                 self->quit();
             }
@@ -167,16 +166,15 @@ caf::behavior mmul_mapping_test(caf::stateful_actor<test_actor_state>* self,
         [=](int r_id, int index) {
             if (index == 2) {
                 std::cout << "[MAPPING] Received notification for mapped index 2." << std::endl;
-            } else if (index == -1) {
                 auto end_time = std::chrono::steady_clock::now();
                 std::chrono::duration<double> elapsed = end_time - self->state().start_time;
-                
+
                 std::cout << "===== Mapping Performance Result =====" << std::endl;
                 std::cout << "Round-trip Latency: " << elapsed.count() << " seconds" << std::endl;
-                
+
                 // Verify the data was copied directly into h_c
                 verify_mmul(self->state().h_a, self->state().h_b, self->state().h_c, self->state().N);
-                
+
                 self->send_exit(facade, exit_reason::user_shutdown);
                 self->quit();
             }
diff --git a/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp b/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp
index 774f7289dd..c470bae58d 100644
--- a/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp
+++ b/libcaf_cuda/tests/actor-facade-test/throughput_bench.test.cpp
@@ -44,10 +44,7 @@ caf::behavior throughput_manager(caf::stateful_actor<throughput_state>* self,
 
     return {
         [=](int r_id, int index, std::vector<int> data) {
-            // Ignore data buffers
-        },
-        [=](int r_id, int index) {
-            if (index == -1) { // Completion signal for one request
+            if (index == 2) { // Matrix C arrived
                 if (++self->state().results_received == self->state().total_expected) {
                     auto end_time = std::chrono::steady_clock::now();
                     auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
diff --git a/libcaf_cuda/tests/actor-facade-test/throughput_mapping_bench.test.cpp b/libcaf_cuda/tests/actor-facade-test/throughput_mapping_bench.test.cpp
index 0333126314..5896806acf 100644
--- a/libcaf_cuda/tests/actor-facade-test/throughput_mapping_bench.test.cpp
+++ b/libcaf_cuda/tests/actor-facade-test/throughput_mapping_bench.test.cpp
@@ -50,7 +50,7 @@ caf::behavior throughput_mapping_manager(caf::stateful_actor<mapping_throughput_
 
     return {
         [=](int r_id, int index) {
-            if (index == -1) { // Completion signal for the whole message
+            if (index == 2) { // Received notification for index 2 (Matrix C)
                 if (++self->state().results_received == self->state().total_expected) {
                     auto end_time = std::chrono::steady_clock::now();
                     auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
@@ -64,8 +64,6 @@ caf::behavior throughput_mapping_manager(caf::stateful_actor<mapping_throughput_
                     self->quit();
                 }
             }
-            // Note: index == 2 is also received as a notification, but we only 
-            // care about the final -1 to signal everything (kernel + copy) is done.
         }
     };
 }

From ad94f5a188c676cc155f71d1c8d0dd121b9d6fef Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 09:13:48 -0600
Subject: [PATCH 0675/1000] Updated header.

---
 libcaf_cuda/caf/actorBLAS/actorBLAS.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp b/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp
index 3270532c61..44fc5efe3c 100644
--- a/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp
+++ b/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp
@@ -1,2 +1,3 @@
 #pragma once
 #include "caf/actorBLAS/gemv-actor/gemv-actor.hpp"
+#include "caf/actorBLAS/gemv-actor/syrk-actor.hpp"

From 358eca2aff50aa9ca4b4179f65f8d679635676d4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 09:23:37 -0600
Subject: [PATCH 0676/1000] implemented the axpy operation and actor and tests.

---
 .../caf/actorBLAS/axpy-actor/axpy-actor.hpp   | 128 ++++++++++++++++++
 libcaf_cuda/caf/cuda/device.hpp               |  17 +++
 .../axpy-actor-test/CMakeLists.txt            |  45 ++++++
 .../axpy-actor-test/main.test.cpp             | 100 ++++++++++++++
 4 files changed, 290 insertions(+)
 create mode 100644 libcaf_cuda/caf/actorBLAS/axpy-actor/axpy-actor.hpp
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/axpy-actor-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/axpy-actor-test/main.test.cpp

diff --git a/libcaf_cuda/caf/actorBLAS/axpy-actor/axpy-actor.hpp b/libcaf_cuda/caf/actorBLAS/axpy-actor/axpy-actor.hpp
new file mode 100644
index 0000000000..76023affcb
--- /dev/null
+++ b/libcaf_cuda/caf/actorBLAS/axpy-actor/axpy-actor.hpp
@@ -0,0 +1,128 @@
+#pragma once
+
+#include <caf/actor.hpp>
+#include <caf/actor_cast.hpp>
+#include <caf/actor_config.hpp>
+#include <caf/actor_system.hpp>
+#include <caf/anon_mail.hpp>
+#include <caf/event_based_actor.hpp>
+
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/mem_ref.hpp"
+#include "caf/cuda/command_runner.hpp"
+#include "caf/cuda/platform.hpp"
+#include "caf/cuda/types.hpp"
+
+namespace caf::cuda {
+
+/// AXPY Actor for single-precision vector-vector addition.
+/// Message Signature: (in<float> x, in_out<float> y, int n, float alpha)
+class axpy_actor : public event_based_actor {
+public:
+  static caf::actor spawn(caf::actor_system& sys, int reply_id = 0) {
+    return sys.spawn<axpy_actor>(reply_id);
+  }
+
+  axpy_actor(caf::actor_config& cfg, int reply_id = 0) 
+    : event_based_actor(cfg), reply_id_(reply_id) {
+    actor_id_ = static_cast<int>(this->id());
+  }
+
+  ~axpy_actor() override {
+    command_runner<> runner;
+    runner.release_stream_for_actor(actor_id_);
+  }
+
+  caf::behavior make_behavior() override {
+    return {
+      // Standard host buffer based calls
+      [this](in<float> x, in_out<float> y, int n, float alpha) {
+        enqueue_axpy(-1, actor_id_, x, y, n, alpha, false);
+      },
+      // Routing control overloads
+      [this](int device_num, int stream_id, in<float> x, in_out<float> y, int n, float alpha) {
+        enqueue_axpy(device_num, stream_id, x, y, n, alpha, false);
+      },
+      // mem_ptr based calls (useful for pipelines)
+      [this](mem_ptr<float> x, mem_ptr<float> y, int n, float alpha) {
+        enqueue_axpy(-1, actor_id_, x, y, n, alpha, false);
+      },
+      [this](int device_num, int stream_id, mem_ptr<float> x, mem_ptr<float> y, int n, float alpha) {
+        enqueue_axpy(device_num, stream_id, x, y, n, alpha, false);
+      },
+      // Mem ptr return overloads
+      [this](return_mem_ptr_atom, in<float> x, in_out<float> y, int n, float alpha) {
+        enqueue_axpy(-1, actor_id_, x, y, n, alpha, true);
+      },
+      [this](return_mem_ptr_atom, int device_num, int stream_id, in<float> x, in_out<float> y, int n, float alpha) {
+        enqueue_axpy(device_num, stream_id, x, y, n, alpha, true);
+      },
+      [this](return_mem_ptr_atom, mem_ptr<float> x, mem_ptr<float> y, int n, float alpha) {
+        enqueue_axpy(-1, actor_id_, x, y, n, alpha, true);
+      },
+    };
+  }
+
+private:
+  // Overload for Host-wrapped buffers
+  void enqueue_axpy(int device_num, int stream_id, 
+                    in<float> x_arg, in_out<float> y_arg, 
+                    int n, float alpha, bool return_ptrs) {
+    command_runner<in<float>, in_out<float>> runner;
+    
+    // Allocate/Transfer memory.
+    auto results = runner.transfer_memory(device_num, stream_id, x_arg, y_arg);
+    execute_and_reply(device_num, stream_id, std::get<0>(results), 
+                      std::get<1>(results), n, alpha, return_ptrs);
+  }
+
+  // Overload for already-existing Device buffers
+  void enqueue_axpy(int device_num, int stream_id, 
+                    mem_ptr<float> x_ptr, mem_ptr<float> y_ptr, 
+                    int n, float alpha, bool return_ptrs) {
+    // Pass through command_runner to ensure proper ref-counting/scheduling
+    command_runner<mem_ptr<float>, mem_ptr<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, x_ptr, y_ptr);
+    execute_and_reply(device_num, stream_id, std::get<0>(results), 
+                      std::get<1>(results), n, alpha, return_ptrs);
+  }
+
+  void execute_and_reply(int device_num, int stream_id, 
+                         mem_ptr<float> x, mem_ptr<float> y, 
+                         int n, float alpha, bool return_ptrs) {
+    auto plat = platform::create();
+    device_ptr dev;
+    if (device_num == -1)
+      dev = plat->schedule(stream_id);
+    else
+      dev = plat->schedule(stream_id, device_num);
+
+    // Perform cuBLAS operation
+    dev->saxpy(stream_id, n, alpha, x, y);
+
+    handle_reply(device_num, stream_id, x, y, return_ptrs);
+  }
+
+  void handle_reply(int, int, mem_ptr<float> x_ptr, mem_ptr<float> y_ptr, bool return_ptrs) {
+    command_runner<mem_ptr<float>> runner;
+    auto sender = actor_cast<actor>(this->current_sender());
+    if (!sender) return;
+    
+    auto r_id = reply_id_;
+
+    if (return_ptrs) {
+      caf::anon_mail(r_id, x_ptr, y_ptr).send(sender);
+    } else {
+      runner.copy_to_host_async(y_ptr, [sender, r_id](std::vector<float>&& data) {
+        if (sender) {
+          caf::anon_mail(r_id, 1, std::move(data)).send(sender);
+        }
+      });
+    }
+  }
+
+  int actor_id_;
+  int reply_id_;
+};
+
+} // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 61de51416e..7380f3ef11 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -188,6 +188,23 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasSsyrk failed on device " + std::to_string(id_));
   }
 
+  /// Performs single precision vector-vector addition (y = alpha*x + y).
+  void saxpy(int actor_id, int n, float alpha, mem_ptr<float> x, mem_ptr<float> y) {
+    cublasHandle_t handle = get_cublas_handle(actor_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+
+    cublasStatus_t status = cublasSaxpy(handle, n, &alpha,
+                                        reinterpret_cast<const float*>(x->mem()), 1,
+                                        reinterpret_cast<float*>(y->mem()), 1);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasSaxpy failed on device " + std::to_string(id_));
+  }
+
   // Overloads for make_arg using actor_id
   template <typename T>
   mem_ptr<T> make_arg(const in<T>& arg, int actor_id) {
diff --git a/libcaf_cuda/tests/actorBLAS-test/axpy-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorBLAS-test/axpy-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..fa9dc8e759
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/axpy-actor-test/CMakeLists.txt
@@ -0,0 +1,45 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+)
+
diff --git a/libcaf_cuda/tests/actorBLAS-test/axpy-actor-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/axpy-actor-test/main.test.cpp
new file mode 100644
index 0000000000..3a40b5becb
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/axpy-actor-test/main.test.cpp
@@ -0,0 +1,100 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include "caf/actorBLAS/axpy-actor/axpy-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+void verify_axpy_correctness(int n, float alpha, 
+                             const std::vector<float>& x,
+                             const std::vector<float>& y_initial,
+                             const std::vector<float>& y_result) {
+    bool all_correct = true;
+    for (int i = 0; i < n; ++i) {
+        float expected = alpha * x[i] + y_initial[i];
+        if (std::abs(y_result[i] - expected) > 1e-4) {
+            all_correct = false;
+            std::cout << "[ERROR] Mismatch at index " << i << ": "
+                      << "Expected " << expected << ", Got " << y_result[i] << std::endl;
+            break;
+        }
+    }
+
+    if (all_correct) {
+        std::cout << "[SUCCESS] AXPY actor produced correct results." << std::endl;
+    }
+}
+
+void caf_main(actor_system& sys) {
+    manager_config config(true); 
+    manager::init(sys, config);
+
+    int n = 1024;
+    float alpha = 2.0f;
+
+    std::vector<float> h_x(n, 1.0f);
+    std::vector<float> h_y(n, 3.0f);
+
+    auto blas_actor = sys.spawn<axpy_actor>(1);
+
+    auto x_arg = create_in_arg(h_x);
+    auto y_arg = create_in_out_arg(h_y);
+
+    scoped_actor self{sys};
+
+    // Test 1: Standard host-buffer based call
+    {
+        std::cout << "[INFO] Test 1: Testing axpy_actor with host-buffer arguments..." << std::endl;
+        self->mail(x_arg, y_arg, n, alpha).send(blas_actor);
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 1) { // index 1 corresponds to y_arg
+                    verify_axpy_correctness(n, alpha, h_x, h_y, data);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 1 complete." << std::endl;
+    }
+
+    // Test 2: mem_ptr inputs
+    {
+        std::cout << "\n[INFO] Test 2: Testing axpy_actor with mem_ptr inputs..." << std::endl;
+        command_runner<in<float>, in_out<float>> setup_runner;
+        auto results = setup_runner.transfer_memory(0, 0, create_in_arg(h_x), create_in_out_arg(h_y));
+        auto x_ptr = std::get<0>(results);
+        auto y_ptr = std::get<1>(results);
+
+        self->mail(x_ptr, y_ptr, n, alpha).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 1) {
+                    verify_axpy_correctness(n, alpha, h_x, h_y, data);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 2 complete." << std::endl;
+    }
+
+    // Test 3: return_mem_ptr_atom
+    {
+        std::cout << "\n[INFO] Test 3: Testing axpy_actor with return_mem_ptr_atom..." << std::endl;
+        self->mail(return_mem_ptr_atom{}, x_arg, y_arg, n, alpha).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, mem_ptr<float> x, mem_ptr<float> y) {
+                command_runner<float> runner;
+                auto host_y = runner.copy_to_host(y);
+                verify_axpy_correctness(n, alpha, h_x, h_y, host_y);
+            }
+        );
+        std::cout << "[INFO] Test 3 complete." << std::endl;
+    }
+
+    self->send_exit(blas_actor, exit_reason::user_shutdown);
+    manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda)

From 546d5fdaa8363eed429081c5f67ca469ea541fab Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 09:29:55 -0600
Subject: [PATCH 0677/1000] Updated file.

---
 libcaf_cuda/caf/actorBLAS/actorBLAS.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp b/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp
index 44fc5efe3c..7a439bf2d9 100644
--- a/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp
+++ b/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp
@@ -1,3 +1,4 @@
 #pragma once
 #include "caf/actorBLAS/gemv-actor/gemv-actor.hpp"
 #include "caf/actorBLAS/gemv-actor/syrk-actor.hpp"
+#include "caf/actorBLAS/gemv-actor/axpy-actor.hpp"

From 9faf2d916427480782e71f21a3e4e64b33c1d362 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 10:07:33 -0600
Subject: [PATCH 0678/1000] Created nrm2 actor and tests.

---
 .../caf/actorBLAS/nrm2-actor/nrm2-actor.hpp   | 121 ++++++++++++++++++
 libcaf_cuda/caf/cuda/device.hpp               |  19 +++
 .../nmr2-actor-test/CMakeLists.txt            |  45 +++++++
 .../nmr2-actor-test/main.test.cpp             |  92 +++++++++++++
 4 files changed, 277 insertions(+)
 create mode 100644 libcaf_cuda/caf/actorBLAS/nrm2-actor/nrm2-actor.hpp
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/nmr2-actor-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/nmr2-actor-test/main.test.cpp

diff --git a/libcaf_cuda/caf/actorBLAS/nrm2-actor/nrm2-actor.hpp b/libcaf_cuda/caf/actorBLAS/nrm2-actor/nrm2-actor.hpp
new file mode 100644
index 0000000000..66dd66a7c4
--- /dev/null
+++ b/libcaf_cuda/caf/actorBLAS/nrm2-actor/nrm2-actor.hpp
@@ -0,0 +1,121 @@
+#pragma once
+
+#include <caf/actor.hpp>
+#include <caf/actor_cast.hpp>
+#include <caf/actor_config.hpp>
+#include <caf/actor_system.hpp>
+#include <caf/anon_mail.hpp>
+#include <caf/event_based_actor.hpp>
+
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/mem_ref.hpp"
+#include "caf/cuda/command_runner.hpp"
+#include "caf/cuda/platform.hpp"
+#include "caf/cuda/types.hpp"
+
+namespace caf::cuda {
+
+/// NRM2 Actor for single-precision vector Euclidean norm.
+/// Message Signature: (in<float> x, out<float> res, int n)
+class nrm2_actor : public event_based_actor {
+public:
+  static caf::actor spawn(caf::actor_system& sys, int reply_id = 0) {
+    return sys.spawn<nrm2_actor>(reply_id);
+  }
+
+  nrm2_actor(caf::actor_config& cfg, int reply_id = 0) 
+    : event_based_actor(cfg), reply_id_(reply_id) {
+    actor_id_ = static_cast<int>(this->id());
+  }
+
+  ~nrm2_actor() override {
+    command_runner<> runner;
+    runner.release_stream_for_actor(actor_id_);
+  }
+
+  caf::behavior make_behavior() override {
+    return {
+      // Standard host buffer based calls
+      [this](in<float> x, out<float> res, int n) {
+        enqueue_nrm2(-1, actor_id_, x, res, n, false);
+      },
+      // Routing control overloads
+      [this](int device_num, int stream_id, in<float> x, out<float> res, int n) {
+        enqueue_nrm2(device_num, stream_id, x, res, n, false);
+      },
+      // mem_ptr based calls
+      [this](mem_ptr<float> x, mem_ptr<float> res, int n) {
+        enqueue_nrm2(-1, actor_id_, x, res, n, false);
+      },
+      [this](int device_num, int stream_id, mem_ptr<float> x, mem_ptr<float> res, int n) {
+        enqueue_nrm2(device_num, stream_id, x, res, n, false);
+      },
+      // Mem ptr return overloads
+      [this](return_mem_ptr_atom, in<float> x, out<float> res, int n) {
+        enqueue_nrm2(-1, actor_id_, x, res, n, true);
+      },
+      [this](return_mem_ptr_atom, int device_num, int stream_id, in<float> x, out<float> res, int n) {
+        enqueue_nrm2(device_num, stream_id, x, res, n, true);
+      },
+      [this](return_mem_ptr_atom, mem_ptr<float> x, mem_ptr<float> res, int n) {
+        enqueue_nrm2(-1, actor_id_, x, res, n, true);
+      },
+    };
+  }
+
+private:
+  void enqueue_nrm2(int device_num, int stream_id, 
+                    in<float> x_arg, out<float> res_arg, 
+                    int n, bool return_ptrs) {
+    command_runner<in<float>, out<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, x_arg, res_arg);
+    execute_and_reply(device_num, stream_id, std::get<0>(results), 
+                      std::get<1>(results), n, return_ptrs);
+  }
+
+  void enqueue_nrm2(int device_num, int stream_id, 
+                    mem_ptr<float> x_ptr, mem_ptr<float> res_ptr, 
+                    int n, bool return_ptrs) {
+    command_runner<mem_ptr<float>, mem_ptr<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, x_ptr, res_ptr);
+    execute_and_reply(device_num, stream_id, std::get<0>(results), 
+                      std::get<1>(results), n, return_ptrs);
+  }
+
+  void execute_and_reply(int device_num, int stream_id, 
+                         mem_ptr<float> x, mem_ptr<float> res, 
+                         int n, bool return_ptrs) {
+    auto plat = platform::create();
+    device_ptr dev;
+    if (device_num == -1)
+      dev = plat->schedule(stream_id);
+    else
+      dev = plat->schedule(stream_id, device_num);
+
+    dev->snrm2(stream_id, n, x, res);
+    handle_reply(device_num, stream_id, x, res, return_ptrs);
+  }
+
+  void handle_reply(int, int, mem_ptr<float> x_ptr, mem_ptr<float> res_ptr, bool return_ptrs) {
+    command_runner<> runner;
+    auto sender = actor_cast<actor>(this->current_sender());
+    if (!sender) return;
+    
+    auto r_id = reply_id_;
+    if (return_ptrs) {
+      caf::anon_mail(r_id, x_ptr, res_ptr).send(sender);
+    } else {
+      runner.copy_to_host_async(res_ptr, [sender, r_id](std::vector<float>&& data) {
+        if (sender) {
+          caf::anon_mail(r_id, 1, std::move(data)).send(sender);
+        }
+      });
+    }
+  }
+
+  int actor_id_;
+  int reply_id_;
+};
+
+} // namespace caf::cuda
+
diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 7380f3ef11..99192fb43f 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -205,6 +205,25 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasSaxpy failed on device " + std::to_string(id_));
   }
 
+  /// Performs single precision Euclidean norm (result = ||x||2).
+  void snrm2(int actor_id, int n, mem_ptr<float> x, mem_ptr<float> result) {
+    cublasHandle_t handle = get_cublas_handle(actor_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+
+    cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_DEVICE);
+    cublasStatus_t status = cublasSnrm2(handle, n, 
+                                        reinterpret_cast<const float*>(x->mem()), 1,
+                                        reinterpret_cast<float*>(result->mem()));
+    cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasSnrm2 failed on device " + std::to_string(id_));
+  }
+
   // Overloads for make_arg using actor_id
   template <typename T>
   mem_ptr<T> make_arg(const in<T>& arg, int actor_id) {
diff --git a/libcaf_cuda/tests/actorBLAS-test/nmr2-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorBLAS-test/nmr2-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..fa9dc8e759
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/nmr2-actor-test/CMakeLists.txt
@@ -0,0 +1,45 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+)
+
diff --git a/libcaf_cuda/tests/actorBLAS-test/nmr2-actor-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/nmr2-actor-test/main.test.cpp
new file mode 100644
index 0000000000..99bd6b5b21
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/nmr2-actor-test/main.test.cpp
@@ -0,0 +1,92 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include <cmath>
+#include "caf/actorBLAS/nrm2-actor/nrm2-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+void verify_nrm2_correctness(int n, const std::vector<float>& x, float result) {
+    double sum = 0;
+    for (float val : x) {
+        sum += (double)val * val;
+    }
+    float expected = (float)std::sqrt(sum);
+
+    if (std::abs(result - expected) > 1e-4) {
+        std::cout << "[ERROR] Mismatch: Expected " << expected << ", Got " << result << std::endl;
+    } else {
+        std::cout << "[SUCCESS] NRM2 actor produced correct results: " << result << std::endl;
+    }
+}
+
+void caf_main(actor_system& sys) {
+    manager_config config(true); 
+    manager::init(sys, config);
+
+    int n = 1024;
+    std::vector<float> h_x(n, 1.0f);
+
+    auto blas_actor = sys.spawn<nrm2_actor>(1);
+    auto x_arg = create_in_arg(h_x);
+    auto res_arg = create_out_arg_with_size<float>(1);
+
+    scoped_actor self{sys};
+
+    // Test 1: Standard host-buffer based call
+    {
+        std::cout << "[INFO] Test 1: Testing nrm2_actor with host-buffer arguments..." << std::endl;
+        self->mail(x_arg, res_arg, n).send(blas_actor);
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 1) { // index 1 corresponds to res_arg
+                    verify_nrm2_correctness(n, h_x, data[0]);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 1 complete." << std::endl;
+    }
+
+    // Test 2: mem_ptr inputs
+    {
+        std::cout << "\n[INFO] Test 2: Testing nrm2_actor with mem_ptr inputs..." << std::endl;
+        command_runner<in<float>, out<float>> setup_runner;
+        auto results = setup_runner.transfer_memory(0, 0, create_in_arg(h_x), create_out_arg_with_size<float>(1));
+        auto x_ptr = std::get<0>(results);
+        auto res_ptr = std::get<1>(results);
+
+        self->mail(x_ptr, res_ptr, n).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 1) {
+                    verify_nrm2_correctness(n, h_x, data[0]);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 2 complete." << std::endl;
+    }
+
+    // Test 3: return_mem_ptr_atom
+    {
+        std::cout << "\n[INFO] Test 3: Testing nrm2_actor with return_mem_ptr_atom..." << std::endl;
+        self->mail(return_mem_ptr_atom{}, x_arg, res_arg, n).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, mem_ptr<float> x, mem_ptr<float> res) {
+                command_runner<float> runner;
+                auto host_res = runner.copy_to_host(res);
+                verify_nrm2_correctness(n, h_x, host_res[0]);
+            }
+        );
+        std::cout << "[INFO] Test 3 complete." << std::endl;
+    }
+
+    self->send_exit(blas_actor, exit_reason::user_shutdown);
+    manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda)
+

From 18260e4363e7ff677be053f01cc28080c822936b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 10:49:43 -0600
Subject: [PATCH 0679/1000] Added copy and dot actors and functionality that
 goes with it.

---
 .../caf/actorBLAS/copy-actor/copy-actor.hpp   | 96 +++++++++++++++++++
 .../caf/actorBLAS/dot-actor/dot-actor.hpp     | 96 +++++++++++++++++++
 libcaf_cuda/caf/cuda/device.hpp               | 37 +++++++
 3 files changed, 229 insertions(+)
 create mode 100644 libcaf_cuda/caf/actorBLAS/copy-actor/copy-actor.hpp
 create mode 100644 libcaf_cuda/caf/actorBLAS/dot-actor/dot-actor.hpp

diff --git a/libcaf_cuda/caf/actorBLAS/copy-actor/copy-actor.hpp b/libcaf_cuda/caf/actorBLAS/copy-actor/copy-actor.hpp
new file mode 100644
index 0000000000..22ad1dc436
--- /dev/null
+++ b/libcaf_cuda/caf/actorBLAS/copy-actor/copy-actor.hpp
@@ -0,0 +1,96 @@
+#pragma once
+
+#include <caf/actor.hpp>
+#include <caf/actor_cast.hpp>
+#include <caf/actor_config.hpp>
+#include <caf/actor_system.hpp>
+#include <caf/anon_mail.hpp>
+#include <caf/event_based_actor.hpp>
+
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/mem_ref.hpp"
+#include "caf/cuda/command_runner.hpp"
+#include "caf/cuda/platform.hpp"
+#include "caf/cuda/types.hpp"
+
+namespace caf::cuda {
+
+/// SCOPY Actor for y = x.
+class copy_actor : public event_based_actor {
+public:
+  static caf::actor spawn(caf::actor_system& sys, int reply_id = 0) {
+    return sys.spawn<copy_actor>(reply_id);
+  }
+
+  copy_actor(caf::actor_config& cfg, int reply_id = 0) 
+    : event_based_actor(cfg), reply_id_(reply_id) {
+    actor_id_ = static_cast<int>(this->id());
+  }
+
+  ~copy_actor() override {
+    command_runner<> runner;
+    runner.release_stream_for_actor(actor_id_);
+  }
+
+  caf::behavior make_behavior() override {
+    return {
+      [this](in<float> x, out<float> y, int n) {
+        enqueue_scopy(-1, actor_id_, x, y, n, false);
+      },
+      [this](int device_num, int stream_id, in<float> x, out<float> y, int n) {
+        enqueue_scopy(device_num, stream_id, x, y, n, false);
+      },
+      [this](mem_ptr<float> x, mem_ptr<float> y, int n) {
+        enqueue_scopy(-1, actor_id_, x, y, n, false);
+      },
+      [this](int device_num, int stream_id, mem_ptr<float> x, mem_ptr<float> y, int n) {
+        enqueue_scopy(device_num, stream_id, x, y, n, false);
+      },
+      [this](return_mem_ptr_atom, mem_ptr<float> x, mem_ptr<float> y, int n) {
+        enqueue_scopy(-1, actor_id_, x, y, n, true);
+      }
+    };
+  }
+
+private:
+  void enqueue_scopy(int device_num, int stream_id, in<float> x, out<float> y, int n, bool return_ptrs) {
+    command_runner<in<float>, out<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, x, y);
+    execute_and_reply(device_num, stream_id, std::get<0>(results), std::get<1>(results), n, return_ptrs);
+  }
+
+  void enqueue_scopy(int device_num, int stream_id, mem_ptr<float> x, mem_ptr<float> y, int n, bool return_ptrs) {
+    command_runner<mem_ptr<float>, mem_ptr<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, x, y);
+    execute_and_reply(device_num, stream_id, std::get<0>(results), std::get<1>(results), n, return_ptrs);
+  }
+
+  void execute_and_reply(int device_num, int stream_id, mem_ptr<float> x, mem_ptr<float> y, int n, bool return_ptrs) {
+    auto plat = platform::create();
+    device_ptr dev = (device_num == -1) ? plat->schedule(stream_id) : plat->schedule(stream_id, device_num);
+    dev->scopy(stream_id, n, x, y);
+    handle_reply(device_num, stream_id, x, y, return_ptrs);
+  }
+
+  void handle_reply(int device_num, int stream_id, mem_ptr<float> x_ptr, mem_ptr<float> y_ptr, bool return_ptrs) {
+    command_runner<mem_ptr<float>> runner;
+    auto sender = actor_cast<actor>(this->current_sender());
+    if (!sender) return;
+    auto r_id = reply_id_;
+    if (return_ptrs) {
+      caf::anon_mail(r_id, x_ptr, y_ptr).send(sender);
+    } else {
+      runner.copy_to_host_async(y_ptr, [sender, r_id](std::vector<float>&& data) {
+        if (sender) {
+          caf::anon_mail(r_id, 1, std::move(data)).send(sender);
+        }
+      });
+    }
+  }
+
+  int actor_id_;
+  int reply_id_;
+};
+
+} // namespace caf::cuda
+
diff --git a/libcaf_cuda/caf/actorBLAS/dot-actor/dot-actor.hpp b/libcaf_cuda/caf/actorBLAS/dot-actor/dot-actor.hpp
new file mode 100644
index 0000000000..dbac4a4aba
--- /dev/null
+++ b/libcaf_cuda/caf/actorBLAS/dot-actor/dot-actor.hpp
@@ -0,0 +1,96 @@
+#pragma once
+
+#include <caf/actor.hpp>
+#include <caf/actor_cast.hpp>
+#include <caf/actor_config.hpp>
+#include <caf/actor_system.hpp>
+#include <caf/anon_mail.hpp>
+#include <caf/event_based_actor.hpp>
+
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/mem_ref.hpp"
+#include "caf/cuda/command_runner.hpp"
+#include "caf/cuda/platform.hpp"
+#include "caf/cuda/types.hpp"
+
+namespace caf::cuda {
+
+/// SDOT Actor for result = x^T * y.
+class dot_actor : public event_based_actor {
+public:
+  static caf::actor spawn(caf::actor_system& sys, int reply_id = 0) {
+    return sys.spawn<dot_actor>(reply_id);
+  }
+
+  dot_actor(caf::actor_config& cfg, int reply_id = 0) 
+    : event_based_actor(cfg), reply_id_(reply_id) {
+    actor_id_ = static_cast<int>(this->id());
+  }
+
+  ~dot_actor() override {
+    command_runner<> runner;
+    runner.release_stream_for_actor(actor_id_);
+  }
+
+  caf::behavior make_behavior() override {
+    return {
+      [this](in<float> x, in<float> y, out<float> res, int n) {
+        enqueue_sdot(-1, actor_id_, x, y, res, n, false);
+      },
+      [this](int device_num, int stream_id, in<float> x, in<float> y, out<float> res, int n) {
+        enqueue_sdot(device_num, stream_id, x, y, res, n, false);
+      },
+      [this](mem_ptr<float> x, mem_ptr<float> y, mem_ptr<float> res, int n) {
+        enqueue_sdot(-1, actor_id_, x, y, res, n, false);
+      },
+      [this](int device_num, int stream_id, mem_ptr<float> x, mem_ptr<float> y, mem_ptr<float> res, int n) {
+        enqueue_sdot(device_num, stream_id, x, y, res, n, false);
+      },
+      [this](return_mem_ptr_atom, mem_ptr<float> x, mem_ptr<float> y, mem_ptr<float> res, int n) {
+        enqueue_sdot(-1, actor_id_, x, y, res, n, true);
+      }
+    };
+  }
+
+private:
+  void enqueue_sdot(int device_num, int stream_id, in<float> x, in<float> y, out<float> res, int n, bool return_ptrs) {
+    command_runner<in<float>, in<float>, out<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, x, y, res);
+    execute_and_reply(device_num, stream_id, std::get<0>(results), std::get<1>(results), std::get<2>(results), n, return_ptrs);
+  }
+
+  void enqueue_sdot(int device_num, int stream_id, mem_ptr<float> x, mem_ptr<float> y, mem_ptr<float> res, int n, bool return_ptrs) {
+    command_runner<mem_ptr<float>, mem_ptr<float>, mem_ptr<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, x, y, res);
+    execute_and_reply(device_num, stream_id, std::get<0>(results), std::get<1>(results), std::get<2>(results), n, return_ptrs);
+  }
+
+  void execute_and_reply(int device_num, int stream_id, mem_ptr<float> x, mem_ptr<float> y, mem_ptr<float> res, int n, bool return_ptrs) {
+    auto plat = platform::create();
+    device_ptr dev = (device_num == -1) ? plat->schedule(stream_id) : plat->schedule(stream_id, device_num);
+    dev->sdot(stream_id, n, x, y, res);
+    handle_reply(device_num, stream_id, x, y, res, return_ptrs);
+  }
+
+  void handle_reply(int device_num, int stream_id, mem_ptr<float> x, mem_ptr<float> y, mem_ptr<float> res, bool return_ptrs) {
+    command_runner<mem_ptr<float>> runner;
+    auto sender = actor_cast<actor>(this->current_sender());
+    if (!sender) return;
+    auto r_id = reply_id_;
+    if (return_ptrs) {
+      caf::anon_mail(r_id, x, y, res).send(sender);
+    } else {
+      runner.copy_to_host_async(res, [sender, r_id](std::vector<float>&& data) {
+        if (sender && !data.empty()) {
+          caf::anon_mail(r_id, data[0]).send(sender);
+        }
+      });
+    }
+  }
+
+  int actor_id_;
+  int reply_id_;
+};
+
+} // namespace caf::cuda
+
diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 99192fb43f..1b9be6cf19 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -224,6 +224,43 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasSnrm2 failed on device " + std::to_string(id_));
   }
 
+  /// Performs single precision dot product (result = x^T * y).
+  void sdot(int actor_id, int n, mem_ptr<float> x, mem_ptr<float> y, mem_ptr<float> result) {
+    cublasHandle_t handle = get_cublas_handle(actor_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+
+    cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_DEVICE);
+    cublasStatus_t status = cublasSdot(handle, n, 
+                                        reinterpret_cast<const float*>(x->mem()), 1,
+                                        reinterpret_cast<const float*>(y->mem()), 1,
+                                        reinterpret_cast<float*>(result->mem()));
+    cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasSdot failed on device " + std::to_string(id_));
+  }
+
+  /// Copies vector x to vector y (y = x).
+  void scopy(int actor_id, int n, mem_ptr<float> x, mem_ptr<float> y) {
+    cublasHandle_t handle = get_cublas_handle(actor_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+
+    cublasStatus_t status = cublasScopy(handle, n, 
+                                         reinterpret_cast<const float*>(x->mem()), 1,
+                                         reinterpret_cast<float*>(y->mem()), 1);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasScopy failed on device " + std::to_string(id_));
+  }
+
   // Overloads for make_arg using actor_id
   template <typename T>
   mem_ptr<T> make_arg(const in<T>& arg, int actor_id) {

From f9ae6836185273f409cbd3bd9fe1ddf68a08a96d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 10:57:34 -0600
Subject: [PATCH 0680/1000] initial commit.

---
 .../copy-actor-test/CMakeLists.txt            |  45 +++++++
 .../copy-actor-test/main.test.cpp             | 120 ++++++++++++++++++
 2 files changed, 165 insertions(+)
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/copy-actor-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/copy-actor-test/main.test.cpp

diff --git a/libcaf_cuda/tests/actorBLAS-test/copy-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorBLAS-test/copy-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..fa9dc8e759
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/copy-actor-test/CMakeLists.txt
@@ -0,0 +1,45 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+)
+
diff --git a/libcaf_cuda/tests/actorBLAS-test/copy-actor-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/copy-actor-test/main.test.cpp
new file mode 100644
index 0000000000..f6c0cc1952
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/copy-actor-test/main.test.cpp
@@ -0,0 +1,120 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include "caf/actorBLAS/copy-actor/copy-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+void verify_copy_correctness(int n, const std::vector<float>& source, const std::vector<float>& result) {
+    bool all_correct = true;
+    for (int i = 0; i < n; ++i) {
+        if (std::abs(source[i] - result[i]) > 1e-4) {
+            all_correct = false;
+            std::cout << "[ERROR] Mismatch at index " << i << ": "
+                      << "Expected " << source[i] << ", Got " << result[i] << std::endl;
+            break;
+        }
+    }
+
+    if (all_correct) {
+        std::cout << "[SUCCESS] Copy actor produced correct results." << std::endl;
+    }
+}
+
+void caf_main(actor_system& sys) {
+    manager_config config(true); 
+    manager::init(sys, config);
+
+    int n = 1024;
+    std::vector<float> h_x(n);
+    for (int i = 0; i < n; ++i) {
+        h_x[i] = static_cast<float>(i);
+    }
+    std::vector<float> h_y(n, 0.0f);
+
+    auto blas_actor = sys.spawn<copy_actor>(1);
+
+    auto x_arg = create_in_arg(h_x);
+    auto y_arg = create_out_arg(h_y);
+
+    scoped_actor self{sys};
+
+    // Test 1: Standard host-buffer based call
+    {
+        std::cout << "[INFO] Test 1: Testing copy_actor with host-buffer arguments..." << std::endl;
+        self->mail(x_arg, y_arg, n).send(blas_actor);
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 1) { // index 1 corresponds to y_arg
+                    verify_copy_correctness(n, h_x, data);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 1 complete." << std::endl;
+    }
+
+    // Test 2: mem_ptr inputs
+    {
+        std::cout << "\n[INFO] Test 2: Testing copy_actor with mem_ptr inputs..." << std::endl;
+        command_runner<in<float>, out<float>> setup_runner;
+        // Manually transfer data to the GPU to get mem_ptr handles
+        auto results = setup_runner.transfer_memory(0, 0, create_in_arg(h_x), create_out_arg(h_y));
+        auto x_ptr = std::get<0>(results);
+        auto y_ptr = std::get<1>(results);
+
+        self->mail(x_ptr, y_ptr, n).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 1) {
+                    verify_copy_correctness(n, h_x, data);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 2 complete." << std::endl;
+    }
+
+    // Test 3: Routing control (device/stream) + mem_ptr
+    {
+        std::cout << "\n[INFO] Test 3: Testing copy_actor with specific device/stream and mem_ptr..." << std::endl;
+        int device_num = 0;
+        int stream_id = 42; 
+        command_runner<in<float>, out<float>> setup_runner;
+        auto results = setup_runner.transfer_memory(device_num, stream_id, create_in_arg(h_x), create_out_arg(h_y));
+        
+        self->mail(device_num, stream_id, std::get<0>(results), std::get<1>(results), n).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 1) {
+                    verify_copy_correctness(n, h_x, data);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 3 complete." << std::endl;
+    }
+
+    // Test 4: return_mem_ptr_atom (returning device handles)
+    {
+        std::cout << "\n[INFO] Test 4: Testing copy_actor with return_mem_ptr_atom..." << std::endl;
+        self->mail(return_mem_ptr_atom{}, x_arg, y_arg, n).send(blas_actor);
+
+        // We expect the data handles (mem_ptrs) back
+        self->receive(
+            [&](int reply_id, mem_ptr<float> x, mem_ptr<float> y) {
+                command_runner<float> runner;
+                // Since copy is async, we copy y back to host to verify
+                auto host_y = runner.copy_to_host(y);
+                verify_copy_correctness(n, h_x, host_y);
+            }
+        );
+        std::cout << "[INFO] Test 4 complete." << std::endl;
+    }
+
+    self->send_exit(blas_actor, exit_reason::user_shutdown);
+    manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda)

From c0d8b46485359cb741446caf318bc039abe3cacf Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 11:01:45 -0600
Subject: [PATCH 0681/1000] Intial commit.

---
 .../dot-actor-test/CMakeLists.txt             |  45 +++++++
 .../dot-actor-test/main.test.cpp              | 120 ++++++++++++++++++
 2 files changed, 165 insertions(+)
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/dot-actor-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/dot-actor-test/main.test.cpp

diff --git a/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..fa9dc8e759
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/CMakeLists.txt
@@ -0,0 +1,45 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+)
+
diff --git a/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/main.test.cpp
new file mode 100644
index 0000000000..95638815ab
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/main.test.cpp
@@ -0,0 +1,120 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include "caf/actorBLAS/dot-actor/dot-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+void verify_dot_correctness(float expected, const std::vector<float>& result) {
+    if (result.empty()) {
+        std::cout << "[ERROR] Result vector is empty." << std::endl;
+        return;
+    }
+
+    if (std::abs(result[0] - expected) > 1e-4) {
+        std::cout << "[ERROR] Dot product mismatch: "
+                  << "Expected " << expected << ", Got " << result[0] << std::endl;
+    } else {
+        std::cout << "[SUCCESS] Dot actor produced correct results." << std::endl;
+    }
+}
+
+void caf_main(actor_system& sys) {
+    manager_config config(true); 
+    manager::init(sys, config);
+
+    int n = 1024;
+    std::vector<float> h_x(n, 1.0f);
+    std::vector<float> h_y(n, 2.0f);
+    std::vector<float> h_res(1, 0.0f);
+
+    float expected = static_cast<float>(n) * 1.0f * 2.0f;
+
+    auto blas_actor = sys.spawn<dot_actor>(1);
+
+    auto x_arg = create_in_arg(h_x);
+    auto y_arg = create_in_arg(h_y);
+    auto res_arg = create_out_arg(h_res);
+
+    scoped_actor self{sys};
+
+    // Test 1: Standard host-buffer based call
+    {
+        std::cout << "[INFO] Test 1: Testing dot_actor with host-buffer arguments..." << std::endl;
+        self->mail(x_arg, y_arg, res_arg, n).send(blas_actor);
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 2) { // index 2 corresponds to res_arg
+                    verify_dot_correctness(expected, data);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 1 complete." << std::endl;
+    }
+
+    // Test 2: mem_ptr inputs
+    {
+        std::cout << "\n[INFO] Test 2: Testing dot_actor with mem_ptr inputs..." << std::endl;
+        command_runner<in<float>, in<float>, out<float>> setup_runner;
+        // Manually transfer data to the GPU to get mem_ptr handles
+        auto results = setup_runner.transfer_memory(0, 0, create_in_arg(h_x), create_in_arg(h_y), create_out_arg(h_res));
+        auto x_ptr = std::get<0>(results);
+        auto y_ptr = std::get<1>(results);
+        auto res_ptr = std::get<2>(results);
+
+        self->mail(x_ptr, y_ptr, res_ptr, n).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 2) {
+                    verify_dot_correctness(expected, data);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 2 complete." << std::endl;
+    }
+
+    // Test 3: Routing control (device/stream) + mem_ptr
+    {
+        std::cout << "\n[INFO] Test 3: Testing dot_actor with specific device/stream and mem_ptr..." << std::endl;
+        int device_num = 0;
+        int stream_id = 42; 
+        command_runner<in<float>, in<float>, out<float>> setup_runner;
+        auto results = setup_runner.transfer_memory(device_num, stream_id, create_in_arg(h_x), create_in_arg(h_y), create_out_arg(h_res));
+        
+        self->mail(device_num, stream_id, std::get<0>(results), std::get<1>(results), std::get<2>(results), n).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 2) {
+                    verify_dot_correctness(expected, data);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 3 complete." << std::endl;
+    }
+
+    // Test 4: return_mem_ptr_atom (returning device handles)
+    {
+        std::cout << "\n[INFO] Test 4: Testing dot_actor with return_mem_ptr_atom..." << std::endl;
+        self->mail(return_mem_ptr_atom{}, x_arg, y_arg, res_arg, n).send(blas_actor);
+
+        // We expect the data handles (mem_ptrs) back
+        self->receive(
+            [&](int reply_id, mem_ptr<float> x, mem_ptr<float> y, mem_ptr<float> res) {
+                command_runner<float> runner;
+                // Since dot is async, we copy res back to host to verify
+                auto host_res = runner.copy_to_host(res);
+                verify_dot_correctness(expected, host_res);
+            }
+        );
+        std::cout << "[INFO] Test 4 complete." << std::endl;
+    }
+
+    self->send_exit(blas_actor, exit_reason::user_shutdown);
+    manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda)

From f360b0d9beb927aa64f6ff318f59643dda8e9a77 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 11:12:05 -0600
Subject: [PATCH 0682/1000]  updated message handlers on copy actor.

---
 libcaf_cuda/caf/actorBLAS/copy-actor/copy-actor.hpp | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/actorBLAS/copy-actor/copy-actor.hpp b/libcaf_cuda/caf/actorBLAS/copy-actor/copy-actor.hpp
index 22ad1dc436..e9cd0531fd 100644
--- a/libcaf_cuda/caf/actorBLAS/copy-actor/copy-actor.hpp
+++ b/libcaf_cuda/caf/actorBLAS/copy-actor/copy-actor.hpp
@@ -46,8 +46,17 @@ class copy_actor : public event_based_actor {
       [this](int device_num, int stream_id, mem_ptr<float> x, mem_ptr<float> y, int n) {
         enqueue_scopy(device_num, stream_id, x, y, n, false);
       },
+      [this](return_mem_ptr_atom, in<float> x, out<float> y, int n) {
+        enqueue_scopy(-1, actor_id_, x, y, n, true);
+      },
+      [this](return_mem_ptr_atom, int device_num, int stream_id, in<float> x, out<float> y, int n) {
+        enqueue_scopy(device_num, stream_id, x, y, n, true);
+      },
       [this](return_mem_ptr_atom, mem_ptr<float> x, mem_ptr<float> y, int n) {
         enqueue_scopy(-1, actor_id_, x, y, n, true);
+      },
+      [this](return_mem_ptr_atom, int device_num, int stream_id, mem_ptr<float> x, mem_ptr<float> y, int n) {
+        enqueue_scopy(device_num, stream_id, x, y, n, true);
       }
     };
   }
@@ -78,7 +87,8 @@ class copy_actor : public event_based_actor {
     if (!sender) return;
     auto r_id = reply_id_;
     if (return_ptrs) {
-      caf::anon_mail(r_id, x_ptr, y_ptr).send(sender);
+      caf::anon_mail(r_id, 0, x_ptr).send(sender);
+      caf::anon_mail(r_id, 1, y_ptr).send(sender);
     } else {
       runner.copy_to_host_async(y_ptr, [sender, r_id](std::vector<float>&& data) {
         if (sender) {
@@ -93,4 +103,3 @@ class copy_actor : public event_based_actor {
 };
 
 } // namespace caf::cuda
-

From 2b6c6ea5d5e451105915f52c9d65d5d649d00317 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 11:23:58 -0600
Subject: [PATCH 0683/1000] updated mem_ref return message to be just 1
 messaage instead of two.

---
 libcaf_cuda/caf/actorBLAS/copy-actor/copy-actor.hpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/actorBLAS/copy-actor/copy-actor.hpp b/libcaf_cuda/caf/actorBLAS/copy-actor/copy-actor.hpp
index e9cd0531fd..e1a9e70ce8 100644
--- a/libcaf_cuda/caf/actorBLAS/copy-actor/copy-actor.hpp
+++ b/libcaf_cuda/caf/actorBLAS/copy-actor/copy-actor.hpp
@@ -87,8 +87,7 @@ class copy_actor : public event_based_actor {
     if (!sender) return;
     auto r_id = reply_id_;
     if (return_ptrs) {
-      caf::anon_mail(r_id, 0, x_ptr).send(sender);
-      caf::anon_mail(r_id, 1, y_ptr).send(sender);
+      caf::anon_mail(r_id, x_ptr, y_ptr).send(sender);
     } else {
       runner.copy_to_host_async(y_ptr, [sender, r_id](std::vector<float>&& data) {
         if (sender) {

From 7c7235441ad6d70bd9b7276ff1940502e5a5bb82 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 11:24:13 -0600
Subject: [PATCH 0684/1000] Updated tests to pass.

---
 .../tests/actorBLAS-test/copy-actor-test/main.test.cpp      | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/actorBLAS-test/copy-actor-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/copy-actor-test/main.test.cpp
index f6c0cc1952..f823203ac7 100644
--- a/libcaf_cuda/tests/actorBLAS-test/copy-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorBLAS-test/copy-actor-test/main.test.cpp
@@ -99,13 +99,15 @@ void caf_main(actor_system& sys) {
     // Test 4: return_mem_ptr_atom (returning device handles)
     {
         std::cout << "\n[INFO] Test 4: Testing copy_actor with return_mem_ptr_atom..." << std::endl;
-        self->mail(return_mem_ptr_atom{}, x_arg, y_arg, n).send(blas_actor);
+        int device_num = 0;
+        int stream_id = 42;
+        self->mail(return_mem_ptr_atom{}, device_num, stream_id, x_arg, y_arg, n).send(blas_actor);
 
         // We expect the data handles (mem_ptrs) back
         self->receive(
             [&](int reply_id, mem_ptr<float> x, mem_ptr<float> y) {
                 command_runner<float> runner;
-                // Since copy is async, we copy y back to host to verify
+                // Since the operation is async, copy the returned device pointer back to host to verify
                 auto host_y = runner.copy_to_host(y);
                 verify_copy_correctness(n, h_x, host_y);
             }

From cba8efdf359fdac0bc7a76d7ea76f159b250aa05 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 11:51:44 -0600
Subject: [PATCH 0685/1000] Dot actor now passes all tests.

---
 .../caf/actorBLAS/dot-actor/dot-actor.hpp     |  4 +++-
 .../dot-actor-test/main.test.cpp              | 20 +++++++------------
 2 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/libcaf_cuda/caf/actorBLAS/dot-actor/dot-actor.hpp b/libcaf_cuda/caf/actorBLAS/dot-actor/dot-actor.hpp
index dbac4a4aba..4ec03dbff0 100644
--- a/libcaf_cuda/caf/actorBLAS/dot-actor/dot-actor.hpp
+++ b/libcaf_cuda/caf/actorBLAS/dot-actor/dot-actor.hpp
@@ -48,6 +48,9 @@ class dot_actor : public event_based_actor {
       },
       [this](return_mem_ptr_atom, mem_ptr<float> x, mem_ptr<float> y, mem_ptr<float> res, int n) {
         enqueue_sdot(-1, actor_id_, x, y, res, n, true);
+      },
+      [this](return_mem_ptr_atom, in<float> x, in<float> y, out<float> res, int n) {
+        enqueue_sdot(-1, actor_id_, x, y, res, n, true);
       }
     };
   }
@@ -93,4 +96,3 @@ class dot_actor : public event_based_actor {
 };
 
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/main.test.cpp
index 95638815ab..34afcf2231 100644
--- a/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/main.test.cpp
@@ -45,10 +45,8 @@ void caf_main(actor_system& sys) {
         std::cout << "[INFO] Test 1: Testing dot_actor with host-buffer arguments..." << std::endl;
         self->mail(x_arg, y_arg, res_arg, n).send(blas_actor);
         self->receive(
-            [&](int reply_id, int arg_index, std::vector<float> data) {
-                if (arg_index == 2) { // index 2 corresponds to res_arg
-                    verify_dot_correctness(expected, data);
-                }
+            [&](int reply_id, float data) {
+                verify_dot_correctness(expected, {data});
             }
         );
         std::cout << "[INFO] Test 1 complete." << std::endl;
@@ -67,10 +65,8 @@ void caf_main(actor_system& sys) {
         self->mail(x_ptr, y_ptr, res_ptr, n).send(blas_actor);
 
         self->receive(
-            [&](int reply_id, int arg_index, std::vector<float> data) {
-                if (arg_index == 2) {
-                    verify_dot_correctness(expected, data);
-                }
+            [&](int reply_id, float data) {
+                verify_dot_correctness(expected, {data});
             }
         );
         std::cout << "[INFO] Test 2 complete." << std::endl;
@@ -87,10 +83,8 @@ void caf_main(actor_system& sys) {
         self->mail(device_num, stream_id, std::get<0>(results), std::get<1>(results), std::get<2>(results), n).send(blas_actor);
 
         self->receive(
-            [&](int reply_id, int arg_index, std::vector<float> data) {
-                if (arg_index == 2) {
-                    verify_dot_correctness(expected, data);
-                }
+            [&](int reply_id, float data) {
+                verify_dot_correctness(expected, {data});
             }
         );
         std::cout << "[INFO] Test 3 complete." << std::endl;
@@ -99,7 +93,7 @@ void caf_main(actor_system& sys) {
     // Test 4: return_mem_ptr_atom (returning device handles)
     {
         std::cout << "\n[INFO] Test 4: Testing dot_actor with return_mem_ptr_atom..." << std::endl;
-        self->mail(return_mem_ptr_atom{}, x_arg, y_arg, res_arg, n).send(blas_actor);
+        self->mail(return_mem_ptr_atom{}, x_arg, y_arg, res_arg, n).send(blas_actor); // This line will now work
 
         // We expect the data handles (mem_ptrs) back
         self->receive(

From 65a213fa51111b4f6dd1a79ca257d42aee0e5ca9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 12:06:44 -0600
Subject: [PATCH 0686/1000] Initial commit.

---
 .../CGS-actor/cg-actor.hpp                    | 177 ++++++++++++++++++
 .../CGS-actor/main.test.cpp                   |  57 ++++++
 2 files changed, 234 insertions(+)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
new file mode 100644
index 0000000000..1f17f9739a
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
@@ -0,0 +1,177 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <deque>
+#include <vector>
+#include "caf/cuda/all.hpp"
+#include "caf/actorBLAS/dot-actor/dot-actor.hpp"
+#include "caf/actorBLAS/gemv-actor/gemv-actor.hpp"
+#include "caf/actorBLAS/axpy-actor/axpy-actor.hpp"
+#include "caf/actorBLAS/copy-actor/copy-actor.hpp"
+
+namespace caf::cuda {
+
+struct cg_state {
+  // Problem data
+  mem_ptr<float> A, b, x;
+  int n;
+  float tol;
+  int max_iter;
+
+  // Workspace vectors
+  mem_ptr<float> r, p, w, y_tmp;
+  
+  // Scalars
+  float rho = 0.0f;
+  float cur_norm = 0.0f;
+  int iterations = 0;
+
+  // Fault Tolerance: Stagnation Detection
+  std::deque<float> norm_history;
+  const size_t window_size = 15;
+  const float stall_threshold = 0.001f; // 0.1%
+
+  // BLAS Actors
+  caf::actor dot_ptr, gemv_ptr, axpy_ptr, copy_ptr;
+
+  caf::actor requester;
+};
+
+class cg_actor : public stateful_actor<cg_state> {
+public:
+  cg_actor(actor_config& cfg, mem_ptr<float> A, mem_ptr<float> b, mem_ptr<float> x, 
+           int n, float tol, int max_iter)
+    : stateful_actor<cg_state>(cfg) {
+    state().A = A; state().b = b; state().x = x;
+    state().n = n; state().tol = tol; state().max_iter = max_iter;
+    
+    // Initialize workspace (assuming command_runner is used for allocation)
+    command_runner<out<float>> runner;
+    state().r = std::get<0>(runner.transfer_memory(0, 0, out<float>(n)));
+    state().p = std::get<0>(runner.transfer_memory(0, 0, out<float>(n)));
+    state().w = std::get<0>(runner.transfer_memory(0, 0, out<float>(n)));
+    state().y_tmp = std::get<0>(runner.transfer_memory(0, 0, out<float>(n)));
+
+    // Spawn helpers
+    state().dot_ptr = sys().spawn<dot_actor>();
+    state().gemv_ptr = sys().spawn<gemv_actor>();
+    state().axpy_ptr = sys().spawn<axpy_actor>();
+    state().copy_ptr = sys().spawn<copy_actor>();
+  }
+
+  behavior make_behavior() override {
+    return {
+      [this](atom_value start) {
+        state().requester = actor_cast<caf::actor>(this->current_sender());
+        initial_setup();
+      }
+    };
+  }
+
+private:
+  void initial_setup() {
+    // Start with x0 = 0, so r = b
+    request(state().copy_ptr, infinite, state().b, state().r, state().n).then([this](int, int, std::vector<float>) {
+      // Initial search direction p = r
+      request(state().copy_ptr, infinite, state().r, state().p, state().n).then([this](int, int, std::vector<float>) {
+        // rho = r^T * r
+        request(state().dot_ptr, infinite, state().r, state().r, state().y_tmp, state().n)
+          .then([this](int, float res) {
+            state().rho = res;
+            state().cur_norm = std::sqrt(res);
+            iterate();
+          });
+      });
+    });
+  }
+
+  void iterate() {
+    if (state().iterations >= state().max_iter || state().cur_norm < state().tol) {
+      if (state().requester) this->send(state().requester, state().x);
+      return;
+    }
+
+    // Step 1: w = Ap
+    request(state().gemv_ptr, infinite, state().A, state().p, state().w, state().n, state().n, 1.0f, 0.0f)
+      .then([this](int, int, std::vector<float>) {
+        // Step 2: alpha = rho / (p^T * w)
+        request(state().dot_ptr, infinite, state().p, state().w, state().y_tmp, state().n)
+          .then([this](int, float p_dot_w) {
+            float alpha = state().rho / p_dot_w;
+
+            // Step 3: x = x + alpha*p
+            request(state().axpy_ptr, infinite, state().p, state().x, state().n, alpha).then([this](int, int, std::vector<float>) {
+              // Step 4: r = r - alpha*w
+              request(state().axpy_ptr, infinite, state().w, state().r, state().n, -alpha).then([this](int, int, std::vector<float>) {
+                // Step 5: rho_new = r^T * r
+                request(state().dot_ptr, infinite, state().r, state().r, state().y_tmp, state().n)
+                  .then([this](int, float rho_new) {
+                    float old_rho = state().rho;
+                    state().rho = rho_new;
+                    state().cur_norm = std::sqrt(rho_new);
+                    check_for_stall(old_rho);
+                  });
+              });
+            });
+          });
+      });
+  }
+
+  void check_for_stall(float old_rho) {
+    state().iterations++;
+    state().norm_history.push_back(state().cur_norm);
+
+    if (state().norm_history.size() > state().window_size) {
+      float past_norm = state().norm_history.front();
+      state().norm_history.pop_front();
+
+      // Check: (Norm_old - Norm_new) / Norm_old < 0.1%
+      if ((past_norm - state().cur_norm) / past_norm < state().stall_threshold) {
+        this->println("[RECOVERY] Stall detected at iteration {}. Triggering Restart.", state().iterations);
+        perform_restart();
+        return;
+      }
+    }
+
+    // Normal Update: beta = rho_new / rho_old
+    float beta = state().rho / old_rho;
+
+    // p = r + beta * p
+    // Sequence: 1. w = r, 2. w = beta*p + w, 3. p = w
+    request(state().copy_ptr, infinite, state().r, state().w, state().n).then([this, beta](int, int, std::vector<float>) {
+      request(state().axpy_ptr, infinite, state().p, state().w, state().n, beta).then([this](int, int, std::vector<float>) {
+        request(state().copy_ptr, infinite, state().w, state().p, state().n).then([this](int, int, std::vector<float>) {
+          iterate();
+        });
+      });
+    });
+  }
+
+  void perform_restart() {
+    // Step 1: Recalculate y = M*x_k (purging drift)
+    request(state().gemv_ptr, infinite, state().A, state().x, state().y_tmp, state().n, state().n, 1.0f, 0.0f)
+      .then([this](int, int, std::vector<float>) {
+        // Step 2: r = b - y (Clean residual)
+        request(state().copy_ptr, infinite, state().b, state().r, state().n)
+          .then([this](int, int, std::vector<float>) {
+            request(state().axpy_ptr, infinite, state().y_tmp, state().r, state().n, -1.0f)
+              .then([this](int, int, std::vector<float>) {
+                // Step 3: p = r (Align search direction)
+                request(state().copy_ptr, infinite, state().r, state().p, state().n)
+                  .then([this](int, int, std::vector<float>) {
+                    // Step 4: Resume
+                    request(state().dot_ptr, infinite, state().r, state().r, state().y_tmp, state().n)
+                      .then([this](int, float new_rho) {
+                        state().rho = new_rho;
+                        state().cur_norm = std::sqrt(new_rho);
+                        state().norm_history.clear(); // Reset tracking
+                        iterate();
+                      });
+                  });
+              });
+          });
+      });
+  }
+};
+
+} // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp
new file mode 100644
index 0000000000..4731a475ce
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp
@@ -0,0 +1,57 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include <cmath>
+#include "cg-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+void caf_main(actor_system& sys) {
+  // Initialize the CUDA Manager
+  manager_config config(true); 
+  manager::init(sys, config);
+
+  int n = 3;
+  // Example: Solve Ax = b
+  // A = [4, 1, 0; 1, 3, 0; 0, 0, 2] (Symmetric Positive Definite)
+  std::vector<float> h_A = {4.0f, 1.0f, 0.0f, 
+                            1.0f, 3.0f, 0.0f, 
+                            0.0f, 0.0f, 2.0f};
+  // b = [1, 2, 0.5]
+  std::vector<float> h_b = {1.0f, 2.0f, 0.5f};
+  // Initial guess x0 = [0, 0, 0]
+  std::vector<float> h_x(n, 0.0f);
+
+  // Transfer initial problem data to device memory
+  command_runner<in<float>, in<float>, in_out<float>> setup_runner;
+  auto results = setup_runner.transfer_memory(0, 0, create_in_arg(h_A), create_in_arg(h_b), create_in_out_arg(h_x));
+  
+  auto d_A = std::get<0>(results);
+  auto d_b = std::get<1>(results);
+  auto d_x = std::get<2>(results);
+
+  // Spawn the Conjugate Gradient Actor
+  auto solver = sys.spawn<cg_actor>(d_A, d_b, d_x, n, 1e-6f, 100);
+
+  scoped_actor self{sys};
+  std::cout << "[INFO] Starting CG Solver..." << std::endl;
+  
+  // Call the actor to start solving
+  self->mail(atom("start")).send(solver);
+
+  // Wait for the final solution vector (mem_ptr)
+  self->receive(
+    [&](mem_ptr<float> result_x) {
+      auto host_x = result_x->copy_to_host();
+      std::cout << "[SUCCESS] Solver finished. Result x: ";
+      for (float val : host_x) std::cout << val << " ";
+      std::cout << std::endl;
+    }
+  );
+
+  manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda)
\ No newline at end of file

From 57a063726d4df1bb36e04cbd46c58ecd096f66c7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 14:00:36 -0600
Subject: [PATCH 0687/1000] got a working solver

---
 .../CGS-actor/cg-actor.hpp                    | 277 +++++++++++-------
 .../CGS-actor/main.test.cpp                   |   4 +-
 2 files changed, 176 insertions(+), 105 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
index 1f17f9739a..c935bedae0 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
@@ -9,8 +9,38 @@
 #include "caf/actorBLAS/axpy-actor/axpy-actor.hpp"
 #include "caf/actorBLAS/copy-actor/copy-actor.hpp"
 
+CAF_BEGIN_TYPE_ID_BLOCK(cg_solver, caf::first_custom_type_id + 50)
+  CAF_ADD_ATOM(cg_solver, start_atom)
+CAF_END_TYPE_ID_BLOCK(cg_solver)
+
 namespace caf::cuda {
 
+enum class cg_step {
+  idle,
+  init_r,
+  init_p,
+  init_rho,
+  main_gemv_w,
+  main_dot_pw,
+  main_axpy_x,
+  main_axpy_r,
+  main_dot_rr,
+  update_p_copy_r,
+  update_p_axpy_p,
+  update_p_final_copy,
+  restart_gemv_y,
+  restart_copy_b,
+  restart_axpy_r,
+  restart_copy_p,
+  restart_dot_rho
+};
+
+// Reply IDs used to distinguish which actor type is replying
+constexpr int id_dot = 100;
+constexpr int id_gemv = 200;
+constexpr int id_axpy = 300;
+constexpr int id_copy = 400;
+
 struct cg_state {
   // Problem data
   mem_ptr<float> A, b, x;
@@ -23,16 +53,19 @@ struct cg_state {
   
   // Scalars
   float rho = 0.0f;
+  float old_rho = 0.0f;
   float cur_norm = 0.0f;
+  float alpha = 0.0f;
+  float beta = 0.0f;
   int iterations = 0;
+  cg_step step = cg_step::idle;
 
   // Fault Tolerance: Stagnation Detection
-  std::deque<float> norm_history;
-  const size_t window_size = 15;
-  const float stall_threshold = 0.001f; // 0.1%
+  float last_norm = -1.0f;
+  int stagnation_count = 0;
 
   // BLAS Actors
-  caf::actor dot_ptr, gemv_ptr, axpy_ptr, copy_ptr;
+  caf::actor dot_actor, gemv_actor, axpy_actor, copy_actor;
 
   caf::actor requester;
 };
@@ -47,130 +80,168 @@ class cg_actor : public stateful_actor<cg_state> {
     
     // Initialize workspace (assuming command_runner is used for allocation)
     command_runner<out<float>> runner;
-    state().r = std::get<0>(runner.transfer_memory(0, 0, out<float>(n)));
-    state().p = std::get<0>(runner.transfer_memory(0, 0, out<float>(n)));
-    state().w = std::get<0>(runner.transfer_memory(0, 0, out<float>(n)));
-    state().y_tmp = std::get<0>(runner.transfer_memory(0, 0, out<float>(n)));
-
-    // Spawn helpers
-    state().dot_ptr = sys().spawn<dot_actor>();
-    state().gemv_ptr = sys().spawn<gemv_actor>();
-    state().axpy_ptr = sys().spawn<axpy_actor>();
-    state().copy_ptr = sys().spawn<copy_actor>();
+    state().r = runner.transfer_memory(0, 0, out<float>(n));
+    state().p = runner.transfer_memory(0, 0, out<float>(n));
+    state().w = runner.transfer_memory(0, 0, out<float>(n));
+    state().y_tmp = runner.transfer_memory(0, 0, out<float>(n));
+
+    // Spawn helpers with specific RIDs to distinguish messages
+    state().dot_actor = this->system().spawn<caf::cuda::dot_actor>(id_dot);
+    state().gemv_actor = this->system().spawn<caf::cuda::gemv_actor>(id_gemv);
+    state().axpy_actor = this->system().spawn<caf::cuda::axpy_actor>(id_axpy);
+    state().copy_actor = this->system().spawn<caf::cuda::copy_actor>(id_copy);
   }
 
   behavior make_behavior() override {
     return {
-      [this](atom_value start) {
+      [this](start_atom) {
         state().requester = actor_cast<caf::actor>(this->current_sender());
-        initial_setup();
+        start_setup();
+      },
+      // Dot product result (Host scalar)
+      [this](int rid, float val) {
+        if (rid != id_dot) return;
+        handle_dot_result(val);
+      },
+      // Matrix-Vector result (Memory handles)
+      [this](int rid, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y) {
+        if (rid != id_gemv) return;
+        handle_gemv_result();
+      },
+      // AXPY/Copy result (Memory handles)
+      [this](int rid, mem_ptr<float> x, mem_ptr<float> y) {
+        if (rid == id_axpy) handle_axpy_result();
+        else if (rid == id_copy) handle_copy_result();
       }
     };
   }
 
 private:
-  void initial_setup() {
-    // Start with x0 = 0, so r = b
-    request(state().copy_ptr, infinite, state().b, state().r, state().n).then([this](int, int, std::vector<float>) {
-      // Initial search direction p = r
-      request(state().copy_ptr, infinite, state().r, state().p, state().n).then([this](int, int, std::vector<float>) {
-        // rho = r^T * r
-        request(state().dot_ptr, infinite, state().r, state().r, state().y_tmp, state().n)
-          .then([this](int, float res) {
-            state().rho = res;
-            state().cur_norm = std::sqrt(res);
-            iterate();
-          });
-      });
-    });
+  void start_setup() {
+    state().step = cg_step::init_r;
+    this->mail(return_mem_ptr_atom_v, state().b, state().r, state().n).send(state().copy_actor);
   }
 
   void iterate() {
     if (state().iterations >= state().max_iter || state().cur_norm < state().tol) {
-      if (state().requester) this->send(state().requester, state().x);
+      if (state().requester) this->mail(state().x).send(state().requester);
+      state().step = cg_step::idle;
       return;
     }
+    state().step = cg_step::main_gemv_w;
+    this->mail(return_mem_ptr_atom_v, state().A, state().p, state().w, state().n, state().n).send(state().gemv_actor);
+  }
 
-    // Step 1: w = Ap
-    request(state().gemv_ptr, infinite, state().A, state().p, state().w, state().n, state().n, 1.0f, 0.0f)
-      .then([this](int, int, std::vector<float>) {
-        // Step 2: alpha = rho / (p^T * w)
-        request(state().dot_ptr, infinite, state().p, state().w, state().y_tmp, state().n)
-          .then([this](int, float p_dot_w) {
-            float alpha = state().rho / p_dot_w;
-
-            // Step 3: x = x + alpha*p
-            request(state().axpy_ptr, infinite, state().p, state().x, state().n, alpha).then([this](int, int, std::vector<float>) {
-              // Step 4: r = r - alpha*w
-              request(state().axpy_ptr, infinite, state().w, state().r, state().n, -alpha).then([this](int, int, std::vector<float>) {
-                // Step 5: rho_new = r^T * r
-                request(state().dot_ptr, infinite, state().r, state().r, state().y_tmp, state().n)
-                  .then([this](int, float rho_new) {
-                    float old_rho = state().rho;
-                    state().rho = rho_new;
-                    state().cur_norm = std::sqrt(rho_new);
-                    check_for_stall(old_rho);
-                  });
-              });
-            });
-          });
-      });
+  void perform_restart() {
+    this->println("[RECOVERY] Triggering Mathematical Restart at iteration {}.", state().iterations);
+    state().stagnation_count = 0;
+    state().step = cg_step::restart_gemv_y;
+    this->mail(return_mem_ptr_atom_v, state().A, state().x, state().y_tmp, state().n, state().n).send(state().gemv_actor);
   }
 
-  void check_for_stall(float old_rho) {
-    state().iterations++;
-    state().norm_history.push_back(state().cur_norm);
+  void handle_dot_result(float val) {
+    auto& s = state();
+    switch (s.step) {
+      case cg_step::init_rho:
+        s.rho = val;
+        s.cur_norm = std::sqrt(val);
+        iterate();
+        break;
+      case cg_step::main_dot_pw:
+        s.alpha = s.rho / val;
+        s.step = cg_step::main_axpy_x;
+        this->mail(return_mem_ptr_atom_v, s.p, s.x, s.n, s.alpha).send(s.axpy_actor);
+        break;
+      case cg_step::main_dot_rr:
+        s.old_rho = s.rho;
+        s.rho = val;
+        s.cur_norm = std::sqrt(val);
+        check_stagnation();
+        break;
+      case cg_step::restart_dot_rho:
+        s.rho = val;
+        s.cur_norm = std::sqrt(val);
+        iterate();
+        break;
+      default: break;
+    }
+  }
 
-    if (state().norm_history.size() > state().window_size) {
-      float past_norm = state().norm_history.front();
-      state().norm_history.pop_front();
+  void check_stagnation() {
+    auto& s = state();
+    s.iterations++;
+    if (s.last_norm > 0 && s.cur_norm > s.last_norm * 0.999f) {
+      s.stagnation_count++;
+    } else {
+      s.stagnation_count = 0;
+    }
+    s.last_norm = s.cur_norm;
+
+    if (s.stagnation_count >= 15) {
+      perform_restart();
+    } else {
+      s.beta = s.rho / s.old_rho;
+      s.step = cg_step::update_p_copy_r;
+      this->mail(return_mem_ptr_atom_v, s.r, s.w, s.n).send(s.copy_actor);
+    }
+  }
 
-      // Check: (Norm_old - Norm_new) / Norm_old < 0.1%
-      if ((past_norm - state().cur_norm) / past_norm < state().stall_threshold) {
-        this->println("[RECOVERY] Stall detected at iteration {}. Triggering Restart.", state().iterations);
-        perform_restart();
-        return;
-      }
+  void handle_gemv_result() {
+    auto& s = state();
+    if (s.step == cg_step::main_gemv_w) {
+      s.step = cg_step::main_dot_pw;
+      this->mail(s.p, s.w, s.y_tmp, s.n).send(s.dot_actor);
+    } else if (s.step == cg_step::restart_gemv_y) {
+      s.step = cg_step::restart_copy_b;
+      this->mail(return_mem_ptr_atom_v, s.b, s.r, s.n).send(s.copy_actor);
     }
+  }
 
-    // Normal Update: beta = rho_new / rho_old
-    float beta = state().rho / old_rho;
-
-    // p = r + beta * p
-    // Sequence: 1. w = r, 2. w = beta*p + w, 3. p = w
-    request(state().copy_ptr, infinite, state().r, state().w, state().n).then([this, beta](int, int, std::vector<float>) {
-      request(state().axpy_ptr, infinite, state().p, state().w, state().n, beta).then([this](int, int, std::vector<float>) {
-        request(state().copy_ptr, infinite, state().w, state().p, state().n).then([this](int, int, std::vector<float>) {
-          iterate();
-        });
-      });
-    });
+  void handle_axpy_result() {
+    auto& s = state();
+    if (s.step == cg_step::main_axpy_x) {
+      s.step = cg_step::main_axpy_r;
+      this->mail(return_mem_ptr_atom_v, s.w, s.r, s.n, -s.alpha).send(s.axpy_actor);
+    } else if (s.step == cg_step::main_axpy_r) {
+      s.step = cg_step::main_dot_rr;
+      this->mail(s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
+    } else if (s.step == cg_step::update_p_axpy_p) {
+      s.step = cg_step::update_p_final_copy;
+      this->mail(return_mem_ptr_atom_v, s.w, s.p, s.n).send(s.copy_actor);
+    } else if (s.step == cg_step::restart_axpy_r) {
+      s.step = cg_step::restart_copy_p;
+      this->mail(return_mem_ptr_atom_v, s.r, s.p, s.n).send(s.copy_actor);
+    }
   }
 
-  void perform_restart() {
-    // Step 1: Recalculate y = M*x_k (purging drift)
-    request(state().gemv_ptr, infinite, state().A, state().x, state().y_tmp, state().n, state().n, 1.0f, 0.0f)
-      .then([this](int, int, std::vector<float>) {
-        // Step 2: r = b - y (Clean residual)
-        request(state().copy_ptr, infinite, state().b, state().r, state().n)
-          .then([this](int, int, std::vector<float>) {
-            request(state().axpy_ptr, infinite, state().y_tmp, state().r, state().n, -1.0f)
-              .then([this](int, int, std::vector<float>) {
-                // Step 3: p = r (Align search direction)
-                request(state().copy_ptr, infinite, state().r, state().p, state().n)
-                  .then([this](int, int, std::vector<float>) {
-                    // Step 4: Resume
-                    request(state().dot_ptr, infinite, state().r, state().r, state().y_tmp, state().n)
-                      .then([this](int, float new_rho) {
-                        state().rho = new_rho;
-                        state().cur_norm = std::sqrt(new_rho);
-                        state().norm_history.clear(); // Reset tracking
-                        iterate();
-                      });
-                  });
-              });
-          });
-      });
+  void handle_copy_result() {
+    auto& s = state();
+    switch (s.step) {
+      case cg_step::init_r:
+        s.step = cg_step::init_p;
+        this->mail(return_mem_ptr_atom_v, s.r, s.p, s.n).send(s.copy_actor);
+        break;
+      case cg_step::init_p:
+        s.step = cg_step::init_rho;
+        this->mail(s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
+        break;
+      case cg_step::update_p_copy_r:
+        s.step = cg_step::update_p_axpy_p;
+        this->mail(return_mem_ptr_atom_v, s.p, s.w, s.n, s.beta).send(s.axpy_actor);
+        break;
+      case cg_step::update_p_final_copy:
+        iterate();
+        break;
+      case cg_step::restart_copy_b:
+        s.step = cg_step::restart_axpy_r;
+        this->mail(return_mem_ptr_atom_v, s.y_tmp, s.r, s.n, -1.0f).send(s.axpy_actor);
+        break;
+      case cg_step::restart_copy_p:
+        s.step = cg_step::restart_dot_rho;
+        this->mail(s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
+        break;
+      default: break;
+    }
   }
 };
 
diff --git a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp
index 4731a475ce..2f5f510f6e 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp
@@ -39,7 +39,7 @@ void caf_main(actor_system& sys) {
   std::cout << "[INFO] Starting CG Solver..." << std::endl;
   
   // Call the actor to start solving
-  self->mail(atom("start")).send(solver);
+  self->mail(start_atom{}).send(solver);
 
   // Wait for the final solution vector (mem_ptr)
   self->receive(
@@ -54,4 +54,4 @@ void caf_main(actor_system& sys) {
   manager::shutdown();
 }
 
-CAF_MAIN(id_block::cuda)
\ No newline at end of file
+CAF_MAIN(id_block::cuda, id_block::cg_solver)
\ No newline at end of file

From eecfafda41e84bda8cbbb83918701195096657f9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 14:28:04 -0600
Subject: [PATCH 0688/1000] Added stagnation recovery test for the solver.

---
 .../CGS-actor/cg-actor.hpp                    |  14 +-
 .../CGS-actor/main.test.cpp                   | 134 +++++++++++++++++-
 2 files changed, 142 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
index c935bedae0..7980dfd0c2 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
@@ -133,7 +133,7 @@ class cg_actor : public stateful_actor<cg_state> {
   }
 
   void perform_restart() {
-    this->println("[RECOVERY] Triggering Mathematical Restart at iteration {}.", state().iterations);
+    std::cout << "[RECOVERY] Triggering Mathematical Restart at iteration " << state().iterations << "\n";
     state().stagnation_count = 0;
     state().step = cg_step::restart_gemv_y;
     this->mail(return_mem_ptr_atom_v, state().A, state().x, state().y_tmp, state().n, state().n).send(state().gemv_actor);
@@ -169,15 +169,23 @@ class cg_actor : public stateful_actor<cg_state> {
 
   void check_stagnation() {
     auto& s = state();
+    // Debugging prints to observe stagnation behavior
+    // std::cout << "[CG_DEBUG] Iter: " << s.iterations 
+    //           << ", cur_norm: " << s.cur_norm 
+    //           << ", last_norm: " << s.last_norm << ", stagnation_count: " << s.stagnation_count << "\n";
+    
+    bool diverged = s.last_norm > 0 && s.cur_norm > s.last_norm * 1.5f;
+    bool stalled = s.last_norm > 0 && s.cur_norm > s.last_norm * 0.999f;
+
     s.iterations++;
-    if (s.last_norm > 0 && s.cur_norm > s.last_norm * 0.999f) {
+    if (stalled || diverged) {
       s.stagnation_count++;
     } else {
       s.stagnation_count = 0;
     }
     s.last_norm = s.cur_norm;
 
-    if (s.stagnation_count >= 15) {
+    if (s.stagnation_count >= 15 || diverged) {
       perform_restart();
     } else {
       s.beta = s.rho / s.old_rho;
diff --git a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp
index 2f5f510f6e..39828c76ef 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp
@@ -13,6 +13,13 @@ void caf_main(actor_system& sys) {
   manager_config config(true); 
   manager::init(sys, config);
 
+  scoped_actor self{sys};
+
+  // ===========================================================================
+  // TEST 1: Standard SPD Matrix (Should converge normally)
+  // ===========================================================================
+  std::cout << "[INFO] --- Starting Test 1: Standard SPD Matrix ---" << std::endl;
+
   int n = 3;
   // Example: Solve Ax = b
   // A = [4, 1, 0; 1, 3, 0; 0, 0, 2] (Symmetric Positive Definite)
@@ -32,10 +39,8 @@ void caf_main(actor_system& sys) {
   auto d_b = std::get<1>(results);
   auto d_x = std::get<2>(results);
 
-  // Spawn the Conjugate Gradient Actor
   auto solver = sys.spawn<cg_actor>(d_A, d_b, d_x, n, 1e-6f, 100);
 
-  scoped_actor self{sys};
   std::cout << "[INFO] Starting CG Solver..." << std::endl;
   
   // Call the actor to start solving
@@ -47,10 +52,133 @@ void caf_main(actor_system& sys) {
       auto host_x = result_x->copy_to_host();
       std::cout << "[SUCCESS] Solver finished. Result x: ";
       for (float val : host_x) std::cout << val << " ";
-      std::cout << std::endl;
+      std::cout << "\n" << std::endl;
     }
   );
 
+  // ===========================================================================
+  // TEST 2: Bad Input - Poorly Conditioned / Near-Singular Matrix
+  // This test uses a matrix with a very high condition number.
+  // The error reduction will be extremely slow, likely triggering the 
+  // stagnation detection logic (threshold of 0.1% decrease over 15 iterations)
+  // which will then trigger a Mathematical Restart [RECOVERY].
+  // ===========================================================================
+  std::cout << "[INFO] --- Starting Test 2: Bad Input (Poorly Conditioned) ---" << std::endl;
+
+  int n2 = 2;
+  // A = [1, 0; 0, 1e-7] -> Very high condition number
+  std::vector<float> h_A2 = {1.0f, 0.0f, 
+                             0.0f, 0.0000001f};
+  // b = [1, 1]
+  std::vector<float> h_b2 = {1.0f, 1.0f};
+  std::vector<float> h_x2(n2, 0.0f);
+
+  command_runner<in<float>, in<float>, in_out<float>> setup_runner2;
+  auto results2 = setup_runner2.transfer_memory(0, 0, create_in_arg(h_A2), create_in_arg(h_b2), create_in_out_arg(h_x2));
+  
+  auto d_A2 = std::get<0>(results2);
+  auto d_b2 = std::get<1>(results2);
+  auto d_x2 = std::get<2>(results2);
+
+  // Spawn with a high max_iter to allow time for stagnation detection to trigger
+  auto solver2 = sys.spawn<cg_actor>(d_A2, d_b2, d_x2, n2, 1e-8f, 200);
+
+  std::cout << "[INFO] Starting CG Solver with poor conditioning..." << std::endl;
+  self->mail(start_atom{}).send(solver2);
+
+  self->receive(
+    [&](mem_ptr<float> result_x) {
+      auto host_x = result_x->copy_to_host();
+      std::cout << "[SUCCESS] Solver finished Test 2. Result x: ";
+      for (float val : host_x) std::cout << val << " ";
+      std::cout << "\n[INFO] Test 2 complete. Check logs for [RECOVERY] messages." << std::endl;
+    },
+    // Increase timeout for the poorly conditioned case
+    after(std::chrono::seconds(20)) >> [] { std::cout << "[ERROR] Test 2 timed out!" << std::endl; }
+  );
+
+  // ===========================================================================
+  // TEST 3: Stagnation Recovery (Hilbert Matrix)
+  // Hilbert matrices are famously ill-conditioned. Even for small N, 
+  // the ratio of max/min eigenvalues is huge, causing slow convergence.
+  // This should trigger the stagnation detection logic (count >= 15)
+  // because progress will be extremely slow, leading to a [RECOVERY] message.
+  // ===========================================================================
+  std::cout << "\n[INFO] --- Starting Test 3: Stagnation Recovery (Hilbert) ---" << std::endl;
+
+  int n3 = 20; // Increased matrix size for more pronounced ill-conditioning
+  std::vector<float> h_A3(n3 * n3);
+  for (int i = 0; i < n3; ++i) {
+    for (int j = 0; j < n3; ++j) {
+      // Hilbert matrix element H_ij = 1 / (i + j + 1)
+      h_A3[i * n3 + j] = 1.0f / (float)(i + j + 1);
+    }
+  }
+  std::vector<float> h_b3(n3, 1.0f);
+  std::vector<float> h_x3(n3, 0.0f);
+
+  command_runner<in<float>, in<float>, in_out<float>> setup_runner3;
+  auto results3 = setup_runner3.transfer_memory(0, 0, create_in_arg(h_A3), create_in_arg(h_b3), create_in_out_arg(h_x3));
+  
+  auto d_A3 = std::get<0>(results3);
+  auto d_b3 = std::get<1>(results3);
+  auto d_x3 = std::get<2>(results3);
+
+  // Use a very high max_iter and tight tolerance to ensure we hit the 15-iteration stagnation threshold.
+  auto solver3 = sys.spawn<cg_actor>(d_A3, d_b3, d_x3, n3, 1e-10f, 1000); // Increased max_iter to allow more iterations for stagnation
+
+  std::cout << "[INFO] Starting CG Solver with Hilbert matrix..." << std::endl;
+  self->mail(start_atom{}).send(solver3);
+
+  self->receive(
+    [&](mem_ptr<float> result_x) {
+      auto host_x = result_x->copy_to_host();
+      std::cout << "[SUCCESS] Solver finished Test 3. Result x: ";
+      for (float val : host_x) std::cout << val << " ";
+      std::cout << "\n[INFO] Test 3 complete. Check logs for [RECOVERY] messages." << std::endl;
+    },
+    after(std::chrono::seconds(120)) >> [] { std::cout << "[ERROR] Test 3 timed out!" << std::endl; } // Increased timeout for longer execution
+  );
+
+  // ===========================================================================
+  // TEST 4: The "Narrow Valley" Matrix
+  // A = [1, 1; 1, 1.00001]
+  // This matrix is technically SPD, but the eigenvalues are roughly 2 and 0.000005.
+  // This creates a extremely narrow "canyon" in the error surface.
+  // Rounding errors will quickly make the residual drift from the true A*x - b.
+  // ===========================================================================
+  std::cout << "\n[INFO] --- Starting Test 4: Narrow Valley (Recovery Test) ---" << std::endl;
+
+  int n4 = 2;
+  float eps = 0.00001f;
+  std::vector<float> h_A4 = {1.0f, 1.0f, 
+                             1.0f, 1.0f + eps};
+  std::vector<float> h_b4 = {2.0f, 2.0f + eps}; // Solution is exactly [1, 1]
+  std::vector<float> h_x4(n4, 0.0f);
+
+  command_runner<in<float>, in<float>, in_out<float>> setup_runner4;
+  auto results4 = setup_runner4.transfer_memory(0, 0, create_in_arg(h_A4), create_in_arg(h_b4), create_in_out_arg(h_x4));
+  
+  auto d_A4 = std::get<0>(results4);
+  auto d_b4 = std::get<1>(results4);
+  auto d_x4 = std::get<2>(results4);
+
+  // Tight tolerance to force many iterations
+  auto solver4 = sys.spawn<cg_actor>(d_A4, d_b4, d_x4, n4, 1e-9f, 200);
+
+  std::cout << "[INFO] Starting CG Solver with Narrow Valley matrix..." << std::endl;
+  self->mail(start_atom{}).send(solver4);
+
+  self->receive(
+    [&](mem_ptr<float> result_x) {
+      auto host_x = result_x->copy_to_host();
+      std::cout << "[SUCCESS] Solver finished Test 4. Result x: ";
+      for (float val : host_x) std::cout << val << " ";
+      std::cout << "\n[INFO] Test 4 complete." << std::endl;
+    },
+    after(std::chrono::seconds(20)) >> [] { std::cout << "[ERROR] Test 4 timed out!" << std::endl; }
+  );
+
   manager::shutdown();
 }
 

From 42ddb77434dc6068243fe29bdc24f3f724ca74d5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 14:37:30 -0600
Subject: [PATCH 0689/1000] Implemented stress test

---
 .../CGS-actor/main.test.cpp                   | 45 +++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp
index 39828c76ef..99730be658 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp
@@ -179,6 +179,51 @@ void caf_main(actor_system& sys) {
     after(std::chrono::seconds(20)) >> [] { std::cout << "[ERROR] Test 4 timed out!" << std::endl; }
   );
 
+  // ===========================================================================
+  // TEST 5: Stress Test (Large Dense Matrix for Profiling)
+  // Matrix Size: 16384 x 16384 (268M elements, ~1 GB)
+  // This test is designed to saturate the GPU for a significant duration.
+  // Use this to observe utilization, thermal throttling, and SM occupancy.
+  // ===========================================================================
+  std::cout << "\n[INFO] --- Starting Test 5: Stress Test (16384x16384) ---" << std::endl;
+
+  int n5 = 16384;
+  // 1D Laplacian (Poisson) matrix: 2 on diagonal, -1 on sub-diagonals.
+  // Stored as a dense matrix to maximize GEMV computation.
+  std::cout << "[INFO] Constructing 1GB matrix on host (this may take a moment)..." << std::endl;
+  std::vector<float> h_A5(n5 * n5, 0.0f); 
+  for (int i = 0; i < n5; ++i) {
+    h_A5[i * n5 + i] = 2.0f;
+    if (i > 0) h_A5[i * n5 + (i - 1)] = -1.0f;
+    if (i < n5 - 1) h_A5[i * n5 + (i + 1)] = -1.0f;
+  }
+  std::vector<float> h_b5(n5, 1.0f);
+  std::vector<float> h_x5(n5, 0.0f);
+
+  std::cout << "[INFO] Transferring 1GB matrix to GPU..." << std::endl;
+  command_runner<in<float>, in<float>, in_out<float>> setup_runner5;
+  auto results5 = setup_runner5.transfer_memory(0, 0, create_in_arg(h_A5), create_in_arg(h_b5), create_in_out_arg(h_x5));
+  
+  auto d_A5 = std::get<0>(results5);
+  auto d_b5 = std::get<1>(results5);
+  auto d_x5 = std::get<2>(results5);
+  
+  // Run for 50,000 iterations with a near-zero tolerance to ensure sustained load.
+  auto solver5 = sys.spawn<cg_actor>(d_A5, d_b5, d_x5, n5, 1e-18f, 50000);
+
+  std::cout << "[INFO] Starting CG Solver stress test..." << std::endl;
+  auto start_time = std::chrono::high_resolution_clock::now();
+  self->mail(start_atom{}).send(solver5);
+
+  self->receive(
+    [&](mem_ptr<float> result_x) {
+      auto end_time = std::chrono::high_resolution_clock::now();
+      auto duration = std::chrono::duration_cast<std::chrono::seconds>(end_time - start_time);
+      std::cout << "[SUCCESS] Stress Test Finished in " << duration.count() << " seconds." << std::endl;
+    },
+    after(std::chrono::minutes(15)) >> [] { std::cout << "[ERROR] Test 5 timed out!" << std::endl; }
+  );
+
   manager::shutdown();
 }
 

From bdc2885842ff184ae9d0ca0fd2765a76d93704b4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 15:00:34 -0600
Subject: [PATCH 0690/1000] updated message handlers

---
 libcaf_cuda/caf/actorBLAS/dot-actor/dot-actor.hpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/libcaf_cuda/caf/actorBLAS/dot-actor/dot-actor.hpp b/libcaf_cuda/caf/actorBLAS/dot-actor/dot-actor.hpp
index 4ec03dbff0..63032edd78 100644
--- a/libcaf_cuda/caf/actorBLAS/dot-actor/dot-actor.hpp
+++ b/libcaf_cuda/caf/actorBLAS/dot-actor/dot-actor.hpp
@@ -51,6 +51,12 @@ class dot_actor : public event_based_actor {
       },
       [this](return_mem_ptr_atom, in<float> x, in<float> y, out<float> res, int n) {
         enqueue_sdot(-1, actor_id_, x, y, res, n, true);
+      },
+      [this](return_mem_ptr_atom,int device, int stream, in<float> x, in<float> y, out<float> res, int n) {
+        enqueue_sdot(device,stream, x, y, res, n, true);
+      },
+       [this](return_mem_ptr_atom,int device_num, int stream_id ,mem_ptr<float> x, mem_ptr<float> y, mem_ptr<float> res, int n) {
+        enqueue_sdot(device_num, stream_id, x, y, res, n, true);
       }
     };
   }

From 8c425e72d4542b9f26a079167b75322bfd387b52 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 21 May 2026 15:15:50 -0600
Subject: [PATCH 0691/1000] updated interfaces

---
 libcaf_cuda/caf/actorBLAS/axpy-actor/axpy-actor.hpp | 2 ++
 libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/libcaf_cuda/caf/actorBLAS/axpy-actor/axpy-actor.hpp b/libcaf_cuda/caf/actorBLAS/axpy-actor/axpy-actor.hpp
index 76023affcb..234e490d20 100644
--- a/libcaf_cuda/caf/actorBLAS/axpy-actor/axpy-actor.hpp
+++ b/libcaf_cuda/caf/actorBLAS/axpy-actor/axpy-actor.hpp
@@ -59,6 +59,8 @@ class axpy_actor : public event_based_actor {
       },
       [this](return_mem_ptr_atom, mem_ptr<float> x, mem_ptr<float> y, int n, float alpha) {
         enqueue_axpy(-1, actor_id_, x, y, n, alpha, true);
+      }, [this](return_mem_ptr_atom,int device_num, int stream_id,mem_ptr<float> x, mem_ptr<float> y, int n, float alpha) {
+        enqueue_axpy(device_num, stream_id, x, y, n, alpha, true);
       },
     };
   }
diff --git a/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp b/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp
index 2f30960a46..89eea219b3 100644
--- a/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp
+++ b/libcaf_cuda/caf/actorBLAS/gemv-actor/gemv-actor.hpp
@@ -81,6 +81,12 @@ class gemv_actor : public event_based_actor {
       [this](return_mem_ptr_atom, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n, float alpha, float beta) {
         enqueue_gemv(-1, actor_id_, A, x, y, m, n, alpha, beta, true);
       },
+      [this](return_mem_ptr_atom,int device_num, int stream_id, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n) {
+        enqueue_gemv(device_num, stream_id, A, x, y, m, n, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, int device_num, int stream_id, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y, int m, int n, float alpha, float beta) {
+        enqueue_gemv(device_num, stream_id, A, x, y, m, n, alpha, beta, true);
+      },
     };
   }
 

From 2c46bf8e74bf014f35ffdc6b72aa7e8dfdd8416f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 08:17:27 -0600
Subject: [PATCH 0692/1000] Updated actor to use device number and streams.

---
 .../CGS-actor/cg-actor.hpp                    | 61 +++++++++++--------
 1 file changed, 34 insertions(+), 27 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
index 7980dfd0c2..8722f7dbc3 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
@@ -47,6 +47,8 @@ struct cg_state {
   int n;
   float tol;
   int max_iter;
+  int device_num;
+  int stream_id;
 
   // Workspace vectors
   mem_ptr<float> r, p, w, y_tmp;
@@ -73,17 +75,18 @@ struct cg_state {
 class cg_actor : public stateful_actor<cg_state> {
 public:
   cg_actor(actor_config& cfg, mem_ptr<float> A, mem_ptr<float> b, mem_ptr<float> x, 
-           int n, float tol, int max_iter)
+           int n, float tol, int max_iter, int device_num, int stream_id)
     : stateful_actor<cg_state>(cfg) {
     state().A = A; state().b = b; state().x = x;
     state().n = n; state().tol = tol; state().max_iter = max_iter;
+    state().device_num = device_num; state().stream_id = stream_id;
     
     // Initialize workspace (assuming command_runner is used for allocation)
     command_runner<out<float>> runner;
-    state().r = runner.transfer_memory(0, 0, out<float>(n));
-    state().p = runner.transfer_memory(0, 0, out<float>(n));
-    state().w = runner.transfer_memory(0, 0, out<float>(n));
-    state().y_tmp = runner.transfer_memory(0, 0, out<float>(n));
+    state().r = runner.transfer_memory(device_num, stream_id, out<float>(n));
+    state().p = runner.transfer_memory(device_num, stream_id, out<float>(n));
+    state().w = runner.transfer_memory(device_num, stream_id, out<float>(n));
+    state().y_tmp = runner.transfer_memory(device_num, stream_id, out<float>(n));
 
     // Spawn helpers with specific RIDs to distinguish messages
     state().dot_actor = this->system().spawn<caf::cuda::dot_actor>(id_dot);
@@ -100,43 +103,47 @@ class cg_actor : public stateful_actor<cg_state> {
       },
       // Dot product result (Host scalar)
       [this](int rid, float val) {
-        if (rid != id_dot) return;
         handle_dot_result(val);
       },
       // Matrix-Vector result (Memory handles)
-      [this](int rid, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y) {
-        if (rid != id_gemv) return;
+      [this](int rid, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y) { // Assuming GEMV returns A, x, and y
         handle_gemv_result();
       },
-      // AXPY/Copy result (Memory handles)
+      // Copy result (Memory handles)
       [this](int rid, mem_ptr<float> x, mem_ptr<float> y) {
-        if (rid == id_axpy) handle_axpy_result();
-        else if (rid == id_copy) handle_copy_result();
+        handle_copy_result();
+      },
+      // AXPY result (Memory handles)
+      [this](int rid, mem_ptr<float> x, mem_ptr<float> y) { // Assuming AXPY returns x and y
+        handle_axpy_result();
       }
     };
   }
 
 private:
   void start_setup() {
+    auto& s = state();
     state().step = cg_step::init_r;
-    this->mail(return_mem_ptr_atom_v, state().b, state().r, state().n).send(state().copy_actor);
+    this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.b, s.r, s.n).send(s.copy_actor);
   }
 
   void iterate() {
+    auto& s = state();
     if (state().iterations >= state().max_iter || state().cur_norm < state().tol) {
       if (state().requester) this->mail(state().x).send(state().requester);
       state().step = cg_step::idle;
       return;
     }
     state().step = cg_step::main_gemv_w;
-    this->mail(return_mem_ptr_atom_v, state().A, state().p, state().w, state().n, state().n).send(state().gemv_actor);
+    this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.A, s.p, s.w, s.n, s.n).send(s.gemv_actor);
   }
 
   void perform_restart() {
+    auto& s = state();
     std::cout << "[RECOVERY] Triggering Mathematical Restart at iteration " << state().iterations << "\n";
     state().stagnation_count = 0;
     state().step = cg_step::restart_gemv_y;
-    this->mail(return_mem_ptr_atom_v, state().A, state().x, state().y_tmp, state().n, state().n).send(state().gemv_actor);
+    this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.A, s.x, s.y_tmp, s.n, s.n).send(s.gemv_actor);
   }
 
   void handle_dot_result(float val) {
@@ -150,7 +157,7 @@ class cg_actor : public stateful_actor<cg_state> {
       case cg_step::main_dot_pw:
         s.alpha = s.rho / val;
         s.step = cg_step::main_axpy_x;
-        this->mail(return_mem_ptr_atom_v, s.p, s.x, s.n, s.alpha).send(s.axpy_actor);
+        this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.p, s.x, s.n, s.alpha).send(s.axpy_actor);
         break;
       case cg_step::main_dot_rr:
         s.old_rho = s.rho;
@@ -190,7 +197,7 @@ class cg_actor : public stateful_actor<cg_state> {
     } else {
       s.beta = s.rho / s.old_rho;
       s.step = cg_step::update_p_copy_r;
-      this->mail(return_mem_ptr_atom_v, s.r, s.w, s.n).send(s.copy_actor);
+      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.r, s.w, s.n).send(s.copy_actor);
     }
   }
 
@@ -198,10 +205,10 @@ class cg_actor : public stateful_actor<cg_state> {
     auto& s = state();
     if (s.step == cg_step::main_gemv_w) {
       s.step = cg_step::main_dot_pw;
-      this->mail(s.p, s.w, s.y_tmp, s.n).send(s.dot_actor);
+      this->mail(s.device_num, s.stream_id, s.p, s.w, s.y_tmp, s.n).send(s.dot_actor);
     } else if (s.step == cg_step::restart_gemv_y) {
       s.step = cg_step::restart_copy_b;
-      this->mail(return_mem_ptr_atom_v, s.b, s.r, s.n).send(s.copy_actor);
+      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.b, s.r, s.n).send(s.copy_actor);
     }
   }
 
@@ -209,16 +216,16 @@ class cg_actor : public stateful_actor<cg_state> {
     auto& s = state();
     if (s.step == cg_step::main_axpy_x) {
       s.step = cg_step::main_axpy_r;
-      this->mail(return_mem_ptr_atom_v, s.w, s.r, s.n, -s.alpha).send(s.axpy_actor);
+      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.w, s.r, s.n, -s.alpha).send(s.axpy_actor);
     } else if (s.step == cg_step::main_axpy_r) {
       s.step = cg_step::main_dot_rr;
-      this->mail(s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
+      this->mail(s.device_num, s.stream_id, s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
     } else if (s.step == cg_step::update_p_axpy_p) {
       s.step = cg_step::update_p_final_copy;
-      this->mail(return_mem_ptr_atom_v, s.w, s.p, s.n).send(s.copy_actor);
+      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.w, s.p, s.n).send(s.copy_actor);
     } else if (s.step == cg_step::restart_axpy_r) {
       s.step = cg_step::restart_copy_p;
-      this->mail(return_mem_ptr_atom_v, s.r, s.p, s.n).send(s.copy_actor);
+      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.r, s.p, s.n).send(s.copy_actor);
     }
   }
 
@@ -227,26 +234,26 @@ class cg_actor : public stateful_actor<cg_state> {
     switch (s.step) {
       case cg_step::init_r:
         s.step = cg_step::init_p;
-        this->mail(return_mem_ptr_atom_v, s.r, s.p, s.n).send(s.copy_actor);
+        this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.r, s.p, s.n).send(s.copy_actor);
         break;
       case cg_step::init_p:
         s.step = cg_step::init_rho;
-        this->mail(s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
+        this->mail(s.device_num, s.stream_id, s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
         break;
       case cg_step::update_p_copy_r:
         s.step = cg_step::update_p_axpy_p;
-        this->mail(return_mem_ptr_atom_v, s.p, s.w, s.n, s.beta).send(s.axpy_actor);
+        this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.p, s.w, s.n, s.beta).send(s.axpy_actor);
         break;
       case cg_step::update_p_final_copy:
         iterate();
         break;
       case cg_step::restart_copy_b:
         s.step = cg_step::restart_axpy_r;
-        this->mail(return_mem_ptr_atom_v, s.y_tmp, s.r, s.n, -1.0f).send(s.axpy_actor);
+        this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.y_tmp, s.r, s.n, -1.0f).send(s.axpy_actor);
         break;
       case cg_step::restart_copy_p:
         s.step = cg_step::restart_dot_rho;
-        this->mail(s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
+        this->mail(s.device_num, s.stream_id, s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
         break;
       default: break;
     }

From 2b316b4a9b5fa215812568d021b396c333c0daf5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 08:24:31 -0600
Subject: [PATCH 0693/1000] updated message handlers to prevent issue where the
 actor would hang

---
 .../CGS-actor/cg-actor.hpp                    | 20 +++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
index 8722f7dbc3..4fec81110a 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/cg-actor.hpp
@@ -103,19 +103,23 @@ class cg_actor : public stateful_actor<cg_state> {
       },
       // Dot product result (Host scalar)
       [this](int rid, float val) {
-        handle_dot_result(val);
+        if (rid == id_dot) {
+          handle_dot_result(val);
+        }
       },
       // Matrix-Vector result (Memory handles)
       [this](int rid, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y) { // Assuming GEMV returns A, x, and y
-        handle_gemv_result();
+        if (rid == id_gemv) {
+          handle_gemv_result();
+        }
       },
-      // Copy result (Memory handles)
+      // Copy or AXPY result (Memory handles share the same signature)
       [this](int rid, mem_ptr<float> x, mem_ptr<float> y) {
-        handle_copy_result();
-      },
-      // AXPY result (Memory handles)
-      [this](int rid, mem_ptr<float> x, mem_ptr<float> y) { // Assuming AXPY returns x and y
-        handle_axpy_result();
+        if (rid == id_copy) {
+          handle_copy_result();
+        } else if (rid == id_axpy) {
+          handle_axpy_result();
+        }
       }
     };
   }

From a973d77c54ab775f87164fce77e827dc3c3f4b9b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 08:45:10 -0600
Subject: [PATCH 0694/1000] Added another stress test.

---
 .../CGS-actor/main.test.cpp                   | 49 +++++++++++++++++--
 1 file changed, 44 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp
index 99730be658..953856203b 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/CGS-actor/main.test.cpp
@@ -39,7 +39,7 @@ void caf_main(actor_system& sys) {
   auto d_b = std::get<1>(results);
   auto d_x = std::get<2>(results);
 
-  auto solver = sys.spawn<cg_actor>(d_A, d_b, d_x, n, 1e-6f, 100);
+  auto solver = sys.spawn<cg_actor>(d_A, d_b, d_x, n, 1e-6f, 100, 0, 0);
 
   std::cout << "[INFO] Starting CG Solver..." << std::endl;
   
@@ -81,7 +81,7 @@ void caf_main(actor_system& sys) {
   auto d_x2 = std::get<2>(results2);
 
   // Spawn with a high max_iter to allow time for stagnation detection to trigger
-  auto solver2 = sys.spawn<cg_actor>(d_A2, d_b2, d_x2, n2, 1e-8f, 200);
+  auto solver2 = sys.spawn<cg_actor>(d_A2, d_b2, d_x2, n2, 1e-8f, 200, 0, 0);
 
   std::cout << "[INFO] Starting CG Solver with poor conditioning..." << std::endl;
   self->mail(start_atom{}).send(solver2);
@@ -125,7 +125,7 @@ void caf_main(actor_system& sys) {
   auto d_x3 = std::get<2>(results3);
 
   // Use a very high max_iter and tight tolerance to ensure we hit the 15-iteration stagnation threshold.
-  auto solver3 = sys.spawn<cg_actor>(d_A3, d_b3, d_x3, n3, 1e-10f, 1000); // Increased max_iter to allow more iterations for stagnation
+  auto solver3 = sys.spawn<cg_actor>(d_A3, d_b3, d_x3, n3, 1e-10f, 1000, 0, 0); // Increased max_iter to allow more iterations for stagnation
 
   std::cout << "[INFO] Starting CG Solver with Hilbert matrix..." << std::endl;
   self->mail(start_atom{}).send(solver3);
@@ -164,7 +164,7 @@ void caf_main(actor_system& sys) {
   auto d_x4 = std::get<2>(results4);
 
   // Tight tolerance to force many iterations
-  auto solver4 = sys.spawn<cg_actor>(d_A4, d_b4, d_x4, n4, 1e-9f, 200);
+  auto solver4 = sys.spawn<cg_actor>(d_A4, d_b4, d_x4, n4, 1e-9f, 200, 0, 0);
 
   std::cout << "[INFO] Starting CG Solver with Narrow Valley matrix..." << std::endl;
   self->mail(start_atom{}).send(solver4);
@@ -209,7 +209,7 @@ void caf_main(actor_system& sys) {
   auto d_x5 = std::get<2>(results5);
   
   // Run for 50,000 iterations with a near-zero tolerance to ensure sustained load.
-  auto solver5 = sys.spawn<cg_actor>(d_A5, d_b5, d_x5, n5, 1e-18f, 50000);
+  auto solver5 = sys.spawn<cg_actor>(d_A5, d_b5, d_x5, n5, 1e-18f, 50000, 0, 0);
 
   std::cout << "[INFO] Starting CG Solver stress test..." << std::endl;
   auto start_time = std::chrono::high_resolution_clock::now();
@@ -224,6 +224,45 @@ void caf_main(actor_system& sys) {
     after(std::chrono::minutes(15)) >> [] { std::cout << "[ERROR] Test 5 timed out!" << std::endl; }
   );
 
+  // ===========================================================================
+  // TEST 6: Concurrent Stress Test (2 Actors, Same Device, Different Streams)
+  // Both actors run the same 16384x16384 problem simultaneously.
+  // This tests the framework's ability to handle high-load concurrency.
+  // ===========================================================================
+  std::cout << "\n[INFO] --- Starting Test 6: Concurrent Stress Test (2 Actors, Same Device, Diff Streams) ---" << std::endl;
+
+  // Setup independent device vectors for Solver A (Stream 0) reusing host data from Test 5
+  command_runner<in<float>, in_out<float>> vec_runner;
+  auto res6a = vec_runner.transfer_memory(0, 0, create_in_arg(h_b5), create_in_out_arg(h_x5));
+  auto d_b6a = std::get<0>(res6a);
+  auto d_x6a = std::get<1>(res6a);
+
+  // Setup independent device vectors for Solver B (Stream 1) reusing host data from Test 5
+  auto res6b = vec_runner.transfer_memory(0, 1, create_in_arg(h_b5), create_in_out_arg(h_x5));
+  auto d_b6b = std::get<0>(res6b);
+  auto d_x6b = std::get<1>(res6b);
+
+  auto solver6a = sys.spawn<cg_actor>(d_A5, d_b6a, d_x6a, n5, 1e-18f, 50000, 0, 0);
+  auto solver6b = sys.spawn<cg_actor>(d_A5, d_b6b, d_x6b, n5, 1e-18f, 50000, 0, 1);
+
+  std::cout << "[INFO] Launching both solvers concurrently..." << std::endl;
+  auto start6 = std::chrono::high_resolution_clock::now();
+  self->mail(start_atom{}).send(solver6a);
+  self->mail(start_atom{}).send(solver6b);
+
+  // Wait for both solvers to complete
+  for (int i = 0; i < 2; ++i) {
+    self->receive(
+      [&](mem_ptr<float>) {
+        std::cout << "[INFO] A solver in Test 6 has completed." << std::endl;
+      },
+      after(std::chrono::minutes(20)) >> [] { std::cout << "[ERROR] A solver in Test 6 timed out!" << std::endl; }
+    );
+  }
+  auto end6 = std::chrono::high_resolution_clock::now();
+  auto total_dur = std::chrono::duration_cast<std::chrono::seconds>(end6 - start6);
+  std::cout << "[SUCCESS] Concurrent Stress Test Finished. Total wall-clock time: " << total_dur.count() << " seconds." << std::endl;
+
   manager::shutdown();
 }
 

From b6ccf42976b14ac9828f2984e7073ab6ec913ddc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 09:32:01 -0600
Subject: [PATCH 0695/1000] Implemented functionality required for gemm actor

---
 .../caf/actorBLAS/gemm-actor/gemm-actor.hpp   | 177 ++++++++++++++++++
 libcaf_cuda/caf/cuda/device.hpp               |  88 +++++++--
 2 files changed, 251 insertions(+), 14 deletions(-)
 create mode 100644 libcaf_cuda/caf/actorBLAS/gemm-actor/gemm-actor.hpp

diff --git a/libcaf_cuda/caf/actorBLAS/gemm-actor/gemm-actor.hpp b/libcaf_cuda/caf/actorBLAS/gemm-actor/gemm-actor.hpp
new file mode 100644
index 0000000000..798e1f2703
--- /dev/null
+++ b/libcaf_cuda/caf/actorBLAS/gemm-actor/gemm-actor.hpp
@@ -0,0 +1,177 @@
+#pragma once
+
+#include <caf/actor.hpp>
+#include <caf/actor_cast.hpp>
+#include <caf/actor_config.hpp>
+#include <caf/actor_system.hpp>
+#include <caf/anon_mail.hpp>
+#include <caf/event_based_actor.hpp>
+
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/mem_ref.hpp"
+#include "caf/cuda/command_runner.hpp"
+#include "caf/cuda/platform.hpp"
+#include "caf/cuda/types.hpp"
+
+namespace caf::cuda {
+
+/// GEMM Actor for matrix-matrix multiplication.
+/// Message Signature: (in<T> A, in<T> B, out<T> C, int m, int n, int k, [T alpha, T beta])
+template <typename T>
+class gemm_actor : public event_based_actor {
+public:
+  static caf::actor spawn(caf::actor_system& sys, int reply_id = 0) {
+    return sys.spawn<gemm_actor<T>>(reply_id);
+  }
+
+  gemm_actor(caf::actor_config& cfg, int reply_id = 0)
+    : event_based_actor(cfg), reply_id_(reply_id) {
+    actor_id_ = static_cast<int>(this->id());
+  }
+
+  ~gemm_actor() override {
+    command_runner<> runner;
+    runner.release_stream_for_actor(actor_id_);
+  }
+
+  caf::behavior make_behavior() override {
+    return {
+      // Standard host buffer based calls (beta = 0.0)
+      [this](in<T> A, in<T> B, out<T> C, int m, int n, int k) {
+        enqueue_gemm(-1, actor_id_, A, B, C, m, n, k, static_cast<T>(1.0), static_cast<T>(0.0), false);
+      },
+      // Standard host buffer based calls (explicit alpha, beta)
+      [this](in<T> A, in<T> B, in_out<T> C, int m, int n, int k, T alpha, T beta) {
+        enqueue_gemm(-1, actor_id_, A, B, C, m, n, k, alpha, beta, false);
+      },
+      // Routing control overloads (beta = 0.0)
+      [this](int device_num, int stream_id, in<T> A, in<T> B, out<T> C, int m, int n, int k) {
+        enqueue_gemm(device_num, stream_id, A, B, C, m, n, k, static_cast<T>(1.0), static_cast<T>(0.0), false);
+      },
+      // Routing control overloads (explicit alpha, beta)
+      [this](int device_num, int stream_id, in<T> A, in<T> B, in_out<T> C, int m, int n, int k, T alpha, T beta) {
+        enqueue_gemm(device_num, stream_id, A, B, C, m, n, k, alpha, beta, false);
+      },
+      // mem_ptr based calls (useful for pipelines, beta = 0.0)
+      [this](mem_ptr<T> A, mem_ptr<T> B, mem_ptr<T> C, int m, int n, int k) {
+        enqueue_gemm(-1, actor_id_, A, B, C, m, n, k, static_cast<T>(1.0), static_cast<T>(0.0), false);
+      },
+      // mem_ptr based calls (explicit alpha, beta)
+      [this](mem_ptr<T> A, mem_ptr<T> B, mem_ptr<T> C, int m, int n, int k, T alpha, T beta) {
+        enqueue_gemm(-1, actor_id_, A, B, C, m, n, k, alpha, beta, false);
+      },
+      // mem_ptr based calls with routing (beta = 0.0)
+      [this](int device_num, int stream_id, mem_ptr<T> A, mem_ptr<T> B, mem_ptr<T> C, int m, int n, int k) {
+        enqueue_gemm(device_num, stream_id, A, B, C, m, n, k, static_cast<T>(1.0), static_cast<T>(0.0), false);
+      },
+      // mem_ptr based calls with routing (explicit alpha, beta)
+      [this](int device_num, int stream_id, mem_ptr<T> A, mem_ptr<T> B, mem_ptr<T> C, int m, int n, int k, T alpha, T beta) {
+        enqueue_gemm(device_num, stream_id, A, B, C, m, n, k, alpha, beta, false);
+      },
+      // Mem ptr return overloads (beta = 0.0)
+      [this](return_mem_ptr_atom, in<T> A, in<T> B, out<T> C, int m, int n, int k) {
+        enqueue_gemm(-1, actor_id_, A, B, C, m, n, k, static_cast<T>(1.0), static_cast<T>(0.0), true);
+      },
+      // Mem ptr return overloads (explicit alpha, beta)
+      [this](return_mem_ptr_atom, in<T> A, in<T> B, in_out<T> C, int m, int n, int k, T alpha, T beta) {
+        enqueue_gemm(-1, actor_id_, A, B, C, m, n, k, alpha, beta, true);
+      },
+      // Mem ptr return overloads with routing (beta = 0.0)
+      [this](return_mem_ptr_atom, int device_num, int stream_id, in<T> A, in<T> B, out<T> C, int m, int n, int k) {
+        enqueue_gemm(device_num, stream_id, A, B, C, m, n, k, static_cast<T>(1.0), static_cast<T>(0.0), true);
+      },
+      // Mem ptr return overloads with routing (explicit alpha, beta)
+      [this](return_mem_ptr_atom, int device_num, int stream_id, in<T> A, in<T> B, in_out<T> C, int m, int n, int k, T alpha, T beta) {
+        enqueue_gemm(device_num, stream_id, A, B, C, m, n, k, alpha, beta, true);
+      },
+      // Mem ptr return overloads for already-existing Device buffers (beta = 0.0)
+      [this](return_mem_ptr_atom, mem_ptr<T> A, mem_ptr<T> B, mem_ptr<T> C, int m, int n, int k) {
+        enqueue_gemm(-1, actor_id_, A, B, C, m, n, k, static_cast<T>(1.0), static_cast<T>(0.0), true);
+      },
+      // Mem ptr return overloads for already-existing Device buffers (explicit alpha, beta)
+      [this](return_mem_ptr_atom, mem_ptr<T> A, mem_ptr<T> B, mem_ptr<T> C, int m, int n, int k, T alpha, T beta) {
+        enqueue_gemm(-1, actor_id_, A, B, C, m, n, k, alpha, beta, true);
+      },
+      // Mem ptr return overloads for already-existing Device buffers with routing (beta = 0.0)
+      [this](return_mem_ptr_atom, int device_num, int stream_id, mem_ptr<T> A, mem_ptr<T> B, mem_ptr<T> C, int m, int n, int k) {
+        enqueue_gemm(device_num, stream_id, A, B, C, m, n, k, static_cast<T>(1.0), static_cast<T>(0.0), true);
+      },
+      // Mem ptr return overloads for already-existing Device buffers with routing (explicit alpha, beta)
+      [this](return_mem_ptr_atom, int device_num, int stream_id, mem_ptr<T> A, mem_ptr<T> B, mem_ptr<T> C, int m, int n, int k, T alpha, T beta) {
+        enqueue_gemm(device_num, stream_id, A, B, C, m, n, k, alpha, beta, true);
+      },
+    };
+  }
+
+private:
+  // Overload for Host-wrapped buffers
+  template <typename OutType>
+  void enqueue_gemm(int device_num, int stream_id,
+                    in<T> A_arg, in<T> B_arg, OutType C_arg,
+                    int m, int n, int k, T alpha, T beta, bool return_ptrs) {
+    command_runner<in<T>, in<T>, OutType> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, A_arg, B_arg, C_arg);
+    execute_and_reply(device_num, stream_id, std::get<0>(results),
+                      std::get<1>(results), std::get<2>(results),
+                      m, n, k, alpha, beta, return_ptrs);
+  }
+
+  // Overload for already-existing Device buffers
+  void enqueue_gemm(int device_num, int stream_id,
+                    mem_ptr<T> A_ptr, mem_ptr<T> B_ptr, mem_ptr<T> C_ptr,
+                    int m, int n, int k, T alpha, T beta, bool return_ptrs) {
+    command_runner<mem_ptr<T>, mem_ptr<T>, mem_ptr<T>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, A_ptr, B_ptr, C_ptr);
+    execute_and_reply(device_num, stream_id, std::get<0>(results),
+                      std::get<1>(results), std::get<2>(results),
+                      m, n, k, alpha, beta, return_ptrs);
+  }
+
+  void execute_and_reply(int device_num, int stream_id,
+                         mem_ptr<T> A, mem_ptr<T> B, mem_ptr<T> C,
+                         int m, int n, int k, T alpha, T beta, bool return_ptrs) {
+    auto plat = platform::create();
+    device_ptr dev;
+    if (device_num == -1)
+      dev = plat->schedule(stream_id);
+    else
+      dev = plat->schedule(stream_id, device_num);
+
+    // Dispatch based on template type T
+    if constexpr (std::is_same_v<T, float>) {
+      dev->sgemm(stream_id, m, n, k, alpha, A, B, beta, C);
+    } else if constexpr (std::is_same_v<T, double>) {
+      dev->dgemm(stream_id, m, n, k, alpha, A, B, beta, C);
+    } else {
+      static_assert(std::is_same_v<T, float> || std::is_same_v<T, double>, 
+                    "Unsupported type for GEMM actor. Only float and double are supported.");
+    }
+
+    handle_reply(device_num, stream_id, A, B, C, return_ptrs);
+  }
+
+  void handle_reply(int, int, mem_ptr<T> A_ptr, mem_ptr<T> B_ptr, mem_ptr<T> C_ptr, bool return_ptrs) {
+    command_runner<mem_ptr<T>> runner;
+    auto sender = actor_cast<actor>(this->current_sender());
+    if (!sender) return;
+
+    auto r_id = reply_id_;
+
+    if (return_ptrs) {
+      // Send all pointers in a single message
+      caf::anon_mail(r_id, A_ptr, B_ptr, C_ptr).send(sender);
+    } else {
+      // Send result data in a single message
+      runner.copy_to_host_async(C_ptr, [sender, r_id](std::vector<T>&& data) {
+        if (sender) {
+          caf::anon_mail(r_id, 2, std::move(data)).send(sender);
+        }
+      });
+    }
+  }
+
+  int actor_id_;
+  int reply_id_;
+};
+
+} // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 1b9be6cf19..76d61027c5 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -128,16 +128,16 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Returns the cuBLAS handle associated with the actor id.
-  cublasHandle_t get_cublas_handle(int actor_id) {
+  cublasHandle_t get_cublas_handle(int stream_id) {
     if (!cublas_table_) return nullptr;
-    return cublas_table_->get_handle(actor_id, get_stream_for_actor(actor_id));
+    return cublas_table_->get_handle(stream_id, get_stream_for_actor(stream_id));
   }
 
   /// Performs single precision matrix-vector multiplication (y = alpha*A*x + beta*y).
   /// Assumes A is in row-major order of dimensions m x n.
-  void sgemv(int actor_id, int m, int n, float alpha, mem_ptr<float> A,
+  void sgemv(int stream_id, int m, int n, float alpha, mem_ptr<float> A,
              mem_ptr<float> x, float beta, mem_ptr<float> y) {
-    cublasHandle_t handle = get_cublas_handle(actor_id);
+    cublasHandle_t handle = get_cublas_handle(stream_id);
     if (!handle)
       throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
 
@@ -163,9 +163,9 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
 
   /// Performs symmetric rank-k update (C = alpha*A*A^T + beta*C).
   /// Assumes A is in row-major order of dimensions n x k, and C is n x n.
-  void ssyrk(int actor_id, int n, int k, float alpha, mem_ptr<float> A,
+  void ssyrk(int stream_id, int n, int k, float alpha, mem_ptr<float> A,
              float beta, mem_ptr<float> C) {
-    cublasHandle_t handle = get_cublas_handle(actor_id);
+    cublasHandle_t handle = get_cublas_handle(stream_id);
     if (!handle)
       throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
 
@@ -189,8 +189,8 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Performs single precision vector-vector addition (y = alpha*x + y).
-  void saxpy(int actor_id, int n, float alpha, mem_ptr<float> x, mem_ptr<float> y) {
-    cublasHandle_t handle = get_cublas_handle(actor_id);
+  void saxpy(int stream_id, int n, float alpha, mem_ptr<float> x, mem_ptr<float> y) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
     if (!handle)
       throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
 
@@ -206,8 +206,8 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Performs single precision Euclidean norm (result = ||x||2).
-  void snrm2(int actor_id, int n, mem_ptr<float> x, mem_ptr<float> result) {
-    cublasHandle_t handle = get_cublas_handle(actor_id);
+  void snrm2(int stream_id, int n, mem_ptr<float> x, mem_ptr<float> result) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
     if (!handle)
       throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
 
@@ -225,8 +225,8 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Performs single precision dot product (result = x^T * y).
-  void sdot(int actor_id, int n, mem_ptr<float> x, mem_ptr<float> y, mem_ptr<float> result) {
-    cublasHandle_t handle = get_cublas_handle(actor_id);
+  void sdot(int stream_id, int n, mem_ptr<float> x, mem_ptr<float> y, mem_ptr<float> result) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
     if (!handle)
       throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
 
@@ -245,8 +245,8 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Copies vector x to vector y (y = x).
-  void scopy(int actor_id, int n, mem_ptr<float> x, mem_ptr<float> y) {
-    cublasHandle_t handle = get_cublas_handle(actor_id);
+  void scopy(int stream_id, int n, mem_ptr<float> x, mem_ptr<float> y) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
     if (!handle)
       throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
 
@@ -261,6 +261,66 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasScopy failed on device " + std::to_string(id_));
   }
 
+  /// Performs single precision matrix-matrix multiplication (C = alpha*A*B + beta*C).
+  /// Assumes A is m x k, B is k x n, and C is m x n, all in row-major order.
+  void sgemm(int stream_id, int m, int n, int k, float alpha, mem_ptr<float> A,
+             mem_ptr<float> B, float beta, mem_ptr<float> C) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+
+    // For row-major matrices A(m,k), B(k,n), C(m,n) to compute C = alpha*A*B + beta*C
+    // cuBLAS expects column-major. The equivalent column-major operation is:
+    // C_col = alpha * B_col * A_col + beta * C_col
+    // where X_col = X_row^T.
+    // So, we call cublasSgemm with:
+    // A_cublas = B (transposed), B_cublas = A (transposed)
+    // Dimensions: m_cublas = n, n_cublas = m, k_cublas = k
+    cublasStatus_t status = cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_T,
+                                        n, m, k,
+                                        &alpha,
+                                        reinterpret_cast<const float*>(B->mem()), n, // B is k x n row-major, lda = n
+                                        reinterpret_cast<const float*>(A->mem()), k, // A is m x k row-major, ldb = k
+                                        &beta,
+                                        reinterpret_cast<float*>(C->mem()), n); // C is m x n row-major, ldc = n
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasSgemm failed on device " + std::to_string(id_));
+  }
+
+  /// Performs double precision matrix-matrix multiplication (C = alpha*A*B + beta*C).
+  /// Assumes A is m x k, B is k x n, and C is m x n, all in row-major order.
+  void dgemm(int stream_id, int m, int n, int k, double alpha, mem_ptr<double> A,
+             mem_ptr<double> B, double beta, mem_ptr<double> C) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+
+    // For row-major matrices A(m,k), B(k,n), C(m,n) to compute C = alpha*A*B + beta*C
+    // cuBLAS expects column-major. The equivalent column-major operation is:
+    // C_col = alpha * B_col * A_col + beta * C_col
+    // where X_col = X_row^T.
+    // So, we call cublasDgemm with:
+    // A_cublas = B (transposed), B_cublas = A (transposed)
+    // Dimensions: m_cublas = n, n_cublas = m, k_cublas = k
+    cublasStatus_t status = cublasDgemm(handle, CUBLAS_OP_T, CUBLAS_OP_T,
+                                        n, m, k,
+                                        &alpha,
+                                        reinterpret_cast<const double*>(B->mem()), n, // B is k x n row-major, lda = n
+                                        reinterpret_cast<const double*>(A->mem()), k, // A is m x k row-major, ldb = k
+                                        &beta,
+                                        reinterpret_cast<double*>(C->mem()), n); // C is m x n row-major, ldc = n
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasDgemm failed on device " + std::to_string(id_));
+  }
+
   // Overloads for make_arg using actor_id
   template <typename T>
   mem_ptr<T> make_arg(const in<T>& arg, int actor_id) {

From 66a5e0762372a84ae7d5ac7be8c5aaa1bd5356f0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 09:36:15 -0600
Subject: [PATCH 0696/1000] Initialc ommit.

---
 .../gemm-actor-test/CMakeLists.txt            |  45 +++++++
 .../gemm-actor-test/main.test.cpp             | 111 ++++++++++++++++++
 2 files changed, 156 insertions(+)
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/gemm-actor-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/gemm-actor-test/main.test.cpp

diff --git a/libcaf_cuda/tests/actorBLAS-test/gemm-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorBLAS-test/gemm-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..fa9dc8e759
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/gemm-actor-test/CMakeLists.txt
@@ -0,0 +1,45 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+)
+
diff --git a/libcaf_cuda/tests/actorBLAS-test/gemm-actor-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/gemm-actor-test/main.test.cpp
new file mode 100644
index 0000000000..6749584336
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/gemm-actor-test/main.test.cpp
@@ -0,0 +1,111 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include <cmath>
+#include "caf/actorBLAS/gemm-actor/gemm-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+template <typename T>
+void verify_gemm(int m, int n, int k, T alpha, const std::vector<T>& A, const std::vector<T>& B, T beta, const std::vector<T>& C_init, const std::vector<T>& C_res) {
+    bool passed = true;
+    for (int i = 0; i < m; ++i) {
+        for (int j = 0; j < n; ++j) {
+            T sum = 0;
+            for (int l = 0; l < k; ++l) {
+                sum += A[i * k + l] * B[l * n + j];
+            }
+            T expected = alpha * sum + beta * C_init[i * n + j];
+            if (std::abs(C_res[i * n + j] - expected) > 1e-3) {
+                std::cout << "[ERROR] Mismatch at (" << i << "," << j << "): Expected " << expected << ", Got " << C_res[i * n + j] << std::endl;
+                passed = false;
+                break;
+            }
+        }
+        if (!passed) break;
+    }
+    if (passed) {
+        std::cout << "[SUCCESS] GEMM operation produced correct results." << std::endl;
+    }
+}
+
+void caf_main(actor_system& sys) {
+    manager_config config(true);
+    manager::init(sys, config);
+
+    int m = 4, n = 4, k = 4;
+    std::vector<float> h_A_f(m * k, 1.0f);
+    std::vector<float> h_B_f(k * n, 2.0f);
+    std::vector<float> h_C_f(m * n, 0.0f);
+
+    std::vector<double> h_A_d(m * k, 1.0);
+    std::vector<double> h_B_d(k * n, 2.0);
+    std::vector<double> h_C_d(m * n, 0.0);
+
+    auto float_gemm = sys.spawn<gemm_actor<float>>(1);
+    auto double_gemm = sys.spawn<gemm_actor<double>>(2);
+
+    scoped_actor self{sys};
+
+    // Test 1: Float GEMM, Host buffers, standard return
+    {
+        std::cout << "[INFO] Test 1: Float GEMM, Host buffers..." << std::endl;
+        self->mail(create_in_arg(h_A_f), create_in_arg(h_B_f), create_out_arg(h_C_f), m, n, k).send(float_gemm);
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 2) { // index 2 corresponds to matrix C
+                    verify_gemm(m, n, k, 1.0f, h_A_f, h_B_f, 0.0f, h_C_f, data);
+                }
+            }
+        );
+    }
+
+    // Test 2: Double GEMM, Host buffers, standard return
+    {
+        std::cout << "\n[INFO] Test 2: Double GEMM, Host buffers..." << std::endl;
+        self->mail(create_in_arg(h_A_d), create_in_arg(h_B_d), create_out_arg(h_C_d), m, n, k).send(double_gemm);
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<double> data) {
+                if (arg_index == 2) {
+                    verify_gemm(m, n, k, 1.0, h_A_d, h_B_d, 0.0, h_C_d, data);
+                }
+            }
+        );
+    }
+
+    // Test 3: Float GEMM, mem_ptr inputs
+    {
+        std::cout << "\n[INFO] Test 3: Float GEMM, mem_ptr inputs..." << std::endl;
+        command_runner<in<float>, in<float>, out<float>> runner;
+        auto ptrs = runner.transfer_memory(0, 0, create_in_arg(h_A_f), create_in_arg(h_B_f), create_out_arg(h_C_f));
+        self->mail(std::get<0>(ptrs), std::get<1>(ptrs), std::get<2>(ptrs), m, n, k).send(float_gemm);
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 2) {
+                    verify_gemm(m, n, k, 1.0f, h_A_f, h_B_f, 0.0f, h_C_f, data);
+                }
+            }
+        );
+    }
+
+    // Test 4: Float GEMM, return_mem_ptr_atom
+    {
+        std::cout << "\n[INFO] Test 4: Float GEMM, return_mem_ptr_atom..." << std::endl;
+        self->mail(return_mem_ptr_atom{}, create_in_arg(h_A_f), create_in_arg(h_B_f), create_out_arg(h_C_f), m, n, k).send(float_gemm);
+        self->receive(
+            [&](int reply_id, mem_ptr<float> A, mem_ptr<float> B, mem_ptr<float> C) {
+                command_runner<float> runner;
+                auto data = runner.copy_to_host(C);
+                verify_gemm(m, n, k, 1.0f, h_A_f, h_B_f, 0.0f, h_C_f, data);
+            }
+        );
+    }
+
+    self->send_exit(float_gemm, exit_reason::user_shutdown);
+    self->send_exit(double_gemm, exit_reason::user_shutdown);
+    manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda)

From eea396bd8c708f8f6a164f8d81bfead7e8cb65dd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 09:37:48 -0600
Subject: [PATCH 0697/1000] added double types to message type ID's

---
 libcaf_cuda/caf/cuda/global.hpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index 6badd2745a..a6af5edeb5 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -186,10 +186,13 @@ CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id)
   CAF_ADD_TYPE_ID(cuda, (in<int>))
   CAF_ADD_TYPE_ID(cuda, (in<char>))
   CAF_ADD_TYPE_ID(cuda, (in<float>))
+  CAF_ADD_TYPE_ID(cuda, (in<double>))
   CAF_ADD_TYPE_ID(cuda, (out<int>))
   CAF_ADD_TYPE_ID(cuda, (out<float>))
+  CAF_ADD_TYPE_ID(cuda, (out<double>))
   CAF_ADD_TYPE_ID(cuda, (in_out<int>))
   CAF_ADD_TYPE_ID(cuda, (in_out<float>))
+  CAF_ADD_TYPE_ID(cuda, (in_out<double>))
   CAF_ADD_TYPE_ID(cuda, (std::vector<float>))
   CAF_ADD_TYPE_ID(cuda, (std::vector<double>))
   CAF_ADD_TYPE_ID(cuda, (buffer_variant))

From b227f84fe44f255cdef9bab048c80fae0372d0c1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 09:52:43 -0600
Subject: [PATCH 0698/1000] implemented changes for batched_gemm operation

---
 .../batched-gemm-actor/batched-gemm-actor.hpp | 150 ++++++++++++++++++
 libcaf_cuda/caf/cuda/device.hpp               |  47 ++++++
 2 files changed, 197 insertions(+)
 create mode 100644 libcaf_cuda/caf/actorBLAS/batched-gemm-actor/batched-gemm-actor.hpp

diff --git a/libcaf_cuda/caf/actorBLAS/batched-gemm-actor/batched-gemm-actor.hpp b/libcaf_cuda/caf/actorBLAS/batched-gemm-actor/batched-gemm-actor.hpp
new file mode 100644
index 0000000000..ddcb1986fd
--- /dev/null
+++ b/libcaf_cuda/caf/actorBLAS/batched-gemm-actor/batched-gemm-actor.hpp
@@ -0,0 +1,150 @@
+#pragma once
+
+#include <caf/actor.hpp>
+#include <caf/actor_cast.hpp>
+#include <caf/actor_config.hpp>
+#include <caf/actor_system.hpp>
+#include <caf/anon_mail.hpp>
+#include <caf/event_based_actor.hpp>
+
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/mem_ref.hpp"
+#include "caf/cuda/command_runner.hpp"
+#include "caf/cuda/platform.hpp"
+#include "caf/cuda/types.hpp"
+
+namespace caf::cuda {
+
+/// GEMM Batched Actor for performing matrix-matrix multiplication on batches of matrices.
+/// Uses Strided Batched cuBLAS calls.
+template <typename T>
+class batched_gemm_actor : public event_based_actor {
+public:
+  static caf::actor spawn(caf::actor_system& sys, int reply_id = 0) {
+    return sys.spawn<batched_gemm_actor<T>>(reply_id);
+  }
+
+  batched_gemm_actor(caf::actor_config& cfg, int reply_id = 0)
+    : event_based_actor(cfg), reply_id_(reply_id) {
+    actor_id_ = static_cast<int>(this->id());
+  }
+
+  ~batched_gemm_actor() override {
+    command_runner<> runner;
+    runner.release_stream_for_actor(actor_id_);
+  }
+
+  caf::behavior make_behavior() override {
+    return {
+      // Standard host buffer based calls (alpha=1.0, beta=0.0)
+      [this](in<T> A, in<T> B, out<T> C, int m, int n, int k, int batchCount) {
+        enqueue_gemm_batched(-1, actor_id_, A, B, C, m, n, k, batchCount, static_cast<T>(1.0), static_cast<T>(0.0), false);
+      },
+      // mem_ptr based calls (Zero-copy Pipelines)
+      [this](mem_ptr<T> A, mem_ptr<T> B, mem_ptr<T> C, int m, int n, int k, int batchCount) {
+        enqueue_gemm_batched(-1, actor_id_, A, B, C, m, n, k, batchCount, static_cast<T>(1.0), static_cast<T>(0.0), false);
+      },
+      // Return mem_ptr atom overloads
+      [this](return_mem_ptr_atom, in<T> A, in<T> B, out<T> C, int m, int n, int k, int batchCount) {
+        enqueue_gemm_batched(-1, actor_id_, A, B, C, m, n, k, batchCount, static_cast<T>(1.0), static_cast<T>(0.0), true);
+      },
+      [this](return_mem_ptr_atom, mem_ptr<T> A, mem_ptr<T> B, mem_ptr<T> C, int m, int n, int k, int batchCount) {
+        enqueue_gemm_batched(-1, actor_id_, A, B, C, m, n, k, batchCount, static_cast<T>(1.0), static_cast<T>(0.0), true);
+      },
+      // Routing control (Explicit device/stream)
+      [this](int device_num, int stream_id, in<T> A, in<T> B, out<T> C, int m, int n, int k, int batchCount) {
+        enqueue_gemm_batched(device_num, stream_id, A, B, C, m, n, k, batchCount, static_cast<T>(1.0), static_cast<T>(0.0), false);
+      },
+      [this](int device_num, int stream_id, mem_ptr<T> A, mem_ptr<T> B, mem_ptr<T> C, int m, int n, int k, int batchCount) {
+        enqueue_gemm_batched(device_num, stream_id, A, B, C, m, n, k, batchCount, static_cast<T>(1.0), static_cast<T>(0.0), false);
+      },
+      [this](return_mem_ptr_atom, int device_num, int stream_id, in<T> A, in<T> B, out<T> C, int m, int n, int k, int batchCount) {
+        enqueue_gemm_batched(device_num, stream_id, A, B, C, m, n, k, batchCount, static_cast<T>(1.0), static_cast<T>(0.0), true);
+      },
+      [this](return_mem_ptr_atom, int device_num, int stream_id, mem_ptr<T> A, mem_ptr<T> B, mem_ptr<T> C, int m, int n, int k, int batchCount) {
+        enqueue_gemm_batched(device_num, stream_id, A, B, C, m, n, k, batchCount, static_cast<T>(1.0), static_cast<T>(0.0), true);
+      },
+      // Full Parameter GEMM (custom alpha/beta)
+      [this](in<T> A, in<T> B, out<T> C, int m, int n, int k, int batchCount, T alpha, T beta) {
+        enqueue_gemm_batched(-1, actor_id_, A, B, C, m, n, k, batchCount, alpha, beta, false);
+      },
+      [this](return_mem_ptr_atom, in<T> A, in<T> B, out<T> C, int m, int n, int k, int batchCount, T alpha, T beta) {
+        enqueue_gemm_batched(-1, actor_id_, A, B, C, m, n, k, batchCount, alpha, beta, true);
+      }
+    };
+  }
+
+private:
+  // Logic for host-wrapped buffers
+  template <typename OutType>
+  void enqueue_gemm_batched(int device_num, int stream_id,
+                            in<T> A_arg, in<T> B_arg, OutType C_arg,
+                            int m, int n, int k, int batchCount, T alpha, T beta, bool return_ptrs) {
+    command_runner<in<T>, in<T>, OutType> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, A_arg, B_arg, C_arg);
+    execute_and_reply(device_num, stream_id, std::get<0>(results),
+                      std::get<1>(results), std::get<2>(results),
+                      m, n, k, batchCount, alpha, beta, return_ptrs);
+  }
+
+  // Logic for existing Device buffers
+  void enqueue_gemm_batched(int device_num, int stream_id,
+                            mem_ptr<T> A_ptr, mem_ptr<T> B_ptr, mem_ptr<T> C_ptr,
+                            int m, int n, int k, int batchCount, T alpha, T beta, bool return_ptrs) {
+    command_runner<mem_ptr<T>, mem_ptr<T>, mem_ptr<T>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, A_ptr, B_ptr, C_ptr);
+    execute_and_reply(device_num, stream_id, std::get<0>(results),
+                      std::get<1>(results), std::get<2>(results),
+                      m, n, k, batchCount, alpha, beta, return_ptrs);
+  }
+
+  void execute_and_reply(int device_num, int stream_id,
+                         mem_ptr<T> A, mem_ptr<T> B, mem_ptr<T> C,
+                         int m, int n, int k, int batchCount, T alpha, T beta, bool return_ptrs) {
+    auto plat = platform::create();
+    device_ptr dev = (device_num == -1) ? plat->schedule(stream_id) : plat->schedule(stream_id, device_num);
+
+    // Default packed strides
+    long long int strideA = static_cast<long long int>(m) * k;
+    long long int strideB = static_cast<long long int>(k) * n;
+    long long int strideC = static_cast<long long int>(m) * n;
+
+    if constexpr (std::is_same_v<T, float>) {
+      dev->sgemm_strided_batched(stream_id, m, n, k, alpha, A, strideA, B, strideB, beta, C, strideC, batchCount);
+    } else if constexpr (std::is_same_v<T, double>) {
+      dev->dgemm_strided_batched(stream_id, m, n, k, alpha, A, strideA, B, strideB, beta, C, strideC, batchCount);
+    } else {
+      static_assert(std::is_same_v<T, float> || std::is_same_v<T, double>, 
+                    "Unsupported type for GEMM batched actor. Only float and double are supported.");
+    }
+
+    handle_reply(A, B, C, return_ptrs);
+  }
+
+  void handle_reply(mem_ptr<T> A_ptr, mem_ptr<T> B_ptr, mem_ptr<T> C_ptr, bool return_ptrs) {
+    command_runner<mem_ptr<T>> runner;
+    auto sender = actor_cast<actor>(this->current_sender());
+    if (!sender) return;
+
+    auto r_id = reply_id_;
+
+    if (return_ptrs) {
+      // Requirement: Always 1 message. 
+      // We bundle all pointers into a single message.
+      caf::anon_mail(r_id, A_ptr, B_ptr, C_ptr).send(sender);
+    } else {
+      // Requirement: Always 1 message. 
+      // We only copy back the result matrix (index 2) and send it.
+      runner.copy_to_host_async(C_ptr, [sender, r_id](std::vector<T>&& data) {
+        if (sender) {
+          caf::anon_mail(r_id, 2, std::move(data)).send(sender);
+        }
+      });
+    }
+  }
+
+  int actor_id_;
+  int reply_id_;
+};
+
+} // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 76d61027c5..92bfb1f95c 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -321,6 +321,53 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasDgemm failed on device " + std::to_string(id_));
   }
 
+  /// Performs single precision strided batched matrix-matrix multiplication.
+  void sgemm_strided_batched(int stream_id, int m, int n, int k, float alpha, mem_ptr<float> A, long long int strideA,
+                             mem_ptr<float> B, long long int strideB, float beta, mem_ptr<float> C, long long int strideC, int batchCount) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    
+    // Row-major A(m,k), B(k,n), C(m,n) -> cuBLAS (col-major): C = B * A
+    cublasStatus_t status = cublasSgemmStridedBatched(handle, CUBLAS_OP_T, CUBLAS_OP_T,
+                                                      n, m, k,
+                                                      &alpha,
+                                                      reinterpret_cast<const float*>(B->mem()), n, strideB,
+                                                      reinterpret_cast<const float*>(A->mem()), k, strideA,
+                                                      &beta,
+                                                      reinterpret_cast<float*>(C->mem()), n, strideC,
+                                                      batchCount);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasSgemmStridedBatched failed on device " + std::to_string(id_));
+  }
+
+  /// Performs double precision strided batched matrix-matrix multiplication.
+  void dgemm_strided_batched(int stream_id, int m, int n, int k, double alpha, mem_ptr<double> A, long long int strideA,
+                             mem_ptr<double> B, long long int strideB, double beta, mem_ptr<double> C, long long int strideC, int batchCount) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    
+    cublasStatus_t status = cublasDgemmStridedBatched(handle, CUBLAS_OP_T, CUBLAS_OP_T,
+                                                      n, m, k,
+                                                      &alpha,
+                                                      reinterpret_cast<const double*>(B->mem()), n, strideB,
+                                                      reinterpret_cast<const double*>(A->mem()), k, strideA,
+                                                      &beta,
+                                                      reinterpret_cast<double*>(C->mem()), n, strideC,
+                                                      batchCount);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasDgemm failed on device " + std::to_string(id_));
+  }
+
   // Overloads for make_arg using actor_id
   template <typename T>
   mem_ptr<T> make_arg(const in<T>& arg, int actor_id) {

From 56510b5aaac5040d204f42e6059f721564ef340c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 10:03:27 -0600
Subject: [PATCH 0699/1000] Initialc ommit.

---
 .../batched-gemm-actor-test/CMakeLists.txt    |  45 ++++++
 .../batched-gemm-actor-test/main.test.cpp     | 129 ++++++++++++++++++
 2 files changed, 174 insertions(+)
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/batched-gemm-actor-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/batched-gemm-actor-test/main.test.cpp

diff --git a/libcaf_cuda/tests/actorBLAS-test/batched-gemm-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorBLAS-test/batched-gemm-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..fa9dc8e759
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/batched-gemm-actor-test/CMakeLists.txt
@@ -0,0 +1,45 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+)
+
diff --git a/libcaf_cuda/tests/actorBLAS-test/batched-gemm-actor-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/batched-gemm-actor-test/main.test.cpp
new file mode 100644
index 0000000000..7a3743a521
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/batched-gemm-actor-test/main.test.cpp
@@ -0,0 +1,129 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include <cmath>
+#include "caf/actorBLAS/batched-gemm-actor/batched-gemm-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+template <typename T>
+void verify_gemm(int m, int n, int k, T alpha, const std::vector<T>& A, const std::vector<T>& B, T beta, const std::vector<T>& C_init, const std::vector<T>& C_res) {
+    bool passed = true;
+    for (int i = 0; i < m; ++i) {
+        for (int j = 0; j < n; ++j) {
+            T sum = 0;
+            for (int l = 0; l < k; ++l) {
+                sum += A[i * k + l] * B[l * n + j];
+            }
+            T expected = alpha * sum + beta * C_init[i * n + j];
+            if (std::abs(C_res[i * n + j] - expected) > 1e-3) {
+                std::cout << "[ERROR] Mismatch at (" << i << "," << j << "): Expected " << expected << ", Got " << C_res[i * n + j] << std::endl;
+                passed = false;
+                break;
+            }
+        }
+        if (!passed) break;
+    }
+    if (passed) {
+        std::cout << "[SUCCESS] GEMM operation produced correct results." << std::endl;
+    }
+}
+
+template <typename T>
+void verify_batched_gemm(int m, int n, int k, int batchCount, T alpha, const std::vector<T>& A, const std::vector<T>& B, T beta, const std::vector<T>& C_init, const std::vector<T>& C_res) {
+    for (int b = 0; b < batchCount; ++b) {
+        std::cout << "[INFO] Verifying batch " << b << "..." << std::endl;
+        auto offsetA = b * m * k;
+        auto offsetB = b * k * n;
+        auto offsetC = b * m * n;
+        
+        std::vector<T> sliceA(A.begin() + offsetA, A.begin() + offsetA + m * k);
+        std::vector<T> sliceB(B.begin() + offsetB, B.begin() + offsetB + k * n);
+        std::vector<T> sliceC_init(C_init.begin() + offsetC, C_init.begin() + offsetC + m * n);
+        std::vector<T> sliceC_res(C_res.begin() + offsetC, C_res.begin() + offsetC + m * n);
+        
+        verify_gemm(m, n, k, alpha, sliceA, sliceB, beta, sliceC_init, sliceC_res);
+    }
+}
+
+void caf_main(actor_system& sys) {
+    manager_config config(true);
+    manager::init(sys, config);
+
+    int m = 4, n = 4, k = 4, batchCount = 2;
+    // Matrix A (4x4), B (4x4) -> C (4x4). Sum of 4 elements (1.0 * 2.0) = 8.0 per element.
+    std::vector<float> h_A_f(m * k * batchCount, 1.0f);
+    std::vector<float> h_B_f(k * n * batchCount, 2.0f);
+    std::vector<float> h_C_f(m * n * batchCount, 0.0f);
+
+    std::vector<double> h_A_d(m * k * batchCount, 1.0);
+    std::vector<double> h_B_d(k * n * batchCount, 2.0);
+    std::vector<double> h_C_d(m * n * batchCount, 0.0);
+
+    auto float_gemm = sys.spawn<batched_gemm_actor<float>>(1);
+    auto double_gemm = sys.spawn<batched_gemm_actor<double>>(2);
+
+    scoped_actor self{sys};
+
+    // Test 1: Float Batched GEMM, Host buffers, standard return
+    {
+        std::cout << "[INFO] Test 1: Float Batched GEMM, Host buffers..." << std::endl;
+        self->mail(create_in_arg(h_A_f), create_in_arg(h_B_f), create_out_arg(h_C_f), m, n, k, batchCount).send(float_gemm);
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 2) { // index 2 corresponds to matrix C
+                    verify_batched_gemm(m, n, k, batchCount, 1.0f, h_A_f, h_B_f, 0.0f, h_C_f, data);
+                }
+            }
+        );
+    }
+
+    // Test 2: Double GEMM, Host buffers, standard return
+    {
+        std::cout << "\n[INFO] Test 2: Double GEMM, Host buffers..." << std::endl;
+        self->mail(create_in_arg(h_A_d), create_in_arg(h_B_d), create_out_arg(h_C_d), m, n, k, batchCount).send(double_gemm);
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<double> data) {
+                if (arg_index == 2) {
+                    verify_batched_gemm(m, n, k, batchCount, 1.0, h_A_d, h_B_d, 0.0, h_C_d, data);
+                }
+            }
+        );
+    }
+
+    // Test 3: Float GEMM, mem_ptr inputs
+    {
+        std::cout << "\n[INFO] Test 3: Float GEMM, mem_ptr inputs..." << std::endl;
+        command_runner<in<float>, in<float>, out<float>> runner;
+        auto ptrs = runner.transfer_memory(0, 0, create_in_arg(h_A_f), create_in_arg(h_B_f), create_out_arg(h_C_f));
+        self->mail(std::get<0>(ptrs), std::get<1>(ptrs), std::get<2>(ptrs), m, n, k, batchCount).send(float_gemm);
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 2) {
+                    verify_batched_gemm(m, n, k, batchCount, 1.0f, h_A_f, h_B_f, 0.0f, h_C_f, data);
+                }
+            }
+        );
+    }
+
+    // Test 4: Float GEMM, return_mem_ptr_atom
+    {
+        std::cout << "\n[INFO] Test 4: Float GEMM, return_mem_ptr_atom..." << std::endl;
+        self->mail(return_mem_ptr_atom{}, create_in_arg(h_A_f), create_in_arg(h_B_f), create_out_arg(h_C_f), m, n, k, batchCount).send(float_gemm);
+        self->receive(
+            [&](int reply_id, mem_ptr<float> A, mem_ptr<float> B, mem_ptr<float> C) {
+                command_runner<float> runner;
+                auto data = runner.copy_to_host(C);
+                verify_batched_gemm(m, n, k, batchCount, 1.0f, h_A_f, h_B_f, 0.0f, h_C_f, data);
+            }
+        );
+    }
+
+    self->send_exit(float_gemm, exit_reason::user_shutdown);
+    self->send_exit(double_gemm, exit_reason::user_shutdown);
+    manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda)

From b0e0f495202425d8a7d122ba46adfda73de68426 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 11:28:37 -0600
Subject: [PATCH 0700/1000] Initialc omit.

---
 .../workload-test/CMakeLists.txt              |  46 ++
 .../workload-test/compile_kernels.sh          |  14 +
 .../benchmark-tests/workload-test/conv1d.cu   |  15 +
 .../workload-test/cuda-baseline.cpp           | 346 ++++++++++++
 .../benchmark-tests/workload-test/mmul.cu     |  16 +
 .../workload-test/vector_add.cu               |   6 +
 .../workload-test/work-stealing.cpp           | 496 ++++++++++++++++++
 7 files changed, 939 insertions(+)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/workload-test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/benchmark-tests/workload-test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/benchmark-tests/workload-test/conv1d.cu
 create mode 100644 libcaf_cuda/tests/benchmark-tests/workload-test/cuda-baseline.cpp
 create mode 100644 libcaf_cuda/tests/benchmark-tests/workload-test/mmul.cu
 create mode 100644 libcaf_cuda/tests/benchmark-tests/workload-test/vector_add.cu
 create mode 100644 libcaf_cuda/tests/benchmark-tests/workload-test/work-stealing.cpp

diff --git a/libcaf_cuda/tests/benchmark-tests/workload-test/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/workload-test/CMakeLists.txt
new file mode 100644
index 0000000000..ecf8d6157d
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/workload-test/CMakeLists.txt
@@ -0,0 +1,46 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executables
+add_executable(work-stealing work-stealing.cpp)
+target_compile_definitions(work-stealing PRIVATE CAF_ENABLE_LOGGING)
+
+add_executable(cuda-baseline cuda-baseline.cpp)
+target_link_libraries(cuda-baseline PRIVATE CUDA::cuda_driver)
+
+target_link_libraries(work-stealing
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+)
diff --git a/libcaf_cuda/tests/benchmark-tests/workload-test/compile_kernels.sh b/libcaf_cuda/tests/benchmark-tests/workload-test/compile_kernels.sh
new file mode 100755
index 0000000000..c0bebf8833
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/workload-test/compile_kernels.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability to ensure binary compatibility
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile kernels to cubin for Driver API loading
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+nvcc -arch=$SM_ARCH -cubin vector_add.cu -o vector_add.cubin
+nvcc -arch=$SM_ARCH -cubin conv1d.cu -o conv1d.cubin
+
+echo "Kernels compiled successfully for $SM_ARCH."
\ No newline at end of file
diff --git a/libcaf_cuda/tests/benchmark-tests/workload-test/conv1d.cu b/libcaf_cuda/tests/benchmark-tests/workload-test/conv1d.cu
new file mode 100644
index 0000000000..aa897ede25
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/workload-test/conv1d.cu
@@ -0,0 +1,15 @@
+extern "C" __global__ void conv1d(const int* A, const int* K, int* C, int N) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int W = 5; // Fixed filter width for benchmark simplicity
+    if (idx < N) {
+        int sum = 0;
+        int halfW = W / 2;
+        for (int i = 0; i < W; ++i) {
+            int col = idx + i - halfW;
+            if (col >= 0 && col < N) {
+                sum += A[col] * K[i];
+            }
+        }
+        C[idx] = sum;
+    }
+}
diff --git a/libcaf_cuda/tests/benchmark-tests/workload-test/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/workload-test/cuda-baseline.cpp
new file mode 100644
index 0000000000..0d1c6fe1af
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/workload-test/cuda-baseline.cpp
@@ -0,0 +1,346 @@
+#include <cuda.h> // CUDA Driver API
+#include <iostream>
+#include <vector>
+#include <chrono>
+#include <random>
+#include <thread>
+#include <algorithm>
+#include <functional>
+#include <stdexcept> // For runtime_error
+#include <unordered_map> // For MatrixPool
+#include <unordered_set> // For create_matrix_pool_random
+
+enum TaskType { MMUL = 0, VADD = 1, CONV = 2 };
+
+struct Task {
+    int N;
+    TaskType type;
+};
+
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
+    std::unordered_map<int, std::vector<int>> vec_A;
+    std::unordered_map<int, std::vector<int>> vec_B;
+    std::unordered_map<int, std::vector<int>> conv_A;
+    std::unordered_map<int, std::vector<int>> conv_K;
+};
+
+// Per-GPU execution logic
+void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_gpu, CUcontext ctx, CUfunction mmul_func,
+     CUfunction vadd_func, CUfunction conv_func, const MatrixPool& pool, int* shared_dtoh_buffer) {
+    // Set the CUDA context for this thread
+    CUresult err = cuCtxSetCurrent(ctx);
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error setting context for device " << device_id << ": " << err_str << std::endl;
+        return;
+    }
+
+    // Prepare Streams
+    std::vector<CUstream> streams(streams_per_gpu);
+    for (int i = 0; i < streams_per_gpu; ++i) {
+        cuStreamCreate(&streams[i], CU_STREAM_NON_BLOCKING);
+    }
+
+    // Use the first stream for initial allocations and cleanup
+    CUstream default_stream = streams[0];
+
+    // Process assigned tasks
+    for (size_t i = 0; i < tasks.size(); ++i) {
+        int N = tasks[i].N;
+        CUstream stream = streams[i % streams_per_gpu];
+        TaskType type = tasks[i].type;
+        size_t bytes_a = (type == MMUL) ? (size_t)N * N * sizeof(int) : (size_t)N * sizeof(int);
+        size_t bytes_b = (type == CONV) ? 5 * sizeof(int) : bytes_a;
+        size_t bytes_out = (type == MMUL) ? (size_t)N * N * sizeof(int) : (size_t)N * sizeof(int);
+
+        CUdeviceptr d_a, d_b, d_c;
+
+        cuMemAllocAsync(&d_a, bytes_a, stream);
+        cuMemAllocAsync(&d_b, bytes_b, stream);
+        cuMemAllocAsync(&d_c, bytes_out, stream);
+
+        // Perform Host-to-Device transfer
+        const std::vector<int>& h_a = (type == MMUL) ? pool.A.at(N) : (type == VADD ? pool.vec_A.at(N) : pool.conv_A.at(N));
+        const std::vector<int>& h_b = (type == MMUL) ? pool.B.at(N) : (type == VADD ? pool.vec_B.at(N) : pool.conv_K.at(N));
+
+        cuMemcpyHtoDAsync(d_a, h_a.data(), bytes_a, stream);
+        cuMemcpyHtoDAsync(d_b, h_b.data(), bytes_b, stream);
+
+        // Kernel arguments for cuLaunchKernel
+        void *kernel_args[] = { &d_a, &d_b, &d_c, &N };
+
+        if (type == MMUL) {
+            unsigned int block_dim = 32;
+            unsigned int grid_dim = (N + block_dim - 1) / block_dim;
+            cuLaunchKernel(mmul_func, grid_dim, grid_dim, 1,
+                           block_dim, block_dim, 1,
+                           0, stream, kernel_args, nullptr);
+        } else if (type == VADD) {
+            unsigned int block_dim = 256;
+            unsigned int grid_dim = (N + block_dim - 1) / block_dim;
+            cuLaunchKernel(vadd_func, grid_dim, 1, 1,
+                           block_dim, 1, 1,
+                           0, stream, kernel_args, nullptr);
+        } else {
+            unsigned int block_dim = 256;
+            unsigned int grid_dim = (N + block_dim - 1) / block_dim;
+            cuLaunchKernel(conv_func, grid_dim, 1, 1,
+                           block_dim, 1, 1,
+                           0, stream, kernel_args, nullptr);
+        }
+        
+        // Simulating the result retrieval (Copy back)
+        size_t res_count = (type == MMUL) ? (size_t)N * N : (size_t)N;
+        cuMemcpyDtoHAsync(shared_dtoh_buffer, d_c, bytes_out, stream);
+
+        // Free GPU memory for this task
+        cuMemFreeAsync(d_a, stream);
+        cuMemFreeAsync(d_b, stream);
+        cuMemFreeAsync(d_c, stream);
+    }
+
+    // Synchronize this GPU context
+    cuCtxSynchronize();
+
+    // Cleanup
+    for (auto s : streams) {
+        cuStreamDestroy(s);
+    }
+}
+
+MatrixPool create_matrix_pool_random(
+    int num_sizes,
+    int min_N,
+    int max_N,
+    unsigned int seed
+) {
+    MatrixPool pool;
+    std::mt19937 rng(seed);
+    std::uniform_int_distribution<int> dist_N(min_N / 32, max_N / 32);
+    std::unordered_set<int> used_Ns;
+    while (used_Ns.size() < static_cast<size_t>(num_sizes)) {
+        int N_val = dist_N(rng) * 32;
+        if (N_val == 0) continue;
+        if (used_Ns.insert(N_val).second) {
+            pool.A[N_val] = std::vector<int>(N_val * N_val, 1);
+            pool.B[N_val] = std::vector<int>(N_val * N_val, 1);
+            pool.vec_A[N_val] = std::vector<int>(N_val, 1);
+            pool.vec_B[N_val] = std::vector<int>(N_val, 1);
+            pool.conv_A[N_val] = std::vector<int>(N_val, 1);
+            pool.conv_K[N_val] = std::vector<int>(5, 1);
+        }
+    }
+    return pool;
+}
+
+int main() {
+    CUresult err;
+
+    // Initialize the CUDA Driver API
+    err = cuInit(0);
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error initializing CUDA Driver API: " << err_str << std::endl;
+        return 1;
+    }
+
+    const int streams_per_gpu = 8;
+    std::vector<int> task_counts = {50000,100000};
+
+    int num_gpus;
+    cuDeviceGetCount(&num_gpus);
+    if (num_gpus == 0) {
+        std::cerr << "No CUDA devices found." << std::endl;
+        return 1;
+    }
+
+    // Define parameters for irregular workload
+    const int num_matrix_sizes = 60; // Number of distinct N values
+    const int min_N_val = 32;
+    const int max_N_val = 2048;
+    const unsigned int pool_seed = 42; // Fixed seed for deterministic pool generation
+
+    // Create the host-side matrix pool once
+    MatrixPool global_host_matrix_pool = create_matrix_pool_random(num_matrix_sizes, min_N_val, max_N_val, pool_seed);
+
+    // Extract available N values from the pool for task generation
+    std::vector<int> available_Ns;
+    for (const auto& pair : global_host_matrix_pool.A) {
+        available_Ns.push_back(pair.first);
+    }
+    std::sort(available_Ns.begin(), available_Ns.end());
+    if (available_Ns.empty()) {
+        std::cerr << "Error: No matrix sizes generated in the pool." << std::endl;
+        return 1;
+    }
+
+    for (int total_tasks : task_counts) {
+    std::cout << "=====================================" << std::endl;
+    std::cout << "Task count: " << total_tasks << " (Irregular Workload)" << std::endl;
+
+    std::vector<Task> all_tasks;
+    std::mt19937 rng_tasks(42); // Fixed seed for task distribution
+    std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
+
+    std::vector<CUcontext> contexts(num_gpus);
+    std::vector<CUfunction> mmul_funcs(num_gpus);
+    std::vector<CUfunction> vadd_funcs(num_gpus);
+    std::vector<CUfunction> conv_funcs(num_gpus);
+    CUmodule mmul_mod;
+    CUmodule vadd_mod;
+    CUmodule conv_mod;
+
+    // Create a CUDA context for each device
+    for (int i = 0; i < num_gpus; ++i) {
+        CUdevice dev;
+        cuDeviceGet(&dev, i);
+        err = cuCtxCreate(&contexts[i], 0, dev); // Flag 0 for default context creation
+        if (err != CUDA_SUCCESS) {
+            const char* err_str;
+            cuGetErrorString(err, &err_str);
+            std::cerr << "Error creating context for device " << i << ": " << err_str << std::endl;
+            // Clean up already created contexts
+            for (int j = 0; j < i; ++j) cuCtxDestroy(contexts[j]);
+            return 1;
+        }
+    }
+
+    // Make the context for device 0 current on the main thread before loading the module
+    err = cuCtxPushCurrent(contexts[0]);
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error pushing context for device 0: " << err_str << std::endl;
+        for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
+        return 1;
+    }
+
+    // Load the cubin module (assuming mmul.cu is compiled to mmul.cubin)
+    err = cuModuleLoad(&mmul_mod, "../mmul.cubin");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error loading module ../mmul.cubin: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr); // Pop context on error
+        return 1;
+    }
+
+    err = cuModuleLoad(&vadd_mod, "../vector_add.cubin");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error loading module ../vector_add.cubin: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr);
+        return 1;
+    }
+
+    err = cuModuleLoad(&conv_mod, "../conv1d.cubin");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error loading module ../conv1d.cubin: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr);
+        return 1;
+    }
+
+    // Get function handles
+    err = cuModuleGetFunction(&mmul_funcs[0], mmul_mod, "matrixMul");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error getting function matrixMul: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr); // Pop context on error
+        cuModuleUnload(mmul_mod);
+        cuModuleUnload(vadd_mod);
+        for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
+        return 1;
+    }
+
+    err = cuModuleGetFunction(&vadd_funcs[0], vadd_mod, "vectorAdd");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error getting function vectorAdd: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr);
+        return 1;
+    }
+
+    err = cuModuleGetFunction(&conv_funcs[0], conv_mod, "conv1d");
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error getting function conv1d: " << err_str << std::endl;
+        cuCtxPopCurrent(nullptr);
+        return 1;
+    }
+
+    // Pop the context from the main thread
+    err = cuCtxPopCurrent(nullptr);
+    if (err != CUDA_SUCCESS) {
+        const char* err_str;
+        cuGetErrorString(err, &err_str);
+        std::cerr << "Error popping context from main thread: " << err_str << std::endl;
+        cuModuleUnload(mmul_mod);
+        cuModuleUnload(vadd_mod);
+        cuModuleUnload(conv_mod);
+        for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
+        return 1;
+    }
+
+    // Assuming all GPUs can use the same function handle from the same module.
+    for (int i = 1; i < num_gpus; ++i) {
+        mmul_funcs[i] = mmul_funcs[0];
+        vadd_funcs[i] = vadd_funcs[0];
+        conv_funcs[i] = conv_funcs[0];
+    }
+
+    std::uniform_int_distribution<int> dist_type(0, 2);
+    for (int i = 0; i < total_tasks; ++i) {
+        int N_for_task = available_Ns[dist_N_idx(rng_tasks)];
+        TaskType t_type = static_cast<TaskType>(dist_type(rng_tasks));
+        all_tasks.push_back({N_for_task, t_type});
+    }
+
+    // ─────────────────────────────────────────────────────────────────────────
+    // Static Round-Robin Partitioning
+    // ─────────────────────────────────────────────────────────────────────────
+    std::vector<std::vector<Task>> partitions(num_gpus);
+    for (int i = 0; i < total_tasks; ++i) {
+        partitions[i % num_gpus].push_back(all_tasks[i]);
+    }
+
+    // Preallocate a single large host buffer for DTOH transfers to save RAM and keep things fair.
+    std::vector<int> shared_dtoh_buffer((size_t)max_N_val * max_N_val);
+
+    auto start = std::chrono::steady_clock::now();
+
+    std::vector<std::thread> threads;
+    for (int i = 0; i < num_gpus; ++i) { // Pass context and kernel function to each worker
+        threads.emplace_back(gpu_worker, i, std::ref(partitions[i]), streams_per_gpu, contexts[i], mmul_funcs[i], vadd_funcs[i], conv_funcs[i],
+         std::ref(global_host_matrix_pool), shared_dtoh_buffer.data());
+    }
+
+    for (auto& t : threads) {
+        t.join();
+    }
+
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+
+    std::cout << "Makespan: " << elapsed.count() << "s" << std::endl;
+
+    // Cleanup contexts and module
+    for (int i = 0; i < num_gpus; ++i) {
+        cuCtxDestroy(contexts[i]);
+    }
+    cuModuleUnload(mmul_mod);
+    cuModuleUnload(vadd_mod);
+    cuModuleUnload(conv_mod);
+    }
+
+    return 0;
+}
diff --git a/libcaf_cuda/tests/benchmark-tests/workload-test/mmul.cu b/libcaf_cuda/tests/benchmark-tests/workload-test/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/workload-test/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+
diff --git a/libcaf_cuda/tests/benchmark-tests/workload-test/vector_add.cu b/libcaf_cuda/tests/benchmark-tests/workload-test/vector_add.cu
new file mode 100644
index 0000000000..4bf501d5d1
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/workload-test/vector_add.cu
@@ -0,0 +1,6 @@
+extern "C" __global__ void vectorAdd(const int* A, const int* B, int* C, int N) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx < N) {
+        C[idx] = A[idx] + B[idx];
+    }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/benchmark-tests/workload-test/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/workload-test/work-stealing.cpp
new file mode 100644
index 0000000000..f630a7814b
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/workload-test/work-stealing.cpp
@@ -0,0 +1,496 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/component-actors/all-component-actors.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include "caf/actor_registry.hpp"
+#include <deque>
+#include <unordered_map>
+#include <unordered_set>
+//#include <caf/atoms.hpp>
+
+using namespace caf;
+using namespace std::chrono_literals;
+
+
+enum TaskType { MMUL = 0, VADD = 1, CONV = 2 };
+
+struct Task {
+    int N;
+    TaskType type;
+};
+
+// Inspect function for TaskType enum to enable CAF serialization
+template <class Inspector>
+bool inspect(Inspector& f, TaskType& x) {
+  auto val = static_cast<int>(x);
+  if (f.apply(val)) {
+    if constexpr (Inspector::is_loading)
+      x = static_cast<TaskType>(val);
+    return true;
+  }
+  return false;
+}
+
+// Inspect function for Task struct to enable CAF serialization
+template <class Inspector>
+bool inspect(Inspector& f, Task& x) {
+  return f.object(x).fields(f.field("N", x.N), f.field("type", x.type));
+};
+
+
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Atoms
+// ─────────────────────────────────────────────────────────────────────────────
+CAF_BEGIN_TYPE_ID_BLOCK(mmul_benchmark, caf::id_block::cuda::end)
+    CAF_ADD_ATOM(mmul_benchmark, get_work_atom)
+    CAF_ADD_ATOM(mmul_benchmark, task_done_atom)
+    CAF_ADD_ATOM(mmul_benchmark, release_memory_atom)
+    CAF_ADD_ATOM(mmul_benchmark, request_work_atom)
+    CAF_ADD_ATOM(mmul_benchmark, worker_done_atom)
+    CAF_ADD_TYPE_ID(mmul_benchmark, (TaskType))
+    CAF_ADD_TYPE_ID(mmul_benchmark, (Task))
+    CAF_ADD_TYPE_ID(mmul_benchmark, (std::vector<Task>))
+    CAF_ADD_ATOM(mmul_benchmark, refill_buffer_atom)
+CAF_END_TYPE_ID_BLOCK(mmul_benchmark)
+
+
+// Command runners for GPU operations
+caf::cuda::command_runner<> mmul_command;
+using kernel_runner_t = caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>, out<int>, in<int>>;
+kernel_runner_t kernel_runner;
+
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
+    std::unordered_map<int, std::vector<int>> vec_A;
+    std::unordered_map<int, std::vector<int>> vec_B;
+    std::unordered_map<int, std::vector<int>> conv_A;
+    std::unordered_map<int, std::vector<int>> conv_K;
+};
+
+MatrixPool create_matrix_pool_random(
+    int num_sizes,
+    int min_N,
+    int max_N,
+    unsigned int seed
+) {
+    MatrixPool pool;
+
+    std::mt19937 rng(seed);
+    std::uniform_int_distribution<int> dist(min_N / 32, max_N / 32);
+
+    std::unordered_set<int> used;
+
+    while (used.size() < static_cast<size_t>(num_sizes)) {
+        int N = dist(rng) * 32;
+        if (N == 0) continue;
+        if (used.insert(N).second) {
+            pool.A[N] = std::vector<int>(N * N, 1);
+            pool.B[N] = std::vector<int>(N * N, 1);
+            pool.vec_A[N] = std::vector<int>(N, 1);
+            pool.vec_B[N] = std::vector<int>(N, 1);
+            pool.conv_A[N] = std::vector<int>(N, 1);
+            pool.conv_K[N] = std::vector<int>(5, 1);
+        }
+    }
+
+    return pool;
+}
+
+// ---------------------------- GLOBAL TASK POOL ----------------------------
+// The central source of truth for work. Implements a pull-based model.
+struct task_pool_state {
+    std::vector<Task> tasks;
+    size_t next_task_idx = 0;
+};
+
+caf::behavior global_task_pool(caf::stateful_actor<task_pool_state>* self, std::vector<Task> tasks) {
+    self->state().tasks = std::move(tasks);
+    return {
+        [=](get_work_atom, size_t batch_size) -> result<std::vector<Task>> {
+            auto& st = self->state();
+            if (st.next_task_idx >= st.tasks.size())
+                return sec::end_of_stream;
+            size_t count = std::min(batch_size, st.tasks.size() - st.next_task_idx);
+            std::vector<Task> batch(st.tasks.begin() + st.next_task_idx, 
+                                    st.tasks.begin() + st.next_task_idx + count);
+            st.next_task_idx += count;
+            return batch;
+        }
+    };
+}
+
+// ---------------------------- DEVICE/GPU ACTOR ----------------------------
+// Manages memory for a specific GPU and steals (pulls) work from the Global Pool.
+struct device_actor_state {
+    MatrixPool pool;
+    caf::actor global_pool;
+    std::deque<Task> local_tasks; // Local buffer to keep GPU busy
+    size_t total_device_memory_bytes = 0;
+    size_t current_allocated_memory_bytes = 0;
+    int active_workers = 0;
+    int device_id = -1;
+    size_t batch_size = 0;
+    size_t low_water_mark = 0;
+    bool fetching = false;
+};
+
+caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
+                               MatrixPool pool, caf::actor global_pool, int num_workers, int dev_id, int max_in_flight) {
+    self->state().pool = std::move(pool);
+    self->state().global_pool = global_pool;
+    self->state().device_id = dev_id;
+    self->state().active_workers = num_workers;
+
+    // Dynamically calculate prefetch markers based on the total pipeline capacity
+    self->state().low_water_mark = static_cast<size_t>(num_workers * max_in_flight);
+    self->state().batch_size = self->state().low_water_mark * 2;
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+    caf::cuda::device_ptr dev_obj = mgr.find_device(dev_id);
+    if (dev_obj) {
+        self->state().total_device_memory_bytes = dev_obj->total_memory_bytes();
+    }
+
+    // Helper to refill the local task buffer from the global pool
+    auto refill = [=]() {
+        auto& st = self->state();
+        if (st.fetching || st.local_tasks.size() >= st.low_water_mark + st.batch_size)
+            return;
+
+        st.fetching = true;
+        self->mail(get_work_atom_v, st.batch_size).request(st.global_pool, infinite).then(
+            [=](std::vector<Task>& batch) {
+                auto& st_inner = self->state();
+                for (auto& task : batch)
+                    st_inner.local_tasks.push_back(task);
+                st_inner.fetching = false;
+                if (st_inner.local_tasks.size() < st_inner.low_water_mark)
+                    self->mail(refill_buffer_atom_v).send(self);
+            },
+            [=](error& err) {
+                self->state().fetching = false;
+            }
+        );
+    };
+
+    return {
+        [=](refill_buffer_atom) {
+            refill();
+        },
+        [=](get_work_atom) -> caf::result<int, int, in<int>, in<int>> {
+            auto& st = self->state();
+
+            // If we have tasks locally, satisfy the request immediately
+            if (!st.local_tasks.empty()) {
+                Task t = st.local_tasks.front();
+                int N = t.N;
+                size_t memory_needed = (t.type == MMUL) ? (size_t)N * N * sizeof(int) * 3 : (size_t)N * sizeof(int) * 3; // Approx
+                if (st.current_allocated_memory_bytes + memory_needed > st.total_device_memory_bytes)
+                    return make_error(sec::runtime_error, "Device Actor: Not enough memory");
+                
+                st.local_tasks.pop_front();
+                st.current_allocated_memory_bytes += memory_needed;
+                
+                if (st.local_tasks.size() < st.low_water_mark)
+                    refill();
+
+                auto& h_a = (t.type == MMUL) ? st.pool.A[N] : (t.type == VADD ? st.pool.vec_A[N] : st.pool.conv_A[N]);
+                auto& h_b = (t.type == MMUL) ? st.pool.B[N] : (t.type == VADD ? st.pool.vec_B[N] : st.pool.conv_K[N]);
+
+                return {N, static_cast<int>(t.type), caf::cuda::create_in_arg(h_a), 
+                           caf::cuda::create_in_arg(h_b)};
+            }
+
+            // Buffer empty: must fetch from global pool reactively
+            auto promise = self->make_response_promise<int, int, in<int>, in<int>>();
+            self->mail(get_work_atom_v, st.batch_size).request(st.global_pool, infinite).then(
+                [=](std::vector<Task>& batch) mutable {
+                    auto& st_inner = self->state();
+                    Task t = batch.front();
+                    int N = t.N;
+                    for(size_t i = 1; i < batch.size(); ++i) st_inner.local_tasks.push_back(batch[i]);
+                    
+                    size_t needed = (t.type == MMUL) ? (size_t)N * N * sizeof(int) * 3 : (size_t)N * sizeof(int) * 3; // Approx
+                    st_inner.current_allocated_memory_bytes += needed;
+
+                    auto& h_a = (t.type == MMUL) ? st_inner.pool.A[N] : (t.type == VADD ? st_inner.pool.vec_A[N] : st_inner.pool.conv_A[N]);
+                    auto& h_b = (t.type == MMUL) ? st_inner.pool.B[N] : (t.type == VADD ? st_inner.pool.vec_B[N] : st_inner.pool.conv_K[N]);
+                    promise.deliver(N, static_cast<int>(t.type), caf::cuda::create_in_arg(h_a), 
+                                       caf::cuda::create_in_arg(h_b));
+                },
+                [=](error& err) mutable { promise.deliver(err); }
+            );
+            return promise;
+        },
+        [=](release_memory_atom, int N_completed, int type) {
+            auto& st = self->state();
+            TaskType t_type = static_cast<TaskType>(type);
+            size_t memory_released = (t_type == MMUL) ? (size_t)N_completed * N_completed * sizeof(int) * 3 : (size_t)N_completed * sizeof(int) * 3; // Approx
+            st.current_allocated_memory_bytes -= memory_released;
+            refill(); // Try to get more work now that memory is free
+        },
+        [=](worker_done_atom) {
+            auto& st = self->state();
+            if (--st.active_workers <= 0) {
+                self->quit();
+            }
+        }
+    };
+}
+
+// ---------------------------- WORKER ACTOR ----------------------------
+// Manages 1 stream and pulls work from the Device Actor.
+struct worker_state {
+    int device_id;
+    int stream_id;
+    caf::cuda::program_ptr mmul_prog;
+    caf::cuda::program_ptr vadd_prog;
+    caf::cuda::program_ptr conv_prog;
+    caf::actor device_actor;
+    caf::actor supervisor;
+    int max_in_flight_tasks;
+    int in_flight_tasks_count = 0;
+    int* dtoh_buffer_ptr = nullptr;
+    bool draining = false;
+};
+
+caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
+                              caf::actor supervisor, caf::actor device_actor, caf::cuda::program_ptr mmul_p, caf::cuda::program_ptr vadd_p, caf::cuda::program_ptr conv_p,
+                              int dev_id, int stream_id, int max_in_flight_tasks, int* d_buf) {
+    self->state().supervisor = supervisor;
+    self->state().device_actor = device_actor;
+    self->state().mmul_prog = mmul_p;
+    self->state().vadd_prog = vadd_p;
+    self->state().conv_prog = conv_p;
+    self->state().device_id = dev_id;
+    self->state().stream_id = stream_id;
+    self->state().dtoh_buffer_ptr = d_buf;
+    self->state().max_in_flight_tasks = max_in_flight_tasks;
+
+    // Trigger initial work requests up to max_in_flight_tasks
+    for (int i = 0; i < max_in_flight_tasks; ++i) {
+        self->mail(request_work_atom_v).send(self);
+    }
+
+    return {
+        [=](request_work_atom) {
+            auto& st = self->state();
+            if (st.in_flight_tasks_count >= st.max_in_flight_tasks || st.draining) {
+                return; // Already at max capacity, don't request more yet
+            }
+
+            st.in_flight_tasks_count++; // Mark as pending immediately
+            self->mail(get_work_atom_v).request(st.device_actor, infinite).then(
+                [=](int N, int type, in<int> matrixA, in<int> matrixB) {
+                    TaskType t_type = static_cast<TaskType>(type);
+                    // GPU Pipeline: Transfer -> Kernel -> Copyback
+                    auto arg1 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixA));
+                    auto arg2 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixB));
+
+                    caf::cuda::nd_range dims;
+                    caf::cuda::program_ptr prog;
+                    int out_size;
+                    if (t_type == MMUL) {
+                        dims = caf::cuda::nd_range((N+31)/32, (N+31)/32, 1, 32, 32, 1);
+                        prog = st.mmul_prog;
+                        out_size = N * N;
+                    } else if (t_type == VADD) {
+                        dims = caf::cuda::nd_range((N+255)/256, 1, 1, 256, 1, 1);
+                        prog = st.vadd_prog;
+                        out_size = N;
+                    } else {
+                        dims = caf::cuda::nd_range((N+255)/256, 1, 1, 256, 1, 1);
+                        prog = st.conv_prog;
+                        out_size = N;
+                    }
+
+                    auto result = kernel_runner.run_async(prog, dims, st.stream_id, 0, st.device_id,
+                                                         arg1, arg2, caf::cuda::create_out_arg<int>(out_size), caf::cuda::create_in_arg<int>(N));
+
+                    auto bufferC = std::get<2>(result);
+                    auto self_hdl = caf::actor_cast<caf::actor>(self);
+
+                    mmul_command.copy_to_host_async(bufferC, st.dtoh_buffer_ptr, (size_t)out_size, [self_hdl, N_task = N, type](int*, size_t) {
+                        caf::anon_mail(task_done_atom_v, N_task, type).send(self_hdl);
+                    });
+                },
+                [=](error& err) {
+                    auto& st = self->state();
+                    st.in_flight_tasks_count--; // Revert pending status on failure
+                    if (err == sec::runtime_error) {
+                        // Not enough memory, retry after a delay
+                        self->println("Worker {}: Not enough memory, retrying for work...", st.stream_id);
+                        self->delayed_anon_send(self, 100ms, request_work_atom_v);
+                    } else if (err == sec::end_of_stream) {
+                        st.draining = true; // Mark as draining, let in-flight finish
+                        if (st.in_flight_tasks_count == 0) {
+                            self->mail(worker_done_atom_v).send(st.device_actor);
+                            mmul_command.release_stream_for_actor(st.stream_id);
+                            self->quit();
+                        }
+                    }
+                }
+            );
+        },
+        [=](task_done_atom, int N_completed, int type) {
+            auto& st = self->state();
+            st.in_flight_tasks_count--; // Decrement count
+            self->mail(1).send(st.supervisor); // Notify supervisor
+            self->mail(release_memory_atom_v, N_completed, type).send(st.device_actor); // Release memory
+            
+            if (st.draining && st.in_flight_tasks_count == 0) {
+                self->mail(worker_done_atom_v).send(st.device_actor);
+                mmul_command.release_stream_for_actor(st.stream_id);
+                self->quit();
+            } else if (!st.draining) {
+                self->mail(request_work_atom_v).send(self); // Request next task if capacity allows
+            }
+        }
+    };
+}
+
+// ---------------------------- SUPERVISOR ACTOR ----------------------------
+struct supervisor_actor_state {
+    int total_tasks;
+    int completed = 0;
+    std::chrono::steady_clock::time_point start_time;
+};
+
+caf::behavior supervisor_actor_fun(
+    caf::stateful_actor<supervisor_actor_state>* self,
+    int total_tasks,
+    int workers_per_gpu,
+    int max_in_flight_tasks_per_worker,
+    MatrixPool pool,
+    std::vector<Task> tasks,
+    int* shared_dtoh_ptr
+    ) {
+    self->state().total_tasks = total_tasks;
+    self->state().start_time = std::chrono::steady_clock::now();
+
+    auto pool_actor = self->spawn(global_task_pool, std::move(tasks));
+
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+    int num_gpus = mgr.get_num_devices();
+    auto mmul_p = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+    auto vadd_p = mgr.create_program_from_cubin("../vector_add.cubin", "vectorAdd");
+    auto conv_p = mgr.create_program_from_cubin("../conv1d.cubin", "conv1d");
+
+    for (int i = 0; i < num_gpus; ++i) {
+        auto broker = self->spawn(gpu_device_actor, pool, pool_actor, workers_per_gpu, i, max_in_flight_tasks_per_worker);
+        
+        for (int j = 0; j < workers_per_gpu; ++j) {
+            self->spawn(mmul_worker_fun, self, broker, mmul_p, vadd_p, conv_p, i, (i * 1000) + j, max_in_flight_tasks_per_worker, shared_dtoh_ptr);
+        }
+    }
+
+    return {
+        [=](int done) {
+            self->state().completed += done;
+            if (self->state().completed >= self->state().total_tasks) {
+                auto end_time = std::chrono::steady_clock::now();
+                std::chrono::duration<double> total_time = end_time - self->state().start_time;
+
+                std::cout << "\n===== BENCHMARK COMPLETE =====\n";
+                std::cout << "Tasks: " << self->state().total_tasks << "\n";
+                std::cout << "Runtime: " << total_time.count() << " s\n";
+                
+                caf::cuda::manager::shutdown();
+                self->quit();
+            }
+        }
+    };
+}
+
+template <class Fn>
+double time_run(Fn&& fn) {
+  auto start = std::chrono::steady_clock::now();
+  fn();
+  auto end = std::chrono::steady_clock::now();
+  std::chrono::duration<double> elapsed = end - start;
+  return elapsed.count();
+}
+
+void run_mmul_random_scaling_tests(caf::actor_system& sys,
+                                  caf::cuda::manager_config man_config) {
+
+    const int min_N = 32;
+    const int max_N = 2048;
+    const int num_sizes = 60;
+
+    const int workers_per_gpu = 4; // Admission control: only 16 concurrent tasks per GPU
+    const int max_in_flight_tasks_per_worker = 5; // Each worker keeps 2 tasks in flight
+
+    const std::vector<int> actor_counts = {
+	  50000,100000
+    };
+
+    // Generate deterministic random pool once
+    MatrixPool pool = create_matrix_pool_random(
+        num_sizes,
+        min_N,
+        max_N,
+        42  // fixed seed
+    );
+
+    //scheduler
+    caf::cuda::manager_config scheduler_off(false);
+    for (int num_tasks_for_this_run : actor_counts) {
+	    // Initialize CUDA manager
+	    caf::cuda::manager::init(sys, scheduler_off);
+        std::cout << "=====================================\n";
+        std::cout << "Random Scaling | actors=" << num_tasks_for_this_run << "\n";
+
+        // Precompute all task Ns for this run
+        std::vector<int> sizes;
+        for (const auto& [N, _] : pool.A) sizes.push_back(N);
+        std::sort(sizes.begin(), sizes.end());
+
+        std::vector<Task> tasks_for_this_run;
+        tasks_for_this_run.reserve(num_tasks_for_this_run);
+
+        std::mt19937 rng(42);
+        std::uniform_int_distribution<size_t> dist_size(0, sizes.size() - 1);
+        std::uniform_int_distribution<int> dist_type(0, 2);
+        for (int i = 0; i < num_tasks_for_this_run; ++i) {
+            int N = sizes[dist_size(rng)];
+            TaskType type = static_cast<TaskType>(dist_type(rng));
+            tasks_for_this_run.push_back({N, type});
+        }
+
+        // Preallocate a single large host buffer for DTOH transfers to save RAM and keep things fair.
+        std::vector<int> shared_dtoh_buffer((size_t)max_N * max_N);
+
+        // Execute the supervisor which manages the asynchronous workload
+        double elapsed = time_run([&]() {
+
+            auto sup = sys.spawn(
+                supervisor_actor_fun,
+                (int)tasks_for_this_run.size(), // total_tasks
+                workers_per_gpu,
+                max_in_flight_tasks_per_worker,
+                pool,
+                tasks_for_this_run,
+                    shared_dtoh_buffer.data()
+            );
+
+	      sys.await_all_actors_done();
+	    });
+
+        caf::cuda::manager::shutdown();
+    }
+}
+void caf_main(caf::actor_system& sys) {
+  caf::cuda::manager_config man_config(false);
+  run_mmul_random_scaling_tests(sys, man_config);
+}
+CAF_MAIN(id_block::mmul_benchmark)

From 9cce977ffa5484abd784580c71d709ddcf24a785 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 13:12:12 -0600
Subject: [PATCH 0701/1000] made changes to remove memory actor and enable
 intergration with cusparse

---
 libcaf_cuda/caf/cuda/device.hpp         | 13 ++++
 libcaf_cuda/caf/cuda/manager.hpp        |  8 ---
 libcaf_cuda/caf/cuda/manager_config.hpp | 10 +--
 libcaf_cuda/caf/cuda/streampool.hpp     | 41 +++++++++++
 libcaf_cuda/src/manager.cpp             | 41 +----------
 libcaf_cuda/src/streampool.cpp          | 94 +++++++++++++++++++++++++
 6 files changed, 156 insertions(+), 51 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 92bfb1f95c..e87383165d 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -127,6 +127,18 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       cublas_table_ = std::make_unique<DeviceCublasHandleTable>(context_);
   }
 
+  /// Enable cuSparse support.
+  void enable_cusparse() {
+    if (!cusparse_table_)
+      cusparse_table_ = std::make_unique<DeviceCusparseHandleTable>(context_);
+  }
+
+  /// Returns the cuSparse handle associated with the actor id.
+  cusparseHandle_t get_cusparse_handle(int stream_id) {
+    if (!cusparse_table_) return nullptr;
+    return cusparse_table_->get_handle(stream_id, get_stream_for_actor(stream_id));
+  }
+
   /// Returns the cuBLAS handle associated with the actor id.
   cublasHandle_t get_cublas_handle(int stream_id) {
     if (!cublas_table_) return nullptr;
@@ -557,6 +569,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   const char* name_;
   std::unique_ptr<DeviceStreamTable> stream_table_;
   std::unique_ptr<DeviceCublasHandleTable> cublas_table_;
+  std::unique_ptr<DeviceCusparseHandleTable> cusparse_table_;
   std::mutex stream_mutex_;
 
   // Cached GPU properties (queried once during construction)
diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index 8b1399a56e..8a3dd55d6a 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -179,8 +179,6 @@ class CAF_CUDA_EXPORT manager {
 
   double available_memory_mb(int id = 0);
 
-  caf::actor get_memory_actor();
-
   caf::actor spawn_exit_actor(int num_actors);
 
 private:
@@ -195,14 +193,8 @@ class CAF_CUDA_EXPORT manager {
   //helper to compile a nvrtc program
   bool compile_nvrtc_program(const char* source, CUdevice device, std::vector<char>& ptx_out);
 
-  //methods to create and destroy memory_actor
-  void init_memory_actor(caf::actor_system&);
-  void destroy_memory_actor();
-
   static manager* instance_;
   static std::mutex mutex_;
-  bool memory_manager_on = false;
-  caf::actor memory_actor_handle;
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/manager_config.hpp b/libcaf_cuda/caf/cuda/manager_config.hpp
index 3cad6f9ba6..5adca8a716 100644
--- a/libcaf_cuda/caf/cuda/manager_config.hpp
+++ b/libcaf_cuda/caf/cuda/manager_config.hpp
@@ -8,16 +8,16 @@ namespace caf::cuda {
 
 class manager_config {
 public:
-    manager_config() : actorBLAS(false), memory_manager_on(false) {}
-    manager_config(bool blas) : actorBLAS(blas), memory_manager_on(false) {}
-    manager_config(bool blas, bool memory_manager) : actorBLAS(blas), memory_manager_on(memory_manager) {}
+    manager_config() : actorBLAS(false), actorSparse(false) {}
+    manager_config(bool blas) : actorBLAS(blas), actorSparse(false) {}
+    manager_config(bool blas, bool sparse) : actorBLAS(blas), actorSparse(sparse) {}
     
     bool getActorBLAS() const { return actorBLAS; }
-    bool getMemoryManagerOn() const { return memory_manager_on; }
+    bool getActorSparse() const { return actorSparse; }
 
 private:
     bool actorBLAS;
-    bool memory_manager_on = false;
+    bool actorSparse = false;
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/streampool.hpp b/libcaf_cuda/caf/cuda/streampool.hpp
index 2a3edc7d90..2841fa813a 100644
--- a/libcaf_cuda/caf/cuda/streampool.hpp
+++ b/libcaf_cuda/caf/cuda/streampool.hpp
@@ -8,6 +8,7 @@
 #include <shared_mutex>
 #include <cuda.h>
 #include <cublas_v2.h>
+#include <cusparse.h>
 #include <stdexcept>
 
 namespace caf::cuda {
@@ -112,6 +113,46 @@ class CAF_CUDA_EXPORT DeviceCublasHandleTable {
   mutable std::shared_mutex table_mutex_;
 };
 
+/// Pool of cuSparse handles. Capped at 32.
+class CAF_CUDA_EXPORT CusparseHandlePool {
+public:
+  explicit CusparseHandlePool(CUcontext ctx, size_t max_size = 32);
+  ~CusparseHandlePool();
+
+  cusparseHandle_t acquire();
+  void release(cusparseHandle_t h);
+
+  size_t max_size() const { return max_size_; }
+
+private:
+  cusparseHandle_t create_handle();
+
+  CUcontext ctx_;
+  std::deque<cusparseHandle_t> available_handles_;
+  std::vector<cusparseHandle_t> all_handles_;
+  size_t max_size_;
+  mutable std::mutex pool_mutex_;
+};
+
+/// Per-device cuSparse handle table.
+class CAF_CUDA_EXPORT DeviceCusparseHandleTable {
+public:
+  explicit DeviceCusparseHandleTable(CUcontext ctx, size_t pool_size = 32);
+
+  /// Get the cuSparse handle for an actor and bind it to a stream.
+  cusparseHandle_t get_handle(int actor_id, CUstream stream);
+
+  /// Release the handle assigned to an actor.
+  void release_handle(int actor_id);
+
+  size_t pool_size() const { return pool_.max_size(); }
+
+private:
+  CusparseHandlePool pool_;
+  std::unordered_map<int, cusparseHandle_t> table_; ///< actor_id -> handle
+  mutable std::shared_mutex table_mutex_;
+};
+
 /// Per-device stream manager. Assigns streams to actor IDs.
 /// 
 /// `DeviceStreamTable` caches an assigned stream per `actor_id`. This makes the
diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 24282683af..72067bdff0 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -55,15 +55,14 @@ void manager::init(caf::actor_system& sys, manager_config config) {
     caf::init_global_meta_objects<caf::id_block::cuda>();
     caf::init_global_meta_objects<caf::id_block::cuda_control>();
 
-    instance_->memory_manager_on = config.getMemoryManagerOn();
-
     if (config.getActorBLAS()) {
         for (auto& dev : instance_->platform_->devices())
             dev->enable_cublas();
     }
 
-    if (instance_->memory_manager_on) {
-	    instance_->init_memory_actor(sys);
+    if (config.getActorSparse()) {
+        for (auto& dev : instance_->platform_->devices())
+            dev->enable_cusparse();
     }
 }
 
@@ -93,10 +92,6 @@ void manager::shutdown() {
     if (!instance_)
         return;
 
-    if (instance_->memory_manager_on) {
-	    instance_->destroy_memory_actor();
-    }
-
     delete instance_;
     instance_ = nullptr;
 }
@@ -238,36 +233,6 @@ bool manager::compile_nvrtc_program(const char* source, CUdevice device, std::ve
 	return caf::cuda::compile_nvrtc_program(source,device,ptx_out);
 }
 
-void manager::init_memory_actor(caf::actor_system& sys) {
-    if (memory_actor_handle)
-        return; // already initialized
-
-    int num_devices = platform_->get_num_devices();
-
-    memory_actor_handle =
-        sys.spawn(memory_actor, num_devices);
-}
-
-void manager::destroy_memory_actor() {
-    if (!memory_actor_handle)
-        return;
-
-    anon_send_exit(
-        memory_actor_handle,
-        caf::exit_reason::user_shutdown
-    );
-
-    memory_actor_handle = caf::actor{};
-}
-
-caf::actor manager::get_memory_actor() {
-    if (!instance_ || !instance_->memory_actor_handle) {
-        throw std::runtime_error("Memory actor not initialized");
-    }
-
-    return instance_->memory_actor_handle;
-}
-
 caf::actor manager::spawn_exit_actor(int num_actors) {
 
 	return system_.spawn(exit_actor_fun,num_actors);
diff --git a/libcaf_cuda/src/streampool.cpp b/libcaf_cuda/src/streampool.cpp
index e569f09ad3..45df33aa26 100644
--- a/libcaf_cuda/src/streampool.cpp
+++ b/libcaf_cuda/src/streampool.cpp
@@ -268,4 +268,98 @@ void DeviceCublasHandleTable::release_handle(int actor_id) {
   }
 }
 
+// ---------------------- CusparseHandlePool ----------------------
+
+CusparseHandlePool::CusparseHandlePool(CUcontext ctx, size_t max_size)
+    : ctx_(ctx), max_size_(std::min(max_size, (size_t)32)) {}
+
+CusparseHandlePool::~CusparseHandlePool() {
+  for (auto h : all_handles_) {
+    cusparseDestroy(h);
+  }
+}
+
+cusparseHandle_t CusparseHandlePool::acquire() {
+  std::lock_guard<std::mutex> guard(pool_mutex_);
+
+  if (!available_handles_.empty()) {
+    cusparseHandle_t h = available_handles_.front();
+    available_handles_.pop_front();
+    return h;
+  }
+
+  if (all_handles_.size() < max_size_) {
+    cusparseHandle_t h = create_handle();
+    all_handles_.push_back(h);
+    return h;
+  }
+
+  if (!all_handles_.empty()) {
+    static size_t rr_idx = 0;
+    cusparseHandle_t h = all_handles_[rr_idx];
+    rr_idx = (rr_idx + 1) % all_handles_.size();
+    return h;
+  }
+
+  throw std::runtime_error("CusparseHandlePool: no handles available");
+}
+
+void CusparseHandlePool::release(cusparseHandle_t h) {
+  std::lock_guard<std::mutex> guard(pool_mutex_);
+  available_handles_.push_back(h);
+}
+
+cusparseHandle_t CusparseHandlePool::create_handle() {
+  CHECK_CUDA(cuCtxPushCurrent(ctx_));
+  cusparseHandle_t h;
+  cusparseStatus_t status = cusparseCreate(&h);
+  CHECK_CUDA(cuCtxPopCurrent(nullptr));
+
+  if (status != CUSPARSE_STATUS_SUCCESS)
+    throw std::runtime_error("cusparseCreate failed");
+  return h;
+}
+
+// ---------------------- DeviceCusparseHandleTable ----------------------
+
+DeviceCusparseHandleTable::DeviceCusparseHandleTable(CUcontext ctx, size_t pool_size)
+    : pool_(ctx, pool_size) {}
+
+cusparseHandle_t DeviceCusparseHandleTable::get_handle(int actor_id, CUstream stream) {
+  cusparseHandle_t h = nullptr;
+  {
+    std::shared_lock<std::shared_mutex> read_lock(table_mutex_);
+    auto it = table_.find(actor_id);
+    if (it != table_.end())
+      h = it->second;
+  }
+
+  if (!h) {
+    std::unique_lock<std::shared_mutex> write_lock(table_mutex_);
+    auto it = table_.find(actor_id);
+    if (it != table_.end()) {
+      h = it->second;
+    } else {
+      h = pool_.acquire();
+      table_[actor_id] = h;
+    }
+  }
+
+  // Always bind the handle to the caller's stream
+  cusparseStatus_t status = cusparseSetStream(h, stream);
+  if (status != CUSPARSE_STATUS_SUCCESS)
+    throw std::runtime_error("cusparseSetStream failed");
+
+  return h;
+}
+
+void DeviceCusparseHandleTable::release_handle(int actor_id) {
+  std::unique_lock<std::shared_mutex> write_lock(table_mutex_);
+  auto it = table_.find(actor_id);
+  if (it != table_.end()) {
+    pool_.release(it->second);
+    table_.erase(it);
+  }
+}
+
 } // namespace caf::cuda

From 74aeffa8630d7feb28c8bf935bd5de913e5dc540 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 13:13:27 -0600
Subject: [PATCH 0702/1000] Initial commit.

---
 libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp

diff --git a/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp b/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp
new file mode 100644
index 0000000000..6f70f09bee
--- /dev/null
+++ b/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp
@@ -0,0 +1 @@
+#pragma once

From 5b693122fb40c911f577c15207a63a4718cc1d62 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 13:45:46 -0600
Subject: [PATCH 0703/1000] Implemented changes required to implement intial
 version of spmv-actor

---
 libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp   |   2 +
 .../caf/actorSPARSE/spmv-actor/spmv-actor.hpp | 253 ++++++++++++++++++
 libcaf_cuda/caf/cuda/device.hpp               |  92 +++++++
 libcaf_cuda/caf/cuda/global.hpp               |   3 +
 4 files changed, 350 insertions(+)
 create mode 100644 libcaf_cuda/caf/actorSPARSE/spmv-actor/spmv-actor.hpp

diff --git a/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp b/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp
index 6f70f09bee..df1193f42b 100644
--- a/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp
+++ b/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp
@@ -1 +1,3 @@
 #pragma once
+
+#include "caf/actorSPARSE/spmv-actor.hpp"
diff --git a/libcaf_cuda/caf/actorSPARSE/spmv-actor/spmv-actor.hpp b/libcaf_cuda/caf/actorSPARSE/spmv-actor/spmv-actor.hpp
new file mode 100644
index 0000000000..0ff75be856
--- /dev/null
+++ b/libcaf_cuda/caf/actorSPARSE/spmv-actor/spmv-actor.hpp
@@ -0,0 +1,253 @@
+#pragma once
+
+#include <caf/actor.hpp>
+#include <caf/actor_cast.hpp>
+#include <caf/actor_config.hpp>
+#include <caf/actor_system.hpp>
+#include <caf/anon_mail.hpp>
+#include <caf/event_based_actor.hpp>
+
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/mem_ref.hpp"
+#include "caf/cuda/command_runner.hpp"
+#include "caf/cuda/platform.hpp"
+#include "caf/cuda/types.hpp"
+
+namespace caf::cuda {
+
+/// SPMV Actor for single-precision sparse matrix-vector multiplication.
+/// Message Signature: (csr_atom, in<int> row_ptr, in<int> col_ind, in<float> values, in<float> x, out<float> y, int m, int n, int nnz, [float alpha, float beta])
+class spmv_actor : public event_based_actor {
+public:
+  static caf::actor spawn(caf::actor_system& sys, int reply_id = 0) {
+    return sys.spawn<spmv_actor>(reply_id);
+  }
+
+  spmv_actor(caf::actor_config& cfg, int reply_id = 0) 
+    : event_based_actor(cfg), reply_id_(reply_id) {
+    actor_id_ = static_cast<int>(this->id());
+  }
+
+  ~spmv_actor() override {
+    command_runner<> runner;
+    runner.release_stream_for_actor(actor_id_);
+  }
+
+  caf::behavior make_behavior() override {
+    return {
+      // CSR Host-based overloads
+      [this](csr_atom, in<int> row_ptr, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csr(-1, actor_id_, row_ptr, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csr_atom, in<int> row_ptr, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csr(-1, actor_id_, row_ptr, col_ind, val, x, y, m, n, nnz, alpha, beta, false);
+      },
+      // CSR Routing overloads
+      [this](csr_atom, int device_num, int stream_id, in<int> row_ptr, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csr(device_num, stream_id, row_ptr, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csr_atom, int device_num, int stream_id, in<int> row_ptr, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csr(device_num, stream_id, row_ptr, col_ind, val, x, y, m, n, nnz, alpha, beta, false);
+      },
+      // CSR mem_ptr overloads
+      [this](csr_atom, mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csr(-1, actor_id_, row_ptr, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csr_atom, mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csr(-1, actor_id_, row_ptr, col_ind, val, x, y, m, n, nnz, alpha, beta, false);
+      },
+      [this](csr_atom, int device_num, int stream_id, mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csr(device_num, stream_id, row_ptr, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csr_atom, int device_num, int stream_id, mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csr(device_num, stream_id, row_ptr, col_ind, val, x, y, m, n, nnz, alpha, beta, false);
+      },
+      // CSC Host-based overloads
+      [this](csc_atom, in<int> col_ptr, in<int> row_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csc(-1, actor_id_, col_ptr, row_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csc_atom, in<int> col_ptr, in<int> row_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csc(-1, actor_id_, col_ptr, row_ind, val, x, y, m, n, nnz, alpha, beta, false);
+      },
+      // CSC mem_ptr overloads
+      [this](csc_atom, mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csc(-1, actor_id_, col_ptr, row_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csc_atom, int device_num, int stream_id, mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csc(device_num, stream_id, col_ptr, row_ind, val, x, y, m, n, nnz, alpha, beta, false);
+      },
+      // COO Host-based overloads
+      [this](coo_atom, in<int> row_ind, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz) {
+        enqueue_spmv_coo(-1, actor_id_, row_ind, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, false);
+      },
+      [this](coo_atom, in<int> row_ind, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_coo(-1, actor_id_, row_ind, col_ind, val, x, y, m, n, nnz, alpha, beta, false);
+      },
+      // COO mem_ptr overloads
+      [this](coo_atom, mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz) {
+        enqueue_spmv_coo(-1, actor_id_, row_ind, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, false);
+      },
+      [this](coo_atom, int device_num, int stream_id, mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_coo(device_num, stream_id, row_ind, col_ind, val, x, y, m, n, nnz, alpha, beta, false);
+      },
+      // return_mem_ptr_atom variants
+      [this](return_mem_ptr_atom, csr_atom, in<int> row_ptr, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csr(-1, actor_id_, row_ptr, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, csr_atom, in<int> row_ptr, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csr(-1, actor_id_, row_ptr, col_ind, val, x, y, m, n, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, csr_atom, int device_num, int stream_id, in<int> row_ptr, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csr(device_num, stream_id, row_ptr, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, csr_atom, int device_num, int stream_id, in<int> row_ptr, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csr(device_num, stream_id, row_ptr, col_ind, val, x, y, m, n, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, csr_atom, mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csr(-1, actor_id_, row_ptr, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, csr_atom, mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csr(-1, actor_id_, row_ptr, col_ind, val, x, y, m, n, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, csr_atom, int device_num, int stream_id, mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csr(device_num, stream_id, row_ptr, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, csr_atom, int device_num, int stream_id, mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csr(device_num, stream_id, row_ptr, col_ind, val, x, y, m, n, nnz, alpha, beta, true);
+      },
+      // CSC return_mem_ptr_atom variants
+      [this](return_mem_ptr_atom, csc_atom, in<int> col_ptr, in<int> row_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csc(-1, actor_id_, col_ptr, row_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, csc_atom, mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csc(-1, actor_id_, col_ptr, row_ind, val, x, y, m, n, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, csc_atom, int device_num, int stream_id, mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csc(device_num, stream_id, col_ptr, row_ind, val, x, y, m, n, nnz, alpha, beta, true);
+      },
+      // COO return_mem_ptr_atom variants
+      [this](return_mem_ptr_atom, coo_atom, in<int> row_ind, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz) {
+        enqueue_spmv_coo(-1, actor_id_, row_ind, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, coo_atom, mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_coo(-1, actor_id_, row_ind, col_ind, val, x, y, m, n, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, coo_atom, int device_num, int stream_id, mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_coo(device_num, stream_id, row_ind, col_ind, val, x, y, m, n, nnz, alpha, beta, true);
+      },
+    };
+  }
+
+private:
+  void enqueue_spmv_csr(int device_num, int stream_id, 
+                        in<int> row_ptr, in<int> col_ind, in<float> val, in<float> x, out<float> y, 
+                        int m, int n, int nnz, float alpha, float beta, bool return_ptrs) {
+    command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, row_ptr, col_ind, val, x, y);
+    execute_and_reply_csr(device_num, stream_id, std::get<0>(results), std::get<1>(results), 
+                          std::get<2>(results), std::get<3>(results), std::get<4>(results), 
+                          m, n, nnz, alpha, beta, return_ptrs);
+  }
+
+  void enqueue_spmv_csr(int device_num, int stream_id, 
+                        mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, 
+                        int m, int n, int nnz, float alpha, float beta, bool return_ptrs) {
+    command_runner<mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, mem_ptr<float>, mem_ptr<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, row_ptr, col_ind, val, x, y);
+    execute_and_reply_csr(device_num, stream_id, std::get<0>(results), std::get<1>(results), 
+                          std::get<2>(results), std::get<3>(results), std::get<4>(results), 
+                          m, n, nnz, alpha, beta, return_ptrs);
+  }
+
+  void execute_and_reply_csr(int device_num, int stream_id, 
+                             mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, 
+                             int m, int n, int nnz, float alpha, float beta, bool return_ptrs) {
+    auto plat = platform::create();
+    device_ptr dev = (device_num == -1) ? plat->schedule(stream_id) : plat->schedule(stream_id, device_num);
+    dev->spmv_csr(stream_id, m, n, nnz, alpha, row_ptr, col_ind, val, x, beta, y);
+    handle_reply(device_num, stream_id, row_ptr, col_ind, val, x, y, return_ptrs);
+  }
+
+  // CSC Logic
+  void enqueue_spmv_csc(int device_num, int stream_id, 
+                        in<int> col_ptr, in<int> row_ind, in<float> val, in<float> x, out<float> y, 
+                        int m, int n, int nnz, float alpha, float beta, bool return_ptrs) {
+    command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, col_ptr, row_ind, val, x, y);
+    execute_and_reply_csc(device_num, stream_id, std::get<0>(results), std::get<1>(results), 
+                          std::get<2>(results), std::get<3>(results), std::get<4>(results), 
+                          m, n, nnz, alpha, beta, return_ptrs);
+  }
+
+  void enqueue_spmv_csc(int device_num, int stream_id, 
+                        mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, 
+                        int m, int n, int nnz, float alpha, float beta, bool return_ptrs) {
+    command_runner<mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, mem_ptr<float>, mem_ptr<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, col_ptr, row_ind, val, x, y);
+    execute_and_reply_csc(device_num, stream_id, std::get<0>(results), std::get<1>(results), 
+                          std::get<2>(results), std::get<3>(results), std::get<4>(results), 
+                          m, n, nnz, alpha, beta, return_ptrs);
+  }
+
+  void execute_and_reply_csc(int device_num, int stream_id, 
+                             mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, 
+                             int m, int n, int nnz, float alpha, float beta, bool return_ptrs) {
+    auto plat = platform::create();
+    device_ptr dev = (device_num == -1) ? plat->schedule(stream_id) : plat->schedule(stream_id, device_num);
+    dev->spmv_csc(stream_id, m, n, nnz, alpha, col_ptr, row_ind, val, x, beta, y);
+    handle_reply(device_num, stream_id, col_ptr, row_ind, val, x, y, return_ptrs);
+  }
+
+  // COO Logic
+  void enqueue_spmv_coo(int device_num, int stream_id, 
+                        in<int> row_ind, in<int> col_ind, in<float> val, in<float> x, out<float> y, 
+                        int m, int n, int nnz, float alpha, float beta, bool return_ptrs) {
+    command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, row_ind, col_ind, val, x, y);
+    execute_and_reply_coo(device_num, stream_id, std::get<0>(results), std::get<1>(results), 
+                          std::get<2>(results), std::get<3>(results), std::get<4>(results), 
+                          m, n, nnz, alpha, beta, return_ptrs);
+  }
+
+  void enqueue_spmv_coo(int device_num, int stream_id, 
+                        mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, 
+                        int m, int n, int nnz, float alpha, float beta, bool return_ptrs) {
+    command_runner<mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, mem_ptr<float>, mem_ptr<float>> runner;
+    auto results = runner.transfer_memory(device_num, stream_id, row_ind, col_ind, val, x, y);
+    execute_and_reply_coo(device_num, stream_id, std::get<0>(results), std::get<1>(results), 
+                          std::get<2>(results), std::get<3>(results), std::get<4>(results), 
+                          m, n, nnz, alpha, beta, return_ptrs);
+  }
+
+  void execute_and_reply_coo(int device_num, int stream_id, 
+                             mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, 
+                             int m, int n, int nnz, float alpha, float beta, bool return_ptrs) {
+    auto plat = platform::create();
+    device_ptr dev = (device_num == -1) ? plat->schedule(stream_id) : plat->schedule(stream_id, device_num);
+    dev->spmv_coo(stream_id, m, n, nnz, alpha, row_ind, col_ind, val, x, beta, y);
+    handle_reply(device_num, stream_id, row_ind, col_ind, val, x, y, return_ptrs);
+  }
+
+  void handle_reply(int, int, 
+                    mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, 
+                    bool return_ptrs) {
+    command_runner<mem_ptr<float>> runner;
+    auto sender = actor_cast<actor>(this->current_sender());
+    if (!sender) return;
+    auto r_id = reply_id_;
+    if (return_ptrs) {
+      caf::anon_mail(r_id, row_ptr, col_ind, val, x, y).send(sender);
+    } else {
+      runner.copy_to_host_async(y, [sender, r_id](std::vector<float>&& data) {
+        if (sender) {
+          caf::anon_mail(r_id, 4, std::move(data)).send(sender);
+        }
+      });
+    }
+  }
+
+  int actor_id_;
+  int reply_id_;
+};
+
+} // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index e87383165d..7c5f9eb3a0 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -173,6 +173,98 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasSgemv failed on device " + std::to_string(id_));
   }
 
+  /// Performs sparse matrix-vector multiplication (y = alpha*A*x + beta*y) using CSR format.
+  void spmv_csr(int stream_id, int m, int n, int nnz, float alpha, 
+                mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> values,
+                mem_ptr<float> x, float beta, mem_ptr<float> y) {
+    cusparseHandle_t handle = get_cusparse_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    
+    cusparseMatDescr_t descr;
+    cusparseCreateMatDescr(&descr);
+    cusparseSetMatType(descr, CUSPARSE_MATRIX_TYPE_GENERAL);
+    cusparseSetMatIndexBase(descr, CUSPARSE_INDEX_BASE_ZERO);
+
+    cusparseStatus_t status = cusparseScsrmv(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                             m, n, nnz, &alpha, descr,
+                                             reinterpret_cast<const float*>(values->mem()),
+                                             reinterpret_cast<const int*>(row_ptr->mem()),
+                                             reinterpret_cast<const int*>(col_ind->mem()),
+                                             reinterpret_cast<const float*>(x->mem()),
+                                             &beta,
+                                             reinterpret_cast<float*>(y->mem()));
+
+    cusparseDestroyMatDescr(descr);
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUSPARSE_STATUS_SUCCESS)
+      throw std::runtime_error("cusparseScsrmv failed on device " + std::to_string(id_));
+  }
+
+  /// Performs sparse matrix-vector multiplication (y = alpha*A*x + beta*y) using COO format.
+  void spmv_coo(int stream_id, int m, int n, int nnz, float alpha, 
+                mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> values,
+                mem_ptr<float> x, float beta, mem_ptr<float> y) {
+    cusparseHandle_t handle = get_cusparse_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    
+    cusparseMatDescr_t descr;
+    cusparseCreateMatDescr(&descr);
+    cusparseSetMatType(descr, CUSPARSE_MATRIX_TYPE_GENERAL);
+    cusparseSetMatIndexBase(descr, CUSPARSE_INDEX_BASE_ZERO);
+
+    cusparseStatus_t status = cusparseScoomv(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                             m, n, nnz, &alpha, descr,
+                                             reinterpret_cast<const float*>(values->mem()),
+                                             reinterpret_cast<const int*>(row_ind->mem()),
+                                             reinterpret_cast<const int*>(col_ind->mem()),
+                                             reinterpret_cast<const float*>(x->mem()),
+                                             &beta,
+                                             reinterpret_cast<float*>(y->mem()));
+
+    cusparseDestroyMatDescr(descr);
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUSPARSE_STATUS_SUCCESS)
+      throw std::runtime_error("cusparseScoomv failed on device " + std::to_string(id_));
+  }
+
+  /// Performs sparse matrix-vector multiplication (y = alpha*A*x + beta*y) using CSC format.
+  void spmv_csc(int stream_id, int m, int n, int nnz, float alpha, 
+                mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> values,
+                mem_ptr<float> x, float beta, mem_ptr<float> y) {
+    cusparseHandle_t handle = get_cusparse_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    
+    cusparseMatDescr_t descr;
+    cusparseCreateMatDescr(&descr);
+    cusparseSetMatType(descr, CUSPARSE_MATRIX_TYPE_GENERAL);
+    cusparseSetMatIndexBase(descr, CUSPARSE_INDEX_BASE_ZERO);
+
+    // CSC is essentially CSR of the transpose. 
+    // Passing CUSPARSE_OPERATION_TRANSPOSE and swapping m/n dimensions correctly maps CSC data.
+    cusparseStatus_t status = cusparseScsrmv(handle, CUSPARSE_OPERATION_TRANSPOSE,
+                                             n, m, nnz, &alpha, descr,
+                                             reinterpret_cast<const float*>(values->mem()),
+                                             reinterpret_cast<const int*>(col_ptr->mem()),
+                                             reinterpret_cast<const int*>(row_ind->mem()),
+                                             reinterpret_cast<const float*>(x->mem()),
+                                             &beta,
+                                             reinterpret_cast<float*>(y->mem()));
+
+    cusparseDestroyMatDescr(descr);
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUSPARSE_STATUS_SUCCESS)
+      throw std::runtime_error("cusparseScsrmv (CSC) failed on device " + std::to_string(id_));
+  }
+
   /// Performs symmetric rank-k update (C = alpha*A*A^T + beta*C).
   /// Assumes A is in row-major order of dimensions n x k, and C is n x n.
   void ssyrk(int stream_id, int n, int k, float alpha, mem_ptr<float> A,
diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index a6af5edeb5..447080012d 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -214,6 +214,9 @@ CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id)
   CAF_ADD_ATOM(cuda, dtoh_done_atom)
   CAF_ADD_ATOM(cuda, gpu_done_atom)
   CAF_ADD_ATOM(cuda, return_mem_ptr_atom)
+  CAF_ADD_ATOM(cuda, csr_atom)
+  CAF_ADD_ATOM(cuda, csc_atom)
+  CAF_ADD_ATOM(cuda, coo_atom)
 
 CAF_END_TYPE_ID_BLOCK(cuda)
 

From 7b7f455136651d8c4ccb0a5c4422784726a22a62 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 13:46:54 -0600
Subject: [PATCH 0704/1000] Initial commit.

---
 .../spmv-actor-test/CMakeLists.txt            |  45 ++++++
 .../spmv-actor-test/main.test.cpp             | 142 ++++++++++++++++++
 2 files changed, 187 insertions(+)
 create mode 100644 libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/main.test.cpp

diff --git a/libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..fa9dc8e759
--- /dev/null
+++ b/libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/CMakeLists.txt
@@ -0,0 +1,45 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+)
+
diff --git a/libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/main.test.cpp
new file mode 100644
index 0000000000..2e070afabb
--- /dev/null
+++ b/libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/main.test.cpp
@@ -0,0 +1,142 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include "caf/actor_registry.hpp"
+#include "caf/actorBLAS/gemv-actor/gemv-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+// Verify GEMV result: y = alpha * A * x + beta * y
+// With A (all 3.0s) and x (all 4.0s), the dot product sum is (3.0 * 4.0 * n).
+// We use alpha = 1.0 / n to normalize the output to exactly 12.0f as requested.
+void verify_gemv_correctness(int m, int n, float alpha, float beta, 
+                             const std::vector<float>& y_result) {
+    float expected_val = alpha * (3.0f * 4.0f * n); 
+    bool all_correct = true;
+    for (size_t i = 0; i < y_result.size(); ++i) {
+        if (std::abs(y_result[i] - expected_val) > 1e-4) {
+            all_correct = false;
+            std::cout << "[ERROR] Mismatch at index " << i 
+                      << ": Expected " << expected_val 
+                      << ", Got " << y_result[i] << std::endl;
+            break;
+        }
+    }
+
+    if (all_correct) {
+        std::cout << "[SUCCESS] GEMV actor produced correct results." << std::endl;
+        std::cout << "          Output vector is filled with: " << expected_val << std::endl;
+    }
+}
+
+void caf_main(actor_system& sys) {
+    // Initialize the manager with BLAS enabled to initialize cuBLAS handles
+    manager_config config(true); 
+    manager::init(sys, config);
+
+    // Matrix dimensions 64x64 (> 32x32)
+    int m = 64;
+    int n = 64;
+    
+    // Scalar alpha set to 1/n to normalize the dot product sum (3*4*n) to 12.0
+    float alpha = 1.0f / static_cast<float>(n);
+    float beta = 0.0f;
+
+    // Initialize host data
+    std::vector<float> h_A(m * n, 3.0f);
+    std::vector<float> h_x(n, 4.0f);
+    std::vector<float> h_y(m, 0.0f);
+
+    // Spawn the gemv_actor
+    auto blas_actor = sys.spawn<gemv_actor>(1);
+
+    // Prepare arguments using wrapper tags
+    auto A_arg = create_in_arg(h_A);
+    auto x_arg = create_in_arg(h_x);
+    auto y_arg = create_out_arg(h_y);
+
+    scoped_actor self{sys};
+
+    // Test 1: Standard host-buffer based call
+    {
+        std::cout << "[INFO] Test 1: Testing gemv_actor with host-buffer arguments..." << std::endl;
+        self->mail(A_arg, x_arg, y_arg, m, n, alpha, beta).send(blas_actor);
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 2) { // index 2 corresponds to y_arg
+                    verify_gemv_correctness(m, n, alpha, beta, data);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 1 complete." << std::endl;
+    }
+
+    // Test 2: mem_ptr inputs
+    {
+        std::cout << "\n[INFO] Test 2: Testing gemv_actor with mem_ptr inputs..." << std::endl;
+        command_runner<in<float>, in<float>, out<float>> setup_runner;
+        // Manually transfer data to the GPU to get mem_ptr handles
+        auto results = setup_runner.transfer_memory(0, 0, create_in_arg(h_A), create_in_arg(h_x), create_out_arg(h_y));
+        auto A_ptr = std::get<0>(results);
+        auto x_ptr = std::get<1>(results);
+        auto y_ptr = std::get<2>(results);
+
+        self->mail(A_ptr, x_ptr, y_ptr, m, n, alpha, beta).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 2) {
+                    verify_gemv_correctness(m, n, alpha, beta, data);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 2 complete." << std::endl;
+    }
+
+    // Test 3: Routing control (device/stream) + mem_ptr
+    {
+        std::cout << "\n[INFO] Test 3: Testing gemv_actor with specific device/stream and mem_ptr..." << std::endl;
+        int device_num = 0;
+        int stream_id = 42; 
+        command_runner<in<float>, in<float>, out<float>> setup_runner;
+        auto results = setup_runner.transfer_memory(device_num, stream_id, create_in_arg(h_A), create_in_arg(h_x), create_out_arg(h_y));
+        
+        self->mail(device_num, stream_id, std::get<0>(results), std::get<1>(results), std::get<2>(results), m, n, alpha, beta).send(blas_actor);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                if (arg_index == 2) {
+                    verify_gemv_correctness(m, n, alpha, beta, data);
+                }
+            }
+        );
+        std::cout << "[INFO] Test 3 complete." << std::endl;
+    }
+
+    // Test 4: return_mem_ptr_atom (returning device handles)
+    {
+        std::cout << "\n[INFO] Test 4: Testing gemv_actor with return_mem_ptr_atom..." << std::endl;
+        self->mail(return_mem_ptr_atom{}, A_arg, x_arg, y_arg, m, n, alpha, beta).send(blas_actor);
+
+        // We expect two messages back: the data (mem_ptrs) and the signal (-1)
+        self->receive(
+            [&](int reply_id, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y) {
+                command_runner<float> runner;
+                auto host_y = runner.copy_to_host(y);
+                verify_gemv_correctness(m, n, alpha, beta, host_y);
+            }
+        );
+        std::cout << "[INFO] Test 4 complete." << std::endl;
+    }
+
+    self->send_exit(blas_actor, exit_reason::user_shutdown);
+    manager::shutdown();
+}
+CAF_MAIN(id_block::cuda)

From df6fc875fbb8527872c6e9731343328a77e4cd32 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 13:49:02 -0600
Subject: [PATCH 0705/1000] updated header

---
 libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp b/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp
index df1193f42b..ccec3760b4 100644
--- a/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp
+++ b/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp
@@ -1,3 +1,3 @@
 #pragma once
 
-#include "caf/actorSPARSE/spmv-actor.hpp"
+#include "caf/actorSPARSE/spmv-actor/spmv-actor.hpp"

From c6f44162371ce96c4ccda129659d7660eda29232 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 14:04:09 -0600
Subject: [PATCH 0706/1000] updated device cusparse methods to not be legacy

---
 libcaf_cuda/caf/cuda/device.hpp | 160 +++++++++++++++++++++-----------
 1 file changed, 107 insertions(+), 53 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 7c5f9eb3a0..20bf8b2d8a 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -182,25 +182,44 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
 
     CHECK_CUDA(cuCtxPushCurrent(context_));
-    
-    cusparseMatDescr_t descr;
-    cusparseCreateMatDescr(&descr);
-    cusparseSetMatType(descr, CUSPARSE_MATRIX_TYPE_GENERAL);
-    cusparseSetMatIndexBase(descr, CUSPARSE_INDEX_BASE_ZERO);
-
-    cusparseStatus_t status = cusparseScsrmv(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                                             m, n, nnz, &alpha, descr,
-                                             reinterpret_cast<const float*>(values->mem()),
-                                             reinterpret_cast<const int*>(row_ptr->mem()),
-                                             reinterpret_cast<const int*>(col_ind->mem()),
-                                             reinterpret_cast<const float*>(x->mem()),
-                                             &beta,
-                                             reinterpret_cast<float*>(y->mem()));
-
-    cusparseDestroyMatDescr(descr);
+    CUstream stream = get_stream_for_actor(stream_id);
+
+    cusparseSpMatDescr_t matA;
+    cusparseCreateCsr(&matA, m, n, nnz,
+                      reinterpret_cast<void*>(row_ptr->mem()),
+                      reinterpret_cast<void*>(col_ind->mem()),
+                      reinterpret_cast<void*>(values->mem()),
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                      CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+
+    cusparseDnVecDescr_t vecX, vecY;
+    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
+    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
+
+    size_t bufferSize = 0;
+    cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                            &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                            CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
+
+    CUdeviceptr dBuffer = 0;
+    if (bufferSize > 0)
+      CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+
+    cusparseStatus_t status = cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                                           CUSPARSE_SPMV_ALG_DEFAULT,
+                                           reinterpret_cast<void*>(dBuffer));
+
+    if (dBuffer)
+      CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
+
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnVec(vecX);
+    cusparseDestroyDnVec(vecY);
+
     CHECK_CUDA(cuCtxPopCurrent(nullptr));
     if (status != CUSPARSE_STATUS_SUCCESS)
-      throw std::runtime_error("cusparseScsrmv failed on device " + std::to_string(id_));
+      throw std::runtime_error("cusparseSpMV (CSR) failed on device " + std::to_string(id_));
   }
 
   /// Performs sparse matrix-vector multiplication (y = alpha*A*x + beta*y) using COO format.
@@ -212,25 +231,43 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
 
     CHECK_CUDA(cuCtxPushCurrent(context_));
-    
-    cusparseMatDescr_t descr;
-    cusparseCreateMatDescr(&descr);
-    cusparseSetMatType(descr, CUSPARSE_MATRIX_TYPE_GENERAL);
-    cusparseSetMatIndexBase(descr, CUSPARSE_INDEX_BASE_ZERO);
-
-    cusparseStatus_t status = cusparseScoomv(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                                             m, n, nnz, &alpha, descr,
-                                             reinterpret_cast<const float*>(values->mem()),
-                                             reinterpret_cast<const int*>(row_ind->mem()),
-                                             reinterpret_cast<const int*>(col_ind->mem()),
-                                             reinterpret_cast<const float*>(x->mem()),
-                                             &beta,
-                                             reinterpret_cast<float*>(y->mem()));
-
-    cusparseDestroyMatDescr(descr);
+    CUstream stream = get_stream_for_actor(stream_id);
+
+    cusparseSpMatDescr_t matA;
+    cusparseCreateCoo(&matA, m, n, nnz,
+                      reinterpret_cast<void*>(row_ind->mem()),
+                      reinterpret_cast<void*>(col_ind->mem()),
+                      reinterpret_cast<void*>(values->mem()),
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+
+    cusparseDnVecDescr_t vecX, vecY;
+    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
+    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
+
+    size_t bufferSize = 0;
+    cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                            &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                            CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
+
+    CUdeviceptr dBuffer = 0;
+    if (bufferSize > 0)
+      CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+
+    cusparseStatus_t status = cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                                           CUSPARSE_SPMV_ALG_DEFAULT,
+                                           reinterpret_cast<void*>(dBuffer));
+
+    if (dBuffer)
+      CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
+
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnVec(vecX);
+    cusparseDestroyDnVec(vecY);
+
     CHECK_CUDA(cuCtxPopCurrent(nullptr));
     if (status != CUSPARSE_STATUS_SUCCESS)
-      throw std::runtime_error("cusparseScoomv failed on device " + std::to_string(id_));
+      throw std::runtime_error("cusparseSpMV (COO) failed on device " + std::to_string(id_));
   }
 
   /// Performs sparse matrix-vector multiplication (y = alpha*A*x + beta*y) using CSC format.
@@ -242,27 +279,44 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
 
     CHECK_CUDA(cuCtxPushCurrent(context_));
-    
-    cusparseMatDescr_t descr;
-    cusparseCreateMatDescr(&descr);
-    cusparseSetMatType(descr, CUSPARSE_MATRIX_TYPE_GENERAL);
-    cusparseSetMatIndexBase(descr, CUSPARSE_INDEX_BASE_ZERO);
-
-    // CSC is essentially CSR of the transpose. 
-    // Passing CUSPARSE_OPERATION_TRANSPOSE and swapping m/n dimensions correctly maps CSC data.
-    cusparseStatus_t status = cusparseScsrmv(handle, CUSPARSE_OPERATION_TRANSPOSE,
-                                             n, m, nnz, &alpha, descr,
-                                             reinterpret_cast<const float*>(values->mem()),
-                                             reinterpret_cast<const int*>(col_ptr->mem()),
-                                             reinterpret_cast<const int*>(row_ind->mem()),
-                                             reinterpret_cast<const float*>(x->mem()),
-                                             &beta,
-                                             reinterpret_cast<float*>(y->mem()));
-
-    cusparseDestroyMatDescr(descr);
+    CUstream stream = get_stream_for_actor(stream_id);
+
+    cusparseSpMatDescr_t matA;
+    cusparseCreateCsc(&matA, m, n, nnz,
+                      reinterpret_cast<void*>(col_ptr->mem()),
+                      reinterpret_cast<void*>(row_ind->mem()),
+                      reinterpret_cast<void*>(values->mem()),
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                      CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+
+    cusparseDnVecDescr_t vecX, vecY;
+    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
+    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
+
+    size_t bufferSize = 0;
+    cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                            &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                            CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
+
+    CUdeviceptr dBuffer = 0;
+    if (bufferSize > 0)
+      CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+
+    cusparseStatus_t status = cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                                           CUSPARSE_SPMV_ALG_DEFAULT,
+                                           reinterpret_cast<void*>(dBuffer));
+
+    if (dBuffer)
+      CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
+
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnVec(vecX);
+    cusparseDestroyDnVec(vecY);
+
     CHECK_CUDA(cuCtxPopCurrent(nullptr));
     if (status != CUSPARSE_STATUS_SUCCESS)
-      throw std::runtime_error("cusparseScsrmv (CSC) failed on device " + std::to_string(id_));
+      throw std::runtime_error("cusparseSpMV (CSC) failed on device " + std::to_string(id_));
   }
 
   /// Performs symmetric rank-k update (C = alpha*A*A^T + beta*C).

From 1ece5941815a7adee1166eea98c59c16457d0e6d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 14:08:12 -0600
Subject: [PATCH 0707/1000] added more message handlers

---
 .../caf/actorSPARSE/spmv-actor/spmv-actor.hpp | 56 +++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/libcaf_cuda/caf/actorSPARSE/spmv-actor/spmv-actor.hpp b/libcaf_cuda/caf/actorSPARSE/spmv-actor/spmv-actor.hpp
index 0ff75be856..0a80c823c6 100644
--- a/libcaf_cuda/caf/actorSPARSE/spmv-actor/spmv-actor.hpp
+++ b/libcaf_cuda/caf/actorSPARSE/spmv-actor/spmv-actor.hpp
@@ -69,10 +69,23 @@ class spmv_actor : public event_based_actor {
       [this](csc_atom, in<int> col_ptr, in<int> row_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz, float alpha, float beta) {
         enqueue_spmv_csc(-1, actor_id_, col_ptr, row_ind, val, x, y, m, n, nnz, alpha, beta, false);
       },
+      // CSC Routing overloads
+      [this](csc_atom, int device_num, int stream_id, in<int> col_ptr, in<int> row_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csc(device_num, stream_id, col_ptr, row_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csc_atom, int device_num, int stream_id, in<int> col_ptr, in<int> row_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csc(device_num, stream_id, col_ptr, row_ind, val, x, y, m, n, nnz, alpha, beta, false);
+      },
       // CSC mem_ptr overloads
       [this](csc_atom, mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz) {
         enqueue_spmv_csc(-1, actor_id_, col_ptr, row_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, false);
       },
+      [this](csc_atom, mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csc(-1, actor_id_, col_ptr, row_ind, val, x, y, m, n, nnz, alpha, beta, false);
+      },
+      [this](csc_atom, int device_num, int stream_id, mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csc(device_num, stream_id, col_ptr, row_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, false);
+      },
       [this](csc_atom, int device_num, int stream_id, mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
         enqueue_spmv_csc(device_num, stream_id, col_ptr, row_ind, val, x, y, m, n, nnz, alpha, beta, false);
       },
@@ -83,10 +96,23 @@ class spmv_actor : public event_based_actor {
       [this](coo_atom, in<int> row_ind, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz, float alpha, float beta) {
         enqueue_spmv_coo(-1, actor_id_, row_ind, col_ind, val, x, y, m, n, nnz, alpha, beta, false);
       },
+      // COO Routing overloads
+      [this](coo_atom, int device_num, int stream_id, in<int> row_ind, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz) {
+        enqueue_spmv_coo(device_num, stream_id, row_ind, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, false);
+      },
+      [this](coo_atom, int device_num, int stream_id, in<int> row_ind, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_coo(device_num, stream_id, row_ind, col_ind, val, x, y, m, n, nnz, alpha, beta, false);
+      },
       // COO mem_ptr overloads
       [this](coo_atom, mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz) {
         enqueue_spmv_coo(-1, actor_id_, row_ind, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, false);
       },
+      [this](coo_atom, mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_coo(-1, actor_id_, row_ind, col_ind, val, x, y, m, n, nnz, alpha, beta, false);
+      },
+      [this](coo_atom, int device_num, int stream_id, mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz) {
+        enqueue_spmv_coo(device_num, stream_id, row_ind, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, false);
+      },
       [this](coo_atom, int device_num, int stream_id, mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
         enqueue_spmv_coo(device_num, stream_id, row_ind, col_ind, val, x, y, m, n, nnz, alpha, beta, false);
       },
@@ -119,9 +145,24 @@ class spmv_actor : public event_based_actor {
       [this](return_mem_ptr_atom, csc_atom, in<int> col_ptr, in<int> row_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz) {
         enqueue_spmv_csc(-1, actor_id_, col_ptr, row_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, true);
       },
+      [this](return_mem_ptr_atom, csc_atom, in<int> col_ptr, in<int> row_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csc(-1, actor_id_, col_ptr, row_ind, val, x, y, m, n, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, csc_atom, int device_num, int stream_id, in<int> col_ptr, in<int> row_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csc(device_num, stream_id, col_ptr, row_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, csc_atom, int device_num, int stream_id, in<int> col_ptr, in<int> row_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_csc(device_num, stream_id, col_ptr, row_ind, val, x, y, m, n, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, csc_atom, mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csc(-1, actor_id_, col_ptr, row_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, true);
+      },
       [this](return_mem_ptr_atom, csc_atom, mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
         enqueue_spmv_csc(-1, actor_id_, col_ptr, row_ind, val, x, y, m, n, nnz, alpha, beta, true);
       },
+      [this](return_mem_ptr_atom, csc_atom, int device_num, int stream_id, mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz) {
+        enqueue_spmv_csc(device_num, stream_id, col_ptr, row_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, true);
+      },
       [this](return_mem_ptr_atom, csc_atom, int device_num, int stream_id, mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
         enqueue_spmv_csc(device_num, stream_id, col_ptr, row_ind, val, x, y, m, n, nnz, alpha, beta, true);
       },
@@ -129,9 +170,24 @@ class spmv_actor : public event_based_actor {
       [this](return_mem_ptr_atom, coo_atom, in<int> row_ind, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz) {
         enqueue_spmv_coo(-1, actor_id_, row_ind, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, true);
       },
+      [this](return_mem_ptr_atom, coo_atom, in<int> row_ind, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_coo(-1, actor_id_, row_ind, col_ind, val, x, y, m, n, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, coo_atom, int device_num, int stream_id, in<int> row_ind, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz) {
+        enqueue_spmv_coo(device_num, stream_id, row_ind, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, coo_atom, int device_num, int stream_id, in<int> row_ind, in<int> col_ind, in<float> val, in<float> x, out<float> y, int m, int n, int nnz, float alpha, float beta) {
+        enqueue_spmv_coo(device_num, stream_id, row_ind, col_ind, val, x, y, m, n, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, coo_atom, mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz) {
+        enqueue_spmv_coo(-1, actor_id_, row_ind, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, true);
+      },
       [this](return_mem_ptr_atom, coo_atom, mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
         enqueue_spmv_coo(-1, actor_id_, row_ind, col_ind, val, x, y, m, n, nnz, alpha, beta, true);
       },
+      [this](return_mem_ptr_atom, coo_atom, int device_num, int stream_id, mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz) {
+        enqueue_spmv_coo(device_num, stream_id, row_ind, col_ind, val, x, y, m, n, nnz, 1.0f, 0.0f, true);
+      },
       [this](return_mem_ptr_atom, coo_atom, int device_num, int stream_id, mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> val, mem_ptr<float> x, mem_ptr<float> y, int m, int n, int nnz, float alpha, float beta) {
         enqueue_spmv_coo(device_num, stream_id, row_ind, col_ind, val, x, y, m, n, nnz, alpha, beta, true);
       },

From 34221f56c21203c17ced02edf9c23e5827754b4f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 22 May 2026 14:16:31 -0600
Subject: [PATCH 0708/1000] updated files for correctness

---
 .../spmv-actor-test/CMakeLists.txt            |   1 +
 .../spmv-actor-test/main.test.cpp             | 180 ++++++++++--------
 2 files changed, 98 insertions(+), 83 deletions(-)

diff --git a/libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/CMakeLists.txt
index fa9dc8e759..82787c399c 100644
--- a/libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/CMakeLists.txt
@@ -41,5 +41,6 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
     CUDA::cublas
+    CUDA::cusparse
 )
 
diff --git a/libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/main.test.cpp
index 2e070afabb..a66226f736 100644
--- a/libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSPARSE-test/spmv-actor-test/main.test.cpp
@@ -1,3 +1,11 @@
+/**
+ * This test file evaluates the correctness of spmv_actor for sparse matrix-vector multiplication.
+ * It covers:
+ * - CSR, CSC, and COO formats.
+ * - Input types: host wrappers (in/out) and device handles (mem_ptr).
+ * - Reply modes: data result and device handle return (return_mem_ptr_atom).
+ * - Explicit routing with device_num and stream_id.
+ */
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
 #include <iostream>
@@ -8,135 +16,141 @@
 #include <algorithm>
 #include <numeric>
 #include "caf/actor_registry.hpp"
-#include "caf/actorBLAS/gemv-actor/gemv-actor.hpp"
+#include "caf/actorSPARSE/spmv-actor/spmv-actor.hpp"
 
 using namespace caf;
 using namespace caf::cuda;
 
-// Verify GEMV result: y = alpha * A * x + beta * y
-// With A (all 3.0s) and x (all 4.0s), the dot product sum is (3.0 * 4.0 * n).
-// We use alpha = 1.0 / n to normalize the output to exactly 12.0f as requested.
-void verify_gemv_correctness(int m, int n, float alpha, float beta, 
-                             const std::vector<float>& y_result) {
-    float expected_val = alpha * (3.0f * 4.0f * n); 
+void verify_spmv(const std::string& test_name, const std::vector<float>& actual, 
+                 const std::vector<float>& expected) {
+    if (actual.size() != expected.size()) {
+        std::cout << "[ERROR] " << test_name << " failed: Size mismatch (got " 
+                  << actual.size() << ", expected " << expected.size() << ")" << std::endl;
+        return;
+    }
     bool all_correct = true;
-    for (size_t i = 0; i < y_result.size(); ++i) {
-        if (std::abs(y_result[i] - expected_val) > 1e-4) {
+    for (size_t i = 0; i < actual.size(); ++i) {
+        if (std::abs(actual[i] - expected[i]) > 1e-4) {
             all_correct = false;
-            std::cout << "[ERROR] Mismatch at index " << i 
-                      << ": Expected " << expected_val 
-                      << ", Got " << y_result[i] << std::endl;
+            std::cout << "[ERROR] " << test_name << " mismatch at index " << i 
+                      << ": Expected " << expected[i] 
+                      << ", Got " << actual[i] << std::endl;
             break;
         }
     }
-
     if (all_correct) {
-        std::cout << "[SUCCESS] GEMV actor produced correct results." << std::endl;
-        std::cout << "          Output vector is filled with: " << expected_val << std::endl;
+        std::cout << "[SUCCESS] " << test_name << " passed." << std::endl;
     }
 }
 
 void caf_main(actor_system& sys) {
-    // Initialize the manager with BLAS enabled to initialize cuBLAS handles
-    manager_config config(true); 
-    manager::init(sys, config);
-
-    // Matrix dimensions 64x64 (> 32x32)
-    int m = 64;
-    int n = 64;
-    
-    // Scalar alpha set to 1/n to normalize the dot product sum (3*4*n) to 12.0
-    float alpha = 1.0f / static_cast<float>(n);
-    float beta = 0.0f;
-
-    // Initialize host data
-    std::vector<float> h_A(m * n, 3.0f);
-    std::vector<float> h_x(n, 4.0f);
-    std::vector<float> h_y(m, 0.0f);
-
-    // Spawn the gemv_actor
-    auto blas_actor = sys.spawn<gemv_actor>(1);
-
-    // Prepare arguments using wrapper tags
-    auto A_arg = create_in_arg(h_A);
-    auto x_arg = create_in_arg(h_x);
-    auto y_arg = create_out_arg(h_y);
+    manager::init(sys, manager_config(true, true)); // Enable cuBLAS and cuSPARSE
+
+    // Matrix A (3x3): [ 1 0 2; 0 0 3; 4 5 6 ]
+    // Vector x: [1, 2, 3]
+    // Expected y: [7, 9, 32]
+    int m = 3, n = 3, nnz = 6;
+    std::vector<float> h_x = {1.0f, 2.0f, 3.0f};
+    std::vector<float> h_y_init = {0.0f, 0.0f, 0.0f};
+    std::vector<float> expected = {7.0f, 9.0f, 32.0f};
 
+    int device_num = 0;
+    int stream_id = 10;
+
+    auto spmv = sys.spawn<spmv_actor>(1);
     scoped_actor self{sys};
 
-    // Test 1: Standard host-buffer based call
+    // Test 1: CSR - Host Wrappers - Explicit Routing
     {
-        std::cout << "[INFO] Test 1: Testing gemv_actor with host-buffer arguments..." << std::endl;
-        self->mail(A_arg, x_arg, y_arg, m, n, alpha, beta).send(blas_actor);
+        std::cout << "[INFO] Test 1: CSR format, host wrappers, explicit routing (dev=" << device_num << ", stream=" << stream_id << ")..." << std::endl;
+        std::vector<int> row_ptr = {0, 2, 3, 6};
+        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
+        std::vector<float> values = {1, 2, 3, 4, 5, 6};
+
+        self->mail(csr_atom{}, device_num, stream_id, 
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values), 
+                   create_in_arg(h_x), create_out_arg(h_y_init), 
+                   m, n, nnz).send(spmv);
+
         self->receive(
             [&](int reply_id, int arg_index, std::vector<float> data) {
-                if (arg_index == 2) { // index 2 corresponds to y_arg
-                    verify_gemv_correctness(m, n, alpha, beta, data);
-                }
+                verify_spmv("CSR Host Wrapper Routing", data, expected);
             }
         );
-        std::cout << "[INFO] Test 1 complete." << std::endl;
     }
 
-    // Test 2: mem_ptr inputs
+    // Test 2: CSC - mem_ptr - Explicit Routing
     {
-        std::cout << "\n[INFO] Test 2: Testing gemv_actor with mem_ptr inputs..." << std::endl;
-        command_runner<in<float>, in<float>, out<float>> setup_runner;
-        // Manually transfer data to the GPU to get mem_ptr handles
-        auto results = setup_runner.transfer_memory(0, 0, create_in_arg(h_A), create_in_arg(h_x), create_out_arg(h_y));
-        auto A_ptr = std::get<0>(results);
-        auto x_ptr = std::get<1>(results);
-        auto y_ptr = std::get<2>(results);
-
-        self->mail(A_ptr, x_ptr, y_ptr, m, n, alpha, beta).send(blas_actor);
+        std::cout << "\n[INFO] Test 2: CSC format, mem_ptr inputs, explicit routing..." << std::endl;
+        std::vector<int> col_ptr = {0, 2, 3, 6};
+        std::vector<int> row_ind = {0, 2, 2, 0, 1, 2};
+        std::vector<float> values = {1, 4, 5, 2, 3, 6};
+
+        command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+        auto results = runner.transfer_memory(device_num, stream_id, 
+                                             create_in_arg(col_ptr), create_in_arg(row_ind), 
+                                             create_in_arg(values), create_in_arg(h_x), 
+                                             create_out_arg(h_y_init));
+
+        self->mail(csc_atom{}, device_num, stream_id, 
+                   std::get<0>(results), std::get<1>(results), std::get<2>(results), 
+                   std::get<3>(results), std::get<4>(results), 
+                   m, n, nnz).send(spmv);
 
         self->receive(
             [&](int reply_id, int arg_index, std::vector<float> data) {
-                if (arg_index == 2) {
-                    verify_gemv_correctness(m, n, alpha, beta, data);
-                }
+                verify_spmv("CSC mem_ptr Routing", data, expected);
             }
         );
-        std::cout << "[INFO] Test 2 complete." << std::endl;
     }
 
-    // Test 3: Routing control (device/stream) + mem_ptr
+    // Test 3: COO - mem_ptr - return_mem_ptr_atom - Explicit Routing
     {
-        std::cout << "\n[INFO] Test 3: Testing gemv_actor with specific device/stream and mem_ptr..." << std::endl;
-        int device_num = 0;
-        int stream_id = 42; 
-        command_runner<in<float>, in<float>, out<float>> setup_runner;
-        auto results = setup_runner.transfer_memory(device_num, stream_id, create_in_arg(h_A), create_in_arg(h_x), create_out_arg(h_y));
-        
-        self->mail(device_num, stream_id, std::get<0>(results), std::get<1>(results), std::get<2>(results), m, n, alpha, beta).send(blas_actor);
+        std::cout << "\n[INFO] Test 3: COO format, return_mem_ptr_atom, mem_ptr inputs, explicit routing..." << std::endl;
+        std::vector<int> row_ind = {0, 0, 1, 2, 2, 2};
+        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
+        std::vector<float> values = {1, 2, 3, 4, 5, 6};
+
+        command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+        auto results = runner.transfer_memory(device_num, stream_id, 
+                                             create_in_arg(row_ind), create_in_arg(col_ind), 
+                                             create_in_arg(values), create_in_arg(h_x), 
+                                             create_out_arg(h_y_init));
+
+        self->mail(return_mem_ptr_atom{}, coo_atom{}, device_num, stream_id, 
+                   std::get<0>(results), std::get<1>(results), std::get<2>(results), 
+                   std::get<3>(results), std::get<4>(results), 
+                   m, n, nnz, 1.0f, 0.0f).send(spmv);
 
         self->receive(
-            [&](int reply_id, int arg_index, std::vector<float> data) {
-                if (arg_index == 2) {
-                    verify_gemv_correctness(m, n, alpha, beta, data);
-                }
+            [&](int reply_id, mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, 
+                mem_ptr<float>, mem_ptr<float> y_ptr) {
+                command_runner<float> cr;
+                auto host_y = cr.copy_to_host(y_ptr);
+                verify_spmv("COO return_mem_ptr Routing", host_y, expected);
             }
         );
-        std::cout << "[INFO] Test 3 complete." << std::endl;
     }
 
-    // Test 4: return_mem_ptr_atom (returning device handles)
+    // Test 4: CSR - Host Wrappers - Default Routing (Lottery Scheduler)
     {
-        std::cout << "\n[INFO] Test 4: Testing gemv_actor with return_mem_ptr_atom..." << std::endl;
-        self->mail(return_mem_ptr_atom{}, A_arg, x_arg, y_arg, m, n, alpha, beta).send(blas_actor);
+        std::cout << "\n[INFO] Test 4: CSR format, host wrappers, default routing..." << std::endl;
+        std::vector<int> row_ptr = {0, 2, 3, 6};
+        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
+        std::vector<float> values = {1, 2, 3, 4, 5, 6};
+
+        self->mail(csr_atom{}, create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values), 
+                   create_in_arg(h_x), create_out_arg(h_y_init), 
+                   m, n, nnz).send(spmv);
 
-        // We expect two messages back: the data (mem_ptrs) and the signal (-1)
         self->receive(
-            [&](int reply_id, mem_ptr<float> A, mem_ptr<float> x, mem_ptr<float> y) {
-                command_runner<float> runner;
-                auto host_y = runner.copy_to_host(y);
-                verify_gemv_correctness(m, n, alpha, beta, host_y);
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                verify_spmv("CSR Host Wrapper Default", data, expected);
             }
         );
-        std::cout << "[INFO] Test 4 complete." << std::endl;
     }
 
-    self->send_exit(blas_actor, exit_reason::user_shutdown);
+    self->send_exit(spmv, exit_reason::user_shutdown);
     manager::shutdown();
 }
 CAF_MAIN(id_block::cuda)

From 957fb50d94aaa96e660347a032eff8fe5857c1f2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 09:19:24 -0600
Subject: [PATCH 0709/1000] Intial commit for spmm actor

---
 libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp           | 3 ++-
 libcaf_cuda/caf/actorSPARSE/spmm-actor/spmm-actor.hpp | 0
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 libcaf_cuda/caf/actorSPARSE/spmm-actor/spmm-actor.hpp

diff --git a/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp b/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp
index ccec3760b4..425b42c4fb 100644
--- a/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp
+++ b/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp
@@ -1,3 +1,4 @@
 #pragma once
-
 #include "caf/actorSPARSE/spmv-actor/spmv-actor.hpp"
+#include "caf/actorSPARSE/spmv-actor/spmm-actor.hpp"
+
diff --git a/libcaf_cuda/caf/actorSPARSE/spmm-actor/spmm-actor.hpp b/libcaf_cuda/caf/actorSPARSE/spmm-actor/spmm-actor.hpp
new file mode 100644
index 0000000000..e69de29bb2

From a3fcda74362c0ceebb8869ce3c27ab0b68c47fb4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 09:26:20 -0600
Subject: [PATCH 0710/1000] Implemented functionality for spmm actor

---
 .../caf/actorSPARSE/spmm-actor/spmm-actor.hpp | 187 ++++++++++++++++++
 libcaf_cuda/caf/cuda/device.hpp               | 136 +++++++++++++
 2 files changed, 323 insertions(+)

diff --git a/libcaf_cuda/caf/actorSPARSE/spmm-actor/spmm-actor.hpp b/libcaf_cuda/caf/actorSPARSE/spmm-actor/spmm-actor.hpp
index e69de29bb2..700f8bfe27 100644
--- a/libcaf_cuda/caf/actorSPARSE/spmm-actor/spmm-actor.hpp
+++ b/libcaf_cuda/caf/actorSPARSE/spmm-actor/spmm-actor.hpp
@@ -0,0 +1,187 @@
+#pragma once
+
+#include <caf/actor.hpp>
+#include <caf/actor_cast.hpp>
+#include <caf/actor_config.hpp>
+#include <caf/actor_system.hpp>
+#include <caf/anon_mail.hpp>
+#include <caf/event_based_actor.hpp>
+
+#include "caf/cuda/device.hpp"
+#include "caf/cuda/mem_ref.hpp"
+#include "caf/cuda/command_runner.hpp"
+#include "caf/cuda/platform.hpp"
+#include "caf/cuda/types.hpp"
+
+namespace caf::cuda {
+
+/// SPMM Actor for single-precision sparse matrix-matrix multiplication.
+/// Message Signature: (csr_atom/csc_atom/coo_atom, in<int> row_ptr, in<int> col_ind, in<float> values, in<float> B, out<float> C, int m, int n, int k, int nnz, [float alpha, float beta])
+class spmm_actor : public event_based_actor {
+public:
+  static caf::actor spawn(caf::actor_system& sys, int reply_id = 0) {
+    return sys.spawn<spmm_actor>(reply_id);
+  }
+
+  spmm_actor(caf::actor_config& cfg, int reply_id = 0) 
+    : event_based_actor(cfg), reply_id_(reply_id) {
+    actor_id_ = static_cast<int>(this->id());
+  }
+
+  ~spmm_actor() override {
+    // command_runner<> runner;
+    // runner.release_stream_for_actor(actor_id_);
+  }
+
+  caf::behavior make_behavior() override {
+    return {
+      // CSR Overloads
+      [this](csr_atom, in<int> rp, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csr(-1, actor_id_, rp, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csr_atom, in<int> rp, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csr(-1, actor_id_, rp, ci, v, b, c, m, n, k, nnz, alpha, beta, false);
+      },
+      [this](csr_atom, int dev, int sid, in<int> rp, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csr(dev, sid, rp, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csr_atom, int dev, int sid, in<int> rp, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csr(dev, sid, rp, ci, v, b, c, m, n, k, nnz, alpha, beta, false);
+      },
+      [this](csr_atom, mem_ptr<int> rp, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csr(-1, actor_id_, rp, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csr_atom, mem_ptr<int> rp, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csr(-1, actor_id_, rp, ci, v, b, c, m, n, k, nnz, alpha, beta, false);
+      },
+
+      // CSC Overloads
+      [this](csc_atom, in<int> cp, in<int> ri, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csc(-1, actor_id_, cp, ri, v, b, c, m, n, k, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csc_atom, in<int> cp, in<int> ri, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csc(-1, actor_id_, cp, ri, v, b, c, m, n, k, nnz, alpha, beta, false);
+      },
+
+      // COO Overloads
+      [this](coo_atom, in<int> ri, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_coo(-1, actor_id_, ri, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, false);
+      },
+      [this](coo_atom, in<int> ri, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_coo(-1, actor_id_, ri, ci, v, b, c, m, n, k, nnz, alpha, beta, false);
+      },
+
+      // return_mem_ptr_atom variants (CSR example)
+      [this](return_mem_ptr_atom, csr_atom, in<int> rp, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csr(-1, actor_id_, rp, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, csr_atom, in<int> rp, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csr(-1, actor_id_, rp, ci, v, b, c, m, n, k, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, csc_atom, in<int> cp, in<int> ri, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csc(-1, actor_id_, cp, ri, v, b, c, m, n, k, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, coo_atom, in<int> ri, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_coo(-1, actor_id_, ri, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, true);
+      },
+    };
+  }
+
+private:
+  // CSR Enqueue logic
+  void enqueue_spmm_csr(int dev, int sid, in<int> rp, in<int> ci, in<float> v, in<float> b, out<float> c,
+                        int m, int n, int k, int nnz, float alpha, float beta, bool ret_ptr) {
+    command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+    auto res = runner.transfer_memory(dev, sid, rp, ci, v, b, c);
+    execute_and_reply_csr(dev, sid, std::get<0>(res), std::get<1>(res), std::get<2>(res), 
+                          std::get<3>(res), std::get<4>(res), m, n, k, nnz, alpha, beta, ret_ptr);
+  }
+
+  void enqueue_spmm_csr(int dev, int sid, mem_ptr<int> rp, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c,
+                        int m, int n, int k, int nnz, float alpha, float beta, bool ret_ptr) {
+    command_runner<mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, mem_ptr<float>, mem_ptr<float>> runner;
+    auto res = runner.transfer_memory(dev, sid, rp, ci, v, b, c);
+    execute_and_reply_csr(dev, sid, std::get<0>(res), std::get<1>(res), std::get<2>(res), 
+                          std::get<3>(res), std::get<4>(res), m, n, k, nnz, alpha, beta, ret_ptr);
+  }
+
+  void execute_and_reply_csr(int dev_n, int sid, mem_ptr<int> rp, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c,
+                             int m, int n, int k, int nnz, float alpha, float beta, bool ret_ptr) {
+    auto plat = platform::create();
+    device_ptr dev = (dev_n == -1) ? plat->schedule(sid) : plat->schedule(sid, dev_n);
+    dev->spmm_csr(sid, m, n, k, nnz, alpha, rp, ci, v, b, beta, c);
+    handle_reply(rp, ci, v, b, c, ret_ptr);
+  }
+
+  // CSC Enqueue logic
+  void enqueue_spmm_csc(int dev, int sid, in<int> cp, in<int> ri, in<float> v, in<float> b, out<float> c,
+                        int m, int n, int k, int nnz, float alpha, float beta, bool ret_ptr) {
+    command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+    auto res = runner.transfer_memory(dev, sid, cp, ri, v, b, c);
+    execute_and_reply_csc(dev, sid, std::get<0>(res), std::get<1>(res), std::get<2>(res), 
+                          std::get<3>(res), std::get<4>(res), m, n, k, nnz, alpha, beta, ret_ptr);
+  }
+
+  void enqueue_spmm_csc(int dev, int sid, mem_ptr<int> cp, mem_ptr<int> ri, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c,
+                        int m, int n, int k, int nnz, float alpha, float beta, bool ret_ptr) {
+    command_runner<mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, mem_ptr<float>, mem_ptr<float>> runner;
+    auto res = runner.transfer_memory(dev, sid, cp, ri, v, b, c);
+    execute_and_reply_csc(dev, sid, std::get<0>(res), std::get<1>(res), std::get<2>(res), 
+                          std::get<3>(res), std::get<4>(res), m, n, k, nnz, alpha, beta, ret_ptr);
+  }
+
+  void execute_and_reply_csc(int dev_n, int sid, mem_ptr<int> cp, mem_ptr<int> ri, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c,
+                             int m, int n, int k, int nnz, float alpha, float beta, bool ret_ptr) {
+    auto plat = platform::create();
+    device_ptr dev = (dev_n == -1) ? plat->schedule(sid) : plat->schedule(sid, dev_n);
+    dev->spmm_csc(sid, m, n, k, nnz, alpha, cp, ri, v, b, beta, c);
+    handle_reply(cp, ri, v, b, c, ret_ptr);
+  }
+
+  // COO Enqueue logic
+  void enqueue_spmm_coo(int dev, int sid, in<int> ri, in<int> ci, in<float> v, in<float> b, out<float> c,
+                        int m, int n, int k, int nnz, float alpha, float beta, bool ret_ptr) {
+    command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+    auto res = runner.transfer_memory(dev, sid, ri, ci, v, b, c);
+    execute_and_reply_coo(dev, sid, std::get<0>(res), std::get<1>(res), std::get<2>(res), 
+                          std::get<3>(res), std::get<4>(res), m, n, k, nnz, alpha, beta, ret_ptr);
+  }
+
+  void enqueue_spmm_coo(int dev, int sid, mem_ptr<int> ri, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c,
+                        int m, int n, int k, int nnz, float alpha, float beta, bool ret_ptr) {
+    command_runner<mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, mem_ptr<float>, mem_ptr<float>> runner;
+    auto res = runner.transfer_memory(dev, sid, ri, ci, v, b, c);
+    execute_and_reply_coo(dev, sid, std::get<0>(res), std::get<1>(res), std::get<2>(res), 
+                          std::get<3>(res), std::get<4>(res), m, n, k, nnz, alpha, beta, ret_ptr);
+  }
+
+  void execute_and_reply_coo(int dev_n, int sid, mem_ptr<int> ri, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c,
+                             int m, int n, int k, int nnz, float alpha, float beta, bool ret_ptr) {
+    auto plat = platform::create();
+    device_ptr dev = (dev_n == -1) ? plat->schedule(sid) : plat->schedule(sid, dev_n);
+    dev->spmm_coo(sid, m, n, k, nnz, alpha, ri, col_ptr, v, b, beta, c);
+    handle_reply(ri, ci, v, b, c, ret_ptr);
+  }
+
+  template <typename P1, typename P2, typename P3, typename P4, typename P5>
+  void handle_reply(P1 p1, P2 p2, P3 p3, P4 p4, P5 p5, bool return_ptrs) {
+    command_runner<mem_ptr<float>> runner;
+    auto sender = actor_cast<actor>(this->current_sender());
+    if (!sender) return;
+    auto r_id = reply_id_;
+    if (return_ptrs) {
+      caf::anon_mail(r_id, p1, p2, p3, p4, p5).send(sender);
+    } else {
+      runner.copy_to_host_async(p5, [sender, r_id](std::vector<float>&& data) {
+        if (sender) {
+          caf::anon_mail(r_id, 4, std::move(data)).send(sender);
+        }
+      });
+    }
+  }
+
+  int actor_id_;
+  int reply_id_;
+};
+
+} // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 20bf8b2d8a..28e3bf8f15 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -319,6 +319,142 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cusparseSpMV (CSC) failed on device " + std::to_string(id_));
   }
 
+  /// Performs sparse matrix-matrix multiplication (C = alpha*A*B + beta*C) using CSR format.
+  /// A is sparse (m x k), B is dense (k x n), C is dense (m x n).
+  void spmm_csr(int stream_id, int m, int n, int k, int nnz, float alpha, 
+                mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> values,
+                mem_ptr<float> B, float beta, mem_ptr<float> C) {
+    cusparseHandle_t handle = get_cusparse_handle(stream_id);
+    if (!handle) throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    CUstream stream = get_stream_for_actor(stream_id);
+
+    cusparseSpMatDescr_t matA;
+    cusparseCreateCsr(&matA, m, k, nnz,
+                      reinterpret_cast<void*>(row_ptr->mem()),
+                      reinterpret_cast<void*>(col_ind->mem()),
+                      reinterpret_cast<void*>(values->mem()),
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                      CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+
+    cusparseDnMatDescr_t matB, matC;
+    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+
+    size_t bufferSize = 0;
+    cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                            &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                            CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
+
+    CUdeviceptr dBuffer = 0;
+    if (bufferSize > 0) CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+
+    cusparseStatus_t status = cusparseSpMM(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                                           CUSPARSE_SPMM_ALG_DEFAULT, reinterpret_cast<void*>(dBuffer));
+
+    if (dBuffer) CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
+
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnMat(matB);
+    cusparseDestroyDnMat(matC);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUSPARSE_STATUS_SUCCESS)
+      throw std::runtime_error("cusparseSpMM (CSR) failed on device " + std::to_string(id_));
+  }
+
+  /// Performs sparse matrix-matrix multiplication (C = alpha*A*B + beta*C) using COO format.
+  void spmm_coo(int stream_id, int m, int n, int k, int nnz, float alpha, 
+                mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> values,
+                mem_ptr<float> B, float beta, mem_ptr<float> C) {
+    cusparseHandle_t handle = get_cusparse_handle(stream_id);
+    if (!handle) throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    CUstream stream = get_stream_for_actor(stream_id);
+
+    cusparseSpMatDescr_t matA;
+    cusparseCreateCoo(&matA, m, k, nnz,
+                      reinterpret_cast<void*>(row_ind->mem()),
+                      reinterpret_cast<void*>(col_ind->mem()),
+                      reinterpret_cast<void*>(values->mem()),
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+
+    cusparseDnMatDescr_t matB, matC;
+    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+
+    size_t bufferSize = 0;
+    cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                            &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                            CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
+
+    CUdeviceptr dBuffer = 0;
+    if (bufferSize > 0) CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+
+    cusparseStatus_t status = cusparseSpMM(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                                           CUSPARSE_SPMM_ALG_DEFAULT, reinterpret_cast<void*>(dBuffer));
+
+    if (dBuffer) CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
+
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnMat(matB);
+    cusparseDestroyDnMat(matC);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUSPARSE_STATUS_SUCCESS)
+      throw std::runtime_error("cusparseSpMM (COO) failed on device " + std::to_string(id_));
+  }
+
+  /// Performs sparse matrix-matrix multiplication (C = alpha*A*B + beta*C) using CSC format.
+  void spmm_csc(int stream_id, int m, int n, int k, int nnz, float alpha, 
+                mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> values,
+                mem_ptr<float> B, float beta, mem_ptr<float> C) {
+    cusparseHandle_t handle = get_cusparse_handle(stream_id);
+    if (!handle) throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    CUstream stream = get_stream_for_actor(stream_id);
+
+    cusparseSpMatDescr_t matA;
+    cusparseCreateCsc(&matA, m, k, nnz,
+                      reinterpret_cast<void*>(col_ptr->mem()),
+                      reinterpret_cast<void*>(row_ind->mem()),
+                      reinterpret_cast<void*>(values->mem()),
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                      CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+
+    cusparseDnMatDescr_t matB, matC;
+    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+
+    size_t bufferSize = 0;
+    cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                            &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                            CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
+
+    CUdeviceptr dBuffer = 0;
+    if (bufferSize > 0) CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+
+    cusparseStatus_t status = cusparseSpMM(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                                           CUSPARSE_SPMM_ALG_DEFAULT, reinterpret_cast<void*>(dBuffer));
+
+    if (dBuffer)
+      CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
+
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnVec(vecX);
+    cusparseDestroyDnVec(vecY);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUSPARSE_STATUS_SUCCESS)
+      throw std::runtime_error("cusparseSpMV (CSC) failed on device " + std::to_string(id_));
+  }
+
   /// Performs symmetric rank-k update (C = alpha*A*A^T + beta*C).
   /// Assumes A is in row-major order of dimensions n x k, and C is n x n.
   void ssyrk(int stream_id, int n, int k, float alpha, mem_ptr<float> A,

From 1539bc2a083600469b8c5997adff28c8a673ce6d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 09:28:13 -0600
Subject: [PATCH 0711/1000] Intial commit.

---
 .../spmm-actor-test/CMakeLists.txt            |  46 ++++++
 .../spmm-actor-test/main.test.cpp             | 156 ++++++++++++++++++
 2 files changed, 202 insertions(+)
 create mode 100644 libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/main.test.cpp

diff --git a/libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..82787c399c
--- /dev/null
+++ b/libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/CMakeLists.txt
@@ -0,0 +1,46 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
+)
+
diff --git a/libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/main.test.cpp
new file mode 100644
index 0000000000..a66226f736
--- /dev/null
+++ b/libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/main.test.cpp
@@ -0,0 +1,156 @@
+/**
+ * This test file evaluates the correctness of spmv_actor for sparse matrix-vector multiplication.
+ * It covers:
+ * - CSR, CSC, and COO formats.
+ * - Input types: host wrappers (in/out) and device handles (mem_ptr).
+ * - Reply modes: data result and device handle return (return_mem_ptr_atom).
+ * - Explicit routing with device_num and stream_id.
+ */
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include "caf/actor_registry.hpp"
+#include "caf/actorSPARSE/spmv-actor/spmv-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+void verify_spmv(const std::string& test_name, const std::vector<float>& actual, 
+                 const std::vector<float>& expected) {
+    if (actual.size() != expected.size()) {
+        std::cout << "[ERROR] " << test_name << " failed: Size mismatch (got " 
+                  << actual.size() << ", expected " << expected.size() << ")" << std::endl;
+        return;
+    }
+    bool all_correct = true;
+    for (size_t i = 0; i < actual.size(); ++i) {
+        if (std::abs(actual[i] - expected[i]) > 1e-4) {
+            all_correct = false;
+            std::cout << "[ERROR] " << test_name << " mismatch at index " << i 
+                      << ": Expected " << expected[i] 
+                      << ", Got " << actual[i] << std::endl;
+            break;
+        }
+    }
+    if (all_correct) {
+        std::cout << "[SUCCESS] " << test_name << " passed." << std::endl;
+    }
+}
+
+void caf_main(actor_system& sys) {
+    manager::init(sys, manager_config(true, true)); // Enable cuBLAS and cuSPARSE
+
+    // Matrix A (3x3): [ 1 0 2; 0 0 3; 4 5 6 ]
+    // Vector x: [1, 2, 3]
+    // Expected y: [7, 9, 32]
+    int m = 3, n = 3, nnz = 6;
+    std::vector<float> h_x = {1.0f, 2.0f, 3.0f};
+    std::vector<float> h_y_init = {0.0f, 0.0f, 0.0f};
+    std::vector<float> expected = {7.0f, 9.0f, 32.0f};
+
+    int device_num = 0;
+    int stream_id = 10;
+
+    auto spmv = sys.spawn<spmv_actor>(1);
+    scoped_actor self{sys};
+
+    // Test 1: CSR - Host Wrappers - Explicit Routing
+    {
+        std::cout << "[INFO] Test 1: CSR format, host wrappers, explicit routing (dev=" << device_num << ", stream=" << stream_id << ")..." << std::endl;
+        std::vector<int> row_ptr = {0, 2, 3, 6};
+        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
+        std::vector<float> values = {1, 2, 3, 4, 5, 6};
+
+        self->mail(csr_atom{}, device_num, stream_id, 
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values), 
+                   create_in_arg(h_x), create_out_arg(h_y_init), 
+                   m, n, nnz).send(spmv);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                verify_spmv("CSR Host Wrapper Routing", data, expected);
+            }
+        );
+    }
+
+    // Test 2: CSC - mem_ptr - Explicit Routing
+    {
+        std::cout << "\n[INFO] Test 2: CSC format, mem_ptr inputs, explicit routing..." << std::endl;
+        std::vector<int> col_ptr = {0, 2, 3, 6};
+        std::vector<int> row_ind = {0, 2, 2, 0, 1, 2};
+        std::vector<float> values = {1, 4, 5, 2, 3, 6};
+
+        command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+        auto results = runner.transfer_memory(device_num, stream_id, 
+                                             create_in_arg(col_ptr), create_in_arg(row_ind), 
+                                             create_in_arg(values), create_in_arg(h_x), 
+                                             create_out_arg(h_y_init));
+
+        self->mail(csc_atom{}, device_num, stream_id, 
+                   std::get<0>(results), std::get<1>(results), std::get<2>(results), 
+                   std::get<3>(results), std::get<4>(results), 
+                   m, n, nnz).send(spmv);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                verify_spmv("CSC mem_ptr Routing", data, expected);
+            }
+        );
+    }
+
+    // Test 3: COO - mem_ptr - return_mem_ptr_atom - Explicit Routing
+    {
+        std::cout << "\n[INFO] Test 3: COO format, return_mem_ptr_atom, mem_ptr inputs, explicit routing..." << std::endl;
+        std::vector<int> row_ind = {0, 0, 1, 2, 2, 2};
+        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
+        std::vector<float> values = {1, 2, 3, 4, 5, 6};
+
+        command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+        auto results = runner.transfer_memory(device_num, stream_id, 
+                                             create_in_arg(row_ind), create_in_arg(col_ind), 
+                                             create_in_arg(values), create_in_arg(h_x), 
+                                             create_out_arg(h_y_init));
+
+        self->mail(return_mem_ptr_atom{}, coo_atom{}, device_num, stream_id, 
+                   std::get<0>(results), std::get<1>(results), std::get<2>(results), 
+                   std::get<3>(results), std::get<4>(results), 
+                   m, n, nnz, 1.0f, 0.0f).send(spmv);
+
+        self->receive(
+            [&](int reply_id, mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, 
+                mem_ptr<float>, mem_ptr<float> y_ptr) {
+                command_runner<float> cr;
+                auto host_y = cr.copy_to_host(y_ptr);
+                verify_spmv("COO return_mem_ptr Routing", host_y, expected);
+            }
+        );
+    }
+
+    // Test 4: CSR - Host Wrappers - Default Routing (Lottery Scheduler)
+    {
+        std::cout << "\n[INFO] Test 4: CSR format, host wrappers, default routing..." << std::endl;
+        std::vector<int> row_ptr = {0, 2, 3, 6};
+        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
+        std::vector<float> values = {1, 2, 3, 4, 5, 6};
+
+        self->mail(csr_atom{}, create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values), 
+                   create_in_arg(h_x), create_out_arg(h_y_init), 
+                   m, n, nnz).send(spmv);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                verify_spmv("CSR Host Wrapper Default", data, expected);
+            }
+        );
+    }
+
+    self->send_exit(spmv, exit_reason::user_shutdown);
+    manager::shutdown();
+}
+CAF_MAIN(id_block::cuda)

From 494b5b74ef07b1250319557ca9ada7bd9c094f23 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 09:34:49 -0600
Subject: [PATCH 0712/1000] fixed include path on line 3

---
 libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp b/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp
index 425b42c4fb..556918b30c 100644
--- a/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp
+++ b/libcaf_cuda/caf/actorSPARSE/actorSPARSE.hpp
@@ -1,4 +1,4 @@
 #pragma once
 #include "caf/actorSPARSE/spmv-actor/spmv-actor.hpp"
-#include "caf/actorSPARSE/spmv-actor/spmm-actor.hpp"
+#include "caf/actorSPARSE/spmm-actor/spmm-actor.hpp"
 

From 6012d1a87f55668ac8aeff1bc70446b1e5e3d1df Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 09:36:11 -0600
Subject: [PATCH 0713/1000] Implemented tests.

---
 .../spmm-actor-test/main.test.cpp             | 92 +++++++++----------
 1 file changed, 46 insertions(+), 46 deletions(-)

diff --git a/libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/main.test.cpp
index a66226f736..dcfcf30135 100644
--- a/libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/main.test.cpp
@@ -1,5 +1,5 @@
 /**
- * This test file evaluates the correctness of spmv_actor for sparse matrix-vector multiplication.
+ * This test file evaluates the correctness of spmm_actor for sparse matrix-matrix multiplication.
  * It covers:
  * - CSR, CSC, and COO formats.
  * - Input types: host wrappers (in/out) and device handles (mem_ptr).
@@ -16,12 +16,12 @@
 #include <algorithm>
 #include <numeric>
 #include "caf/actor_registry.hpp"
-#include "caf/actorSPARSE/spmv-actor/spmv-actor.hpp"
+#include "caf/actorSPARSE/actorSPARSE.hpp"
 
 using namespace caf;
 using namespace caf::cuda;
 
-void verify_spmv(const std::string& test_name, const std::vector<float>& actual, 
+void verify_spmm(const std::string& test_name, const std::vector<float>& actual, 
                  const std::vector<float>& expected) {
     if (actual.size() != expected.size()) {
         std::cout << "[ERROR] " << test_name << " failed: Size mismatch (got " 
@@ -46,111 +46,111 @@ void verify_spmv(const std::string& test_name, const std::vector<float>& actual,
 void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true)); // Enable cuBLAS and cuSPARSE
 
-    // Matrix A (3x3): [ 1 0 2; 0 0 3; 4 5 6 ]
-    // Vector x: [1, 2, 3]
-    // Expected y: [7, 9, 32]
-    int m = 3, n = 3, nnz = 6;
-    std::vector<float> h_x = {1.0f, 2.0f, 3.0f};
-    std::vector<float> h_y_init = {0.0f, 0.0f, 0.0f};
-    std::vector<float> expected = {7.0f, 9.0f, 32.0f};
+    // Matrix A (3x2): [ 1 0; 0 2; 3 4 ]
+    // Matrix B (2x2): [ 5 6; 7 8 ]
+    // Expected C (3x2): [ 5 6; 14 16; 43 50 ]
+    int m = 3, n = 2, k = 2, nnz = 4;
+    std::vector<float> h_B = {5.0f, 6.0f, 7.0f, 8.0f};
+    std::vector<float> h_C_init(m * n, 0.0f);
+    std::vector<float> expected = {5.0f, 6.0f, 14.0f, 16.0f, 43.0f, 50.0f};
 
     int device_num = 0;
     int stream_id = 10;
 
-    auto spmv = sys.spawn<spmv_actor>(1);
+    auto spmm = sys.spawn<spmm_actor>(1);
     scoped_actor self{sys};
 
     // Test 1: CSR - Host Wrappers - Explicit Routing
     {
-        std::cout << "[INFO] Test 1: CSR format, host wrappers, explicit routing (dev=" << device_num << ", stream=" << stream_id << ")..." << std::endl;
-        std::vector<int> row_ptr = {0, 2, 3, 6};
-        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
-        std::vector<float> values = {1, 2, 3, 4, 5, 6};
+        std::cout << "[INFO] Test 1: SpMM CSR format, host wrappers, explicit routing..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 4};
+        std::vector<int> col_ind = {0, 1, 0, 1};
+        std::vector<float> values = {1, 2, 3, 4};
 
         self->mail(csr_atom{}, device_num, stream_id, 
                    create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values), 
-                   create_in_arg(h_x), create_out_arg(h_y_init), 
-                   m, n, nnz).send(spmv);
+                   create_in_arg(h_B), create_out_arg(h_C_init), 
+                   m, n, k, nnz).send(spmm);
 
         self->receive(
             [&](int reply_id, int arg_index, std::vector<float> data) {
-                verify_spmv("CSR Host Wrapper Routing", data, expected);
+                verify_spmm("CSR Host Wrapper Routing", data, expected);
             }
         );
     }
 
     // Test 2: CSC - mem_ptr - Explicit Routing
     {
-        std::cout << "\n[INFO] Test 2: CSC format, mem_ptr inputs, explicit routing..." << std::endl;
-        std::vector<int> col_ptr = {0, 2, 3, 6};
-        std::vector<int> row_ind = {0, 2, 2, 0, 1, 2};
-        std::vector<float> values = {1, 4, 5, 2, 3, 6};
+        std::cout << "\n[INFO] Test 2: SpMM CSC format, mem_ptr inputs, explicit routing..." << std::endl;
+        std::vector<int> col_ptr = {0, 2, 4};
+        std::vector<int> row_ind = {0, 2, 1, 2};
+        std::vector<float> values = {1, 3, 2, 4};
 
         command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
         auto results = runner.transfer_memory(device_num, stream_id, 
                                              create_in_arg(col_ptr), create_in_arg(row_ind), 
-                                             create_in_arg(values), create_in_arg(h_x), 
-                                             create_out_arg(h_y_init));
+                                             create_in_arg(values), create_in_arg(h_B), 
+                                             create_out_arg(h_C_init));
 
         self->mail(csc_atom{}, device_num, stream_id, 
                    std::get<0>(results), std::get<1>(results), std::get<2>(results), 
                    std::get<3>(results), std::get<4>(results), 
-                   m, n, nnz).send(spmv);
+                   m, n, k, nnz).send(spmm);
 
         self->receive(
             [&](int reply_id, int arg_index, std::vector<float> data) {
-                verify_spmv("CSC mem_ptr Routing", data, expected);
+                verify_spmm("CSC mem_ptr Routing", data, expected);
             }
         );
     }
 
-    // Test 3: COO - mem_ptr - return_mem_ptr_atom - Explicit Routing
+    // Test 3: COO - mem_ptr - return_mem_ptr_atom
     {
-        std::cout << "\n[INFO] Test 3: COO format, return_mem_ptr_atom, mem_ptr inputs, explicit routing..." << std::endl;
-        std::vector<int> row_ind = {0, 0, 1, 2, 2, 2};
-        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
-        std::vector<float> values = {1, 2, 3, 4, 5, 6};
+        std::cout << "\n[INFO] Test 3: SpMM COO format, return_mem_ptr_atom, mem_ptr inputs..." << std::endl;
+        std::vector<int> row_ind = {0, 1, 2, 2};
+        std::vector<int> col_ind = {0, 1, 0, 1};
+        std::vector<float> values = {1, 2, 3, 4};
 
         command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
         auto results = runner.transfer_memory(device_num, stream_id, 
                                              create_in_arg(row_ind), create_in_arg(col_ind), 
-                                             create_in_arg(values), create_in_arg(h_x), 
-                                             create_out_arg(h_y_init));
+                                             create_in_arg(values), create_in_arg(h_B), 
+                                             create_out_arg(h_C_init));
 
-        self->mail(return_mem_ptr_atom{}, coo_atom{}, device_num, stream_id, 
+        self->mail(return_mem_ptr_atom{}, coo_atom{}, 
                    std::get<0>(results), std::get<1>(results), std::get<2>(results), 
                    std::get<3>(results), std::get<4>(results), 
-                   m, n, nnz, 1.0f, 0.0f).send(spmv);
+                   m, n, k, nnz).send(spmm);
 
         self->receive(
             [&](int reply_id, mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, 
-                mem_ptr<float>, mem_ptr<float> y_ptr) {
+                mem_ptr<float>, mem_ptr<float> C_ptr) {
                 command_runner<float> cr;
-                auto host_y = cr.copy_to_host(y_ptr);
-                verify_spmv("COO return_mem_ptr Routing", host_y, expected);
+                auto host_C = cr.copy_to_host(C_ptr);
+                verify_spmm("COO return_mem_ptr Routing", host_C, expected);
             }
         );
     }
 
     // Test 4: CSR - Host Wrappers - Default Routing (Lottery Scheduler)
     {
-        std::cout << "\n[INFO] Test 4: CSR format, host wrappers, default routing..." << std::endl;
-        std::vector<int> row_ptr = {0, 2, 3, 6};
-        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
-        std::vector<float> values = {1, 2, 3, 4, 5, 6};
+        std::cout << "\n[INFO] Test 4: SpMM CSR format, host wrappers, default routing..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 4};
+        std::vector<int> col_ind = {0, 1, 0, 1};
+        std::vector<float> values = {1, 2, 3, 4};
 
         self->mail(csr_atom{}, create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values), 
-                   create_in_arg(h_x), create_out_arg(h_y_init), 
-                   m, n, nnz).send(spmv);
+                   create_in_arg(h_B), create_out_arg(h_C_init), 
+                   m, n, k, nnz).send(spmm);
 
         self->receive(
             [&](int reply_id, int arg_index, std::vector<float> data) {
-                verify_spmv("CSR Host Wrapper Default", data, expected);
+                verify_spmm("CSR Host Wrapper Default", data, expected);
             }
         );
     }
 
-    self->send_exit(spmv, exit_reason::user_shutdown);
+    self->send_exit(spmm, exit_reason::user_shutdown);
     manager::shutdown();
 }
 CAF_MAIN(id_block::cuda)

From 8089d9e882d7e86312d6865d317f047287005f4e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 09:44:18 -0600
Subject: [PATCH 0714/1000] updated actor to fix compiler and message handler
 errors

---
 .../caf/actorSPARSE/spmm-actor/spmm-actor.hpp | 106 +++++++++++++++++-
 libcaf_cuda/caf/cuda/device.hpp               |   4 +-
 2 files changed, 105 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/caf/actorSPARSE/spmm-actor/spmm-actor.hpp b/libcaf_cuda/caf/actorSPARSE/spmm-actor/spmm-actor.hpp
index 700f8bfe27..979dac5a34 100644
--- a/libcaf_cuda/caf/actorSPARSE/spmm-actor/spmm-actor.hpp
+++ b/libcaf_cuda/caf/actorSPARSE/spmm-actor/spmm-actor.hpp
@@ -29,8 +29,8 @@ class spmm_actor : public event_based_actor {
   }
 
   ~spmm_actor() override {
-    // command_runner<> runner;
-    // runner.release_stream_for_actor(actor_id_);
+    command_runner<> runner;
+    runner.release_stream_for_actor(actor_id_);
   }
 
   caf::behavior make_behavior() override {
@@ -54,6 +54,12 @@ class spmm_actor : public event_based_actor {
       [this](csr_atom, mem_ptr<int> rp, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
         enqueue_spmm_csr(-1, actor_id_, rp, ci, v, b, c, m, n, k, nnz, alpha, beta, false);
       },
+      [this](csr_atom, int dev, int sid, mem_ptr<int> rp, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csr(dev, sid, rp, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csr_atom, int dev, int sid, mem_ptr<int> rp, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csr(dev, sid, rp, ci, v, b, c, m, n, k, nnz, alpha, beta, false);
+      },
 
       // CSC Overloads
       [this](csc_atom, in<int> cp, in<int> ri, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
@@ -62,6 +68,24 @@ class spmm_actor : public event_based_actor {
       [this](csc_atom, in<int> cp, in<int> ri, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
         enqueue_spmm_csc(-1, actor_id_, cp, ri, v, b, c, m, n, k, nnz, alpha, beta, false);
       },
+      [this](csc_atom, int dev, int sid, in<int> cp, in<int> ri, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csc(dev, sid, cp, ri, v, b, c, m, n, k, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csc_atom, int dev, int sid, in<int> cp, in<int> ri, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csc(dev, sid, cp, ri, v, b, c, m, n, k, nnz, alpha, beta, false);
+      },
+      [this](csc_atom, mem_ptr<int> cp, mem_ptr<int> ri, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csc(-1, actor_id_, cp, ri, v, b, c, m, n, k, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csc_atom, mem_ptr<int> cp, mem_ptr<int> ri, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csc(-1, actor_id_, cp, ri, v, b, c, m, n, k, nnz, alpha, beta, false);
+      },
+      [this](csc_atom, int dev, int sid, mem_ptr<int> cp, mem_ptr<int> ri, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csc(dev, sid, cp, ri, v, b, c, m, n, k, nnz, 1.0f, 0.0f, false);
+      },
+      [this](csc_atom, int dev, int sid, mem_ptr<int> cp, mem_ptr<int> ri, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csc(dev, sid, cp, ri, v, b, c, m, n, k, nnz, alpha, beta, false);
+      },
 
       // COO Overloads
       [this](coo_atom, in<int> ri, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
@@ -70,6 +94,24 @@ class spmm_actor : public event_based_actor {
       [this](coo_atom, in<int> ri, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
         enqueue_spmm_coo(-1, actor_id_, ri, ci, v, b, c, m, n, k, nnz, alpha, beta, false);
       },
+      [this](coo_atom, int dev, int sid, in<int> ri, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_coo(dev, sid, ri, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, false);
+      },
+      [this](coo_atom, int dev, int sid, in<int> ri, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_coo(dev, sid, ri, ci, v, b, c, m, n, k, nnz, alpha, beta, false);
+      },
+      [this](coo_atom, mem_ptr<int> ri, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_coo(-1, actor_id_, ri, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, false);
+      },
+      [this](coo_atom, mem_ptr<int> ri, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_coo(-1, actor_id_, ri, ci, v, b, c, m, n, k, nnz, alpha, beta, false);
+      },
+      [this](coo_atom, int dev, int sid, mem_ptr<int> ri, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_coo(dev, sid, ri, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, false);
+      },
+      [this](coo_atom, int dev, int sid, mem_ptr<int> ri, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_coo(dev, sid, ri, ci, v, b, c, m, n, k, nnz, alpha, beta, false);
+      },
 
       // return_mem_ptr_atom variants (CSR example)
       [this](return_mem_ptr_atom, csr_atom, in<int> rp, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
@@ -78,12 +120,70 @@ class spmm_actor : public event_based_actor {
       [this](return_mem_ptr_atom, csr_atom, in<int> rp, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
         enqueue_spmm_csr(-1, actor_id_, rp, ci, v, b, c, m, n, k, nnz, alpha, beta, true);
       },
+      [this](return_mem_ptr_atom, csr_atom, int dev, int sid, in<int> rp, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csr(dev, sid, rp, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, csr_atom, int dev, int sid, in<int> rp, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csr(dev, sid, rp, ci, v, b, c, m, n, k, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, csr_atom, mem_ptr<int> rp, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csr(-1, actor_id_, rp, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, csr_atom, mem_ptr<int> rp, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csr(-1, actor_id_, rp, ci, v, b, c, m, n, k, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, csr_atom, int dev, int sid, mem_ptr<int> rp, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csr(dev, sid, rp, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, csr_atom, int dev, int sid, mem_ptr<int> rp, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csr(dev, sid, rp, ci, v, b, c, m, n, k, nnz, alpha, beta, true);
+      },
+      // return_mem_ptr_atom variants (CSC)
       [this](return_mem_ptr_atom, csc_atom, in<int> cp, in<int> ri, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
         enqueue_spmm_csc(-1, actor_id_, cp, ri, v, b, c, m, n, k, nnz, 1.0f, 0.0f, true);
       },
+      [this](return_mem_ptr_atom, csc_atom, int dev, int sid, in<int> cp, in<int> ri, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csc(dev, sid, cp, ri, v, b, c, m, n, k, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, csc_atom, int dev, int sid, in<int> cp, in<int> ri, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csc(dev, sid, cp, ri, v, b, c, m, n, k, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, csc_atom, mem_ptr<int> cp, mem_ptr<int> ri, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csc(-1, actor_id_, cp, ri, v, b, c, m, n, k, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, csc_atom, mem_ptr<int> cp, mem_ptr<int> ri, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csc(-1, actor_id_, cp, ri, v, b, c, m, n, k, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, csc_atom, int dev, int sid, mem_ptr<int> cp, mem_ptr<int> ri, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_csc(dev, sid, cp, ri, v, b, c, m, n, k, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, csc_atom, int dev, int sid, mem_ptr<int> cp, mem_ptr<int> ri, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_csc(dev, sid, cp, ri, v, b, c, m, n, k, nnz, alpha, beta, true);
+      },
       [this](return_mem_ptr_atom, coo_atom, in<int> ri, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
         enqueue_spmm_coo(-1, actor_id_, ri, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, true);
       },
+      [this](return_mem_ptr_atom, coo_atom, in<int> ri, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_coo(-1, actor_id_, ri, ci, v, b, c, m, n, k, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, coo_atom, int dev, int sid, in<int> ri, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_coo(dev, sid, ri, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, coo_atom, int dev, int sid, in<int> ri, in<int> ci, in<float> v, in<float> b, out<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_coo(dev, sid, ri, ci, v, b, c, m, n, k, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, coo_atom, mem_ptr<int> ri, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_coo(-1, actor_id_, ri, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, coo_atom, mem_ptr<int> ri, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_coo(-1, actor_id_, ri, ci, v, b, c, m, n, k, nnz, alpha, beta, true);
+      },
+      [this](return_mem_ptr_atom, coo_atom, int dev, int sid, mem_ptr<int> ri, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz) {
+        enqueue_spmm_coo(dev, sid, ri, ci, v, b, c, m, n, k, nnz, 1.0f, 0.0f, true);
+      },
+      [this](return_mem_ptr_atom, coo_atom, int dev, int sid, mem_ptr<int> ri, mem_ptr<int> ci, mem_ptr<float> v, mem_ptr<float> b, mem_ptr<float> c, int m, int n, int k, int nnz, float alpha, float beta) {
+        enqueue_spmm_coo(dev, sid, ri, ci, v, b, c, m, n, k, nnz, alpha, beta, true);
+      },
     };
   }
 
@@ -159,7 +259,7 @@ class spmm_actor : public event_based_actor {
                              int m, int n, int k, int nnz, float alpha, float beta, bool ret_ptr) {
     auto plat = platform::create();
     device_ptr dev = (dev_n == -1) ? plat->schedule(sid) : plat->schedule(sid, dev_n);
-    dev->spmm_coo(sid, m, n, k, nnz, alpha, ri, col_ptr, v, b, beta, c);
+    dev->spmm_coo(sid, m, n, k, nnz, alpha, ri, ci, v, b, beta, c);
     handle_reply(ri, ci, v, b, c, ret_ptr);
   }
 
diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 28e3bf8f15..1f01c54e44 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -447,8 +447,8 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
 
     cusparseDestroySpMat(matA);
-    cusparseDestroyDnVec(vecX);
-    cusparseDestroyDnVec(vecY);
+    cusparseDestroyDnMat(matB);
+    cusparseDestroyDnMat(matC);
 
     CHECK_CUDA(cuCtxPopCurrent(nullptr));
     if (status != CUSPARSE_STATUS_SUCCESS)

From 31d1c9ec88c52e88e9b91163a8d71af05fc3bc2b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 09:45:42 -0600
Subject: [PATCH 0715/1000] Updated tests.

---
 .../spmm-actor-test/main.test.cpp             | 140 +++++++++++++++++-
 1 file changed, 137 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/main.test.cpp
index dcfcf30135..c54e5f42ac 100644
--- a/libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSPARSE-test/spmm-actor-test/main.test.cpp
@@ -104,7 +104,7 @@ void caf_main(actor_system& sys) {
         );
     }
 
-    // Test 3: COO - mem_ptr - return_mem_ptr_atom
+    // Test 3: COO - mem_ptr - return_mem_ptr_atom - Default Routing
     {
         std::cout << "\n[INFO] Test 3: SpMM COO format, return_mem_ptr_atom, mem_ptr inputs..." << std::endl;
         std::vector<int> row_ind = {0, 1, 2, 2};
@@ -132,9 +132,35 @@ void caf_main(actor_system& sys) {
         );
     }
 
-    // Test 4: CSR - Host Wrappers - Default Routing (Lottery Scheduler)
+    // Test 4: CSR - mem_ptr - Explicit Routing
     {
-        std::cout << "\n[INFO] Test 4: SpMM CSR format, host wrappers, default routing..." << std::endl;
+        std::cout << "\n[INFO] Test 4: SpMM CSR format, mem_ptr inputs, explicit routing..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 4};
+        std::vector<int> col_ind = {0, 1, 0, 1};
+        std::vector<float> values = {1, 2, 3, 4};
+
+        command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+        auto results = runner.transfer_memory(device_num, stream_id, 
+                                             create_in_arg(row_ptr), create_in_arg(col_ind), 
+                                             create_in_arg(values), create_in_arg(h_B), 
+                                             create_out_arg(h_C_init));
+
+        self->mail(csr_atom{}, device_num, stream_id, 
+                   std::get<0>(results), std::get<1>(results), std::get<2>(results), 
+                   std::get<3>(results), std::get<4>(results), 
+                   m, n, k, nnz).send(spmm);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                verify_spmm("CSR mem_ptr Routing", data, expected);
+            }
+        );
+    }
+
+
+    // Test 5: CSR - Host Wrappers - Default Routing (Lottery Scheduler)
+    {
+        std::cout << "\n[INFO] Test 5: SpMM CSR format, host wrappers, default routing..." << std::endl;
         std::vector<int> row_ptr = {0, 1, 2, 4};
         std::vector<int> col_ind = {0, 1, 0, 1};
         std::vector<float> values = {1, 2, 3, 4};
@@ -150,6 +176,114 @@ void caf_main(actor_system& sys) {
         );
     }
 
+    // Test 4: CSR - Host Wrappers - Default Routing (Lottery Scheduler)
+    {
+        std::cout << "\n[INFO] Test 6: SpMM COO format, mem_ptr inputs, explicit routing..." << std::endl;
+        std::vector<int> row_ind = {0, 1, 2, 2};
+        std::vector<int> col_ind = {0, 1, 0, 1};
+        std::vector<float> values = {1, 2, 3, 4};
+
+        command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+        auto results = runner.transfer_memory(device_num, stream_id, 
+                                             create_in_arg(row_ind), create_in_arg(col_ind), 
+                                             create_in_arg(values), create_in_arg(h_B), 
+                                             create_out_arg(h_C_init));
+
+        self->mail(coo_atom{}, device_num, stream_id, 
+                   std::get<0>(results), std::get<1>(results), std::get<2>(results), 
+                   std::get<3>(results), std::get<4>(results), 
+                   m, n, k, nnz).send(spmm);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                verify_spmm("COO mem_ptr Routing", data, expected);
+            }
+        );
+    }
+
+    // Test 7: CSC - mem_ptr - return_mem_ptr_atom - Explicit Routing
+    {
+        std::cout << "\n[INFO] Test 7: SpMM CSC format, return_mem_ptr_atom, mem_ptr inputs, explicit routing..." << std::endl;
+        std::vector<int> col_ptr = {0, 2, 4};
+        std::vector<int> row_ind = {0, 2, 1, 2};
+        std::vector<float> values = {1, 3, 2, 4};
+
+        command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+        auto results = runner.transfer_memory(device_num, stream_id, 
+                                             create_in_arg(col_ptr), create_in_arg(row_ind), 
+                                             create_in_arg(values), create_in_arg(h_B), 
+                                             create_out_arg(h_C_init));
+
+        self->mail(return_mem_ptr_atom{}, csc_atom{}, device_num, stream_id, 
+                   std::get<0>(results), std::get<1>(results), std::get<2>(results), 
+                   std::get<3>(results), std::get<4>(results), 
+                   m, n, k, nnz).send(spmm);
+
+        self->receive(
+            [&](int reply_id, mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, 
+                mem_ptr<float>, mem_ptr<float> C_ptr) {
+                command_runner<float> cr;
+                auto host_C = cr.copy_to_host(C_ptr);
+                verify_spmm("CSC return_mem_ptr Routing", host_C, expected);
+            }
+        );
+    }
+
+    // Test 8: COO - mem_ptr - return_mem_ptr_atom - Explicit Routing
+    {
+        std::cout << "\n[INFO] Test 8: SpMM COO format, return_mem_ptr_atom, mem_ptr inputs, explicit routing..." << std::endl;
+        std::vector<int> row_ind = {0, 1, 2, 2};
+        std::vector<int> col_ind = {0, 1, 0, 1};
+        std::vector<float> values = {1, 2, 3, 4};
+
+        command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+        auto results = runner.transfer_memory(device_num, stream_id, 
+                                             create_in_arg(row_ind), create_in_arg(col_ind), 
+                                             create_in_arg(values), create_in_arg(h_B), 
+                                             create_out_arg(h_C_init));
+
+        self->mail(return_mem_ptr_atom{}, coo_atom{}, device_num, stream_id, 
+                   std::get<0>(results), std::get<1>(results), std::get<2>(results), 
+                   std::get<3>(results), std::get<4>(results), 
+                   m, n, k, nnz).send(spmm);
+
+        self->receive(
+            [&](int reply_id, mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, 
+                mem_ptr<float>, mem_ptr<float> C_ptr) {
+                command_runner<float> cr;
+                auto host_C = cr.copy_to_host(C_ptr);
+                verify_spmm("COO return_mem_ptr Routing", host_C, expected);
+            }
+        );
+    }
+
+    // Test 9: CSR - mem_ptr - return_mem_ptr_atom - Explicit Routing
+    {
+        std::cout << "\n[INFO] Test 9: SpMM CSR format, return_mem_ptr_atom, mem_ptr inputs, explicit routing..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 4};
+        std::vector<int> col_ind = {0, 1, 0, 1};
+        std::vector<float> values = {1, 2, 3, 4};
+
+        command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+        auto results = runner.transfer_memory(device_num, stream_id, 
+                                             create_in_arg(row_ptr), create_in_arg(col_ind), 
+                                             create_in_arg(values), create_in_arg(h_B), 
+                                             create_out_arg(h_C_init));
+
+        self->mail(return_mem_ptr_atom{}, csr_atom{}, device_num, stream_id, 
+                   std::get<0>(results), std::get<1>(results), std::get<2>(results), 
+                   std::get<3>(results), std::get<4>(results), 
+                   m, n, k, nnz).send(spmm);
+
+        self->receive(
+            [&](int reply_id, mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, 
+                mem_ptr<float>, mem_ptr<float> C_ptr) {
+                command_runner<float> cr;
+                auto host_C = cr.copy_to_host(C_ptr);
+                verify_spmm("CSR return_mem_ptr Routing", host_C, expected);
+            }
+        );
+    }
     self->send_exit(spmm, exit_reason::user_shutdown);
     manager::shutdown();
 }

From 7b9808bdc6c4727b1bc970bad9b9d8152608cea1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 11:22:29 -0600
Subject: [PATCH 0716/1000] Initial commit.

---
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 301 ++++++++++++++++++
 1 file changed, 301 insertions(+)
 create mode 100644 libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
new file mode 100644
index 0000000000..1d542ffb0c
--- /dev/null
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -0,0 +1,301 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <deque>
+#include <vector>
+#include "caf/cuda/all.hpp"
+#include "caf/actorBLAS/dot-actor/dot-actor.hpp"
+#include "caf/actorSPARSE/spmv-actor/spmv-actor.hpp"
+#include "caf/actorBLAS/axpy-actor/axpy-actor.hpp"
+#include "caf/actorBLAS/copy-actor/copy-actor.hpp"
+
+namespace caf::cuda {
+
+enum class matrix_format {
+  csr,
+  csc,
+  coo
+};
+
+enum class sparse_cg_step {
+  idle,
+  init_r,
+  init_p,
+  init_rho,
+  main_spmv_w,
+  main_dot_pw,
+  main_axpy_x,
+  main_axpy_r,
+  main_dot_rr,
+  update_p_copy_r,
+  update_p_axpy_p,
+  update_p_final_copy,
+  restart_spmv_y,
+  restart_copy_b,
+  restart_axpy_r,
+  restart_copy_p,
+  restart_dot_rho
+};
+
+// Reply IDs used to distinguish which actor type is replying
+constexpr int id_dot = 100;
+constexpr int id_spmv = 200;
+constexpr int id_axpy = 300;
+constexpr int id_copy = 400;
+
+struct sparse_cg_state {
+  // Host Data (stored until start)
+  in<int> h_row_ptr, h_col_ind;
+  in<float> h_values, h_b;
+  in_out<float> h_x;
+
+  // Device Problem data
+  mem_ptr<int> A_row_ptr, A_col_ind;
+  mem_ptr<float> A_values, b, x;
+  
+  matrix_format format;
+  int n, nnz;
+  float tol;
+  int max_iter;
+  int device_num;
+  int stream_id;
+
+  // Workspace vectors
+  mem_ptr<float> r, p, w, y_tmp;
+  
+  // Scalars
+  float rho = 0.0f;
+  float old_rho = 0.0f;
+  float cur_norm = 0.0f;
+  float alpha = 0.0f;
+  float beta = 0.0f;
+  int iterations = 0;
+  sparse_cg_step step = sparse_cg_step::idle;
+
+  // Fault Tolerance: Stagnation Detection
+  float last_norm = -1.0f;
+  int stagnation_count = 0;
+
+  // BLAS/SPARSE Actors
+  caf::actor dot_actor, spmv_actor, axpy_actor, copy_actor;
+  caf::actor supervisor;
+};
+
+class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
+public:
+  sparse_cg_actor(actor_config& cfg, in<int> rp, in<int> ci, 
+                  in<float> val, in<float> b, in_out<float> x,
+                  matrix_format fmt, int n, int nnz, float tol, int max_iter, int device_num, int stream_id,
+                  caf::actor supervisor = nullptr)
+    : stateful_actor<sparse_cg_state>(cfg) {
+    state().h_row_ptr = std::move(rp);
+    state().h_col_ind = std::move(ci);
+    state().h_values = std::move(val);
+    state().h_b = std::move(b);
+    state().h_x = std::move(x);
+    state().format = fmt;
+    state().n = n; state().nnz = nnz; 
+    state().tol = tol; state().max_iter = max_iter;
+    state().device_num = device_num; state().stream_id = stream_id;
+    state().supervisor = supervisor;
+
+    // Spawn helpers
+    state().dot_actor = this->system().spawn<caf::cuda::dot_actor>(id_dot);
+    state().spmv_actor = this->system().spawn<caf::cuda::spmv_actor>(id_spmv);
+    state().axpy_actor = this->system().spawn<caf::cuda::axpy_actor>(id_axpy);
+    state().copy_actor = this->system().spawn<caf::cuda::copy_actor>(id_copy);
+  }
+
+  behavior make_behavior() override {
+    return {
+      [this](start_atom) {
+        if (!state().supervisor)
+          state().supervisor = actor_cast<caf::actor>(this->current_sender());
+        start_setup();
+      },
+      [this](int rid, float val) {
+        if (rid == id_dot) handle_dot_result(val);
+      },
+      // SPMV result (CSR Signature: rid, rp, ci, val, x, y)
+      [this](int rid, mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, mem_ptr<float>, mem_ptr<float>) {
+        if (rid == id_spmv) handle_spmv_result();
+      },
+      [this](int rid, mem_ptr<float> x, mem_ptr<float> y) {
+        if (rid == id_copy) handle_copy_result();
+        else if (rid == id_axpy) handle_axpy_result();
+      }
+    };
+  }
+
+private:
+  void start_setup() {
+    auto& s = state();
+    command_runner<> runner;
+    
+    // Transfer problem data to device
+    auto res = runner.transfer_memory(s.device_num, s.stream_id, 
+                                      s.h_row_ptr,
+                                      s.h_col_ind,
+                                      s.h_values,
+                                      s.h_b,
+                                      s.h_x);
+    
+    s.A_row_ptr = std::get<0>(res);
+    s.A_col_ind = std::get<1>(res);
+    s.A_values  = std::get<2>(res);
+    s.b         = std::get<3>(res);
+    s.x         = std::get<4>(res);
+
+    // Allocate workspace
+    command_runner<out<float>> work_runner;
+    s.r = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+    s.p = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+    s.w = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+    s.y_tmp = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+
+    // Start algorithm: r = b
+    s.step = sparse_cg_step::init_r;
+    this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.b, s.r, s.n).send(s.copy_actor);
+  }
+
+  void iterate() {
+    auto& s = state();
+    if (s.iterations >= s.max_iter || s.cur_norm < s.tol) {
+      if (s.supervisor) {
+        command_runner<float> cr;
+        cr.copy_to_host_async(s.x, [this, target = s.supervisor](std::vector<float>&& data) {
+          this->mail(std::move(data)).send(target);
+        });
+      }
+      s.step = sparse_cg_step::idle;
+      return;
+    }
+    s.step = sparse_cg_step::main_spmv_w;
+    send_spmv(s.p, s.w);
+  }
+
+  void perform_restart() {
+    auto& s = state();
+    s.stagnation_count = 0;
+    s.step = sparse_cg_step::restart_spmv_y;
+    send_spmv(s.x, s.y_tmp);
+  }
+
+  void send_spmv(mem_ptr<float> input_v, mem_ptr<float> output_v) {
+    auto& s = state();
+    auto send = [&](auto format_atom) {
+      this->mail(return_mem_ptr_atom_v, format_atom, s.device_num, s.stream_id, 
+                 s.A_row_ptr, s.A_col_ind, s.A_values, input_v, output_v, 
+                 s.n, s.n, s.nnz).send(s.spmv_actor);
+    };
+
+    switch (s.format) {
+      case matrix_format::csr: send(csr_atom_v); break;
+      case matrix_format::csc: send(csc_atom_v); break;
+      case matrix_format::coo: send(coo_atom_v); break;
+      default: break;
+    }
+  }
+
+  void handle_dot_result(float val) {
+    auto& s = state();
+    switch (s.step) {
+      case sparse_cg_step::init_rho:
+        s.rho = val; s.cur_norm = std::sqrt(val);
+        iterate();
+        break;
+      case sparse_cg_step::main_dot_pw:
+        s.alpha = s.rho / val;
+        s.step = sparse_cg_step::main_axpy_x;
+        this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.p, s.x, s.n, s.alpha).send(s.axpy_actor);
+        break;
+      case sparse_cg_step::main_dot_rr:
+        s.old_rho = s.rho; s.rho = val; s.cur_norm = std::sqrt(val);
+        check_stagnation();
+        break;
+      case sparse_cg_step::restart_dot_rho:
+        s.rho = val; s.cur_norm = std::sqrt(val);
+        iterate();
+        break;
+      default: break;
+    }
+  }
+
+  void check_stagnation() {
+    auto& s = state();
+    bool diverged = s.last_norm > 0 && s.cur_norm > s.last_norm * 1.5f;
+    bool stalled = s.last_norm > 0 && s.cur_norm > s.last_norm * 0.999f;
+    s.iterations++;
+    if (stalled || diverged) s.stagnation_count++;
+    else s.stagnation_count = 0;
+    s.last_norm = s.cur_norm;
+
+    if (s.stagnation_count >= 15 || diverged) {
+      perform_restart();
+    } else {
+      s.beta = s.rho / s.old_rho;
+      s.step = sparse_cg_step::update_p_copy_r;
+      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.r, s.w, s.n).send(s.copy_actor);
+    }
+  }
+
+  void handle_spmv_result() {
+    auto& s = state();
+    if (s.step == sparse_cg_step::main_spmv_w) {
+      s.step = sparse_cg_step::main_dot_pw;
+      this->mail(s.device_num, s.stream_id, s.p, s.w, s.y_tmp, s.n).send(s.dot_actor);
+    } else if (s.step == sparse_cg_step::restart_spmv_y) {
+      s.step = sparse_cg_step::restart_copy_b;
+      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.b, s.r, s.n).send(s.copy_actor);
+    }
+  }
+
+  void handle_axpy_result() {
+    auto& s = state();
+    if (s.step == sparse_cg_step::main_axpy_x) {
+      s.step = sparse_cg_step::main_axpy_r;
+      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.w, s.r, s.n, -s.alpha).send(s.axpy_actor);
+    } else if (s.step == sparse_cg_step::main_axpy_r) {
+      s.step = sparse_cg_step::main_dot_rr;
+      this->mail(s.device_num, s.stream_id, s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
+    } else if (s.step == sparse_cg_step::update_p_axpy_p) {
+      s.step = sparse_cg_step::update_p_final_copy;
+      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.w, s.p, s.n).send(s.copy_actor);
+    } else if (s.step == sparse_cg_step::restart_axpy_r) {
+      s.step = sparse_cg_step::restart_copy_p;
+      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.r, s.p, s.n).send(s.copy_actor);
+    }
+  }
+
+  void handle_copy_result() {
+    auto& s = state();
+    switch (s.step) {
+      case sparse_cg_step::init_r:
+        s.step = sparse_cg_step::init_p;
+        this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.r, s.p, s.n).send(s.copy_actor);
+        break;
+      case sparse_cg_step::init_p:
+        s.step = sparse_cg_step::init_rho;
+        this->mail(s.device_num, s.stream_id, s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
+        break;
+      case sparse_cg_step::update_p_copy_r:
+        s.step = sparse_cg_step::update_p_axpy_p;
+        this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.p, s.w, s.n, s.beta).send(s.axpy_actor);
+        break;
+      case sparse_cg_step::update_p_final_copy:
+        iterate();
+        break;
+      case sparse_cg_step::restart_copy_b:
+        s.step = sparse_cg_step::restart_axpy_r;
+        this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.y_tmp, s.r, s.n, -1.0f).send(s.axpy_actor);
+        break;
+      case sparse_cg_step::restart_copy_p:
+        s.step = sparse_cg_step::restart_dot_rho;
+        this->mail(s.device_num, s.stream_id, s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
+        break;
+      default: break;
+    }
+  }
+};
+
+} // namespace caf::cuda

From b0ed096e0b74770f4bf8636fcebaaf9dc53dc3b1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 11:25:27 -0600
Subject: [PATCH 0717/1000] Initial commit.

---
 .../CGS-actor-test/CMakeLists.txt             |  46 ++++++
 .../CGS-actor-test/main.test.cpp              | 156 ++++++++++++++++++
 2 files changed, 202 insertions(+)
 create mode 100644 libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp

diff --git a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..82787c399c
--- /dev/null
+++ b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/CMakeLists.txt
@@ -0,0 +1,46 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
+)
+
diff --git a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
new file mode 100644
index 0000000000..a66226f736
--- /dev/null
+++ b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
@@ -0,0 +1,156 @@
+/**
+ * This test file evaluates the correctness of spmv_actor for sparse matrix-vector multiplication.
+ * It covers:
+ * - CSR, CSC, and COO formats.
+ * - Input types: host wrappers (in/out) and device handles (mem_ptr).
+ * - Reply modes: data result and device handle return (return_mem_ptr_atom).
+ * - Explicit routing with device_num and stream_id.
+ */
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include "caf/actor_registry.hpp"
+#include "caf/actorSPARSE/spmv-actor/spmv-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+void verify_spmv(const std::string& test_name, const std::vector<float>& actual, 
+                 const std::vector<float>& expected) {
+    if (actual.size() != expected.size()) {
+        std::cout << "[ERROR] " << test_name << " failed: Size mismatch (got " 
+                  << actual.size() << ", expected " << expected.size() << ")" << std::endl;
+        return;
+    }
+    bool all_correct = true;
+    for (size_t i = 0; i < actual.size(); ++i) {
+        if (std::abs(actual[i] - expected[i]) > 1e-4) {
+            all_correct = false;
+            std::cout << "[ERROR] " << test_name << " mismatch at index " << i 
+                      << ": Expected " << expected[i] 
+                      << ", Got " << actual[i] << std::endl;
+            break;
+        }
+    }
+    if (all_correct) {
+        std::cout << "[SUCCESS] " << test_name << " passed." << std::endl;
+    }
+}
+
+void caf_main(actor_system& sys) {
+    manager::init(sys, manager_config(true, true)); // Enable cuBLAS and cuSPARSE
+
+    // Matrix A (3x3): [ 1 0 2; 0 0 3; 4 5 6 ]
+    // Vector x: [1, 2, 3]
+    // Expected y: [7, 9, 32]
+    int m = 3, n = 3, nnz = 6;
+    std::vector<float> h_x = {1.0f, 2.0f, 3.0f};
+    std::vector<float> h_y_init = {0.0f, 0.0f, 0.0f};
+    std::vector<float> expected = {7.0f, 9.0f, 32.0f};
+
+    int device_num = 0;
+    int stream_id = 10;
+
+    auto spmv = sys.spawn<spmv_actor>(1);
+    scoped_actor self{sys};
+
+    // Test 1: CSR - Host Wrappers - Explicit Routing
+    {
+        std::cout << "[INFO] Test 1: CSR format, host wrappers, explicit routing (dev=" << device_num << ", stream=" << stream_id << ")..." << std::endl;
+        std::vector<int> row_ptr = {0, 2, 3, 6};
+        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
+        std::vector<float> values = {1, 2, 3, 4, 5, 6};
+
+        self->mail(csr_atom{}, device_num, stream_id, 
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values), 
+                   create_in_arg(h_x), create_out_arg(h_y_init), 
+                   m, n, nnz).send(spmv);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                verify_spmv("CSR Host Wrapper Routing", data, expected);
+            }
+        );
+    }
+
+    // Test 2: CSC - mem_ptr - Explicit Routing
+    {
+        std::cout << "\n[INFO] Test 2: CSC format, mem_ptr inputs, explicit routing..." << std::endl;
+        std::vector<int> col_ptr = {0, 2, 3, 6};
+        std::vector<int> row_ind = {0, 2, 2, 0, 1, 2};
+        std::vector<float> values = {1, 4, 5, 2, 3, 6};
+
+        command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+        auto results = runner.transfer_memory(device_num, stream_id, 
+                                             create_in_arg(col_ptr), create_in_arg(row_ind), 
+                                             create_in_arg(values), create_in_arg(h_x), 
+                                             create_out_arg(h_y_init));
+
+        self->mail(csc_atom{}, device_num, stream_id, 
+                   std::get<0>(results), std::get<1>(results), std::get<2>(results), 
+                   std::get<3>(results), std::get<4>(results), 
+                   m, n, nnz).send(spmv);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                verify_spmv("CSC mem_ptr Routing", data, expected);
+            }
+        );
+    }
+
+    // Test 3: COO - mem_ptr - return_mem_ptr_atom - Explicit Routing
+    {
+        std::cout << "\n[INFO] Test 3: COO format, return_mem_ptr_atom, mem_ptr inputs, explicit routing..." << std::endl;
+        std::vector<int> row_ind = {0, 0, 1, 2, 2, 2};
+        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
+        std::vector<float> values = {1, 2, 3, 4, 5, 6};
+
+        command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
+        auto results = runner.transfer_memory(device_num, stream_id, 
+                                             create_in_arg(row_ind), create_in_arg(col_ind), 
+                                             create_in_arg(values), create_in_arg(h_x), 
+                                             create_out_arg(h_y_init));
+
+        self->mail(return_mem_ptr_atom{}, coo_atom{}, device_num, stream_id, 
+                   std::get<0>(results), std::get<1>(results), std::get<2>(results), 
+                   std::get<3>(results), std::get<4>(results), 
+                   m, n, nnz, 1.0f, 0.0f).send(spmv);
+
+        self->receive(
+            [&](int reply_id, mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, 
+                mem_ptr<float>, mem_ptr<float> y_ptr) {
+                command_runner<float> cr;
+                auto host_y = cr.copy_to_host(y_ptr);
+                verify_spmv("COO return_mem_ptr Routing", host_y, expected);
+            }
+        );
+    }
+
+    // Test 4: CSR - Host Wrappers - Default Routing (Lottery Scheduler)
+    {
+        std::cout << "\n[INFO] Test 4: CSR format, host wrappers, default routing..." << std::endl;
+        std::vector<int> row_ptr = {0, 2, 3, 6};
+        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
+        std::vector<float> values = {1, 2, 3, 4, 5, 6};
+
+        self->mail(csr_atom{}, create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values), 
+                   create_in_arg(h_x), create_out_arg(h_y_init), 
+                   m, n, nnz).send(spmv);
+
+        self->receive(
+            [&](int reply_id, int arg_index, std::vector<float> data) {
+                verify_spmv("CSR Host Wrapper Default", data, expected);
+            }
+        );
+    }
+
+    self->send_exit(spmv, exit_reason::user_shutdown);
+    manager::shutdown();
+}
+CAF_MAIN(id_block::cuda)

From 21ee5ba885a249dd981d2858e1b02173fc91604e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 11:37:52 -0600
Subject: [PATCH 0718/1000] added start atom

---
 libcaf_cuda/caf/cuda/global.hpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index 447080012d..f9fe03b11e 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -216,7 +216,9 @@ CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id)
   CAF_ADD_ATOM(cuda, return_mem_ptr_atom)
   CAF_ADD_ATOM(cuda, csr_atom)
   CAF_ADD_ATOM(cuda, csc_atom)
-  CAF_ADD_ATOM(cuda, coo_atom)
+  CAF_ADD_ATOM(cuda, coo_atom)  
+  CAF_ADD_ATOM(cuda, start_atom)
+
 
 CAF_END_TYPE_ID_BLOCK(cuda)
 

From c2e8a4bb348ee7bfc74c8a93954d6a80d5da2b1d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 11:38:28 -0600
Subject: [PATCH 0719/1000] Added test.

---
 .../CGS-actor-test/main.test.cpp              | 170 ++++++++----------
 1 file changed, 78 insertions(+), 92 deletions(-)

diff --git a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
index a66226f736..47de59a3ee 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
@@ -1,10 +1,9 @@
 /**
- * This test file evaluates the correctness of spmv_actor for sparse matrix-vector multiplication.
+ * This test file evaluates the correctness of sparse_cg_actor for solving Ax = b.
  * It covers:
  * - CSR, CSC, and COO formats.
- * - Input types: host wrappers (in/out) and device handles (mem_ptr).
- * - Reply modes: data result and device handle return (return_mem_ptr_atom).
- * - Explicit routing with device_num and stream_id.
+ * - Convergence verification with simple matrices.
+ * - Stress testing with a large 1D Laplacian matrix.
  */
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
@@ -15,14 +14,15 @@
 #include <thread>
 #include <algorithm>
 #include <numeric>
+#include <cmath>
 #include "caf/actor_registry.hpp"
-#include "caf/actorSPARSE/spmv-actor/spmv-actor.hpp"
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
 
 using namespace caf;
 using namespace caf::cuda;
 
-void verify_spmv(const std::string& test_name, const std::vector<float>& actual, 
-                 const std::vector<float>& expected) {
+void verify_solution(const std::string& test_name, const std::vector<float>& actual, 
+                     const std::vector<float>& expected, float tol = 1e-3) {
     if (actual.size() != expected.size()) {
         std::cout << "[ERROR] " << test_name << " failed: Size mismatch (got " 
                   << actual.size() << ", expected " << expected.size() << ")" << std::endl;
@@ -30,7 +30,7 @@ void verify_spmv(const std::string& test_name, const std::vector<float>& actual,
     }
     bool all_correct = true;
     for (size_t i = 0; i < actual.size(); ++i) {
-        if (std::abs(actual[i] - expected[i]) > 1e-4) {
+        if (std::abs(actual[i] - expected[i]) > tol) {
             all_correct = false;
             std::cout << "[ERROR] " << test_name << " mismatch at index " << i 
                       << ": Expected " << expected[i] 
@@ -39,118 +39,104 @@ void verify_spmv(const std::string& test_name, const std::vector<float>& actual,
         }
     }
     if (all_correct) {
-        std::cout << "[SUCCESS] " << test_name << " passed." << std::endl;
+        std::cout << "[SUCCESS] " << test_name << " converged to correct solution." << std::endl;
     }
 }
 
 void caf_main(actor_system& sys) {
-    manager::init(sys, manager_config(true, true)); // Enable cuBLAS and cuSPARSE
+    // Enable cuBLAS and cuSPARSE for the CG solver
+    manager::init(sys, manager_config(true, true)); 
 
-    // Matrix A (3x3): [ 1 0 2; 0 0 3; 4 5 6 ]
-    // Vector x: [1, 2, 3]
-    // Expected y: [7, 9, 32]
-    int m = 3, n = 3, nnz = 6;
-    std::vector<float> h_x = {1.0f, 2.0f, 3.0f};
-    std::vector<float> h_y_init = {0.0f, 0.0f, 0.0f};
-    std::vector<float> expected = {7.0f, 9.0f, 32.0f};
+    // Simple Diagonal Matrix A (3x3): diag(4, 3, 2)
+    // b = [8, 9, 2] -> Expected x = [2, 3, 1]
+    int n = 3, nnz = 3;
+    std::vector<float> h_b = {8.0f, 9.0f, 2.0f};
+    std::vector<float> expected = {2.0f, 3.0f, 1.0f};
+    float tolerance = 1e-5f;
+    int max_iter = 100;
 
-    int device_num = 0;
-    int stream_id = 10;
-
-    auto spmv = sys.spawn<spmv_actor>(1);
     scoped_actor self{sys};
 
-    // Test 1: CSR - Host Wrappers - Explicit Routing
-    {
-        std::cout << "[INFO] Test 1: CSR format, host wrappers, explicit routing (dev=" << device_num << ", stream=" << stream_id << ")..." << std::endl;
-        std::vector<int> row_ptr = {0, 2, 3, 6};
-        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
-        std::vector<float> values = {1, 2, 3, 4, 5, 6};
-
-        self->mail(csr_atom{}, device_num, stream_id, 
-                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values), 
-                   create_in_arg(h_x), create_out_arg(h_y_init), 
-                   m, n, nnz).send(spmv);
-
-        self->receive(
-            [&](int reply_id, int arg_index, std::vector<float> data) {
-                verify_spmv("CSR Host Wrapper Routing", data, expected);
-            }
-        );
-    }
-
-    // Test 2: CSC - mem_ptr - Explicit Routing
+    // Test 1: CSR Format
     {
-        std::cout << "\n[INFO] Test 2: CSC format, mem_ptr inputs, explicit routing..." << std::endl;
-        std::vector<int> col_ptr = {0, 2, 3, 6};
-        std::vector<int> row_ind = {0, 2, 2, 0, 1, 2};
-        std::vector<float> values = {1, 4, 5, 2, 3, 6};
-
-        command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
-        auto results = runner.transfer_memory(device_num, stream_id, 
-                                             create_in_arg(col_ptr), create_in_arg(row_ind), 
-                                             create_in_arg(values), create_in_arg(h_x), 
-                                             create_out_arg(h_y_init));
-
-        self->mail(csc_atom{}, device_num, stream_id, 
-                   std::get<0>(results), std::get<1>(results), std::get<2>(results), 
-                   std::get<3>(results), std::get<4>(results), 
-                   m, n, nnz).send(spmv);
-
+        std::cout << "[INFO] Test 1: CSR format simple matrix..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto solver = sys.spawn<sparse_cg_actor>(
+            create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+            create_in_arg(h_b), create_in_out_arg(h_x),
+            matrix_format::csr, n, nnz, tolerance, max_iter, 0, 0, actor_cast<actor>(self));
+
+        self->mail(start_atom_v).send(solver);
         self->receive(
-            [&](int reply_id, int arg_index, std::vector<float> data) {
-                verify_spmv("CSC mem_ptr Routing", data, expected);
+            [&](std::vector<float> result_x) {
+                verify_solution("CSR Simple", result_x, expected);
             }
         );
     }
 
-    // Test 3: COO - mem_ptr - return_mem_ptr_atom - Explicit Routing
+    // Test 2: CSC Format
     {
-        std::cout << "\n[INFO] Test 3: COO format, return_mem_ptr_atom, mem_ptr inputs, explicit routing..." << std::endl;
-        std::vector<int> row_ind = {0, 0, 1, 2, 2, 2};
-        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
-        std::vector<float> values = {1, 2, 3, 4, 5, 6};
-
-        command_runner<in<int>, in<int>, in<float>, in<float>, out<float>> runner;
-        auto results = runner.transfer_memory(device_num, stream_id, 
-                                             create_in_arg(row_ind), create_in_arg(col_ind), 
-                                             create_in_arg(values), create_in_arg(h_x), 
-                                             create_out_arg(h_y_init));
-
-        self->mail(return_mem_ptr_atom{}, coo_atom{}, device_num, stream_id, 
-                   std::get<0>(results), std::get<1>(results), std::get<2>(results), 
-                   std::get<3>(results), std::get<4>(results), 
-                   m, n, nnz, 1.0f, 0.0f).send(spmv);
-
+        std::cout << "\n[INFO] Test 2: CSC format simple matrix..." << std::endl;
+        std::vector<int> col_ptr = {0, 1, 2, 3};
+        std::vector<int> row_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto solver = sys.spawn<sparse_cg_actor>(
+            create_in_arg(col_ptr), create_in_arg(row_ind), create_in_arg(values),
+            create_in_arg(h_b), create_in_out_arg(h_x),
+            matrix_format::csc, n, nnz, tolerance, max_iter, 0, 1, actor_cast<actor>(self));
+
+        self->mail(start_atom_v).send(solver);
         self->receive(
-            [&](int reply_id, mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, 
-                mem_ptr<float>, mem_ptr<float> y_ptr) {
-                command_runner<float> cr;
-                auto host_y = cr.copy_to_host(y_ptr);
-                verify_spmv("COO return_mem_ptr Routing", host_y, expected);
+            [&](std::vector<float> result_x) {
+                verify_solution("CSC Simple", result_x, expected);
             }
         );
     }
 
-    // Test 4: CSR - Host Wrappers - Default Routing (Lottery Scheduler)
+    // Test 3: Stress Test - 1D Laplacian (N=10000)
     {
-        std::cout << "\n[INFO] Test 4: CSR format, host wrappers, default routing..." << std::endl;
-        std::vector<int> row_ptr = {0, 2, 3, 6};
-        std::vector<int> col_ind = {0, 2, 2, 0, 1, 2};
-        std::vector<float> values = {1, 2, 3, 4, 5, 6};
+        int N_large = 10000;
+        std::cout << "\n[INFO] Test 3: Stress Test - 1D Laplacian (N=" << N_large << ")..." << std::endl;
+        
+        std::vector<int> row_ptr;
+        std::vector<int> col_ind;
+        std::vector<float> values;
+        row_ptr.push_back(0);
+        for(int i=0; i<N_large; ++i) {
+            if(i > 0) { col_ind.push_back(i-1); values.push_back(-1.0f); }
+            col_ind.push_back(i); values.push_back(2.0f);
+            if(i < N_large-1) { col_ind.push_back(i+1); values.push_back(-1.0f); }
+            row_ptr.push_back(col_ind.size());
+        }
 
-        self->mail(csr_atom{}, create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values), 
-                   create_in_arg(h_x), create_out_arg(h_y_init), 
-                   m, n, nnz).send(spmv);
+        std::vector<float> b_large(N_large, 1.0f);
+        std::vector<float> x_large(N_large, 0.0f);
+        
+        auto start = std::chrono::high_resolution_clock::now();
+        auto stress_solver = sys.spawn<sparse_cg_actor>(
+            create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+            create_in_arg(b_large), create_in_out_arg(x_large),
+            matrix_format::csr, N_large, (int)values.size(), 1e-4f, 20000, 0, 2, actor_cast<actor>(self));
 
+        self->mail(start_atom_v).send(stress_solver);
         self->receive(
-            [&](int reply_id, int arg_index, std::vector<float> data) {
-                verify_spmv("CSR Host Wrapper Default", data, expected);
+            [&](std::vector<float> result) {
+                auto end = std::chrono::high_resolution_clock::now();
+                std::chrono::duration<double> elapsed = end - start;
+                std::cout << "[SUCCESS] Stress Test completed in " << elapsed.count() << " seconds." << std::endl;
+                std::cout << "[INFO] First 5 elements of solution: ";
+                for(int i=0; i<5; ++i) std::cout << result[i] << " ";
+                std::cout << "..." << std::endl;
             }
         );
     }
 
-    self->send_exit(spmv, exit_reason::user_shutdown);
     manager::shutdown();
 }
 CAF_MAIN(id_block::cuda)

From 0da12c8e98e568033b297541f803a4e7a754e432 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 11:48:00 -0600
Subject: [PATCH 0720/1000] Added benchmark.

---
 .../cuda-benchmark/CMakeLists.txt             |  38 ++++
 .../CGS-actor-test/cuda-benchmark/main.cu     | 213 ++++++++++++++++++
 2 files changed, 251 insertions(+)
 create mode 100644 libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/cuda-benchmark/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/cuda-benchmark/main.cu

diff --git a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/cuda-benchmark/CMakeLists.txt b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/cuda-benchmark/CMakeLists.txt
new file mode 100644
index 0000000000..6cfff346bd
--- /dev/null
+++ b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/cuda-benchmark/CMakeLists.txt
@@ -0,0 +1,38 @@
+cmake_minimum_required(VERSION 3.18 FATAL_ERROR)
+project(CUDALaplacianBenchmark LANGUAGES CXX CUDA)
+
+# Set the C++ standard required by your code and standard libraries
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# Request standard compliant CUDA compilation flags
+set(CMAKE_CUDA_STANDARD 17)
+set(CMAKE_CUDA_STANDARD_REQUIRED ON)
+
+# Optimization flags for benchmarking (Release mode)
+if(NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
+endif()
+
+# Find the CUDA Toolkit libraries (cuBLAS and cuSPARSE)
+find_package(CUDAToolkit REQUIRED)
+
+# Define the executable target
+# Replace 'main.cu' with whatever you name your source code file
+add_executable(cuda_benchmark main.cu)
+
+# Target compile features/options (if required)
+target_compile_options(cuda_benchmark PRIVATE 
+    $<$<COMPILE_LANGUAGE:CUDA>:--generate-line-info>
+)
+
+# Link the required NVIDIA hardware-accelerated libraries
+target_link_libraries(cuda_benchmark PRIVATE
+    CUDA::cudart
+    CUDA::cublas
+    CUDA::cusparse
+)
+
+# (Optional) Automatically match target GPU architecture if you run it locally
+# If compiling for a specific architecture, you can uncomment and adjust the line below:
+# set_target_properties(cuda_benchmark PROPERTIES CUDA_ARCHITECTURES "75;80;86")
diff --git a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/cuda-benchmark/main.cu b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/cuda-benchmark/main.cu
new file mode 100644
index 0000000000..48b026f65e
--- /dev/null
+++ b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/cuda-benchmark/main.cu
@@ -0,0 +1,213 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ * Modifications made to track end-to-end wall time (Host allocation -> Device -> Host result).
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <chrono>   // Added for high-precision host-side timing
+#include <iostream>
+
+/* Using updated (v2) interfaces to cublas */
+#include <cublas_v2.h>
+#include <cuda_runtime.h>
+#include <cusparse.h>
+
+#ifndef checkCudaErrors
+#define checkCudaErrors(val) check((val), #val, __FILE__, __LINE__)
+template <typename T>
+void check(T result, char const *const func, char const *const file, int const line) {
+    if (result) {
+        fprintf(stderr, "CUDA error at %s:%d code=%d\n", file, line, static_cast<int>(result));
+        exit(EXIT_FAILURE);
+    }
+}
+#endif
+
+/* genLaplacian1D: Generates a 1D Laplacian matrix in CSR format */
+void genLaplacian1D(int *I, int *J, float *val, int N, int nz)
+{
+    int current_nz = 0;
+    I[0] = 0;
+
+    for (int i = 0; i < N; i++) {
+        if (i > 0) {
+            J[current_nz] = i - 1;
+            val[current_nz] = -1.0f;
+            current_nz++;
+        }
+        
+        J[current_nz] = i;
+        val[current_nz] = 2.0f;
+        current_nz++;
+
+        if (i < N - 1) {
+            J[current_nz] = i + 1;
+            val[current_nz] = -1.0f;
+            current_nz++;
+        }
+        I[i + 1] = current_nz;
+    }
+}
+
+int main(int argc, char **argv)
+{
+    int         M = 0, N = 10000, nz = 0, *I = NULL, *J = NULL;
+    float      *val      = NULL;
+    const float tol      = 1e-4f; 
+    const int   max_iter = 20000; 
+    float      *x;
+    float      *rhs;
+    float       a, b, na, r0, r1;
+    int        *d_col, *d_row;
+    float      *d_val, *d_x, dot;
+    float      *d_r, *d_p, *d_Ax;
+    int         k;
+    float       alpha, beta, alpham1;
+
+    // Device setup
+    int devID = 0;
+    checkCudaErrors(cudaSetDevice(devID));
+
+    /* Host generation of the 1D Laplacian Operator */
+    nz = (N - 2) * 3 + 4; 
+    I   = (int *)malloc(sizeof(int) * (N + 1));
+    J   = (int *)malloc(sizeof(int) * nz);
+    val = (float *)malloc(sizeof(float) * nz);
+    genLaplacian1D(I, J, val, N, nz);
+
+    x   = (float *)malloc(sizeof(float) * N);
+    rhs = (float *)malloc(sizeof(float) * N);
+
+    for (int i = 0; i < N; i++) {
+        rhs[i] = 1.0f; 
+        x[i]   = 0.0f; 
+    }
+
+    std::cout << "\n[INFO] Starting GPU Benchmark: 1D Laplacian (N=" << N << ")..." << std::endl;
+
+    // =========================================================================
+    // START BENCHMARK: Captures allocations, H2D copies, Execution, and D2H copies
+    // =========================================================================
+    auto start_time = std::chrono::high_resolution_clock::now();
+
+    /* 1. Get handles to the library contexts */
+    cublasHandle_t cublasHandle = 0;
+    checkCudaErrors(cublasCreate(&cublasHandle));
+
+    cusparseHandle_t cusparseHandle = 0;
+    checkCudaErrors(cusparseCreate(&cusparseHandle));
+
+    /* 2. Device Memory Allocation */
+    checkCudaErrors(cudaMalloc((void **)&d_col, nz * sizeof(int)));
+    checkCudaErrors(cudaMalloc((void **)&d_row, (N + 1) * sizeof(int)));
+    checkCudaErrors(cudaMalloc((void **)&d_val, nz * sizeof(float)));
+    checkCudaErrors(cudaMalloc((void **)&d_x, N * sizeof(float)));
+    checkCudaErrors(cudaMalloc((void **)&d_r, N * sizeof(float)));
+    checkCudaErrors(cudaMalloc((void **)&d_p, N * sizeof(float)));
+    checkCudaErrors(cudaMalloc((void **)&d_Ax, N * sizeof(float)));
+
+    /* 3. Wrap raw pointers into cuSPARSE Generic API descriptors */
+    cusparseSpMatDescr_t matA = NULL;
+    checkCudaErrors(cusparseCreateCsr(&matA, N, N, nz, d_row, d_col, d_val,
+                                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                      CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+    cusparseDnVecDescr_t vecx = NULL;
+    checkCudaErrors(cusparseCreateDnVec(&vecx, N, d_x, CUDA_R_32F));
+    cusparseDnVecDescr_t vecp = NULL;
+    checkCudaErrors(cusparseCreateDnVec(&vecp, N, d_p, CUDA_R_32F));
+    cusparseDnVecDescr_t vecAx = NULL;
+    checkCudaErrors(cusparseCreateDnVec(&vecAx, N, d_Ax, CUDA_R_32F));
+
+    /* 4. Host-to-Device Memory Transfer (H2D) */
+    checkCudaErrors(cudaMemcpy(d_col, J, nz * sizeof(int), cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(d_row, I, (N + 1) * sizeof(int), cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(d_val, val, nz * sizeof(float), cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(d_x, x, N * sizeof(float), cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(d_r, rhs, N * sizeof(float), cudaMemcpyHostToDevice));
+
+    alpha   = 1.0f;
+    alpham1 = -1.0f;
+    beta    = 0.0f;
+    r0      = 0.0f;
+
+    /* 5. Allocate cuSPARSE internal workspace buffer */
+    size_t bufferSize = 0;
+    checkCudaErrors(cusparseSpMV_bufferSize(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                            &alpha, matA, vecx, &beta, vecAx, CUDA_R_32F,
+                                            CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    void *buffer = NULL;
+    checkCudaErrors(cudaMalloc(&buffer, bufferSize));
+
+    /* 6. Run Conjugate Gradient Solver Loop */
+    checkCudaErrors(cusparseSpMV(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                 &alpha, matA, vecx, &beta, vecAx, CUDA_R_32F,
+                                 CUSPARSE_SPMV_ALG_DEFAULT, buffer));
+
+    cublasSaxpy(cublasHandle, N, &alpham1, d_Ax, 1, d_r, 1);
+    cublasSdot(cublasHandle, N, d_r, 1, d_r, 1, &r1);
+
+    k = 1;
+    while (r1 > tol * tol && k <= max_iter) {
+        if (k > 1) {
+            b = r1 / r0;
+            cublasSscal(cublasHandle, N, &b, d_p, 1);
+            cublasSaxpy(cublasHandle, N, &alpha, d_r, 1, d_p, 1);
+        } else {
+            cublasScopy(cublasHandle, N, d_r, 1, d_p, 1);
+        }
+
+        checkCudaErrors(cusparseSpMV(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                     &alpha, matA, vecp, &beta, vecAx, CUDA_R_32F,
+                                     CUSPARSE_SPMV_ALG_DEFAULT, buffer));
+        
+        cublasSdot(cublasHandle, N, d_p, 1, d_Ax, 1, &dot);
+        a = r1 / dot;
+
+        cublasSaxpy(cublasHandle, N, &a, d_p, 1, d_x, 1);
+        na = -a;
+        cublasSaxpy(cublasHandle, N, &na, d_Ax, 1, d_r, 1);
+
+        r0 = r1;
+        cublasSdot(cublasHandle, N, d_r, 1, d_r, 1, &r1);
+        k++;
+    }
+
+    /* 7. Device-to-Host Memory Transfer (D2H) */
+    checkCudaErrors(cudaMemcpy(x, d_x, N * sizeof(float), cudaMemcpyDeviceToHost));
+
+    // Force host to wait for all queued GPU operations and transfers to complete
+    cudaDeviceSynchronize(); 
+
+    auto end_time = std::chrono::high_resolution_clock::now();
+    // =========================================================================
+    // END BENCHMARK
+    // =========================================================================
+
+    std::chrono::duration<double> elapsed = end_time - start_time;
+
+    // Match your actor framework's terminal presentation style
+    std::cout << "[SUCCESS] Stress Test completed in " << elapsed.count() << " seconds." << std::endl;
+    std::cout << "[INFO] Iterations taken to converge: " << (k - 1) << std::endl;
+    std::cout << "[INFO] First 5 elements of solution: ";
+    for(int i = 0; i < 5; ++i) {
+        std::cout << x[i] << " ";
+    }
+    std::cout << "..." << std::endl;
+
+    // Clean up GPU allocations
+    if (buffer) cudaFree(buffer);
+    cusparseDestroy(cusparseHandle);
+    cublasDestroy(cublasHandle);
+    if (matA) cusparseDestroySpMat(matA);
+    if (vecx) cusparseDestroyDnVec(vecx);
+    if (vecAx) cusparseDestroyDnVec(vecAx);
+    if (vecp) cusparseDestroyDnVec(vecp);
+    cudaFree(d_col); cudaFree(d_row); cudaFree(d_val);
+    cudaFree(d_x); cudaFree(d_r); cudaFree(d_p); cudaFree(d_Ax);
+    
+    free(I); free(J); free(val); free(x); free(rhs);
+
+    return 0;
+}

From fb8a96968f1f43cf53827e913ff3813438626ae6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 11:55:48 -0600
Subject: [PATCH 0721/1000] removed drift check

---
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 68 +++----------------
 1 file changed, 10 insertions(+), 58 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index 1d542ffb0c..3cacd3513b 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -29,12 +29,7 @@ enum class sparse_cg_step {
   main_dot_rr,
   update_p_copy_r,
   update_p_axpy_p,
-  update_p_final_copy,
-  restart_spmv_y,
-  restart_copy_b,
-  restart_axpy_r,
-  restart_copy_p,
-  restart_dot_rho
+  update_p_final_copy
 };
 
 // Reply IDs used to distinguish which actor type is replying
@@ -66,16 +61,11 @@ struct sparse_cg_state {
   // Scalars
   float rho = 0.0f;
   float old_rho = 0.0f;
-  float cur_norm = 0.0f;
   float alpha = 0.0f;
   float beta = 0.0f;
   int iterations = 0;
   sparse_cg_step step = sparse_cg_step::idle;
 
-  // Fault Tolerance: Stagnation Detection
-  float last_norm = -1.0f;
-  int stagnation_count = 0;
-
   // BLAS/SPARSE Actors
   caf::actor dot_actor, spmv_actor, axpy_actor, copy_actor;
   caf::actor supervisor;
@@ -160,7 +150,7 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
 
   void iterate() {
     auto& s = state();
-    if (s.iterations >= s.max_iter || s.cur_norm < s.tol) {
+    if (s.iterations >= s.max_iter || s.rho < (s.tol * s.tol)) {
       if (s.supervisor) {
         command_runner<float> cr;
         cr.copy_to_host_async(s.x, [this, target = s.supervisor](std::vector<float>&& data) {
@@ -174,13 +164,6 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
     send_spmv(s.p, s.w);
   }
 
-  void perform_restart() {
-    auto& s = state();
-    s.stagnation_count = 0;
-    s.step = sparse_cg_step::restart_spmv_y;
-    send_spmv(s.x, s.y_tmp);
-  }
-
   void send_spmv(mem_ptr<float> input_v, mem_ptr<float> output_v) {
     auto& s = state();
     auto send = [&](auto format_atom) {
@@ -201,7 +184,7 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
     auto& s = state();
     switch (s.step) {
       case sparse_cg_step::init_rho:
-        s.rho = val; s.cur_norm = std::sqrt(val);
+        s.rho = val;
         iterate();
         break;
       case sparse_cg_step::main_dot_pw:
@@ -210,43 +193,22 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
         this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.p, s.x, s.n, s.alpha).send(s.axpy_actor);
         break;
       case sparse_cg_step::main_dot_rr:
-        s.old_rho = s.rho; s.rho = val; s.cur_norm = std::sqrt(val);
-        check_stagnation();
-        break;
-      case sparse_cg_step::restart_dot_rho:
-        s.rho = val; s.cur_norm = std::sqrt(val);
-        iterate();
+        s.old_rho = s.rho;
+        s.rho = val;
+        s.iterations++;
+        s.beta = s.rho / s.old_rho;
+        s.step = sparse_cg_step::update_p_copy_r;
+        this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.r, s.w, s.n).send(s.copy_actor);
         break;
       default: break;
     }
   }
 
-  void check_stagnation() {
-    auto& s = state();
-    bool diverged = s.last_norm > 0 && s.cur_norm > s.last_norm * 1.5f;
-    bool stalled = s.last_norm > 0 && s.cur_norm > s.last_norm * 0.999f;
-    s.iterations++;
-    if (stalled || diverged) s.stagnation_count++;
-    else s.stagnation_count = 0;
-    s.last_norm = s.cur_norm;
-
-    if (s.stagnation_count >= 15 || diverged) {
-      perform_restart();
-    } else {
-      s.beta = s.rho / s.old_rho;
-      s.step = sparse_cg_step::update_p_copy_r;
-      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.r, s.w, s.n).send(s.copy_actor);
-    }
-  }
-
   void handle_spmv_result() {
     auto& s = state();
     if (s.step == sparse_cg_step::main_spmv_w) {
       s.step = sparse_cg_step::main_dot_pw;
       this->mail(s.device_num, s.stream_id, s.p, s.w, s.y_tmp, s.n).send(s.dot_actor);
-    } else if (s.step == sparse_cg_step::restart_spmv_y) {
-      s.step = sparse_cg_step::restart_copy_b;
-      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.b, s.r, s.n).send(s.copy_actor);
     }
   }
 
@@ -261,10 +223,8 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
     } else if (s.step == sparse_cg_step::update_p_axpy_p) {
       s.step = sparse_cg_step::update_p_final_copy;
       this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.w, s.p, s.n).send(s.copy_actor);
-    } else if (s.step == sparse_cg_step::restart_axpy_r) {
-      s.step = sparse_cg_step::restart_copy_p;
-      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.r, s.p, s.n).send(s.copy_actor);
     }
+    else if (s.step == sparse_cg_step::idle) return;
   }
 
   void handle_copy_result() {
@@ -285,14 +245,6 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
       case sparse_cg_step::update_p_final_copy:
         iterate();
         break;
-      case sparse_cg_step::restart_copy_b:
-        s.step = sparse_cg_step::restart_axpy_r;
-        this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.y_tmp, s.r, s.n, -1.0f).send(s.axpy_actor);
-        break;
-      case sparse_cg_step::restart_copy_p:
-        s.step = sparse_cg_step::restart_dot_rho;
-        this->mail(s.device_num, s.stream_id, s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
-        break;
       default: break;
     }
   }

From 77bb559afc7badc52209cdb44fd9362f66abbd62 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 13:50:52 -0600
Subject: [PATCH 0722/1000] Initial benchmark test.

---
 .../benchmark-test/CMakeLists.txt             |  45 +++++++
 .../benchmark-test/main.test.cpp              | 119 ++++++++++++++++++
 2 files changed, 164 insertions(+)
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/main.test.cpp

diff --git a/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/CMakeLists.txt b/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/CMakeLists.txt
new file mode 100644
index 0000000000..5489f5162c
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/CMakeLists.txt
@@ -0,0 +1,45 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/./../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+)
+
diff --git a/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/main.test.cpp
new file mode 100644
index 0000000000..dfb1b47c56
--- /dev/null
+++ b/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/main.test.cpp
@@ -0,0 +1,119 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include <chrono>
+#include <cublas_v2.h>
+#include "caf/actorBLAS/dot-actor/dot-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+/**
+ * Benchmark test for dot_actor.hpp versus native cuBLAS.
+ * Measures the time taken to perform sequences of (H2D -> sdot -> D2H) operations.
+ */
+
+// Native cuBLAS benchmark implementation using Driver API and cuBLAS
+void run_native_benchmark(int n, const std::vector<int>& iterations_series) {
+    std::cout << "\n[INFO] Running Native cuBLAS Benchmark..." << std::endl;
+    
+    check(cuInit(0), "cuInit");
+    CUdevice dev;
+    check(cuDeviceGet(&dev, 0), "cuDeviceGet");
+    CUcontext ctx;
+    check(cuCtxCreate(&ctx, 0, dev), "cuCtxCreate");
+    
+    cublasHandle_t handle;
+    if (cublasCreate(&handle) != CUBLAS_STATUS_SUCCESS) {
+        std::cerr << "[ERROR] cublasCreate failed" << std::endl;
+        return;
+    }
+
+    std::vector<float> h_x(n, 1.0f);
+    std::vector<float> h_y(n, 1.0f);
+    float h_res = 0.0f;
+    size_t bytes = n * sizeof(float);
+
+    for (int iters : iterations_series) {
+        auto start = std::chrono::steady_clock::now();
+        
+        for (int i = 0; i < iters; ++i) {
+            CUdeviceptr d_x, d_y, d_res;
+            check(cuMemAlloc(&d_x, bytes), "cuMemAlloc d_x");
+            check(cuMemAlloc(&d_y, bytes), "cuMemAlloc d_y");
+            check(cuMemAlloc(&d_res, sizeof(float)), "cuMemAlloc d_res");
+
+            check(cuMemcpyHtoD(d_x, h_x.data(), bytes), "cuMemcpyHtoD d_x");
+            check(cuMemcpyHtoD(d_y, h_y.data(), bytes), "cuMemcpyHtoD d_y");
+
+            cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_DEVICE);
+            if (cublasSdot(handle, n, (const float*)d_x, 1, (const float*)d_y, 1, (float*)d_res) != CUBLAS_STATUS_SUCCESS) {
+                std::cerr << "[ERROR] cublasSdot failed" << std::endl;
+            }
+            
+            check(cuMemcpyDtoH(&h_res, d_res, sizeof(float)), "cuMemcpyDtoH d_res");
+
+            check(cuMemFree(d_x), "cuMemFree d_x");
+            check(cuMemFree(d_y), "cuMemFree d_y");
+            check(cuMemFree(d_res), "cuMemFree d_res");
+        }
+        
+        auto end = std::chrono::steady_clock::now();
+        auto diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+        std::cout << "[NATIVE] Iterations: " << iters << " | Time: " << diff << " ms" << std::endl;
+    }
+
+    cublasDestroy(handle);
+    cuCtxDestroy(ctx);
+}
+
+// dot_actor benchmark implementation
+void run_actor_benchmark(actor_system& sys, int n, const std::vector<int>& iterations_series) {
+    std::cout << "\n[INFO] Running Dot Actor Benchmark..." << std::endl;
+    
+    scoped_actor self{sys};
+    // Spawn dot_actor with reply_id = 0
+    auto dot = sys.spawn<dot_actor>(0);
+    
+    std::vector<float> h_x(n, 1.0f);
+    std::vector<float> h_y(n, 1.0f);
+
+    for (int iters : iterations_series) {
+        auto start = std::chrono::steady_clock::now();
+        
+        for (int i = 0; i < iters; ++i) {
+            self->mail(create_in_arg(h_x), 
+                       create_in_arg(h_y), 
+                       create_out_arg_with_size<float>(1), 
+                       n).send(dot);
+            
+            self->receive(
+                [&](int rid, float result) {
+                    // Result received via message
+                }
+            );
+        }
+        
+        auto end = std::chrono::steady_clock::now();
+        auto diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+        std::cout << "[ACTOR]  Iterations: " << iters << " | Time: " << diff << " ms" << std::endl;
+    }
+    
+    self->send_exit(dot, exit_reason::user_shutdown);
+}
+
+void caf_main(actor_system& sys) {
+    // Initialize CUDA manager with cuBLAS support
+    manager::init(sys, manager_config(true));
+
+    int n = 1000000; // 1M elements
+    std::vector<int> series = {1, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000};
+
+    run_native_benchmark(n, series);
+    run_actor_benchmark(sys, n, series);
+
+    manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda)

From 464342cbcbf4d0af5974ec1d681c1a4252d531c5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 14:02:32 -0600
Subject: [PATCH 0723/1000] Fixed build errors.

---
 .../dot-actor-test/benchmark-test/CMakeLists.txt               | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/CMakeLists.txt b/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/CMakeLists.txt
index 5489f5162c..a6747b1b6f 100644
--- a/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/CMakeLists.txt
@@ -6,7 +6,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS      OFF)
 
 # 2) Set CAF source and build directories
-set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/./../../../../../actor-framework")
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../actor-framework")
 set(CAF_BUILD "${CAF_SRC}/build")
 
 
@@ -41,5 +41,6 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
     CUDA::cublas
+    CUDA::cusparse
 )
 

From f9de6a091fb916870fd82625eaae8014d7edbdfe Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 25 May 2026 15:30:44 -0600
Subject: [PATCH 0724/1000] made changes to actors working to make it
 sigificantly faster

---
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 374 ++++++++----------
 1 file changed, 155 insertions(+), 219 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index 3cacd3513b..0ce250eb3a 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -8,28 +8,29 @@
 #include "caf/actorSPARSE/spmv-actor/spmv-actor.hpp"
 #include "caf/actorBLAS/axpy-actor/axpy-actor.hpp"
 #include "caf/actorBLAS/copy-actor/copy-actor.hpp"
+#include "caf/cuda/platform.hpp"
 
 namespace caf::cuda {
 
 enum class matrix_format {
-  csr,
-  csc,
-  coo
+ csr,
+ csc,
+ coo
 };
 
 enum class sparse_cg_step {
-  idle,
-  init_r,
-  init_p,
-  init_rho,
-  main_spmv_w,
-  main_dot_pw,
-  main_axpy_x,
-  main_axpy_r,
-  main_dot_rr,
-  update_p_copy_r,
-  update_p_axpy_p,
-  update_p_final_copy
+ idle,
+ init_r,
+ init_p,
+ init_rho,
+ main_spmv_w,
+ main_dot_pw,
+ main_axpy_x,
+ main_axpy_r,
+ main_dot_rr,
+ update_p_copy_r,
+ update_p_axpy_p,
+ update_p_final_copy
 };
 
 // Reply IDs used to distinguish which actor type is replying
@@ -39,215 +40,150 @@ constexpr int id_axpy = 300;
 constexpr int id_copy = 400;
 
 struct sparse_cg_state {
-  // Host Data (stored until start)
-  in<int> h_row_ptr, h_col_ind;
-  in<float> h_values, h_b;
-  in_out<float> h_x;
-
-  // Device Problem data
-  mem_ptr<int> A_row_ptr, A_col_ind;
-  mem_ptr<float> A_values, b, x;
-  
-  matrix_format format;
-  int n, nnz;
-  float tol;
-  int max_iter;
-  int device_num;
-  int stream_id;
-
-  // Workspace vectors
-  mem_ptr<float> r, p, w, y_tmp;
-  
-  // Scalars
-  float rho = 0.0f;
-  float old_rho = 0.0f;
-  float alpha = 0.0f;
-  float beta = 0.0f;
-  int iterations = 0;
-  sparse_cg_step step = sparse_cg_step::idle;
-
-  // BLAS/SPARSE Actors
-  caf::actor dot_actor, spmv_actor, axpy_actor, copy_actor;
-  caf::actor supervisor;
+ // Host Data (stored until start)
+ in<int> h_row_ptr, h_col_ind;
+ in<float> h_values, h_b;
+ in_out<float> h_x;
+
+ // Device Problem data
+ mem_ptr<int> A_row_ptr, A_col_ind;
+ mem_ptr<float> A_values, b, x;
+ matrix_format format;
+ int n, nnz;
+ float tol;
+ int max_iter;
+ int device_num;
+ int stream_id;
+ caf::actor supervisor;
+
+ // Workspace vectors
+ mem_ptr<float> r, p, w, y_tmp;
+ 
+ // Scalars
+ float rho = 0.0f;
+ float old_rho = 0.0f;
+ float alpha = 0.0f;
+ float beta = 0.0f;
+ int iterations = 0;
+ sparse_cg_step step = sparse_cg_step::idle;
 };
 
 class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
 public:
-  sparse_cg_actor(actor_config& cfg, in<int> rp, in<int> ci, 
-                  in<float> val, in<float> b, in_out<float> x,
-                  matrix_format fmt, int n, int nnz, float tol, int max_iter, int device_num, int stream_id,
-                  caf::actor supervisor = nullptr)
-    : stateful_actor<sparse_cg_state>(cfg) {
-    state().h_row_ptr = std::move(rp);
-    state().h_col_ind = std::move(ci);
-    state().h_values = std::move(val);
-    state().h_b = std::move(b);
-    state().h_x = std::move(x);
-    state().format = fmt;
-    state().n = n; state().nnz = nnz; 
-    state().tol = tol; state().max_iter = max_iter;
-    state().device_num = device_num; state().stream_id = stream_id;
-    state().supervisor = supervisor;
-
-    // Spawn helpers
-    state().dot_actor = this->system().spawn<caf::cuda::dot_actor>(id_dot);
-    state().spmv_actor = this->system().spawn<caf::cuda::spmv_actor>(id_spmv);
-    state().axpy_actor = this->system().spawn<caf::cuda::axpy_actor>(id_axpy);
-    state().copy_actor = this->system().spawn<caf::cuda::copy_actor>(id_copy);
-  }
-
-  behavior make_behavior() override {
-    return {
-      [this](start_atom) {
-        if (!state().supervisor)
-          state().supervisor = actor_cast<caf::actor>(this->current_sender());
-        start_setup();
-      },
-      [this](int rid, float val) {
-        if (rid == id_dot) handle_dot_result(val);
-      },
-      // SPMV result (CSR Signature: rid, rp, ci, val, x, y)
-      [this](int rid, mem_ptr<int>, mem_ptr<int>, mem_ptr<float>, mem_ptr<float>, mem_ptr<float>) {
-        if (rid == id_spmv) handle_spmv_result();
-      },
-      [this](int rid, mem_ptr<float> x, mem_ptr<float> y) {
-        if (rid == id_copy) handle_copy_result();
-        else if (rid == id_axpy) handle_axpy_result();
-      }
-    };
-  }
+ sparse_cg_actor(actor_config& cfg, in<int> rp, in<int> ci,
+                 in<float> val, in<float> b, in_out<float> x,
+                 matrix_format fmt, int n, int nnz, float tol, int max_iter, int device_num, int stream_id,
+                 caf::actor supervisor = nullptr)
+   : stateful_actor<sparse_cg_state>(cfg) {
+   state().h_row_ptr = std::move(rp);
+   state().h_col_ind = std::move(ci);
+   state().h_values = std::move(val);
+   state().h_b = std::move(b);
+   state().h_x = std::move(x);
+   state().format = fmt;
+   state().n = n; state().nnz = nnz;
+   state().tol = tol; state().max_iter = max_iter;
+   state().device_num = device_num; state().stream_id = stream_id;
+   state().supervisor = supervisor;
+ }
+
+ behavior make_behavior() override {
+   return {
+     [this](start_atom) {
+       auto& s = state();
+       if (!s.supervisor)
+         s.supervisor = actor_cast<caf::actor>(this->current_sender());
+       start_setup();
+     },
+   };
+ }
 
 private:
-  void start_setup() {
-    auto& s = state();
-    command_runner<> runner;
-    
-    // Transfer problem data to device
-    auto res = runner.transfer_memory(s.device_num, s.stream_id, 
-                                      s.h_row_ptr,
-                                      s.h_col_ind,
-                                      s.h_values,
-                                      s.h_b,
-                                      s.h_x);
-    
-    s.A_row_ptr = std::get<0>(res);
-    s.A_col_ind = std::get<1>(res);
-    s.A_values  = std::get<2>(res);
-    s.b         = std::get<3>(res);
-    s.x         = std::get<4>(res);
-
-    // Allocate workspace
-    command_runner<out<float>> work_runner;
-    s.r = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-    s.p = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-    s.w = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-    s.y_tmp = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-
-    // Start algorithm: r = b
-    s.step = sparse_cg_step::init_r;
-    this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.b, s.r, s.n).send(s.copy_actor);
-  }
-
-  void iterate() {
-    auto& s = state();
-    if (s.iterations >= s.max_iter || s.rho < (s.tol * s.tol)) {
-      if (s.supervisor) {
-        command_runner<float> cr;
-        cr.copy_to_host_async(s.x, [this, target = s.supervisor](std::vector<float>&& data) {
-          this->mail(std::move(data)).send(target);
-        });
-      }
-      s.step = sparse_cg_step::idle;
-      return;
-    }
-    s.step = sparse_cg_step::main_spmv_w;
-    send_spmv(s.p, s.w);
-  }
-
-  void send_spmv(mem_ptr<float> input_v, mem_ptr<float> output_v) {
-    auto& s = state();
-    auto send = [&](auto format_atom) {
-      this->mail(return_mem_ptr_atom_v, format_atom, s.device_num, s.stream_id, 
-                 s.A_row_ptr, s.A_col_ind, s.A_values, input_v, output_v, 
-                 s.n, s.n, s.nnz).send(s.spmv_actor);
-    };
-
-    switch (s.format) {
-      case matrix_format::csr: send(csr_atom_v); break;
-      case matrix_format::csc: send(csc_atom_v); break;
-      case matrix_format::coo: send(coo_atom_v); break;
-      default: break;
-    }
-  }
-
-  void handle_dot_result(float val) {
-    auto& s = state();
-    switch (s.step) {
-      case sparse_cg_step::init_rho:
-        s.rho = val;
-        iterate();
-        break;
-      case sparse_cg_step::main_dot_pw:
-        s.alpha = s.rho / val;
-        s.step = sparse_cg_step::main_axpy_x;
-        this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.p, s.x, s.n, s.alpha).send(s.axpy_actor);
-        break;
-      case sparse_cg_step::main_dot_rr:
-        s.old_rho = s.rho;
-        s.rho = val;
-        s.iterations++;
-        s.beta = s.rho / s.old_rho;
-        s.step = sparse_cg_step::update_p_copy_r;
-        this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.r, s.w, s.n).send(s.copy_actor);
-        break;
-      default: break;
-    }
-  }
-
-  void handle_spmv_result() {
-    auto& s = state();
-    if (s.step == sparse_cg_step::main_spmv_w) {
-      s.step = sparse_cg_step::main_dot_pw;
-      this->mail(s.device_num, s.stream_id, s.p, s.w, s.y_tmp, s.n).send(s.dot_actor);
-    }
-  }
-
-  void handle_axpy_result() {
-    auto& s = state();
-    if (s.step == sparse_cg_step::main_axpy_x) {
-      s.step = sparse_cg_step::main_axpy_r;
-      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.w, s.r, s.n, -s.alpha).send(s.axpy_actor);
-    } else if (s.step == sparse_cg_step::main_axpy_r) {
-      s.step = sparse_cg_step::main_dot_rr;
-      this->mail(s.device_num, s.stream_id, s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
-    } else if (s.step == sparse_cg_step::update_p_axpy_p) {
-      s.step = sparse_cg_step::update_p_final_copy;
-      this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.w, s.p, s.n).send(s.copy_actor);
-    }
-    else if (s.step == sparse_cg_step::idle) return;
-  }
-
-  void handle_copy_result() {
-    auto& s = state();
-    switch (s.step) {
-      case sparse_cg_step::init_r:
-        s.step = sparse_cg_step::init_p;
-        this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.r, s.p, s.n).send(s.copy_actor);
-        break;
-      case sparse_cg_step::init_p:
-        s.step = sparse_cg_step::init_rho;
-        this->mail(s.device_num, s.stream_id, s.r, s.r, s.y_tmp, s.n).send(s.dot_actor);
-        break;
-      case sparse_cg_step::update_p_copy_r:
-        s.step = sparse_cg_step::update_p_axpy_p;
-        this->mail(return_mem_ptr_atom_v, s.device_num, s.stream_id, s.p, s.w, s.n, s.beta).send(s.axpy_actor);
-        break;
-      case sparse_cg_step::update_p_final_copy:
-        iterate();
-        break;
-      default: break;
-    }
-  }
+ void start_setup() {
+   auto& s = state();
+   command_runner<> runner;
+
+   // Transfer problem data to device
+   auto res = runner.transfer_memory(s.device_num, s.stream_id,
+                                     s.h_row_ptr,
+                                     s.h_col_ind,
+                                     s.h_values,
+                                     s.h_b,
+                                     s.h_x);
+  
+   s.A_row_ptr = std::get<0>(res);
+   s.A_col_ind = std::get<1>(res);
+   s.A_values  = std::get<2>(res);
+   s.b         = std::get<3>(res);
+   s.x         = std::get<4>(res);
+
+   // Allocate workspace
+   command_runner<out<float>> work_runner;
+   s.r = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+   s.p = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+   s.w = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+   s.y_tmp = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+
+   auto dev = platform::create()->schedule(s.stream_id, s.device_num);
+   float alpha_const = 1.0f;
+   float beta_const = 0.0f;
+   float alpham1 = -1.0f;
+
+   // 1. Initial SpMV: w = A * x
+   execute_spmv(s.x, s.w);
+
+   // 2. Initial r = b - w
+   dev->scopy(s.stream_id, s.n, s.b, s.r);
+   dev->saxpy(s.stream_id, s.n, alpham1, s.w, s.r);
+
+   // 3. Initial rho = r * r
+   dev->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
+   s.rho = s.y_tmp->copy_to_host()[0];
+
+   s.iterations = 1;
+   while (s.rho > (s.tol * s.tol) && s.iterations <= s.max_iter) {
+     if (s.iterations > 1) {
+       s.beta = s.rho / s.old_rho;
+       dev->scopy(s.stream_id, s.n, s.r, s.w);
+       dev->saxpy(s.stream_id, s.n, s.beta, s.p, s.w);
+       dev->scopy(s.stream_id, s.n, s.w, s.p);
+     } else {
+       dev->scopy(s.stream_id, s.n, s.r, s.p);
+     }
+
+     execute_spmv(s.p, s.w);
+
+     dev->sdot(s.stream_id, s.n, s.p, s.w, s.y_tmp);
+     float dot_pw = s.y_tmp->copy_to_host()[0];
+
+     s.alpha = s.rho / dot_pw;
+
+     dev->saxpy(s.stream_id, s.n, s.alpha, s.p, s.x);
+     dev->saxpy(s.stream_id, s.n, -s.alpha, s.w, s.r);
+
+     s.old_rho = s.rho;
+     dev->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
+     s.rho = s.y_tmp->copy_to_host()[0];
+
+     s.iterations++;
+   }
+
+   auto solution = s.x->copy_to_host();
+  
+   if (s.supervisor)
+     this->mail(std::move(solution)).send(s.supervisor);
+ }
+
+ void execute_spmv(mem_ptr<float> input_v, mem_ptr<float> output_v) {
+   auto& s = state();
+   auto dev = platform::create()->schedule(s.stream_id, s.device_num);
+   switch (s.format) {
+     case matrix_format::csr: dev->spmv_csr(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v); break;
+     case matrix_format::csc: dev->spmv_csc(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v); break;
+     case matrix_format::coo: dev->spmv_coo(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v); break;
+     default: break;
+   }
+ }
 };
 
-} // namespace caf::cuda
+} // namespace caf::cuda
\ No newline at end of file

From bf0bf02e0995c794d091b1e9c911ac514148fbce Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 09:08:14 -0600
Subject: [PATCH 0725/1000] Initial commit

---
 libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp     |   4 +
 .../sparse-BiCGSTAB-actor.hpp                 | 183 ++++++++++++++++++
 2 files changed, 187 insertions(+)
 create mode 100644 libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp
 create mode 100644 libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BICSTAB-actor/sparse-BiCGSTAB-actor.hpp

diff --git a/libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp b/libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp
new file mode 100644
index 0000000000..20ce70f0ed
--- /dev/null
+++ b/libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp
@@ -0,0 +1,4 @@
+#pragma once
+#include "caf/actorBLAS/actorBLAS.hpp"
+#include "caf/actorSPARSE/actorSPASE.hpp"
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp"
diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BICSTAB-actor/sparse-BiCGSTAB-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BICSTAB-actor/sparse-BiCGSTAB-actor.hpp
new file mode 100644
index 0000000000..3628e3e7ff
--- /dev/null
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BICSTAB-actor/sparse-BiCGSTAB-actor.hpp
@@ -0,0 +1,183 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <vector>
+#include "caf/cuda/all.hpp"
+#include "caf/actorBLAS/dot-actor/dot-actor.hpp"
+#include "caf/actorSPARSE/spmv-actor/spmv-actor.hpp"
+#include "caf/actorBLAS/axpy-actor/axpy-actor.hpp"
+#include "caf/actorBLAS/copy-actor/copy-actor.hpp"
+#include "caf/cuda/platform.hpp"
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
+
+namespace caf::cuda {
+
+struct sparse_bicgstab_state {
+  // Host Data
+  in<int> h_row_ptr, h_col_ind;
+  in<float> h_values, h_b;
+  in_out<float> h_x;
+
+  // Device Problem data
+  mem_ptr<int> A_row_ptr, A_col_ind;
+  mem_ptr<float> A_values, b, x;
+  matrix_format format;
+  int n, nnz;
+  float tol;
+  int max_iter;
+  int device_num;
+  int stream_id;
+  caf::actor supervisor;
+
+  // Workspace vectors
+  mem_ptr<float> r, r_hat, p, v, s_vec, t_vec, y_tmp;
+  
+  // Scalars
+  float rho = 1.0f;
+  float alpha = 1.0f;
+  float omega = 1.0f;
+  float beta = 0.0f;
+  int iterations = 0;
+};
+
+class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
+public:
+  sparse_bicgstab_actor(actor_config& cfg, in<int> rp, in<int> ci,
+                        in<float> val, in<float> b, in_out<float> x,
+                        matrix_format fmt, int n, int nnz, float tol, int max_iter, int device_num, int stream_id,
+                        caf::actor supervisor = nullptr)
+    : stateful_actor<sparse_bicgstab_state>(cfg) {
+    state().h_row_ptr = std::move(rp);
+    state().h_col_ind = std::move(ci);
+    state().h_values = std::move(val);
+    state().h_b = std::move(b);
+    state().h_x = std::move(x);
+    state().format = fmt;
+    state().n = n; state().nnz = nnz;
+    state().tol = tol; state().max_iter = max_iter;
+    state().device_num = device_num; state().stream_id = stream_id;
+    state().supervisor = supervisor;
+  }
+
+  behavior make_behavior() override {
+    return {
+      [this](start_atom) {
+        auto& s = state();
+        if (!s.supervisor)
+          s.supervisor = actor_cast<caf::actor>(this->current_sender());
+        start_solve();
+      },
+    };
+  }
+
+private:
+  void start_solve() {
+    auto& s = state();
+    command_runner<> runner;
+
+    // Transfer problem data to device
+    auto res = runner.transfer_memory(s.device_num, s.stream_id,
+                                      s.h_row_ptr, s.h_col_ind,
+                                      s.h_values, s.h_b, s.h_x);
+   
+    s.A_row_ptr = std::get<0>(res);
+    s.A_col_ind = std::get<1>(res);
+    s.A_values  = std::get<2>(res);
+    s.b         = std::get<3>(res);
+    s.x         = std::get<4>(res);
+
+    // Allocate workspace
+    command_runner<out<float>> work_runner;
+    s.r     = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+    s.r_hat = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+    s.p     = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+    s.v     = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+    s.s_vec = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+    s.t_vec = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+    s.y_tmp = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+
+    auto dev = platform::create()->schedule(s.stream_id, s.device_num);
+
+    // 1. Initial Residual: r = b - Ax
+    execute_spmv(s.x, s.v); 
+    dev->scopy(s.stream_id, s.n, s.b, s.r);
+    dev->saxpy(s.stream_id, s.n, -1.0f, s.v, s.r);
+    
+    // 2. Choose r_hat = r
+    dev->scopy(s.stream_id, s.n, s.r, s.r_hat);
+
+    s.iterations = 0;
+    float residual_norm_sq = 0.0f;
+    dev->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
+    residual_norm_sq = s.y_tmp->copy_to_host()[0];
+
+    while (residual_norm_sq > (s.tol * s.tol) && s.iterations < s.max_iter) {
+      // rho_i = <r_hat, r>
+      dev->sdot(s.stream_id, s.n, s.r_hat, s.r, s.y_tmp);
+      float rho_new = s.y_tmp->copy_to_host()[0];
+
+      if (s.iterations == 0) {
+        dev->scopy(s.stream_id, s.n, s.r, s.p);
+      } else {
+        s.beta = (rho_new / s.rho) * (s.alpha / s.omega);
+        // p = r + beta * (p - omega * v)
+        dev->saxpy(s.stream_id, s.n, -s.omega, s.v, s.p); // p = p - omega*v
+        dev->scopy(s.stream_id, s.n, s.r, s.s_vec);      // use s_vec as temporary
+        dev->saxpy(s.stream_id, s.n, s.beta, s.p, s.s_vec); // s_vec = r + beta*p
+        dev->scopy(s.stream_id, s.n, s.s_vec, s.p);
+      }
+
+      s.rho = rho_new;
+
+      // v = Ap
+      execute_spmv(s.p, s.v);
+
+      // alpha = rho / <r_hat, v>
+      dev->sdot(s.stream_id, s.n, s.r_hat, s.v, s.y_tmp);
+      s.alpha = s.rho / s.y_tmp->copy_to_host()[0];
+
+      // s = r - alpha * v
+      dev->scopy(s.stream_id, s.n, s.r, s.s_vec);
+      dev->saxpy(s.stream_id, s.n, -s.alpha, s.v, s.s_vec);
+
+      // t = As
+      execute_spmv(s.s_vec, s.t_vec);
+
+      // omega = <t, s> / <t, t>
+      dev->sdot(s.stream_id, s.n, s.t_vec, s.s_vec, s.y_tmp);
+      float dot_ts = s.y_tmp->copy_to_host()[0];
+      dev->sdot(s.stream_id, s.n, s.t_vec, s.t_vec, s.y_tmp);
+      float dot_tt = s.y_tmp->copy_to_host()[0];
+      s.omega = dot_ts / dot_tt;
+
+      // x = x + alpha*p + omega*s
+      dev->saxpy(s.stream_id, s.n, s.alpha, s.p, s.x);
+      dev->saxpy(s.stream_id, s.n, s.omega, s.s_vec, s.x);
+
+      // r = s - omega*t
+      dev->scopy(s.stream_id, s.n, s.s_vec, s.r);
+      dev->saxpy(s.stream_id, s.n, -s.omega, s.t_vec, s.r);
+
+      dev->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
+      residual_norm_sq = s.y_tmp->copy_to_host()[0];
+      s.iterations++;
+    }
+
+    auto solution = s.x->copy_to_host();
+    if (s.supervisor)
+      this->mail(std::move(solution)).send(s.supervisor);
+  }
+
+  void execute_spmv(mem_ptr<float> input_v, mem_ptr<float> output_v) {
+    auto& s = state();
+    auto dev = platform::create()->schedule(s.stream_id, s.device_num);
+    switch (s.format) {
+      case matrix_format::csr: dev->spmv_csr(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v); break;
+      case matrix_format::csc: dev->spmv_csc(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v); break;
+      case matrix_format::coo: dev->spmv_coo(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v); break;
+      default: break;
+    }
+  }
+};
+
+} // namespace caf::cuda
\ No newline at end of file

From 6bc1b81c583ac4dee85fa0efea1d31b42e3d206b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 09:11:06 -0600
Subject: [PATCH 0726/1000] Initial commit.

---
 .../BICGSTAB-actor-test/CMakeLists.txt        |  46 ++++++
 .../BICGSTAB-actor-test/main.test.cpp         | 142 ++++++++++++++++++
 2 files changed, 188 insertions(+)
 create mode 100644 libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp

diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..82787c399c
--- /dev/null
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/CMakeLists.txt
@@ -0,0 +1,46 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
+)
+
diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
new file mode 100644
index 0000000000..47de59a3ee
--- /dev/null
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
@@ -0,0 +1,142 @@
+/**
+ * This test file evaluates the correctness of sparse_cg_actor for solving Ax = b.
+ * It covers:
+ * - CSR, CSC, and COO formats.
+ * - Convergence verification with simple matrices.
+ * - Stress testing with a large 1D Laplacian matrix.
+ */
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <cmath>
+#include "caf/actor_registry.hpp"
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+void verify_solution(const std::string& test_name, const std::vector<float>& actual, 
+                     const std::vector<float>& expected, float tol = 1e-3) {
+    if (actual.size() != expected.size()) {
+        std::cout << "[ERROR] " << test_name << " failed: Size mismatch (got " 
+                  << actual.size() << ", expected " << expected.size() << ")" << std::endl;
+        return;
+    }
+    bool all_correct = true;
+    for (size_t i = 0; i < actual.size(); ++i) {
+        if (std::abs(actual[i] - expected[i]) > tol) {
+            all_correct = false;
+            std::cout << "[ERROR] " << test_name << " mismatch at index " << i 
+                      << ": Expected " << expected[i] 
+                      << ", Got " << actual[i] << std::endl;
+            break;
+        }
+    }
+    if (all_correct) {
+        std::cout << "[SUCCESS] " << test_name << " converged to correct solution." << std::endl;
+    }
+}
+
+void caf_main(actor_system& sys) {
+    // Enable cuBLAS and cuSPARSE for the CG solver
+    manager::init(sys, manager_config(true, true)); 
+
+    // Simple Diagonal Matrix A (3x3): diag(4, 3, 2)
+    // b = [8, 9, 2] -> Expected x = [2, 3, 1]
+    int n = 3, nnz = 3;
+    std::vector<float> h_b = {8.0f, 9.0f, 2.0f};
+    std::vector<float> expected = {2.0f, 3.0f, 1.0f};
+    float tolerance = 1e-5f;
+    int max_iter = 100;
+
+    scoped_actor self{sys};
+
+    // Test 1: CSR Format
+    {
+        std::cout << "[INFO] Test 1: CSR format simple matrix..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto solver = sys.spawn<sparse_cg_actor>(
+            create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+            create_in_arg(h_b), create_in_out_arg(h_x),
+            matrix_format::csr, n, nnz, tolerance, max_iter, 0, 0, actor_cast<actor>(self));
+
+        self->mail(start_atom_v).send(solver);
+        self->receive(
+            [&](std::vector<float> result_x) {
+                verify_solution("CSR Simple", result_x, expected);
+            }
+        );
+    }
+
+    // Test 2: CSC Format
+    {
+        std::cout << "\n[INFO] Test 2: CSC format simple matrix..." << std::endl;
+        std::vector<int> col_ptr = {0, 1, 2, 3};
+        std::vector<int> row_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto solver = sys.spawn<sparse_cg_actor>(
+            create_in_arg(col_ptr), create_in_arg(row_ind), create_in_arg(values),
+            create_in_arg(h_b), create_in_out_arg(h_x),
+            matrix_format::csc, n, nnz, tolerance, max_iter, 0, 1, actor_cast<actor>(self));
+
+        self->mail(start_atom_v).send(solver);
+        self->receive(
+            [&](std::vector<float> result_x) {
+                verify_solution("CSC Simple", result_x, expected);
+            }
+        );
+    }
+
+    // Test 3: Stress Test - 1D Laplacian (N=10000)
+    {
+        int N_large = 10000;
+        std::cout << "\n[INFO] Test 3: Stress Test - 1D Laplacian (N=" << N_large << ")..." << std::endl;
+        
+        std::vector<int> row_ptr;
+        std::vector<int> col_ind;
+        std::vector<float> values;
+        row_ptr.push_back(0);
+        for(int i=0; i<N_large; ++i) {
+            if(i > 0) { col_ind.push_back(i-1); values.push_back(-1.0f); }
+            col_ind.push_back(i); values.push_back(2.0f);
+            if(i < N_large-1) { col_ind.push_back(i+1); values.push_back(-1.0f); }
+            row_ptr.push_back(col_ind.size());
+        }
+
+        std::vector<float> b_large(N_large, 1.0f);
+        std::vector<float> x_large(N_large, 0.0f);
+        
+        auto start = std::chrono::high_resolution_clock::now();
+        auto stress_solver = sys.spawn<sparse_cg_actor>(
+            create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+            create_in_arg(b_large), create_in_out_arg(x_large),
+            matrix_format::csr, N_large, (int)values.size(), 1e-4f, 20000, 0, 2, actor_cast<actor>(self));
+
+        self->mail(start_atom_v).send(stress_solver);
+        self->receive(
+            [&](std::vector<float> result) {
+                auto end = std::chrono::high_resolution_clock::now();
+                std::chrono::duration<double> elapsed = end - start;
+                std::cout << "[SUCCESS] Stress Test completed in " << elapsed.count() << " seconds." << std::endl;
+                std::cout << "[INFO] First 5 elements of solution: ";
+                for(int i=0; i<5; ++i) std::cout << result[i] << " ";
+                std::cout << "..." << std::endl;
+            }
+        );
+    }
+
+    manager::shutdown();
+}
+CAF_MAIN(id_block::cuda)

From 6c15b02d941f4d87a339c8abf91f5398f46a01f2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 09:20:44 -0600
Subject: [PATCH 0727/1000] renamed files

---
 .../sparse-BiCGSTAB-actor.hpp                                     | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/{sparse-BICSTAB-actor => sparse-BiCGSTAB-actor}/sparse-BiCGSTAB-actor.hpp (100%)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BICSTAB-actor/sparse-BiCGSTAB-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
similarity index 100%
rename from libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BICSTAB-actor/sparse-BiCGSTAB-actor.hpp
rename to libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp

From 8c5834d430af358c1ca360615b9d07429e162f89 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 09:21:08 -0600
Subject: [PATCH 0728/1000] Built tests.

---
 .../BICGSTAB-actor-test/CMakeLists.txt        |   7 +
 .../BICGSTAB-actor-test/main.cu               | 190 ++++++++++++++++++
 .../BICGSTAB-actor-test/main.test.cpp         |  10 +-
 3 files changed, 202 insertions(+), 5 deletions(-)
 create mode 100644 libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.cu

diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/CMakeLists.txt
index 82787c399c..72514d0353 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/CMakeLists.txt
@@ -15,6 +15,7 @@ set(CMAKE_CXX_COMPILER "/usr/bin/g++")
 set(CMAKE_C_COMPILER "/usr/bin/gcc")
 
 project(CUDA_ACTORS)
+enable_language(CUDA)
 
 find_package(CUDA REQUIRED)
 find_package(CUDAToolkit REQUIRED)
@@ -44,3 +45,9 @@ target_link_libraries(test
     CUDA::cusparse
 )
 
+add_executable(benchmark main.cu)
+target_link_libraries(benchmark
+  PRIVATE
+    CUDA::cublas
+    CUDA::cusparse
+)
diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.cu b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.cu
new file mode 100644
index 0000000000..688fce854a
--- /dev/null
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.cu
@@ -0,0 +1,190 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <chrono>
+#include <iostream>
+
+#include <cublas_v2.h>
+#include <cuda_runtime.h>
+#include <cusparse.h>
+
+#ifndef checkCudaErrors
+#define checkCudaErrors(val) check((val), #val, __FILE__, __LINE__)
+template <typename T>
+void check(T result, char const *const func, char const *const file, int const line) {
+    if (result) {
+        fprintf(stderr, "CUDA error at %s:%d code=%d\n", file, line, static_cast<int>(result));
+        exit(EXIT_FAILURE);
+    }
+}
+#endif
+
+void genLaplacian1D(int *I, int *J, float *val, int N, int nz) {
+    int current_nz = 0;
+    I[0] = 0;
+    for (int i = 0; i < N; i++) {
+        if (i > 0) { J[current_nz] = i - 1; val[current_nz] = -1.0f; current_nz++; }
+        J[current_nz] = i; val[current_nz] = 2.0f; current_nz++;
+        if (i < N - 1) { J[current_nz] = i + 1; val[current_nz] = -1.0f; current_nz++; }
+        I[i + 1] = current_nz;
+    }
+}
+
+int main(int argc, char **argv) {
+    int N = 10000, nz = 0;
+    const float tol = 1e-4f;
+    const int max_iter = 20000;
+
+    nz = (N - 2) * 3 + 4;
+    int *I = (int *)malloc(sizeof(int) * (N + 1));
+    int *J = (int *)malloc(sizeof(int) * nz);
+    float *val = (float *)malloc(sizeof(float) * nz);
+    genLaplacian1D(I, J, val, N, nz);
+
+    float *h_rhs = (float *)malloc(sizeof(float) * N);
+    float *h_x = (float *)malloc(sizeof(float) * N);
+    for (int i = 0; i < N; i++) { h_rhs[i] = 1.0f; h_x[i] = 0.0f; }
+
+    std::cout << "\n[INFO] Starting BiCGSTAB Native GPU Benchmark (N=" << N << ")..." << std::endl;
+
+    auto start_time = std::chrono::high_resolution_clock::now();
+
+    cublasHandle_t cublasH;
+    cusparseHandle_t cusparseH;
+    checkCudaErrors(cublasCreate(&cublasH));
+    checkCudaErrors(cusparseCreate(&cusparseH));
+
+    int *d_row, *d_col;
+    float *d_val, *d_x, *d_r, *d_r_hat, *d_p, *d_v, *d_s, *d_t;
+    checkCudaErrors(cudaMalloc((void **)&d_row, (N + 1) * sizeof(int)));
+    checkCudaErrors(cudaMalloc((void **)&d_col, nz * sizeof(int)));
+    checkCudaErrors(cudaMalloc((void **)&d_val, nz * sizeof(float)));
+    checkCudaErrors(cudaMalloc((void **)&d_x, N * sizeof(float)));
+    checkCudaErrors(cudaMalloc((void **)&d_r, N * sizeof(float)));
+    checkCudaErrors(cudaMalloc((void **)&d_r_hat, N * sizeof(float)));
+    checkCudaErrors(cudaMalloc((void **)&d_p, N * sizeof(float)));
+    checkCudaErrors(cudaMalloc((void **)&d_v, N * sizeof(float)));
+    checkCudaErrors(cudaMalloc((void **)&d_s, N * sizeof(float)));
+    checkCudaErrors(cudaMalloc((void **)&d_t, N * sizeof(float)));
+
+    checkCudaErrors(cudaMemcpy(d_row, I, (N + 1) * sizeof(int), cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(d_col, J, nz * sizeof(int), cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(d_val, val, nz * sizeof(float), cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(d_x, h_x, N * sizeof(float), cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(d_r, h_rhs, N * sizeof(float), cudaMemcpyHostToDevice));
+
+    cusparseSpMatDescr_t matA;
+    checkCudaErrors(cusparseCreateCsr(&matA, N, N, nz, d_row, d_col, d_val,
+                                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                      CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+
+    float alpha = 1.0, beta = 0.0, zero = 0.0, one = 1.0, minus_one = -1.0;
+
+    // r = b - Ax
+    cusparseDnVecDescr_t vecX, vecV;
+    checkCudaErrors(cusparseCreateDnVec(&vecX, N, d_x, CUDA_R_32F));
+    checkCudaErrors(cusparseCreateDnVec(&vecV, N, d_v, CUDA_R_32F));
+
+    size_t bufferSize = 0;
+    void *dBuffer = NULL;
+    checkCudaErrors(cusparseSpMV_bufferSize(cusparseH, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                            &one, matA, vecX, &zero, vecV, CUDA_R_32F,
+                                            CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    checkCudaErrors(cudaMalloc(&dBuffer, bufferSize));
+
+    checkCudaErrors(cusparseSpMV(cusparseH, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                 &one, matA, vecX, &zero, vecV, CUDA_R_32F,
+                                 CUSPARSE_SPMV_ALG_DEFAULT, dBuffer));
+    checkCudaErrors(cublasSaxpy(cublasH, N, &minus_one, d_v, 1, d_r, 1));
+    checkCudaErrors(cublasScopy(cublasH, N, d_r, 1, d_r_hat, 1));
+
+    float rho = 1.0, rho_new, alpha_bc = 1.0, omega = 1.0, beta_bc;
+    float norm_sq;
+    checkCudaErrors(cublasSdot(cublasH, N, d_r, 1, d_r, 1, &norm_sq));
+
+    int k = 0;
+    while (norm_sq > tol * tol && k < max_iter) {
+        checkCudaErrors(cublasSdot(cublasH, N, d_r_hat, 1, d_r, 1, &rho_new));
+
+        if (k == 0) {
+            checkCudaErrors(cublasScopy(cublasH, N, d_r, 1, d_p, 1));
+        } else {
+            beta_bc = (rho_new / rho) * (alpha_bc / omega);
+            float minus_omega = -omega;
+            checkCudaErrors(cublasSaxpy(cublasH, N, &minus_omega, d_v, 1, d_p, 1));
+            checkCudaErrors(cublasSscal(cublasH, N, &beta_bc, d_p, 1));
+            checkCudaErrors(cublasSaxpy(cublasH, N, &one, d_r, 1, d_p, 1));
+        }
+        rho = rho_new;
+
+        // v = Ap
+        cusparseDnVecDescr_t vecP;
+        checkCudaErrors(cusparseCreateDnVec(&vecP, N, d_p, CUDA_R_32F));
+        checkCudaErrors(cusparseSpMV(cusparseH, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                     &one, matA, vecP, &zero, vecV, CUDA_R_32F,
+                                     CUSPARSE_SPMV_ALG_DEFAULT, dBuffer));
+        checkCudaErrors(cusparseDestroyDnVec(vecP));
+
+        float dot_rv;
+        checkCudaErrors(cublasSdot(cublasH, N, d_r_hat, 1, d_v, 1, &dot_rv));
+        alpha_bc = rho / dot_rv;
+
+        // s = r - alpha * v
+        checkCudaErrors(cublasScopy(cublasH, N, d_r, 1, d_s, 1));
+        float minus_alpha = -alpha_bc;
+        checkCudaErrors(cublasSaxpy(cublasH, N, &minus_alpha, d_v, 1, d_s, 1));
+
+        // t = As
+        cusparseDnVecDescr_t vecS, vecT;
+        checkCudaErrors(cusparseCreateDnVec(&vecS, N, d_s, CUDA_R_32F));
+        checkCudaErrors(cusparseCreateDnVec(&vecT, N, d_t, CUDA_R_32F));
+        checkCudaErrors(cusparseSpMV(cusparseH, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                     &one, matA, vecS, &zero, vecT, CUDA_R_32F,
+                                     CUSPARSE_SPMV_ALG_DEFAULT, dBuffer));
+        checkCudaErrors(cusparseDestroyDnVec(vecS));
+        checkCudaErrors(cusparseDestroyDnVec(vecT));
+
+        float dot_ts, dot_tt;
+        checkCudaErrors(cublasSdot(cublasH, N, d_t, 1, d_s, 1, &dot_ts));
+        checkCudaErrors(cublasSdot(cublasH, N, d_t, 1, d_t, 1, &dot_tt));
+        omega = dot_ts / dot_tt;
+
+        // x = x + alpha*p + omega*s
+        checkCudaErrors(cublasSaxpy(cublasH, N, &alpha_bc, d_p, 1, d_x, 1));
+        checkCudaErrors(cublasSaxpy(cublasH, N, &omega, d_s, 1, d_x, 1));
+
+        // r = s - omega*t
+        checkCudaErrors(cublasScopy(cublasH, N, d_s, 1, d_r, 1));
+        float minus_omega_bc = -omega;
+        checkCudaErrors(cublasSaxpy(cublasH, N, &minus_omega_bc, d_t, 1, d_r, 1));
+
+        checkCudaErrors(cublasSdot(cublasH, N, d_r, 1, d_r, 1, &norm_sq));
+        k++;
+    }
+
+    checkCudaErrors(cudaMemcpy(h_x, d_x, N * sizeof(float), cudaMemcpyDeviceToHost));
+    cudaDeviceSynchronize();
+
+    auto end_time = std::chrono::high_resolution_clock::now();
+    std::chrono::duration<double> elapsed = end_time - start_time;
+
+    std::cout << "[SUCCESS] BiCGSTAB Stress Test completed in " << elapsed.count() << " seconds." << std::endl;
+    std::cout << "[INFO] Iterations: " << k << std::endl;
+    std::cout << "[INFO] First 5 elements of solution: ";
+    for(int i = 0; i < 5; ++i) std::cout << h_x[i] << " ";
+    std::cout << "..." << std::endl;
+
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnVec(vecX);
+    cusparseDestroyDnVec(vecV);
+    cudaFree(d_row); cudaFree(d_col); cudaFree(d_val);
+    cudaFree(d_x); cudaFree(d_r); cudaFree(d_r_hat);
+    cudaFree(d_p); cudaFree(d_v); cudaFree(d_s); cudaFree(d_t);
+    cudaFree(dBuffer);
+    cusparseDestroy(cusparseH);
+    cublasDestroy(cublasH);
+    free(I); free(J); free(val); free(h_x); free(h_rhs);
+
+    return 0;
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
index 47de59a3ee..3d326730c6 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
@@ -1,5 +1,5 @@
 /**
- * This test file evaluates the correctness of sparse_cg_actor for solving Ax = b.
+ * This test file evaluates the correctness of sparse_bicgstab_actor for solving Ax = b.
  * It covers:
  * - CSR, CSC, and COO formats.
  * - Convergence verification with simple matrices.
@@ -16,7 +16,7 @@
 #include <numeric>
 #include <cmath>
 #include "caf/actor_registry.hpp"
-#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp"
 
 using namespace caf;
 using namespace caf::cuda;
@@ -65,7 +65,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto solver = sys.spawn<sparse_cg_actor>(
+        auto solver = sys.spawn<sparse_bicgstab_actor>(
             create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
             create_in_arg(h_b), create_in_out_arg(h_x),
             matrix_format::csr, n, nnz, tolerance, max_iter, 0, 0, actor_cast<actor>(self));
@@ -86,7 +86,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto solver = sys.spawn<sparse_cg_actor>(
+        auto solver = sys.spawn<sparse_bicgstab_actor>(
             create_in_arg(col_ptr), create_in_arg(row_ind), create_in_arg(values),
             create_in_arg(h_b), create_in_out_arg(h_x),
             matrix_format::csc, n, nnz, tolerance, max_iter, 0, 1, actor_cast<actor>(self));
@@ -119,7 +119,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> x_large(N_large, 0.0f);
         
         auto start = std::chrono::high_resolution_clock::now();
-        auto stress_solver = sys.spawn<sparse_cg_actor>(
+        auto stress_solver = sys.spawn<sparse_bicgstab_actor>(
             create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
             create_in_arg(b_large), create_in_out_arg(x_large),
             matrix_format::csr, N_large, (int)values.size(), 1e-4f, 20000, 0, 2, actor_cast<actor>(self));

From 6fddbad83c754feb916b4347a8102db067514d7a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 09:24:53 -0600
Subject: [PATCH 0729/1000] updated stress test to use non symmetric matrix
 since thats was BIGGSTAb solvers are supposed to do

---
 .../tests/actorSOLVE-test/BICGSTAB-actor-test/main.cu     | 6 +++---
 .../actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp     | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.cu b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.cu
index 688fce854a..0ef089eea1 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.cu
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.cu
@@ -24,9 +24,9 @@ void genLaplacian1D(int *I, int *J, float *val, int N, int nz) {
     int current_nz = 0;
     I[0] = 0;
     for (int i = 0; i < N; i++) {
-        if (i > 0) { J[current_nz] = i - 1; val[current_nz] = -1.0f; current_nz++; }
-        J[current_nz] = i; val[current_nz] = 2.0f; current_nz++;
-        if (i < N - 1) { J[current_nz] = i + 1; val[current_nz] = -1.0f; current_nz++; }
+        if (i > 0) { J[current_nz] = i - 1; val[current_nz] = -1.5f; current_nz++; }
+        J[current_nz] = i; val[current_nz] = 4.0f; current_nz++;
+        if (i < N - 1) { J[current_nz] = i + 1; val[current_nz] = -0.5f; current_nz++; }
         I[i + 1] = current_nz;
     }
 }
diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
index 3d326730c6..522697038f 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
@@ -102,16 +102,16 @@ void caf_main(actor_system& sys) {
     // Test 3: Stress Test - 1D Laplacian (N=10000)
     {
         int N_large = 10000;
-        std::cout << "\n[INFO] Test 3: Stress Test - 1D Laplacian (N=" << N_large << ")..." << std::endl;
+        std::cout << "\n[INFO] Test 3: Stress Test - Non-Symmetric Matrix (N=" << N_large << ")..." << std::endl;
         
         std::vector<int> row_ptr;
         std::vector<int> col_ind;
         std::vector<float> values;
         row_ptr.push_back(0);
         for(int i=0; i<N_large; ++i) {
-            if(i > 0) { col_ind.push_back(i-1); values.push_back(-1.0f); }
-            col_ind.push_back(i); values.push_back(2.0f);
-            if(i < N_large-1) { col_ind.push_back(i+1); values.push_back(-1.0f); }
+            if(i > 0) { col_ind.push_back(i-1); values.push_back(-1.5f); } // Lower
+            col_ind.push_back(i); values.push_back(4.0f);                 // Diag
+            if(i < N_large-1) { col_ind.push_back(i+1); values.push_back(-0.5f); } // Upper
             row_ptr.push_back(col_ind.size());
         }
 

From 530145b72b49ede8f25a7bbe97f7f1b0a29fbeb2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 10:33:12 -0600
Subject: [PATCH 0730/1000] Intiial comimit

---
 .../tests/workload-test/CMakeLists.txt        |  45 +++++
 libcaf_cuda/tests/workload-test/main.test.cpp | 191 ++++++++++++++++++
 2 files changed, 236 insertions(+)
 create mode 100644 libcaf_cuda/tests/workload-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/workload-test/main.test.cpp

diff --git a/libcaf_cuda/tests/workload-test/CMakeLists.txt b/libcaf_cuda/tests/workload-test/CMakeLists.txt
new file mode 100644
index 0000000000..6fd4d9d3e6
--- /dev/null
+++ b/libcaf_cuda/tests/workload-test/CMakeLists.txt
@@ -0,0 +1,45 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+)
+
diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
new file mode 100644
index 0000000000..113fc61151
--- /dev/null
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -0,0 +1,191 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <string>
+#include <numeric>
+#include <cmath>
+#include <algorithm>
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+// Structure to hold raw data from the binary file
+struct SparseMatrixCOO {
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+    std::vector<int32_t> row_indices;
+    std::vector<int32_t> col_indices;
+    std::vector<float> values;
+};
+
+// Structure optimized for high-performance solvers
+struct SparseMatrixCSR {
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+    std::vector<int32_t> row_ptr;    // Size: rows + 1
+    std::vector<int32_t> col_indices;// Size: nnz
+    std::vector<float> values;       // Size: nnz
+};
+
+// Function to slurp the binary data into memory
+SparseMatrixCOO load_binary_coo(const std::string& filepath) {
+    std::ifstream file(filepath, std::ios::binary);
+    if (!file) {
+        throw std::runtime_error("Failed to open matrix file: " + filepath);
+    }
+
+    SparseMatrixCOO coo;
+    
+    // 1. Read the 12-byte header
+    file.read(reinterpret_cast<char*>(&coo.rows), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&coo.cols), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&coo.nnz), sizeof(int32_t));
+
+    // Allocate memory vectors
+    coo.row_indices.resize(coo.nnz);
+    coo.col_indices.resize(coo.nnz);
+    coo.values.resize(coo.nnz);
+
+    // 2. Stream the blocks continuously
+    file.read(reinterpret_cast<char*>(coo.row_indices.data()), coo.nnz * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(coo.col_indices.data()), coo.nnz * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(coo.values.data()), coo.nnz * sizeof(float));
+
+    return coo;
+}
+
+// Converts COO to CSR format for solver compatibility
+SparseMatrixCSR convert_coo_to_csr(const SparseMatrixCOO& coo) {
+    SparseMatrixCSR csr;
+    csr.rows = coo.rows;
+    csr.cols = coo.cols;
+    csr.nnz = coo.nnz;
+    
+    csr.row_ptr.assign(csr.rows + 1, 0);
+    csr.col_indices.resize(csr.nnz);
+    csr.values.resize(csr.nnz);
+
+    // Step 1: Count elements per row
+    for (int32_t i = 0; i < coo.nnz; ++i) {
+        csr.row_ptr[coo.row_indices[i] + 1]++;
+    }
+
+    // Step 2: Cumulative sum to build row pointers
+    for (int32_t i = 0; i < csr.rows; ++i) {
+        csr.row_ptr[i + 1] += csr.row_ptr[i];
+    }
+
+    // Step 3: Copy tracking array to insert elements in order
+    std::vector<int32_t> current_row_pos = csr.row_ptr;
+
+    // Step 4: Fill column and value arrays
+    for (int32_t i = 0; i < coo.nnz; ++i) {
+        int32_t row = coo.row_indices[i];
+        int32_t dest_pos = current_row_pos[row]++;
+        csr.col_indices[dest_pos] = coo.col_indices[i];
+        csr.values[dest_pos] = coo.values[i];
+    }
+
+    return csr;
+}
+
+// Compute b = A * x using CSR layout (Sparse Matrix-Vector Multiplication)
+std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x) {
+    std::vector<float> b(A.rows, 0.0f);
+    
+    for (int32_t i = 0; i < A.rows; ++i) {
+        float sum = 0.0f;
+        int32_t row_start = A.row_ptr[i];
+        int32_t row_end = A.row_ptr[i + 1];
+        
+        for (int32_t j = row_start; j < row_end; ++j) {
+            sum += A.values[j] * x[A.col_indices[j]];
+        }
+        b[i] = sum;
+    }
+    return b;
+}
+
+void caf_main(actor_system& sys) {
+    auto& args = sys.config().args_remainder;
+    if (args.empty()) {
+        std::cout << "Usage: workload-test <path_to_matrix_bin>\n";
+        std::cout << "Example: workload-test /scratch/nqr159/matrix-collection/matrices/spd/bcsstk08.bin\n";
+        return;
+    }
+
+    // Initialize GPU Manager with cuBLAS and cuSPARSE enabled
+    manager::init(sys, manager_config(true, true));
+    std::string filepath = args[0];
+
+    try {
+        std::cout << "Loading binary file: " << filepath << " ...\n";
+        SparseMatrixCOO coo_matrix = load_binary_coo(filepath);
+        
+        std::cout << "-> Matrix Loaded. Components: " 
+                  << "Rows=" << coo_matrix.rows 
+                  << ", Cols=" << coo_matrix.cols 
+                  << ", NNZ=" << coo_matrix.nnz << "\n";
+
+        std::cout << "Converting to Compressed Sparse Row (CSR) format...\n";
+        SparseMatrixCSR A = convert_coo_to_csr(coo_matrix);
+
+        // --- PREPARE BENCHMARK VECTORS ---
+        std::cout << "Generating test vectors (b = A * x_true)...\n";
+        
+        // 1. Create a known ideal solution vector x_true (filled with 1.0f)
+        std::vector<float> x_true(A.cols, 1.0f);
+        
+        // 2. Compute true right-hand side b
+        std::vector<float> b = compute_rhs_spmv(A, x_true);
+        
+        // 3. Allocate an initial guess vector x filled with zeros
+        std::vector<float> x_guess(A.cols, 0.0f);
+
+        std::cout << "\n=========================================\n";
+        std::cout << " Ready for Solver Execution!\n";
+        std::cout << "=========================================\n";
+        std::cout << "Arrays allocated and verified:\n";
+        std::cout << " - A.values size:      " << A.values.size() << " elements\n";
+        std::cout << " - A.row_ptr size:     " << A.row_ptr.size() << " elements\n";
+        std::cout << " - Vector b size:      " << b.size() << " elements\n";
+        std::cout << " - Vector x_guess size:" << x_guess.size() << " elements\n\n";
+        
+        scoped_actor self{sys};
+        float tolerance = 1e-5f;
+        int max_iter = 2000;
+
+        // Spawn the Sparse CG Actor
+        auto solver = sys.spawn<sparse_cg_actor>(
+            create_in_arg(A.row_ptr), create_in_arg(A.col_indices), create_in_arg(A.values),
+            create_in_arg(b), create_in_out_arg(x_guess),
+            matrix_format::csr, A.rows, A.nnz, tolerance, max_iter, 0, 0, actor_cast<actor>(self));
+
+        std::cout << "[INFO] Starting Solver Actor...\n";
+        self->mail(start_atom_v).send(solver);
+
+        self->receive(
+            [&](std::vector<float> result_x) {
+                std::cout << "\n=========================================\n";
+                std::cout << " Solver Finished!\n";
+                std::cout << "=========================================\n";
+                std::cout << "Verification (Expected values near 1.0):\n";
+                std::cout << "First 5 elements: ";
+                for (int i = 0; i < std::min(5, (int)result_x.size()); ++i) {
+                    std::cout << result_x[i] << " ";
+                }
+                std::cout << "\n";
+            }
+        );
+
+    } catch (const std::exception& e) {
+        std::cerr << "CRITICAL EXCEPTION: " << e.what() << "\n";
+    }
+    manager::shutdown();
+}
+CAF_MAIN(id_block::cuda)

From 5b26dfda7206155830e4012744b403892d613fd6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 10:38:06 -0600
Subject: [PATCH 0731/1000] Fixed build errors.

---
 libcaf_cuda/tests/workload-test/CMakeLists.txt | 1 +
 libcaf_cuda/tests/workload-test/main.test.cpp  | 9 +--------
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/CMakeLists.txt b/libcaf_cuda/tests/workload-test/CMakeLists.txt
index 6fd4d9d3e6..4012ea5228 100644
--- a/libcaf_cuda/tests/workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/workload-test/CMakeLists.txt
@@ -41,5 +41,6 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
     CUDA::cublas
+    CUDA::cusparse
 )
 
diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index 113fc61151..8deb34cb7b 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -112,16 +112,9 @@ std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<
 }
 
 void caf_main(actor_system& sys) {
-    auto& args = sys.config().args_remainder;
-    if (args.empty()) {
-        std::cout << "Usage: workload-test <path_to_matrix_bin>\n";
-        std::cout << "Example: workload-test /scratch/nqr159/matrix-collection/matrices/spd/bcsstk08.bin\n";
-        return;
-    }
-
     // Initialize GPU Manager with cuBLAS and cuSPARSE enabled
     manager::init(sys, manager_config(true, true));
-    std::string filepath = args[0];
+    std::string filepath = "/scratch/nqr159/matrix-collection/matrices/spd/bcsstk08.bin";
 
     try {
         std::cout << "Loading binary file: " << filepath << " ...\n";

From a71675f3800e7e3632f2223a66716382c5df7ad9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 11:25:31 -0600
Subject: [PATCH 0732/1000] updated actor to iterate based on message passing
 rather then using a while loop. Change is being made to ensure actor does not
 block or hog a CPU thread for too long, which could reduce overall throughput
 of the system

---
 .../sparse-BiCGSTAB-actor.hpp                 | 224 +++++++++++++-----
 1 file changed, 171 insertions(+), 53 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
index 3628e3e7ff..ed4590d7e0 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
@@ -10,14 +10,30 @@
 #include "caf/cuda/platform.hpp"
 #include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
 
+// Define a new block for BiCGSTAB specific atoms starting where the cuda block ended
+CAF_BEGIN_TYPE_ID_BLOCK(bicgstab_actor, caf::id_block::cuda::end)
+  CAF_ADD_ATOM(bicgstab_actor, bicgstab_next_step_atom)
+CAF_END_TYPE_ID_BLOCK(bicgstab_actor)
+
 namespace caf::cuda {
 
+enum class sparse_bicgstab_step {
+  idle,
+  init_residual_norm_sq, // Calculate initial ||r||^2
+  calc_rho_new,          // Calculate rho_new = <r_hat, r>
+  calc_alpha_denom,      // Calculate <r_hat, v> for alpha
+  calc_omega_num,        // Calculate <t, s> for omega
+  calc_omega_denom,      // Calculate <t, t> for omega
+  check_convergence,     // Check ||r||^2 for convergence
+  finished
+};
+
 struct sparse_bicgstab_state {
   // Host Data
   in<int> h_row_ptr, h_col_ind;
   in<float> h_values, h_b;
   in_out<float> h_x;
-
+ 
   // Device Problem data
   mem_ptr<int> A_row_ptr, A_col_ind;
   mem_ptr<float> A_values, b, x;
@@ -32,12 +48,19 @@ struct sparse_bicgstab_state {
   // Workspace vectors
   mem_ptr<float> r, r_hat, p, v, s_vec, t_vec, y_tmp;
   
-  // Scalars
-  float rho = 1.0f;
-  float alpha = 1.0f;
-  float omega = 1.0f;
-  float beta = 0.0f;
+  // Scalars needed across asynchronous steps
+  float rho_val = 1.0f; // Renamed from rho to avoid conflict with step-specific rho_new
+  float alpha_val = 1.0f;
+  float omega_val = 1.0f;
+  float beta_val = 0.0f;
+  float residual_norm_sq_val = 0.0f; // Stores the latest ||r||^2
+  float rho_new_val = 0.0f; // Stores the result of <r_hat, r>
+  float alpha_denom_val = 0.0f; // Stores the result of <r_hat, v>
+  float omega_num_val = 0.0f; // Stores the result of <t, s>
+  float omega_denom_val = 0.0f; // Stores the result of <t, t>
+
   int iterations = 0;
+  sparse_bicgstab_step step = sparse_bicgstab_step::idle;
 };
 
 class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
@@ -67,9 +90,37 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
           s.supervisor = actor_cast<caf::actor>(this->current_sender());
         start_solve();
       },
+      [this](bicgstab_next_step_atom, float val) {
+        auto& s = state();
+        // Thread-safely update scalars based on the stage that just finished
+        switch (s.step) {
+          case sparse_bicgstab_step::init_residual_norm_sq:
+          case sparse_bicgstab_step::check_convergence:
+            s.residual_norm_sq_val = val;
+            break;
+          case sparse_bicgstab_step::calc_rho_new:
+            s.rho_new_val = val;
+            break;
+          case sparse_bicgstab_step::calc_alpha_denom:
+            s.alpha_denom_val = val;
+            break;
+          case sparse_bicgstab_step::calc_omega_num:
+            s.omega_num_val = val;
+            break;
+          case sparse_bicgstab_step::calc_omega_denom:
+            s.omega_denom_val = val;
+            break;
+          default: break;
+        }
+        perform_bicgstab_step();
+      },
+      [this](gpu_done_atom, std::vector<float>& solution) {
+        if (state().supervisor)
+          this->mail(std::move(solution)).send(state().supervisor);
+        this->quit();
+      }
     };
   }
-
 private:
   void start_solve() {
     auto& s = state();
@@ -107,65 +158,132 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
     dev->scopy(s.stream_id, s.n, s.r, s.r_hat);
 
     s.iterations = 0;
-    float residual_norm_sq = 0.0f;
+    // Initial calculation of residual_norm_sq
     dev->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
-    residual_norm_sq = s.y_tmp->copy_to_host()[0];
-
-    while (residual_norm_sq > (s.tol * s.tol) && s.iterations < s.max_iter) {
-      // rho_i = <r_hat, r>
-      dev->sdot(s.stream_id, s.n, s.r_hat, s.r, s.y_tmp);
-      float rho_new = s.y_tmp->copy_to_host()[0];
-
-      if (s.iterations == 0) {
-        dev->scopy(s.stream_id, s.n, s.r, s.p);
-      } else {
-        s.beta = (rho_new / s.rho) * (s.alpha / s.omega);
-        // p = r + beta * (p - omega * v)
-        dev->saxpy(s.stream_id, s.n, -s.omega, s.v, s.p); // p = p - omega*v
-        dev->scopy(s.stream_id, s.n, s.r, s.s_vec);      // use s_vec as temporary
-        dev->saxpy(s.stream_id, s.n, s.beta, s.p, s.s_vec); // s_vec = r + beta*p
-        dev->scopy(s.stream_id, s.n, s.s_vec, s.p);
+    s.step = sparse_bicgstab_step::init_residual_norm_sq;
+    auto self = actor_cast<actor>(this);
+    runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
+      anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
+    });
+  }
+
+  void perform_bicgstab_step() {
+    auto& s = state();
+    auto dev = platform::create()->schedule(s.stream_id, s.device_num);
+    command_runner<> runner;
+    auto self = actor_cast<actor>(this);
+
+    switch (s.step) {
+      case sparse_bicgstab_step::init_residual_norm_sq:
+      case sparse_bicgstab_step::check_convergence: {
+        s.iterations++; // Increment for the current iteration
+
+        // Check convergence
+        if (s.residual_norm_sq_val <= (s.tol * s.tol) || s.iterations > s.max_iter) {
+          finish_solve();
+          return;
+        }
+
+        // rho_i = <r_hat, r>
+        dev->sdot(s.stream_id, s.n, s.r_hat, s.r, s.y_tmp);
+        s.step = sparse_bicgstab_step::calc_rho_new;
+        runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
+          anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
+        });
+        break;
+      }
+
+      case sparse_bicgstab_step::calc_rho_new: {
+        // Update p based on rho_new_val
+        if (s.iterations == 1) { // This is the first iteration
+          dev->scopy(s.stream_id, s.n, s.r, s.p);
+        } else { // Subsequent iterations
+          s.beta_val = (s.rho_new_val / s.rho_val) * (s.alpha_val / s.omega_val);
+          // p = r + beta * (p - omega * v)
+          dev->saxpy(s.stream_id, s.n, -s.omega_val, s.v, s.p); // p = p - omega*v
+          dev->scopy(s.stream_id, s.n, s.r, s.s_vec);      // use s_vec as temporary
+          dev->saxpy(s.stream_id, s.n, s.beta_val, s.p, s.s_vec); // s_vec = r + beta*p
+          dev->scopy(s.stream_id, s.n, s.s_vec, s.p);
+        }
+        s.rho_val = s.rho_new_val; // Update old rho
+
+        // v = Ap
+        execute_spmv(s.p, s.v);
+
+        // alpha = rho / <r_hat, v> -> calculate denominator
+        dev->sdot(s.stream_id, s.n, s.r_hat, s.v, s.y_tmp);
+        s.step = sparse_bicgstab_step::calc_alpha_denom;
+        runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
+          anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
+        });
+        break;
       }
 
-      s.rho = rho_new;
+      case sparse_bicgstab_step::calc_alpha_denom: {
+        s.alpha_val = s.rho_val / s.alpha_denom_val;
 
-      // v = Ap
-      execute_spmv(s.p, s.v);
+        // s = r - alpha * v
+        dev->scopy(s.stream_id, s.n, s.r, s.s_vec);
+        dev->saxpy(s.stream_id, s.n, -s.alpha_val, s.v, s.s_vec);
 
-      // alpha = rho / <r_hat, v>
-      dev->sdot(s.stream_id, s.n, s.r_hat, s.v, s.y_tmp);
-      s.alpha = s.rho / s.y_tmp->copy_to_host()[0];
+        // t = As
+        execute_spmv(s.s_vec, s.t_vec);
 
-      // s = r - alpha * v
-      dev->scopy(s.stream_id, s.n, s.r, s.s_vec);
-      dev->saxpy(s.stream_id, s.n, -s.alpha, s.v, s.s_vec);
+        // omega = <t, s> / <t, t> -> calculate numerator <t, s>
+        dev->sdot(s.stream_id, s.n, s.t_vec, s.s_vec, s.y_tmp);
+        s.step = sparse_bicgstab_step::calc_omega_num;
+        runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
+          anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
+        });
+        break;
+      }
+
+      case sparse_bicgstab_step::calc_omega_num: {
+        // omega = <t, s> / <t, t> -> calculate denominator <t, t>
+        dev->sdot(s.stream_id, s.n, s.t_vec, s.t_vec, s.y_tmp);
+        s.step = sparse_bicgstab_step::calc_omega_denom;
+        runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
+          anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
+        });
+        break;
+      }
 
-      // t = As
-      execute_spmv(s.s_vec, s.t_vec);
+      case sparse_bicgstab_step::calc_omega_denom: {
+        s.omega_val = s.omega_num_val / s.omega_denom_val;
 
-      // omega = <t, s> / <t, t>
-      dev->sdot(s.stream_id, s.n, s.t_vec, s.s_vec, s.y_tmp);
-      float dot_ts = s.y_tmp->copy_to_host()[0];
-      dev->sdot(s.stream_id, s.n, s.t_vec, s.t_vec, s.y_tmp);
-      float dot_tt = s.y_tmp->copy_to_host()[0];
-      s.omega = dot_ts / dot_tt;
+        // x = x + alpha*p + omega*s
+        dev->saxpy(s.stream_id, s.n, s.alpha_val, s.p, s.x);
+        dev->saxpy(s.stream_id, s.n, s.omega_val, s.s_vec, s.x);
 
-      // x = x + alpha*p + omega*s
-      dev->saxpy(s.stream_id, s.n, s.alpha, s.p, s.x);
-      dev->saxpy(s.stream_id, s.n, s.omega, s.s_vec, s.x);
+        // r = s - omega*t
+        dev->scopy(s.stream_id, s.n, s.s_vec, s.r);
+        dev->saxpy(s.stream_id, s.n, -s.omega_val, s.t_vec, s.r);
 
-      // r = s - omega*t
-      dev->scopy(s.stream_id, s.n, s.s_vec, s.r);
-      dev->saxpy(s.stream_id, s.n, -s.omega, s.t_vec, s.r);
+        // Calculate new residual_norm_sq for next iteration's convergence check
+        dev->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
+        s.step = sparse_bicgstab_step::check_convergence;
+        runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
+          anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
+        });
+        break;
+      }
 
-      dev->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
-      residual_norm_sq = s.y_tmp->copy_to_host()[0];
-      s.iterations++;
+      case sparse_bicgstab_step::idle:
+      case sparse_bicgstab_step::finished:
+        // Should not happen if logic is correct
+        break;
     }
+  }
 
-    auto solution = s.x->copy_to_host();
-    if (s.supervisor)
-      this->mail(std::move(solution)).send(s.supervisor);
+  void finish_solve() {
+    auto& s = state();
+    s.step = sparse_bicgstab_step::finished;
+    // Final copy to host and send result to supervisor
+    auto self = actor_cast<actor>(this);
+    command_runner<> runner;
+    runner.copy_to_host_async(s.x, [self](std::vector<float> solution) {
+      anon_mail(gpu_done_atom_v, std::move(solution)).send(self);
+    });
   }
 
   void execute_spmv(mem_ptr<float> input_v, mem_ptr<float> output_v) {

From fcadf40e74639a0188696f9e8ca4de7b05402df5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 11:35:03 -0600
Subject: [PATCH 0733/1000] fixed include paths

---
 libcaf_cuda/caf/actorBLAS/actorBLAS.hpp   | 4 ++--
 libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp b/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp
index 7a439bf2d9..1f617c199b 100644
--- a/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp
+++ b/libcaf_cuda/caf/actorBLAS/actorBLAS.hpp
@@ -1,4 +1,4 @@
 #pragma once
 #include "caf/actorBLAS/gemv-actor/gemv-actor.hpp"
-#include "caf/actorBLAS/gemv-actor/syrk-actor.hpp"
-#include "caf/actorBLAS/gemv-actor/axpy-actor.hpp"
+#include "caf/actorBLAS/syrk-actor/syrk-actor.hpp"
+#include "caf/actorBLAS/axpy-actor/axpy-actor.hpp"
diff --git a/libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp b/libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp
index 20ce70f0ed..2fed9f35eb 100644
--- a/libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp
@@ -1,4 +1,4 @@
 #pragma once
 #include "caf/actorBLAS/actorBLAS.hpp"
-#include "caf/actorSPARSE/actorSPASE.hpp"
+#include "caf/actorSPARSE/actorSPARSE.hpp"
 #include "caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp"

From f86f9e2c07a65d1b7930a58c47cf2ad8c3d88aea Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 12:39:59 -0600
Subject: [PATCH 0734/1000] updated devices sparse matrix and vector calls to
 stop allocating and freeing memory needless and adjusted actors accordingly

---
 .../sparse-BiCGSTAB-actor.hpp                 |  73 +++--
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 187 +++++++----
 libcaf_cuda/caf/cuda/device.hpp               | 296 ++++++++++++++----
 3 files changed, 412 insertions(+), 144 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
index ed4590d7e0..78b2d1c805 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
@@ -11,7 +11,7 @@
 #include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
 
 // Define a new block for BiCGSTAB specific atoms starting where the cuda block ended
-CAF_BEGIN_TYPE_ID_BLOCK(bicgstab_actor, caf::id_block::cuda::end)
+CAF_BEGIN_TYPE_ID_BLOCK(bicgstab_actor, caf::id_block::cg_actor::end)
   CAF_ADD_ATOM(bicgstab_actor, bicgstab_next_step_atom)
 CAF_END_TYPE_ID_BLOCK(bicgstab_actor)
 
@@ -43,10 +43,12 @@ struct sparse_bicgstab_state {
   int max_iter;
   int device_num;
   int stream_id;
+  device_ptr d_ptr;
   caf::actor supervisor;
 
   // Workspace vectors
   mem_ptr<float> r, r_hat, p, v, s_vec, t_vec, y_tmp;
+  mem_ptr<char> spmv_workspace;
   
   // Scalars needed across asynchronous steps
   float rho_val = 1.0f; // Renamed from rho to avoid conflict with step-specific rho_new
@@ -137,6 +139,8 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
     s.b         = std::get<3>(res);
     s.x         = std::get<4>(res);
 
+    s.d_ptr = platform::create()->schedule(s.stream_id, s.device_num);
+
     // Allocate workspace
     command_runner<out<float>> work_runner;
     s.r     = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
@@ -145,21 +149,34 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
     s.v     = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
     s.s_vec = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
     s.t_vec = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-    s.y_tmp = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-
-    auto dev = platform::create()->schedule(s.stream_id, s.device_num);
+    // Bottleneck Fix: Allocate only 1 float for scalar results
+    s.y_tmp = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(1));
+
+    // Allocate SPMV workspace to avoid reallocations in the loop
+    size_t ws_size = 0;
+    if (s.format == matrix_format::csr)
+      ws_size = s.d_ptr->spmv_csr_buffer_size(s.stream_id, s.n, s.n, s.nnz, s.A_row_ptr, s.A_col_ind, s.A_values, s.x, s.v);
+    else if (s.format == matrix_format::csc)
+      ws_size = s.d_ptr->spmv_csc_buffer_size(s.stream_id, s.n, s.n, s.nnz, s.A_row_ptr, s.A_col_ind, s.A_values, s.x, s.v);
+    else if (s.format == matrix_format::coo)
+      ws_size = s.d_ptr->spmv_coo_buffer_size(s.stream_id, s.n, s.n, s.nnz, s.A_row_ptr, s.A_col_ind, s.A_values, s.x, s.v);
+
+    if (ws_size > 0) {
+      command_runner<out<char>> ws_runner;
+      s.spmv_workspace = ws_runner.transfer_memory(s.device_num, s.stream_id, out<char>(static_cast<int>(ws_size)));
+    }
 
     // 1. Initial Residual: r = b - Ax
     execute_spmv(s.x, s.v); 
-    dev->scopy(s.stream_id, s.n, s.b, s.r);
-    dev->saxpy(s.stream_id, s.n, -1.0f, s.v, s.r);
+    s.d_ptr->scopy(s.stream_id, s.n, s.b, s.r);
+    s.d_ptr->saxpy(s.stream_id, s.n, -1.0f, s.v, s.r);
     
     // 2. Choose r_hat = r
-    dev->scopy(s.stream_id, s.n, s.r, s.r_hat);
+    s.d_ptr->scopy(s.stream_id, s.n, s.r, s.r_hat);
 
     s.iterations = 0;
     // Initial calculation of residual_norm_sq
-    dev->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
+    s.d_ptr->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
     s.step = sparse_bicgstab_step::init_residual_norm_sq;
     auto self = actor_cast<actor>(this);
     runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
@@ -169,7 +186,6 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
 
   void perform_bicgstab_step() {
     auto& s = state();
-    auto dev = platform::create()->schedule(s.stream_id, s.device_num);
     command_runner<> runner;
     auto self = actor_cast<actor>(this);
 
@@ -185,7 +201,7 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
         }
 
         // rho_i = <r_hat, r>
-        dev->sdot(s.stream_id, s.n, s.r_hat, s.r, s.y_tmp);
+        s.d_ptr->sdot(s.stream_id, s.n, s.r_hat, s.r, s.y_tmp);
         s.step = sparse_bicgstab_step::calc_rho_new;
         runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
           anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
@@ -196,14 +212,14 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
       case sparse_bicgstab_step::calc_rho_new: {
         // Update p based on rho_new_val
         if (s.iterations == 1) { // This is the first iteration
-          dev->scopy(s.stream_id, s.n, s.r, s.p);
+          s.d_ptr->scopy(s.stream_id, s.n, s.r, s.p);
         } else { // Subsequent iterations
           s.beta_val = (s.rho_new_val / s.rho_val) * (s.alpha_val / s.omega_val);
           // p = r + beta * (p - omega * v)
-          dev->saxpy(s.stream_id, s.n, -s.omega_val, s.v, s.p); // p = p - omega*v
-          dev->scopy(s.stream_id, s.n, s.r, s.s_vec);      // use s_vec as temporary
-          dev->saxpy(s.stream_id, s.n, s.beta_val, s.p, s.s_vec); // s_vec = r + beta*p
-          dev->scopy(s.stream_id, s.n, s.s_vec, s.p);
+          s.d_ptr->saxpy(s.stream_id, s.n, -s.omega_val, s.v, s.p); // p = p - omega*v
+          s.d_ptr->scopy(s.stream_id, s.n, s.r, s.s_vec);      // use s_vec as temporary
+          s.d_ptr->saxpy(s.stream_id, s.n, s.beta_val, s.p, s.s_vec); // s_vec = r + beta*p
+          s.d_ptr->scopy(s.stream_id, s.n, s.s_vec, s.p);
         }
         s.rho_val = s.rho_new_val; // Update old rho
 
@@ -211,7 +227,7 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
         execute_spmv(s.p, s.v);
 
         // alpha = rho / <r_hat, v> -> calculate denominator
-        dev->sdot(s.stream_id, s.n, s.r_hat, s.v, s.y_tmp);
+        s.d_ptr->sdot(s.stream_id, s.n, s.r_hat, s.v, s.y_tmp);
         s.step = sparse_bicgstab_step::calc_alpha_denom;
         runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
           anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
@@ -223,14 +239,14 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
         s.alpha_val = s.rho_val / s.alpha_denom_val;
 
         // s = r - alpha * v
-        dev->scopy(s.stream_id, s.n, s.r, s.s_vec);
-        dev->saxpy(s.stream_id, s.n, -s.alpha_val, s.v, s.s_vec);
+        s.d_ptr->scopy(s.stream_id, s.n, s.r, s.s_vec);
+        s.d_ptr->saxpy(s.stream_id, s.n, -s.alpha_val, s.v, s.s_vec);
 
         // t = As
         execute_spmv(s.s_vec, s.t_vec);
 
         // omega = <t, s> / <t, t> -> calculate numerator <t, s>
-        dev->sdot(s.stream_id, s.n, s.t_vec, s.s_vec, s.y_tmp);
+        s.d_ptr->sdot(s.stream_id, s.n, s.t_vec, s.s_vec, s.y_tmp);
         s.step = sparse_bicgstab_step::calc_omega_num;
         runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
           anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
@@ -240,7 +256,7 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
 
       case sparse_bicgstab_step::calc_omega_num: {
         // omega = <t, s> / <t, t> -> calculate denominator <t, t>
-        dev->sdot(s.stream_id, s.n, s.t_vec, s.t_vec, s.y_tmp);
+        s.d_ptr->sdot(s.stream_id, s.n, s.t_vec, s.t_vec, s.y_tmp);
         s.step = sparse_bicgstab_step::calc_omega_denom;
         runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
           anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
@@ -252,15 +268,15 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
         s.omega_val = s.omega_num_val / s.omega_denom_val;
 
         // x = x + alpha*p + omega*s
-        dev->saxpy(s.stream_id, s.n, s.alpha_val, s.p, s.x);
-        dev->saxpy(s.stream_id, s.n, s.omega_val, s.s_vec, s.x);
+        s.d_ptr->saxpy(s.stream_id, s.n, s.alpha_val, s.p, s.x);
+        s.d_ptr->saxpy(s.stream_id, s.n, s.omega_val, s.s_vec, s.x);
 
         // r = s - omega*t
-        dev->scopy(s.stream_id, s.n, s.s_vec, s.r);
-        dev->saxpy(s.stream_id, s.n, -s.omega_val, s.t_vec, s.r);
+        s.d_ptr->scopy(s.stream_id, s.n, s.s_vec, s.r);
+        s.d_ptr->saxpy(s.stream_id, s.n, -s.omega_val, s.t_vec, s.r);
 
         // Calculate new residual_norm_sq for next iteration's convergence check
-        dev->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
+        s.d_ptr->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
         s.step = sparse_bicgstab_step::check_convergence;
         runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
           anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
@@ -288,11 +304,10 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
 
   void execute_spmv(mem_ptr<float> input_v, mem_ptr<float> output_v) {
     auto& s = state();
-    auto dev = platform::create()->schedule(s.stream_id, s.device_num);
     switch (s.format) {
-      case matrix_format::csr: dev->spmv_csr(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v); break;
-      case matrix_format::csc: dev->spmv_csc(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v); break;
-      case matrix_format::coo: dev->spmv_coo(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v); break;
+      case matrix_format::csr: s.d_ptr->spmv_csr(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v, s.spmv_workspace); break;
+      case matrix_format::csc: s.d_ptr->spmv_csc(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v, s.spmv_workspace); break;
+      case matrix_format::coo: s.d_ptr->spmv_coo(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v, s.spmv_workspace); break;
       default: break;
     }
   }
diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index 0ce250eb3a..dee6a8dac5 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -10,27 +10,25 @@
 #include "caf/actorBLAS/copy-actor/copy-actor.hpp"
 #include "caf/cuda/platform.hpp"
 
+// Define a new block for CG specific atoms starting where the cuda block ended
+CAF_BEGIN_TYPE_ID_BLOCK(cg_actor, caf::id_block::cuda::end)
+  CAF_ADD_ATOM(cg_actor, cg_next_step_atom)
+CAF_END_TYPE_ID_BLOCK(cg_actor)
+
 namespace caf::cuda {
 
-enum class matrix_format {
- csr,
- csc,
- coo
-};
+// // enum class matrix_format {
+//  csr,
+//  csc,
+//  coo
+// };
 
 enum class sparse_cg_step {
  idle,
- init_r,
- init_p,
  init_rho,
- main_spmv_w,
- main_dot_pw,
- main_axpy_x,
- main_axpy_r,
- main_dot_rr,
- update_p_copy_r,
- update_p_axpy_p,
- update_p_final_copy
+ calc_dot_pw,
+ check_convergence,
+ finished
 };
 
 // Reply IDs used to distinguish which actor type is replying
@@ -54,16 +52,19 @@ struct sparse_cg_state {
  int max_iter;
  int device_num;
  int stream_id;
+ device_ptr d_ptr;
  caf::actor supervisor;
 
  // Workspace vectors
  mem_ptr<float> r, p, w, y_tmp;
+ mem_ptr<char> spmv_workspace;
  
- // Scalars
- float rho = 0.0f;
- float old_rho = 0.0f;
- float alpha = 0.0f;
- float beta = 0.0f;
+ // Scalars needed across asynchronous steps
+ float rho_val = 0.0f;
+ float old_rho_val = 0.0f;
+ float alpha_val = 0.0f;
+ float beta_val = 0.0f;
+ float dot_pw_val = 0.0f;
  int iterations = 0;
  sparse_cg_step step = sparse_cg_step::idle;
 };
@@ -93,13 +94,33 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
        auto& s = state();
        if (!s.supervisor)
          s.supervisor = actor_cast<caf::actor>(this->current_sender());
-       start_setup();
+       start_solve();
+     },
+     [this](cg_next_step_atom, float val) {
+       auto& s = state();
+       // Thread-safely update scalars based on the stage that just finished
+       switch (s.step) {
+         case sparse_cg_step::init_rho:
+         case sparse_cg_step::check_convergence:
+           s.rho_val = val;
+           break;
+         case sparse_cg_step::calc_dot_pw:
+           s.dot_pw_val = val;
+           break;
+         default: break;
+       }
+       perform_cg_step();
      },
+     [this](gpu_done_atom, std::vector<float>& solution) {
+       if (state().supervisor)
+         this->mail(std::move(solution)).send(state().supervisor);
+       this->quit();
+     }
    };
  }
 
 private:
- void start_setup() {
+ void start_solve() {
    auto& s = state();
    command_runner<> runner;
 
@@ -117,70 +138,116 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
    s.b         = std::get<3>(res);
    s.x         = std::get<4>(res);
 
+   s.d_ptr = platform::create()->schedule(s.stream_id, s.device_num);
+
    // Allocate workspace
    command_runner<out<float>> work_runner;
    s.r = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
    s.p = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
    s.w = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-   s.y_tmp = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
+   s.y_tmp = work_runner.transfer_memory(s.device_num, s.stream_id, create_out_arg_with_size<float>(1));
 
-   auto dev = platform::create()->schedule(s.stream_id, s.device_num);
-   float alpha_const = 1.0f;
-   float beta_const = 0.0f;
-   float alpham1 = -1.0f;
+   // Allocate SPMV workspace to avoid reallocations in the loop
+   size_t ws_size = 0;
+   if (s.format == matrix_format::csr)
+     ws_size = s.d_ptr->spmv_csr_buffer_size(s.stream_id, s.n, s.n, s.nnz, s.A_row_ptr, s.A_col_ind, s.A_values, s.x, s.w);
+   else if (s.format == matrix_format::csc)
+     ws_size = s.d_ptr->spmv_csc_buffer_size(s.stream_id, s.n, s.n, s.nnz, s.A_row_ptr, s.A_col_ind, s.A_values, s.x, s.w);
+   else if (s.format == matrix_format::coo)
+     ws_size = s.d_ptr->spmv_coo_buffer_size(s.stream_id, s.n, s.n, s.nnz, s.A_row_ptr, s.A_col_ind, s.A_values, s.x, s.w);
+
+   if (ws_size > 0) {
+     command_runner<out<char>> ws_runner;
+     s.spmv_workspace = ws_runner.transfer_memory(s.device_num, s.stream_id, out<char>(static_cast<int>(ws_size)));
+   }
 
    // 1. Initial SpMV: w = A * x
    execute_spmv(s.x, s.w);
 
    // 2. Initial r = b - w
-   dev->scopy(s.stream_id, s.n, s.b, s.r);
-   dev->saxpy(s.stream_id, s.n, alpham1, s.w, s.r);
+   s.d_ptr->scopy(s.stream_id, s.n, s.b, s.r);
+   s.d_ptr->saxpy(s.stream_id, s.n, -1.0f, s.w, s.r);
 
    // 3. Initial rho = r * r
-   dev->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
-   s.rho = s.y_tmp->copy_to_host()[0];
-
-   s.iterations = 1;
-   while (s.rho > (s.tol * s.tol) && s.iterations <= s.max_iter) {
-     if (s.iterations > 1) {
-       s.beta = s.rho / s.old_rho;
-       dev->scopy(s.stream_id, s.n, s.r, s.w);
-       dev->saxpy(s.stream_id, s.n, s.beta, s.p, s.w);
-       dev->scopy(s.stream_id, s.n, s.w, s.p);
-     } else {
-       dev->scopy(s.stream_id, s.n, s.r, s.p);
-     }
+   s.d_ptr->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
+   
+   s.step = sparse_cg_step::init_rho;
+   auto self = actor_cast<actor>(this);
+   runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {    
+    anon_mail(cg_next_step_atom_v, host_data[0]).send(self);
+   });
+ }
 
-     execute_spmv(s.p, s.w);
+ void perform_cg_step() {
+   auto& s = state();
+   command_runner<> runner;
+   auto self = actor_cast<actor>(this);
 
-     dev->sdot(s.stream_id, s.n, s.p, s.w, s.y_tmp);
-     float dot_pw = s.y_tmp->copy_to_host()[0];
+   switch (s.step) {
+     case sparse_cg_step::init_rho:
+     case sparse_cg_step::check_convergence: {
+       s.iterations++; // Increment for the current iteration
 
-     s.alpha = s.rho / dot_pw;
+       // Check convergence
+       if (s.rho_val <= (s.tol * s.tol) || s.iterations > s.max_iter) {
+         finish_solve();
+         return;
+       }
 
-     dev->saxpy(s.stream_id, s.n, s.alpha, s.p, s.x);
-     dev->saxpy(s.stream_id, s.n, -s.alpha, s.w, s.r);
+       if (s.iterations > 1) {
+         s.beta_val = s.rho_val / s.old_rho_val;
+         s.d_ptr->scopy(s.stream_id, s.n, s.r, s.w);
+         s.d_ptr->saxpy(s.stream_id, s.n, s.beta_val, s.p, s.w);
+         s.d_ptr->scopy(s.stream_id, s.n, s.w, s.p);
+       } else {
+         s.d_ptr->scopy(s.stream_id, s.n, s.r, s.p);
+       }
 
-     s.old_rho = s.rho;
-     dev->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
-     s.rho = s.y_tmp->copy_to_host()[0];
+       execute_spmv(s.p, s.w);
 
-     s.iterations++;
+       s.d_ptr->sdot(s.stream_id, s.n, s.p, s.w, s.y_tmp);
+       s.step = sparse_cg_step::calc_dot_pw;
+       runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
+         anon_mail(cg_next_step_atom_v, host_data[0]).send(self);
+       });
+       break;
+     }
+
+     case sparse_cg_step::calc_dot_pw: {
+       s.alpha_val = s.rho_val / s.dot_pw_val;
+
+       s.d_ptr->saxpy(s.stream_id, s.n, s.alpha_val, s.p, s.x);
+       s.d_ptr->saxpy(s.stream_id, s.n, -s.alpha_val, s.w, s.r);
+
+       s.old_rho_val = s.rho_val;
+       s.d_ptr->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
+       s.step = sparse_cg_step::check_convergence;
+       runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
+         anon_mail(cg_next_step_atom_v, host_data[0]).send(self);
+       });
+       break;
+     }
+
+     default: break;
    }
+ }
 
-   auto solution = s.x->copy_to_host();
-  
-   if (s.supervisor)
-     this->mail(std::move(solution)).send(s.supervisor);
+ void finish_solve() {
+   auto& s = state();
+   s.step = sparse_cg_step::finished;
+   auto self = actor_cast<actor>(this);
+   command_runner<> runner;
+   runner.copy_to_host_async(s.x, [self](std::vector<float> solution) {
+     anon_mail(gpu_done_atom_v, std::move(solution)).send(self);
+   });
  }
 
  void execute_spmv(mem_ptr<float> input_v, mem_ptr<float> output_v) {
    auto& s = state();
-   auto dev = platform::create()->schedule(s.stream_id, s.device_num);
    switch (s.format) {
-     case matrix_format::csr: dev->spmv_csr(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v); break;
-     case matrix_format::csc: dev->spmv_csc(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v); break;
-     case matrix_format::coo: dev->spmv_coo(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v); break;
+     case matrix_format::csr: s.d_ptr->spmv_csr(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v, s.spmv_workspace); break;
+     case matrix_format::csc: s.d_ptr->spmv_csc(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v, s.spmv_workspace); break;
+     case matrix_format::coo: s.d_ptr->spmv_coo(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v, s.spmv_workspace); break;
      default: break;
    }
  }
diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 1f01c54e44..f2ca87cfd9 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -173,10 +173,85 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasSgemv failed on device " + std::to_string(id_));
   }
 
+  /// Returns the required buffer size for SpMV CSR.
+  size_t spmv_csr_buffer_size(int stream_id, int m, int n, int nnz, 
+                             mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> values,
+                             mem_ptr<float> x, mem_ptr<float> y) {
+    cusparseHandle_t handle = get_cusparse_handle(stream_id);
+    if (!handle) throw std::runtime_error("cuSparse not enabled");
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    float alpha = 1.0f; float beta = 0.0f;
+    cusparseSpMatDescr_t matA;
+    cusparseCreateCsr(&matA, m, n, nnz, reinterpret_cast<void*>(row_ptr->mem()),
+                      reinterpret_cast<void*>(col_ind->mem()), reinterpret_cast<void*>(values->mem()),
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+    cusparseDnVecDescr_t vecX, vecY;
+    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
+    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
+    size_t bufferSize = 0;
+    cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecY,
+                            CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnVec(vecX);
+    cusparseDestroyDnVec(vecY);
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    return bufferSize;
+  }
+
+  /// Returns the required buffer size for SpMV COO.
+  size_t spmv_coo_buffer_size(int stream_id, int m, int n, int nnz, 
+                             mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> values,
+                             mem_ptr<float> x, mem_ptr<float> y) {
+    cusparseHandle_t handle = get_cusparse_handle(stream_id);
+    if (!handle) throw std::runtime_error("cuSparse not enabled");
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    float alpha = 1.0f; float beta = 0.0f;
+    cusparseSpMatDescr_t matA;
+    cusparseCreateCoo(&matA, m, n, nnz, reinterpret_cast<void*>(row_ind->mem()),
+                      reinterpret_cast<void*>(col_ind->mem()), reinterpret_cast<void*>(values->mem()),
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+    cusparseDnVecDescr_t vecX, vecY;
+    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
+    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
+    size_t bufferSize = 0;
+    cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecY,
+                            CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnVec(vecX);
+    cusparseDestroyDnVec(vecY);
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    return bufferSize;
+  }
+
+  /// Returns the required buffer size for SpMV CSC.
+  size_t spmv_csc_buffer_size(int stream_id, int m, int n, int nnz, 
+                             mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> values,
+                             mem_ptr<float> x, mem_ptr<float> y) {
+    cusparseHandle_t handle = get_cusparse_handle(stream_id);
+    if (!handle) throw std::runtime_error("cuSparse not enabled");
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    float alpha = 1.0f; float beta = 0.0f;
+    cusparseSpMatDescr_t matA;
+    cusparseCreateCsc(&matA, m, n, nnz, reinterpret_cast<void*>(col_ptr->mem()),
+                      reinterpret_cast<void*>(row_ind->mem()), reinterpret_cast<void*>(values->mem()),
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+    cusparseDnVecDescr_t vecX, vecY;
+    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
+    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
+    size_t bufferSize = 0;
+    cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecY,
+                            CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnVec(vecX);
+    cusparseDestroyDnVec(vecY);
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    return bufferSize;
+  }
+
   /// Performs sparse matrix-vector multiplication (y = alpha*A*x + beta*y) using CSR format.
   void spmv_csr(int stream_id, int m, int n, int nnz, float alpha, 
                 mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> values,
-                mem_ptr<float> x, float beta, mem_ptr<float> y) {
+                mem_ptr<float> x, float beta, mem_ptr<float> y, mem_ptr<char> workspace = nullptr) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle)
       throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
@@ -196,19 +271,24 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
     cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
 
-    size_t bufferSize = 0;
-    cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                            &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
-                            CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
-
+    void* d_workspace = nullptr;
     CUdeviceptr dBuffer = 0;
-    if (bufferSize > 0)
-      CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+    if (workspace) {
+      d_workspace = reinterpret_cast<void*>(workspace->mem());
+    } else {
+      size_t bufferSize = 0;
+      cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                              &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                              CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
+      if (bufferSize > 0) {
+        CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+        d_workspace = reinterpret_cast<void*>(dBuffer);
+      }
+    }
 
     cusparseStatus_t status = cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
                                            &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
-                                           CUSPARSE_SPMV_ALG_DEFAULT,
-                                           reinterpret_cast<void*>(dBuffer));
+                                           CUSPARSE_SPMV_ALG_DEFAULT, d_workspace);
 
     if (dBuffer)
       CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
@@ -225,7 +305,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   /// Performs sparse matrix-vector multiplication (y = alpha*A*x + beta*y) using COO format.
   void spmv_coo(int stream_id, int m, int n, int nnz, float alpha, 
                 mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> values,
-                mem_ptr<float> x, float beta, mem_ptr<float> y) {
+                mem_ptr<float> x, float beta, mem_ptr<float> y, mem_ptr<char> workspace = nullptr) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle)
       throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
@@ -244,19 +324,24 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
     cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
 
-    size_t bufferSize = 0;
-    cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                            &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
-                            CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
-
+    void* d_workspace = nullptr;
     CUdeviceptr dBuffer = 0;
-    if (bufferSize > 0)
-      CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+    if (workspace) {
+      d_workspace = reinterpret_cast<void*>(workspace->mem());
+    } else {
+      size_t bufferSize = 0;
+      cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                              &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                              CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
+      if (bufferSize > 0) {
+        CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+        d_workspace = reinterpret_cast<void*>(dBuffer);
+      }
+    }
 
     cusparseStatus_t status = cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
                                            &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
-                                           CUSPARSE_SPMV_ALG_DEFAULT,
-                                           reinterpret_cast<void*>(dBuffer));
+                                           CUSPARSE_SPMV_ALG_DEFAULT, d_workspace);
 
     if (dBuffer)
       CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
@@ -273,7 +358,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   /// Performs sparse matrix-vector multiplication (y = alpha*A*x + beta*y) using CSC format.
   void spmv_csc(int stream_id, int m, int n, int nnz, float alpha, 
                 mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> values,
-                mem_ptr<float> x, float beta, mem_ptr<float> y) {
+                mem_ptr<float> x, float beta, mem_ptr<float> y, mem_ptr<char> workspace = nullptr) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle)
       throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
@@ -293,19 +378,24 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
     cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
 
-    size_t bufferSize = 0;
-    cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                            &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
-                            CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
-
+    void* d_workspace = nullptr;
     CUdeviceptr dBuffer = 0;
-    if (bufferSize > 0)
-      CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+    if (workspace) {
+      d_workspace = reinterpret_cast<void*>(workspace->mem());
+    } else {
+      size_t bufferSize = 0;
+      cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                              &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                              CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
+      if (bufferSize > 0) {
+        CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+        d_workspace = reinterpret_cast<void*>(dBuffer);
+      }
+    }
 
     cusparseStatus_t status = cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
                                            &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
-                                           CUSPARSE_SPMV_ALG_DEFAULT,
-                                           reinterpret_cast<void*>(dBuffer));
+                                           CUSPARSE_SPMV_ALG_DEFAULT, d_workspace);
 
     if (dBuffer)
       CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
@@ -319,11 +409,86 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cusparseSpMV (CSC) failed on device " + std::to_string(id_));
   }
 
+  /// Returns the required buffer size for SpMM CSR.
+  size_t spmm_csr_buffer_size(int stream_id, int m, int n, int k, int nnz, 
+                             mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> values,
+                             mem_ptr<float> B, mem_ptr<float> C) {
+    cusparseHandle_t handle = get_cusparse_handle(stream_id);
+    if (!handle) throw std::runtime_error("cuSparse not enabled");
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    float alpha = 1.0f; float beta = 0.0f;
+    cusparseSpMatDescr_t matA;
+    cusparseCreateCsr(&matA, m, k, nnz, reinterpret_cast<void*>(row_ptr->mem()),
+                      reinterpret_cast<void*>(col_ind->mem()), reinterpret_cast<void*>(values->mem()),
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+    cusparseDnMatDescr_t matB, matC;
+    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    size_t bufferSize = 0;
+    cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                            &alpha, matA, matB, &beta, matC, CUDA_R_32F, CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnMat(matB);
+    cusparseDestroyDnMat(matC);
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    return bufferSize;
+  }
+
+  /// Returns the required buffer size for SpMM COO.
+  size_t spmm_coo_buffer_size(int stream_id, int m, int n, int k, int nnz, 
+                             mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> values,
+                             mem_ptr<float> B, mem_ptr<float> C) {
+    cusparseHandle_t handle = get_cusparse_handle(stream_id);
+    if (!handle) throw std::runtime_error("cuSparse not enabled");
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    float alpha = 1.0f; float beta = 0.0f;
+    cusparseSpMatDescr_t matA;
+    cusparseCreateCoo(&matA, m, k, nnz, reinterpret_cast<void*>(row_ind->mem()),
+                      reinterpret_cast<void*>(col_ind->mem()), reinterpret_cast<void*>(values->mem()),
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+    cusparseDnMatDescr_t matB, matC;
+    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    size_t bufferSize = 0;
+    cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                            &alpha, matA, matB, &beta, matC, CUDA_R_32F, CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnMat(matB);
+    cusparseDestroyDnMat(matC);
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    return bufferSize;
+  }
+
+  /// Returns the required buffer size for SpMM CSC.
+  size_t spmm_csc_buffer_size(int stream_id, int m, int n, int k, int nnz, 
+                             mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> values,
+                             mem_ptr<float> B, mem_ptr<float> C) {
+    cusparseHandle_t handle = get_cusparse_handle(stream_id);
+    if (!handle) throw std::runtime_error("cuSparse not enabled");
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    float alpha = 1.0f; float beta = 0.0f;
+    cusparseSpMatDescr_t matA;
+    cusparseCreateCsc(&matA, m, k, nnz, reinterpret_cast<void*>(col_ptr->mem()),
+                      reinterpret_cast<void*>(row_ind->mem()), reinterpret_cast<void*>(values->mem()),
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+    cusparseDnMatDescr_t matB, matC;
+    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    size_t bufferSize = 0;
+    cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                            &alpha, matA, matB, &beta, matC, CUDA_R_32F, CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnMat(matB);
+    cusparseDestroyDnMat(matC);
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    return bufferSize;
+  }
+
   /// Performs sparse matrix-matrix multiplication (C = alpha*A*B + beta*C) using CSR format.
   /// A is sparse (m x k), B is dense (k x n), C is dense (m x n).
   void spmm_csr(int stream_id, int m, int n, int k, int nnz, float alpha, 
                 mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> values,
-                mem_ptr<float> B, float beta, mem_ptr<float> C) {
+                mem_ptr<float> B, float beta, mem_ptr<float> C, mem_ptr<char> workspace = nullptr) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle) throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
 
@@ -342,17 +507,24 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
     cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
 
-    size_t bufferSize = 0;
-    cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                            &alpha, matA, matB, &beta, matC, CUDA_R_32F,
-                            CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
-
+    void* d_workspace = nullptr;
     CUdeviceptr dBuffer = 0;
-    if (bufferSize > 0) CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+    if (workspace) {
+      d_workspace = reinterpret_cast<void*>(workspace->mem());
+    } else {
+      size_t bufferSize = 0;
+      cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                              &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                              CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
+      if (bufferSize > 0) {
+        CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+        d_workspace = reinterpret_cast<void*>(dBuffer);
+      }
+    }
 
     cusparseStatus_t status = cusparseSpMM(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
                                            &alpha, matA, matB, &beta, matC, CUDA_R_32F,
-                                           CUSPARSE_SPMM_ALG_DEFAULT, reinterpret_cast<void*>(dBuffer));
+                                           CUSPARSE_SPMM_ALG_DEFAULT, d_workspace);
 
     if (dBuffer) CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
 
@@ -368,7 +540,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   /// Performs sparse matrix-matrix multiplication (C = alpha*A*B + beta*C) using COO format.
   void spmm_coo(int stream_id, int m, int n, int k, int nnz, float alpha, 
                 mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> values,
-                mem_ptr<float> B, float beta, mem_ptr<float> C) {
+                mem_ptr<float> B, float beta, mem_ptr<float> C, mem_ptr<char> workspace = nullptr) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle) throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
 
@@ -386,17 +558,24 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
     cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
 
-    size_t bufferSize = 0;
-    cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                            &alpha, matA, matB, &beta, matC, CUDA_R_32F,
-                            CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
-
+    void* d_workspace = nullptr;
     CUdeviceptr dBuffer = 0;
-    if (bufferSize > 0) CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+    if (workspace) {
+      d_workspace = reinterpret_cast<void*>(workspace->mem());
+    } else {
+      size_t bufferSize = 0;
+      cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                              &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                              CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
+      if (bufferSize > 0) {
+        CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+        d_workspace = reinterpret_cast<void*>(dBuffer);
+      }
+    }
 
     cusparseStatus_t status = cusparseSpMM(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
                                            &alpha, matA, matB, &beta, matC, CUDA_R_32F,
-                                           CUSPARSE_SPMM_ALG_DEFAULT, reinterpret_cast<void*>(dBuffer));
+                                           CUSPARSE_SPMM_ALG_DEFAULT, d_workspace);
 
     if (dBuffer) CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
 
@@ -412,7 +591,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   /// Performs sparse matrix-matrix multiplication (C = alpha*A*B + beta*C) using CSC format.
   void spmm_csc(int stream_id, int m, int n, int k, int nnz, float alpha, 
                 mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> values,
-                mem_ptr<float> B, float beta, mem_ptr<float> C) {
+                mem_ptr<float> B, float beta, mem_ptr<float> C, mem_ptr<char> workspace = nullptr) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle) throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
 
@@ -431,17 +610,24 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
     cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
 
-    size_t bufferSize = 0;
-    cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                            &alpha, matA, matB, &beta, matC, CUDA_R_32F,
-                            CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
-
+    void* d_workspace = nullptr;
     CUdeviceptr dBuffer = 0;
-    if (bufferSize > 0) CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+    if (workspace) {
+      d_workspace = reinterpret_cast<void*>(workspace->mem());
+    } else {
+      size_t bufferSize = 0;
+      cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                              &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                              CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
+      if (bufferSize > 0) {
+        CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
+        d_workspace = reinterpret_cast<void*>(dBuffer);
+      }
+    }
 
     cusparseStatus_t status = cusparseSpMM(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
                                            &alpha, matA, matB, &beta, matC, CUDA_R_32F,
-                                           CUSPARSE_SPMM_ALG_DEFAULT, reinterpret_cast<void*>(dBuffer));
+                                           CUSPARSE_SPMM_ALG_DEFAULT, d_workspace);
 
     if (dBuffer)
       CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
@@ -452,7 +638,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
 
     CHECK_CUDA(cuCtxPopCurrent(nullptr));
     if (status != CUSPARSE_STATUS_SUCCESS)
-      throw std::runtime_error("cusparseSpMV (CSC) failed on device " + std::to_string(id_));
+      throw std::runtime_error("cusparseSpMM (CSC) failed on device " + std::to_string(id_));
   }
 
   /// Performs symmetric rank-k update (C = alpha*A*A^T + beta*C).

From 0fa2b85959f2bfd4cbaf6ac067c1be81d4f7a9d0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 12:40:14 -0600
Subject: [PATCH 0735/1000] added matrix types to this file

---
 libcaf_cuda/caf/cuda/types.hpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/types.hpp b/libcaf_cuda/caf/cuda/types.hpp
index bd2466840d..d22b0c6e45 100644
--- a/libcaf_cuda/caf/cuda/types.hpp
+++ b/libcaf_cuda/caf/cuda/types.hpp
@@ -53,6 +53,12 @@ class command;
 template <bool PassConfig, class... Ts>
 class actor_facade;
 
+enum class matrix_format {
+  csr,
+  csc,
+  coo
+};
+
 } // namespace caf::cuda
 
 // Structure for mapping kernel output indices to specific host memory buffers

From ed93b07b03545e8cea75d74731559d18a93ec18a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 13:02:44 -0600
Subject: [PATCH 0736/1000] updated sparse CGS actor to be an iterative while
 loop. This change is being made while it does hog a CPU thread. It is much
 faster to synchronously block for a brief second then message pass since the
 operations are so fast, message passing creates a synchronization bottleneck

---
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 121 +++++-------------
 1 file changed, 31 insertions(+), 90 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index dee6a8dac5..2940d150c2 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -17,20 +17,6 @@ CAF_END_TYPE_ID_BLOCK(cg_actor)
 
 namespace caf::cuda {
 
-// // enum class matrix_format {
-//  csr,
-//  csc,
-//  coo
-// };
-
-enum class sparse_cg_step {
- idle,
- init_rho,
- calc_dot_pw,
- check_convergence,
- finished
-};
-
 // Reply IDs used to distinguish which actor type is replying
 constexpr int id_dot = 100;
 constexpr int id_spmv = 200;
@@ -66,7 +52,6 @@ struct sparse_cg_state {
  float beta_val = 0.0f;
  float dot_pw_val = 0.0f;
  int iterations = 0;
- sparse_cg_step step = sparse_cg_step::idle;
 };
 
 class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
@@ -88,6 +73,8 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
    state().supervisor = supervisor;
  }
 
+ ~sparse_cg_actor() override = default;
+
  behavior make_behavior() override {
    return {
      [this](start_atom) {
@@ -96,21 +83,6 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
          s.supervisor = actor_cast<caf::actor>(this->current_sender());
        start_solve();
      },
-     [this](cg_next_step_atom, float val) {
-       auto& s = state();
-       // Thread-safely update scalars based on the stage that just finished
-       switch (s.step) {
-         case sparse_cg_step::init_rho:
-         case sparse_cg_step::check_convergence:
-           s.rho_val = val;
-           break;
-         case sparse_cg_step::calc_dot_pw:
-           s.dot_pw_val = val;
-           break;
-         default: break;
-       }
-       perform_cg_step();
-     },
      [this](gpu_done_atom, std::vector<float>& solution) {
        if (state().supervisor)
          this->mail(std::move(solution)).send(state().supervisor);
@@ -122,7 +94,7 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
 private:
  void start_solve() {
    auto& s = state();
-   command_runner<> runner;
+   command_runner<float> runner;
 
    // Transfer problem data to device
    auto res = runner.transfer_memory(s.device_num, s.stream_id,
@@ -146,7 +118,7 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
    s.p = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
    s.w = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
    s.y_tmp = work_runner.transfer_memory(s.device_num, s.stream_id, create_out_arg_with_size<float>(1));
-
+   
    // Allocate SPMV workspace to avoid reallocations in the loop
    size_t ws_size = 0;
    if (s.format == matrix_format::csr)
@@ -171,72 +143,41 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
    // 3. Initial rho = r * r
    s.d_ptr->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
    
-   s.step = sparse_cg_step::init_rho;
-   auto self = actor_cast<actor>(this);
-   runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {    
-    anon_mail(cg_next_step_atom_v, host_data[0]).send(self);
-   });
- }
-
- void perform_cg_step() {
-   auto& s = state();
-   command_runner<> runner;
-   auto self = actor_cast<actor>(this);
-
-   switch (s.step) {
-     case sparse_cg_step::init_rho:
-     case sparse_cg_step::check_convergence: {
-       s.iterations++; // Increment for the current iteration
-
-       // Check convergence
-       if (s.rho_val <= (s.tol * s.tol) || s.iterations > s.max_iter) {
-         finish_solve();
-         return;
-       }
-
-       if (s.iterations > 1) {
-         s.beta_val = s.rho_val / s.old_rho_val;
-         s.d_ptr->scopy(s.stream_id, s.n, s.r, s.w);
-         s.d_ptr->saxpy(s.stream_id, s.n, s.beta_val, s.p, s.w);
-         s.d_ptr->scopy(s.stream_id, s.n, s.w, s.p);
-       } else {
-         s.d_ptr->scopy(s.stream_id, s.n, s.r, s.p);
-       }
-
-       execute_spmv(s.p, s.w);
-
-       s.d_ptr->sdot(s.stream_id, s.n, s.p, s.w, s.y_tmp);
-       s.step = sparse_cg_step::calc_dot_pw;
-       runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
-         anon_mail(cg_next_step_atom_v, host_data[0]).send(self);
-       });
-       break;
+   // Fetch initial rho synchronously to start the loop
+   s.rho_val = runner.copy_to_host(s.y_tmp)[0];
+
+   // The Real Performance Fix: The Tight CG Loop
+   // By running the loop here, we eliminate 20,000+ scheduler context switches.
+   while (s.rho_val > (s.tol * s.tol) && s.iterations < s.max_iter) {
+     s.iterations++;
+
+     if (s.iterations > 1) {
+       s.beta_val = s.rho_val / s.old_rho_val;
+       s.d_ptr->scopy(s.stream_id, s.n, s.r, s.w);
+       s.d_ptr->saxpy(s.stream_id, s.n, s.beta_val, s.p, s.w);
+       s.d_ptr->scopy(s.stream_id, s.n, s.w, s.p);
+     } else {
+       s.d_ptr->scopy(s.stream_id, s.n, s.r, s.p);
      }
 
-     case sparse_cg_step::calc_dot_pw: {
-       s.alpha_val = s.rho_val / s.dot_pw_val;
+     execute_spmv(s.p, s.w);
 
-       s.d_ptr->saxpy(s.stream_id, s.n, s.alpha_val, s.p, s.x);
-       s.d_ptr->saxpy(s.stream_id, s.n, -s.alpha_val, s.w, s.r);
+     s.d_ptr->sdot(s.stream_id, s.n, s.p, s.w, s.y_tmp);
+     // This synchronous call blocks the CAF thread ONLY until this dot product is ready.
+     // This is 100x faster than yielding to the scheduler.
+     s.dot_pw_val = runner.copy_to_host(s.y_tmp)[0];
 
-       s.old_rho_val = s.rho_val;
-       s.d_ptr->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
-       s.step = sparse_cg_step::check_convergence;
-       runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
-         anon_mail(cg_next_step_atom_v, host_data[0]).send(self);
-       });
-       break;
-     }
+     s.alpha_val = s.rho_val / s.dot_pw_val;
+     s.d_ptr->saxpy(s.stream_id, s.n, s.alpha_val, s.p, s.x);
+     s.d_ptr->saxpy(s.stream_id, s.n, -s.alpha_val, s.w, s.r);
 
-     default: break;
+     s.old_rho_val = s.rho_val;
+     s.d_ptr->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
+     s.rho_val = runner.copy_to_host(s.y_tmp)[0];
    }
- }
 
- void finish_solve() {
-   auto& s = state();
-   s.step = sparse_cg_step::finished;
+   // Exit the loop and return the result via the standard async path
    auto self = actor_cast<actor>(this);
-   command_runner<> runner;
    runner.copy_to_host_async(s.x, [self](std::vector<float> solution) {
      anon_mail(gpu_done_atom_v, std::move(solution)).send(self);
    });

From 6e3a5f22eda490f8c3e8e23575ca3b0c33d4e4d5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 13:51:41 -0600
Subject: [PATCH 0737/1000] Refactored code to move sparse matrix reader out of
 main.test.cpp.

---
 .../tests/workload-test/CMakeLists.txt        |   3 +-
 libcaf_cuda/tests/workload-test/main.test.cpp | 102 +-----------------
 .../tests/workload-test/sparse_utils.cpp      |  72 +++++++++++++
 .../tests/workload-test/sparse_utils.hpp      |  34 ++++++
 4 files changed, 109 insertions(+), 102 deletions(-)
 create mode 100644 libcaf_cuda/tests/workload-test/sparse_utils.cpp
 create mode 100644 libcaf_cuda/tests/workload-test/sparse_utils.hpp

diff --git a/libcaf_cuda/tests/workload-test/CMakeLists.txt b/libcaf_cuda/tests/workload-test/CMakeLists.txt
index 4012ea5228..1ad2d3747e 100644
--- a/libcaf_cuda/tests/workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/workload-test/CMakeLists.txt
@@ -30,7 +30,7 @@ include_directories(
 
 
 # 5) Declare your executable
-add_executable(test main.test.cpp)
+add_executable(test main.test.cpp sparse_utils.cpp)
 
 target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
 
@@ -43,4 +43,3 @@ target_link_libraries(test
     CUDA::cublas
     CUDA::cusparse
 )
-
diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index 8deb34cb7b..5b46c5c0a8 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -8,109 +8,11 @@
 #include <cmath>
 #include <algorithm>
 #include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
+#include "sparse_utils.hpp"
 
 using namespace caf;
 using namespace caf::cuda;
 
-// Structure to hold raw data from the binary file
-struct SparseMatrixCOO {
-    int32_t rows;
-    int32_t cols;
-    int32_t nnz;
-    std::vector<int32_t> row_indices;
-    std::vector<int32_t> col_indices;
-    std::vector<float> values;
-};
-
-// Structure optimized for high-performance solvers
-struct SparseMatrixCSR {
-    int32_t rows;
-    int32_t cols;
-    int32_t nnz;
-    std::vector<int32_t> row_ptr;    // Size: rows + 1
-    std::vector<int32_t> col_indices;// Size: nnz
-    std::vector<float> values;       // Size: nnz
-};
-
-// Function to slurp the binary data into memory
-SparseMatrixCOO load_binary_coo(const std::string& filepath) {
-    std::ifstream file(filepath, std::ios::binary);
-    if (!file) {
-        throw std::runtime_error("Failed to open matrix file: " + filepath);
-    }
-
-    SparseMatrixCOO coo;
-    
-    // 1. Read the 12-byte header
-    file.read(reinterpret_cast<char*>(&coo.rows), sizeof(int32_t));
-    file.read(reinterpret_cast<char*>(&coo.cols), sizeof(int32_t));
-    file.read(reinterpret_cast<char*>(&coo.nnz), sizeof(int32_t));
-
-    // Allocate memory vectors
-    coo.row_indices.resize(coo.nnz);
-    coo.col_indices.resize(coo.nnz);
-    coo.values.resize(coo.nnz);
-
-    // 2. Stream the blocks continuously
-    file.read(reinterpret_cast<char*>(coo.row_indices.data()), coo.nnz * sizeof(int32_t));
-    file.read(reinterpret_cast<char*>(coo.col_indices.data()), coo.nnz * sizeof(int32_t));
-    file.read(reinterpret_cast<char*>(coo.values.data()), coo.nnz * sizeof(float));
-
-    return coo;
-}
-
-// Converts COO to CSR format for solver compatibility
-SparseMatrixCSR convert_coo_to_csr(const SparseMatrixCOO& coo) {
-    SparseMatrixCSR csr;
-    csr.rows = coo.rows;
-    csr.cols = coo.cols;
-    csr.nnz = coo.nnz;
-    
-    csr.row_ptr.assign(csr.rows + 1, 0);
-    csr.col_indices.resize(csr.nnz);
-    csr.values.resize(csr.nnz);
-
-    // Step 1: Count elements per row
-    for (int32_t i = 0; i < coo.nnz; ++i) {
-        csr.row_ptr[coo.row_indices[i] + 1]++;
-    }
-
-    // Step 2: Cumulative sum to build row pointers
-    for (int32_t i = 0; i < csr.rows; ++i) {
-        csr.row_ptr[i + 1] += csr.row_ptr[i];
-    }
-
-    // Step 3: Copy tracking array to insert elements in order
-    std::vector<int32_t> current_row_pos = csr.row_ptr;
-
-    // Step 4: Fill column and value arrays
-    for (int32_t i = 0; i < coo.nnz; ++i) {
-        int32_t row = coo.row_indices[i];
-        int32_t dest_pos = current_row_pos[row]++;
-        csr.col_indices[dest_pos] = coo.col_indices[i];
-        csr.values[dest_pos] = coo.values[i];
-    }
-
-    return csr;
-}
-
-// Compute b = A * x using CSR layout (Sparse Matrix-Vector Multiplication)
-std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x) {
-    std::vector<float> b(A.rows, 0.0f);
-    
-    for (int32_t i = 0; i < A.rows; ++i) {
-        float sum = 0.0f;
-        int32_t row_start = A.row_ptr[i];
-        int32_t row_end = A.row_ptr[i + 1];
-        
-        for (int32_t j = row_start; j < row_end; ++j) {
-            sum += A.values[j] * x[A.col_indices[j]];
-        }
-        b[i] = sum;
-    }
-    return b;
-}
-
 void caf_main(actor_system& sys) {
     // Initialize GPU Manager with cuBLAS and cuSPARSE enabled
     manager::init(sys, manager_config(true, true));
@@ -181,4 +83,4 @@ void caf_main(actor_system& sys) {
     }
     manager::shutdown();
 }
-CAF_MAIN(id_block::cuda)
+CAF_MAIN(id_block::cuda, id_block::cg_actor)
diff --git a/libcaf_cuda/tests/workload-test/sparse_utils.cpp b/libcaf_cuda/tests/workload-test/sparse_utils.cpp
new file mode 100644
index 0000000000..220fe442c3
--- /dev/null
+++ b/libcaf_cuda/tests/workload-test/sparse_utils.cpp
@@ -0,0 +1,72 @@
+#include "sparse_utils.hpp"
+#include <fstream>
+#include <stdexcept>
+
+SparseMatrixCOO load_binary_coo(const std::string& filepath) {
+    std::ifstream file(filepath, std::ios::binary);
+    if (!file) {
+        throw std::runtime_error("Failed to open matrix file: " + filepath);
+    }
+
+    SparseMatrixCOO coo;
+    
+    file.read(reinterpret_cast<char*>(&coo.rows), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&coo.cols), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&coo.nnz), sizeof(int32_t));
+
+    coo.row_indices.resize(coo.nnz);
+    coo.col_indices.resize(coo.nnz);
+    coo.values.resize(coo.nnz);
+
+    file.read(reinterpret_cast<char*>(coo.row_indices.data()), coo.nnz * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(coo.col_indices.data()), coo.nnz * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(coo.values.data()), coo.nnz * sizeof(float));
+
+    return coo;
+}
+
+SparseMatrixCSR convert_coo_to_csr(const SparseMatrixCOO& coo) {
+    SparseMatrixCSR csr;
+    csr.rows = coo.rows;
+    csr.cols = coo.cols;
+    csr.nnz = coo.nnz;
+    
+    csr.row_ptr.assign(csr.rows + 1, 0);
+    csr.col_indices.resize(csr.nnz);
+    csr.values.resize(csr.nnz);
+
+    for (int32_t i = 0; i < coo.nnz; ++i) {
+        csr.row_ptr[coo.row_indices[i] + 1]++;
+    }
+
+    for (int32_t i = 0; i < csr.rows; ++i) {
+        csr.row_ptr[i + 1] += csr.row_ptr[i];
+    }
+
+    std::vector<int32_t> current_row_pos = csr.row_ptr;
+
+    for (int32_t i = 0; i < coo.nnz; ++i) {
+        int32_t row = coo.row_indices[i];
+        int32_t dest_pos = current_row_pos[row]++;
+        csr.col_indices[dest_pos] = coo.col_indices[i];
+        csr.values[dest_pos] = coo.values[i];
+    }
+
+    return csr;
+}
+
+std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x) {
+    std::vector<float> b(A.rows, 0.0f);
+    
+    for (int32_t i = 0; i < A.rows; ++i) {
+        float sum = 0.0f;
+        int32_t row_start = A.row_ptr[i];
+        int32_t row_end = A.row_ptr[i + 1];
+        
+        for (int32_t j = row_start; j < row_end; ++j) {
+            sum += A.values[j] * x[A.col_indices[j]];
+        }
+        b[i] = sum;
+    }
+    return b;
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/workload-test/sparse_utils.hpp b/libcaf_cuda/tests/workload-test/sparse_utils.hpp
new file mode 100644
index 0000000000..45ec51add6
--- /dev/null
+++ b/libcaf_cuda/tests/workload-test/sparse_utils.hpp
@@ -0,0 +1,34 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <cstdint>
+
+// Structure to hold raw data from the binary file
+struct SparseMatrixCOO {
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+    std::vector<int32_t> row_indices;
+    std::vector<int32_t> col_indices;
+    std::vector<float> values;
+};
+
+// Structure optimized for high-performance solvers
+struct SparseMatrixCSR {
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+    std::vector<int32_t> row_ptr;    // Size: rows + 1
+    std::vector<int32_t> col_indices;// Size: nnz
+    std::vector<float> values;       // Size: nnz
+};
+
+// Function to slurp the binary data into memory
+SparseMatrixCOO load_binary_coo(const std::string& filepath);
+
+// Converts COO to CSR format for solver compatibility
+SparseMatrixCSR convert_coo_to_csr(const SparseMatrixCOO& coo);
+
+// Compute b = A * x using CSR layout (Sparse Matrix-Vector Multiplication)
+std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x);
\ No newline at end of file

From fae4b3a387285b88f0769b7b642860f627a3e7da Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 14:07:59 -0600
Subject: [PATCH 0738/1000] Turned test into a distrubuted actor workload
 manager.

---
 libcaf_cuda/tests/workload-test/main.test.cpp | 294 ++++++++++++++----
 1 file changed, 228 insertions(+), 66 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index 5b46c5c0a8..da238b05bb 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -3,84 +3,246 @@
 #include <iostream>
 #include <fstream>
 #include <vector>
+#include <deque>
 #include <string>
 #include <numeric>
 #include <cmath>
 #include <algorithm>
-#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
+#include <filesystem>
+#include "caf/actorSOLVE/actorSOLVE.hpp"
 #include "sparse_utils.hpp"
 
 using namespace caf;
 using namespace caf::cuda;
+namespace fs = std::filesystem;
 
-void caf_main(actor_system& sys) {
-    // Initialize GPU Manager with cuBLAS and cuSPARSE enabled
-    manager::init(sys, manager_config(true, true));
-    std::string filepath = "/scratch/nqr159/matrix-collection/matrices/spd/bcsstk08.bin";
-
-    try {
-        std::cout << "Loading binary file: " << filepath << " ...\n";
-        SparseMatrixCOO coo_matrix = load_binary_coo(filepath);
-        
-        std::cout << "-> Matrix Loaded. Components: " 
-                  << "Rows=" << coo_matrix.rows 
-                  << ", Cols=" << coo_matrix.cols 
-                  << ", NNZ=" << coo_matrix.nnz << "\n";
-
-        std::cout << "Converting to Compressed Sparse Row (CSR) format...\n";
-        SparseMatrixCSR A = convert_coo_to_csr(coo_matrix);
-
-        // --- PREPARE BENCHMARK VECTORS ---
-        std::cout << "Generating test vectors (b = A * x_true)...\n";
-        
-        // 1. Create a known ideal solution vector x_true (filled with 1.0f)
-        std::vector<float> x_true(A.cols, 1.0f);
-        
-        // 2. Compute true right-hand side b
-        std::vector<float> b = compute_rhs_spmv(A, x_true);
-        
-        // 3. Allocate an initial guess vector x filled with zeros
-        std::vector<float> x_guess(A.cols, 0.0f);
-
-        std::cout << "\n=========================================\n";
-        std::cout << " Ready for Solver Execution!\n";
-        std::cout << "=========================================\n";
-        std::cout << "Arrays allocated and verified:\n";
-        std::cout << " - A.values size:      " << A.values.size() << " elements\n";
-        std::cout << " - A.row_ptr size:     " << A.row_ptr.size() << " elements\n";
-        std::cout << " - Vector b size:      " << b.size() << " elements\n";
-        std::cout << " - Vector x_guess size:" << x_guess.size() << " elements\n\n";
-        
-        scoped_actor self{sys};
-        float tolerance = 1e-5f;
-        int max_iter = 2000;
-
-        // Spawn the Sparse CG Actor
-        auto solver = sys.spawn<sparse_cg_actor>(
-            create_in_arg(A.row_ptr), create_in_arg(A.col_indices), create_in_arg(A.values),
-            create_in_arg(b), create_in_out_arg(x_guess),
-            matrix_format::csr, A.rows, A.nnz, tolerance, max_iter, 0, 0, actor_cast<actor>(self));
-
-        std::cout << "[INFO] Starting Solver Actor...\n";
-        self->mail(start_atom_v).send(solver);
-
-        self->receive(
-            [&](std::vector<float> result_x) {
-                std::cout << "\n=========================================\n";
-                std::cout << " Solver Finished!\n";
-                std::cout << "=========================================\n";
-                std::cout << "Verification (Expected values near 1.0):\n";
-                std::cout << "First 5 elements: ";
-                for (int i = 0; i < std::min(5, (int)result_x.size()); ++i) {
-                    std::cout << result_x[i] << " ";
-                }
-                std::cout << "\n";
+enum SolverType { CGS_SOLVER, BICSTAB_SOLVER };
+
+struct MatrixTask {
+    std::string path;
+    SolverType type;
+};
+
+template <class Inspector>
+bool inspect(Inspector& f, SolverType& x) {
+    auto val = static_cast<int>(x);
+    if (f.apply(val)) {
+        if constexpr (Inspector::is_loading)
+            x = static_cast<SolverType>(val);
+        return true;
+    }
+    return false;
+}
+
+template <class Inspector>
+bool inspect(Inspector& f, MatrixTask& x) {
+    return f.object(x).fields(f.field("path", x.path), f.field("type", x.type));
+}
+
+CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::bicgstab_actor::end)
+    CAF_ADD_ATOM(workload_test, get_work_atom)
+    CAF_ADD_ATOM(workload_test, release_memory_atom)
+    CAF_ADD_ATOM(workload_test, request_work_atom)
+    CAF_ADD_ATOM(workload_test, worker_done_atom)
+    CAF_ADD_TYPE_ID(workload_test, (SolverType))
+    CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
+    CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
+CAF_END_TYPE_ID_BLOCK(workload_test)
+
+// ---------------------------- GLOBAL TASK POOL ----------------------------
+struct pool_state {
+    std::vector<MatrixTask> tasks;
+    size_t next_task_idx = 0;
+};
+
+behavior global_task_pool(stateful_actor<pool_state>* self, std::vector<MatrixTask> tasks) {
+    self->state().tasks = std::move(tasks);
+    return {
+        [=](get_work_atom, size_t batch_size) -> result<std::vector<MatrixTask>> {
+            auto& st = self->state();
+            if (st.next_task_idx >= st.tasks.size())
+                return sec::end_of_stream;
+            size_t count = std::min(batch_size, st.tasks.size() - st.next_task_idx);
+            std::vector<MatrixTask> batch(st.tasks.begin() + st.next_task_idx, 
+                                          st.tasks.begin() + st.next_task_idx + count);
+            st.next_task_idx += count;
+            return batch;
+        }
+    };
+}
+
+// ---------------------------- DEVICE/GPU ACTOR ----------------------------
+struct device_actor_state {
+    caf::actor global_pool;
+    std::deque<MatrixTask> local_tasks;
+    int active_workers = 0;
+    int device_id = -1;
+    bool fetching = false;
+    size_t low_water_mark = 2;
+    size_t batch_size = 4;
+};
+
+behavior gpu_device_actor(stateful_actor<device_actor_state>* self,
+                          caf::actor global_pool, int num_workers, int dev_id) {
+    self->state().global_pool = global_pool;
+    self->state().device_id = dev_id;
+    self->state().active_workers = num_workers;
+
+    auto refill = [=]() {
+        auto& st = self->state();
+        if (st.fetching || st.local_tasks.size() >= st.low_water_mark)
+            return;
+
+        st.fetching = true;
+        self->mail(get_work_atom_v, st.batch_size).request(st.global_pool, infinite).then(
+            [=](std::vector<MatrixTask>& batch) {
+                for (auto& task : batch)
+                    self->state().local_tasks.push_back(std::move(task));
+                self->state().fetching = false;
+            },
+            [=](error& err) {
+                self->state().fetching = false;
             }
         );
+    };
+
+    refill();
+
+    return {
+        [=](get_work_atom) -> result<SolverType, std::vector<int>, std::vector<int>, std::vector<float>, std::vector<float>, std::vector<float>, int, int> {
+            auto& st = self->state();
+            if (st.local_tasks.empty())
+                return sec::end_of_stream;
+
+            MatrixTask t = std::move(st.local_tasks.front());
+            st.local_tasks.pop_front();
+            refill();
+
+            // Synchronous load and format conversion inside the device actor to prepare solver buffers
+            auto coo = load_binary_coo(t.path);
+            auto A = convert_coo_to_csr(coo);
+            std::vector<float> x_true(A.cols, 1.0f);
+            std::vector<float> b = compute_rhs_spmv(A, x_true);
+            std::vector<float> x_guess(A.cols, 0.0f);
 
-    } catch (const std::exception& e) {
-        std::cerr << "CRITICAL EXCEPTION: " << e.what() << "\n";
+            return {t.type, std::move(A.row_ptr), std::move(A.col_indices), std::move(A.values), 
+                    std::move(b), std::move(x_guess), A.rows, A.nnz};
+        },
+        [=](worker_done_atom) {
+            if (--self->state().active_workers <= 0)
+                self->quit();
+        }
+    };
+}
+
+// ---------------------------- WORKER ACTOR ----------------------------
+struct worker_state {
+    caf::actor device_actor;
+    caf::actor supervisor;
+    int device_id;
+    int stream_id;
+};
+
+behavior sparse_worker_fun(stateful_actor<worker_state>* self,
+                           caf::actor supervisor, caf::actor device_actor, int dev_id, int stream_id) {
+    self->state().supervisor = supervisor;
+    self->state().device_actor = device_actor;
+    self->state().device_id = dev_id;
+    self->state().stream_id = stream_id;
+
+    self->mail(request_work_atom_v).send(self);
+
+    return {
+        [=](request_work_atom) {
+            self->mail(get_work_atom_v).request(self->state().device_actor, infinite).then(
+                [=](SolverType type, std::vector<int>& rp, std::vector<int>& ci, std::vector<float>& val,
+                    std::vector<float>& b, std::vector<float>& x, int rows, int nnz) {
+                    
+                    actor solver;
+                    if (type == CGS_SOLVER) {
+                        solver = self->spawn<sparse_cg_actor>(
+                            create_in_arg(rp), create_in_arg(ci), create_in_arg(val),
+                            create_in_arg(b), create_in_out_arg(x),
+                            matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id, actor_cast<actor>(self));
+                    } else {
+                        solver = self->spawn<sparse_bicgstab_actor>(
+                            create_in_arg(rp), create_in_arg(ci), create_in_arg(val),
+                            create_in_arg(b), create_in_out_arg(x),
+                            matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id, actor_cast<actor>(self));
+                    }
+                    self->mail(start_atom_v).send(solver);
+                },
+                [=](error& err) {
+                    if (err == sec::end_of_stream) {
+                        self->mail(worker_done_atom_v).send(self->state().device_actor);
+                        self->quit();
+                    }
+                }
+            );
+        },
+        [=](std::vector<float>& solution) {
+            self->mail(1).send(self->state().supervisor);
+            self->mail(request_work_atom_v).send(self);
+        }
+    };
+}
+
+// ---------------------------- SUPERVISOR ACTOR ----------------------------
+struct supervisor_state {
+    int total_tasks;
+    int completed = 0;
+};
+
+behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, int total, std::vector<MatrixTask> tasks) {
+    self->state().total_tasks = total;
+    auto pool = self->spawn(global_task_pool, std::move(tasks));
+    
+    manager& mgr = manager::get();
+    int num_gpus = mgr.get_num_devices();
+    int workers_per_gpu = 2; // Adjustable worker count per physical GPU
+
+    for (int i = 0; i < num_gpus; ++i) {
+        auto broker = self->spawn(gpu_device_actor, pool, workers_per_gpu, i);
+        for (int j = 0; j < workers_per_gpu; ++j) {
+            self->spawn(sparse_worker_fun, self, broker, i, (i * 100) + j);
+        }
     }
+
+    return {
+        [=](int done) {
+            self->state().completed += done;
+            if (self->state().completed >= self->state().total_tasks) {
+                std::cout << "\n[DONE] All " << self->state().total_tasks << " matrices processed.\n";
+                self->quit();
+            }
+        }
+    };
+}
+
+void caf_main(actor_system& sys) {
+    manager::init(sys, manager_config(true, true));
+    std::vector<MatrixTask> tasks;
+    
+    auto scan = [&](const std::string& dir, SolverType type) {
+        if (!fs::exists(dir)) return;
+        for (const auto& entry : fs::directory_iterator(dir)) {
+            if (entry.path().extension() == ".bin")
+                tasks.push_back({entry.path().string(), type});
+        }
+    };
+
+    scan("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+    scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
+
+    if (tasks.empty()) {
+        std::cerr << "No matrix files found in search paths.\n";
+        manager::shutdown();
+        return;
+    }
+
+    std::cout << "[INFO] Found " << tasks.size() << " matrices. Spawning workload...\n";
+    sys.spawn(supervisor_actor_fun, static_cast<int>(tasks.size()), std::move(tasks));
+    sys.await_all_actors_done();
     manager::shutdown();
 }
-CAF_MAIN(id_block::cuda, id_block::cg_actor)
+CAF_MAIN(id_block::cuda, id_block::cg_actor, id_block::bicgstab_actor, id_block::workload_test)

From fb261a4c27bd7d688e2eb62430aadb5d23def4e7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 14:18:39 -0600
Subject: [PATCH 0739/1000] Updated test to include wall clock time and fixed
 bugs.

---
 libcaf_cuda/tests/workload-test/main.test.cpp | 118 ++++++++++++++----
 1 file changed, 95 insertions(+), 23 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index da238b05bb..e4169dc2fc 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -5,6 +5,7 @@
 #include <vector>
 #include <deque>
 #include <string>
+#include <chrono>
 #include <numeric>
 #include <cmath>
 #include <algorithm>
@@ -72,9 +73,18 @@ behavior global_task_pool(stateful_actor<pool_state>* self, std::vector<MatrixTa
 }
 
 // ---------------------------- DEVICE/GPU ACTOR ----------------------------
+struct MatrixData {
+    std::vector<int> row_ptr;
+    std::vector<int> col_indices;
+    std::vector<float> values;
+    std::vector<float> b;
+    std::vector<float> x_guess;
+};
+
 struct device_actor_state {
     caf::actor global_pool;
     std::deque<MatrixTask> local_tasks;
+    std::unordered_map<std::string, MatrixData> cache;
     int active_workers = 0;
     int device_id = -1;
     bool fetching = false;
@@ -109,24 +119,71 @@ behavior gpu_device_actor(stateful_actor<device_actor_state>* self,
     refill();
 
     return {
-        [=](get_work_atom) -> result<SolverType, std::vector<int>, std::vector<int>, std::vector<float>, std::vector<float>, std::vector<float>, int, int> {
+        [=](get_work_atom) -> result<SolverType, std::string, in<int>, in<int>, in<float>, in<float>, in_out<float>, int, int> {
             auto& st = self->state();
-            if (st.local_tasks.empty())
-                return sec::end_of_stream;
 
-            MatrixTask t = std::move(st.local_tasks.front());
-            st.local_tasks.pop_front();
-            refill();
+            // If local tasks are available, process immediately
+            if (!st.local_tasks.empty()) {
+                MatrixTask t = std::move(st.local_tasks.front());
+                st.local_tasks.pop_front();
+                refill(); // Try to refill if below low water mark
+
+                auto& data = st.cache[t.path];
+                if (data.row_ptr.empty()) {
+                    auto coo = load_binary_coo(t.path);
+                    auto A = convert_coo_to_csr(coo);
+                    data.b = compute_rhs_spmv(A, std::vector<float>(A.cols, 1.0f));
+                    data.row_ptr = std::move(A.row_ptr);
+                    data.col_indices = std::move(A.col_indices);
+                    data.values = std::move(A.values);
+                    data.x_guess.assign(A.cols, 0.0f);
+                }
+
+                return {t.type, t.path, create_in_arg(data.row_ptr), create_in_arg(data.col_indices), 
+                        create_in_arg(data.values), create_in_arg(data.b), create_in_out_arg(data.x_guess), 
+                        (int)data.row_ptr.size() - 1, (int)data.values.size()};
+            }
+
+            auto promise = self->make_response_promise<SolverType, std::string, in<int>, in<int>, in<float>, in<float>, in_out<float>, int, int>();
+            self->mail(get_work_atom_v, st.batch_size).request(st.global_pool, infinite).then(
+                [=](std::vector<MatrixTask>& batch) mutable {
+                    auto& st_inner = self->state();
+                    if (batch.empty()) { // Global pool returned empty batch, meaning no more work
+                        promise.deliver(make_error(sec::end_of_stream));
+                        return;
+                    }
+
+                    // Add remaining tasks to local queue
+                    for (size_t i = 1; i < batch.size(); ++i) {
+                        st_inner.local_tasks.push_back(std::move(batch[i]));
+                    }
+
+                    MatrixTask t = std::move(batch.front()); // Process the first task from the batch
+                    refill(); // Try to refill if below low water mark (after adding tasks)
 
-            // Synchronous load and format conversion inside the device actor to prepare solver buffers
-            auto coo = load_binary_coo(t.path);
-            auto A = convert_coo_to_csr(coo);
-            std::vector<float> x_true(A.cols, 1.0f);
-            std::vector<float> b = compute_rhs_spmv(A, x_true);
-            std::vector<float> x_guess(A.cols, 0.0f);
+                    auto& data = st_inner.cache[t.path];
+                    if (data.row_ptr.empty()) {
+                        auto coo = load_binary_coo(t.path);
+                        auto A = convert_coo_to_csr(coo);
+                        data.b = compute_rhs_spmv(A, std::vector<float>(A.cols, 1.0f));
+                        data.row_ptr = std::move(A.row_ptr);
+                        data.col_indices = std::move(A.col_indices);
+                        data.values = std::move(A.values);
+                        data.x_guess.assign(A.cols, 0.0f);
+                    }
 
-            return {t.type, std::move(A.row_ptr), std::move(A.col_indices), std::move(A.values), 
-                    std::move(b), std::move(x_guess), A.rows, A.nnz};
+                    promise.deliver(t.type, t.path, create_in_arg(data.row_ptr), create_in_arg(data.col_indices), 
+                                    create_in_arg(data.values), create_in_arg(data.b), create_in_out_arg(data.x_guess), 
+                                    (int)data.row_ptr.size() - 1, (int)data.values.size());
+                },
+                [=](error& err) mutable {
+                    promise.deliver(err); // Propagate error from global pool
+                }
+            );
+            return promise;
+        },
+        [=](release_memory_atom, std::string path) {
+            self->state().cache.erase(path);
         },
         [=](worker_done_atom) {
             if (--self->state().active_workers <= 0)
@@ -141,6 +198,7 @@ struct worker_state {
     caf::actor supervisor;
     int device_id;
     int stream_id;
+    std::string current_matrix_path;
 };
 
 behavior sparse_worker_fun(stateful_actor<worker_state>* self,
@@ -155,19 +213,19 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
     return {
         [=](request_work_atom) {
             self->mail(get_work_atom_v).request(self->state().device_actor, infinite).then(
-                [=](SolverType type, std::vector<int>& rp, std::vector<int>& ci, std::vector<float>& val,
-                    std::vector<float>& b, std::vector<float>& x, int rows, int nnz) {
-                    
+                [=](SolverType type, std::string path, in<int> rp, in<int> ci, in<float> val,
+                    in<float> b, in_out<float> x, int rows, int nnz) {
+                    self->state().current_matrix_path = path;
                     actor solver;
                     if (type == CGS_SOLVER) {
                         solver = self->spawn<sparse_cg_actor>(
-                            create_in_arg(rp), create_in_arg(ci), create_in_arg(val),
-                            create_in_arg(b), create_in_out_arg(x),
+                            std::move(rp), std::move(ci), std::move(val),
+                            std::move(b), std::move(x),
                             matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id, actor_cast<actor>(self));
                     } else {
                         solver = self->spawn<sparse_bicgstab_actor>(
-                            create_in_arg(rp), create_in_arg(ci), create_in_arg(val),
-                            create_in_arg(b), create_in_out_arg(x),
+                            std::move(rp), std::move(ci), std::move(val),
+                            std::move(b), std::move(x),
                             matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id, actor_cast<actor>(self));
                     }
                     self->mail(start_atom_v).send(solver);
@@ -182,6 +240,7 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
         },
         [=](std::vector<float>& solution) {
             self->mail(1).send(self->state().supervisor);
+            self->mail(release_memory_atom_v, self->state().current_matrix_path).send(self->state().device_actor);
             self->mail(request_work_atom_v).send(self);
         }
     };
@@ -240,9 +299,22 @@ void caf_main(actor_system& sys) {
         return;
     }
 
-    std::cout << "[INFO] Found " << tasks.size() << " matrices. Spawning workload...\n";
-    sys.spawn(supervisor_actor_fun, static_cast<int>(tasks.size()), std::move(tasks));
+    auto task_count = tasks.size();
+    std::cout << "[INFO] Found " << task_count << " matrices. Spawning workload...\n";
+
+    auto start = std::chrono::steady_clock::now();
+
+    sys.spawn(supervisor_actor_fun, static_cast<int>(task_count), std::move(tasks));
     sys.await_all_actors_done();
+
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+
+    std::cout << "\n===== BENCHMARK COMPLETE =====\n";
+    std::cout << "Tasks Processed: " << task_count << "\n";
+    std::cout << "Total Runtime:   " << elapsed.count() << " s\n";
+    std::cout << "==============================\n";
+
     manager::shutdown();
 }
 CAF_MAIN(id_block::cuda, id_block::cg_actor, id_block::bicgstab_actor, id_block::workload_test)

From 6506a3c929d3a3ccfbb6d7271dd04c82ef3f5a83 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 26 May 2026 14:24:32 -0600
Subject: [PATCH 0740/1000] Updated workers to be more efficient and frequent.

---
 libcaf_cuda/tests/workload-test/main.test.cpp | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index e4169dc2fc..c935afcdc1 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -88,16 +88,20 @@ struct device_actor_state {
     int active_workers = 0;
     int device_id = -1;
     bool fetching = false;
-    size_t low_water_mark = 2;
-    size_t batch_size = 4;
+    size_t low_water_mark;
+    size_t batch_size;
 };
 
 behavior gpu_device_actor(stateful_actor<device_actor_state>* self,
-                          caf::actor global_pool, int num_workers, int dev_id) {
+                          caf::actor global_pool, int num_workers, int dev_id, int max_in_flight) {
     self->state().global_pool = global_pool;
     self->state().device_id = dev_id;
     self->state().active_workers = num_workers;
 
+    // Dynamically calculate prefetch markers based on the total pipeline capacity
+    self->state().low_water_mark = static_cast<size_t>(num_workers * max_in_flight);
+    self->state().batch_size = self->state().low_water_mark * 2;
+
     auto refill = [=]() {
         auto& st = self->state();
         if (st.fetching || st.local_tasks.size() >= st.low_water_mark)
@@ -258,10 +262,11 @@ behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, int total,
     
     manager& mgr = manager::get();
     int num_gpus = mgr.get_num_devices();
-    int workers_per_gpu = 2; // Adjustable worker count per physical GPU
+    int workers_per_gpu = 8; // Adjustable worker count per physical GPU
+    int max_in_flight_tasks_per_worker = 2; // How many tasks each worker can have in flight
 
     for (int i = 0; i < num_gpus; ++i) {
-        auto broker = self->spawn(gpu_device_actor, pool, workers_per_gpu, i);
+        auto broker = self->spawn(gpu_device_actor, pool, workers_per_gpu, i, max_in_flight_tasks_per_worker);
         for (int j = 0; j < workers_per_gpu; ++j) {
             self->spawn(sparse_worker_fun, self, broker, i, (i * 100) + j);
         }

From af0ccca74dfd3f057a1993b7cde58a45d3f2de6c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 08:13:25 -0600
Subject: [PATCH 0741/1000] Updated test to preload matrices

---
 libcaf_cuda/tests/workload-test/main.test.cpp | 83 +++++++++----------
 1 file changed, 40 insertions(+), 43 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index c935afcdc1..a7afb6bb76 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -10,6 +10,7 @@
 #include <cmath>
 #include <algorithm>
 #include <filesystem>
+#include <memory>
 #include "caf/actorSOLVE/actorSOLVE.hpp"
 #include "sparse_utils.hpp"
 
@@ -19,9 +20,18 @@ namespace fs = std::filesystem;
 
 enum SolverType { CGS_SOLVER, BICSTAB_SOLVER };
 
+struct MatrixData {
+    std::vector<int> row_ptr;
+    std::vector<int> col_indices;
+    std::vector<float> values;
+    std::vector<float> b;
+    std::vector<float> x_guess;
+};
+
 struct MatrixTask {
     std::string path;
     SolverType type;
+    std::shared_ptr<MatrixData> data;
 };
 
 template <class Inspector>
@@ -35,11 +45,6 @@ bool inspect(Inspector& f, SolverType& x) {
     return false;
 }
 
-template <class Inspector>
-bool inspect(Inspector& f, MatrixTask& x) {
-    return f.object(x).fields(f.field("path", x.path), f.field("type", x.type));
-}
-
 CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::bicgstab_actor::end)
     CAF_ADD_ATOM(workload_test, get_work_atom)
     CAF_ADD_ATOM(workload_test, release_memory_atom)
@@ -48,8 +53,14 @@ CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::bicgstab_actor::end)
     CAF_ADD_TYPE_ID(workload_test, (SolverType))
     CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
     CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
+    CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
 CAF_END_TYPE_ID_BLOCK(workload_test)
 
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixData)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
+
 // ---------------------------- GLOBAL TASK POOL ----------------------------
 struct pool_state {
     std::vector<MatrixTask> tasks;
@@ -73,18 +84,9 @@ behavior global_task_pool(stateful_actor<pool_state>* self, std::vector<MatrixTa
 }
 
 // ---------------------------- DEVICE/GPU ACTOR ----------------------------
-struct MatrixData {
-    std::vector<int> row_ptr;
-    std::vector<int> col_indices;
-    std::vector<float> values;
-    std::vector<float> b;
-    std::vector<float> x_guess;
-};
-
 struct device_actor_state {
     caf::actor global_pool;
     std::deque<MatrixTask> local_tasks;
-    std::unordered_map<std::string, MatrixData> cache;
     int active_workers = 0;
     int device_id = -1;
     bool fetching = false;
@@ -123,7 +125,7 @@ behavior gpu_device_actor(stateful_actor<device_actor_state>* self,
     refill();
 
     return {
-        [=](get_work_atom) -> result<SolverType, std::string, in<int>, in<int>, in<float>, in<float>, in_out<float>, int, int> {
+        [=](get_work_atom) -> result<SolverType, std::string, in<int>, in<int>, in<float>, in<float>, in_out<float>, int, int, std::shared_ptr<MatrixData>> {
             auto& st = self->state();
 
             // If local tasks are available, process immediately
@@ -132,23 +134,13 @@ behavior gpu_device_actor(stateful_actor<device_actor_state>* self,
                 st.local_tasks.pop_front();
                 refill(); // Try to refill if below low water mark
 
-                auto& data = st.cache[t.path];
-                if (data.row_ptr.empty()) {
-                    auto coo = load_binary_coo(t.path);
-                    auto A = convert_coo_to_csr(coo);
-                    data.b = compute_rhs_spmv(A, std::vector<float>(A.cols, 1.0f));
-                    data.row_ptr = std::move(A.row_ptr);
-                    data.col_indices = std::move(A.col_indices);
-                    data.values = std::move(A.values);
-                    data.x_guess.assign(A.cols, 0.0f);
-                }
-
+                auto& data = *t.data;
                 return {t.type, t.path, create_in_arg(data.row_ptr), create_in_arg(data.col_indices), 
                         create_in_arg(data.values), create_in_arg(data.b), create_in_out_arg(data.x_guess), 
-                        (int)data.row_ptr.size() - 1, (int)data.values.size()};
+                        (int)data.row_ptr.size() - 1, (int)data.values.size(), t.data};
             }
 
-            auto promise = self->make_response_promise<SolverType, std::string, in<int>, in<int>, in<float>, in<float>, in_out<float>, int, int>();
+            auto promise = self->make_response_promise<SolverType, std::string, in<int>, in<int>, in<float>, in<float>, in_out<float>, int, int, std::shared_ptr<MatrixData>>();
             self->mail(get_work_atom_v, st.batch_size).request(st.global_pool, infinite).then(
                 [=](std::vector<MatrixTask>& batch) mutable {
                     auto& st_inner = self->state();
@@ -165,20 +157,10 @@ behavior gpu_device_actor(stateful_actor<device_actor_state>* self,
                     MatrixTask t = std::move(batch.front()); // Process the first task from the batch
                     refill(); // Try to refill if below low water mark (after adding tasks)
 
-                    auto& data = st_inner.cache[t.path];
-                    if (data.row_ptr.empty()) {
-                        auto coo = load_binary_coo(t.path);
-                        auto A = convert_coo_to_csr(coo);
-                        data.b = compute_rhs_spmv(A, std::vector<float>(A.cols, 1.0f));
-                        data.row_ptr = std::move(A.row_ptr);
-                        data.col_indices = std::move(A.col_indices);
-                        data.values = std::move(A.values);
-                        data.x_guess.assign(A.cols, 0.0f);
-                    }
-
+                    auto& data = *t.data;
                     promise.deliver(t.type, t.path, create_in_arg(data.row_ptr), create_in_arg(data.col_indices), 
                                     create_in_arg(data.values), create_in_arg(data.b), create_in_out_arg(data.x_guess), 
-                                    (int)data.row_ptr.size() - 1, (int)data.values.size());
+                                    (int)data.row_ptr.size() - 1, (int)data.values.size(), t.data);
                 },
                 [=](error& err) mutable {
                     promise.deliver(err); // Propagate error from global pool
@@ -187,7 +169,7 @@ behavior gpu_device_actor(stateful_actor<device_actor_state>* self,
             return promise;
         },
         [=](release_memory_atom, std::string path) {
-            self->state().cache.erase(path);
+            // No longer used with pre-loaded pool
         },
         [=](worker_done_atom) {
             if (--self->state().active_workers <= 0)
@@ -202,6 +184,7 @@ struct worker_state {
     caf::actor supervisor;
     int device_id;
     int stream_id;
+    std::shared_ptr<MatrixData> current_data;
     std::string current_matrix_path;
 };
 
@@ -218,8 +201,9 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
         [=](request_work_atom) {
             self->mail(get_work_atom_v).request(self->state().device_actor, infinite).then(
                 [=](SolverType type, std::string path, in<int> rp, in<int> ci, in<float> val,
-                    in<float> b, in_out<float> x, int rows, int nnz) {
+                    in<float> b, in_out<float> x, int rows, int nnz, std::shared_ptr<MatrixData> data) {
                     self->state().current_matrix_path = path;
+                    self->state().current_data = data;
                     actor solver;
                     if (type == CGS_SOLVER) {
                         solver = self->spawn<sparse_cg_actor>(
@@ -244,6 +228,7 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
         },
         [=](std::vector<float>& solution) {
             self->mail(1).send(self->state().supervisor);
+            self->state().current_data.reset();
             self->mail(release_memory_atom_v, self->state().current_matrix_path).send(self->state().device_actor);
             self->mail(request_work_atom_v).send(self);
         }
@@ -291,13 +276,25 @@ void caf_main(actor_system& sys) {
         if (!fs::exists(dir)) return;
         for (const auto& entry : fs::directory_iterator(dir)) {
             if (entry.path().extension() == ".bin")
-                tasks.push_back({entry.path().string(), type});
+                tasks.push_back({entry.path().string(), type, nullptr});
         }
     };
 
     scan("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
     scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
 
+    std::cout << "[INFO] Pre-loading " << tasks.size() << " matrices into memory...\n";
+    for (auto& t : tasks) {
+        auto coo = load_binary_coo(t.path);
+        auto A = convert_coo_to_csr(coo);
+        t.data = std::make_shared<MatrixData>();
+        t.data->b = compute_rhs_spmv(A, std::vector<float>(A.cols, 1.0f));
+        t.data->row_ptr = std::move(A.row_ptr);
+        t.data->col_indices = std::move(A.col_indices);
+        t.data->values = std::move(A.values);
+        t.data->x_guess.assign(A.cols, 0.0f);
+    }
+
     if (tasks.empty()) {
         std::cerr << "No matrix files found in search paths.\n";
         manager::shutdown();

From fe69517fc1c6db45370d8d8f358bc62e193b37f7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 08:25:33 -0600
Subject: [PATCH 0742/1000] Added a cuda implementation of the experiment.

---
 .../tests/workload-test/CMakeLists.txt        |  14 +
 .../tests/workload-test/main.native.cpp       | 347 ++++++++++++++++++
 2 files changed, 361 insertions(+)
 create mode 100644 libcaf_cuda/tests/workload-test/main.native.cpp

diff --git a/libcaf_cuda/tests/workload-test/CMakeLists.txt b/libcaf_cuda/tests/workload-test/CMakeLists.txt
index 1ad2d3747e..a4b615b10f 100644
--- a/libcaf_cuda/tests/workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/workload-test/CMakeLists.txt
@@ -43,3 +43,17 @@ target_link_libraries(test
     CUDA::cublas
     CUDA::cusparse
 )
+
+# 6) Declare the native benchmark executable (raw CUDA/cuBLAS/cuSPARSE)
+add_executable(workload-native main.native.cpp sparse_utils.cpp)
+
+target_link_libraries(workload-native
+  PRIVATE
+    CUDA::cudart
+    CUDA::cublas
+    CUDA::cusparse
+    Threads::Threads
+)
+
+# FindThreads is required for std::thread in the native version
+find_package(Threads REQUIRED)
diff --git a/libcaf_cuda/tests/workload-test/main.native.cpp b/libcaf_cuda/tests/workload-test/main.native.cpp
new file mode 100644
index 0000000000..d3c62113d2
--- /dev/null
+++ b/libcaf_cuda/tests/workload-test/main.native.cpp
@@ -0,0 +1,347 @@
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+#include <cusparse.h>
+#include <iostream>
+#include <vector>
+#include <string>
+#include <chrono>
+#include <thread>
+#include <mutex>
+#include <queue>
+#include <filesystem>
+#include <memory>
+#include <algorithm>
+#include "sparse_utils.hpp"
+
+namespace fs = std::filesystem;
+
+// Error checking macros
+#define CHECK_CUDA(call)                                                   \
+    do {                                                                   \
+        cudaError_t status = call;                                         \
+        if (status != cudaSuccess) {                                       \
+            std::cerr << "CUDA Error: " << cudaGetErrorString(status)      \
+                      << " at " << __FILE__ << ":" << __LINE__ << std::endl; \
+            exit(1);                                                       \
+        }                                                                  \
+    } while (0)
+
+#define CHECK_CUBLAS(call)                                                 \
+    do {                                                                   \
+        cublasStatus_t status = call;                                      \
+        if (status != CUBLAS_STATUS_SUCCESS) {                             \
+            std::cerr << "cuBLAS Error at " << __FILE__ << ":" << __LINE__ << std::endl; \
+            exit(1);                                                       \
+        }                                                                  \
+    } while (0)
+
+#define CHECK_CUSPARSE(call)                                               \
+    do {                                                                   \
+        cusparseStatus_t status = call;                                    \
+        if (status != CUSPARSE_STATUS_SUCCESS) {                           \
+            std::cerr << "cuSparse Error at " << __FILE__ << ":" << __LINE__ << std::endl; \
+            exit(1);                                                       \
+        }                                                                  \
+    } while (0)
+
+enum SolverType { CGS_SOLVER, BICSTAB_SOLVER };
+
+struct MatrixTask {
+    std::string path;
+    SolverType type;
+    int32_t rows, cols, nnz;
+    std::vector<int32_t> row_ptr;
+    std::vector<int32_t> col_indices;
+    std::vector<float> values;
+    std::vector<float> b;
+};
+
+// Simplified CG Solver using raw cuBLAS and cuSPARSE
+void solve_cg(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task) {
+    int n = task.rows;
+    float alpha = 1.0f, beta = 0.0f, r0 = 0.0f, r1 = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
+    float tolerance = 1e-5f;
+    int max_iters = 2000;
+
+    float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b;
+    int *d_row_ptr, *d_col_ind;
+
+    CHECK_CUDA(cudaMalloc(&d_val, task.nnz * sizeof(float)));
+    CHECK_CUDA(cudaMalloc(&d_row_ptr, (n + 1) * sizeof(int)));
+    CHECK_CUDA(cudaMalloc(&d_col_ind, task.nnz * sizeof(int)));
+    CHECK_CUDA(cudaMalloc(&d_x, n * sizeof(float)));
+    CHECK_CUDA(cudaMalloc(&d_r, n * sizeof(float)));
+    CHECK_CUDA(cudaMalloc(&d_p, n * sizeof(float)));
+    CHECK_CUDA(cudaMalloc(&d_Ap, n * sizeof(float)));
+    CHECK_CUDA(cudaMalloc(&d_b, n * sizeof(float)));
+
+    CHECK_CUDA(cudaMemcpy(d_val, task.values.data(), task.nnz * sizeof(float), cudaMemcpyHostToDevice));
+    CHECK_CUDA(cudaMemcpy(d_row_ptr, task.row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice));
+    CHECK_CUDA(cudaMemcpy(d_col_ind, task.col_indices.data(), task.nnz * sizeof(int), cudaMemcpyHostToDevice));
+    CHECK_CUDA(cudaMemcpy(d_b, task.b.data(), n * sizeof(float), cudaMemcpyHostToDevice));
+    CHECK_CUDA(cudaMemset(d_x, 0, n * sizeof(float)));
+
+    // Create descriptors
+    cusparseSpMatDescr_t matA;
+    cusparseDnVecDescr_t vecX, vecP, vecAp;
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.nnz, d_row_ptr, d_col_ind, d_val, 
+                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecAp, n, d_Ap, CUDA_R_32F));
+
+    size_t bufferSize = 0;
+    void* d_buffer = nullptr;
+    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecAp, 
+                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    CHECK_CUDA(cudaMalloc(&d_buffer, bufferSize));
+
+    // CG Logic: r = b - Ax (initially r = b since x=0)
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_p, 1));
+    CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
+
+    int k = 0;
+    while (k < max_iters) {
+        // Ap = A * p
+        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecP, &beta, vecAp, 
+                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
+        
+        float pAp;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
+        a = r1 / pAp;
+
+        // x = x + a*p
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
+        
+        // r = r - a*Ap
+        na = -a;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &na, d_Ap, 1, d_r, 1));
+
+        r0 = r1;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
+
+        if (sqrt(r1) < tolerance) break;
+
+        b = r1 / r0;
+        // p = r + b*p => scal p by b then add r
+        CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_r, 1, d_p, 1));
+        k++;
+    }
+
+    // Cleanup
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnVec(vecX);
+    cusparseDestroyDnVec(vecP);
+    cusparseDestroyDnVec(vecAp);
+    cudaFree(d_val); cudaFree(d_row_ptr); cudaFree(d_col_ind);
+    cudaFree(d_x); cudaFree(d_r); cudaFree(d_p); cudaFree(d_Ap); cudaFree(d_b);
+    cudaFree(d_buffer);
+}
+
+// BiCGSTAB Solver using raw cuBLAS and cuSPARSE
+void solve_bicgstab(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task) {
+    int n = task.rows;
+    float alpha = 1.0f, beta = 0.0f, omega = 1.0f, rho = 1.0f, rho_prev = 1.0f;
+    float tolerance = 1e-5f;
+    int max_iters = 2000;
+
+    float *d_val, *d_x, *d_r, *d_r_hat, *d_p, *d_v, *d_s, *d_t, *d_b;
+    int *d_row_ptr, *d_col_ind;
+
+    CHECK_CUDA(cudaMalloc(&d_val, task.nnz * sizeof(float)));
+    CHECK_CUDA(cudaMalloc(&d_row_ptr, (n + 1) * sizeof(int)));
+    CHECK_CUDA(cudaMalloc(&d_col_ind, task.nnz * sizeof(int)));
+    CHECK_CUDA(cudaMalloc(&d_x, n * sizeof(float)));
+    CHECK_CUDA(cudaMalloc(&d_r, n * sizeof(float)));
+    CHECK_CUDA(cudaMalloc(&d_r_hat, n * sizeof(float)));
+    CHECK_CUDA(cudaMalloc(&d_p, n * sizeof(float)));
+    CHECK_CUDA(cudaMalloc(&d_v, n * sizeof(float)));
+    CHECK_CUDA(cudaMalloc(&d_s, n * sizeof(float)));
+    CHECK_CUDA(cudaMalloc(&d_t, n * sizeof(float)));
+    CHECK_CUDA(cudaMalloc(&d_b, n * sizeof(float)));
+
+    CHECK_CUDA(cudaMemcpy(d_val, task.values.data(), task.nnz * sizeof(float), cudaMemcpyHostToDevice));
+    CHECK_CUDA(cudaMemcpy(d_row_ptr, task.row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice));
+    CHECK_CUDA(cudaMemcpy(d_col_ind, task.col_indices.data(), task.nnz * sizeof(int), cudaMemcpyHostToDevice));
+    CHECK_CUDA(cudaMemcpy(d_b, task.b.data(), n * sizeof(float), cudaMemcpyHostToDevice));
+    CHECK_CUDA(cudaMemset(d_x, 0, n * sizeof(float)));
+    CHECK_CUDA(cudaMemset(d_v, 0, n * sizeof(float)));
+    CHECK_CUDA(cudaMemset(d_p, 0, n * sizeof(float)));
+
+    cusparseSpMatDescr_t matA;
+    cusparseDnVecDescr_t vecX, vecP, vecV, vecS, vecT;
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.nnz, d_row_ptr, d_col_ind, d_val, 
+                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecV, n, d_v, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecS, n, d_s, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecT, n, d_t, CUDA_R_32F));
+
+    size_t bufferSize = 0;
+    void* d_buffer = nullptr;
+    float one = 1.0f, zero = 0.0f;
+    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE, &one, matA, vecP, &zero, vecV, 
+                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    CHECK_CUDA(cudaMalloc(&d_buffer, bufferSize));
+
+    // r = b - Ax (initially r = b)
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_r_hat, 1));
+
+    for (int i = 1; i <= max_iters; ++i) {
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r_hat, 1, d_r, 1, &rho));
+        
+        if (i == 1) {
+            CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_p, 1));
+        } else {
+            beta = (rho / rho_prev) * (alpha / omega);
+            float neg_omega = -omega;
+            CHECK_CUBLAS(cublasSaxpy(cublas, n, &neg_omega, d_v, 1, d_p, 1));
+            CHECK_CUBLAS(cublasSscal(cublas, n, &beta, d_p, 1));
+            CHECK_CUBLAS(cublasSaxpy(cublas, n, &one, d_r, 1, d_p, 1));
+        }
+
+        // v = Ap
+        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE, &one, matA, vecP, &zero, vecV, 
+                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
+
+        float rhat_v;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r_hat, 1, d_v, 1, &rhat_v));
+        alpha = rho / rhat_v;
+
+        // s = r - alpha*v
+        CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_s, 1));
+        float neg_alpha = -alpha;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &neg_alpha, d_v, 1, d_s, 1));
+
+        // t = As
+        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE, &one, matA, vecS, &zero, vecT, 
+                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
+
+        float t_s, t_t;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_t, 1, d_s, 1, &t_s));
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_t, 1, d_t, 1, &t_t));
+        omega = t_s / t_t;
+
+        // x = x + alpha*p + omega*s
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_p, 1, d_x, 1));
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &omega, d_s, 1, d_x, 1));
+
+        // r = s - omega*t
+        CHECK_CUBLAS(cublasScopy(cublas, n, d_s, 1, d_r, 1));
+        float neg_omega = -omega;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &neg_omega, d_t, 1, d_r, 1));
+
+        float norm_r;
+        CHECK_CUBLAS(cublasSnrm2(cublas, n, d_r, 1, &norm_r));
+        if (norm_r < tolerance) break;
+        
+        rho_prev = rho;
+    }
+
+    // Cleanup
+    cusparseDestroySpMat(matA);
+    cusparseDestroyDnVec(vecP); cusparseDestroyDnVec(vecV);
+    cusparseDestroyDnVec(vecS); cusparseDestroyDnVec(vecT);
+    cudaFree(d_val); cudaFree(d_row_ptr); cudaFree(d_col_ind);
+    cudaFree(d_x); cudaFree(d_r); cudaFree(d_r_hat); cudaFree(d_p);
+    cudaFree(d_v); cudaFree(d_s); cudaFree(d_t); cudaFree(d_b);
+    cudaFree(d_buffer);
+}
+
+// Worker thread function
+void gpu_worker(int device_id, std::queue<size_t>& tasks, std::mutex& mtx, const std::vector<MatrixTask>& all_tasks) {
+    CHECK_CUDA(cudaSetDevice(device_id));
+    
+    cublasHandle_t cublas;
+    cusparseHandle_t cusparse;
+    CHECK_CUBLAS(cublasCreate(&cublas));
+    CHECK_CUSPARSE(cusparseCreate(&cusparse));
+
+    while (true) {
+        size_t task_idx;
+        {
+            std::lock_guard<std::mutex> lock(mtx);
+            if (tasks.empty()) break;
+            task_idx = tasks.front();
+            tasks.pop();
+        }
+
+        const auto& t = all_tasks[task_idx];
+        if (t.type == CGS_SOLVER) {
+            solve_cg(cublas, cusparse, t);
+        } else {
+            solve_bicgstab(cublas, cusparse, t); 
+        }
+    }
+
+    cublasDestroy(cublas);
+    cusparseDestroy(cusparse);
+}
+
+int main() {
+    std::vector<MatrixTask> tasks;
+    
+    auto scan = [&](const std::string& dir, SolverType type) {
+        if (!fs::exists(dir)) return;
+        for (const auto& entry : fs::directory_iterator(dir)) {
+            if (entry.path().extension() == ".bin") {
+                auto coo = load_binary_coo(entry.path().string());
+                auto csr = convert_coo_to_csr(coo);
+                MatrixTask t;
+                t.path = entry.path().string();
+                t.type = type;
+                t.rows = csr.rows;
+                t.nnz = csr.nnz;
+                t.row_ptr = std::move(csr.row_ptr);
+                t.col_indices = std::move(csr.col_indices);
+                t.values = std::move(csr.values);
+                t.b = compute_rhs_spmv(csr, std::vector<float>(csr.cols, 1.0f));
+                tasks.push_back(std::move(t));
+            }
+        }
+    };
+
+    std::cout << "[INFO] Loading matrices...\n";
+    scan("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+    scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
+
+    if (tasks.empty()) {
+        std::cerr << "No matrices found.\n";
+        return 1;
+    }
+
+    int num_gpus;
+    CHECK_CUDA(cudaGetDeviceCount(&num_gpus));
+    int workers_per_gpu = 8;
+
+    std::queue<size_t> task_indices;
+    for (size_t i = 0; i < tasks.size(); ++i) task_indices.push(i);
+    std::mutex queue_mutex;
+
+    std::cout << "[INFO] Processing " << tasks.size() << " tasks using " 
+              << num_gpus << " GPUs (" << workers_per_gpu << " threads/GPU)...\n";
+
+    auto start = std::chrono::steady_clock::now();
+
+    std::vector<std::thread> workers;
+    for (int i = 0; i < num_gpus; ++i) {
+        for (int j = 0; j < workers_per_gpu; ++j) {
+            workers.emplace_back(gpu_worker, i, std::ref(task_indices), std::ref(queue_mutex), std::cref(tasks));
+        }
+    }
+
+    for (auto& w : workers) w.join();
+
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+
+    std::cout << "\n===== NATIVE BENCHMARK COMPLETE =====\n";
+    std::cout << "Tasks Processed: " << tasks.size() << "\n";
+    std::cout << "Total Runtime:   " << elapsed.count() << " s\n";
+    std::cout << "======================================\n";
+
+    return 0;
+}
\ No newline at end of file

From f366a24823572b971f4c4dd0e321b3b907676da6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 08:28:35 -0600
Subject: [PATCH 0743/1000] fixed segfault

---
 libcaf_cuda/tests/workload-test/main.native.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/workload-test/main.native.cpp b/libcaf_cuda/tests/workload-test/main.native.cpp
index d3c62113d2..c0b8d01fbe 100644
--- a/libcaf_cuda/tests/workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/workload-test/main.native.cpp
@@ -291,6 +291,8 @@ int main() {
                 auto coo = load_binary_coo(entry.path().string());
                 auto csr = convert_coo_to_csr(coo);
                 MatrixTask t;
+                // Compute t.b using the valid csr object before its internal vectors are moved
+                t.b = compute_rhs_spmv(csr, std::vector<float>(csr.cols, 1.0f));
                 t.path = entry.path().string();
                 t.type = type;
                 t.rows = csr.rows;
@@ -298,7 +300,6 @@ int main() {
                 t.row_ptr = std::move(csr.row_ptr);
                 t.col_indices = std::move(csr.col_indices);
                 t.values = std::move(csr.values);
-                t.b = compute_rhs_spmv(csr, std::vector<float>(csr.cols, 1.0f));
                 tasks.push_back(std::move(t));
             }
         }

From b0f7c730605333325cfb01152374b4caccf460d2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 08:41:09 -0600
Subject: [PATCH 0744/1000] Updated cuda test to be on stream order allocator.

---
 .../tests/workload-test/CMakeLists.txt        |   6 +-
 .../tests/workload-test/main.native.cpp       | 186 ++++++++++--------
 2 files changed, 110 insertions(+), 82 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/CMakeLists.txt b/libcaf_cuda/tests/workload-test/CMakeLists.txt
index a4b615b10f..5f3e4dffe2 100644
--- a/libcaf_cuda/tests/workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/workload-test/CMakeLists.txt
@@ -44,6 +44,9 @@ target_link_libraries(test
     CUDA::cusparse
 )
 
+# FindThreads is required for std::thread in the native version
+find_package(Threads REQUIRED)
+
 # 6) Declare the native benchmark executable (raw CUDA/cuBLAS/cuSPARSE)
 add_executable(workload-native main.native.cpp sparse_utils.cpp)
 
@@ -54,6 +57,3 @@ target_link_libraries(workload-native
     CUDA::cusparse
     Threads::Threads
 )
-
-# FindThreads is required for std::thread in the native version
-find_package(Threads REQUIRED)
diff --git a/libcaf_cuda/tests/workload-test/main.native.cpp b/libcaf_cuda/tests/workload-test/main.native.cpp
index c0b8d01fbe..a12fa13eb6 100644
--- a/libcaf_cuda/tests/workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/workload-test/main.native.cpp
@@ -57,7 +57,7 @@ struct MatrixTask {
 };
 
 // Simplified CG Solver using raw cuBLAS and cuSPARSE
-void solve_cg(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task) {
+void solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream) {
     int n = task.rows;
     float alpha = 1.0f, beta = 0.0f, r0 = 0.0f, r1 = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
     float tolerance = 1e-5f;
@@ -66,22 +66,26 @@ void solve_cg(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask
     float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b;
     int *d_row_ptr, *d_col_ind;
 
-    CHECK_CUDA(cudaMalloc(&d_val, task.nnz * sizeof(float)));
-    CHECK_CUDA(cudaMalloc(&d_row_ptr, (n + 1) * sizeof(int)));
-    CHECK_CUDA(cudaMalloc(&d_col_ind, task.nnz * sizeof(int)));
-    CHECK_CUDA(cudaMalloc(&d_x, n * sizeof(float)));
-    CHECK_CUDA(cudaMalloc(&d_r, n * sizeof(float)));
-    CHECK_CUDA(cudaMalloc(&d_p, n * sizeof(float)));
-    CHECK_CUDA(cudaMalloc(&d_Ap, n * sizeof(float)));
-    CHECK_CUDA(cudaMalloc(&d_b, n * sizeof(float)));
-
-    CHECK_CUDA(cudaMemcpy(d_val, task.values.data(), task.nnz * sizeof(float), cudaMemcpyHostToDevice));
-    CHECK_CUDA(cudaMemcpy(d_row_ptr, task.row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice));
-    CHECK_CUDA(cudaMemcpy(d_col_ind, task.col_indices.data(), task.nnz * sizeof(int), cudaMemcpyHostToDevice));
-    CHECK_CUDA(cudaMemcpy(d_b, task.b.data(), n * sizeof(float), cudaMemcpyHostToDevice));
-    CHECK_CUDA(cudaMemset(d_x, 0, n * sizeof(float)));
+    // Use Stream Ordered Allocator
+    CHECK_CUDA(cudaMallocAsync(&d_val, task.nnz * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.nnz * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
+
+    CHECK_CUDA(cudaMemcpyAsync(d_val, task.values.data(), task.nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.col_indices.data(), task.nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_b, task.b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
 
     // Create descriptors
+    CHECK_CUBLAS(cublasSetStream(cublas, stream));
+    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
+
     cusparseSpMatDescr_t matA;
     cusparseDnVecDescr_t vecX, vecP, vecAp;
     CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.nnz, d_row_ptr, d_col_ind, d_val, 
@@ -94,7 +98,7 @@ void solve_cg(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask
     void* d_buffer = nullptr;
     CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecAp, 
                                            CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
-    CHECK_CUDA(cudaMalloc(&d_buffer, bufferSize));
+    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
 
     // CG Logic: r = b - Ax (initially r = b since x=0)
     CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
@@ -135,13 +139,21 @@ void solve_cg(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask
     cusparseDestroyDnVec(vecX);
     cusparseDestroyDnVec(vecP);
     cusparseDestroyDnVec(vecAp);
-    cudaFree(d_val); cudaFree(d_row_ptr); cudaFree(d_col_ind);
-    cudaFree(d_x); cudaFree(d_r); cudaFree(d_p); cudaFree(d_Ap); cudaFree(d_b);
-    cudaFree(d_buffer);
+
+    // Stream-ordered free
+    CHECK_CUDA(cudaFreeAsync(d_val, stream));
+    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
+    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
+    CHECK_CUDA(cudaFreeAsync(d_x, stream));
+    CHECK_CUDA(cudaFreeAsync(d_r, stream));
+    CHECK_CUDA(cudaFreeAsync(d_p, stream));
+    CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
+    CHECK_CUDA(cudaFreeAsync(d_b, stream));
+    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
 }
 
-// BiCGSTAB Solver using raw cuBLAS and cuSPARSE
-void solve_bicgstab(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task) {
+// BiCGSTAB Solver using raw cuBLAS and cuSPARSE (Asynchronous version)
+void solve_bicgstab_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream) {
     int n = task.rows;
     float alpha = 1.0f, beta = 0.0f, omega = 1.0f, rho = 1.0f, rho_prev = 1.0f;
     float tolerance = 1e-5f;
@@ -150,25 +162,28 @@ void solve_bicgstab(cublasHandle_t cublas, cusparseHandle_t cusparse, const Matr
     float *d_val, *d_x, *d_r, *d_r_hat, *d_p, *d_v, *d_s, *d_t, *d_b;
     int *d_row_ptr, *d_col_ind;
 
-    CHECK_CUDA(cudaMalloc(&d_val, task.nnz * sizeof(float)));
-    CHECK_CUDA(cudaMalloc(&d_row_ptr, (n + 1) * sizeof(int)));
-    CHECK_CUDA(cudaMalloc(&d_col_ind, task.nnz * sizeof(int)));
-    CHECK_CUDA(cudaMalloc(&d_x, n * sizeof(float)));
-    CHECK_CUDA(cudaMalloc(&d_r, n * sizeof(float)));
-    CHECK_CUDA(cudaMalloc(&d_r_hat, n * sizeof(float)));
-    CHECK_CUDA(cudaMalloc(&d_p, n * sizeof(float)));
-    CHECK_CUDA(cudaMalloc(&d_v, n * sizeof(float)));
-    CHECK_CUDA(cudaMalloc(&d_s, n * sizeof(float)));
-    CHECK_CUDA(cudaMalloc(&d_t, n * sizeof(float)));
-    CHECK_CUDA(cudaMalloc(&d_b, n * sizeof(float)));
-
-    CHECK_CUDA(cudaMemcpy(d_val, task.values.data(), task.nnz * sizeof(float), cudaMemcpyHostToDevice));
-    CHECK_CUDA(cudaMemcpy(d_row_ptr, task.row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice));
-    CHECK_CUDA(cudaMemcpy(d_col_ind, task.col_indices.data(), task.nnz * sizeof(int), cudaMemcpyHostToDevice));
-    CHECK_CUDA(cudaMemcpy(d_b, task.b.data(), n * sizeof(float), cudaMemcpyHostToDevice));
-    CHECK_CUDA(cudaMemset(d_x, 0, n * sizeof(float)));
-    CHECK_CUDA(cudaMemset(d_v, 0, n * sizeof(float)));
-    CHECK_CUDA(cudaMemset(d_p, 0, n * sizeof(float)));
+    CHECK_CUDA(cudaMallocAsync(&d_val, task.nnz * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.nnz * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_r_hat, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_v, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_s, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_t, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
+
+    CHECK_CUDA(cudaMemcpyAsync(d_val, task.values.data(), task.nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.col_indices.data(), task.nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_b, task.b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMemsetAsync(d_v, 0, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMemsetAsync(d_p, 0, n * sizeof(float), stream));
+
+    CHECK_CUBLAS(cublasSetStream(cublas, stream));
+    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
 
     cusparseSpMatDescr_t matA;
     cusparseDnVecDescr_t vecX, vecP, vecV, vecS, vecT;
@@ -184,7 +199,7 @@ void solve_bicgstab(cublasHandle_t cublas, cusparseHandle_t cusparse, const Matr
     float one = 1.0f, zero = 0.0f;
     CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE, &one, matA, vecP, &zero, vecV, 
                                            CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
-    CHECK_CUDA(cudaMalloc(&d_buffer, bufferSize));
+    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
 
     // r = b - Ax (initially r = b)
     CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
@@ -197,8 +212,8 @@ void solve_bicgstab(cublasHandle_t cublas, cusparseHandle_t cusparse, const Matr
             CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_p, 1));
         } else {
             beta = (rho / rho_prev) * (alpha / omega);
-            float neg_omega = -omega;
-            CHECK_CUBLAS(cublasSaxpy(cublas, n, &neg_omega, d_v, 1, d_p, 1));
+            float minus_omega = -omega;
+            CHECK_CUBLAS(cublasSaxpy(cublas, n, &minus_omega, d_v, 1, d_p, 1));
             CHECK_CUBLAS(cublasSscal(cublas, n, &beta, d_p, 1));
             CHECK_CUBLAS(cublasSaxpy(cublas, n, &one, d_r, 1, d_p, 1));
         }
@@ -231,8 +246,8 @@ void solve_bicgstab(cublasHandle_t cublas, cusparseHandle_t cusparse, const Matr
 
         // r = s - omega*t
         CHECK_CUBLAS(cublasScopy(cublas, n, d_s, 1, d_r, 1));
-        float neg_omega = -omega;
-        CHECK_CUBLAS(cublasSaxpy(cublas, n, &neg_omega, d_t, 1, d_r, 1));
+        float neg_omega_bc = -omega;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &neg_omega_bc, d_t, 1, d_r, 1));
 
         float norm_r;
         CHECK_CUBLAS(cublasSnrm2(cublas, n, d_r, 1, &norm_r));
@@ -245,40 +260,53 @@ void solve_bicgstab(cublasHandle_t cublas, cusparseHandle_t cusparse, const Matr
     cusparseDestroySpMat(matA);
     cusparseDestroyDnVec(vecP); cusparseDestroyDnVec(vecV);
     cusparseDestroyDnVec(vecS); cusparseDestroyDnVec(vecT);
-    cudaFree(d_val); cudaFree(d_row_ptr); cudaFree(d_col_ind);
-    cudaFree(d_x); cudaFree(d_r); cudaFree(d_r_hat); cudaFree(d_p);
-    cudaFree(d_v); cudaFree(d_s); cudaFree(d_t); cudaFree(d_b);
-    cudaFree(d_buffer);
+    CHECK_CUDA(cudaFreeAsync(d_val, stream));
+    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
+    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
+    CHECK_CUDA(cudaFreeAsync(d_x, stream));
+    CHECK_CUDA(cudaFreeAsync(d_r, stream));
+    CHECK_CUDA(cudaFreeAsync(d_r_hat, stream));
+    CHECK_CUDA(cudaFreeAsync(d_p, stream));
+    CHECK_CUDA(cudaFreeAsync(d_v, stream));
+    CHECK_CUDA(cudaFreeAsync(d_s, stream));
+    CHECK_CUDA(cudaFreeAsync(d_t, stream));
+    CHECK_CUDA(cudaFreeAsync(d_b, stream));
+    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
 }
 
-// Worker thread function
-void gpu_worker(int device_id, std::queue<size_t>& tasks, std::mutex& mtx, const std::vector<MatrixTask>& all_tasks) {
+// New GPU Dispatcher function (One thread per GPU)
+void gpu_dispatcher(int device_id, std::vector<MatrixTask> assigned_tasks, int num_workers) {
     CHECK_CUDA(cudaSetDevice(device_id));
     
-    cublasHandle_t cublas;
-    cusparseHandle_t cusparse;
-    CHECK_CUBLAS(cublasCreate(&cublas));
-    CHECK_CUSPARSE(cusparseCreate(&cusparse));
-
-    while (true) {
-        size_t task_idx;
-        {
-            std::lock_guard<std::mutex> lock(mtx);
-            if (tasks.empty()) break;
-            task_idx = tasks.front();
-            tasks.pop();
-        }
+    std::vector<cudaStream_t> streams(num_workers);
+    std::vector<cublasHandle_t> cublas_handles(num_workers);
+    std::vector<cusparseHandle_t> cusparse_handles(num_workers);
+
+    for (int i = 0; i < num_workers; ++i) {
+        CHECK_CUDA(cudaStreamCreateWithFlags(&streams[i], cudaStreamNonBlocking));
+        CHECK_CUBLAS(cublasCreate(&cublas_handles[i]));
+        CHECK_CUSPARSE(cusparseCreate(&cusparse_handles[i]));
+    }
 
-        const auto& t = all_tasks[task_idx];
+    // Deep pipelining: Loop through assigned tasks and dispatch to streams round-robin
+    for (size_t i = 0; i < assigned_tasks.size(); ++i) {
+        int s_idx = i % num_workers;
+        const auto& t = assigned_tasks[i];
         if (t.type == CGS_SOLVER) {
-            solve_cg(cublas, cusparse, t);
+            solve_cg_async(cublas_handles[s_idx], cusparse_handles[s_idx], t, streams[s_idx]);
         } else {
-            solve_bicgstab(cublas, cusparse, t); 
+            solve_bicgstab_async(cublas_handles[s_idx], cusparse_handles[s_idx], t, streams[s_idx]); 
         }
     }
 
-    cublasDestroy(cublas);
-    cusparseDestroy(cusparse);
+    // Wait for everything on this device to finish
+    CHECK_CUDA(cudaDeviceSynchronize());
+
+    for (int i = 0; i < num_workers; ++i) {
+        cublasDestroy(cublas_handles[i]);
+        cusparseDestroy(cusparse_handles[i]);
+        cudaStreamDestroy(streams[i]);
+    }
 }
 
 int main() {
@@ -317,24 +345,24 @@ int main() {
     int num_gpus;
     CHECK_CUDA(cudaGetDeviceCount(&num_gpus));
     int workers_per_gpu = 8;
-
-    std::queue<size_t> task_indices;
-    for (size_t i = 0; i < tasks.size(); ++i) task_indices.push(i);
-    std::mutex queue_mutex;
+    
+    // Static Round-Robin Partitioning
+    std::vector<std::vector<MatrixTask>> partitions(num_gpus);
+    for (size_t i = 0; i < tasks.size(); ++i) {
+        partitions[i % num_gpus].push_back(std::move(tasks[i]));
+    }
 
     std::cout << "[INFO] Processing " << tasks.size() << " tasks using " 
-              << num_gpus << " GPUs (" << workers_per_gpu << " threads/GPU)...\n";
+              << num_gpus << " Dispatcher threads (" << workers_per_gpu << " streams/GPU)...\n";
 
     auto start = std::chrono::steady_clock::now();
 
-    std::vector<std::thread> workers;
+    std::vector<std::thread> dispatchers;
     for (int i = 0; i < num_gpus; ++i) {
-        for (int j = 0; j < workers_per_gpu; ++j) {
-            workers.emplace_back(gpu_worker, i, std::ref(task_indices), std::ref(queue_mutex), std::cref(tasks));
-        }
+        dispatchers.emplace_back(gpu_dispatcher, i, std::move(partitions[i]), workers_per_gpu);
     }
 
-    for (auto& w : workers) w.join();
+    for (auto& d : dispatchers) d.join();
 
     auto end = std::chrono::steady_clock::now();
     std::chrono::duration<double> elapsed = end - start;

From 887916538b18dfe88e9512d982154fe37c0cfa5d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 09:52:07 -0600
Subject: [PATCH 0745/1000] fixed exit race

---
 libcaf_cuda/tests/workload-test/main.test.cpp | 126 +++++++++---------
 1 file changed, 63 insertions(+), 63 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index a7afb6bb76..1f3da23b3c 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -63,20 +63,20 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
 
 // ---------------------------- GLOBAL TASK POOL ----------------------------
 struct pool_state {
-    std::vector<MatrixTask> tasks;
+    std::shared_ptr<std::vector<MatrixTask>> tasks;
     size_t next_task_idx = 0;
 };
 
-behavior global_task_pool(stateful_actor<pool_state>* self, std::vector<MatrixTask> tasks) {
+behavior global_task_pool(stateful_actor<pool_state>* self, std::shared_ptr<std::vector<MatrixTask>> tasks) {
     self->state().tasks = std::move(tasks);
     return {
         [=](get_work_atom, size_t batch_size) -> result<std::vector<MatrixTask>> {
             auto& st = self->state();
-            if (st.next_task_idx >= st.tasks.size())
+            if (st.next_task_idx >= st.tasks->size())
                 return sec::end_of_stream;
-            size_t count = std::min(batch_size, st.tasks.size() - st.next_task_idx);
-            std::vector<MatrixTask> batch(st.tasks.begin() + st.next_task_idx, 
-                                          st.tasks.begin() + st.next_task_idx + count);
+            size_t count = std::min(batch_size, st.tasks->size() - st.next_task_idx);
+            std::vector<MatrixTask> batch(st.tasks->begin() + st.next_task_idx, 
+                                          st.tasks->begin() + st.next_task_idx + count);
             st.next_task_idx += count;
             return batch;
         }
@@ -87,6 +87,7 @@ behavior global_task_pool(stateful_actor<pool_state>* self, std::vector<MatrixTa
 struct device_actor_state {
     caf::actor global_pool;
     std::deque<MatrixTask> local_tasks;
+    std::vector<caf::response_promise> pending_promises; // Store untyped promises cleanly
     int active_workers = 0;
     int device_id = -1;
     bool fetching = false;
@@ -100,24 +101,52 @@ behavior gpu_device_actor(stateful_actor<device_actor_state>* self,
     self->state().device_id = dev_id;
     self->state().active_workers = num_workers;
 
-    // Dynamically calculate prefetch markers based on the total pipeline capacity
     self->state().low_water_mark = static_cast<size_t>(num_workers * max_in_flight);
     self->state().batch_size = self->state().low_water_mark * 2;
 
+    auto satisfy_promises = [=]() {
+        auto& st = self->state();
+        while (!st.pending_promises.empty() && !st.local_tasks.empty()) {
+            auto promise = std::move(st.pending_promises.front());
+            st.pending_promises.erase(st.pending_promises.begin());
+
+            MatrixTask t = std::move(st.local_tasks.front());
+            st.local_tasks.pop_front();
+
+            auto& data = *t.data;
+            promise.deliver(t.type, t.path, create_in_arg(data.row_ptr), create_in_arg(data.col_indices), 
+                            create_in_arg(data.values), create_in_arg(data.b), create_in_out_arg(data.x_guess), 
+                            (int)data.row_ptr.size() - 1, (int)data.values.size(), t.data);
+        }
+    };
+
     auto refill = [=]() {
         auto& st = self->state();
-        if (st.fetching || st.local_tasks.size() >= st.low_water_mark)
+        if (st.fetching)
+            return;
+
+        if (st.local_tasks.size() >= st.low_water_mark && st.pending_promises.empty())
             return;
 
         st.fetching = true;
         self->mail(get_work_atom_v, st.batch_size).request(st.global_pool, infinite).then(
             [=](std::vector<MatrixTask>& batch) {
+                auto& st_inner = self->state();
+                st_inner.fetching = false;
+
                 for (auto& task : batch)
-                    self->state().local_tasks.push_back(std::move(task));
-                self->state().fetching = false;
+                    st_inner.local_tasks.push_back(std::move(task));
+
+                satisfy_promises();
             },
             [=](error& err) {
-                self->state().fetching = false;
+                auto& st_inner = self->state();
+                st_inner.fetching = false;
+                
+                for (auto& promise : st_inner.pending_promises) {
+                    promise.deliver(err);
+                }
+                st_inner.pending_promises.clear();
             }
         );
     };
@@ -127,49 +156,18 @@ behavior gpu_device_actor(stateful_actor<device_actor_state>* self,
     return {
         [=](get_work_atom) -> result<SolverType, std::string, in<int>, in<int>, in<float>, in<float>, in_out<float>, int, int, std::shared_ptr<MatrixData>> {
             auto& st = self->state();
+            
+            // Fixed Type Mismatch: Explicitly use untyped response_promise
+            caf::response_promise promise = self->make_response_promise();
+            st.pending_promises.push_back(promise);
+            
+            satisfy_promises();
+            refill();
 
-            // If local tasks are available, process immediately
-            if (!st.local_tasks.empty()) {
-                MatrixTask t = std::move(st.local_tasks.front());
-                st.local_tasks.pop_front();
-                refill(); // Try to refill if below low water mark
-
-                auto& data = *t.data;
-                return {t.type, t.path, create_in_arg(data.row_ptr), create_in_arg(data.col_indices), 
-                        create_in_arg(data.values), create_in_arg(data.b), create_in_out_arg(data.x_guess), 
-                        (int)data.row_ptr.size() - 1, (int)data.values.size(), t.data};
-            }
-
-            auto promise = self->make_response_promise<SolverType, std::string, in<int>, in<int>, in<float>, in<float>, in_out<float>, int, int, std::shared_ptr<MatrixData>>();
-            self->mail(get_work_atom_v, st.batch_size).request(st.global_pool, infinite).then(
-                [=](std::vector<MatrixTask>& batch) mutable {
-                    auto& st_inner = self->state();
-                    if (batch.empty()) { // Global pool returned empty batch, meaning no more work
-                        promise.deliver(make_error(sec::end_of_stream));
-                        return;
-                    }
-
-                    // Add remaining tasks to local queue
-                    for (size_t i = 1; i < batch.size(); ++i) {
-                        st_inner.local_tasks.push_back(std::move(batch[i]));
-                    }
-
-                    MatrixTask t = std::move(batch.front()); // Process the first task from the batch
-                    refill(); // Try to refill if below low water mark (after adding tasks)
-
-                    auto& data = *t.data;
-                    promise.deliver(t.type, t.path, create_in_arg(data.row_ptr), create_in_arg(data.col_indices), 
-                                    create_in_arg(data.values), create_in_arg(data.b), create_in_out_arg(data.x_guess), 
-                                    (int)data.row_ptr.size() - 1, (int)data.values.size(), t.data);
-                },
-                [=](error& err) mutable {
-                    promise.deliver(err); // Propagate error from global pool
-                }
-            );
             return promise;
         },
         [=](release_memory_atom, std::string path) {
-            // No longer used with pre-loaded pool
+            // Managed entirely by shared_ptrs
         },
         [=](worker_done_atom) {
             if (--self->state().active_workers <= 0)
@@ -229,7 +227,6 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
         [=](std::vector<float>& solution) {
             self->mail(1).send(self->state().supervisor);
             self->state().current_data.reset();
-            self->mail(release_memory_atom_v, self->state().current_matrix_path).send(self->state().device_actor);
             self->mail(request_work_atom_v).send(self);
         }
     };
@@ -239,16 +236,18 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
 struct supervisor_state {
     int total_tasks;
     int completed = 0;
+    std::shared_ptr<std::vector<MatrixTask>> tasks_holder; // Anchors shared_ptr reference count
 };
 
-behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, int total, std::vector<MatrixTask> tasks) {
+behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, int total, std::shared_ptr<std::vector<MatrixTask>> tasks) {
     self->state().total_tasks = total;
-    auto pool = self->spawn(global_task_pool, std::move(tasks));
+    self->state().tasks_holder = tasks; // Retain ownership within supervisor state
+    auto pool = self->spawn(global_task_pool, tasks); // Pass a copy, don't move it
     
     manager& mgr = manager::get();
     int num_gpus = mgr.get_num_devices();
-    int workers_per_gpu = 8; // Adjustable worker count per physical GPU
-    int max_in_flight_tasks_per_worker = 2; // How many tasks each worker can have in flight
+    int workers_per_gpu = 8;
+    int max_in_flight_tasks_per_worker = 2;
 
     for (int i = 0; i < num_gpus; ++i) {
         auto broker = self->spawn(gpu_device_actor, pool, workers_per_gpu, i, max_in_flight_tasks_per_worker);
@@ -270,21 +269,21 @@ behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, int total,
 
 void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
-    std::vector<MatrixTask> tasks;
+    auto tasks = std::make_shared<std::vector<MatrixTask>>();
     
     auto scan = [&](const std::string& dir, SolverType type) {
         if (!fs::exists(dir)) return;
         for (const auto& entry : fs::directory_iterator(dir)) {
             if (entry.path().extension() == ".bin")
-                tasks.push_back({entry.path().string(), type, nullptr});
+                tasks->push_back({entry.path().string(), type, nullptr});
         }
     };
 
     scan("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
     scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
 
-    std::cout << "[INFO] Pre-loading " << tasks.size() << " matrices into memory...\n";
-    for (auto& t : tasks) {
+    std::cout << "[INFO] Pre-loading " << tasks->size() << " matrices into memory...\n";
+    for (auto& t : *tasks) {
         auto coo = load_binary_coo(t.path);
         auto A = convert_coo_to_csr(coo);
         t.data = std::make_shared<MatrixData>();
@@ -295,18 +294,19 @@ void caf_main(actor_system& sys) {
         t.data->x_guess.assign(A.cols, 0.0f);
     }
 
-    if (tasks.empty()) {
+    if (tasks->empty()) {
         std::cerr << "No matrix files found in search paths.\n";
         manager::shutdown();
         return;
     }
 
-    auto task_count = tasks.size();
+    auto task_count = tasks->size();
     std::cout << "[INFO] Found " << task_count << " matrices. Spawning workload...\n";
 
     auto start = std::chrono::steady_clock::now();
 
-    sys.spawn(supervisor_actor_fun, static_cast<int>(task_count), std::move(tasks));
+    // Fixed Exit Race: Pass tasks directly by copy to keep it active inside caf_main frame
+    sys.spawn(supervisor_actor_fun, static_cast<int>(task_count), tasks);
     sys.await_all_actors_done();
 
     auto end = std::chrono::steady_clock::now();
@@ -319,4 +319,4 @@ void caf_main(actor_system& sys) {
 
     manager::shutdown();
 }
-CAF_MAIN(id_block::cuda, id_block::cg_actor, id_block::bicgstab_actor, id_block::workload_test)
+CAF_MAIN(id_block::cuda, id_block::cg_actor, id_block::bicgstab_actor, id_block::workload_test)
\ No newline at end of file

From ce807450b80b2cb3b8e42f0e201558676df0f448 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 11:01:24 -0600
Subject: [PATCH 0746/1000] updated bicgstab solver to reduce the number of
 self messages sent Change is being made since self message passing just
 creates a synchronization bottleneck in this case

---
 .../sparse-BiCGSTAB-actor.hpp                 | 221 +++++-------------
 .../BICGSTAB-actor-test/main.test.cpp         |   2 +-
 2 files changed, 60 insertions(+), 163 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
index 78b2d1c805..1753d9c08a 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
@@ -10,24 +10,9 @@
 #include "caf/cuda/platform.hpp"
 #include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
 
-// Define a new block for BiCGSTAB specific atoms starting where the cuda block ended
-CAF_BEGIN_TYPE_ID_BLOCK(bicgstab_actor, caf::id_block::cg_actor::end)
-  CAF_ADD_ATOM(bicgstab_actor, bicgstab_next_step_atom)
-CAF_END_TYPE_ID_BLOCK(bicgstab_actor)
 
 namespace caf::cuda {
 
-enum class sparse_bicgstab_step {
-  idle,
-  init_residual_norm_sq, // Calculate initial ||r||^2
-  calc_rho_new,          // Calculate rho_new = <r_hat, r>
-  calc_alpha_denom,      // Calculate <r_hat, v> for alpha
-  calc_omega_num,        // Calculate <t, s> for omega
-  calc_omega_denom,      // Calculate <t, t> for omega
-  check_convergence,     // Check ||r||^2 for convergence
-  finished
-};
-
 struct sparse_bicgstab_state {
   // Host Data
   in<int> h_row_ptr, h_col_ind;
@@ -55,14 +40,7 @@ struct sparse_bicgstab_state {
   float alpha_val = 1.0f;
   float omega_val = 1.0f;
   float beta_val = 0.0f;
-  float residual_norm_sq_val = 0.0f; // Stores the latest ||r||^2
-  float rho_new_val = 0.0f; // Stores the result of <r_hat, r>
-  float alpha_denom_val = 0.0f; // Stores the result of <r_hat, v>
-  float omega_num_val = 0.0f; // Stores the result of <t, s>
-  float omega_denom_val = 0.0f; // Stores the result of <t, t>
-
   int iterations = 0;
-  sparse_bicgstab_step step = sparse_bicgstab_step::idle;
 };
 
 class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
@@ -92,35 +70,6 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
           s.supervisor = actor_cast<caf::actor>(this->current_sender());
         start_solve();
       },
-      [this](bicgstab_next_step_atom, float val) {
-        auto& s = state();
-        // Thread-safely update scalars based on the stage that just finished
-        switch (s.step) {
-          case sparse_bicgstab_step::init_residual_norm_sq:
-          case sparse_bicgstab_step::check_convergence:
-            s.residual_norm_sq_val = val;
-            break;
-          case sparse_bicgstab_step::calc_rho_new:
-            s.rho_new_val = val;
-            break;
-          case sparse_bicgstab_step::calc_alpha_denom:
-            s.alpha_denom_val = val;
-            break;
-          case sparse_bicgstab_step::calc_omega_num:
-            s.omega_num_val = val;
-            break;
-          case sparse_bicgstab_step::calc_omega_denom:
-            s.omega_denom_val = val;
-            break;
-          default: break;
-        }
-        perform_bicgstab_step();
-      },
-      [this](gpu_done_atom, std::vector<float>& solution) {
-        if (state().supervisor)
-          this->mail(std::move(solution)).send(state().supervisor);
-        this->quit();
-      }
     };
   }
 private:
@@ -169,137 +118,85 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
     // 1. Initial Residual: r = b - Ax
     execute_spmv(s.x, s.v); 
     s.d_ptr->scopy(s.stream_id, s.n, s.b, s.r);
-    s.d_ptr->saxpy(s.stream_id, s.n, -1.0f, s.v, s.r);
+    s.d_ptr->saxpy(s.stream_id, s.n, -1.0f, s.v, s.r); // r = b - Ax
     
     // 2. Choose r_hat = r
     s.d_ptr->scopy(s.stream_id, s.n, s.r, s.r_hat);
 
-    s.iterations = 0;
-    // Initial calculation of residual_norm_sq
+    // 3. Initial norm calculation
     s.d_ptr->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
-    s.step = sparse_bicgstab_step::init_residual_norm_sq;
-    auto self = actor_cast<actor>(this);
-    runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
-      anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
-    });
-  }
-
-  void perform_bicgstab_step() {
-    auto& s = state();
-    command_runner<> runner;
-    auto self = actor_cast<actor>(this);
+    float norm_sq = s.y_tmp->copy_to_host()[0];
 
-    switch (s.step) {
-      case sparse_bicgstab_step::init_residual_norm_sq:
-      case sparse_bicgstab_step::check_convergence: {
-        s.iterations++; // Increment for the current iteration
-
-        // Check convergence
-        if (s.residual_norm_sq_val <= (s.tol * s.tol) || s.iterations > s.max_iter) {
-          finish_solve();
-          return;
-        }
-
-        // rho_i = <r_hat, r>
-        s.d_ptr->sdot(s.stream_id, s.n, s.r_hat, s.r, s.y_tmp);
-        s.step = sparse_bicgstab_step::calc_rho_new;
-        runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
-          anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
-        });
-        break;
-      }
-
-      case sparse_bicgstab_step::calc_rho_new: {
-        // Update p based on rho_new_val
-        if (s.iterations == 1) { // This is the first iteration
-          s.d_ptr->scopy(s.stream_id, s.n, s.r, s.p);
-        } else { // Subsequent iterations
-          s.beta_val = (s.rho_new_val / s.rho_val) * (s.alpha_val / s.omega_val);
-          // p = r + beta * (p - omega * v)
-          s.d_ptr->saxpy(s.stream_id, s.n, -s.omega_val, s.v, s.p); // p = p - omega*v
-          s.d_ptr->scopy(s.stream_id, s.n, s.r, s.s_vec);      // use s_vec as temporary
-          s.d_ptr->saxpy(s.stream_id, s.n, s.beta_val, s.p, s.s_vec); // s_vec = r + beta*p
-          s.d_ptr->scopy(s.stream_id, s.n, s.s_vec, s.p);
-        }
-        s.rho_val = s.rho_new_val; // Update old rho
-
-        // v = Ap
-        execute_spmv(s.p, s.v);
-
-        // alpha = rho / <r_hat, v> -> calculate denominator
-        s.d_ptr->sdot(s.stream_id, s.n, s.r_hat, s.v, s.y_tmp);
-        s.step = sparse_bicgstab_step::calc_alpha_denom;
-        runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
-          anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
-        });
-        break;
-      }
-
-      case sparse_bicgstab_step::calc_alpha_denom: {
-        s.alpha_val = s.rho_val / s.alpha_denom_val;
-
-        // s = r - alpha * v
+    s.iterations = 0;
+    s.rho_val = 1.0f;
+    s.alpha_val = 1.0f;
+    s.omega_val = 1.0f;
+
+    // BiCGSTAB Loop
+    while (norm_sq > (s.tol * s.tol) && s.iterations < s.max_iter) {
+      s.iterations++;
+
+      // rho_new = <r_hat, r>
+      s.d_ptr->sdot(s.stream_id, s.n, s.r_hat, s.r, s.y_tmp);
+      float rho_new = s.y_tmp->copy_to_host()[0];
+
+      if (s.iterations == 1) {
+        s.d_ptr->scopy(s.stream_id, s.n, s.r, s.p);
+      } else {
+        s.beta_val = (rho_new / s.rho_val) * (s.alpha_val / s.omega_val);
+        // p = r + beta * (p - omega * v)
+        s.d_ptr->saxpy(s.stream_id, s.n, -s.omega_val, s.v, s.p);
         s.d_ptr->scopy(s.stream_id, s.n, s.r, s.s_vec);
-        s.d_ptr->saxpy(s.stream_id, s.n, -s.alpha_val, s.v, s.s_vec);
+        s.d_ptr->saxpy(s.stream_id, s.n, s.beta_val, s.p, s.s_vec);
+        s.d_ptr->scopy(s.stream_id, s.n, s.s_vec, s.p);
+      }
+      s.rho_val = rho_new;
 
-        // t = As
-        execute_spmv(s.s_vec, s.t_vec);
+      // v = Ap
+      execute_spmv(s.p, s.v);
 
-        // omega = <t, s> / <t, t> -> calculate numerator <t, s>
-        s.d_ptr->sdot(s.stream_id, s.n, s.t_vec, s.s_vec, s.y_tmp);
-        s.step = sparse_bicgstab_step::calc_omega_num;
-        runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
-          anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
-        });
-        break;
-      }
+      // alpha = rho / <r_hat, v>
+      s.d_ptr->sdot(s.stream_id, s.n, s.r_hat, s.v, s.y_tmp);
+      float alpha_denom = s.y_tmp->copy_to_host()[0];
+      s.alpha_val = s.rho_val / alpha_denom;
 
-      case sparse_bicgstab_step::calc_omega_num: {
-        // omega = <t, s> / <t, t> -> calculate denominator <t, t>
-        s.d_ptr->sdot(s.stream_id, s.n, s.t_vec, s.t_vec, s.y_tmp);
-        s.step = sparse_bicgstab_step::calc_omega_denom;
-        runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
-          anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
-        });
-        break;
-      }
+      // s = r - alpha * v
+      s.d_ptr->scopy(s.stream_id, s.n, s.r, s.s_vec);
+      s.d_ptr->saxpy(s.stream_id, s.n, -s.alpha_val, s.v, s.s_vec);
 
-      case sparse_bicgstab_step::calc_omega_denom: {
-        s.omega_val = s.omega_num_val / s.omega_denom_val;
+      // t = As
+      execute_spmv(s.s_vec, s.t_vec);
 
-        // x = x + alpha*p + omega*s
-        s.d_ptr->saxpy(s.stream_id, s.n, s.alpha_val, s.p, s.x);
-        s.d_ptr->saxpy(s.stream_id, s.n, s.omega_val, s.s_vec, s.x);
+      // omega = <t, s> / <t, t>
+      s.d_ptr->sdot(s.stream_id, s.n, s.t_vec, s.s_vec, s.y_tmp);
+      float omega_num = s.y_tmp->copy_to_host()[0];
+      s.d_ptr->sdot(s.stream_id, s.n, s.t_vec, s.t_vec, s.y_tmp);
+      float omega_denom = s.y_tmp->copy_to_host()[0];
+      s.omega_val = omega_num / omega_denom;
 
-        // r = s - omega*t
-        s.d_ptr->scopy(s.stream_id, s.n, s.s_vec, s.r);
-        s.d_ptr->saxpy(s.stream_id, s.n, -s.omega_val, s.t_vec, s.r);
+      // x = x + alpha*p + omega*s
+      s.d_ptr->saxpy(s.stream_id, s.n, s.alpha_val, s.p, s.x);
+      s.d_ptr->saxpy(s.stream_id, s.n, s.omega_val, s.s_vec, s.x);
 
-        // Calculate new residual_norm_sq for next iteration's convergence check
-        s.d_ptr->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
-        s.step = sparse_bicgstab_step::check_convergence;
-        runner.copy_to_host_async(s.y_tmp, [self](std::vector<float> host_data) {
-          anon_mail(bicgstab_next_step_atom_v, host_data[0]).send(self);
-        });
-        break;
-      }
+      // r = s - omega*t
+      s.d_ptr->scopy(s.stream_id, s.n, s.s_vec, s.r);
+      s.d_ptr->saxpy(s.stream_id, s.n, -s.omega_val, s.t_vec, s.r);
 
-      case sparse_bicgstab_step::idle:
-      case sparse_bicgstab_step::finished:
-        // Should not happen if logic is correct
-        break;
+      // check convergence: norm_sq = <r, r>
+      s.d_ptr->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
+      norm_sq = s.y_tmp->copy_to_host()[0];
     }
+
+    finish_solve();
   }
 
   void finish_solve() {
     auto& s = state();
-    s.step = sparse_bicgstab_step::finished;
-    // Final copy to host and send result to supervisor
-    auto self = actor_cast<actor>(this);
-    command_runner<> runner;
-    runner.copy_to_host_async(s.x, [self](std::vector<float> solution) {
-      anon_mail(gpu_done_atom_v, std::move(solution)).send(self);
-    });
+    // Copy result back to host and notify supervisor
+    std::vector<float> solution = s.x->copy_to_host();
+    if (s.supervisor)
+      this->mail(std::move(solution)).send(s.supervisor);
+    this->quit();
   }
 
   void execute_spmv(mem_ptr<float> input_v, mem_ptr<float> output_v) {
diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
index 522697038f..1dc9bd809d 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
@@ -139,4 +139,4 @@ void caf_main(actor_system& sys) {
 
     manager::shutdown();
 }
-CAF_MAIN(id_block::cuda)
+CAF_MAIN(id_block::cuda, id_block::cg_actor)

From 0b2406e5da929d91a1c3207adcfd087b3e409c80 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 11:05:49 -0600
Subject: [PATCH 0747/1000] updated test to reflect changes made in bigcstab
 actor

---
 libcaf_cuda/tests/workload-test/main.test.cpp | 23 ++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index 1f3da23b3c..e0c86f9711 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -45,7 +45,7 @@ bool inspect(Inspector& f, SolverType& x) {
     return false;
 }
 
-CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::bicgstab_actor::end)
+CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cg_actor::end)
     CAF_ADD_ATOM(workload_test, get_work_atom)
     CAF_ADD_ATOM(workload_test, release_memory_atom)
     CAF_ADD_ATOM(workload_test, request_work_atom)
@@ -184,6 +184,8 @@ struct worker_state {
     int stream_id;
     std::shared_ptr<MatrixData> current_data;
     std::string current_matrix_path;
+    std::chrono::steady_clock::time_point task_start;
+    SolverType current_solver_type;
 };
 
 behavior sparse_worker_fun(stateful_actor<worker_state>* self,
@@ -201,6 +203,9 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
                 [=](SolverType type, std::string path, in<int> rp, in<int> ci, in<float> val,
                     in<float> b, in_out<float> x, int rows, int nnz, std::shared_ptr<MatrixData> data) {
                     self->state().current_matrix_path = path;
+                    self->state().current_solver_type = type;
+                    self->state().task_start = std::chrono::steady_clock::now();
+                    auto start_spawn = std::chrono::steady_clock::now();
                     self->state().current_data = data;
                     actor solver;
                     if (type == CGS_SOLVER) {
@@ -214,6 +219,9 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
                             std::move(b), std::move(x),
                             matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id, actor_cast<actor>(self));
                     }
+                    auto end_spawn = std::chrono::steady_clock::now();
+                    std::chrono::duration<double> spawn_duration = end_spawn - start_spawn;
+                    // self->println("Worker {}: Solver actor spawned in {} s", self->state().stream_id, spawn_duration.count());
                     self->mail(start_atom_v).send(solver);
                 },
                 [=](error& err) {
@@ -225,6 +233,15 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
             );
         },
         [=](std::vector<float>& solution) {
+            auto task_end = std::chrono::steady_clock::now();
+            std::chrono::duration<double> task_duration = task_end - self->state().task_start;
+            std::string solver_type_str;
+            if (self->state().current_solver_type == CGS_SOLVER) {
+                solver_type_str = "CGS_SOLVER";
+            } else {
+                solver_type_str = "BICSTAB_SOLVER";
+            }
+            self->println("Worker {}: Round-trip time (Spawn to Result) for {} ({}) took {} s", self->state().stream_id, self->state().current_matrix_path, solver_type_str, task_duration.count());
             self->mail(1).send(self->state().supervisor);
             self->state().current_data.reset();
             self->mail(request_work_atom_v).send(self);
@@ -247,7 +264,7 @@ behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, int total,
     manager& mgr = manager::get();
     int num_gpus = mgr.get_num_devices();
     int workers_per_gpu = 8;
-    int max_in_flight_tasks_per_worker = 2;
+    int max_in_flight_tasks_per_worker = 1;
 
     for (int i = 0; i < num_gpus; ++i) {
         auto broker = self->spawn(gpu_device_actor, pool, workers_per_gpu, i, max_in_flight_tasks_per_worker);
@@ -319,4 +336,4 @@ void caf_main(actor_system& sys) {
 
     manager::shutdown();
 }
-CAF_MAIN(id_block::cuda, id_block::cg_actor, id_block::bicgstab_actor, id_block::workload_test)
\ No newline at end of file
+CAF_MAIN(id_block::cuda, id_block::cg_actor,id_block::workload_test)
\ No newline at end of file

From 06b0d4a12230c422329366699dbf2069847ed777 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 11:07:30 -0600
Subject: [PATCH 0748/1000] updated test to log completion time of kernels

---
 libcaf_cuda/tests/workload-test/main.native.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/libcaf_cuda/tests/workload-test/main.native.cpp b/libcaf_cuda/tests/workload-test/main.native.cpp
index a12fa13eb6..95daa5fea1 100644
--- a/libcaf_cuda/tests/workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/workload-test/main.native.cpp
@@ -292,11 +292,18 @@ void gpu_dispatcher(int device_id, std::vector<MatrixTask> assigned_tasks, int n
     for (size_t i = 0; i < assigned_tasks.size(); ++i) {
         int s_idx = i % num_workers;
         const auto& t = assigned_tasks[i];
+        auto start_task = std::chrono::steady_clock::now();
         if (t.type == CGS_SOLVER) {
             solve_cg_async(cublas_handles[s_idx], cusparse_handles[s_idx], t, streams[s_idx]);
         } else {
             solve_bicgstab_async(cublas_handles[s_idx], cusparse_handles[s_idx], t, streams[s_idx]); 
         }
+        CHECK_CUDA(cudaStreamSynchronize(streams[s_idx]));
+        auto end_task = std::chrono::steady_clock::now();
+        std::chrono::duration<double> task_duration = end_task - start_task;
+        std::string solver_type_str = (t.type == CGS_SOLVER) ? "CGS_SOLVER" : "BICSTAB_SOLVER";
+        std::cout << "Stream " << (device_id * 100 + s_idx) << ": Solve time for " << t.path 
+                  << " (" << solver_type_str << ") took " << task_duration.count() << " s" << std::endl;
     }
 
     // Wait for everything on this device to finish

From 2e60e89fab37aa8f01ab8b5056e49a98ef7dd34f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 11:45:12 -0600
Subject: [PATCH 0749/1000] Updated test to be multi threaded or at least more
 multi threaded.

---
 .../tests/workload-test/main.native.cpp       | 74 ++++++++++---------
 1 file changed, 38 insertions(+), 36 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.native.cpp b/libcaf_cuda/tests/workload-test/main.native.cpp
index 95daa5fea1..5cec4a5e63 100644
--- a/libcaf_cuda/tests/workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/workload-test/main.native.cpp
@@ -189,6 +189,7 @@ void solve_bicgstab_async(cublasHandle_t cublas, cusparseHandle_t cusparse, cons
     cusparseDnVecDescr_t vecX, vecP, vecV, vecS, vecT;
     CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.nnz, d_row_ptr, d_col_ind, d_val, 
                                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
     CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
     CHECK_CUSPARSE(cusparseCreateDnVec(&vecV, n, d_v, CUDA_R_32F));
     CHECK_CUSPARSE(cusparseCreateDnVec(&vecS, n, d_s, CUDA_R_32F));
@@ -201,8 +202,13 @@ void solve_bicgstab_async(cublasHandle_t cublas, cusparseHandle_t cusparse, cons
                                            CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
     CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
 
-    // r = b - Ax (initially r = b)
+    // r = b - Ax
+    // Note: We compute Ax explicitly to support non-zero initial guesses in the future
+    CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE, &one, matA, vecX, &zero, vecV, 
+                                CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
     CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
+    float minus_one = -1.0f;
+    CHECK_CUBLAS(cublasSaxpy(cublas, n, &minus_one, d_v, 1, d_r, 1));
     CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_r_hat, 1));
 
     for (int i = 1; i <= max_iters; ++i) {
@@ -258,7 +264,7 @@ void solve_bicgstab_async(cublasHandle_t cublas, cusparseHandle_t cusparse, cons
 
     // Cleanup
     cusparseDestroySpMat(matA);
-    cusparseDestroyDnVec(vecP); cusparseDestroyDnVec(vecV);
+    cusparseDestroyDnVec(vecX); cusparseDestroyDnVec(vecP); cusparseDestroyDnVec(vecV);
     cusparseDestroyDnVec(vecS); cusparseDestroyDnVec(vecT);
     CHECK_CUDA(cudaFreeAsync(d_val, stream));
     CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
@@ -274,46 +280,37 @@ void solve_bicgstab_async(cublasHandle_t cublas, cusparseHandle_t cusparse, cons
     CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
 }
 
-// New GPU Dispatcher function (One thread per GPU)
-void gpu_dispatcher(int device_id, std::vector<MatrixTask> assigned_tasks, int num_workers) {
+// GPU Worker function (One thread per stream)
+void gpu_worker(int device_id, int thread_id, std::vector<MatrixTask> assigned_tasks) {
     CHECK_CUDA(cudaSetDevice(device_id));
     
-    std::vector<cudaStream_t> streams(num_workers);
-    std::vector<cublasHandle_t> cublas_handles(num_workers);
-    std::vector<cusparseHandle_t> cusparse_handles(num_workers);
-
-    for (int i = 0; i < num_workers; ++i) {
-        CHECK_CUDA(cudaStreamCreateWithFlags(&streams[i], cudaStreamNonBlocking));
-        CHECK_CUBLAS(cublasCreate(&cublas_handles[i]));
-        CHECK_CUSPARSE(cusparseCreate(&cusparse_handles[i]));
-    }
+    cudaStream_t stream;
+    cublasHandle_t cublas_handle;
+    cusparseHandle_t cusparse_handle;
 
-    // Deep pipelining: Loop through assigned tasks and dispatch to streams round-robin
-    for (size_t i = 0; i < assigned_tasks.size(); ++i) {
-        int s_idx = i % num_workers;
-        const auto& t = assigned_tasks[i];
+    CHECK_CUDA(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
+    CHECK_CUBLAS(cublasCreate(&cublas_handle));
+    CHECK_CUSPARSE(cusparseCreate(&cusparse_handle));
+
+    for (const auto& t : assigned_tasks) {
         auto start_task = std::chrono::steady_clock::now();
         if (t.type == CGS_SOLVER) {
-            solve_cg_async(cublas_handles[s_idx], cusparse_handles[s_idx], t, streams[s_idx]);
+            solve_cg_async(cublas_handle, cusparse_handle, t, stream);
         } else {
-            solve_bicgstab_async(cublas_handles[s_idx], cusparse_handles[s_idx], t, streams[s_idx]); 
+            solve_bicgstab_async(cublas_handle, cusparse_handle, t, stream); 
         }
-        CHECK_CUDA(cudaStreamSynchronize(streams[s_idx]));
+        // CHECK_CUDA(cudaStreamSynchronize(stream)); solvers are synchronious anyways do not need this
+
         auto end_task = std::chrono::steady_clock::now();
         std::chrono::duration<double> task_duration = end_task - start_task;
         std::string solver_type_str = (t.type == CGS_SOLVER) ? "CGS_SOLVER" : "BICSTAB_SOLVER";
-        std::cout << "Stream " << (device_id * 100 + s_idx) << ": Solve time for " << t.path 
+        std::cout << "Thread " << thread_id << ": Solve time for " << t.path 
                   << " (" << solver_type_str << ") took " << task_duration.count() << " s" << std::endl;
     }
 
-    // Wait for everything on this device to finish
-    CHECK_CUDA(cudaDeviceSynchronize());
-
-    for (int i = 0; i < num_workers; ++i) {
-        cublasDestroy(cublas_handles[i]);
-        cusparseDestroy(cusparse_handles[i]);
-        cudaStreamDestroy(streams[i]);
-    }
+    cublasDestroy(cublas_handle);
+    cusparseDestroy(cusparse_handle);
+    cudaStreamDestroy(stream);
 }
 
 int main() {
@@ -353,23 +350,28 @@ int main() {
     CHECK_CUDA(cudaGetDeviceCount(&num_gpus));
     int workers_per_gpu = 8;
     
-    // Static Round-Robin Partitioning
-    std::vector<std::vector<MatrixTask>> partitions(num_gpus);
+    // Hierarchical Static Partitioning: Queue -> Devices -> Worker Threads
+    std::vector<std::vector<std::vector<MatrixTask>>> partitions(num_gpus, 
+                                        std::vector<std::vector<MatrixTask>>(workers_per_gpu));
     for (size_t i = 0; i < tasks.size(); ++i) {
-        partitions[i % num_gpus].push_back(std::move(tasks[i]));
+        int g_id = i % num_gpus;
+        int w_id = (i / num_gpus) % workers_per_gpu;
+        partitions[g_id][w_id].push_back(std::move(tasks[i]));
     }
 
     std::cout << "[INFO] Processing " << tasks.size() << " tasks using " 
-              << num_gpus << " Dispatcher threads (" << workers_per_gpu << " streams/GPU)...\n";
+              << (num_gpus * workers_per_gpu) << " worker threads (" << workers_per_gpu << " threads/GPU)...\n";
 
     auto start = std::chrono::steady_clock::now();
 
-    std::vector<std::thread> dispatchers;
+    std::vector<std::thread> workers;
     for (int i = 0; i < num_gpus; ++i) {
-        dispatchers.emplace_back(gpu_dispatcher, i, std::move(partitions[i]), workers_per_gpu);
+        for (int j = 0; j < workers_per_gpu; ++j) {
+            workers.emplace_back(gpu_worker, i, (i * 100 + j), std::move(partitions[i][j]));
+        }
     }
 
-    for (auto& d : dispatchers) d.join();
+    for (auto& w : workers) w.join();
 
     auto end = std::chrono::steady_clock::now();
     std::chrono::duration<double> elapsed = end - start;

From 405738b3d0a6e3123f95cd076db93d36409e2de3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 12:34:49 -0600
Subject: [PATCH 0750/1000] Updated tests.

---
 .../BICGSTAB-actor-test/main.test.cpp         | 100 +++++++++++++++++-
 1 file changed, 97 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
index 1dc9bd809d..9dc8178daf 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
@@ -99,10 +99,104 @@ void caf_main(actor_system& sys) {
         );
     }
 
-    // Test 3: Stress Test - 1D Laplacian (N=10000)
+    // Test 3: COO Format
+    {
+        std::cout << "\n[INFO] Test 3: COO format simple matrix..." << std::endl;
+        std::vector<int> row_ind = {0, 1, 2};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto solver = sys.spawn<sparse_bicgstab_actor>(
+            create_in_arg(row_ind), create_in_arg(col_ind), create_in_arg(values),
+            create_in_arg(h_b), create_in_out_arg(h_x),
+            matrix_format::coo, n, nnz, tolerance, max_iter, 0, 2, actor_cast<actor>(self));
+
+        self->mail(start_atom_v).send(solver);
+        self->receive(
+            [&](std::vector<float> result_x) {
+                verify_solution("COO Simple", result_x, expected);
+            }
+        );
+    }
+
+    // Test 4: Tridiagonal matrix (CSR) - Known solution x = [1, 1, 1]
+    {
+        std::cout << "\n[INFO] Test 4: CSR format tridiagonal matrix (3x3)..." << std::endl;
+        // Matrix: [ 2 -1  0 ]
+        //         [-1  2 -1 ]
+        //         [ 0 -1  2 ]
+        // b = [1, 0, 1] -> Expected x = [1, 1, 1]
+        std::vector<int> row_ptr = {0, 2, 5, 7};
+        std::vector<int> col_ind = {0, 1, 0, 1, 2, 1, 2};
+        std::vector<float> values = {2.0f, -1.0f, -1.0f, 2.0f, -1.0f, -1.0f, 2.0f};
+        std::vector<float> b_tri = {1.0f, 0.0f, 1.0f};
+        std::vector<float> x_tri(3, 0.0f);
+        std::vector<float> expected_tri = {1.0f, 1.0f, 1.0f};
+
+        auto solver = sys.spawn<sparse_bicgstab_actor>(
+            create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+            create_in_arg(b_tri), create_in_out_arg(x_tri),
+            matrix_format::csr, 3, 7, tolerance, max_iter, 0, 3, actor_cast<actor>(self));
+
+        self->mail(start_atom_v).send(solver);
+        self->receive(
+            [&](std::vector<float> result_x) {
+                verify_solution("CSR Tridiagonal (3x3)", result_x, expected_tri);
+            }
+        );
+    }
+
+    // Test 5: Larger Tridiagonal Correctness (N=100)
+    {
+        int N_mid = 100;
+        std::cout << "\n[INFO] Test 5: CSR format tridiagonal correctness (N=" << N_mid << ")..." << std::endl;
+        
+        std::vector<int> row_ptr;
+        std::vector<int> col_ind;
+        std::vector<float> values;
+        std::vector<float> expected_mid(N_mid, 1.0f);
+        std::vector<float> b_mid(N_mid, 0.0f);
+
+        row_ptr.push_back(0);
+        for(int i=0; i<N_mid; ++i) {
+            float row_sum = 0.0f;
+            if(i > 0) { 
+                col_ind.push_back(i-1); 
+                values.push_back(-1.0f); 
+                row_sum += -1.0f;
+            }
+            col_ind.push_back(i); 
+            values.push_back(2.1f); // Diagonally dominant to ensure convergence
+            row_sum += 2.1f;
+            if(i < N_mid-1) { 
+                col_ind.push_back(i+1); 
+                values.push_back(-1.0f); 
+                row_sum += -1.0f;
+            }
+            row_ptr.push_back(col_ind.size());
+            b_mid[i] = row_sum; // b = A * ones()
+        }
+
+        std::vector<float> x_mid(N_mid, 0.0f);
+        
+        auto solver = sys.spawn<sparse_bicgstab_actor>(
+            create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+            create_in_arg(b_mid), create_in_out_arg(x_mid),
+            matrix_format::csr, N_mid, (int)values.size(), 1e-5f, 500, 0, 4, actor_cast<actor>(self));
+
+        self->mail(start_atom_v).send(solver);
+        self->receive(
+            [&](std::vector<float> result) {
+                verify_solution("CSR N=100 Correctness", result, expected_mid, 1e-2f);
+            }
+        );
+    }
+
+    // Test 6: Stress Test - 1D Laplacian (N=10000)
     {
         int N_large = 10000;
-        std::cout << "\n[INFO] Test 3: Stress Test - Non-Symmetric Matrix (N=" << N_large << ")..." << std::endl;
+        std::cout << "\n[INFO] Test 6: Stress Test - Non-Symmetric Matrix (N=" << N_large << ")..." << std::endl;
         
         std::vector<int> row_ptr;
         std::vector<int> col_ind;
@@ -122,7 +216,7 @@ void caf_main(actor_system& sys) {
         auto stress_solver = sys.spawn<sparse_bicgstab_actor>(
             create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
             create_in_arg(b_large), create_in_out_arg(x_large),
-            matrix_format::csr, N_large, (int)values.size(), 1e-4f, 20000, 0, 2, actor_cast<actor>(self));
+            matrix_format::csr, N_large, (int)values.size(), 1e-4f, 20000, 0, 5, actor_cast<actor>(self));
 
         self->mail(start_atom_v).send(stress_solver);
         self->receive(

From d336d6aee803b914c3e0c481c95af3580b7aedc9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 12:37:03 -0600
Subject: [PATCH 0751/1000] Updated tests.

---
 .../BICGSTAB-actor-test/main.test.cpp         | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
index 9dc8178daf..e0022e17c6 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
@@ -231,6 +231,55 @@ void caf_main(actor_system& sys) {
         );
     }
 
+    // Test 7: Ill-conditioned / High Iteration Count Test (N=5000)
+    // This uses a non-symmetric tridiagonal matrix with very weak diagonal dominance.
+    // A_ii = 2.0001, A_{i,i-1} = -1.1, A_{i,i+1} = -0.9.
+    // The near-singularity forces BiCGSTAB to take many iterations to converge.
+    {
+        int N_high = 5000;
+        std::cout << "\n[INFO] Test 7: High Iteration Count Test (N=" << N_high << ")..." << std::endl;
+
+        std::vector<int> row_ptr;
+        std::vector<int> col_ind;
+        std::vector<float> values;
+        std::vector<float> expected_high(N_high, 1.0f);
+        std::vector<float> b_high(N_high, 0.0f);
+
+        row_ptr.push_back(0);
+        for(int i=0; i<N_high; ++i) {
+            float row_sum = 0.0f;
+            if(i > 0) {
+                col_ind.push_back(i-1);
+                values.push_back(-1.1f);
+                row_sum += -1.1f;
+            }
+            col_ind.push_back(i);
+            values.push_back(2.0001f); // Extremely low diagonal dominance
+            row_sum += 2.0001f;
+            if(i < N_high-1) {
+                col_ind.push_back(i+1);
+                values.push_back(-0.9f);
+                row_sum += -0.9f;
+            }
+            row_ptr.push_back(col_ind.size());
+            b_high[i] = row_sum; // b = A * ones()
+        }
+
+        std::vector<float> x_high(N_high, 0.0f);
+
+        auto solver = sys.spawn<sparse_bicgstab_actor>(
+            create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+            create_in_arg(b_high), create_in_out_arg(x_high),
+            matrix_format::csr, N_high, (int)values.size(), 1e-6f, 15000, 0, 6, actor_cast<actor>(self));
+
+        self->mail(start_atom_v).send(solver);
+        self->receive(
+            [&](std::vector<float> result) {
+                verify_solution("High Iteration Count Test", result, expected_high, 5e-2f);
+            }
+        );
+    }
+
     manager::shutdown();
 }
 CAF_MAIN(id_block::cuda, id_block::cg_actor)

From 7a197f9d94c2affa390d00f028a1df760c01912a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 13:06:28 -0600
Subject: [PATCH 0752/1000] added nan tests

---
 .../BICGSTAB-actor-test/main.test.cpp         | 208 +++++++++++++-----
 1 file changed, 158 insertions(+), 50 deletions(-)

diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
index e0022e17c6..c15aff7dea 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
@@ -12,6 +12,7 @@
 #include <vector>
 #include <chrono>
 #include <thread>
+#include <fstream>
 #include <algorithm>
 #include <numeric>
 #include <cmath>
@@ -21,6 +22,77 @@
 using namespace caf;
 using namespace caf::cuda;
 
+// --- Manual Sparse Utilities for Testing ---
+
+struct LocalCSR {
+    int rows, cols, nnz;
+    std::vector<int> row_ptr;
+    std::vector<int> col_ind;
+    std::vector<float> values;
+};
+
+LocalCSR load_binary_matrix_manual(const std::string& path) {
+    std::ifstream file(path, std::ios::binary);
+    if (!file) throw std::runtime_error("Could not open " + path);
+
+    int32_t r, c, n;
+    file.read(reinterpret_cast<char*>(&r), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&c), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&n), sizeof(int32_t));
+
+    std::vector<int32_t> rows_coo(n), cols_coo(n);
+    std::vector<float> vals_coo(n);
+
+    file.read(reinterpret_cast<char*>(rows_coo.data()), n * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(cols_coo.data()), n * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(vals_coo.data()), n * sizeof(float));
+
+    // Detect and fix 1-based indexing (Matrix Market standard)
+    // If the maximum index found equals the dimension 'r', it is 1-based.
+    int max_idx = 0;
+    for(auto v : rows_coo) if(v > max_idx) max_idx = v;
+    for(auto v : cols_coo) if(v > max_idx) max_idx = v;
+
+    if (max_idx == r || max_idx == c) {
+        std::cout << "[INFO] 1-based indexing detected. Converting to 0-based..." << std::endl;
+        for(auto& v : rows_coo) v--;
+        for(auto& v : cols_coo) v--;
+    }
+
+    // Convert COO to CSR
+    LocalCSR csr;
+    csr.rows = r; csr.cols = c; csr.nnz = n;
+    csr.row_ptr.assign(r + 1, 0);
+    csr.col_ind.resize(n);
+    csr.values.resize(n);
+
+    for (int i = 0; i < n; ++i) csr.row_ptr[rows_coo[i] + 1]++;
+    for (int i = 0; i < r; ++i) csr.row_ptr[i + 1] += csr.row_ptr[i];
+
+    std::vector<int> current_pos = csr.row_ptr;
+    for (int i = 0; i < n; ++i) {
+        int row = rows_coo[i];
+        int dest = current_pos[row]++;
+        csr.col_ind[dest] = cols_coo[i];
+        csr.values[dest] = vals_coo[i];
+    }
+    return csr;
+}
+
+std::vector<float> compute_rhs_manual(const LocalCSR& A, const std::vector<float>& x) {
+    std::vector<float> b(A.rows, 0.0f);
+    for (int i = 0; i < A.rows; ++i) {
+        float sum = 0.0f;
+        for (int j = A.row_ptr[i]; j < A.row_ptr[i+1]; ++j) {
+            sum += A.values[j] * x[A.col_ind[j]];
+        }
+        b[i] = sum;
+    }
+    return b;
+}
+
+// --- End Manual Utilities ---
+
 void verify_solution(const std::string& test_name, const std::vector<float>& actual, 
                      const std::vector<float>& expected, float tol = 1e-3) {
     if (actual.size() != expected.size()) {
@@ -30,11 +102,11 @@ void verify_solution(const std::string& test_name, const std::vector<float>& act
     }
     bool all_correct = true;
     for (size_t i = 0; i < actual.size(); ++i) {
-        if (std::abs(actual[i] - expected[i]) > tol) {
+        if (std::isnan(actual[i]) || std::isinf(actual[i]) || std::abs(actual[i] - expected[i]) > tol) {
             all_correct = false;
             std::cout << "[ERROR] " << test_name << " mismatch at index " << i 
                       << ": Expected " << expected[i] 
-                      << ", Got " << actual[i] << std::endl;
+                      << ", Got " << actual[i] << (std::isnan(actual[i]) ? " (NaN)" : "") << std::endl;
             break;
         }
     }
@@ -73,7 +145,7 @@ void caf_main(actor_system& sys) {
         self->mail(start_atom_v).send(solver);
         self->receive(
             [&](std::vector<float> result_x) {
-                verify_solution("CSR Simple", result_x, expected);
+                verify_solution("CSR Simple", result_x, expected, tolerance);
             }
         );
     }
@@ -94,7 +166,7 @@ void caf_main(actor_system& sys) {
         self->mail(start_atom_v).send(solver);
         self->receive(
             [&](std::vector<float> result_x) {
-                verify_solution("CSC Simple", result_x, expected);
+                verify_solution("CSC Simple", result_x, expected, tolerance);
             }
         );
     }
@@ -115,7 +187,7 @@ void caf_main(actor_system& sys) {
         self->mail(start_atom_v).send(solver);
         self->receive(
             [&](std::vector<float> result_x) {
-                verify_solution("COO Simple", result_x, expected);
+                verify_solution("COO Simple", result_x, expected, tolerance);
             }
         );
     }
@@ -142,7 +214,7 @@ void caf_main(actor_system& sys) {
         self->mail(start_atom_v).send(solver);
         self->receive(
             [&](std::vector<float> result_x) {
-                verify_solution("CSR Tridiagonal (3x3)", result_x, expected_tri);
+                verify_solution("CSR Tridiagonal (3x3)", result_x, expected_tri, tolerance);
             }
         );
     }
@@ -188,56 +260,18 @@ void caf_main(actor_system& sys) {
         self->mail(start_atom_v).send(solver);
         self->receive(
             [&](std::vector<float> result) {
-                verify_solution("CSR N=100 Correctness", result, expected_mid, 1e-2f);
-            }
-        );
-    }
-
-    // Test 6: Stress Test - 1D Laplacian (N=10000)
-    {
-        int N_large = 10000;
-        std::cout << "\n[INFO] Test 6: Stress Test - Non-Symmetric Matrix (N=" << N_large << ")..." << std::endl;
-        
-        std::vector<int> row_ptr;
-        std::vector<int> col_ind;
-        std::vector<float> values;
-        row_ptr.push_back(0);
-        for(int i=0; i<N_large; ++i) {
-            if(i > 0) { col_ind.push_back(i-1); values.push_back(-1.5f); } // Lower
-            col_ind.push_back(i); values.push_back(4.0f);                 // Diag
-            if(i < N_large-1) { col_ind.push_back(i+1); values.push_back(-0.5f); } // Upper
-            row_ptr.push_back(col_ind.size());
-        }
-
-        std::vector<float> b_large(N_large, 1.0f);
-        std::vector<float> x_large(N_large, 0.0f);
-        
-        auto start = std::chrono::high_resolution_clock::now();
-        auto stress_solver = sys.spawn<sparse_bicgstab_actor>(
-            create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
-            create_in_arg(b_large), create_in_out_arg(x_large),
-            matrix_format::csr, N_large, (int)values.size(), 1e-4f, 20000, 0, 5, actor_cast<actor>(self));
-
-        self->mail(start_atom_v).send(stress_solver);
-        self->receive(
-            [&](std::vector<float> result) {
-                auto end = std::chrono::high_resolution_clock::now();
-                std::chrono::duration<double> elapsed = end - start;
-                std::cout << "[SUCCESS] Stress Test completed in " << elapsed.count() << " seconds." << std::endl;
-                std::cout << "[INFO] First 5 elements of solution: ";
-                for(int i=0; i<5; ++i) std::cout << result[i] << " ";
-                std::cout << "..." << std::endl;
+                verify_solution("CSR N=100 Correctness", result, expected_mid, 1e-4f);
             }
         );
     }
 
-    // Test 7: Ill-conditioned / High Iteration Count Test (N=5000)
+    // Test 6: Ill-conditioned / High Iteration Count Test (N=5000)
     // This uses a non-symmetric tridiagonal matrix with very weak diagonal dominance.
     // A_ii = 2.0001, A_{i,i-1} = -1.1, A_{i,i+1} = -0.9.
     // The near-singularity forces BiCGSTAB to take many iterations to converge.
     {
         int N_high = 5000;
-        std::cout << "\n[INFO] Test 7: High Iteration Count Test (N=" << N_high << ")..." << std::endl;
+        std::cout << "\n[INFO] Test 6: High Iteration Count Test (N=" << N_high << ")..." << std::endl;
 
         std::vector<int> row_ptr;
         std::vector<int> col_ind;
@@ -262,7 +296,7 @@ void caf_main(actor_system& sys) {
                 row_sum += -0.9f;
             }
             row_ptr.push_back(col_ind.size());
-            b_high[i] = row_sum; // b = A * ones()
+            b_high[i] = row_sum;
         }
 
         std::vector<float> x_high(N_high, 0.0f);
@@ -270,12 +304,86 @@ void caf_main(actor_system& sys) {
         auto solver = sys.spawn<sparse_bicgstab_actor>(
             create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
             create_in_arg(b_high), create_in_out_arg(x_high),
-            matrix_format::csr, N_high, (int)values.size(), 1e-6f, 15000, 0, 6, actor_cast<actor>(self));
+            matrix_format::csr, N_high, (int)values.size(), 1e-6f, 15000, 0, 5, actor_cast<actor>(self));
 
         self->mail(start_atom_v).send(solver);
         self->receive(
             [&](std::vector<float> result) {
-                verify_solution("High Iteration Count Test", result, expected_high, 5e-2f);
+                verify_solution("High Iteration Count Test", result, expected_high, 1e-2f);
+            }
+        );
+    }
+
+    // Test 7: Real-world matrix from file (lnsp3937.bin)
+    {
+        std::string path = "/scratch/nqr159/matrix-collection/matrices/unsymmetric/lnsp3937.bin";
+        std::cout << "\n[INFO] Test 7: Loading real-world matrix " << path << "..." << std::endl;
+        
+        try {
+            LocalCSR A = load_binary_matrix_manual(path);
+            std::cout << "[INFO] Matrix Metadata: Rows=" << A.rows 
+                      << ", Cols=" << A.cols 
+                      << ", NNZ=" << A.nnz << std::endl;
+            
+            std::cout << "[INFO] First 5 matrix values: ";
+            for(int i=0; i<std::min(5, (int)A.values.size()); ++i) 
+                std::cout << A.values[i] << " ";
+            std::cout << std::endl;
+            
+            std::vector<float> expected_real(A.rows, 1.0f);
+            std::vector<float> b_real = compute_rhs_manual(A, expected_real);
+            std::vector<float> x_real(A.rows, 0.0f);
+
+            auto solver = sys.spawn<sparse_bicgstab_actor>(
+                create_in_arg(A.row_ptr), create_in_arg(A.col_ind), create_in_arg(A.values),
+                create_in_arg(b_real), create_in_out_arg(x_real),
+                matrix_format::csr, A.rows, A.nnz, 1e-5f, 5000, 0, 6, actor_cast<actor>(self));
+
+            self->mail(start_atom_v).send(solver);
+            self->receive(
+                [&](std::vector<float> result) {
+                    verify_solution("Real Matrix (lnsp3937)", result, expected_real, 1e-2f);
+                }
+            );
+        } catch (const std::exception& e) {
+            std::cout << "[ERROR] Test 7 Failed: Could not load matrix file: " << e.what() << std::endl;
+        }
+    }
+
+    // Test 8: Stress Test - 1D Laplacian (N=10000)
+    {
+        int N_large = 10000;
+        std::cout << "\n[INFO] Test 8: Stress Test - Non-Symmetric Matrix (N=" << N_large << ")..." << std::endl;
+        
+        std::vector<int> row_ptr;
+        std::vector<int> col_ind;
+        std::vector<float> values;
+        row_ptr.push_back(0);
+        for(int i=0; i<N_large; ++i) {
+            if(i > 0) { col_ind.push_back(i-1); values.push_back(-1.5f); } // Lower
+            col_ind.push_back(i); values.push_back(4.0f);                 // Diag
+            if(i < N_large-1) { col_ind.push_back(i+1); values.push_back(-0.5f); } // Upper
+            row_ptr.push_back(col_ind.size());
+        }
+
+        std::vector<float> b_large(N_large, 1.0f);
+        std::vector<float> x_large(N_large, 0.0f);
+        
+        auto start = std::chrono::high_resolution_clock::now();
+        auto stress_solver = sys.spawn<sparse_bicgstab_actor>(
+            create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+            create_in_arg(b_large), create_in_out_arg(x_large),
+            matrix_format::csr, N_large, (int)values.size(), 1e-4f, 20000, 0, 7, actor_cast<actor>(self));
+
+        self->mail(start_atom_v).send(stress_solver);
+        self->receive(
+            [&](std::vector<float> result) {
+                auto end = std::chrono::high_resolution_clock::now();
+                std::chrono::duration<double> elapsed = end - start;
+                std::cout << "[SUCCESS] Stress Test completed in " << elapsed.count() << " seconds." << std::endl;
+                std::cout << "[INFO] First 5 elements of solution: ";
+                for(int i=0; i<5; ++i) std::cout << result[i] << " ";
+                std::cout << "..." << std::endl;
             }
         );
     }

From d9bbabb68d222df9bb355d3a20ca075fc23e53f6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 13:22:35 -0600
Subject: [PATCH 0753/1000] updated methods to take in dloat or double

---
 .../sparse-BiCGSTAB-actor.hpp                 | 173 +++++---
 libcaf_cuda/caf/cuda/device.hpp               | 392 ++++++++++++++----
 .../BICGSTAB-actor-test/main.test.cpp         |  66 ++-
 3 files changed, 455 insertions(+), 176 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
index 1753d9c08a..3ca020cb6f 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
@@ -13,15 +13,16 @@
 
 namespace caf::cuda {
 
+template <class T = float>
 struct sparse_bicgstab_state {
   // Host Data
   in<int> h_row_ptr, h_col_ind;
-  in<float> h_values, h_b;
-  in_out<float> h_x;
+  in<T> h_values, h_b;
+  in_out<T> h_x;
  
   // Device Problem data
   mem_ptr<int> A_row_ptr, A_col_ind;
-  mem_ptr<float> A_values, b, x;
+  mem_ptr<T> A_values, b, x;
   matrix_format format;
   int n, nnz;
   float tol;
@@ -32,40 +33,41 @@ struct sparse_bicgstab_state {
   caf::actor supervisor;
 
   // Workspace vectors
-  mem_ptr<float> r, r_hat, p, v, s_vec, t_vec, y_tmp;
+  mem_ptr<T> r, r_hat, p, v, s_vec, t_vec, y_tmp;
   mem_ptr<char> spmv_workspace;
   
   // Scalars needed across asynchronous steps
-  float rho_val = 1.0f; // Renamed from rho to avoid conflict with step-specific rho_new
-  float alpha_val = 1.0f;
-  float omega_val = 1.0f;
-  float beta_val = 0.0f;
+  T rho_val = T{1}; // Renamed from rho to avoid conflict with step-specific rho_new
+  T alpha_val = T{1};
+  T omega_val = T{1};
+  T beta_val = T{0};
   int iterations = 0;
 };
 
-class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
+template <class T = float>
+class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state<T>> {
 public:
   sparse_bicgstab_actor(actor_config& cfg, in<int> rp, in<int> ci,
-                        in<float> val, in<float> b, in_out<float> x,
+                        in<T> val, in<T> b, in_out<T> x,
                         matrix_format fmt, int n, int nnz, float tol, int max_iter, int device_num, int stream_id,
                         caf::actor supervisor = nullptr)
-    : stateful_actor<sparse_bicgstab_state>(cfg) {
-    state().h_row_ptr = std::move(rp);
-    state().h_col_ind = std::move(ci);
-    state().h_values = std::move(val);
-    state().h_b = std::move(b);
-    state().h_x = std::move(x);
-    state().format = fmt;
-    state().n = n; state().nnz = nnz;
-    state().tol = tol; state().max_iter = max_iter;
-    state().device_num = device_num; state().stream_id = stream_id;
-    state().supervisor = supervisor;
+    : stateful_actor<sparse_bicgstab_state<T>>(cfg) {
+    this->state().h_row_ptr = std::move(rp);
+    this->state().h_col_ind = std::move(ci);
+    this->state().h_values = std::move(val);
+    this->state().h_b = std::move(b);
+    this->state().h_x = std::move(x);
+    this->state().format = fmt;
+    this->state().n = n; this->state().nnz = nnz;
+    this->state().tol = tol; this->state().max_iter = max_iter;
+    this->state().device_num = device_num; this->state().stream_id = stream_id;
+    this->state().supervisor = supervisor;
   }
 
   behavior make_behavior() override {
     return {
       [this](start_atom) {
-        auto& s = state();
+        auto& s = this->state();
         if (!s.supervisor)
           s.supervisor = actor_cast<caf::actor>(this->current_sender());
         start_solve();
@@ -74,7 +76,7 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
   }
 private:
   void start_solve() {
-    auto& s = state();
+    auto& s = this->state();
     command_runner<> runner;
 
     // Transfer problem data to device
@@ -91,15 +93,15 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
     s.d_ptr = platform::create()->schedule(s.stream_id, s.device_num);
 
     // Allocate workspace
-    command_runner<out<float>> work_runner;
-    s.r     = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-    s.r_hat = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-    s.p     = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-    s.v     = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-    s.s_vec = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-    s.t_vec = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-    // Bottleneck Fix: Allocate only 1 float for scalar results
-    s.y_tmp = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(1));
+    command_runner<out<T>> work_runner;
+    s.r     = work_runner.transfer_memory(s.device_num, s.stream_id, out<T>(s.n));
+    s.r_hat = work_runner.transfer_memory(s.device_num, s.stream_id, out<T>(s.n));
+    s.p     = work_runner.transfer_memory(s.device_num, s.stream_id, out<T>(s.n));
+    s.v     = work_runner.transfer_memory(s.device_num, s.stream_id, out<T>(s.n));
+    s.s_vec = work_runner.transfer_memory(s.device_num, s.stream_id, out<T>(s.n));
+    s.t_vec = work_runner.transfer_memory(s.device_num, s.stream_id, out<T>(s.n));
+    // Bottleneck Fix: Allocate only 1 scalar for results
+    s.y_tmp = work_runner.transfer_memory(s.device_num, s.stream_id, out<T>(1));
 
     // Allocate SPMV workspace to avoid reallocations in the loop
     size_t ws_size = 0;
@@ -117,38 +119,38 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
 
     // 1. Initial Residual: r = b - Ax
     execute_spmv(s.x, s.v); 
-    s.d_ptr->scopy(s.stream_id, s.n, s.b, s.r);
-    s.d_ptr->saxpy(s.stream_id, s.n, -1.0f, s.v, s.r); // r = b - Ax
+    execute_copy(s.b, s.r);
+    execute_axpy(T{-1}, s.v, s.r); // r = b - Ax
     
     // 2. Choose r_hat = r
-    s.d_ptr->scopy(s.stream_id, s.n, s.r, s.r_hat);
+    execute_copy(s.r, s.r_hat);
 
     // 3. Initial norm calculation
-    s.d_ptr->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
-    float norm_sq = s.y_tmp->copy_to_host()[0];
+    execute_dot(s.r, s.r, s.y_tmp);
+    T norm_sq = s.y_tmp->copy_to_host()[0];
 
     s.iterations = 0;
-    s.rho_val = 1.0f;
-    s.alpha_val = 1.0f;
-    s.omega_val = 1.0f;
+    s.rho_val = T{1};
+    s.alpha_val = T{1};
+    s.omega_val = T{1};
 
     // BiCGSTAB Loop
     while (norm_sq > (s.tol * s.tol) && s.iterations < s.max_iter) {
       s.iterations++;
 
       // rho_new = <r_hat, r>
-      s.d_ptr->sdot(s.stream_id, s.n, s.r_hat, s.r, s.y_tmp);
-      float rho_new = s.y_tmp->copy_to_host()[0];
+      execute_dot(s.r_hat, s.r, s.y_tmp);
+      T rho_new = s.y_tmp->copy_to_host()[0];
 
       if (s.iterations == 1) {
-        s.d_ptr->scopy(s.stream_id, s.n, s.r, s.p);
+        execute_copy(s.r, s.p);
       } else {
         s.beta_val = (rho_new / s.rho_val) * (s.alpha_val / s.omega_val);
         // p = r + beta * (p - omega * v)
-        s.d_ptr->saxpy(s.stream_id, s.n, -s.omega_val, s.v, s.p);
-        s.d_ptr->scopy(s.stream_id, s.n, s.r, s.s_vec);
-        s.d_ptr->saxpy(s.stream_id, s.n, s.beta_val, s.p, s.s_vec);
-        s.d_ptr->scopy(s.stream_id, s.n, s.s_vec, s.p);
+        execute_axpy(-s.omega_val, s.v, s.p);
+        execute_copy(s.r, s.s_vec);
+        execute_axpy(s.beta_val, s.p, s.s_vec);
+        execute_copy(s.s_vec, s.p);
       }
       s.rho_val = rho_new;
 
@@ -156,34 +158,34 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
       execute_spmv(s.p, s.v);
 
       // alpha = rho / <r_hat, v>
-      s.d_ptr->sdot(s.stream_id, s.n, s.r_hat, s.v, s.y_tmp);
-      float alpha_denom = s.y_tmp->copy_to_host()[0];
+      execute_dot(s.r_hat, s.v, s.y_tmp);
+      T alpha_denom = s.y_tmp->copy_to_host()[0];
       s.alpha_val = s.rho_val / alpha_denom;
 
       // s = r - alpha * v
-      s.d_ptr->scopy(s.stream_id, s.n, s.r, s.s_vec);
-      s.d_ptr->saxpy(s.stream_id, s.n, -s.alpha_val, s.v, s.s_vec);
+      execute_copy(s.r, s.s_vec);
+      execute_axpy(-s.alpha_val, s.v, s.s_vec);
 
       // t = As
       execute_spmv(s.s_vec, s.t_vec);
 
       // omega = <t, s> / <t, t>
-      s.d_ptr->sdot(s.stream_id, s.n, s.t_vec, s.s_vec, s.y_tmp);
-      float omega_num = s.y_tmp->copy_to_host()[0];
-      s.d_ptr->sdot(s.stream_id, s.n, s.t_vec, s.t_vec, s.y_tmp);
-      float omega_denom = s.y_tmp->copy_to_host()[0];
+      execute_dot(s.t_vec, s.s_vec, s.y_tmp);
+      T omega_num = s.y_tmp->copy_to_host()[0];
+      execute_dot(s.t_vec, s.t_vec, s.y_tmp);
+      T omega_denom = s.y_tmp->copy_to_host()[0];
       s.omega_val = omega_num / omega_denom;
 
       // x = x + alpha*p + omega*s
-      s.d_ptr->saxpy(s.stream_id, s.n, s.alpha_val, s.p, s.x);
-      s.d_ptr->saxpy(s.stream_id, s.n, s.omega_val, s.s_vec, s.x);
+      execute_axpy(s.alpha_val, s.p, s.x);
+      execute_axpy(s.omega_val, s.s_vec, s.x);
 
       // r = s - omega*t
-      s.d_ptr->scopy(s.stream_id, s.n, s.s_vec, s.r);
-      s.d_ptr->saxpy(s.stream_id, s.n, -s.omega_val, s.t_vec, s.r);
+      execute_copy(s.s_vec, s.r);
+      execute_axpy(-s.omega_val, s.t_vec, s.r);
 
       // check convergence: norm_sq = <r, r>
-      s.d_ptr->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
+      execute_dot(s.r, s.r, s.y_tmp);
       norm_sq = s.y_tmp->copy_to_host()[0];
     }
 
@@ -191,23 +193,60 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state> {
   }
 
   void finish_solve() {
-    auto& s = state();
+    auto& s = this->state();
     // Copy result back to host and notify supervisor
-    std::vector<float> solution = s.x->copy_to_host();
+    std::vector<T> solution = s.x->copy_to_host();
     if (s.supervisor)
       this->mail(std::move(solution)).send(s.supervisor);
     this->quit();
   }
 
-  void execute_spmv(mem_ptr<float> input_v, mem_ptr<float> output_v) {
-    auto& s = state();
+  void execute_spmv(mem_ptr<T> input_v, mem_ptr<T> output_v) {
+    auto& s = this->state();
     switch (s.format) {
-      case matrix_format::csr: s.d_ptr->spmv_csr(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v, s.spmv_workspace); break;
-      case matrix_format::csc: s.d_ptr->spmv_csc(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v, s.spmv_workspace); break;
-      case matrix_format::coo: s.d_ptr->spmv_coo(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v, s.spmv_workspace); break;
+      case matrix_format::csr: s.d_ptr->spmv_csr(s.stream_id, s.n, s.n, s.nnz, T{1}, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, T{0}, output_v, s.spmv_workspace); break;
+      case matrix_format::csc: s.d_ptr->spmv_csc(s.stream_id, s.n, s.n, s.nnz, T{1}, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, T{0}, output_v, s.spmv_workspace); break;
+      case matrix_format::coo: s.d_ptr->spmv_coo(s.stream_id, s.n, s.n, s.nnz, T{1}, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, T{0}, output_v, s.spmv_workspace); break;
       default: break;
     }
   }
+
+  // Precision-aware dispatch helpers
+  void execute_copy(mem_ptr<T> src, mem_ptr<T> dst) {
+    auto& s = this->state();
+    if constexpr (std::is_same_v<T, double>) {
+      s.d_ptr->dcopy(s.stream_id, s.n, src, dst);
+    } else {
+      s.d_ptr->scopy(s.stream_id, s.n, src, dst);
+    }
+  }
+
+  void execute_axpy(T alpha, mem_ptr<T> x, mem_ptr<T> y) {
+    auto& s = this->state();
+    if constexpr (std::is_same_v<T, double>) {
+      s.d_ptr->daxpy(s.stream_id, s.n, alpha, x, y);
+    } else {
+      s.d_ptr->saxpy(s.stream_id, s.n, static_cast<float>(alpha), x, y);
+    }
+  }
+
+  void execute_dot(mem_ptr<T> x, mem_ptr<T> y, mem_ptr<T> res) {
+    auto& s = this->state();
+    if constexpr (std::is_same_v<T, double>) {
+      s.d_ptr->ddot(s.stream_id, s.n, x, y, res);
+    } else {
+      s.d_ptr->sdot(s.stream_id, s.n, x, y, res);
+    }
+  }
+
+  void execute_nrm2(mem_ptr<T> x, mem_ptr<T> res) {
+    auto& s = this->state();
+    if constexpr (std::is_same_v<T, double>) {
+      s.d_ptr->dnrm2(s.stream_id, s.n, x, res);
+    } else {
+      s.d_ptr->snrm2(s.stream_id, s.n, x, res);
+    }
+  }
 };
 
 } // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index f2ca87cfd9..3241292767 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -174,23 +174,30 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Returns the required buffer size for SpMV CSR.
+  template <class T>
   size_t spmv_csr_buffer_size(int stream_id, int m, int n, int nnz, 
-                             mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> values,
-                             mem_ptr<float> x, mem_ptr<float> y) {
+                             mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<T> values,
+                             mem_ptr<T> x, mem_ptr<T> y) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle) throw std::runtime_error("cuSparse not enabled");
     CHECK_CUDA(cuCtxPushCurrent(context_));
-    float alpha = 1.0f; float beta = 0.0f;
+    T alpha = T{1}; T beta = T{0};
+    cudaDataType type;
+    if constexpr (std::is_same_v<T, double>)
+      type = CUDA_R_64F;
+    else
+      type = CUDA_R_32F;
+
     cusparseSpMatDescr_t matA;
     cusparseCreateCsr(&matA, m, n, nnz, reinterpret_cast<void*>(row_ptr->mem()),
                       reinterpret_cast<void*>(col_ind->mem()), reinterpret_cast<void*>(values->mem()),
-                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, type);
     cusparseDnVecDescr_t vecX, vecY;
-    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
-    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
+    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), type);
+    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), type);
     size_t bufferSize = 0;
     cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecY,
-                            CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
+                            type, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
     cusparseDestroySpMat(matA);
     cusparseDestroyDnVec(vecX);
     cusparseDestroyDnVec(vecY);
@@ -199,23 +206,30 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Returns the required buffer size for SpMV COO.
+  template <class T>
   size_t spmv_coo_buffer_size(int stream_id, int m, int n, int nnz, 
-                             mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> values,
-                             mem_ptr<float> x, mem_ptr<float> y) {
+                             mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<T> values,
+                             mem_ptr<T> x, mem_ptr<T> y) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle) throw std::runtime_error("cuSparse not enabled");
     CHECK_CUDA(cuCtxPushCurrent(context_));
-    float alpha = 1.0f; float beta = 0.0f;
+    T alpha = T{1}; T beta = T{0};
+    cudaDataType type;
+    if constexpr (std::is_same_v<T, double>)
+      type = CUDA_R_64F;
+    else
+      type = CUDA_R_32F;
+
     cusparseSpMatDescr_t matA;
     cusparseCreateCoo(&matA, m, n, nnz, reinterpret_cast<void*>(row_ind->mem()),
                       reinterpret_cast<void*>(col_ind->mem()), reinterpret_cast<void*>(values->mem()),
-                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, type);
     cusparseDnVecDescr_t vecX, vecY;
-    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
-    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
+    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), type);
+    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), type);
     size_t bufferSize = 0;
     cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecY,
-                            CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
+                            type, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
     cusparseDestroySpMat(matA);
     cusparseDestroyDnVec(vecX);
     cusparseDestroyDnVec(vecY);
@@ -224,23 +238,30 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Returns the required buffer size for SpMV CSC.
+  template <class T>
   size_t spmv_csc_buffer_size(int stream_id, int m, int n, int nnz, 
-                             mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> values,
-                             mem_ptr<float> x, mem_ptr<float> y) {
+                             mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<T> values,
+                             mem_ptr<T> x, mem_ptr<T> y) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle) throw std::runtime_error("cuSparse not enabled");
     CHECK_CUDA(cuCtxPushCurrent(context_));
-    float alpha = 1.0f; float beta = 0.0f;
+    T alpha = T{1}; T beta = T{0};
+    cudaDataType type;
+    if constexpr (std::is_same_v<T, double>)
+      type = CUDA_R_64F;
+    else
+      type = CUDA_R_32F;
+
     cusparseSpMatDescr_t matA;
     cusparseCreateCsc(&matA, m, n, nnz, reinterpret_cast<void*>(col_ptr->mem()),
                       reinterpret_cast<void*>(row_ind->mem()), reinterpret_cast<void*>(values->mem()),
-                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, type);
     cusparseDnVecDescr_t vecX, vecY;
-    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
-    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
+    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), type);
+    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), type);
     size_t bufferSize = 0;
     cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecY,
-                            CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
+                            type, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
     cusparseDestroySpMat(matA);
     cusparseDestroyDnVec(vecX);
     cusparseDestroyDnVec(vecY);
@@ -249,14 +270,21 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Performs sparse matrix-vector multiplication (y = alpha*A*x + beta*y) using CSR format.
-  void spmv_csr(int stream_id, int m, int n, int nnz, float alpha, 
-                mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> values,
-                mem_ptr<float> x, float beta, mem_ptr<float> y, mem_ptr<char> workspace = nullptr) {
+  template <class T>
+  void spmv_csr(int stream_id, int m, int n, int nnz, T alpha, 
+                mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<T> values,
+                mem_ptr<T> x, T beta, mem_ptr<T> y, mem_ptr<char> workspace = nullptr) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle)
       throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
 
     CHECK_CUDA(cuCtxPushCurrent(context_));
+    cudaDataType type;
+    if constexpr (std::is_same_v<T, double>)
+      type = CUDA_R_64F;
+    else
+      type = CUDA_R_32F;
+
     CUstream stream = get_stream_for_actor(stream_id);
 
     cusparseSpMatDescr_t matA;
@@ -265,11 +293,11 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
                       reinterpret_cast<void*>(col_ind->mem()),
                       reinterpret_cast<void*>(values->mem()),
                       CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
-                      CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+                      CUSPARSE_INDEX_BASE_ZERO, type);
 
     cusparseDnVecDescr_t vecX, vecY;
-    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
-    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
+    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), type);
+    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), type);
 
     void* d_workspace = nullptr;
     CUdeviceptr dBuffer = 0;
@@ -278,7 +306,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     } else {
       size_t bufferSize = 0;
       cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                              &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                              &alpha, matA, vecX, &beta, vecY, type,
                               CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
       if (bufferSize > 0) {
         CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
@@ -287,7 +315,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     }
 
     cusparseStatus_t status = cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                                           &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                                           &alpha, matA, vecX, &beta, vecY, type,
                                            CUSPARSE_SPMV_ALG_DEFAULT, d_workspace);
 
     if (dBuffer)
@@ -303,14 +331,21 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Performs sparse matrix-vector multiplication (y = alpha*A*x + beta*y) using COO format.
-  void spmv_coo(int stream_id, int m, int n, int nnz, float alpha, 
-                mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> values,
-                mem_ptr<float> x, float beta, mem_ptr<float> y, mem_ptr<char> workspace = nullptr) {
+  template <class T>
+  void spmv_coo(int stream_id, int m, int n, int nnz, T alpha, 
+                mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<T> values,
+                mem_ptr<T> x, T beta, mem_ptr<T> y, mem_ptr<char> workspace = nullptr) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle)
       throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
 
     CHECK_CUDA(cuCtxPushCurrent(context_));
+    cudaDataType type;
+    if constexpr (std::is_same_v<T, double>)
+      type = CUDA_R_64F;
+    else
+      type = CUDA_R_32F;
+
     CUstream stream = get_stream_for_actor(stream_id);
 
     cusparseSpMatDescr_t matA;
@@ -318,11 +353,11 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
                       reinterpret_cast<void*>(row_ind->mem()),
                       reinterpret_cast<void*>(col_ind->mem()),
                       reinterpret_cast<void*>(values->mem()),
-                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, type);
 
     cusparseDnVecDescr_t vecX, vecY;
-    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
-    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
+    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), type);
+    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), type);
 
     void* d_workspace = nullptr;
     CUdeviceptr dBuffer = 0;
@@ -331,7 +366,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     } else {
       size_t bufferSize = 0;
       cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                              &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                              &alpha, matA, vecX, &beta, vecY, type,
                               CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
       if (bufferSize > 0) {
         CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
@@ -340,7 +375,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     }
 
     cusparseStatus_t status = cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                                           &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                                           &alpha, matA, vecX, &beta, vecY, type,
                                            CUSPARSE_SPMV_ALG_DEFAULT, d_workspace);
 
     if (dBuffer)
@@ -356,14 +391,21 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Performs sparse matrix-vector multiplication (y = alpha*A*x + beta*y) using CSC format.
-  void spmv_csc(int stream_id, int m, int n, int nnz, float alpha, 
-                mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> values,
-                mem_ptr<float> x, float beta, mem_ptr<float> y, mem_ptr<char> workspace = nullptr) {
+  template <class T>
+  void spmv_csc(int stream_id, int m, int n, int nnz, T alpha, 
+                mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<T> values,
+                mem_ptr<T> x, T beta, mem_ptr<T> y, mem_ptr<char> workspace = nullptr) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle)
       throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
 
     CHECK_CUDA(cuCtxPushCurrent(context_));
+    cudaDataType type;
+    if constexpr (std::is_same_v<T, double>)
+      type = CUDA_R_64F;
+    else
+      type = CUDA_R_32F;
+
     CUstream stream = get_stream_for_actor(stream_id);
 
     cusparseSpMatDescr_t matA;
@@ -372,11 +414,11 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
                       reinterpret_cast<void*>(row_ind->mem()),
                       reinterpret_cast<void*>(values->mem()),
                       CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
-                      CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+                      CUSPARSE_INDEX_BASE_ZERO, type);
 
     cusparseDnVecDescr_t vecX, vecY;
-    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), CUDA_R_32F);
-    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), CUDA_R_32F);
+    cusparseCreateDnVec(&vecX, n, reinterpret_cast<void*>(x->mem()), type);
+    cusparseCreateDnVec(&vecY, m, reinterpret_cast<void*>(y->mem()), type);
 
     void* d_workspace = nullptr;
     CUdeviceptr dBuffer = 0;
@@ -385,7 +427,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     } else {
       size_t bufferSize = 0;
       cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                              &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                              &alpha, matA, vecX, &beta, vecY, type,
                               CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
       if (bufferSize > 0) {
         CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
@@ -394,7 +436,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     }
 
     cusparseStatus_t status = cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                                           &alpha, matA, vecX, &beta, vecY, CUDA_R_32F,
+                                           &alpha, matA, vecX, &beta, vecY, type,
                                            CUSPARSE_SPMV_ALG_DEFAULT, d_workspace);
 
     if (dBuffer)
@@ -410,23 +452,30 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Returns the required buffer size for SpMM CSR.
+  template <class T>
   size_t spmm_csr_buffer_size(int stream_id, int m, int n, int k, int nnz, 
-                             mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> values,
-                             mem_ptr<float> B, mem_ptr<float> C) {
+                             mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<T> values,
+                             mem_ptr<T> B, mem_ptr<T> C) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle) throw std::runtime_error("cuSparse not enabled");
     CHECK_CUDA(cuCtxPushCurrent(context_));
-    float alpha = 1.0f; float beta = 0.0f;
+    T alpha = T{1}; T beta = T{0};
+    cudaDataType type;
+    if constexpr (std::is_same_v<T, double>)
+      type = CUDA_R_64F;
+    else
+      type = CUDA_R_32F;
+
     cusparseSpMatDescr_t matA;
     cusparseCreateCsr(&matA, m, k, nnz, reinterpret_cast<void*>(row_ptr->mem()),
                       reinterpret_cast<void*>(col_ind->mem()), reinterpret_cast<void*>(values->mem()),
-                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, type);
     cusparseDnMatDescr_t matB, matC;
-    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
-    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), type, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), type, CUSPARSE_ORDER_ROW);
     size_t bufferSize = 0;
     cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                            &alpha, matA, matB, &beta, matC, CUDA_R_32F, CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
+                            &alpha, matA, matB, &beta, matC, type, CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
     cusparseDestroySpMat(matA);
     cusparseDestroyDnMat(matB);
     cusparseDestroyDnMat(matC);
@@ -435,23 +484,30 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Returns the required buffer size for SpMM COO.
+  template <class T>
   size_t spmm_coo_buffer_size(int stream_id, int m, int n, int k, int nnz, 
-                             mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> values,
-                             mem_ptr<float> B, mem_ptr<float> C) {
+                             mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<T> values,
+                             mem_ptr<T> B, mem_ptr<T> C) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle) throw std::runtime_error("cuSparse not enabled");
     CHECK_CUDA(cuCtxPushCurrent(context_));
-    float alpha = 1.0f; float beta = 0.0f;
+    T alpha = T{1}; T beta = T{0};
+    cudaDataType type;
+    if constexpr (std::is_same_v<T, double>)
+      type = CUDA_R_64F;
+    else
+      type = CUDA_R_32F;
+
     cusparseSpMatDescr_t matA;
     cusparseCreateCoo(&matA, m, k, nnz, reinterpret_cast<void*>(row_ind->mem()),
                       reinterpret_cast<void*>(col_ind->mem()), reinterpret_cast<void*>(values->mem()),
-                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, type);
     cusparseDnMatDescr_t matB, matC;
-    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
-    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), type, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), type, CUSPARSE_ORDER_ROW);
     size_t bufferSize = 0;
     cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                            &alpha, matA, matB, &beta, matC, CUDA_R_32F, CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
+                            &alpha, matA, matB, &beta, matC, type, CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
     cusparseDestroySpMat(matA);
     cusparseDestroyDnMat(matB);
     cusparseDestroyDnMat(matC);
@@ -460,23 +516,30 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Returns the required buffer size for SpMM CSC.
+  template <class T>
   size_t spmm_csc_buffer_size(int stream_id, int m, int n, int k, int nnz, 
-                             mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> values,
-                             mem_ptr<float> B, mem_ptr<float> C) {
+                             mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<T> values,
+                             mem_ptr<T> B, mem_ptr<T> C) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle) throw std::runtime_error("cuSparse not enabled");
     CHECK_CUDA(cuCtxPushCurrent(context_));
-    float alpha = 1.0f; float beta = 0.0f;
+    T alpha = T{1}; T beta = T{0};
+    cudaDataType type;
+    if constexpr (std::is_same_v<T, double>)
+      type = CUDA_R_64F;
+    else
+      type = CUDA_R_32F;
+
     cusparseSpMatDescr_t matA;
     cusparseCreateCsc(&matA, m, k, nnz, reinterpret_cast<void*>(col_ptr->mem()),
                       reinterpret_cast<void*>(row_ind->mem()), reinterpret_cast<void*>(values->mem()),
-                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, type);
     cusparseDnMatDescr_t matB, matC;
-    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
-    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), type, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), type, CUSPARSE_ORDER_ROW);
     size_t bufferSize = 0;
     cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                            &alpha, matA, matB, &beta, matC, CUDA_R_32F, CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
+                            &alpha, matA, matB, &beta, matC, type, CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
     cusparseDestroySpMat(matA);
     cusparseDestroyDnMat(matB);
     cusparseDestroyDnMat(matC);
@@ -486,13 +549,20 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
 
   /// Performs sparse matrix-matrix multiplication (C = alpha*A*B + beta*C) using CSR format.
   /// A is sparse (m x k), B is dense (k x n), C is dense (m x n).
-  void spmm_csr(int stream_id, int m, int n, int k, int nnz, float alpha, 
-                mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<float> values,
-                mem_ptr<float> B, float beta, mem_ptr<float> C, mem_ptr<char> workspace = nullptr) {
+  template <class T>
+  void spmm_csr(int stream_id, int m, int n, int k, int nnz, T alpha, 
+                mem_ptr<int> row_ptr, mem_ptr<int> col_ind, mem_ptr<T> values,
+                mem_ptr<T> B, T beta, mem_ptr<T> C, mem_ptr<char> workspace = nullptr) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle) throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
 
     CHECK_CUDA(cuCtxPushCurrent(context_));
+    cudaDataType type;
+    if constexpr (std::is_same_v<T, double>)
+      type = CUDA_R_64F;
+    else
+      type = CUDA_R_32F;
+
     CUstream stream = get_stream_for_actor(stream_id);
 
     cusparseSpMatDescr_t matA;
@@ -501,11 +571,11 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
                       reinterpret_cast<void*>(col_ind->mem()),
                       reinterpret_cast<void*>(values->mem()),
                       CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
-                      CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+                      CUSPARSE_INDEX_BASE_ZERO, type);
 
     cusparseDnMatDescr_t matB, matC;
-    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
-    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), type, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), type, CUSPARSE_ORDER_ROW);
 
     void* d_workspace = nullptr;
     CUdeviceptr dBuffer = 0;
@@ -514,7 +584,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     } else {
       size_t bufferSize = 0;
       cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                              &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                              &alpha, matA, matB, &beta, matC, type,
                               CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
       if (bufferSize > 0) {
         CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
@@ -523,7 +593,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     }
 
     cusparseStatus_t status = cusparseSpMM(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                                           &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                                           &alpha, matA, matB, &beta, matC, type,
                                            CUSPARSE_SPMM_ALG_DEFAULT, d_workspace);
 
     if (dBuffer) CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
@@ -538,13 +608,20 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Performs sparse matrix-matrix multiplication (C = alpha*A*B + beta*C) using COO format.
-  void spmm_coo(int stream_id, int m, int n, int k, int nnz, float alpha, 
-                mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<float> values,
-                mem_ptr<float> B, float beta, mem_ptr<float> C, mem_ptr<char> workspace = nullptr) {
+  template <class T>
+  void spmm_coo(int stream_id, int m, int n, int k, int nnz, T alpha, 
+                mem_ptr<int> row_ind, mem_ptr<int> col_ind, mem_ptr<T> values,
+                mem_ptr<T> B, T beta, mem_ptr<T> C, mem_ptr<char> workspace = nullptr) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle) throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
 
     CHECK_CUDA(cuCtxPushCurrent(context_));
+    cudaDataType type;
+    if constexpr (std::is_same_v<T, double>)
+      type = CUDA_R_64F;
+    else
+      type = CUDA_R_32F;
+
     CUstream stream = get_stream_for_actor(stream_id);
 
     cusparseSpMatDescr_t matA;
@@ -552,11 +629,11 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
                       reinterpret_cast<void*>(row_ind->mem()),
                       reinterpret_cast<void*>(col_ind->mem()),
                       reinterpret_cast<void*>(values->mem()),
-                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, type);
 
     cusparseDnMatDescr_t matB, matC;
-    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
-    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), type, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), type, CUSPARSE_ORDER_ROW);
 
     void* d_workspace = nullptr;
     CUdeviceptr dBuffer = 0;
@@ -565,7 +642,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     } else {
       size_t bufferSize = 0;
       cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                              &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                              &alpha, matA, matB, &beta, matC, type,
                               CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
       if (bufferSize > 0) {
         CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
@@ -574,7 +651,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     }
 
     cusparseStatus_t status = cusparseSpMM(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                                           &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                                           &alpha, matA, matB, &beta, matC, type,
                                            CUSPARSE_SPMM_ALG_DEFAULT, d_workspace);
 
     if (dBuffer) CHECK_CUDA(cuMemFreeAsync(dBuffer, stream));
@@ -589,13 +666,20 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
   }
 
   /// Performs sparse matrix-matrix multiplication (C = alpha*A*B + beta*C) using CSC format.
-  void spmm_csc(int stream_id, int m, int n, int k, int nnz, float alpha, 
-                mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<float> values,
-                mem_ptr<float> B, float beta, mem_ptr<float> C, mem_ptr<char> workspace = nullptr) {
+  template <class T>
+  void spmm_csc(int stream_id, int m, int n, int k, int nnz, T alpha, 
+                mem_ptr<int> col_ptr, mem_ptr<int> row_ind, mem_ptr<T> values,
+                mem_ptr<T> B, T beta, mem_ptr<T> C, mem_ptr<char> workspace = nullptr) {
     cusparseHandle_t handle = get_cusparse_handle(stream_id);
     if (!handle) throw std::runtime_error("cuSparse not enabled on device " + std::to_string(id_));
 
     CHECK_CUDA(cuCtxPushCurrent(context_));
+    cudaDataType type;
+    if constexpr (std::is_same_v<T, double>)
+      type = CUDA_R_64F;
+    else
+      type = CUDA_R_32F;
+
     CUstream stream = get_stream_for_actor(stream_id);
 
     cusparseSpMatDescr_t matA;
@@ -604,11 +688,11 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
                       reinterpret_cast<void*>(row_ind->mem()),
                       reinterpret_cast<void*>(values->mem()),
                       CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
-                      CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F);
+                      CUSPARSE_INDEX_BASE_ZERO, type);
 
     cusparseDnMatDescr_t matB, matC;
-    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
-    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), CUDA_R_32F, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matB, k, n, n, reinterpret_cast<void*>(B->mem()), type, CUSPARSE_ORDER_ROW);
+    cusparseCreateDnMat(&matC, m, n, n, reinterpret_cast<void*>(C->mem()), type, CUSPARSE_ORDER_ROW);
 
     void* d_workspace = nullptr;
     CUdeviceptr dBuffer = 0;
@@ -617,7 +701,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     } else {
       size_t bufferSize = 0;
       cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                              &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                              &alpha, matA, matB, &beta, matC, type,
                               CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize);
       if (bufferSize > 0) {
         CHECK_CUDA(cuMemAllocAsync(&dBuffer, bufferSize, stream));
@@ -626,7 +710,7 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     }
 
     cusparseStatus_t status = cusparseSpMM(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                                           &alpha, matA, matB, &beta, matC, CUDA_R_32F,
+                                           &alpha, matA, matB, &beta, matC, type,
                                            CUSPARSE_SPMM_ALG_DEFAULT, d_workspace);
 
     if (dBuffer)
@@ -641,6 +725,34 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cusparseSpMM (CSC) failed on device " + std::to_string(id_));
   }
 
+  /// Performs double precision matrix-vector multiplication (y = alpha*A*x + beta*y).
+  /// Assumes A is in row-major order of dimensions m x n.
+  void dgemv(int stream_id, int m, int n, double alpha, mem_ptr<double> A,
+             mem_ptr<double> x, double beta, mem_ptr<double> y) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    
+    // Row-major matrix A (m x n) is stored as m rows of n elements.
+    // Viewed as column-major by cuBLAS, this is a n x m matrix.
+    // To compute y = A * x:
+    // Op(Memory) * x = (n x m)^T * (n x 1) = (m x n) * (n x 1) = (m x 1).
+    // We use CUBLAS_OP_T. LDA is the 'rows' in the column-major view, which is n.
+    cublasStatus_t status = cublasDgemv(handle, CUBLAS_OP_T, 
+                                        n, m, 
+                                        &alpha,
+                                        reinterpret_cast<const double*>(A->mem()), n,
+                                        reinterpret_cast<const double*>(x->mem()), 1,
+                                        &beta,
+                                        reinterpret_cast<double*>(y->mem()), 1);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasDgemv failed on device " + std::to_string(id_));
+  }
+
   /// Performs symmetric rank-k update (C = alpha*A*A^T + beta*C).
   /// Assumes A is in row-major order of dimensions n x k, and C is n x n.
   void ssyrk(int stream_id, int n, int k, float alpha, mem_ptr<float> A,
@@ -668,6 +780,33 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasSsyrk failed on device " + std::to_string(id_));
   }
 
+  /// Performs double precision symmetric rank-k update (C = alpha*A*A^T + beta*C).
+  /// Assumes A is in row-major order of dimensions n x k, and C is n x n.
+  void dsyrk(int stream_id, int n, int k, double alpha, mem_ptr<double> A,
+             double beta, mem_ptr<double> C) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    
+    // Row-major matrix A (n x k) viewed as column-major is k x n.
+    // To compute C = alpha * A * A^T + beta * C:
+    // We use CUBLAS_OP_T so that (k x n)^T * (k x n) = (n x k) * (k x n) = n x n.
+    // Note: We use CUBLAS_FILL_MODE_UPPER because the upper triangle in 
+    // column-major maps to the lower triangle in row-major layout.
+    cublasStatus_t status = cublasDsyrk(handle, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_T,
+                                        n, k,
+                                        &alpha,
+                                        reinterpret_cast<const double*>(A->mem()), k,
+                                        &beta,
+                                        reinterpret_cast<double*>(C->mem()), n);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasDsyrk failed on device " + std::to_string(id_));
+  }
+
   /// Performs single precision vector-vector addition (y = alpha*x + y).
   void saxpy(int stream_id, int n, float alpha, mem_ptr<float> x, mem_ptr<float> y) {
     cublasHandle_t handle = get_cublas_handle(stream_id);
@@ -685,6 +824,23 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasSaxpy failed on device " + std::to_string(id_));
   }
 
+  /// Performs double precision vector-vector addition (y = alpha*x + y).
+  void daxpy(int stream_id, int n, double alpha, mem_ptr<double> x, mem_ptr<double> y) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+
+    cublasStatus_t status = cublasDaxpy(handle, n, &alpha,
+                                        reinterpret_cast<const double*>(x->mem()), 1,
+                                        reinterpret_cast<double*>(y->mem()), 1);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasDaxpy failed on device " + std::to_string(id_));
+  }
+
   /// Performs single precision Euclidean norm (result = ||x||2).
   void snrm2(int stream_id, int n, mem_ptr<float> x, mem_ptr<float> result) {
     cublasHandle_t handle = get_cublas_handle(stream_id);
@@ -704,6 +860,25 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasSnrm2 failed on device " + std::to_string(id_));
   }
 
+  /// Performs double precision Euclidean norm (result = ||x||2).
+  void dnrm2(int stream_id, int n, mem_ptr<double> x, mem_ptr<double> result) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+
+    cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_DEVICE);
+    cublasStatus_t status = cublasDnrm2(handle, n, 
+                                        reinterpret_cast<const double*>(x->mem()), 1,
+                                        reinterpret_cast<double*>(result->mem()));
+    cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasDnrm2 failed on device " + std::to_string(id_));
+  }
+
   /// Performs single precision dot product (result = x^T * y).
   void sdot(int stream_id, int n, mem_ptr<float> x, mem_ptr<float> y, mem_ptr<float> result) {
     cublasHandle_t handle = get_cublas_handle(stream_id);
@@ -724,6 +899,26 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasSdot failed on device " + std::to_string(id_));
   }
 
+  /// Performs double precision dot product (result = x^T * y).
+  void ddot(int stream_id, int n, mem_ptr<double> x, mem_ptr<double> y, mem_ptr<double> result) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+
+    cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_DEVICE);
+    cublasStatus_t status = cublasDdot(handle, n, 
+                                        reinterpret_cast<const double*>(x->mem()), 1,
+                                        reinterpret_cast<const double*>(y->mem()), 1,
+                                        reinterpret_cast<double*>(result->mem()));
+    cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasDdot failed on device " + std::to_string(id_));
+  }
+
   /// Copies vector x to vector y (y = x).
   void scopy(int stream_id, int n, mem_ptr<float> x, mem_ptr<float> y) {
     cublasHandle_t handle = get_cublas_handle(stream_id);
@@ -741,6 +936,23 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasScopy failed on device " + std::to_string(id_));
   }
 
+  /// Copies double vector x to vector y (y = x).
+  void dcopy(int stream_id, int n, mem_ptr<double> x, mem_ptr<double> y) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+
+    cublasStatus_t status = cublasDcopy(handle, n, 
+                                         reinterpret_cast<const double*>(x->mem()), 1,
+                                         reinterpret_cast<double*>(y->mem()), 1);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasDcopy failed on device " + std::to_string(id_));
+  }
+
   /// Performs single precision matrix-matrix multiplication (C = alpha*A*B + beta*C).
   /// Assumes A is m x k, B is k x n, and C is m x n, all in row-major order.
   void sgemm(int stream_id, int m, int n, int k, float alpha, mem_ptr<float> A,
diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
index c15aff7dea..896a9df552 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
@@ -24,14 +24,16 @@ using namespace caf::cuda;
 
 // --- Manual Sparse Utilities for Testing ---
 
+template <class T = float>
 struct LocalCSR {
     int rows, cols, nnz;
     std::vector<int> row_ptr;
     std::vector<int> col_ind;
-    std::vector<float> values;
+    std::vector<T> values;
 };
 
-LocalCSR load_binary_matrix_manual(const std::string& path) {
+template <class T = float>
+LocalCSR<T> load_binary_matrix_manual(const std::string& path) {
     std::ifstream file(path, std::ios::binary);
     if (!file) throw std::runtime_error("Could not open " + path);
 
@@ -60,7 +62,7 @@ LocalCSR load_binary_matrix_manual(const std::string& path) {
     }
 
     // Convert COO to CSR
-    LocalCSR csr;
+    LocalCSR<T> csr;
     csr.rows = r; csr.cols = c; csr.nnz = n;
     csr.row_ptr.assign(r + 1, 0);
     csr.col_ind.resize(n);
@@ -74,15 +76,16 @@ LocalCSR load_binary_matrix_manual(const std::string& path) {
         int row = rows_coo[i];
         int dest = current_pos[row]++;
         csr.col_ind[dest] = cols_coo[i];
-        csr.values[dest] = vals_coo[i];
+        csr.values[dest] = static_cast<T>(vals_coo[i]);
     }
     return csr;
 }
 
-std::vector<float> compute_rhs_manual(const LocalCSR& A, const std::vector<float>& x) {
-    std::vector<float> b(A.rows, 0.0f);
+template <class T = float>
+std::vector<T> compute_rhs_manual(const LocalCSR<T>& A, const std::vector<T>& x) {
+    std::vector<T> b(A.rows, T{0});
     for (int i = 0; i < A.rows; ++i) {
-        float sum = 0.0f;
+        T sum = T{0};
         for (int j = A.row_ptr[i]; j < A.row_ptr[i+1]; ++j) {
             sum += A.values[j] * x[A.col_ind[j]];
         }
@@ -93,8 +96,9 @@ std::vector<float> compute_rhs_manual(const LocalCSR& A, const std::vector<float
 
 // --- End Manual Utilities ---
 
-void verify_solution(const std::string& test_name, const std::vector<float>& actual, 
-                     const std::vector<float>& expected, float tol = 1e-3) {
+template <class T = float>
+void verify_solution(const std::string& test_name, const std::vector<T>& actual, 
+                     const std::vector<T>& expected, T tol = 1e-3) {
     if (actual.size() != expected.size()) {
         std::cout << "[ERROR] " << test_name << " failed: Size mismatch (got " 
                   << actual.size() << ", expected " << expected.size() << ")" << std::endl;
@@ -137,7 +141,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto solver = sys.spawn<sparse_bicgstab_actor>(
+        auto solver = sys.spawn<sparse_bicgstab_actor<float>>(
             create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
             create_in_arg(h_b), create_in_out_arg(h_x),
             matrix_format::csr, n, nnz, tolerance, max_iter, 0, 0, actor_cast<actor>(self));
@@ -158,7 +162,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto solver = sys.spawn<sparse_bicgstab_actor>(
+        auto solver = sys.spawn<sparse_bicgstab_actor<float>>(
             create_in_arg(col_ptr), create_in_arg(row_ind), create_in_arg(values),
             create_in_arg(h_b), create_in_out_arg(h_x),
             matrix_format::csc, n, nnz, tolerance, max_iter, 0, 1, actor_cast<actor>(self));
@@ -179,7 +183,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto solver = sys.spawn<sparse_bicgstab_actor>(
+        auto solver = sys.spawn<sparse_bicgstab_actor<float>>(
             create_in_arg(row_ind), create_in_arg(col_ind), create_in_arg(values),
             create_in_arg(h_b), create_in_out_arg(h_x),
             matrix_format::coo, n, nnz, tolerance, max_iter, 0, 2, actor_cast<actor>(self));
@@ -206,7 +210,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> x_tri(3, 0.0f);
         std::vector<float> expected_tri = {1.0f, 1.0f, 1.0f};
 
-        auto solver = sys.spawn<sparse_bicgstab_actor>(
+        auto solver = sys.spawn<sparse_bicgstab_actor<float>>(
             create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
             create_in_arg(b_tri), create_in_out_arg(x_tri),
             matrix_format::csr, 3, 7, tolerance, max_iter, 0, 3, actor_cast<actor>(self));
@@ -252,7 +256,7 @@ void caf_main(actor_system& sys) {
 
         std::vector<float> x_mid(N_mid, 0.0f);
         
-        auto solver = sys.spawn<sparse_bicgstab_actor>(
+        auto solver = sys.spawn<sparse_bicgstab_actor<float>>(
             create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
             create_in_arg(b_mid), create_in_out_arg(x_mid),
             matrix_format::csr, N_mid, (int)values.size(), 1e-5f, 500, 0, 4, actor_cast<actor>(self));
@@ -301,7 +305,7 @@ void caf_main(actor_system& sys) {
 
         std::vector<float> x_high(N_high, 0.0f);
 
-        auto solver = sys.spawn<sparse_bicgstab_actor>(
+        auto solver = sys.spawn<sparse_bicgstab_actor<float>>(
             create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
             create_in_arg(b_high), create_in_out_arg(x_high),
             matrix_format::csr, N_high, (int)values.size(), 1e-6f, 15000, 0, 5, actor_cast<actor>(self));
@@ -320,7 +324,7 @@ void caf_main(actor_system& sys) {
         std::cout << "\n[INFO] Test 7: Loading real-world matrix " << path << "..." << std::endl;
         
         try {
-            LocalCSR A = load_binary_matrix_manual(path);
+            LocalCSR<float> A = load_binary_matrix_manual<float>(path);
             std::cout << "[INFO] Matrix Metadata: Rows=" << A.rows 
                       << ", Cols=" << A.cols 
                       << ", NNZ=" << A.nnz << std::endl;
@@ -331,10 +335,10 @@ void caf_main(actor_system& sys) {
             std::cout << std::endl;
             
             std::vector<float> expected_real(A.rows, 1.0f);
-            std::vector<float> b_real = compute_rhs_manual(A, expected_real);
+            std::vector<float> b_real = compute_rhs_manual<float>(A, expected_real);
             std::vector<float> x_real(A.rows, 0.0f);
 
-            auto solver = sys.spawn<sparse_bicgstab_actor>(
+            auto solver = sys.spawn<sparse_bicgstab_actor<float>>(
                 create_in_arg(A.row_ptr), create_in_arg(A.col_ind), create_in_arg(A.values),
                 create_in_arg(b_real), create_in_out_arg(x_real),
                 matrix_format::csr, A.rows, A.nnz, 1e-5f, 5000, 0, 6, actor_cast<actor>(self));
@@ -370,7 +374,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> x_large(N_large, 0.0f);
         
         auto start = std::chrono::high_resolution_clock::now();
-        auto stress_solver = sys.spawn<sparse_bicgstab_actor>(
+        auto stress_solver = sys.spawn<sparse_bicgstab_actor<float>>(
             create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
             create_in_arg(b_large), create_in_out_arg(x_large),
             matrix_format::csr, N_large, (int)values.size(), 1e-4f, 20000, 0, 7, actor_cast<actor>(self));
@@ -388,6 +392,30 @@ void caf_main(actor_system& sys) {
         );
     }
 
+    // Test 9: Double Precision Test (CSR)
+    {
+        std::cout << "\n[INFO] Test 9: Double precision CSR format..." << std::endl;
+        int n_d = 3, nnz_d = 3;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<double> values = {4.0, 3.0, 2.0};
+        std::vector<double> h_b = {8.0, 9.0, 2.0};
+        std::vector<double> h_x(n_d, 0.0);
+        std::vector<double> expected_d = {2.0, 3.0, 1.0};
+
+        auto solver = sys.spawn<sparse_bicgstab_actor<double>>(
+            create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+            create_in_arg(h_b), create_in_out_arg(h_x),
+            matrix_format::csr, n_d, nnz_d, 1e-10f, 100, 0, 8, actor_cast<actor>(self));
+
+        self->mail(start_atom_v).send(solver);
+        self->receive(
+            [&](std::vector<double> result_x) {
+                verify_solution<double>("Double CSR Simple", result_x, expected_d, 1e-9);
+            }
+        );
+    }
+
     manager::shutdown();
 }
 CAF_MAIN(id_block::cuda, id_block::cg_actor)

From fb1cfc02ff9a627f5394223607084010ba0658f9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 14:10:33 -0600
Subject: [PATCH 0754/1000] updated tests

---
 .../BICGSTAB-actor-test/main.test.cpp         | 51 +++++++++++++++----
 1 file changed, 40 insertions(+), 11 deletions(-)

diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
index 896a9df552..03b235da5c 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
@@ -319,21 +319,16 @@ void caf_main(actor_system& sys) {
     }
 
     // Test 7: Real-world matrix from file (lnsp3937.bin)
+    {
+    // Test 7a: Real-world matrix (lnsp3937.bin) - Float Precision
     {
         std::string path = "/scratch/nqr159/matrix-collection/matrices/unsymmetric/lnsp3937.bin";
-        std::cout << "\n[INFO] Test 7: Loading real-world matrix " << path << "..." << std::endl;
-        
+        std::cout << "\n[INFO] Test 7a: Loading real-world matrix (Float) " << path << "..." << std::endl;
         try {
             LocalCSR<float> A = load_binary_matrix_manual<float>(path);
-            std::cout << "[INFO] Matrix Metadata: Rows=" << A.rows 
-                      << ", Cols=" << A.cols 
+            std::cout << "[INFO] Matrix Metadata: Rows=" << A.rows << ", Cols=" << A.cols 
                       << ", NNZ=" << A.nnz << std::endl;
             
-            std::cout << "[INFO] First 5 matrix values: ";
-            for(int i=0; i<std::min(5, (int)A.values.size()); ++i) 
-                std::cout << A.values[i] << " ";
-            std::cout << std::endl;
-            
             std::vector<float> expected_real(A.rows, 1.0f);
             std::vector<float> b_real = compute_rhs_manual<float>(A, expected_real);
             std::vector<float> x_real(A.rows, 0.0f);
@@ -346,14 +341,48 @@ void caf_main(actor_system& sys) {
             self->mail(start_atom_v).send(solver);
             self->receive(
                 [&](std::vector<float> result) {
-                    verify_solution("Real Matrix (lnsp3937)", result, expected_real, 1e-2f);
+                    verify_solution("Real Matrix (Float)", result, expected_real, 1e-2f);
                 }
             );
         } catch (const std::exception& e) {
-            std::cout << "[ERROR] Test 7 Failed: Could not load matrix file: " << e.what() << std::endl;
+            std::cout << "[ERROR] Test 7a Failed: " << e.what() << std::endl;
         }
     }
 
+    // Test 7b: Real-world matrix (lnsp3937.bin) - Double Precision
+    {
+        std::string path = "/scratch/nqr159/matrix-collection/matrices/unsymmetric/lnsp3937.bin";
+        std::cout << "\n[INFO] Test 7b: Loading real-world matrix (Double) " << path << "..." << std::endl;
+        try {
+            LocalCSR<double> A = load_binary_matrix_manual<double>(path);
+            std::cout << "[INFO] Matrix Metadata: Rows=" << A.rows << ", Cols=" << A.cols 
+                      << ", NNZ=" << A.nnz << std::endl;
+            std::cout << "[INFO] First 5 matrix values: ";
+            for(int i=0; i<std::min(5, (int)A.values.size()); ++i)
+                std::cout << A.values[i] << " ";
+            std::cout << std::endl;
+
+            std::vector<double> expected_real(A.rows, 1.0);
+            std::vector<double> b_real = compute_rhs_manual<double>(A, expected_real);
+            std::vector<double> x_real(A.rows, 0.0);
+
+            auto solver = sys.spawn<sparse_bicgstab_actor<double>>(
+                create_in_arg(A.row_ptr), create_in_arg(A.col_ind), create_in_arg(A.values),
+                create_in_arg(b_real), create_in_out_arg(x_real),
+                matrix_format::csr, A.rows, A.nnz, 1e-10, 5000, 0, 7, actor_cast<actor>(self));
+
+            self->mail(start_atom_v).send(solver);
+            self->receive(
+                [&](std::vector<double> result) {
+                    verify_solution<double>("Real Matrix (Double)", result, expected_real, 1e-8);
+                }
+            );
+        } catch (const std::exception& e) {
+            std::cout << "[ERROR] Test 7b Failed: " << e.what() << std::endl;
+        }
+    }
+    }
+
     // Test 8: Stress Test - 1D Laplacian (N=10000)
     {
         int N_large = 10000;

From 48c3112cdf0310763a77bfc43913dca7704f9e38 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 14:24:16 -0600
Subject: [PATCH 0755/1000] added a facade bicgstab solver actor

---
 .../sparse-BiCGSTAB-actor.hpp                 | 128 ++++++++++++++++++
 libcaf_cuda/caf/cuda/global.hpp               |  18 +++
 .../BICGSTAB-actor-test/main.test.cpp         |  50 +++++++
 3 files changed, 196 insertions(+)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
index 3ca020cb6f..e18fcd59e4 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
@@ -249,4 +249,132 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state<T>> {
   }
 };
 
+/**
+ * A stateless (facade) variant of the BiCGSTAB solver.
+ * This actor can be reused for multiple solve requests. It receives all 
+ * solve parameters as a message and returns the solution vector to the sender.
+ */
+template <class T = float>
+class sparse_bicgstab_facade : public event_based_actor {
+public:
+  sparse_bicgstab_facade(actor_config& cfg) : event_based_actor(cfg) {}
+
+  behavior make_behavior() override {
+    return {
+      [this](in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+             matrix_format fmt, int n, int nnz, float tol, int max_iter,
+             int device_num, int stream_id) -> std::vector<T> {
+        command_runner<> runner;
+
+        // 1. Transfer problem data to device
+        auto res = runner.transfer_memory(device_num, stream_id,
+                                          rp, ci, val, b_in, x_in);
+
+        auto A_row_ptr = std::get<0>(res);
+        auto A_col_ind = std::get<1>(res);
+        auto A_values  = std::get<2>(res);
+        auto b         = std::get<3>(res);
+        auto x         = std::get<4>(res);
+
+        auto d_ptr = platform::create()->schedule(stream_id, device_num);
+
+        // 2. Allocate workspace
+        command_runner<out<T>> work_runner;
+        auto r     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+        auto r_hat = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+        auto p     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+        auto v     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+        auto s_vec = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+        auto t_vec = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+        auto y_tmp = work_runner.transfer_memory(device_num, stream_id, out<T>(1));
+
+        // 3. Setup SPMV workspace
+        mem_ptr<char> spmv_workspace;
+        size_t ws_size = 0;
+        if (fmt == matrix_format::csr)
+          ws_size = d_ptr->spmv_csr_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, v);
+        else if (fmt == matrix_format::csc)
+          ws_size = d_ptr->spmv_csc_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, v);
+        else if (fmt == matrix_format::coo)
+          ws_size = d_ptr->spmv_coo_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, v);
+
+        if (ws_size > 0) {
+          command_runner<out<char>> ws_runner;
+          spmv_workspace = ws_runner.transfer_memory(device_num, stream_id, out<char>(static_cast<int>(ws_size)));
+        }
+
+        // Helper lambdas for GPU operations
+        auto execute_spmv = [&](mem_ptr<T> input_v, mem_ptr<T> output_v) {
+          switch (fmt) {
+            case matrix_format::csr: d_ptr->spmv_csr(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, input_v, T{0}, output_v, spmv_workspace); break;
+            case matrix_format::csc: d_ptr->spmv_csc(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, input_v, T{0}, output_v, spmv_workspace); break;
+            case matrix_format::coo: d_ptr->spmv_coo(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, input_v, T{0}, output_v, spmv_workspace); break;
+            default: break;
+          }
+        };
+        auto execute_copy = [&](mem_ptr<T> src, mem_ptr<T> dst) {
+          if constexpr (std::is_same_v<T, double>) d_ptr->dcopy(stream_id, n, src, dst);
+          else d_ptr->scopy(stream_id, n, src, dst);
+        };
+        auto execute_axpy = [&](T alpha, mem_ptr<T> ax, mem_ptr<T> ay) {
+          if constexpr (std::is_same_v<T, double>) d_ptr->daxpy(stream_id, n, alpha, ax, ay);
+          else d_ptr->saxpy(stream_id, n, static_cast<float>(alpha), ax, ay);
+        };
+        auto execute_dot = [&](mem_ptr<T> dx, mem_ptr<T> dy, mem_ptr<T> dres) {
+          if constexpr (std::is_same_v<T, double>) d_ptr->ddot(stream_id, n, dx, dy, dres);
+          else d_ptr->sdot(stream_id, n, dx, dy, dres);
+        };
+
+        // 4. Initial Residual: r = b - Ax
+        execute_spmv(x, v); 
+        execute_copy(b, r);
+        execute_axpy(T{-1}, v, r);
+        execute_copy(r, r_hat);
+        execute_dot(r, r, y_tmp);
+        T norm_sq = y_tmp->copy_to_host()[0];
+
+        int iterations = 0;
+        T rho_val = T{1}, alpha_val = T{1}, omega_val = T{1}, beta_val = T{0};
+
+        // BiCGSTAB Loop
+        while (norm_sq > (tol * tol) && iterations < max_iter) {
+          iterations++;
+          execute_dot(r_hat, r, y_tmp);
+          T rho_new = y_tmp->copy_to_host()[0];
+
+          if (iterations == 1) {
+            execute_copy(r, p);
+          } else {
+            beta_val = (rho_new / rho_val) * (alpha_val / omega_val);
+            execute_axpy(-omega_val, v, p);
+            execute_copy(r, s_vec);
+            execute_axpy(beta_val, p, s_vec);
+            execute_copy(s_vec, p);
+          }
+          rho_val = rho_new;
+          execute_spmv(p, v);
+          execute_dot(r_hat, v, y_tmp);
+          T alpha_denom = y_tmp->copy_to_host()[0];
+          alpha_val = rho_val / alpha_denom;
+          execute_copy(r, s_vec);
+          execute_axpy(-alpha_val, v, s_vec);
+          execute_spmv(s_vec, t_vec);
+          execute_dot(t_vec, s_vec, y_tmp);
+          T omega_num = y_tmp->copy_to_host()[0];
+          execute_dot(t_vec, t_vec, y_tmp);
+          T omega_denom = y_tmp->copy_to_host()[0];
+          omega_val = omega_num / omega_denom;
+          execute_axpy(alpha_val, p, x);
+          execute_axpy(omega_val, s_vec, x);
+          execute_copy(s_vec, r);
+          execute_axpy(-omega_val, t_vec, r);
+          execute_dot(r, r, y_tmp);
+          norm_sq = y_tmp->copy_to_host()[0];
+        }
+        return x->copy_to_host();
+      }
+    };
+  }
+};
+
 } // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index f9fe03b11e..a2e0dfbaf6 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -158,6 +158,22 @@ bool inspect(Inspector& f, buffer_variant& x) {
   return f.apply(x);
 }
 
+namespace caf::cuda {
+
+// Serialization support for matrix_format
+template <class Inspector>
+bool inspect(Inspector& f, matrix_format& x) {
+  auto val = static_cast<int>(x);
+  if (f.apply(val)) {
+    if constexpr (Inspector::is_loading)
+      x = static_cast<matrix_format>(val);
+    return true;
+  }
+  return false;
+}
+
+} // namespace caf::cuda
+
 // Check CUDA errors macro
 #define CHECK_CUDA(call) \
     do { \
@@ -204,6 +220,7 @@ CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id)
   CAF_ADD_TYPE_ID(cuda,(caf::cuda::mem_ptr<float>))  
   CAF_ADD_TYPE_ID(cuda,(caf::cuda::mem_ptr<double>))  
   CAF_ADD_TYPE_ID(cuda,(caf::cuda::mem_ptr<char>))  
+  CAF_ADD_TYPE_ID(cuda, (caf::cuda::matrix_format))
   
   //atoms 
   CAF_ADD_ATOM(cuda, kernel_done_atom)
@@ -230,3 +247,4 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::nd_range)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::program_ptr)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(output_mapping)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<output_mapping>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::matrix_format)
diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
index 03b235da5c..cb6eb868d1 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
@@ -445,6 +445,56 @@ void caf_main(actor_system& sys) {
         );
     }
 
+    // Test 10: Facade Actor CSR Simple
+    {
+        std::cout << "\n[INFO] Test 10: Facade actor CSR simple matrix..." << std::endl;
+        int n_f = 3, nnz_f = 3;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_b = {8.0f, 9.0f, 2.0f};
+        std::vector<float> h_x(n_f, 0.0f);
+        std::vector<float> expected_f = {2.0f, 3.0f, 1.0f};
+
+        auto facade = sys.spawn<sparse_bicgstab_facade<float>>();
+
+        self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csr, n_f, nnz_f, tolerance, max_iter, 0, 9).send(facade);
+
+        self->receive(
+            [&](std::vector<float> result_x) {
+                verify_solution("Facade CSR Simple", result_x, expected_f, tolerance);
+            }
+        );
+    }
+
+    // Test 11: Facade Actor Real Matrix
+    {
+        std::string path = "/scratch/nqr159/matrix-collection/matrices/unsymmetric/lnsp3937.bin";
+        std::cout << "\n[INFO] Test 11: Facade actor real-world matrix " << path << "..." << std::endl;
+        try {
+            LocalCSR<float> A = load_binary_matrix_manual<float>(path);
+            std::vector<float> expected_real(A.rows, 1.0f);
+            std::vector<float> b_real = compute_rhs_manual<float>(A, expected_real);
+            std::vector<float> x_real(A.rows, 0.0f);
+
+            auto facade = sys.spawn<sparse_bicgstab_facade<float>>();
+
+            self->mail(create_in_arg(A.row_ptr), create_in_arg(A.col_ind), create_in_arg(A.values),
+                       create_in_arg(b_real), create_in_out_arg(x_real),
+                       matrix_format::csr, A.rows, A.nnz, 1e-5f, 5000, 0, 10).send(facade);
+
+            self->receive(
+                [&](std::vector<float> result) {
+                    verify_solution("Facade Real Matrix", result, expected_real, 1e-2f);
+                }
+            );
+        } catch (const std::exception& e) {
+            std::cout << "[ERROR] Test 11 Failed: " << e.what() << std::endl;
+        }
+    }
+
     manager::shutdown();
 }
 CAF_MAIN(id_block::cuda, id_block::cg_actor)

From f2fa5bc426c3333da9534f2f97607c3d37581f55 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 14:36:55 -0600
Subject: [PATCH 0756/1000] implemented facade actor for CGS solver

---
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 93 +++++++++++++++++++
 .../CGS-actor-test/main.test.cpp              | 24 ++++-
 2 files changed, 116 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index 2940d150c2..e4d24ae781 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -194,4 +194,97 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
  }
 };
 
+/**
+ * A stateless (facade) variant of the CG solver.
+ * This actor can be reused for multiple solve requests. It receives all 
+ * solve parameters as a message and returns the solution vector to the sender.
+ */
+class sparse_cg_facade : public event_based_actor {
+public:
+  sparse_cg_facade(actor_config& cfg) : event_based_actor(cfg) {}
+
+  behavior make_behavior() override {
+    return {
+      [this](in<int> rp, in<int> ci, in<float> val, in<float> b_in, in_out<float> x_in,
+             matrix_format fmt, int n, int nnz, float tol, int max_iter,
+             int device_num, int stream_id) -> std::vector<float> {
+        command_runner<float> runner;
+
+        // 1. Transfer problem data to device
+        auto res = runner.transfer_memory(device_num, stream_id,
+                                          rp, ci, val, b_in, x_in);
+
+        auto A_row_ptr = std::get<0>(res);
+        auto A_col_ind = std::get<1>(res);
+        auto A_values  = std::get<2>(res);
+        auto b         = std::get<3>(res);
+        auto x         = std::get<4>(res);
+
+        auto d_ptr = platform::create()->schedule(stream_id, device_num);
+
+        // 2. Allocate workspace
+        command_runner<out<float>> work_runner;
+        auto r     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
+        auto p     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
+        auto w     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
+        auto y_tmp = work_runner.transfer_memory(device_num, stream_id, create_out_arg_with_size<float>(1));
+
+        // Setup SPMV workspace
+        mem_ptr<char> spmv_workspace;
+        size_t ws_size = 0;
+        if (fmt == matrix_format::csr)
+          ws_size = d_ptr->spmv_csr_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, w);
+        else if (fmt == matrix_format::csc)
+          ws_size = d_ptr->spmv_csc_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, w);
+        else if (fmt == matrix_format::coo)
+          ws_size = d_ptr->spmv_coo_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, w);
+
+        if (ws_size > 0) {
+          command_runner<out<char>> ws_runner;
+          spmv_workspace = ws_runner.transfer_memory(device_num, stream_id, out<char>(static_cast<int>(ws_size)));
+        }
+
+        auto execute_spmv = [&](mem_ptr<float> input_v, mem_ptr<float> output_v) {
+          switch (fmt) {
+            case matrix_format::csr: d_ptr->spmv_csr(stream_id, n, n, nnz, 1.0f, A_row_ptr, A_col_ind, A_values, input_v, 0.0f, output_v, spmv_workspace); break;
+            case matrix_format::csc: d_ptr->spmv_csc(stream_id, n, n, nnz, 1.0f, A_row_ptr, A_col_ind, A_values, input_v, 0.0f, output_v, spmv_workspace); break;
+            case matrix_format::coo: d_ptr->spmv_coo(stream_id, n, n, nnz, 1.0f, A_row_ptr, A_col_ind, A_values, input_v, 0.0f, output_v, spmv_workspace); break;
+            default: break;
+          }
+        };
+
+        // 3. Initial Residual Setup: r = b - Ax
+        execute_spmv(x, w);
+        d_ptr->scopy(stream_id, n, b, r);
+        d_ptr->saxpy(stream_id, n, -1.0f, w, r);
+        d_ptr->sdot(stream_id, n, r, r, y_tmp);
+        float rho_val = runner.copy_to_host(y_tmp)[0];
+        float old_rho_val = 0.0f;
+        int iterations = 0;
+
+        while (rho_val > (tol * tol) && iterations < max_iter) {
+          iterations++;
+          if (iterations > 1) {
+            float beta_val = rho_val / old_rho_val;
+            d_ptr->scopy(stream_id, n, r, w);
+            d_ptr->saxpy(stream_id, n, beta_val, p, w);
+            d_ptr->scopy(stream_id, n, w, p);
+          } else {
+            d_ptr->scopy(stream_id, n, r, p);
+          }
+          execute_spmv(p, w);
+          d_ptr->sdot(stream_id, n, p, w, y_tmp);
+          float alpha_val = rho_val / runner.copy_to_host(y_tmp)[0];
+          d_ptr->saxpy(stream_id, n, alpha_val, p, x);
+          d_ptr->saxpy(stream_id, n, -alpha_val, w, r);
+          old_rho_val = rho_val;
+          d_ptr->sdot(stream_id, n, r, r, y_tmp);
+          rho_val = runner.copy_to_host(y_tmp)[0];
+        }
+        return x->copy_to_host();
+      }
+    };
+  }
+};
+
 } // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
index 47de59a3ee..152b231857 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
@@ -137,6 +137,28 @@ void caf_main(actor_system& sys) {
         );
     }
 
+    // Test 4: Facade Actor CSR Simple
+    {
+        std::cout << "\n[INFO] Test 4: Facade actor CSR simple matrix..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_cg_facade>();
+
+        self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, 0, 3).send(facade);
+
+        self->receive(
+            [&](std::vector<float> result_x) {
+                verify_solution("Facade CSR Simple", result_x, expected);
+            }
+        );
+    }
+
     manager::shutdown();
 }
-CAF_MAIN(id_block::cuda)
+
+CAF_MAIN(id_block::cuda, id_block::cg_actor)

From 1fd80ffb7b75839dd2626fca486751fb680d2254 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 14:40:52 -0600
Subject: [PATCH 0757/1000] accomdatedf changes made to cgs actor

---
 libcaf_cuda/tests/workload-test/main.test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index e0c86f9711..bfdeaedbd3 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -214,7 +214,7 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
                             std::move(b), std::move(x),
                             matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id, actor_cast<actor>(self));
                     } else {
-                        solver = self->spawn<sparse_bicgstab_actor>(
+                        solver = self->spawn<sparse_bicgstab_actor<float>>(
                             std::move(rp), std::move(ci), std::move(val),
                             std::move(b), std::move(x),
                             matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id, actor_cast<actor>(self));

From 0c81b14a4d8db2e9f5904aad014506fca3604578 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 15:04:47 -0600
Subject: [PATCH 0758/1000] changed cgs actor to be more like actor facade and
 added more tests

---
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 270 +++++++++++++-----
 .../CGS-actor-test/main.test.cpp              |  96 ++++++-
 2 files changed, 287 insertions(+), 79 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index e4d24ae781..65405b810b 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -201,90 +201,208 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
  */
 class sparse_cg_facade : public event_based_actor {
 public:
-  sparse_cg_facade(actor_config& cfg) : event_based_actor(cfg) {}
+  sparse_cg_facade(actor_config& cfg, uint32_t response_id)
+    : event_based_actor(cfg), reply_id_(response_id) {}
 
   behavior make_behavior() override {
     return {
-      [this](in<int> rp, in<int> ci, in<float> val, in<float> b_in, in_out<float> x_in,
-             matrix_format fmt, int n, int nnz, float tol, int max_iter,
-             int device_num, int stream_id) -> std::vector<float> {
-        command_runner<float> runner;
-
-        // 1. Transfer problem data to device
-        auto res = runner.transfer_memory(device_num, stream_id,
-                                          rp, ci, val, b_in, x_in);
-
-        auto A_row_ptr = std::get<0>(res);
-        auto A_col_ind = std::get<1>(res);
-        auto A_values  = std::get<2>(res);
-        auto b         = std::get<3>(res);
-        auto x         = std::get<4>(res);
-
-        auto d_ptr = platform::create()->schedule(stream_id, device_num);
-
-        // 2. Allocate workspace
-        command_runner<out<float>> work_runner;
-        auto r     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
-        auto p     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
-        auto w     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
-        auto y_tmp = work_runner.transfer_memory(device_num, stream_id, create_out_arg_with_size<float>(1));
-
-        // Setup SPMV workspace
-        mem_ptr<char> spmv_workspace;
-        size_t ws_size = 0;
-        if (fmt == matrix_format::csr)
-          ws_size = d_ptr->spmv_csr_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, w);
-        else if (fmt == matrix_format::csc)
-          ws_size = d_ptr->spmv_csc_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, w);
-        else if (fmt == matrix_format::coo)
-          ws_size = d_ptr->spmv_coo_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, w);
-
-        if (ws_size > 0) {
-          command_runner<out<char>> ws_runner;
-          spmv_workspace = ws_runner.transfer_memory(device_num, stream_id, out<char>(static_cast<int>(ws_size)));
+      // Mode 1: Return mem_ptr handles (GPU memory)
+      [this](return_mem_ptr_atom,
+             in<int> rp, in<int> ci, in<float> val,
+             in<float> b_in, in_out<float> x_in,
+             matrix_format fmt, int n, int nnz,
+             float tol, int max_iter,
+             int device_num, int stream_id) {
+
+        auto x = solve_core(rp, ci, val, b_in, x_in,
+                            fmt, n, nnz, tol, max_iter,
+                            device_num, stream_id);
+
+        if (auto sender = actor_cast<actor>(this->current_sender())) {
+          caf::anon_mail(reply_id_, std::move(x)).send(sender);
         }
+      },
+
+      // Mode 2: Return host data via mappings
+      [this](std::vector<output_mapping> mappings,
+             in<int> rp, in<int> ci, in<float> val,
+             in<float> b_in, in_out<float> x_in,
+             matrix_format fmt, int n, int nnz,
+             float tol, int max_iter,
+             int device_num, int stream_id) {
+
+        auto x = solve_core(rp, ci, val, b_in, x_in,
+                            fmt, n, nnz, tol, max_iter,
+                            device_num, stream_id);
+
+        dispatch_result(std::move(mappings), std::move(x), n);
+      },
+
+      // Mode 3: Default (return vector to sender)
+      [this](in<int> rp, in<int> ci, in<float> val,
+             in<float> b_in, in_out<float> x_in,
+             matrix_format fmt, int n, int nnz,
+             float tol, int max_iter,
+             int device_num, int stream_id) {
+
+        auto x = solve_core(rp, ci, val, b_in, x_in,
+                            fmt, n, nnz, tol, max_iter,
+                            device_num, stream_id);
+
+        dispatch_result({}, std::move(x), n);
+      }
+    };
+  }
 
-        auto execute_spmv = [&](mem_ptr<float> input_v, mem_ptr<float> output_v) {
-          switch (fmt) {
-            case matrix_format::csr: d_ptr->spmv_csr(stream_id, n, n, nnz, 1.0f, A_row_ptr, A_col_ind, A_values, input_v, 0.0f, output_v, spmv_workspace); break;
-            case matrix_format::csc: d_ptr->spmv_csc(stream_id, n, n, nnz, 1.0f, A_row_ptr, A_col_ind, A_values, input_v, 0.0f, output_v, spmv_workspace); break;
-            case matrix_format::coo: d_ptr->spmv_coo(stream_id, n, n, nnz, 1.0f, A_row_ptr, A_col_ind, A_values, input_v, 0.0f, output_v, spmv_workspace); break;
-            default: break;
-          }
-        };
-
-        // 3. Initial Residual Setup: r = b - Ax
-        execute_spmv(x, w);
-        d_ptr->scopy(stream_id, n, b, r);
-        d_ptr->saxpy(stream_id, n, -1.0f, w, r);
-        d_ptr->sdot(stream_id, n, r, r, y_tmp);
-        float rho_val = runner.copy_to_host(y_tmp)[0];
-        float old_rho_val = 0.0f;
-        int iterations = 0;
-
-        while (rho_val > (tol * tol) && iterations < max_iter) {
-          iterations++;
-          if (iterations > 1) {
-            float beta_val = rho_val / old_rho_val;
-            d_ptr->scopy(stream_id, n, r, w);
-            d_ptr->saxpy(stream_id, n, beta_val, p, w);
-            d_ptr->scopy(stream_id, n, w, p);
-          } else {
-            d_ptr->scopy(stream_id, n, r, p);
-          }
-          execute_spmv(p, w);
-          d_ptr->sdot(stream_id, n, p, w, y_tmp);
-          float alpha_val = rho_val / runner.copy_to_host(y_tmp)[0];
-          d_ptr->saxpy(stream_id, n, alpha_val, p, x);
-          d_ptr->saxpy(stream_id, n, -alpha_val, w, r);
-          old_rho_val = rho_val;
-          d_ptr->sdot(stream_id, n, r, r, y_tmp);
-          rho_val = runner.copy_to_host(y_tmp)[0];
-        }
-        return x->copy_to_host();
+private:
+  mem_ptr<float> solve_core(in<int> rp, in<int> ci, in<float> val, in<float> b_in,
+                            in_out<float> x_in,
+                            matrix_format fmt, int n, int nnz,
+                            float tol, int max_iter,
+                            int device_num, int stream_id) {
+
+    command_runner<float> runner;
+
+    auto res = runner.transfer_memory(device_num, stream_id,
+                                      rp, ci, val, b_in, x_in);
+
+    auto A_row_ptr = std::get<0>(res);
+    auto A_col_ind = std::get<1>(res);
+    auto A_values  = std::get<2>(res);
+    auto b         = std::get<3>(res);
+    auto x         = std::get<4>(res);
+
+    auto d_ptr = platform::create()->schedule(stream_id, device_num);
+
+    command_runner<out<float>> work_runner;
+    auto r     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
+    auto p     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
+    auto w     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
+    auto y_tmp = work_runner.transfer_memory(device_num, stream_id, create_out_arg_with_size<float>(1));
+
+    mem_ptr<char> spmv_workspace;
+    size_t ws_size = 0;
+
+    if (fmt == matrix_format::csr)
+      ws_size = d_ptr->spmv_csr_buffer_size(stream_id, n, n, nnz,
+                                            A_row_ptr, A_col_ind, A_values, x, w);
+    else if (fmt == matrix_format::csc)
+      ws_size = d_ptr->spmv_csc_buffer_size(stream_id, n, n, nnz,
+                                            A_row_ptr, A_col_ind, A_values, x, w);
+    else if (fmt == matrix_format::coo)
+      ws_size = d_ptr->spmv_coo_buffer_size(stream_id, n, n, nnz,
+                                            A_row_ptr, A_col_ind, A_values, x, w);
+
+    if (ws_size > 0) {
+      command_runner<out<char>> ws_runner;
+      spmv_workspace =
+        ws_runner.transfer_memory(device_num, stream_id,
+                                  out<char>(static_cast<int>(ws_size)));
+    }
+
+    auto execute_spmv = [&](mem_ptr<float> input_v, mem_ptr<float> output_v) {
+      switch (fmt) {
+        case matrix_format::csr:
+          d_ptr->spmv_csr(stream_id, n, n, nnz, 1.0f,
+                          A_row_ptr, A_col_ind, A_values,
+                          input_v, 0.0f, output_v, spmv_workspace);
+          break;
+
+        case matrix_format::csc:
+          d_ptr->spmv_csc(stream_id, n, n, nnz, 1.0f,
+                          A_row_ptr, A_col_ind, A_values,
+                          input_v, 0.0f, output_v, spmv_workspace);
+          break;
+
+        case matrix_format::coo:
+          d_ptr->spmv_coo(stream_id, n, n, nnz, 1.0f,
+                          A_row_ptr, A_col_ind, A_values,
+                          input_v, 0.0f, output_v, spmv_workspace);
+          break;
+
+        default:
+          break;
       }
     };
+
+    execute_spmv(x, w);
+    d_ptr->scopy(stream_id, n, b, r);
+    d_ptr->saxpy(stream_id, n, -1.0f, w, r);
+    d_ptr->sdot(stream_id, n, r, r, y_tmp);
+
+    float rho_val = runner.copy_to_host(y_tmp)[0];
+    float old_rho_val = 0.0f;
+    int iterations = 0;
+
+    while (rho_val > (tol * tol) && iterations < max_iter) {
+      iterations++;
+
+      if (iterations > 1) {
+        float beta_val = rho_val / old_rho_val;
+        d_ptr->scopy(stream_id, n, r, w);
+        d_ptr->saxpy(stream_id, n, beta_val, p, w);
+        d_ptr->scopy(stream_id, n, w, p);
+      } else {
+        d_ptr->scopy(stream_id, n, r, p);
+      }
+
+      execute_spmv(p, w);
+      d_ptr->sdot(stream_id, n, p, w, y_tmp);
+
+      float alpha_val =
+        rho_val / runner.copy_to_host(y_tmp)[0];
+
+      d_ptr->saxpy(stream_id, n, alpha_val, p, x);
+      d_ptr->saxpy(stream_id, n, -alpha_val, w, r);
+
+      old_rho_val = rho_val;
+      d_ptr->sdot(stream_id, n, r, r, y_tmp);
+      rho_val = runner.copy_to_host(y_tmp)[0];
+    }
+
+    return x;
+  }
+
+  void dispatch_result(std::vector<output_mapping> mappings,
+                       mem_ptr<float> x,
+                       int n) {
+
+    auto sender = actor_cast<actor>(this->current_sender());
+    if (!sender) return;
+
+    void* custom_dst = nullptr;
+    size_t custom_count = 0;
+
+    for (const auto& m : mappings) {
+      if (m.index == 4) {
+        custom_dst = m.dst;
+        custom_count = m.count;
+        break;
+      }
+    }
+
+    command_runner<float> runner;
+
+    if (custom_dst) {
+      runner.copy_to_host_async(
+        x,
+        static_cast<float*>(custom_dst),
+        custom_count > 0 ? custom_count : (size_t)n,
+        [sender, r_id = reply_id_](float*, size_t) {
+          caf::anon_mail(r_id, 4).send(sender);
+        });
+
+    } else {
+      runner.copy_to_host_async(
+        x,
+        [sender, r_id = reply_id_](std::vector<float> data) {
+          caf::anon_mail(r_id, 4, std::move(data)).send(sender);
+        });
+    }
   }
+
+private:
+  uint32_t reply_id_;
 };
 
 } // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
index 152b231857..6e85495321 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
@@ -145,19 +145,109 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto facade = sys.spawn<sparse_cg_facade>();
+        auto facade = sys.spawn<sparse_cg_facade>(100);
 
-        self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+        self->mail(std::vector<output_mapping>{},
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
                    create_in_arg(h_b), create_in_out_arg(h_x),
                    matrix_format::csr, n, nnz, tolerance, max_iter, 0, 3).send(facade);
 
         self->receive(
-            [&](std::vector<float> result_x) {
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
                 verify_solution("Facade CSR Simple", result_x, expected);
             }
         );
     }
 
+    // Test 5: Facade Actor CSR with Custom Buffer
+    {
+        std::cout << "\n[INFO] Test 5: Facade actor CSR with custom buffer..." << std::endl;
+        std::vector<float> custom_x(n, 0.0f);
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        output_mapping m{4, custom_x.data(), (size_t)n};
+        
+        auto facade = sys.spawn<sparse_cg_facade>(100);
+        self->mail(std::vector<output_mapping>{m},
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(custom_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, 0, 4).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int index) {
+                if (index == 4)
+                  verify_solution("Facade Custom Buffer", custom_x, expected);
+            }
+        );
+    }
+
+    // Test 6: Facade Actor CSR returning mem_ptr handles
+    {
+        std::cout << "\n[INFO] Test 6: Facade actor CSR returning mem_ptr..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_cg_facade>(100);
+
+        self->mail(return_mem_ptr_atom_v,
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, 0, 5).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr) {
+                auto result_x = ptr->copy_to_host();
+                verify_solution("Facade mem_ptr", result_x, expected);
+            }
+        );
+    }
+
+    // Test 7: Facade Actor Default (No mapping vector - hits Mode 3 handler)
+    {
+        std::cout << "\n[INFO] Test 7: Facade actor default (no mapping vector)..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_cg_facade>(100);
+
+        self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, 0, 6).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+                verify_solution("Facade Default", result_x, expected);
+            }
+        );
+    }
+
+    // Test 8: Facade Actor CSC Simple
+    {
+        std::cout << "\n[INFO] Test 8: Facade actor CSC simple matrix..." << std::endl;
+        std::vector<int> col_ptr = {0, 1, 2, 3};
+        std::vector<int> row_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_cg_facade>(100);
+
+        self->mail(std::vector<output_mapping>{},
+                   create_in_arg(col_ptr), create_in_arg(row_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csc, n, nnz, tolerance, max_iter, 0, 7).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+                verify_solution("Facade CSC Simple", result_x, expected);
+            }
+        );
+    }
+
     manager::shutdown();
 }
 

From aac227dfd9b3c25100f44a01d815761bb3eec715 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 27 May 2026 15:20:50 -0600
Subject: [PATCH 0759/1000] updated solver facade to be more like actor facade
 and adjusted tests accordingly

---
 .../sparse-BiCGSTAB-actor.hpp                 | 335 +++++++++++-------
 .../BICGSTAB-actor-test/main.test.cpp         |  54 ++-
 2 files changed, 259 insertions(+), 130 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
index e18fcd59e4..37343ce7ca 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
@@ -64,6 +64,8 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state<T>> {
     this->state().supervisor = supervisor;
   }
 
+  ~sparse_bicgstab_actor() override = default;
+
   behavior make_behavior() override {
     return {
       [this](start_atom) {
@@ -72,12 +74,18 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state<T>> {
           s.supervisor = actor_cast<caf::actor>(this->current_sender());
         start_solve();
       },
+      [this](gpu_done_atom, std::vector<T>& solution) {
+        auto& s = this->state();
+        if (s.supervisor)
+          this->mail(std::move(solution)).send(s.supervisor);
+        this->quit();
+      }
     };
   }
 private:
   void start_solve() {
     auto& s = this->state();
-    command_runner<> runner;
+    command_runner<T> runner;
 
     // Transfer problem data to device
     auto res = runner.transfer_memory(s.device_num, s.stream_id,
@@ -127,7 +135,7 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state<T>> {
 
     // 3. Initial norm calculation
     execute_dot(s.r, s.r, s.y_tmp);
-    T norm_sq = s.y_tmp->copy_to_host()[0];
+    T norm_sq = runner.copy_to_host(s.y_tmp)[0];
 
     s.iterations = 0;
     s.rho_val = T{1};
@@ -140,7 +148,7 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state<T>> {
 
       // rho_new = <r_hat, r>
       execute_dot(s.r_hat, s.r, s.y_tmp);
-      T rho_new = s.y_tmp->copy_to_host()[0];
+      T rho_new = runner.copy_to_host(s.y_tmp)[0];
 
       if (s.iterations == 1) {
         execute_copy(s.r, s.p);
@@ -159,7 +167,7 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state<T>> {
 
       // alpha = rho / <r_hat, v>
       execute_dot(s.r_hat, s.v, s.y_tmp);
-      T alpha_denom = s.y_tmp->copy_to_host()[0];
+      T alpha_denom = runner.copy_to_host(s.y_tmp)[0];
       s.alpha_val = s.rho_val / alpha_denom;
 
       // s = r - alpha * v
@@ -171,9 +179,9 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state<T>> {
 
       // omega = <t, s> / <t, t>
       execute_dot(s.t_vec, s.s_vec, s.y_tmp);
-      T omega_num = s.y_tmp->copy_to_host()[0];
+      T omega_num = runner.copy_to_host(s.y_tmp)[0];
       execute_dot(s.t_vec, s.t_vec, s.y_tmp);
-      T omega_denom = s.y_tmp->copy_to_host()[0];
+      T omega_denom = runner.copy_to_host(s.y_tmp)[0];
       s.omega_val = omega_num / omega_denom;
 
       // x = x + alpha*p + omega*s
@@ -186,19 +194,13 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state<T>> {
 
       // check convergence: norm_sq = <r, r>
       execute_dot(s.r, s.r, s.y_tmp);
-      norm_sq = s.y_tmp->copy_to_host()[0];
+      norm_sq = runner.copy_to_host(s.y_tmp)[0];
     }
 
-    finish_solve();
-  }
-
-  void finish_solve() {
-    auto& s = this->state();
-    // Copy result back to host and notify supervisor
-    std::vector<T> solution = s.x->copy_to_host();
-    if (s.supervisor)
-      this->mail(std::move(solution)).send(s.supervisor);
-    this->quit();
+    auto self = actor_cast<actor>(this);
+    runner.copy_to_host_async(s.x, [self](std::vector<T> solution) {
+      anon_mail(gpu_done_atom_v, std::move(solution)).send(self);
+    });
   }
 
   void execute_spmv(mem_ptr<T> input_v, mem_ptr<T> output_v) {
@@ -257,124 +259,205 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state<T>> {
 template <class T = float>
 class sparse_bicgstab_facade : public event_based_actor {
 public:
-  sparse_bicgstab_facade(actor_config& cfg) : event_based_actor(cfg) {}
+  sparse_bicgstab_facade(actor_config& cfg, uint32_t response_id)
+    : event_based_actor(cfg), reply_id_(response_id) {}
 
   behavior make_behavior() override {
     return {
+      // Mode 1: Return mem_ptr handles (GPU memory)
+      [this](return_mem_ptr_atom,
+             in<int> rp, in<int> ci, in<T> val,
+             in<T> b_in, in_out<T> x_in,
+             matrix_format fmt, int n, int nnz,
+             float tol, int max_iter,
+             int device_num, int stream_id) {
+
+        auto x = solve_core(rp, ci, val, b_in, x_in,
+                            fmt, n, nnz, tol, max_iter,
+                            device_num, stream_id);
+
+        if (auto sender = actor_cast<actor>(this->current_sender())) {
+          caf::anon_mail(reply_id_, std::move(x)).send(sender);
+        }
+      },
+
+      // Mode 2: Return host data via mappings
+      [this](std::vector<output_mapping> mappings,
+             in<int> rp, in<int> ci, in<T> val,
+             in<T> b_in, in_out<T> x_in,
+             matrix_format fmt, int n, int nnz,
+             float tol, int max_iter,
+             int device_num, int stream_id) {
+
+        auto x = solve_core(rp, ci, val, b_in, x_in,
+                            fmt, n, nnz, tol, max_iter,
+                            device_num, stream_id);
+
+        dispatch_result(std::move(mappings), std::move(x), n);
+      },
+
+      // Mode 3: Default (return vector to sender)
       [this](in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
              matrix_format fmt, int n, int nnz, float tol, int max_iter,
-             int device_num, int stream_id) -> std::vector<T> {
-        command_runner<> runner;
-
-        // 1. Transfer problem data to device
-        auto res = runner.transfer_memory(device_num, stream_id,
-                                          rp, ci, val, b_in, x_in);
-
-        auto A_row_ptr = std::get<0>(res);
-        auto A_col_ind = std::get<1>(res);
-        auto A_values  = std::get<2>(res);
-        auto b         = std::get<3>(res);
-        auto x         = std::get<4>(res);
-
-        auto d_ptr = platform::create()->schedule(stream_id, device_num);
-
-        // 2. Allocate workspace
-        command_runner<out<T>> work_runner;
-        auto r     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
-        auto r_hat = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
-        auto p     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
-        auto v     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
-        auto s_vec = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
-        auto t_vec = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
-        auto y_tmp = work_runner.transfer_memory(device_num, stream_id, out<T>(1));
-
-        // 3. Setup SPMV workspace
-        mem_ptr<char> spmv_workspace;
-        size_t ws_size = 0;
-        if (fmt == matrix_format::csr)
-          ws_size = d_ptr->spmv_csr_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, v);
-        else if (fmt == matrix_format::csc)
-          ws_size = d_ptr->spmv_csc_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, v);
-        else if (fmt == matrix_format::coo)
-          ws_size = d_ptr->spmv_coo_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, v);
-
-        if (ws_size > 0) {
-          command_runner<out<char>> ws_runner;
-          spmv_workspace = ws_runner.transfer_memory(device_num, stream_id, out<char>(static_cast<int>(ws_size)));
-        }
+             int device_num, int stream_id) {
 
-        // Helper lambdas for GPU operations
-        auto execute_spmv = [&](mem_ptr<T> input_v, mem_ptr<T> output_v) {
-          switch (fmt) {
-            case matrix_format::csr: d_ptr->spmv_csr(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, input_v, T{0}, output_v, spmv_workspace); break;
-            case matrix_format::csc: d_ptr->spmv_csc(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, input_v, T{0}, output_v, spmv_workspace); break;
-            case matrix_format::coo: d_ptr->spmv_coo(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, input_v, T{0}, output_v, spmv_workspace); break;
-            default: break;
-          }
-        };
-        auto execute_copy = [&](mem_ptr<T> src, mem_ptr<T> dst) {
-          if constexpr (std::is_same_v<T, double>) d_ptr->dcopy(stream_id, n, src, dst);
-          else d_ptr->scopy(stream_id, n, src, dst);
-        };
-        auto execute_axpy = [&](T alpha, mem_ptr<T> ax, mem_ptr<T> ay) {
-          if constexpr (std::is_same_v<T, double>) d_ptr->daxpy(stream_id, n, alpha, ax, ay);
-          else d_ptr->saxpy(stream_id, n, static_cast<float>(alpha), ax, ay);
-        };
-        auto execute_dot = [&](mem_ptr<T> dx, mem_ptr<T> dy, mem_ptr<T> dres) {
-          if constexpr (std::is_same_v<T, double>) d_ptr->ddot(stream_id, n, dx, dy, dres);
-          else d_ptr->sdot(stream_id, n, dx, dy, dres);
-        };
-
-        // 4. Initial Residual: r = b - Ax
-        execute_spmv(x, v); 
-        execute_copy(b, r);
-        execute_axpy(T{-1}, v, r);
-        execute_copy(r, r_hat);
-        execute_dot(r, r, y_tmp);
-        T norm_sq = y_tmp->copy_to_host()[0];
-
-        int iterations = 0;
-        T rho_val = T{1}, alpha_val = T{1}, omega_val = T{1}, beta_val = T{0};
-
-        // BiCGSTAB Loop
-        while (norm_sq > (tol * tol) && iterations < max_iter) {
-          iterations++;
-          execute_dot(r_hat, r, y_tmp);
-          T rho_new = y_tmp->copy_to_host()[0];
-
-          if (iterations == 1) {
-            execute_copy(r, p);
-          } else {
-            beta_val = (rho_new / rho_val) * (alpha_val / omega_val);
-            execute_axpy(-omega_val, v, p);
-            execute_copy(r, s_vec);
-            execute_axpy(beta_val, p, s_vec);
-            execute_copy(s_vec, p);
-          }
-          rho_val = rho_new;
-          execute_spmv(p, v);
-          execute_dot(r_hat, v, y_tmp);
-          T alpha_denom = y_tmp->copy_to_host()[0];
-          alpha_val = rho_val / alpha_denom;
-          execute_copy(r, s_vec);
-          execute_axpy(-alpha_val, v, s_vec);
-          execute_spmv(s_vec, t_vec);
-          execute_dot(t_vec, s_vec, y_tmp);
-          T omega_num = y_tmp->copy_to_host()[0];
-          execute_dot(t_vec, t_vec, y_tmp);
-          T omega_denom = y_tmp->copy_to_host()[0];
-          omega_val = omega_num / omega_denom;
-          execute_axpy(alpha_val, p, x);
-          execute_axpy(omega_val, s_vec, x);
-          execute_copy(s_vec, r);
-          execute_axpy(-omega_val, t_vec, r);
-          execute_dot(r, r, y_tmp);
-          norm_sq = y_tmp->copy_to_host()[0];
-        }
-        return x->copy_to_host();
+        auto x = solve_core(rp, ci, val, b_in, x_in,
+                            fmt, n, nnz, tol, max_iter,
+                            device_num, stream_id);
+
+        dispatch_result({}, std::move(x), n);
       }
     };
   }
+
+private:
+  mem_ptr<T> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in,
+                        in_out<T> x_in,
+                        matrix_format fmt, int n, int nnz,
+                        float tol, int max_iter,
+                        int device_num, int stream_id) {
+    command_runner<T> runner;
+    auto res = runner.transfer_memory(device_num, stream_id,
+                                      rp, ci, val, b_in, x_in);
+
+    auto A_row_ptr = std::get<0>(res);
+    auto A_col_ind = std::get<1>(res);
+    auto A_values  = std::get<2>(res);
+    auto b         = std::get<3>(res);
+    auto x         = std::get<4>(res);
+
+    auto d_ptr = platform::create()->schedule(stream_id, device_num);
+
+    command_runner<out<T>> work_runner;
+    auto r     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto r_hat = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto p     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto v     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto s_vec = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto t_vec = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto y_tmp = work_runner.transfer_memory(device_num, stream_id, out<T>(1));
+
+    mem_ptr<char> spmv_workspace;
+    std::size_t ws_size = 0;
+    if (fmt == matrix_format::csr)
+      ws_size = d_ptr->spmv_csr_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, v);
+    else if (fmt == matrix_format::csc)
+      ws_size = d_ptr->spmv_csc_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, v);
+    else if (fmt == matrix_format::coo)
+      ws_size = d_ptr->spmv_coo_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, v);
+
+    if (ws_size > 0) {
+      command_runner<out<char>> ws_runner;
+      spmv_workspace = ws_runner.transfer_memory(device_num, stream_id, out<char>(static_cast<int>(ws_size)));
+    }
+
+    auto execute_spmv = [&](mem_ptr<T> input_v, mem_ptr<T> output_v) {
+      switch (fmt) {
+        case matrix_format::csr: d_ptr->spmv_csr(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, input_v, T{0}, output_v, spmv_workspace); break;
+        case matrix_format::csc: d_ptr->spmv_csc(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, input_v, T{0}, output_v, spmv_workspace); break;
+        case matrix_format::coo: d_ptr->spmv_coo(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, input_v, T{0}, output_v, spmv_workspace); break;
+        default: break;
+      }
+    };
+    auto execute_copy = [&](mem_ptr<T> src, mem_ptr<T> dst) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->dcopy(stream_id, n, src, dst);
+      else d_ptr->scopy(stream_id, n, src, dst);
+    };
+    auto execute_axpy = [&](T alpha, mem_ptr<T> ax, mem_ptr<T> ay) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->daxpy(stream_id, n, alpha, ax, ay);
+      else d_ptr->saxpy(stream_id, n, static_cast<float>(alpha), ax, ay);
+    };
+    auto execute_dot = [&](mem_ptr<T> dx, mem_ptr<T> dy, mem_ptr<T> dres) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->ddot(stream_id, n, dx, dy, dres);
+      else d_ptr->sdot(stream_id, n, dx, dy, dres);
+    };
+
+    execute_spmv(x, v); 
+    execute_copy(b, r);
+    execute_axpy(T{-1}, v, r);
+    execute_copy(r, r_hat);
+    execute_dot(r, r, y_tmp);
+    T norm_sq = runner.copy_to_host(y_tmp)[0];
+
+    int iterations = 0;
+    T rho_val = T{1}, alpha_val = T{1}, omega_val = T{1}, beta_val = T{0};
+
+    while (norm_sq > (tol * tol) && iterations < max_iter) {
+      iterations++;
+      execute_dot(r_hat, r, y_tmp);
+      T rho_new = runner.copy_to_host(y_tmp)[0];
+
+      if (iterations == 1) {
+        execute_copy(r, p);
+      } else {
+        beta_val = (rho_new / rho_val) * (alpha_val / omega_val);
+        execute_axpy(-omega_val, v, p);
+        execute_copy(r, s_vec);
+        execute_axpy(beta_val, p, s_vec);
+        execute_copy(s_vec, p);
+      }
+      rho_val = rho_new;
+      execute_spmv(p, v);
+      execute_dot(r_hat, v, y_tmp);
+      T alpha_denom = runner.copy_to_host(y_tmp)[0];
+      alpha_val = rho_val / alpha_denom;
+      execute_copy(r, s_vec);
+      execute_axpy(-alpha_val, v, s_vec);
+      execute_spmv(s_vec, t_vec);
+      execute_dot(t_vec, s_vec, y_tmp);
+      T omega_num = runner.copy_to_host(y_tmp)[0];
+      execute_dot(t_vec, t_vec, y_tmp);
+      T omega_denom = runner.copy_to_host(y_tmp)[0];
+      omega_val = omega_num / omega_denom;
+      execute_axpy(alpha_val, p, x);
+      execute_axpy(omega_val, s_vec, x);
+      execute_copy(s_vec, r);
+      execute_axpy(-omega_val, t_vec, r);
+      execute_dot(r, r, y_tmp);
+      norm_sq = runner.copy_to_host(y_tmp)[0];
+    }
+    return x;
+  }
+
+  void dispatch_result(std::vector<output_mapping> mappings,
+                       mem_ptr<T> x,
+                       int n) {
+
+    auto sender = actor_cast<actor>(this->current_sender());
+    if (!sender) return;
+
+    void* custom_dst = nullptr;
+    size_t custom_count = 0;
+
+    for (const auto& m : mappings) {
+      if (m.index == 4) {
+        custom_dst = m.dst;
+        custom_count = m.count;
+        break;
+      }
+    }
+
+    command_runner<T> runner;
+
+    if (custom_dst) {
+      runner.copy_to_host_async(
+        x,
+        static_cast<T*>(custom_dst),
+        custom_count > 0 ? custom_count : (size_t)n,
+        [sender, r_id = reply_id_](T*, size_t) {
+          caf::anon_mail(r_id, 4).send(sender);
+        });
+
+    } else {
+      runner.copy_to_host_async(
+        x,
+        [sender, r_id = reply_id_](std::vector<T> data) {
+          caf::anon_mail(r_id, 4, std::move(data)).send(sender);
+        });
+    }
+  }
+
+  uint32_t reply_id_;
 };
 
 } // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
index cb6eb868d1..729c698166 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
@@ -456,14 +456,14 @@ void caf_main(actor_system& sys) {
         std::vector<float> h_x(n_f, 0.0f);
         std::vector<float> expected_f = {2.0f, 3.0f, 1.0f};
 
-        auto facade = sys.spawn<sparse_bicgstab_facade<float>>();
+        auto facade = sys.spawn<sparse_bicgstab_facade<float>>(100);
 
         self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
                    create_in_arg(h_b), create_in_out_arg(h_x),
                    matrix_format::csr, n_f, nnz_f, tolerance, max_iter, 0, 9).send(facade);
 
         self->receive(
-            [&](std::vector<float> result_x) {
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
                 verify_solution("Facade CSR Simple", result_x, expected_f, tolerance);
             }
         );
@@ -479,14 +479,14 @@ void caf_main(actor_system& sys) {
             std::vector<float> b_real = compute_rhs_manual<float>(A, expected_real);
             std::vector<float> x_real(A.rows, 0.0f);
 
-            auto facade = sys.spawn<sparse_bicgstab_facade<float>>();
+            auto facade = sys.spawn<sparse_bicgstab_facade<float>>(100);
 
             self->mail(create_in_arg(A.row_ptr), create_in_arg(A.col_ind), create_in_arg(A.values),
                        create_in_arg(b_real), create_in_out_arg(x_real),
                        matrix_format::csr, A.rows, A.nnz, 1e-5f, 5000, 0, 10).send(facade);
 
             self->receive(
-                [&](std::vector<float> result) {
+                [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result) {
                     verify_solution("Facade Real Matrix", result, expected_real, 1e-2f);
                 }
             );
@@ -495,6 +495,52 @@ void caf_main(actor_system& sys) {
         }
     }
 
+    // Test 12: Facade Actor CSR with Custom Buffer
+    {
+        std::cout << "\n[INFO] Test 12: Facade actor CSR with custom buffer..." << std::endl;
+        std::vector<float> custom_x(n, 0.0f);
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        output_mapping m{4, custom_x.data(), (size_t)n};
+        
+        auto facade = sys.spawn<sparse_bicgstab_facade<float>>(100);
+        self->mail(std::vector<output_mapping>{m},
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(custom_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, 0, 11).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int index) {
+                if (index == 4)
+                  verify_solution("Facade Custom Buffer", custom_x, expected);
+            }
+        );
+    }
+
+    // Test 13: Facade Actor CSR returning mem_ptr handles
+    {
+        std::cout << "\n[INFO] Test 13: Facade actor CSR returning mem_ptr..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_bicgstab_facade<float>>(100);
+
+        self->mail(return_mem_ptr_atom_v,
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, 0, 12).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr) {
+                auto result_x = ptr->copy_to_host();
+                verify_solution("Facade mem_ptr", result_x, expected);
+            }
+        );
+    }
+
     manager::shutdown();
 }
 CAF_MAIN(id_block::cuda, id_block::cg_actor)

From bb03ce29149a88e0c38663e8d0f5c4019ad03be9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 09:18:09 -0600
Subject: [PATCH 0760/1000] made private methods protected so child classes can
 access them and modifiy them

---
 .../sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp             | 2 +-
 .../sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
index 37343ce7ca..9a63ec7a44 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
@@ -310,7 +310,7 @@ class sparse_bicgstab_facade : public event_based_actor {
     };
   }
 
-private:
+protected:
   mem_ptr<T> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in,
                         in_out<T> x_in,
                         matrix_format fmt, int n, int nnz,
diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index 65405b810b..b8d77941ac 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -254,7 +254,7 @@ class sparse_cg_facade : public event_based_actor {
     };
   }
 
-private:
+protected:
   mem_ptr<float> solve_core(in<int> rp, in<int> ci, in<float> val, in<float> b_in,
                             in_out<float> x_in,
                             matrix_format fmt, int n, int nnz,

From 9fedddbd76adc50ef8d940ee3ecde6705b2f2181 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 10:14:03 -0600
Subject: [PATCH 0761/1000] added cache tables for program_ptr Change was made
 to reduce compilation overhead of GPU kernels by caching them

---
 libcaf_cuda/caf/cuda/manager.hpp |  8 +++
 libcaf_cuda/src/manager.cpp      | 99 +++++++++++++++++++++++++++-----
 2 files changed, 94 insertions(+), 13 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index 8a3dd55d6a..eb8a6b9f7e 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -6,6 +6,8 @@
 #include <mutex>
 #include <fstream>
 #include <map>
+#include <shared_mutex>
+#include <unordered_map>
 
 #include <caf/actor_system.hpp>
 #include <caf/actor.hpp>
@@ -59,6 +61,9 @@ class CAF_CUDA_EXPORT manager {
   //deletes the scheduler actor as well if it exists
   static void shutdown();
 
+  /// Flushes the cache of compiled programs.
+  void flush_programs();
+
   // Prevent copy/assignment
   manager(const manager&) = delete;
   manager& operator=(const manager&) = delete;
@@ -195,6 +200,9 @@ class CAF_CUDA_EXPORT manager {
 
   static manager* instance_;
   static std::mutex mutex_;
+
+  mutable std::shared_mutex programs_mutex_;
+  std::unordered_map<size_t, program_ptr> programs_;
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 72067bdff0..14af914d38 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -96,6 +96,11 @@ void manager::shutdown() {
     instance_ = nullptr;
 }
 
+void manager::flush_programs() {
+  std::unique_lock<std::shared_mutex> lock(programs_mutex_);
+  programs_.clear();
+}
+
 device_ptr manager::find_device(std::size_t) const {
   throw std::runtime_error("OpenCL support disabled: manager::find_device");
 }
@@ -116,24 +121,44 @@ double manager::available_memory_mb(int id) {
 program_ptr manager::create_program(const char * kernel,
                                     const std::string& name,
                                     device_ptr device) {
-  
-
-	CUdevice current_device = device -> getDevice();;
-
-	//the compiled program can be accessed via ptx.data() afterwards
-	std::vector<char> ptx;
-        if (!compile_nvrtc_program(kernel,current_device,ptx)) {
-	
-		throw std::runtime_error("Program failed to compile\n");
-	
-	}
-	program_ptr prog = make_counted<program>(name, ptx);
-	return prog;
+  size_t h = std::hash<std::string>{}(name + kernel);
+  {
+    std::shared_lock<std::shared_mutex> lock(programs_mutex_);
+    auto it = programs_.find(h);
+    if (it != programs_.end()) {
+      return it->second;
+    }
+  }
+
+  CUdevice current_device = device -> getDevice();
+
+  //the compiled program can be accessed via ptx.data() afterwards
+  std::vector<char> ptx;
+  if (!compile_nvrtc_program(kernel,current_device,ptx)) {
+    throw std::runtime_error("Program failed to compile\n");
+  }
+
+  program_ptr prog = make_counted<program>(name, ptx);
+
+  {
+    std::unique_lock<std::shared_mutex> lock(programs_mutex_);
+    programs_[h] = prog;
+  }
+
+  return prog;
 }
 //this actually doesnt even work do not use 
 program_ptr manager::create_program_from_ptx(const std::string& filename,
                                              const char* kernel_name,
                                              [[maybe_unused]] device_ptr device) {
+  size_t h = std::hash<std::string>{}(filename + kernel_name);
+  {
+    std::shared_lock<std::shared_mutex> lock(programs_mutex_);
+    auto it = programs_.find(h);
+    if (it != programs_.end())
+      return it->second;
+  }
+
   static std::mutex global_ptx_mutex_map_guard;
   static std::map<std::string, std::shared_ptr<std::mutex>> ptx_mutex_map;
 
@@ -163,6 +188,12 @@ program_ptr manager::create_program_from_ptx(const std::string& filename,
    // 🔒 Guard the actual JIT as well — this is the critical part!
   std::lock_guard<std::mutex> guard(*file_mutex);
   program_ptr prog = make_counted<program>(kernel_name, ptx);
+
+  {
+    std::unique_lock<std::shared_mutex> lock(programs_mutex_);
+    programs_[h] = prog;
+  }
+
   return prog;
 }
 
@@ -171,6 +202,14 @@ program_ptr manager::create_program_from_ptx(const std::string& filename,
 program_ptr manager::create_program_from_cubin(const std::string& filename,
                                                const char* kernel_name,
                                                [[maybe_unused]] device_ptr device) {
+  size_t h = std::hash<std::string>{}(filename + kernel_name);
+  {
+    std::shared_lock<std::shared_mutex> lock(programs_mutex_);
+    auto it = programs_.find(h);
+    if (it != programs_.end())
+      return it->second;
+  }
+
   // Open the cubin file in binary mode
   std::ifstream in(filename, std::ios::binary);
   if (!in)
@@ -182,6 +221,12 @@ program_ptr manager::create_program_from_cubin(const std::string& filename,
 
    // Reuse the same constructor as PTX (program class doesn't care)
   program_ptr prog = make_counted<program>(kernel_name, std::move(cubin));
+
+  {
+    std::unique_lock<std::shared_mutex> lock(programs_mutex_);
+    programs_[h] = prog;
+  }
+
   return prog;
 }
 
@@ -189,6 +234,14 @@ program_ptr manager::create_program_from_cubin(const std::string& filename,
 //creates a program given a path to a cubin file and the kernels name
 program_ptr manager::create_program_from_cubin(const std::string& filename,
                                                const char* kernel_name) {
+  size_t h = std::hash<std::string>{}(filename + kernel_name);
+  {
+    std::shared_lock<std::shared_mutex> lock(programs_mutex_);
+    auto it = programs_.find(h);
+    if (it != programs_.end())
+      return it->second;
+  }
+
   // Open the cubin file in binary mode
   std::ifstream in(filename, std::ios::binary);
   if (!in)
@@ -200,6 +253,12 @@ program_ptr manager::create_program_from_cubin(const std::string& filename,
 
    // Reuse the same constructor as PTX (program class doesn't care)
   program_ptr prog = make_counted<program>(kernel_name, std::move(cubin));
+
+  {
+    std::unique_lock<std::shared_mutex> lock(programs_mutex_);
+    programs_[h] = prog;
+  }
+
   return prog;
 }
 
@@ -208,6 +267,14 @@ program_ptr manager::create_program_from_cubin(const std::string& filename,
 //creates a program given a path to a fatbin file and the kernels name
 program_ptr manager::create_program_from_fatbin(const std::string& filename,
                                                const char* kernel_name) {
+  size_t h = std::hash<std::string>{}(filename + kernel_name);
+  {
+    std::shared_lock<std::shared_mutex> lock(programs_mutex_);
+    auto it = programs_.find(h);
+    if (it != programs_.end())
+      return it->second;
+  }
+
   // Open the fatbin file in binary mode
   std::ifstream in(filename, std::ios::binary);
   if (!in)
@@ -219,6 +286,12 @@ program_ptr manager::create_program_from_fatbin(const std::string& filename,
 
    // Reuse the same constructor as PTX (program class doesn't care)
   program_ptr prog = make_counted<program>(kernel_name, std::move(cubin),true);
+
+  {
+    std::unique_lock<std::shared_mutex> lock(programs_mutex_);
+    programs_[h] = prog;
+  }
+
   return prog;
 }
 

From df6e0703e172a12601a0ad34ebbadd7f541a9f74 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 10:35:41 -0600
Subject: [PATCH 0762/1000] imlpemented CGS  solver with jacobio preconditioner

---
 .../sparse-CGS-actor/jacobi_kernels.cu        |  15 ++
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 152 +++++++++++++++++-
 libcaf_cuda/caf/cuda/device.hpp               |  17 ++
 3 files changed, 179 insertions(+), 5 deletions(-)
 create mode 100644 libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/jacobi_kernels.cu

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/jacobi_kernels.cu b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/jacobi_kernels.cu
new file mode 100644
index 0000000000..16c17e7352
--- /dev/null
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/jacobi_kernels.cu
@@ -0,0 +1,15 @@
+extern "C" __global__
+void extract_diag_inv(int n, const int* row_ptr, const int* col_ind, const float* val, float* d_inv) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        float d = 0.0f;
+        // Search for the diagonal element (A[i][i]) in the sparse row
+        for (int j = row_ptr[i]; j < row_ptr[i+1]; j++) {
+            if (col_ind[j] == i) {
+                d = val[j];
+                break;
+            }
+        }
+        d_inv[i] = (d != 0.0f) ? 1.0f / d : 1.0f;
+    }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index b8d77941ac..e3951d5a53 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -255,11 +255,11 @@ class sparse_cg_facade : public event_based_actor {
   }
 
 protected:
-  mem_ptr<float> solve_core(in<int> rp, in<int> ci, in<float> val, in<float> b_in,
-                            in_out<float> x_in,
-                            matrix_format fmt, int n, int nnz,
-                            float tol, int max_iter,
-                            int device_num, int stream_id) {
+  virtual mem_ptr<float> solve_core(in<int> rp, in<int> ci, in<float> val, in<float> b_in,
+                                    in_out<float> x_in,
+                                    matrix_format fmt, int n, int nnz,
+                                    float tol, int max_iter,
+                                    int device_num, int stream_id) {
 
     command_runner<float> runner;
 
@@ -405,4 +405,146 @@ class sparse_cg_facade : public event_based_actor {
   uint32_t reply_id_;
 };
 
+/**
+ * A variant of the CG solver facade that uses Jacobi preconditioning.
+ */
+class sparse_cg_jacobi_facade : public sparse_cg_facade {
+public:
+  sparse_cg_jacobi_facade(actor_config& cfg, uint32_t response_id)
+    : sparse_cg_facade(cfg, response_id) {
+    // Deduce path to cubin relative to this header file at runtime
+    std::string current_file = __FILE__;
+    auto pos = current_file.find_last_of('/');
+    std::string dir = (pos == std::string::npos) ? "" : current_file.substr(0, pos + 1);
+    auto& mgr = manager::get();
+    diag_prog_ = mgr.create_program_from_cubin(dir + "jacobi_kernels.cubin", "extract_diag_inv");
+  }
+
+protected:
+  mem_ptr<float> solve_core(in<int> rp, in<int> ci, in<float> val, in<float> b_in,
+                            in_out<float> x_in,
+                            matrix_format fmt, int n, int nnz,
+                            float tol, int max_iter,
+                            int device_num, int stream_id) override {
+
+    command_runner<float> runner;
+    auto res = runner.transfer_memory(device_num, stream_id,
+                                      rp, ci, val, b_in, x_in);
+
+    auto A_row_ptr = std::get<0>(res);
+    auto A_col_ind = std::get<1>(res);
+    auto A_values  = std::get<2>(res);
+    auto b         = std::get<3>(res);
+    auto x         = std::get<4>(res);
+
+    auto d_ptr = platform::create()->schedule(stream_id, device_num);
+
+    command_runner<out<float>> work_runner;
+    auto r     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
+    auto p     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
+    auto w     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
+    auto z     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
+    auto D_inv = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
+    auto y_tmp = work_runner.transfer_memory(device_num, stream_id, create_out_arg_with_size<float>(1));
+
+    mem_ptr<char> spmv_workspace;
+    size_t ws_size = 0;
+
+    if (fmt == matrix_format::csr)
+      ws_size = d_ptr->spmv_csr_buffer_size(stream_id, n, n, nnz,
+                                            A_row_ptr, A_col_ind, A_values, x, w);
+    else if (fmt == matrix_format::csc)
+      ws_size = d_ptr->spmv_csc_buffer_size(stream_id, n, n, nnz,
+                                            A_row_ptr, A_col_ind, A_values, x, w);
+    else if (fmt == matrix_format::coo)
+      ws_size = d_ptr->spmv_coo_buffer_size(stream_id, n, n, nnz,
+                                            A_row_ptr, A_col_ind, A_values, x, w);
+
+    if (ws_size > 0) {
+      command_runner<out<char>> ws_runner;
+      spmv_workspace =
+        ws_runner.transfer_memory(device_num, stream_id,
+                                  out<char>(static_cast<int>(ws_size)));
+    }
+
+    auto execute_spmv = [&](mem_ptr<float> input_v, mem_ptr<float> output_v) {
+      switch (fmt) {
+        case matrix_format::csr:
+          d_ptr->spmv_csr(stream_id, n, n, nnz, 1.0f,
+                          A_row_ptr, A_col_ind, A_values,
+                          input_v, 0.0f, output_v, spmv_workspace);
+          break;
+        case matrix_format::csc:
+          d_ptr->spmv_csc(stream_id, n, n, nnz, 1.0f,
+                          A_row_ptr, A_col_ind, A_values,
+                          input_v, 0.0f, output_v, spmv_workspace);
+          break;
+        case matrix_format::coo:
+          d_ptr->spmv_coo(stream_id, n, n, nnz, 1.0f,
+                          A_row_ptr, A_col_ind, A_values,
+                          input_v, 0.0f, output_v, spmv_workspace);
+          break;
+        default:
+          break;
+      }
+    };
+
+    // 0. Preconditioning setup: Extract diagonal inverse using custom kernel
+    int threads = 256;
+    int blocks = (n + threads - 1) / threads;
+    nd_range range(blocks, 1, 1, threads, 1, 1);
+
+    d_ptr->launch_kernel_mem_ref(diag_prog_->get_kernel(d_ptr->getId()), range,
+                                 std::make_tuple(in<int>(n), A_row_ptr, A_col_ind, A_values, D_inv),
+                                 stream_id);
+
+    // 1. Initial Residual: r = b - Ax
+    execute_spmv(x, w);
+    d_ptr->scopy(stream_id, n, b, r);
+    d_ptr->saxpy(stream_id, n, -1.0f, w, r);
+    
+    // 2. Initial Preconditioned Residual: z = D_inv * r
+    d_ptr->s_elementwise_multiply(stream_id, n, D_inv, r, z);
+    
+    // 3. Initial rho = r * z
+    d_ptr->sdot(stream_id, n, r, z, y_tmp);
+
+    float rho_val = runner.copy_to_host(y_tmp)[0];
+    float old_rho_val = 0.0f;
+    int iterations = 0;
+
+    while (rho_val > (tol * tol) && iterations < max_iter) {
+      iterations++;
+
+      if (iterations > 1) {
+        float beta_val = rho_val / old_rho_val;
+        // p = z + beta * p
+        d_ptr->scopy(stream_id, n, z, w);
+        d_ptr->saxpy(stream_id, n, beta_val, p, w);
+        d_ptr->scopy(stream_id, n, w, p);
+      } else {
+        // p = z
+        d_ptr->scopy(stream_id, n, z, p);
+      }
+
+      execute_spmv(p, w);
+      d_ptr->sdot(stream_id, n, p, w, y_tmp);
+      float alpha_val = rho_val / runner.copy_to_host(y_tmp)[0];
+
+      d_ptr->saxpy(stream_id, n, alpha_val, p, x);
+      d_ptr->saxpy(stream_id, n, -alpha_val, w, r);
+
+      old_rho_val = rho_val;
+      d_ptr->s_elementwise_multiply(stream_id, n, D_inv, r, z); // z_new = D_inv * r
+      d_ptr->sdot(stream_id, n, r, z, y_tmp); // rho_new = r * z_new
+      rho_val = runner.copy_to_host(y_tmp)[0];
+    }
+
+    return x;
+  }
+
+private:
+  program_ptr diag_prog_;
+};
+
 } // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 3241292767..63fbd3b779 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -173,6 +173,23 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasSgemv failed on device " + std::to_string(id_));
   }
 
+  /// Performs element-wise multiplication of two vectors: result = x .* y.
+  /// This is implemented using cublasSdgmm (Diagonal Matrix-Vector Multiplication).
+  void s_elementwise_multiply(int stream_id, int n, mem_ptr<float> x, mem_ptr<float> y, mem_ptr<float> result) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
+    if (!handle) throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    // cublasSdgmm: C = diag(X) * Y. When Y is a vector (n x 1), this is element-wise mult.
+    cublasStatus_t status = cublasSdgmm(handle, CUBLAS_SIDE_LEFT, n, 1,
+                                        reinterpret_cast<const float*>(x->mem()), n,
+                                        reinterpret_cast<const float*>(y->mem()), n,
+                                        reinterpret_cast<float*>(result->mem()), n);
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasSdgmm failed on device " + std::to_string(id_));
+  }
+
   /// Returns the required buffer size for SpMV CSR.
   template <class T>
   size_t spmv_csr_buffer_size(int stream_id, int m, int n, int nnz, 

From 52336e96f1dabac81904c34477b9cca16da8a79f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 10:45:12 -0600
Subject: [PATCH 0763/1000] updated Cmakelists to auto compile .cu files found
 in the caf directoryu Change is being made so we can premake certain
 functionallity easily

---
 libcaf_cuda/CMakeLists.txt | 40 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 38 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/CMakeLists.txt b/libcaf_cuda/CMakeLists.txt
index 8a95fcf535..d1282578e2 100644
--- a/libcaf_cuda/CMakeLists.txt
+++ b/libcaf_cuda/CMakeLists.txt
@@ -1,10 +1,46 @@
 #project(caf_opencl C CXX)
 include(CMakePrintHelpers)
 
-
-#get the cuda related dependencies 
+# Get the cuda related dependencies first so we have the nvcc path
 find_package(CUDAToolkit REQUIRED)
 
+# Set the GPU architecture. 'native' detects the local GPU capability.
+# You can override this with -DCAF_CUDA_ARCH=sm_80 for example.
+set(CAF_CUDA_ARCH "native" CACHE STRING "Target CUDA architecture (e.g., sm_70, native)")
+
+# Find all .cu files recursively in the caf/ directory relative to this CMakeLists.txt
+file(GLOB_RECURSE CAF_CUDA_KERNELS "${CMAKE_CURRENT_SOURCE_DIR}/caf/*.cu")
+
+foreach(CU_FILE ${CAF_CUDA_KERNELS})
+    # Get the base name of the .cu file (e.g., "some_kernel")
+    get_filename_component(BASENAME ${CU_FILE} NAME_WE)
+    # Get the directory of the .cu file (e.g., "/path/to/libcaf_cuda/caf/cuda")
+    get_filename_component(DIRNAME ${CU_FILE} DIRECTORY)
+    
+    # Construct the output .cubin file path (e.g., "/path/to/libcaf_cuda/caf/cuda/some_kernel.cubin")
+    set(CUBIN_FILE "${DIRNAME}/${BASENAME}.cubin")
+
+    # Create a unique target name based on the relative path to avoid collisions
+    file(RELATIVE_PATH REL_PATH "${CMAKE_CURRENT_SOURCE_DIR}/caf" ${CU_FILE})
+    string(MAKE_C_IDENTIFIER "cubin_${REL_PATH}" TARGET_NAME)
+
+    # Add a custom command to compile the .cu file into a .cubin
+    add_custom_command(
+        OUTPUT ${CUBIN_FILE}
+        COMMAND ${CUDAToolkit_NVCC_EXECUTABLE}
+            -cubin
+            -arch=${CAF_CUDA_ARCH}
+            -o ${CUBIN_FILE}
+            ${CU_FILE}
+        DEPENDS ${CU_FILE}
+        COMMENT "Compiling CUDA kernel ${CU_FILE} to ${CUBIN_FILE}"
+        VERBATIM
+    )
+    
+    # Add a custom target to ensure the cubin is built.
+    add_custom_target(${TARGET_NAME} ALL DEPENDS ${CUBIN_FILE})
+endforeach()
+
 # get header files; only needed by CMake generators,
 # e.g., for creating proper Xcode projects
 file(GLOB_RECURSE CAF_CUDA_HEADERS "caf/*.hpp")

From 482b7bb70a156c5fb3bdaf87be71e54d084b98f4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 10:50:52 -0600
Subject: [PATCH 0764/1000] fixed bug in s_elementwise_multiply

---
 libcaf_cuda/caf/cuda/device.hpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 63fbd3b779..d8f983d7f3 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -181,9 +181,10 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
 
     CHECK_CUDA(cuCtxPushCurrent(context_));
     // cublasSdgmm: C = diag(X) * Y. When Y is a vector (n x 1), this is element-wise mult.
-    cublasStatus_t status = cublasSdgmm(handle, CUBLAS_SIDE_LEFT, n, 1,
-                                        reinterpret_cast<const float*>(x->mem()), n,
+    cublasStatus_t status = cublasSdgmm(handle, CUBLAS_SIDE_LEFT,
+                                        n, 1,
                                         reinterpret_cast<const float*>(y->mem()), n,
+                                        reinterpret_cast<const float*>(x->mem()), 1,
                                         reinterpret_cast<float*>(result->mem()), n);
     CHECK_CUDA(cuCtxPopCurrent(nullptr));
     if (status != CUBLAS_STATUS_SUCCESS)

From e1abe38e0ddf79b5aeabf38f2b336bbe03252a41 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 10:51:28 -0600
Subject: [PATCH 0765/1000] updated tests to test the jacobi actor facade

---
 .../CGS-actor-test/main.test.cpp              | 111 ++++++++++++++++++
 1 file changed, 111 insertions(+)

diff --git a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
index 6e85495321..bd4359a4ee 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
@@ -248,6 +248,117 @@ void caf_main(actor_system& sys) {
         );
     }
 
+    // Test 9: Jacobi Facade Actor CSR Simple
+    {
+        std::cout << "\n[INFO] Test 9: Jacobi Facade actor CSR simple matrix..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_cg_jacobi_facade>(200);
+
+        self->mail(std::vector<output_mapping>{},
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, 0, 8).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+                verify_solution("Jacobi Facade CSR Simple", result_x, expected);
+            }
+        );
+    }
+
+    // Test 10: Jacobi Facade Actor CSR with Custom Buffer
+    {
+        std::cout << "\n[INFO] Test 10: Jacobi Facade actor CSR with custom buffer..." << std::endl;
+        std::vector<float> custom_x(n, 0.0f);
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        output_mapping m{4, custom_x.data(), (size_t)n};
+        
+        auto facade = sys.spawn<sparse_cg_jacobi_facade>(200);
+        self->mail(std::vector<output_mapping>{m},
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(custom_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, 0, 9).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int index) {
+                if (index == 4)
+                  verify_solution("Jacobi Facade Custom Buffer", custom_x, expected);
+            }
+        );
+    }
+
+    // Test 11: Jacobi Facade Actor CSR returning mem_ptr handles
+    {
+        std::cout << "\n[INFO] Test 11: Jacobi Facade actor CSR returning mem_ptr..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_cg_jacobi_facade>(200);
+
+        self->mail(return_mem_ptr_atom_v,
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, 0, 10).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr) {
+                auto result_x = ptr->copy_to_host();
+                verify_solution("Jacobi Facade mem_ptr", result_x, expected);
+            }
+        );
+    }
+
+    // Test 12: Jacobi Facade Actor Default (No mapping vector)
+    {
+        std::cout << "\n[INFO] Test 12: Jacobi Facade actor default (no mapping vector)..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_cg_jacobi_facade>(200);
+
+        self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, 0, 11).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+                verify_solution("Jacobi Facade Default", result_x, expected);
+            }
+        );
+    }
+
+    // Test 13: Jacobi Facade Actor CSC Simple
+    {
+        std::cout << "\n[INFO] Test 13: Jacobi Facade actor CSC simple matrix..." << std::endl;
+        std::vector<int> col_ptr = {0, 1, 2, 3};
+        std::vector<int> row_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_cg_jacobi_facade>(200);
+
+        self->mail(std::vector<output_mapping>{},
+                   create_in_arg(col_ptr), create_in_arg(row_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csc, n, nnz, tolerance, max_iter, 0, 12).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+                verify_solution("Jacobi Facade CSC Simple", result_x, expected);
+            }
+        );
+    }
+
     manager::shutdown();
 }
 

From 66bd1d9b65769258fc63db85d1e528260396caa3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 10:58:34 -0600
Subject: [PATCH 0766/1000] implemented jacobi preconditioner solver

---
 .../sparse-BiCGSTAB-actor/jacobi_kernels.cu   |  15 ++
 .../sparse-BiCGSTAB-actor.hpp                 | 152 ++++++++++++++++++
 2 files changed, 167 insertions(+)
 create mode 100644 libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/jacobi_kernels.cu

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/jacobi_kernels.cu b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/jacobi_kernels.cu
new file mode 100644
index 0000000000..16c17e7352
--- /dev/null
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/jacobi_kernels.cu
@@ -0,0 +1,15 @@
+extern "C" __global__
+void extract_diag_inv(int n, const int* row_ptr, const int* col_ind, const float* val, float* d_inv) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        float d = 0.0f;
+        // Search for the diagonal element (A[i][i]) in the sparse row
+        for (int j = row_ptr[i]; j < row_ptr[i+1]; j++) {
+            if (col_ind[j] == i) {
+                d = val[j];
+                break;
+            }
+        }
+        d_inv[i] = (d != 0.0f) ? 1.0f / d : 1.0f;
+    }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
index 9a63ec7a44..df55c26297 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
@@ -460,4 +460,156 @@ class sparse_bicgstab_facade : public event_based_actor {
   uint32_t reply_id_;
 };
 
+/**
+ * A variant of the BiCGSTAB solver facade that uses Jacobi preconditioning.
+ */
+template <class T = float>
+class sparse_bicgstab_jacobi_facade : public sparse_bicgstab_facade<T> {
+public:
+  sparse_bicgstab_jacobi_facade(actor_config& cfg, uint32_t response_id)
+    : sparse_bicgstab_facade<T>(cfg, response_id) {
+    // Deduce path to cubin relative to this header file at runtime
+    std::string current_file = __FILE__;
+    auto pos = current_file.find_last_of('/');
+    std::string dir = (pos == std::string::npos) ? "" : current_file.substr(0, pos + 1);
+    auto& mgr = manager::get();
+    diag_prog_ = mgr.create_program_from_cubin(dir + "jacobi_kernels.cubin", "extract_diag_inv");
+  }
+
+protected:
+  mem_ptr<T> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in,
+                        in_out<T> x_in,
+                        matrix_format fmt, int n, int nnz,
+                        float tol, int max_iter,
+                        int device_num, int stream_id) override {
+
+    command_runner<T> runner;
+    auto res = runner.transfer_memory(device_num, stream_id,
+                                      rp, ci, val, b_in, x_in);
+
+    auto A_row_ptr = std::get<0>(res);
+    auto A_col_ind = std::get<1>(res);
+    auto A_values  = std::get<2>(res);
+    auto b         = std::get<3>(res);
+    auto x         = std::get<4>(res);
+
+    auto d_ptr = platform::create()->schedule(stream_id, device_num);
+
+    command_runner<out<T>> work_runner;
+    auto r     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto r_hat = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto p     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto v     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto s_vec = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto t_vec = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto D_inv = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto p_hat = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto s_hat = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto y_tmp = work_runner.transfer_memory(device_num, stream_id, out<T>(1));
+
+    mem_ptr<char> spmv_workspace;
+    size_t ws_size = 0;
+
+    if (fmt == matrix_format::csr)
+      ws_size = d_ptr->spmv_csr_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, v);
+    else if (fmt == matrix_format::csc)
+      ws_size = d_ptr->spmv_csc_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, v);
+    else if (fmt == matrix_format::coo)
+      ws_size = d_ptr->spmv_coo_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, v);
+
+    if (ws_size > 0) {
+      command_runner<out<char>> ws_runner;
+      spmv_workspace = ws_runner.transfer_memory(device_num, stream_id, out<char>(static_cast<int>(ws_size)));
+    }
+
+    auto execute_spmv = [&](mem_ptr<T> input_v, mem_ptr<T> output_v) {
+      switch (fmt) {
+        case matrix_format::csr: d_ptr->spmv_csr(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, input_v, T{0}, output_v, spmv_workspace); break;
+        case matrix_format::csc: d_ptr->spmv_csc(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, input_v, T{0}, output_v, spmv_workspace); break;
+        case matrix_format::coo: d_ptr->spmv_coo(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, input_v, T{0}, output_v, spmv_workspace); break;
+        default: break;
+      }
+    };
+    auto execute_copy = [&](mem_ptr<T> src, mem_ptr<T> dst) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->dcopy(stream_id, n, src, dst);
+      else d_ptr->scopy(stream_id, n, src, dst);
+    };
+    auto execute_axpy = [&](T alpha, mem_ptr<T> ax, mem_ptr<T> ay) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->daxpy(stream_id, n, alpha, ax, ay);
+      else d_ptr->saxpy(stream_id, n, static_cast<float>(alpha), ax, ay);
+    };
+    auto execute_dot = [&](mem_ptr<T> dx, mem_ptr<T> dy, mem_ptr<T> dres) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->ddot(stream_id, n, dx, dy, dres);
+      else d_ptr->sdot(stream_id, n, dx, dy, dres);
+    };
+
+    // Preconditioning setup: Extract diagonal inverse
+    int threads = 256;
+    int blocks = (n + threads - 1) / threads;
+    nd_range range(blocks, 1, 1, threads, 1, 1);
+    d_ptr->launch_kernel_mem_ref(diag_prog_->get_kernel(d_ptr->getId()), range,
+                                 std::make_tuple(in<int>(n), A_row_ptr, A_col_ind, A_values, D_inv),
+                                 stream_id);
+
+    // Initial Residual: r = b - Ax
+    execute_spmv(x, v); 
+    execute_copy(b, r);
+    execute_axpy(T{-1}, v, r);
+    execute_copy(r, r_hat);
+    execute_dot(r, r, y_tmp);
+    T norm_sq = runner.copy_to_host(y_tmp)[0];
+
+    int iterations = 0;
+    T rho_val = T{1}, alpha_val = T{1}, omega_val = T{1}, beta_val = T{0};
+
+    while (norm_sq > (tol * tol) && iterations < max_iter) {
+      iterations++;
+      execute_dot(r_hat, r, y_tmp);
+      T rho_new = runner.copy_to_host(y_tmp)[0];
+
+      if (iterations == 1) {
+        execute_copy(r, p);
+      } else {
+        beta_val = (rho_new / rho_val) * (alpha_val / omega_val);
+        execute_axpy(-omega_val, v, p);
+        execute_copy(r, s_vec);
+        execute_axpy(beta_val, p, s_vec);
+        execute_copy(s_vec, p);
+      }
+      rho_val = rho_new;
+
+      // Apply Preconditioner: p_hat = D_inv * p
+      if constexpr (std::is_same_v<T, float>) d_ptr->s_elementwise_multiply(stream_id, n, D_inv, p, p_hat);
+      else execute_copy(p, p_hat); // Fallback if double elementwise mult is not in device.hpp
+
+      execute_spmv(p_hat, v);
+      execute_dot(r_hat, v, y_tmp);
+      T alpha_denom = runner.copy_to_host(y_tmp)[0];
+      alpha_val = rho_val / alpha_denom;
+      execute_copy(r, s_vec);
+      execute_axpy(-alpha_val, v, s_vec);
+      
+      // Apply Preconditioner: s_hat = D_inv * s
+      if constexpr (std::is_same_v<T, float>) d_ptr->s_elementwise_multiply(stream_id, n, D_inv, s_vec, s_hat);
+      else execute_copy(s_vec, s_hat);
+
+      execute_spmv(s_hat, t_vec);
+      execute_dot(t_vec, s_hat, y_tmp);
+      T omega_num = runner.copy_to_host(y_tmp)[0];
+      execute_dot(t_vec, t_vec, y_tmp);
+      T omega_denom = runner.copy_to_host(y_tmp)[0];
+      omega_val = omega_num / omega_denom;
+      execute_axpy(alpha_val, p_hat, x);
+      execute_axpy(omega_val, s_hat, x);
+      execute_copy(s_vec, r);
+      execute_axpy(-omega_val, t_vec, r);
+      execute_dot(r, r, y_tmp);
+      norm_sq = runner.copy_to_host(y_tmp)[0];
+    }
+    return x;
+  }
+private:
+  program_ptr diag_prog_;
+};
+
 } // namespace caf::cuda
\ No newline at end of file

From 9207f72de443018d046b5315975f58ff9c23867a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 11:15:08 -0600
Subject: [PATCH 0767/1000] modified jacobi actor to pass tests

---
 .../sparse-BiCGSTAB-actor.hpp                 | 54 +++++++----
 libcaf_cuda/caf/cuda/device.hpp               | 19 ++++
 .../BICGSTAB-actor-test/main.test.cpp         | 90 +++++++++++++++++++
 3 files changed, 145 insertions(+), 18 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
index df55c26297..78cf2b24cd 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
@@ -25,7 +25,7 @@ struct sparse_bicgstab_state {
   mem_ptr<T> A_values, b, x;
   matrix_format format;
   int n, nnz;
-  float tol;
+  T tol;
   int max_iter;
   int device_num;
   int stream_id;
@@ -47,9 +47,10 @@ struct sparse_bicgstab_state {
 template <class T = float>
 class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state<T>> {
 public:
-  sparse_bicgstab_actor(actor_config& cfg, in<int> rp, in<int> ci,
-                        in<T> val, in<T> b, in_out<T> x,
-                        matrix_format fmt, int n, int nnz, float tol, int max_iter, int device_num, int stream_id,
+  sparse_bicgstab_actor(actor_config& cfg, in<int> rp, in<int> ci, in<T> val,
+                        in<T> b, in_out<T> x, matrix_format fmt, int n,
+                        int nnz, T tol, int max_iter, int device_num,
+                        int stream_id,
                         caf::actor supervisor = nullptr)
     : stateful_actor<sparse_bicgstab_state<T>>(cfg) {
     this->state().h_row_ptr = std::move(rp);
@@ -174,6 +175,13 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state<T>> {
       execute_copy(s.r, s.s_vec);
       execute_axpy(-s.alpha_val, s.v, s.s_vec);
 
+      // Convergence check: Exit if the solution is reached after the alpha update
+      execute_dot(s.s_vec, s.s_vec, s.y_tmp);
+      if (runner.copy_to_host(s.y_tmp)[0] < (s.tol * s.tol)) {
+        execute_axpy(s.alpha_val, s.p, s.x);
+        break;
+      }
+
       // t = As
       execute_spmv(s.s_vec, s.t_vec);
 
@@ -269,7 +277,7 @@ class sparse_bicgstab_facade : public event_based_actor {
              in<int> rp, in<int> ci, in<T> val,
              in<T> b_in, in_out<T> x_in,
              matrix_format fmt, int n, int nnz,
-             float tol, int max_iter,
+             T tol, int max_iter,
              int device_num, int stream_id) {
 
         auto x = solve_core(rp, ci, val, b_in, x_in,
@@ -286,7 +294,7 @@ class sparse_bicgstab_facade : public event_based_actor {
              in<int> rp, in<int> ci, in<T> val,
              in<T> b_in, in_out<T> x_in,
              matrix_format fmt, int n, int nnz,
-             float tol, int max_iter,
+             T tol, int max_iter,
              int device_num, int stream_id) {
 
         auto x = solve_core(rp, ci, val, b_in, x_in,
@@ -298,7 +306,7 @@ class sparse_bicgstab_facade : public event_based_actor {
 
       // Mode 3: Default (return vector to sender)
       [this](in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
-             matrix_format fmt, int n, int nnz, float tol, int max_iter,
+             matrix_format fmt, int n, int nnz, T tol, int max_iter,
              int device_num, int stream_id) {
 
         auto x = solve_core(rp, ci, val, b_in, x_in,
@@ -311,11 +319,10 @@ class sparse_bicgstab_facade : public event_based_actor {
   }
 
 protected:
-  mem_ptr<T> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in,
-                        in_out<T> x_in,
-                        matrix_format fmt, int n, int nnz,
-                        float tol, int max_iter,
-                        int device_num, int stream_id) {
+  virtual mem_ptr<T> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in,
+                                in_out<T> x_in, matrix_format fmt, int n,
+                                int nnz, T tol, int max_iter,
+                                int device_num, int stream_id) {
     command_runner<T> runner;
     auto res = runner.transfer_memory(device_num, stream_id,
                                       rp, ci, val, b_in, x_in);
@@ -480,7 +487,7 @@ class sparse_bicgstab_jacobi_facade : public sparse_bicgstab_facade<T> {
   mem_ptr<T> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in,
                         in_out<T> x_in,
                         matrix_format fmt, int n, int nnz,
-                        float tol, int max_iter,
+                        T tol, int max_iter,
                         int device_num, int stream_id) override {
 
     command_runner<T> runner;
@@ -579,8 +586,10 @@ class sparse_bicgstab_jacobi_facade : public sparse_bicgstab_facade<T> {
       rho_val = rho_new;
 
       // Apply Preconditioner: p_hat = D_inv * p
-      if constexpr (std::is_same_v<T, float>) d_ptr->s_elementwise_multiply(stream_id, n, D_inv, p, p_hat);
-      else execute_copy(p, p_hat); // Fallback if double elementwise mult is not in device.hpp
+      if constexpr (std::is_same_v<T, float>)
+        d_ptr->s_elementwise_multiply(stream_id, n, D_inv, p, p_hat);
+      else
+        d_ptr->d_elementwise_multiply(stream_id, n, D_inv, p, p_hat);
 
       execute_spmv(p_hat, v);
       execute_dot(r_hat, v, y_tmp);
@@ -589,12 +598,21 @@ class sparse_bicgstab_jacobi_facade : public sparse_bicgstab_facade<T> {
       execute_copy(r, s_vec);
       execute_axpy(-alpha_val, v, s_vec);
       
+      // Convergence check: Exit if the solution is reached after the alpha update
+      execute_dot(s_vec, s_vec, y_tmp);
+      if (runner.copy_to_host(y_tmp)[0] < (tol * tol)) {
+        execute_axpy(alpha_val, p_hat, x);
+        return x;
+      }
+
       // Apply Preconditioner: s_hat = D_inv * s
-      if constexpr (std::is_same_v<T, float>) d_ptr->s_elementwise_multiply(stream_id, n, D_inv, s_vec, s_hat);
-      else execute_copy(s_vec, s_hat);
+      if constexpr (std::is_same_v<T, float>)
+        d_ptr->s_elementwise_multiply(stream_id, n, D_inv, s_vec, s_hat);
+      else
+        d_ptr->d_elementwise_multiply(stream_id, n, D_inv, s_vec, s_hat);
 
       execute_spmv(s_hat, t_vec);
-      execute_dot(t_vec, s_hat, y_tmp);
+      execute_dot(t_vec, s_vec, y_tmp); // Corrected: Numerator uses unpreconditioned residual s
       T omega_num = runner.copy_to_host(y_tmp)[0];
       execute_dot(t_vec, t_vec, y_tmp);
       T omega_denom = runner.copy_to_host(y_tmp)[0];
diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index d8f983d7f3..073374d761 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -191,6 +191,25 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasSdgmm failed on device " + std::to_string(id_));
   }
 
+  /// Performs element-wise multiplication of two vectors: result = x .* y.
+  /// This is implemented using cublasDdgmm (Diagonal Matrix-Vector Multiplication).
+  void d_elementwise_multiply(int stream_id, int n, mem_ptr<double> x, mem_ptr<double> y, mem_ptr<double> result) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
+    if (!handle) throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    // cublasDdgmm: C = diag(X) * Y. When Y is a vector (n x 1), this is element-wise mult.
+    cublasStatus_t status = cublasDdgmm(handle, CUBLAS_SIDE_LEFT,
+                                        n, 1,
+                                        reinterpret_cast<const double*>(y->mem()), n,
+                                        reinterpret_cast<const double*>(x->mem()), 1,
+                                        reinterpret_cast<double*>(result->mem()), n);
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasDdgmm failed on device " + std::to_string(id_));
+  }
+
+
   /// Returns the required buffer size for SpMV CSR.
   template <class T>
   size_t spmv_csr_buffer_size(int stream_id, int m, int n, int nnz, 
diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
index 729c698166..9839024571 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
@@ -541,6 +541,96 @@ void caf_main(actor_system& sys) {
         );
     }
 
+    // Test 14: Jacobi Facade Actor CSR Simple
+    {
+        std::cout << "\n[INFO] Test 14: Jacobi Facade actor CSR simple matrix..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_bicgstab_jacobi_facade<float>>(200);
+
+        self->mail(std::vector<output_mapping>{},
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, 0, 13).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+                verify_solution("Jacobi Facade CSR Simple", result_x, expected);
+            }
+        );
+    }
+
+    // Test 15: Jacobi Facade Actor CSR with Custom Buffer
+    {
+        std::cout << "\n[INFO] Test 15: Jacobi Facade actor CSR with custom buffer..." << std::endl;
+        std::vector<float> custom_x(n, 0.0f);
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        output_mapping m{4, custom_x.data(), (size_t)n};
+        
+        auto facade = sys.spawn<sparse_bicgstab_jacobi_facade<float>>(200);
+        self->mail(std::vector<output_mapping>{m},
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(custom_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, 0, 14).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int index) {
+                if (index == 4)
+                  verify_solution("Jacobi Facade Custom Buffer", custom_x, expected);
+            }
+        );
+    }
+
+    // Test 16: Jacobi Facade Actor CSR returning mem_ptr handles
+    {
+        std::cout << "\n[INFO] Test 16: Jacobi Facade actor CSR returning mem_ptr..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_bicgstab_jacobi_facade<float>>(200);
+
+        self->mail(return_mem_ptr_atom_v,
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, 0, 15).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr) {
+                auto result_x = ptr->copy_to_host();
+                verify_solution("Jacobi Facade mem_ptr", result_x, expected);
+            }
+        );
+    }
+
+    // Test 17: Jacobi Facade Actor CSC Simple
+    {
+        std::cout << "\n[INFO] Test 17: Jacobi Facade actor CSC simple matrix..." << std::endl;
+        std::vector<int> col_ptr = {0, 1, 2, 3};
+        std::vector<int> row_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_bicgstab_jacobi_facade<float>>(200);
+
+        self->mail(std::vector<output_mapping>{},
+                   create_in_arg(col_ptr), create_in_arg(row_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csc, n, nnz, tolerance, max_iter, 0, 16).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+                verify_solution("Jacobi Facade CSC Simple", result_x, expected);
+            }
+        );
+    }
+
     manager::shutdown();
 }
 CAF_MAIN(id_block::cuda, id_block::cg_actor)

From 56bd1f3fc8d04dbb647219c64a2270182e11d6a7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 11:16:51 -0600
Subject: [PATCH 0768/1000] added more tests

---
 .../BICGSTAB-actor-test/main.test.cpp         | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
index 9839024571..a035a432fb 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
@@ -631,6 +631,32 @@ void caf_main(actor_system& sys) {
         );
     }
 
+    // Test 18: Jacobi Facade Actor Real Matrix
+    {
+        std::string path = "/scratch/nqr159/matrix-collection/matrices/unsymmetric/lnsp3937.bin";
+        std::cout << "\n[INFO] Test 18: Jacobi Facade actor real-world matrix " << path << "..." << std::endl;
+        try {
+            LocalCSR<float> A = load_binary_matrix_manual<float>(path);
+            std::vector<float> expected_real(A.rows, 1.0f);
+            std::vector<float> b_real = compute_rhs_manual<float>(A, expected_real);
+            std::vector<float> x_real(A.rows, 0.0f);
+
+            auto facade = sys.spawn<sparse_bicgstab_jacobi_facade<float>>(200);
+
+            self->mail(create_in_arg(A.row_ptr), create_in_arg(A.col_ind), create_in_arg(A.values),
+                       create_in_arg(b_real), create_in_out_arg(x_real),
+                       matrix_format::csr, A.rows, A.nnz, 1e-5f, 5000, 0, 17).send(facade);
+
+            self->receive(
+                [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result) {
+                    verify_solution("Jacobi Facade Real Matrix", result, expected_real, 1e-2f);
+                }
+            );
+        } catch (const std::exception& e) {
+            std::cout << "[ERROR] Test 18 Failed: " << e.what() << std::endl;
+        }
+    }
+
     manager::shutdown();
 }
 CAF_MAIN(id_block::cuda, id_block::cg_actor)

From ad15c8ef592722b5458dbe6d835b4d411e36c92a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 12:12:52 -0600
Subject: [PATCH 0769/1000] Updated cmakelists to compile correctly.

---
 .../benchmark-tests/mmul-actor-benchmarking/CMakeLists.txt      | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/CMakeLists.txt
index 89bcf5ba7c..59d7388f4a 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/CMakeLists.txt
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/CMakeLists.txt
@@ -39,6 +39,8 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
 )
 
 

From 1124f8b2881c205c7b3c103394354b4b4861f7a8 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 12:14:46 -0600
Subject: [PATCH 0770/1000] Fixed build.

---
 .../baseline-comparison/actors/CMakeLists.txt                   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/CMakeLists.txt
index d79166c1fb..e0377a78b2 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/CMakeLists.txt
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/CMakeLists.txt
@@ -39,6 +39,8 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
 )
 
 

From dc83f2ddab7f0ab3ea7be9602cbbf80b1f85dcae Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 14:31:11 -0600
Subject: [PATCH 0771/1000] created initial version of GMRES actor

---
 .../sparse-GMRES-actor/sparse-GMRES-actor.hpp | 285 ++++++++++++++++++
 1 file changed, 285 insertions(+)
 create mode 100644 libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp
new file mode 100644
index 0000000000..821d9dc8bf
--- /dev/null
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp
@@ -0,0 +1,285 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <vector>
+#include <cmath>
+#include <algorithm>
+#include "caf/cuda/all.hpp"
+#include "caf/cuda/platform.hpp"
+
+namespace caf::cuda {
+
+/**
+ * State for the GMRES solver.
+ * GMRES(m) requires storing m basis vectors, making it memory-intensive.
+ */
+template <class T = float>
+struct sparse_gmres_state {
+  // Host Data
+  in<int> h_row_ptr, h_col_ind;
+  in<T> h_values, h_b;
+  in_out<T> h_x;
+ 
+  // Device Problem data
+  mem_ptr<int> A_row_ptr, A_col_ind;
+  mem_ptr<T> A_values, b, x;
+  matrix_format format;
+  int n, nnz;
+  T tol;
+  int max_iter;
+  int m; // Restart parameter (Krylov subspace dimension)
+  int device_num;
+  int stream_id;
+  device_ptr d_ptr;
+  caf::actor supervisor;
+
+  // Workspace
+  std::vector<mem_ptr<T>> V; // Krylov basis vectors v_1 ... v_{m+1}
+  mem_ptr<T> w;              // Temporary vector for Arnoldi
+  mem_ptr<T> y_tmp;          // Scalar workspace (dot products/norms)
+  mem_ptr<char> spmv_workspace;
+
+  int iterations = 0;
+};
+
+template <class T = float>
+class sparse_gmres_actor : public stateful_actor<sparse_gmres_state<T>> {
+public:
+  sparse_gmres_actor(actor_config& cfg, in<int> rp, in<int> ci, in<T> val,
+                     in<T> b, in_out<T> x, matrix_format fmt, int n,
+                     int nnz, T tol, int max_iter, int m, int device_num,
+                     int stream_id, caf::actor supervisor = nullptr)
+    : stateful_actor<sparse_gmres_state<T>>(cfg) {
+    auto& s = this->state();
+    s.h_row_ptr = std::move(rp);
+    s.h_col_ind = std::move(ci);
+    s.h_values = std::move(val);
+    s.h_b = std::move(b);
+    s.h_x = std::move(x);
+    s.format = fmt;
+    s.n = n; s.nnz = nnz;
+    s.tol = tol; s.max_iter = max_iter; s.m = m;
+    s.device_num = device_num; s.stream_id = stream_id;
+    s.supervisor = supervisor;
+  }
+
+  behavior make_behavior() override {
+    return {
+      [this](start_atom) {
+        if (!this->state().supervisor)
+          this->state().supervisor = actor_cast<caf::actor>(this->current_sender());
+        start_solve();
+      }
+    };
+  }
+
+private:
+  void start_solve() {
+    // Implementation would mirror facade's solve_core logic for consistency.
+    // For brevity in this combined file, the core logic is encapsulated in the facade's 
+    // virtual method which can be called by both.
+  }
+};
+
+/**
+ * Stateless facade for the GMRES(m) solver.
+ */
+template <class T = float>
+class sparse_gmres_facade : public event_based_actor {
+public:
+  sparse_gmres_facade(actor_config& cfg, uint32_t response_id)
+    : event_based_actor(cfg), reply_id_(response_id) {}
+
+  behavior make_behavior() override {
+    return {
+      // Mode 1: Return mem_ptr handles (GPU memory)
+      [this](return_mem_ptr_atom, in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+             matrix_format fmt, int n, int nnz, T tol, int max_iter, int m,
+             int device_num, int stream_id) {
+        auto x = solve_core(rp, ci, val, b_in, x_in, fmt, n, nnz, tol, max_iter, m, device_num, stream_id);
+        if (auto sender = actor_cast<actor>(this->current_sender()))
+          caf::anon_mail(reply_id_, std::move(x)).send(sender);
+      },
+
+      // Mode 2: Return host data via mappings
+      [this](std::vector<output_mapping> mappings, in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+             matrix_format fmt, int n, int nnz, T tol, int max_iter, int m,
+             int device_num, int stream_id) {
+        auto x = solve_core(rp, ci, val, b_in, x_in, fmt, n, nnz, tol, max_iter, m, device_num, stream_id);
+        dispatch_result(std::move(mappings), std::move(x), n);
+      },
+
+      // Mode 3: Default (return vector to sender)
+      [this](in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+             matrix_format fmt, int n, int nnz, T tol, int max_iter, int m,
+             int device_num, int stream_id) {
+        auto x = solve_core(rp, ci, val, b_in, x_in, fmt, n, nnz, tol, max_iter, m, device_num, stream_id);
+        dispatch_result({}, std::move(x), n);
+      }
+    };
+  }
+
+protected:
+  virtual mem_ptr<T> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+                                matrix_format fmt, int n, int nnz, T tol, int max_iter, int m,
+                                int device_num, int stream_id) {
+    command_runner<T> runner;
+    auto res = runner.transfer_memory(device_num, stream_id, rp, ci, val, b_in, x_in);
+    auto A_row_ptr = std::get<0>(res), A_col_ind = std::get<1>(res), A_values = std::get<2>(res);
+    auto b = std::get<3>(res), x = std::get<4>(res);
+
+    auto d_ptr = platform::create()->schedule(stream_id, device_num);
+    command_runner<out<T>> work_runner;
+    auto w = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto y_tmp = work_runner.transfer_memory(device_num, stream_id, out<T>(1));
+    
+    // Basis vectors V_1...V_{m+1}
+    std::vector<mem_ptr<T>> V;
+    for (int i = 0; i <= m; ++i) 
+      V.push_back(work_runner.transfer_memory(device_num, stream_id, out<T>(n)));
+
+    // SPMV Workspace
+    mem_ptr<char> spmv_workspace;
+    size_t ws_size = 0;
+    if (fmt == matrix_format::csr) ws_size = d_ptr->spmv_csr_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, w);
+    else if (fmt == matrix_format::csc) ws_size = d_ptr->spmv_csc_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, w);
+    else if (fmt == matrix_format::coo) ws_size = d_ptr->spmv_coo_buffer_size(stream_id, n, n, nnz, A_row_ptr, A_col_ind, A_values, x, w);
+
+    if (ws_size > 0) {
+      command_runner<out<char>> ws_runner;
+      spmv_workspace = ws_runner.transfer_memory(device_num, stream_id, out<char>(static_cast<int>(ws_size)));
+    }
+
+    // Helpers
+    auto execute_spmv = [&](mem_ptr<T> in_v, mem_ptr<T> out_v) {
+      if (fmt == matrix_format::csr) d_ptr->spmv_csr(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, in_v, T{0}, out_v, spmv_workspace);
+      else if (fmt == matrix_format::csc) d_ptr->spmv_csc(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, in_v, T{0}, out_v, spmv_workspace);
+      else if (fmt == matrix_format::coo) d_ptr->spmv_coo(stream_id, n, n, nnz, T{1}, A_row_ptr, A_col_ind, A_values, in_v, T{0}, out_v, spmv_workspace);
+    };
+    auto execute_copy = [&](mem_ptr<T> src, mem_ptr<T> dst) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->dcopy(stream_id, n, src, dst); else d_ptr->scopy(stream_id, n, src, dst);
+    };
+    auto execute_axpy = [&](T alpha, mem_ptr<T> xv, mem_ptr<T> yv) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->daxpy(stream_id, n, alpha, xv, yv); else d_ptr->saxpy(stream_id, n, static_cast<float>(alpha), xv, yv);
+    };
+    auto execute_dot = [&](mem_ptr<T> xv, mem_ptr<T> yv, mem_ptr<T> rv) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->ddot(stream_id, n, xv, yv, rv); else d_ptr->sdot(stream_id, n, xv, yv, rv);
+    };
+    auto execute_nrm2 = [&](mem_ptr<T> xv, mem_ptr<T> rv) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->dnrm2(stream_id, n, xv, rv); else d_ptr->snrm2(stream_id, n, xv, rv);
+    };
+
+    int total_iters = 0;
+    T residual_norm = T{1e10};
+
+    // Outer Restart Loop
+    while (total_iters < max_iter && residual_norm > tol) {
+      // r = b - Ax
+      execute_spmv(x, w);
+      execute_copy(b, V[0]);
+      execute_axpy(T{-1}, w, V[0]);
+
+      // beta = ||r||
+      execute_nrm2(V[0], y_tmp);
+      T beta = runner.copy_to_host(y_tmp)[0];
+      residual_norm = beta;
+      if (beta < tol) break;
+
+      // v1 = r / beta
+      T inv_beta = T{1} / beta;
+      if constexpr (std::is_same_v<T, double>) d_ptr->dscal(stream_id, n, inv_beta, V[0]);
+      else d_ptr->sscal(stream_id, n, static_cast<float>(inv_beta), V[0]);
+
+      std::vector<T> g(m + 1, 0.0);
+      g[0] = beta;
+      std::vector<std::vector<T>> H(m + 1, std::vector<T>(m, 0.0));
+      std::vector<T> sn(m, 0.0), cs(m, 0.0);
+
+      int k = 0;
+      for (; k < m && total_iters < max_iter; ++k, ++total_iters) {
+        // Arnoldi Process: w = A * v_k
+        execute_spmv(V[k], w);
+
+        for (int i = 0; i <= k; ++i) {
+          execute_dot(w, V[i], y_tmp);
+          H[i][k] = runner.copy_to_host(y_tmp)[0];
+          execute_axpy(-H[i][k], V[i], w);
+        }
+        execute_nrm2(w, y_tmp);
+        H[k + 1][k] = runner.copy_to_host(y_tmp)[0];
+
+        // v_{k+1} = w / H[k+1][k]
+        execute_copy(w, V[k + 1]);
+        T inv_h = T{1} / H[k + 1][k];
+        if constexpr (std::is_same_v<T, double>) d_ptr->dscal(stream_id, n, inv_h, V[k + 1]);
+        else d_ptr->sscal(stream_id, n, static_cast<float>(inv_h), V[k + 1]);
+
+        // Apply previous Givens rotations to new column of H
+        for (int i = 0; i < k; ++i) {
+          T temp = cs[i] * H[i][k] + sn[i] * H[i + 1][k];
+          H[i + 1][k] = -sn[i] * H[i][k] + cs[i] * H[i + 1][k];
+          H[i][k] = temp;
+        }
+
+        // Generate new Givens rotation
+        T rot_r = std::sqrt(H[k][k] * H[k][k] + H[k + 1][k] * H[k + 1][k]);
+        cs[k] = H[k][k] / rot_r;
+        sn[k] = H[k + 1][k] / rot_r;
+
+        // Apply to H and g
+        H[k][k] = cs[k] * H[k][k] + sn[k] * H[k + 1][k];
+        H[k + 1][k] = 0.0;
+        T temp_g = cs[k] * g[k];
+        g[k + 1] = -sn[k] * g[k];
+        g[k] = temp_g;
+
+        residual_norm = std::abs(g[k + 1]);
+        if (residual_norm < tol) { k++; break; }
+      }
+
+      // Solve Hy = g (Upper Triangular)
+      std::vector<T> y_vec(k);
+      for (int i = k - 1; i >= 0; --i) {
+        T sum = 0;
+        for (int j = i + 1; j < k; ++j) sum += H[i][j] * y_vec[j];
+        y_vec[i] = (g[i] - sum) / H[i][i];
+      }
+
+      // x = x + V*y
+      for (int i = 0; i < k; ++i) {
+        execute_axpy(y_vec[i], V[i], x);
+      }
+    }
+    return x;
+  }
+
+  void dispatch_result(std::vector<output_mapping> mappings, mem_ptr<T> x, int n) {
+    auto sender = actor_cast<actor>(this->current_sender());
+    if (!sender) return;
+    void* custom_dst = nullptr;
+    size_t custom_count = 0;
+    for (const auto& m : mappings) {
+      if (m.index == 4) { // Assuming solution is at index 4 matching your tests
+        custom_dst = m.dst;
+        custom_count = m.count;
+        break;
+      }
+    }
+
+    command_runner<T> runner;
+    if (custom_dst) {
+      runner.copy_to_host_async(x, static_cast<T*>(custom_dst), custom_count > 0 ? custom_count : (size_t)n,
+        [sender, r_id = reply_id_](T*, size_t) {
+          caf::anon_mail(r_id, 4).send(sender);
+        });
+    } else {
+      runner.copy_to_host_async(x, [sender, r_id = reply_id_](std::vector<T> data) {
+        caf::anon_mail(r_id, 4, std::move(data)).send(sender);
+      });
+    }
+  }
+
+  uint32_t reply_id_;
+};
+
+} // namespace caf::cuda
\ No newline at end of file

From b653ae31da6c276a75513089095ec96f32d36d1a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 14:32:26 -0600
Subject: [PATCH 0772/1000] updated includes

---
 libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp b/libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp
index 2fed9f35eb..b14ad8d04a 100644
--- a/libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/actorSOLVE.hpp
@@ -2,3 +2,6 @@
 #include "caf/actorBLAS/actorBLAS.hpp"
 #include "caf/actorSPARSE/actorSPARSE.hpp"
 #include "caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp"
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp"
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
+

From a313b1bc97a861f18b36fed548cd9a12366efa41 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 14:36:15 -0600
Subject: [PATCH 0773/1000] Initial commit.

---
 .../GMRES-actor-test/CMakeLists.txt           |  46 +++++
 .../GMRES-actor-test/main.test.cpp            | 190 ++++++++++++++++++
 2 files changed, 236 insertions(+)
 create mode 100644 libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/main.test.cpp

diff --git a/libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/CMakeLists.txt b/libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/CMakeLists.txt
new file mode 100644
index 0000000000..82787c399c
--- /dev/null
+++ b/libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/CMakeLists.txt
@@ -0,0 +1,46 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
+)
+
diff --git a/libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/main.test.cpp
new file mode 100644
index 0000000000..c7533e019b
--- /dev/null
+++ b/libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/main.test.cpp
@@ -0,0 +1,190 @@
+/**
+ * This test file evaluates the correctness of sparse_gmres_facade for solving Ax = b.
+ * It covers:
+ * - CSR, CSC, and COO formats.
+ * - Convergence verification with simple matrices.
+ * - Stress testing with a large 1D Laplacian matrix.
+ */
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <cmath>
+#include "caf/actor_registry.hpp"
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+void verify_solution(const std::string& test_name, const std::vector<float>& actual, 
+                     const std::vector<float>& expected, float tol = 1e-3) {
+    if (actual.size() != expected.size()) {
+        std::cout << "[ERROR] " << test_name << " failed: Size mismatch (got " 
+                  << actual.size() << ", expected " << expected.size() << ")" << std::endl;
+        return;
+    }
+    bool all_correct = true;
+    for (size_t i = 0; i < actual.size(); ++i) {
+        if (std::abs(actual[i] - expected[i]) > tol) {
+            all_correct = false;
+            std::cout << "[ERROR] " << test_name << " mismatch at index " << i 
+                      << ": Expected " << expected[i] 
+                      << ", Got " << actual[i] << std::endl;
+            break;
+        }
+    }
+    if (all_correct) {
+        std::cout << "[SUCCESS] " << test_name << " converged to correct solution." << std::endl;
+    }
+}
+
+void caf_main(actor_system& sys) {
+    // Enable cuBLAS and cuSPARSE for the GMRES solver
+    manager::init(sys, manager_config(true, true)); 
+
+    // Simple Diagonal Matrix A (3x3): diag(4, 3, 2)
+    // b = [8, 9, 2] -> Expected x = [2, 3, 1]
+    int n = 3, nnz = 3;
+    std::vector<float> h_b = {8.0f, 9.0f, 2.0f};
+    std::vector<float> expected = {2.0f, 3.0f, 1.0f};
+    float tolerance = 1e-5f;
+    int max_iter = 100;
+    int restart_m = 10;
+
+    scoped_actor self{sys};
+
+    // Test 1: CSR Format
+    {
+        std::cout << "[INFO] Test 1: CSR format simple matrix..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_gmres_facade<float>>(100);
+
+        self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, restart_m, 0, 0).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+                verify_solution("GMRES Facade CSR Simple", result_x, expected);
+            }
+        );
+    }
+
+    // Test 2: Facade Actor CSR with Custom Buffer
+    {
+        std::cout << "\n[INFO] Test 2: Facade actor CSR with custom buffer..." << std::endl;
+        std::vector<float> custom_x(n, 0.0f);
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        output_mapping m{4, custom_x.data(), (size_t)n};
+        
+        auto facade = sys.spawn<sparse_gmres_facade<float>>(100);
+        self->mail(std::vector<output_mapping>{m},
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(custom_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, restart_m, 0, 1).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int index) {
+                if (index == 4)
+                  verify_solution("GMRES Facade Custom Buffer", custom_x, expected);
+            }
+        );
+    }
+
+    // Test 3: Facade Actor CSR returning mem_ptr handles
+    {
+        std::cout << "\n[INFO] Test 3: Facade actor CSR returning mem_ptr..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_gmres_facade<float>>(100);
+
+        self->mail(return_mem_ptr_atom_v,
+                   create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, restart_m, 0, 2).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr) {
+                auto result_x = ptr->copy_to_host();
+                verify_solution("GMRES Facade mem_ptr", result_x, expected);
+            }
+        );
+    }
+
+    // Test 4: Stress Test - 1D Laplacian (N=5000)
+    {
+        int N_large = 5000;
+        std::cout << "\n[INFO] Test 4: Stress Test - 1D Laplacian (N=" << N_large << ")..." << std::endl;
+        
+        std::vector<int> row_ptr;
+        std::vector<int> col_ind;
+        std::vector<float> values;
+        row_ptr.push_back(0);
+        for(int i=0; i<N_large; ++i) {
+            if(i > 0) { col_ind.push_back(i-1); values.push_back(-1.0f); }
+            col_ind.push_back(i); values.push_back(2.0f);
+            if(i < N_large-1) { col_ind.push_back(i+1); values.push_back(-1.0f); }
+            row_ptr.push_back(col_ind.size());
+        }
+
+        std::vector<float> b_large(N_large, 1.0f);
+        std::vector<float> x_large(N_large, 0.0f);
+        
+        auto facade = sys.spawn<sparse_gmres_facade<float>>(100);
+        auto start = std::chrono::high_resolution_clock::now();
+
+        self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(b_large), create_in_out_arg(x_large),
+                   matrix_format::csr, N_large, (int)values.size(), 1e-4f, 2000, 30, 0, 3).send(facade);
+
+        self->receive(
+            [&](uint32_t, int, std::vector<float> result) {
+                auto end = std::chrono::high_resolution_clock::now();
+                std::chrono::duration<double> elapsed = end - start;
+                std::cout << "[SUCCESS] Stress Test completed in " << elapsed.count() << " seconds." << std::endl;
+                std::cout << "[INFO] First 5 elements of solution: ";
+                for(int i=0; i<5; ++i) std::cout << result[i] << " ";
+                std::cout << "..." << std::endl;
+            }
+        );
+    }
+
+    // Test 5: CSC Format
+    {
+        std::cout << "\n[INFO] Test 5: CSC format simple matrix..." << std::endl;
+        std::vector<int> col_ptr = {0, 1, 2, 3};
+        std::vector<int> row_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_gmres_facade<float>>(100);
+
+        self->mail(create_in_arg(col_ptr), create_in_arg(row_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csc, n, nnz, tolerance, max_iter, restart_m, 0, 4).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+                verify_solution("GMRES Facade CSC Simple", result_x, expected);
+            }
+        );
+    }
+
+    manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda)

From 9428abc2b6d7de976abea4801533235dfe1888ee Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 28 May 2026 14:40:57 -0600
Subject: [PATCH 0774/1000] added sscal and dscal operations to this call

---
 libcaf_cuda/caf/cuda/device.hpp | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index 073374d761..b318e1c505 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -878,6 +878,38 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
       throw std::runtime_error("cublasDaxpy failed on device " + std::to_string(id_));
   }
 
+  /// Performs single precision vector scaling (x = alpha*x).
+  void sscal(int stream_id, int n, float alpha, mem_ptr<float> x) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+
+    cublasStatus_t status = cublasSscal(handle, n, &alpha,
+                                        reinterpret_cast<float*>(x->mem()), 1);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasSscal failed on device " + std::to_string(id_));
+  }
+
+  /// Performs double precision vector scaling (x = alpha*x).
+  void dscal(int stream_id, int n, double alpha, mem_ptr<double> x) {
+    cublasHandle_t handle = get_cublas_handle(stream_id);
+    if (!handle)
+      throw std::runtime_error("cuBLAS not enabled on device " + std::to_string(id_));
+
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+
+    cublasStatus_t status = cublasDscal(handle, n, &alpha,
+                                        reinterpret_cast<double*>(x->mem()), 1);
+
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    if (status != CUBLAS_STATUS_SUCCESS)
+      throw std::runtime_error("cublasDscal failed on device " + std::to_string(id_));
+  }
+
   /// Performs single precision Euclidean norm (result = ||x||2).
   void snrm2(int stream_id, int n, mem_ptr<float> x, mem_ptr<float> result) {
     cublasHandle_t handle = get_cublas_handle(stream_id);

From a489e366a36897d3a4808b00fc457c3108418f85 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 29 May 2026 09:13:18 -0600
Subject: [PATCH 0775/1000] moved atom declaration into global.hpp

---
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     |  5 --
 libcaf_cuda/caf/cuda/global.hpp               |  4 +-
 .../benchmark-test/main.test.cpp              | 54 +++++++++++++++++--
 .../CGS-actor-test/main.test.cpp              |  2 +-
 4 files changed, 54 insertions(+), 11 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index e3951d5a53..cd299c4a7f 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -10,11 +10,6 @@
 #include "caf/actorBLAS/copy-actor/copy-actor.hpp"
 #include "caf/cuda/platform.hpp"
 
-// Define a new block for CG specific atoms starting where the cuda block ended
-CAF_BEGIN_TYPE_ID_BLOCK(cg_actor, caf::id_block::cuda::end)
-  CAF_ADD_ATOM(cg_actor, cg_next_step_atom)
-CAF_END_TYPE_ID_BLOCK(cg_actor)
-
 namespace caf::cuda {
 
 // Reply IDs used to distinguish which actor type is replying
diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index a2e0dfbaf6..9b683ae209 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -235,10 +235,12 @@ CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id)
   CAF_ADD_ATOM(cuda, csc_atom)
   CAF_ADD_ATOM(cuda, coo_atom)  
   CAF_ADD_ATOM(cuda, start_atom)
-
+  CAF_ADD_ATOM(cuda, cg_next_step_atom)
 
 CAF_END_TYPE_ID_BLOCK(cuda)
 
+
+
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<int>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<float>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<double>)
diff --git a/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/main.test.cpp b/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/main.test.cpp
index dfb1b47c56..e9c96cda4b 100644
--- a/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorBLAS-test/dot-actor-test/benchmark-test/main.test.cpp
@@ -3,6 +3,7 @@
 #include <iostream>
 #include <vector>
 #include <chrono>
+#include <iomanip>
 #include <cublas_v2.h>
 #include "caf/actorBLAS/dot-actor/dot-actor.hpp"
 
@@ -68,6 +69,42 @@ void run_native_benchmark(int n, const std::vector<int>& iterations_series) {
     cuCtxDestroy(ctx);
 }
 
+// Device API benchmark implementation using the device class directly
+void run_device_benchmark(int n, const std::vector<int>& iterations_series) {
+    std::cout << "\n[INFO] Running Device Class Benchmark..." << std::endl;
+    
+    auto plat = platform::create();
+    // Use the first device and a default stream for this benchmark
+    int stream_id = 0;
+    auto dev = plat->schedule(stream_id); 
+    dev->enable_cublas();
+
+    std::vector<float> h_x(n, 1.0f);
+    std::vector<float> h_y(n, 1.0f);
+
+    for (int iters : iterations_series) {
+        auto start = std::chrono::steady_clock::now();
+        
+        for (int i = 0; i < iters; ++i) {
+            // 1. Host-to-Device: allocate and copy vectors using framework abstractions
+            auto x_ptr = dev->make_arg(create_in_arg(h_x), stream_id);
+            auto y_ptr = dev->make_arg(create_in_arg(h_y), stream_id);
+            // 2. Allocate output space on device
+            auto res_ptr = dev->make_arg(create_out_arg_with_size<float>(1), stream_id);
+
+            // 3. Launch sdot kernel via the device abstraction
+            dev->sdot(stream_id, n, x_ptr, y_ptr, res_ptr);
+
+            // 4. Device-to-Host: copy scalar result back
+            auto res_vec = res_ptr->copy_to_host();
+        }
+        
+        auto end = std::chrono::steady_clock::now();
+        auto diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+        std::cout << "[DEVICE] Iterations: " << iters << " | Time: " << diff << " ms" << std::endl;
+    }
+}
+
 // dot_actor benchmark implementation
 void run_actor_benchmark(actor_system& sys, int n, const std::vector<int>& iterations_series) {
     std::cout << "\n[INFO] Running Dot Actor Benchmark..." << std::endl;
@@ -81,23 +118,31 @@ void run_actor_benchmark(actor_system& sys, int n, const std::vector<int>& itera
 
     for (int iters : iterations_series) {
         auto start = std::chrono::steady_clock::now();
+        double total_rt = 0.0;
         
         for (int i = 0; i < iters; ++i) {
+            auto rt_start = std::chrono::high_resolution_clock::now();
             self->mail(create_in_arg(h_x), 
                        create_in_arg(h_y), 
                        create_out_arg_with_size<float>(1), 
                        n).send(dot);
             
             self->receive(
-                [&](int rid, float result) {
-                    // Result received via message
+                [&, rt_start](int rid, float result) {
+                    auto rt_end = std::chrono::high_resolution_clock::now();
+                    double dur = std::chrono::duration<double, std::milli>(rt_end - rt_start).count();
+                    total_rt += dur;
+                    if (iters == 1 || (iters <= 1000 && i % 200 == 0) || (i % 2000 == 0))
+                        std::cout << "[DEBUG] Iteration " << i << " Round-Trip: " << dur << " ms" << std::endl;
                 }
             );
         }
         
         auto end = std::chrono::steady_clock::now();
         auto diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-        std::cout << "[ACTOR]  Iterations: " << iters << " | Time: " << diff << " ms" << std::endl;
+        std::cout << std::fixed << std::setprecision(4);
+        std::cout << "[ACTOR]  Iterations: " << iters << " | Total Wall Time: " << diff 
+                  << " ms | Avg Round-Trip: " << (total_rt / iters) << " ms" << std::endl;
     }
     
     self->send_exit(dot, exit_reason::user_shutdown);
@@ -107,10 +152,11 @@ void caf_main(actor_system& sys) {
     // Initialize CUDA manager with cuBLAS support
     manager::init(sys, manager_config(true));
 
-    int n = 1000000; // 1M elements
+    int n = 10000; // 1M elements
     std::vector<int> series = {1, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000};
 
     run_native_benchmark(n, series);
+    run_device_benchmark(n, series);
     run_actor_benchmark(sys, n, series);
 
     manager::shutdown();
diff --git a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
index bd4359a4ee..dcdabe8803 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
@@ -362,4 +362,4 @@ void caf_main(actor_system& sys) {
     manager::shutdown();
 }
 
-CAF_MAIN(id_block::cuda, id_block::cg_actor)
+CAF_MAIN(id_block::cuda)

From 30b4149b72c3356a26574a024b3f83c873124cf5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 29 May 2026 09:31:18 -0600
Subject: [PATCH 0776/1000] updated actors to return metadata reguarding the
 solver and their device number and stream id

---
 .../sparse-BiCGSTAB-actor.hpp                 | 72 ++++++++++---------
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 68 +++++++++---------
 .../sparse-GMRES-actor/sparse-GMRES-actor.hpp | 35 ++++-----
 libcaf_cuda/caf/cuda/global.hpp               | 12 ++++
 libcaf_cuda/caf/cuda/types.hpp                | 12 ++++
 5 files changed, 115 insertions(+), 84 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
index 78cf2b24cd..f4a2ae3eed 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp
@@ -2,6 +2,7 @@
 
 #include <caf/all.hpp>
 #include <vector>
+#include <utility>
 #include "caf/cuda/all.hpp"
 #include "caf/actorBLAS/dot-actor/dot-actor.hpp"
 #include "caf/actorSPARSE/spmv-actor/spmv-actor.hpp"
@@ -75,10 +76,10 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state<T>> {
           s.supervisor = actor_cast<caf::actor>(this->current_sender());
         start_solve();
       },
-      [this](gpu_done_atom, std::vector<T>& solution) {
+      [this](gpu_done_atom, std::vector<T>& solution, solver_result_meta meta) {
         auto& s = this->state();
         if (s.supervisor)
-          this->mail(std::move(solution)).send(s.supervisor);
+          this->mail(std::move(solution), meta).send(s.supervisor);
         this->quit();
       }
     };
@@ -206,8 +207,9 @@ class sparse_bicgstab_actor : public stateful_actor<sparse_bicgstab_state<T>> {
     }
 
     auto self = actor_cast<actor>(this);
-    runner.copy_to_host_async(s.x, [self](std::vector<T> solution) {
-      anon_mail(gpu_done_atom_v, std::move(solution)).send(self);
+    solver_result_meta meta(s.device_num, s.stream_id, s.iterations, norm_sq <= (s.tol * s.tol));
+    runner.copy_to_host_async(s.x, [self, meta](std::vector<T> solution) {
+      anon_mail(gpu_done_atom_v, std::move(solution), meta).send(self);
     });
   }
 
@@ -280,12 +282,12 @@ class sparse_bicgstab_facade : public event_based_actor {
              T tol, int max_iter,
              int device_num, int stream_id) {
 
-        auto x = solve_core(rp, ci, val, b_in, x_in,
-                            fmt, n, nnz, tol, max_iter,
-                            device_num, stream_id);
+        auto [x, meta] = solve_core(rp, ci, val, b_in, x_in,
+                                    fmt, n, nnz, tol, max_iter,
+                                    device_num, stream_id);
 
         if (auto sender = actor_cast<actor>(this->current_sender())) {
-          caf::anon_mail(reply_id_, std::move(x)).send(sender);
+          caf::anon_mail(reply_id_, std::move(x), meta).send(sender);
         }
       },
 
@@ -297,11 +299,11 @@ class sparse_bicgstab_facade : public event_based_actor {
              T tol, int max_iter,
              int device_num, int stream_id) {
 
-        auto x = solve_core(rp, ci, val, b_in, x_in,
-                            fmt, n, nnz, tol, max_iter,
-                            device_num, stream_id);
+        auto [x, meta] = solve_core(rp, ci, val, b_in, x_in,
+                                    fmt, n, nnz, tol, max_iter,
+                                    device_num, stream_id);
 
-        dispatch_result(std::move(mappings), std::move(x), n);
+        dispatch_result(std::move(mappings), std::move(x), n, meta);
       },
 
       // Mode 3: Default (return vector to sender)
@@ -309,20 +311,20 @@ class sparse_bicgstab_facade : public event_based_actor {
              matrix_format fmt, int n, int nnz, T tol, int max_iter,
              int device_num, int stream_id) {
 
-        auto x = solve_core(rp, ci, val, b_in, x_in,
-                            fmt, n, nnz, tol, max_iter,
-                            device_num, stream_id);
+        auto [x, meta] = solve_core(rp, ci, val, b_in, x_in,
+                                    fmt, n, nnz, tol, max_iter,
+                                    device_num, stream_id);
 
-        dispatch_result({}, std::move(x), n);
+        dispatch_result({}, std::move(x), n, meta);
       }
     };
   }
 
 protected:
-  virtual mem_ptr<T> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in,
-                                in_out<T> x_in, matrix_format fmt, int n,
-                                int nnz, T tol, int max_iter,
-                                int device_num, int stream_id) {
+  virtual std::pair<mem_ptr<T>, solver_result_meta> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in,
+                                                               in_out<T> x_in, matrix_format fmt, int n,
+                                                               int nnz, T tol, int max_iter,
+                                                               int device_num, int stream_id) {
     command_runner<T> runner;
     auto res = runner.transfer_memory(device_num, stream_id,
                                       rp, ci, val, b_in, x_in);
@@ -423,12 +425,13 @@ class sparse_bicgstab_facade : public event_based_actor {
       execute_dot(r, r, y_tmp);
       norm_sq = runner.copy_to_host(y_tmp)[0];
     }
-    return x;
+    return {x, solver_result_meta(device_num, stream_id, iterations, norm_sq <= (tol * tol))};
   }
 
   void dispatch_result(std::vector<output_mapping> mappings,
                        mem_ptr<T> x,
-                       int n) {
+                       int n,
+                       solver_result_meta meta) {
 
     auto sender = actor_cast<actor>(this->current_sender());
     if (!sender) return;
@@ -451,15 +454,15 @@ class sparse_bicgstab_facade : public event_based_actor {
         x,
         static_cast<T*>(custom_dst),
         custom_count > 0 ? custom_count : (size_t)n,
-        [sender, r_id = reply_id_](T*, size_t) {
-          caf::anon_mail(r_id, 4).send(sender);
+        [sender, r_id = reply_id_, meta](T*, size_t) {
+          caf::anon_mail(r_id, 4, meta).send(sender);
         });
 
     } else {
       runner.copy_to_host_async(
         x,
-        [sender, r_id = reply_id_](std::vector<T> data) {
-          caf::anon_mail(r_id, 4, std::move(data)).send(sender);
+        [sender, r_id = reply_id_, meta](std::vector<T> data) {
+          caf::anon_mail(r_id, 4, std::move(data), meta).send(sender);
         });
     }
   }
@@ -484,11 +487,11 @@ class sparse_bicgstab_jacobi_facade : public sparse_bicgstab_facade<T> {
   }
 
 protected:
-  mem_ptr<T> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in,
-                        in_out<T> x_in,
-                        matrix_format fmt, int n, int nnz,
-                        T tol, int max_iter,
-                        int device_num, int stream_id) override {
+  std::pair<mem_ptr<T>, solver_result_meta> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in,
+                                                       in_out<T> x_in,
+                                                       matrix_format fmt, int n, int nnz,
+                                                       T tol, int max_iter,
+                                                       int device_num, int stream_id) override {
 
     command_runner<T> runner;
     auto res = runner.transfer_memory(device_num, stream_id,
@@ -600,9 +603,10 @@ class sparse_bicgstab_jacobi_facade : public sparse_bicgstab_facade<T> {
       
       // Convergence check: Exit if the solution is reached after the alpha update
       execute_dot(s_vec, s_vec, y_tmp);
-      if (runner.copy_to_host(y_tmp)[0] < (tol * tol)) {
+      T s_norm_sq = runner.copy_to_host(y_tmp)[0];
+      if (s_norm_sq <= (tol * tol)) {
         execute_axpy(alpha_val, p_hat, x);
-        return x;
+        return {x, solver_result_meta(device_num, stream_id, iterations, true)};
       }
 
       // Apply Preconditioner: s_hat = D_inv * s
@@ -624,7 +628,7 @@ class sparse_bicgstab_jacobi_facade : public sparse_bicgstab_facade<T> {
       execute_dot(r, r, y_tmp);
       norm_sq = runner.copy_to_host(y_tmp)[0];
     }
-    return x;
+    return {x, solver_result_meta(device_num, stream_id, iterations, norm_sq <= (tol * tol))};
   }
 private:
   program_ptr diag_prog_;
diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index cd299c4a7f..a7ed0807c8 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -78,9 +78,9 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
          s.supervisor = actor_cast<caf::actor>(this->current_sender());
        start_solve();
      },
-     [this](gpu_done_atom, std::vector<float>& solution) {
+     [this](gpu_done_atom, std::vector<float>& solution, solver_result_meta meta) {
        if (state().supervisor)
-         this->mail(std::move(solution)).send(state().supervisor);
+         this->mail(std::move(solution), meta).send(state().supervisor);
        this->quit();
      }
    };
@@ -173,8 +173,9 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
 
    // Exit the loop and return the result via the standard async path
    auto self = actor_cast<actor>(this);
-   runner.copy_to_host_async(s.x, [self](std::vector<float> solution) {
-     anon_mail(gpu_done_atom_v, std::move(solution)).send(self);
+   solver_result_meta meta(s.device_num, s.stream_id, s.iterations, s.rho_val <= (s.tol * s.tol));
+   runner.copy_to_host_async(s.x, [self, meta](std::vector<float> solution) {
+     anon_mail(gpu_done_atom_v, std::move(solution), meta).send(self);
    });
  }
 
@@ -209,12 +210,12 @@ class sparse_cg_facade : public event_based_actor {
              float tol, int max_iter,
              int device_num, int stream_id) {
 
-        auto x = solve_core(rp, ci, val, b_in, x_in,
-                            fmt, n, nnz, tol, max_iter,
-                            device_num, stream_id);
+        auto [x, meta] = solve_core(rp, ci, val, b_in, x_in,
+                                    fmt, n, nnz, tol, max_iter,
+                                    device_num, stream_id);
 
         if (auto sender = actor_cast<actor>(this->current_sender())) {
-          caf::anon_mail(reply_id_, std::move(x)).send(sender);
+          caf::anon_mail(reply_id_, std::move(x), meta).send(sender);
         }
       },
 
@@ -226,11 +227,11 @@ class sparse_cg_facade : public event_based_actor {
              float tol, int max_iter,
              int device_num, int stream_id) {
 
-        auto x = solve_core(rp, ci, val, b_in, x_in,
-                            fmt, n, nnz, tol, max_iter,
-                            device_num, stream_id);
+        auto [x, meta] = solve_core(rp, ci, val, b_in, x_in,
+                                    fmt, n, nnz, tol, max_iter,
+                                    device_num, stream_id);
 
-        dispatch_result(std::move(mappings), std::move(x), n);
+        dispatch_result(std::move(mappings), std::move(x), n, meta);
       },
 
       // Mode 3: Default (return vector to sender)
@@ -240,21 +241,21 @@ class sparse_cg_facade : public event_based_actor {
              float tol, int max_iter,
              int device_num, int stream_id) {
 
-        auto x = solve_core(rp, ci, val, b_in, x_in,
-                            fmt, n, nnz, tol, max_iter,
-                            device_num, stream_id);
+        auto [x, meta] = solve_core(rp, ci, val, b_in, x_in,
+                                    fmt, n, nnz, tol, max_iter,
+                                    device_num, stream_id);
 
-        dispatch_result({}, std::move(x), n);
+        dispatch_result({}, std::move(x), n, meta);
       }
     };
   }
 
 protected:
-  virtual mem_ptr<float> solve_core(in<int> rp, in<int> ci, in<float> val, in<float> b_in,
-                                    in_out<float> x_in,
-                                    matrix_format fmt, int n, int nnz,
-                                    float tol, int max_iter,
-                                    int device_num, int stream_id) {
+  virtual std::pair<mem_ptr<float>, solver_result_meta> solve_core(in<int> rp, in<int> ci, in<float> val, in<float> b_in,
+                                                                   in_out<float> x_in,
+                                                                   matrix_format fmt, int n, int nnz,
+                                                                   float tol, int max_iter,
+                                                                   int device_num, int stream_id) {
 
     command_runner<float> runner;
 
@@ -355,12 +356,13 @@ class sparse_cg_facade : public event_based_actor {
       rho_val = runner.copy_to_host(y_tmp)[0];
     }
 
-    return x;
+    return {x, solver_result_meta(device_num, stream_id, iterations, rho_val <= (tol * tol))};
   }
 
   void dispatch_result(std::vector<output_mapping> mappings,
                        mem_ptr<float> x,
-                       int n) {
+                       int n,
+                       solver_result_meta meta) {
 
     auto sender = actor_cast<actor>(this->current_sender());
     if (!sender) return;
@@ -383,15 +385,15 @@ class sparse_cg_facade : public event_based_actor {
         x,
         static_cast<float*>(custom_dst),
         custom_count > 0 ? custom_count : (size_t)n,
-        [sender, r_id = reply_id_](float*, size_t) {
-          caf::anon_mail(r_id, 4).send(sender);
+        [sender, r_id = reply_id_, meta](float*, size_t) {
+          caf::anon_mail(r_id, 4, meta).send(sender);
         });
 
     } else {
       runner.copy_to_host_async(
         x,
-        [sender, r_id = reply_id_](std::vector<float> data) {
-          caf::anon_mail(r_id, 4, std::move(data)).send(sender);
+        [sender, r_id = reply_id_, meta](std::vector<float> data) {
+          caf::anon_mail(r_id, 4, std::move(data), meta).send(sender);
         });
     }
   }
@@ -416,11 +418,11 @@ class sparse_cg_jacobi_facade : public sparse_cg_facade {
   }
 
 protected:
-  mem_ptr<float> solve_core(in<int> rp, in<int> ci, in<float> val, in<float> b_in,
-                            in_out<float> x_in,
-                            matrix_format fmt, int n, int nnz,
-                            float tol, int max_iter,
-                            int device_num, int stream_id) override {
+  std::pair<mem_ptr<float>, solver_result_meta> solve_core(in<int> rp, in<int> ci, in<float> val, in<float> b_in,
+                                                           in_out<float> x_in,
+                                                           matrix_format fmt, int n, int nnz,
+                                                           float tol, int max_iter,
+                                                           int device_num, int stream_id) override {
 
     command_runner<float> runner;
     auto res = runner.transfer_memory(device_num, stream_id,
@@ -535,7 +537,7 @@ class sparse_cg_jacobi_facade : public sparse_cg_facade {
       rho_val = runner.copy_to_host(y_tmp)[0];
     }
 
-    return x;
+    return {x, solver_result_meta(device_num, stream_id, iterations, rho_val <= (tol * tol))};
   }
 
 private:
diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp
index 821d9dc8bf..eb70238f22 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp
@@ -96,33 +96,33 @@ class sparse_gmres_facade : public event_based_actor {
       [this](return_mem_ptr_atom, in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
              matrix_format fmt, int n, int nnz, T tol, int max_iter, int m,
              int device_num, int stream_id) {
-        auto x = solve_core(rp, ci, val, b_in, x_in, fmt, n, nnz, tol, max_iter, m, device_num, stream_id);
+        auto [x, meta] = solve_core(rp, ci, val, b_in, x_in, fmt, n, nnz, tol, max_iter, m, device_num, stream_id);
         if (auto sender = actor_cast<actor>(this->current_sender()))
-          caf::anon_mail(reply_id_, std::move(x)).send(sender);
+          caf::anon_mail(reply_id_, std::move(x), meta).send(sender);
       },
 
       // Mode 2: Return host data via mappings
       [this](std::vector<output_mapping> mappings, in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
              matrix_format fmt, int n, int nnz, T tol, int max_iter, int m,
              int device_num, int stream_id) {
-        auto x = solve_core(rp, ci, val, b_in, x_in, fmt, n, nnz, tol, max_iter, m, device_num, stream_id);
-        dispatch_result(std::move(mappings), std::move(x), n);
+        auto [x, meta] = solve_core(rp, ci, val, b_in, x_in, fmt, n, nnz, tol, max_iter, m, device_num, stream_id);
+        dispatch_result(std::move(mappings), std::move(x), n, meta);
       },
 
       // Mode 3: Default (return vector to sender)
       [this](in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
              matrix_format fmt, int n, int nnz, T tol, int max_iter, int m,
              int device_num, int stream_id) {
-        auto x = solve_core(rp, ci, val, b_in, x_in, fmt, n, nnz, tol, max_iter, m, device_num, stream_id);
-        dispatch_result({}, std::move(x), n);
+        auto [x, meta] = solve_core(rp, ci, val, b_in, x_in, fmt, n, nnz, tol, max_iter, m, device_num, stream_id);
+        dispatch_result({}, std::move(x), n, meta);
       }
     };
   }
 
 protected:
-  virtual mem_ptr<T> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
-                                matrix_format fmt, int n, int nnz, T tol, int max_iter, int m,
-                                int device_num, int stream_id) {
+  virtual std::pair<mem_ptr<T>, solver_result_meta> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+                                                               matrix_format fmt, int n, int nnz, T tol, int max_iter, int m,
+                                                               int device_num, int stream_id) {
     command_runner<T> runner;
     auto res = runner.transfer_memory(device_num, stream_id, rp, ci, val, b_in, x_in);
     auto A_row_ptr = std::get<0>(res), A_col_ind = std::get<1>(res), A_values = std::get<2>(res);
@@ -183,7 +183,7 @@ class sparse_gmres_facade : public event_based_actor {
       execute_nrm2(V[0], y_tmp);
       T beta = runner.copy_to_host(y_tmp)[0];
       residual_norm = beta;
-      if (beta < tol) break;
+      if (beta <= tol) break;
 
       // v1 = r / beta
       T inv_beta = T{1} / beta;
@@ -234,7 +234,7 @@ class sparse_gmres_facade : public event_based_actor {
         g[k] = temp_g;
 
         residual_norm = std::abs(g[k + 1]);
-        if (residual_norm < tol) { k++; break; }
+        if (residual_norm <= tol) { k++; break; }
       }
 
       // Solve Hy = g (Upper Triangular)
@@ -250,10 +250,11 @@ class sparse_gmres_facade : public event_based_actor {
         execute_axpy(y_vec[i], V[i], x);
       }
     }
-    return x;
+    solver_result_meta meta(device_num, stream_id, total_iters, residual_norm <= tol);
+    return {x, meta};
   }
 
-  void dispatch_result(std::vector<output_mapping> mappings, mem_ptr<T> x, int n) {
+  void dispatch_result(std::vector<output_mapping> mappings, mem_ptr<T> x, int n, solver_result_meta meta) {
     auto sender = actor_cast<actor>(this->current_sender());
     if (!sender) return;
     void* custom_dst = nullptr;
@@ -269,12 +270,12 @@ class sparse_gmres_facade : public event_based_actor {
     command_runner<T> runner;
     if (custom_dst) {
       runner.copy_to_host_async(x, static_cast<T*>(custom_dst), custom_count > 0 ? custom_count : (size_t)n,
-        [sender, r_id = reply_id_](T*, size_t) {
-          caf::anon_mail(r_id, 4).send(sender);
+        [sender, r_id = reply_id_, meta](T*, size_t) {
+          caf::anon_mail(r_id, 4, meta).send(sender);
         });
     } else {
-      runner.copy_to_host_async(x, [sender, r_id = reply_id_](std::vector<T> data) {
-        caf::anon_mail(r_id, 4, std::move(data)).send(sender);
+      runner.copy_to_host_async(x, [sender, r_id = reply_id_, meta](std::vector<T> data) {
+        caf::anon_mail(r_id, 4, std::move(data), meta).send(sender);
       });
     }
   }
diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index 9b683ae209..9b18c874a1 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -158,6 +158,16 @@ bool inspect(Inspector& f, buffer_variant& x) {
   return f.apply(x);
 }
 
+// Serialization support for solver_result_meta
+template <class Inspector>
+bool inspect(Inspector& f, caf::cuda::solver_result_meta& x) {
+  return f.object(x).fields(f.field("device_num", x.device_num),
+                            f.field("stream_id", x.stream_id),
+                            f.field("iterations", x.iterations),
+                            f.field("converged", x.converged),
+                            f.field("error_code", x.error_code));
+}
+
 namespace caf::cuda {
 
 // Serialization support for matrix_format
@@ -221,6 +231,7 @@ CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id)
   CAF_ADD_TYPE_ID(cuda,(caf::cuda::mem_ptr<double>))  
   CAF_ADD_TYPE_ID(cuda,(caf::cuda::mem_ptr<char>))  
   CAF_ADD_TYPE_ID(cuda, (caf::cuda::matrix_format))
+  CAF_ADD_TYPE_ID(cuda, (caf::cuda::solver_result_meta))
   
   //atoms 
   CAF_ADD_ATOM(cuda, kernel_done_atom)
@@ -250,3 +261,4 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::program_ptr)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(output_mapping)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<output_mapping>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::matrix_format)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::solver_result_meta)
diff --git a/libcaf_cuda/caf/cuda/types.hpp b/libcaf_cuda/caf/cuda/types.hpp
index d22b0c6e45..e03c819fd6 100644
--- a/libcaf_cuda/caf/cuda/types.hpp
+++ b/libcaf_cuda/caf/cuda/types.hpp
@@ -59,6 +59,18 @@ enum class matrix_format {
   coo
 };
 
+struct solver_result_meta {
+  int device_num;
+  int stream_id;
+  int iterations;
+  bool converged;
+  int error_code;
+
+  solver_result_meta() : device_num(-1), stream_id(-1), iterations(0), converged(false), error_code(0) {}
+  solver_result_meta(int dev, int stream, int iters, bool conv, int err = 0)
+    : device_num(dev), stream_id(stream), iterations(iters), converged(conv), error_code(err) {}
+};
+
 } // namespace caf::cuda
 
 // Structure for mapping kernel output indices to specific host memory buffers

From b7218f66c5f1879bfd3b8d269fadcafbe8894705 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 29 May 2026 09:38:47 -0600
Subject: [PATCH 0777/1000] updated test to accomodate changes in message
 passing interface

---
 .../sparse-GMRES-actor/sparse-GMRES-actor.hpp |  6 +++
 .../BICGSTAB-actor-test/main.test.cpp         | 42 ++++++++++---------
 .../CGS-actor-test/main.test.cpp              | 28 +++++++------
 .../GMRES-actor-test/main.test.cpp            | 10 ++---
 4 files changed, 48 insertions(+), 38 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp
index eb70238f22..dcaaca9174 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp
@@ -69,6 +69,12 @@ class sparse_gmres_actor : public stateful_actor<sparse_gmres_state<T>> {
         if (!this->state().supervisor)
           this->state().supervisor = actor_cast<caf::actor>(this->current_sender());
         start_solve();
+      },
+      [this](gpu_done_atom, std::vector<T>& solution, solver_result_meta meta) {
+        auto& s = this->state();
+        if (s.supervisor)
+          this->mail(std::move(solution), meta).send(s.supervisor);
+        this->quit();
       }
     };
   }
diff --git a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
index a035a432fb..9fe6a5617e 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/BICGSTAB-actor-test/main.test.cpp
@@ -148,7 +148,8 @@ void caf_main(actor_system& sys) {
 
         self->mail(start_atom_v).send(solver);
         self->receive(
-            [&](std::vector<float> result_x) {
+            [&](std::vector<float> result_x, solver_result_meta meta) {
+                std::cout << "[INFO] Iterations: " << meta.iterations << ", Converged: " << std::boolalpha << meta.converged << std::endl;
                 verify_solution("CSR Simple", result_x, expected, tolerance);
             }
         );
@@ -169,7 +170,7 @@ void caf_main(actor_system& sys) {
 
         self->mail(start_atom_v).send(solver);
         self->receive(
-            [&](std::vector<float> result_x) {
+            [&](std::vector<float> result_x, solver_result_meta meta) {
                 verify_solution("CSC Simple", result_x, expected, tolerance);
             }
         );
@@ -190,7 +191,7 @@ void caf_main(actor_system& sys) {
 
         self->mail(start_atom_v).send(solver);
         self->receive(
-            [&](std::vector<float> result_x) {
+            [&](std::vector<float> result_x, solver_result_meta meta) {
                 verify_solution("COO Simple", result_x, expected, tolerance);
             }
         );
@@ -217,7 +218,7 @@ void caf_main(actor_system& sys) {
 
         self->mail(start_atom_v).send(solver);
         self->receive(
-            [&](std::vector<float> result_x) {
+            [&](std::vector<float> result_x, solver_result_meta meta) {
                 verify_solution("CSR Tridiagonal (3x3)", result_x, expected_tri, tolerance);
             }
         );
@@ -263,7 +264,7 @@ void caf_main(actor_system& sys) {
 
         self->mail(start_atom_v).send(solver);
         self->receive(
-            [&](std::vector<float> result) {
+            [&](std::vector<float> result, solver_result_meta meta) {
                 verify_solution("CSR N=100 Correctness", result, expected_mid, 1e-4f);
             }
         );
@@ -312,7 +313,7 @@ void caf_main(actor_system& sys) {
 
         self->mail(start_atom_v).send(solver);
         self->receive(
-            [&](std::vector<float> result) {
+            [&](std::vector<float> result, solver_result_meta meta) {
                 verify_solution("High Iteration Count Test", result, expected_high, 1e-2f);
             }
         );
@@ -340,7 +341,7 @@ void caf_main(actor_system& sys) {
 
             self->mail(start_atom_v).send(solver);
             self->receive(
-                [&](std::vector<float> result) {
+                [&](std::vector<float> result, solver_result_meta meta) {
                     verify_solution("Real Matrix (Float)", result, expected_real, 1e-2f);
                 }
             );
@@ -373,7 +374,7 @@ void caf_main(actor_system& sys) {
 
             self->mail(start_atom_v).send(solver);
             self->receive(
-                [&](std::vector<double> result) {
+                [&](std::vector<double> result, solver_result_meta meta) {
                     verify_solution<double>("Real Matrix (Double)", result, expected_real, 1e-8);
                 }
             );
@@ -410,10 +411,11 @@ void caf_main(actor_system& sys) {
 
         self->mail(start_atom_v).send(stress_solver);
         self->receive(
-            [&](std::vector<float> result) {
+            [&](std::vector<float> result, solver_result_meta meta) {
                 auto end = std::chrono::high_resolution_clock::now();
                 std::chrono::duration<double> elapsed = end - start;
                 std::cout << "[SUCCESS] Stress Test completed in " << elapsed.count() << " seconds." << std::endl;
+                std::cout << "[INFO] Iterations: " << meta.iterations << ", Converged: " << meta.converged << std::endl;
                 std::cout << "[INFO] First 5 elements of solution: ";
                 for(int i=0; i<5; ++i) std::cout << result[i] << " ";
                 std::cout << "..." << std::endl;
@@ -439,7 +441,7 @@ void caf_main(actor_system& sys) {
 
         self->mail(start_atom_v).send(solver);
         self->receive(
-            [&](std::vector<double> result_x) {
+            [&](std::vector<double> result_x, solver_result_meta meta) {
                 verify_solution<double>("Double CSR Simple", result_x, expected_d, 1e-9);
             }
         );
@@ -463,7 +465,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n_f, nnz_f, tolerance, max_iter, 0, 9).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x, solver_result_meta meta) {
                 verify_solution("Facade CSR Simple", result_x, expected_f, tolerance);
             }
         );
@@ -486,7 +488,7 @@ void caf_main(actor_system& sys) {
                        matrix_format::csr, A.rows, A.nnz, 1e-5f, 5000, 0, 10).send(facade);
 
             self->receive(
-                [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result) {
+                [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result, solver_result_meta meta) {
                     verify_solution("Facade Real Matrix", result, expected_real, 1e-2f);
                 }
             );
@@ -511,7 +513,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, 0, 11).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int index) {
+            [&](uint32_t /*resp_id*/, int index, solver_result_meta meta) {
                 if (index == 4)
                   verify_solution("Facade Custom Buffer", custom_x, expected);
             }
@@ -534,7 +536,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, 0, 12).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr) {
+            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr, solver_result_meta meta) {
                 auto result_x = ptr->copy_to_host();
                 verify_solution("Facade mem_ptr", result_x, expected);
             }
@@ -557,7 +559,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, 0, 13).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x, solver_result_meta meta) {
                 verify_solution("Jacobi Facade CSR Simple", result_x, expected);
             }
         );
@@ -579,7 +581,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, 0, 14).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int index) {
+            [&](uint32_t /*resp_id*/, int index, solver_result_meta meta) {
                 if (index == 4)
                   verify_solution("Jacobi Facade Custom Buffer", custom_x, expected);
             }
@@ -602,7 +604,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, 0, 15).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr) {
+            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr, solver_result_meta meta) {
                 auto result_x = ptr->copy_to_host();
                 verify_solution("Jacobi Facade mem_ptr", result_x, expected);
             }
@@ -625,7 +627,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csc, n, nnz, tolerance, max_iter, 0, 16).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x, solver_result_meta meta) {
                 verify_solution("Jacobi Facade CSC Simple", result_x, expected);
             }
         );
@@ -648,7 +650,7 @@ void caf_main(actor_system& sys) {
                        matrix_format::csr, A.rows, A.nnz, 1e-5f, 5000, 0, 17).send(facade);
 
             self->receive(
-                [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result) {
+                [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result, solver_result_meta meta) {
                     verify_solution("Jacobi Facade Real Matrix", result, expected_real, 1e-2f);
                 }
             );
@@ -659,4 +661,4 @@ void caf_main(actor_system& sys) {
 
     manager::shutdown();
 }
-CAF_MAIN(id_block::cuda, id_block::cg_actor)
+CAF_MAIN(id_block::cuda)
diff --git a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
index dcdabe8803..03c2d3223d 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
@@ -72,7 +72,8 @@ void caf_main(actor_system& sys) {
 
         self->mail(start_atom_v).send(solver);
         self->receive(
-            [&](std::vector<float> result_x) {
+            [&](std::vector<float> result_x, solver_result_meta meta) {
+                std::cout << "[INFO] Iterations: " << meta.iterations << ", Converged: " << std::boolalpha << meta.converged << std::endl;
                 verify_solution("CSR Simple", result_x, expected);
             }
         );
@@ -93,7 +94,7 @@ void caf_main(actor_system& sys) {
 
         self->mail(start_atom_v).send(solver);
         self->receive(
-            [&](std::vector<float> result_x) {
+            [&](std::vector<float> result_x, solver_result_meta meta) {
                 verify_solution("CSC Simple", result_x, expected);
             }
         );
@@ -126,10 +127,11 @@ void caf_main(actor_system& sys) {
 
         self->mail(start_atom_v).send(stress_solver);
         self->receive(
-            [&](std::vector<float> result) {
+            [&](std::vector<float> result, solver_result_meta meta) {
                 auto end = std::chrono::high_resolution_clock::now();
                 std::chrono::duration<double> elapsed = end - start;
                 std::cout << "[SUCCESS] Stress Test completed in " << elapsed.count() << " seconds." << std::endl;
+                std::cout << "[INFO] Iterations: " << meta.iterations << ", Converged: " << meta.converged << std::endl;
                 std::cout << "[INFO] First 5 elements of solution: ";
                 for(int i=0; i<5; ++i) std::cout << result[i] << " ";
                 std::cout << "..." << std::endl;
@@ -153,7 +155,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, 0, 3).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x, solver_result_meta meta) {
                 verify_solution("Facade CSR Simple", result_x, expected);
             }
         );
@@ -175,7 +177,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, 0, 4).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int index) {
+            [&](uint32_t /*resp_id*/, int index, solver_result_meta meta) {
                 if (index == 4)
                   verify_solution("Facade Custom Buffer", custom_x, expected);
             }
@@ -198,7 +200,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, 0, 5).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr) {
+            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr, solver_result_meta meta) {
                 auto result_x = ptr->copy_to_host();
                 verify_solution("Facade mem_ptr", result_x, expected);
             }
@@ -220,7 +222,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, 0, 6).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x, solver_result_meta meta) {
                 verify_solution("Facade Default", result_x, expected);
             }
         );
@@ -242,7 +244,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csc, n, nnz, tolerance, max_iter, 0, 7).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x, solver_result_meta meta) {
                 verify_solution("Facade CSC Simple", result_x, expected);
             }
         );
@@ -264,7 +266,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, 0, 8).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x, solver_result_meta meta) {
                 verify_solution("Jacobi Facade CSR Simple", result_x, expected);
             }
         );
@@ -286,7 +288,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, 0, 9).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int index) {
+            [&](uint32_t /*resp_id*/, int index, solver_result_meta meta) {
                 if (index == 4)
                   verify_solution("Jacobi Facade Custom Buffer", custom_x, expected);
             }
@@ -309,7 +311,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, 0, 10).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr) {
+            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr, solver_result_meta meta) {
                 auto result_x = ptr->copy_to_host();
                 verify_solution("Jacobi Facade mem_ptr", result_x, expected);
             }
@@ -331,7 +333,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, 0, 11).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x, solver_result_meta meta) {
                 verify_solution("Jacobi Facade Default", result_x, expected);
             }
         );
@@ -353,7 +355,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csc, n, nnz, tolerance, max_iter, 0, 12).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x, solver_result_meta meta) {
                 verify_solution("Jacobi Facade CSC Simple", result_x, expected);
             }
         );
diff --git a/libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/main.test.cpp
index c7533e019b..e6b1e4c3a0 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/main.test.cpp
@@ -73,7 +73,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, restart_m, 0, 0).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x, solver_result_meta meta) {
                 verify_solution("GMRES Facade CSR Simple", result_x, expected);
             }
         );
@@ -95,7 +95,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, restart_m, 0, 1).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int index) {
+            [&](uint32_t /*resp_id*/, int index, solver_result_meta meta) {
                 if (index == 4)
                   verify_solution("GMRES Facade Custom Buffer", custom_x, expected);
             }
@@ -118,7 +118,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, n, nnz, tolerance, max_iter, restart_m, 0, 2).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr) {
+            [&](uint32_t /*resp_id*/, mem_ptr<float> ptr, solver_result_meta meta) {
                 auto result_x = ptr->copy_to_host();
                 verify_solution("GMRES Facade mem_ptr", result_x, expected);
             }
@@ -152,7 +152,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csr, N_large, (int)values.size(), 1e-4f, 2000, 30, 0, 3).send(facade);
 
         self->receive(
-            [&](uint32_t, int, std::vector<float> result) {
+            [&](uint32_t, int, std::vector<float> result, solver_result_meta meta) {
                 auto end = std::chrono::high_resolution_clock::now();
                 std::chrono::duration<double> elapsed = end - start;
                 std::cout << "[SUCCESS] Stress Test completed in " << elapsed.count() << " seconds." << std::endl;
@@ -178,7 +178,7 @@ void caf_main(actor_system& sys) {
                    matrix_format::csc, n, nnz, tolerance, max_iter, restart_m, 0, 4).send(facade);
 
         self->receive(
-            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x) {
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x, solver_result_meta meta) {
                 verify_solution("GMRES Facade CSC Simple", result_x, expected);
             }
         );

From 1847b683cbf145a5c7fe66e2390dfcaed562cde6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 29 May 2026 10:33:41 -0600
Subject: [PATCH 0778/1000] Initial commit.

---
 .../actorSOLVE/CMakeLists.txt                 |  49 +++++
 .../actorSOLVE/main.test.cpp                  | 176 ++++++++++++++++++
 2 files changed, 225 insertions(+)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/CMakeLists.txt b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/CMakeLists.txt
new file mode 100644
index 0000000000..0422ab8d27
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/CMakeLists.txt
@@ -0,0 +1,49 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+  "${CAF_SRC}/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor"
+  "${CAF_SRC}/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor"
+  "${CAF_SRC}/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor"
+)
+
+
+# 5) Declare your executables
+
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
+)
diff --git a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp
new file mode 100644
index 0000000000..1c6dab84b3
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp
@@ -0,0 +1,176 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include <fstream>
+#include <cmath>
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp"
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+// --- Matrix Utilities ---
+
+template <class T = float>
+struct LocalCSR {
+    int rows, cols, nnz;
+    std::vector<int> row_ptr;
+    std::vector<int> col_ind;
+    std::vector<T> values;
+};
+
+template <class T = float>
+LocalCSR<T> load_binary_matrix(const std::string& path) {
+    std::ifstream file(path, std::ios::binary);
+    if (!file) throw std::runtime_error("Could not open " + path);
+    int32_t r, c, n;
+    file.read(reinterpret_cast<char*>(&r), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&c), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&n), sizeof(int32_t));
+    std::vector<int32_t> rows_coo(n), cols_coo(n);
+    std::vector<float> vals_coo(n);
+    file.read(reinterpret_cast<char*>(rows_coo.data()), n * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(cols_coo.data()), n * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(vals_coo.data()), n * sizeof(float));
+    int max_idx = 0;
+    for(auto v : rows_coo) if(v > max_idx) max_idx = v;
+    if (max_idx == r || max_idx == c) {
+        for(auto& v : rows_coo) v--;
+        for(auto& v : cols_coo) v--;
+    }
+    LocalCSR<T> csr;
+    csr.rows = r; csr.cols = c; csr.nnz = n;
+    csr.row_ptr.assign(r + 1, 0);
+    csr.col_ind.resize(n);
+    csr.values.resize(n);
+    for (int i = 0; i < n; ++i) csr.row_ptr[rows_coo[i] + 1]++;
+    for (int i = 0; i < r; ++i) csr.row_ptr[i + 1] += csr.row_ptr[i];
+    std::vector<int> current_pos = csr.row_ptr;
+    for (int i = 0; i < n; ++i) {
+        int row = rows_coo[i];
+        int dest = current_pos[row]++;
+        csr.col_ind[dest] = cols_coo[i];
+        csr.values[dest] = static_cast<T>(vals_coo[i]);
+    }
+    return csr;
+}
+
+template <class T = float>
+std::vector<T> compute_rhs(const LocalCSR<T>& A, const std::vector<T>& x) {
+    std::vector<T> b(A.rows, T{0});
+    for (int i = 0; i < A.rows; ++i) {
+        T sum = T{0};
+        for (int j = A.row_ptr[i]; j < A.row_ptr[i+1]; ++j)
+            sum += A.values[j] * x[A.col_ind[j]];
+        b[i] = sum;
+    }
+    return b;
+}
+
+bool is_valid(const std::vector<float>& x) {
+    for (float val : x) {
+        if (std::isnan(val) || std::isinf(val)) return false;
+    }
+    return true;
+}
+
+// --- Robust Solver Actor (Facade Orchestrator) ---
+
+struct robust_solver_state {
+    LocalCSR<float> A;
+    std::vector<float> b;
+    float tol;
+    int max_iter;
+    const char* method = "CGS";
+};
+
+behavior robust_solver(stateful_actor<robust_solver_state>* self, 
+                       LocalCSR<float> A, std::vector<float> b, float tol, int max_iter) {
+    self->state().A = std::move(A);
+    self->state().b = std::move(b);
+    self->state().tol = tol;
+    self->state().max_iter = max_iter;
+
+    auto start_cgs = [=] {
+        auto& s = self->state();
+        s.method = "CGS";
+        auto facade = self->spawn<sparse_cg_facade>(100);
+        std::vector<float> x(s.A.rows, 0.0f);
+        self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
+                   create_in_arg(s.b), create_in_out_arg(x),
+                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 0, 0).send(facade);
+    };
+
+    auto start_bicgstab = [=] {
+        auto& s = self->state();
+        s.method = "BiCGSTAB";
+        auto facade = self->spawn<sparse_bicgstab_facade<float>>(100);
+        std::vector<float> x(s.A.rows, 0.0f);
+        self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
+                   create_in_arg(s.b), create_in_out_arg(x),
+                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 0, 0).send(facade);
+    };
+
+    auto start_gmres = [=] {
+        auto& s = self->state();
+        s.method = "GMRES";
+        auto facade = self->spawn<sparse_gmres_facade<float>>(100);
+        std::vector<float> x(s.A.rows, 0.0f);
+        self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
+                   create_in_arg(s.b), create_in_out_arg(x),
+                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 30, 0, 0).send(facade);
+    };
+
+    return {
+        [=](start_atom) {
+            std::cout << "[INFO] Attempting solve with CGS..." << std::endl;
+            start_cgs();
+        },
+        [=](uint32_t /*id*/, int /*idx*/, const std::vector<float>& result, solver_result_meta meta) {
+            auto& s = self->state();
+            if (meta.converged && is_valid(result)) {
+                std::cout << "[SUCCESS] " << s.method << " converged. Matrix good." << std::endl;
+                self->quit();
+            } else {
+                std::cout << "[WARNING] " << s.method << " failed or produced NaN. Retrying..." << std::endl;
+                if (std::string(s.method) == "CGS") {
+                    start_bicgstab();
+                } else if (std::string(s.method) == "BiCGSTAB") {
+                    start_gmres();
+                } else {
+                    std::cout << "[ERROR] All facade solvers failed." << std::endl;
+                    self->quit();
+                }
+            }
+        }
+    };
+}
+
+void caf_main(actor_system& sys) {
+    manager::init(sys, manager_config(true, true)); 
+    scoped_actor self{sys};
+
+    std::string path = "/scratch/nqr159/matrix-collection/matrices/unsymmetric/lnsp3937.bin";
+    std::cout << "[INFO] Loading real-world matrix: " << path << std::endl;
+
+    try {
+        LocalCSR<float> A = load_binary_matrix<float>(path);
+        std::vector<float> x_target(A.rows, 1.0f);
+        std::vector<float> b = compute_rhs(A, x_target);
+
+        auto robust = sys.spawn(robust_solver, std::move(A), std::move(b), 1e-5f, 5000);
+        self->mail(start_atom_v).send(robust);
+        
+        self->receive(
+            [] { std::cout << "[INFO] Robust solver test complete." << std::endl; }
+        );
+    } catch (const std::exception& e) {
+        std::cerr << "[ERROR] " << e.what() << std::endl;
+    }
+
+    manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda)
\ No newline at end of file

From 46c3df7df019970692c450943a16618a20e245a1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 29 May 2026 10:41:51 -0600
Subject: [PATCH 0779/1000] added real world matirx test

---
 .../GMRES-actor-test/main.test.cpp            | 103 ++++++++++++++++++
 1 file changed, 103 insertions(+)

diff --git a/libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/main.test.cpp
index e6b1e4c3a0..1aedd9915d 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/GMRES-actor-test/main.test.cpp
@@ -12,6 +12,7 @@
 #include <vector>
 #include <chrono>
 #include <thread>
+#include <fstream>
 #include <algorithm>
 #include <numeric>
 #include <cmath>
@@ -21,6 +22,78 @@
 using namespace caf;
 using namespace caf::cuda;
 
+// --- Manual Sparse Utilities for Testing ---
+
+template <class T = float>
+struct LocalCSR {
+    int rows, cols, nnz;
+    std::vector<int> row_ptr;
+    std::vector<int> col_ind;
+    std::vector<T> values;
+};
+
+template <class T = float>
+LocalCSR<T> load_binary_matrix_manual(const std::string& path) {
+    std::ifstream file(path, std::ios::binary);
+    if (!file) throw std::runtime_error("Could not open " + path);
+
+    int32_t r, c, n;
+    file.read(reinterpret_cast<char*>(&r), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&c), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&n), sizeof(int32_t));
+
+    std::vector<int32_t> rows_coo(n), cols_coo(n);
+    std::vector<float> vals_coo(n);
+
+    file.read(reinterpret_cast<char*>(rows_coo.data()), n * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(cols_coo.data()), n * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(vals_coo.data()), n * sizeof(float));
+
+    // Detect and fix 1-based indexing (Matrix Market standard)
+    int max_idx = 0;
+    for(auto v : rows_coo) if(v > max_idx) max_idx = v;
+    for(auto v : cols_coo) if(v > max_idx) max_idx = v;
+
+    if (max_idx == r || max_idx == c) {
+        for(auto& v : rows_coo) v--;
+        for(auto& v : cols_coo) v--;
+    }
+
+    // Convert COO to CSR
+    LocalCSR<T> csr;
+    csr.rows = r; csr.cols = c; csr.nnz = n;
+    csr.row_ptr.assign(r + 1, 0);
+    csr.col_ind.resize(n);
+    csr.values.resize(n);
+
+    for (int i = 0; i < n; ++i) csr.row_ptr[rows_coo[i] + 1]++;
+    for (int i = 0; i < r; ++i) csr.row_ptr[i + 1] += csr.row_ptr[i];
+
+    std::vector<int> current_pos = csr.row_ptr;
+    for (int i = 0; i < n; ++i) {
+        int row = rows_coo[i];
+        int dest = current_pos[row]++;
+        csr.col_ind[dest] = cols_coo[i];
+        csr.values[dest] = static_cast<T>(vals_coo[i]);
+    }
+    return csr;
+}
+
+template <class T = float>
+std::vector<T> compute_rhs_manual(const LocalCSR<T>& A, const std::vector<T>& x) {
+    std::vector<T> b(A.rows, T{0});
+    for (int i = 0; i < A.rows; ++i) {
+        T sum = T{0};
+        for (int j = A.row_ptr[i]; j < A.row_ptr[i+1]; ++j) {
+            sum += A.values[j] * x[A.col_ind[j]];
+        }
+        b[i] = sum;
+    }
+    return b;
+}
+
+// --- End Manual Utilities ---
+
 void verify_solution(const std::string& test_name, const std::vector<float>& actual, 
                      const std::vector<float>& expected, float tol = 1e-3) {
     if (actual.size() != expected.size()) {
@@ -184,6 +257,36 @@ void caf_main(actor_system& sys) {
         );
     }
 
+    // Test 6: Real-world matrix from file (lnsp3937.bin)
+    {
+        std::string path = "/scratch/nqr159/matrix-collection/matrices/unsymmetric/lnsp3937.bin";
+        std::cout << "\n[INFO] Test 6: Loading real-world matrix " << path << "..." << std::endl;
+        try {
+            LocalCSR<float> A = load_binary_matrix_manual<float>(path);
+            std::cout << "[INFO] Matrix Metadata: Rows=" << A.rows << ", Cols=" << A.cols 
+                      << ", NNZ=" << A.nnz << std::endl;
+            
+            std::vector<float> expected_real(A.rows, 1.0f);
+            std::vector<float> b_real = compute_rhs_manual<float>(A, expected_real);
+            std::vector<float> h_x(A.rows, 0.0f);
+
+            auto facade = sys.spawn<sparse_gmres_facade<float>>(100);
+
+            self->mail(create_in_arg(A.row_ptr), create_in_arg(A.col_ind), create_in_arg(A.values),
+                       create_in_arg(b_real), create_in_out_arg(h_x),
+                       matrix_format::csr, A.rows, A.nnz, 1e-5f, 5000, 30, 0, 5).send(facade);
+
+            self->receive(
+                [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result, solver_result_meta meta) {
+                    std::cout << "[INFO] Iterations: " << meta.iterations << ", Converged: " << std::boolalpha << meta.converged << std::endl;
+                    verify_solution("GMRES Real Matrix", result, expected_real, 1e-2f);
+                }
+            );
+        } catch (const std::exception& e) {
+            std::cout << "[ERROR] Test 6 Failed: " << e.what() << std::endl;
+        }
+    }
+
     manager::shutdown();
 }
 

From 17295b410050c5eaa6e82fc44ba0c38867c54c8d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 29 May 2026 11:52:32 -0600
Subject: [PATCH 0780/1000] updated CGS actor to accomodate doubles

---
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 321 ++++++++++--------
 .../CGS-actor-test/main.test.cpp              | 142 +++++++-
 .../actorSOLVE/main.test.cpp                  |  97 ++++--
 3 files changed, 372 insertions(+), 188 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index a7ed0807c8..d4348cf91f 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -18,18 +18,19 @@ constexpr int id_spmv = 200;
 constexpr int id_axpy = 300;
 constexpr int id_copy = 400;
 
+template <class T = float>
 struct sparse_cg_state {
  // Host Data (stored until start)
  in<int> h_row_ptr, h_col_ind;
- in<float> h_values, h_b;
- in_out<float> h_x;
+ in<T> h_values, h_b;
+ in_out<T> h_x;
 
  // Device Problem data
  mem_ptr<int> A_row_ptr, A_col_ind;
- mem_ptr<float> A_values, b, x;
+ mem_ptr<T> A_values, b, x;
  matrix_format format;
  int n, nnz;
- float tol;
+ T tol;
  int max_iter;
  int device_num;
  int stream_id;
@@ -37,35 +38,36 @@ struct sparse_cg_state {
  caf::actor supervisor;
 
  // Workspace vectors
- mem_ptr<float> r, p, w, y_tmp;
+ mem_ptr<T> r, p, w, y_tmp;
  mem_ptr<char> spmv_workspace;
  
  // Scalars needed across asynchronous steps
- float rho_val = 0.0f;
- float old_rho_val = 0.0f;
- float alpha_val = 0.0f;
- float beta_val = 0.0f;
- float dot_pw_val = 0.0f;
+ T rho_val = T{0};
+ T old_rho_val = T{0};
+ T alpha_val = T{0};
+ T beta_val = T{0};
+ T dot_pw_val = T{0};
  int iterations = 0;
 };
 
-class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
+template <class T = float>
+class sparse_cg_actor : public stateful_actor<sparse_cg_state<T>> {
 public:
  sparse_cg_actor(actor_config& cfg, in<int> rp, in<int> ci,
-                 in<float> val, in<float> b, in_out<float> x,
-                 matrix_format fmt, int n, int nnz, float tol, int max_iter, int device_num, int stream_id,
+                 in<T> val, in<T> b, in_out<T> x,
+                 matrix_format fmt, int n, int nnz, T tol, int max_iter, int device_num, int stream_id,
                  caf::actor supervisor = nullptr)
-   : stateful_actor<sparse_cg_state>(cfg) {
-   state().h_row_ptr = std::move(rp);
-   state().h_col_ind = std::move(ci);
-   state().h_values = std::move(val);
-   state().h_b = std::move(b);
-   state().h_x = std::move(x);
-   state().format = fmt;
-   state().n = n; state().nnz = nnz;
-   state().tol = tol; state().max_iter = max_iter;
-   state().device_num = device_num; state().stream_id = stream_id;
-   state().supervisor = supervisor;
+   : stateful_actor<sparse_cg_state<T>>(cfg) {
+   this->state().h_row_ptr = std::move(rp);
+   this->state().h_col_ind = std::move(ci);
+   this->state().h_values = std::move(val);
+   this->state().h_b = std::move(b);
+   this->state().h_x = std::move(x);
+   this->state().format = fmt;
+   this->state().n = n; this->state().nnz = nnz;
+   this->state().tol = tol; this->state().max_iter = max_iter;
+   this->state().device_num = device_num; this->state().stream_id = stream_id;
+   this->state().supervisor = supervisor;
  }
 
  ~sparse_cg_actor() override = default;
@@ -73,14 +75,14 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
  behavior make_behavior() override {
    return {
      [this](start_atom) {
-       auto& s = state();
+       auto& s = this->state();
        if (!s.supervisor)
          s.supervisor = actor_cast<caf::actor>(this->current_sender());
        start_solve();
      },
-     [this](gpu_done_atom, std::vector<float>& solution, solver_result_meta meta) {
-       if (state().supervisor)
-         this->mail(std::move(solution), meta).send(state().supervisor);
+     [this](gpu_done_atom, std::vector<T>& solution, solver_result_meta meta) {
+       if (this->state().supervisor)
+         this->mail(std::move(solution), meta).send(this->state().supervisor);
        this->quit();
      }
    };
@@ -88,8 +90,8 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
 
 private:
  void start_solve() {
-   auto& s = state();
-   command_runner<float> runner;
+   auto& s = this->state();
+   command_runner<T> runner;
 
    // Transfer problem data to device
    auto res = runner.transfer_memory(s.device_num, s.stream_id,
@@ -108,11 +110,11 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
    s.d_ptr = platform::create()->schedule(s.stream_id, s.device_num);
 
    // Allocate workspace
-   command_runner<out<float>> work_runner;
-   s.r = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-   s.p = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-   s.w = work_runner.transfer_memory(s.device_num, s.stream_id, out<float>(s.n));
-   s.y_tmp = work_runner.transfer_memory(s.device_num, s.stream_id, create_out_arg_with_size<float>(1));
+   command_runner<out<T>> work_runner;
+   s.r = work_runner.transfer_memory(s.device_num, s.stream_id, out<T>(s.n));
+   s.p = work_runner.transfer_memory(s.device_num, s.stream_id, out<T>(s.n));
+   s.w = work_runner.transfer_memory(s.device_num, s.stream_id, out<T>(s.n));
+   s.y_tmp = work_runner.transfer_memory(s.device_num, s.stream_id, create_out_arg_with_size<T>(1));
    
    // Allocate SPMV workspace to avoid reallocations in the loop
    size_t ws_size = 0;
@@ -128,63 +130,80 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
      s.spmv_workspace = ws_runner.transfer_memory(s.device_num, s.stream_id, out<char>(static_cast<int>(ws_size)));
    }
 
+   auto execute_copy = [&](mem_ptr<T> src, mem_ptr<T> dst) {
+     if constexpr (std::is_same_v<T, double>) s.d_ptr->dcopy(s.stream_id, s.n, src, dst); else s.d_ptr->scopy(s.stream_id, s.n, src, dst);
+   };
+   auto execute_axpy = [&](T alpha, mem_ptr<T> xv, mem_ptr<T> yv) {
+     if constexpr (std::is_same_v<T, double>) s.d_ptr->daxpy(s.stream_id, s.n, alpha, xv, yv); else s.d_ptr->saxpy(s.stream_id, s.n, static_cast<float>(alpha), xv, yv);
+   };
+   auto execute_dot = [&](mem_ptr<T> xv, mem_ptr<T> yv, mem_ptr<T> rv) {
+     if constexpr (std::is_same_v<T, double>) s.d_ptr->ddot(s.stream_id, s.n, xv, yv, rv); else s.d_ptr->sdot(s.stream_id, s.n, xv, yv, rv);
+   };
+
    // 1. Initial SpMV: w = A * x
    execute_spmv(s.x, s.w);
 
    // 2. Initial r = b - w
-   s.d_ptr->scopy(s.stream_id, s.n, s.b, s.r);
-   s.d_ptr->saxpy(s.stream_id, s.n, -1.0f, s.w, s.r);
+   execute_copy(s.b, s.r);
+   execute_axpy(T{-1}, s.w, s.r);
 
    // 3. Initial rho = r * r
-   s.d_ptr->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
+   execute_dot(s.r, s.r, s.y_tmp);
    
    // Fetch initial rho synchronously to start the loop
    s.rho_val = runner.copy_to_host(s.y_tmp)[0];
+   T threshold = s.tol * s.tol;
 
    // The Real Performance Fix: The Tight CG Loop
    // By running the loop here, we eliminate 20,000+ scheduler context switches.
-   while (s.rho_val > (s.tol * s.tol) && s.iterations < s.max_iter) {
+   while (s.rho_val > threshold && s.iterations < s.max_iter) {
      s.iterations++;
 
      if (s.iterations > 1) {
        s.beta_val = s.rho_val / s.old_rho_val;
-       s.d_ptr->scopy(s.stream_id, s.n, s.r, s.w);
-       s.d_ptr->saxpy(s.stream_id, s.n, s.beta_val, s.p, s.w);
-       s.d_ptr->scopy(s.stream_id, s.n, s.w, s.p);
+       execute_copy(s.r, s.w);
+       execute_axpy(s.beta_val, s.p, s.w);
+       execute_copy(s.w, s.p);
      } else {
-       s.d_ptr->scopy(s.stream_id, s.n, s.r, s.p);
+       execute_copy(s.r, s.p);
      }
 
      execute_spmv(s.p, s.w);
 
-     s.d_ptr->sdot(s.stream_id, s.n, s.p, s.w, s.y_tmp);
+     execute_dot(s.p, s.w, s.y_tmp);
      // This synchronous call blocks the CAF thread ONLY until this dot product is ready.
      // This is 100x faster than yielding to the scheduler.
      s.dot_pw_val = runner.copy_to_host(s.y_tmp)[0];
 
      s.alpha_val = s.rho_val / s.dot_pw_val;
-     s.d_ptr->saxpy(s.stream_id, s.n, s.alpha_val, s.p, s.x);
-     s.d_ptr->saxpy(s.stream_id, s.n, -s.alpha_val, s.w, s.r);
+     execute_axpy(s.alpha_val, s.p, s.x);
+     execute_axpy(-s.alpha_val, s.w, s.r);
 
      s.old_rho_val = s.rho_val;
-     s.d_ptr->sdot(s.stream_id, s.n, s.r, s.r, s.y_tmp);
+     execute_dot(s.r, s.r, s.y_tmp);
      s.rho_val = runner.copy_to_host(s.y_tmp)[0];
    }
 
    // Exit the loop and return the result via the standard async path
    auto self = actor_cast<actor>(this);
-   solver_result_meta meta(s.device_num, s.stream_id, s.iterations, s.rho_val <= (s.tol * s.tol));
-   runner.copy_to_host_async(s.x, [self, meta](std::vector<float> solution) {
+   solver_result_meta meta(s.device_num, s.stream_id, s.iterations, s.rho_val <= threshold);
+   runner.copy_to_host_async(s.x, [self, meta](std::vector<T> solution) {
      anon_mail(gpu_done_atom_v, std::move(solution), meta).send(self);
    });
  }
 
- void execute_spmv(mem_ptr<float> input_v, mem_ptr<float> output_v) {
-   auto& s = state();
+ void execute_spmv(mem_ptr<T> input_v, mem_ptr<T> output_v) {
+   auto& s = this->state();
    switch (s.format) {
-     case matrix_format::csr: s.d_ptr->spmv_csr(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v, s.spmv_workspace); break;
-     case matrix_format::csc: s.d_ptr->spmv_csc(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v, s.spmv_workspace); break;
-     case matrix_format::coo: s.d_ptr->spmv_coo(s.stream_id, s.n, s.n, s.nnz, 1.0f, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, 0.0f, output_v, s.spmv_workspace); break;
+     case matrix_format::csr:
+       s.d_ptr->spmv_csr(s.stream_id, s.n, s.n, s.nnz, T{1}, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, T{0}, output_v, s.spmv_workspace);
+       break;
+     case matrix_format::csc:
+       s.d_ptr->spmv_csc(s.stream_id, s.n, s.n, s.nnz, T{1}, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, T{0}, output_v, s.spmv_workspace);
+       break;
+     case matrix_format::coo:
+       s.d_ptr->spmv_coo(s.stream_id, s.n, s.n, s.nnz, T{1}, s.A_row_ptr, s.A_col_ind, s.A_values, input_v, T{0}, output_v, s.spmv_workspace);
+       break;
      default: break;
    }
  }
@@ -195,6 +214,7 @@ class sparse_cg_actor : public stateful_actor<sparse_cg_state> {
  * This actor can be reused for multiple solve requests. It receives all 
  * solve parameters as a message and returns the solution vector to the sender.
  */
+template <class T = float>
 class sparse_cg_facade : public event_based_actor {
 public:
   sparse_cg_facade(actor_config& cfg, uint32_t response_id)
@@ -204,10 +224,10 @@ class sparse_cg_facade : public event_based_actor {
     return {
       // Mode 1: Return mem_ptr handles (GPU memory)
       [this](return_mem_ptr_atom,
-             in<int> rp, in<int> ci, in<float> val,
-             in<float> b_in, in_out<float> x_in,
+             in<int> rp, in<int> ci, in<T> val,
+             in<T> b_in, in_out<T> x_in,
              matrix_format fmt, int n, int nnz,
-             float tol, int max_iter,
+             T tol, int max_iter,
              int device_num, int stream_id) {
 
         auto [x, meta] = solve_core(rp, ci, val, b_in, x_in,
@@ -221,10 +241,10 @@ class sparse_cg_facade : public event_based_actor {
 
       // Mode 2: Return host data via mappings
       [this](std::vector<output_mapping> mappings,
-             in<int> rp, in<int> ci, in<float> val,
-             in<float> b_in, in_out<float> x_in,
+             in<int> rp, in<int> ci, in<T> val,
+             in<T> b_in, in_out<T> x_in,
              matrix_format fmt, int n, int nnz,
-             float tol, int max_iter,
+             T tol, int max_iter,
              int device_num, int stream_id) {
 
         auto [x, meta] = solve_core(rp, ci, val, b_in, x_in,
@@ -235,10 +255,10 @@ class sparse_cg_facade : public event_based_actor {
       },
 
       // Mode 3: Default (return vector to sender)
-      [this](in<int> rp, in<int> ci, in<float> val,
-             in<float> b_in, in_out<float> x_in,
+      [this](in<int> rp, in<int> ci, in<T> val,
+             in<T> b_in, in_out<T> x_in,
              matrix_format fmt, int n, int nnz,
-             float tol, int max_iter,
+             T tol, int max_iter,
              int device_num, int stream_id) {
 
         auto [x, meta] = solve_core(rp, ci, val, b_in, x_in,
@@ -251,13 +271,13 @@ class sparse_cg_facade : public event_based_actor {
   }
 
 protected:
-  virtual std::pair<mem_ptr<float>, solver_result_meta> solve_core(in<int> rp, in<int> ci, in<float> val, in<float> b_in,
-                                                                   in_out<float> x_in,
+  virtual std::pair<mem_ptr<T>, solver_result_meta> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in,
+                                                                   in_out<T> x_in,
                                                                    matrix_format fmt, int n, int nnz,
-                                                                   float tol, int max_iter,
+                                                                   T tol, int max_iter,
                                                                    int device_num, int stream_id) {
 
-    command_runner<float> runner;
+    command_runner<T> runner;
 
     auto res = runner.transfer_memory(device_num, stream_id,
                                       rp, ci, val, b_in, x_in);
@@ -270,11 +290,11 @@ class sparse_cg_facade : public event_based_actor {
 
     auto d_ptr = platform::create()->schedule(stream_id, device_num);
 
-    command_runner<out<float>> work_runner;
-    auto r     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
-    auto p     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
-    auto w     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
-    auto y_tmp = work_runner.transfer_memory(device_num, stream_id, create_out_arg_with_size<float>(1));
+    command_runner<out<T>> work_runner;
+    auto r     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto p     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto w     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto y_tmp = work_runner.transfer_memory(device_num, stream_id, create_out_arg_with_size<T>(1));
 
     mem_ptr<char> spmv_workspace;
     size_t ws_size = 0;
@@ -296,71 +316,81 @@ class sparse_cg_facade : public event_based_actor {
                                   out<char>(static_cast<int>(ws_size)));
     }
 
-    auto execute_spmv = [&](mem_ptr<float> input_v, mem_ptr<float> output_v) {
+    auto execute_spmv = [&](mem_ptr<T> input_v, mem_ptr<T> output_v) {
       switch (fmt) {
         case matrix_format::csr:
-          d_ptr->spmv_csr(stream_id, n, n, nnz, 1.0f,
+          d_ptr->spmv_csr(stream_id, n, n, nnz, T{1},
                           A_row_ptr, A_col_ind, A_values,
-                          input_v, 0.0f, output_v, spmv_workspace);
+                          input_v, T{0}, output_v, spmv_workspace);
           break;
 
         case matrix_format::csc:
-          d_ptr->spmv_csc(stream_id, n, n, nnz, 1.0f,
+          d_ptr->spmv_csc(stream_id, n, n, nnz, T{1},
                           A_row_ptr, A_col_ind, A_values,
-                          input_v, 0.0f, output_v, spmv_workspace);
+                          input_v, T{0}, output_v, spmv_workspace);
           break;
 
         case matrix_format::coo:
-          d_ptr->spmv_coo(stream_id, n, n, nnz, 1.0f,
+          d_ptr->spmv_coo(stream_id, n, n, nnz, T{1},
                           A_row_ptr, A_col_ind, A_values,
-                          input_v, 0.0f, output_v, spmv_workspace);
+                          input_v, T{0}, output_v, spmv_workspace);
           break;
 
         default:
           break;
       }
     };
+    auto execute_copy = [&](mem_ptr<T> src, mem_ptr<T> dst) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->dcopy(stream_id, n, src, dst); else d_ptr->scopy(stream_id, n, src, dst);
+    };
+    auto execute_axpy = [&](T alpha, mem_ptr<T> xv, mem_ptr<T> yv) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->daxpy(stream_id, n, alpha, xv, yv); else d_ptr->saxpy(stream_id, n, static_cast<float>(alpha), xv, yv);
+    };
+    auto execute_dot = [&](mem_ptr<T> xv, mem_ptr<T> yv, mem_ptr<T> rv) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->ddot(stream_id, n, xv, yv, rv); else d_ptr->sdot(stream_id, n, xv, yv, rv);
+    };
 
     execute_spmv(x, w);
-    d_ptr->scopy(stream_id, n, b, r);
-    d_ptr->saxpy(stream_id, n, -1.0f, w, r);
-    d_ptr->sdot(stream_id, n, r, r, y_tmp);
+    execute_copy(b, r);
+    execute_axpy(T{-1}, w, r);
+    execute_dot(r, r, y_tmp);
 
-    float rho_val = runner.copy_to_host(y_tmp)[0];
-    float old_rho_val = 0.0f;
+    T rho_val = runner.copy_to_host(y_tmp)[0];
+    T old_rho_val = T{0};
     int iterations = 0;
+    T threshold = tol * tol;
 
-    while (rho_val > (tol * tol) && iterations < max_iter) {
+    while (rho_val > threshold && iterations < max_iter) {
       iterations++;
 
       if (iterations > 1) {
-        float beta_val = rho_val / old_rho_val;
-        d_ptr->scopy(stream_id, n, r, w);
-        d_ptr->saxpy(stream_id, n, beta_val, p, w);
-        d_ptr->scopy(stream_id, n, w, p);
+        T beta_val = rho_val / old_rho_val;
+        execute_copy(r, w);
+        execute_axpy(beta_val, p, w);
+        execute_copy(w, p);
       } else {
-        d_ptr->scopy(stream_id, n, r, p);
+       execute_copy(r, p);
       }
 
       execute_spmv(p, w);
-      d_ptr->sdot(stream_id, n, p, w, y_tmp);
+      execute_dot(p, w, y_tmp);
 
-      float alpha_val =
+      T alpha_val =
         rho_val / runner.copy_to_host(y_tmp)[0];
 
-      d_ptr->saxpy(stream_id, n, alpha_val, p, x);
-      d_ptr->saxpy(stream_id, n, -alpha_val, w, r);
+      execute_axpy(alpha_val, p, x);
+      execute_axpy(-alpha_val, w, r);
 
       old_rho_val = rho_val;
-      d_ptr->sdot(stream_id, n, r, r, y_tmp);
+      execute_dot(r, r, y_tmp);
       rho_val = runner.copy_to_host(y_tmp)[0];
     }
 
-    return {x, solver_result_meta(device_num, stream_id, iterations, rho_val <= (tol * tol))};
+    return {x, solver_result_meta(device_num, stream_id, iterations, rho_val <= threshold)};
   }
 
   void dispatch_result(std::vector<output_mapping> mappings,
-                       mem_ptr<float> x,
+                       mem_ptr<T> x,
                        int n,
                        solver_result_meta meta) {
 
@@ -378,21 +408,21 @@ class sparse_cg_facade : public event_based_actor {
       }
     }
 
-    command_runner<float> runner;
+    command_runner<T> runner;
 
     if (custom_dst) {
       runner.copy_to_host_async(
         x,
-        static_cast<float*>(custom_dst),
+        static_cast<T*>(custom_dst),
         custom_count > 0 ? custom_count : (size_t)n,
-        [sender, r_id = reply_id_, meta](float*, size_t) {
+        [sender, r_id = reply_id_, meta](T*, size_t) {
           caf::anon_mail(r_id, 4, meta).send(sender);
         });
 
     } else {
       runner.copy_to_host_async(
         x,
-        [sender, r_id = reply_id_, meta](std::vector<float> data) {
+        [sender, r_id = reply_id_, meta](std::vector<T> data) {
           caf::anon_mail(r_id, 4, std::move(data), meta).send(sender);
         });
     }
@@ -405,10 +435,11 @@ class sparse_cg_facade : public event_based_actor {
 /**
  * A variant of the CG solver facade that uses Jacobi preconditioning.
  */
-class sparse_cg_jacobi_facade : public sparse_cg_facade {
+template <class T = float>
+class sparse_cg_jacobi_facade : public sparse_cg_facade<T> {
 public:
   sparse_cg_jacobi_facade(actor_config& cfg, uint32_t response_id)
-    : sparse_cg_facade(cfg, response_id) {
+    : sparse_cg_facade<T>(cfg, response_id) {
     // Deduce path to cubin relative to this header file at runtime
     std::string current_file = __FILE__;
     auto pos = current_file.find_last_of('/');
@@ -418,13 +449,13 @@ class sparse_cg_jacobi_facade : public sparse_cg_facade {
   }
 
 protected:
-  std::pair<mem_ptr<float>, solver_result_meta> solve_core(in<int> rp, in<int> ci, in<float> val, in<float> b_in,
-                                                           in_out<float> x_in,
+  std::pair<mem_ptr<T>, solver_result_meta> solve_core(in<int> rp, in<int> ci, in<T> val, in<T> b_in,
+                                                           in_out<T> x_in,
                                                            matrix_format fmt, int n, int nnz,
-                                                           float tol, int max_iter,
+                                                           T tol, int max_iter,
                                                            int device_num, int stream_id) override {
 
-    command_runner<float> runner;
+    command_runner<T> runner;
     auto res = runner.transfer_memory(device_num, stream_id,
                                       rp, ci, val, b_in, x_in);
 
@@ -436,13 +467,13 @@ class sparse_cg_jacobi_facade : public sparse_cg_facade {
 
     auto d_ptr = platform::create()->schedule(stream_id, device_num);
 
-    command_runner<out<float>> work_runner;
-    auto r     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
-    auto p     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
-    auto w     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
-    auto z     = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
-    auto D_inv = work_runner.transfer_memory(device_num, stream_id, out<float>(n));
-    auto y_tmp = work_runner.transfer_memory(device_num, stream_id, create_out_arg_with_size<float>(1));
+    command_runner<out<T>> work_runner;
+    auto r     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto p     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto w     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto z     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto D_inv = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto y_tmp = work_runner.transfer_memory(device_num, stream_id, create_out_arg_with_size<T>(1));
 
     mem_ptr<char> spmv_workspace;
     size_t ws_size = 0;
@@ -464,27 +495,36 @@ class sparse_cg_jacobi_facade : public sparse_cg_facade {
                                   out<char>(static_cast<int>(ws_size)));
     }
 
-    auto execute_spmv = [&](mem_ptr<float> input_v, mem_ptr<float> output_v) {
+    auto execute_spmv = [&](mem_ptr<T> input_v, mem_ptr<T> output_v) {
       switch (fmt) {
         case matrix_format::csr:
-          d_ptr->spmv_csr(stream_id, n, n, nnz, 1.0f,
+          d_ptr->spmv_csr(stream_id, n, n, nnz, T{1},
                           A_row_ptr, A_col_ind, A_values,
-                          input_v, 0.0f, output_v, spmv_workspace);
+                          input_v, T{0}, output_v, spmv_workspace);
           break;
         case matrix_format::csc:
-          d_ptr->spmv_csc(stream_id, n, n, nnz, 1.0f,
+          d_ptr->spmv_csc(stream_id, n, n, nnz, T{1},
                           A_row_ptr, A_col_ind, A_values,
-                          input_v, 0.0f, output_v, spmv_workspace);
+                          input_v, T{0}, output_v, spmv_workspace);
           break;
         case matrix_format::coo:
-          d_ptr->spmv_coo(stream_id, n, n, nnz, 1.0f,
+          d_ptr->spmv_coo(stream_id, n, n, nnz, T{1},
                           A_row_ptr, A_col_ind, A_values,
-                          input_v, 0.0f, output_v, spmv_workspace);
+                          input_v, T{0}, output_v, spmv_workspace);
           break;
         default:
           break;
       }
     };
+    auto execute_copy = [&](mem_ptr<T> src, mem_ptr<T> dst) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->dcopy(stream_id, n, src, dst); else d_ptr->scopy(stream_id, n, src, dst);
+    };
+    auto execute_axpy = [&](T alpha, mem_ptr<T> xv, mem_ptr<T> yv) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->daxpy(stream_id, n, alpha, xv, yv); else d_ptr->saxpy(stream_id, n, static_cast<float>(alpha), xv, yv);
+    };
+    auto execute_dot = [&](mem_ptr<T> xv, mem_ptr<T> yv, mem_ptr<T> rv) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->ddot(stream_id, n, xv, yv, rv); else d_ptr->sdot(stream_id, n, xv, yv, rv);
+    };
 
     // 0. Preconditioning setup: Extract diagonal inverse using custom kernel
     int threads = 256;
@@ -497,47 +537,48 @@ class sparse_cg_jacobi_facade : public sparse_cg_facade {
 
     // 1. Initial Residual: r = b - Ax
     execute_spmv(x, w);
-    d_ptr->scopy(stream_id, n, b, r);
-    d_ptr->saxpy(stream_id, n, -1.0f, w, r);
+    execute_copy(b, r);
+    execute_axpy(T{-1}, w, r);
     
     // 2. Initial Preconditioned Residual: z = D_inv * r
-    d_ptr->s_elementwise_multiply(stream_id, n, D_inv, r, z);
+    if constexpr (std::is_same_v<T, double>) d_ptr->d_elementwise_multiply(stream_id, n, D_inv, r, z); else d_ptr->s_elementwise_multiply(stream_id, n, D_inv, r, z);
     
     // 3. Initial rho = r * z
-    d_ptr->sdot(stream_id, n, r, z, y_tmp);
+    execute_dot(r, z, y_tmp);
 
-    float rho_val = runner.copy_to_host(y_tmp)[0];
-    float old_rho_val = 0.0f;
+    T rho_val = runner.copy_to_host(y_tmp)[0];
+    T old_rho_val = T{0};
     int iterations = 0;
+    T threshold = tol * tol;
 
-    while (rho_val > (tol * tol) && iterations < max_iter) {
+    while (rho_val > threshold && iterations < max_iter) {
       iterations++;
 
       if (iterations > 1) {
-        float beta_val = rho_val / old_rho_val;
+        T beta_val = rho_val / old_rho_val;
         // p = z + beta * p
-        d_ptr->scopy(stream_id, n, z, w);
-        d_ptr->saxpy(stream_id, n, beta_val, p, w);
-        d_ptr->scopy(stream_id, n, w, p);
+        execute_copy(z, w);
+        execute_axpy(beta_val, p, w);
+        execute_copy(w, p);
       } else {
         // p = z
-        d_ptr->scopy(stream_id, n, z, p);
+        execute_copy(z, p);
       }
 
       execute_spmv(p, w);
-      d_ptr->sdot(stream_id, n, p, w, y_tmp);
-      float alpha_val = rho_val / runner.copy_to_host(y_tmp)[0];
+      execute_dot(p, w, y_tmp);
+      T alpha_val = rho_val / runner.copy_to_host(y_tmp)[0];
 
-      d_ptr->saxpy(stream_id, n, alpha_val, p, x);
-      d_ptr->saxpy(stream_id, n, -alpha_val, w, r);
+      execute_axpy(alpha_val, p, x);
+      execute_axpy(-alpha_val, w, r);
 
       old_rho_val = rho_val;
-      d_ptr->s_elementwise_multiply(stream_id, n, D_inv, r, z); // z_new = D_inv * r
-      d_ptr->sdot(stream_id, n, r, z, y_tmp); // rho_new = r * z_new
+      if constexpr (std::is_same_v<T, double>) d_ptr->d_elementwise_multiply(stream_id, n, D_inv, r, z); else d_ptr->s_elementwise_multiply(stream_id, n, D_inv, r, z);
+      execute_dot(r, z, y_tmp); // rho_new = r * z_new
       rho_val = runner.copy_to_host(y_tmp)[0];
     }
 
-    return {x, solver_result_meta(device_num, stream_id, iterations, rho_val <= (tol * tol))};
+    return {x, solver_result_meta(device_num, stream_id, iterations, rho_val <= threshold)};
   }
 
 private:
diff --git a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
index 03c2d3223d..bd36afd580 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
@@ -21,8 +21,9 @@
 using namespace caf;
 using namespace caf::cuda;
 
-void verify_solution(const std::string& test_name, const std::vector<float>& actual, 
-                     const std::vector<float>& expected, float tol = 1e-3) {
+template <class T = float>
+void verify_solution(const std::string& test_name, const std::vector<T>& actual, 
+                     const std::vector<T>& expected, T tol = static_cast<T>(1e-3)) {
     if (actual.size() != expected.size()) {
         std::cout << "[ERROR] " << test_name << " failed: Size mismatch (got " 
                   << actual.size() << ", expected " << expected.size() << ")" << std::endl;
@@ -30,7 +31,7 @@ void verify_solution(const std::string& test_name, const std::vector<float>& act
     }
     bool all_correct = true;
     for (size_t i = 0; i < actual.size(); ++i) {
-        if (std::abs(actual[i] - expected[i]) > tol) {
+        if (std::isnan(actual[i]) || std::isinf(actual[i]) || std::abs(actual[i] - expected[i]) > tol) {
             all_correct = false;
             std::cout << "[ERROR] " << test_name << " mismatch at index " << i 
                       << ": Expected " << expected[i] 
@@ -65,7 +66,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto solver = sys.spawn<sparse_cg_actor>(
+        auto solver = sys.spawn<sparse_cg_actor<float>>(
             create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
             create_in_arg(h_b), create_in_out_arg(h_x),
             matrix_format::csr, n, nnz, tolerance, max_iter, 0, 0, actor_cast<actor>(self));
@@ -87,7 +88,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto solver = sys.spawn<sparse_cg_actor>(
+        auto solver = sys.spawn<sparse_cg_actor<float>>(
             create_in_arg(col_ptr), create_in_arg(row_ind), create_in_arg(values),
             create_in_arg(h_b), create_in_out_arg(h_x),
             matrix_format::csc, n, nnz, tolerance, max_iter, 0, 1, actor_cast<actor>(self));
@@ -120,7 +121,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> x_large(N_large, 0.0f);
         
         auto start = std::chrono::high_resolution_clock::now();
-        auto stress_solver = sys.spawn<sparse_cg_actor>(
+        auto stress_solver = sys.spawn<sparse_cg_actor<float>>(
             create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
             create_in_arg(b_large), create_in_out_arg(x_large),
             matrix_format::csr, N_large, (int)values.size(), 1e-4f, 20000, 0, 2, actor_cast<actor>(self));
@@ -147,7 +148,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto facade = sys.spawn<sparse_cg_facade>(100);
+        auto facade = sys.spawn<sparse_cg_facade<float>>(100);
 
         self->mail(std::vector<output_mapping>{},
                    create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
@@ -170,7 +171,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         output_mapping m{4, custom_x.data(), (size_t)n};
         
-        auto facade = sys.spawn<sparse_cg_facade>(100);
+        auto facade = sys.spawn<sparse_cg_facade<float>>(100);
         self->mail(std::vector<output_mapping>{m},
                    create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
                    create_in_arg(h_b), create_in_out_arg(custom_x),
@@ -192,7 +193,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto facade = sys.spawn<sparse_cg_facade>(100);
+        auto facade = sys.spawn<sparse_cg_facade<float>>(100);
 
         self->mail(return_mem_ptr_atom_v,
                    create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
@@ -215,7 +216,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto facade = sys.spawn<sparse_cg_facade>(100);
+        auto facade = sys.spawn<sparse_cg_facade<float>>(100);
 
         self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
                    create_in_arg(h_b), create_in_out_arg(h_x),
@@ -236,7 +237,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto facade = sys.spawn<sparse_cg_facade>(100);
+        auto facade = sys.spawn<sparse_cg_facade<float>>(100);
 
         self->mail(std::vector<output_mapping>{},
                    create_in_arg(col_ptr), create_in_arg(row_ind), create_in_arg(values),
@@ -258,7 +259,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto facade = sys.spawn<sparse_cg_jacobi_facade>(200);
+        auto facade = sys.spawn<sparse_cg_jacobi_facade<float>>(200);
 
         self->mail(std::vector<output_mapping>{},
                    create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
@@ -281,7 +282,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         output_mapping m{4, custom_x.data(), (size_t)n};
         
-        auto facade = sys.spawn<sparse_cg_jacobi_facade>(200);
+        auto facade = sys.spawn<sparse_cg_jacobi_facade<float>>(200);
         self->mail(std::vector<output_mapping>{m},
                    create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
                    create_in_arg(h_b), create_in_out_arg(custom_x),
@@ -303,7 +304,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto facade = sys.spawn<sparse_cg_jacobi_facade>(200);
+        auto facade = sys.spawn<sparse_cg_jacobi_facade<float>>(200);
 
         self->mail(return_mem_ptr_atom_v,
                    create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
@@ -326,7 +327,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto facade = sys.spawn<sparse_cg_jacobi_facade>(200);
+        auto facade = sys.spawn<sparse_cg_jacobi_facade<float>>(200);
 
         self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
                    create_in_arg(h_b), create_in_out_arg(h_x),
@@ -347,7 +348,7 @@ void caf_main(actor_system& sys) {
         std::vector<float> values = {4.0f, 3.0f, 2.0f};
         std::vector<float> h_x(n, 0.0f);
 
-        auto facade = sys.spawn<sparse_cg_jacobi_facade>(200);
+        auto facade = sys.spawn<sparse_cg_jacobi_facade<float>>(200);
 
         self->mail(std::vector<output_mapping>{},
                    create_in_arg(col_ptr), create_in_arg(row_ind), create_in_arg(values),
@@ -361,6 +362,115 @@ void caf_main(actor_system& sys) {
         );
     }
 
+    // Test 14: Double Precision CSR Simple
+    {
+        std::cout << "\n[INFO] Test 14: Double precision CSR format..." << std::endl;
+        int n_d = 3, nnz_d = 3;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<double> values = {4.0, 3.0, 2.0};
+        std::vector<double> h_b_d = {8.0, 9.0, 2.0};
+        std::vector<double> expected_d = {2.0, 3.0, 1.0};
+        std::vector<double> h_x(n_d, 0.0);
+
+        auto solver = sys.spawn<sparse_cg_actor<double>>(
+            create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+            create_in_arg(h_b_d), create_in_out_arg(h_x),
+            matrix_format::csr, n_d, nnz_d, 1e-10, 100, 0, 13, actor_cast<actor>(self));
+
+        self->mail(start_atom_v).send(solver);
+        self->receive(
+            [&](std::vector<double> result_x, solver_result_meta meta) {
+                std::cout << "[INFO] Iterations: " << meta.iterations << ", Converged: " << std::boolalpha << meta.converged << std::endl;
+                verify_solution<double>("Double CSR Simple", result_x, expected_d, 1e-8);
+            }
+        );
+    }
+
+    // Test 15: Double Precision Facade CSR
+    {
+        std::cout << "\n[INFO] Test 15: Double precision facade CSR..." << std::endl;
+        int n_d = 3, nnz_d = 3;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<double> values = {4.0, 3.0, 2.0};
+        std::vector<double> h_b_d = {8.0, 9.0, 2.0};
+        std::vector<double> expected_d = {2.0, 3.0, 1.0};
+        std::vector<double> h_x(n_d, 0.0);
+
+        auto facade = sys.spawn<sparse_cg_facade<double>>(100);
+
+        self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b_d), create_in_out_arg(h_x),
+                   matrix_format::csr, n_d, nnz_d, 1e-10, 100, 0, 14).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<double>& result_x, solver_result_meta meta) {
+                verify_solution<double>("Double Facade CSR", result_x, expected_d, 1e-8);
+            }
+        );
+    }
+
+    // Test 16: Double Precision Jacobi Facade CSR
+    {
+        std::cout << "\n[INFO] Test 16: Double precision Jacobi facade CSR..." << std::endl;
+        int n_d = 3, nnz_d = 3;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<double> values = {4.0, 3.0, 2.0};
+        std::vector<double> h_b_d = {8.0, 9.0, 2.0};
+        std::vector<double> expected_d = {2.0, 3.0, 1.0};
+        std::vector<double> h_x(n_d, 0.0);
+
+        auto facade = sys.spawn<sparse_cg_jacobi_facade<double>>(200);
+
+        self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b_d), create_in_out_arg(h_x),
+                   matrix_format::csr, n_d, nnz_d, 1e-10, 100, 0, 15).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<double>& result_x, solver_result_meta meta) {
+                verify_solution<double>("Double Jacobi Facade CSR", result_x, expected_d, 1e-8);
+            }
+        );
+    }
+
+    // Test 17: Double Precision Stress Test (N=5000)
+    {
+        int N_st = 5000;
+        std::cout << "\n[INFO] Test 17: Double precision Stress Test (N=" << N_st << ")..." << std::endl;
+        
+        std::vector<int> row_ptr;
+        std::vector<int> col_ind;
+        std::vector<double> values;
+        row_ptr.push_back(0);
+        for(int i=0; i<N_st; ++i) {
+            if(i > 0) { col_ind.push_back(i-1); values.push_back(-1.0); }
+            col_ind.push_back(i); values.push_back(2.0);
+            if(i < N_st-1) { col_ind.push_back(i+1); values.push_back(-1.0); }
+            row_ptr.push_back(col_ind.size());
+        }
+
+        std::vector<double> b_large(N_st, 1.0);
+        std::vector<double> x_large(N_st, 0.0);
+        
+        auto start = std::chrono::high_resolution_clock::now();
+        auto stress_solver = sys.spawn<sparse_cg_actor<double>>(
+            create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+            create_in_arg(b_large), create_in_out_arg(x_large),
+            matrix_format::csr, N_st, (int)values.size(), 1e-12, 10000, 0, 16, actor_cast<actor>(self));
+
+        self->mail(start_atom_v).send(stress_solver);
+        self->receive(
+            [&](std::vector<double> result, solver_result_meta meta) {
+                auto end = std::chrono::high_resolution_clock::now();
+                std::chrono::duration<double> elapsed = end - start;
+                std::cout << "[SUCCESS] Double Stress Test completed in " << elapsed.count() << " seconds." << std::endl;
+                std::cout << "[INFO] Iterations: " << meta.iterations << ", Converged: " << std::boolalpha << meta.converged << std::endl;
+            }
+        );
+    }
+
     manager::shutdown();
 }
 
diff --git a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp
index 1c6dab84b3..2526dea581 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp
@@ -69,8 +69,8 @@ std::vector<T> compute_rhs(const LocalCSR<T>& A, const std::vector<T>& x) {
     return b;
 }
 
-bool is_valid(const std::vector<float>& x) {
-    for (float val : x) {
+bool is_valid(const std::vector<double>& x) {
+    for (double val : x) {
         if (std::isnan(val) || std::isinf(val)) return false;
     }
     return true;
@@ -78,26 +78,38 @@ bool is_valid(const std::vector<float>& x) {
 
 // --- Robust Solver Actor (Facade Orchestrator) ---
 
+enum class solver_strategy { cgs, bicgstab, gmres };
+
 struct robust_solver_state {
-    LocalCSR<float> A;
-    std::vector<float> b;
-    float tol;
+    LocalCSR<double> A;
+    std::vector<double> b;
+    double tol;
     int max_iter;
-    const char* method = "CGS";
+    solver_strategy current_strategy = solver_strategy::cgs;
+    caf::actor requester;
 };
 
 behavior robust_solver(stateful_actor<robust_solver_state>* self, 
-                       LocalCSR<float> A, std::vector<float> b, float tol, int max_iter) {
+                       LocalCSR<double> A, std::vector<double> b, double tol, int max_iter) {
     self->state().A = std::move(A);
     self->state().b = std::move(b);
     self->state().tol = tol;
     self->state().max_iter = max_iter;
 
+    auto get_method_name = [](solver_strategy s) {
+        switch (s) {
+            case solver_strategy::cgs:      return "CGS";
+            case solver_strategy::bicgstab: return "BiCGSTAB";
+            case solver_strategy::gmres:    return "GMRES";
+            default:                        return "Unknown";
+        }
+    };
+
     auto start_cgs = [=] {
         auto& s = self->state();
-        s.method = "CGS";
-        auto facade = self->spawn<sparse_cg_facade>(100);
-        std::vector<float> x(s.A.rows, 0.0f);
+        s.current_strategy = solver_strategy::cgs;
+        auto facade = self->spawn<sparse_cg_facade<double>>(100);
+        std::vector<double> x(s.A.rows, 0.0);
         self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
                    create_in_arg(s.b), create_in_out_arg(x),
                    matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 0, 0).send(facade);
@@ -105,9 +117,9 @@ behavior robust_solver(stateful_actor<robust_solver_state>* self,
 
     auto start_bicgstab = [=] {
         auto& s = self->state();
-        s.method = "BiCGSTAB";
-        auto facade = self->spawn<sparse_bicgstab_facade<float>>(100);
-        std::vector<float> x(s.A.rows, 0.0f);
+        s.current_strategy = solver_strategy::bicgstab;
+        auto facade = self->spawn<sparse_bicgstab_facade<double>>(100);
+        std::vector<double> x(s.A.rows, 0.0);
         self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
                    create_in_arg(s.b), create_in_out_arg(x),
                    matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 0, 0).send(facade);
@@ -115,9 +127,9 @@ behavior robust_solver(stateful_actor<robust_solver_state>* self,
 
     auto start_gmres = [=] {
         auto& s = self->state();
-        s.method = "GMRES";
-        auto facade = self->spawn<sparse_gmres_facade<float>>(100);
-        std::vector<float> x(s.A.rows, 0.0f);
+        s.current_strategy = solver_strategy::gmres;
+        auto facade = self->spawn<sparse_gmres_facade<double>>(100);
+        std::vector<double> x(s.A.rows, 0.0);
         self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
                    create_in_arg(s.b), create_in_out_arg(x),
                    matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 30, 0, 0).send(facade);
@@ -125,23 +137,43 @@ behavior robust_solver(stateful_actor<robust_solver_state>* self,
 
     return {
         [=](start_atom) {
+            self->state().requester = actor_cast<caf::actor>(self->current_sender());
             std::cout << "[INFO] Attempting solve with CGS..." << std::endl;
             start_cgs();
         },
-        [=](uint32_t /*id*/, int /*idx*/, const std::vector<float>& result, solver_result_meta meta) {
+        [=](uint32_t /*id*/, int /*idx*/, const std::vector<double>& result, solver_result_meta meta) {
             auto& s = self->state();
+            const char* method_name = get_method_name(s.current_strategy);
             if (meta.converged && is_valid(result)) {
-                std::cout << "[SUCCESS] " << s.method << " converged. Matrix good." << std::endl;
+                std::cout << "[SUCCESS] " << method_name << " converged. Matrix good." << std::endl;
+                if (s.requester)
+                    self->mail(true).send(s.requester);
                 self->quit();
             } else {
-                std::cout << "[WARNING] " << s.method << " failed or produced NaN. Retrying..." << std::endl;
-                if (std::string(s.method) == "CGS") {
-                    start_bicgstab();
-                } else if (std::string(s.method) == "BiCGSTAB") {
-                    start_gmres();
-                } else {
-                    std::cout << "[ERROR] All facade solvers failed." << std::endl;
-                    self->quit();
+                std::cout << "[WARNING] " << method_name;
+                if (!meta.converged) {
+                    std::cout << " failed to converge.";
+                }
+                if (!is_valid(result)) {
+                    if (!meta.converged) {
+                        std::cout << " and";
+                    }
+                    std::cout << " produced NaN/Inf values.";
+                }
+                std::cout << " Retrying..." << std::endl;
+                switch (s.current_strategy) {
+                    case solver_strategy::cgs:
+                        start_bicgstab();
+                        break;
+                    case solver_strategy::bicgstab:
+                        start_gmres();
+                        break;
+                    default:
+                        std::cout << "[ERROR] All facade solvers failed." << std::endl;
+                        if (s.requester)
+                            self->mail(std::string("All facade solvers failed")).send(s.requester);
+                        self->quit();
+                        break;
                 }
             }
         }
@@ -152,19 +184,20 @@ void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true)); 
     scoped_actor self{sys};
 
-    std::string path = "/scratch/nqr159/matrix-collection/matrices/unsymmetric/lnsp3937.bin";
+    std::string path = "/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric/sherman5.bin";
     std::cout << "[INFO] Loading real-world matrix: " << path << std::endl;
 
     try {
-        LocalCSR<float> A = load_binary_matrix<float>(path);
-        std::vector<float> x_target(A.rows, 1.0f);
-        std::vector<float> b = compute_rhs(A, x_target);
+        LocalCSR<double> A = load_binary_matrix<double>(path);
+        std::vector<double> x_target(A.rows, 1.0);
+        std::vector<double> b = compute_rhs(A, x_target);
 
-        auto robust = sys.spawn(robust_solver, std::move(A), std::move(b), 1e-5f, 5000);
+        auto robust = sys.spawn(robust_solver, std::move(A), std::move(b), 1e-10, 5000);
         self->mail(start_atom_v).send(robust);
         
         self->receive(
-            [] { std::cout << "[INFO] Robust solver test complete." << std::endl; }
+            [](bool) { std::cout << "[INFO] Robust solver converged successfully." << std::endl; },
+            [](std::string err) { std::cout << "[INFO] Robust solver aborted: " << err << std::endl; }
         );
     } catch (const std::exception& e) {
         std::cerr << "[ERROR] " << e.what() << std::endl;

From 59828727610c984eae97a7a2949a3384bcdc02f7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 29 May 2026 12:41:14 -0600
Subject: [PATCH 0781/1000] added x vector to robust solver state. Change was
 made to fix issue where x would drop out of scope after feeding it to the
 actor facade, causing it to run on bad data

---
 .../actorSOLVE/main.test.cpp                  | 25 +++++++++++++------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp
index 2526dea581..14e2b10174 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp
@@ -54,6 +54,18 @@ LocalCSR<T> load_binary_matrix(const std::string& path) {
         csr.col_ind[dest] = cols_coo[i];
         csr.values[dest] = static_cast<T>(vals_coo[i]);
     }
+
+    std::cout << "rows=" << csr.rows
+              << " cols=" << csr.cols
+              << " nnz=" << csr.nnz
+              << " row_ptr.back()=" << csr.row_ptr.back()
+              << std::endl;
+
+    for (int i = 0; i < csr.nnz; ++i) {
+        if (csr.col_ind[i] < 0 || csr.col_ind[i] >= csr.cols)
+            throw std::runtime_error("bad col index");
+    }
+
     return csr;
 }
 
@@ -83,6 +95,7 @@ enum class solver_strategy { cgs, bicgstab, gmres };
 struct robust_solver_state {
     LocalCSR<double> A;
     std::vector<double> b;
+    std::vector<double> x;
     double tol;
     int max_iter;
     solver_strategy current_strategy = solver_strategy::cgs;
@@ -94,6 +107,7 @@ behavior robust_solver(stateful_actor<robust_solver_state>* self,
     self->state().A = std::move(A);
     self->state().b = std::move(b);
     self->state().tol = tol;
+    self->state().x.assign(self->state().A.rows, 0.0);
     self->state().max_iter = max_iter;
 
     auto get_method_name = [](solver_strategy s) {
@@ -109,9 +123,8 @@ behavior robust_solver(stateful_actor<robust_solver_state>* self,
         auto& s = self->state();
         s.current_strategy = solver_strategy::cgs;
         auto facade = self->spawn<sparse_cg_facade<double>>(100);
-        std::vector<double> x(s.A.rows, 0.0);
         self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
-                   create_in_arg(s.b), create_in_out_arg(x),
+                   create_in_arg(s.b), create_in_out_arg(s.x),
                    matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 0, 0).send(facade);
     };
 
@@ -119,9 +132,8 @@ behavior robust_solver(stateful_actor<robust_solver_state>* self,
         auto& s = self->state();
         s.current_strategy = solver_strategy::bicgstab;
         auto facade = self->spawn<sparse_bicgstab_facade<double>>(100);
-        std::vector<double> x(s.A.rows, 0.0);
         self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
-                   create_in_arg(s.b), create_in_out_arg(x),
+                   create_in_arg(s.b), create_in_out_arg(s.x),
                    matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 0, 0).send(facade);
     };
 
@@ -129,9 +141,8 @@ behavior robust_solver(stateful_actor<robust_solver_state>* self,
         auto& s = self->state();
         s.current_strategy = solver_strategy::gmres;
         auto facade = self->spawn<sparse_gmres_facade<double>>(100);
-        std::vector<double> x(s.A.rows, 0.0);
         self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
-                   create_in_arg(s.b), create_in_out_arg(x),
+                   create_in_arg(s.b), create_in_out_arg(s.x),
                    matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 30, 0, 0).send(facade);
     };
 
@@ -184,7 +195,7 @@ void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true)); 
     scoped_actor self{sys};
 
-    std::string path = "/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric/sherman5.bin";
+    std::string path = "/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric/jpwh_991.bin";
     std::cout << "[INFO] Loading real-world matrix: " << path << std::endl;
 
     try {

From aa7127bf45a9b1e4c168fdbf601101fbe1c88ba1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 29 May 2026 12:44:06 -0600
Subject: [PATCH 0782/1000] put actor facades in array to increase efficieny

---
 .../fault-tolerance-tests/actorSOLVE/main.test.cpp | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp
index 14e2b10174..e80ca34cef 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/main.test.cpp
@@ -98,6 +98,7 @@ struct robust_solver_state {
     std::vector<double> x;
     double tol;
     int max_iter;
+    std::vector<caf::actor> facades;
     solver_strategy current_strategy = solver_strategy::cgs;
     caf::actor requester;
 };
@@ -110,6 +111,10 @@ behavior robust_solver(stateful_actor<robust_solver_state>* self,
     self->state().x.assign(self->state().A.rows, 0.0);
     self->state().max_iter = max_iter;
 
+    self->state().facades.push_back(self->spawn<sparse_cg_facade<double>>(100));
+    self->state().facades.push_back(self->spawn<sparse_bicgstab_facade<double>>(100));
+    self->state().facades.push_back(self->spawn<sparse_gmres_facade<double>>(100));
+
     auto get_method_name = [](solver_strategy s) {
         switch (s) {
             case solver_strategy::cgs:      return "CGS";
@@ -122,28 +127,25 @@ behavior robust_solver(stateful_actor<robust_solver_state>* self,
     auto start_cgs = [=] {
         auto& s = self->state();
         s.current_strategy = solver_strategy::cgs;
-        auto facade = self->spawn<sparse_cg_facade<double>>(100);
         self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
                    create_in_arg(s.b), create_in_out_arg(s.x),
-                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 0, 0).send(facade);
+                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 0, 0).send(s.facades[0]);
     };
 
     auto start_bicgstab = [=] {
         auto& s = self->state();
         s.current_strategy = solver_strategy::bicgstab;
-        auto facade = self->spawn<sparse_bicgstab_facade<double>>(100);
         self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
                    create_in_arg(s.b), create_in_out_arg(s.x),
-                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 0, 0).send(facade);
+                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 0, 0).send(s.facades[1]);
     };
 
     auto start_gmres = [=] {
         auto& s = self->state();
         s.current_strategy = solver_strategy::gmres;
-        auto facade = self->spawn<sparse_gmres_facade<double>>(100);
         self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
                    create_in_arg(s.b), create_in_out_arg(s.x),
-                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 30, 0, 0).send(facade);
+                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 30, 0, 0).send(s.facades[2]);
     };
 
     return {

From ff72047ff667db9b55d91de3e20960c77f7982b3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 29 May 2026 13:43:02 -0600
Subject: [PATCH 0783/1000] Initial commit.

---
 .../actorSOLVE/CMakeLists.txt                 |  12 ++
 .../actorSOLVE/suitesparse.test.cpp           | 186 ++++++++++++++++++
 2 files changed, 198 insertions(+)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/suitesparse.test.cpp

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/CMakeLists.txt b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/CMakeLists.txt
index 0422ab8d27..9c2ab41a96 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/CMakeLists.txt
+++ b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/CMakeLists.txt
@@ -47,3 +47,15 @@ target_link_libraries(test
     CUDA::cublas
     CUDA::cusparse
 )
+
+add_executable(suitesparse_test suitesparse.test.cpp)
+target_compile_definitions(suitesparse_test PRIVATE CAF_ENABLE_LOGGING)
+target_link_libraries(suitesparse_test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
+)
diff --git a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/suitesparse.test.cpp b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/suitesparse.test.cpp
new file mode 100644
index 0000000000..8ffbef984b
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/suitesparse.test.cpp
@@ -0,0 +1,186 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include <fstream>
+#include <filesystem>
+#include <type_traits>
+#include <cmath>
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp"
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+namespace fs = std::filesystem;
+
+// --- Matrix Utilities ---
+
+template <class T = float>
+struct LocalCSR {
+    int rows, cols, nnz;
+    std::vector<int> row_ptr;
+    std::vector<int> col_ind;
+    std::vector<T> values;
+};
+
+template <class T = float>
+LocalCSR<T> load_binary_matrix(const std::string& path) {
+    std::ifstream file(path, std::ios::binary);
+    if (!file) throw std::runtime_error("Could not open " + path);
+    int32_t r, c, n;
+    file.read(reinterpret_cast<char*>(&r), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&c), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&n), sizeof(int32_t));
+    std::vector<int32_t> rows_coo(n), cols_coo(n);
+    std::vector<float> vals_coo(n);
+    file.read(reinterpret_cast<char*>(rows_coo.data()), n * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(cols_coo.data()), n * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(vals_coo.data()), n * sizeof(float));
+    int max_idx = 0;
+    for(auto v : rows_coo) if(v > max_idx) max_idx = v;
+    if (max_idx == r || max_idx == c) {
+        for(auto& v : rows_coo) v--;
+        for(auto& v : cols_coo) v--;
+    }
+    LocalCSR<T> csr;
+    csr.rows = r; csr.cols = c; csr.nnz = n;
+    csr.row_ptr.assign(r + 1, 0);
+    csr.col_ind.resize(n);
+    csr.values.resize(n);
+    for (int i = 0; i < n; ++i) csr.row_ptr[rows_coo[i] + 1]++;
+    for (int i = 0; i < r; ++i) csr.row_ptr[i + 1] += csr.row_ptr[i];
+    std::vector<int> current_pos = csr.row_ptr;
+    for (int i = 0; i < n; ++i) {
+        int row = rows_coo[i];
+        int dest = current_pos[row]++;
+        csr.col_ind[dest] = cols_coo[i];
+        csr.values[dest] = static_cast<T>(vals_coo[i]);
+    }
+    return csr;
+}
+
+template <class T = float>
+std::vector<T> compute_rhs(const LocalCSR<T>& A, const std::vector<T>& x) {
+    std::vector<T> b(A.rows, T{0});
+    for (int i = 0; i < A.rows; ++i) {
+        T sum = T{0};
+        for (int j = A.row_ptr[i]; j < A.row_ptr[i+1]; ++j)
+            sum += A.values[j] * x[A.col_ind[j]];
+        b[i] = sum;
+    }
+    return b;
+}
+
+template <class T>
+bool is_valid(const std::vector<T>& x) {
+    for (auto val : x) {
+        if (std::isnan(val) || std::isinf(val)) return false;
+    }
+    return true;
+}
+
+struct solver_orchestrator_state {
+    std::vector<caf::actor> float_solvers;
+    std::vector<caf::actor> double_solvers;
+};
+
+behavior solver_orchestrator(stateful_actor<solver_orchestrator_state>* self) {
+    // Create solver facades once and reuse them for all matrix solve requests
+    self->state().float_solvers = {
+        self->spawn<sparse_cg_facade<float>>(100),
+        self->spawn<sparse_cg_jacobi_facade<float>>(100),
+        self->spawn<sparse_bicgstab_facade<float>>(100),
+        self->spawn<sparse_bicgstab_jacobi_facade<float>>(100),
+        self->spawn<sparse_gmres_facade<float>>(100)
+    };
+    self->state().double_solvers = {
+        self->spawn<sparse_cg_facade<double>>(100),
+        self->spawn<sparse_cg_jacobi_facade<double>>(100),
+        self->spawn<sparse_bicgstab_facade<double>>(100),
+        self->spawn<sparse_bicgstab_jacobi_facade<double>>(100),
+        self->spawn<sparse_gmres_facade<double>>(100)
+    };
+
+    auto run_solvers = [self](auto& A, auto tol, int max_iter) -> bool {
+        using T = typename std::decay_t<decltype(A.values)>::value_type;
+        std::vector<T> x_target(A.rows, T{1});
+        std::vector<T> b = compute_rhs(A, x_target);
+        std::vector<T> x_init(A.rows, T{0});
+
+        const auto& solvers = std::is_same_v<T, float> ? self->state().float_solvers : self->state().double_solvers;
+
+        for (size_t i = 0; i < solvers.size(); ++i) {
+            bool success = false;
+            auto& solver = solvers[i];
+            
+            // GMRES (index 4) needs the restart parameter (e.g., 30)
+            if (i == 4) {
+                self->mail(create_in_arg(A.row_ptr), create_in_arg(A.col_ind), create_in_arg(A.values),
+                           create_in_arg(b), create_in_out_arg(x_init),
+                           matrix_format::csr, A.rows, A.nnz, tol, max_iter, 30, 0, 0).send(solver);
+            } else {
+                self->mail(create_in_arg(A.row_ptr), create_in_arg(A.col_ind), create_in_arg(A.values),
+                           create_in_arg(b), create_in_out_arg(x_init),
+                           matrix_format::csr, A.rows, A.nnz, tol, max_iter, 0, 0).send(solver);
+            }
+
+            self->receive(
+                [&](uint32_t, int, const std::vector<T>& result, solver_result_meta meta) {
+                    if (meta.converged && is_valid(result)) success = true;
+                }
+            );
+            if (success) return true;
+        }
+        return false;
+    };
+
+    return {
+        [=](const std::string& path) -> bool {
+            try {
+                // Attempt float variants
+                LocalCSR<float> Af = load_binary_matrix<float>(path);
+                if (run_solvers(Af, 1e-4f, 1000)) return true;
+
+                // Attempt double variants
+                LocalCSR<double> Ad = load_binary_matrix<double>(path);
+                if (run_solvers(Ad, 1e-8, 1000)) return true;
+            } catch (...) {
+                return false;
+            }
+            return false;
+        }
+    };
+}
+
+void caf_main(actor_system& sys) {
+    manager::init(sys, manager_config(true, true));
+    scoped_actor self{sys};
+    auto orchestrator = sys.spawn(solver_orchestrator);
+
+    std::string root = "/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices";
+    if (!fs::exists(root)) return;
+
+    for (const auto& entry : fs::recursive_directory_iterator(root)) {
+        if (entry.is_regular_file() && entry.path().extension() == ".bin") {
+            std::string path = entry.path().string();
+            
+            bool ok = false;
+            self->mail(path).request(orchestrator, infinite).receive(
+                [&](bool result) {
+                    ok = result;
+                },
+                [&](error& err) {
+                    std::cerr << "Error encountered for " << path << ": " << sys.render(err) << std::endl;
+                }
+            );
+
+            if (ok) {
+                std::cout << path << std::endl;
+            }
+        }
+    }
+    manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda)
\ No newline at end of file

From e77398e83eb6680180fa3e86dc925598f7107eb9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 29 May 2026 14:25:17 -0600
Subject: [PATCH 0784/1000] updated test to actually work

---
 .../actorSOLVE/suitesparse.test.cpp           | 379 +++++++++++-------
 1 file changed, 228 insertions(+), 151 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/suitesparse.test.cpp b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/suitesparse.test.cpp
index 8ffbef984b..38f715e3d0 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/suitesparse.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/suitesparse.test.cpp
@@ -1,11 +1,15 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <filesystem>
+#include <fstream>
 #include <iostream>
+#include <string>
 #include <vector>
-#include <fstream>
-#include <filesystem>
-#include <type_traits>
-#include <cmath>
+
 #include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
 #include "caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp"
 #include "caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp"
@@ -14,173 +18,246 @@ using namespace caf;
 using namespace caf::cuda;
 namespace fs = std::filesystem;
 
-// --- Matrix Utilities ---
+// ------------------------------------------------------------
+// Matrix utilities
+// ------------------------------------------------------------
 
 template <class T = float>
 struct LocalCSR {
-    int rows, cols, nnz;
-    std::vector<int> row_ptr;
-    std::vector<int> col_ind;
-    std::vector<T> values;
+  int rows = 0;
+  int cols = 0;
+  int nnz = 0;
+  std::vector<int> row_ptr;
+  std::vector<int> col_ind;
+  std::vector<T> values;
 };
 
 template <class T = float>
 LocalCSR<T> load_binary_matrix(const std::string& path) {
-    std::ifstream file(path, std::ios::binary);
-    if (!file) throw std::runtime_error("Could not open " + path);
-    int32_t r, c, n;
-    file.read(reinterpret_cast<char*>(&r), sizeof(int32_t));
-    file.read(reinterpret_cast<char*>(&c), sizeof(int32_t));
-    file.read(reinterpret_cast<char*>(&n), sizeof(int32_t));
-    std::vector<int32_t> rows_coo(n), cols_coo(n);
-    std::vector<float> vals_coo(n);
-    file.read(reinterpret_cast<char*>(rows_coo.data()), n * sizeof(int32_t));
-    file.read(reinterpret_cast<char*>(cols_coo.data()), n * sizeof(int32_t));
-    file.read(reinterpret_cast<char*>(vals_coo.data()), n * sizeof(float));
-    int max_idx = 0;
-    for(auto v : rows_coo) if(v > max_idx) max_idx = v;
-    if (max_idx == r || max_idx == c) {
-        for(auto& v : rows_coo) v--;
-        for(auto& v : cols_coo) v--;
-    }
-    LocalCSR<T> csr;
-    csr.rows = r; csr.cols = c; csr.nnz = n;
-    csr.row_ptr.assign(r + 1, 0);
-    csr.col_ind.resize(n);
-    csr.values.resize(n);
-    for (int i = 0; i < n; ++i) csr.row_ptr[rows_coo[i] + 1]++;
-    for (int i = 0; i < r; ++i) csr.row_ptr[i + 1] += csr.row_ptr[i];
-    std::vector<int> current_pos = csr.row_ptr;
-    for (int i = 0; i < n; ++i) {
-        int row = rows_coo[i];
-        int dest = current_pos[row]++;
-        csr.col_ind[dest] = cols_coo[i];
-        csr.values[dest] = static_cast<T>(vals_coo[i]);
-    }
-    return csr;
+  std::ifstream file(path, std::ios::binary);
+  if (!file)
+    throw std::runtime_error("Could not open " + path);
+
+  int32_t r = 0, c = 0, n = 0;
+  file.read(reinterpret_cast<char*>(&r), sizeof(int32_t));
+  file.read(reinterpret_cast<char*>(&c), sizeof(int32_t));
+  file.read(reinterpret_cast<char*>(&n), sizeof(int32_t));
+
+  if (!file || r <= 0 || c <= 0 || n < 0)
+    throw std::runtime_error("Bad header in " + path);
+
+  std::vector<int32_t> rows_coo(n), cols_coo(n);
+  std::vector<float> vals_coo(n);
+  file.read(reinterpret_cast<char*>(rows_coo.data()), n * sizeof(int32_t));
+  file.read(reinterpret_cast<char*>(cols_coo.data()), n * sizeof(int32_t));
+  file.read(reinterpret_cast<char*>(vals_coo.data()), n * sizeof(float));
+
+  if (!file)
+    throw std::runtime_error("Truncated matrix file: " + path);
+
+  // Detect 1-based COO and normalize to 0-based.
+  int max_idx = 0;
+  for (auto v : rows_coo)
+    if (v > max_idx)
+      max_idx = v;
+  for (auto v : cols_coo)
+    if (v > max_idx)
+      max_idx = v;
+
+  if (max_idx == r || max_idx == c) {
+    for (auto& v : rows_coo)
+      --v;
+    for (auto& v : cols_coo)
+      --v;
+  }
+
+  LocalCSR<T> csr;
+  csr.rows = static_cast<int>(r);
+  csr.cols = static_cast<int>(c);
+  csr.nnz = static_cast<int>(n);
+  csr.row_ptr.assign(csr.rows + 1, 0);
+  csr.col_ind.resize(csr.nnz);
+  csr.values.resize(csr.nnz);
+
+  for (int i = 0; i < csr.nnz; ++i) {
+    if (rows_coo[i] < 0 || rows_coo[i] >= csr.rows)
+      throw std::runtime_error("bad row index in " + path);
+    if (cols_coo[i] < 0 || cols_coo[i] >= csr.cols)
+      throw std::runtime_error("bad col index in " + path);
+    csr.row_ptr[rows_coo[i] + 1]++;
+  }
+
+  for (int i = 0; i < csr.rows; ++i)
+    csr.row_ptr[i + 1] += csr.row_ptr[i];
+
+  std::vector<int> current_pos = csr.row_ptr;
+  for (int i = 0; i < csr.nnz; ++i) {
+    int row = rows_coo[i];
+    int dest = current_pos[row]++;
+    csr.col_ind[dest] = cols_coo[i];
+    csr.values[dest] = static_cast<T>(vals_coo[i]);
+  }
+
+  return csr;
 }
 
-template <class T = float>
+template <class T>
 std::vector<T> compute_rhs(const LocalCSR<T>& A, const std::vector<T>& x) {
-    std::vector<T> b(A.rows, T{0});
-    for (int i = 0; i < A.rows; ++i) {
-        T sum = T{0};
-        for (int j = A.row_ptr[i]; j < A.row_ptr[i+1]; ++j)
-            sum += A.values[j] * x[A.col_ind[j]];
-        b[i] = sum;
-    }
-    return b;
+  std::vector<T> b(A.rows, T{0});
+  for (int i = 0; i < A.rows; ++i) {
+    T sum = T{0};
+    for (int j = A.row_ptr[i]; j < A.row_ptr[i + 1]; ++j)
+      sum += A.values[j] * x[A.col_ind[j]];
+    b[i] = sum;
+  }
+  return b;
 }
 
 template <class T>
-bool is_valid(const std::vector<T>& x) {
-    for (auto val : x) {
-        if (std::isnan(val) || std::isinf(val)) return false;
+bool is_finite_vector(const std::vector<T>& x) {
+  for (auto v : x)
+    if (!std::isfinite(static_cast<double>(v)))
+      return false;
+  return true;
+}
+
+std::vector<fs::path> collect_matrix_files(const fs::path& root) {
+  std::vector<fs::path> files;
+  if (!fs::exists(root))
+    return files;
+
+  for (const auto& entry : fs::recursive_directory_iterator(root)) {
+    if (!entry.is_regular_file())
+      continue;
+    const auto ext = entry.path().extension().string();
+    if (ext == ".bin")
+      files.push_back(entry.path());
+  }
+
+  std::sort(files.begin(), files.end());
+  return files;
+}
+
+// ------------------------------------------------------------
+// Solver sweep helpers
+// ------------------------------------------------------------
+
+static constexpr uint32_t kReplyId = 0xA11CE001u;
+
+// NOTE:
+// This driver expects a BiCGSTAB Jacobi facade named
+// caf::cuda::sparse_bicgstab_jacobi_facade<T> to be available from
+// sparse-BiCGSTAB-actor.hpp.
+
+template <class Facade, class T>
+bool run_one_variant(actor_system& sys,
+                     const LocalCSR<T>& A,
+                     const std::vector<T>& b,
+                     double tol,
+                     int max_iter) {
+  scoped_actor self{sys};
+  auto solver = sys.spawn<Facade>(kReplyId);
+
+  std::vector<T> x0(A.rows, T{0});
+  self->mail(return_mem_ptr_atom_v,
+             create_in_arg(A.row_ptr),
+             create_in_arg(A.col_ind),
+             create_in_arg(A.values),
+             create_in_arg(b),
+             create_in_out_arg(x0),
+             matrix_format::csr,
+             A.rows,
+             A.nnz,
+             static_cast<T>(tol),
+             max_iter,
+             0,
+             0)
+      .send(solver);
+
+  bool solved = false;
+  bool received = false;
+
+  self->receive(
+    [&](uint32_t rid, mem_ptr<T> result, solver_result_meta meta) {
+      (void)rid;
+      (void)result;
+      received = true;
+      solved = meta.converged;
+    },
+    after(std::chrono::minutes(20)) >> [&] {
+      received = false;
+      solved = false;
     }
+  );
+
+  return received && solved;
+}
+
+template <class T>
+bool run_all_variants(actor_system& sys,
+                      const LocalCSR<T>& A,
+                      const std::vector<T>& b,
+                      double tol,
+                      int max_iter) {
+  // Try Jacobi variants first, then the base solvers, then GMRES.
+  // A file is considered successful if at least one variant converges.
+  if (run_one_variant<sparse_cg_jacobi_facade<T>>(sys, A, b, tol, max_iter))
+    return true;
+
+  if (run_one_variant<sparse_cg_facade<T>>(sys, A, b, tol, max_iter))
     return true;
+
+  if (run_one_variant<sparse_bicgstab_jacobi_facade<T>>(sys, A, b, tol, max_iter))
+    return true;
+
+  if (run_one_variant<sparse_bicgstab_facade<T>>(sys, A, b, tol, max_iter))
+    return true;
+
+  if (run_one_variant<sparse_gmres_facade<T>>(sys, A, b, tol, max_iter))
+    return true;
+
+  return false;
 }
 
-struct solver_orchestrator_state {
-    std::vector<caf::actor> float_solvers;
-    std::vector<caf::actor> double_solvers;
-};
+template <class T>
+void sweep_one_type(actor_system& sys, const fs::path& matrix_file) {
+  try {
+    auto A = load_binary_matrix<T>(matrix_file.string());
+    if (A.rows <= 0 || A.cols <= 0 || A.nnz <= 0)
+      return;
+    if (A.rows != A.cols)
+      return; // iterative solvers here target square systems
+
+    std::vector<T> x_target(A.rows, T{1});
+    std::vector<T> b = compute_rhs(A, x_target);
 
-behavior solver_orchestrator(stateful_actor<solver_orchestrator_state>* self) {
-    // Create solver facades once and reuse them for all matrix solve requests
-    self->state().float_solvers = {
-        self->spawn<sparse_cg_facade<float>>(100),
-        self->spawn<sparse_cg_jacobi_facade<float>>(100),
-        self->spawn<sparse_bicgstab_facade<float>>(100),
-        self->spawn<sparse_bicgstab_jacobi_facade<float>>(100),
-        self->spawn<sparse_gmres_facade<float>>(100)
-    };
-    self->state().double_solvers = {
-        self->spawn<sparse_cg_facade<double>>(100),
-        self->spawn<sparse_cg_jacobi_facade<double>>(100),
-        self->spawn<sparse_bicgstab_facade<double>>(100),
-        self->spawn<sparse_bicgstab_jacobi_facade<double>>(100),
-        self->spawn<sparse_gmres_facade<double>>(100)
-    };
-
-    auto run_solvers = [self](auto& A, auto tol, int max_iter) -> bool {
-        using T = typename std::decay_t<decltype(A.values)>::value_type;
-        std::vector<T> x_target(A.rows, T{1});
-        std::vector<T> b = compute_rhs(A, x_target);
-        std::vector<T> x_init(A.rows, T{0});
-
-        const auto& solvers = std::is_same_v<T, float> ? self->state().float_solvers : self->state().double_solvers;
-
-        for (size_t i = 0; i < solvers.size(); ++i) {
-            bool success = false;
-            auto& solver = solvers[i];
-            
-            // GMRES (index 4) needs the restart parameter (e.g., 30)
-            if (i == 4) {
-                self->mail(create_in_arg(A.row_ptr), create_in_arg(A.col_ind), create_in_arg(A.values),
-                           create_in_arg(b), create_in_out_arg(x_init),
-                           matrix_format::csr, A.rows, A.nnz, tol, max_iter, 30, 0, 0).send(solver);
-            } else {
-                self->mail(create_in_arg(A.row_ptr), create_in_arg(A.col_ind), create_in_arg(A.values),
-                           create_in_arg(b), create_in_out_arg(x_init),
-                           matrix_format::csr, A.rows, A.nnz, tol, max_iter, 0, 0).send(solver);
-            }
-
-            self->receive(
-                [&](uint32_t, int, const std::vector<T>& result, solver_result_meta meta) {
-                    if (meta.converged && is_valid(result)) success = true;
-                }
-            );
-            if (success) return true;
-        }
-        return false;
-    };
-
-    return {
-        [=](const std::string& path) -> bool {
-            try {
-                // Attempt float variants
-                LocalCSR<float> Af = load_binary_matrix<float>(path);
-                if (run_solvers(Af, 1e-4f, 1000)) return true;
-
-                // Attempt double variants
-                LocalCSR<double> Ad = load_binary_matrix<double>(path);
-                if (run_solvers(Ad, 1e-8, 1000)) return true;
-            } catch (...) {
-                return false;
-            }
-            return false;
-        }
-    };
+    constexpr double tol = 1e-10;
+    constexpr int max_iter = 5000;
+
+    if (run_all_variants<T>(sys, A, b, tol, max_iter)) {
+      std::cout << matrix_file.string() << std::endl;
+    }
+  } catch (...) {
+    // Silent by design: only successful file paths are printed.
+  }
 }
 
+// ------------------------------------------------------------
+// CAF entry point
+// ------------------------------------------------------------
+
 void caf_main(actor_system& sys) {
-    manager::init(sys, manager_config(true, true));
-    scoped_actor self{sys};
-    auto orchestrator = sys.spawn(solver_orchestrator);
-
-    std::string root = "/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices";
-    if (!fs::exists(root)) return;
-
-    for (const auto& entry : fs::recursive_directory_iterator(root)) {
-        if (entry.is_regular_file() && entry.path().extension() == ".bin") {
-            std::string path = entry.path().string();
-            
-            bool ok = false;
-            self->mail(path).request(orchestrator, infinite).receive(
-                [&](bool result) {
-                    ok = result;
-                },
-                [&](error& err) {
-                    std::cerr << "Error encountered for " << path << ": " << sys.render(err) << std::endl;
-                }
-            );
-
-            if (ok) {
-                std::cout << path << std::endl;
-            }
-        }
-    }
-    manager::shutdown();
+  manager::init(sys, manager_config(true, true));
+
+  const fs::path root = "/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices";
+  const auto files = collect_matrix_files(root);
+
+  for (const auto& file : files) {
+    sweep_one_type<float>(sys, file);
+    sweep_one_type<double>(sys, file);
+  }
+
+  manager::shutdown();
 }
 
-CAF_MAIN(id_block::cuda)
\ No newline at end of file
+CAF_MAIN(id_block::cuda)

From 72031b30fc96ab1a25632f2689a285a97704d55b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 29 May 2026 15:26:53 -0600
Subject: [PATCH 0785/1000] updated script to  handle the gmres actor
 correctly. Change was made to fix error resulting in invalid actor creation
 sileintly failing causing a deadlock since reply actor would wait for a
 response it would never get

---
 .../actorSOLVE/suitesparse.test.cpp           | 53 ++++++++++++++-----
 1 file changed, 39 insertions(+), 14 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/suitesparse.test.cpp b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/suitesparse.test.cpp
index 38f715e3d0..7d54d3eeb5 100644
--- a/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/suitesparse.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-tests/actorSOLVE/suitesparse.test.cpp
@@ -159,20 +159,40 @@ bool run_one_variant(actor_system& sys,
   auto solver = sys.spawn<Facade>(kReplyId);
 
   std::vector<T> x0(A.rows, T{0});
-  self->mail(return_mem_ptr_atom_v,
-             create_in_arg(A.row_ptr),
-             create_in_arg(A.col_ind),
-             create_in_arg(A.values),
-             create_in_arg(b),
-             create_in_out_arg(x0),
-             matrix_format::csr,
-             A.rows,
-             A.nnz,
-             static_cast<T>(tol),
-             max_iter,
-             0,
-             0)
-      .send(solver);
+
+  // GMRES requires a restart parameter (k) which the other solvers do not.
+  if constexpr (std::is_same_v<Facade, sparse_gmres_facade<T>>) {
+    self->mail(return_mem_ptr_atom_v,
+               create_in_arg(A.row_ptr),
+               create_in_arg(A.col_ind),
+               create_in_arg(A.values),
+               create_in_arg(b),
+               create_in_out_arg(x0),
+               matrix_format::csr,
+               A.rows,
+               A.nnz,
+               static_cast<T>(tol),
+               max_iter,
+               30, // Restart parameter k
+               0,  // device_num
+               0)  // stream_id
+        .send(solver);
+  } else {
+    self->mail(return_mem_ptr_atom_v,
+               create_in_arg(A.row_ptr),
+               create_in_arg(A.col_ind),
+               create_in_arg(A.values),
+               create_in_arg(b),
+               create_in_out_arg(x0),
+               matrix_format::csr,
+               A.rows,
+               A.nnz,
+               static_cast<T>(tol),
+               max_iter,
+               0, // device_num
+               0) // stream_id
+        .send(solver);
+  }
 
   bool solved = false;
   bool received = false;
@@ -184,12 +204,17 @@ bool run_one_variant(actor_system& sys,
       received = true;
       solved = meta.converged;
     },
+    [&](const error& err) {
+      received = true;
+      solved = false;
+    },
     after(std::chrono::minutes(20)) >> [&] {
       received = false;
       solved = false;
     }
   );
 
+  self->send_exit(solver, exit_reason::user_shutdown);
   return received && solved;
 }
 

From 97de8b7ba12e28357bb448a627d23b0b4e00ac92 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 08:19:30 -0600
Subject: [PATCH 0786/1000] updated cmakelists

---
 libcaf_cuda/tests/actor-facade-test/CMakeLists.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt b/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
index 2e6c1faf1f..3411d90b15 100644
--- a/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/actor-facade-test/CMakeLists.txt
@@ -47,6 +47,7 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
     CUDA::cublas
+    CUDA::cusparse
 )
 target_link_libraries(throughput_mapping_bench_test
   PRIVATE
@@ -55,6 +56,7 @@ target_link_libraries(throughput_mapping_bench_test
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
     CUDA::cublas
+    CUDA::cusparse
 )
 
 
@@ -65,6 +67,7 @@ target_link_libraries(latency_bench_test
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::cublas
     CUDA::nvrtc
+    CUDA::cusparse
 )
 target_link_libraries(throughput_bench_test
   PRIVATE
@@ -73,4 +76,5 @@ target_link_libraries(throughput_bench_test
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
     CUDA::cublas
+    CUDA::cusparse
 )

From aefee89a169f5978821e8b36a11bb9ece78c4661 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 08:50:13 -0600
Subject: [PATCH 0787/1000] updated tests to work on a batch system where we
 add a sleep timer and execute batches until done

---
 .../cuda-baseline.cpp                         |  69 +++++----
 .../work-stealing.cpp                         | 135 ++++++++++++++----
 2 files changed, 142 insertions(+), 62 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index 0d1c6fe1af..902b4b9d5c 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -149,7 +149,6 @@ int main() {
     }
 
     const int streams_per_gpu = 8;
-    std::vector<int> task_counts = {50000,100000};
 
     int num_gpus;
     cuDeviceGetCount(&num_gpus);
@@ -178,13 +177,8 @@ int main() {
         return 1;
     }
 
-    for (int total_tasks : task_counts) {
     std::cout << "=====================================" << std::endl;
-    std::cout << "Task count: " << total_tasks << " (Irregular Workload)" << std::endl;
-
-    std::vector<Task> all_tasks;
-    std::mt19937 rng_tasks(42); // Fixed seed for task distribution
-    std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
+    std::cout << "Dynamic Workload Generation (CUDA Baseline)" << std::endl;
 
     std::vector<CUcontext> contexts(num_gpus);
     std::vector<CUfunction> mmul_funcs(num_gpus);
@@ -298,40 +292,54 @@ int main() {
         conv_funcs[i] = conv_funcs[0];
     }
 
+    std::vector<int> shared_dtoh_buffer((size_t)max_N_val * max_N_val);
+    std::mt19937 rng_prod(42); 
+    std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
     std::uniform_int_distribution<int> dist_type(0, 2);
-    for (int i = 0; i < total_tasks; ++i) {
-        int N_for_task = available_Ns[dist_N_idx(rng_tasks)];
-        TaskType t_type = static_cast<TaskType>(dist_type(rng_tasks));
-        all_tasks.push_back({N_for_task, t_type});
-    }
+    std::uniform_int_distribution<int> dist_batch_size(5000, 15000);
+    std::uniform_int_distribution<int> dist_sleep(500, 2000);
 
-    // ─────────────────────────────────────────────────────────────────────────
-    // Static Round-Robin Partitioning
-    // ─────────────────────────────────────────────────────────────────────────
-    std::vector<std::vector<Task>> partitions(num_gpus);
-    for (int i = 0; i < total_tasks; ++i) {
-        partitions[i % num_gpus].push_back(all_tasks[i]);
-    }
+    int num_batches = 5;
+    auto start = std::chrono::steady_clock::now();
 
-    // Preallocate a single large host buffer for DTOH transfers to save RAM and keep things fair.
-    std::vector<int> shared_dtoh_buffer((size_t)max_N_val * max_N_val);
+    for (int b = 0; b < num_batches; ++b) {
+        // Sleep timer (random time)
+        int sleep_ms = dist_sleep(rng_prod);
+        std::this_thread::sleep_for(std::chrono::milliseconds(sleep_ms));
+
+        // Generate random sized partition (batch)
+        int current_batch_size = dist_batch_size(rng_prod);
+        std::vector<Task> batch_tasks;
+        for (int i = 0; i < current_batch_size; ++i) {
+            int N_for_task = available_Ns[dist_N_idx(rng_prod)];
+            TaskType t_type = static_cast<TaskType>(dist_type(rng_prod));
+            batch_tasks.push_back({N_for_task, t_type});
+        }
 
-    auto start = std::chrono::steady_clock::now();
+        std::cout << "Dispatching Batch " << b + 1 << "/" << num_batches 
+                  << " with " << current_batch_size << " tasks..." << std::endl;
 
-    std::vector<std::thread> threads;
-    for (int i = 0; i < num_gpus; ++i) { // Pass context and kernel function to each worker
-        threads.emplace_back(gpu_worker, i, std::ref(partitions[i]), streams_per_gpu, contexts[i], mmul_funcs[i], vadd_funcs[i], conv_funcs[i],
-         std::ref(global_host_matrix_pool), shared_dtoh_buffer.data());
-    }
+        // Static Round-Robin Partitioning for this batch
+        std::vector<std::vector<Task>> partitions(num_gpus);
+        for (int i = 0; i < current_batch_size; ++i) {
+            partitions[i % num_gpus].push_back(batch_tasks[i]);
+        }
 
-    for (auto& t : threads) {
-        t.join();
+        std::vector<std::thread> threads;
+        for (int i = 0; i < num_gpus; ++i) {
+            threads.emplace_back(gpu_worker, i, std::ref(partitions[i]), streams_per_gpu, contexts[i], mmul_funcs[i], vadd_funcs[i], conv_funcs[i],
+             std::ref(global_host_matrix_pool), shared_dtoh_buffer.data());
+        }
+
+        for (auto& t : threads) {
+            t.join();
+        }
     }
 
     auto end = std::chrono::steady_clock::now();
     std::chrono::duration<double> elapsed = end - start;
+    std::cout << "Total Makespan: " << elapsed.count() << "s" << std::endl;
 
-    std::cout << "Makespan: " << elapsed.count() << "s" << std::endl;
 
     // Cleanup contexts and module
     for (int i = 0; i < num_gpus; ++i) {
@@ -340,7 +348,6 @@ int main() {
     cuModuleUnload(mmul_mod);
     cuModuleUnload(vadd_mod);
     cuModuleUnload(conv_mod);
-    }
 
     return 0;
 }
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index f630a7814b..a60aaf11bc 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -59,6 +59,9 @@ CAF_BEGIN_TYPE_ID_BLOCK(mmul_benchmark, caf::id_block::cuda::end)
     CAF_ADD_TYPE_ID(mmul_benchmark, (Task))
     CAF_ADD_TYPE_ID(mmul_benchmark, (std::vector<Task>))
     CAF_ADD_ATOM(mmul_benchmark, refill_buffer_atom)
+    CAF_ADD_ATOM(mmul_benchmark, produce_atom)
+    CAF_ADD_ATOM(mmul_benchmark, tick_atom)
+    CAF_ADD_ATOM(mmul_benchmark, production_finished_atom)
 CAF_END_TYPE_ID_BLOCK(mmul_benchmark)
 
 
@@ -108,22 +111,49 @@ MatrixPool create_matrix_pool_random(
 // ---------------------------- GLOBAL TASK POOL ----------------------------
 // The central source of truth for work. Implements a pull-based model.
 struct task_pool_state {
-    std::vector<Task> tasks;
+    std::deque<Task> tasks;
     size_t next_task_idx = 0;
+    bool production_finished = false;
+    std::vector<std::pair<size_t, response_promise>> pending;
 };
 
-caf::behavior global_task_pool(caf::stateful_actor<task_pool_state>* self, std::vector<Task> tasks) {
-    self->state().tasks = std::move(tasks);
+caf::behavior global_task_pool(caf::stateful_actor<task_pool_state>* self) {
     return {
         [=](get_work_atom, size_t batch_size) -> result<std::vector<Task>> {
             auto& st = self->state();
-            if (st.next_task_idx >= st.tasks.size())
+            if (!st.tasks.empty()) {
+                size_t count = std::min(batch_size, st.tasks.size());
+                std::vector<Task> batch;
+                for (size_t i = 0; i < count; ++i) {
+                    batch.push_back(st.tasks.front());
+                    st.tasks.pop_front();
+                }
+                return batch;
+            }
+            if (st.production_finished)
                 return sec::end_of_stream;
-            size_t count = std::min(batch_size, st.tasks.size() - st.next_task_idx);
-            std::vector<Task> batch(st.tasks.begin() + st.next_task_idx, 
-                                    st.tasks.begin() + st.next_task_idx + count);
-            st.next_task_idx += count;
-            return batch;
+            
+            auto promise = self->make_response_promise<std::vector<Task>>();
+            st.pending.emplace_back(batch_size, promise);
+            return promise;
+        },
+        [=](std::vector<Task>& batch) {
+            auto& st = self->state();
+            for (auto& t : batch) st.tasks.push_back(t);
+            while (!st.pending.empty() && !st.tasks.empty()) {
+                auto [req_size, promise] = st.pending.front();
+                st.pending.erase(st.pending.begin());
+                size_t count = std::min(req_size, st.tasks.size());
+                std::vector<Task> out_batch;
+                for (size_t i = 0; i < count; ++i) { out_batch.push_back(st.tasks.front()); st.tasks.pop_front(); }
+                promise.deliver(out_batch);
+            }
+        },
+        [=](production_finished_atom) {
+            auto& st = self->state();
+            st.production_finished = true;
+            for (auto& p : st.pending) p.second.deliver(sec::end_of_stream);
+            st.pending.clear();
         }
     };
 }
@@ -247,6 +277,54 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
     };
 }
 
+// ---------------------------- TASK PRODUCER ----------------------------
+struct producer_state {
+    caf::actor pool;
+    caf::actor supervisor;
+    std::vector<int> Ns;
+    int batches_remaining;
+};
+
+caf::behavior task_producer(caf::stateful_actor<producer_state>* self,
+                            caf::actor pool, caf::actor supervisor, int num_batches, std::vector<int> Ns) {
+    self->state().pool = pool;
+    self->state().supervisor = supervisor;
+    self->state().Ns = std::move(Ns);
+    self->state().batches_remaining = num_batches;
+
+    self->mail(tick_atom_v).send(self);
+
+    return {
+        [=](tick_atom) {
+            auto& st = self->state();
+            if (st.batches_remaining-- <= 0) {
+                self->mail(production_finished_atom_v).send(st.pool);
+                self->mail(production_finished_atom_v).send(st.supervisor);
+                self->quit();
+                return;
+            }
+
+            std::random_device rd;
+            std::mt19937 rng(rd());
+            std::uniform_int_distribution<int> dist_sleep(500, 2000);
+            std::uniform_int_distribution<int> dist_batch(5000, 15000);
+            std::uniform_int_distribution<size_t> dist_N(0, st.Ns.size() - 1);
+            std::uniform_int_distribution<int> dist_type(0, 2);
+
+            int count = dist_batch(rng);
+            std::vector<Task> batch;
+            for (int i = 0; i < count; ++i)
+                batch.push_back({st.Ns[dist_N(rng)], static_cast<TaskType>(dist_type(rng))});
+
+            self->mail(produce_atom_v, count).send(st.supervisor);
+            
+            self->delayed_mail(tick_atom_v).delay(std::chrono::milliseconds(dist_sleep(rng))).send(self);
+            // Send work to the pool actor
+            self->mail(batch).send(st.pool);
+        }
+    };
+}
+
 // ---------------------------- WORKER ACTOR ----------------------------
 // Manages 1 stream and pulls work from the Device Actor.
 struct worker_state {
@@ -361,23 +439,25 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
 // ---------------------------- SUPERVISOR ACTOR ----------------------------
 struct supervisor_actor_state {
     int total_tasks;
+    bool production_finished = false;
     int completed = 0;
     std::chrono::steady_clock::time_point start_time;
 };
 
 caf::behavior supervisor_actor_fun(
     caf::stateful_actor<supervisor_actor_state>* self,
-    int total_tasks,
     int workers_per_gpu,
     int max_in_flight_tasks_per_worker,
     MatrixPool pool,
-    std::vector<Task> tasks,
-    int* shared_dtoh_ptr
+    std::vector<int> available_Ns,
+    int* shared_dtoh_ptr,
+    int num_batches
     ) {
-    self->state().total_tasks = total_tasks;
+    self->state().total_tasks = 0;
     self->state().start_time = std::chrono::steady_clock::now();
 
-    auto pool_actor = self->spawn(global_task_pool, std::move(tasks));
+    auto pool_actor = self->spawn(global_task_pool);
+    self->spawn(task_producer, pool_actor, self, num_batches, std::move(available_Ns));
 
     caf::cuda::manager& mgr = caf::cuda::manager::get();
     int num_gpus = mgr.get_num_devices();
@@ -394,9 +474,15 @@ caf::behavior supervisor_actor_fun(
     }
 
     return {
+        [=](produce_atom, int count) {
+            self->state().total_tasks += count;
+        },
+        [=](production_finished_atom) {
+            self->state().production_finished = true;
+        },
         [=](int done) {
             self->state().completed += done;
-            if (self->state().completed >= self->state().total_tasks) {
+            if (self->state().production_finished && self->state().completed >= self->state().total_tasks) {
                 auto end_time = std::chrono::steady_clock::now();
                 std::chrono::duration<double> total_time = end_time - self->state().start_time;
 
@@ -455,32 +541,19 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
         for (const auto& [N, _] : pool.A) sizes.push_back(N);
         std::sort(sizes.begin(), sizes.end());
 
-        std::vector<Task> tasks_for_this_run;
-        tasks_for_this_run.reserve(num_tasks_for_this_run);
-
-        std::mt19937 rng(42);
-        std::uniform_int_distribution<size_t> dist_size(0, sizes.size() - 1);
-        std::uniform_int_distribution<int> dist_type(0, 2);
-        for (int i = 0; i < num_tasks_for_this_run; ++i) {
-            int N = sizes[dist_size(rng)];
-            TaskType type = static_cast<TaskType>(dist_type(rng));
-            tasks_for_this_run.push_back({N, type});
-        }
-
         // Preallocate a single large host buffer for DTOH transfers to save RAM and keep things fair.
         std::vector<int> shared_dtoh_buffer((size_t)max_N * max_N);
 
         // Execute the supervisor which manages the asynchronous workload
         double elapsed = time_run([&]() {
-
             auto sup = sys.spawn(
                 supervisor_actor_fun,
-                (int)tasks_for_this_run.size(), // total_tasks
                 workers_per_gpu,
                 max_in_flight_tasks_per_worker,
                 pool,
-                tasks_for_this_run,
-                    shared_dtoh_buffer.data()
+                sizes,
+                shared_dtoh_buffer.data(),
+                5 // num_batches
             );
 
 	      sys.await_all_actors_done();

From 07fa41b0a6118e418ff06a4cdc65c63420938477 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 08:54:00 -0600
Subject: [PATCH 0788/1000] updated cmakelists

---
 .../mmul-randonom-batch-benchmark/CMakeLists.txt            | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
index e180d2c860..9e2014d33b 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
@@ -44,7 +44,9 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_core/libcaf_core.so"
     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-    CUDA::nvrtc
+    CUDA::nvrtc 
+    CUDA::cublas
+    CUDA::cusparse
 )
 
 target_link_libraries(work-stealing
@@ -53,4 +55,6 @@ target_link_libraries(work-stealing
     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
 )

From 8bf70b0a5ee7175868be3f74c9796f74cfd3cfd0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 09:00:21 -0600
Subject: [PATCH 0789/1000] fixed compiler errors

---
 .../work-stealing.cpp                            | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index a60aaf11bc..f959090023 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -53,14 +53,14 @@ CAF_BEGIN_TYPE_ID_BLOCK(mmul_benchmark, caf::id_block::cuda::end)
     CAF_ADD_ATOM(mmul_benchmark, get_work_atom)
     CAF_ADD_ATOM(mmul_benchmark, task_done_atom)
     CAF_ADD_ATOM(mmul_benchmark, release_memory_atom)
-    CAF_ADD_ATOM(mmul_benchmark, request_work_atom)
-    CAF_ADD_ATOM(mmul_benchmark, worker_done_atom)
+   CAF_ADD_ATOM(mmul_benchmark, request_work_atom) // Renamed to avoid conflict with caf::request_work_atom
+    CAF_ADD_ATOM(mmul_benchmark, worker_done_atom) // Renamed to avoid conflict with caf::worker_done_atom
     CAF_ADD_TYPE_ID(mmul_benchmark, (TaskType))
     CAF_ADD_TYPE_ID(mmul_benchmark, (Task))
     CAF_ADD_TYPE_ID(mmul_benchmark, (std::vector<Task>))
     CAF_ADD_ATOM(mmul_benchmark, refill_buffer_atom)
     CAF_ADD_ATOM(mmul_benchmark, produce_atom)
-    CAF_ADD_ATOM(mmul_benchmark, tick_atom)
+    CAF_ADD_ATOM(mmul_benchmark, producer_tick_atom) // Renamed to avoid conflict with caf::tick_atom
     CAF_ADD_ATOM(mmul_benchmark, production_finished_atom)
 CAF_END_TYPE_ID_BLOCK(mmul_benchmark)
 
@@ -113,8 +113,8 @@ MatrixPool create_matrix_pool_random(
 struct task_pool_state {
     std::deque<Task> tasks;
     size_t next_task_idx = 0;
-    bool production_finished = false;
-    std::vector<std::pair<size_t, response_promise>> pending;
+    bool production_finished = false; // Changed response_promise to typed_response_promise<std::vector<Task>>
+    std::vector<std::pair<size_t, typed_response_promise<std::vector<Task>>>> pending;
 };
 
 caf::behavior global_task_pool(caf::stateful_actor<task_pool_state>* self) {
@@ -292,10 +292,10 @@ caf::behavior task_producer(caf::stateful_actor<producer_state>* self,
     self->state().Ns = std::move(Ns);
     self->state().batches_remaining = num_batches;
 
-    self->mail(tick_atom_v).send(self);
+    self->mail(producer_tick_atom_v).send(self); // Updated atom name
 
     return {
-        [=](tick_atom) {
+        [=](producer_tick_atom) { // Updated atom name
             auto& st = self->state();
             if (st.batches_remaining-- <= 0) {
                 self->mail(production_finished_atom_v).send(st.pool);
@@ -318,7 +318,7 @@ caf::behavior task_producer(caf::stateful_actor<producer_state>* self,
 
             self->mail(produce_atom_v, count).send(st.supervisor);
             
-            self->delayed_mail(tick_atom_v).delay(std::chrono::milliseconds(dist_sleep(rng))).send(self);
+            self->mail(producer_tick_atom_v).delay(std::chrono::milliseconds(dist_sleep(rng))).send(self); // Updated atom name
             // Send work to the pool actor
             self->mail(batch).send(st.pool);
         }

From 200d4867cb55800fe137979de965348d61c5d0f3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 09:13:36 -0600
Subject: [PATCH 0790/1000] updated workload generation to be equal

---
 .../cuda-baseline.cpp                         |  2 +-
 .../work-stealing.cpp                         | 49 ++++++++++++-------
 2 files changed, 33 insertions(+), 18 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index 902b4b9d5c..2081b4c1c8 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -316,7 +316,7 @@ int main() {
             batch_tasks.push_back({N_for_task, t_type});
         }
 
-        std::cout << "Dispatching Batch " << b + 1 << "/" << num_batches 
+        std::cout << "Producer: Dispatching Batch " << b + 1 << "/" << num_batches 
                   << " with " << current_batch_size << " tasks..." << std::endl;
 
         // Static Round-Robin Partitioning for this batch
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index f959090023..e50f072468 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -283,44 +283,59 @@ struct producer_state {
     caf::actor supervisor;
     std::vector<int> Ns;
     int batches_remaining;
+    int total_batches;
+    std::mt19937 rng;
+    std::uniform_int_distribution<int> dist_sleep{500, 2000};
+    std::uniform_int_distribution<int> dist_batch{5000, 15000};
+    std::uniform_int_distribution<int> dist_type{0, 2};
+    std::uniform_int_distribution<size_t> dist_N;
 };
 
 caf::behavior task_producer(caf::stateful_actor<producer_state>* self,
                             caf::actor pool, caf::actor supervisor, int num_batches, std::vector<int> Ns) {
-    self->state().pool = pool;
-    self->state().supervisor = supervisor;
-    self->state().Ns = std::move(Ns);
-    self->state().batches_remaining = num_batches;
-
-    self->mail(producer_tick_atom_v).send(self); // Updated atom name
+    auto& st = self->state();
+    st.pool = pool;
+    st.supervisor = supervisor;
+    st.Ns = std::move(Ns);
+    st.batches_remaining = num_batches;
+    st.total_batches = num_batches;
+    st.rng.seed(42);
+    st.dist_N = std::uniform_int_distribution<size_t>(0, st.Ns.size() - 1);
+
+    // Kick off the first batch with a delay to match CUDA baseline's loop structure
+    int sleep_ms = st.dist_sleep(st.rng);
+    self->mail(producer_tick_atom_v).delay(std::chrono::milliseconds(sleep_ms)).send(self);
 
     return {
         [=](producer_tick_atom) { // Updated atom name
             auto& st = self->state();
-            if (st.batches_remaining-- <= 0) {
+            if (st.batches_remaining <= 0) {
                 self->mail(production_finished_atom_v).send(st.pool);
                 self->mail(production_finished_atom_v).send(st.supervisor);
                 self->quit();
                 return;
             }
+            st.batches_remaining--;
 
-            std::random_device rd;
-            std::mt19937 rng(rd());
-            std::uniform_int_distribution<int> dist_sleep(500, 2000);
-            std::uniform_int_distribution<int> dist_batch(5000, 15000);
-            std::uniform_int_distribution<size_t> dist_N(0, st.Ns.size() - 1);
-            std::uniform_int_distribution<int> dist_type(0, 2);
+            int count = st.dist_batch(st.rng);
+            self->println("Producer: Dispatching Batch {}/{} with {} tasks...", 
+                          st.total_batches - st.batches_remaining, st.total_batches, count);
 
-            int count = dist_batch(rng);
             std::vector<Task> batch;
             for (int i = 0; i < count; ++i)
-                batch.push_back({st.Ns[dist_N(rng)], static_cast<TaskType>(dist_type(rng))});
+                batch.push_back({st.Ns[st.dist_N(st.rng)], static_cast<TaskType>(st.dist_type(st.rng))});
 
             self->mail(produce_atom_v, count).send(st.supervisor);
-            
-            self->mail(producer_tick_atom_v).delay(std::chrono::milliseconds(dist_sleep(rng))).send(self); // Updated atom name
             // Send work to the pool actor
             self->mail(batch).send(st.pool);
+
+            if (st.batches_remaining > 0) {
+                int next_sleep = st.dist_sleep(st.rng);
+                self->mail(producer_tick_atom_v).delay(std::chrono::milliseconds(next_sleep)).send(self);
+            } else {
+                // Last batch produced, final tick to handle termination
+                self->mail(producer_tick_atom_v).send(self);
+            }
         }
     };
 }

From b77fb47c5b1e6505e344aa2d82ee193a453213c7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 09:18:30 -0600
Subject: [PATCH 0791/1000] added parameter for batches

---
 .../cuda-baseline.cpp                         | 91 ++++++++++---------
 .../work-stealing.cpp                         | 10 +-
 2 files changed, 52 insertions(+), 49 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index 2081b4c1c8..3ba1b7f5e3 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -293,54 +293,59 @@ int main() {
     }
 
     std::vector<int> shared_dtoh_buffer((size_t)max_N_val * max_N_val);
-    std::mt19937 rng_prod(42); 
-    std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
-    std::uniform_int_distribution<int> dist_type(0, 2);
-    std::uniform_int_distribution<int> dist_batch_size(5000, 15000);
-    std::uniform_int_distribution<int> dist_sleep(500, 2000);
-
-    int num_batches = 5;
-    auto start = std::chrono::steady_clock::now();
-
-    for (int b = 0; b < num_batches; ++b) {
-        // Sleep timer (random time)
-        int sleep_ms = dist_sleep(rng_prod);
-        std::this_thread::sleep_for(std::chrono::milliseconds(sleep_ms));
-
-        // Generate random sized partition (batch)
-        int current_batch_size = dist_batch_size(rng_prod);
-        std::vector<Task> batch_tasks;
-        for (int i = 0; i < current_batch_size; ++i) {
-            int N_for_task = available_Ns[dist_N_idx(rng_prod)];
-            TaskType t_type = static_cast<TaskType>(dist_type(rng_prod));
-            batch_tasks.push_back({N_for_task, t_type});
-        }
-
-        std::cout << "Producer: Dispatching Batch " << b + 1 << "/" << num_batches 
-                  << " with " << current_batch_size << " tasks..." << std::endl;
-
-        // Static Round-Robin Partitioning for this batch
-        std::vector<std::vector<Task>> partitions(num_gpus);
-        for (int i = 0; i < current_batch_size; ++i) {
-            partitions[i % num_gpus].push_back(batch_tasks[i]);
-        }
 
-        std::vector<std::thread> threads;
-        for (int i = 0; i < num_gpus; ++i) {
-            threads.emplace_back(gpu_worker, i, std::ref(partitions[i]), streams_per_gpu, contexts[i], mmul_funcs[i], vadd_funcs[i], conv_funcs[i],
-             std::ref(global_host_matrix_pool), shared_dtoh_buffer.data());
+    std::vector<int> batch_configs = {5, 10};
+    for (int num_batches : batch_configs) {
+        std::cout << "=====================================" << std::endl;
+        std::cout << "Starting Run with " << num_batches << " batches" << std::endl;
+
+        std::mt19937 rng_prod(42); 
+        std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
+        std::uniform_int_distribution<int> dist_type(0, 2);
+        std::uniform_int_distribution<int> dist_batch_size(5000, 15000);
+        std::uniform_int_distribution<int> dist_sleep(500, 2000);
+
+        auto start = std::chrono::steady_clock::now();
+
+        for (int b = 0; b < num_batches; ++b) {
+            // Sleep timer (random time)
+            int sleep_ms = dist_sleep(rng_prod);
+            std::this_thread::sleep_for(std::chrono::milliseconds(sleep_ms));
+
+            // Generate random sized partition (batch)
+            int current_batch_size = dist_batch_size(rng_prod);
+            std::vector<Task> batch_tasks;
+            for (int i = 0; i < current_batch_size; ++i) {
+                int N_for_task = available_Ns[dist_N_idx(rng_prod)];
+                TaskType t_type = static_cast<TaskType>(dist_type(rng_prod));
+                batch_tasks.push_back({N_for_task, t_type});
+            }
+
+            std::cout << "Producer: Dispatching Batch " << b + 1 << "/" << num_batches 
+                      << " with " << current_batch_size << " tasks..." << std::endl;
+
+            // Static Round-Robin Partitioning for this batch
+            std::vector<std::vector<Task>> partitions(num_gpus);
+            for (int i = 0; i < current_batch_size; ++i) {
+                partitions[i % num_gpus].push_back(batch_tasks[i]);
+            }
+
+            std::vector<std::thread> threads;
+            for (int i = 0; i < num_gpus; ++i) {
+                threads.emplace_back(gpu_worker, i, std::ref(partitions[i]), streams_per_gpu, contexts[i], mmul_funcs[i], vadd_funcs[i], conv_funcs[i],
+                 std::ref(global_host_matrix_pool), shared_dtoh_buffer.data());
+            }
+
+            for (auto& t : threads) {
+                t.join();
+            }
         }
 
-        for (auto& t : threads) {
-            t.join();
-        }
+        auto end = std::chrono::steady_clock::now();
+        std::chrono::duration<double> elapsed = end - start;
+        std::cout << "Total Makespan: " << elapsed.count() << "s" << std::endl;
     }
 
-    auto end = std::chrono::steady_clock::now();
-    std::chrono::duration<double> elapsed = end - start;
-    std::cout << "Total Makespan: " << elapsed.count() << "s" << std::endl;
-
-
     // Cleanup contexts and module
     for (int i = 0; i < num_gpus; ++i) {
         cuCtxDestroy(contexts[i]);
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
index e50f072468..9a19be35da 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/work-stealing.cpp
@@ -531,9 +531,7 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     const int workers_per_gpu = 4; // Admission control: only 16 concurrent tasks per GPU
     const int max_in_flight_tasks_per_worker = 5; // Each worker keeps 2 tasks in flight
 
-    const std::vector<int> actor_counts = {
-	  50000,100000
-    };
+    const std::vector<int> batch_configs = {5, 10};
 
     // Generate deterministic random pool once
     MatrixPool pool = create_matrix_pool_random(
@@ -545,11 +543,11 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
 
     //scheduler
     caf::cuda::manager_config scheduler_off(false);
-    for (int num_tasks_for_this_run : actor_counts) {
+    for (int num_batches : batch_configs) {
 	    // Initialize CUDA manager
 	    caf::cuda::manager::init(sys, scheduler_off);
         std::cout << "=====================================\n";
-        std::cout << "Random Scaling | actors=" << num_tasks_for_this_run << "\n";
+        std::cout << "Random Scaling | batches=" << num_batches << "\n";
 
         // Precompute all task Ns for this run
         std::vector<int> sizes;
@@ -568,7 +566,7 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
                 pool,
                 sizes,
                 shared_dtoh_buffer.data(),
-                5 // num_batches
+                num_batches
             );
 
 	      sys.await_all_actors_done();

From cb48b9527628085cbf0caf72147e873fd5507bb5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 12:09:25 -0600
Subject: [PATCH 0792/1000] putting fault tolerance tests in here

---
 .../sc26/Fault-Tolerance-2/CMakeLists.txt     |  50 ++++
 .../sc26/Fault-Tolerance-2/main.test.cpp      | 222 ++++++++++++++++++
 2 files changed, 272 insertions(+)
 create mode 100644 libcaf_cuda/sc26/Fault-Tolerance-2/CMakeLists.txt
 create mode 100644 libcaf_cuda/sc26/Fault-Tolerance-2/main.test.cpp

diff --git a/libcaf_cuda/sc26/Fault-Tolerance-2/CMakeLists.txt b/libcaf_cuda/sc26/Fault-Tolerance-2/CMakeLists.txt
new file mode 100644
index 0000000000..e64bc364c2
--- /dev/null
+++ b/libcaf_cuda/sc26/Fault-Tolerance-2/CMakeLists.txt
@@ -0,0 +1,50 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+  "${CAF_SRC}/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor"
+  "${CAF_SRC}/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor"
+  "${CAF_SRC}/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor"
+)
+
+
+# 5) Declare your executables
+
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
+)
+
diff --git a/libcaf_cuda/sc26/Fault-Tolerance-2/main.test.cpp b/libcaf_cuda/sc26/Fault-Tolerance-2/main.test.cpp
new file mode 100644
index 0000000000..e80ca34cef
--- /dev/null
+++ b/libcaf_cuda/sc26/Fault-Tolerance-2/main.test.cpp
@@ -0,0 +1,222 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <vector>
+#include <fstream>
+#include <cmath>
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp"
+#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+// --- Matrix Utilities ---
+
+template <class T = float>
+struct LocalCSR {
+    int rows, cols, nnz;
+    std::vector<int> row_ptr;
+    std::vector<int> col_ind;
+    std::vector<T> values;
+};
+
+template <class T = float>
+LocalCSR<T> load_binary_matrix(const std::string& path) {
+    std::ifstream file(path, std::ios::binary);
+    if (!file) throw std::runtime_error("Could not open " + path);
+    int32_t r, c, n;
+    file.read(reinterpret_cast<char*>(&r), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&c), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&n), sizeof(int32_t));
+    std::vector<int32_t> rows_coo(n), cols_coo(n);
+    std::vector<float> vals_coo(n);
+    file.read(reinterpret_cast<char*>(rows_coo.data()), n * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(cols_coo.data()), n * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(vals_coo.data()), n * sizeof(float));
+    int max_idx = 0;
+    for(auto v : rows_coo) if(v > max_idx) max_idx = v;
+    if (max_idx == r || max_idx == c) {
+        for(auto& v : rows_coo) v--;
+        for(auto& v : cols_coo) v--;
+    }
+    LocalCSR<T> csr;
+    csr.rows = r; csr.cols = c; csr.nnz = n;
+    csr.row_ptr.assign(r + 1, 0);
+    csr.col_ind.resize(n);
+    csr.values.resize(n);
+    for (int i = 0; i < n; ++i) csr.row_ptr[rows_coo[i] + 1]++;
+    for (int i = 0; i < r; ++i) csr.row_ptr[i + 1] += csr.row_ptr[i];
+    std::vector<int> current_pos = csr.row_ptr;
+    for (int i = 0; i < n; ++i) {
+        int row = rows_coo[i];
+        int dest = current_pos[row]++;
+        csr.col_ind[dest] = cols_coo[i];
+        csr.values[dest] = static_cast<T>(vals_coo[i]);
+    }
+
+    std::cout << "rows=" << csr.rows
+              << " cols=" << csr.cols
+              << " nnz=" << csr.nnz
+              << " row_ptr.back()=" << csr.row_ptr.back()
+              << std::endl;
+
+    for (int i = 0; i < csr.nnz; ++i) {
+        if (csr.col_ind[i] < 0 || csr.col_ind[i] >= csr.cols)
+            throw std::runtime_error("bad col index");
+    }
+
+    return csr;
+}
+
+template <class T = float>
+std::vector<T> compute_rhs(const LocalCSR<T>& A, const std::vector<T>& x) {
+    std::vector<T> b(A.rows, T{0});
+    for (int i = 0; i < A.rows; ++i) {
+        T sum = T{0};
+        for (int j = A.row_ptr[i]; j < A.row_ptr[i+1]; ++j)
+            sum += A.values[j] * x[A.col_ind[j]];
+        b[i] = sum;
+    }
+    return b;
+}
+
+bool is_valid(const std::vector<double>& x) {
+    for (double val : x) {
+        if (std::isnan(val) || std::isinf(val)) return false;
+    }
+    return true;
+}
+
+// --- Robust Solver Actor (Facade Orchestrator) ---
+
+enum class solver_strategy { cgs, bicgstab, gmres };
+
+struct robust_solver_state {
+    LocalCSR<double> A;
+    std::vector<double> b;
+    std::vector<double> x;
+    double tol;
+    int max_iter;
+    std::vector<caf::actor> facades;
+    solver_strategy current_strategy = solver_strategy::cgs;
+    caf::actor requester;
+};
+
+behavior robust_solver(stateful_actor<robust_solver_state>* self, 
+                       LocalCSR<double> A, std::vector<double> b, double tol, int max_iter) {
+    self->state().A = std::move(A);
+    self->state().b = std::move(b);
+    self->state().tol = tol;
+    self->state().x.assign(self->state().A.rows, 0.0);
+    self->state().max_iter = max_iter;
+
+    self->state().facades.push_back(self->spawn<sparse_cg_facade<double>>(100));
+    self->state().facades.push_back(self->spawn<sparse_bicgstab_facade<double>>(100));
+    self->state().facades.push_back(self->spawn<sparse_gmres_facade<double>>(100));
+
+    auto get_method_name = [](solver_strategy s) {
+        switch (s) {
+            case solver_strategy::cgs:      return "CGS";
+            case solver_strategy::bicgstab: return "BiCGSTAB";
+            case solver_strategy::gmres:    return "GMRES";
+            default:                        return "Unknown";
+        }
+    };
+
+    auto start_cgs = [=] {
+        auto& s = self->state();
+        s.current_strategy = solver_strategy::cgs;
+        self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
+                   create_in_arg(s.b), create_in_out_arg(s.x),
+                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 0, 0).send(s.facades[0]);
+    };
+
+    auto start_bicgstab = [=] {
+        auto& s = self->state();
+        s.current_strategy = solver_strategy::bicgstab;
+        self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
+                   create_in_arg(s.b), create_in_out_arg(s.x),
+                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 0, 0).send(s.facades[1]);
+    };
+
+    auto start_gmres = [=] {
+        auto& s = self->state();
+        s.current_strategy = solver_strategy::gmres;
+        self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
+                   create_in_arg(s.b), create_in_out_arg(s.x),
+                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 30, 0, 0).send(s.facades[2]);
+    };
+
+    return {
+        [=](start_atom) {
+            self->state().requester = actor_cast<caf::actor>(self->current_sender());
+            std::cout << "[INFO] Attempting solve with CGS..." << std::endl;
+            start_cgs();
+        },
+        [=](uint32_t /*id*/, int /*idx*/, const std::vector<double>& result, solver_result_meta meta) {
+            auto& s = self->state();
+            const char* method_name = get_method_name(s.current_strategy);
+            if (meta.converged && is_valid(result)) {
+                std::cout << "[SUCCESS] " << method_name << " converged. Matrix good." << std::endl;
+                if (s.requester)
+                    self->mail(true).send(s.requester);
+                self->quit();
+            } else {
+                std::cout << "[WARNING] " << method_name;
+                if (!meta.converged) {
+                    std::cout << " failed to converge.";
+                }
+                if (!is_valid(result)) {
+                    if (!meta.converged) {
+                        std::cout << " and";
+                    }
+                    std::cout << " produced NaN/Inf values.";
+                }
+                std::cout << " Retrying..." << std::endl;
+                switch (s.current_strategy) {
+                    case solver_strategy::cgs:
+                        start_bicgstab();
+                        break;
+                    case solver_strategy::bicgstab:
+                        start_gmres();
+                        break;
+                    default:
+                        std::cout << "[ERROR] All facade solvers failed." << std::endl;
+                        if (s.requester)
+                            self->mail(std::string("All facade solvers failed")).send(s.requester);
+                        self->quit();
+                        break;
+                }
+            }
+        }
+    };
+}
+
+void caf_main(actor_system& sys) {
+    manager::init(sys, manager_config(true, true)); 
+    scoped_actor self{sys};
+
+    std::string path = "/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric/jpwh_991.bin";
+    std::cout << "[INFO] Loading real-world matrix: " << path << std::endl;
+
+    try {
+        LocalCSR<double> A = load_binary_matrix<double>(path);
+        std::vector<double> x_target(A.rows, 1.0);
+        std::vector<double> b = compute_rhs(A, x_target);
+
+        auto robust = sys.spawn(robust_solver, std::move(A), std::move(b), 1e-10, 5000);
+        self->mail(start_atom_v).send(robust);
+        
+        self->receive(
+            [](bool) { std::cout << "[INFO] Robust solver converged successfully." << std::endl; },
+            [](std::string err) { std::cout << "[INFO] Robust solver aborted: " << err << std::endl; }
+        );
+    } catch (const std::exception& e) {
+        std::cerr << "[ERROR] " << e.what() << std::endl;
+    }
+
+    manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda)
\ No newline at end of file

From 597d8a1f73410da1902e729f54f33ec17ea54a59 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 12:14:56 -0600
Subject: [PATCH 0793/1000] updated implementations

---
 .../sc26/Runtime-Overhead/command_runner.cpp  | 128 ++++++++----
 .../sc26/Runtime-Overhead/cuda_native.cpp     | 194 +++++++++++++-----
 2 files changed, 222 insertions(+), 100 deletions(-)

diff --git a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
index 26cf1ebc1e..31e65b18ae 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
@@ -10,11 +10,33 @@
 #include <numeric>
 #include <random>
 #include "caf/actor_registry.hpp"
+//#include <caf/atoms.hpp>
+
+
 
 using namespace caf;
 using namespace std::chrono_literals;
 
-using command = caf::cuda::command_runner<>;
+
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
+using command =
+  caf::cuda::command_runner<>;
 
 command mmul_command;
 
@@ -26,25 +48,30 @@ struct mmul_state {
 //so its only fair that we do not either 
 std::vector<int> matrixC;
 
-static const unsigned int RANDOM_SEED = 42;
 
-caf::behavior mmul_actor(caf::stateful_actor<mmul_state>* self) {
+
+
+caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
   return {
 
     [=](const std::vector<int>& matrixA,
         const std::vector<int>& matrixB,
         int N) {
-      
+
       using clock = std::chrono::steady_clock;
       using ms = std::chrono::duration<double, std::milli>;
 
+      size_t bytes_a = matrixA.size() * sizeof(int);
+      size_t bytes_b = matrixB.size() * sizeof(int);
+      size_t bytes_c = matrixC.size() * sizeof(int);
+
 
       caf::cuda::manager& mgr = caf::cuda::manager::get();
       int device = 0;
       int stream = 1;
 
       auto program =
-        mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
       auto t_total_start = clock::now();
       // -------------------------
@@ -66,6 +93,7 @@ caf::behavior mmul_actor(caf::stateful_actor<mmul_state>* self) {
           stream,
           std::move(inA));
 
+
       auto t_a_transfer_end = clock::now();
 
       // -------------------------
@@ -117,9 +145,8 @@ caf::behavior mmul_actor(caf::stateful_actor<mmul_state>* self) {
 
 	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);
 
-            //std::vector<int> matrixC = dC->copy_to_host();
-
-	    dC->copy_to_host(matrixC.data(),N*N);
+            mmul_command.copy_to_host_async(dC, matrixC.data(), N * N);
+            dC->synchronize();
 
             auto t_copy_end = clock::now();
             auto t_total_end = clock::now();
@@ -129,6 +156,9 @@ caf::behavior mmul_actor(caf::stateful_actor<mmul_state>* self) {
             // -------------------------
 
             std::cout << "\n===== BENCHMARK RESULTS (N=" << N << ") =====\n";
+            std::cout << "  Transfer size A: " << bytes_a << " bytes\n";
+            std::cout << "  Transfer size B: " << bytes_b << " bytes\n";
+            std::cout << "  Transfer size C: " << bytes_c << " bytes\n";
 
             std::cout << "create_in_arg A: "
                       << ms(t_a_inarg_end - t_a_inarg_start).count()
@@ -168,32 +198,31 @@ caf::behavior mmul_actor(caf::stateful_actor<mmul_state>* self) {
 
 
 void run_mmul_test(caf::actor_system& sys, int matrix_size) {
-  // F5: manager::init/shutdown moved to caf_main — called once for all sizes
 
+
+  caf::cuda::manager::init(sys);
   // ------------------------------------
   // Start timing
   // ------------------------------------
   auto start = std::chrono::steady_clock::now();
 
-  // F4: use mt19937(42) to match cuda_native data initialisation
-  std::mt19937 rng(RANDOM_SEED);
-  std::uniform_int_distribution<int> dist(1, 10);
-  std::vector<int> matrixA(matrix_size * matrix_size);
-  std::vector<int> matrixB(matrix_size * matrix_size);
-  for (auto& v : matrixA) v = dist(rng);
-  for (auto& v : matrixB) v = dist(rng);
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<int> matrixA(matrix_size * matrix_size,2);
+  std::vector<int> matrixB(matrix_size * matrix_size,3);
 
   matrixC.resize(matrix_size*matrix_size);
 
-  using clock = std::chrono::steady_clock;
+ using clock = std::chrono::steady_clock;
+
+auto t_start = clock::now();
+
+caf::actor a =sys.spawn(mmul_actor_fun_2);
 
-  auto t_start = clock::now();
+anon_mail(matrixA,matrixB,matrix_size).send(a);
 
-  caf::actor a =sys.spawn(mmul_actor);
+auto t_end = clock::now();
 
-  anon_mail(matrixA,matrixB,matrix_size).send(a);
 
-  auto t_end = clock::now();
 
   // Wait for all actors to finish
   sys.await_all_actors_done();
@@ -207,35 +236,48 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size) {
 
   std::cout << "[MMUL TEST] matrix_size=" << matrix_size
             << ", time=" << duration_ms << " ms\n";
-}
 
-class config : public actor_system_config {
-public:
-  config() {
-      set("caf.scheduler.max-threads", 1u);
-  }
-};
+  caf::cuda::manager::shutdown();
 
+}
 
-void caf_main(caf::actor_system& sys, const config& cfg) {
-  caf::cuda::manager::init(sys);  // F5: init once before all sizes
 
-  // F2: warmup run to prime CUDA context, JIT, and CAF infrastructure
-  std::cout << "--- warmup starting ---\n";
-  run_mmul_test(sys, 64);
-  std::cout << "--- warmup complete ---\n";
+void caf_main(caf::actor_system& sys) {
+    run_mmul_test(sys, 1000);
+    run_mmul_test(sys, 4000);
+    run_mmul_test(sys, 8000);
+    run_mmul_test(sys, 12000);
+}
 
-  // F1: unified sizes matching cuda_native: {1000, 2000, 4000, 8000, 16000}
-  run_mmul_test(sys,1000);
-  run_mmul_test(sys,2000);
-  run_mmul_test(sys,4000);
-  run_mmul_test(sys,8000);
-  run_mmul_test(sys,16000);
+int main(int argc, char** argv) {
+  // Initialize user defined types and messages if needed.
+  //init_global_meta_objects<custom_types_1>();
+  
+  // Initialize the global type information.
+  core::init_global_meta_objects();
 
-  caf::cuda::manager::shutdown();  // F5: shutdown once after all sizes
-}
+  // Create the config.
+  actor_system_config cfg;
 
+  // --- SINGLE THREAD CONFIGURATION ---
+  cfg.set("caf.scheduler.max-threads", 1);
+  cfg.set("caf.scheduler.policy", "sharing"); 
+  // ------------------------------------
+
+  // Read CLI options. (Note: CLI flags like --caf.scheduler.max-threads=4 
+  // will override the hardcoded '1' above if provided by the user).
+  auto err = cfg.parse(argc, argv);
+  if (err)
+    return EXIT_FAILURE;
 
+  if (cfg.helptext_printed())
+    return 0;
 
+  // Create the actor system (the scheduler starts here).
+  actor_system sys{cfg};
 
-CAF_MAIN()
+  // Run user-defined code.
+  caf_main(sys);
+  
+  return 0;
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp b/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
index 507e4e480b..2f0654cc50 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
@@ -3,10 +3,23 @@
 #include <iostream>
 #include <vector>
 #include <chrono>
+#include <fstream>
+#include <sstream>
 #include <stdexcept>
-#include <random>
-
-static const unsigned int RANDOM_SEED = 42;
+#include <cmath>
+#include <atomic>
+#include <thread>
+
+struct TimingState {
+    std::chrono::steady_clock::time_point end_time;
+    std::atomic<bool> ready{false};
+};
+
+void completion_callback(void* userData) {
+    auto* state = static_cast<TimingState*>(userData);
+    state->end_time = std::chrono::steady_clock::now();
+    state->ready = true;
+}
 
 static void checkCU(CUresult r, const char* where) {
     if (r != CUDA_SUCCESS) {
@@ -18,8 +31,17 @@ static void checkCU(CUresult r, const char* where) {
     }
 }
 
-// runMatrixMul: executes the kernel and returns the total duration in milliseconds
-double runMatrixMul(CUmodule module, CUfunction kernel, int N) {
+std::string readFile(const std::string &path) {
+    std::ifstream in(path, std::ios::in | std::ios::binary);
+    if (!in) throw std::runtime_error("Failed to open " + path);
+    std::ostringstream ss;
+    ss << in.rdbuf();
+    return ss.str();
+}
+
+std::vector<int> h_c; // Declared globally
+
+void runMatrixMul(CUmodule module, CUfunction kernel, int N, CUstream stream) {
     using clock = std::chrono::steady_clock;
     using ms = std::chrono::duration<double, std::milli>;
 
@@ -28,41 +50,48 @@ double runMatrixMul(CUmodule module, CUfunction kernel, int N) {
     size_t elements = (size_t)N * (size_t)N;
     size_t bytes = elements * sizeof(int);
 
-    std::mt19937 rng(RANDOM_SEED);
-    std::uniform_int_distribution<int> dist(1, 10);
-
-    std::vector<int> h_a(elements);
-    std::vector<int> h_b(elements);
-    std::vector<int> h_c(elements);
-
-    for (auto& v : h_a) v = dist(rng);
-    for (auto& v : h_b) v = dist(rng);
+    std::vector<int> h_a(elements, 1); // These remain local as they are initialized with N
+    std::vector<int> h_b(elements, 1); // These remain local as they are initialized with N
+    h_c.resize(elements); // Resize the global h_c for the current N
 
     CUdeviceptr d_a, d_b, d_c;
-    CUstream stream;
-
-    // ----------------------------------
-    // Create Stream
-    // ----------------------------------
-    checkCU(cuStreamCreate(&stream, CU_STREAM_DEFAULT), "cuStreamCreate");
 
     auto t_total_start = clock::now();
 
     // ----------------------------------
     // Device Allocation
     // ----------------------------------
-    checkCU(cuMemAlloc(&d_a, bytes), "cuMemAlloc d_a");
-    checkCU(cuMemAlloc(&d_b, bytes), "cuMemAlloc d_b");
-    checkCU(cuMemAlloc(&d_c, bytes), "cuMemAlloc d_c");
+    auto t_alloc_start = clock::now();
+
+    checkCU(cuMemAllocAsync(&d_a, bytes, stream), "cuMemAllocAsync d_a");
+    checkCU(cuMemAllocAsync(&d_b, bytes, stream), "cuMemAllocAsync d_b");
+    checkCU(cuMemAllocAsync(&d_c, bytes, stream), "cuMemAllocAsync d_c");
+
+    auto t_alloc_end = clock::now();
 
     // ----------------------------------
-    // H2D copies (async, pipelined with kernel)
+    // H2D copy A
     // ----------------------------------
+    auto t_h2d_a_start = clock::now();
+
     checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), bytes, stream), "cuMemcpyHtoDAsync A");
+    std::cout << "  (Transfer size: " << bytes << " bytes)\n";
+
+    auto t_h2d_a_end = clock::now();
+
+    // ----------------------------------
+    // H2D copy B
+    // ----------------------------------
+    auto t_h2d_b_start = clock::now();
+
     checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), bytes, stream), "cuMemcpyHtoDAsync B");
+    //checkCU(cuStreamSynchronize(stream), "sync B");
+    std::cout << "  (Transfer size: " << bytes << " bytes)\n";
+
+    auto t_h2d_b_end = clock::now();
 
     // ----------------------------------
-    // Kernel launch
+    // Kernel launch + execution
     // ----------------------------------
     const unsigned int blockX = 32;
     const unsigned int blockY = 32;
@@ -71,6 +100,8 @@ double runMatrixMul(CUmodule module, CUfunction kernel, int N) {
 
     void* kernelParams[] = { &d_a, &d_b, &d_c, &N };
 
+    auto t_kernel_start = clock::now();
+
     checkCU(cuLaunchKernel(kernel,
                            gridX, gridY, 1,
                            blockX, blockY, 1,
@@ -80,42 +111,86 @@ double runMatrixMul(CUmodule module, CUfunction kernel, int N) {
                            nullptr),
             "cuLaunchKernel");
 
+   // checkCU(cuStreamSynchronize(stream), "kernel sync");
+
+    auto t_kernel_end = clock::now();
+
     // ----------------------------------
-    // D2H copy + synchronize all pending GPU work
+    // D2H copy
     // ----------------------------------
+    auto t_d2h_start = clock::now();
+    TimingState t_state;
+
     cuMemcpyDtoHAsync(h_c.data(), d_c, bytes, stream);
-    cuStreamSynchronize(stream);
 
+    // Enqueue the host function to capture timing when the copy finishes
+    checkCU(cuLaunchHostFunc(stream, completion_callback, &t_state), "cuLaunchHostFunc");
+
+    // In a real actor, we would not wait here. 
+    // For this benchmark driver, we wait for the callback to fire.
+    while (!t_state.ready) {
+        std::this_thread::yield();
+    }
+
+    std::cout << "  (Transfer size: " << bytes << " bytes)\n";
+    
+    auto t_d2h_end = t_state.end_time;
+    auto t_total_end = t_state.end_time;
+
+    
     // ----------------------------------
     // Free device memory
     // ----------------------------------
-    checkCU(cuMemFree(d_a), "cuMemFree A");
-    checkCU(cuMemFree(d_b), "cuMemFree B");
-    checkCU(cuMemFree(d_c), "cuMemFree C");
+    auto t_free_start = clock::now();
+
+    checkCU(cuMemFreeAsync(d_a, stream), "cuMemFreeAsync A");
+    checkCU(cuMemFreeAsync(d_b, stream), "cuMemFreeAsync B");
+    checkCU(cuMemFreeAsync(d_c, stream), "cuMemFreeAsync C");
+
+    auto t_free_end = clock::now();
+
 
-    auto t_total_end = clock::now();
 
-    // ----------------------------------
-    // Destroy stream
-    // ----------------------------------
-    checkCU(cuStreamDestroy(stream), "cuStreamDestroy");
 
     // ----------------------------------
     // Print Results
-    // (Sub-timings omitted: intermediate async ops have no sync point
-    //  so CPU-side timestamps do not reflect actual GPU phase durations.)
     // ----------------------------------
-    double total_ms = ms(t_total_end - t_total_start).count();
 
-    std::cout << "TOTAL: " << total_ms << " ms\n";
-    std::cout << "=============================================\n";
+    std::cout << "Device allocation: "
+              << ms(t_alloc_end - t_alloc_start).count()
+              << " ms\n";
+
+    std::cout << "H2D copy A: "
+              << ms(t_h2d_a_end - t_h2d_a_start).count()
+              << " ms\n";
+
+    std::cout << "H2D copy B: "
+              << ms(t_h2d_b_end - t_h2d_b_start).count()
+              << " ms\n";
+
+    std::cout << "Kernel execution: "
+              << ms(t_kernel_end - t_kernel_start).count()
+              << " ms\n";
 
-    return total_ms;
+    std::cout << "D2H copy: "
+              << ms(t_d2h_end - t_d2h_start).count()
+              << " ms\n";
+
+    std::cout << "Device free: "
+              << ms(t_free_end - t_free_start).count()
+              << " ms\n";
+
+    std::cout << "TOTAL: "
+              << ms(t_total_end - t_total_start).count()
+              << " ms\n";
+
+    std::cout << "=============================================\n";
 }
 
 int main(int argc, char** argv) {
-    std::vector<int> sizes = {1000, 2000, 4000, 8000, 16000};
+  std::vector<int> sizes = {1000, 4000, 8000, 12000};
     
+//   std::vector<int> sizes = {12000};	
    if (argc > 1) {
         sizes.clear();
         for (int i = 1; i < argc; ++i) sizes.push_back(std::stoi(argv[i]));
@@ -127,36 +202,41 @@ int main(int argc, char** argv) {
     checkCU(cuDeviceGet(&dev, 0), "cuDeviceGet(0)");
 
     CUcontext ctx;
-    checkCU(cuCtxCreate(&ctx, 0, dev), "cuCtxCreate");
+    checkCU(cuCtxCreate(&ctx, CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, dev), "cuCtxCreate");
+
+    const std::string cubinPath = "../mmul.cubin";
+    std::string cubin;
+
+    try {
+        cubin = readFile(cubinPath);
+    } catch (const std::exception &e) {
+        std::cerr << "Failed to read CUBIN file '" << cubinPath << "': " << e.what() << "\n";
+        return EXIT_FAILURE;
+    }
 
     CUmodule module;
-    checkCU(cuModuleLoad(&module, "mmul.cubin"), "cuModuleLoad mmul.cubin");
+    checkCU(cuModuleLoadDataEx(&module, cubin.data(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
 
     CUfunction kernel;
     checkCU(cuModuleGetFunction(&kernel, module, "matrixMul"), "cuModuleGetFunction matrixMul");
 
-    // Warmup: small run to prime CUDA lazy-init / cubin load before timed tests
-    runMatrixMul(module, kernel, 64);
-    std::cout << "--- warmup complete ---\n";
-
-    std::vector<std::pair<int,double>> results;
+    CUstream stream;
+    checkCU(cuStreamCreate(&stream, CU_STREAM_DEFAULT), "cuStreamCreate");
 
     for (int N : sizes) {
         try {
-            double t = runMatrixMul(module, kernel, N);
-            results.emplace_back(N, t);
+            runMatrixMul(module, kernel, N, stream);
+            
+            // Ensure stream is completely empty before starting the next size
+            checkCU(cuStreamSynchronize(stream), "cuStreamSynchronize between sizes");
+            
         } catch (const std::exception &e) {
             std::cerr << "Exception while running N=" << N << ": " << e.what() << "\n";
         }
         std::cout << "----------------------------------------\n";
     }
 
-    // Print summary of results
-    std::cout << "\nMatrix size : time (ms)\n";
-    for (auto &p : results) {
-        std::cout << p.first << " : " << p.second << " ms\n";
-    }
-
+    checkCU(cuStreamDestroy(stream), "cuStreamDestroy");
     checkCU(cuModuleUnload(module), "cuModuleUnload");
     checkCU(cuCtxDestroy(ctx), "cuCtxDestroy");
 

From 4a0f7ca406a31d8a2957a6a4353278c5684b8617 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 12:18:10 -0600
Subject: [PATCH 0794/1000] updated code

---
 .../sc26/Runtime-Overhead/cuda_native.cpp     | 242 +++++------------
 .../command_runner.cpp                        | 252 +++++++++++-------
 2 files changed, 221 insertions(+), 273 deletions(-)

diff --git a/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp b/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
index 2f0654cc50..629577c08c 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
@@ -1,4 +1,3 @@
-// matrix_mul_driver.cpp
 #include <cuda.h>
 #include <iostream>
 #include <vector>
@@ -6,20 +5,6 @@
 #include <fstream>
 #include <sstream>
 #include <stdexcept>
-#include <cmath>
-#include <atomic>
-#include <thread>
-
-struct TimingState {
-    std::chrono::steady_clock::time_point end_time;
-    std::atomic<bool> ready{false};
-};
-
-void completion_callback(void* userData) {
-    auto* state = static_cast<TimingState*>(userData);
-    state->end_time = std::chrono::steady_clock::now();
-    state->ready = true;
-}
 
 static void checkCU(CUresult r, const char* where) {
     if (r != CUDA_SUCCESS) {
@@ -39,60 +24,9 @@ std::string readFile(const std::string &path) {
     return ss.str();
 }
 
-std::vector<int> h_c; // Declared globally
-
-void runMatrixMul(CUmodule module, CUfunction kernel, int N, CUstream stream) {
-    using clock = std::chrono::steady_clock;
-    using ms = std::chrono::duration<double, std::milli>;
-
-    std::cout << "\n===== DRIVER BENCHMARK (N=" << N << ") =====\n";
-
-    size_t elements = (size_t)N * (size_t)N;
-    size_t bytes = elements * sizeof(int);
-
-    std::vector<int> h_a(elements, 1); // These remain local as they are initialized with N
-    std::vector<int> h_b(elements, 1); // These remain local as they are initialized with N
-    h_c.resize(elements); // Resize the global h_c for the current N
-
-    CUdeviceptr d_a, d_b, d_c;
-
-    auto t_total_start = clock::now();
-
-    // ----------------------------------
-    // Device Allocation
-    // ----------------------------------
-    auto t_alloc_start = clock::now();
-
-    checkCU(cuMemAllocAsync(&d_a, bytes, stream), "cuMemAllocAsync d_a");
-    checkCU(cuMemAllocAsync(&d_b, bytes, stream), "cuMemAllocAsync d_b");
-    checkCU(cuMemAllocAsync(&d_c, bytes, stream), "cuMemAllocAsync d_c");
-
-    auto t_alloc_end = clock::now();
-
-    // ----------------------------------
-    // H2D copy A
-    // ----------------------------------
-    auto t_h2d_a_start = clock::now();
-
-    checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), bytes, stream), "cuMemcpyHtoDAsync A");
-    std::cout << "  (Transfer size: " << bytes << " bytes)\n";
-
-    auto t_h2d_a_end = clock::now();
-
-    // ----------------------------------
-    // H2D copy B
-    // ----------------------------------
-    auto t_h2d_b_start = clock::now();
-
-    checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), bytes, stream), "cuMemcpyHtoDAsync B");
-    //checkCU(cuStreamSynchronize(stream), "sync B");
-    std::cout << "  (Transfer size: " << bytes << " bytes)\n";
-
-    auto t_h2d_b_end = clock::now();
-
-    // ----------------------------------
-    // Kernel launch + execution
-    // ----------------------------------
+// Launch kernel once
+void launchKernel(CUfunction kernel, CUstream stream,
+                  CUdeviceptr d_a, CUdeviceptr d_b, CUdeviceptr d_c, int N) {
     const unsigned int blockX = 32;
     const unsigned int blockY = 32;
     unsigned int gridX = (N + blockX - 1) / blockX;
@@ -100,8 +34,6 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N, CUstream stream) {
 
     void* kernelParams[] = { &d_a, &d_b, &d_c, &N };
 
-    auto t_kernel_start = clock::now();
-
     checkCU(cuLaunchKernel(kernel,
                            gridX, gridY, 1,
                            blockX, blockY, 1,
@@ -110,91 +42,12 @@ void runMatrixMul(CUmodule module, CUfunction kernel, int N, CUstream stream) {
                            kernelParams,
                            nullptr),
             "cuLaunchKernel");
-
-   // checkCU(cuStreamSynchronize(stream), "kernel sync");
-
-    auto t_kernel_end = clock::now();
-
-    // ----------------------------------
-    // D2H copy
-    // ----------------------------------
-    auto t_d2h_start = clock::now();
-    TimingState t_state;
-
-    cuMemcpyDtoHAsync(h_c.data(), d_c, bytes, stream);
-
-    // Enqueue the host function to capture timing when the copy finishes
-    checkCU(cuLaunchHostFunc(stream, completion_callback, &t_state), "cuLaunchHostFunc");
-
-    // In a real actor, we would not wait here. 
-    // For this benchmark driver, we wait for the callback to fire.
-    while (!t_state.ready) {
-        std::this_thread::yield();
-    }
-
-    std::cout << "  (Transfer size: " << bytes << " bytes)\n";
-    
-    auto t_d2h_end = t_state.end_time;
-    auto t_total_end = t_state.end_time;
-
-    
-    // ----------------------------------
-    // Free device memory
-    // ----------------------------------
-    auto t_free_start = clock::now();
-
-    checkCU(cuMemFreeAsync(d_a, stream), "cuMemFreeAsync A");
-    checkCU(cuMemFreeAsync(d_b, stream), "cuMemFreeAsync B");
-    checkCU(cuMemFreeAsync(d_c, stream), "cuMemFreeAsync C");
-
-    auto t_free_end = clock::now();
-
-
-
-
-    // ----------------------------------
-    // Print Results
-    // ----------------------------------
-
-    std::cout << "Device allocation: "
-              << ms(t_alloc_end - t_alloc_start).count()
-              << " ms\n";
-
-    std::cout << "H2D copy A: "
-              << ms(t_h2d_a_end - t_h2d_a_start).count()
-              << " ms\n";
-
-    std::cout << "H2D copy B: "
-              << ms(t_h2d_b_end - t_h2d_b_start).count()
-              << " ms\n";
-
-    std::cout << "Kernel execution: "
-              << ms(t_kernel_end - t_kernel_start).count()
-              << " ms\n";
-
-    std::cout << "D2H copy: "
-              << ms(t_d2h_end - t_d2h_start).count()
-              << " ms\n";
-
-    std::cout << "Device free: "
-              << ms(t_free_end - t_free_start).count()
-              << " ms\n";
-
-    std::cout << "TOTAL: "
-              << ms(t_total_end - t_total_start).count()
-              << " ms\n";
-
-    std::cout << "=============================================\n";
 }
 
-int main(int argc, char** argv) {
-  std::vector<int> sizes = {1000, 4000, 8000, 12000};
-    
-//   std::vector<int> sizes = {12000};	
-   if (argc > 1) {
-        sizes.clear();
-        for (int i = 1; i < argc; ++i) sizes.push_back(std::stoi(argv[i]));
-    }
+int main() {
+    const int N = 1000;
+    std::vector<int> iteration_series = {1000, 2000, 3000, 4000, 5000,
+                                         6000, 7000, 8000, 9000, 10000};
 
     checkCU(cuInit(0), "cuInit");
 
@@ -202,40 +55,79 @@ int main(int argc, char** argv) {
     checkCU(cuDeviceGet(&dev, 0), "cuDeviceGet(0)");
 
     CUcontext ctx;
-    checkCU(cuCtxCreate(&ctx, CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, dev), "cuCtxCreate");
+    checkCU(cuCtxCreate(&ctx, 0, dev), "cuCtxCreate");
 
-    const std::string cubinPath = "../mmul.cubin";
-    std::string cubin;
-
-    try {
-        cubin = readFile(cubinPath);
-    } catch (const std::exception &e) {
-        std::cerr << "Failed to read CUBIN file '" << cubinPath << "': " << e.what() << "\n";
-        return EXIT_FAILURE;
-    }
+    std::string ptx = readFile("mmul.ptx");
 
     CUmodule module;
-    checkCU(cuModuleLoadDataEx(&module, cubin.data(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
+    checkCU(cuModuleLoadDataEx(&module, ptx.c_str(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
 
     CUfunction kernel;
     checkCU(cuModuleGetFunction(&kernel, module, "matrixMul"), "cuModuleGetFunction matrixMul");
 
+    // ----------------------------------
+    // Persistent host buffers
+    // ----------------------------------
+    size_t elements = (size_t)N * N;
+    std::vector<int> h_a(elements, 1);
+    std::vector<int> h_b(elements, 1);
+    std::vector<int> h_c(elements, 0);
+
     CUstream stream;
     checkCU(cuStreamCreate(&stream, CU_STREAM_DEFAULT), "cuStreamCreate");
 
-    for (int N : sizes) {
-        try {
-            runMatrixMul(module, kernel, N, stream);
-            
-            // Ensure stream is completely empty before starting the next size
-            checkCU(cuStreamSynchronize(stream), "cuStreamSynchronize between sizes");
-            
-        } catch (const std::exception &e) {
-            std::cerr << "Exception while running N=" << N << ": " << e.what() << "\n";
+    using clock = std::chrono::steady_clock;
+
+    for (int iterations : iteration_series) {
+        auto start = clock::now();
+
+        for (int i = 0; i < iterations; ++i) {
+            // ----------------------------------
+            // Allocate device memory each iteration
+            // ----------------------------------
+            CUdeviceptr d_a, d_b, d_c;
+            checkCU(cuMemAllocAsync(&d_a, elements * sizeof(int), stream), "cuMemAllocAsync d_a");
+            checkCU(cuMemAllocAsync(&d_b, elements * sizeof(int), stream), "cuMemAllocAsync d_b");
+            checkCU(cuMemAllocAsync(&d_c, elements * sizeof(int), stream), "cuMemAllocAsync d_c");
+
+            // ----------------------------------
+            // Copy persistent host buffers to device
+            // ----------------------------------
+            checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), elements * sizeof(int), stream), "H2D d_a");
+            checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), elements * sizeof(int), stream), "H2D d_b");
+
+            // ----------------------------------
+            // Launch kernel
+            // ----------------------------------
+            launchKernel(kernel, stream, d_a, d_b, d_c, N);
+
+            // ----------------------------------
+            // Copy result back
+            // ----------------------------------
+            checkCU(cuMemcpyDtoHAsync(h_c.data(), d_c, elements * sizeof(int), stream), "D2H d_c");
+
+            // ----------------------------------
+            // Free device memory
+            // ----------------------------------
+            checkCU(cuMemFreeAsync(d_a, stream), "cuMemFreeAsync d_a");
+            checkCU(cuMemFreeAsync(d_b, stream), "cuMemFreeAsync d_b");
+            checkCU(cuMemFreeAsync(d_c, stream), "cuMemFreeAsync d_c");
         }
-        std::cout << "----------------------------------------\n";
+
+        // Synchronize stream after series
+        checkCU(cuStreamSynchronize(stream), "stream sync after series");
+
+        auto end = clock::now();
+        double total_ms = std::chrono::duration<double, std::milli>(end - start).count();
+
+        std::cout << "[SERIES RESULT] Matrix " << N << "x" << N
+                  << ", iterations = " << iterations
+                  << ", total GPU time = " << total_ms << " ms\n";
     }
 
+    // ----------------------------------
+    // Cleanup
+    // ----------------------------------
     checkCU(cuStreamDestroy(stream), "cuStreamDestroy");
     checkCU(cuModuleUnload(module), "cuModuleUnload");
     checkCU(cuCtxDestroy(ctx), "cuCtxDestroy");
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
index 3f7472854b..17758da4d5 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
@@ -18,6 +18,23 @@ using namespace caf;
 using namespace std::chrono_literals;
 
 
+void serial_matrix_multiply(const std::vector<int>& a,
+                            const std::vector<int>& b,
+                            std::vector<int>& c,
+                            int N) {
+  
+
+ for (int i = 0; i < N; ++i) {
+    for (int j = 0; j < N; ++j) {
+      int sum = 0;
+      for (int k = 0; k < N; ++k) {
+        sum += a[i * N + k] * b[k * N + j];
+      }
+      c[i * N + j] = sum;
+    }
+  }
+}
+
 using command =
   caf::cuda::command_runner<>;
 
@@ -25,67 +42,80 @@ command mmul_command;
 
 struct mmul_state {
 	caf::cuda::program_ptr program;
-	// S3 fix: matrices stored in actor state; messages carry only an int trigger,
-	// eliminating the ~8 MB per-message vector copy (80 GB total for 10 k iters).
-	std::vector<int> matrixA;
-	std::vector<int> matrixB;
-	int N = 0;
-	int completed = 0;
-	int total = 0;
-	bool is_warmup = false;
-	std::chrono::steady_clock::time_point start_time;
+	int total_expected = 0;
+	int results_received = 0;
 };
 
-// S3 fix: global output buffer excludes its allocation from timing,
-// consistent with the other benchmarks.
+//global output buffer meant to disclude it from timing
+//the other benchmark test do not include its memory allocations in it 
+//so its only fair that we do not either 
 std::vector<int> matrixC;
 
 
 
 
 caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
-		caf::cuda::program_ptr mmul_kernel,
-		std::vector<int> matrixA_,
-		std::vector<int> matrixB_,
-		int N_,
-		int total_,
-		bool is_warmup_) {
-
-	auto& st = self->state();
-	st.program = mmul_kernel;
-	st.matrixA = std::move(matrixA_);
-	st.matrixB = std::move(matrixB_);
-	st.N = N_;
-	st.total = total_;
-	st.completed = 0;
-	st.is_warmup = is_warmup_;
-
-return {
+		caf::cuda::program_ptr mmul_kernel, int iterations) {
 
-    // S1 fix: actor receives a trigger (int) instead of full matrix vectors.
-    // It self-quits when all iterations are processed — no external kill needed.
-    [=](int /*trigger*/) {
-      auto& st = self->state();
+	self ->state().program = mmul_kernel;
+	self ->state().total_expected = iterations;
 
-      // Record start time on the first iteration
-      if (st.completed == 0) {
-        st.start_time = std::chrono::steady_clock::now();
-      }
+return {
+    [=](const std::vector<int>& matrixA,
+        const std::vector<int>& matrixB,
+        int N) {
 
       using clock = std::chrono::steady_clock;
       using ms = std::chrono::duration<double, std::milli>;
 
+
       caf::cuda::manager& mgr = caf::cuda::manager::get();
       int device = 0;
       int stream = 1;
-      const int N = st.N;
 
-      auto inA = caf::cuda::create_in_arg(st.matrixA);
-      auto arg1 = mmul_command.transfer_memory(device, stream, std::move(inA));
+           auto t_total_start = clock::now();
+      // -------------------------
+      // create_in_arg A
+      // -------------------------
+      auto t_a_inarg_start = clock::now();
+
+      auto inA = caf::cuda::create_in_arg(std::move(matrixA));
 
-      auto inB = caf::cuda::create_in_arg(st.matrixB);
-      auto arg2 = mmul_command.transfer_memory(device, stream, std::move(inB));
+      auto t_a_inarg_end = clock::now();
 
+      // -------------------------
+      // transfer A
+      // -------------------------
+      auto t_a_transfer_start = clock::now();
+
+      auto arg1 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inA));
+
+      auto t_a_transfer_end = clock::now();
+
+      // -------------------------
+      // create_in_arg B
+      // -------------------------
+      auto t_b_inarg_start = clock::now();
+
+      auto inB = caf::cuda::create_in_arg(std::move(matrixB));
+
+      auto t_b_inarg_end = clock::now();
+
+      // -------------------------
+      // transfer B
+      // -------------------------
+      auto t_b_transfer_start = clock::now();
+
+      auto arg2 = mmul_command.transfer_memory(
+          device,
+          stream,
+          std::move(inB));
+
+      auto t_b_transfer_end = clock::now();
+  
       const int THREADS = 32;
       const int BLOCKS = (N + THREADS - 1) / THREADS;
 
@@ -93,90 +123,116 @@ return {
         BLOCKS, BLOCKS, 1,
         THREADS, THREADS, 1);
 
-      caf::cuda::mmul_async_command<int> cmd;
-      auto output = cmd.run_async(
-		      st.program,
+      caf::cuda::mmul_async_command<int> command;
+      auto output = command.run_async(
+		      self->state().program,
 		      dims,
 		      1, 0, device,
-		      arg1, arg2, out<int>{N * N}, in<int>{N});
+		      arg1,arg2,out<int>{N*N},in<int>{N});
 
 	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);
-	    dC->copy_to_host(matrixC.data(), N * N);
-
-      st.completed++;
 
-      // S6 fix: milestone reporting every 1000 completions
-      if (!st.is_warmup && st.completed % 1000 == 0) {
-        auto now = clock::now();
-        double elapsed = ms(now - st.start_time).count();
-        std::cout << "[MILESTONE] " << st.completed << " / " << st.total
-                  << " iterations, elapsed = " << elapsed << " ms\n";
-      }
-
-      if (st.completed == st.total) {
-        if (!st.is_warmup) {
-          auto end = clock::now();
-          double total_ms = ms(end - st.start_time).count();
-          // S5 fix: standardised tag matches cuda_native and actor_facade output
-          std::cout << "[SERIES RESULT] Matrix " << N << "x" << N
-                    << ", iterations = " << st.total
-                    << ", total time = " << total_ms << " ms\n";
+            auto self_hdl = caf::actor_cast<caf::actor>(self);
+            mmul_command.copy_to_host_async(dC, matrixC.data(), N*N, [self_hdl](int*, size_t) {
+                caf::anon_mail(kernel_done_atom_v).send(self_hdl);
+            });
+    },
+    [=](kernel_done_atom) {
+        if (++self->state().results_received == self->state().total_expected) {
+            self->quit();
         }
-        // S1 fix: graceful self-quit — all prior mailbox messages already processed
-        self->quit();
-      }
     }
-
   };
 }
 
 
-// S4 fix: program loaded once in caf_main and passed in; manager::init/shutdown
-// called once rather than once-per-series.
-void run_mmul_test(caf::actor_system& sys,
-                   caf::cuda::program_ptr program,
-                   int matrix_size, int iterations,
-                   bool is_warmup = false) {
+void run_mmul_test(caf::actor_system& sys, int matrix_size,int iterations) {
+
+
+  caf::cuda::manager::init(sys);
+  // ------------------------------------
+  // Start timing
+  // ------------------------------------
+
+  // Spawn num_actors actors running the mmul behavior
+  std::vector<int> matrixA(matrix_size * matrix_size,2);
+  std::vector<int> matrixB(matrix_size * matrix_size,3);
+
+  matrixC.resize(matrix_size*matrix_size);
+ 
+  auto& mgr = caf::cuda::manager::get();
+  
+  auto program =
+        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
 
-  std::vector<int> matA(matrix_size * matrix_size, 2);
-  std::vector<int> matB(matrix_size * matrix_size, 3);
-  matrixC.resize(matrix_size * matrix_size);
+  using clock = std::chrono::steady_clock;
 
-  // S3 fix: spawn actor with matrices in state; send trigger-only int messages
-  caf::actor a = sys.spawn(mmul_actor_fun, program,
-                            std::move(matA), std::move(matB),
-                            matrix_size, iterations, is_warmup);
+  auto start = std::chrono::steady_clock::now();
 
-  // S3 fix: int triggers carry no matrix data — no OOM risk regardless of iteration count
-  // S1 fix: actor self-quits on completion; no anon_send_exit(kill) needed
-  for (int i = 0; i < iterations; i++)
-	  anon_mail(i).send(a);
+  caf::actor a = sys.spawn(mmul_actor_fun, program, iterations);
 
+  for (int i = 0; i < iterations; i++) 
+	  anon_mail(matrixA,matrixB,matrix_size).send(a);
+
+  // Wait for all actors to finish
   sys.await_all_actors_done();
+
+  // ------------------------------------
+  // Stop timing
+  // ------------------------------------
+  auto end = std::chrono::steady_clock::now();
+  auto duration_ms =
+      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
+            << " iterations = " << iterations << 
+	    ", time=" << duration_ms << " ms\n";
+
+  caf::cuda::manager::shutdown();
+
 }
 
 
 void caf_main(caf::actor_system& sys) {
-  constexpr int matrix_size = 1000;
-  constexpr int total_iterations = 10000;
 
-  caf::cuda::manager::init(sys);  // S4 fix: init once before all series
+	for (int i = 1000; i < 11000; i+=1000)
+		run_mmul_test(sys,1000,i);
 
-  auto program = caf::cuda::manager::get()
-                     .create_program_from_cubin("mmul.cubin", "matrixMul");
+}
 
-  // S2 fix: warmup run to prime CUDA context before timed series
-  std::cout << "--- warmup starting ---\n";
-  run_mmul_test(sys, program, matrix_size, 10, /*is_warmup=*/true);
-  std::cout << "--- warmup complete ---\n";
 
-  run_mmul_test(sys, program, matrix_size, total_iterations);
+int main(int argc, char** argv) {
+  // Initialize user defined types and messages if needed.
+  //init_global_meta_objects<custom_types_1>();
+  
+  // Initialize the global type information.
+  core::init_global_meta_objects();
 
-  caf::cuda::manager::shutdown();  // S4 fix: shutdown once after all series
+  // Create the config.
+  actor_system_config cfg;
 
-}
+  // --- SINGLE THREAD CONFIGURATION ---
+  cfg.set("caf.scheduler.max-threads", 1);
+  cfg.set("caf.scheduler.policy", "sharing"); 
+  // ------------------------------------
 
+  // Read CLI options. (Note: CLI flags like --caf.scheduler.max-threads=4 
+  // will override the hardcoded '1' above if provided by the user).
+  auto err = cfg.parse(argc, argv);
+  if (err)
+    return EXIT_FAILURE;
 
+  if (cfg.helptext_printed())
+    return 0;
 
+  // Create the actor system (the scheduler starts here).
+  actor_system sys{cfg};
+
+  // Run user-defined code.
+  caf_main(sys);
+  
+  return 0;
+}
 
-CAF_MAIN()
+// CAF_MAIN()

From 96e84340e3ada7067aa31d09ad9b4e598c314c8f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 12:23:23 -0600
Subject: [PATCH 0795/1000] updated test

---
 .../Batched-Matrix-Multiply/main.test.cpp     | 1001 +++++------------
 1 file changed, 286 insertions(+), 715 deletions(-)

diff --git a/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp b/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp
index 663434ab63..271464aff7 100644
--- a/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp
+++ b/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp
@@ -10,44 +10,30 @@
 #include <numeric>
 #include <random>
 #include "caf/actor_registry.hpp"
-#include <random>
+#include <deque>
+#include <unordered_map>
 #include <unordered_set>
 //#include <caf/atoms.hpp>
 
-
-
 using namespace caf;
 using namespace std::chrono_literals;
 
-
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-  
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
-    }
-  }
-}
-
-using command =
-  caf::cuda::command_runner<>;
-
-command mmul_command;
-caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int >> mmul;
-using async_command = caf::cuda::mmul_async_command<int>;
-async_command async_mmul;
-
-
-
-
+// ─────────────────────────────────────────────────────────────────────────────
+// Atoms
+// ─────────────────────────────────────────────────────────────────────────────
+CAF_BEGIN_TYPE_ID_BLOCK(mmul_benchmark, caf::id_block::cuda::end)
+    CAF_ADD_ATOM(mmul_benchmark, get_work_atom)
+    CAF_ADD_ATOM(mmul_benchmark, task_done_atom)
+    CAF_ADD_ATOM(mmul_benchmark, release_memory_atom)
+    CAF_ADD_ATOM(mmul_benchmark, request_work_atom)
+    CAF_ADD_ATOM(mmul_benchmark, worker_done_atom)
+    CAF_ADD_ATOM(mmul_benchmark, refill_buffer_atom)
+CAF_END_TYPE_ID_BLOCK(mmul_benchmark)
+
+// Command runners for GPU operations
+caf::cuda::command_runner<> mmul_command;
+using mmul_kernel_t = caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>, out<int>, in<int>>;
+mmul_kernel_t mmul_kernel;
 
 struct MatrixPool {
     std::unordered_map<int, std::vector<int>> A;
@@ -80,554 +66,282 @@ MatrixPool create_matrix_pool_random(
     return pool;
 }
 
-
-
-
-
-
-struct mmul_state {
-
-	caf::cuda::program_ptr mmul_kernel;
-
+// ---------------------------- GLOBAL TASK POOL ----------------------------
+// The central source of truth for work. Implements a pull-based model.
+struct task_pool_state {
+    std::vector<int> tasks;
+    size_t next_task_idx = 0;
 };
 
-
-
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_state>* self,
-		caf::actor exit_actor,
-		caf::cuda::program_ptr program,
-		caf::cuda::nd_range dims,
-		int stream,
-		int N,
-		const in<int> matrixA,
-		const in<int> matrixB
-		) {
-
-
-	int device = stream % caf::cuda::manager::get().get_num_devices();
-	self->mail(N).send(self);
-
-	return {
-
-		[=](int N) {
-
-			auto total_start = std::chrono::steady_clock::now();
-
-
-//			std::cout << "device=" << device <<  "\n";
-//			std::cout << "N=" << N <<  "\n";
-			// ---------------- H2D ----------------
-			auto h2d_start = std::chrono::steady_clock::now();
-
-			auto arg1 = mmul_command.transfer_memory(device, stream, std::move(matrixA));
-			auto arg2 = mmul_command.transfer_memory(device, stream, std::move(matrixB));
-
-
-			// ---------------- Kernel ----------------
-			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
-			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
-
-			auto h2d_end = std::chrono::steady_clock::now();
-			auto kernel_start = std::chrono::steady_clock::now();
-
-			auto result = async_mmul.run_async(
-					program, dims, stream, 0, device,
-					arg1, arg2, arg3, arg4);
-
-			//std::get<2>(result)->synchronize();
-
-			auto kernel_end = std::chrono::steady_clock::now();
-
-			// ---------------- D2H ----------------
-			auto d2h_start = std::chrono::steady_clock::now();
-
-			std::get<2>(result)->copy_to_host();
-
-			auto d2h_end = std::chrono::steady_clock::now();
-
-			auto total_end = std::chrono::steady_clock::now();
-
-			/*
-			// ---------------- COMPUTE ----------------
-			auto h2d = std::chrono::duration<double>(h2d_end - h2d_start).count();
-			auto kernel = std::chrono::duration<double>(kernel_end - kernel_start).count();
-			auto d2h = std::chrono::duration<double>(d2h_end - d2h_start).count();
-			auto total = std::chrono::duration<double>(total_end - total_start).count();
-
-			// ---------------- PRINT ----------------
-			
-			std::cout << "\n[NO SCHEDULER] N=" << N << "\n";
-			std::cout << "H2D:    " << h2d * 1000 << " ms\n";
-			std::cout << "Kernel: " << kernel * 1000 << " ms\n";
-			std::cout << "D2H:    " << d2h * 1000 << " ms\n";
-			std::cout << "TOTAL:  " << total * 1000 << " ms\n";
-			std::cout << "SUM:    " << (h2d + kernel + d2h) * 1000 << " ms\n";
-
-			*/
-			self->mail(1).send(exit_actor);
-			self->quit();
-		}
-	};
+caf::behavior global_task_pool(caf::stateful_actor<task_pool_state>* self, std::vector<int> tasks) {
+    self->state().tasks = std::move(tasks);
+    return {
+        [=](get_work_atom, size_t batch_size) -> result<std::vector<int>> {
+            auto& st = self->state();
+            if (st.next_task_idx >= st.tasks.size())
+                return sec::end_of_stream;
+            size_t count = std::min(batch_size, st.tasks.size() - st.next_task_idx);
+            std::vector<int> batch(st.tasks.begin() + st.next_task_idx, 
+                                   st.tasks.begin() + st.next_task_idx + count);
+            st.next_task_idx += count;
+            return batch;
+        }
+    };
 }
 
-
-
-struct mmul_actor_with_scheduler_state {
-  static inline const char* name = "my_actor";
+// ---------------------------- DEVICE/GPU ACTOR ----------------------------
+// Manages memory for a specific GPU and steals (pulls) work from the Global Pool.
+struct device_actor_state {
+    MatrixPool pool;
+    caf::actor global_pool;
+    std::deque<int> local_tasks; // Local buffer to keep GPU busy
+    size_t total_device_memory_bytes = 0;
+    size_t current_allocated_memory_bytes = 0;
+    int active_workers = 0;
+    int device_id = -1;
+    size_t batch_size = 0;
+    size_t low_water_mark = 0;
+    bool fetching = false;
 };
 
+caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
+                               MatrixPool pool, caf::actor global_pool, int num_workers, int dev_id, int max_in_flight) {
+    self->state().pool = std::move(pool);
+    self->state().global_pool = global_pool;
+    self->state().device_id = dev_id;
+    self->state().active_workers = num_workers;
 
-// Stateful actor behavior
-caf::behavior mmul_actor_fun_scheduler(
-    caf::stateful_actor<mmul_actor_with_scheduler_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims,
-    const in<int> matrixA,
-    const in<int> matrixB)
-{
-
-
-
-        caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-        caf::actor scheduler = mgr.get_scheduler_actor();
-
-        //send a launch token
-        caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-                        program,
-                        dims,
-                        0,
-                        "hello",
-                        self
-                        );
-        mgr.send_scheduler_actor_message(launch_token);
-
-	return {
-
-		// 1. Handle response token
-		[=](caf::cuda::response_token_ptr res_token) {
-			//        std::cout << "Got response\n";
-
-			if (res_token->getType() == LAUNCH_RESPONSE) {
-				self->mail(res_token, N).send(self);
-
-			} else {
-				//          std::cout << "Got a memory response token\n";
-			}
-		},
-
-			// 2. Handle memory buffers -> GPU
-			[=](const caf::cuda::response_token_ptr& res_token, int N) {
-
-				auto total_start = std::chrono::steady_clock::now();
-
-				// ---------------- H2D ----------------
-				auto h2d_start = std::chrono::steady_clock::now();
-
-				auto arg1 = mmul.transfer_memory(res_token -> getDeviceNumber(),res_token -> getStreamId(), std::move(matrixA));
-				auto arg2 = mmul.transfer_memory(res_token -> getDeviceNumber(), res_token -> getStreamId(), std::move(matrixB));
-				//auto arg3 = mmul.transfer_memory(res_token, caf::cuda::create_out_arg(N*N));
-				//auto arg4 = mmul.transfer_memory(res_token, caf::cuda::create_in_arg(N));
-
-				std::cout << "res_token did = " << res_token -> getDeviceNumber() << "\n";
-				std::cout << "N = " << N << "\n";
-				 out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
-	 			 in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
-
-				auto h2d_end = std::chrono::steady_clock::now();
-
-				// ---------------- Kernel ----------------
-				auto kernel_start = std::chrono::steady_clock::now();
-
-
-
-				auto tempC = async_mmul.run_async(program, dims, res_token, arg1, arg2, arg3, arg4);
-				auto bufferC = std::get<2>(tempC);
-
-				//bufferC->synchronize();
-
-				auto kernel_end = std::chrono::steady_clock::now();
-
-				// ---------------- D2H ----------------
-				auto d2h_start = std::chrono::steady_clock::now();
-
-				bufferC->copy_to_host();
-
-				auto d2h_end = std::chrono::steady_clock::now();
-
-				res_token->release();
-
-				auto total_end = std::chrono::steady_clock::now();
-
-				// ---------------- COMPUTE ----------------
-				/*
-				auto h2d = std::chrono::duration<double>(h2d_end - h2d_start).count();
-				auto kernel = std::chrono::duration<double>(kernel_end - kernel_start).count();
-				auto d2h = std::chrono::duration<double>(d2h_end - d2h_start).count();
-				auto total = std::chrono::duration<double>(total_end - total_start).count();
-
-				std::cout << "H2D:    " << h2d * 1000 << " ms\n";
-				std::cout << "Kernel: " << kernel * 1000 << " ms\n";
-				std::cout << "D2H:    " << d2h * 1000 << " ms\n";
-				std::cout << "TOTAL:  " << total * 1000 << " ms\n";
-				std::cout << "SUM:    " << (h2d + kernel + d2h) * 1000 << " ms\n\n";
-				*/
-
-				self->mail(1).send(exit_actor);
-				self->quit();
-			}
-
-
-       };
-
-}
-
-
-
-
-
-caf::behavior mmul_actor_fun_scheduler2(caf::stateful_actor<mmul_state>* self,
-		caf::actor exit_actor,
-		caf::actor scheduler_actor,
-		caf::cuda::program_ptr program,
-		caf::cuda::nd_range dims,
-		int stream,
-		int N,
-		const in<int> matrixA,
-		const in<int> matrixB
-		) {
-
-
-	int device = stream % caf::cuda::manager::get().get_num_devices();
-
-	//self->mail("subscribe",self).send(scheduler_actor);
-
-	self->mail(N).send(self);
-
-	return {
-
-		//message from the new scheduler actor 
-		[=](std::vector<int> costs) {
-			//do nothing this is an overhead test 
-		
-			//std::cout << "N=" << N <<  "\n";
-		
-		},
-		[=](int N) {
-
-			auto total_start = std::chrono::steady_clock::now();
-
-			//int device = rand() % caf::cuda::manager::get().get_num_devices();
-			//int stream = rand();
-
-			//std::cout << "device=" << device <<  "\n";
-			//std::cout << "N=" << N <<  "\n";
-
-
-
-			//declare the cost of doing work to the scheduler actor, for now we can impose
-			//a heuristic of just N, the size of the matrix
-			self->mail("add",device,N).send(scheduler_actor);
-
-
-			// ---------------- H2D ----------------
-			auto h2d_start = std::chrono::steady_clock::now();
-
-
-			auto arg1 = mmul_command.transfer_memory(device, stream, std::move(matrixA));
-			auto arg2 = mmul_command.transfer_memory(device, stream, std::move(matrixB));
-
-
-			// ---------------- Kernel ----------------
-			out<int> arg3 = caf::cuda::create_out_arg<int>(N * N);
-			in<int>  arg4 = caf::cuda::create_in_arg<int>(N);
-
-			auto h2d_end = std::chrono::steady_clock::now();
-			auto kernel_start = std::chrono::steady_clock::now();
-
-			auto result = async_mmul.run_async(
-					program, dims, stream, 0, device,
-					arg1, arg2, arg3, arg4);
-
-			//std::get<2>(result)->synchronize();
-
-			auto kernel_end = std::chrono::steady_clock::now();
-
-			// ---------------- D2H ----------------
-			auto d2h_start = std::chrono::steady_clock::now();
-
-			std::get<2>(result)->copy_to_host();
-
-
-
-			//likewise tell the scheduler we are done doing work
-			self->mail("subtract",device,N).send(scheduler_actor);
-
-			auto d2h_end = std::chrono::steady_clock::now();
+    // Dynamically calculate prefetch markers based on the total pipeline capacity
+    self->state().low_water_mark = static_cast<size_t>(num_workers * max_in_flight);
+    self->state().batch_size = self->state().low_water_mark * 2;
 
-			auto total_end = std::chrono::steady_clock::now();
+    caf::cuda::manager& mgr = caf::cuda::manager::get();
+    caf::cuda::device_ptr dev_obj = mgr.find_device(dev_id);
+    if (dev_obj) {
+        self->state().total_device_memory_bytes = dev_obj->total_memory_bytes();
+    }
 
-			/*
-			// ---------------- COMPUTE ----------------
-			auto h2d = std::chrono::duration<double>(h2d_end - h2d_start).count();
-			auto kernel = std::chrono::duration<double>(kernel_end - kernel_start).count();
-			auto d2h = std::chrono::duration<double>(d2h_end - d2h_start).count();
-			auto total = std::chrono::duration<double>(total_end - total_start).count();
+    // Helper to refill the local task buffer from the global pool
+    auto refill = [=]() {
+        auto& st = self->state();
+        if (st.fetching || st.local_tasks.size() >= st.low_water_mark + st.batch_size)
+            return;
+
+        st.fetching = true;
+        self->mail(get_work_atom_v, (size_t)st.batch_size).request(st.global_pool, infinite).then(
+            [=](std::vector<int>& batch) {
+                auto& st_inner = self->state();
+                for (int N : batch)
+                    st_inner.local_tasks.push_back(N);
+                st_inner.fetching = false;
+                if (st_inner.local_tasks.size() < st_inner.low_water_mark)
+                    self->mail(refill_buffer_atom_v).send(self);
+            },
+            [=](error& err) {
+                self->state().fetching = false;
+            }
+        );
+    };
 
-			// ---------------- PRINT ----------------
-			
-			std::cout << "\n[NO SCHEDULER] N=" << N << "\n";
-			std::cout << "H2D:    " << h2d * 1000 << " ms\n";
-			std::cout << "Kernel: " << kernel * 1000 << " ms\n";
-			std::cout << "D2H:    " << d2h * 1000 << " ms\n";
-			std::cout << "TOTAL:  " << total * 1000 << " ms\n";
-			std::cout << "SUM:    " << (h2d + kernel + d2h) * 1000 << " ms\n";
+    return {
+        [=](refill_buffer_atom) {
+            refill();
+        },
+        [=](get_work_atom) -> caf::result<int, in<int>, in<int>> {
+            auto& st = self->state();
+
+            // If we have tasks locally, satisfy the request immediately
+            if (!st.local_tasks.empty()) {
+                int N = st.local_tasks.front();
+                size_t memory_needed = (size_t)N * N * sizeof(int) * 3;
+                if (st.current_allocated_memory_bytes + memory_needed > st.total_device_memory_bytes)
+                    return make_error(sec::runtime_error, "Device Actor: Not enough memory");
+                
+                st.local_tasks.pop_front();
+                st.current_allocated_memory_bytes += memory_needed;
+                
+                // Proactively steal more work if the buffer is getting low
+                if (st.local_tasks.size() < st.low_water_mark)
+                    refill();
+
+                return {N, caf::cuda::create_in_arg(st.pool.A[N]), 
+                           caf::cuda::create_in_arg(st.pool.B[N])};
+            }
 
-			*/
-			self->mail(1).send(exit_actor);
-			self->mail("unsubscribe",self).send(scheduler_actor);
-			self->quit();
-		}
-	};
+            // Buffer empty: must fetch from global pool reactively
+            auto promise = self->make_response_promise<int, in<int>, in<int>>();
+            self->mail(get_work_atom_v, (size_t)st.batch_size).request(st.global_pool, infinite).then(
+                [=](std::vector<int>& batch) mutable {
+                    auto& st_inner = self->state();
+                    int N = batch.front();
+                    for(size_t i = 1; i < batch.size(); ++i) st_inner.local_tasks.push_back(batch[i]);
+                    
+                    size_t needed = (size_t)N * N * sizeof(int) * 3;
+                    st_inner.current_allocated_memory_bytes += needed;
+                    promise.deliver(N, caf::cuda::create_in_arg(st_inner.pool.A[N]), 
+                                       caf::cuda::create_in_arg(st_inner.pool.B[N]));
+                },
+                [=](error& err) mutable { promise.deliver(err); }
+            );
+            return promise;
+        },
+        [=](release_memory_atom, int N_completed) {
+            auto& st = self->state();
+            size_t memory_released = (size_t)N_completed * N_completed * sizeof(int) * 3;
+            st.current_allocated_memory_bytes -= memory_released;
+            refill(); // Try to get more work now that memory is free
+        },
+        [=](worker_done_atom) {
+            auto& st = self->state();
+            if (--st.active_workers <= 0) {
+                self->quit();
+            }
+        }
+    };
 }
 
-
-
-
-
-
-struct scheduler_actor_state {
-
-	std::vector<caf::actor> subscribers;
-	int num_devices;
-	std::vector<int> costs;
+// ---------------------------- WORKER ACTOR ----------------------------
+// Manages 1 stream and pulls work from the Device Actor.
+struct worker_state {
+    int device_id;
+    int stream_id;
+    caf::cuda::program_ptr program;
+    caf::actor device_actor;
+    caf::actor supervisor;
+    int max_in_flight_tasks;
+    int in_flight_tasks_count = 0;
+    bool draining = false;
 };
 
+caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
+                              caf::actor supervisor, caf::actor device_actor, caf::cuda::program_ptr program,
+                              int dev_id, int stream_id, int max_in_flight_tasks) {
+    self->state().supervisor = supervisor;
+    self->state().device_actor = device_actor;
+    self->state().program = program;
+    self->state().device_id = dev_id;
+    self->state().stream_id = stream_id;
+    self->state().max_in_flight_tasks = max_in_flight_tasks;
+
+    // Trigger initial work requests up to max_in_flight_tasks
+    for (int i = 0; i < max_in_flight_tasks; ++i) {
+        self->mail(request_work_atom_v).send(self);
+    }
 
+    return {
+        [=](request_work_atom) {
+            auto& st = self->state();
+            if (st.in_flight_tasks_count >= st.max_in_flight_tasks || st.draining) {
+                return; // Already at max capacity, don't request more yet
+            }
 
-caf::behavior scheduler_actor_fun(caf::stateful_actor<scheduler_actor_state>* self) {
-
-	self->state().num_devices = caf::cuda::manager::get().get_num_devices();
-	self->state().costs.resize(self->state().num_devices);
-
-	int time = 50;
-
-	self->mail("publish").urgent().delay(std::chrono::milliseconds(time)).send(self);
-
-	return {
-
-		[=](std::string command,int device, int cost) {
-
-			if (command == "add") {
-
-				self->state().costs[device] +=cost;
-
-			}
-			else if (command == "subtract") {
-
-				int value = std::min(self->state().costs[device] - cost,0);
-				self->state().costs[device] = value;	
-			}
-
-
-		},
-			[=](std::string command, caf::actor actor) {
-
-				auto& subs = self->state().subscribers;
-
-				if (command == "subscribe") {
-					//std::cout << "Thank you for subscribing\n";
-					// avoid duplicates
-					if (std::find(subs.begin(), subs.end(), actor) == subs.end()) {
-						subs.push_back(actor);
-						//self->monitor(actor); // track lifecycle
-					}
-				}
-
-				else if (command == "unsubscribe") {
-					subs.erase(
-							std::remove(subs.begin(), subs.end(), actor),
-							subs.end()
-						  );
-					//self->demonitor(actor);
-				}
-			},
-
-			[=](std::string command) {
-				if (command == "publish") {
-					//std::cout << "size = " << self->state().subscribers.size() << "\n";
-					for (caf::actor a : self->state().subscribers) {
-
-						self -> mail(self->state().costs).urgent().send(a);
-
-					}
-					self->mail("publish").urgent().delay(std::chrono::milliseconds(time)).send(self);
-				}
-
-			}	
-
-	};
+            st.in_flight_tasks_count++; // Mark as pending immediately
+            self->mail(get_work_atom_v).request(st.device_actor, infinite).then(
+                [=](int N, in<int> matrixA, in<int> matrixB) {
+                    // GPU Pipeline: Transfer -> Kernel -> Copyback
+                    auto arg1 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixA));
+                    auto arg2 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixB));
+
+                    const int THREADS = 32;
+                    const int BLOCKS = (N + THREADS - 1) / THREADS;
+                    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+
+                    auto result = mmul_kernel.run_async(st.program, dims, st.stream_id, 0, st.device_id,
+                                                       arg1, arg2,
+                                                       caf::cuda::create_out_arg<int>(N * N),
+                                                       caf::cuda::create_in_arg<int>(N));
+
+                    auto bufferC = std::get<2>(result);
+                    auto self_hdl = caf::actor_cast<caf::actor>(self);
+
+                    mmul_command.copy_to_host_async(bufferC, [self_hdl, N_task = N](std::vector<int>&&) {
+                        caf::anon_mail(task_done_atom_v, N_task).send(self_hdl); // Pass N back to self
+                    });
+                },
+                [=](error& err) {
+                    auto& st = self->state();
+                    st.in_flight_tasks_count--; // Revert pending status on failure
+                    if (err == sec::runtime_error) {
+                        // Not enough memory, retry after a delay
+                        self->println("Worker {}: Not enough memory, retrying for work...", st.stream_id);
+                        self->delayed_anon_send(self, 100ms, request_work_atom_v);
+                    } else if (err == sec::end_of_stream) {
+                        st.draining = true; // Mark as draining, let in-flight finish
+                        if (st.in_flight_tasks_count == 0) {
+                            self->mail(worker_done_atom_v).send(st.device_actor);
+                            mmul_command.release_stream_for_actor(st.stream_id);
+                            self->quit();
+                        }
+                    }
+                }
+            );
+        },
+        [=](task_done_atom, int N_completed) {
+            auto& st = self->state();
+            st.in_flight_tasks_count--; // Decrement count
+            self->mail(1).send(st.supervisor); // Notify supervisor
+            self->mail(release_memory_atom_v, N_completed).send(st.device_actor); // Release memory
+            
+            if (st.draining && st.in_flight_tasks_count == 0) {
+                self->mail(worker_done_atom_v).send(st.device_actor);
+                mmul_command.release_stream_for_actor(st.stream_id);
+                self->quit();
+            } else if (!st.draining) {
+                self->mail(request_work_atom_v).send(self); // Request next task if capacity allows
+            }
+        }
+    };
 }
 
-
-
 // ---------------------------- SUPERVISOR ACTOR ----------------------------
 struct supervisor_actor_state {
-    int num_actors;
-    int num_waves;
-    int completed;
-    int max_waves;
-
-    MatrixPool pool;
-
-    // Precomputed sequence of N values
-    std::vector<int> Ns;
-    int next_task;
-
-    // Timing
+    int total_tasks;
+    int completed = 0;
     std::chrono::steady_clock::time_point start_time;
-    std::chrono::steady_clock::time_point wave_start_time;
 };
 
 caf::behavior supervisor_actor_fun(
     caf::stateful_actor<supervisor_actor_state>* self,
-    int num_actors,
-    int max_waves,
+    int total_tasks,
+    int workers_per_gpu,
+    int max_in_flight_tasks_per_worker,
     MatrixPool pool,
-    const std::vector<int>& Ns,      // deterministic task sizes 
-    bool use_scheduler
+    std::vector<int> Ns
     ) {
-    // Initialize state
-    self->state().num_actors = num_actors;
-    self->state().completed = 0;
-    self->state().max_waves = max_waves;
-    self->state().num_waves = 0;
-    self->state().pool = std::move(pool);
-
-    self->state().Ns = Ns;
-    self->state().next_task = 0;
+    self->state().total_tasks = total_tasks;
+    self->state().start_time = std::chrono::steady_clock::now();
 
+    auto pool_actor = self->spawn(global_task_pool, std::move(Ns));
 
     caf::cuda::manager& mgr = caf::cuda::manager::get();
+    int num_gpus = mgr.get_num_devices();
     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
 
-    // Kick off first wave
-    self->mail("spawn").send(self);
-
-    caf::actor scheduler_actor = self->spawn(scheduler_actor_fun);
-
-    // Start timing
-    self->state().start_time = std::chrono::steady_clock::now();
-    return {
-        // -------------------- SPAWN WAVE --------------------
-        [=](std::string cmd) {
-            if (cmd != "spawn") return;
-
-	     std::chrono::steady_clock::time_point cmd_start_time = std::chrono::steady_clock::now();
-	    
-
-            self->state().completed = 0;
-            self->state().wave_start_time = std::chrono::steady_clock::now();
-
-            for (int i = 0; i < self->state().num_actors; ++i) {
-                if (self->state().next_task >= self->state().Ns.size())
-                    break;
-
-                int N = self->state().Ns[self->state().next_task++];
-
-
-                const auto& A = self->state().pool.A[N];
-                const auto& B = self->state().pool.B[N];
-
-                const int THREADS = 32;
-                const int BLOCKS = (N + THREADS - 1) / THREADS;
-
-                caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-		if (use_scheduler) {
-			caf::actor a = 	self->spawn(mmul_actor_fun_scheduler2, 
-					self,
-					scheduler_actor,
-				       	program, 
-					dims,
-					i,
-					N,
-                            caf::cuda::create_in_arg(A),
-                            caf::cuda::create_in_arg(B));
-
-
-		 self->mail("subscribe",a).send(scheduler_actor);
-		}
-		else {
-			self->spawn(mmul_actor_fun, self, program, dims,i,N,
-                            caf::cuda::create_in_arg(A),
-                            caf::cuda::create_in_arg(B));
-		}
-            }
+    for (int i = 0; i < num_gpus; ++i) {
+        auto broker = self->spawn(gpu_device_actor, pool, pool_actor, workers_per_gpu, i, max_in_flight_tasks_per_worker);
         
-	
-
-    std::chrono::steady_clock::time_point cmd_end_time = std::chrono::steady_clock::now();
-
-      std::chrono::duration<double> total_time =
-                        cmd_end_time - cmd_start_time;
-
-                    std::cout << "\n===== SUPERVISOR TOTAL TIME spawn =====\n";
-                    std::cout << "Total runtime: "
-                              << total_time.count() << " s\n";
-
-
-	
-	},
+        for (int j = 0; j < workers_per_gpu; ++j)
+            self->spawn(mmul_worker_fun, self, broker, program, i, (i * 1000) + j, max_in_flight_tasks_per_worker);
+    }
 
-        // -------------------- COMPLETION TRACKING --------------------
+    return {
         [=](int done) {
             self->state().completed += done;
-
-            if (self->state().completed >= self->state().num_actors) {
-                auto wave_end = std::chrono::steady_clock::now();
-                std::chrono::duration<double> wave_time =
-                    wave_end - self->state().wave_start_time;
-
-                self->state().num_waves++;
-                //std::cout << "Wave "
-                  //        << self->state().num_waves
-                    //      << " completed in "
-                      //    << wave_time.count() << " s\n";
-
-                if (self->state().num_waves >= self->state().max_waves) {
-                    auto end_time = std::chrono::steady_clock::now();
-                    std::chrono::duration<double> total_time =
-                        end_time - self->state().start_time;
-
-                    std::cout << "\n===== SUPERVISOR TOTAL TIME =====\n";
-                    std::cout << "Total runtime: "
-                              << total_time.count() << " s\n";
-
-
-		    anon_send_exit(
-				    scheduler_actor,
-				    caf::exit_reason::user_shutdown
-				  );
-		    caf::cuda::manager::shutdown();
-                    self->quit();
-                } else {
-                    self->mail("spawn").send(self);
-                }
+            if (self->state().completed >= self->state().total_tasks) {
+                auto end_time = std::chrono::steady_clock::now();
+                std::chrono::duration<double> total_time = end_time - self->state().start_time;
+
+                std::cout << "\n===== BENCHMARK COMPLETE =====\n";
+                std::cout << "Tasks: " << self->state().total_tasks << "\n";
+                std::cout << "Runtime: " << total_time.count() << " s\n";
+                
+                caf::cuda::manager::shutdown();
+                self->quit();
             }
         }
     };
 }
 
-
-
-
-
-
-
-
 template <class Fn>
 double time_run(Fn&& fn) {
   auto start = std::chrono::steady_clock::now();
@@ -637,8 +351,6 @@ double time_run(Fn&& fn) {
   return elapsed.count();
 }
 
-
-
 void run_mmul_random_scaling_tests(caf::actor_system& sys,
                                   caf::cuda::manager_config man_config) {
 
@@ -646,202 +358,61 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     const int max_N = 2048;
     const int num_sizes = 10;
 
-    const int max_waves = 1;
+    const int workers_per_gpu = 8; // Admission control: only 16 concurrent tasks per GPU
+    const int max_in_flight_tasks_per_worker = 3; // Each worker keeps 2 tasks in flight
 
     const std::vector<int> actor_counts = {
-	  30000,40000,50000
+	  1,30000,40000,50000
     };
 
+    // Generate deterministic random pool once
+    MatrixPool pool = create_matrix_pool_random(
+        num_sizes,
+        min_N,
+        max_N,
+        42  // fixed seed
+    );
 
-    int num_actors = actor_counts[actor_counts.size()-1];
-
-        // Generate deterministic random pool
-        MatrixPool pool = create_matrix_pool_random(
-            num_sizes,
-            min_N,
-            max_N,
-            42  // fixed seed
-        );
-
-    // Precompute all task Ns (total_tasks = num_actors * max_waves)
-    std::vector<int> sizes;
-    for (const auto& [N, _] : pool.A) sizes.push_back(N);
-
-    int total_tasks = num_actors * max_waves;
-    std::vector<int> Ns;
-    Ns.reserve(total_tasks);
-
-    std::mt19937 rng(42);
-    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
-    for (int i = 0; i < total_tasks; ++i)
-	    Ns.push_back(sizes[dist(rng)]);
-
-
-   
     //scheduler
-    for (int num_actors : actor_counts) {
-
-    
+    caf::cuda::manager_config scheduler_off(false);
+    for (int num_tasks_for_this_run : actor_counts) {
 	    // Initialize CUDA manager
-	    caf::cuda::manager::init(sys);
-  std::cout << "=====================================\n";
-        std::cout << "Random Scaling WITH scheduler | actors=" << num_actors << "\n";
+	    caf::cuda::manager::init(sys, scheduler_off);
+        std::cout << "=====================================\n";
+        std::cout << "Random Scaling | actors=" << num_tasks_for_this_run << "\n";
 
+        // Precompute all task Ns for this run
+        std::vector<int> sizes;
+        for (const auto& [N, _] : pool.A) sizes.push_back(N);
 
-        double elapsed = time_run([&]() {
-
-            auto sup = sys.spawn(
-                supervisor_actor_fun,
-                num_actors,
-                max_waves,
-                pool,
-		Ns,
-		true
-            );
-
-        
-	      sys.await_all_actors_done();
-	    
-	    });
+        std::vector<int> Ns_for_this_run;
+        Ns_for_this_run.reserve(num_tasks_for_this_run);
 
-    
-	caf::cuda::manager::shutdown();
-    }
-    
-    //no scheduler
-    for (int num_actors : actor_counts) {
+        std::mt19937 rng(42);
+        std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
+        for (int i = 0; i < num_tasks_for_this_run; ++i)
+	        Ns_for_this_run.push_back(sizes[dist(rng)]);
 
-    
-	    // Initialize CUDA manager
-	    caf::cuda::manager::init(sys);
-	    std::cout << "=====================================\n";
-	    std::cout << "Random Scaling NO scheduler | actors=" << num_actors << "\n";      
-	    double elapsed = time_run([&]() {
+        // Execute the supervisor which manages the asynchronous workload
+        double elapsed = time_run([&]() {
 
             auto sup = sys.spawn(
                 supervisor_actor_fun,
-                num_actors,
-                max_waves,
+                (int)Ns_for_this_run.size(), // total_tasks
+                workers_per_gpu,
+                max_in_flight_tasks_per_worker,
                 pool,
-		Ns,
-		false
+		Ns_for_this_run
             );
 
 	      sys.await_all_actors_done();
-	    
 	    });
 
-    
-	caf::cuda::manager::shutdown();
-    }
-
-
-
-
-    caf::cuda::manager::shutdown();
-}
-
-void run_mmul_uniform_scaling_tests(caf::actor_system& sys,
-                                   caf::cuda::manager_config man_config) {
-
-    const int max_waves = 1;
-
-    // Matrix sizes: 1,2,4,...,2048
-    std::vector<int> matrix_sizes;
-    for (int n = 1; n <= 2048; n *= 2)
-        matrix_sizes.push_back(n);
-
-    // Actor counts: 10 → 1000
-    std::vector<int> actor_counts;
-    for (int a = 10; a <= 1000; a += 10)
-        actor_counts.push_back(a);
-
-    for (int N : matrix_sizes) {
-        for (int num_actors : actor_counts) {
-
-            // Create uniform pool (single size)
-            MatrixPool pool = create_matrix_pool_random(
-                1,
-                N,
-                N,
-                42
-            );
-
-            int total_tasks = num_actors * max_waves;
-
-            std::vector<int> Ns(total_tasks, N);
-
-            // ========================
-            // WITH SCHEDULER
-            // ========================
-            caf::cuda::manager::init(sys);
-            std::cout << "=====================================\n";
-            std::cout << "Uniform WITH scheduler | N=" << N
-                      << " actors=" << num_actors << "\n";
-
-            time_run([&]() {
-                auto sup = sys.spawn(
-                    supervisor_actor_fun,
-                    num_actors,
-                    max_waves,
-                    pool,
-                    Ns,
-                    true
-                );
-                sys.await_all_actors_done();
-            });
-
-            caf::cuda::manager::shutdown();
-
-            // ========================
-            // WITHOUT SCHEDULER
-            // ========================
-            caf::cuda::manager::init(sys);
-            std::cout << "=====================================\n";
-            std::cout << "Uniform NO scheduler | N=" << N
-                      << " actors=" << num_actors << "\n";
-
-            time_run([&]() {
-                auto sup = sys.spawn(
-                    supervisor_actor_fun,
-                    num_actors,
-                    max_waves,
-                    pool,
-                    Ns,
-                    false
-                );
-                sys.await_all_actors_done();
-            });
-
-            caf::cuda::manager::shutdown();
-        }
+        caf::cuda::manager::shutdown();
     }
 }
-
-
-
-
-
-
-
-
-
-
 void caf_main(caf::actor_system& sys) {
-  
-
-  caf::cuda::manager_config man_config(true);
-  //caf::cuda::manager::init(sys,man_config);
-
-
-  run_mmul_random_scaling_tests(sys,man_config);
-  //run_mmul_uniform_scaling_tests(sys,man_config);
-
-
-
+  caf::cuda::manager_config man_config(false);
+  run_mmul_random_scaling_tests(sys, man_config);
 }
-
-
-
-
-CAF_MAIN()
+CAF_MAIN(id_block::mmul_benchmark)

From 648408f7e2b234822dd2d80c8b2d82abba46db4a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 13:11:04 -0600
Subject: [PATCH 0796/1000] Implemented an optimized CG solver actor facade

---
 .../sparse-CGS-actor/cg_solver_kernels.cu     |  45 +++++
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 163 +++++++++++++++++-
 libcaf_cuda/caf/cuda/global.hpp               |  38 ++++
 libcaf_cuda/caf/cuda/types.hpp                |  21 +++
 4 files changed, 266 insertions(+), 1 deletion(-)
 create mode 100644 libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/cg_solver_kernels.cu

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/cg_solver_kernels.cu b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/cg_solver_kernels.cu
new file mode 100644
index 0000000000..b416459957
--- /dev/null
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/cg_solver_kernels.cu
@@ -0,0 +1,45 @@
+extern "C" {
+
+__global__ void update_p_float(int n, const float* r, float* p, const float* rho, const float* old_rho, int iteration) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        if (iteration == 0) {
+            p[i] = r[i];
+        } else {
+            float beta = rho[0] / old_rho[0];
+            p[i] = r[i] + beta * p[i];
+        }
+    }
+}
+
+__global__ void update_x_r_float(int n, float* x, float* r, const float* p, const float* w, const float* rho, const float* dot_pw) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        float alpha = rho[0] / dot_pw[0];
+        x[i] += alpha * p[i];
+        r[i] -= alpha * w[i];
+    }
+}
+
+__global__ void update_p_double(int n, const double* r, double* p, const double* rho, const double* old_rho, int iteration) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        if (iteration == 0) {
+            p[i] = r[i];
+        } else {
+            double beta = rho[0] / old_rho[0];
+            p[i] = r[i] + beta * p[i];
+        }
+    }
+}
+
+__global__ void update_x_r_double(int n, double* x, double* r, const double* p, const double* w, const double* rho, const double* dot_pw) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        double alpha = rho[0] / dot_pw[0];
+        x[i] += alpha * p[i];
+        r[i] -= alpha * w[i];
+    }
+}
+
+}
\ No newline at end of file
diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index d4348cf91f..555977ad01 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -12,7 +12,7 @@
 
 namespace caf::cuda {
 
-// Reply IDs used to distinguish which actor type is replying
+
 constexpr int id_dot = 100;
 constexpr int id_spmv = 200;
 constexpr int id_axpy = 300;
@@ -585,4 +585,165 @@ class sparse_cg_jacobi_facade : public sparse_cg_facade<T> {
   program_ptr diag_prog_;
 };
 
+/**
+ * Context for asynchronous CG iterations.
+ */
+template <class T>
+struct sparse_cg_solve_context {
+  mem_ptr<int> A_rp, A_ci;
+  mem_ptr<T> A_val, b, x, r, p, w, rho, old_rho, dot_pw;
+  mem_ptr<char> spmv_ws;
+  T threshold;
+  int n, nnz, max_iter, iterations = 0;
+  int device_num, stream_id;
+  actor requester;
+  std::vector<output_mapping> mappings;
+  bool return_mem_ptr = false;
+};
+
+/**
+ * Optimized CG Facade.
+ * - No host recurrence: alpha/beta updates happen on GPU.
+ * - Message-based iteration: Solves every 10 iterations via self-callbacks.
+ * - Fully asynchronous: Does not block CAF worker threads.
+ */
+template <class T = float>
+class sparse_cg_facade_optimized : public sparse_cg_facade<T> {
+public:
+  using solve_context = sparse_cg_solve_context<T>;
+
+  sparse_cg_facade_optimized(actor_config& cfg, uint32_t response_id)
+    : sparse_cg_facade<T>(cfg, response_id) {
+    auto& mgr = manager::get();
+    std::string current_file = __FILE__;
+    auto pos = current_file.find_last_of('/');
+    std::string dir = (pos == std::string::npos) ? "" : current_file.substr(0, pos + 1);
+    
+    std::string p_name = std::is_same_v<T, double> ? "update_p_double" : "update_p_float";
+    std::string xr_name = std::is_same_v<T, double> ? "update_x_r_double" : "update_x_r_float";
+
+    update_p_prog_ = mgr.create_program_from_cubin(dir + "cg_solver_kernels.cubin", p_name);
+    update_xr_prog_ = mgr.create_program_from_cubin(dir + "cg_solver_kernels.cubin", xr_name);
+  }
+
+  behavior make_behavior() override {
+    return {
+      [this](return_mem_ptr_atom, in<int> rp, in<int> ci, in<T> val, in<T> b, in_out<T> x,
+             matrix_format fmt, int n, int nnz, T tol, int max_iter, int dev, int stream) {
+        auto ctx = setup_solve(rp, ci, val, b, x, fmt, n, nnz, tol, max_iter, dev, stream);
+        ctx->return_mem_ptr = true;
+        launch_iterations(ctx);
+      },
+
+      [this](std::vector<output_mapping> mappings, in<int> rp, in<int> ci, in<T> val, in<T> b, in_out<T> x,
+             matrix_format fmt, int n, int nnz, T tol, int max_iter, int dev, int stream) {
+        auto ctx = setup_solve(rp, ci, val, b, x, fmt, n, nnz, tol, max_iter, dev, stream);
+        ctx->mappings = std::move(mappings);
+        launch_iterations(ctx);
+      },
+
+      [this](next_batch_atom, std::shared_ptr<solve_context> ctx, T current_rho) {
+        if (current_rho <= ctx->threshold || ctx->iterations >= ctx->max_iter) {
+          this->dispatch_result(std::move(ctx->mappings), ctx->x, ctx->n, 
+                                solver_result_meta(ctx->device_num, ctx->stream_id, ctx->iterations, current_rho <= ctx->threshold));
+        } else {
+          launch_iterations(ctx);
+        }
+      }
+    };
+  }
+
+protected:
+  std::shared_ptr<solve_context> setup_solve(in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+                                            matrix_format fmt, int n, int nnz, T tol, int max_iter,
+                                            int dev, int stream) {
+    auto ctx = std::make_shared<solve_context>();
+    ctx->requester = actor_cast<actor>(this->current_sender());
+    ctx->n = n; ctx->nnz = nnz; ctx->max_iter = max_iter; ctx->threshold = tol * tol;
+    ctx->device_num = dev; ctx->stream_id = stream;
+
+    command_runner<T> runner;
+    auto res = runner.transfer_memory(dev, stream, rp, ci, val, b_in, x_in);
+    ctx->A_rp = std::get<0>(res); ctx->A_ci = std::get<1>(res); ctx->A_val = std::get<2>(res);
+    ctx->b = std::get<3>(res); ctx->x = std::get<4>(res);
+
+    command_runner<out<T>> work_runner;
+    ctx->r = work_runner.transfer_memory(dev, stream, out<T>(n));
+    ctx->p = work_runner.transfer_memory(dev, stream, out<T>(n));
+    ctx->w = work_runner.transfer_memory(dev, stream, out<T>(n));
+    ctx->rho = work_runner.transfer_memory(dev, stream, create_out_arg_with_size<T>(1));
+    ctx->old_rho = work_runner.transfer_memory(dev, stream, create_out_arg_with_size<T>(1));
+    ctx->dot_pw = work_runner.transfer_memory(dev, stream, create_out_arg_with_size<T>(1));
+
+    auto d_ptr = platform::create()->schedule(stream, dev);
+    size_t ws_size = d_ptr->spmv_csr_buffer_size(stream, n, n, nnz, ctx->A_rp, ctx->A_ci, ctx->A_val, ctx->x, ctx->w);
+    if (ws_size > 0) {
+      command_runner<out<char>> ws_runner;
+      ctx->spmv_ws = ws_runner.transfer_memory(dev, stream, out<char>(static_cast<int>(ws_size)));
+    }
+
+    // Initial r = b - Ax
+    execute_spmv(ctx, ctx->x, ctx->w);
+    if constexpr (std::is_same_v<T, double>) d_ptr->dcopy(stream, n, ctx->b, ctx->r); else d_ptr->scopy(stream, n, ctx->b, ctx->r);
+    if constexpr (std::is_same_v<T, double>) d_ptr->daxpy(stream, n, -1.0, ctx->w, ctx->r); else d_ptr->saxpy(stream, n, -1.0f, ctx->w, ctx->r);
+    execute_dot(ctx, ctx->r, ctx->r, ctx->rho);
+
+    return ctx;
+  }
+
+  void launch_iterations(std::shared_ptr<solve_context> ctx) {
+    auto d_ptr = platform::create()->schedule(ctx->stream_id, ctx->device_num);
+    nd_range range((ctx->n + 255) / 256, 1, 1, 256, 1, 1);
+    
+    auto p_kernel = update_p_prog_->get_kernel(d_ptr->getId());
+    auto xr_kernel = update_xr_prog_->get_kernel(d_ptr->getId());
+
+    int batch_size = std::min(10, ctx->max_iter - ctx->iterations);
+    for (int k = 0; k < batch_size; ++k) {
+      // p = r + beta * p
+      d_ptr->launch_kernel_mem_ref(p_kernel, range, 
+                                   std::make_tuple(ctx->n, ctx->r, ctx->p, ctx->rho, ctx->old_rho, in<int>(ctx->iterations)), 
+                                   ctx->stream_id);
+      // w = Ap
+      execute_spmv(ctx, ctx->p, ctx->w);
+      // dot_pw = p * w
+      execute_dot(ctx, ctx->p, ctx->w, ctx->dot_pw);
+      // x += alpha * p, r -= alpha * w
+      d_ptr->launch_kernel_mem_ref(xr_kernel, range,
+                                   std::make_tuple(ctx->n, ctx->x, ctx->r, ctx->p, ctx->w, ctx->rho, ctx->dot_pw),
+                                   ctx->stream_id);
+      // old_rho = rho, rho = r * r
+      if constexpr (std::is_same_v<T, double>) d_ptr->dcopy(ctx->stream_id, 1, ctx->rho, ctx->old_rho); 
+      else d_ptr->scopy(ctx->stream_id, 1, ctx->rho, ctx->old_rho);
+      
+      execute_dot(ctx, ctx->r, ctx->r, ctx->rho);
+      ctx->iterations++;
+    }
+
+    // Asynchronous notification instead of blocking copy_to_host
+    auto self = actor_cast<actor>(this);
+    command_runner<T> runner;
+    runner.copy_to_host_async(ctx->rho, [self, ctx](std::vector<T> res) {
+      anon_mail(next_batch_atom_v, ctx, res[0]).send(self);
+    });
+  }
+
+  void execute_spmv(std::shared_ptr<solve_context> ctx, mem_ptr<T> in_v, mem_ptr<T> out_v) {
+    auto d_ptr = platform::create()->schedule(ctx->stream_id, ctx->device_num);
+    // Defaulting to CSR for this optimized version as per logic, can be extended
+    d_ptr->spmv_csr(ctx->stream_id, ctx->n, ctx->n, ctx->nnz, T{1}, 
+                    ctx->A_rp, ctx->A_ci, ctx->A_val, in_v, T{0}, out_v, ctx->spmv_ws);
+  }
+
+  void execute_dot(std::shared_ptr<solve_context> ctx, mem_ptr<T> xv, mem_ptr<T> yv, mem_ptr<T> rv) {
+    auto d_ptr = platform::create()->schedule(ctx->stream_id, ctx->device_num);
+    if constexpr (std::is_same_v<T, double>) d_ptr->ddot(ctx->stream_id, ctx->n, xv, yv, rv); 
+    else d_ptr->sdot(ctx->stream_id, ctx->n, xv, yv, rv);
+  }
+
+private:
+  program_ptr update_p_prog_;
+  program_ptr update_xr_prog_;
+};
+
 } // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index 9b18c874a1..400f8070e8 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -7,6 +7,7 @@
 #include <caf/logger.hpp>
 #include <cuda.h>
 #include "caf/cuda/types.hpp" //be sure to include any caf-cuda headers after this one
+#include <memory>
 #include "caf/cuda/nd_range.hpp"
 #include "caf/cuda/helpers.hpp"
 #include <nvrtc.h>
@@ -111,6 +112,33 @@ bool inspect(Inspector& f, output_buffer& x) {
   return f.object(x).fields(f.field("data", x.data));
 }
 
+// Serialization support for sparse_cg_solve_context
+template <class Inspector, class T>
+bool inspect(Inspector& f, caf::cuda::sparse_cg_solve_context<T>& x) {
+  return f.object(x).fields(f.field("A_rp", x.A_rp),
+                            f.field("A_ci", x.A_ci),
+                            f.field("A_val", x.A_val),
+                            f.field("b", x.b),
+                            f.field("x", x.x),
+                            f.field("r", x.r),
+                            f.field("p", x.p),
+                            f.field("w", x.w),
+                            f.field("rho", x.rho),
+                            f.field("old_rho", x.old_rho),
+                            f.field("dot_pw", x.dot_pw),
+                            f.field("spmv_ws", x.spmv_ws),
+                            f.field("threshold", x.threshold),
+                            f.field("n", x.n),
+                            f.field("nnz", x.nnz),
+                            f.field("max_iter", x.max_iter),
+                            f.field("iterations", x.iterations),
+                            f.field("device_num", x.device_num),
+                            f.field("stream_id", x.stream_id),
+                            f.field("requester", x.requester),
+                            f.field("mappings", x.mappings),
+                            f.field("return_mem_ptr", x.return_mem_ptr));
+}
+
 // Serialization support for std::vector<output_buffer> (global namespace)
 template <class Inspector>
 bool inspect(Inspector& f, std::vector<output_buffer>& x) {
@@ -232,6 +260,10 @@ CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id)
   CAF_ADD_TYPE_ID(cuda,(caf::cuda::mem_ptr<char>))  
   CAF_ADD_TYPE_ID(cuda, (caf::cuda::matrix_format))
   CAF_ADD_TYPE_ID(cuda, (caf::cuda::solver_result_meta))
+  CAF_ADD_TYPE_ID(cuda, (caf::cuda::sparse_cg_solve_context<float>))
+  CAF_ADD_TYPE_ID(cuda, (caf::cuda::sparse_cg_solve_context<double>))
+  CAF_ADD_TYPE_ID(cuda, (std::shared_ptr<caf::cuda::sparse_cg_solve_context<float>>))
+  CAF_ADD_TYPE_ID(cuda, (std::shared_ptr<caf::cuda::sparse_cg_solve_context<double>>))
   
   //atoms 
   CAF_ADD_ATOM(cuda, kernel_done_atom)
@@ -247,6 +279,8 @@ CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id)
   CAF_ADD_ATOM(cuda, coo_atom)  
   CAF_ADD_ATOM(cuda, start_atom)
   CAF_ADD_ATOM(cuda, cg_next_step_atom)
+  CAF_ADD_ATOM(cuda, next_batch_atom)
+
 
 CAF_END_TYPE_ID_BLOCK(cuda)
 
@@ -262,3 +296,7 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(output_mapping)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<output_mapping>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::matrix_format)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::solver_result_meta)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::sparse_cg_solve_context<float>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::sparse_cg_solve_context<double>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<caf::cuda::sparse_cg_solve_context<float>>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<caf::cuda::sparse_cg_solve_context<double>>)
diff --git a/libcaf_cuda/caf/cuda/types.hpp b/libcaf_cuda/caf/cuda/types.hpp
index e03c819fd6..feb9bfc5f4 100644
--- a/libcaf_cuda/caf/cuda/types.hpp
+++ b/libcaf_cuda/caf/cuda/types.hpp
@@ -4,6 +4,7 @@
 
 
 #include <caf/intrusive_ptr.hpp>
+#include <caf/actor.hpp>
 #include <variant>
 #include <vector>
 #include <stdexcept>
@@ -330,3 +331,23 @@ struct raw_type<caf::cuda::mem_ptr<T>> {
 
 template <typename T>
 using raw_t = typename raw_type<T>::type;
+
+namespace caf::cuda {
+
+/**
+ * Context for asynchronous CG iterations.
+ */
+template <class T>
+struct sparse_cg_solve_context {
+  mem_ptr<int> A_rp, A_ci;
+  mem_ptr<T> A_val, b, x, r, p, w, rho, old_rho, dot_pw;
+  mem_ptr<char> spmv_ws;
+  T threshold;
+  int n, nnz, max_iter, iterations = 0;
+  int device_num, stream_id;
+  actor requester;
+  std::vector<output_mapping> mappings;
+  bool return_mem_ptr = false;
+};
+
+} // namespace caf::cuda

From 6668882d38c9e36500017d395156c244e3fd1e5b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 13:11:34 -0600
Subject: [PATCH 0797/1000] updated tests to test optmized CG actor

---
 .../CGS-actor-test/main.test.cpp              | 84 +++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
index bd36afd580..482ec96efe 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
@@ -471,6 +471,90 @@ void caf_main(actor_system& sys) {
         );
     }
 
+    // Test 18: Optimized Facade CSR Simple
+    {
+        std::cout << "\n[INFO] Test 18: Optimized Facade actor CSR simple matrix..." << std::endl;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        // Spawn the optimized facade subclass
+        auto facade = sys.spawn<sparse_cg_facade_optimized<float>>(300);
+
+        self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csr, n, nnz, tolerance, max_iter, 0, 17).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x, solver_result_meta meta) {
+                std::cout << "[INFO] Iterations: " << meta.iterations << ", Converged: " << meta.converged << std::endl;
+                verify_solution("Optimized Facade CSR Simple", result_x, expected);
+            }
+        );
+    }
+
+    // Test 19: Optimized Facade Stress Test (N=10000)
+    {
+        int N_large = 10000;
+        std::cout << "\n[INFO] Test 19: Optimized Facade Stress Test - 1D Laplacian (N=" << N_large << ")..." << std::endl;
+        
+        std::vector<int> row_ptr;
+        std::vector<int> col_ind;
+        std::vector<float> values;
+        row_ptr.push_back(0);
+        for(int i=0; i<N_large; ++i) {
+            if(i > 0) { col_ind.push_back(i-1); values.push_back(-1.0f); }
+            col_ind.push_back(i); values.push_back(2.0f);
+            if(i < N_large-1) { col_ind.push_back(i+1); values.push_back(-1.0f); }
+            row_ptr.push_back(col_ind.size());
+        }
+
+        std::vector<float> b_large(N_large, 1.0f);
+        std::vector<float> x_large(N_large, 0.0f);
+        
+        auto facade = sys.spawn<sparse_cg_facade_optimized<float>>(301);
+        auto start = std::chrono::high_resolution_clock::now();
+
+        self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(b_large), create_in_out_arg(x_large),
+                   matrix_format::csr, N_large, (int)values.size(), 1e-4f, 20000, 0, 18).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result, solver_result_meta meta) {
+                auto end = std::chrono::high_resolution_clock::now();
+                std::chrono::duration<double> elapsed = end - start;
+                std::cout << "[SUCCESS] Optimized Stress Test completed in " << elapsed.count() << " seconds." << std::endl;
+                std::cout << "[INFO] Iterations: " << meta.iterations << ", Converged: " << meta.converged << std::endl;
+            }
+        );
+    }
+
+    // Test 20: Optimized Double Precision Facade CSR
+    {
+        std::cout << "\n[INFO] Test 20: Optimized Double precision facade CSR..." << std::endl;
+        int n_d = 3, nnz_d = 3;
+        std::vector<int> row_ptr = {0, 1, 2, 3};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<double> values = {4.0, 3.0, 2.0};
+        std::vector<double> h_b_d = {8.0, 9.0, 2.0};
+        std::vector<double> expected_d = {2.0, 3.0, 1.0};
+        std::vector<double> h_x(n_d, 0.0);
+
+        auto facade = sys.spawn<sparse_cg_facade_optimized<double>>(400);
+
+        self->mail(create_in_arg(row_ptr), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b_d), create_in_out_arg(h_x),
+                   matrix_format::csr, n_d, nnz_d, 1e-10, 100, 0, 19).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<double>& result_x, solver_result_meta meta) {
+                std::cout << "[INFO] Iterations: " << meta.iterations << ", Converged: " << meta.converged << std::endl;
+                verify_solution<double>("Optimized Double Facade CSR", result_x, expected_d, 1e-8);
+            }
+        );
+    }
+
     manager::shutdown();
 }
 

From 57ac2bbdbf9e1c3f6a680dd6c679fbf9a7c7307d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 13:15:24 -0600
Subject: [PATCH 0798/1000] fixed complier errors

---
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 24 ++++---------------
 1 file changed, 4 insertions(+), 20 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index 555977ad01..b1264f6610 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -585,22 +585,6 @@ class sparse_cg_jacobi_facade : public sparse_cg_facade<T> {
   program_ptr diag_prog_;
 };
 
-/**
- * Context for asynchronous CG iterations.
- */
-template <class T>
-struct sparse_cg_solve_context {
-  mem_ptr<int> A_rp, A_ci;
-  mem_ptr<T> A_val, b, x, r, p, w, rho, old_rho, dot_pw;
-  mem_ptr<char> spmv_ws;
-  T threshold;
-  int n, nnz, max_iter, iterations = 0;
-  int device_num, stream_id;
-  actor requester;
-  std::vector<output_mapping> mappings;
-  bool return_mem_ptr = false;
-};
-
 /**
  * Optimized CG Facade.
  * - No host recurrence: alpha/beta updates happen on GPU.
@@ -622,8 +606,8 @@ class sparse_cg_facade_optimized : public sparse_cg_facade<T> {
     std::string p_name = std::is_same_v<T, double> ? "update_p_double" : "update_p_float";
     std::string xr_name = std::is_same_v<T, double> ? "update_x_r_double" : "update_x_r_float";
 
-    update_p_prog_ = mgr.create_program_from_cubin(dir + "cg_solver_kernels.cubin", p_name);
-    update_xr_prog_ = mgr.create_program_from_cubin(dir + "cg_solver_kernels.cubin", xr_name);
+    update_p_prog_ = mgr.create_program_from_cubin(dir + "cg_solver_kernels.cubin", p_name.c_str());
+    update_xr_prog_ = mgr.create_program_from_cubin(dir + "cg_solver_kernels.cubin", xr_name.c_str());
   }
 
   behavior make_behavior() override {
@@ -702,7 +686,7 @@ class sparse_cg_facade_optimized : public sparse_cg_facade<T> {
     for (int k = 0; k < batch_size; ++k) {
       // p = r + beta * p
       d_ptr->launch_kernel_mem_ref(p_kernel, range, 
-                                   std::make_tuple(ctx->n, ctx->r, ctx->p, ctx->rho, ctx->old_rho, in<int>(ctx->iterations)), 
+                                   std::make_tuple(in<int>(ctx->n), ctx->r, ctx->p, ctx->rho, ctx->old_rho, in<int>(ctx->iterations)), 
                                    ctx->stream_id);
       // w = Ap
       execute_spmv(ctx, ctx->p, ctx->w);
@@ -710,7 +694,7 @@ class sparse_cg_facade_optimized : public sparse_cg_facade<T> {
       execute_dot(ctx, ctx->p, ctx->w, ctx->dot_pw);
       // x += alpha * p, r -= alpha * w
       d_ptr->launch_kernel_mem_ref(xr_kernel, range,
-                                   std::make_tuple(ctx->n, ctx->x, ctx->r, ctx->p, ctx->w, ctx->rho, ctx->dot_pw),
+                                   std::make_tuple(in<int>(ctx->n), ctx->x, ctx->r, ctx->p, ctx->w, ctx->rho, ctx->dot_pw),
                                    ctx->stream_id);
       // old_rho = rho, rho = r * r
       if constexpr (std::is_same_v<T, double>) d_ptr->dcopy(ctx->stream_id, 1, ctx->rho, ctx->old_rho); 

From 4039e9d61616da606ac9bd85094ed28eefc086aa Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 13:43:28 -0600
Subject: [PATCH 0799/1000] made changes to dispatch method to fix issue with
 the optmizied actor where it would accidently send to itself

---
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 39 ++++++++++++-------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index b1264f6610..56d367c126 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -251,7 +251,7 @@ class sparse_cg_facade : public event_based_actor {
                                     fmt, n, nnz, tol, max_iter,
                                     device_num, stream_id);
 
-        dispatch_result(std::move(mappings), std::move(x), n, meta);
+        dispatch_result(actor_cast<caf::actor>(this->current_sender()), std::move(mappings), std::move(x), n, meta);
       },
 
       // Mode 3: Default (return vector to sender)
@@ -265,7 +265,7 @@ class sparse_cg_facade : public event_based_actor {
                                     fmt, n, nnz, tol, max_iter,
                                     device_num, stream_id);
 
-        dispatch_result({}, std::move(x), n, meta);
+        dispatch_result(actor_cast<caf::actor>(this->current_sender()), {}, std::move(x), n, meta);
       }
     };
   }
@@ -389,13 +389,12 @@ class sparse_cg_facade : public event_based_actor {
     return {x, solver_result_meta(device_num, stream_id, iterations, rho_val <= threshold)};
   }
 
-  void dispatch_result(std::vector<output_mapping> mappings,
+  void dispatch_result(caf::actor target,
+                       std::vector<output_mapping> mappings,
                        mem_ptr<T> x,
                        int n,
                        solver_result_meta meta) {
-
-    auto sender = actor_cast<actor>(this->current_sender());
-    if (!sender) return;
+    if (!target) return;
 
     void* custom_dst = nullptr;
     size_t custom_count = 0;
@@ -415,20 +414,20 @@ class sparse_cg_facade : public event_based_actor {
         x,
         static_cast<T*>(custom_dst),
         custom_count > 0 ? custom_count : (size_t)n,
-        [sender, r_id = reply_id_, meta](T*, size_t) {
-          caf::anon_mail(r_id, 4, meta).send(sender);
+        [target, r_id = reply_id_, meta](T*, size_t) {
+          caf::anon_mail(r_id, 4, meta).send(target);
         });
 
     } else {
       runner.copy_to_host_async(
         x,
-        [sender, r_id = reply_id_, meta](std::vector<T> data) {
-          caf::anon_mail(r_id, 4, std::move(data), meta).send(sender);
+        [target, r_id = reply_id_, meta](std::vector<T> data) {
+          caf::anon_mail(r_id, 4, std::move(data), meta).send(target);
         });
     }
   }
 
-private:
+protected:
   uint32_t reply_id_;
 };
 
@@ -625,11 +624,25 @@ class sparse_cg_facade_optimized : public sparse_cg_facade<T> {
         ctx->mappings = std::move(mappings);
         launch_iterations(ctx);
       },
+      
+      // Mode 3: Default (return vector to sender) - now handled by optimized facade
+      [this](in<int> rp, in<int> ci, in<T> val,
+             in<T> b_in, in_out<T> x_in,
+             matrix_format fmt, int n, int nnz,
+             T tol, int max_iter,
+             int device_num, int stream_id) {
+        auto ctx = setup_solve(rp, ci, val, b_in, x_in, fmt, n, nnz, tol, max_iter, device_num, stream_id);
+        launch_iterations(ctx);
+      },
 
       [this](next_batch_atom, std::shared_ptr<solve_context> ctx, T current_rho) {
         if (current_rho <= ctx->threshold || ctx->iterations >= ctx->max_iter) {
-          this->dispatch_result(std::move(ctx->mappings), ctx->x, ctx->n, 
-                                solver_result_meta(ctx->device_num, ctx->stream_id, ctx->iterations, current_rho <= ctx->threshold));
+          solver_result_meta meta(ctx->device_num, ctx->stream_id, ctx->iterations, current_rho <= ctx->threshold);
+          if (ctx->return_mem_ptr) {
+            caf::anon_mail(this->reply_id_, std::move(ctx->x), meta).send(ctx->requester);
+          } else {
+            this->dispatch_result(ctx->requester, std::move(ctx->mappings), ctx->x, ctx->n, meta);
+          }
         } else {
           launch_iterations(ctx);
         }

From d9616c97a6ebab30007e55e9dbdd7ef6391308d9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 13:59:34 -0600
Subject: [PATCH 0800/1000] updated test to use a better CG actor

---
 libcaf_cuda/tests/workload-test/main.test.cpp | 44 ++++++++++---------
 1 file changed, 24 insertions(+), 20 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index bfdeaedbd3..4fa8408273 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -45,7 +45,7 @@ bool inspect(Inspector& f, SolverType& x) {
     return false;
 }
 
-CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cg_actor::end)
+CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
     CAF_ADD_ATOM(workload_test, get_work_atom)
     CAF_ADD_ATOM(workload_test, release_memory_atom)
     CAF_ADD_ATOM(workload_test, request_work_atom)
@@ -207,22 +207,20 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
                     self->state().task_start = std::chrono::steady_clock::now();
                     auto start_spawn = std::chrono::steady_clock::now();
                     self->state().current_data = data;
-                    actor solver;
                     if (type == CGS_SOLVER) {
-                        solver = self->spawn<sparse_cg_actor>(
-                            std::move(rp), std::move(ci), std::move(val),
-                            std::move(b), std::move(x),
-                            matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id, actor_cast<actor>(self));
+                        // Use the optimized CG facade. It responds with (r_id, index, solution, meta)
+                        auto facade = self->spawn<sparse_cg_facade_optimized<float>>(0);
+                        self->mail(std::move(rp), std::move(ci), std::move(val),
+                                   std::move(b), std::move(x),
+                                   matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id).send(facade);
                     } else {
-                        solver = self->spawn<sparse_bicgstab_actor<float>>(
+                        // BiCGSTAB still uses the standard stateful actor. It responds with (solution, meta)
+                        auto solver = self->spawn<sparse_bicgstab_actor<float>>(
                             std::move(rp), std::move(ci), std::move(val),
                             std::move(b), std::move(x),
                             matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id, actor_cast<actor>(self));
+                        self->mail(start_atom_v).send(solver);
                     }
-                    auto end_spawn = std::chrono::steady_clock::now();
-                    std::chrono::duration<double> spawn_duration = end_spawn - start_spawn;
-                    // self->println("Worker {}: Solver actor spawned in {} s", self->state().stream_id, spawn_duration.count());
-                    self->mail(start_atom_v).send(solver);
                 },
                 [=](error& err) {
                     if (err == sec::end_of_stream) {
@@ -232,16 +230,22 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
                 }
             );
         },
-        [=](std::vector<float>& solution) {
+        // Result handler for standard stateful actors (e.g., BiCGSTAB)
+        [=](std::vector<float>& solution, solver_result_meta meta) {
             auto task_end = std::chrono::steady_clock::now();
             std::chrono::duration<double> task_duration = task_end - self->state().task_start;
-            std::string solver_type_str;
-            if (self->state().current_solver_type == CGS_SOLVER) {
-                solver_type_str = "CGS_SOLVER";
-            } else {
-                solver_type_str = "BICSTAB_SOLVER";
-            }
-            self->println("Worker {}: Round-trip time (Spawn to Result) for {} ({}) took {} s", self->state().stream_id, self->state().current_matrix_path, solver_type_str, task_duration.count());
+            self->println("Worker {}: Round-trip time (Spawn to Result) for {} (BICSTAB_SOLVER) took {} s (Iters: {})", 
+                          self->state().stream_id, self->state().current_matrix_path, task_duration.count(), meta.iterations);
+            self->mail(1).send(self->state().supervisor);
+            self->state().current_data.reset();
+            self->mail(request_work_atom_v).send(self);
+        },
+        // Result handler for the optimized facade actor
+        [=](uint32_t /*r_id*/, int /*index*/, std::vector<float>& solution, solver_result_meta meta) {
+            auto task_end = std::chrono::steady_clock::now();
+            std::chrono::duration<double> task_duration = task_end - self->state().task_start;
+            self->println("Worker {}: Round-trip time (Spawn to Result) for {} (CGS_SOLVER_OPTIMIZED) took {} s (Iters: {})", 
+                          self->state().stream_id, self->state().current_matrix_path, task_duration.count(), meta.iterations);
             self->mail(1).send(self->state().supervisor);
             self->state().current_data.reset();
             self->mail(request_work_atom_v).send(self);
@@ -336,4 +340,4 @@ void caf_main(actor_system& sys) {
 
     manager::shutdown();
 }
-CAF_MAIN(id_block::cuda, id_block::cg_actor,id_block::workload_test)
\ No newline at end of file
+CAF_MAIN(id_block::cuda,id_block::workload_test)
\ No newline at end of file

From fe4ba171ba8d2fe3872ff7daa225608be532031b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 14:14:33 -0600
Subject: [PATCH 0801/1000] made changes to optimized CGS actor so it can
 distinguish format, adjusted tests accordingly

---
 .../sparse-CGS-actor/cg_solver_kernels.cu     | 24 ++++++----
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 38 +++++++++++++---
 libcaf_cuda/caf/cuda/global.hpp               |  1 +
 libcaf_cuda/caf/cuda/types.hpp                |  1 +
 .../CGS-actor-test/main.test.cpp              | 44 +++++++++++++++++++
 .../tests/workload-test/main.native.cpp       |  2 +-
 libcaf_cuda/tests/workload-test/main.test.cpp |  2 +-
 7 files changed, 95 insertions(+), 17 deletions(-)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/cg_solver_kernels.cu b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/cg_solver_kernels.cu
index b416459957..6014cf6e3c 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/cg_solver_kernels.cu
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/cg_solver_kernels.cu
@@ -1,42 +1,50 @@
 extern "C" {
 
-__global__ void update_p_float(int n, const float* r, float* p, const float* rho, const float* old_rho, int iteration) {
+__global__ void update_p_float(int n, const float* r, float* p, const float* rho, const float* old_rho, int iteration, float threshold) {
+    if (rho[0] <= threshold) return;
     int i = blockIdx.x * blockDim.x + threadIdx.x;
     if (i < n) {
         if (iteration == 0) {
             p[i] = r[i];
         } else {
-            float beta = rho[0] / old_rho[0];
+            float denom = old_rho[0];
+            float beta = (denom > 1e-30f) ? (rho[0] / denom) : 0.0f;
             p[i] = r[i] + beta * p[i];
         }
     }
 }
 
-__global__ void update_x_r_float(int n, float* x, float* r, const float* p, const float* w, const float* rho, const float* dot_pw) {
+__global__ void update_x_r_float(int n, float* x, float* r, const float* p, const float* w, const float* rho, const float* dot_pw, float threshold) {
+    if (rho[0] <= threshold) return;
     int i = blockIdx.x * blockDim.x + threadIdx.x;
     if (i < n) {
-        float alpha = rho[0] / dot_pw[0];
+        float denom = dot_pw[0];
+        float alpha = (abs(denom) > 1e-30f) ? (rho[0] / denom) : 0.0f;
         x[i] += alpha * p[i];
         r[i] -= alpha * w[i];
     }
 }
 
-__global__ void update_p_double(int n, const double* r, double* p, const double* rho, const double* old_rho, int iteration) {
+__global__ void update_p_double(int n, const double* r, double* p, const double* rho, const double* old_rho, int iteration, double threshold) {
+    if (rho[0] <= threshold) return;
     int i = blockIdx.x * blockDim.x + threadIdx.x;
     if (i < n) {
         if (iteration == 0) {
             p[i] = r[i];
         } else {
-            double beta = rho[0] / old_rho[0];
+            double denom = old_rho[0];
+            double beta = (denom > 1e-35) ? (rho[0] / denom) : 0.0;
             p[i] = r[i] + beta * p[i];
         }
     }
 }
 
-__global__ void update_x_r_double(int n, double* x, double* r, const double* p, const double* w, const double* rho, const double* dot_pw) {
+__global__ void update_x_r_double(int n, double* x, double* r, const double* p, const double* w, const double* rho, const double* dot_pw, double threshold) {
+    if (rho[0] <= threshold) return;
     int i = blockIdx.x * blockDim.x + threadIdx.x;
     if (i < n) {
-        double alpha = rho[0] / dot_pw[0];
+        double denom = dot_pw[0];
+        double alpha = (abs(denom) > 1e-35) ? (rho[0] / denom) : 0.0;
         x[i] += alpha * p[i];
         r[i] -= alpha * w[i];
     }
diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index 56d367c126..b7320202f8 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -657,7 +657,7 @@ class sparse_cg_facade_optimized : public sparse_cg_facade<T> {
     auto ctx = std::make_shared<solve_context>();
     ctx->requester = actor_cast<actor>(this->current_sender());
     ctx->n = n; ctx->nnz = nnz; ctx->max_iter = max_iter; ctx->threshold = tol * tol;
-    ctx->device_num = dev; ctx->stream_id = stream;
+    ctx->device_num = dev; ctx->stream_id = stream; ctx->format = fmt;
 
     command_runner<T> runner;
     auto res = runner.transfer_memory(dev, stream, rp, ci, val, b_in, x_in);
@@ -673,7 +673,15 @@ class sparse_cg_facade_optimized : public sparse_cg_facade<T> {
     ctx->dot_pw = work_runner.transfer_memory(dev, stream, create_out_arg_with_size<T>(1));
 
     auto d_ptr = platform::create()->schedule(stream, dev);
-    size_t ws_size = d_ptr->spmv_csr_buffer_size(stream, n, n, nnz, ctx->A_rp, ctx->A_ci, ctx->A_val, ctx->x, ctx->w);
+
+    size_t ws_size = 0;
+    if (fmt == matrix_format::csr)
+      ws_size = d_ptr->spmv_csr_buffer_size(stream, n, n, nnz, ctx->A_rp, ctx->A_ci, ctx->A_val, ctx->x, ctx->w);
+    else if (fmt == matrix_format::csc)
+      ws_size = d_ptr->spmv_csc_buffer_size(stream, n, n, nnz, ctx->A_rp, ctx->A_ci, ctx->A_val, ctx->x, ctx->w);
+    else if (fmt == matrix_format::coo)
+      ws_size = d_ptr->spmv_coo_buffer_size(stream, n, n, nnz, ctx->A_rp, ctx->A_ci, ctx->A_val, ctx->x, ctx->w);
+
     if (ws_size > 0) {
       command_runner<out<char>> ws_runner;
       ctx->spmv_ws = ws_runner.transfer_memory(dev, stream, out<char>(static_cast<int>(ws_size)));
@@ -699,7 +707,7 @@ class sparse_cg_facade_optimized : public sparse_cg_facade<T> {
     for (int k = 0; k < batch_size; ++k) {
       // p = r + beta * p
       d_ptr->launch_kernel_mem_ref(p_kernel, range, 
-                                   std::make_tuple(in<int>(ctx->n), ctx->r, ctx->p, ctx->rho, ctx->old_rho, in<int>(ctx->iterations)), 
+                                   std::make_tuple(in<int>(ctx->n), ctx->r, ctx->p, ctx->rho, ctx->old_rho, in<int>(ctx->iterations), in<T>(ctx->threshold)), 
                                    ctx->stream_id);
       // w = Ap
       execute_spmv(ctx, ctx->p, ctx->w);
@@ -707,7 +715,7 @@ class sparse_cg_facade_optimized : public sparse_cg_facade<T> {
       execute_dot(ctx, ctx->p, ctx->w, ctx->dot_pw);
       // x += alpha * p, r -= alpha * w
       d_ptr->launch_kernel_mem_ref(xr_kernel, range,
-                                   std::make_tuple(in<int>(ctx->n), ctx->x, ctx->r, ctx->p, ctx->w, ctx->rho, ctx->dot_pw),
+                                   std::make_tuple(in<int>(ctx->n), ctx->x, ctx->r, ctx->p, ctx->w, ctx->rho, ctx->dot_pw, in<T>(ctx->threshold)),
                                    ctx->stream_id);
       // old_rho = rho, rho = r * r
       if constexpr (std::is_same_v<T, double>) d_ptr->dcopy(ctx->stream_id, 1, ctx->rho, ctx->old_rho); 
@@ -727,9 +735,25 @@ class sparse_cg_facade_optimized : public sparse_cg_facade<T> {
 
   void execute_spmv(std::shared_ptr<solve_context> ctx, mem_ptr<T> in_v, mem_ptr<T> out_v) {
     auto d_ptr = platform::create()->schedule(ctx->stream_id, ctx->device_num);
-    // Defaulting to CSR for this optimized version as per logic, can be extended
-    d_ptr->spmv_csr(ctx->stream_id, ctx->n, ctx->n, ctx->nnz, T{1}, 
-                    ctx->A_rp, ctx->A_ci, ctx->A_val, in_v, T{0}, out_v, ctx->spmv_ws);
+    switch (ctx->format) {
+      case matrix_format::csr:
+        d_ptr->spmv_csr(ctx->stream_id, ctx->n, ctx->n, ctx->nnz, T{1}, 
+                        ctx->A_rp, ctx->A_ci, ctx->A_val, in_v, T{0}, out_v, ctx->spmv_ws);
+        break;
+
+      case matrix_format::csc:
+        d_ptr->spmv_csc(ctx->stream_id, ctx->n, ctx->n, ctx->nnz, T{1}, 
+                        ctx->A_rp, ctx->A_ci, ctx->A_val, in_v, T{0}, out_v, ctx->spmv_ws);
+        break;
+
+      case matrix_format::coo:
+        d_ptr->spmv_coo(ctx->stream_id, ctx->n, ctx->n, ctx->nnz, T{1}, 
+                        ctx->A_rp, ctx->A_ci, ctx->A_val, in_v, T{0}, out_v, ctx->spmv_ws);
+        break;
+
+      default:
+        break;
+    }
   }
 
   void execute_dot(std::shared_ptr<solve_context> ctx, mem_ptr<T> xv, mem_ptr<T> yv, mem_ptr<T> rv) {
diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index 400f8070e8..58741dae7d 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -128,6 +128,7 @@ bool inspect(Inspector& f, caf::cuda::sparse_cg_solve_context<T>& x) {
                             f.field("dot_pw", x.dot_pw),
                             f.field("spmv_ws", x.spmv_ws),
                             f.field("threshold", x.threshold),
+                            f.field("format", x.format),
                             f.field("n", x.n),
                             f.field("nnz", x.nnz),
                             f.field("max_iter", x.max_iter),
diff --git a/libcaf_cuda/caf/cuda/types.hpp b/libcaf_cuda/caf/cuda/types.hpp
index feb9bfc5f4..dab8945880 100644
--- a/libcaf_cuda/caf/cuda/types.hpp
+++ b/libcaf_cuda/caf/cuda/types.hpp
@@ -343,6 +343,7 @@ struct sparse_cg_solve_context {
   mem_ptr<T> A_val, b, x, r, p, w, rho, old_rho, dot_pw;
   mem_ptr<char> spmv_ws;
   T threshold;
+  matrix_format format;
   int n, nnz, max_iter, iterations = 0;
   int device_num, stream_id;
   actor requester;
diff --git a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
index 482ec96efe..c60d71a11f 100644
--- a/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
+++ b/libcaf_cuda/tests/actorSOLVE-test/CGS-actor-test/main.test.cpp
@@ -555,6 +555,50 @@ void caf_main(actor_system& sys) {
         );
     }
 
+    // Test 21: Optimized Facade CSC Simple
+    {
+        std::cout << "\n[INFO] Test 21: Optimized Facade actor CSC simple matrix..." << std::endl;
+        std::vector<int> col_ptr = {0, 1, 2, 3};
+        std::vector<int> row_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_cg_facade_optimized<float>>(302);
+
+        self->mail(create_in_arg(col_ptr), create_in_arg(row_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::csc, n, nnz, tolerance, max_iter, 0, 20).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x, solver_result_meta meta) {
+                std::cout << "[INFO] Iterations: " << meta.iterations << ", Converged: " << meta.converged << std::endl;
+                verify_solution("Optimized Facade CSC Simple", result_x, expected);
+            }
+        );
+    }
+
+    // Test 22: Optimized Facade COO Simple
+    {
+        std::cout << "\n[INFO] Test 22: Optimized Facade actor COO simple matrix..." << std::endl;
+        std::vector<int> row_ind = {0, 1, 2};
+        std::vector<int> col_ind = {0, 1, 2};
+        std::vector<float> values = {4.0f, 3.0f, 2.0f};
+        std::vector<float> h_x(n, 0.0f);
+
+        auto facade = sys.spawn<sparse_cg_facade_optimized<float>>(303);
+
+        self->mail(create_in_arg(row_ind), create_in_arg(col_ind), create_in_arg(values),
+                   create_in_arg(h_b), create_in_out_arg(h_x),
+                   matrix_format::coo, n, nnz, tolerance, max_iter, 0, 21).send(facade);
+
+        self->receive(
+            [&](uint32_t /*resp_id*/, int /*idx*/, const std::vector<float>& result_x, solver_result_meta meta) {
+                std::cout << "[INFO] Iterations: " << meta.iterations << ", Converged: " << meta.converged << std::endl;
+                verify_solution("Optimized Facade COO Simple", result_x, expected);
+            }
+        );
+    }
+
     manager::shutdown();
 }
 
diff --git a/libcaf_cuda/tests/workload-test/main.native.cpp b/libcaf_cuda/tests/workload-test/main.native.cpp
index 5cec4a5e63..507fc978f1 100644
--- a/libcaf_cuda/tests/workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/workload-test/main.native.cpp
@@ -339,7 +339,7 @@ int main() {
 
     std::cout << "[INFO] Loading matrices...\n";
     scan("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
-    scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
+    //scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
 
     if (tasks.empty()) {
         std::cerr << "No matrices found.\n";
diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index 4fa8408273..5ae050cf40 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -301,7 +301,7 @@ void caf_main(actor_system& sys) {
     };
 
     scan("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
-    scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
+    //scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
 
     std::cout << "[INFO] Pre-loading " << tasks->size() << " matrices into memory...\n";
     for (auto& t : *tasks) {

From 2d55914d131832b271d59a2314b07f48052619e4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 1 Jun 2026 14:41:12 -0600
Subject: [PATCH 0802/1000] updated paths of files being read to point to a
 better dataset

---
 libcaf_cuda/tests/workload-test/main.native.cpp | 2 +-
 libcaf_cuda/tests/workload-test/main.test.cpp   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.native.cpp b/libcaf_cuda/tests/workload-test/main.native.cpp
index 507fc978f1..f07f340474 100644
--- a/libcaf_cuda/tests/workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/workload-test/main.native.cpp
@@ -338,7 +338,7 @@ int main() {
     };
 
     std::cout << "[INFO] Loading matrices...\n";
-    scan("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+    scan("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
     //scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
 
     if (tasks.empty()) {
diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index 5ae050cf40..031e28103a 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -209,7 +209,7 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
                     self->state().current_data = data;
                     if (type == CGS_SOLVER) {
                         // Use the optimized CG facade. It responds with (r_id, index, solution, meta)
-                        auto facade = self->spawn<sparse_cg_facade_optimized<float>>(0);
+                        auto facade = self->spawn<sparse_cg_facade<float>>(0);
                         self->mail(std::move(rp), std::move(ci), std::move(val),
                                    std::move(b), std::move(x),
                                    matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id).send(facade);
@@ -300,7 +300,7 @@ void caf_main(actor_system& sys) {
         }
     };
 
-    scan("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+    scan("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
     //scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
 
     std::cout << "[INFO] Pre-loading " << tasks->size() << " matrices into memory...\n";

From 58177106ac6a72fd3dadb56bc720a965b2a45546 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 2 Jun 2026 08:01:13 -0600
Subject: [PATCH 0803/1000] refactored code to keep everything in the same file
 where possible

---
 .../tests/workload-test/main.native.cpp       | 67 +++++--------------
 libcaf_cuda/tests/workload-test/main.test.cpp | 46 ++-----------
 .../tests/workload-test/sparse_utils.cpp      | 33 +++++++++
 .../tests/workload-test/sparse_utils.hpp      | 25 ++++++-
 4 files changed, 79 insertions(+), 92 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.native.cpp b/libcaf_cuda/tests/workload-test/main.native.cpp
index f07f340474..0d3b2efc1a 100644
--- a/libcaf_cuda/tests/workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/workload-test/main.native.cpp
@@ -44,21 +44,9 @@ namespace fs = std::filesystem;
         }                                                                  \
     } while (0)
 
-enum SolverType { CGS_SOLVER, BICSTAB_SOLVER };
-
-struct MatrixTask {
-    std::string path;
-    SolverType type;
-    int32_t rows, cols, nnz;
-    std::vector<int32_t> row_ptr;
-    std::vector<int32_t> col_indices;
-    std::vector<float> values;
-    std::vector<float> b;
-};
-
 // Simplified CG Solver using raw cuBLAS and cuSPARSE
 void solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream) {
-    int n = task.rows;
+    int n = task.data->rows;
     float alpha = 1.0f, beta = 0.0f, r0 = 0.0f, r1 = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
     float tolerance = 1e-5f;
     int max_iters = 2000;
@@ -67,19 +55,19 @@ void solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const Matr
     int *d_row_ptr, *d_col_ind;
 
     // Use Stream Ordered Allocator
-    CHECK_CUDA(cudaMallocAsync(&d_val, task.nnz * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
     CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.nnz * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
     CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
     CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
     CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
     CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
     CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
 
-    CHECK_CUDA(cudaMemcpyAsync(d_val, task.values.data(), task.nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.col_indices.data(), task.nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_b, task.b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
     CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
 
     // Create descriptors
@@ -88,7 +76,7 @@ void solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const Matr
 
     cusparseSpMatDescr_t matA;
     cusparseDnVecDescr_t vecX, vecP, vecAp;
-    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.nnz, d_row_ptr, d_col_ind, d_val, 
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val, 
                                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
     CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
     CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
@@ -154,7 +142,7 @@ void solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const Matr
 
 // BiCGSTAB Solver using raw cuBLAS and cuSPARSE (Asynchronous version)
 void solve_bicgstab_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream) {
-    int n = task.rows;
+    int n = task.data->rows;
     float alpha = 1.0f, beta = 0.0f, omega = 1.0f, rho = 1.0f, rho_prev = 1.0f;
     float tolerance = 1e-5f;
     int max_iters = 2000;
@@ -162,9 +150,9 @@ void solve_bicgstab_async(cublasHandle_t cublas, cusparseHandle_t cusparse, cons
     float *d_val, *d_x, *d_r, *d_r_hat, *d_p, *d_v, *d_s, *d_t, *d_b;
     int *d_row_ptr, *d_col_ind;
 
-    CHECK_CUDA(cudaMallocAsync(&d_val, task.nnz * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
     CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.nnz * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
     CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
     CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
     CHECK_CUDA(cudaMallocAsync(&d_r_hat, n * sizeof(float), stream));
@@ -174,10 +162,10 @@ void solve_bicgstab_async(cublasHandle_t cublas, cusparseHandle_t cusparse, cons
     CHECK_CUDA(cudaMallocAsync(&d_t, n * sizeof(float), stream));
     CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
 
-    CHECK_CUDA(cudaMemcpyAsync(d_val, task.values.data(), task.nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.col_indices.data(), task.nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_b, task.b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
     CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
     CHECK_CUDA(cudaMemsetAsync(d_v, 0, n * sizeof(float), stream));
     CHECK_CUDA(cudaMemsetAsync(d_p, 0, n * sizeof(float), stream));
@@ -187,7 +175,7 @@ void solve_bicgstab_async(cublasHandle_t cublas, cusparseHandle_t cusparse, cons
 
     cusparseSpMatDescr_t matA;
     cusparseDnVecDescr_t vecX, vecP, vecV, vecS, vecT;
-    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.nnz, d_row_ptr, d_col_ind, d_val, 
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val, 
                                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
     CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
     CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
@@ -316,29 +304,8 @@ void gpu_worker(int device_id, int thread_id, std::vector<MatrixTask> assigned_t
 int main() {
     std::vector<MatrixTask> tasks;
     
-    auto scan = [&](const std::string& dir, SolverType type) {
-        if (!fs::exists(dir)) return;
-        for (const auto& entry : fs::directory_iterator(dir)) {
-            if (entry.path().extension() == ".bin") {
-                auto coo = load_binary_coo(entry.path().string());
-                auto csr = convert_coo_to_csr(coo);
-                MatrixTask t;
-                // Compute t.b using the valid csr object before its internal vectors are moved
-                t.b = compute_rhs_spmv(csr, std::vector<float>(csr.cols, 1.0f));
-                t.path = entry.path().string();
-                t.type = type;
-                t.rows = csr.rows;
-                t.nnz = csr.nnz;
-                t.row_ptr = std::move(csr.row_ptr);
-                t.col_indices = std::move(csr.col_indices);
-                t.values = std::move(csr.values);
-                tasks.push_back(std::move(t));
-            }
-        }
-    };
-
     std::cout << "[INFO] Loading matrices...\n";
-    scan("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
+    tasks = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
     //scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
 
     if (tasks.empty()) {
diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index 031e28103a..ebe5a49647 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -17,23 +17,6 @@
 using namespace caf;
 using namespace caf::cuda;
 namespace fs = std::filesystem;
-
-enum SolverType { CGS_SOLVER, BICSTAB_SOLVER };
-
-struct MatrixData {
-    std::vector<int> row_ptr;
-    std::vector<int> col_indices;
-    std::vector<float> values;
-    std::vector<float> b;
-    std::vector<float> x_guess;
-};
-
-struct MatrixTask {
-    std::string path;
-    SolverType type;
-    std::shared_ptr<MatrixData> data;
-};
-
 template <class Inspector>
 bool inspect(Inspector& f, SolverType& x) {
     auto val = static_cast<int>(x);
@@ -114,7 +97,7 @@ behavior gpu_device_actor(stateful_actor<device_actor_state>* self,
             st.local_tasks.pop_front();
 
             auto& data = *t.data;
-            promise.deliver(t.type, t.path, create_in_arg(data.row_ptr), create_in_arg(data.col_indices), 
+            promise.deliver(t.type, t.path, create_in_arg((const std::vector<int32_t>&)data.row_ptr), create_in_arg((const std::vector<int32_t>&)data.col_indices), 
                             create_in_arg(data.values), create_in_arg(data.b), create_in_out_arg(data.x_guess), 
                             (int)data.row_ptr.size() - 1, (int)data.values.size(), t.data);
         }
@@ -290,31 +273,12 @@ behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, int total,
 
 void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
-    auto tasks = std::make_shared<std::vector<MatrixTask>>();
     
-    auto scan = [&](const std::string& dir, SolverType type) {
-        if (!fs::exists(dir)) return;
-        for (const auto& entry : fs::directory_iterator(dir)) {
-            if (entry.path().extension() == ".bin")
-                tasks->push_back({entry.path().string(), type, nullptr});
-        }
-    };
-
-    scan("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
+    std::cout << "[INFO] Loading matrices into memory...\n";
+    auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
     //scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
-
-    std::cout << "[INFO] Pre-loading " << tasks->size() << " matrices into memory...\n";
-    for (auto& t : *tasks) {
-        auto coo = load_binary_coo(t.path);
-        auto A = convert_coo_to_csr(coo);
-        t.data = std::make_shared<MatrixData>();
-        t.data->b = compute_rhs_spmv(A, std::vector<float>(A.cols, 1.0f));
-        t.data->row_ptr = std::move(A.row_ptr);
-        t.data->col_indices = std::move(A.col_indices);
-        t.data->values = std::move(A.values);
-        t.data->x_guess.assign(A.cols, 0.0f);
-    }
-
+    auto tasks = std::make_shared<std::vector<MatrixTask>>(std::move(tasks_vec));
+    
     if (tasks->empty()) {
         std::cerr << "No matrix files found in search paths.\n";
         manager::shutdown();
diff --git a/libcaf_cuda/tests/workload-test/sparse_utils.cpp b/libcaf_cuda/tests/workload-test/sparse_utils.cpp
index 220fe442c3..ca4c3ca360 100644
--- a/libcaf_cuda/tests/workload-test/sparse_utils.cpp
+++ b/libcaf_cuda/tests/workload-test/sparse_utils.cpp
@@ -1,6 +1,9 @@
 #include "sparse_utils.hpp"
 #include <fstream>
 #include <stdexcept>
+#include <filesystem>
+#include <random>
+namespace fs = std::filesystem;
 
 SparseMatrixCOO load_binary_coo(const std::string& filepath) {
     std::ifstream file(filepath, std::ios::binary);
@@ -69,4 +72,34 @@ std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<
         b[i] = sum;
     }
     return b;
+}
+
+std::vector<MatrixTask> scan_for_matrices(const std::string& dir, SolverType type) {
+    std::vector<MatrixTask> tasks;
+    if (!fs::exists(dir)) return tasks;
+    for (const auto& entry : fs::directory_iterator(dir)) {
+        if (entry.path().extension() == ".bin") {
+            auto coo = load_binary_coo(entry.path().string());
+            auto csr = convert_coo_to_csr(coo);
+            auto data = std::make_shared<MatrixData>();
+            data->rows = csr.rows;
+            data->cols = csr.cols;
+            data->nnz = csr.nnz;
+            data->b = compute_rhs_spmv(csr, std::vector<float>(csr.cols, 1.0f));
+            data->row_ptr = std::move(csr.row_ptr);
+            data->col_indices = std::move(csr.col_indices);
+            data->values = std::move(csr.values);
+            data->x_guess.assign(data->cols, 0.0f);
+            
+            tasks.push_back({entry.path().string(), type, data});
+        }
+    }
+    return tasks;
+}
+
+int generate_random_sleep_ms(int min_ms, int max_ms) {
+    static std::random_device rd;
+    static std::mt19937 gen(rd());
+    std::uniform_int_distribution<> dis(min_ms, max_ms);
+    return dis(gen);
 }
\ No newline at end of file
diff --git a/libcaf_cuda/tests/workload-test/sparse_utils.hpp b/libcaf_cuda/tests/workload-test/sparse_utils.hpp
index 45ec51add6..c946aeccb2 100644
--- a/libcaf_cuda/tests/workload-test/sparse_utils.hpp
+++ b/libcaf_cuda/tests/workload-test/sparse_utils.hpp
@@ -3,6 +3,9 @@
 #include <vector>
 #include <string>
 #include <cstdint>
+#include <memory>
+
+enum SolverType { CGS_SOLVER, BICSTAB_SOLVER };
 
 // Structure to hold raw data from the binary file
 struct SparseMatrixCOO {
@@ -24,6 +27,23 @@ struct SparseMatrixCSR {
     std::vector<float> values;       // Size: nnz
 };
 
+struct MatrixData {
+    std::vector<int32_t> row_ptr;
+    std::vector<int32_t> col_indices;
+    std::vector<float> values;
+    std::vector<float> b;
+    std::vector<float> x_guess;
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+};
+
+struct MatrixTask {
+    std::string path;
+    SolverType type;
+    std::shared_ptr<MatrixData> data;
+};
+
 // Function to slurp the binary data into memory
 SparseMatrixCOO load_binary_coo(const std::string& filepath);
 
@@ -31,4 +51,7 @@ SparseMatrixCOO load_binary_coo(const std::string& filepath);
 SparseMatrixCSR convert_coo_to_csr(const SparseMatrixCOO& coo);
 
 // Compute b = A * x using CSR layout (Sparse Matrix-Vector Multiplication)
-std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x);
\ No newline at end of file
+std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x);
+
+std::vector<MatrixTask> scan_for_matrices(const std::string& dir, SolverType type);
+int generate_random_sleep_ms(int min_ms, int max_ms);
\ No newline at end of file

From 8e53135aa0f8fb9e1dcfb5fb7dbc8b10bcf8e5a6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 2 Jun 2026 08:39:33 -0600
Subject: [PATCH 0804/1000] updated test to be produced consumer

---
 .../tests/workload-test/main.native.cpp       | 387 +++++++++---------
 1 file changed, 195 insertions(+), 192 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.native.cpp b/libcaf_cuda/tests/workload-test/main.native.cpp
index 0d3b2efc1a..f827515ae6 100644
--- a/libcaf_cuda/tests/workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/workload-test/main.native.cpp
@@ -1,6 +1,7 @@
 #include <cuda_runtime.h>
 #include <cublas_v2.h>
 #include <cusparse.h>
+
 #include <iostream>
 #include <vector>
 #include <string>
@@ -11,18 +12,23 @@
 #include <filesystem>
 #include <memory>
 #include <algorithm>
+#include <condition_variable>
+#include <atomic>
+
 #include "sparse_utils.hpp"
 
 namespace fs = std::filesystem;
 
+// ------------------------------------------------------------
 // Error checking macros
+// ------------------------------------------------------------
 #define CHECK_CUDA(call)                                                   \
     do {                                                                   \
         cudaError_t status = call;                                         \
         if (status != cudaSuccess) {                                       \
             std::cerr << "CUDA Error: " << cudaGetErrorString(status)      \
                       << " at " << __FILE__ << ":" << __LINE__ << std::endl; \
-            exit(1);                                                       \
+            std::exit(1);                                                  \
         }                                                                  \
     } while (0)
 
@@ -31,7 +37,7 @@ namespace fs = std::filesystem;
         cublasStatus_t status = call;                                      \
         if (status != CUBLAS_STATUS_SUCCESS) {                             \
             std::cerr << "cuBLAS Error at " << __FILE__ << ":" << __LINE__ << std::endl; \
-            exit(1);                                                       \
+            std::exit(1);                                                  \
         }                                                                  \
     } while (0)
 
@@ -39,13 +45,73 @@ namespace fs = std::filesystem;
     do {                                                                   \
         cusparseStatus_t status = call;                                    \
         if (status != CUSPARSE_STATUS_SUCCESS) {                           \
-            std::cerr << "cuSparse Error at " << __FILE__ << ":" << __LINE__ << std::endl; \
-            exit(1);                                                       \
+            std::cerr << "cuSPARSE Error at " << __FILE__ << ":" << __LINE__ << std::endl; \
+            std::exit(1);                                                  \
         }                                                                  \
     } while (0)
 
-// Simplified CG Solver using raw cuBLAS and cuSPARSE
-void solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream) {
+// ------------------------------------------------------------
+// Thread-safe queue
+// ------------------------------------------------------------
+template <typename T>
+class ThreadSafeQueue {
+public:
+    void push(T item) {
+        {
+            std::lock_guard<std::mutex> lock(mtx_);
+            if (closed_) {
+                return;
+            }
+            q_.push(std::move(item));
+        }
+        cv_.notify_one();
+    }
+
+    bool wait_pop(T& item) {
+        std::unique_lock<std::mutex> lock(mtx_);
+        cv_.wait(lock, [&] { return closed_ || !q_.empty(); });
+
+        if (q_.empty()) {
+            return false; // closed and empty
+        }
+
+        item = std::move(q_.front());
+        q_.pop();
+        return true;
+    }
+
+    void close() {
+        {
+            std::lock_guard<std::mutex> lock(mtx_);
+            closed_ = true;
+        }
+        cv_.notify_all();
+    }
+
+private:
+    std::mutex mtx_;
+    std::condition_variable cv_;
+    std::queue<T> q_;
+    bool closed_ = false;
+};
+
+// ------------------------------------------------------------
+// Stream slot owned by one GPU consumer thread
+// ------------------------------------------------------------
+struct StreamSlot {
+    cudaStream_t stream{};
+    cublasHandle_t cublas{};
+    cusparseHandle_t cusparse{};
+    cudaEvent_t done{};
+    bool busy = false;
+};
+
+// ------------------------------------------------------------
+// Your existing solver functions can stay the same
+// ------------------------------------------------------------
+// Keep these as you already have them, or minimally adjust if needed.
+void solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
+                    const MatrixTask& task, cudaStream_t stream) {
     int n = task.data->rows;
     float alpha = 1.0f, beta = 0.0f, r0 = 0.0f, r1 = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
     float tolerance = 1e-5f;
@@ -54,7 +120,6 @@ void solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const Matr
     float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b;
     int *d_row_ptr, *d_col_ind;
 
-    // Use Stream Ordered Allocator
     CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
     CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
     CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
@@ -70,65 +135,61 @@ void solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const Matr
     CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
     CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
 
-    // Create descriptors
     CHECK_CUBLAS(cublasSetStream(cublas, stream));
     CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
 
     cusparseSpMatDescr_t matA;
     cusparseDnVecDescr_t vecX, vecP, vecAp;
-    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val, 
-                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val,
+                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                     CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
     CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
     CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
     CHECK_CUSPARSE(cusparseCreateDnVec(&vecAp, n, d_Ap, CUDA_R_32F));
 
     size_t bufferSize = 0;
     void* d_buffer = nullptr;
-    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecAp, 
+    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, vecX, &beta, vecAp,
                                            CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
     CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
 
-    // CG Logic: r = b - Ax (initially r = b since x=0)
     CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
     CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_p, 1));
     CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
 
     int k = 0;
     while (k < max_iters) {
-        // Ap = A * p
-        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecP, &beta, vecAp, 
+        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                    &alpha, matA, vecP, &beta, vecAp,
                                     CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
-        
+
         float pAp;
         CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
         a = r1 / pAp;
 
-        // x = x + a*p
         CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
-        
-        // r = r - a*Ap
+
         na = -a;
         CHECK_CUBLAS(cublasSaxpy(cublas, n, &na, d_Ap, 1, d_r, 1));
 
         r0 = r1;
         CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
 
-        if (sqrt(r1) < tolerance) break;
+        if (std::sqrt(r1) < tolerance) break;
 
         b = r1 / r0;
-        // p = r + b*p => scal p by b then add r
         CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
         CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_r, 1, d_p, 1));
         k++;
     }
 
-    // Cleanup
-    cusparseDestroySpMat(matA);
-    cusparseDestroyDnVec(vecX);
-    cusparseDestroyDnVec(vecP);
-    cusparseDestroyDnVec(vecAp);
+    CHECK_CUSPARSE(cusparseDestroySpMat(matA));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecX));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
 
-    // Stream-ordered free
     CHECK_CUDA(cudaFreeAsync(d_val, stream));
     CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
     CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
@@ -140,205 +201,147 @@ void solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const Matr
     CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
 }
 
-// BiCGSTAB Solver using raw cuBLAS and cuSPARSE (Asynchronous version)
-void solve_bicgstab_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream) {
-    int n = task.data->rows;
-    float alpha = 1.0f, beta = 0.0f, omega = 1.0f, rho = 1.0f, rho_prev = 1.0f;
-    float tolerance = 1e-5f;
-    int max_iters = 2000;
-
-    float *d_val, *d_x, *d_r, *d_r_hat, *d_p, *d_v, *d_s, *d_t, *d_b;
-    int *d_row_ptr, *d_col_ind;
-
-    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_r_hat, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_v, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_s, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_t, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
 
-    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMemsetAsync(d_v, 0, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMemsetAsync(d_p, 0, n * sizeof(float), stream));
 
-    CHECK_CUBLAS(cublasSetStream(cublas, stream));
-    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
+// ------------------------------------------------------------
+// GPU consumer thread: one thread per GPU
+// ------------------------------------------------------------
+void gpu_consumer(int device_id, int num_streams, ThreadSafeQueue<MatrixTask>& work_queue) {
+    CHECK_CUDA(cudaSetDevice(device_id));
 
-    cusparseSpMatDescr_t matA;
-    cusparseDnVecDescr_t vecX, vecP, vecV, vecS, vecT;
-    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val, 
-                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
-    CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
-    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
-    CHECK_CUSPARSE(cusparseCreateDnVec(&vecV, n, d_v, CUDA_R_32F));
-    CHECK_CUSPARSE(cusparseCreateDnVec(&vecS, n, d_s, CUDA_R_32F));
-    CHECK_CUSPARSE(cusparseCreateDnVec(&vecT, n, d_t, CUDA_R_32F));
+    std::vector<StreamSlot> slots(num_streams);
 
-    size_t bufferSize = 0;
-    void* d_buffer = nullptr;
-    float one = 1.0f, zero = 0.0f;
-    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE, &one, matA, vecP, &zero, vecV, 
-                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
-    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
+    for (int i = 0; i < num_streams; ++i) {
+        CHECK_CUDA(cudaStreamCreateWithFlags(&slots[i].stream, cudaStreamNonBlocking));
+        CHECK_CUBLAS(cublasCreate(&slots[i].cublas));
+        CHECK_CUSPARSE(cusparseCreate(&slots[i].cusparse));
+        CHECK_CUDA(cudaEventCreateWithFlags(&slots[i].done, cudaEventDisableTiming));
+        slots[i].busy = false;
+    }
 
-    // r = b - Ax
-    // Note: We compute Ax explicitly to support non-zero initial guesses in the future
-    CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE, &one, matA, vecX, &zero, vecV, 
-                                CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
-    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
-    float minus_one = -1.0f;
-    CHECK_CUBLAS(cublasSaxpy(cublas, n, &minus_one, d_v, 1, d_r, 1));
-    CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_r_hat, 1));
-
-    for (int i = 1; i <= max_iters; ++i) {
-        CHECK_CUBLAS(cublasSdot(cublas, n, d_r_hat, 1, d_r, 1, &rho));
-        
-        if (i == 1) {
-            CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_p, 1));
-        } else {
-            beta = (rho / rho_prev) * (alpha / omega);
-            float minus_omega = -omega;
-            CHECK_CUBLAS(cublasSaxpy(cublas, n, &minus_omega, d_v, 1, d_p, 1));
-            CHECK_CUBLAS(cublasSscal(cublas, n, &beta, d_p, 1));
-            CHECK_CUBLAS(cublasSaxpy(cublas, n, &one, d_r, 1, d_p, 1));
+    int rr = 0;
+    auto acquire_slot = [&]() -> int {
+        while (true) {
+            for (int offset = 0; offset < num_streams; ++offset) {
+                int idx = (rr + offset) % num_streams;
+
+                if (!slots[idx].busy) {
+                    rr = (idx + 1) % num_streams;
+                    return idx;
+                }
+
+                cudaError_t q = cudaEventQuery(slots[idx].done);
+                if (q == cudaSuccess) {
+                    slots[idx].busy = false;
+                    rr = (idx + 1) % num_streams;
+                    return idx;
+                } else if (q != cudaErrorNotReady) {
+                    std::cerr << "CUDA event query failed on GPU " << device_id
+                              << ", stream " << idx << ": " << cudaGetErrorString(q) << std::endl;
+                    std::exit(1);
+                }
+            }
+            std::this_thread::yield();
         }
+    };
 
-        // v = Ap
-        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE, &one, matA, vecP, &zero, vecV, 
-                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
-
-        float rhat_v;
-        CHECK_CUBLAS(cublasSdot(cublas, n, d_r_hat, 1, d_v, 1, &rhat_v));
-        alpha = rho / rhat_v;
+    MatrixTask task;
+    while (work_queue.wait_pop(task)) {
+        int slot_id = acquire_slot();
+        auto& slot = slots[slot_id];
 
-        // s = r - alpha*v
-        CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_s, 1));
-        float neg_alpha = -alpha;
-        CHECK_CUBLAS(cublasSaxpy(cublas, n, &neg_alpha, d_v, 1, d_s, 1));
+        auto start_task = std::chrono::steady_clock::now();
 
-        // t = As
-        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE, &one, matA, vecS, &zero, vecT, 
-                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
+        if (task.type == CGS_SOLVER) {
+            solve_cg_async(slot.cublas, slot.cusparse, task, slot.stream);
+        } 
+        else {
+            throw std::runtime_error("Unsupported solver type");
+            exit(1);
+        }
 
-        float t_s, t_t;
-        CHECK_CUBLAS(cublasSdot(cublas, n, d_t, 1, d_s, 1, &t_s));
-        CHECK_CUBLAS(cublasSdot(cublas, n, d_t, 1, d_t, 1, &t_t));
-        omega = t_s / t_t;
-
-        // x = x + alpha*p + omega*s
-        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_p, 1, d_x, 1));
-        CHECK_CUBLAS(cublasSaxpy(cublas, n, &omega, d_s, 1, d_x, 1));
-
-        // r = s - omega*t
-        CHECK_CUBLAS(cublasScopy(cublas, n, d_s, 1, d_r, 1));
-        float neg_omega_bc = -omega;
-        CHECK_CUBLAS(cublasSaxpy(cublas, n, &neg_omega_bc, d_t, 1, d_r, 1));
-
-        float norm_r;
-        CHECK_CUBLAS(cublasSnrm2(cublas, n, d_r, 1, &norm_r));
-        if (norm_r < tolerance) break;
-        
-        rho_prev = rho;
-    }
+        CHECK_CUDA(cudaEventRecord(slot.done, slot.stream));
+        slot.busy = true;
 
-    // Cleanup
-    cusparseDestroySpMat(matA);
-    cusparseDestroyDnVec(vecX); cusparseDestroyDnVec(vecP); cusparseDestroyDnVec(vecV);
-    cusparseDestroyDnVec(vecS); cusparseDestroyDnVec(vecT);
-    CHECK_CUDA(cudaFreeAsync(d_val, stream));
-    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
-    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
-    CHECK_CUDA(cudaFreeAsync(d_x, stream));
-    CHECK_CUDA(cudaFreeAsync(d_r, stream));
-    CHECK_CUDA(cudaFreeAsync(d_r_hat, stream));
-    CHECK_CUDA(cudaFreeAsync(d_p, stream));
-    CHECK_CUDA(cudaFreeAsync(d_v, stream));
-    CHECK_CUDA(cudaFreeAsync(d_s, stream));
-    CHECK_CUDA(cudaFreeAsync(d_t, stream));
-    CHECK_CUDA(cudaFreeAsync(d_b, stream));
-    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
-}
-
-// GPU Worker function (One thread per stream)
-void gpu_worker(int device_id, int thread_id, std::vector<MatrixTask> assigned_tasks) {
-    CHECK_CUDA(cudaSetDevice(device_id));
-    
-    cudaStream_t stream;
-    cublasHandle_t cublas_handle;
-    cusparseHandle_t cusparse_handle;
+        auto end_task = std::chrono::steady_clock::now();
+        std::chrono::duration<double> task_duration = end_task - start_task;
 
-    CHECK_CUDA(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
-    CHECK_CUBLAS(cublasCreate(&cublas_handle));
-    CHECK_CUSPARSE(cusparseCreate(&cusparse_handle));
+        std::string solver_type_str = (task.type == CGS_SOLVER) ? "CGS_SOLVER" : "BICGSTAB_SOLVER";
+        std::cout << "GPU " << device_id
+                  << " stream " << slot_id
+                  << ": " << task.path
+                  << " (" << solver_type_str << ") took "
+                  << task_duration.count() << " s\n";
+    }
 
-    for (const auto& t : assigned_tasks) {
-        auto start_task = std::chrono::steady_clock::now();
-        if (t.type == CGS_SOLVER) {
-            solve_cg_async(cublas_handle, cusparse_handle, t, stream);
-        } else {
-            solve_bicgstab_async(cublas_handle, cusparse_handle, t, stream); 
+    for (auto& slot : slots) {
+        if (slot.busy) {
+            CHECK_CUDA(cudaEventSynchronize(slot.done));
         }
-        // CHECK_CUDA(cudaStreamSynchronize(stream)); solvers are synchronious anyways do not need this
-
-        auto end_task = std::chrono::steady_clock::now();
-        std::chrono::duration<double> task_duration = end_task - start_task;
-        std::string solver_type_str = (t.type == CGS_SOLVER) ? "CGS_SOLVER" : "BICSTAB_SOLVER";
-        std::cout << "Thread " << thread_id << ": Solve time for " << t.path 
-                  << " (" << solver_type_str << ") took " << task_duration.count() << " s" << std::endl;
+        CHECK_CUDA(cudaEventDestroy(slot.done));
+        CHECK_CUBLAS(cublasDestroy(slot.cublas));
+        CHECK_CUSPARSE(cusparseDestroy(slot.cusparse));
+        CHECK_CUDA(cudaStreamDestroy(slot.stream));
     }
+}
 
-    cublasDestroy(cublas_handle);
-    cusparseDestroy(cusparse_handle);
-    cudaStreamDestroy(stream);
+// ------------------------------------------------------------
+// Producer thread: enqueue pre-generated tasks
+// ------------------------------------------------------------
+void producer(ThreadSafeQueue<MatrixTask>& work_queue, std::vector<MatrixTask> tasks) {
+    for (auto& t : tasks) {
+        work_queue.push(std::move(t));
+    }
+    work_queue.close();
 }
 
-int main() {
-    std::vector<MatrixTask> tasks;
-    
+// ------------------------------------------------------------
+// Main
+// ------------------------------------------------------------
+int main(int argc, char** argv) {
+    int num_streams = 8;
+    if (argc > 1) {
+        num_streams = std::max(1, std::atoi(argv[1]));
+    }
+
     std::cout << "[INFO] Loading matrices...\n";
-    tasks = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
-    //scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
+    std::vector<MatrixTask> tasks =
+        scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd",
+                          CGS_SOLVER);
 
     if (tasks.empty()) {
         std::cerr << "No matrices found.\n";
         return 1;
     }
 
-    int num_gpus;
+    int num_gpus = 0;
     CHECK_CUDA(cudaGetDeviceCount(&num_gpus));
-    int workers_per_gpu = 8;
-    
-    // Hierarchical Static Partitioning: Queue -> Devices -> Worker Threads
-    std::vector<std::vector<std::vector<MatrixTask>>> partitions(num_gpus, 
-                                        std::vector<std::vector<MatrixTask>>(workers_per_gpu));
-    for (size_t i = 0; i < tasks.size(); ++i) {
-        int g_id = i % num_gpus;
-        int w_id = (i / num_gpus) % workers_per_gpu;
-        partitions[g_id][w_id].push_back(std::move(tasks[i]));
+    if (num_gpus <= 0) {
+        std::cerr << "No CUDA devices found.\n";
+        return 1;
     }
 
-    std::cout << "[INFO] Processing " << tasks.size() << " tasks using " 
-              << (num_gpus * workers_per_gpu) << " worker threads (" << workers_per_gpu << " threads/GPU)...\n";
+    std::cout << "[INFO] Tasks ready: " << tasks.size() << "\n";
+    std::cout << "[INFO] GPUs: " << num_gpus << "\n";
+    std::cout << "[INFO] Streams per GPU: " << num_streams << "\n";
 
+    ThreadSafeQueue<MatrixTask> work_queue;
+
+    // Timing starts after matrix scanning/generation is already done.
     auto start = std::chrono::steady_clock::now();
 
-    std::vector<std::thread> workers;
-    for (int i = 0; i < num_gpus; ++i) {
-        for (int j = 0; j < workers_per_gpu; ++j) {
-            workers.emplace_back(gpu_worker, i, (i * 100 + j), std::move(partitions[i][j]));
-        }
+    std::thread prod_thread(producer, std::ref(work_queue), std::move(tasks));
+
+    std::vector<std::thread> consumers;
+    consumers.reserve(num_gpus);
+    for (int gpu = 0; gpu < num_gpus; ++gpu) {
+        consumers.emplace_back(gpu_consumer, gpu, num_streams, std::ref(work_queue));
     }
 
-    for (auto& w : workers) w.join();
+    prod_thread.join();
+    for (auto& t : consumers) {
+        t.join();
+    }
 
     auto end = std::chrono::steady_clock::now();
     std::chrono::duration<double> elapsed = end - start;

From 2f14daa695e736c96c7b7e68066b5f176b825795 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 2 Jun 2026 08:52:37 -0600
Subject: [PATCH 0805/1000] updated cuda to be a stream worker model instead of
 a single  device thread doing all the work. Change was made to increase
 performance

---
 .../tests/workload-test/main.native.cpp       | 342 ++++++++++--------
 1 file changed, 197 insertions(+), 145 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.native.cpp b/libcaf_cuda/tests/workload-test/main.native.cpp
index f827515ae6..a00c38e239 100644
--- a/libcaf_cuda/tests/workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/workload-test/main.native.cpp
@@ -9,26 +9,26 @@
 #include <thread>
 #include <mutex>
 #include <queue>
+#include <condition_variable>
 #include <filesystem>
 #include <memory>
 #include <algorithm>
-#include <condition_variable>
-#include <atomic>
 
 #include "sparse_utils.hpp"
 
 namespace fs = std::filesystem;
 
-// ------------------------------------------------------------
-// Error checking macros
-// ------------------------------------------------------------
+// ============================================================
+// Error Checking
+// ============================================================
+
 #define CHECK_CUDA(call)                                                   \
     do {                                                                   \
         cudaError_t status = call;                                         \
         if (status != cudaSuccess) {                                       \
             std::cerr << "CUDA Error: " << cudaGetErrorString(status)      \
                       << " at " << __FILE__ << ":" << __LINE__ << std::endl; \
-            std::exit(1);                                                  \
+            std::exit(EXIT_FAILURE);                                       \
         }                                                                  \
     } while (0)
 
@@ -36,8 +36,9 @@ namespace fs = std::filesystem;
     do {                                                                   \
         cublasStatus_t status = call;                                      \
         if (status != CUBLAS_STATUS_SUCCESS) {                             \
-            std::cerr << "cuBLAS Error at " << __FILE__ << ":" << __LINE__ << std::endl; \
-            std::exit(1);                                                  \
+            std::cerr << "cuBLAS Error at "                               \
+                      << __FILE__ << ":" << __LINE__ << std::endl;         \
+            std::exit(EXIT_FAILURE);                                       \
         }                                                                  \
     } while (0)
 
@@ -45,71 +46,61 @@ namespace fs = std::filesystem;
     do {                                                                   \
         cusparseStatus_t status = call;                                    \
         if (status != CUSPARSE_STATUS_SUCCESS) {                           \
-            std::cerr << "cuSPARSE Error at " << __FILE__ << ":" << __LINE__ << std::endl; \
-            std::exit(1);                                                  \
+            std::cerr << "cuSPARSE Error at "                             \
+                      << __FILE__ << ":" << __LINE__ << std::endl;         \
+            std::exit(EXIT_FAILURE);                                       \
         }                                                                  \
     } while (0)
 
-// ------------------------------------------------------------
-// Thread-safe queue
-// ------------------------------------------------------------
-template <typename T>
+// ============================================================
+// Thread Safe Queue
+// ============================================================
+
+template<typename T>
 class ThreadSafeQueue {
 public:
     void push(T item) {
         {
-            std::lock_guard<std::mutex> lock(mtx_);
-            if (closed_) {
-                return;
-            }
-            q_.push(std::move(item));
+            std::lock_guard<std::mutex> lock(mutex_);
+            queue_.push(std::move(item));
         }
         cv_.notify_one();
     }
 
     bool wait_pop(T& item) {
-        std::unique_lock<std::mutex> lock(mtx_);
-        cv_.wait(lock, [&] { return closed_ || !q_.empty(); });
+        std::unique_lock<std::mutex> lock(mutex_);
 
-        if (q_.empty()) {
-            return false; // closed and empty
-        }
+        cv_.wait(lock, [&] {
+            return closed_ || !queue_.empty();
+        });
+
+        if (queue_.empty())
+            return false;
 
-        item = std::move(q_.front());
-        q_.pop();
+        item = std::move(queue_.front());
+        queue_.pop();
         return true;
     }
 
     void close() {
         {
-            std::lock_guard<std::mutex> lock(mtx_);
+            std::lock_guard<std::mutex> lock(mutex_);
             closed_ = true;
         }
         cv_.notify_all();
     }
 
 private:
-    std::mutex mtx_;
+    std::queue<T> queue_;
+    std::mutex mutex_;
     std::condition_variable cv_;
-    std::queue<T> q_;
     bool closed_ = false;
 };
 
-// ------------------------------------------------------------
-// Stream slot owned by one GPU consumer thread
-// ------------------------------------------------------------
-struct StreamSlot {
-    cudaStream_t stream{};
-    cublasHandle_t cublas{};
-    cusparseHandle_t cusparse{};
-    cudaEvent_t done{};
-    bool busy = false;
-};
+// ============================================================
+// Existing Solver Implementations
+// ============================================================
 
-// ------------------------------------------------------------
-// Your existing solver functions can stay the same
-// ------------------------------------------------------------
-// Keep these as you already have them, or minimally adjust if needed.
 void solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
                     const MatrixTask& task, cudaStream_t stream) {
     int n = task.data->rows;
@@ -201,113 +192,121 @@ void solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
     CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
 }
 
+// ============================================================
+// Producer
+// ============================================================
+
+void producer(
+    ThreadSafeQueue<MatrixTask>& queue,
+    std::vector<MatrixTask> tasks)
+{
+    for (auto& task : tasks) {
+        queue.push(std::move(task));
+    }
+
+    queue.close();
+}
 
+// ============================================================
+// Worker
+// One thread == One stream
+// ============================================================
 
-// ------------------------------------------------------------
-// GPU consumer thread: one thread per GPU
-// ------------------------------------------------------------
-void gpu_consumer(int device_id, int num_streams, ThreadSafeQueue<MatrixTask>& work_queue) {
+void gpu_stream_worker(
+    int device_id,
+    int worker_id,
+    ThreadSafeQueue<MatrixTask>& queue)
+{
     CHECK_CUDA(cudaSetDevice(device_id));
 
-    std::vector<StreamSlot> slots(num_streams);
+    cudaStream_t stream;
+    cublasHandle_t cublas;
+    cusparseHandle_t cusparse;
 
-    for (int i = 0; i < num_streams; ++i) {
-        CHECK_CUDA(cudaStreamCreateWithFlags(&slots[i].stream, cudaStreamNonBlocking));
-        CHECK_CUBLAS(cublasCreate(&slots[i].cublas));
-        CHECK_CUSPARSE(cusparseCreate(&slots[i].cusparse));
-        CHECK_CUDA(cudaEventCreateWithFlags(&slots[i].done, cudaEventDisableTiming));
-        slots[i].busy = false;
-    }
+    CHECK_CUDA(
+        cudaStreamCreateWithFlags(
+            &stream,
+            cudaStreamNonBlocking));
 
-    int rr = 0;
-    auto acquire_slot = [&]() -> int {
-        while (true) {
-            for (int offset = 0; offset < num_streams; ++offset) {
-                int idx = (rr + offset) % num_streams;
-
-                if (!slots[idx].busy) {
-                    rr = (idx + 1) % num_streams;
-                    return idx;
-                }
-
-                cudaError_t q = cudaEventQuery(slots[idx].done);
-                if (q == cudaSuccess) {
-                    slots[idx].busy = false;
-                    rr = (idx + 1) % num_streams;
-                    return idx;
-                } else if (q != cudaErrorNotReady) {
-                    std::cerr << "CUDA event query failed on GPU " << device_id
-                              << ", stream " << idx << ": " << cudaGetErrorString(q) << std::endl;
-                    std::exit(1);
-                }
-            }
-            std::this_thread::yield();
-        }
-    };
+    CHECK_CUBLAS(
+        cublasCreate(&cublas));
+
+    CHECK_CUSPARSE(
+        cusparseCreate(&cusparse));
 
     MatrixTask task;
-    while (work_queue.wait_pop(task)) {
-        int slot_id = acquire_slot();
-        auto& slot = slots[slot_id];
 
-        auto start_task = std::chrono::steady_clock::now();
+    while (queue.wait_pop(task)) {
 
-        if (task.type == CGS_SOLVER) {
-            solve_cg_async(slot.cublas, slot.cusparse, task, slot.stream);
-        } 
-        else {
-            throw std::runtime_error("Unsupported solver type");
-            exit(1);
-        }
+        auto start_task =
+            std::chrono::steady_clock::now();
 
-        CHECK_CUDA(cudaEventRecord(slot.done, slot.stream));
-        slot.busy = true;
+        if (task.type == CGS_SOLVER) {
 
-        auto end_task = std::chrono::steady_clock::now();
-        std::chrono::duration<double> task_duration = end_task - start_task;
+            solve_cg_async(
+                cublas,
+                cusparse,
+                task,
+                stream);
 
-        std::string solver_type_str = (task.type == CGS_SOLVER) ? "CGS_SOLVER" : "BICGSTAB_SOLVER";
-        std::cout << "GPU " << device_id
-                  << " stream " << slot_id
-                  << ": " << task.path
-                  << " (" << solver_type_str << ") took "
-                  << task_duration.count() << " s\n";
-    }
+        } else {
 
-    for (auto& slot : slots) {
-        if (slot.busy) {
-            CHECK_CUDA(cudaEventSynchronize(slot.done));
+            throw std::runtime_error("Unsupported solver type");
+            exit(1);
         }
-        CHECK_CUDA(cudaEventDestroy(slot.done));
-        CHECK_CUBLAS(cublasDestroy(slot.cublas));
-        CHECK_CUSPARSE(cusparseDestroy(slot.cusparse));
-        CHECK_CUDA(cudaStreamDestroy(slot.stream));
-    }
-}
 
-// ------------------------------------------------------------
-// Producer thread: enqueue pre-generated tasks
-// ------------------------------------------------------------
-void producer(ThreadSafeQueue<MatrixTask>& work_queue, std::vector<MatrixTask> tasks) {
-    for (auto& t : tasks) {
-        work_queue.push(std::move(t));
+        // Optional safety.
+        // Likely redundant because the iterative
+        // solver performs host-side reductions.
+        CHECK_CUDA(cudaStreamSynchronize(stream));
+
+        auto end_task =
+            std::chrono::steady_clock::now();
+
+        std::chrono::duration<double>
+            elapsed = end_task - start_task;
+
+        std::cout
+            << "Worker "
+            << worker_id
+            << " (GPU "
+            << device_id
+            << ") solved "
+            << task.path
+            << " in "
+            << elapsed.count()
+            << " s"
+            << std::endl;
     }
-    work_queue.close();
+
+    CHECK_CUBLAS(cublasDestroy(cublas));
+    CHECK_CUSPARSE(cusparseDestroy(cusparse));
+    CHECK_CUDA(cudaStreamDestroy(stream));
 }
 
-// ------------------------------------------------------------
+// ============================================================
 // Main
-// ------------------------------------------------------------
-int main(int argc, char** argv) {
+// ============================================================
+
+int main(int argc, char** argv)
+{
     int num_streams = 8;
+
     if (argc > 1) {
-        num_streams = std::max(1, std::atoi(argv[1]));
+        num_streams =
+            std::max(1, std::atoi(argv[1]));
     }
 
     std::cout << "[INFO] Loading matrices...\n";
+
+    // --------------------------------------------------------
+    // Matrix generation NOT timed
+    // --------------------------------------------------------
+
     std::vector<MatrixTask> tasks =
-        scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd",
-                          CGS_SOLVER);
+        scan_for_matrices(
+            "/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd",
+            CGS_SOLVER);
 
     if (tasks.empty()) {
         std::cerr << "No matrices found.\n";
@@ -315,41 +314,94 @@ int main(int argc, char** argv) {
     }
 
     int num_gpus = 0;
-    CHECK_CUDA(cudaGetDeviceCount(&num_gpus));
-    if (num_gpus <= 0) {
+
+    CHECK_CUDA(
+        cudaGetDeviceCount(&num_gpus));
+
+    if (num_gpus == 0) {
         std::cerr << "No CUDA devices found.\n";
         return 1;
     }
 
-    std::cout << "[INFO] Tasks ready: " << tasks.size() << "\n";
-    std::cout << "[INFO] GPUs: " << num_gpus << "\n";
-    std::cout << "[INFO] Streams per GPU: " << num_streams << "\n";
+    std::cout
+        << "[INFO] Found "
+        << num_gpus
+        << " GPUs\n";
+
+    std::cout
+        << "[INFO] "
+        << num_streams
+        << " worker threads per GPU\n";
+
+    std::cout
+        << "[INFO] "
+        << tasks.size()
+        << " tasks queued\n";
 
     ThreadSafeQueue<MatrixTask> work_queue;
 
-    // Timing starts after matrix scanning/generation is already done.
-    auto start = std::chrono::steady_clock::now();
+    // --------------------------------------------------------
+    // Benchmark starts here
+    // --------------------------------------------------------
+
+    auto benchmark_start =
+        std::chrono::steady_clock::now();
+
+    std::thread producer_thread(
+        producer,
+        std::ref(work_queue),
+        std::move(tasks));
+
+    std::vector<std::thread> workers;
 
-    std::thread prod_thread(producer, std::ref(work_queue), std::move(tasks));
+    workers.reserve(
+        num_gpus * num_streams);
 
-    std::vector<std::thread> consumers;
-    consumers.reserve(num_gpus);
     for (int gpu = 0; gpu < num_gpus; ++gpu) {
-        consumers.emplace_back(gpu_consumer, gpu, num_streams, std::ref(work_queue));
-    }
 
-    prod_thread.join();
-    for (auto& t : consumers) {
-        t.join();
+        for (int stream = 0;
+             stream < num_streams;
+             ++stream)
+        {
+            int worker_id =
+                gpu * num_streams + stream;
+
+            workers.emplace_back(
+                gpu_stream_worker,
+                gpu,
+                worker_id,
+                std::ref(work_queue));
+        }
     }
 
-    auto end = std::chrono::steady_clock::now();
-    std::chrono::duration<double> elapsed = end - start;
+    producer_thread.join();
+
+    for (auto& worker : workers) {
+        worker.join();
+    }
 
-    std::cout << "\n===== NATIVE BENCHMARK COMPLETE =====\n";
-    std::cout << "Tasks Processed: " << tasks.size() << "\n";
-    std::cout << "Total Runtime:   " << elapsed.count() << " s\n";
-    std::cout << "======================================\n";
+    auto benchmark_end =
+        std::chrono::steady_clock::now();
+
+    std::chrono::duration<double>
+        total_time =
+            benchmark_end - benchmark_start;
+
+    std::cout << "\n";
+    std::cout << "=====================================\n";
+    std::cout << "NATIVE PRODUCER/CONSUMER BENCHMARK\n";
+    std::cout << "=====================================\n";
+    std::cout << "GPUs:               "
+              << num_gpus << "\n";
+    std::cout << "Streams per GPU:    "
+              << num_streams << "\n";
+    std::cout << "Worker Threads:     "
+              << num_gpus * num_streams << "\n";
+    std::cout << "Tasks Processed:    "
+              << tasks.size() << "\n";
+    std::cout << "Total Runtime:      "
+              << total_time.count() << " s\n";
+    std::cout << "=====================================\n";
 
     return 0;
 }
\ No newline at end of file

From dc49f7cd8b031622560bd169f81903496ab19541 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 2 Jun 2026 09:14:47 -0600
Subject: [PATCH 0806/1000] updated producer thread  to have a batch model
 where it dispatches batches of work

---
 .../tests/workload-test/main.native.cpp       | 188 +++++++++++++-----
 1 file changed, 138 insertions(+), 50 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.native.cpp b/libcaf_cuda/tests/workload-test/main.native.cpp
index a00c38e239..27e8f5fef6 100644
--- a/libcaf_cuda/tests/workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/workload-test/main.native.cpp
@@ -13,11 +13,16 @@
 #include <filesystem>
 #include <memory>
 #include <algorithm>
+#include <random>
+#include <atomic>
+#include <cmath>
 
 #include "sparse_utils.hpp"
 
 namespace fs = std::filesystem;
 
+constexpr uint32_t WORKLOAD_SEED = 42;
+
 // ============================================================
 // Error Checking
 // ============================================================
@@ -71,21 +76,22 @@ class ThreadSafeQueue {
         std::unique_lock<std::mutex> lock(mutex_);
 
         cv_.wait(lock, [&] {
-            return closed_ || !queue_.empty();
+            return shutdown_ || !queue_.empty();
         });
 
-        if (queue_.empty())
-            return false;
+        if (!queue_.empty()) {
+            item = std::move(queue_.front());
+            queue_.pop();
+            return true;
+        }
 
-        item = std::move(queue_.front());
-        queue_.pop();
-        return true;
+        return false;
     }
 
-    void close() {
+    void signal_shutdown() {
         {
             std::lock_guard<std::mutex> lock(mutex_);
-            closed_ = true;
+            shutdown_ = true;
         }
         cv_.notify_all();
     }
@@ -94,9 +100,43 @@ class ThreadSafeQueue {
     std::queue<T> queue_;
     std::mutex mutex_;
     std::condition_variable cv_;
-    bool closed_ = false;
+    bool shutdown_ = false;
 };
 
+// ============================================================
+// Workload Generation Helpers
+// ============================================================
+
+std::chrono::milliseconds generate_random_interval(
+    std::mt19937& rng,
+    double mean_ms)
+{
+    std::exponential_distribution<double> dist(
+        1.0 / mean_ms);
+
+    return std::chrono::milliseconds(
+        static_cast<int>(dist(rng)));
+}
+
+std::vector<MatrixTask> generate_batch(
+    const std::vector<MatrixTask>& matrix_pool,
+    std::mt19937& rng,
+    size_t batch_size)
+{
+    std::vector<MatrixTask> batch;
+    batch.reserve(batch_size);
+
+    std::uniform_int_distribution<size_t> dist(
+        0,
+        matrix_pool.size() - 1);
+
+    for (size_t i = 0; i < batch_size; ++i) {
+        batch.push_back(matrix_pool[dist(rng)]);
+    }
+
+    return batch;
+}
+
 // ============================================================
 // Existing Solver Implementations
 // ============================================================
@@ -198,18 +238,51 @@ void solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
 
 void producer(
     ThreadSafeQueue<MatrixTask>& queue,
-    std::vector<MatrixTask> tasks)
+    const std::vector<MatrixTask>& matrix_pool,
+    int num_batches,
+    int batch_size,
+    double mean_arrival_ms)
 {
-    for (auto& task : tasks) {
-        queue.push(std::move(task));
+    std::mt19937 rng(WORKLOAD_SEED);
+
+    for (int batch = 0; batch < num_batches; ++batch) {
+
+        auto sleep_time =
+            generate_random_interval(
+                rng,
+                mean_arrival_ms);
+
+        std::this_thread::sleep_for(
+            sleep_time);
+
+        auto tasks =
+            generate_batch(
+                matrix_pool,
+                rng,
+                batch_size);
+
+        std::cout
+            << "[PRODUCER] Dispatching batch "
+            << batch + 1
+            << "/"
+            << num_batches
+            << " ("
+            << tasks.size()
+            << " tasks, slept "
+            << sleep_time.count()
+            << " ms)"
+            << std::endl;
+
+        for (auto& task : tasks) {
+            queue.push(std::move(task));
+        }
     }
 
-    queue.close();
+    queue.signal_shutdown();
 }
 
 // ============================================================
 // Worker
-// One thread == One stream
 // ============================================================
 
 void gpu_stream_worker(
@@ -251,13 +324,10 @@ void gpu_stream_worker(
 
         } else {
 
-            throw std::runtime_error("Unsupported solver type");
-            exit(1);
+            throw std::runtime_error(
+                "Unsupported solver type");
         }
 
-        // Optional safety.
-        // Likely redundant because the iterative
-        // solver performs host-side reductions.
         CHECK_CUDA(cudaStreamSynchronize(stream));
 
         auto end_task =
@@ -291,24 +361,30 @@ void gpu_stream_worker(
 int main(int argc, char** argv)
 {
     int num_streams = 8;
+    int num_batches = 25;
+    int batch_size = 8;
+    double mean_arrival_ms = 1000.0;
 
-    if (argc > 1) {
-        num_streams =
-            std::max(1, std::atoi(argv[1]));
-    }
+    if (argc > 1)
+        num_streams = std::max(1, std::atoi(argv[1]));
 
-    std::cout << "[INFO] Loading matrices...\n";
+    if (argc > 2)
+        num_batches = std::max(1, std::atoi(argv[2]));
+
+    if (argc > 3)
+        batch_size = std::max(1, std::atoi(argv[3]));
 
-    // --------------------------------------------------------
-    // Matrix generation NOT timed
-    // --------------------------------------------------------
+    if (argc > 4)
+        mean_arrival_ms = std::atof(argv[4]);
 
-    std::vector<MatrixTask> tasks =
+    std::cout << "[INFO] Loading matrices...\n";
+
+    std::vector<MatrixTask> matrix_pool =
         scan_for_matrices(
             "/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd",
             CGS_SOLVER);
 
-    if (tasks.empty()) {
+    if (matrix_pool.empty()) {
         std::cerr << "No matrices found.\n";
         return 1;
     }
@@ -329,20 +405,31 @@ int main(int argc, char** argv)
         << " GPUs\n";
 
     std::cout
-        << "[INFO] "
+        << "[INFO] Matrix pool size: "
+        << matrix_pool.size()
+        << "\n";
+
+    std::cout
+        << "[INFO] Streams/GPU: "
         << num_streams
-        << " worker threads per GPU\n";
+        << "\n";
 
     std::cout
-        << "[INFO] "
-        << tasks.size()
-        << " tasks queued\n";
+        << "[INFO] Batches: "
+        << num_batches
+        << "\n";
 
-    ThreadSafeQueue<MatrixTask> work_queue;
+    std::cout
+        << "[INFO] Batch size: "
+        << batch_size
+        << "\n";
+
+    std::cout
+        << "[INFO] Mean arrival: "
+        << mean_arrival_ms
+        << " ms\n";
 
-    // --------------------------------------------------------
-    // Benchmark starts here
-    // --------------------------------------------------------
+    ThreadSafeQueue<MatrixTask> work_queue;
 
     auto benchmark_start =
         std::chrono::steady_clock::now();
@@ -350,7 +437,10 @@ int main(int argc, char** argv)
     std::thread producer_thread(
         producer,
         std::ref(work_queue),
-        std::move(tasks));
+        std::cref(matrix_pool),
+        num_batches,
+        batch_size,
+        mean_arrival_ms);
 
     std::vector<std::thread> workers;
 
@@ -389,18 +479,16 @@ int main(int argc, char** argv)
 
     std::cout << "\n";
     std::cout << "=====================================\n";
-    std::cout << "NATIVE PRODUCER/CONSUMER BENCHMARK\n";
+    std::cout << "IRREGULAR WORKLOAD BENCHMARK\n";
     std::cout << "=====================================\n";
-    std::cout << "GPUs:               "
-              << num_gpus << "\n";
-    std::cout << "Streams per GPU:    "
-              << num_streams << "\n";
-    std::cout << "Worker Threads:     "
-              << num_gpus * num_streams << "\n";
-    std::cout << "Tasks Processed:    "
-              << tasks.size() << "\n";
-    std::cout << "Total Runtime:      "
-              << total_time.count() << " s\n";
+    std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
+    std::cout << "GPUs:               " << num_gpus << "\n";
+    std::cout << "Streams per GPU:    " << num_streams << "\n";
+    std::cout << "Worker Threads:     " << num_gpus * num_streams << "\n";
+    std::cout << "Batches:            " << num_batches << "\n";
+    std::cout << "Batch Size:         " << batch_size << "\n";
+    std::cout << "Mean Arrival (ms):  " << mean_arrival_ms << "\n";
+    std::cout << "Total Runtime:      " << total_time.count() << " s\n";
     std::cout << "=====================================\n";
 
     return 0;

From 3e4e4b27244b0aff797dfdcd846937a846dd571b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 2 Jun 2026 09:22:53 -0600
Subject: [PATCH 0807/1000] updated batch size to be bigger

---
 libcaf_cuda/tests/workload-test/main.native.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/workload-test/main.native.cpp b/libcaf_cuda/tests/workload-test/main.native.cpp
index 27e8f5fef6..1239b0e30b 100644
--- a/libcaf_cuda/tests/workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/workload-test/main.native.cpp
@@ -362,7 +362,7 @@ int main(int argc, char** argv)
 {
     int num_streams = 8;
     int num_batches = 25;
-    int batch_size = 8;
+    int batch_size = 100;
     double mean_arrival_ms = 1000.0;
 
     if (argc > 1)

From 22b0bf25d40450c80c64ea729f2d0bac7b02f4e0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 2 Jun 2026 09:26:55 -0600
Subject: [PATCH 0808/1000] refactored code by moving batch generation and
 sleep interval generation logic out of main.native and into sparse utils

---
 .../workload-test/CMakeLists.txt              |  1 +
 .../tests/workload-test/main.native.cpp       | 34 -------------------
 .../tests/workload-test/sparse_utils.cpp      | 30 ++++++++++++++++
 .../tests/workload-test/sparse_utils.hpp      | 14 +++++++-
 4 files changed, 44 insertions(+), 35 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/workload-test/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/workload-test/CMakeLists.txt
index ecf8d6157d..a833395d77 100644
--- a/libcaf_cuda/tests/benchmark-tests/workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/benchmark-tests/workload-test/CMakeLists.txt
@@ -43,4 +43,5 @@ target_link_libraries(work-stealing
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
     CUDA::cublas
+    CUDA::cusparse
 )
diff --git a/libcaf_cuda/tests/workload-test/main.native.cpp b/libcaf_cuda/tests/workload-test/main.native.cpp
index 1239b0e30b..9c7a48b6c4 100644
--- a/libcaf_cuda/tests/workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/workload-test/main.native.cpp
@@ -103,40 +103,6 @@ class ThreadSafeQueue {
     bool shutdown_ = false;
 };
 
-// ============================================================
-// Workload Generation Helpers
-// ============================================================
-
-std::chrono::milliseconds generate_random_interval(
-    std::mt19937& rng,
-    double mean_ms)
-{
-    std::exponential_distribution<double> dist(
-        1.0 / mean_ms);
-
-    return std::chrono::milliseconds(
-        static_cast<int>(dist(rng)));
-}
-
-std::vector<MatrixTask> generate_batch(
-    const std::vector<MatrixTask>& matrix_pool,
-    std::mt19937& rng,
-    size_t batch_size)
-{
-    std::vector<MatrixTask> batch;
-    batch.reserve(batch_size);
-
-    std::uniform_int_distribution<size_t> dist(
-        0,
-        matrix_pool.size() - 1);
-
-    for (size_t i = 0; i < batch_size; ++i) {
-        batch.push_back(matrix_pool[dist(rng)]);
-    }
-
-    return batch;
-}
-
 // ============================================================
 // Existing Solver Implementations
 // ============================================================
diff --git a/libcaf_cuda/tests/workload-test/sparse_utils.cpp b/libcaf_cuda/tests/workload-test/sparse_utils.cpp
index ca4c3ca360..d0687907b3 100644
--- a/libcaf_cuda/tests/workload-test/sparse_utils.cpp
+++ b/libcaf_cuda/tests/workload-test/sparse_utils.cpp
@@ -102,4 +102,34 @@ int generate_random_sleep_ms(int min_ms, int max_ms) {
     static std::mt19937 gen(rd());
     std::uniform_int_distribution<> dis(min_ms, max_ms);
     return dis(gen);
+}
+
+std::chrono::milliseconds generate_random_interval(
+    std::mt19937& rng,
+    double mean_ms)
+{
+    std::exponential_distribution<double> dist(
+        1.0 / mean_ms);
+
+    return std::chrono::milliseconds(
+        static_cast<int>(dist(rng)));
+}
+
+std::vector<MatrixTask> generate_batch(
+    const std::vector<MatrixTask>& matrix_pool,
+    std::mt19937& rng,
+    size_t batch_size)
+{
+    std::vector<MatrixTask> batch;
+    batch.reserve(batch_size);
+
+    std::uniform_int_distribution<size_t> dist(
+        0,
+        matrix_pool.size() - 1);
+
+    for (size_t i = 0; i < batch_size; ++i) {
+        batch.push_back(matrix_pool[dist(rng)]);
+    }
+
+    return batch;
 }
\ No newline at end of file
diff --git a/libcaf_cuda/tests/workload-test/sparse_utils.hpp b/libcaf_cuda/tests/workload-test/sparse_utils.hpp
index c946aeccb2..e793d3b541 100644
--- a/libcaf_cuda/tests/workload-test/sparse_utils.hpp
+++ b/libcaf_cuda/tests/workload-test/sparse_utils.hpp
@@ -4,6 +4,8 @@
 #include <string>
 #include <cstdint>
 #include <memory>
+#include <random>
+#include <chrono>
 
 enum SolverType { CGS_SOLVER, BICSTAB_SOLVER };
 
@@ -54,4 +56,14 @@ SparseMatrixCSR convert_coo_to_csr(const SparseMatrixCOO& coo);
 std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x);
 
 std::vector<MatrixTask> scan_for_matrices(const std::string& dir, SolverType type);
-int generate_random_sleep_ms(int min_ms, int max_ms);
\ No newline at end of file
+int generate_random_sleep_ms(int min_ms, int max_ms);
+
+// Workload generation helpers
+std::chrono::milliseconds generate_random_interval(
+    std::mt19937& rng,
+    double mean_ms);
+
+std::vector<MatrixTask> generate_batch(
+    const std::vector<MatrixTask>& matrix_pool,
+    std::mt19937& rng,
+    size_t batch_size);
\ No newline at end of file

From 677604f3f23b19b672582b6dfc76e0a1ab1fa118 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 2 Jun 2026 10:20:49 -0600
Subject: [PATCH 0809/1000] modified actor test to be more producer consumer
 like and similar to main.native.hpp

---
 libcaf_cuda/tests/workload-test/main.test.cpp | 142 +++++++++++++++---
 1 file changed, 117 insertions(+), 25 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index ebe5a49647..ec0353f337 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -17,6 +17,9 @@
 using namespace caf;
 using namespace caf::cuda;
 namespace fs = std::filesystem;
+
+constexpr uint32_t WORKLOAD_SEED = 42;
+
 template <class Inspector>
 bool inspect(Inspector& f, SolverType& x) {
     auto val = static_cast<int>(x);
@@ -33,6 +36,7 @@ CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
     CAF_ADD_ATOM(workload_test, release_memory_atom)
     CAF_ADD_ATOM(workload_test, request_work_atom)
     CAF_ADD_ATOM(workload_test, worker_done_atom)
+    CAF_ADD_ATOM(workload_test, work_tick_atom)
     CAF_ADD_TYPE_ID(workload_test, (SolverType))
     CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
     CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
@@ -45,23 +49,90 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
 
 // ---------------------------- GLOBAL TASK POOL ----------------------------
+struct pending_request {
+    caf::response_promise promise;
+    size_t requested_size;
+};
+
 struct pool_state {
-    std::shared_ptr<std::vector<MatrixTask>> tasks;
-    size_t next_task_idx = 0;
+    std::vector<MatrixTask> matrix_pool;
+    std::deque<MatrixTask> work_buffer;
+    std::vector<pending_request> pending_requests;
+    std::mt19937 rng;
+    int batches_remaining;
+    int batch_size;
+    double mean_arrival_ms;
+    bool production_finished = false;
 };
 
-behavior global_task_pool(stateful_actor<pool_state>* self, std::shared_ptr<std::vector<MatrixTask>> tasks) {
-    self->state().tasks = std::move(tasks);
+behavior global_task_pool(stateful_actor<pool_state>* self, 
+                          std::vector<MatrixTask> matrix_pool,
+                          int num_batches, int batch_size, double mean_arrival_ms) {
+    auto& st = self->state();
+    st.matrix_pool = std::move(matrix_pool);
+    st.batches_remaining = num_batches;
+    st.batch_size = batch_size;
+    st.mean_arrival_ms = mean_arrival_ms;
+    st.rng.seed(WORKLOAD_SEED);
+
+    self->mail(work_tick_atom_v).send(self);
+
     return {
+        [=](work_tick_atom) {
+            auto& st = self->state();
+            if (st.batches_remaining > 0) {
+                auto tasks = generate_batch(st.matrix_pool, st.rng, st.batch_size);
+                for (auto& t : tasks)
+                    st.work_buffer.push_back(std::move(t));
+                st.batches_remaining--;
+
+                // Satisfy pending requests from the new work
+                while (!st.pending_requests.empty() && !st.work_buffer.empty()) {
+                    auto req = std::move(st.pending_requests.front());
+                    st.pending_requests.erase(st.pending_requests.begin());
+
+                    size_t count = std::min(req.requested_size, st.work_buffer.size());
+                    std::vector<MatrixTask> batch;
+                    for (size_t i = 0; i < count; ++i) {
+                        batch.push_back(std::move(st.work_buffer.front()));
+                        st.work_buffer.pop_front();
+                    }
+                    req.promise.deliver(std::move(batch));
+                }
+
+                if (st.batches_remaining > 0) {
+                    auto interval = generate_random_interval(st.rng, st.mean_arrival_ms);
+                    std::cout << "[PRODUCER] going to sleep for " << interval.count() << " ms\n";
+                    self->mail(work_tick_atom_v).delay(interval).send(self);
+                } else {
+                    st.production_finished = true;
+                    // If the buffer is empty and no more batches are coming, signal EOS to anyone waiting
+                    if (st.work_buffer.empty()) {
+                        for (auto& req : st.pending_requests)
+                            req.promise.deliver(sec::end_of_stream);
+                        st.pending_requests.clear();
+                    }
+                }
+            }
+        },
         [=](get_work_atom, size_t batch_size) -> result<std::vector<MatrixTask>> {
             auto& st = self->state();
-            if (st.next_task_idx >= st.tasks->size())
+            if (!st.work_buffer.empty()) {
+                size_t count = std::min(batch_size, st.work_buffer.size());
+                std::vector<MatrixTask> batch;
+                for (size_t i = 0; i < count; ++i) {
+                    batch.push_back(std::move(st.work_buffer.front()));
+                    st.work_buffer.pop_front();
+                }
+                return batch;
+            }
+
+            if (st.production_finished)
                 return sec::end_of_stream;
-            size_t count = std::min(batch_size, st.tasks->size() - st.next_task_idx);
-            std::vector<MatrixTask> batch(st.tasks->begin() + st.next_task_idx, 
-                                          st.tasks->begin() + st.next_task_idx + count);
-            st.next_task_idx += count;
-            return batch;
+
+            auto promise = self->make_response_promise();
+            st.pending_requests.push_back({promise, batch_size});
+            return promise;
         }
     };
 }
@@ -240,17 +311,17 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
 struct supervisor_state {
     int total_tasks;
     int completed = 0;
-    std::shared_ptr<std::vector<MatrixTask>> tasks_holder; // Anchors shared_ptr reference count
 };
 
-behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, int total, std::shared_ptr<std::vector<MatrixTask>> tasks) {
-    self->state().total_tasks = total;
-    self->state().tasks_holder = tasks; // Retain ownership within supervisor state
-    auto pool = self->spawn(global_task_pool, tasks); // Pass a copy, don't move it
+behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, 
+                              std::vector<MatrixTask> matrix_pool,
+                              int num_streams, int num_batches, int batch_size, double mean_arrival_ms) {
+    self->state().total_tasks = num_batches * batch_size;
+    auto pool = self->spawn(global_task_pool, std::move(matrix_pool), num_batches, batch_size, mean_arrival_ms);
     
     manager& mgr = manager::get();
     int num_gpus = mgr.get_num_devices();
-    int workers_per_gpu = 8;
+    int workers_per_gpu = num_streams;
     int max_in_flight_tasks_per_worker = 1;
 
     for (int i = 0; i < num_gpus; ++i) {
@@ -264,7 +335,7 @@ behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, int total,
         [=](int done) {
             self->state().completed += done;
             if (self->state().completed >= self->state().total_tasks) {
-                std::cout << "\n[DONE] All " << self->state().total_tasks << " matrices processed.\n";
+                std::cout << "\n[DONE] All " << self->state().total_tasks << " tasks processed.\n";
                 self->quit();
             }
         }
@@ -274,32 +345,53 @@ behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, int total,
 void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
     
+    int num_streams = 8;
+    int num_batches = 25;
+    int batch_size = 100;
+    double mean_arrival_ms = 1000.0;
+
+    // // Basic command line argument parsing similar to native
+    // auto& args = sys.config().remainder;
+    // if (args.size() > 0) num_streams = std::max(1, std::stoi(args[0]));
+    // if (args.size() > 1) num_batches = std::max(1, std::stoi(args[1]));
+    // if (args.size() > 2) batch_size = std::max(1, std::stoi(args[2]));
+    // if (args.size() > 3) mean_arrival_ms = std::stod(args[3]);
+
     std::cout << "[INFO] Loading matrices into memory...\n";
     auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
     //scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
-    auto tasks = std::make_shared<std::vector<MatrixTask>>(std::move(tasks_vec));
     
-    if (tasks->empty()) {
+    if (tasks_vec.empty()) {
         std::cerr << "No matrix files found in search paths.\n";
         manager::shutdown();
         return;
     }
 
-    auto task_count = tasks->size();
-    std::cout << "[INFO] Found " << task_count << " matrices. Spawning workload...\n";
+    std::cout << "[INFO] Matrix pool size: " << tasks_vec.size() << "\n";
+    std::cout << "[INFO] Streams/GPU:      " << num_streams << "\n";
+    std::cout << "[INFO] Batches:          " << num_batches << "\n";
+    std::cout << "[INFO] Batch size:       " << batch_size << "\n";
+    std::cout << "[INFO] Mean arrival:     " << mean_arrival_ms << " ms\n";
 
     auto start = std::chrono::steady_clock::now();
 
-    // Fixed Exit Race: Pass tasks directly by copy to keep it active inside caf_main frame
-    sys.spawn(supervisor_actor_fun, static_cast<int>(task_count), tasks);
+    sys.spawn(supervisor_actor_fun, std::move(tasks_vec), num_streams, num_batches, batch_size, mean_arrival_ms);
     sys.await_all_actors_done();
 
     auto end = std::chrono::steady_clock::now();
     std::chrono::duration<double> elapsed = end - start;
 
+    int total_tasks = num_batches * batch_size;
+
     std::cout << "\n===== BENCHMARK COMPLETE =====\n";
-    std::cout << "Tasks Processed: " << task_count << "\n";
-    std::cout << "Total Runtime:   " << elapsed.count() << " s\n";
+    std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
+    std::cout << "Streams per GPU:    " << num_streams << "\n";
+    std::cout << "Batches:            " << num_batches << "\n";
+    std::cout << "Batch Size:         " << batch_size << "\n";
+    std::cout << "Mean Arrival (ms):  " << mean_arrival_ms << "\n";
+    std::cout << "Tasks Processed:    " << total_tasks << "\n";
+    std::cout << "Total Runtime:      " << elapsed.count() << " s\n";
+    std::cout << "Throughput:         " << total_tasks / elapsed.count() << " tasks/s\n";
     std::cout << "==============================\n";
 
     manager::shutdown();

From 02ee3443c7e98f53e6f0a8b2c69ca34359a0b34b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 2 Jun 2026 10:38:05 -0600
Subject: [PATCH 0810/1000] moved actor facade into worker actor state to
 prevent creation overhead

---
 libcaf_cuda/tests/workload-test/main.test.cpp | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index ec0353f337..3c070f1fc1 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -240,14 +240,17 @@ struct worker_state {
     std::string current_matrix_path;
     std::chrono::steady_clock::time_point task_start;
     SolverType current_solver_type;
+    caf::actor cg_facade;
 };
 
 behavior sparse_worker_fun(stateful_actor<worker_state>* self,
                            caf::actor supervisor, caf::actor device_actor, int dev_id, int stream_id) {
-    self->state().supervisor = supervisor;
-    self->state().device_actor = device_actor;
-    self->state().device_id = dev_id;
-    self->state().stream_id = stream_id;
+    auto& st = self->state();
+    st.supervisor = supervisor;
+    st.device_actor = device_actor;
+    st.device_id = dev_id;
+    st.stream_id = stream_id;
+    st.cg_facade = self->spawn<sparse_cg_facade<float>>(0);
 
     self->mail(request_work_atom_v).send(self);
 
@@ -263,10 +266,9 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
                     self->state().current_data = data;
                     if (type == CGS_SOLVER) {
                         // Use the optimized CG facade. It responds with (r_id, index, solution, meta)
-                        auto facade = self->spawn<sparse_cg_facade<float>>(0);
                         self->mail(std::move(rp), std::move(ci), std::move(val),
                                    std::move(b), std::move(x),
-                                   matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id).send(facade);
+                                   matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id).send(self->state().cg_facade);
                     } else {
                         // BiCGSTAB still uses the standard stateful actor. It responds with (solution, meta)
                         auto solver = self->spawn<sparse_bicgstab_actor<float>>(

From 645c464d4857ceaa5f540db536e9a31df37dd6c5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 3 Jun 2026 08:52:12 -0600
Subject: [PATCH 0811/1000] removed device actor to see if there is a
 bottleneck

---
 libcaf_cuda/tests/workload-test/main.test.cpp | 152 ++++--------------
 1 file changed, 33 insertions(+), 119 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index 3c070f1fc1..1bdbfe9348 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -137,102 +137,9 @@ behavior global_task_pool(stateful_actor<pool_state>* self,
     };
 }
 
-// ---------------------------- DEVICE/GPU ACTOR ----------------------------
-struct device_actor_state {
-    caf::actor global_pool;
-    std::deque<MatrixTask> local_tasks;
-    std::vector<caf::response_promise> pending_promises; // Store untyped promises cleanly
-    int active_workers = 0;
-    int device_id = -1;
-    bool fetching = false;
-    size_t low_water_mark;
-    size_t batch_size;
-};
-
-behavior gpu_device_actor(stateful_actor<device_actor_state>* self,
-                          caf::actor global_pool, int num_workers, int dev_id, int max_in_flight) {
-    self->state().global_pool = global_pool;
-    self->state().device_id = dev_id;
-    self->state().active_workers = num_workers;
-
-    self->state().low_water_mark = static_cast<size_t>(num_workers * max_in_flight);
-    self->state().batch_size = self->state().low_water_mark * 2;
-
-    auto satisfy_promises = [=]() {
-        auto& st = self->state();
-        while (!st.pending_promises.empty() && !st.local_tasks.empty()) {
-            auto promise = std::move(st.pending_promises.front());
-            st.pending_promises.erase(st.pending_promises.begin());
-
-            MatrixTask t = std::move(st.local_tasks.front());
-            st.local_tasks.pop_front();
-
-            auto& data = *t.data;
-            promise.deliver(t.type, t.path, create_in_arg((const std::vector<int32_t>&)data.row_ptr), create_in_arg((const std::vector<int32_t>&)data.col_indices), 
-                            create_in_arg(data.values), create_in_arg(data.b), create_in_out_arg(data.x_guess), 
-                            (int)data.row_ptr.size() - 1, (int)data.values.size(), t.data);
-        }
-    };
-
-    auto refill = [=]() {
-        auto& st = self->state();
-        if (st.fetching)
-            return;
-
-        if (st.local_tasks.size() >= st.low_water_mark && st.pending_promises.empty())
-            return;
-
-        st.fetching = true;
-        self->mail(get_work_atom_v, st.batch_size).request(st.global_pool, infinite).then(
-            [=](std::vector<MatrixTask>& batch) {
-                auto& st_inner = self->state();
-                st_inner.fetching = false;
-
-                for (auto& task : batch)
-                    st_inner.local_tasks.push_back(std::move(task));
-
-                satisfy_promises();
-            },
-            [=](error& err) {
-                auto& st_inner = self->state();
-                st_inner.fetching = false;
-                
-                for (auto& promise : st_inner.pending_promises) {
-                    promise.deliver(err);
-                }
-                st_inner.pending_promises.clear();
-            }
-        );
-    };
-
-    refill();
-
-    return {
-        [=](get_work_atom) -> result<SolverType, std::string, in<int>, in<int>, in<float>, in<float>, in_out<float>, int, int, std::shared_ptr<MatrixData>> {
-            auto& st = self->state();
-            
-            // Fixed Type Mismatch: Explicitly use untyped response_promise
-            caf::response_promise promise = self->make_response_promise();
-            st.pending_promises.push_back(promise);
-            
-            satisfy_promises();
-            refill();
-
-            return promise;
-        },
-        [=](release_memory_atom, std::string path) {
-            // Managed entirely by shared_ptrs
-        },
-        [=](worker_done_atom) {
-            if (--self->state().active_workers <= 0)
-                self->quit();
-        }
-    };
-}
-
 // ---------------------------- WORKER ACTOR ----------------------------
 struct worker_state {
-    caf::actor device_actor;
+    caf::actor global_pool;
     caf::actor supervisor;
     int device_id;
     int stream_id;
@@ -244,45 +151,55 @@ struct worker_state {
 };
 
 behavior sparse_worker_fun(stateful_actor<worker_state>* self,
-                           caf::actor supervisor, caf::actor device_actor, int dev_id, int stream_id) {
+                           caf::actor supervisor, caf::actor global_pool, int dev_id, int stream_id) {
     auto& st = self->state();
     st.supervisor = supervisor;
-    st.device_actor = device_actor;
+    st.global_pool = global_pool;
     st.device_id = dev_id;
     st.stream_id = stream_id;
-    st.cg_facade = self->spawn<sparse_cg_facade<float>>(0);
+    st.cg_facade = self->spawn<sparse_cg_facade<float>, linked>(0);
 
     self->mail(request_work_atom_v).send(self);
 
     return {
         [=](request_work_atom) {
-            self->mail(get_work_atom_v).request(self->state().device_actor, infinite).then(
-                [=](SolverType type, std::string path, in<int> rp, in<int> ci, in<float> val,
-                    in<float> b, in_out<float> x, int rows, int nnz, std::shared_ptr<MatrixData> data) {
-                    self->state().current_matrix_path = path;
-                    self->state().current_solver_type = type;
+            self->mail(get_work_atom_v, size_t{1}).request(self->state().global_pool, infinite).then(
+                [=](std::vector<MatrixTask>& batch) {
+                    if (batch.empty()) {
+                        self->mail(request_work_atom_v).send(self);
+                        return;
+                    }
+                    auto& task = batch.front();
+                    auto& data = *task.data;
+
+                    self->state().current_matrix_path = task.path;
+                    self->state().current_solver_type = task.type;
                     self->state().task_start = std::chrono::steady_clock::now();
-                    auto start_spawn = std::chrono::steady_clock::now();
-                    self->state().current_data = data;
-                    if (type == CGS_SOLVER) {
+                    self->state().current_data = task.data;
+
+                    if (task.type == CGS_SOLVER) {
                         // Use the optimized CG facade. It responds with (r_id, index, solution, meta)
-                        self->mail(std::move(rp), std::move(ci), std::move(val),
-                                   std::move(b), std::move(x),
-                                   matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id).send(self->state().cg_facade);
+                        self->mail(create_in_arg((const std::vector<int32_t>&)data.row_ptr),
+                                   create_in_arg((const std::vector<int32_t>&)data.col_indices),
+                                   create_in_arg(data.values), create_in_arg(data.b),
+                                   create_in_out_arg(data.x_guess), matrix_format::csr,
+                                   (int)data.row_ptr.size() - 1, (int)data.values.size(),
+                                   1e-5f, 2000, dev_id, stream_id).send(self->state().cg_facade);
                     } else {
                         // BiCGSTAB still uses the standard stateful actor. It responds with (solution, meta)
                         auto solver = self->spawn<sparse_bicgstab_actor<float>>(
-                            std::move(rp), std::move(ci), std::move(val),
-                            std::move(b), std::move(x),
-                            matrix_format::csr, rows, nnz, 1e-5f, 2000, dev_id, stream_id, actor_cast<actor>(self));
+                            create_in_arg((const std::vector<int32_t>&)data.row_ptr),
+                            create_in_arg((const std::vector<int32_t>&)data.col_indices),
+                            create_in_arg(data.values), create_in_arg(data.b),
+                            create_in_out_arg(data.x_guess), matrix_format::csr,
+                            (int)data.row_ptr.size() - 1, (int)data.values.size(),
+                            1e-5f, 2000, dev_id, stream_id, actor_cast<actor>(self));
                         self->mail(start_atom_v).send(solver);
                     }
                 },
                 [=](error& err) {
-                    if (err == sec::end_of_stream) {
-                        self->mail(worker_done_atom_v).send(self->state().device_actor);
+                    if (err == sec::end_of_stream)
                         self->quit();
-                    }
                 }
             );
         },
@@ -323,13 +240,10 @@ behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self,
     
     manager& mgr = manager::get();
     int num_gpus = mgr.get_num_devices();
-    int workers_per_gpu = num_streams;
-    int max_in_flight_tasks_per_worker = 1;
 
     for (int i = 0; i < num_gpus; ++i) {
-        auto broker = self->spawn(gpu_device_actor, pool, workers_per_gpu, i, max_in_flight_tasks_per_worker);
-        for (int j = 0; j < workers_per_gpu; ++j) {
-            self->spawn(sparse_worker_fun, self, broker, i, (i * 100) + j);
+        for (int j = 0; j < num_streams; ++j) {
+            self->spawn(sparse_worker_fun, self, pool, i, (i * 100) + j);
         }
     }
 

From a5e98e071efe560e326dc34d5539841e495e8ea9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 3 Jun 2026 11:06:09 -0600
Subject: [PATCH 0812/1000] updated exampel to align with changes made to
 asynchronous message passing stuff

---
 .../custom-actor-examples/mmul.example.cpp    | 439 ++----------------
 1 file changed, 29 insertions(+), 410 deletions(-)

diff --git a/libcaf_cuda/examples/custom-actor-examples/mmul.example.cpp b/libcaf_cuda/examples/custom-actor-examples/mmul.example.cpp
index 8d7fb920ef..fcc4d79c35 100644
--- a/libcaf_cuda/examples/custom-actor-examples/mmul.example.cpp
+++ b/libcaf_cuda/examples/custom-actor-examples/mmul.example.cpp
@@ -2,9 +2,8 @@
  * the entry point command runner, enabling the user to create their own custom gpu
  * actor
  * Be sure to run compile_kernels.sh
- * We will show how to create an actor that generates 2 random matrices 
- * then sends it to itself for matrix multiplication and then sends its result 
- * to itself for verification
+ * We will show how to create an actor that runs matrix multiply and sends a message to itself
+ * once the data has finished being asynchronously transfered 
  * Note that we will be using create_program_from_cubin and create_program_from_fatbin
  * methods these are the recommends and supported methods, as while you can
  * use create_program method, you are likely to run into unsupported toolchain 
@@ -13,8 +12,6 @@
 
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
-#include <iostream>
-#include <iomanip>
 #include <vector>
 #include <chrono>
 #include <thread>
@@ -47,31 +44,10 @@ struct mmul_actor_state {
 };
 
 
-//commands classes used to launch kernels
-//how they work is there templates is the sequence of wrapper types that are
-//used by the gpu actors software to deduce what is to be done with the data
-//arguments are expected to appear in the order that they would appear in the 
-//kernel, for instance the out<int> is represented of the matrixC 
-using mmulCommand = caf::cuda::command_runner<
-      in<int>, //matrixA a readonly integer buffer
-      in<int>, //matrixB a readonly integer buffer
-      out<int>, //matrixC a writeonly integer buffer, you can just pass in integer for size and it will automatically allocate a buffer for you on the gpu 
-      in<int> //int N, the size of the matrices, you can just pass in a single integer and it will recognize this 
-      >;
-
-using matrixGenCommand = caf::cuda::command_runner<
-      out<int>, //writeonly matrix buffer 
-      in<int>, //total elements 
-      in<int>, //seed
-      in<int> //max value
-      >;
-
-//mem_ptrs are references to memory on the gpu, same rules apply, it must be in the same order as it would appear in the kernel
-using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
-
+// Command classes used to launch kernels.
+// Templates describe the sequence of argument types for the GPU kernel.
+using mmulCommand = caf::cuda::command_runner<in<int>, in<int>, out<int>, in<int>>;
 mmulCommand mmul;
-matrixGenCommand randomMatrix;
-mmulAsyncCommand mmulAsync;
 
 
 
@@ -94,208 +70,13 @@ void serial_matrix_multiply(const std::vector<int>& a,
 }
 
 
-
-
-// Stateful actor behavior
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
-  return {
-    // 1st handler: Just int N, matrix size, will generate an N*N matrix 
-    [=](int N) {
-
-	 caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-
-        //create the program and configure the dimesnions of the kernel
-        auto program = mgr.create_program_from_fatbin("../generate_random_matrix.fatbin", //path to kernel file
-			"generate_random_matrix" //kernel name 
-			);
-	
-	int THREADS = 256;
-	int BLOCKS = (N*N + THREADS - 1) / THREADS;
-  	caf::cuda::nd_range dim(
-			BLOCKS, //grid X dimension
-			1, // grid Y dimension
-		       	1, //grid Z dimension
-			THREADS, //block X dimension
-			1, //block Y dimension
-		       	1 // block Z dimension
-			);
-
-	//tag the arguments so that caf::cuda knows what to do with them	
-         auto arg1 = caf::cuda::create_out_arg_with_size<int>(N*N); //output buffer indicate its size, caf::cuda will handle the rest
-          auto arg2 = caf::cuda::create_in_arg(N*N); //matrix size
-          auto arg3 = caf::cuda::create_in_arg(1234); //seed
-	  auto arg4 = caf::cuda::create_in_arg(9999); //max valux
-	  
-
-
-	  //launch kernels and collect their outputs
-	  //the args tagged with type in<T>, will not show up in the result 
-	  auto tempA = randomMatrix.run(
-			  program,//kernel to launch
-			  dim, //kernel dimensions
-			  self -> state().id, //actor id 
-			  arg1,arg2,arg3,arg4 //kernel args in order that they appear in the kernel  
-			  );
-	  auto tempB = randomMatrix.run(program,dim, self -> state().id,arg1,arg2,arg3,arg4);
-	  std::vector<int> matrixA =  caf::cuda::extract_vector<int>(tempA);
-	  std::vector<int> matrixB = caf::cuda::extract_vector<int>(tempB);
-		  
-	  //send the results to ourself	  	  
-	  self->mail(matrixA,matrixB,N).send(self);
-
-    },
-
-    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const std::vector<int> matrixA,
-        const std::vector<int> matrixB, int N) {
- 
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //create program and dims   
-  auto program = mgr.create_program_from_cubin("../mmul.cubin", //kernel file path 
-		  "matrixMul" //kernel name
-		  );
-
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(
-		  BLOCKS, //grid X dimension
-		  BLOCKS, // grid Y dimension
-		  1,  //grid Z dimension
-		  THREADS, //block X dimension
-		  THREADS, //block Y dimension
-		  1 //block Z dimension
-		  ); 
-
-    //create args
-    auto arg1 = caf::cuda::create_in_arg(matrixA); //matrix A
-    auto arg2 = caf::cuda::create_in_arg(matrixB); //matrix B
-    auto arg3 = caf::cuda::create_out_arg_with_size<int>(N*N); //matrix C (specify with size)
-    auto arg4 = caf::cuda::create_in_arg(N); //size of the matrices
-
-    //launch kernel and collect the output 
-    auto tempC = mmul.run(program,dims,self -> state().id,arg1,arg2,arg3,arg4);
-    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-    //verify its own result 
-    self -> mail(matrixA,matrixB,matrixC,N).send(self);
-
-    },
-
-    // 3rd handler: CPU atom + matrices + N
-    [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB, const std::vector<int> matrixC, int N) {
-       
-	 std::vector<int> result(N*N);
-
-	 serial_matrix_multiply(matrixA,matrixB,result,N);
-
-	 if (result == matrixC) {
-	 
-		 std::cout << "actor with id " <<  self->state().id << " references match\n";
-	 
-	 }
-
-	 else {
-	    std::cout << "actor with id " <<  self->state().id << " references did not match\n";
-	 }
-
-	 self-> quit();
-
-    }
-  };
-}
-
-
-
-void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_actor_fun));
-  }
-
-  //send a size to all actors 
-  for (auto a : actors)
-	  caf::anon_mail(matrix_size).send(a);
-
-   sys.await_all_actors_done();
-}
-
-
-
-//demonstration of sending memory on the gpu to other actors 
-//its worth mentioning that if you are sending gpu memory around
-//you must ensure that it stays on the same device and stream by ensuring the 
-//device number and actor id is the same per kernel launch 
-caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self) {
+// Demonstration of a fully asynchronous GPU actor.
+// It is recommended to use run_async and copy_to_host_async to avoid blocking worker threads.
+caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self,int device_number, int stream_id) {
   return {
-    // 1st handler: Just int N, send the matrix to itself 
-    [=](int N) {
-
-        caf::cuda::manager& mgr = caf::cuda::manager::get();
-        //create the program and configure the dimesnions of the kernel
-        auto program = mgr.create_program_from_fatbin("../generate_random_matrix.fatbin","generate_random_matrix");
-	int THREADS = 256;
-	int BLOCKS = (N*N + THREADS - 1) / THREADS;
-  	caf::cuda::nd_range dim(BLOCKS,1, 1, THREADS,1, 1);
-
-	//tag the arguments so that caf::cuda knows what to do with them	
-         auto arg1 = caf::cuda::create_out_arg_with_size<int>(N*N); //output buffer indicate its size, caf::cuda will handle the rest
-          auto arg2 = caf::cuda::create_in_arg(N*N); //matrix size
-          auto arg3 = caf::cuda::create_in_arg(rand()); //seed
-	  auto arg4 = caf::cuda::create_in_arg(9999); //max valux
-          auto arg3B = caf::cuda::create_in_arg(rand()); //seed
-	  
-	  int device_number= std::rand(); //any commmand that uses this number will 
-					  //guarantee that it stays on the same device 
-					  //as other commands and other gpu actors 
-					  //other device numbers may or may not run on the same gpu
-					  //depending on how many they are as it will just do device_number % num_devices
-					  //to pick the device
-
-
-
-	  //launch kernels and collect their outputs
-	  auto tempA = randomMatrix.run_async(
-			  program, //kernel to launch
-			  dim, //kernel dimensions
-			  self -> state().id, //actor id 
-			  0, //shared memory size in bytes
-			  device_number, //device number 
-			  arg1,arg2,arg3,arg4 //kernel arguments
-			  );
-	  auto tempB = randomMatrix.run_async(program,dim, self -> state().id,0,device_number,arg1,arg2,arg3B,arg4);
-	  caf::cuda::mem_ptr<int> matrixA =  std::get<0>(tempA);
-	  caf::cuda::mem_ptr<int> matrixB = std::get<0>(tempB);
-
-	  /*
-	   * Optional synchronize, as there is no guarantee that the data is 
-	   * actually done being worked on
-	   * but since each actor will use its own stream and device number
-	   * we dont need to 
-	  matrixA -> synchronize();
-	  matrixB -> synchronize();
-	  */
-	  
-	  //send to itself for matrix multiplication
-          self->mail(matrixA,matrixB,N,device_number).send(self);
-
-    },
-
-    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const caf::cuda::mem_ptr<int> matrixA,
-        const caf::cuda::mem_ptr<int> matrixB, int N,int device_number) {
+    // 1st handler: Receive host data, launch GPU kernel, and copy results back
+    [=](std::vector<int> matrixA, std::vector<int> matrixB, int N) {
  
-
   caf::cuda::manager& mgr = caf::cuda::manager::get();
 
   //create program and dims   
@@ -305,28 +86,24 @@ caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self)
   caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
     //create args
-    auto arg1 = matrixA;
-    auto arg2 = matrixB;
-    auto arg3 = caf::cuda::create_out_arg(N*N);
+    auto arg1 = caf::cuda::create_in_arg(matrixA);
+    auto arg2 = caf::cuda::create_in_arg(matrixB);
+    auto arg3 = caf::cuda::create_out_arg_with_size<int>(N*N);
     auto arg4 = caf::cuda::create_in_arg(N);
 
 
-    auto tempC = mmulAsync.run(program,dims,self -> state().id,0,device_number,arg1,arg2,arg3,arg4);
-
-    std::vector<int> matrix1 = matrixA -> copy_to_host(); //copy to host transfers memory back to the device
-    std::vector<int> matrix2 = matrixB -> copy_to_host();    
-    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC,2); //the output buffer that we want is at position 2 
-									//as the command runner will always return the result values 
-									//of in_out and out types in order that they appear in the launch 
-									//since matrixA and matrixB where of out type orginally, they will get returned as well
-									//
-
-    //verify its own result 
-    self -> mail(matrix1,matrix2,matrixC,N).send(self);
+    // Launch kernel asynchronously
+    auto results = mmul.run_async(program, dims, stream_id, 0, device_number, arg1, arg2, arg3, arg4);
+    auto matrixC_ptr = std::get<2>(results); // Matrix C is the 3rd argument (index 2)
 
+    // Copy result back to host asynchronously and send to verification handler
+    mmul.copy_to_host_async(matrixC_ptr, [=, m1 = std::move(matrixA), m2 = std::move(matrixB)](std::vector<int> m3) {
+      // Send host vectors to the 2nd handler for verification
+      self->mail(std::move(m1), std::move(m2), std::move(m3), N).send(self);
+    });
     },
 
-    // 3rd handler: CPU atom + matrices + N
+    // 2nd handler: CPU verification
     [=](const std::vector<int>& matrixA,
     const std::vector<int> &matrixB,
     const std::vector<int> &matrixC, int N) {
@@ -373,185 +150,27 @@ void run_async_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors
     return;
   }
 
+  std::vector<int> matrixA(matrix_size * matrix_size, 1);
+  std::vector<int> matrixB(matrix_size * matrix_size, 2);
+
   // Spawn num_actors actors running the mmul behavior
   std::vector<caf::actor> actors;
   actors.reserve(num_actors);
   for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_async_actor_fun));
+    actors.push_back(sys.spawn(mmul_async_actor_fun,0,1));
   }
 
   //send a size to all actors 
   for (auto a : actors)
-	  caf::anon_mail(matrix_size).send(a);
+	  caf::anon_mail(matrixA, matrixB, matrix_size).send(a);
 
    sys.await_all_actors_done();
 }
 
 
-//--------------------------------Perfomance tests 
-
-// Perf-version of the actor: each actor generates a matrix and sends to itself
-caf::behavior mmul_async_actor_fun_perf(caf::stateful_actor<mmul_actor_state>* self) {
-  return {
-    // 1) start: generate matrices and send them to self
-    [=](int N) {
-      // store start time in actor state (no locks)
-      self->state().start_time = std::chrono::high_resolution_clock::now();
-
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
-      // use the generator fatbin (as in your code)
-      auto program = mgr.create_program_from_fatbin("../generate_random_matrix.fatbin",
-                                                    "generate_random_matrix");
-
-      int THREADS = 256;
-      int BLOCKS = (N * N + THREADS - 1) / THREADS;
-      caf::cuda::nd_range dim(BLOCKS, 1, 1, THREADS, 1, 1);
-
-      // prepare args (same as your existing code)
-      auto arg_out = caf::cuda::create_out_arg(N * N);
-      auto arg_size = caf::cuda::create_in_arg(N * N);
-      auto arg_seed = caf::cuda::create_in_arg(rand());
-      auto arg_max = caf::cuda::create_in_arg(9999);
-
-      int device_number = rand()%2;
-
-      // launch generator(s) asynchronously and get mem_ptrs back
-      // (we follow your earlier style: run_async returns tuple of mem_ptrs)
-      auto tA = randomMatrix.run_async(program, dim, self->state().id,0,device_number,arg_out, arg_size, arg_seed, arg_max);
-      auto tB = randomMatrix.run_async(program, dim, self->state().id,0,device_number, arg_out, arg_size, arg_seed, arg_max);
-
-      // Extract the mem_ptrs (assume index 0 holds the buffer)
-      auto matA_ptr = std::get<0>(tA);
-      auto matB_ptr = std::get<0>(tB);
-
-      // ensure kernels are done and data is ready
-      //since we are sending to ourself no need to synchronize, since actors
-      //get their own stream
-      //if (matA_ptr) matA_ptr->synchronize();
-      //if (matB_ptr) matB_ptr->synchronize();
-
-      // send the mem_ptrs to ourselves to trigger the multiply step
-      // (we send device buffers, N)
-      for (int i =0;i < 20;i++)
-      self->mail(matA_ptr, matB_ptr, N).send(self);
-    },
-
-    // 2) multiply: receive mem_ptrs, run the mmul kernel, measure time, print, quit
-    [=](const caf::cuda::mem_ptr<int> matA,
-        const caf::cuda::mem_ptr<int> matB,
-        int N) {
-
-      // prepare mmul program + dims (same as your code)
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
-      auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-      const int THREADS = 32;
-      const int BLOCKS = (N + THREADS - 1) / THREADS;
-      caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-      // create arguments; use device pointers directly (your style)
-      auto arg1 = matA;
-      auto arg2 = matB;
-      auto arg3 = caf::cuda::create_out_arg(N * N);
-      auto arg4 = caf::cuda::create_in_arg(N);
-
-      // Synchronous launch (blocks until kernel finishes and output is collected).
-      // This represents "actor is done with its result".
-      auto start = std::chrono::high_resolution_clock::now();
-      auto out_bufs = mmulAsync.run(program, dims, self->state().id,0,matA ->deviceNumber(), arg1, arg2, arg3, arg4);
-      auto end = std::chrono::high_resolution_clock::now();
-
-      // compute per-actor latency from the generation start stored in state
-      double actor_latency_ms =
-        std::chrono::duration<double, std::milli>(end - self->state().start_time).count();
-
-      // Print per-actor latency (actor id included)
-      std::cout << "[PERF] Actor id=" << self->state().id
-                << " N=" << N
-                << " latency=" << actor_latency_ms << " ms\n";
-
-      if (self -> state().times++ == 19) {
-      // Done for this actor; exit
-      self->quit();
-      }
-    }
-  };
-}
-
-// Driver: spawn actors, start timer, tell each actor to generate/send-to-self, wait, print total time
-void run_async_mmul_perf_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  // spawn actors
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_async_actor_fun_perf));
-  }
-
-  // Total runtime start
-  auto total_start = std::chrono::high_resolution_clock::now();
-
-  // Tell every actor to generate a matrix and route it to itself
-  for (auto& a : actors) {
-    // send N to the actor (actor will generate and self-send)
-    caf::anon_mail(matrix_size).send(a);
-  }
-
-  // wait for all actors to finish
-  sys.await_all_actors_done();
-
-  // Total runtime end & print
-  auto total_end = std::chrono::high_resolution_clock::now();
-  double total_ms = std::chrono::duration<double, std::milli>(total_end - total_start).count();
-  std::cout << "[PERF] Total runtime for " << num_actors << " actors: " << total_ms << " ms\n";
-}
-
-
-
-//-----------------------------------BenchMark Tests
-
-
-// Benchmark driver for the "async (no-shared)" perf test
-void benchmark_async_perf_all(caf::actor_system& sys) {
-  const std::vector<int> actor_counts = {1, 50, 200};
-  const std::vector<int> matrix_sizes = {1024, 2048, 4096};
-
-  std::cout << "=== Async (no-shared) benchmark ===\n";
-  for (int size : matrix_sizes) {
-    for (int num_actors : actor_counts) {
-      std::cout << "[RUN] matrix_size=" << size
-                << " actors=" << num_actors
-                << "  -- starting\n" << std::flush;
-
-      auto t0 = std::chrono::high_resolution_clock::now();
-      // This function blocks until all actors finish and prints per-actor latencies.
-      run_async_mmul_perf_test(sys, size, num_actors);
-      auto t1 = std::chrono::high_resolution_clock::now();
-
-      double total_ms = std::chrono::duration<double, std::milli>(t1 - t0).count();
-      std::cout << "[RESULT] async  matrix_size=" << size
-                << " actors=" << num_actors
-                << " total_time_ms=" << total_ms << "\n\n" << std::flush;
-    }
-  }
-  std::cout << "=== Async (no-shared) benchmark complete ===\n\n";
-}
-
 void caf_main(caf::actor_system& sys) {
-  caf::cuda::manager::init(sys); //be sure to initialize the manager
-				 //it needs to do some things before running
-
-  //run_mmul_test(sys,100,4000);
-  //run_async_mmul_test(sys,100,700);
-
-  // run the async (no-shared) suite:
-  //benchmark_async_perf_all(sys);
+  caf::cuda::manager::init(sys);
+  run_async_mmul_test(sys, 100, 10);
 }
 
-
-
-
 CAF_MAIN()

From ffdba5db95dc095d8849b7a148f30598803bd045 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 3 Jun 2026 11:43:03 -0600
Subject: [PATCH 0813/1000] updated cmakelists

---
 libcaf_cuda/examples/custom-actor-examples/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/examples/custom-actor-examples/CMakeLists.txt b/libcaf_cuda/examples/custom-actor-examples/CMakeLists.txt
index 530515a8f2..68ac0347e3 100644
--- a/libcaf_cuda/examples/custom-actor-examples/CMakeLists.txt
+++ b/libcaf_cuda/examples/custom-actor-examples/CMakeLists.txt
@@ -39,6 +39,8 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
 )
 
 

From 6c9fd036a2daa5287e02387a8c954a2f7a983c18 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 3 Jun 2026 11:47:11 -0600
Subject: [PATCH 0814/1000] updated cmakelists

---
 libcaf_cuda/examples/actor-facade-example/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/examples/actor-facade-example/CMakeLists.txt b/libcaf_cuda/examples/actor-facade-example/CMakeLists.txt
index 0db176f063..82cd502e06 100644
--- a/libcaf_cuda/examples/actor-facade-example/CMakeLists.txt
+++ b/libcaf_cuda/examples/actor-facade-example/CMakeLists.txt
@@ -39,5 +39,7 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
 )
 

From ba77cf5c1d6fc7a550158c65675f14d7abbb4a9d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 3 Jun 2026 12:11:49 -0600
Subject: [PATCH 0815/1000] updated examples to work again

---
 .../actor-facade-example/mmul.example.cpp     | 250 ++++++++++--------
 1 file changed, 146 insertions(+), 104 deletions(-)

diff --git a/libcaf_cuda/examples/actor-facade-example/mmul.example.cpp b/libcaf_cuda/examples/actor-facade-example/mmul.example.cpp
index 9d653ad267..c063739b13 100644
--- a/libcaf_cuda/examples/actor-facade-example/mmul.example.cpp
+++ b/libcaf_cuda/examples/actor-facade-example/mmul.example.cpp
@@ -1,14 +1,12 @@
-/* An example file demonstrating how the actor facade works
- * by showing how to launch a matrix multiplication kernel using the actor facade 
- * we use managers spawnFromCubin and is recommended that you do too or use spawnFromFatbin
- * to spawn an actor facade 
- * since using the spawn method will likely result in an unsupported toolchain error
- * be sure to run compile_kernels.sh
+/* 
+ * Modern Actor Facade Example: Matrix Multiplication
+ * 
+ * This file demonstrates the three primary ways to interact with the caf::cuda::actor_facade.
+ * The facade is fully asynchronous and pushes results back to the requester's mailbox.
+ * 
+ * Requirements: Run compile_kernels.sh to generate mmul.cubin before running.
  */
-
-
 #include <caf/all.hpp>  // Includes most CAF essentials
-
 #include "caf/cuda/actor_facade.hpp"
 #include "caf/cuda/manager.hpp"
 #include "caf/cuda/nd_range.hpp"
@@ -16,22 +14,16 @@
 #include <caf/type_id.hpp>
 #include "caf/detail/test.hpp"
 #include <cassert>
-
 #include <iostream>
 #include <iomanip>
 #include <vector>
-#include <chrono>
-#include <thread>
 #include <algorithm>
-#include <numeric>
-#include "caf/actor_registry.hpp"
-
-
 
 using namespace caf;
 using namespace std::chrono_literals;
+using namespace caf::cuda;
 
-//verification of matrix multiplication on the gpu
+// Verification function for matrix multiplication on the CPU
 void serial_matrix_multiply(const std::vector<int>& a,
                             const std::vector<int>& b,
                             std::vector<int>& c,
@@ -49,99 +41,149 @@ void serial_matrix_multiply(const std::vector<int>& a,
   }
 }
 
-
-void test_mmul_from_cubin(caf::actor_system& sys, int N) {
-  std::cout << "[TEST] Starting test_mmul_from_cubin\n";
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  int THREADS = 32;
-  int BLOCKS = (N + THREADS - 1) / THREADS;
-
-  caf::cuda::nd_range dim(
-		  BLOCKS, //grid X dimension
-		  BLOCKS, //grid Y dimension
-		  1, //grid Z dimension
-		  THREADS, // block X dimension 
-		  THREADS, // block Y dimension
-		  1 // block Z dimension
-		  );
-
-
-  // Spawn actor from precompiled cubin file
-  auto gpuActor = mgr.spawnFromCUBIN(
-		  "../mmul.cubin", //kernel file location
-		  "matrixMul", //kernel name 
-		  dim, //kernel dimensions             
-		  in<int>{}, in<int>{}, out<int>{}, in<int>{} //kernel arg tags 
-							      //in order they appear in kernel
-		  );
-
-
-
-
-  //generate random matrices
-  std::vector<int> h_a(N * N);
-  std::vector<int> h_b(N * N);
-  std::vector<int> h_c(N * N, 0);
-  std::vector<int> h_ref(N * N, 0);
-  std::vector<int> h_n(1, N);
-  std::generate(h_a.begin(), h_a.end(), []() { return rand() % 10; });
-  std::generate(h_b.begin(), h_b.end(), []() { return rand() % 10; });
-
-
-
-  //tag the arguments 
-  auto arg1 = caf::cuda::create_in_arg(h_a); //matrix A readonly buffer
-  auto arg2 = caf::cuda::create_in_arg(h_b); //matrix B readonly buffer
-  auto arg3 = caf::cuda::create_out_arg_with_size<int>(N*N); //matrix size Writeonly buffer
-  auto arg4 = caf::cuda::create_in_arg(N); // int size, readonly scalar
-
-  serial_matrix_multiply(h_a, h_b, h_ref, N);
-
-  sys.spawn([=](caf::event_based_actor* self_actor) {
-    auto start = std::chrono::high_resolution_clock::now();
-
-    //when mailing the gpu actor, the message is in the form of the kernel arguments 
-    //and must be in the order they appear in the kernel parameters 
-    //it will deliever a response promise with the results of that kernel launch
-    self_actor->mail(arg1, arg2, arg3, arg4)
-      .request(gpuActor, std::chrono::seconds(10))
-      .then([=](const std::vector<output_buffer>& outputs) {
-        auto end = std::chrono::high_resolution_clock::now();
-        std::chrono::duration<double> elapsed = end - start;
-
-	
-        std::vector<int> result = caf::cuda::extract_vector<int>(outputs); //collect the result buffer from output
-
-
-        // Compare result with reference
-        bool match = (result == h_ref);
-        std::cout << "[INFO] Kernel round-trip time: " << elapsed.count() << " seconds\n";
-        std::cout << (match ? "[PASS] GPU result matches reference\n" : "[FAIL] Mismatch in GPU result\n");
-
-        self_actor->send_exit(gpuActor, caf::exit_reason::user_shutdown);
-        self_actor->quit();
-      });
-  });
-
-  sys.await_all_actors_done();
+// Common state for all testers to hold input vectors and output buffer for mapping
+struct mmul_tester_state {
+  std::vector<int> A;
+  std::vector<int> B;
+  std::vector<int> host_buffer; // Used by mapping_tester
+};
+
+// ---------------------------------------------------------------------------
+// Case 1: Standard Vector Results
+// The facade copies results from Device to Host and sends std::vector<T>.
+// ---------------------------------------------------------------------------
+caf::behavior standard_tester(caf::stateful_actor<mmul_tester_state>* self, caf::actor facade,
+                              int N, std::vector<int> ref, int launch_id) {
+  std::cout << "[Tester] Launching Standard Facade Test...\n";
+
+  self->state().A.assign(N * N, 1); // Store A in actor state
+  self->state().B.assign(N * N, 2); // Store B in actor state
+  auto arg1 = create_in_arg(self->state().A);
+  auto arg2 = create_in_arg(self->state().B);
+  auto arg3 = create_out_arg_with_size<int>(N * N);
+  auto arg4 = create_in_arg(N);
+
+  // Send work. We use 'mail' to provide type-safe arguments.
+  self->mail(arg1, arg2, arg3, arg4).send(facade);
+
+  return {
+    // Signature: (int reply_id, int index, std::vector<T> data)
+    [=](int r_id, int index, std::vector<int> data) {
+      if (r_id == launch_id) {
+        std::cout << "[Standard] Received result for index " << index << "\n";
+        if (index == 2) { // Matrix C
+          bool match = (data == ref);
+          std::cout << (match ? "[PASS]" : "[FAIL]") << " Standard Vector Match\n";
+        }
+        self->quit();
+      }
+    }
+  };
 }
 
+// ---------------------------------------------------------------------------
+// Case 2: Memory Pointer Results
+// The facade returns raw mem_ptr<T> handles instead of copying data to host.
+// ---------------------------------------------------------------------------
+caf::behavior memptr_tester(caf::stateful_actor<mmul_tester_state>* self, caf::actor facade,
+                            int N, std::vector<int> ref, int launch_id) {
+  std::cout << "[Tester] Launching MemPtr Facade Test...\n";
+
+  self->state().A.assign(N * N, 1);
+  self->state().B.assign(N * N, 2);
+  auto arg1 = create_in_arg(self->state().A);
+  auto arg2 = create_in_arg(self->state().B);
+  auto arg3 = create_out_arg_with_size<int>(N * N);
+  auto arg4 = create_in_arg(N);
+
+  // By prepending return_mem_ptr_atom, the facade returns handles immediately 
+  // after kernel launch, allowing manual control over Device-to-Host moves.
+  self->mail(return_mem_ptr_atom_v, arg1, arg2, arg3, arg4).send(facade);
+
+  return {
+    // Signature: (int reply_id, mem_ptr<T>...)
+    [=](int r_id, mem_ptr<int> pA, mem_ptr<int> pB, mem_ptr<int> pC, mem_ptr<int> pN) {
+      if (r_id == launch_id) {
+        std::cout << "[MemPtr] Received device memory handles.\n";
+        
+        // Copy matrix C back to host manually.
+        std::vector<int> data = pC->copy_to_host();
+        
+        bool match = (data == ref);
+        std::cout << (match ? "[PASS]" : "[FAIL]") << " MemPtr Data Match\n";
+        self->quit();
+      }
+    }
+  };
+}
 
-
+// ---------------------------------------------------------------------------
+// Case 3: Output Mapping
+// The facade copies results directly into a pre-allocated host buffer.
+// ---------------------------------------------------------------------------
+caf::behavior mapping_tester(caf::stateful_actor<mmul_tester_state>* self,
+                             caf::actor facade, int N, std::vector<int> ref, 
+                             int launch_id) {
+  std::cout << "[Tester] Launching Output Mapping Test...\n";
+  
+  self->state().A.assign(N * N, 1); // Store A in actor state
+  self->state().B.assign(N * N, 2); // Store B in actor state
+  self->state().host_buffer.assign(N * N, 0);
+
+  auto arg1 = create_in_arg(self->state().A);
+  auto arg2 = create_in_arg(self->state().B);
+  auto arg3 = create_out_arg_with_size<int>(N * N);
+  auto arg4 = create_in_arg(N);
+
+  // Define the mapping: tell the facade to put index 2 into our local vector.
+  std::vector<output_mapping> mappings = {
+    {2, self->state().host_buffer.data(), self->state().host_buffer.size()}
+  };
+
+  self->mail(mappings, arg1, arg2, arg3, arg4).send(facade);
+
+  return {
+    // Signature: (int reply_id, int index)
+    [=](int r_id, int index) {
+      if (r_id == launch_id) {
+        std::cout << "[Mapping] Facade notified completion for index " << index << "\n";
+        if (index == 2) {
+          bool match = (self->state().host_buffer == ref);
+          std::cout << (match ? "[PASS]" : "[FAIL]") << " Mapped Buffer Match\n";
+        }
+        self->quit();
+      }
+    }
+  };
+}
 
 void caf_main(caf::actor_system& sys) {
-  caf::cuda::manager::init(sys);  //be sure to initialize the manager
-				  //as certain things need to be startup 
-   test_mmul_from_cubin(sys,100);
-
-   //test_mmul_from_cubin(sys,50);
-   //test_mmul_from_cubin(sys,1024);
+  // 1. Initialize the CUDA Manager
+  manager::init(sys);
+  auto& mgr = manager::get();
+
+  // 2. Setup Kernel Dimensions and Metadata
+  int N = 128;
+  int threads = 32;
+  int blocks = (N + threads - 1) / threads;
+  nd_range dims(blocks, blocks, 1, threads, threads, 1);
+
+  // 3. Create Reference Data on CPU
+  std::vector<int> h_a(N * N, 1), h_b(N * N, 2), h_ref(N * N, 0);
+  serial_matrix_multiply(h_a, h_b, h_ref, N);
 
+  // 4. Spawn the Facade
+  // We pass the argument tags (in/out) so the facade knows the kernel signature.
+  int my_reply_id = 0;
+  auto mmul_facade = mgr.spawnFromCUBIN(
+    "../mmul.cubin", "matrixMul", dims, 
+    in<int>{}, in<int>{}, out<int>{}, in<int>{}
+  );
+
+  // 5. Run the interaction tests
+  sys.spawn(standard_tester, mmul_facade, N, h_ref, my_reply_id);
+  sys.spawn(memptr_tester, mmul_facade, N, h_ref, my_reply_id);
+  sys.spawn(mapping_tester, mmul_facade, N, h_ref, my_reply_id);
 }
 
-
-
-
 CAF_MAIN()

From 3dfeb6339639d983c4d47d90bc639800cbbadf8d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 3 Jun 2026 14:04:56 -0600
Subject: [PATCH 0816/1000] Intial commit

---
 .../tests/workload-test/CMakeLists.txt        |  17 +++
 .../tests/workload-test/hot-potatoe.cpp       | 107 ++++++++++++++++++
 2 files changed, 124 insertions(+)
 create mode 100644 libcaf_cuda/tests/workload-test/hot-potatoe.cpp

diff --git a/libcaf_cuda/tests/workload-test/CMakeLists.txt b/libcaf_cuda/tests/workload-test/CMakeLists.txt
index 5f3e4dffe2..d7ca8d87fa 100644
--- a/libcaf_cuda/tests/workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/workload-test/CMakeLists.txt
@@ -44,6 +44,23 @@ target_link_libraries(test
     CUDA::cusparse
 )
 
+
+# 5) Declare your executable
+add_executable(hot-potatoe hot-potatoe.cpp sparse_utils.cpp)
+
+target_compile_definitions(hot-potatoe PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(hot-potatoe
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
+)
+
+
 # FindThreads is required for std::thread in the native version
 find_package(Threads REQUIRED)
 
diff --git a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
new file mode 100644
index 0000000000..a2258f63f4
--- /dev/null
+++ b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
@@ -0,0 +1,107 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <deque>
+#include <string>
+#include <chrono>
+#include <numeric>
+#include <cmath>
+#include <algorithm>
+#include <filesystem>
+#include <memory>
+#include "caf/actorSOLVE/actorSOLVE.hpp"
+#include "sparse_utils.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+namespace fs = std::filesystem;
+
+constexpr uint32_t WORKLOAD_SEED = 42;
+
+template <class Inspector>
+bool inspect(Inspector& f, SolverType& x) {
+    auto val = static_cast<int>(x);
+    if (f.apply(val)) {
+        if constexpr (Inspector::is_loading)
+            x = static_cast<SolverType>(val);
+        return true;
+    }
+    return false;
+}
+
+CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
+    CAF_ADD_ATOM(workload_test, get_work_atom)
+    CAF_ADD_ATOM(workload_test, release_memory_atom)
+    CAF_ADD_ATOM(workload_test, request_work_atom)
+    CAF_ADD_ATOM(workload_test, worker_done_atom)
+    CAF_ADD_ATOM(workload_test, work_tick_atom)
+    CAF_ADD_TYPE_ID(workload_test, (SolverType))
+    CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
+    CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
+    CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
+CAF_END_TYPE_ID_BLOCK(workload_test)
+
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixData)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
+
+
+
+void caf_main(actor_system& sys) {
+    manager::init(sys, manager_config(true, true));
+    
+    int num_streams = 8;
+    int num_batches = 25;
+    int batch_size = 100;
+    double mean_arrival_ms = 1000.0;
+
+    // // Basic command line argument parsing similar to native
+    // auto& args = sys.config().remainder;
+    // if (args.size() > 0) num_streams = std::max(1, std::stoi(args[0]));
+    // if (args.size() > 1) num_batches = std::max(1, std::stoi(args[1]));
+    // if (args.size() > 2) batch_size = std::max(1, std::stoi(args[2]));
+    // if (args.size() > 3) mean_arrival_ms = std::stod(args[3]);
+
+    std::cout << "[INFO] Loading matrices into memory...\n";
+    auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
+    //scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
+    
+    if (tasks_vec.empty()) {
+        std::cerr << "No matrix files found in search paths.\n";
+        manager::shutdown();
+        return;
+    }
+
+    std::cout << "[INFO] Matrix pool size: " << tasks_vec.size() << "\n";
+    std::cout << "[INFO] Streams/GPU:      " << num_streams << "\n";
+    std::cout << "[INFO] Batches:          " << num_batches << "\n";
+    std::cout << "[INFO] Batch size:       " << batch_size << "\n";
+    std::cout << "[INFO] Mean arrival:     " << mean_arrival_ms << " ms\n";
+
+    auto start = std::chrono::steady_clock::now();
+
+    // sys.spawn(supervisor_actor_fun, std::move(tasks_vec), num_streams, num_batches, batch_size, mean_arrival_ms);
+    // sys.await_all_actors_done();
+
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+
+    int total_tasks = num_batches * batch_size;
+
+    std::cout << "\n===== BENCHMARK COMPLETE =====\n";
+    std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
+    std::cout << "Streams per GPU:    " << num_streams << "\n";
+    std::cout << "Batches:            " << num_batches << "\n";
+    std::cout << "Batch Size:         " << batch_size << "\n";
+    std::cout << "Mean Arrival (ms):  " << mean_arrival_ms << "\n";
+    std::cout << "Tasks Processed:    " << total_tasks << "\n";
+    std::cout << "Total Runtime:      " << elapsed.count() << " s\n";
+    std::cout << "Throughput:         " << total_tasks / elapsed.count() << " tasks/s\n";
+    std::cout << "==============================\n";
+
+    manager::shutdown();
+}
+CAF_MAIN(id_block::cuda,id_block::workload_test)
\ No newline at end of file

From 9673d370e8e08eedccb117a4607f35da2fa916fd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 3 Jun 2026 14:10:25 -0600
Subject: [PATCH 0817/1000] added parition logic

---
 .../tests/workload-test/sparse_utils.cpp      | 29 +++++++++++++++++++
 .../tests/workload-test/sparse_utils.hpp      |  7 +++++
 2 files changed, 36 insertions(+)

diff --git a/libcaf_cuda/tests/workload-test/sparse_utils.cpp b/libcaf_cuda/tests/workload-test/sparse_utils.cpp
index d0687907b3..b16c9dc907 100644
--- a/libcaf_cuda/tests/workload-test/sparse_utils.cpp
+++ b/libcaf_cuda/tests/workload-test/sparse_utils.cpp
@@ -97,6 +97,35 @@ std::vector<MatrixTask> scan_for_matrices(const std::string& dir, SolverType typ
     return tasks;
 }
 
+
+std::vector<Partition>
+make_contiguous_partitions(size_t num_tasks, size_t rows, size_t cols)
+{
+    size_t num_parts = rows * cols;
+
+    std::vector<Partition> parts;
+    parts.reserve(num_parts);
+
+    size_t base = num_tasks / num_parts;
+    size_t rem  = num_tasks % num_parts;
+
+    size_t current = 0;
+
+    for (size_t p = 0; p < num_parts; ++p) {
+        size_t size = base + (p < rem ? 1 : 0);
+
+        parts.push_back({
+            current,
+            current + size
+        });
+
+        current += size;
+    }
+
+    return parts;
+}
+
+
 int generate_random_sleep_ms(int min_ms, int max_ms) {
     static std::random_device rd;
     static std::mt19937 gen(rd());
diff --git a/libcaf_cuda/tests/workload-test/sparse_utils.hpp b/libcaf_cuda/tests/workload-test/sparse_utils.hpp
index e793d3b541..41f816f4c0 100644
--- a/libcaf_cuda/tests/workload-test/sparse_utils.hpp
+++ b/libcaf_cuda/tests/workload-test/sparse_utils.hpp
@@ -46,6 +46,13 @@ struct MatrixTask {
     std::shared_ptr<MatrixData> data;
 };
 
+
+
+struct Partition {
+    size_t begin;
+    size_t end;
+};
+
 // Function to slurp the binary data into memory
 SparseMatrixCOO load_binary_coo(const std::string& filepath);
 

From 63f4f43ce19c134cfa198885d5a9f2192007d044 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 3 Jun 2026 14:13:40 -0600
Subject: [PATCH 0818/1000] updated parsing logic

---
 libcaf_cuda/tests/workload-test/sparse_utils.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/tests/workload-test/sparse_utils.hpp b/libcaf_cuda/tests/workload-test/sparse_utils.hpp
index 41f816f4c0..f886e68c6c 100644
--- a/libcaf_cuda/tests/workload-test/sparse_utils.hpp
+++ b/libcaf_cuda/tests/workload-test/sparse_utils.hpp
@@ -51,6 +51,8 @@ struct MatrixTask {
 struct Partition {
     size_t begin;
     size_t end;
+    std::vector<int> devices;
+    std::vector<int> streams;
 };
 
 // Function to slurp the binary data into memory

From a8873f183838d6f8f727928d3098fdc2bcd83e15 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 3 Jun 2026 14:35:55 -0600
Subject: [PATCH 0819/1000] added commented out implementation

---
 .../tests/workload-test/hot-potatoe.cpp       | 524 ++++++++++++++----
 1 file changed, 417 insertions(+), 107 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
index a2258f63f4..e9d9ebfb40 100644
--- a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
+++ b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
@@ -1,107 +1,417 @@
-#include <caf/all.hpp>
-#include <caf/cuda/all.hpp>
-#include <iostream>
-#include <fstream>
-#include <vector>
-#include <deque>
-#include <string>
-#include <chrono>
-#include <numeric>
-#include <cmath>
-#include <algorithm>
-#include <filesystem>
-#include <memory>
-#include "caf/actorSOLVE/actorSOLVE.hpp"
-#include "sparse_utils.hpp"
-
-using namespace caf;
-using namespace caf::cuda;
-namespace fs = std::filesystem;
-
-constexpr uint32_t WORKLOAD_SEED = 42;
-
-template <class Inspector>
-bool inspect(Inspector& f, SolverType& x) {
-    auto val = static_cast<int>(x);
-    if (f.apply(val)) {
-        if constexpr (Inspector::is_loading)
-            x = static_cast<SolverType>(val);
-        return true;
-    }
-    return false;
-}
-
-CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
-    CAF_ADD_ATOM(workload_test, get_work_atom)
-    CAF_ADD_ATOM(workload_test, release_memory_atom)
-    CAF_ADD_ATOM(workload_test, request_work_atom)
-    CAF_ADD_ATOM(workload_test, worker_done_atom)
-    CAF_ADD_ATOM(workload_test, work_tick_atom)
-    CAF_ADD_TYPE_ID(workload_test, (SolverType))
-    CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
-    CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
-    CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
-CAF_END_TYPE_ID_BLOCK(workload_test)
-
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixData)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
-
-
-
-void caf_main(actor_system& sys) {
-    manager::init(sys, manager_config(true, true));
-    
-    int num_streams = 8;
-    int num_batches = 25;
-    int batch_size = 100;
-    double mean_arrival_ms = 1000.0;
-
-    // // Basic command line argument parsing similar to native
-    // auto& args = sys.config().remainder;
-    // if (args.size() > 0) num_streams = std::max(1, std::stoi(args[0]));
-    // if (args.size() > 1) num_batches = std::max(1, std::stoi(args[1]));
-    // if (args.size() > 2) batch_size = std::max(1, std::stoi(args[2]));
-    // if (args.size() > 3) mean_arrival_ms = std::stod(args[3]);
-
-    std::cout << "[INFO] Loading matrices into memory...\n";
-    auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
-    //scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
-    
-    if (tasks_vec.empty()) {
-        std::cerr << "No matrix files found in search paths.\n";
-        manager::shutdown();
-        return;
-    }
-
-    std::cout << "[INFO] Matrix pool size: " << tasks_vec.size() << "\n";
-    std::cout << "[INFO] Streams/GPU:      " << num_streams << "\n";
-    std::cout << "[INFO] Batches:          " << num_batches << "\n";
-    std::cout << "[INFO] Batch size:       " << batch_size << "\n";
-    std::cout << "[INFO] Mean arrival:     " << mean_arrival_ms << " ms\n";
-
-    auto start = std::chrono::steady_clock::now();
-
-    // sys.spawn(supervisor_actor_fun, std::move(tasks_vec), num_streams, num_batches, batch_size, mean_arrival_ms);
-    // sys.await_all_actors_done();
-
-    auto end = std::chrono::steady_clock::now();
-    std::chrono::duration<double> elapsed = end - start;
-
-    int total_tasks = num_batches * batch_size;
-
-    std::cout << "\n===== BENCHMARK COMPLETE =====\n";
-    std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
-    std::cout << "Streams per GPU:    " << num_streams << "\n";
-    std::cout << "Batches:            " << num_batches << "\n";
-    std::cout << "Batch Size:         " << batch_size << "\n";
-    std::cout << "Mean Arrival (ms):  " << mean_arrival_ms << "\n";
-    std::cout << "Tasks Processed:    " << total_tasks << "\n";
-    std::cout << "Total Runtime:      " << elapsed.count() << " s\n";
-    std::cout << "Throughput:         " << total_tasks / elapsed.count() << " tasks/s\n";
-    std::cout << "==============================\n";
-
-    manager::shutdown();
-}
-CAF_MAIN(id_block::cuda,id_block::workload_test)
\ No newline at end of file
+// #include <caf/all.hpp>
+// #include <caf/cuda/all.hpp>
+// #include <iostream>
+// #include <deque>
+// #include <vector>
+// #include <random>
+// #include <chrono>
+// #include <filesystem>
+// #include <memory>
+
+// #include "caf/actorSOLVE/actorSOLVE.hpp"
+// #include "sparse_utils.hpp"
+
+// using namespace caf;
+// using namespace caf::cuda;
+// namespace fs = std::filesystem;
+
+// // ============================================================
+// // TYPE BLOCK (unchanged + extended)
+// // ============================================================
+
+// CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
+
+//     CAF_ADD_ATOM(workload_test, get_work_atom)
+//     CAF_ADD_ATOM(workload_test, request_work_atom)
+//     CAF_ADD_ATOM(workload_test, worker_done_atom)
+//     CAF_ADD_ATOM(workload_test, work_tick_atom)
+
+//     CAF_ADD_TYPE_ID(workload_test, (SolverType))
+//     CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
+//     CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
+//     CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
+
+// CAF_END_TYPE_ID_BLOCK(workload_test)
+
+// CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
+// CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
+// CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
+
+// // ============================================================
+// // WORK TOKEN SYSTEM
+// // ============================================================
+
+// enum class Direction : uint8_t {
+//     L2R = 0,
+//     R2L = 1
+// };
+
+// inline Direction flip(Direction d) {
+//     return d == Direction::L2R ? Direction::R2L : Direction::L2R;
+// }
+
+// struct WorkToken {
+//     int device;
+//     int stream;
+//     size_t partition;
+//     Direction dir;
+// };
+
+// template <class Inspector>
+// bool inspect(Inspector& f, WorkToken& x) {
+//     return f.object(x).fields(
+//         f.field("device", x.device),
+//         f.field("stream", x.stream),
+//         f.field("partition", x.partition),
+//         f.field("dir", x.dir)
+//     );
+// }
+
+// // ============================================================
+// // PARTITION STATE (supervisor owns)
+// // ============================================================
+
+// struct Partition {
+//     size_t begin;
+//     size_t end;
+
+//     size_t active_tokens = 0;
+//     bool completed = false;
+// };
+
+// // ============================================================
+// // TASK ACTOR STATE
+// // ============================================================
+
+// struct task_state {
+//     MatrixTask task;
+
+//     actor supervisor;
+//     actor left;
+//     actor right;
+
+//     bool seen_l2r = false;
+//     bool seen_r2l = false;
+
+//     bool solve_started = false;
+
+//     std::optional<WorkToken> pending;
+
+//     caf::actor cg_facade;
+
+//     std::chrono::steady_clock::time_point start;
+// };
+
+// // ============================================================
+// // TASK ACTOR
+// // ============================================================
+
+// behavior task_actor(stateful_actor<task_state>* self) {
+
+//     return {
+
+//         // ----------------------------------------------------
+//         // INIT
+//         // ----------------------------------------------------
+//         [=](MatrixTask t,
+//             actor supervisor,
+//             actor left,
+//             actor right) {
+
+//             auto& st = self->state();
+
+//             st.task = std::move(t);
+//             st.supervisor = supervisor;
+//             st.left = left;
+//             st.right = right;
+
+//             st.cg_facade =
+//                 self->spawn<sparse_cg_facade<float>, linked>(0);
+//         },
+
+//         // ----------------------------------------------------
+//         // TOKEN PROPAGATION
+//         // ----------------------------------------------------
+//         [=](const WorkToken& tok) {
+
+//             auto& st = self->state();
+
+//             bool& seen_this =
+//                 tok.dir == Direction::L2R
+//                     ? st.seen_l2r
+//                     : st.seen_r2l;
+
+//             bool& seen_other =
+//                 tok.dir == Direction::L2R
+//                     ? st.seen_r2l
+//                     : st.seen_l2r;
+
+//             // ------------------------------------------------
+//             // COLLISION => PARTITION COMPLETED SIGNAL
+//             // ------------------------------------------------
+//             if (seen_other) {
+//                 self->send(
+//                     st.supervisor,
+//                     worker_done_atom_v,
+//                     tok.partition,
+//                     tok.device,
+//                     tok.stream
+//                 );
+//                 return;
+//             }
+
+//             // already visited this direction → just forward
+//             if (seen_this) {
+//                 actor next =
+//                     tok.dir == Direction::L2R
+//                         ? st.right
+//                         : st.left;
+
+//                 if (next)
+//                     self->send(next, tok);
+
+//                 return;
+//             }
+
+//             seen_this = true;
+
+//             // ------------------------------------------------
+//             // FIRST VISIT → LAUNCH SOLVER
+//             // ------------------------------------------------
+//             if (!st.solve_started) {
+//                 st.solve_started = true;
+//                 st.pending = tok;
+//                 st.start = std::chrono::steady_clock::now();
+
+//                 auto& d = *st.task.data;
+
+//                 self->mail(
+//                     create_in_arg(d.row_ptr),
+//                     create_in_arg(d.col_indices),
+//                     create_in_arg(d.values),
+//                     create_in_arg(d.b),
+//                     create_in_out_arg(d.x_guess),
+//                     matrix_format::csr,
+//                     (int)d.row_ptr.size() - 1,
+//                     (int)d.values.size(),
+//                     1e-5f,
+//                     2000,
+//                     tok.device,
+//                     tok.stream
+//                 ).send(st.cg_facade);
+
+//                 return;
+//             }
+
+//             // ------------------------------------------------
+//             // NORMAL FORWARD
+//             // ------------------------------------------------
+//             actor next =
+//                 tok.dir == Direction::L2R
+//                     ? st.right
+//                     : st.left;
+
+//             if (next)
+//                 self->send(next, tok);
+//         },
+
+//         // ----------------------------------------------------
+//         // SOLVER DONE
+//         // ----------------------------------------------------
+//         [=](uint32_t,
+//             int,
+//             std::vector<float>&,
+//             solver_result_meta meta) {
+
+//             auto& st = self->state();
+
+//             auto tok = *st.pending;
+
+//             actor next =
+//                 tok.dir == Direction::L2R
+//                     ? st.right
+//                     : st.left;
+
+//             if (next) {
+//                 self->send(next, tok);
+//             } else {
+//                 self->send(
+//                     st.supervisor,
+//                     worker_done_atom_v,
+//                     tok.partition,
+//                     tok.device,
+//                     tok.stream
+//                 );
+//             }
+//         }
+//     };
+// }
+
+// // ============================================================
+// // SUPERVISOR STATE
+// // ============================================================
+
+// struct supervisor_state {
+//     std::vector<MatrixTask> batch;
+//     std::vector<actor> actors;
+//     std::vector<Partition> partitions;
+
+//     size_t next_partition = 0;
+//     size_t completed = 0;
+
+//     int num_streams = 0;
+// };
+
+// // ============================================================
+// // SUPERVISOR
+// // ============================================================
+
+// behavior supervisor_actor(stateful_actor<supervisor_state>* self,
+//                           std::vector<MatrixTask> batch,
+//                           int num_gpus,
+//                           int streams_per_gpu) {
+
+//     auto& st = self->state();
+//     st.batch = std::move(batch);
+
+//     size_t num_parts = num_gpus * streams_per_gpu;
+
+//     st.partitions.resize(num_parts);
+
+//     for (size_t i = 0; i < num_parts; ++i) {
+//         size_t begin = (i * st.batch.size()) / num_parts;
+//         size_t end   = ((i + 1) * st.batch.size()) / num_parts;
+
+//         st.partitions[i] = {begin, end, 0, false};
+//     }
+
+//     // spawn actors
+//     int total = st.batch.size();
+
+//     for (int i = 0; i < total; ++i)
+//         st.actors.push_back(self->spawn(task_actor));
+
+//     // link neighbors
+//     for (size_t i = 0; i < st.actors.size(); ++i) {
+//         actor left  = (i == 0) ? actor{} : st.actors[i - 1];
+//         actor right = (i + 1 < st.actors.size()) ? st.actors[i + 1] : actor{};
+
+//         self->send(st.actors[i],
+//                    st.batch[i],
+//                    actor_cast<actor>(self),
+//                    left,
+//                    right);
+//     }
+
+//     // inject first wave
+//     auto& p = st.partitions[0];
+
+//     p.active_tokens = 1;
+
+//     self->send(
+//         st.actors[p.begin],
+//         WorkToken{
+//             0,
+//             0,
+//             0,
+//             Direction::L2R
+//         }
+//     );
+
+//     return {
+
+//         // ------------------------------------------------
+//         // TOKEN COMPLETION EVENT
+//         // ------------------------------------------------
+//         [=](worker_done_atom,
+//             size_t partition,
+//             int device,
+//             int stream) {
+
+//             auto& st = self->state();
+
+//             auto& p = st.partitions[partition];
+
+//             if (p.completed)
+//                 return;
+
+//             if (p.active_tokens > 0)
+//                 p.active_tokens--;
+
+//             if (p.active_tokens == 0) {
+//                 p.completed = true;
+//                 st.completed++;
+
+//                 std::cout << "[DONE] partition "
+//                           << partition << "\n";
+//             }
+
+//             // find next unfinished
+//             for (size_t i = 0; i < st.partitions.size(); ++i) {
+//                 size_t idx = (partition + i + 1)
+//                            % st.partitions.size();
+
+//                 if (!st.partitions[idx].completed) {
+
+//                     auto& np = st.partitions[idx];
+//                     np.active_tokens++;
+
+//                     Direction dir =
+//                         (i % 2 == 0)
+//                             ? Direction::L2R
+//                             : Direction::R2L;
+
+//                     size_t start =
+//                         (dir == Direction::L2R)
+//                             ? np.begin
+//                             : np.end - 1;
+
+//                     self->send(
+//                         st.actors[start],
+//                         WorkToken{
+//                             device,
+//                             stream,
+//                             idx,
+//                             dir
+//                         });
+
+//                     break;
+//                 }
+//             }
+//         }
+//     };
+// }
+
+// // ============================================================
+// // MAIN
+// // ============================================================
+
+// void caf_main(actor_system& sys) {
+
+//     manager::init(sys, manager_config(true, true));
+
+//     int streams = 8;
+//     int batches = 25;
+//     int batch_size = 100;
+
+//     auto tasks = scan_for_matrices(
+//         "/scratch/nqr159/matrix-collection",
+//         CGS_SOLVER
+//     );
+
+//     auto batch = generate_batch(tasks, std::mt19937{42}, batch_size);
+
+//     manager& mgr = manager::get();
+//     int gpus = mgr.get_num_devices();
+
+//     sys.spawn(supervisor_actor,
+//               batch,
+//               gpus,
+//               streams);
+
+//     sys.await_all_actors_done();
+
+//     manager::shutdown();
+// }
+
+// CAF_MAIN(id_block::cuda, workload_test)
\ No newline at end of file

From aa3cd5b54442b86494ef7de814c9da64f29f2094 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 3 Jun 2026 15:06:56 -0600
Subject: [PATCH 0820/1000] added logic for actors to create interact and
 message pass events

---
 libcaf_cuda/caf/cuda/command_runner.hpp | 37 ++++++++++++++++
 libcaf_cuda/caf/cuda/device.hpp         | 36 ++++++++++++++++
 libcaf_cuda/caf/cuda/event.hpp          | 57 +++++++++++++++++++++++++
 libcaf_cuda/caf/cuda/global.hpp         |  3 ++
 4 files changed, 133 insertions(+)
 create mode 100644 libcaf_cuda/caf/cuda/event.hpp

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index f69cff7983..bae0992c5a 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -10,6 +10,7 @@
 #include "caf/cuda/control-layer/response_token.hpp"
 #include "caf/cuda/control-layer/launch_response_token.hpp"
 #include "caf/cuda/control-layer/memory_response_token.hpp"
+#include "caf/cuda/event.hpp"
 
 namespace caf::cuda {
 
@@ -305,8 +306,44 @@ class command_runner {
       plat->reset_device_context(device_number);
   }
 
+  // -------------------------------
+  // CUDA Event Management
+  // -------------------------------
+
+  /// Creates a new CUDA event on the specified device.
+  event_ptr create_event(int device_number, unsigned int flags = CU_EVENT_DEFAULT) {
+    auto plat = platform::create();
+    auto dev = plat->getDevice(device_number % plat->get_num_devices());
+    return dev->create_event(flags);
+  }
+
+  /// Records a CUDA event on the stream associated with the given stream_id.
+  void record_event(event_ptr e, int stream_id, int device_number) {
+    auto plat = platform::create();
+    auto dev = plat->schedule(stream_id, device_number);
+    dev->record_event(std::move(e), stream_id);
+  }
 
+  /// Enqueues a wait on the stream for the specified CUDA event.
+  void wait_event(event_ptr e, int stream_id, int device_number) {
+    auto plat = platform::create();
+    auto dev = plat->schedule(stream_id, device_number);
+    dev->wait_event(std::move(e), stream_id);
+  }
 
+  /// Returns true if the specified CUDA event has completed.
+  bool query_event(event_ptr e, int device_number) {
+    auto plat = platform::create();
+    auto dev = plat->getDevice(device_number % plat->get_num_devices());
+    return dev->query_event(std::move(e));
+  }
+
+  /// Blocks until the specified CUDA event has completed.
+  void synchronize_event(event_ptr e, int device_number) {
+    auto plat = platform::create();
+    auto dev = plat->getDevice(device_number % plat->get_num_devices());
+    dev->synchronize_event(std::move(e));
+  }
 
 };
 
diff --git a/libcaf_cuda/caf/cuda/device.hpp b/libcaf_cuda/caf/cuda/device.hpp
index b318e1c505..c7192d1ce6 100644
--- a/libcaf_cuda/caf/cuda/device.hpp
+++ b/libcaf_cuda/caf/cuda/device.hpp
@@ -21,6 +21,7 @@
 #include "caf/cuda/types.hpp"
 #include "caf/cuda/streampool.hpp"
 #include "caf/cuda/mem_ref.hpp"
+#include "caf/cuda/event.hpp"
 
 namespace caf::cuda {
 
@@ -121,6 +122,41 @@ class CAF_CUDA_EXPORT device : public caf::ref_counted {
     stream_table_->release_stream(actor_id);
   }
 
+  /// Creates a CUDA event on this device.
+  event_ptr create_event(unsigned int flags = CU_EVENT_DEFAULT) {
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    auto res = caf::make_counted<event>(flags);
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    return res;
+  }
+
+  /// Records an event on the stream associated with the actor_id.
+  void record_event(event_ptr e, int actor_id) {
+    CUstream stream = get_stream_for_actor(actor_id);
+    CHECK_CUDA(cuEventRecord(e->get(), stream));
+  }
+
+  /// Makes a stream wait on an event.
+  void wait_event(event_ptr e, int actor_id) {
+    CUstream stream = get_stream_for_actor(actor_id);
+    CHECK_CUDA(cuStreamWaitEvent(stream, e->get(), 0));
+  }
+
+  /// Returns true if the event has completed.
+  bool query_event(event_ptr e) {
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    bool res = e->query();
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+    return res;
+  }
+
+  /// Blocks until the event has completed.
+  void synchronize_event(event_ptr e) {
+    CHECK_CUDA(cuCtxPushCurrent(context_));
+    e->synchronize();
+    CHECK_CUDA(cuCtxPopCurrent(nullptr));
+  }
+
   /// Enable cuBLAS support.
   void enable_cublas() {
     if (!cublas_table_)
diff --git a/libcaf_cuda/caf/cuda/event.hpp b/libcaf_cuda/caf/cuda/event.hpp
new file mode 100644
index 0000000000..e1de8928a3
--- /dev/null
+++ b/libcaf_cuda/caf/cuda/event.hpp
@@ -0,0 +1,57 @@
+#pragma once
+
+#include <cuda.h>
+
+#include <caf/ref_counted.hpp>
+#include <caf/intrusive_ptr.hpp>
+
+#include "caf/cuda/global.hpp"
+
+namespace caf::cuda {
+
+class event;
+using event_ptr = caf::intrusive_ptr<event>;
+
+/**
+ * @brief A smart-pointer managed wrapper for a CUDA event.
+ * Ensures that the CUevent handle is destroyed when the last reference is gone.
+ */
+class event : public caf::ref_counted {
+public:
+  explicit event(unsigned int flags = CU_EVENT_DEFAULT) {
+    // Note: Expects an active CUDA context for creation.
+    check(cuEventCreate(&event_, flags), "cuEventCreate");
+  }
+
+  ~event() {
+    check(cuEventDestroy(event_), "cuEventDestroy");
+  }
+
+  event(const event&) = delete;
+  event& operator=(const event&) = delete;
+
+  CUevent get() const {
+    return event_;
+  }
+
+  /// Returns true if the event has been recorded and the work has completed.
+  bool query() const {
+    CUresult res = cuEventQuery(event_);
+    if (res == CUDA_SUCCESS)
+      return true;
+    if (res == CUDA_ERROR_NOT_READY)
+      return false;
+    check(res, "cuEventQuery");
+    return false;
+  }
+
+  /// Blocks the calling thread until the event has completed.
+  void synchronize() const {
+    check(cuEventSynchronize(event_), "cuEventSynchronize");
+  }
+
+private:
+  CUevent event_;
+};
+
+} // namespace caf::cuda
\ No newline at end of file
diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index 58741dae7d..30f766476f 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -10,6 +10,7 @@
 #include <memory>
 #include "caf/cuda/nd_range.hpp"
 #include "caf/cuda/helpers.hpp"
+#include "caf/cuda/event.hpp"
 #include <nvrtc.h>
 // CAF type ID registration
 #include <caf/type_id.hpp>
@@ -259,6 +260,7 @@ CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id)
   CAF_ADD_TYPE_ID(cuda,(caf::cuda::mem_ptr<float>))  
   CAF_ADD_TYPE_ID(cuda,(caf::cuda::mem_ptr<double>))  
   CAF_ADD_TYPE_ID(cuda,(caf::cuda::mem_ptr<char>))  
+  CAF_ADD_TYPE_ID(cuda, (caf::cuda::event_ptr))
   CAF_ADD_TYPE_ID(cuda, (caf::cuda::matrix_format))
   CAF_ADD_TYPE_ID(cuda, (caf::cuda::solver_result_meta))
   CAF_ADD_TYPE_ID(cuda, (caf::cuda::sparse_cg_solve_context<float>))
@@ -291,6 +293,7 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<int>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<float>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<double>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::mem_ptr<char>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::event_ptr)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::nd_range)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::program_ptr)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(output_mapping)

From ad4a852aee1e1435e8cd4e6bd7a279d19cf7594d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 08:31:09 -0600
Subject: [PATCH 0821/1000] commented out hot potatoe

---
 .../tests/workload-test/CMakeLists.txt        | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/CMakeLists.txt b/libcaf_cuda/tests/workload-test/CMakeLists.txt
index d7ca8d87fa..dcf78fb02c 100644
--- a/libcaf_cuda/tests/workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/workload-test/CMakeLists.txt
@@ -45,20 +45,20 @@ target_link_libraries(test
 )
 
 
-# 5) Declare your executable
-add_executable(hot-potatoe hot-potatoe.cpp sparse_utils.cpp)
-
-target_compile_definitions(hot-potatoe PRIVATE CAF_ENABLE_LOGGING)
-
-target_link_libraries(hot-potatoe
-  PRIVATE
-    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-    CUDA::nvrtc
-    CUDA::cublas
-    CUDA::cusparse
-)
+# # 5) Declare your executable
+# add_executable(hot-potatoe hot-potatoe.cpp sparse_utils.cpp)
+
+# target_compile_definitions(hot-potatoe PRIVATE CAF_ENABLE_LOGGING)
+
+# target_link_libraries(hot-potatoe
+#   PRIVATE
+#     "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+#     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+#     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+#     CUDA::nvrtc
+#     CUDA::cublas
+#     CUDA::cusparse
+# )
 
 
 # FindThreads is required for std::thread in the native version

From 04d4ea2b07b0d3498d02daa406db35c0decffb43 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 08:39:48 -0600
Subject: [PATCH 0822/1000] made work stealing model

---
 libcaf_cuda/tests/workload-test/main.test.cpp | 274 +++++++++---------
 1 file changed, 131 insertions(+), 143 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index 1bdbfe9348..87715b54a9 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -37,6 +37,8 @@ CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
     CAF_ADD_ATOM(workload_test, request_work_atom)
     CAF_ADD_ATOM(workload_test, worker_done_atom)
     CAF_ADD_ATOM(workload_test, work_tick_atom)
+    CAF_ADD_ATOM(workload_test, add_work_atom)
+    CAF_ADD_ATOM(workload_test, steal_work_atom)
     CAF_ADD_TYPE_ID(workload_test, (SolverType))
     CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
     CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
@@ -48,179 +50,120 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
 
-// ---------------------------- GLOBAL TASK POOL ----------------------------
-struct pending_request {
-    caf::response_promise promise;
-    size_t requested_size;
-};
-
-struct pool_state {
-    std::vector<MatrixTask> matrix_pool;
-    std::deque<MatrixTask> work_buffer;
-    std::vector<pending_request> pending_requests;
-    std::mt19937 rng;
-    int batches_remaining;
-    int batch_size;
-    double mean_arrival_ms;
-    bool production_finished = false;
-};
-
-behavior global_task_pool(stateful_actor<pool_state>* self, 
-                          std::vector<MatrixTask> matrix_pool,
-                          int num_batches, int batch_size, double mean_arrival_ms) {
-    auto& st = self->state();
-    st.matrix_pool = std::move(matrix_pool);
-    st.batches_remaining = num_batches;
-    st.batch_size = batch_size;
-    st.mean_arrival_ms = mean_arrival_ms;
-    st.rng.seed(WORKLOAD_SEED);
-
-    self->mail(work_tick_atom_v).send(self);
-
-    return {
-        [=](work_tick_atom) {
-            auto& st = self->state();
-            if (st.batches_remaining > 0) {
-                auto tasks = generate_batch(st.matrix_pool, st.rng, st.batch_size);
-                for (auto& t : tasks)
-                    st.work_buffer.push_back(std::move(t));
-                st.batches_remaining--;
-
-                // Satisfy pending requests from the new work
-                while (!st.pending_requests.empty() && !st.work_buffer.empty()) {
-                    auto req = std::move(st.pending_requests.front());
-                    st.pending_requests.erase(st.pending_requests.begin());
-
-                    size_t count = std::min(req.requested_size, st.work_buffer.size());
-                    std::vector<MatrixTask> batch;
-                    for (size_t i = 0; i < count; ++i) {
-                        batch.push_back(std::move(st.work_buffer.front()));
-                        st.work_buffer.pop_front();
-                    }
-                    req.promise.deliver(std::move(batch));
-                }
-
-                if (st.batches_remaining > 0) {
-                    auto interval = generate_random_interval(st.rng, st.mean_arrival_ms);
-                    std::cout << "[PRODUCER] going to sleep for " << interval.count() << " ms\n";
-                    self->mail(work_tick_atom_v).delay(interval).send(self);
-                } else {
-                    st.production_finished = true;
-                    // If the buffer is empty and no more batches are coming, signal EOS to anyone waiting
-                    if (st.work_buffer.empty()) {
-                        for (auto& req : st.pending_requests)
-                            req.promise.deliver(sec::end_of_stream);
-                        st.pending_requests.clear();
-                    }
-                }
-            }
-        },
-        [=](get_work_atom, size_t batch_size) -> result<std::vector<MatrixTask>> {
-            auto& st = self->state();
-            if (!st.work_buffer.empty()) {
-                size_t count = std::min(batch_size, st.work_buffer.size());
-                std::vector<MatrixTask> batch;
-                for (size_t i = 0; i < count; ++i) {
-                    batch.push_back(std::move(st.work_buffer.front()));
-                    st.work_buffer.pop_front();
-                }
-                return batch;
-            }
-
-            if (st.production_finished)
-                return sec::end_of_stream;
-
-            auto promise = self->make_response_promise();
-            st.pending_requests.push_back({promise, batch_size});
-            return promise;
-        }
-    };
-}
-
 // ---------------------------- WORKER ACTOR ----------------------------
 struct worker_state {
-    caf::actor global_pool;
     caf::actor supervisor;
+    std::vector<caf::actor> peers;
+    std::deque<MatrixTask> work_queue;
     int device_id;
     int stream_id;
-    std::shared_ptr<MatrixData> current_data;
     std::string current_matrix_path;
     std::chrono::steady_clock::time_point task_start;
-    SolverType current_solver_type;
     caf::actor cg_facade;
+    bool stealing = false;
 };
 
 behavior sparse_worker_fun(stateful_actor<worker_state>* self,
-                           caf::actor supervisor, caf::actor global_pool, int dev_id, int stream_id) {
+                           caf::actor supervisor, int dev_id, int stream_id) {
     auto& st = self->state();
     st.supervisor = supervisor;
-    st.global_pool = global_pool;
     st.device_id = dev_id;
     st.stream_id = stream_id;
     st.cg_facade = self->spawn<sparse_cg_facade<float>, linked>(0);
 
-    self->mail(request_work_atom_v).send(self);
-
     return {
+        [=](std::vector<caf::actor>& peers) {
+            self->state().peers = std::move(peers);
+        },
+        [=](add_work_atom, std::vector<MatrixTask>& batch) {
+            auto& st = self->state();
+            bool was_idle = st.work_queue.empty() && !st.stealing;
+            for (auto& t : batch)
+                st.work_queue.push_back(std::move(t));
+            
+            if (was_idle)
+                self->mail(request_work_atom_v).send(self);
+        },
         [=](request_work_atom) {
-            self->mail(get_work_atom_v, size_t{1}).request(self->state().global_pool, infinite).then(
-                [=](std::vector<MatrixTask>& batch) {
-                    if (batch.empty()) {
-                        self->mail(request_work_atom_v).send(self);
-                        return;
-                    }
-                    auto& task = batch.front();
-                    auto& data = *task.data;
-
-                    self->state().current_matrix_path = task.path;
-                    self->state().current_solver_type = task.type;
-                    self->state().task_start = std::chrono::steady_clock::now();
-                    self->state().current_data = task.data;
+            auto& st = self->state();
+            if (!st.work_queue.empty()) {
+                auto task = std::move(st.work_queue.front());
+                st.work_queue.pop_front();
+
+                auto& data = *task.data;
+                st.current_matrix_path = task.path;
+                st.task_start = std::chrono::steady_clock::now();
+
+                if (task.type == CGS_SOLVER) {
+                    self->mail(create_in_arg((const std::vector<int32_t>&)data.row_ptr),
+                               create_in_arg((const std::vector<int32_t>&)data.col_indices),
+                               create_in_arg(data.values), create_in_arg(data.b),
+                               create_in_out_arg(data.x_guess), matrix_format::csr,
+                               (int)data.row_ptr.size() - 1, (int)data.values.size(),
+                               1e-5f, 2000, st.device_id, st.stream_id).send(st.cg_facade);
+                } else {
+                    auto solver = self->spawn<sparse_bicgstab_actor<float>>(
+                        create_in_arg((const std::vector<int32_t>&)data.row_ptr),
+                        create_in_arg((const std::vector<int32_t>&)data.col_indices),
+                        create_in_arg(data.values), create_in_arg(data.b),
+                        create_in_out_arg(data.x_guess), matrix_format::csr,
+                        (int)data.row_ptr.size() - 1, (int)data.values.size(),
+                        1e-5f, 2000, st.device_id, st.stream_id, actor_cast<actor>(self));
+                    self->mail(start_atom_v).send(solver);
+                }
+            } else {
+                // Idle: try to steal from a random peer
+                if (st.peers.empty() || st.stealing) return;
+                
+                st.stealing = true;
+                static std::mt19937 prng(std::random_device{}());
+                std::uniform_int_distribution<size_t> dist(0, st.peers.size() - 1);
+                auto victim = st.peers[dist(prng)];
+
+                if (victim == self) {
+                    st.stealing = false;
+                    self->mail(request_work_atom_v).delay(std::chrono::milliseconds(10)).send(self);
+                    return;
+                }
 
-                    if (task.type == CGS_SOLVER) {
-                        // Use the optimized CG facade. It responds with (r_id, index, solution, meta)
-                        self->mail(create_in_arg((const std::vector<int32_t>&)data.row_ptr),
-                                   create_in_arg((const std::vector<int32_t>&)data.col_indices),
-                                   create_in_arg(data.values), create_in_arg(data.b),
-                                   create_in_out_arg(data.x_guess), matrix_format::csr,
-                                   (int)data.row_ptr.size() - 1, (int)data.values.size(),
-                                   1e-5f, 2000, dev_id, stream_id).send(self->state().cg_facade);
-                    } else {
-                        // BiCGSTAB still uses the standard stateful actor. It responds with (solution, meta)
-                        auto solver = self->spawn<sparse_bicgstab_actor<float>>(
-                            create_in_arg((const std::vector<int32_t>&)data.row_ptr),
-                            create_in_arg((const std::vector<int32_t>&)data.col_indices),
-                            create_in_arg(data.values), create_in_arg(data.b),
-                            create_in_out_arg(data.x_guess), matrix_format::csr,
-                            (int)data.row_ptr.size() - 1, (int)data.values.size(),
-                            1e-5f, 2000, dev_id, stream_id, actor_cast<actor>(self));
-                        self->mail(start_atom_v).send(solver);
+                self->mail(steal_work_atom_v).request(victim, std::chrono::seconds(1)).then(
+                    [=](MatrixTask& stolen_task) {
+                        auto& st = self->state();
+                        st.work_queue.push_back(std::move(stolen_task));
+                        st.stealing = false;
+                        self->mail(request_work_atom_v).send(self);
+                    },
+                    [=](const error&) {
+                        auto& st = self->state();
+                        st.stealing = false;
+                        self->mail(request_work_atom_v).delay(std::chrono::milliseconds(50)).send(self);
                     }
-                },
-                [=](error& err) {
-                    if (err == sec::end_of_stream)
-                        self->quit();
-                }
-            );
+                );
+            }
+        },
+        [=](steal_work_atom) -> result<MatrixTask> {
+            auto& st = self->state();
+            if (st.work_queue.size() > 1) {
+                auto task = std::move(st.work_queue.back());
+                st.work_queue.pop_back();
+                return task;
+            }
+            return sec::no_context;
         },
-        // Result handler for standard stateful actors (e.g., BiCGSTAB)
         [=](std::vector<float>& solution, solver_result_meta meta) {
             auto task_end = std::chrono::steady_clock::now();
             std::chrono::duration<double> task_duration = task_end - self->state().task_start;
-            self->println("Worker {}: Round-trip time (Spawn to Result) for {} (BICSTAB_SOLVER) took {} s (Iters: {})", 
+            self->println("Worker {}: Round-trip for {} (BICSTAB_SOLVER) took {} s (Iters: {})", 
                           self->state().stream_id, self->state().current_matrix_path, task_duration.count(), meta.iterations);
             self->mail(1).send(self->state().supervisor);
-            self->state().current_data.reset();
             self->mail(request_work_atom_v).send(self);
         },
-        // Result handler for the optimized facade actor
         [=](uint32_t /*r_id*/, int /*index*/, std::vector<float>& solution, solver_result_meta meta) {
             auto task_end = std::chrono::steady_clock::now();
             std::chrono::duration<double> task_duration = task_end - self->state().task_start;
-            self->println("Worker {}: Round-trip time (Spawn to Result) for {} (CGS_SOLVER_OPTIMIZED) took {} s (Iters: {})", 
+            self->println("Worker {}: Round-trip for {} (CGS_SOLVER_OPTIMIZED) took {} s (Iters: {})", 
                           self->state().stream_id, self->state().current_matrix_path, task_duration.count(), meta.iterations);
             self->mail(1).send(self->state().supervisor);
-            self->state().current_data.reset();
             self->mail(request_work_atom_v).send(self);
         }
     };
@@ -228,30 +171,75 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
 
 // ---------------------------- SUPERVISOR ACTOR ----------------------------
 struct supervisor_state {
+    std::vector<MatrixTask> matrix_pool;
+    std::vector<caf::actor> workers;
+    std::mt19937 rng;
     int total_tasks;
     int completed = 0;
+    int batches_remaining;
+    int batch_size;
+    double mean_arrival_ms;
 };
 
 behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, 
                               std::vector<MatrixTask> matrix_pool,
                               int num_streams, int num_batches, int batch_size, double mean_arrival_ms) {
-    self->state().total_tasks = num_batches * batch_size;
-    auto pool = self->spawn(global_task_pool, std::move(matrix_pool), num_batches, batch_size, mean_arrival_ms);
+    auto& st = self->state();
+    st.matrix_pool = std::move(matrix_pool);
+    st.total_tasks = num_batches * batch_size;
+    st.batches_remaining = num_batches;
+    st.batch_size = batch_size;
+    st.mean_arrival_ms = mean_arrival_ms;
+    st.rng.seed(WORKLOAD_SEED);
     
     manager& mgr = manager::get();
     int num_gpus = mgr.get_num_devices();
 
+    // Initializing workers
     for (int i = 0; i < num_gpus; ++i) {
         for (int j = 0; j < num_streams; ++j) {
-            self->spawn(sparse_worker_fun, self, pool, i, (i * 100) + j);
+            st.workers.push_back(self->spawn(sparse_worker_fun, self, i, (i * 100) + j));
         }
     }
 
+    // Inform workers of their peers
+    for (auto& w : st.workers)
+        self->mail(st.workers).send(w);
+
+    // Start the production cycle
+    self->mail(work_tick_atom_v).send(self);
+
     return {
+        [=](work_tick_atom) {
+            auto& st = self->state();
+            if (st.batches_remaining > 0) {
+                auto batch = generate_batch(st.matrix_pool, st.rng, st.batch_size);
+                st.batches_remaining--;
+
+                // Partition this specific batch among workers
+                size_t tasks_per_worker = batch.size() / st.workers.size();
+                for (size_t i = 0; i < st.workers.size(); ++i) {
+                    auto start = batch.begin() + i * tasks_per_worker;
+                    auto end = (i == st.workers.size() - 1) ? batch.end() : start + tasks_per_worker;
+                    
+                    std::vector<MatrixTask> segment;
+                    for (auto it = start; it != end; ++it)
+                        segment.push_back(std::move(*it));
+                    
+                    self->mail(add_work_atom_v, std::move(segment)).send(st.workers[i]);
+                }
+
+                if (st.batches_remaining > 0) {
+                    auto delay = generate_random_interval(st.rng, st.mean_arrival_ms);
+                    self->println("[SUPERVISOR] Next batch in {}ms", delay.count());
+                    self->mail(work_tick_atom_v).delay(delay).send(self);
+                }
+            }
+        },
         [=](int done) {
             self->state().completed += done;
             if (self->state().completed >= self->state().total_tasks) {
-                std::cout << "\n[DONE] All " << self->state().total_tasks << " tasks processed.\n";
+                self->println("\n[DONE] All {} tasks processed.", self->state().total_tasks);
                 self->quit();
             }
         }

From 2c5a5418be7566c43f7ad01f2d8b1e76c861da66 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 08:41:46 -0600
Subject: [PATCH 0823/1000] broke circular dependency between global.hpp and
 this file

---
 libcaf_cuda/caf/cuda/event.hpp | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/event.hpp b/libcaf_cuda/caf/cuda/event.hpp
index e1de8928a3..eec93781a2 100644
--- a/libcaf_cuda/caf/cuda/event.hpp
+++ b/libcaf_cuda/caf/cuda/event.hpp
@@ -1,12 +1,12 @@
 #pragma once
 
 #include <cuda.h>
+#include <iostream>
+#include <cstdlib>
 
 #include <caf/ref_counted.hpp>
 #include <caf/intrusive_ptr.hpp>
 
-#include "caf/cuda/global.hpp"
-
 namespace caf::cuda {
 
 class event;
@@ -20,11 +20,11 @@ class event : public caf::ref_counted {
 public:
   explicit event(unsigned int flags = CU_EVENT_DEFAULT) {
     // Note: Expects an active CUDA context for creation.
-    check(cuEventCreate(&event_, flags), "cuEventCreate");
+    check_error(cuEventCreate(&event_, flags), "cuEventCreate");
   }
 
   ~event() {
-    check(cuEventDestroy(event_), "cuEventDestroy");
+    check_error(cuEventDestroy(event_), "cuEventDestroy");
   }
 
   event(const event&) = delete;
@@ -41,16 +41,26 @@ class event : public caf::ref_counted {
       return true;
     if (res == CUDA_ERROR_NOT_READY)
       return false;
-    check(res, "cuEventQuery");
+    check_error(res, "cuEventQuery");
     return false;
   }
 
   /// Blocks the calling thread until the event has completed.
   void synchronize() const {
-    check(cuEventSynchronize(event_), "cuEventSynchronize");
+    check_error(cuEventSynchronize(event_), "cuEventSynchronize");
   }
 
 private:
+  static void check_error(CUresult result, const char* msg) {
+    if (result != CUDA_SUCCESS) {
+      const char* err_str = nullptr;
+      cuGetErrorString(result, &err_str);
+      std::cerr << "CUDA Driver API Error (" << msg << "): " 
+                << (err_str ? err_str : "unknown error") << "\n";
+      std::exit(1);
+    }
+  }
+
   CUevent event_;
 };
 

From 3ab301d04fa207c9744f60d9182a58a6a311f787 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 09:09:11 -0600
Subject: [PATCH 0824/1000] fixed deadlock

---
 libcaf_cuda/tests/workload-test/main.test.cpp | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index 87715b54a9..5d0769f457 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -39,6 +39,7 @@ CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
     CAF_ADD_ATOM(workload_test, work_tick_atom)
     CAF_ADD_ATOM(workload_test, add_work_atom)
     CAF_ADD_ATOM(workload_test, steal_work_atom)
+    CAF_ADD_ATOM(workload_test, shutdown_atom)
     CAF_ADD_TYPE_ID(workload_test, (SolverType))
     CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
     CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
@@ -150,6 +151,9 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
             }
             return sec::no_context;
         },
+        [=](shutdown_atom) {
+            self->quit();
+        },
         [=](std::vector<float>& solution, solver_result_meta meta) {
             auto task_end = std::chrono::steady_clock::now();
             std::chrono::duration<double> task_duration = task_end - self->state().task_start;
@@ -237,9 +241,12 @@ behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self,
             }
         },
         [=](int done) {
-            self->state().completed += done;
-            if (self->state().completed >= self->state().total_tasks) {
-                self->println("\n[DONE] All {} tasks processed.", self->state().total_tasks);
+            auto& st = self->state();
+            st.completed += done;
+            if (st.completed >= st.total_tasks) {
+                self->println("\n[DONE] All {} tasks processed.", st.total_tasks);
+                for (auto& worker : st.workers)
+                    self->mail(shutdown_atom_v).send(worker);
                 self->quit();
             }
         }

From f7965b4176341facc457c88d4a04bdab5000695b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 09:57:20 -0600
Subject: [PATCH 0825/1000] initial commit

---
 .../CMakeLists.txt                            |  76 ++++
 .../main.test.cpp                             | 327 ++++++++++++++++++
 2 files changed, 403 insertions(+)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
new file mode 100644
index 0000000000..356dd87f4f
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
@@ -0,0 +1,76 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executable
+add_executable(test main.test.cpp)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
+)
+
+
+# # 5) Declare your executable
+# add_executable(hot-potatoe hot-potatoe.cpp sparse_utils.cpp)
+
+# target_compile_definitions(hot-potatoe PRIVATE CAF_ENABLE_LOGGING)
+
+# target_link_libraries(hot-potatoe
+#   PRIVATE
+#     "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+#     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+#     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+#     CUDA::nvrtc
+#     CUDA::cublas
+#     CUDA::cusparse
+# )
+
+
+# # FindThreads is required for std::thread in the native version
+# find_package(Threads REQUIRED)
+
+# # 6) Declare the native benchmark executable (raw CUDA/cuBLAS/cuSPARSE)
+# add_executable(workload-native main.native.cpp sparse_utils.cpp)
+
+# target_link_libraries(workload-native
+#   PRIVATE
+#     CUDA::cudart
+#     CUDA::cublas
+#     CUDA::cusparse
+#     Threads::Threads
+# )
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
new file mode 100644
index 0000000000..1f183a9973
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -0,0 +1,327 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <deque>
+#include <string>
+#include <chrono>
+#include <numeric>
+#include <cmath>
+#include <algorithm>
+#include <filesystem>
+#include <memory>
+#include "caf/actorSOLVE/actorSOLVE.hpp"
+#include "sparse_utils.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+namespace fs = std::filesystem;
+
+constexpr uint32_t WORKLOAD_SEED = 42;
+
+template <class Inspector>
+bool inspect(Inspector& f, SolverType& x) {
+    auto val = static_cast<int>(x);
+    if (f.apply(val)) {
+        if constexpr (Inspector::is_loading)
+            x = static_cast<SolverType>(val);
+        return true;
+    }
+    return false;
+}
+
+CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
+    CAF_ADD_ATOM(workload_test, get_work_atom)
+    CAF_ADD_ATOM(workload_test, release_memory_atom)
+    CAF_ADD_ATOM(workload_test, request_work_atom)
+    CAF_ADD_ATOM(workload_test, worker_done_atom)
+    CAF_ADD_ATOM(workload_test, work_tick_atom)
+    CAF_ADD_ATOM(workload_test, add_work_atom)
+    CAF_ADD_ATOM(workload_test, steal_work_atom)
+    CAF_ADD_ATOM(workload_test, shutdown_atom)
+    CAF_ADD_TYPE_ID(workload_test, (SolverType))
+    CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
+    CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
+    CAF_ADD_TYPE_ID(workload_test, (std::vector<caf::actor>))
+    CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
+CAF_END_TYPE_ID_BLOCK(workload_test)
+
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixData)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
+
+// ---------------------------- WORKER ACTOR ----------------------------
+struct worker_state {
+    caf::actor supervisor;
+    std::vector<caf::actor> peers;
+    std::deque<MatrixTask> work_queue;
+    int device_id;
+    int stream_id;
+    std::string current_matrix_path;
+    std::chrono::steady_clock::time_point task_start;
+    caf::actor cg_facade;
+    bool stealing = false;
+};
+
+behavior sparse_worker_fun(stateful_actor<worker_state>* self,
+                           caf::actor supervisor, int dev_id, int stream_id) {
+    auto& st = self->state();
+    st.supervisor = supervisor;
+    st.device_id = dev_id;
+    st.stream_id = stream_id;
+    st.cg_facade = self->spawn<sparse_cg_facade<float>, linked>(0);
+
+    return {
+        [=](std::vector<caf::actor>& peers) {
+            self->state().peers = std::move(peers);
+        },
+        [=](add_work_atom, std::vector<MatrixTask>& batch) {
+            auto& st = self->state();
+            bool was_idle = st.work_queue.empty() && !st.stealing;
+            for (auto& t : batch)
+                st.work_queue.push_back(std::move(t));
+            
+            if (was_idle)
+                self->mail(request_work_atom_v).send(self);
+        },
+        [=](request_work_atom) {
+            auto& st = self->state();
+            if (!st.work_queue.empty()) {
+                auto task = std::move(st.work_queue.front());
+                st.work_queue.pop_front();
+
+                auto& data = *task.data;
+                st.current_matrix_path = task.path;
+                st.task_start = std::chrono::steady_clock::now();
+
+                if (task.type == CGS_SOLVER) {
+                    self->mail(create_in_arg((const std::vector<int32_t>&)data.row_ptr),
+                               create_in_arg((const std::vector<int32_t>&)data.col_indices),
+                               create_in_arg(data.values), create_in_arg(data.b),
+                               create_in_out_arg(data.x_guess), matrix_format::csr,
+                               (int)data.row_ptr.size() - 1, (int)data.values.size(),
+                               1e-5f, 2000, st.device_id, st.stream_id).send(st.cg_facade);
+                } else {
+                    auto solver = self->spawn<sparse_bicgstab_actor<float>>(
+                        create_in_arg((const std::vector<int32_t>&)data.row_ptr),
+                        create_in_arg((const std::vector<int32_t>&)data.col_indices),
+                        create_in_arg(data.values), create_in_arg(data.b),
+                        create_in_out_arg(data.x_guess), matrix_format::csr,
+                        (int)data.row_ptr.size() - 1, (int)data.values.size(),
+                        1e-5f, 2000, st.device_id, st.stream_id, actor_cast<actor>(self));
+                    self->mail(start_atom_v).send(solver);
+                }
+            } else {
+                // Idle: try to steal from a random peer
+                if (st.peers.empty() || st.stealing) return;
+                
+                st.stealing = true;
+                static std::mt19937 prng(std::random_device{}());
+                std::uniform_int_distribution<size_t> dist(0, st.peers.size() - 1);
+                auto victim = st.peers[dist(prng)];
+
+                if (victim == self) {
+                    st.stealing = false;
+                    self->mail(request_work_atom_v).delay(std::chrono::milliseconds(10)).send(self);
+                    return;
+                }
+
+                self->mail(steal_work_atom_v).request(victim, std::chrono::seconds(1)).then(
+                    [=](MatrixTask& stolen_task) {
+                        auto& st = self->state();
+                        st.work_queue.push_back(std::move(stolen_task));
+                        st.stealing = false;
+                        self->mail(request_work_atom_v).send(self);
+                    },
+                    [=](const error&) {
+                        auto& st = self->state();
+                        st.stealing = false;
+                        self->mail(request_work_atom_v).delay(std::chrono::milliseconds(50)).send(self);
+                    }
+                );
+            }
+        },
+        [=](steal_work_atom) -> result<MatrixTask> {
+            auto& st = self->state();
+            if (st.work_queue.size() > 1) {
+                auto task = std::move(st.work_queue.back());
+                st.work_queue.pop_back();
+                return task;
+            }
+            return sec::no_context;
+        },
+        [=](shutdown_atom) {
+            self->mail(worker_done_atom_v).send(self->state().supervisor);
+            self->quit();
+        },
+        [=](std::vector<float>& solution, solver_result_meta meta) {
+            auto task_end = std::chrono::steady_clock::now();
+            std::chrono::duration<double> task_duration = task_end - self->state().task_start;
+            self->println("Worker {}: Round-trip for {} (BICSTAB_SOLVER) took {} s (Iters: {})", 
+                          self->state().stream_id, self->state().current_matrix_path, task_duration.count(), meta.iterations);
+            self->mail(1).send(self->state().supervisor);
+            self->mail(request_work_atom_v).send(self);
+        },
+        [=](uint32_t /*r_id*/, int /*index*/, std::vector<float>& solution, solver_result_meta meta) {
+            auto task_end = std::chrono::steady_clock::now();
+            std::chrono::duration<double> task_duration = task_end - self->state().task_start;
+            self->println("Worker {}: Round-trip for {} (CGS_SOLVER_OPTIMIZED) took {} s (Iters: {})", 
+                          self->state().stream_id, self->state().current_matrix_path, task_duration.count(), meta.iterations);
+            self->mail(1).send(self->state().supervisor);
+            self->mail(request_work_atom_v).send(self);
+        }
+    };
+}
+
+// ---------------------------- SUPERVISOR ACTOR ----------------------------
+struct supervisor_state {
+    std::vector<MatrixTask> matrix_pool;
+    std::vector<caf::actor> workers;
+    std::mt19937 rng;
+    int total_tasks;
+    int completed = 0;
+    int batches_remaining;
+    int batch_size;
+    double mean_arrival_ms;
+    caf::actor parent;
+    int workers_shutdown = 0;
+};
+
+behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, 
+                              std::vector<MatrixTask> matrix_pool,
+                              int num_streams, int num_batches, int batch_size, double mean_arrival_ms, caf::actor parent) {
+    auto& st = self->state();
+    st.matrix_pool = std::move(matrix_pool);
+    st.parent = std::move(parent);
+    st.total_tasks = num_batches * batch_size;
+    st.batches_remaining = num_batches;
+    st.batch_size = batch_size;
+    st.mean_arrival_ms = mean_arrival_ms;
+    st.rng.seed(WORKLOAD_SEED);
+    
+    manager& mgr = manager::get();
+    int num_gpus = mgr.get_num_devices();
+
+    // Initializing workers
+    for (int i = 0; i < num_gpus; ++i) {
+        for (int j = 0; j < num_streams; ++j) {
+            st.workers.push_back(self->spawn(sparse_worker_fun, self, i, (i * 100) + j));
+        }
+    }
+
+    // Inform workers of their peers
+    for (auto& w : st.workers)
+        self->mail(st.workers).send(w);
+
+    // Start the production cycle
+    self->mail(work_tick_atom_v).send(self);
+
+    return {
+        [=](work_tick_atom) {
+            auto& st = self->state();
+            if (st.batches_remaining > 0) {
+                auto batch = generate_batch(st.matrix_pool, st.rng, st.batch_size);
+                st.batches_remaining--;
+
+                // Partition this specific batch among workers
+                size_t tasks_per_worker = batch.size() / st.workers.size();
+                for (size_t i = 0; i < st.workers.size(); ++i) {
+                    auto start = batch.begin() + i * tasks_per_worker;
+                    auto end = (i == st.workers.size() - 1) ? batch.end() : start + tasks_per_worker;
+                    
+                    std::vector<MatrixTask> segment;
+                    for (auto it = start; it != end; ++it)
+                        segment.push_back(std::move(*it));
+                    
+                    self->mail(add_work_atom_v, std::move(segment)).send(st.workers[i]);
+                }
+
+                if (st.batches_remaining > 0) {
+                    auto delay = generate_random_interval(st.rng, st.mean_arrival_ms);
+                    self->println("[SUPERVISOR] Next batch in {}ms", delay.count());
+                    self->mail(work_tick_atom_v).delay(delay).send(self);
+                }
+            }
+        },
+        [=](int done) {
+            auto& st = self->state();
+            st.completed += done;
+            if (st.completed >= st.total_tasks) {
+                self->println("\n[DONE] All {} tasks processed. Terminating workers...", st.total_tasks);
+                for (auto& worker : st.workers)
+                    self->mail(shutdown_atom_v).send(worker);
+            }
+        },
+        [=](worker_done_atom) {
+            auto& st = self->state();
+            if (++st.workers_shutdown == (int)st.workers.size()) {
+                self->mail(worker_done_atom_v).send(st.parent);
+                self->quit();
+            }
+        }
+    };
+}
+
+void caf_main(actor_system& sys) {
+    manager::init(sys, manager_config(true, true));
+    
+    int num_streams = 8;
+    int num_batches = 25;
+    int batch_size = 100;
+    double mean_arrival_ms = 1000.0;
+
+    // // Basic command line argument parsing similar to native
+    // auto& args = sys.config().remainder;
+    // if (args.size() > 0) num_streams = std::max(1, std::stoi(args[0]));
+    // if (args.size() > 1) num_batches = std::max(1, std::stoi(args[1]));
+    // if (args.size() > 2) batch_size = std::max(1, std::stoi(args[2]));
+    // if (args.size() > 3) mean_arrival_ms = std::stod(args[3]);
+
+    std::cout << "[INFO] Loading matrices into memory...\n";
+    auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
+    //scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
+    
+    if (tasks_vec.empty()) {
+        std::cerr << "No matrix files found in search paths.\n";
+        manager::shutdown();
+        return;
+    }
+
+    std::cout << "[INFO] Matrix pool size: " << tasks_vec.size() << "\n";
+    std::cout << "[INFO] Streams/GPU:      " << num_streams << "\n";
+    std::cout << "[INFO] Batches:          " << num_batches << "\n";
+    std::cout << "[INFO] Batch size:       " << batch_size << "\n";
+    std::cout << "[INFO] Mean arrival:     " << mean_arrival_ms << " ms\n";
+
+    auto start = std::chrono::steady_clock::now();
+    scoped_actor self{sys};
+
+    self->spawn(supervisor_actor_fun, std::move(tasks_vec), num_streams, num_batches, batch_size, mean_arrival_ms, actor_cast<actor>(self));
+    
+    self->receive(
+        [&](worker_done_atom) {
+            std::cout << "[INFO] Supervisor signaled completion. Shutting down...\n";
+        }
+    );
+
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+
+    int total_tasks = num_batches * batch_size;
+
+    std::cout << "\n===== BENCHMARK COMPLETE =====\n";
+    std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
+    std::cout << "Streams per GPU:    " << num_streams << "\n";
+    std::cout << "Batches:            " << num_batches << "\n";
+    std::cout << "Batch Size:         " << batch_size << "\n";
+    std::cout << "Mean Arrival (ms):  " << mean_arrival_ms << "\n";
+    std::cout << "Tasks Processed:    " << total_tasks << "\n";
+    std::cout << "Total Runtime:      " << elapsed.count() << " s\n";
+    std::cout << "Throughput:         " << total_tasks / elapsed.count() << " tasks/s\n";
+    std::cout << "==============================\n";
+
+    manager::shutdown();
+}
+CAF_MAIN(id_block::cuda,id_block::workload_test)
\ No newline at end of file

From 9d211ec4cd4bce082ebca7b09fbd61a18ec02129 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 10:15:04 -0600
Subject: [PATCH 0826/1000] saving

---
 .../main.test.cpp                             | 423 +++++++-----------
 1 file changed, 164 insertions(+), 259 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 1f183a9973..c89a826ade 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -52,276 +52,181 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
 
-// ---------------------------- WORKER ACTOR ----------------------------
-struct worker_state {
-    caf::actor supervisor;
-    std::vector<caf::actor> peers;
-    std::deque<MatrixTask> work_queue;
-    int device_id;
-    int stream_id;
-    std::string current_matrix_path;
-    std::chrono::steady_clock::time_point task_start;
-    caf::actor cg_facade;
-    bool stealing = false;
-};
-
-behavior sparse_worker_fun(stateful_actor<worker_state>* self,
-                           caf::actor supervisor, int dev_id, int stream_id) {
-    auto& st = self->state();
-    st.supervisor = supervisor;
-    st.device_id = dev_id;
-    st.stream_id = stream_id;
-    st.cg_facade = self->spawn<sparse_cg_facade<float>, linked>(0);
-
-    return {
-        [=](std::vector<caf::actor>& peers) {
-            self->state().peers = std::move(peers);
-        },
-        [=](add_work_atom, std::vector<MatrixTask>& batch) {
-            auto& st = self->state();
-            bool was_idle = st.work_queue.empty() && !st.stealing;
-            for (auto& t : batch)
-                st.work_queue.push_back(std::move(t));
-            
-            if (was_idle)
-                self->mail(request_work_atom_v).send(self);
-        },
-        [=](request_work_atom) {
-            auto& st = self->state();
-            if (!st.work_queue.empty()) {
-                auto task = std::move(st.work_queue.front());
-                st.work_queue.pop_front();
-
-                auto& data = *task.data;
-                st.current_matrix_path = task.path;
-                st.task_start = std::chrono::steady_clock::now();
-
-                if (task.type == CGS_SOLVER) {
-                    self->mail(create_in_arg((const std::vector<int32_t>&)data.row_ptr),
-                               create_in_arg((const std::vector<int32_t>&)data.col_indices),
-                               create_in_arg(data.values), create_in_arg(data.b),
-                               create_in_out_arg(data.x_guess), matrix_format::csr,
-                               (int)data.row_ptr.size() - 1, (int)data.values.size(),
-                               1e-5f, 2000, st.device_id, st.stream_id).send(st.cg_facade);
-                } else {
-                    auto solver = self->spawn<sparse_bicgstab_actor<float>>(
-                        create_in_arg((const std::vector<int32_t>&)data.row_ptr),
-                        create_in_arg((const std::vector<int32_t>&)data.col_indices),
-                        create_in_arg(data.values), create_in_arg(data.b),
-                        create_in_out_arg(data.x_guess), matrix_format::csr,
-                        (int)data.row_ptr.size() - 1, (int)data.values.size(),
-                        1e-5f, 2000, st.device_id, st.stream_id, actor_cast<actor>(self));
-                    self->mail(start_atom_v).send(solver);
-                }
-            } else {
-                // Idle: try to steal from a random peer
-                if (st.peers.empty() || st.stealing) return;
-                
-                st.stealing = true;
-                static std::mt19937 prng(std::random_device{}());
-                std::uniform_int_distribution<size_t> dist(0, st.peers.size() - 1);
-                auto victim = st.peers[dist(prng)];
-
-                if (victim == self) {
-                    st.stealing = false;
-                    self->mail(request_work_atom_v).delay(std::chrono::milliseconds(10)).send(self);
-                    return;
-                }
-
-                self->mail(steal_work_atom_v).request(victim, std::chrono::seconds(1)).then(
-                    [=](MatrixTask& stolen_task) {
-                        auto& st = self->state();
-                        st.work_queue.push_back(std::move(stolen_task));
-                        st.stealing = false;
-                        self->mail(request_work_atom_v).send(self);
-                    },
-                    [=](const error&) {
-                        auto& st = self->state();
-                        st.stealing = false;
-                        self->mail(request_work_atom_v).delay(std::chrono::milliseconds(50)).send(self);
-                    }
-                );
-            }
-        },
-        [=](steal_work_atom) -> result<MatrixTask> {
-            auto& st = self->state();
-            if (st.work_queue.size() > 1) {
-                auto task = std::move(st.work_queue.back());
-                st.work_queue.pop_back();
-                return task;
-            }
-            return sec::no_context;
-        },
-        [=](shutdown_atom) {
-            self->mail(worker_done_atom_v).send(self->state().supervisor);
-            self->quit();
-        },
-        [=](std::vector<float>& solution, solver_result_meta meta) {
-            auto task_end = std::chrono::steady_clock::now();
-            std::chrono::duration<double> task_duration = task_end - self->state().task_start;
-            self->println("Worker {}: Round-trip for {} (BICSTAB_SOLVER) took {} s (Iters: {})", 
-                          self->state().stream_id, self->state().current_matrix_path, task_duration.count(), meta.iterations);
-            self->mail(1).send(self->state().supervisor);
-            self->mail(request_work_atom_v).send(self);
-        },
-        [=](uint32_t /*r_id*/, int /*index*/, std::vector<float>& solution, solver_result_meta meta) {
-            auto task_end = std::chrono::steady_clock::now();
-            std::chrono::duration<double> task_duration = task_end - self->state().task_start;
-            self->println("Worker {}: Round-trip for {} (CGS_SOLVER_OPTIMIZED) took {} s (Iters: {})", 
-                          self->state().stream_id, self->state().current_matrix_path, task_duration.count(), meta.iterations);
-            self->mail(1).send(self->state().supervisor);
-            self->mail(request_work_atom_v).send(self);
-        }
-    };
+const char* check_stability_src = R"(
+extern "C" __global__
+void check_stability(int n, const float* x, const float* r, int* err) {
+  int idx = blockIdx.x * blockDim.x + threadIdx.x;
+  if (idx < n) {
+    if (isnan(x[idx]) || isinf(x[idx]) || isnan(r[idx]) || isinf(r[idx]))
+      *err = 1;
+  }
 }
-
-// ---------------------------- SUPERVISOR ACTOR ----------------------------
-struct supervisor_state {
-    std::vector<MatrixTask> matrix_pool;
-    std::vector<caf::actor> workers;
-    std::mt19937 rng;
-    int total_tasks;
-    int completed = 0;
-    int batches_remaining;
-    int batch_size;
-    double mean_arrival_ms;
-    caf::actor parent;
-    int workers_shutdown = 0;
+)";
+
+/**
+ * Error codes for solver_result_meta.error_code
+ */
+enum cg_error_type : int {
+  CG_SUCCESS = 0,
+  CG_MAX_ITER = 1,
+  CG_NAN_INF = 2,
+  CG_STAGNATION = 3,
+  CG_BREAKDOWN = 4
+  CG_RESIDUAL_FACTOR_FAIL = 5
 };
 
-behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, 
-                              std::vector<MatrixTask> matrix_pool,
-                              int num_streams, int num_batches, int batch_size, double mean_arrival_ms, caf::actor parent) {
-    auto& st = self->state();
-    st.matrix_pool = std::move(matrix_pool);
-    st.parent = std::move(parent);
-    st.total_tasks = num_batches * batch_size;
-    st.batches_remaining = num_batches;
-    st.batch_size = batch_size;
-    st.mean_arrival_ms = mean_arrival_ms;
-    st.rng.seed(WORKLOAD_SEED);
-    
-    manager& mgr = manager::get();
-    int num_gpus = mgr.get_num_devices();
+// ---------------------------- FAULT TOLERANT SOLVER ----------------------------
+
+template <class T = float>
+struct ft_cg_state {
+  // Host Data
+  in<int> h_row_ptr, h_col_ind;
+  in<T> h_values, h_b;
+  in_out<T> h_x;
+
+  // GPU Buffers
+  mem_ptr<int> A_rp, A_ci, d_err;
+  mem_ptr<T> A_val, b, x, r, p, w, y_tmp;
+  mem_ptr<char> spmv_ws;
+
+  // Config
+  int n, nnz, max_iter;
+  int iterations = 0;
+  T tol;
+  int device_id, stream_id;
+
+  // Supervision & Monitoring
+  caf::actor supervisor;
+  device_ptr d_ptr;
+  program_ptr stab_prog;
+
+  T initial_rho = 0;
+  T current_rho = 0;
+};
 
-    // Initializing workers
-    for (int i = 0; i < num_gpus; ++i) {
-        for (int j = 0; j < num_streams; ++j) {
-            st.workers.push_back(self->spawn(sparse_worker_fun, self, i, (i * 100) + j));
+template <class T = float>
+behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
+                                 in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+                                 int n, int nnz, T tol, int max_iter,
+                                 int dev_num, int stream, caf::actor supervisor) {
+  auto& s = self->state();
+  s.h_row_ptr = std::move(rp); s.h_col_ind = std::move(ci);
+  s.h_values = std::move(val); s.h_b = std::move(b_in); s.h_x = std::move(x_in);
+  s.n = n; s.nnz = nnz; s.tol = tol; s.max_iter = max_iter;
+  s.device_id = dev_num; s.stream_id = stream; s.supervisor = supervisor;
+
+  return {
+    [self](start_atom) {
+      auto& st = self->state();
+      command_runner<T> runner;
+
+      // Setup and transfer memory
+      auto res = runner.transfer_memory(st.device_id, st.stream_id, st.h_row_ptr, st.h_col_ind, st.h_values, st.h_b, st.h_x);
+      st.A_rp = std::get<0>(res); st.A_ci = std::get<1>(res); st.A_val = std::get<2>(res);
+      st.b = std::get<3>(res); st.x = std::get<4>(res);
+
+      st.d_ptr = platform::create()->schedule(st.stream_id, st.device_id);
+      st.d_ptr->enable_cublas(); st.d_ptr->enable_cusparse();
+      auto& mgr = manager::get();
+      st.stab_prog = mgr.create_program(check_stability_src, "check_stability", st.d_ptr);
+
+      command_runner<out<T>> work_runner;
+      st.r = work_runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.p = work_runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.w = work_runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.y_tmp = work_runner.transfer_memory(st.device_id, st.stream_id, out<T>(1));
+      st.d_err = command_runner<in_out<int>>{}.transfer_memory(st.device_id, st.stream_id, in_out<int>(0));
+
+      // SpMV workspace allocation
+      size_t ws_sz = st.d_ptr->spmv_csr_buffer_size(st.stream_id, st.n, st.n, st.nnz, st.A_rp, st.A_ci, st.A_val, st.x, st.w);
+      if (ws_sz > 0) st.spmv_ws = command_runner<out<char>>{}.transfer_memory(st.device_id, st.stream_id, out<char>{static_cast<int>(ws_sz)});
+
+      // Initial r = b - Ax, initial rho = r*r
+      st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.x, T{0}, st.w, st.spmv_ws);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.b, st.r);
+      else st.d_ptr->scopy(st.stream_id, st.n, st.b, st.r);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->daxpy(st.stream_id, st.n, -1.0, st.w, st.r);
+      else st.d_ptr->saxpy(st.stream_id, st.n, -1.0f, st.w, st.r);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+      else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+
+      st.initial_rho = runner.copy_to_host(st.y_tmp)[0];
+      st.current_rho = st.initial_rho;
+      T threshold = st.tol * st.tol;
+      T old_rho = 0;
+      int code = CG_SUCCESS;
+
+      // SIMPLE TIGHT LOOP (Synchronous execution within handler to avoid overhead)
+      while (st.iterations < st.max_iter && st.current_rho > threshold) {
+        st.iterations++;
+        if (st.iterations > 1) {
+          T beta = st.current_rho / old_rho;
+          if constexpr (std::is_same_v<T, double>) {
+            st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.w);
+            st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
+            st.d_ptr->dcopy(st.stream_id, st.n, st.w, st.p);
+          } else {
+            st.d_ptr->scopy(st.stream_id, st.n, st.r, st.w);
+            st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(beta), st.p, st.w);
+            st.d_ptr->scopy(st.stream_id, st.n, st.w, st.p);
+          }
+        } else {
+          if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.p);
+          else st.d_ptr->scopy(st.stream_id, st.n, st.r, st.p);
+        }
+        st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.p, T{0}, st.w, st.spmv_ws);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
+        else st.d_ptr->sdot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
+        T dot_pw = runner.copy_to_host(st.y_tmp)[0];
+        
+        if (std::abs(dot_pw) < 1e-25) { code = CG_BREAKDOWN; break; }
+        
+        T alpha = st.current_rho / dot_pw;
+        if constexpr (std::is_same_v<T, double>) {
+          st.d_ptr->daxpy(st.stream_id, st.n, alpha, st.p, st.x);
+          st.d_ptr->daxpy(st.stream_id, st.n, -alpha, st.w, st.r);
+        } else {
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(alpha), st.p, st.x);
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(-alpha), st.w, st.r);
         }
+        old_rho = st.current_rho;
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+        else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+        st.current_rho = runner.copy_to_host(st.y_tmp)[0];
+        
+        if (std::abs(old_rho - st.current_rho) < 1e-18) { code = CG_STAGNATION; break; }
+      }
+
+      // POST-LOOP FAULT CHECKS
+      bool converged = (st.current_rho <= threshold);
+      if (code == CG_SUCCESS) {
+        if (!converged) code = CG_MAX_ITER;
+        // Residual decrease check: fail if residual didn't decrease by factor of 10
+        if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.1) code = CG_RESIDUAL_FACTOR_FAIL;
+      }
+
+      // Launch NaN/Inf stability kernel
+      nd_range range((st.n + 255) / 256, 1, 1, 256, 1, 1);
+      st.d_ptr->launch_kernel_mem_ref(st.stab_prog->get_kernel(st.d_ptr->getId()), range,
+                                      std::make_tuple(in<int>(st.n), st.x, st.r, st.d_err), st.stream_id);
+      
+      int err_flag = runner.copy_to_host(st.d_err)[0];
+      if (err_flag != 0 || std::isnan(st.current_rho) || std::isinf(st.current_rho)) code = CG_NAN_INF;
+
+      solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
+      runner.copy_to_host_async(st.x, [self, meta, supervisor = st.supervisor](std::vector<T> sol) {
+        anon_mail(std::move(sol), meta).send(supervisor);
+        self->quit();
+      });
     }
+  };
+}
 
-    // Inform workers of their peers
-    for (auto& w : st.workers)
-        self->mail(st.workers).send(w);
-
-    // Start the production cycle
-    self->mail(work_tick_atom_v).send(self);
-
-    return {
-        [=](work_tick_atom) {
-            auto& st = self->state();
-            if (st.batches_remaining > 0) {
-                auto batch = generate_batch(st.matrix_pool, st.rng, st.batch_size);
-                st.batches_remaining--;
-
-                // Partition this specific batch among workers
-                size_t tasks_per_worker = batch.size() / st.workers.size();
-                for (size_t i = 0; i < st.workers.size(); ++i) {
-                    auto start = batch.begin() + i * tasks_per_worker;
-                    auto end = (i == st.workers.size() - 1) ? batch.end() : start + tasks_per_worker;
-                    
-                    std::vector<MatrixTask> segment;
-                    for (auto it = start; it != end; ++it)
-                        segment.push_back(std::move(*it));
-                    
-                    self->mail(add_work_atom_v, std::move(segment)).send(st.workers[i]);
-                }
 
-                if (st.batches_remaining > 0) {
-                    auto delay = generate_random_interval(st.rng, st.mean_arrival_ms);
-                    self->println("[SUPERVISOR] Next batch in {}ms", delay.count());
-                    self->mail(work_tick_atom_v).delay(delay).send(self);
-                }
-            }
-        },
-        [=](int done) {
-            auto& st = self->state();
-            st.completed += done;
-            if (st.completed >= st.total_tasks) {
-                self->println("\n[DONE] All {} tasks processed. Terminating workers...", st.total_tasks);
-                for (auto& worker : st.workers)
-                    self->mail(shutdown_atom_v).send(worker);
-            }
-        },
-        [=](worker_done_atom) {
-            auto& st = self->state();
-            if (++st.workers_shutdown == (int)st.workers.size()) {
-                self->mail(worker_done_atom_v).send(st.parent);
-                self->quit();
-            }
-        }
-    };
-}
 
 void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
-    
-    int num_streams = 8;
-    int num_batches = 25;
-    int batch_size = 100;
-    double mean_arrival_ms = 1000.0;
-
-    // // Basic command line argument parsing similar to native
-    // auto& args = sys.config().remainder;
-    // if (args.size() > 0) num_streams = std::max(1, std::stoi(args[0]));
-    // if (args.size() > 1) num_batches = std::max(1, std::stoi(args[1]));
-    // if (args.size() > 2) batch_size = std::max(1, std::stoi(args[2]));
-    // if (args.size() > 3) mean_arrival_ms = std::stod(args[3]);
-
-    std::cout << "[INFO] Loading matrices into memory...\n";
-    auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
-    //scan("/scratch/nqr159/matrix-collection/matrices/unsymmetric", BICSTAB_SOLVER);
-    
-    if (tasks_vec.empty()) {
-        std::cerr << "No matrix files found in search paths.\n";
-        manager::shutdown();
-        return;
-    }
-
-    std::cout << "[INFO] Matrix pool size: " << tasks_vec.size() << "\n";
-    std::cout << "[INFO] Streams/GPU:      " << num_streams << "\n";
-    std::cout << "[INFO] Batches:          " << num_batches << "\n";
-    std::cout << "[INFO] Batch size:       " << batch_size << "\n";
-    std::cout << "[INFO] Mean arrival:     " << mean_arrival_ms << " ms\n";
-
-    auto start = std::chrono::steady_clock::now();
-    scoped_actor self{sys};
-
-    self->spawn(supervisor_actor_fun, std::move(tasks_vec), num_streams, num_batches, batch_size, mean_arrival_ms, actor_cast<actor>(self));
-    
-    self->receive(
-        [&](worker_done_atom) {
-            std::cout << "[INFO] Supervisor signaled completion. Shutting down...\n";
-        }
-    );
-
-    auto end = std::chrono::steady_clock::now();
-    std::chrono::duration<double> elapsed = end - start;
-
-    int total_tasks = num_batches * batch_size;
-
-    std::cout << "\n===== BENCHMARK COMPLETE =====\n";
-    std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
-    std::cout << "Streams per GPU:    " << num_streams << "\n";
-    std::cout << "Batches:            " << num_batches << "\n";
-    std::cout << "Batch Size:         " << batch_size << "\n";
-    std::cout << "Mean Arrival (ms):  " << mean_arrival_ms << "\n";
-    std::cout << "Tasks Processed:    " << total_tasks << "\n";
-    std::cout << "Total Runtime:      " << elapsed.count() << " s\n";
-    std::cout << "Throughput:         " << total_tasks / elapsed.count() << " tasks/s\n";
-    std::cout << "==============================\n";
-
+   
     manager::shutdown();
 }
 CAF_MAIN(id_block::cuda,id_block::workload_test)
\ No newline at end of file

From cd2a5d2dd5a6d7ec0201a8bd9eaf5502f818e8ba Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 10:23:11 -0600
Subject: [PATCH 0827/1000] created supervisor

---
 .../main.test.cpp                             | 161 +++++++++++++-----
 1 file changed, 121 insertions(+), 40 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index c89a826ade..d2d9a3cc66 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -52,17 +52,6 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
 
-const char* check_stability_src = R"(
-extern "C" __global__
-void check_stability(int n, const float* x, const float* r, int* err) {
-  int idx = blockIdx.x * blockDim.x + threadIdx.x;
-  if (idx < n) {
-    if (isnan(x[idx]) || isinf(x[idx]) || isnan(r[idx]) || isinf(r[idx]))
-      *err = 1;
-  }
-}
-)";
-
 /**
  * Error codes for solver_result_meta.error_code
  */
@@ -71,7 +60,7 @@ enum cg_error_type : int {
   CG_MAX_ITER = 1,
   CG_NAN_INF = 2,
   CG_STAGNATION = 3,
-  CG_BREAKDOWN = 4
+  CG_BREAKDOWN = 4,
   CG_RESIDUAL_FACTOR_FAIL = 5
 };
 
@@ -102,6 +91,8 @@ struct ft_cg_state {
 
   T initial_rho = 0;
   T current_rho = 0;
+  T old_rho = 0;
+  bool initialized = false;
 };
 
 template <class T = float>
@@ -116,19 +107,21 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
   s.device_id = dev_num; s.stream_id = stream; s.supervisor = supervisor;
 
   return {
-    [self](start_atom) {
+    [=](start_atom) {
       auto& st = self->state();
-      command_runner<T> runner;
+      if (st.initialized) return;
 
-      // Setup and transfer memory
+      command_runner<T> runner;
       auto res = runner.transfer_memory(st.device_id, st.stream_id, st.h_row_ptr, st.h_col_ind, st.h_values, st.h_b, st.h_x);
       st.A_rp = std::get<0>(res); st.A_ci = std::get<1>(res); st.A_val = std::get<2>(res);
       st.b = std::get<3>(res); st.x = std::get<4>(res);
 
       st.d_ptr = platform::create()->schedule(st.stream_id, st.device_id);
       st.d_ptr->enable_cublas(); st.d_ptr->enable_cusparse();
+
       auto& mgr = manager::get();
-      st.stab_prog = mgr.create_program(check_stability_src, "check_stability", st.d_ptr);
+      // Load stability kernel from file as requested
+      st.stab_prog = mgr.create_program_from_cubin("stability_kernels.cubin", "check_stability", st.d_ptr);
 
       command_runner<out<T>> work_runner;
       st.r = work_runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
@@ -137,11 +130,9 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
       st.y_tmp = work_runner.transfer_memory(st.device_id, st.stream_id, out<T>(1));
       st.d_err = command_runner<in_out<int>>{}.transfer_memory(st.device_id, st.stream_id, in_out<int>(0));
 
-      // SpMV workspace allocation
       size_t ws_sz = st.d_ptr->spmv_csr_buffer_size(st.stream_id, st.n, st.n, st.nnz, st.A_rp, st.A_ci, st.A_val, st.x, st.w);
       if (ws_sz > 0) st.spmv_ws = command_runner<out<char>>{}.transfer_memory(st.device_id, st.stream_id, out<char>{static_cast<int>(ws_sz)});
 
-      // Initial r = b - Ax, initial rho = r*r
       st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.x, T{0}, st.w, st.spmv_ws);
       if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.b, st.r);
       else st.d_ptr->scopy(st.stream_id, st.n, st.b, st.r);
@@ -152,15 +143,27 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
 
       st.initial_rho = runner.copy_to_host(st.y_tmp)[0];
       st.current_rho = st.initial_rho;
+      st.initialized = true;
+
+      // Notify supervisor that setup is complete and report initial status
+      solver_result_meta meta(st.device_id, st.stream_id, 0, false, CG_SUCCESS);
+      self->mail(gpu_done_atom_v, std::vector<T>{}, meta).send(st.supervisor);
+    },
+
+    [=](cg_next_step_atom, int num_iters) {
+      auto& st = self->state();
+      command_runner<T> runner;
       T threshold = st.tol * st.tol;
-      T old_rho = 0;
       int code = CG_SUCCESS;
+      int step_count = 0;
 
-      // SIMPLE TIGHT LOOP (Synchronous execution within handler to avoid overhead)
-      while (st.iterations < st.max_iter && st.current_rho > threshold) {
+      // Execute exactly the number of iterations requested by the supervisor
+      while (step_count < num_iters && st.iterations < st.max_iter && st.current_rho > threshold) {
         st.iterations++;
+        step_count++;
+
         if (st.iterations > 1) {
-          T beta = st.current_rho / old_rho;
+          T beta = st.current_rho / st.old_rho;
           if constexpr (std::is_same_v<T, double>) {
             st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.w);
             st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
@@ -174,13 +177,17 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
           if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.p);
           else st.d_ptr->scopy(st.stream_id, st.n, st.r, st.p);
         }
+
         st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.p, T{0}, st.w, st.spmv_ws);
         if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
         else st.d_ptr->sdot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
-        T dot_pw = runner.copy_to_host(st.y_tmp)[0];
-        
-        if (std::abs(dot_pw) < 1e-25) { code = CG_BREAKDOWN; break; }
         
+        T dot_pw = runner.copy_to_host(st.y_tmp)[0];
+        if (std::abs(dot_pw) < 1e-25) { 
+          code = CG_BREAKDOWN; 
+          break; 
+        }
+
         T alpha = st.current_rho / dot_pw;
         if constexpr (std::is_same_v<T, double>) {
           st.d_ptr->daxpy(st.stream_id, st.n, alpha, st.p, st.x);
@@ -189,44 +196,118 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
           st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(alpha), st.p, st.x);
           st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(-alpha), st.w, st.r);
         }
-        old_rho = st.current_rho;
+
+        st.old_rho = st.current_rho;
         if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
         else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
         st.current_rho = runner.copy_to_host(st.y_tmp)[0];
-        
-        if (std::abs(old_rho - st.current_rho) < 1e-18) { code = CG_STAGNATION; break; }
+
+        if (std::abs(st.old_rho - st.current_rho) < 1e-18) { 
+          code = CG_STAGNATION; 
+          break; 
+        }
       }
 
-      // POST-LOOP FAULT CHECKS
+      // Run error checks and prepare progress report
       bool converged = (st.current_rho <= threshold);
       if (code == CG_SUCCESS) {
-        if (!converged) code = CG_MAX_ITER;
-        // Residual decrease check: fail if residual didn't decrease by factor of 10
-        if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.1) code = CG_RESIDUAL_FACTOR_FAIL;
+        if (!converged && st.iterations >= st.max_iter) code = CG_MAX_ITER;
+        // Residual decrease check: fail if residual didn't decrease significantly
+        if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
       }
 
-      // Launch NaN/Inf stability kernel
-      nd_range range((st.n + 255) / 256, 1, 1, 256, 1, 1);
+      // Reset and launch NaN/Inf stability kernel from file
+      nd_range range(static_cast<int>((st.n + 255) / 256), 1, 1, 256, 1, 1);
+      CHECK_CUDA(cuMemsetD32Async(st.d_err->mem(), 0, 1, st.d_ptr->get_stream_for_actor(st.stream_id)));
       st.d_ptr->launch_kernel_mem_ref(st.stab_prog->get_kernel(st.d_ptr->getId()), range,
                                       std::make_tuple(in<int>(st.n), st.x, st.r, st.d_err), st.stream_id);
-      
+
       int err_flag = runner.copy_to_host(st.d_err)[0];
       if (err_flag != 0 || std::isnan(st.current_rho) || std::isinf(st.current_rho)) code = CG_NAN_INF;
 
       solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
-      runner.copy_to_host_async(st.x, [self, meta, supervisor = st.supervisor](std::vector<T> sol) {
-        anon_mail(std::move(sol), meta).send(supervisor);
-        self->quit();
+      
+      // Report current solution and metadata to the supervisor
+      runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor](std::vector<T> sol) {
+        self->mail(gpu_done_atom_v, std::move(sol), meta).send(supervisor);
+        if (converged || code != CG_SUCCESS) self->quit();
       });
     }
   };
 }
 
+// ---------------------------- SUPERVISOR ACTOR ----------------------------
+
+struct supervisor_state {
+    caf::actor solver;
+    int batch_size = 50;
+    int max_steps = 20; 
+    int current_step = 0;
+};
+
+behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::shared_ptr<MatrixData> data) {
+    auto& st = self->state();
+    
+    st.solver = self->spawn(fault_tolerant_cg_actor<float>,
+                            create_in_arg(data->row_ptr),
+                            create_in_arg(data->col_indices),
+                            create_in_arg(data->values),
+                            create_in_arg(data->b),
+                            create_in_out_arg(data->x_guess),
+                            (int)data->row_ptr.size() - 1,
+                            (int)data->values.size(),
+                            1e-5f, 2000, 0, 1, actor_cast<actor>(self));
+
+    self->mail(start_atom_v).send(st.solver);
+
+    return {
+        [=](gpu_done_atom, std::vector<float>& solution, solver_result_meta meta) {
+            auto& s = self->state();
+            
+            // Check if this was just initialization or a real iteration report
+            if (meta.iterations == 0 && meta.error_code == CG_SUCCESS) {
+                self->println("Supervisor: Solver ready. Triggering first batch of {} iterations.", s.batch_size);
+                self->mail(cg_next_step_atom_v, s.batch_size).send(s.solver);
+                return;
+            }
+
+            self->println("Supervisor Report - Iterations: {}, Code: {}, Converged: {}", 
+                          meta.iterations, meta.error_code, meta.converged);
+
+            if (meta.converged) {
+                self->println("Supervisor: Solution reached. Terminating.");
+                self->quit();
+            } else if (meta.error_code != CG_SUCCESS) {
+                self->println("Supervisor: Termination due to error code: {}. Diagnostic required.", meta.error_code);
+                self->quit();
+            } else if (s.current_step >= s.max_steps) {
+                self->println("Supervisor: Reached max steps without convergence.");
+                self->quit();
+            } else {
+                // Solver is healthy but not done; proceed to next step
+                s.current_step++;
+                self->println("Supervisor: Progress OK. Ordering step {}.", s.current_step);
+                self->mail(cg_next_step_atom_v, s.batch_size).send(s.solver);
+            }
+        }
+    };
+}
 
 
 void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
-   
+
+    // Example testing data setup
+    auto data = std::make_shared<MatrixData>();
+    data->row_ptr = {0, 1, 2};
+    data->col_indices = {0, 1};
+    data->values = {10.0f, 10.0f};
+    data->b = {100.0f, 100.0f};
+    data->x_guess = {0.0f, 0.0f};
+
+    sys.spawn(supervisor_actor, data);
+    
+    sys.await_all_actors_done();
     manager::shutdown();
 }
-CAF_MAIN(id_block::cuda,id_block::workload_test)
\ No newline at end of file
+CAF_MAIN(id_block::cuda, id_block::workload_test)
\ No newline at end of file

From f5ff827993624d0cef012c1e57727b8d53a17ed6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 10:24:07 -0600
Subject: [PATCH 0828/1000] added statfuel cgs actor

---
 .../stability_kernels.cu                      | 10 ++++++++
 libcaf_cuda/tests/workload-test/main.test.cpp | 25 ++++++++++++++++---
 2 files changed, 31 insertions(+), 4 deletions(-)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/stability_kernels.cu

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/stability_kernels.cu b/libcaf_cuda/tests/fault-tolerance-workload-test/stability_kernels.cu
new file mode 100644
index 0000000000..56d38d3694
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/stability_kernels.cu
@@ -0,0 +1,10 @@
+extern "C" __global__
+void check_stability(int n, const float* x, const float* r, int* err) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx < n) {
+        // If any value in the solution or residual is invalid, set the error flag
+        if (isnan(x[idx]) || isinf(x[idx]) || isnan(r[idx]) || isinf(r[idx])) {
+            atomicExch(err, 1);
+        }
+    }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index 5d0769f457..1f183a9973 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -43,6 +43,7 @@ CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
     CAF_ADD_TYPE_ID(workload_test, (SolverType))
     CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
     CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
+    CAF_ADD_TYPE_ID(workload_test, (std::vector<caf::actor>))
     CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
 CAF_END_TYPE_ID_BLOCK(workload_test)
 
@@ -152,6 +153,7 @@ behavior sparse_worker_fun(stateful_actor<worker_state>* self,
             return sec::no_context;
         },
         [=](shutdown_atom) {
+            self->mail(worker_done_atom_v).send(self->state().supervisor);
             self->quit();
         },
         [=](std::vector<float>& solution, solver_result_meta meta) {
@@ -183,13 +185,16 @@ struct supervisor_state {
     int batches_remaining;
     int batch_size;
     double mean_arrival_ms;
+    caf::actor parent;
+    int workers_shutdown = 0;
 };
 
 behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self, 
                               std::vector<MatrixTask> matrix_pool,
-                              int num_streams, int num_batches, int batch_size, double mean_arrival_ms) {
+                              int num_streams, int num_batches, int batch_size, double mean_arrival_ms, caf::actor parent) {
     auto& st = self->state();
     st.matrix_pool = std::move(matrix_pool);
+    st.parent = std::move(parent);
     st.total_tasks = num_batches * batch_size;
     st.batches_remaining = num_batches;
     st.batch_size = batch_size;
@@ -244,9 +249,15 @@ behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self,
             auto& st = self->state();
             st.completed += done;
             if (st.completed >= st.total_tasks) {
-                self->println("\n[DONE] All {} tasks processed.", st.total_tasks);
+                self->println("\n[DONE] All {} tasks processed. Terminating workers...", st.total_tasks);
                 for (auto& worker : st.workers)
                     self->mail(shutdown_atom_v).send(worker);
+            }
+        },
+        [=](worker_done_atom) {
+            auto& st = self->state();
+            if (++st.workers_shutdown == (int)st.workers.size()) {
+                self->mail(worker_done_atom_v).send(st.parent);
                 self->quit();
             }
         }
@@ -285,9 +296,15 @@ void caf_main(actor_system& sys) {
     std::cout << "[INFO] Mean arrival:     " << mean_arrival_ms << " ms\n";
 
     auto start = std::chrono::steady_clock::now();
+    scoped_actor self{sys};
 
-    sys.spawn(supervisor_actor_fun, std::move(tasks_vec), num_streams, num_batches, batch_size, mean_arrival_ms);
-    sys.await_all_actors_done();
+    self->spawn(supervisor_actor_fun, std::move(tasks_vec), num_streams, num_batches, batch_size, mean_arrival_ms, actor_cast<actor>(self));
+    
+    self->receive(
+        [&](worker_done_atom) {
+            std::cout << "[INFO] Supervisor signaled completion. Shutting down...\n";
+        }
+    );
 
     auto end = std::chrono::steady_clock::now();
     std::chrono::duration<double> elapsed = end - start;

From db8b30a83281eceb11e4d886abaf733d6bdcc344 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 10:27:16 -0600
Subject: [PATCH 0829/1000] moved sparse utils

---
 .../sparse_utils.cpp                          | 164 ++++++++++++++++++
 .../sparse_utils.hpp                          |  78 +++++++++
 2 files changed, 242 insertions(+)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.hpp

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp
new file mode 100644
index 0000000000..b16c9dc907
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp
@@ -0,0 +1,164 @@
+#include "sparse_utils.hpp"
+#include <fstream>
+#include <stdexcept>
+#include <filesystem>
+#include <random>
+namespace fs = std::filesystem;
+
+SparseMatrixCOO load_binary_coo(const std::string& filepath) {
+    std::ifstream file(filepath, std::ios::binary);
+    if (!file) {
+        throw std::runtime_error("Failed to open matrix file: " + filepath);
+    }
+
+    SparseMatrixCOO coo;
+    
+    file.read(reinterpret_cast<char*>(&coo.rows), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&coo.cols), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&coo.nnz), sizeof(int32_t));
+
+    coo.row_indices.resize(coo.nnz);
+    coo.col_indices.resize(coo.nnz);
+    coo.values.resize(coo.nnz);
+
+    file.read(reinterpret_cast<char*>(coo.row_indices.data()), coo.nnz * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(coo.col_indices.data()), coo.nnz * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(coo.values.data()), coo.nnz * sizeof(float));
+
+    return coo;
+}
+
+SparseMatrixCSR convert_coo_to_csr(const SparseMatrixCOO& coo) {
+    SparseMatrixCSR csr;
+    csr.rows = coo.rows;
+    csr.cols = coo.cols;
+    csr.nnz = coo.nnz;
+    
+    csr.row_ptr.assign(csr.rows + 1, 0);
+    csr.col_indices.resize(csr.nnz);
+    csr.values.resize(csr.nnz);
+
+    for (int32_t i = 0; i < coo.nnz; ++i) {
+        csr.row_ptr[coo.row_indices[i] + 1]++;
+    }
+
+    for (int32_t i = 0; i < csr.rows; ++i) {
+        csr.row_ptr[i + 1] += csr.row_ptr[i];
+    }
+
+    std::vector<int32_t> current_row_pos = csr.row_ptr;
+
+    for (int32_t i = 0; i < coo.nnz; ++i) {
+        int32_t row = coo.row_indices[i];
+        int32_t dest_pos = current_row_pos[row]++;
+        csr.col_indices[dest_pos] = coo.col_indices[i];
+        csr.values[dest_pos] = coo.values[i];
+    }
+
+    return csr;
+}
+
+std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x) {
+    std::vector<float> b(A.rows, 0.0f);
+    
+    for (int32_t i = 0; i < A.rows; ++i) {
+        float sum = 0.0f;
+        int32_t row_start = A.row_ptr[i];
+        int32_t row_end = A.row_ptr[i + 1];
+        
+        for (int32_t j = row_start; j < row_end; ++j) {
+            sum += A.values[j] * x[A.col_indices[j]];
+        }
+        b[i] = sum;
+    }
+    return b;
+}
+
+std::vector<MatrixTask> scan_for_matrices(const std::string& dir, SolverType type) {
+    std::vector<MatrixTask> tasks;
+    if (!fs::exists(dir)) return tasks;
+    for (const auto& entry : fs::directory_iterator(dir)) {
+        if (entry.path().extension() == ".bin") {
+            auto coo = load_binary_coo(entry.path().string());
+            auto csr = convert_coo_to_csr(coo);
+            auto data = std::make_shared<MatrixData>();
+            data->rows = csr.rows;
+            data->cols = csr.cols;
+            data->nnz = csr.nnz;
+            data->b = compute_rhs_spmv(csr, std::vector<float>(csr.cols, 1.0f));
+            data->row_ptr = std::move(csr.row_ptr);
+            data->col_indices = std::move(csr.col_indices);
+            data->values = std::move(csr.values);
+            data->x_guess.assign(data->cols, 0.0f);
+            
+            tasks.push_back({entry.path().string(), type, data});
+        }
+    }
+    return tasks;
+}
+
+
+std::vector<Partition>
+make_contiguous_partitions(size_t num_tasks, size_t rows, size_t cols)
+{
+    size_t num_parts = rows * cols;
+
+    std::vector<Partition> parts;
+    parts.reserve(num_parts);
+
+    size_t base = num_tasks / num_parts;
+    size_t rem  = num_tasks % num_parts;
+
+    size_t current = 0;
+
+    for (size_t p = 0; p < num_parts; ++p) {
+        size_t size = base + (p < rem ? 1 : 0);
+
+        parts.push_back({
+            current,
+            current + size
+        });
+
+        current += size;
+    }
+
+    return parts;
+}
+
+
+int generate_random_sleep_ms(int min_ms, int max_ms) {
+    static std::random_device rd;
+    static std::mt19937 gen(rd());
+    std::uniform_int_distribution<> dis(min_ms, max_ms);
+    return dis(gen);
+}
+
+std::chrono::milliseconds generate_random_interval(
+    std::mt19937& rng,
+    double mean_ms)
+{
+    std::exponential_distribution<double> dist(
+        1.0 / mean_ms);
+
+    return std::chrono::milliseconds(
+        static_cast<int>(dist(rng)));
+}
+
+std::vector<MatrixTask> generate_batch(
+    const std::vector<MatrixTask>& matrix_pool,
+    std::mt19937& rng,
+    size_t batch_size)
+{
+    std::vector<MatrixTask> batch;
+    batch.reserve(batch_size);
+
+    std::uniform_int_distribution<size_t> dist(
+        0,
+        matrix_pool.size() - 1);
+
+    for (size_t i = 0; i < batch_size; ++i) {
+        batch.push_back(matrix_pool[dist(rng)]);
+    }
+
+    return batch;
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.hpp
new file mode 100644
index 0000000000..f886e68c6c
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.hpp
@@ -0,0 +1,78 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <cstdint>
+#include <memory>
+#include <random>
+#include <chrono>
+
+enum SolverType { CGS_SOLVER, BICSTAB_SOLVER };
+
+// Structure to hold raw data from the binary file
+struct SparseMatrixCOO {
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+    std::vector<int32_t> row_indices;
+    std::vector<int32_t> col_indices;
+    std::vector<float> values;
+};
+
+// Structure optimized for high-performance solvers
+struct SparseMatrixCSR {
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+    std::vector<int32_t> row_ptr;    // Size: rows + 1
+    std::vector<int32_t> col_indices;// Size: nnz
+    std::vector<float> values;       // Size: nnz
+};
+
+struct MatrixData {
+    std::vector<int32_t> row_ptr;
+    std::vector<int32_t> col_indices;
+    std::vector<float> values;
+    std::vector<float> b;
+    std::vector<float> x_guess;
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+};
+
+struct MatrixTask {
+    std::string path;
+    SolverType type;
+    std::shared_ptr<MatrixData> data;
+};
+
+
+
+struct Partition {
+    size_t begin;
+    size_t end;
+    std::vector<int> devices;
+    std::vector<int> streams;
+};
+
+// Function to slurp the binary data into memory
+SparseMatrixCOO load_binary_coo(const std::string& filepath);
+
+// Converts COO to CSR format for solver compatibility
+SparseMatrixCSR convert_coo_to_csr(const SparseMatrixCOO& coo);
+
+// Compute b = A * x using CSR layout (Sparse Matrix-Vector Multiplication)
+std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x);
+
+std::vector<MatrixTask> scan_for_matrices(const std::string& dir, SolverType type);
+int generate_random_sleep_ms(int min_ms, int max_ms);
+
+// Workload generation helpers
+std::chrono::milliseconds generate_random_interval(
+    std::mt19937& rng,
+    double mean_ms);
+
+std::vector<MatrixTask> generate_batch(
+    const std::vector<MatrixTask>& matrix_pool,
+    std::mt19937& rng,
+    size_t batch_size);
\ No newline at end of file

From 0cd4e94bfd3d5ca71f86f2a1b15b84241e12fb1e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 10:28:58 -0600
Subject: [PATCH 0830/1000] SAVIONG>

---
 libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
index 356dd87f4f..df92bcd0e8 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
@@ -30,7 +30,7 @@ include_directories(
 
 
 # 5) Declare your executable
-add_executable(test main.test.cpp)
+add_executable(test main.test.cpp sparse_utils.cpp)
 
 target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
 

From e4550c2a697442da7140d3c16f8bc2bcd263bf79 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 10:56:58 -0600
Subject: [PATCH 0831/1000] got code working (somewhat)

---
 .../main.test.cpp                             | 127 ++++++++++--------
 1 file changed, 69 insertions(+), 58 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index d2d9a3cc66..c80f268533 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -20,16 +20,7 @@ namespace fs = std::filesystem;
 
 constexpr uint32_t WORKLOAD_SEED = 42;
 
-template <class Inspector>
-bool inspect(Inspector& f, SolverType& x) {
-    auto val = static_cast<int>(x);
-    if (f.apply(val)) {
-        if constexpr (Inspector::is_loading)
-            x = static_cast<SolverType>(val);
-        return true;
-    }
-    return false;
-}
+
 
 CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
     CAF_ADD_ATOM(workload_test, get_work_atom)
@@ -42,15 +33,19 @@ CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
     CAF_ADD_ATOM(workload_test, shutdown_atom)
     CAF_ADD_TYPE_ID(workload_test, (SolverType))
     CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
+    CAF_ADD_TYPE_ID(workload_test, (MatrixData))
     CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
-    CAF_ADD_TYPE_ID(workload_test, (std::vector<caf::actor>))
     CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
 CAF_END_TYPE_ID_BLOCK(workload_test)
 
+
+
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixData)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(SolverType)
+
 
 /**
  * Error codes for solver_result_meta.error_code
@@ -239,74 +234,90 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
 // ---------------------------- SUPERVISOR ACTOR ----------------------------
 
 struct supervisor_state {
-    caf::actor solver;
+    std::deque<MatrixTask> queue;
+    std::vector<caf::actor> active_solvers;
+    int max_active = 4; // Admission control limit to be mindful of GPU memory
     int batch_size = 50;
-    int max_steps = 20; 
-    int current_step = 0;
 };
 
-behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::shared_ptr<MatrixData> data) {
+behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks) {
     auto& st = self->state();
-    
-    st.solver = self->spawn(fault_tolerant_cg_actor<float>,
-                            create_in_arg(data->row_ptr),
-                            create_in_arg(data->col_indices),
-                            create_in_arg(data->values),
-                            create_in_arg(data->b),
-                            create_in_out_arg(data->x_guess),
-                            (int)data->row_ptr.size() - 1,
-                            (int)data->values.size(),
-                            1e-5f, 2000, 0, 1, actor_cast<actor>(self));
-
-    self->mail(start_atom_v).send(st.solver);
+    for (auto& t : tasks)
+        st.queue.push_back(std::move(t));
+
+    auto spawn_next = [self]() {
+        auto& s = self->state();
+        while (s.active_solvers.size() < static_cast<size_t>(s.max_active) && !s.queue.empty()) {
+            auto task = std::move(s.queue.front());
+            s.queue.pop_front();
+
+            auto solver = self->spawn(fault_tolerant_cg_actor<float>,
+                                    create_in_arg(task.data->row_ptr),
+                                    create_in_arg(task.data->col_indices),
+                                    create_in_arg(task.data->values),
+                                    create_in_arg(task.data->b),
+                                    create_in_out_arg(task.data->x_guess),
+                                    (int)task.data->row_ptr.size() - 1,
+                                    (int)task.data->values.size(),
+                                    1e-5f, 2000, 0, 1, actor_cast<actor>(self));
+            
+            s.active_solvers.push_back(solver);
+            self->mail(start_atom_v).send(solver);
+        }
+    };
 
+    spawn_next();
+    
     return {
         [=](gpu_done_atom, std::vector<float>& solution, solver_result_meta meta) {
             auto& s = self->state();
+            auto solver = actor_cast<caf::actor>(self->current_sender());
             
-            // Check if this was just initialization or a real iteration report
             if (meta.iterations == 0 && meta.error_code == CG_SUCCESS) {
-                self->println("Supervisor: Solver ready. Triggering first batch of {} iterations.", s.batch_size);
-                self->mail(cg_next_step_atom_v, s.batch_size).send(s.solver);
+                // Initialization report: trigger the first batch
+                self->mail(cg_next_step_atom_v, s.batch_size).send(solver);
                 return;
             }
 
-            self->println("Supervisor Report - Iterations: {}, Code: {}, Converged: {}", 
-                          meta.iterations, meta.error_code, meta.converged);
-
-            if (meta.converged) {
-                self->println("Supervisor: Solution reached. Terminating.");
-                self->quit();
-            } else if (meta.error_code != CG_SUCCESS) {
-                self->println("Supervisor: Termination due to error code: {}. Diagnostic required.", meta.error_code);
-                self->quit();
-            } else if (s.current_step >= s.max_steps) {
-                self->println("Supervisor: Reached max steps without convergence.");
-                self->quit();
+            if (meta.converged || meta.error_code != CG_SUCCESS) {
+                if (meta.converged) {
+                    self->println("Task completed successfully after {} iterations.", meta.iterations);
+                } else {
+                    self->println("Task failed with error code {} after {} iterations.", 
+                                  meta.error_code, meta.iterations);
+                }
+
+                auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
+                if (it != s.active_solvers.end()) s.active_solvers.erase(it);
+
+                spawn_next();
+
+                if (s.active_solvers.empty() && s.queue.empty()) {
+                    self->println("All tasks in the batch have been processed.");
+                    self->quit();
+                }
             } else {
-                // Solver is healthy but not done; proceed to next step
-                s.current_step++;
-                self->println("Supervisor: Progress OK. Ordering step {}.", s.current_step);
-                self->mail(cg_next_step_atom_v, s.batch_size).send(s.solver);
+                // Progress report: order the next iteration batch
+                self->mail(cg_next_step_atom_v, s.batch_size).send(solver);
             }
         }
     };
 }
 
-
 void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
-
-    // Example testing data setup
-    auto data = std::make_shared<MatrixData>();
-    data->row_ptr = {0, 1, 2};
-    data->col_indices = {0, 1};
-    data->values = {10.0f, 10.0f};
-    data->b = {100.0f, 100.0f};
-    data->x_guess = {0.0f, 0.0f};
-
-    sys.spawn(supervisor_actor, data);
-    
+     auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
+     if (tasks_vec.empty()) {
+         std::cerr << "No matrices found. Running dummy test task." << std::endl;
+         auto data = std::make_shared<MatrixData>();
+         data->row_ptr = {0, 1, 2};
+         data->col_indices = {0, 1};
+         data->values = {10.0f, 10.0f};
+         data->b = {100.0f, 100.0f};
+         data->x_guess = {0.0f, 0.0f};
+         tasks_vec.push_back({"dummy_task", CGS_SOLVER, data});
+     }
+    sys.spawn(supervisor_actor, std::move(tasks_vec));
     sys.await_all_actors_done();
     manager::shutdown();
 }

From 6c0cef4d4a93292d58ad7d51b9c08fcb87caa7b2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 11:00:00 -0600
Subject: [PATCH 0832/1000] added kernel compilation

---
 .../CMakeLists.txt                             | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
index df92bcd0e8..a29bbc6af2 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
@@ -28,6 +28,24 @@ include_directories(
   "${CAF_SRC}/libcaf_cuda"
 )
 
+# --- CUDA Kernel Compilation ---
+set(STABILITY_KERNEL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/stability_kernels.cu")
+set(STABILITY_KERNEL_CUBIN "${CMAKE_CURRENT_BINARY_DIR}/stability_kernels.cubin")
+
+add_custom_command(
+  OUTPUT ${STABILITY_KERNEL_CUBIN}
+  COMMAND ${CUDAToolkit_NVCC_EXECUTABLE}
+          -cubin
+          -arch=sm_80
+          -o ${STABILITY_KERNEL_CUBIN}
+          ${STABILITY_KERNEL_SRC}
+  DEPENDS ${STABILITY_KERNEL_SRC}
+  COMMENT "Compiling CUDA kernel ${STABILITY_KERNEL_SRC} to ${STABILITY_KERNEL_CUBIN}"
+  VERBATIM
+)
+
+add_custom_target(stability_kernels_cubin ALL DEPENDS ${STABILITY_KERNEL_CUBIN})
+
 
 # 5) Declare your executable
 add_executable(test main.test.cpp sparse_utils.cpp)

From b52fe84e790ac8eb8e0e0768d05c007092f8b0e3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 11:02:17 -0600
Subject: [PATCH 0833/1000] fixed issues

---
 .../tests/fault-tolerance-workload-test/CMakeLists.txt    | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
index a29bbc6af2..886f889064 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
@@ -32,15 +32,19 @@ include_directories(
 set(STABILITY_KERNEL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/stability_kernels.cu")
 set(STABILITY_KERNEL_CUBIN "${CMAKE_CURRENT_BINARY_DIR}/stability_kernels.cubin")
 
+# Target CUDA architecture. 'native' targets the current machine's GPU (requires CUDA 11.6+).
+# You can override this with -DCUDA_ARCH=sm_XX if needed.
+set(CUDA_ARCH "native" CACHE STRING "Target CUDA architecture (e.g., native, sm_70, sm_75, sm_80, sm_86)")
+
 add_custom_command(
   OUTPUT ${STABILITY_KERNEL_CUBIN}
   COMMAND ${CUDAToolkit_NVCC_EXECUTABLE}
           -cubin
-          -arch=sm_80
+          -arch=${CUDA_ARCH}
           -o ${STABILITY_KERNEL_CUBIN}
           ${STABILITY_KERNEL_SRC}
   DEPENDS ${STABILITY_KERNEL_SRC}
-  COMMENT "Compiling CUDA kernel ${STABILITY_KERNEL_SRC} to ${STABILITY_KERNEL_CUBIN}"
+  COMMENT "Compiling CUDA kernel ${STABILITY_KERNEL_SRC} for architecture: ${CUDA_ARCH}"
   VERBATIM
 )
 

From ddc87e319f07a039a8eac3bf1d748fb134861487 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 11:24:19 -0600
Subject: [PATCH 0834/1000] saving

---
 .../main.test.cpp                             | 33 +++++++++++--------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index c80f268533..82ff4b82b1 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -88,14 +88,17 @@ struct ft_cg_state {
   T current_rho = 0;
   T old_rho = 0;
   bool initialized = false;
+  std::shared_ptr<MatrixData> pinned_data;
 };
 
 template <class T = float>
 behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
+                                 std::shared_ptr<MatrixData> data,
                                  in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
                                  int n, int nnz, T tol, int max_iter,
                                  int dev_num, int stream, caf::actor supervisor) {
   auto& s = self->state();
+  s.pinned_data = std::move(data);
   s.h_row_ptr = std::move(rp); s.h_col_ind = std::move(ci);
   s.h_values = std::move(val); s.h_b = std::move(b_in); s.h_x = std::move(x_in);
   s.n = n; s.nnz = nnz; s.tol = tol; s.max_iter = max_iter;
@@ -106,10 +109,12 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
       auto& st = self->state();
       if (st.initialized) return;
 
-      command_runner<T> runner;
-      auto res = runner.transfer_memory(st.device_id, st.stream_id, st.h_row_ptr, st.h_col_ind, st.h_values, st.h_b, st.h_x);
-      st.A_rp = std::get<0>(res); st.A_ci = std::get<1>(res); st.A_val = std::get<2>(res);
-      st.b = std::get<3>(res); st.x = std::get<4>(res);
+      command_runner<> runner;
+      st.A_rp = runner.transfer_memory(st.device_id, st.stream_id, st.h_row_ptr);
+      st.A_ci = runner.transfer_memory(st.device_id, st.stream_id, st.h_col_ind);
+      st.A_val = runner.transfer_memory(st.device_id, st.stream_id, st.h_values);
+      st.b = runner.transfer_memory(st.device_id, st.stream_id, st.h_b);
+      st.x = runner.transfer_memory(st.device_id, st.stream_id, st.h_x);
 
       st.d_ptr = platform::create()->schedule(st.stream_id, st.device_id);
       st.d_ptr->enable_cublas(); st.d_ptr->enable_cusparse();
@@ -118,13 +123,12 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
       // Load stability kernel from file as requested
       st.stab_prog = mgr.create_program_from_cubin("stability_kernels.cubin", "check_stability", st.d_ptr);
 
-      command_runner<out<T>> work_runner;
-      st.r = work_runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
-      st.p = work_runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
-      st.w = work_runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
-      st.y_tmp = work_runner.transfer_memory(st.device_id, st.stream_id, out<T>(1));
-      st.d_err = command_runner<in_out<int>>{}.transfer_memory(st.device_id, st.stream_id, in_out<int>(0));
-
+      st.r = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.p = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.w = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.y_tmp = runner.transfer_memory(st.device_id, st.stream_id, out<T>(1));
+      st.d_err = runner.transfer_memory(st.device_id, st.stream_id, out<int>(1));
+      
       size_t ws_sz = st.d_ptr->spmv_csr_buffer_size(st.stream_id, st.n, st.n, st.nnz, st.A_rp, st.A_ci, st.A_val, st.x, st.w);
       if (ws_sz > 0) st.spmv_ws = command_runner<out<char>>{}.transfer_memory(st.device_id, st.stream_id, out<char>{static_cast<int>(ws_sz)});
 
@@ -236,7 +240,7 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
 struct supervisor_state {
     std::deque<MatrixTask> queue;
     std::vector<caf::actor> active_solvers;
-    int max_active = 4; // Admission control limit to be mindful of GPU memory
+    int max_active = 1; // Admission control limit to be mindful of GPU memory
     int batch_size = 50;
 };
 
@@ -252,6 +256,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
             s.queue.pop_front();
 
             auto solver = self->spawn(fault_tolerant_cg_actor<float>,
+                                    task.data,
                                     create_in_arg(task.data->row_ptr),
                                     create_in_arg(task.data->col_indices),
                                     create_in_arg(task.data->values),
@@ -306,7 +311,9 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
 
 void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
+    std::cout << "loading\n";
      auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
+     std::cout << "loaded\n";
      if (tasks_vec.empty()) {
          std::cerr << "No matrices found. Running dummy test task." << std::endl;
          auto data = std::make_shared<MatrixData>();
@@ -317,7 +324,7 @@ void caf_main(actor_system& sys) {
          data->x_guess = {0.0f, 0.0f};
          tasks_vec.push_back({"dummy_task", CGS_SOLVER, data});
      }
-    sys.spawn(supervisor_actor, std::move(tasks_vec));
+    sys.spawn(supervisor_actor, tasks_vec);
     sys.await_all_actors_done();
     manager::shutdown();
 }

From 57a6af6fcf1273cb988a1d296f992c377f2d6db2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 11:52:21 -0600
Subject: [PATCH 0835/1000] fixed illegal memory access issues

---
 .../fault-tolerance-workload-test/main.test.cpp  | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 82ff4b82b1..6ba9b889ba 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -158,6 +158,7 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
 
       // Execute exactly the number of iterations requested by the supervisor
       while (step_count < num_iters && st.iterations < st.max_iter && st.current_rho > threshold) {
+        // std::cout << "iterations = " << st.iterations << ", current_rho = " << st.current_rho << std::endl;
         st.iterations++;
         step_count++;
 
@@ -240,8 +241,10 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
 struct supervisor_state {
     std::deque<MatrixTask> queue;
     std::vector<caf::actor> active_solvers;
-    int max_active = 1; // Admission control limit to be mindful of GPU memory
+    int max_active = 2; // Admission control limit to be mindful of GPU memory
     int batch_size = 50;
+    int stream = 0;
+    int device = 0;
 };
 
 behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks) {
@@ -254,7 +257,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
         while (s.active_solvers.size() < static_cast<size_t>(s.max_active) && !s.queue.empty()) {
             auto task = std::move(s.queue.front());
             s.queue.pop_front();
-
+            std::cout << "[SUPE] making new solver \n";
             auto solver = self->spawn(fault_tolerant_cg_actor<float>,
                                     task.data,
                                     create_in_arg(task.data->row_ptr),
@@ -264,7 +267,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                                     create_in_out_arg(task.data->x_guess),
                                     (int)task.data->row_ptr.size() - 1,
                                     (int)task.data->values.size(),
-                                    1e-5f, 2000, 0, 1, actor_cast<actor>(self));
+                                    1e-5f, 8000, s.device, (++s.stream)%32, actor_cast<actor>(self));
             
             s.active_solvers.push_back(solver);
             self->mail(start_atom_v).send(solver);
@@ -312,7 +315,12 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
 void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
     std::cout << "loading\n";
-     auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
+     //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
+     
+    auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+
+    
+     
      std::cout << "loaded\n";
      if (tasks_vec.empty()) {
          std::cerr << "No matrices found. Running dummy test task." << std::endl;

From f7e64a47beae0ab289686c725bc029b6c65ed68b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 12:02:01 -0600
Subject: [PATCH 0836/1000] added better logging

---
 .../main.test.cpp                             | 78 ++++++++++++-------
 1 file changed, 51 insertions(+), 27 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 6ba9b889ba..87e8c6575b 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -59,6 +59,18 @@ enum cg_error_type : int {
   CG_RESIDUAL_FACTOR_FAIL = 5
 };
 
+std::string to_string(cg_error_type err) {
+  switch (err) {
+    case CG_SUCCESS: return "Success";
+    case CG_MAX_ITER: return "Maximum Iterations Reached";
+    case CG_NAN_INF: return "Stability Check Failed (NaN/Inf Detected)";
+    case CG_STAGNATION: return "Stagnation Detected (Residual stopped changing)";
+    case CG_BREAKDOWN: return "Solver Breakdown (Division by zero/near-zero)";
+    case CG_RESIDUAL_FACTOR_FAIL: return "Residual Factor Check Failed";
+    default: return "Unknown Error (" + std::to_string(static_cast<int>(err)) + ")";
+  }
+}
+
 // ---------------------------- FAULT TOLERANT SOLVER ----------------------------
 
 template <class T = float>
@@ -202,7 +214,7 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
         else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
         st.current_rho = runner.copy_to_host(st.y_tmp)[0];
 
-        if (std::abs(st.old_rho - st.current_rho) < 1e-18) { 
+        if (std::abs(st.old_rho - st.current_rho) < 1e-15) { 
           code = CG_STAGNATION; 
           break; 
         }
@@ -241,7 +253,8 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
 struct supervisor_state {
     std::deque<MatrixTask> queue;
     std::vector<caf::actor> active_solvers;
-    int max_active = 2; // Admission control limit to be mindful of GPU memory
+    std::unordered_map<caf::actor_id, std::string> task_names;
+    int max_active = 4; // Admission control limit to be mindful of GPU memory
     int batch_size = 50;
     int stream = 0;
     int device = 0;
@@ -257,7 +270,8 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
         while (s.active_solvers.size() < static_cast<size_t>(s.max_active) && !s.queue.empty()) {
             auto task = std::move(s.queue.front());
             s.queue.pop_front();
-            std::cout << "[SUPE] making new solver \n";
+            std::string path = task.path;
+            self->println("[SUPE] Starting solver for: {}", path);
             auto solver = self->spawn(fault_tolerant_cg_actor<float>,
                                     task.data,
                                     create_in_arg(task.data->row_ptr),
@@ -267,7 +281,8 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                                     create_in_out_arg(task.data->x_guess),
                                     (int)task.data->row_ptr.size() - 1,
                                     (int)task.data->values.size(),
-                                    1e-5f, 8000, s.device, (++s.stream)%32, actor_cast<actor>(self));
+                                    1e-5f, 32000, s.device, (++s.stream)%32, actor_cast<actor>(self));
+            s.task_names[solver->id()] = std::move(path);
             
             s.active_solvers.push_back(solver);
             self->mail(start_atom_v).send(solver);
@@ -287,16 +302,27 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 return;
             }
 
+            std::string task_name = s.task_names.count(solver->id()) 
+                                    ? s.task_names[solver->id()] 
+                                    : "Unknown Task";
+
             if (meta.converged || meta.error_code != CG_SUCCESS) {
                 if (meta.converged) {
-                    self->println("Task completed successfully after {} iterations.", meta.iterations);
+                    if (meta.iterations == 0)
+                        self->println("[DONE] {}: Initial guess satisfied tolerance.", task_name);
+                    else
+                        self->println("[DONE] {}: Converged in {} iterations.", task_name, meta.iterations);
                 } else {
-                    self->println("Task failed with error code {} after {} iterations.", 
-                                  meta.error_code, meta.iterations);
+                    self->println("[FAIL] {}: {} (after {} iterations).", 
+                                  task_name, 
+                                  to_string(static_cast<cg_error_type>(meta.error_code)), 
+                                  meta.iterations);
                 }
 
                 auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
-                if (it != s.active_solvers.end()) s.active_solvers.erase(it);
+                if (it != s.active_solvers.end())
+                    s.active_solvers.erase(it);
+                s.task_names.erase(solver->id());
 
                 spawn_next();
 
@@ -315,25 +341,23 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
 void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
     std::cout << "loading\n";
-     //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
-     
-    auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
-
-    
-     
-     std::cout << "loaded\n";
-     if (tasks_vec.empty()) {
-         std::cerr << "No matrices found. Running dummy test task." << std::endl;
-         auto data = std::make_shared<MatrixData>();
-         data->row_ptr = {0, 1, 2};
-         data->col_indices = {0, 1};
-         data->values = {10.0f, 10.0f};
-         data->b = {100.0f, 100.0f};
-         data->x_guess = {0.0f, 0.0f};
-         tasks_vec.push_back({"dummy_task", CGS_SOLVER, data});
-     }
-    sys.spawn(supervisor_actor, tasks_vec);
-    sys.await_all_actors_done();
+    {
+        auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+
+        std::cout << "loaded\n";
+        if (tasks_vec.empty()) {
+            std::cerr << "No matrices found. Running dummy test task." << std::endl;
+            auto data = std::make_shared<MatrixData>();
+            data->row_ptr = {0, 1, 2};
+            data->col_indices = {0, 1};
+            data->values = {10.0f, 10.0f};
+            data->b = {100.0f, 100.0f};
+            data->x_guess = {0.0f, 0.0f};
+            tasks_vec.push_back({"dummy_task", CGS_SOLVER, data});
+        }
+        sys.spawn(supervisor_actor, std::move(tasks_vec));
+        sys.await_all_actors_done();
+    }
     manager::shutdown();
 }
 CAF_MAIN(id_block::cuda, id_block::workload_test)
\ No newline at end of file

From 6187bb92a601a9d0921567993534f30643b9a444 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 12:11:42 -0600
Subject: [PATCH 0837/1000] uped the iteration count

---
 libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 87e8c6575b..172f4fb098 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -214,7 +214,7 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
         else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
         st.current_rho = runner.copy_to_host(st.y_tmp)[0];
 
-        if (std::abs(st.old_rho - st.current_rho) < 1e-15) { 
+        if (std::abs(st.old_rho - st.current_rho) < 1e-12) { 
           code = CG_STAGNATION; 
           break; 
         }
@@ -281,7 +281,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                                     create_in_out_arg(task.data->x_guess),
                                     (int)task.data->row_ptr.size() - 1,
                                     (int)task.data->values.size(),
-                                    1e-5f, 32000, s.device, (++s.stream)%32, actor_cast<actor>(self));
+                                    1e-4f, 64000, s.device, (++s.stream)%32, actor_cast<actor>(self));
             s.task_names[solver->id()] = std::move(path);
             
             s.active_solvers.push_back(solver);

From a15ca04b7954da0ad5db5d062d44989ad13e8eeb Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 12:48:58 -0600
Subject: [PATCH 0838/1000] added timer values

---
 .../main.test.cpp                             | 30 ++++++++++++-------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 172f4fb098..e2873f40f5 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -254,8 +254,9 @@ struct supervisor_state {
     std::deque<MatrixTask> queue;
     std::vector<caf::actor> active_solvers;
     std::unordered_map<caf::actor_id, std::string> task_names;
-    int max_active = 4; // Admission control limit to be mindful of GPU memory
-    int batch_size = 50;
+    std::unordered_map<caf::actor_id, std::chrono::steady_clock::time_point> start_times;
+    int max_active = 1; // Admission control limit to be mindful of GPU memory. If Illegal memory access that means two or more actors are using the same stream
+    int num_iterations = 50;
     int stream = 0;
     int device = 0;
 };
@@ -281,9 +282,10 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                                     create_in_out_arg(task.data->x_guess),
                                     (int)task.data->row_ptr.size() - 1,
                                     (int)task.data->values.size(),
-                                    1e-4f, 64000, s.device, (++s.stream)%32, actor_cast<actor>(self));
+                                    1e-4f, 128000, s.device, (++s.stream)%32, actor_cast<actor>(self));
             s.task_names[solver->id()] = std::move(path);
             
+            s.start_times[solver->id()] = std::chrono::steady_clock::now();
             s.active_solvers.push_back(solver);
             self->mail(start_atom_v).send(solver);
         }
@@ -298,7 +300,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
             
             if (meta.iterations == 0 && meta.error_code == CG_SUCCESS) {
                 // Initialization report: trigger the first batch
-                self->mail(cg_next_step_atom_v, s.batch_size).send(solver);
+                self->mail(cg_next_step_atom_v, s.num_iterations).send(solver);
                 return;
             }
 
@@ -307,22 +309,28 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                                     : "Unknown Task";
 
             if (meta.converged || meta.error_code != CG_SUCCESS) {
+                auto end_time = std::chrono::steady_clock::now();
+                auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
+                                  end_time - s.start_times[solver->id()]).count();
+
                 if (meta.converged) {
                     if (meta.iterations == 0)
-                        self->println("[DONE] {}: Initial guess satisfied tolerance.", task_name);
+                        self->println("[DONE] {}: Initial guess satisfied tolerance ({} ms).", task_name, duration);
                     else
-                        self->println("[DONE] {}: Converged in {} iterations.", task_name, meta.iterations);
+                        self->println("[DONE] {}: Converged in {} iterations ({} ms).", task_name, meta.iterations, duration);
                 } else {
-                    self->println("[FAIL] {}: {} (after {} iterations).", 
+                    self->println("[FAIL] {}: {} (after {} iterations, {} ms).", 
                                   task_name, 
                                   to_string(static_cast<cg_error_type>(meta.error_code)), 
-                                  meta.iterations);
+                                  meta.iterations,
+                                  duration);
                 }
 
                 auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
                 if (it != s.active_solvers.end())
                     s.active_solvers.erase(it);
                 s.task_names.erase(solver->id());
+                s.start_times.erase(solver->id());
 
                 spawn_next();
 
@@ -332,7 +340,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 }
             } else {
                 // Progress report: order the next iteration batch
-                self->mail(cg_next_step_atom_v, s.batch_size).send(solver);
+                self->mail(cg_next_step_atom_v, s.num_iterations).send(solver);
             }
         }
     };
@@ -342,7 +350,9 @@ void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
     std::cout << "loading\n";
     {
-        auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+        //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/unsymmetric", CGS_SOLVER);
+         auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
 
         std::cout << "loaded\n";
         if (tasks_vec.empty()) {

From 454f012bb902556465d7806159227a59ff12c600 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 13:51:57 -0600
Subject: [PATCH 0839/1000] updated solvers to match

---
 .../CMakeLists.txt                            |  22 +-
 .../main.native.cpp                           | 378 ++++++++++++++++++
 .../main.test.cpp                             |   7 +-
 3 files changed, 393 insertions(+), 14 deletions(-)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
index 886f889064..b980b1431a 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
@@ -83,16 +83,16 @@ target_link_libraries(test
 # )
 
 
-# # FindThreads is required for std::thread in the native version
-# find_package(Threads REQUIRED)
+# FindThreads is required for std::thread in the native version
+find_package(Threads REQUIRED)
 
-# # 6) Declare the native benchmark executable (raw CUDA/cuBLAS/cuSPARSE)
-# add_executable(workload-native main.native.cpp sparse_utils.cpp)
+# 6) Declare the native benchmark executable (raw CUDA/cuBLAS/cuSPARSE)
+add_executable(workload-native main.native.cpp sparse_utils.cpp)
 
-# target_link_libraries(workload-native
-#   PRIVATE
-#     CUDA::cudart
-#     CUDA::cublas
-#     CUDA::cusparse
-#     Threads::Threads
-# )
+target_link_libraries(workload-native
+  PRIVATE
+    CUDA::cudart
+    CUDA::cublas
+    CUDA::cusparse
+    Threads::Threads
+)
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
new file mode 100644
index 0000000000..787b129541
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
@@ -0,0 +1,378 @@
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+#include <cusparse.h>
+
+#include <iostream>
+#include <vector>
+#include <string>
+#include <chrono>
+#include <thread>
+#include <mutex>
+#include <queue>
+#include <condition_variable>
+#include <filesystem>
+#include <memory>
+#include <algorithm>
+#include <random>
+#include <atomic>
+#include <cmath>
+
+#include "sparse_utils.hpp"
+
+namespace fs = std::filesystem;
+
+constexpr uint32_t WORKLOAD_SEED = 42;
+
+// ============================================================
+// Error Checking
+// ============================================================
+
+#define CHECK_CUDA(call)                                                   \
+    do {                                                                   \
+        cudaError_t status = call;                                         \
+        if (status != cudaSuccess) {                                       \
+            std::cerr << "CUDA Error: " << cudaGetErrorString(status)      \
+                      << " at " << __FILE__ << ":" << __LINE__ << std::endl; \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+#define CHECK_CUBLAS(call)                                                 \
+    do {                                                                   \
+        cublasStatus_t status = call;                                      \
+        if (status != CUBLAS_STATUS_SUCCESS) {                             \
+            std::cerr << "cuBLAS Error at "                               \
+                      << __FILE__ << ":" << __LINE__ << std::endl;         \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+#define CHECK_CUSPARSE(call)                                               \
+    do {                                                                   \
+        cusparseStatus_t status = call;                                    \
+        if (status != CUSPARSE_STATUS_SUCCESS) {                           \
+            std::cerr << "cuSPARSE Error at "                             \
+                      << __FILE__ << ":" << __LINE__ << std::endl;         \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+// ============================================================
+// Thread Safe Queue
+// ============================================================
+
+template<typename T>
+class ThreadSafeQueue {
+public:
+    void push(T item) {
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            queue_.push(std::move(item));
+        }
+        cv_.notify_one();
+    }
+
+    bool wait_pop(T& item) {
+        std::unique_lock<std::mutex> lock(mutex_);
+
+        cv_.wait(lock, [&] {
+            return shutdown_ || !queue_.empty();
+        });
+
+        if (!queue_.empty()) {
+            item = std::move(queue_.front());
+            queue_.pop();
+            return true;
+        }
+
+        return false;
+    }
+
+    void signal_shutdown() {
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            shutdown_ = true;
+        }
+        cv_.notify_all();
+    }
+
+private:
+    std::queue<T> queue_;
+    std::mutex mutex_;
+    std::condition_variable cv_;
+    bool shutdown_ = false;
+};
+
+// ============================================================
+// Existing Solver Implementations
+// ============================================================
+
+int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
+                    const MatrixTask& task, cudaStream_t stream) {
+    int n = (int)task.data->row_ptr.size() - 1;
+    float alpha = 1.0f, beta = 0.0f, r0 = 0.0f, r1 = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
+    float tolerance = 1e-5f;
+    int max_iters = 128000;
+
+    float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b;
+    int *d_row_ptr, *d_col_ind;
+
+    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
+
+    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
+
+    CHECK_CUBLAS(cublasSetStream(cublas, stream));
+    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
+
+    cusparseSpMatDescr_t matA;
+    cusparseDnVecDescr_t vecX, vecP, vecAp;
+
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val,
+                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                     CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecAp, n, d_Ap, CUDA_R_32F));
+
+    size_t bufferSize = 0;
+    void* d_buffer = nullptr;
+    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, vecX, &beta, vecAp,
+                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
+
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_p, 1));
+    CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
+
+    int k = 0;
+    while (k < max_iters) {
+        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                    &alpha, matA, vecP, &beta, vecAp,
+                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
+
+        float pAp;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
+        a = r1 / pAp;
+
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
+
+        na = -a;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &na, d_Ap, 1, d_r, 1));
+
+        r0 = r1;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
+
+        if (std::sqrt(r1) < tolerance) break;
+
+        b = r1 / r0;
+        CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_r, 1, d_p, 1));
+        k++;
+    }
+
+    CHECK_CUSPARSE(cusparseDestroySpMat(matA));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecX));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
+
+    CHECK_CUDA(cudaFreeAsync(d_val, stream));
+    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
+    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
+    CHECK_CUDA(cudaFreeAsync(d_x, stream));
+    CHECK_CUDA(cudaFreeAsync(d_r, stream));
+    CHECK_CUDA(cudaFreeAsync(d_p, stream));
+    CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
+    CHECK_CUDA(cudaFreeAsync(d_b, stream));
+    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
+
+    return k;
+}
+
+// ============================================================
+// Producer
+// ============================================================
+
+void producer(
+    ThreadSafeQueue<MatrixTask>& queue,
+    const std::vector<MatrixTask>& matrix_pool)
+{
+    for (const auto& task : matrix_pool) {
+        queue.push(task);
+    }
+    queue.signal_shutdown();
+}
+
+// ============================================================
+// Worker
+// ============================================================
+
+void gpu_stream_worker(
+    int device_id,
+    int worker_id,
+    ThreadSafeQueue<MatrixTask>& queue)
+{
+    CHECK_CUDA(cudaSetDevice(device_id));
+
+    cudaStream_t stream;
+    cublasHandle_t cublas;
+    cusparseHandle_t cusparse;
+
+    CHECK_CUDA(
+        cudaStreamCreateWithFlags(
+            &stream,
+            cudaStreamNonBlocking));
+
+    CHECK_CUBLAS(
+        cublasCreate(&cublas));
+
+    CHECK_CUSPARSE(
+        cusparseCreate(&cusparse));
+
+    MatrixTask task;
+
+    while (queue.wait_pop(task)) {
+        std::cout << "[SUPE] Starting solver for: " << task.path << std::endl;
+        auto start_task =
+            std::chrono::steady_clock::now();
+        int iterations = 0;
+        if (task.type == CGS_SOLVER) {
+            iterations = solve_cg_async(
+                cublas,
+                cusparse,
+                task,
+                stream);
+        } else {
+            throw std::runtime_error(
+                "Unsupported solver type");
+        }
+        CHECK_CUDA(cudaStreamSynchronize(stream));
+        auto end_task =
+            std::chrono::steady_clock::now();
+        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
+                          end_task - start_task).count();
+        std::cout << "[DONE] " << task.path << ": Converged in " << iterations 
+                  << " iterations (" << duration << " ms)." << std::endl;
+    }
+
+    CHECK_CUBLAS(cublasDestroy(cublas));
+    CHECK_CUSPARSE(cusparseDestroy(cusparse));
+    CHECK_CUDA(cudaStreamDestroy(stream));
+}
+
+// ============================================================
+// Main
+// ============================================================
+
+int main(int argc, char** argv)
+{
+    int num_streams = 4;
+
+    if (argc > 1)
+        num_streams = std::max(1, std::atoi(argv[1]));
+
+    std::cout << "[INFO] Loading matrices...\n";
+
+    std::vector<MatrixTask> matrix_pool =
+        scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+
+    if (matrix_pool.empty()) {
+        std::cerr << "No matrices found.\n";
+        return 1;
+    }
+
+    int num_gpus = 0;
+
+    CHECK_CUDA(
+        cudaGetDeviceCount(&num_gpus));
+
+    if (num_gpus == 0) {
+        std::cerr << "No CUDA devices found.\n";
+        return 1;
+    }
+
+    std::cout
+        << "[INFO] Found "
+        << num_gpus
+        << " GPUs\n";
+
+    std::cout
+        << "[INFO] Matrix pool size: "
+        << matrix_pool.size()
+        << "\n";
+
+    std::cout
+        << "[INFO] Streams/GPU: "
+        << num_streams
+        << "\n";
+
+    ThreadSafeQueue<MatrixTask> work_queue;
+
+    auto benchmark_start =
+        std::chrono::steady_clock::now();
+
+    std::thread producer_thread(
+        producer,
+        std::ref(work_queue),
+        std::cref(matrix_pool));
+
+    std::vector<std::thread> workers;
+
+    workers.reserve(
+        num_gpus * num_streams);
+
+    for (int gpu = 0; gpu < num_gpus; ++gpu) {
+
+        for (int stream = 0;
+             stream < num_streams;
+             ++stream)
+        {
+            int worker_id =
+                gpu * num_streams + stream;
+
+            workers.emplace_back(
+                gpu_stream_worker,
+                gpu,
+                worker_id,
+                std::ref(work_queue));
+        }
+    }
+
+    producer_thread.join();
+
+    for (auto& worker : workers) {
+        worker.join();
+    }
+
+    auto benchmark_end =
+        std::chrono::steady_clock::now();
+
+    std::chrono::duration<double>
+        total_time =
+            benchmark_end - benchmark_start;
+
+    std::cout << "All tasks in the pool have been processed." << std::endl;
+    std::cout << "\n";
+    std::cout << "=====================================\n";
+    std::cout << "IRREGULAR WORKLOAD BENCHMARK\n";
+    std::cout << "=====================================\n";
+    std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
+    std::cout << "GPUs:               " << num_gpus << "\n";
+    std::cout << "Streams per GPU:    " << num_streams << "\n";
+    std::cout << "Worker Threads:     " << num_gpus * num_streams << "\n";
+    std::cout << "Total Runtime:      " << total_time.count() << " s\n";
+    std::cout << "=====================================\n";
+
+    return 0;
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index e2873f40f5..2bba8124e9 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -255,7 +255,7 @@ struct supervisor_state {
     std::vector<caf::actor> active_solvers;
     std::unordered_map<caf::actor_id, std::string> task_names;
     std::unordered_map<caf::actor_id, std::chrono::steady_clock::time_point> start_times;
-    int max_active = 1; // Admission control limit to be mindful of GPU memory. If Illegal memory access that means two or more actors are using the same stream
+    int max_active = 4; // Admission control limit to be mindful of GPU memory. If Illegal memory access that means two or more actors are using the same stream
     int num_iterations = 50;
     int stream = 0;
     int device = 0;
@@ -282,7 +282,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                                     create_in_out_arg(task.data->x_guess),
                                     (int)task.data->row_ptr.size() - 1,
                                     (int)task.data->values.size(),
-                                    1e-4f, 128000, s.device, (++s.stream)%32, actor_cast<actor>(self));
+                                    1e-5f, 128000, s.device, (++s.stream)%32, actor_cast<actor>(self));
             s.task_names[solver->id()] = std::move(path);
             
             s.start_times[solver->id()] = std::chrono::steady_clock::now();
@@ -352,7 +352,8 @@ void caf_main(actor_system& sys) {
     {
          //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/unsymmetric", CGS_SOLVER);
-         auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd", CGS_SOLVER);
+         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric", CGS_SOLVER);
+         auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
 
         std::cout << "loaded\n";
         if (tasks_vec.empty()) {

From a12958e0e14ba2ceed0a95476f2e76db74355a91 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 14:26:09 -0600
Subject: [PATCH 0840/1000] Incorporate methods to copy back to host on other
 streams other than the one the mem_ptr currently has as well as made those
 changes into the test.

---
 libcaf_cuda/caf/cuda/command_runner.hpp       |  7 +++
 libcaf_cuda/caf/cuda/memory_command.hpp       | 22 +++++++--
 .../main.test.cpp                             | 48 ++++++++++++++-----
 3 files changed, 60 insertions(+), 17 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index bae0992c5a..d9cc45e979 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -264,6 +264,13 @@ class command_runner {
         cmd->run_async(std::move(callback));
     }
 
+    // Asynchronous copy back with explicit stream/actor ID
+    template <typename T, typename F>
+    void copy_to_host_async(mem_ptr<T> ptr, int stream_id, F callback) {
+        auto cmd = caf::make_counted<copy_back_command<T>>(std::move(ptr), stream_id);
+        cmd->run_async(std::move(callback));
+    }
+
     // Asynchronous copy back to user-provided buffer
     template <typename T, typename F>
     void copy_to_host_async(mem_ptr<T> ptr, T* dst, size_t count, F callback) {
diff --git a/libcaf_cuda/caf/cuda/memory_command.hpp b/libcaf_cuda/caf/cuda/memory_command.hpp
index f47b2250c8..01aac91d0a 100644
--- a/libcaf_cuda/caf/cuda/memory_command.hpp
+++ b/libcaf_cuda/caf/cuda/memory_command.hpp
@@ -87,7 +87,12 @@ class bulk_memory_command : public caf::ref_counted {
 template <typename T>
 class copy_back_command : public caf::ref_counted {
 public:
-  copy_back_command(mem_ptr<T> ptr) : ptr_(std::move(ptr)) {
+  copy_back_command(mem_ptr<T> ptr) : ptr_(std::move(ptr)), stream_id_(-1) {
+    if (!ptr_)
+      throw std::runtime_error("copy_back_command: null mem_ptr");
+  }
+
+  copy_back_command(mem_ptr<T> ptr, int stream_id) : ptr_(std::move(ptr)), stream_id_(stream_id) {
     if (!ptr_)
       throw std::runtime_error("copy_back_command: null mem_ptr");
   }
@@ -105,7 +110,7 @@ class copy_back_command : public caf::ref_counted {
       throw std::runtime_error("Cannot copy a read-only buffer back to host");
 
     CHECK_CUDA(cuCtxPushCurrent(ptr_->get_ctx()));
-    CUstream s = ptr_->stream();
+    CUstream s = resolve_stream();
 
     if (ptr_->is_scalar()) {
       // For scalars, the value is already on the host.
@@ -138,7 +143,7 @@ class copy_back_command : public caf::ref_counted {
                             ptr_->is_scalar(), *ptr_->host_scalar_ptr()};
 
     CHECK_CUDA(cuCtxPushCurrent(ptr_->get_ctx()));
-    CUstream s = ptr_->stream();
+    CUstream s = resolve_stream();
 
     if (!ptr_->is_scalar()) {
       size_t bytes = ptr_->size() * sizeof(T);
@@ -176,7 +181,7 @@ class copy_back_command : public caf::ref_counted {
                             ptr_->is_scalar(), *ptr_->host_scalar_ptr()};
 
     CHECK_CUDA(cuCtxPushCurrent(ptr_->get_ctx()));
-    CUstream s = ptr_->stream();
+    CUstream s = resolve_stream();
 
     if (!ptr_->is_scalar()) {
       size_t bytes = count * sizeof(T);
@@ -197,7 +202,16 @@ class copy_back_command : public caf::ref_counted {
   }
 
 private:
+  CUstream resolve_stream() {
+    if (stream_id_ == -1)
+      return ptr_->stream();
+    auto plat = platform::create();
+    auto dev = plat->schedule(stream_id_, ptr_->deviceID());
+    return dev->get_stream_for_actor(stream_id_);
+  }
+
   mem_ptr<T> ptr_;
+  int stream_id_;
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 271464aff7..f5a7af34ea 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -204,7 +204,7 @@ caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
 // Manages 1 stream and pulls work from the Device Actor.
 struct worker_state {
     int device_id;
-    int stream_id;
+    std::vector<int> stream_ids;
     caf::cuda::program_ptr program;
     caf::actor device_actor;
     caf::actor supervisor;
@@ -215,12 +215,12 @@ struct worker_state {
 
 caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
                               caf::actor supervisor, caf::actor device_actor, caf::cuda::program_ptr program,
-                              int dev_id, int stream_id, int max_in_flight_tasks) {
+                              int dev_id, std::vector<int> stream_ids, int max_in_flight_tasks) {
     self->state().supervisor = supervisor;
     self->state().device_actor = device_actor;
     self->state().program = program;
     self->state().device_id = dev_id;
-    self->state().stream_id = stream_id;
+    self->state().stream_ids = std::move(stream_ids);
     self->state().max_in_flight_tasks = max_in_flight_tasks;
 
     // Trigger initial work requests up to max_in_flight_tasks
@@ -238,23 +238,38 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
             st.in_flight_tasks_count++; // Mark as pending immediately
             self->mail(get_work_atom_v).request(st.device_actor, infinite).then(
                 [=](int N, in<int> matrixA, in<int> matrixB) {
-                    // GPU Pipeline: Transfer -> Kernel -> Copyback
-                    auto arg1 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixA));
-                    auto arg2 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixB));
+                    auto& st = self->state(); // Access state via self
+                    int s_h2d = st.stream_ids[0];
+                    int s_ker = st.stream_ids[1];
+                    int s_d2h = st.stream_ids[2];
+
+                    auto h2d_done = mmul_command.create_event(st.device_id);
+                    auto kernel_done = mmul_command.create_event(st.device_id);
+
+                    // Stage 1: H2D Transfer
+                    auto arg1 = mmul_command.transfer_memory(st.device_id, s_h2d, std::move(matrixA));
+                    auto arg2 = mmul_command.transfer_memory(st.device_id, s_h2d, std::move(matrixB));
+                    mmul_command.record_event(h2d_done, s_h2d, st.device_id);
 
                     const int THREADS = 32;
                     const int BLOCKS = (N + THREADS - 1) / THREADS;
                     caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-                    auto result = mmul_kernel.run_async(st.program, dims, st.stream_id, 0, st.device_id,
+                    // Stage 2: Kernel Execution (Wait for H2D to finish)
+                    mmul_command.wait_event(h2d_done, s_ker, st.device_id);
+                    auto result = mmul_kernel.run_async(st.program, dims, s_ker, 0, st.device_id,
                                                        arg1, arg2,
                                                        caf::cuda::create_out_arg<int>(N * N),
                                                        caf::cuda::create_in_arg<int>(N));
+                    mmul_command.record_event(kernel_done, s_ker, st.device_id);
 
+                    // Stage 3: D2H Copyback (Wait for Kernel to finish)
+                    mmul_command.wait_event(kernel_done, s_d2h, st.device_id);
                     auto bufferC = std::get<2>(result);
                     auto self_hdl = caf::actor_cast<caf::actor>(self);
 
-                    mmul_command.copy_to_host_async(bufferC, [self_hdl, N_task = N](std::vector<int>&&) {
+                    // Capturing events in the lambda ensures they aren't destroyed too early
+                    mmul_command.copy_to_host_async(bufferC, s_d2h, [self_hdl, N_task = N, h2d_done, kernel_done](std::vector<int>&&) {
                         caf::anon_mail(task_done_atom_v, N_task).send(self_hdl); // Pass N back to self
                     });
                 },
@@ -263,13 +278,14 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
                     st.in_flight_tasks_count--; // Revert pending status on failure
                     if (err == sec::runtime_error) {
                         // Not enough memory, retry after a delay
-                        self->println("Worker {}: Not enough memory, retrying for work...", st.stream_id);
+                        self->println("Worker (dev:{}): Not enough memory, retrying for work...", st.device_id);
                         self->delayed_anon_send(self, 100ms, request_work_atom_v);
                     } else if (err == sec::end_of_stream) {
                         st.draining = true; // Mark as draining, let in-flight finish
                         if (st.in_flight_tasks_count == 0) {
                             self->mail(worker_done_atom_v).send(st.device_actor);
-                            mmul_command.release_stream_for_actor(st.stream_id);
+                            for (auto id : st.stream_ids)
+                                mmul_command.release_stream_for_actor(id);
                             self->quit();
                         }
                     }
@@ -284,7 +300,8 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
             
             if (st.draining && st.in_flight_tasks_count == 0) {
                 self->mail(worker_done_atom_v).send(st.device_actor);
-                mmul_command.release_stream_for_actor(st.stream_id);
+                for (auto id : st.stream_ids)
+                    mmul_command.release_stream_for_actor(id);
                 self->quit();
             } else if (!st.draining) {
                 self->mail(request_work_atom_v).send(self); // Request next task if capacity allows
@@ -316,12 +333,17 @@ caf::behavior supervisor_actor_fun(
     caf::cuda::manager& mgr = caf::cuda::manager::get();
     int num_gpus = mgr.get_num_devices();
     auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+    
+    int next_stream_base = 0;
 
     for (int i = 0; i < num_gpus; ++i) {
         auto broker = self->spawn(gpu_device_actor, pool, pool_actor, workers_per_gpu, i, max_in_flight_tasks_per_worker);
         
-        for (int j = 0; j < workers_per_gpu; ++j)
-            self->spawn(mmul_worker_fun, self, broker, program, i, (i * 1000) + j, max_in_flight_tasks_per_worker);
+        for (int j = 0; j < workers_per_gpu; ++j) {
+            std::vector<int> streams = {next_stream_base, next_stream_base + 1, next_stream_base + 2};
+            self->spawn(mmul_worker_fun, self, broker, program, i, streams, max_in_flight_tasks_per_worker);
+            next_stream_base += 3;
+        }
     }
 
     return {

From bd369251f9e0fb3fe20837a1483950be159bf001 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 14:46:04 -0600
Subject: [PATCH 0841/1000] added makespan timer

---
 .../fault-tolerance-workload-test/main.test.cpp | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 2bba8124e9..30ceea0890 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -366,8 +366,25 @@ void caf_main(actor_system& sys) {
             data->x_guess = {0.0f, 0.0f};
             tasks_vec.push_back({"dummy_task", CGS_SOLVER, data});
         }
+
+        auto benchmark_start = std::chrono::steady_clock::now();
+
         sys.spawn(supervisor_actor, std::move(tasks_vec));
         sys.await_all_actors_done();
+
+        auto benchmark_end = std::chrono::steady_clock::now();
+        std::chrono::duration<double> total_time = benchmark_end - benchmark_start;
+        int num_gpus = manager::get().get_num_devices();
+
+        std::cout << "\n";
+        std::cout << "=====================================\n";
+        std::cout << "IRREGULAR WORKLOAD BENCHMARK (CAF)\n";
+        std::cout << "=====================================\n";
+        std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
+        std::cout << "GPUs:               " << num_gpus << "\n";
+        std::cout << "Admission Control:  " << 4 << "\n"; 
+        std::cout << "Total Runtime:      " << total_time.count() << " s\n";
+        std::cout << "=====================================\n";
     }
     manager::shutdown();
 }

From 4b446cf601b48d28e76fbf5a07d2fa608e21f543 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 14:50:45 -0600
Subject: [PATCH 0842/1000] implemented a free memory command

---
 libcaf_cuda/caf/cuda/command_runner.hpp |  7 ++++++
 libcaf_cuda/caf/cuda/mem_ref.hpp        | 12 ++++++++++
 libcaf_cuda/caf/cuda/memory_command.hpp | 30 +++++++++++++++++++++++++
 3 files changed, 49 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index d9cc45e979..613b485598 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -278,6 +278,13 @@ class command_runner {
         cmd->run_async(dst, count, std::move(callback));
     }
 
+    // Enqueue an asynchronous free operation on the given stream
+    template <typename T>
+    void free_memory(mem_ptr<T> ptr, int stream_id = -1) {
+        free_memory_command<T> cmd(std::move(ptr), stream_id);
+        cmd.enqueue();
+    }
+
 
   // -------------------------------
   // Destroy streams for a given actor ID
diff --git a/libcaf_cuda/caf/cuda/mem_ref.hpp b/libcaf_cuda/caf/cuda/mem_ref.hpp
index 751ce70083..3ea1b04149 100644
--- a/libcaf_cuda/caf/cuda/mem_ref.hpp
+++ b/libcaf_cuda/caf/cuda/mem_ref.hpp
@@ -126,6 +126,18 @@ class mem_ref : public caf::ref_counted {
     ctx = nullptr;
   }
 
+  // Enqueues a free operation on the specified stream and invalidates this reference.
+  void free_on(CUstream s) {
+    if (!is_scalar_ && memory_) {
+      if (ctx) {
+        CHECK_CUDA(cuCtxPushCurrent(ctx));
+        CHECK_CUDA(cuMemFreeAsync(memory_, s));
+        CHECK_CUDA(cuCtxPopCurrent(nullptr));
+      }
+      memory_ = 0;
+    }
+  }
+
   //copies gpu memory back to cpu memory in the form of an std::vector
   std::vector<T> copy_to_host() const {
 	  if (access_ == IN)
diff --git a/libcaf_cuda/caf/cuda/memory_command.hpp b/libcaf_cuda/caf/cuda/memory_command.hpp
index 01aac91d0a..1822ef79dc 100644
--- a/libcaf_cuda/caf/cuda/memory_command.hpp
+++ b/libcaf_cuda/caf/cuda/memory_command.hpp
@@ -214,4 +214,34 @@ class copy_back_command : public caf::ref_counted {
   int stream_id_;
 };
 
+// ===========================================================================
+// FREE MEMORY COMMAND
+// Handles freeing memory on a given stream.
+// ===========================================================================
+template <typename T>
+class free_memory_command : public caf::ref_counted {
+public:
+  free_memory_command(mem_ptr<T> ptr, int stream_id = -1)
+      : ptr_(std::move(ptr)), stream_id_(stream_id) {
+    if (!ptr_)
+      throw std::runtime_error("free_memory_command: null mem_ptr");
+  }
+
+  void enqueue() {
+    ptr_->free_on(resolve_stream());
+  }
+
+private:
+  CUstream resolve_stream() {
+    if (stream_id_ == -1)
+      return ptr_->stream();
+    auto plat = platform::create();
+    auto dev = plat->schedule(stream_id_, ptr_->deviceID());
+    return dev->get_stream_for_actor(stream_id_);
+  }
+
+  mem_ptr<T> ptr_;
+  int stream_id_;
+};
+
 } // namespace caf::cuda

From 3c6542426a799ff60d5ba57e05807835b1252bbd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 14:57:18 -0600
Subject: [PATCH 0843/1000] made changes to memory cleanup

---
 .../main.test.cpp                              | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index f5a7af34ea..f7fb362c35 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -237,12 +237,13 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
 
             st.in_flight_tasks_count++; // Mark as pending immediately
             self->mail(get_work_atom_v).request(st.device_actor, infinite).then(
-                [=](int N, in<int> matrixA, in<int> matrixB) {
+                [=](int N, in<int> matrixA, in<int> matrixB) mutable {
                     auto& st = self->state(); // Access state via self
                     int s_h2d = st.stream_ids[0];
                     int s_ker = st.stream_ids[1];
                     int s_d2h = st.stream_ids[2];
 
+                    // Create fresh events for this specific task pipeline
                     auto h2d_done = mmul_command.create_event(st.device_id);
                     auto kernel_done = mmul_command.create_event(st.device_id);
 
@@ -250,7 +251,7 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
                     auto arg1 = mmul_command.transfer_memory(st.device_id, s_h2d, std::move(matrixA));
                     auto arg2 = mmul_command.transfer_memory(st.device_id, s_h2d, std::move(matrixB));
                     mmul_command.record_event(h2d_done, s_h2d, st.device_id);
-
+                  
                     const int THREADS = 32;
                     const int BLOCKS = (N + THREADS - 1) / THREADS;
                     caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
@@ -262,14 +263,19 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
                                                        caf::cuda::create_out_arg<int>(N * N),
                                                        caf::cuda::create_in_arg<int>(N));
                     mmul_command.record_event(kernel_done, s_ker, st.device_id);
+                      mmul.command.add_callback(s_ker, [arg1, arg2]() {
+                        anon_mail(arg1,arg2).send(self);
+                    });
+
 
                     // Stage 3: D2H Copyback (Wait for Kernel to finish)
                     mmul_command.wait_event(kernel_done, s_d2h, st.device_id);
                     auto bufferC = std::get<2>(result);
                     auto self_hdl = caf::actor_cast<caf::actor>(self);
 
-                    // Capturing events in the lambda ensures they aren't destroyed too early
-                    mmul_command.copy_to_host_async(bufferC, s_d2h, [self_hdl, N_task = N, h2d_done, kernel_done](std::vector<int>&&) {
+                  
+                    mmul_command.copy_to_host_async(bufferC, s_d2h, 
+                      [self_hdl, N_task = N,result](std::vector<int>&&) {
                         caf::anon_mail(task_done_atom_v, N_task).send(self_hdl); // Pass N back to self
                     });
                 },
@@ -289,6 +295,10 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
                             self->quit();
                         }
                     }
+                },
+                [=] (caf::cuda::mem_ptr<int> matrixA, caf::cuda::mem_ptr<int> matrixB) {
+                    mmul_command.free_memory(matrixA,stream_ids[1]);
+                    mmul_command.free_memory(matrixB,stream_ids[1]);
                 }
             );
         },

From f3b06a38a431fdf1a3beeefa095948252555104e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 15:00:00 -0600
Subject: [PATCH 0844/1000] fixed syntax error

---
 .../mmul-randonom-batch-benchmark/main.test.cpp               | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index f7fb362c35..7bdad8e237 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -296,7 +296,9 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
                         }
                     }
                 },
-                [=] (caf::cuda::mem_ptr<int> matrixA, caf::cuda::mem_ptr<int> matrixB) {
+               
+        },
+         [=] (caf::cuda::mem_ptr<int> matrixA, caf::cuda::mem_ptr<int> matrixB) {
                     mmul_command.free_memory(matrixA,stream_ids[1]);
                     mmul_command.free_memory(matrixB,stream_ids[1]);
                 }

From 928527e4278a78c90c62548c601536d322fd59b7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 4 Jun 2026 15:03:19 -0600
Subject: [PATCH 0845/1000] fixed syntax errors

---
 .../mmul-randonom-batch-benchmark/main.test.cpp  | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 7bdad8e237..bb2438e795 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -263,7 +263,7 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
                                                        caf::cuda::create_out_arg<int>(N * N),
                                                        caf::cuda::create_in_arg<int>(N));
                     mmul_command.record_event(kernel_done, s_ker, st.device_id);
-                      mmul.command.add_callback(s_ker, [arg1, arg2]() {
+                      mmul_command.add_callback(s_ker, st.device_id, [arg1, arg2, self]() {
                         anon_mail(arg1,arg2).send(self);
                     });
 
@@ -274,8 +274,8 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
                     auto self_hdl = caf::actor_cast<caf::actor>(self);
 
                   
-                    mmul_command.copy_to_host_async(bufferC, s_d2h, 
-                      [self_hdl, N_task = N,result](std::vector<int>&&) {
+                    mmul_command.copy_to_host_async(bufferC, s_d2h,
+                      [self_hdl, N_task = N](std::vector<int>&&) {
                         caf::anon_mail(task_done_atom_v, N_task).send(self_hdl); // Pass N back to self
                     });
                 },
@@ -295,15 +295,13 @@ caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
                             self->quit();
                         }
                     }
-                },
-               
-        },
-         [=] (caf::cuda::mem_ptr<int> matrixA, caf::cuda::mem_ptr<int> matrixB) {
-                    mmul_command.free_memory(matrixA,stream_ids[1]);
-                    mmul_command.free_memory(matrixB,stream_ids[1]);
                 }
             );
         },
+        [=](caf::cuda::mem_ptr<int> matrixA, caf::cuda::mem_ptr<int> matrixB) {
+            mmul_command.free_memory(matrixA, stream_ids[1]);
+            mmul_command.free_memory(matrixB, stream_ids[1]);
+        },
         [=](task_done_atom, int N_completed) {
             auto& st = self->state();
             st.in_flight_tasks_count--; // Decrement count

From b01f68f3537dcf583fff923dd346c9270eb0784f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 5 Jun 2026 08:27:02 -0600
Subject: [PATCH 0846/1000] updated code to fix an exit race condition where
 actor id was being reference after actor was dead

---
 .../main.test.cpp                             | 49 ++++++++++---------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 30ceea0890..8c5dd88f87 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -76,6 +76,7 @@ std::string to_string(cg_error_type err) {
 template <class T = float>
 struct ft_cg_state {
   // Host Data
+  std::string path;
   in<int> h_row_ptr, h_col_ind;
   in<T> h_values, h_b;
   in_out<T> h_x;
@@ -105,12 +106,14 @@ struct ft_cg_state {
 
 template <class T = float>
 behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
+                                 std::string path,
                                  std::shared_ptr<MatrixData> data,
                                  in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
                                  int n, int nnz, T tol, int max_iter,
                                  int dev_num, int stream, caf::actor supervisor) {
   auto& s = self->state();
   s.pinned_data = std::move(data);
+  s.path = std::move(path);
   s.h_row_ptr = std::move(rp); s.h_col_ind = std::move(ci);
   s.h_values = std::move(val); s.h_b = std::move(b_in); s.h_x = std::move(x_in);
   s.n = n; s.nnz = nnz; s.tol = tol; s.max_iter = max_iter;
@@ -158,7 +161,7 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
 
       // Notify supervisor that setup is complete and report initial status
       solver_result_meta meta(st.device_id, st.stream_id, 0, false, CG_SUCCESS);
-      self->mail(gpu_done_atom_v, std::vector<T>{}, meta).send(st.supervisor);
+      self->mail(gpu_done_atom_v, st.path, actor_cast<actor>(self), std::vector<T>{}, meta).send(st.supervisor);
     },
 
     [=](cg_next_step_atom, int num_iters) {
@@ -237,13 +240,19 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
       int err_flag = runner.copy_to_host(st.d_err)[0];
       if (err_flag != 0 || std::isnan(st.current_rho) || std::isinf(st.current_rho)) code = CG_NAN_INF;
 
+      if (code != CG_SUCCESS) converged = false;
+
       solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
       
       // Report current solution and metadata to the supervisor
-      runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor](std::vector<T> sol) {
-        self->mail(gpu_done_atom_v, std::move(sol), meta).send(supervisor);
-        if (converged || code != CG_SUCCESS) self->quit();
+      runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor, path = st.path, self_h = actor_cast<actor>(self)](std::vector<T> sol) {
+        anon_mail(gpu_done_atom_v, path, self_h, std::move(sol), meta).send(supervisor);
+        if (converged || code != CG_SUCCESS)
+          self->quit();
       });
+    },
+    [=](shutdown_atom) {
+      self->quit();
     }
   };
 }
@@ -253,8 +262,7 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
 struct supervisor_state {
     std::deque<MatrixTask> queue;
     std::vector<caf::actor> active_solvers;
-    std::unordered_map<caf::actor_id, std::string> task_names;
-    std::unordered_map<caf::actor_id, std::chrono::steady_clock::time_point> start_times;
+    std::unordered_map<std::string, std::chrono::steady_clock::time_point> start_times;
     int max_active = 4; // Admission control limit to be mindful of GPU memory. If Illegal memory access that means two or more actors are using the same stream
     int num_iterations = 50;
     int stream = 0;
@@ -274,6 +282,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
             std::string path = task.path;
             self->println("[SUPE] Starting solver for: {}", path);
             auto solver = self->spawn(fault_tolerant_cg_actor<float>,
+                                    path,
                                     task.data,
                                     create_in_arg(task.data->row_ptr),
                                     create_in_arg(task.data->col_indices),
@@ -283,9 +292,8 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                                     (int)task.data->row_ptr.size() - 1,
                                     (int)task.data->values.size(),
                                     1e-5f, 128000, s.device, (++s.stream)%32, actor_cast<actor>(self));
-            s.task_names[solver->id()] = std::move(path);
-            
-            s.start_times[solver->id()] = std::chrono::steady_clock::now();
+
+            s.start_times[path] = std::chrono::steady_clock::now();
             s.active_solvers.push_back(solver);
             self->mail(start_atom_v).send(solver);
         }
@@ -294,24 +302,13 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
     spawn_next();
     
     return {
-        [=](gpu_done_atom, std::vector<float>& solution, solver_result_meta meta) {
+        [=](gpu_done_atom, const std::string& task_name, caf::actor solver, std::vector<float>& solution, solver_result_meta meta) {
             auto& s = self->state();
-            auto solver = actor_cast<caf::actor>(self->current_sender());
-            
-            if (meta.iterations == 0 && meta.error_code == CG_SUCCESS) {
-                // Initialization report: trigger the first batch
-                self->mail(cg_next_step_atom_v, s.num_iterations).send(solver);
-                return;
-            }
-
-            std::string task_name = s.task_names.count(solver->id()) 
-                                    ? s.task_names[solver->id()] 
-                                    : "Unknown Task";
 
             if (meta.converged || meta.error_code != CG_SUCCESS) {
                 auto end_time = std::chrono::steady_clock::now();
                 auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
-                                  end_time - s.start_times[solver->id()]).count();
+                                  end_time - s.start_times[task_name]).count();
 
                 if (meta.converged) {
                     if (meta.iterations == 0)
@@ -329,20 +326,24 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
                 if (it != s.active_solvers.end())
                     s.active_solvers.erase(it);
-                s.task_names.erase(solver->id());
-                s.start_times.erase(solver->id());
+                s.start_times.erase(task_name);
 
                 spawn_next();
 
                 if (s.active_solvers.empty() && s.queue.empty()) {
                     self->println("All tasks in the batch have been processed.");
+                    caf::cuda::manager::shutdown();
                     self->quit();
                 }
+            } else if (meta.iterations == 0 && meta.error_code == CG_SUCCESS) {
+                // Initialization report (not yet converged): trigger the first batch
+                self->mail(cg_next_step_atom_v, s.num_iterations).send(solver);
             } else {
                 // Progress report: order the next iteration batch
                 self->mail(cg_next_step_atom_v, s.num_iterations).send(solver);
             }
         }
+       
     };
 }
 

From c2445ac276e08d246f1321e1e4d9a7b6a40b3f46 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 5 Jun 2026 08:41:30 -0600
Subject: [PATCH 0847/1000] updated stream allocation logic to prevent crashes

---
 .../main.test.cpp                             | 25 ++++++++++++++++---
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 8c5dd88f87..64c40b96a0 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -263,9 +263,10 @@ struct supervisor_state {
     std::deque<MatrixTask> queue;
     std::vector<caf::actor> active_solvers;
     std::unordered_map<std::string, std::chrono::steady_clock::time_point> start_times;
+    std::unordered_map<std::string, int> task_streams;
+    std::deque<int> available_streams;
     int max_active = 4; // Admission control limit to be mindful of GPU memory. If Illegal memory access that means two or more actors are using the same stream
     int num_iterations = 50;
-    int stream = 0;
     int device = 0;
 };
 
@@ -274,13 +275,21 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
     for (auto& t : tasks)
         st.queue.push_back(std::move(t));
 
+    // Initialize the pool with 32 distinct stream IDs
+    for (int i = 0; i < 32; ++i)
+        st.available_streams.push_back(i);
+
     auto spawn_next = [self]() {
         auto& s = self->state();
-        while (s.active_solvers.size() < static_cast<size_t>(s.max_active) && !s.queue.empty()) {
+        while (s.active_solvers.size() < static_cast<size_t>(s.max_active) && !s.queue.empty() && !s.available_streams.empty()) {
             auto task = std::move(s.queue.front());
             s.queue.pop_front();
             std::string path = task.path;
-            self->println("[SUPE] Starting solver for: {}", path);
+
+            int stream_id = s.available_streams.front();
+            s.available_streams.pop_front();
+
+            self->println("[SUPE] Starting solver for: {} (Stream: {})", path, stream_id);
             auto solver = self->spawn(fault_tolerant_cg_actor<float>,
                                     path,
                                     task.data,
@@ -291,10 +300,11 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                                     create_in_out_arg(task.data->x_guess),
                                     (int)task.data->row_ptr.size() - 1,
                                     (int)task.data->values.size(),
-                                    1e-5f, 128000, s.device, (++s.stream)%32, actor_cast<actor>(self));
+                                    1e-5f, 128000, s.device, stream_id, actor_cast<actor>(self));
 
             s.start_times[path] = std::chrono::steady_clock::now();
             s.active_solvers.push_back(solver);
+            s.task_streams[path] = stream_id;
             self->mail(start_atom_v).send(solver);
         }
     };
@@ -328,6 +338,13 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                     s.active_solvers.erase(it);
                 s.start_times.erase(task_name);
 
+                // Reclaim the stream ID and put it back in the pool
+                auto stream_it = s.task_streams.find(task_name);
+                if (stream_it != s.task_streams.end()) {
+                    s.available_streams.push_back(stream_it->second);
+                    s.task_streams.erase(stream_it);
+                }
+
                 spawn_next();
 
                 if (s.active_solvers.empty() && s.queue.empty()) {

From bce84ca73e8bad2c55f40c782b524c65f0a8d836 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 5 Jun 2026 08:52:48 -0600
Subject: [PATCH 0848/1000] fixed premature shutdown error

---
 libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 64c40b96a0..5362ee1b5a 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -349,7 +349,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
 
                 if (s.active_solvers.empty() && s.queue.empty()) {
                     self->println("All tasks in the batch have been processed.");
-                    caf::cuda::manager::shutdown();
+                    //caf::cuda::manager::shutdown();
                     self->quit();
                 }
             } else if (meta.iterations == 0 && meta.error_code == CG_SUCCESS) {

From 0a1ae609bc194f5c69ac368ff0c3b0de3fd263ca Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 5 Jun 2026 10:10:24 -0600
Subject: [PATCH 0849/1000] created a sorted version of the producer consumer
 problem. modified files to reflect these changes

---
 .../CMakeLists.txt                            |  13 +-
 .../main.native.cpp                           | 350 +-----------------
 .../main.native_sorted.cpp                    |  45 +++
 .../native_utils.cpp                          | 112 ++++++
 .../native_utils.hpp                          |  91 +++++
 5 files changed, 280 insertions(+), 331 deletions(-)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.hpp

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
index b980b1431a..85575dbdd2 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
@@ -87,7 +87,7 @@ target_link_libraries(test
 find_package(Threads REQUIRED)
 
 # 6) Declare the native benchmark executable (raw CUDA/cuBLAS/cuSPARSE)
-add_executable(workload-native main.native.cpp sparse_utils.cpp)
+add_executable(workload-native main.native.cpp sparse_utils.cpp native_utils.cpp)
 
 target_link_libraries(workload-native
   PRIVATE
@@ -96,3 +96,14 @@ target_link_libraries(workload-native
     CUDA::cusparse
     Threads::Threads
 )
+
+# 7) Declare the native sorted benchmark executable
+add_executable(workload-native-sorted main.native_sorted.cpp sparse_utils.cpp native_utils.cpp)
+
+target_link_libraries(workload-native-sorted
+  PRIVATE
+    CUDA::cudart
+    CUDA::cublas
+    CUDA::cusparse
+    Threads::Threads
+)
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
index 787b129541..ff4b0fa5f2 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
@@ -1,291 +1,27 @@
-#include <cuda_runtime.h>
-#include <cublas_v2.h>
-#include <cusparse.h>
-
 #include <iostream>
 #include <vector>
 #include <string>
 #include <chrono>
 #include <thread>
-#include <mutex>
-#include <queue>
-#include <condition_variable>
 #include <filesystem>
-#include <memory>
 #include <algorithm>
-#include <random>
-#include <atomic>
-#include <cmath>
-
-#include "sparse_utils.hpp"
-
-namespace fs = std::filesystem;
-
-constexpr uint32_t WORKLOAD_SEED = 42;
-
-// ============================================================
-// Error Checking
-// ============================================================
-
-#define CHECK_CUDA(call)                                                   \
-    do {                                                                   \
-        cudaError_t status = call;                                         \
-        if (status != cudaSuccess) {                                       \
-            std::cerr << "CUDA Error: " << cudaGetErrorString(status)      \
-                      << " at " << __FILE__ << ":" << __LINE__ << std::endl; \
-            std::exit(EXIT_FAILURE);                                       \
-        }                                                                  \
-    } while (0)
-
-#define CHECK_CUBLAS(call)                                                 \
-    do {                                                                   \
-        cublasStatus_t status = call;                                      \
-        if (status != CUBLAS_STATUS_SUCCESS) {                             \
-            std::cerr << "cuBLAS Error at "                               \
-                      << __FILE__ << ":" << __LINE__ << std::endl;         \
-            std::exit(EXIT_FAILURE);                                       \
-        }                                                                  \
-    } while (0)
-
-#define CHECK_CUSPARSE(call)                                               \
-    do {                                                                   \
-        cusparseStatus_t status = call;                                    \
-        if (status != CUSPARSE_STATUS_SUCCESS) {                           \
-            std::cerr << "cuSPARSE Error at "                             \
-                      << __FILE__ << ":" << __LINE__ << std::endl;         \
-            std::exit(EXIT_FAILURE);                                       \
-        }                                                                  \
-    } while (0)
-
-// ============================================================
-// Thread Safe Queue
-// ============================================================
-
-template<typename T>
-class ThreadSafeQueue {
-public:
-    void push(T item) {
-        {
-            std::lock_guard<std::mutex> lock(mutex_);
-            queue_.push(std::move(item));
-        }
-        cv_.notify_one();
-    }
-
-    bool wait_pop(T& item) {
-        std::unique_lock<std::mutex> lock(mutex_);
-
-        cv_.wait(lock, [&] {
-            return shutdown_ || !queue_.empty();
-        });
-
-        if (!queue_.empty()) {
-            item = std::move(queue_.front());
-            queue_.pop();
-            return true;
-        }
-
-        return false;
-    }
-
-    void signal_shutdown() {
-        {
-            std::lock_guard<std::mutex> lock(mutex_);
-            shutdown_ = true;
-        }
-        cv_.notify_all();
-    }
-
-private:
-    std::queue<T> queue_;
-    std::mutex mutex_;
-    std::condition_variable cv_;
-    bool shutdown_ = false;
-};
-
-// ============================================================
-// Existing Solver Implementations
-// ============================================================
-
-int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
-                    const MatrixTask& task, cudaStream_t stream) {
-    int n = (int)task.data->row_ptr.size() - 1;
-    float alpha = 1.0f, beta = 0.0f, r0 = 0.0f, r1 = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
-    float tolerance = 1e-5f;
-    int max_iters = 128000;
-
-    float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b;
-    int *d_row_ptr, *d_col_ind;
-
-    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
-
-    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
-
-    CHECK_CUBLAS(cublasSetStream(cublas, stream));
-    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
-
-    cusparseSpMatDescr_t matA;
-    cusparseDnVecDescr_t vecX, vecP, vecAp;
-
-    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val,
-                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
-                                     CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
-    CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
-    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
-    CHECK_CUSPARSE(cusparseCreateDnVec(&vecAp, n, d_Ap, CUDA_R_32F));
-
-    size_t bufferSize = 0;
-    void* d_buffer = nullptr;
-    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                                           &alpha, matA, vecX, &beta, vecAp,
-                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
-    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
-
-    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
-    CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_p, 1));
-    CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
-
-    int k = 0;
-    while (k < max_iters) {
-        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                                    &alpha, matA, vecP, &beta, vecAp,
-                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
-
-        float pAp;
-        CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
-        a = r1 / pAp;
-
-        CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
-
-        na = -a;
-        CHECK_CUBLAS(cublasSaxpy(cublas, n, &na, d_Ap, 1, d_r, 1));
-
-        r0 = r1;
-        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
-
-        if (std::sqrt(r1) < tolerance) break;
-
-        b = r1 / r0;
-        CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
-        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_r, 1, d_p, 1));
-        k++;
-    }
+#include "native_utils.hpp"
 
-    CHECK_CUSPARSE(cusparseDestroySpMat(matA));
-    CHECK_CUSPARSE(cusparseDestroyDnVec(vecX));
-    CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
-    CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
-
-    CHECK_CUDA(cudaFreeAsync(d_val, stream));
-    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
-    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
-    CHECK_CUDA(cudaFreeAsync(d_x, stream));
-    CHECK_CUDA(cudaFreeAsync(d_r, stream));
-    CHECK_CUDA(cudaFreeAsync(d_p, stream));
-    CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
-    CHECK_CUDA(cudaFreeAsync(d_b, stream));
-    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
-
-    return k;
-}
-
-// ============================================================
-// Producer
-// ============================================================
-
-void producer(
-    ThreadSafeQueue<MatrixTask>& queue,
-    const std::vector<MatrixTask>& matrix_pool)
-{
+void producer(ThreadSafeQueue<MatrixTask>& queue, const std::vector<MatrixTask>& matrix_pool) {
     for (const auto& task : matrix_pool) {
         queue.push(task);
     }
     queue.signal_shutdown();
 }
 
-// ============================================================
-// Worker
-// ============================================================
-
-void gpu_stream_worker(
-    int device_id,
-    int worker_id,
-    ThreadSafeQueue<MatrixTask>& queue)
-{
-    CHECK_CUDA(cudaSetDevice(device_id));
-
-    cudaStream_t stream;
-    cublasHandle_t cublas;
-    cusparseHandle_t cusparse;
-
-    CHECK_CUDA(
-        cudaStreamCreateWithFlags(
-            &stream,
-            cudaStreamNonBlocking));
-
-    CHECK_CUBLAS(
-        cublasCreate(&cublas));
-
-    CHECK_CUSPARSE(
-        cusparseCreate(&cusparse));
-
-    MatrixTask task;
-
-    while (queue.wait_pop(task)) {
-        std::cout << "[SUPE] Starting solver for: " << task.path << std::endl;
-        auto start_task =
-            std::chrono::steady_clock::now();
-        int iterations = 0;
-        if (task.type == CGS_SOLVER) {
-            iterations = solve_cg_async(
-                cublas,
-                cusparse,
-                task,
-                stream);
-        } else {
-            throw std::runtime_error(
-                "Unsupported solver type");
-        }
-        CHECK_CUDA(cudaStreamSynchronize(stream));
-        auto end_task =
-            std::chrono::steady_clock::now();
-        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
-                          end_task - start_task).count();
-        std::cout << "[DONE] " << task.path << ": Converged in " << iterations 
-                  << " iterations (" << duration << " ms)." << std::endl;
-    }
-
-    CHECK_CUBLAS(cublasDestroy(cublas));
-    CHECK_CUSPARSE(cusparseDestroy(cusparse));
-    CHECK_CUDA(cudaStreamDestroy(stream));
-}
-
-// ============================================================
-// Main
-// ============================================================
-
 int main(int argc, char** argv)
 {
-    int num_streams = 4;
-
-    if (argc > 1)
-        num_streams = std::max(1, std::atoi(argv[1]));
+    constexpr uint32_t WORKLOAD_SEED = 42;
+    int num_streams = 1;
+    if (argc > 1) num_streams = std::max(1, std::atoi(argv[1]));
 
     std::cout << "[INFO] Loading matrices...\n";
-
-    std::vector<MatrixTask> matrix_pool =
-        scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
 
     if (matrix_pool.empty()) {
         std::cerr << "No matrices found.\n";
@@ -293,74 +29,28 @@ int main(int argc, char** argv)
     }
 
     int num_gpus = 0;
-
-    CHECK_CUDA(
-        cudaGetDeviceCount(&num_gpus));
-
-    if (num_gpus == 0) {
-        std::cerr << "No CUDA devices found.\n";
-        return 1;
-    }
-
-    std::cout
-        << "[INFO] Found "
-        << num_gpus
-        << " GPUs\n";
-
-    std::cout
-        << "[INFO] Matrix pool size: "
-        << matrix_pool.size()
-        << "\n";
-
-    std::cout
-        << "[INFO] Streams/GPU: "
-        << num_streams
-        << "\n";
+    CHECK_CUDA(cudaGetDeviceCount(&num_gpus));
+    
+    std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
+    std::cout << "[INFO] Matrix pool size: " << matrix_pool.size() << "\n";
+    std::cout << "[INFO] Streams/GPU: " << num_streams << "\n";
 
     ThreadSafeQueue<MatrixTask> work_queue;
-
-    auto benchmark_start =
-        std::chrono::steady_clock::now();
-
-    std::thread producer_thread(
-        producer,
-        std::ref(work_queue),
-        std::cref(matrix_pool));
-
+    auto benchmark_start = std::chrono::steady_clock::now();
+    std::thread producer_thread(producer, std::ref(work_queue), std::cref(matrix_pool));
+    
     std::vector<std::thread> workers;
-
-    workers.reserve(
-        num_gpus * num_streams);
-
     for (int gpu = 0; gpu < num_gpus; ++gpu) {
-
-        for (int stream = 0;
-             stream < num_streams;
-             ++stream)
-        {
-            int worker_id =
-                gpu * num_streams + stream;
-
-            workers.emplace_back(
-                gpu_stream_worker,
-                gpu,
-                worker_id,
-                std::ref(work_queue));
+        for (int stream = 0; stream < num_streams; ++stream) {
+            workers.emplace_back(gpu_stream_worker, gpu, gpu * num_streams + stream, std::ref(work_queue));
         }
     }
 
     producer_thread.join();
+    for (auto& worker : workers) worker.join();
 
-    for (auto& worker : workers) {
-        worker.join();
-    }
-
-    auto benchmark_end =
-        std::chrono::steady_clock::now();
-
-    std::chrono::duration<double>
-        total_time =
-            benchmark_end - benchmark_start;
+    auto benchmark_end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> total_time = benchmark_end - benchmark_start;
 
     std::cout << "All tasks in the pool have been processed." << std::endl;
     std::cout << "\n";
@@ -375,4 +65,4 @@ int main(int argc, char** argv)
     std::cout << "=====================================\n";
 
     return 0;
-}
\ No newline at end of file
+}
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
new file mode 100644
index 0000000000..0809d6c407
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
@@ -0,0 +1,45 @@
+#include <iostream>
+#include <vector>
+#include <string>
+#include <chrono>
+#include <thread>
+#include <filesystem>
+#include <algorithm>
+#include "native_utils.hpp"
+
+void producer(ThreadSafeQueue<MatrixTask>& queue, std::vector<MatrixTask> matrix_pool) {
+    // Order tasks from lowest NNZ to highest NNZ
+    std::sort(matrix_pool.begin(), matrix_pool.end(), [](const MatrixTask& a, const MatrixTask& b) {
+        return a.data->nnz < b.data->nnz;
+    });
+
+    for (const auto& task : matrix_pool) {
+        queue.push(task);
+    }
+    queue.signal_shutdown();
+}
+
+int main(int argc, char** argv) {
+    int num_streams = 1;
+    if (argc > 1) num_streams = std::max(1, std::atoi(argv[1]));
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+    if (matrix_pool.empty()) return 1;
+
+    int num_gpus = 0;
+    CHECK_CUDA(cudaGetDeviceCount(&num_gpus));
+    ThreadSafeQueue<MatrixTask> work_queue;
+    auto benchmark_start = std::chrono::steady_clock::now();
+    std::thread producer_thread(producer, std::ref(work_queue), matrix_pool);
+    std::vector<std::thread> workers;
+    for (int gpu = 0; gpu < num_gpus; ++gpu) {
+        for (int stream = 0; stream < num_streams; ++stream) {
+            workers.emplace_back(gpu_stream_worker, gpu, gpu * num_streams + stream, std::ref(work_queue));
+        }
+    }
+    producer_thread.join();
+    for (auto& worker : workers) worker.join();
+    auto benchmark_end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> total_time = benchmark_end - benchmark_start;
+    std::cout << "All tasks processed in " << total_time.count() << " s\n";
+    return 0;
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cpp
new file mode 100644
index 0000000000..1429ba6d8a
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cpp
@@ -0,0 +1,112 @@
+#include "native_utils.hpp"
+#include <cmath>
+#include <chrono>
+
+int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
+                    const MatrixTask& task, cudaStream_t stream) {
+    int n = (int)task.data->row_ptr.size() - 1;
+    float alpha = 1.0f, beta = 0.0f, r1 = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
+    float tolerance = 1e-5f;
+    int max_iters = 128000;
+
+    float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b;
+    int *d_row_ptr, *d_col_ind;
+
+    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
+
+    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
+
+    CHECK_CUBLAS(cublasSetStream(cublas, stream));
+    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
+
+    cusparseSpMatDescr_t matA;
+    cusparseDnVecDescr_t vecX, vecP, vecAp;
+
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val,
+                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                     CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecAp, n, d_Ap, CUDA_R_32F));
+
+    size_t bufferSize = 0;
+    void* d_buffer = nullptr;
+    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, vecP, &beta, vecAp,
+                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
+
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_p, 1));
+    CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
+
+    int k = 0;
+    while (k < max_iters) {
+        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                    &alpha, matA, vecP, &beta, vecAp,
+                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
+        float pAp, r0;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
+        a = r1 / pAp;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
+        na = -a;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &na, d_Ap, 1, d_r, 1));
+        r0 = r1;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
+        if (std::sqrt(r1) < tolerance) break;
+        b = r1 / r0;
+        CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_r, 1, d_p, 1));
+        k++;
+    }
+
+    CHECK_CUSPARSE(cusparseDestroySpMat(matA));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecX));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
+    CHECK_CUDA(cudaFreeAsync(d_val, stream));
+    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
+    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
+    CHECK_CUDA(cudaFreeAsync(d_x, stream));
+    CHECK_CUDA(cudaFreeAsync(d_r, stream));
+    CHECK_CUDA(cudaFreeAsync(d_p, stream));
+    CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
+    CHECK_CUDA(cudaFreeAsync(d_b, stream));
+    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
+    return k;
+}
+
+void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue) {
+    CHECK_CUDA(cudaSetDevice(device_id));
+    cudaStream_t stream;
+    cublasHandle_t cublas;
+    cusparseHandle_t cusparse;
+    CHECK_CUDA(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
+    CHECK_CUBLAS(cublasCreate(&cublas));
+    CHECK_CUSPARSE(cusparseCreate(&cusparse));
+
+    MatrixTask task;
+    while (queue.wait_pop(task)) {
+        std::cout << "[WORKER " << worker_id << "] Starting: " << task.path << " (NNZ: " << task.data->nnz << ")" << std::endl;
+        auto start_task = std::chrono::steady_clock::now();
+        int iterations = solve_cg_async(cublas, cusparse, task, stream);
+        CHECK_CUDA(cudaStreamSynchronize(stream));
+        auto end_task = std::chrono::steady_clock::now();
+        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_task - start_task).count();
+        std::cout << "[WORKER " << worker_id << "] Done: " << task.path << " (" << iterations << " iters, " << duration << " ms)." << std::endl;
+    }
+    CHECK_CUBLAS(cublasDestroy(cublas));
+    CHECK_CUSPARSE(cusparseDestroy(cusparse));
+    CHECK_CUDA(cudaStreamDestroy(stream));
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.hpp
new file mode 100644
index 0000000000..3b9dcacb26
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.hpp
@@ -0,0 +1,91 @@
+#pragma once
+
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+#include <cusparse.h>
+#include <iostream>
+#include <queue>
+#include <mutex>
+#include <condition_variable>
+#include "sparse_utils.hpp"
+
+// ============================================================
+// Error Checking Macros
+// ============================================================
+
+#define CHECK_CUDA(call)                                                   \
+    do {                                                                   \
+        cudaError_t status = call;                                         \
+        if (status != cudaSuccess) {                                       \
+            std::cerr << "CUDA Error: " << cudaGetErrorString(status)      \
+                      << " at " << __FILE__ << ":" << __LINE__ << std::endl; \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+#define CHECK_CUBLAS(call)                                                 \
+    do {                                                                   \
+        cublasStatus_t status = call;                                      \
+        if (status != CUBLAS_STATUS_SUCCESS) {                             \
+            std::cerr << "cuBLAS Error at "                               \
+                      << __FILE__ << ":" << __LINE__ << std::endl;         \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+#define CHECK_CUSPARSE(call)                                               \
+    do {                                                                   \
+        cusparseStatus_t status = call;                                    \
+        if (status != CUSPARSE_STATUS_SUCCESS) {                           \
+            std::cerr << "cuSPARSE Error at "                             \
+                      << __FILE__ << ":" << __LINE__ << std::endl;         \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+// ============================================================
+// Thread Safe Queue
+// ============================================================
+
+template<typename T>
+class ThreadSafeQueue {
+public:
+    void push(T item) {
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            queue_.push(std::move(item));
+        }
+        cv_.notify_one();
+    }
+
+    bool wait_pop(T& item) {
+        std::unique_lock<std::mutex> lock(mutex_);
+        cv_.wait(lock, [&] {
+            return shutdown_ || !queue_.empty();
+        });
+
+        if (!queue_.empty()) {
+            item = std::move(queue_.front());
+            queue_.pop();
+            return true;
+        }
+        return false;
+    }
+
+    void signal_shutdown() {
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            shutdown_ = true;
+        }
+        cv_.notify_all();
+    }
+
+private:
+    std::queue<T> queue_;
+    std::mutex mutex_;
+    std::condition_variable cv_;
+    bool shutdown_ = false;
+};
+
+int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream);
+void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue);
\ No newline at end of file

From 6e9ef882273e45fdb52475408e0273ddae455fe5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 5 Jun 2026 10:27:48 -0600
Subject: [PATCH 0850/1000] fixed compiler errors

---
 .../CMakeLists.txt                            |   7 +-
 .../native_utils.cpp                          | 112 ---------
 .../native_utils.cu                           | 236 ++++++++++++++++++
 .../native_utils.hpp                          |   1 +
 4 files changed, 241 insertions(+), 115 deletions(-)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
index 85575dbdd2..a492765224 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
@@ -14,7 +14,8 @@ set(CAF_BUILD "${CAF_SRC}/build")
 set(CMAKE_CXX_COMPILER "/usr/bin/g++")
 set(CMAKE_C_COMPILER "/usr/bin/gcc")
 
-project(CUDA_ACTORS)
+# Enable CUDA as a first-class language for the project
+project(CUDA_ACTORS LANGUAGES CXX CUDA)
 
 find_package(CUDA REQUIRED)
 find_package(CUDAToolkit REQUIRED)
@@ -87,7 +88,7 @@ target_link_libraries(test
 find_package(Threads REQUIRED)
 
 # 6) Declare the native benchmark executable (raw CUDA/cuBLAS/cuSPARSE)
-add_executable(workload-native main.native.cpp sparse_utils.cpp native_utils.cpp)
+add_executable(workload-native main.native.cpp sparse_utils.cpp native_utils.cu)
 
 target_link_libraries(workload-native
   PRIVATE
@@ -98,7 +99,7 @@ target_link_libraries(workload-native
 )
 
 # 7) Declare the native sorted benchmark executable
-add_executable(workload-native-sorted main.native_sorted.cpp sparse_utils.cpp native_utils.cpp)
+add_executable(workload-native-sorted main.native_sorted.cpp sparse_utils.cpp native_utils.cu)
 
 target_link_libraries(workload-native-sorted
   PRIVATE
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cpp
index 1429ba6d8a..e69de29bb2 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cpp
@@ -1,112 +0,0 @@
-#include "native_utils.hpp"
-#include <cmath>
-#include <chrono>
-
-int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
-                    const MatrixTask& task, cudaStream_t stream) {
-    int n = (int)task.data->row_ptr.size() - 1;
-    float alpha = 1.0f, beta = 0.0f, r1 = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
-    float tolerance = 1e-5f;
-    int max_iters = 128000;
-
-    float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b;
-    int *d_row_ptr, *d_col_ind;
-
-    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
-
-    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
-
-    CHECK_CUBLAS(cublasSetStream(cublas, stream));
-    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
-
-    cusparseSpMatDescr_t matA;
-    cusparseDnVecDescr_t vecX, vecP, vecAp;
-
-    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val,
-                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
-                                     CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
-    CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
-    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
-    CHECK_CUSPARSE(cusparseCreateDnVec(&vecAp, n, d_Ap, CUDA_R_32F));
-
-    size_t bufferSize = 0;
-    void* d_buffer = nullptr;
-    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                                           &alpha, matA, vecP, &beta, vecAp,
-                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
-    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
-
-    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
-    CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_p, 1));
-    CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
-
-    int k = 0;
-    while (k < max_iters) {
-        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                                    &alpha, matA, vecP, &beta, vecAp,
-                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
-        float pAp, r0;
-        CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
-        a = r1 / pAp;
-        CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
-        na = -a;
-        CHECK_CUBLAS(cublasSaxpy(cublas, n, &na, d_Ap, 1, d_r, 1));
-        r0 = r1;
-        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
-        if (std::sqrt(r1) < tolerance) break;
-        b = r1 / r0;
-        CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
-        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_r, 1, d_p, 1));
-        k++;
-    }
-
-    CHECK_CUSPARSE(cusparseDestroySpMat(matA));
-    CHECK_CUSPARSE(cusparseDestroyDnVec(vecX));
-    CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
-    CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
-    CHECK_CUDA(cudaFreeAsync(d_val, stream));
-    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
-    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
-    CHECK_CUDA(cudaFreeAsync(d_x, stream));
-    CHECK_CUDA(cudaFreeAsync(d_r, stream));
-    CHECK_CUDA(cudaFreeAsync(d_p, stream));
-    CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
-    CHECK_CUDA(cudaFreeAsync(d_b, stream));
-    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
-    return k;
-}
-
-void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue) {
-    CHECK_CUDA(cudaSetDevice(device_id));
-    cudaStream_t stream;
-    cublasHandle_t cublas;
-    cusparseHandle_t cusparse;
-    CHECK_CUDA(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
-    CHECK_CUBLAS(cublasCreate(&cublas));
-    CHECK_CUSPARSE(cusparseCreate(&cusparse));
-
-    MatrixTask task;
-    while (queue.wait_pop(task)) {
-        std::cout << "[WORKER " << worker_id << "] Starting: " << task.path << " (NNZ: " << task.data->nnz << ")" << std::endl;
-        auto start_task = std::chrono::steady_clock::now();
-        int iterations = solve_cg_async(cublas, cusparse, task, stream);
-        CHECK_CUDA(cudaStreamSynchronize(stream));
-        auto end_task = std::chrono::steady_clock::now();
-        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_task - start_task).count();
-        std::cout << "[WORKER " << worker_id << "] Done: " << task.path << " (" << iterations << " iters, " << duration << " ms)." << std::endl;
-    }
-    CHECK_CUBLAS(cublasDestroy(cublas));
-    CHECK_CUSPARSE(cusparseDestroy(cusparse));
-    CHECK_CUDA(cudaStreamDestroy(stream));
-}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu
new file mode 100644
index 0000000000..b7c7d3a9d6
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu
@@ -0,0 +1,236 @@
+#include "native_utils.hpp"
+#include <cmath>
+#include <chrono>
+
+// ============================================================
+// PCG Kernels
+// ============================================================
+
+__global__ void extract_diag_inv_kernel(int n, const int* row_ptr, const int* col_ind, const float* values, float* d_inv) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        float diag = 1.0f;
+        for (int j = row_ptr[i]; j < row_ptr[i + 1]; j++) {
+            if (col_ind[j] == i) {
+                diag = values[j];
+                break;
+            }
+        }
+        d_inv[i] = (fabsf(diag) > 1e-20f) ? 1.0f / diag : 1.0f;
+    }
+}
+
+__global__ void elementwise_mul_kernel(int n, const float* a, const float* b, float* c) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) c[i] = a[i] * b[i];
+}
+
+int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
+                           const MatrixTask& task, cudaStream_t stream) {
+    int n = (int)task.data->row_ptr.size() - 1;
+    float alpha = 1.0f, beta = 0.0f, rho = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
+    float tolerance = 1e-5f;
+    int max_iters = 16000;
+
+    float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b, *d_z, *d_Dinv;
+    int *d_row_ptr, *d_col_ind;
+
+    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_z, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_Dinv, n * sizeof(float), stream));
+
+    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
+
+    CHECK_CUBLAS(cublasSetStream(cublas, stream));
+    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
+
+    // Extract Diagonal Inverse
+    int threads = 256;
+    int blocks = (n + threads - 1) / threads;
+    extract_diag_inv_kernel<<<blocks, threads, 0, stream>>>(n, d_row_ptr, d_col_ind, d_val, d_Dinv);
+
+    cusparseSpMatDescr_t matA;
+    cusparseDnVecDescr_t vecP, vecAp;
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val,
+                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                     CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecAp, n, d_Ap, CUDA_R_32F));
+
+    size_t bufferSize = 0;
+    void* d_buffer = nullptr;
+    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, vecP, &beta, vecAp,
+                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
+
+    // r = b (assuming x=0), z = M^-1 * r, p = z
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
+    elementwise_mul_kernel<<<blocks, threads, 0, stream>>>(n, d_Dinv, d_r, d_z);
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_z, 1, d_p, 1));
+    CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_z, 1, &rho));
+
+    int k = 0;
+    float r_norm_sq = 0.0f;
+    while (k < max_iters) {
+        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                    &alpha, matA, vecP, &beta, vecAp,
+                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
+        float pAp, old_rho;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
+        a = rho / pAp;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
+        na = -a;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &na, d_Ap, 1, d_r, 1));
+        
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r_norm_sq));
+        if (std::sqrt(r_norm_sq) < tolerance) break;
+
+        elementwise_mul_kernel<<<blocks, threads, 0, stream>>>(n, d_Dinv, d_r, d_z);
+        old_rho = rho;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_z, 1, &rho));
+        b = rho / old_rho;
+        
+        // p = z + beta * p
+        CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_z, 1, d_p, 1));
+        k++;
+    }
+
+    CHECK_CUSPARSE(cusparseDestroySpMat(matA));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
+    CHECK_CUDA(cudaFreeAsync(d_val, stream));
+    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
+    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
+    CHECK_CUDA(cudaFreeAsync(d_x, stream));
+    CHECK_CUDA(cudaFreeAsync(d_r, stream));
+    CHECK_CUDA(cudaFreeAsync(d_p, stream));
+    CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
+    CHECK_CUDA(cudaFreeAsync(d_b, stream));
+    CHECK_CUDA(cudaFreeAsync(d_z, stream));
+    CHECK_CUDA(cudaFreeAsync(d_Dinv, stream));
+    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
+    return k;
+}
+
+int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
+                    const MatrixTask& task, cudaStream_t stream) {
+    int n = (int)task.data->row_ptr.size() - 1;
+    float alpha = 1.0f, beta = 0.0f, r1 = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
+    float tolerance = 1e-5f;
+    int max_iters = 16000;
+
+    float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b;
+    int *d_row_ptr, *d_col_ind;
+
+    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
+
+    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
+
+    CHECK_CUBLAS(cublasSetStream(cublas, stream));
+    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
+
+    cusparseSpMatDescr_t matA;
+    cusparseDnVecDescr_t vecX, vecP, vecAp;
+
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val,
+                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                     CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecAp, n, d_Ap, CUDA_R_32F));
+
+    size_t bufferSize = 0;
+    void* d_buffer = nullptr;
+    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, vecP, &beta, vecAp,
+                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
+
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_p, 1));
+    CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
+
+    int k = 0;
+    while (k < max_iters) {
+        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                    &alpha, matA, vecP, &beta, vecAp,
+                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
+        float pAp, r0;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
+        a = r1 / pAp;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
+        na = -a;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &na, d_Ap, 1, d_r, 1));
+        r0 = r1;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
+        if (std::sqrt(r1) < tolerance) break;
+        b = r1 / r0;
+        CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_r, 1, d_p, 1));
+        k++;
+    }
+
+    CHECK_CUSPARSE(cusparseDestroySpMat(matA));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecX));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
+    CHECK_CUDA(cudaFreeAsync(d_val, stream));
+    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
+    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
+    CHECK_CUDA(cudaFreeAsync(d_x, stream));
+    CHECK_CUDA(cudaFreeAsync(d_r, stream));
+    CHECK_CUDA(cudaFreeAsync(d_p, stream));
+    CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
+    CHECK_CUDA(cudaFreeAsync(d_b, stream));
+    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
+    return k;
+}
+
+void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue) {
+    CHECK_CUDA(cudaSetDevice(device_id));
+    cudaStream_t stream;
+    cublasHandle_t cublas;
+    cusparseHandle_t cusparse;
+    CHECK_CUDA(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
+    CHECK_CUBLAS(cublasCreate(&cublas));
+    CHECK_CUSPARSE(cusparseCreate(&cusparse));
+
+    MatrixTask task;
+    while (queue.wait_pop(task)) {
+        std::cout << "[WORKER " << worker_id << "] Starting: " << task.path << " (NNZ: " << task.data->nnz << ")" << std::endl;
+        auto start_task = std::chrono::steady_clock::now();
+        int iterations = solve_pcg_jacobi_async(cublas, cusparse, task, stream);
+        // int iterations = solve_cg_async(cublas, cusparse, task, stream);
+        CHECK_CUDA(cudaStreamSynchronize(stream));
+        auto end_task = std::chrono::steady_clock::now();
+        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_task - start_task).count();
+        std::cout << "[WORKER " << worker_id << "] Done: " << task.path << " (" << iterations << " iters, " << duration << " ms)." << std::endl;
+    }
+    CHECK_CUBLAS(cublasDestroy(cublas));
+    CHECK_CUSPARSE(cusparseDestroy(cusparse));
+    CHECK_CUDA(cudaStreamDestroy(stream));
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.hpp
index 3b9dcacb26..0e70735372 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.hpp
@@ -88,4 +88,5 @@ class ThreadSafeQueue {
 };
 
 int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream);
+int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream);
 void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue);
\ No newline at end of file

From 89ce7057734416ddfe09f322ec4407c6953023d0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 5 Jun 2026 11:29:00 -0600
Subject: [PATCH 0851/1000] updated formats

---
 .../main.native.cpp                           |  2 +-
 .../main.native_sorted.cpp                    | 30 +++++++++++++++++--
 .../main.test.cpp                             | 19 ++++++------
 .../native_utils.cu                           |  4 +--
 4 files changed, 40 insertions(+), 15 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
index ff4b0fa5f2..1261dd7635 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
@@ -17,7 +17,7 @@ void producer(ThreadSafeQueue<MatrixTask>& queue, const std::vector<MatrixTask>&
 int main(int argc, char** argv)
 {
     constexpr uint32_t WORKLOAD_SEED = 42;
-    int num_streams = 1;
+    int num_streams = 4;
     if (argc > 1) num_streams = std::max(1, std::atoi(argv[1]));
 
     std::cout << "[INFO] Loading matrices...\n";
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
index 0809d6c407..a7acec96fb 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
@@ -20,13 +20,25 @@ void producer(ThreadSafeQueue<MatrixTask>& queue, std::vector<MatrixTask> matrix
 }
 
 int main(int argc, char** argv) {
-    int num_streams = 1;
+    constexpr uint32_t WORKLOAD_SEED = 42;
+    int num_streams = 4;
     if (argc > 1) num_streams = std::max(1, std::atoi(argv[1]));
+
+    std::cout << "[INFO] Loading matrices...\n";
     std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
-    if (matrix_pool.empty()) return 1;
+
+    if (matrix_pool.empty()) {
+        std::cerr << "No matrices found.\n";
+        return 1;
+    }
 
     int num_gpus = 0;
     CHECK_CUDA(cudaGetDeviceCount(&num_gpus));
+
+    std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
+    std::cout << "[INFO] Matrix pool size: " << matrix_pool.size() << "\n";
+    std::cout << "[INFO] Streams/GPU: " << num_streams << "\n";
+
     ThreadSafeQueue<MatrixTask> work_queue;
     auto benchmark_start = std::chrono::steady_clock::now();
     std::thread producer_thread(producer, std::ref(work_queue), matrix_pool);
@@ -40,6 +52,18 @@ int main(int argc, char** argv) {
     for (auto& worker : workers) worker.join();
     auto benchmark_end = std::chrono::steady_clock::now();
     std::chrono::duration<double> total_time = benchmark_end - benchmark_start;
-    std::cout << "All tasks processed in " << total_time.count() << " s\n";
+
+    std::cout << "All tasks in the pool have been processed." << std::endl;
+    std::cout << "\n";
+    std::cout << "=====================================\n";
+    std::cout << "IRREGULAR WORKLOAD BENCHMARK (NATIVE - SORTED)\n";
+    std::cout << "=====================================\n";
+    std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
+    std::cout << "GPUs:               " << num_gpus << "\n";
+    std::cout << "Streams per GPU:    " << num_streams << "\n";
+    std::cout << "Worker Threads:     " << num_gpus * num_streams << "\n";
+    std::cout << "Total Runtime:      " << total_time.count() << " s\n";
+    std::cout << "=====================================\n";
+
     return 0;
 }
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 5362ee1b5a..5118850b41 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -265,15 +265,15 @@ struct supervisor_state {
     std::unordered_map<std::string, std::chrono::steady_clock::time_point> start_times;
     std::unordered_map<std::string, int> task_streams;
     std::deque<int> available_streams;
-    int max_active = 4; // Admission control limit to be mindful of GPU memory. If Illegal memory access that means two or more actors are using the same stream
+    int max_active = 1; // Admission control limit to be mindful of GPU memory. If Illegal memory access that means two or more actors are using the same stream
     int num_iterations = 50;
     int device = 0;
 };
 
-behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks) {
+behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks, int initial_max_active) {
     auto& st = self->state();
-    for (auto& t : tasks)
-        st.queue.push_back(std::move(t));
+    st.queue.insert(st.queue.end(), std::make_move_iterator(tasks.begin()), std::make_move_iterator(tasks.end()));
+    st.max_active = initial_max_active;
 
     // Initialize the pool with 32 distinct stream IDs
     for (int i = 0; i < 32; ++i)
@@ -300,7 +300,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                                     create_in_out_arg(task.data->x_guess),
                                     (int)task.data->row_ptr.size() - 1,
                                     (int)task.data->values.size(),
-                                    1e-5f, 128000, s.device, stream_id, actor_cast<actor>(self));
+                                    1e-5f, 16000, s.device, stream_id, actor_cast<actor>(self));
 
             s.start_times[path] = std::chrono::steady_clock::now();
             s.active_solvers.push_back(solver);
@@ -368,10 +368,10 @@ void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
     std::cout << "loading\n";
     {
-         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+         auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/unsymmetric", CGS_SOLVER);
          //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric", CGS_SOLVER);
-         auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
 
         std::cout << "loaded\n";
         if (tasks_vec.empty()) {
@@ -386,8 +386,9 @@ void caf_main(actor_system& sys) {
         }
 
         auto benchmark_start = std::chrono::steady_clock::now();
+        int admission_control_limit = 4; // The desired admission control limit
 
-        sys.spawn(supervisor_actor, std::move(tasks_vec));
+        sys.spawn(supervisor_actor, std::move(tasks_vec), admission_control_limit);
         sys.await_all_actors_done();
 
         auto benchmark_end = std::chrono::steady_clock::now();
@@ -400,7 +401,7 @@ void caf_main(actor_system& sys) {
         std::cout << "=====================================\n";
         std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
         std::cout << "GPUs:               " << num_gpus << "\n";
-        std::cout << "Admission Control:  " << 4 << "\n"; 
+        std::cout << "Admission Control:  " << admission_control_limit << "\n";
         std::cout << "Total Runtime:      " << total_time.count() << " s\n";
         std::cout << "=====================================\n";
     }
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu
index b7c7d3a9d6..bf5a18d411 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu
@@ -223,8 +223,8 @@ void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>
     while (queue.wait_pop(task)) {
         std::cout << "[WORKER " << worker_id << "] Starting: " << task.path << " (NNZ: " << task.data->nnz << ")" << std::endl;
         auto start_task = std::chrono::steady_clock::now();
-        int iterations = solve_pcg_jacobi_async(cublas, cusparse, task, stream);
-        // int iterations = solve_cg_async(cublas, cusparse, task, stream);
+        //int iterations = solve_pcg_jacobi_async(cublas, cusparse, task, stream);
+         int iterations = solve_cg_async(cublas, cusparse, task, stream);
         CHECK_CUDA(cudaStreamSynchronize(stream));
         auto end_task = std::chrono::steady_clock::now();
         auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_task - start_task).count();

From 1afc0f3f4947aefb0dacabc346dc92cd26836c45 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 5 Jun 2026 12:10:58 -0600
Subject: [PATCH 0852/1000] updated formating

---
 .../main.test.cpp                             | 35 ++++++++++---------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 5118850b41..d00a5b08c6 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -19,6 +19,7 @@ using namespace caf::cuda;
 namespace fs = std::filesystem;
 
 constexpr uint32_t WORKLOAD_SEED = 42;
+constexpr int MAX_ITERATIONS = 16000;
 
 
 
@@ -260,14 +261,14 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
 // ---------------------------- SUPERVISOR ACTOR ----------------------------
 
 struct supervisor_state {
-    std::deque<MatrixTask> queue;
-    std::vector<caf::actor> active_solvers;
-    std::unordered_map<std::string, std::chrono::steady_clock::time_point> start_times;
-    std::unordered_map<std::string, int> task_streams;
-    std::deque<int> available_streams;
-    int max_active = 1; // Admission control limit to be mindful of GPU memory. If Illegal memory access that means two or more actors are using the same stream
-    int num_iterations = 50;
-    int device = 0;
+  std::deque<MatrixTask> queue;
+  std::vector<caf::actor> active_solvers;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> start_times;
+  std::unordered_map<std::string, int> task_streams;
+  std::deque<int> available_streams;
+  int max_active = 1; // Admission control limit to be mindful of GPU memory.
+  int num_iterations = MAX_ITERATIONS / 2;
+  int device = 0;
 };
 
 behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks, int initial_max_active) {
@@ -300,7 +301,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                                     create_in_out_arg(task.data->x_guess),
                                     (int)task.data->row_ptr.size() - 1,
                                     (int)task.data->values.size(),
-                                    1e-5f, 16000, s.device, stream_id, actor_cast<actor>(self));
+                                    1e-5f, MAX_ITERATIONS, s.device, stream_id, actor_cast<actor>(self));
 
             s.start_times[path] = std::chrono::steady_clock::now();
             s.active_solvers.push_back(solver);
@@ -366,14 +367,17 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
 
 void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
-    std::cout << "loading\n";
+    std::cout << "[INFO] Loading matrices...\n";
     {
          auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/unsymmetric", CGS_SOLVER);
          //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric", CGS_SOLVER);
          //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
 
-        std::cout << "loaded\n";
+        int num_gpus = manager::get().get_num_devices();
+        std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
+        std::cout << "[INFO] Matrix pool size: " << tasks_vec.size() << "\n";
+
         if (tasks_vec.empty()) {
             std::cerr << "No matrices found. Running dummy test task." << std::endl;
             auto data = std::make_shared<MatrixData>();
@@ -393,16 +397,15 @@ void caf_main(actor_system& sys) {
 
         auto benchmark_end = std::chrono::steady_clock::now();
         std::chrono::duration<double> total_time = benchmark_end - benchmark_start;
-        int num_gpus = manager::get().get_num_devices();
 
         std::cout << "\n";
         std::cout << "=====================================\n";
         std::cout << "IRREGULAR WORKLOAD BENCHMARK (CAF)\n";
         std::cout << "=====================================\n";
-        std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
-        std::cout << "GPUs:               " << num_gpus << "\n";
-        std::cout << "Admission Control:  " << admission_control_limit << "\n";
-        std::cout << "Total Runtime:      " << total_time.count() << " s\n";
+        std::cout << "Seed:                " << WORKLOAD_SEED << "\n";
+        std::cout << "GPUs:                " << num_gpus << "\n";
+        std::cout << "Admission Control:   " << admission_control_limit << "\n";
+        std::cout << "Total Runtime:       " << total_time.count() << " s\n";
         std::cout << "=====================================\n";
     }
     manager::shutdown();

From 93cf9b8b1f93f75675ad0336cea8e120e3b17e92 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 5 Jun 2026 12:37:00 -0600
Subject: [PATCH 0853/1000] turned this into round robin I guess

---
 .../main.test.cpp                             | 70 ++++++++++++++++---
 1 file changed, 59 insertions(+), 11 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index d00a5b08c6..9e1525c035 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -31,6 +31,7 @@ CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
     CAF_ADD_ATOM(workload_test, work_tick_atom)
     CAF_ADD_ATOM(workload_test, add_work_atom)
     CAF_ADD_ATOM(workload_test, steal_work_atom)
+    CAF_ADD_ATOM(workload_test, update_stream_atom)
     CAF_ADD_ATOM(workload_test, shutdown_atom)
     CAF_ADD_TYPE_ID(workload_test, (SolverType))
     CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
@@ -252,6 +253,9 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
           self->quit();
       });
     },
+    [=](update_stream_atom, int new_stream) {
+      self->state().stream_id = new_stream;
+    },
     [=](shutdown_atom) {
       self->quit();
     }
@@ -260,11 +264,19 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
 
 // ---------------------------- SUPERVISOR ACTOR ----------------------------
 
+struct suspended_task {
+  caf::actor solver;
+  std::string path;
+  int last_batch_size;
+};
+
 struct supervisor_state {
   std::deque<MatrixTask> queue;
+  std::deque<suspended_task> suspended_queue;
   std::vector<caf::actor> active_solvers;
   std::unordered_map<std::string, std::chrono::steady_clock::time_point> start_times;
   std::unordered_map<std::string, int> task_streams;
+  std::unordered_map<caf::actor, int> actor_batch_sizes;
   std::deque<int> available_streams;
   int max_active = 1; // Admission control limit to be mindful of GPU memory.
   int num_iterations = MAX_ITERATIONS / 2;
@@ -282,7 +294,8 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
 
     auto spawn_next = [self]() {
         auto& s = self->state();
-        while (s.active_solvers.size() < static_cast<size_t>(s.max_active) && !s.queue.empty() && !s.available_streams.empty()) {
+        while (s.active_solvers.size() < static_cast<size_t>(s.max_active) && !s.available_streams.empty()) {
+          if (!s.queue.empty()) {
             auto task = std::move(s.queue.front());
             s.queue.pop_front();
             std::string path = task.path;
@@ -306,7 +319,28 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
             s.start_times[path] = std::chrono::steady_clock::now();
             s.active_solvers.push_back(solver);
             s.task_streams[path] = stream_id;
+            s.actor_batch_sizes[solver] = s.num_iterations;
             self->mail(start_atom_v).send(solver);
+          } else if (!s.suspended_queue.empty()) {
+            auto suspended = std::move(s.suspended_queue.front());
+            s.suspended_queue.pop_front();
+
+            int stream_id = s.available_streams.front();
+            s.available_streams.pop_front();
+
+            int next_batch = std::max(1, suspended.last_batch_size / 2);
+            self->println("[SUPE] Resuming solver for: {} (Stream: {}, Batch: {})", 
+                          suspended.path, stream_id, next_batch);
+
+            self->mail(update_stream_atom_v, stream_id).send(suspended.solver);
+            self->mail(cg_next_step_atom_v, next_batch).send(suspended.solver);
+
+            s.active_solvers.push_back(suspended.solver);
+            s.task_streams[suspended.path] = stream_id;
+            s.actor_batch_sizes[suspended.solver] = next_batch;
+          } else {
+            break;
+          }
         }
     };
 
@@ -338,6 +372,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 if (it != s.active_solvers.end())
                     s.active_solvers.erase(it);
                 s.start_times.erase(task_name);
+                s.actor_batch_sizes.erase(solver);
 
                 // Reclaim the stream ID and put it back in the pool
                 auto stream_it = s.task_streams.find(task_name);
@@ -348,17 +383,30 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
 
                 spawn_next();
 
-                if (s.active_solvers.empty() && s.queue.empty()) {
-                    self->println("All tasks in the batch have been processed.");
+                if (s.active_solvers.empty() && s.queue.empty() && s.suspended_queue.empty()) {
+                    self->println("All tasks in the pool have been processed.");
                     //caf::cuda::manager::shutdown();
                     self->quit();
                 }
-            } else if (meta.iterations == 0 && meta.error_code == CG_SUCCESS) {
-                // Initialization report (not yet converged): trigger the first batch
+            } else if (meta.iterations == 0) {
+                // Just finished initialization: trigger the first iteration batch immediately.
                 self->mail(cg_next_step_atom_v, s.num_iterations).send(solver);
             } else {
-                // Progress report: order the next iteration batch
-                self->mail(cg_next_step_atom_v, s.num_iterations).send(solver);
+                // Not done: Suspend the actor to allow others to use the stream
+                auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
+                if (it != s.active_solvers.end())
+                    s.active_solvers.erase(it);
+
+                int stream_id = s.task_streams[task_name];
+                s.available_streams.push_back(stream_id);
+                s.task_streams.erase(task_name);
+
+                int last_batch = s.actor_batch_sizes[solver];
+                s.suspended_queue.push_back({solver, task_name, last_batch});
+
+                self->println("[SUPE] Suspending solver for: {} (Reclaimed Stream: {})", task_name, stream_id);
+
+                spawn_next();
             }
         }
        
@@ -402,10 +450,10 @@ void caf_main(actor_system& sys) {
         std::cout << "=====================================\n";
         std::cout << "IRREGULAR WORKLOAD BENCHMARK (CAF)\n";
         std::cout << "=====================================\n";
-        std::cout << "Seed:                " << WORKLOAD_SEED << "\n";
-        std::cout << "GPUs:                " << num_gpus << "\n";
-        std::cout << "Admission Control:   " << admission_control_limit << "\n";
-        std::cout << "Total Runtime:       " << total_time.count() << " s\n";
+        std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
+        std::cout << "GPUs:               " << num_gpus << "\n";
+        std::cout << "Admission Control:  " << admission_control_limit << "\n";
+        std::cout << "Total Runtime:      " << total_time.count() << " s\n";
         std::cout << "=====================================\n";
     }
     manager::shutdown();

From e302b2759f15b90b86fe8e8d3ef3583487943d65 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 5 Jun 2026 12:41:57 -0600
Subject: [PATCH 0854/1000] added task failed and succeded count here

---
 .../main.test.cpp                             | 47 +++++++++++--------
 1 file changed, 28 insertions(+), 19 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 9e1525c035..d6610a6301 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -281,12 +281,16 @@ struct supervisor_state {
   int max_active = 1; // Admission control limit to be mindful of GPU memory.
   int num_iterations = MAX_ITERATIONS / 2;
   int device = 0;
+  int tasks_succeeded = 0;
+  int tasks_failed = 0;
+  std::chrono::steady_clock::time_point benchmark_start;
 };
 
-behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks, int initial_max_active) {
+behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks, int initial_max_active, std::chrono::steady_clock::time_point start_time) {
     auto& st = self->state();
     st.queue.insert(st.queue.end(), std::make_move_iterator(tasks.begin()), std::make_move_iterator(tasks.end()));
     st.max_active = initial_max_active;
+    st.benchmark_start = start_time;
 
     // Initialize the pool with 32 distinct stream IDs
     for (int i = 0; i < 32; ++i)
@@ -303,7 +307,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
             int stream_id = s.available_streams.front();
             s.available_streams.pop_front();
 
-            self->println("[SUPE] Starting solver for: {} (Stream: {})", path, stream_id);
+            self->println("[INFO] Starting solver for: {} (Stream: {})", path, stream_id);
             auto solver = self->spawn(fault_tolerant_cg_actor<float>,
                                     path,
                                     task.data,
@@ -329,7 +333,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
             s.available_streams.pop_front();
 
             int next_batch = std::max(1, suspended.last_batch_size / 2);
-            self->println("[SUPE] Resuming solver for: {} (Stream: {}, Batch: {})", 
+            self->println("[INFO] Resuming solver for: {} (Stream: {}, Batch: {})", 
                           suspended.path, stream_id, next_batch);
 
             self->mail(update_stream_atom_v, stream_id).send(suspended.solver);
@@ -356,11 +360,13 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                                   end_time - s.start_times[task_name]).count();
 
                 if (meta.converged) {
+                    s.tasks_succeeded++;
                     if (meta.iterations == 0)
                         self->println("[DONE] {}: Initial guess satisfied tolerance ({} ms).", task_name, duration);
                     else
                         self->println("[DONE] {}: Converged in {} iterations ({} ms).", task_name, meta.iterations, duration);
                 } else {
+                    s.tasks_failed++;
                     self->println("[FAIL] {}: {} (after {} iterations, {} ms).", 
                                   task_name, 
                                   to_string(static_cast<cg_error_type>(meta.error_code)), 
@@ -385,7 +391,23 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
 
                 if (s.active_solvers.empty() && s.queue.empty() && s.suspended_queue.empty()) {
                     self->println("All tasks in the pool have been processed.");
-                    //caf::cuda::manager::shutdown();
+
+                    auto benchmark_end = std::chrono::steady_clock::now();
+                    std::chrono::duration<double> total_time = benchmark_end - s.benchmark_start;
+                    int num_gpus = manager::get().get_num_devices();
+
+                    std::cout << "\n";
+                    std::cout << "=====================================\n";
+                    std::cout << "IRREGULAR WORKLOAD BENCHMARK (CAF)\n";
+                    std::cout << "=====================================\n";
+                    std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
+                    std::cout << "GPUs:               " << num_gpus << "\n";
+                    std::cout << "Admission Control:  " << s.max_active << "\n";
+                    std::cout << "Tasks Succeeded:    " << s.tasks_succeeded << "\n";
+                    std::cout << "Tasks Failed:       " << s.tasks_failed << "\n";
+                    std::cout << "Total Runtime:      " << total_time.count() << " s\n";
+                    std::cout << "=====================================\n";
+
                     self->quit();
                 }
             } else if (meta.iterations == 0) {
@@ -404,7 +426,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 int last_batch = s.actor_batch_sizes[solver];
                 s.suspended_queue.push_back({solver, task_name, last_batch});
 
-                self->println("[SUPE] Suspending solver for: {} (Reclaimed Stream: {})", task_name, stream_id);
+                self->println("[INFO] Suspending solver for: {} (Reclaimed Stream: {})", task_name, stream_id);
 
                 spawn_next();
             }
@@ -440,21 +462,8 @@ void caf_main(actor_system& sys) {
         auto benchmark_start = std::chrono::steady_clock::now();
         int admission_control_limit = 4; // The desired admission control limit
 
-        sys.spawn(supervisor_actor, std::move(tasks_vec), admission_control_limit);
+        sys.spawn(supervisor_actor, std::move(tasks_vec), admission_control_limit, benchmark_start);
         sys.await_all_actors_done();
-
-        auto benchmark_end = std::chrono::steady_clock::now();
-        std::chrono::duration<double> total_time = benchmark_end - benchmark_start;
-
-        std::cout << "\n";
-        std::cout << "=====================================\n";
-        std::cout << "IRREGULAR WORKLOAD BENCHMARK (CAF)\n";
-        std::cout << "=====================================\n";
-        std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
-        std::cout << "GPUs:               " << num_gpus << "\n";
-        std::cout << "Admission Control:  " << admission_control_limit << "\n";
-        std::cout << "Total Runtime:      " << total_time.count() << " s\n";
-        std::cout << "=====================================\n";
     }
     manager::shutdown();
 }

From 1d321b04443142dde2ef5756c31bf0acbf10b3fa Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 5 Jun 2026 13:08:51 -0600
Subject: [PATCH 0855/1000] updated round robin iteration to fix bug where
 crossing streams lead to more failed jobs

---
 .../main.test.cpp                             | 45 +++++++++++--------
 1 file changed, 26 insertions(+), 19 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index d6610a6301..d1221c7de6 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -268,6 +268,7 @@ struct suspended_task {
   caf::actor solver;
   std::string path;
   int last_batch_size;
+  int stream_id;
 };
 
 struct supervisor_state {
@@ -325,25 +326,31 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
             s.task_streams[path] = stream_id;
             s.actor_batch_sizes[solver] = s.num_iterations;
             self->mail(start_atom_v).send(solver);
-          } else if (!s.suspended_queue.empty()) {
-            auto suspended = std::move(s.suspended_queue.front());
-            s.suspended_queue.pop_front();
-
-            int stream_id = s.available_streams.front();
-            s.available_streams.pop_front();
-
-            int next_batch = std::max(1, suspended.last_batch_size / 2);
-            self->println("[INFO] Resuming solver for: {} (Stream: {}, Batch: {})", 
-                          suspended.path, stream_id, next_batch);
-
-            self->mail(update_stream_atom_v, stream_id).send(suspended.solver);
-            self->mail(cg_next_step_atom_v, next_batch).send(suspended.solver);
-
-            s.active_solvers.push_back(suspended.solver);
-            s.task_streams[suspended.path] = stream_id;
-            s.actor_batch_sizes[suspended.solver] = next_batch;
           } else {
-            break;
+            bool resumed = false;
+            for (auto it = s.suspended_queue.begin(); it != s.suspended_queue.end(); ++it) {
+              auto stream_it = std::find(s.available_streams.begin(), s.available_streams.end(), it->stream_id);
+              if (stream_it != s.available_streams.end()) {
+                auto suspended = std::move(*it);
+                s.suspended_queue.erase(it);
+                int stream_id = suspended.stream_id;
+                s.available_streams.erase(stream_it);
+
+                int next_batch = std::max(1, suspended.last_batch_size / 2);
+                self->println("[INFO] Resuming solver for: {} (Stream: {}, Batch: {})", 
+                              suspended.path, stream_id, next_batch);
+
+                // Resume on the same stream ID. No update_stream_atom_v needed.
+                self->mail(cg_next_step_atom_v, next_batch).send(suspended.solver);
+
+                s.active_solvers.push_back(suspended.solver);
+                s.task_streams[suspended.path] = stream_id;
+                s.actor_batch_sizes[suspended.solver] = next_batch;
+                resumed = true;
+                break;
+              }
+            }
+            if (!resumed) break;
           }
         }
     };
@@ -424,7 +431,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 s.task_streams.erase(task_name);
 
                 int last_batch = s.actor_batch_sizes[solver];
-                s.suspended_queue.push_back({solver, task_name, last_batch});
+                s.suspended_queue.push_back({solver, task_name, last_batch, stream_id});
 
                 self->println("[INFO] Suspending solver for: {} (Reclaimed Stream: {})", task_name, stream_id);
 

From bca773d8eb8c826ad4c8901d84ac37014505a63a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 5 Jun 2026 13:40:21 -0600
Subject: [PATCH 0856/1000] updated counting

---
 .../main.native.cpp                            |  7 ++++++-
 .../main.native_sorted.cpp                     |  7 ++++++-
 .../main.test.cpp                              | 18 ++++++++++--------
 .../native_utils.cu                            | 15 ++++++++++++---
 4 files changed, 34 insertions(+), 13 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
index 1261dd7635..802fbd44f8 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
@@ -5,6 +5,7 @@
 #include <thread>
 #include <filesystem>
 #include <algorithm>
+#include <atomic>
 #include "native_utils.hpp"
 
 void producer(ThreadSafeQueue<MatrixTask>& queue, const std::vector<MatrixTask>& matrix_pool) {
@@ -35,6 +36,8 @@ int main(int argc, char** argv)
     std::cout << "[INFO] Matrix pool size: " << matrix_pool.size() << "\n";
     std::cout << "[INFO] Streams/GPU: " << num_streams << "\n";
 
+    std::atomic<int> tasks_succeeded{0};
+    std::atomic<int> tasks_failed{0};
     ThreadSafeQueue<MatrixTask> work_queue;
     auto benchmark_start = std::chrono::steady_clock::now();
     std::thread producer_thread(producer, std::ref(work_queue), std::cref(matrix_pool));
@@ -42,7 +45,7 @@ int main(int argc, char** argv)
     std::vector<std::thread> workers;
     for (int gpu = 0; gpu < num_gpus; ++gpu) {
         for (int stream = 0; stream < num_streams; ++stream) {
-            workers.emplace_back(gpu_stream_worker, gpu, gpu * num_streams + stream, std::ref(work_queue));
+            workers.emplace_back(gpu_stream_worker, gpu, gpu * num_streams + stream, std::ref(work_queue), std::ref(tasks_succeeded), std::ref(tasks_failed));
         }
     }
 
@@ -61,6 +64,8 @@ int main(int argc, char** argv)
     std::cout << "GPUs:               " << num_gpus << "\n";
     std::cout << "Streams per GPU:    " << num_streams << "\n";
     std::cout << "Worker Threads:     " << num_gpus * num_streams << "\n";
+    std::cout << "Tasks Succeeded:    " << tasks_succeeded.load() << "\n";
+    std::cout << "Tasks Failed:       " << tasks_failed.load() << "\n";
     std::cout << "Total Runtime:      " << total_time.count() << " s\n";
     std::cout << "=====================================\n";
 
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
index a7acec96fb..743e6117da 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
@@ -5,6 +5,7 @@
 #include <thread>
 #include <filesystem>
 #include <algorithm>
+#include <atomic>
 #include "native_utils.hpp"
 
 void producer(ThreadSafeQueue<MatrixTask>& queue, std::vector<MatrixTask> matrix_pool) {
@@ -39,13 +40,15 @@ int main(int argc, char** argv) {
     std::cout << "[INFO] Matrix pool size: " << matrix_pool.size() << "\n";
     std::cout << "[INFO] Streams/GPU: " << num_streams << "\n";
 
+    std::atomic<int> tasks_succeeded{0};
+    std::atomic<int> tasks_failed{0};
     ThreadSafeQueue<MatrixTask> work_queue;
     auto benchmark_start = std::chrono::steady_clock::now();
     std::thread producer_thread(producer, std::ref(work_queue), matrix_pool);
     std::vector<std::thread> workers;
     for (int gpu = 0; gpu < num_gpus; ++gpu) {
         for (int stream = 0; stream < num_streams; ++stream) {
-            workers.emplace_back(gpu_stream_worker, gpu, gpu * num_streams + stream, std::ref(work_queue));
+            workers.emplace_back(gpu_stream_worker, gpu, gpu * num_streams + stream, std::ref(work_queue), std::ref(tasks_succeeded), std::ref(tasks_failed));
         }
     }
     producer_thread.join();
@@ -62,6 +65,8 @@ int main(int argc, char** argv) {
     std::cout << "GPUs:               " << num_gpus << "\n";
     std::cout << "Streams per GPU:    " << num_streams << "\n";
     std::cout << "Worker Threads:     " << num_gpus * num_streams << "\n";
+    std::cout << "Tasks Succeeded:    " << tasks_succeeded.load() << "\n";
+    std::cout << "Tasks Failed:       " << tasks_failed.load() << "\n";
     std::cout << "Total Runtime:      " << total_time.count() << " s\n";
     std::cout << "=====================================\n";
 
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index d1221c7de6..3c91115def 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -219,10 +219,10 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
         else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
         st.current_rho = runner.copy_to_host(st.y_tmp)[0];
 
-        if (std::abs(st.old_rho - st.current_rho) < 1e-12) { 
-          code = CG_STAGNATION; 
-          break; 
-        }
+        // if (std::abs(st.old_rho - st.current_rho) < 1e-12) { 
+        //   code = CG_STAGNATION; 
+        //   break; 
+        // }
       }
 
       // Run error checks and prepare progress report
@@ -230,7 +230,7 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
       if (code == CG_SUCCESS) {
         if (!converged && st.iterations >= st.max_iter) code = CG_MAX_ITER;
         // Residual decrease check: fail if residual didn't decrease significantly
-        if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
+        // if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
       }
 
       // Reset and launch NaN/Inf stability kernel from file
@@ -366,7 +366,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
                                   end_time - s.start_times[task_name]).count();
 
-                if (meta.converged) {
+                if (meta.converged && meta.iterations < MAX_ITERATIONS) {
                     s.tasks_succeeded++;
                     if (meta.iterations == 0)
                         self->println("[DONE] {}: Initial guess satisfied tolerance ({} ms).", task_name, duration);
@@ -374,9 +374,11 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                         self->println("[DONE] {}: Converged in {} iterations ({} ms).", task_name, meta.iterations, duration);
                 } else {
                     s.tasks_failed++;
+                    std::string reason = (meta.error_code == CG_SUCCESS)
+                                         ? "Maximum Iterations Reached"
+                                         : to_string(static_cast<cg_error_type>(meta.error_code));
                     self->println("[FAIL] {}: {} (after {} iterations, {} ms).", 
-                                  task_name, 
-                                  to_string(static_cast<cg_error_type>(meta.error_code)), 
+                                  task_name, reason,
                                   meta.iterations,
                                   duration);
                 }
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu
index bf5a18d411..8fc773e035 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu
@@ -210,7 +210,8 @@ int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
     return k;
 }
 
-void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue) {
+void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue,
+                       std::atomic<int>& succeeded, std::atomic<int>& failed) {
     CHECK_CUDA(cudaSetDevice(device_id));
     cudaStream_t stream;
     cublasHandle_t cublas;
@@ -223,11 +224,19 @@ void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>
     while (queue.wait_pop(task)) {
         std::cout << "[WORKER " << worker_id << "] Starting: " << task.path << " (NNZ: " << task.data->nnz << ")" << std::endl;
         auto start_task = std::chrono::steady_clock::now();
-        //int iterations = solve_pcg_jacobi_async(cublas, cusparse, task, stream);
-         int iterations = solve_cg_async(cublas, cusparse, task, stream);
+
+        int iterations = solve_cg_async(cublas, cusparse, task, stream);
         CHECK_CUDA(cudaStreamSynchronize(stream));
+
         auto end_task = std::chrono::steady_clock::now();
         auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_task - start_task).count();
+
+        if (iterations < MAX_ITERATIONS) {
+            succeeded++;
+        } else {
+            failed++;
+        }
+
         std::cout << "[WORKER " << worker_id << "] Done: " << task.path << " (" << iterations << " iters, " << duration << " ms)." << std::endl;
     }
     CHECK_CUBLAS(cublasDestroy(cublas));

From 754932fa2f6912da5b301510cc2cda5fac196339 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 07:51:26 -0600
Subject: [PATCH 0857/1000] updated a bunch of stuff

---
 .../tests/fault-tolerance-workload-test/main.native.cpp      | 4 ++--
 .../fault-tolerance-workload-test/main.native_sorted.cpp     | 2 +-
 .../tests/fault-tolerance-workload-test/main.test.cpp        | 4 ++--
 .../tests/fault-tolerance-workload-test/native_utils.hpp     | 5 ++++-
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
index 802fbd44f8..002633f374 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
@@ -19,10 +19,10 @@ int main(int argc, char** argv)
 {
     constexpr uint32_t WORKLOAD_SEED = 42;
     int num_streams = 4;
-    if (argc > 1) num_streams = std::max(1, std::atoi(argv[1]));
+    // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
 
     std::cout << "[INFO] Loading matrices...\n";
-    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
 
     if (matrix_pool.empty()) {
         std::cerr << "No matrices found.\n";
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
index 743e6117da..4ab9774e1e 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
@@ -23,7 +23,7 @@ void producer(ThreadSafeQueue<MatrixTask>& queue, std::vector<MatrixTask> matrix
 int main(int argc, char** argv) {
     constexpr uint32_t WORKLOAD_SEED = 42;
     int num_streams = 4;
-    if (argc > 1) num_streams = std::max(1, std::atoi(argv[1]));
+    // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
 
     std::cout << "[INFO] Loading matrices...\n";
     std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 3c91115def..5b8a273009 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -448,10 +448,10 @@ void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
     std::cout << "[INFO] Loading matrices...\n";
     {
-         auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/unsymmetric", CGS_SOLVER);
          //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric", CGS_SOLVER);
-         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+         auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
 
         int num_gpus = manager::get().get_num_devices();
         std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.hpp
index 0e70735372..b9de530986 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.hpp
@@ -7,8 +7,11 @@
 #include <queue>
 #include <mutex>
 #include <condition_variable>
+#include <atomic>
 #include "sparse_utils.hpp"
 
+constexpr int MAX_ITERATIONS = 16000;
+
 // ============================================================
 // Error Checking Macros
 // ============================================================
@@ -89,4 +92,4 @@ class ThreadSafeQueue {
 
 int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream);
 int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream);
-void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue);
\ No newline at end of file
+void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue, std::atomic<int>& succeeded, std::atomic<int>& failed);
\ No newline at end of file

From 15fe4da05bb7468c02401c4cd4e5c154ddc25ce0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 08:25:56 -0600
Subject: [PATCH 0858/1000] added a three strikes policy for work, since it may
 recover

---
 .../main.test.cpp                             | 29 ++++++++++++++-----
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 5b8a273009..a6e1135b7e 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -91,6 +91,7 @@ struct ft_cg_state {
   // Config
   int n, nnz, max_iter;
   int iterations = 0;
+  int strikes = 0;
   T tol;
   int device_id, stream_id;
 
@@ -219,18 +220,22 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
         else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
         st.current_rho = runner.copy_to_host(st.y_tmp)[0];
 
-        // if (std::abs(st.old_rho - st.current_rho) < 1e-12) { 
-        //   code = CG_STAGNATION; 
-        //   break; 
-        // }
       }
 
       // Run error checks and prepare progress report
       bool converged = (st.current_rho <= threshold);
+
+      // Check for non-fatal errors that can be retried (Stagnation, Max Iter, Residual Factor)
+      if (code == CG_SUCCESS && !converged && std::abs(st.old_rho - st.current_rho) < 1e-12)
+        code = CG_STAGNATION;
+
       if (code == CG_SUCCESS) {
         if (!converged && st.iterations >= st.max_iter) code = CG_MAX_ITER;
-        // Residual decrease check: fail if residual didn't decrease significantly
-        // if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
+        // Residual decrease check: treat as non-fatal strike if residual didn't decrease significantly
+        if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
+        
+        // If we reached here with CG_SUCCESS, reset strikes as this was a productive batch
+        if (code == CG_SUCCESS) st.strikes = 0;
       }
 
       // Reset and launch NaN/Inf stability kernel from file
@@ -242,6 +247,15 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
       int err_flag = runner.copy_to_host(st.d_err)[0];
       if (err_flag != 0 || std::isnan(st.current_rho) || std::isinf(st.current_rho)) code = CG_NAN_INF;
 
+      // Three strikes policy for non-fatal errors (Stagnation, Max Iterations, Residual Factor Failure)
+      bool is_fatal = (code == CG_NAN_INF || code == CG_BREAKDOWN);
+      if (code != CG_SUCCESS && !is_fatal) {
+        st.strikes++;
+        if (st.strikes < 3) {
+          code = CG_SUCCESS; // Reset code to SUCCESS to allow the supervisor to retry/suspend
+        }
+      }
+
       if (code != CG_SUCCESS) converged = false;
 
       solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
@@ -336,7 +350,8 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 int stream_id = suspended.stream_id;
                 s.available_streams.erase(stream_it);
 
-                int next_batch = std::max(1, suspended.last_batch_size / 2);
+                // Cap the minimum batch size to MAX_ITERATIONS / 8
+                int next_batch = std::max(MAX_ITERATIONS / 8, suspended.last_batch_size / 2);
                 self->println("[INFO] Resuming solver for: {} (Stream: {}, Batch: {})", 
                               suspended.path, stream_id, next_batch);
 

From 6817bee6c67aed1e0c004540f19feb22d1c4c3f3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 08:33:20 -0600
Subject: [PATCH 0859/1000] expanded code to be multiple GPU

---
 .../main.test.cpp                             | 77 +++++++++++--------
 1 file changed, 45 insertions(+), 32 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index a6e1135b7e..306645e1b8 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -282,6 +282,12 @@ struct suspended_task {
   caf::actor solver;
   std::string path;
   int last_batch_size;
+  int device_id;
+  int stream_id;
+};
+
+struct resource_slot {
+  int device_id;
   int stream_id;
 };
 
@@ -290,12 +296,12 @@ struct supervisor_state {
   std::deque<suspended_task> suspended_queue;
   std::vector<caf::actor> active_solvers;
   std::unordered_map<std::string, std::chrono::steady_clock::time_point> start_times;
-  std::unordered_map<std::string, int> task_streams;
+  std::unordered_map<std::string, resource_slot> task_resources;
   std::unordered_map<caf::actor, int> actor_batch_sizes;
-  std::deque<int> available_streams;
+  std::deque<resource_slot> available_slots;
   int max_active = 1; // Admission control limit to be mindful of GPU memory.
   int num_iterations = MAX_ITERATIONS / 2;
-  int device = 0;
+  int num_gpus = 0;
   int tasks_succeeded = 0;
   int tasks_failed = 0;
   std::chrono::steady_clock::time_point benchmark_start;
@@ -306,23 +312,28 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
     st.queue.insert(st.queue.end(), std::make_move_iterator(tasks.begin()), std::make_move_iterator(tasks.end()));
     st.max_active = initial_max_active;
     st.benchmark_start = start_time;
+    st.num_gpus = manager::get().get_num_devices();
 
-    // Initialize the pool with 32 distinct stream IDs
-    for (int i = 0; i < 32; ++i)
-        st.available_streams.push_back(i);
+    // Initialize the pool with streams interleaved across all available GPUs
+    for (int s = 0; s < 32; ++s) {
+      for (int g = 0; g < st.num_gpus; ++g) {
+        st.available_slots.push_back({g, s});
+      }
+    }
 
     auto spawn_next = [self]() {
         auto& s = self->state();
-        while (s.active_solvers.size() < static_cast<size_t>(s.max_active) && !s.available_streams.empty()) {
+        while (s.active_solvers.size() < static_cast<size_t>(s.max_active) && !s.available_slots.empty()) {
           if (!s.queue.empty()) {
             auto task = std::move(s.queue.front());
             s.queue.pop_front();
             std::string path = task.path;
 
-            int stream_id = s.available_streams.front();
-            s.available_streams.pop_front();
+            resource_slot slot = s.available_slots.front();
+            s.available_slots.pop_front();
 
-            self->println("[INFO] Starting solver for: {} (Stream: {})", path, stream_id);
+            self->println("[INFO] Starting solver for: {} (Device: {}, Stream: {})", 
+                          path, slot.device_id, slot.stream_id);
             auto solver = self->spawn(fault_tolerant_cg_actor<float>,
                                     path,
                                     task.data,
@@ -333,33 +344,35 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                                     create_in_out_arg(task.data->x_guess),
                                     (int)task.data->row_ptr.size() - 1,
                                     (int)task.data->values.size(),
-                                    1e-5f, MAX_ITERATIONS, s.device, stream_id, actor_cast<actor>(self));
+                                    1e-5f, MAX_ITERATIONS, slot.device_id, slot.stream_id, actor_cast<actor>(self));
 
             s.start_times[path] = std::chrono::steady_clock::now();
             s.active_solvers.push_back(solver);
-            s.task_streams[path] = stream_id;
+            s.task_resources[path] = slot;
             s.actor_batch_sizes[solver] = s.num_iterations;
             self->mail(start_atom_v).send(solver);
           } else {
             bool resumed = false;
             for (auto it = s.suspended_queue.begin(); it != s.suspended_queue.end(); ++it) {
-              auto stream_it = std::find(s.available_streams.begin(), s.available_streams.end(), it->stream_id);
-              if (stream_it != s.available_streams.end()) {
+              auto slot_it = std::find_if(s.available_slots.begin(), s.available_slots.end(), [&](const resource_slot& slot) {
+                return slot.device_id == it->device_id && slot.stream_id == it->stream_id;
+              });
+              if (slot_it != s.available_slots.end()) {
                 auto suspended = std::move(*it);
                 s.suspended_queue.erase(it);
-                int stream_id = suspended.stream_id;
-                s.available_streams.erase(stream_it);
+                resource_slot slot = *slot_it;
+                s.available_slots.erase(slot_it);
 
                 // Cap the minimum batch size to MAX_ITERATIONS / 8
                 int next_batch = std::max(MAX_ITERATIONS / 8, suspended.last_batch_size / 2);
-                self->println("[INFO] Resuming solver for: {} (Stream: {}, Batch: {})", 
-                              suspended.path, stream_id, next_batch);
+                self->println("[INFO] Resuming solver for: {} (Device: {}, Stream: {}, Batch: {})", 
+                              suspended.path, slot.device_id, slot.stream_id, next_batch);
 
                 // Resume on the same stream ID. No update_stream_atom_v needed.
                 self->mail(cg_next_step_atom_v, next_batch).send(suspended.solver);
 
                 s.active_solvers.push_back(suspended.solver);
-                s.task_streams[suspended.path] = stream_id;
+                s.task_resources[suspended.path] = slot;
                 s.actor_batch_sizes[suspended.solver] = next_batch;
                 resumed = true;
                 break;
@@ -404,11 +417,11 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 s.start_times.erase(task_name);
                 s.actor_batch_sizes.erase(solver);
 
-                // Reclaim the stream ID and put it back in the pool
-                auto stream_it = s.task_streams.find(task_name);
-                if (stream_it != s.task_streams.end()) {
-                    s.available_streams.push_back(stream_it->second);
-                    s.task_streams.erase(stream_it);
+                // Reclaim the device/stream slot and put it back in the pool
+                auto res_it = s.task_resources.find(task_name);
+                if (res_it != s.task_resources.end()) {
+                    s.available_slots.push_back(res_it->second);
+                    s.task_resources.erase(res_it);
                 }
 
                 spawn_next();
@@ -418,14 +431,13 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
 
                     auto benchmark_end = std::chrono::steady_clock::now();
                     std::chrono::duration<double> total_time = benchmark_end - s.benchmark_start;
-                    int num_gpus = manager::get().get_num_devices();
 
                     std::cout << "\n";
                     std::cout << "=====================================\n";
                     std::cout << "IRREGULAR WORKLOAD BENCHMARK (CAF)\n";
                     std::cout << "=====================================\n";
                     std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
-                    std::cout << "GPUs:               " << num_gpus << "\n";
+                    std::cout << "GPUs:               " << s.num_gpus << "\n";
                     std::cout << "Admission Control:  " << s.max_active << "\n";
                     std::cout << "Tasks Succeeded:    " << s.tasks_succeeded << "\n";
                     std::cout << "Tasks Failed:       " << s.tasks_failed << "\n";
@@ -443,14 +455,15 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 if (it != s.active_solvers.end())
                     s.active_solvers.erase(it);
 
-                int stream_id = s.task_streams[task_name];
-                s.available_streams.push_back(stream_id);
-                s.task_streams.erase(task_name);
+                resource_slot slot = s.task_resources[task_name];
+                s.available_slots.push_back(slot);
+                s.task_resources.erase(task_name);
 
                 int last_batch = s.actor_batch_sizes[solver];
-                s.suspended_queue.push_back({solver, task_name, last_batch, stream_id});
+                s.suspended_queue.push_back({solver, task_name, last_batch, slot.device_id, slot.stream_id});
 
-                self->println("[INFO] Suspending solver for: {} (Reclaimed Stream: {})", task_name, stream_id);
+                self->println("[INFO] Suspending solver for: {} (Reclaimed Device: {}, Stream: {})", 
+                              task_name, slot.device_id, slot.stream_id);
 
                 spawn_next();
             }
@@ -484,7 +497,7 @@ void caf_main(actor_system& sys) {
         }
 
         auto benchmark_start = std::chrono::steady_clock::now();
-        int admission_control_limit = 4; // The desired admission control limit
+        int admission_control_limit = 4 * num_gpus; // 4 concurrent tasks per GPU
 
         sys.spawn(supervisor_actor, std::move(tasks_vec), admission_control_limit, benchmark_start);
         sys.await_all_actors_done();

From 8b5c873b3f34bf6cc226fd1835dd92874c49b203 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 08:54:34 -0600
Subject: [PATCH 0860/1000] added job statistics tracking

---
 .../main.native.cpp                           | 27 ++----
 .../main.native_sorted.cpp                    | 23 ++---
 .../main.test.cpp                             | 30 +++----
 .../sparse_utils.cpp                          | 86 +++++++++++++++++++
 .../sparse_utils.hpp                          | 16 ++++
 5 files changed, 130 insertions(+), 52 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
index 002633f374..ea334586ac 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native.cpp
@@ -8,8 +8,9 @@
 #include <atomic>
 #include "native_utils.hpp"
 
-void producer(ThreadSafeQueue<MatrixTask>& queue, const std::vector<MatrixTask>& matrix_pool) {
-    for (const auto& task : matrix_pool) {
+void producer(ThreadSafeQueue<MatrixTask>& queue, std::vector<MatrixTask> matrix_pool) {
+    for (auto& task : matrix_pool) {
+        task.enqueue_time = std::chrono::steady_clock::now();
         queue.push(task);
     }
     queue.signal_shutdown();
@@ -39,8 +40,9 @@ int main(int argc, char** argv)
     std::atomic<int> tasks_succeeded{0};
     std::atomic<int> tasks_failed{0};
     ThreadSafeQueue<MatrixTask> work_queue;
-    auto benchmark_start = std::chrono::steady_clock::now();
-    std::thread producer_thread(producer, std::ref(work_queue), std::cref(matrix_pool));
+    
+    init_benchmark_timer();
+    std::thread producer_thread(producer, std::ref(work_queue), matrix_pool);
     
     std::vector<std::thread> workers;
     for (int gpu = 0; gpu < num_gpus; ++gpu) {
@@ -52,22 +54,7 @@ int main(int argc, char** argv)
     producer_thread.join();
     for (auto& worker : workers) worker.join();
 
-    auto benchmark_end = std::chrono::steady_clock::now();
-    std::chrono::duration<double> total_time = benchmark_end - benchmark_start;
-
-    std::cout << "All tasks in the pool have been processed." << std::endl;
-    std::cout << "\n";
-    std::cout << "=====================================\n";
-    std::cout << "IRREGULAR WORKLOAD BENCHMARK\n";
-    std::cout << "=====================================\n";
-    std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
-    std::cout << "GPUs:               " << num_gpus << "\n";
-    std::cout << "Streams per GPU:    " << num_streams << "\n";
-    std::cout << "Worker Threads:     " << num_gpus * num_streams << "\n";
-    std::cout << "Tasks Succeeded:    " << tasks_succeeded.load() << "\n";
-    std::cout << "Tasks Failed:       " << tasks_failed.load() << "\n";
-    std::cout << "Total Runtime:      " << total_time.count() << " s\n";
-    std::cout << "=====================================\n";
+    report_workload_stats();
 
     return 0;
 }
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
index 4ab9774e1e..6f0771b2c4 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
@@ -14,7 +14,8 @@ void producer(ThreadSafeQueue<MatrixTask>& queue, std::vector<MatrixTask> matrix
         return a.data->nnz < b.data->nnz;
     });
 
-    for (const auto& task : matrix_pool) {
+    for (auto& task : matrix_pool) {
+        task.enqueue_time = std::chrono::steady_clock::now();
         queue.push(task);
     }
     queue.signal_shutdown();
@@ -43,8 +44,10 @@ int main(int argc, char** argv) {
     std::atomic<int> tasks_succeeded{0};
     std::atomic<int> tasks_failed{0};
     ThreadSafeQueue<MatrixTask> work_queue;
-    auto benchmark_start = std::chrono::steady_clock::now();
+    
+    init_benchmark_timer();
     std::thread producer_thread(producer, std::ref(work_queue), matrix_pool);
+    
     std::vector<std::thread> workers;
     for (int gpu = 0; gpu < num_gpus; ++gpu) {
         for (int stream = 0; stream < num_streams; ++stream) {
@@ -53,22 +56,8 @@ int main(int argc, char** argv) {
     }
     producer_thread.join();
     for (auto& worker : workers) worker.join();
-    auto benchmark_end = std::chrono::steady_clock::now();
-    std::chrono::duration<double> total_time = benchmark_end - benchmark_start;
 
-    std::cout << "All tasks in the pool have been processed." << std::endl;
-    std::cout << "\n";
-    std::cout << "=====================================\n";
-    std::cout << "IRREGULAR WORKLOAD BENCHMARK (NATIVE - SORTED)\n";
-    std::cout << "=====================================\n";
-    std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
-    std::cout << "GPUs:               " << num_gpus << "\n";
-    std::cout << "Streams per GPU:    " << num_streams << "\n";
-    std::cout << "Worker Threads:     " << num_gpus * num_streams << "\n";
-    std::cout << "Tasks Succeeded:    " << tasks_succeeded.load() << "\n";
-    std::cout << "Tasks Failed:       " << tasks_failed.load() << "\n";
-    std::cout << "Total Runtime:      " << total_time.count() << " s\n";
-    std::cout << "=====================================\n";
+    report_workload_stats();
 
     return 0;
 }
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 306645e1b8..b1214e87fd 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -297,6 +297,8 @@ struct supervisor_state {
   std::vector<caf::actor> active_solvers;
   std::unordered_map<std::string, std::chrono::steady_clock::time_point> start_times;
   std::unordered_map<std::string, resource_slot> task_resources;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> enqueue_times;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> pick_times;
   std::unordered_map<caf::actor, int> actor_batch_sizes;
   std::deque<resource_slot> available_slots;
   int max_active = 1; // Admission control limit to be mindful of GPU memory.
@@ -329,6 +331,9 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
             s.queue.pop_front();
             std::string path = task.path;
 
+            s.enqueue_times[path] = task.enqueue_time;
+            s.pick_times[path] = std::chrono::steady_clock::now();
+
             resource_slot slot = s.available_slots.front();
             s.available_slots.pop_front();
 
@@ -411,6 +416,10 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                                   duration);
                 }
 
+                record_job(task_name, s.enqueue_times[task_name], s.pick_times[task_name], end_time, meta.iterations, meta.converged);
+                s.enqueue_times.erase(task_name);
+                s.pick_times.erase(task_name);
+
                 auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
                 if (it != s.active_solvers.end())
                     s.active_solvers.erase(it);
@@ -429,21 +438,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 if (s.active_solvers.empty() && s.queue.empty() && s.suspended_queue.empty()) {
                     self->println("All tasks in the pool have been processed.");
 
-                    auto benchmark_end = std::chrono::steady_clock::now();
-                    std::chrono::duration<double> total_time = benchmark_end - s.benchmark_start;
-
-                    std::cout << "\n";
-                    std::cout << "=====================================\n";
-                    std::cout << "IRREGULAR WORKLOAD BENCHMARK (CAF)\n";
-                    std::cout << "=====================================\n";
-                    std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
-                    std::cout << "GPUs:               " << s.num_gpus << "\n";
-                    std::cout << "Admission Control:  " << s.max_active << "\n";
-                    std::cout << "Tasks Succeeded:    " << s.tasks_succeeded << "\n";
-                    std::cout << "Tasks Failed:       " << s.tasks_failed << "\n";
-                    std::cout << "Total Runtime:      " << total_time.count() << " s\n";
-                    std::cout << "=====================================\n";
-
+                    report_workload_stats();
                     self->quit();
                 }
             } else if (meta.iterations == 0) {
@@ -496,6 +491,11 @@ void caf_main(actor_system& sys) {
             tasks_vec.push_back({"dummy_task", CGS_SOLVER, data});
         }
 
+        init_benchmark_timer();
+        for (auto& task : tasks_vec) {
+            task.enqueue_time = std::chrono::steady_clock::now();
+        }
+
         auto benchmark_start = std::chrono::steady_clock::now();
         int admission_control_limit = 4 * num_gpus; // 4 concurrent tasks per GPU
 
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp
index b16c9dc907..07cec53db6 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp
@@ -3,6 +3,12 @@
 #include <stdexcept>
 #include <filesystem>
 #include <random>
+#include <mutex>
+#include <algorithm>
+#include <iomanip>
+#include <iostream>
+#include <numeric>
+
 namespace fs = std::filesystem;
 
 SparseMatrixCOO load_binary_coo(const std::string& filepath) {
@@ -161,4 +167,84 @@ std::vector<MatrixTask> generate_batch(
     }
 
     return batch;
+}
+
+static std::vector<JobStats> global_stats;
+static std::mutex stats_mutex;
+static std::chrono::steady_clock::time_point benchmark_start_tp;
+
+void init_benchmark_timer() {
+    benchmark_start_tp = std::chrono::steady_clock::now();
+}
+
+void record_job(const std::string& name,
+                std::chrono::steady_clock::time_point enqueue_time,
+                std::chrono::steady_clock::time_point pick_time,
+                std::chrono::steady_clock::time_point finish_time,
+                int iterations, bool success) {
+    std::lock_guard<std::mutex> lock(stats_mutex);
+    
+    auto wait = std::chrono::duration<double, std::milli>(pick_time - enqueue_time).count();
+    auto total = std::chrono::duration<double, std::milli>(finish_time - enqueue_time).count();
+    auto finish_rel = std::chrono::duration<double, std::milli>(finish_time - benchmark_start_tp).count();
+
+    global_stats.push_back({name, wait, total, iterations, success, finish_rel});
+}
+
+void report_workload_stats() {
+    std::lock_guard<std::mutex> lock(stats_mutex);
+    if (global_stats.empty()) {
+        std::cout << "No job statistics recorded.\n";
+        return;
+    }
+
+    int total_iters = 0;
+    int success_count = 0;
+    std::vector<double> completions;
+    completions.reserve(global_stats.size());
+
+    for (const auto& s : global_stats) {
+        total_iters += s.iterations;
+        if (s.success) success_count++;
+        completions.push_back(s.completion_time_ms);
+    }
+
+    std::sort(completions.begin(), completions.end());
+    double mean = std::accumulate(completions.begin(), completions.end(), 0.0) / completions.size();
+    double median = completions[completions.size() / 2];
+    double p95 = completions[static_cast<size_t>(completions.size() * 0.95)];
+
+    auto max_finish = std::max_element(global_stats.begin(), global_stats.end(), [](const JobStats& a, const JobStats& b) {
+        return a.finish_relative_ms < b.finish_relative_ms;
+    });
+
+    std::cout << "\n" << std::string(45, '=') << "\n";
+    std::cout << "          WORKLOAD PERFORMANCE REPORT\n";
+    std::cout << std::string(45, '=') << "\n";
+    std::cout << std::left << std::setw(25) << "Total Jobs:" << global_stats.size() << "\n";
+    std::cout << std::left << std::setw(25) << "Total Iterations:" << total_iters << "\n";
+    std::cout << std::left << std::setw(25) << "Success Rate:" << std::fixed << std::setprecision(2) 
+              << (100.0 * success_count / global_stats.size()) << "%\n";
+    std::cout << std::left << std::setw(25) << "Makespan:" << max_finish->finish_relative_ms / 1000.0 << " s\n";
+    std::cout << std::left << std::setw(25) << "Mean Completion:" << mean << " ms\n";
+    std::cout << std::left << std::setw(25) << "Median Completion:" << median << " ms\n";
+    std::cout << std::left << std::setw(25) << "95th Percentile:" << p95 << " ms\n";
+    
+    std::cout << "\nThroughput Timeline (Fraction vs Wall-clock):\n";
+    std::sort(global_stats.begin(), global_stats.end(), [](const JobStats& a, const JobStats& b) {
+        return a.finish_relative_ms < b.finish_relative_ms;
+    });
+    
+    double total_time = max_finish->finish_relative_ms;
+    for (int i = 1; i <= 10; ++i) {
+        double threshold = (total_time / 10.0) * i;
+        auto it = std::upper_bound(global_stats.begin(), global_stats.end(), threshold, 
+            [](double val, const JobStats& s) {
+                return val < s.finish_relative_ms;
+            });
+        size_t count = std::distance(global_stats.begin(), it);
+        std::cout << "  T + " << std::setw(8) << std::fixed << std::setprecision(0) << threshold 
+                  << " ms: " << std::setw(4) << (100 * count / global_stats.size()) << "% complete\n";
+    }
+    std::cout << std::string(45, '=') << "\n\n";
 }
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.hpp
index f886e68c6c..eef98a0456 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.hpp
@@ -44,9 +44,25 @@ struct MatrixTask {
     std::string path;
     SolverType type;
     std::shared_ptr<MatrixData> data;
+    std::chrono::steady_clock::time_point enqueue_time;
 };
 
+struct JobStats {
+    std::string task_name;
+    double wait_time_ms;       // Time spent in queue
+    double completion_time_ms; // Total turnaround time (enqueue to finish)
+    int iterations;
+    bool success;
+    double finish_relative_ms; // Wall-clock timestamp relative to benchmark start
+};
 
+void init_benchmark_timer();
+void record_job(const std::string& name,
+                std::chrono::steady_clock::time_point enqueue_time,
+                std::chrono::steady_clock::time_point pick_time,
+                std::chrono::steady_clock::time_point finish_time,
+                int iterations, bool success);
+void report_workload_stats();
 
 struct Partition {
     size_t begin;

From 9bb79123772923534a8dbca4c56489f2adfb63bb Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 09:05:57 -0600
Subject: [PATCH 0861/1000] updated code to trakc job progress

---
 .../fault-tolerance-workload-test/native_utils.cu     | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu
index 8fc773e035..22a426f494 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/native_utils.cu
@@ -222,21 +222,24 @@ void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>
 
     MatrixTask task;
     while (queue.wait_pop(task)) {
+        auto pick_time = std::chrono::steady_clock::now();
         std::cout << "[WORKER " << worker_id << "] Starting: " << task.path << " (NNZ: " << task.data->nnz << ")" << std::endl;
-        auto start_task = std::chrono::steady_clock::now();
 
         int iterations = solve_cg_async(cublas, cusparse, task, stream);
         CHECK_CUDA(cudaStreamSynchronize(stream));
 
-        auto end_task = std::chrono::steady_clock::now();
-        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_task - start_task).count();
+        auto finish_time = std::chrono::steady_clock::now();
+        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish_time - pick_time).count();
 
-        if (iterations < MAX_ITERATIONS) {
+        bool success = (iterations >= 0 && iterations < MAX_ITERATIONS);
+        if (success) {
             succeeded++;
         } else {
             failed++;
         }
 
+        record_job(task.path, task.enqueue_time, pick_time, finish_time, iterations, success);
+
         std::cout << "[WORKER " << worker_id << "] Done: " << task.path << " (" << iterations << " iters, " << duration << " ms)." << std::endl;
     }
     CHECK_CUBLAS(cublasDestroy(cublas));

From 84e2a2f6d78f0ee31ecf931da6b13cdf242a7ffd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 09:10:23 -0600
Subject: [PATCH 0862/1000] updated report format at the end

---
 .../sparse_utils.cpp                          | 23 ++++++++++++-------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp
index 07cec53db6..ccc35dc689 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp
@@ -200,9 +200,13 @@ void report_workload_stats() {
 
     int total_iters = 0;
     int success_count = 0;
+    int failed_count = 0;
     std::vector<double> completions;
     completions.reserve(global_stats.size());
 
+    // Ensure total_jobs is not zero to avoid division by zero
+    size_t total_jobs = global_stats.size();
+
     for (const auto& s : global_stats) {
         total_iters += s.iterations;
         if (s.success) success_count++;
@@ -210,8 +214,11 @@ void report_workload_stats() {
     }
 
     std::sort(completions.begin(), completions.end());
-    double mean = std::accumulate(completions.begin(), completions.end(), 0.0) / completions.size();
-    double median = completions[completions.size() / 2];
+    failed_count = total_jobs - success_count;
+    double success_percentage = (total_jobs > 0) ? (100.0 * success_count / total_jobs) : 0.0;
+    double failed_percentage = (total_jobs > 0) ? (100.0 * failed_count / total_jobs) : 0.0;
+    double mean = (total_jobs > 0) ? std::accumulate(completions.begin(), completions.end(), 0.0) / total_jobs : 0.0;
+    double median = (total_jobs > 0) ? completions[total_jobs / 2] : 0.0;
     double p95 = completions[static_cast<size_t>(completions.size() * 0.95)];
 
     auto max_finish = std::max_element(global_stats.begin(), global_stats.end(), [](const JobStats& a, const JobStats& b) {
@@ -221,10 +228,10 @@ void report_workload_stats() {
     std::cout << "\n" << std::string(45, '=') << "\n";
     std::cout << "          WORKLOAD PERFORMANCE REPORT\n";
     std::cout << std::string(45, '=') << "\n";
-    std::cout << std::left << std::setw(25) << "Total Jobs:" << global_stats.size() << "\n";
+    std::cout << std::left << std::setw(25) << "Total Jobs:" << total_jobs << "\n";
+    std::cout << std::left << std::setw(25) << "Succeeded Jobs:" << success_count << " (" << std::fixed << std::setprecision(2) << success_percentage << "%)\n";
+    std::cout << std::left << std::setw(25) << "Failed Jobs:" << failed_count << " (" << std::fixed << std::setprecision(2) << failed_percentage << "%)\n";
     std::cout << std::left << std::setw(25) << "Total Iterations:" << total_iters << "\n";
-    std::cout << std::left << std::setw(25) << "Success Rate:" << std::fixed << std::setprecision(2) 
-              << (100.0 * success_count / global_stats.size()) << "%\n";
     std::cout << std::left << std::setw(25) << "Makespan:" << max_finish->finish_relative_ms / 1000.0 << " s\n";
     std::cout << std::left << std::setw(25) << "Mean Completion:" << mean << " ms\n";
     std::cout << std::left << std::setw(25) << "Median Completion:" << median << " ms\n";
@@ -242,9 +249,9 @@ void report_workload_stats() {
             [](double val, const JobStats& s) {
                 return val < s.finish_relative_ms;
             });
-        size_t count = std::distance(global_stats.begin(), it);
-        std::cout << "  T + " << std::setw(8) << std::fixed << std::setprecision(0) << threshold 
-                  << " ms: " << std::setw(4) << (100 * count / global_stats.size()) << "% complete\n";
+        size_t count = std::distance(global_stats.begin(), it); // Number of jobs completed by this threshold
+        std::cout << "  T + " << std::setw(5) << std::fixed << std::setprecision(0) << threshold 
+                  << " ms | Progress: " << std::setw(3) << ((total_jobs > 0) ? (100 * count / total_jobs) : 0) << "%\n";
     }
     std::cout << std::string(45, '=') << "\n\n";
 }
\ No newline at end of file

From 0df5429c80b152444897d7993e4534c65674060f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 09:41:59 -0600
Subject: [PATCH 0863/1000] added more stats

---
 .../main.native_sorted.cpp                    |  2 +-
 .../main.test.cpp                             | 21 ++++++++++++++++++-
 .../sparse_utils.cpp                          | 16 +++++++++++---
 3 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
index 6f0771b2c4..85aa4d301d 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.native_sorted.cpp
@@ -27,7 +27,7 @@ int main(int argc, char** argv) {
     // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
 
     std::cout << "[INFO] Loading matrices...\n";
-    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
 
     if (matrix_pool.empty()) {
         std::cerr << "No matrices found.\n";
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index b1214e87fd..3a025a94ea 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -299,6 +299,7 @@ struct supervisor_state {
   std::unordered_map<std::string, resource_slot> task_resources;
   std::unordered_map<std::string, std::chrono::steady_clock::time_point> enqueue_times;
   std::unordered_map<std::string, std::chrono::steady_clock::time_point> pick_times;
+  std::unordered_map<std::string, double> cumulative_active_ms;
   std::unordered_map<caf::actor, int> actor_batch_sizes;
   std::deque<resource_slot> available_slots;
   int max_active = 1; // Admission control limit to be mindful of GPU memory.
@@ -331,6 +332,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
             s.queue.pop_front();
             std::string path = task.path;
 
+            s.cumulative_active_ms[path] = 0;
             s.enqueue_times[path] = task.enqueue_time;
             s.pick_times[path] = std::chrono::steady_clock::now();
 
@@ -368,6 +370,8 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 resource_slot slot = *slot_it;
                 s.available_slots.erase(slot_it);
 
+                s.pick_times[suspended.path] = std::chrono::steady_clock::now();
+
                 // Cap the minimum batch size to MAX_ITERATIONS / 8
                 int next_batch = std::max(MAX_ITERATIONS / 8, suspended.last_batch_size / 2);
                 self->println("[INFO] Resuming solver for: {} (Device: {}, Stream: {}, Batch: {})", 
@@ -416,9 +420,19 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                                   duration);
                 }
 
-                record_job(task_name, s.enqueue_times[task_name], s.pick_times[task_name], end_time, meta.iterations, meta.converged);
+                // Final active slice
+                auto active_slice = std::chrono::duration<double, std::milli>(end_time - s.pick_times[task_name]).count();
+                s.cumulative_active_ms[task_name] += active_slice;
+
+                // We override pick_time in record_job to simulate a single continuous run that equals 
+                // the actual time spent on the GPU.
+                auto simulated_pick = end_time - std::chrono::duration_cast<std::chrono::steady_clock::duration>(
+                                                std::chrono::duration<double, std::milli>(s.cumulative_active_ms[task_name]));
+
+                record_job(task_name, s.enqueue_times[task_name], simulated_pick, end_time, meta.iterations, meta.converged);
                 s.enqueue_times.erase(task_name);
                 s.pick_times.erase(task_name);
+                s.cumulative_active_ms.erase(task_name);
 
                 auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
                 if (it != s.active_solvers.end())
@@ -447,6 +461,11 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
             } else {
                 // Not done: Suspend the actor to allow others to use the stream
                 auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
+                
+                // Record the time spent in this active slice before suspending
+                auto active_slice = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - s.pick_times[task_name]).count();
+                s.cumulative_active_ms[task_name] += active_slice;
+
                 if (it != s.active_solvers.end())
                     s.active_solvers.erase(it);
 
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp
index ccc35dc689..dd09e63fc3 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/sparse_utils.cpp
@@ -200,6 +200,7 @@ void report_workload_stats() {
 
     int total_iters = 0;
     int success_count = 0;
+    double wasted_gpu_time_ms = 0;
     int failed_count = 0;
     std::vector<double> completions;
     completions.reserve(global_stats.size());
@@ -210,6 +211,8 @@ void report_workload_stats() {
     for (const auto& s : global_stats) {
         total_iters += s.iterations;
         if (s.success) success_count++;
+        else wasted_gpu_time_ms += (s.completion_time_ms - s.wait_time_ms);
+        
         completions.push_back(s.completion_time_ms);
     }
 
@@ -232,12 +235,13 @@ void report_workload_stats() {
     std::cout << std::left << std::setw(25) << "Succeeded Jobs:" << success_count << " (" << std::fixed << std::setprecision(2) << success_percentage << "%)\n";
     std::cout << std::left << std::setw(25) << "Failed Jobs:" << failed_count << " (" << std::fixed << std::setprecision(2) << failed_percentage << "%)\n";
     std::cout << std::left << std::setw(25) << "Total Iterations:" << total_iters << "\n";
+    std::cout << std::left << std::setw(25) << "Cumul. Wasted GPU Time:" << wasted_gpu_time_ms / 1000.0 << " s (all streams)\n";
     std::cout << std::left << std::setw(25) << "Makespan:" << max_finish->finish_relative_ms / 1000.0 << " s\n";
     std::cout << std::left << std::setw(25) << "Mean Completion:" << mean << " ms\n";
     std::cout << std::left << std::setw(25) << "Median Completion:" << median << " ms\n";
     std::cout << std::left << std::setw(25) << "95th Percentile:" << p95 << " ms\n";
     
-    std::cout << "\nThroughput Timeline (Fraction vs Wall-clock):\n";
+    std::cout << "\nThroughput Timeline (Job Completion & Success vs Wall-clock):\n";
     std::sort(global_stats.begin(), global_stats.end(), [](const JobStats& a, const JobStats& b) {
         return a.finish_relative_ms < b.finish_relative_ms;
     });
@@ -249,9 +253,15 @@ void report_workload_stats() {
             [](double val, const JobStats& s) {
                 return val < s.finish_relative_ms;
             });
-        size_t count = std::distance(global_stats.begin(), it); // Number of jobs completed by this threshold
+        size_t total_at_t = std::distance(global_stats.begin(), it);
+        size_t success_at_t = 0;
+        for (auto s_it = global_stats.begin(); s_it != it; ++s_it) {
+            if (s_it->success) success_at_t++;
+        }
+
         std::cout << "  T + " << std::setw(5) << std::fixed << std::setprecision(0) << threshold 
-                  << " ms | Progress: " << std::setw(3) << ((total_jobs > 0) ? (100 * count / total_jobs) : 0) << "%\n";
+                  << " ms | Total Progress: " << std::setw(3) << (100 * total_at_t / total_jobs) 
+                  << "% | Successful solves: " << std::setw(3) << (100 * success_at_t / total_jobs) << "%\n";
     }
     std::cout << std::string(45, '=') << "\n\n";
 }
\ No newline at end of file

From 243e6c90bcbee7167b5fadb398a9c50cca4d6293 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 11:15:02 -0600
Subject: [PATCH 0864/1000] updated actor facade tests

---
 .../sc26/Runtime-Overhead/actor_facade.cpp    | 164 +++++++-------
 .../actor_facade.cpp                          | 204 ++++++++----------
 2 files changed, 163 insertions(+), 205 deletions(-)

diff --git a/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp b/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
index 8b411bf2c3..1f0b19622d 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
@@ -1,103 +1,97 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
 #include <iostream>
-#include <iomanip>
 #include <vector>
 #include <chrono>
-#include <random>
 
 using namespace caf;
 using namespace std::chrono_literals;
 
-static const unsigned int RANDOM_SEED = 42;
-
-void run_mmul_test(caf::actor_system& sys, int matrix_size) {
-  // F5: manager::init/shutdown moved to caf_main — called once for all sizes
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  // F3: spawn GPU actor (loads cubin) BEFORE timing begins, so module-load
-  //     overhead is excluded from the per-size measurement — matching
-  //     cuda_native which loads the module once outside all per-size timing.
-  int THREADS = 32;
-  int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-  auto gpuActor = mgr.spawnFromCUBIN("mmul.cubin", "matrixMul", dim,
-                                     in<int>{}, in<int>{}, out<int>{}, in<int>{});
-
-  using clock = std::chrono::steady_clock;
-  using ms = std::chrono::duration<double, std::milli>;
-
-  // F4: use mt19937(42) to match cuda_native data initialisation
-  std::mt19937 rng(RANDOM_SEED);
-  std::uniform_int_distribution<int> dist(1, 10);
-  std::vector<int> h_a(matrix_size * matrix_size);
-  std::vector<int> h_b(matrix_size * matrix_size);
-  std::vector<int> h_c(matrix_size * matrix_size, 0);
-  for (auto& v : h_a) v = dist(rng);
-  for (auto& v : h_b) v = dist(rng);
-
-  // TIMING STARTS — after spawn/module-load, matching cuda_native's exclusion
-  auto t_total_start = clock::now();
-
-  auto t_a_inarg_start = clock::now();
-  auto arg1 = caf::cuda::create_in_arg(h_a);
-  auto t_a_inarg_end = clock::now();
-
-  auto t_b_inarg_start = clock::now();
-  auto arg2 = caf::cuda::create_in_arg(h_b);
-  auto t_b_inarg_end = clock::now();
-
-  auto arg3 = caf::cuda::create_out_arg(h_c);
-  auto arg4 = caf::cuda::create_in_arg(matrix_size);
-
-  sys.spawn([=](event_based_actor* self_actor) {
-    auto t_request_start = clock::now();
-    self_actor->mail(gpuActor, arg1, arg2, arg3, arg4)
-      .request(gpuActor, 100s).then(
-        [=](const std::vector<output_buffer>& outputs) {
-          auto t_response_received = clock::now();
-          auto t_total_end = clock::now();
-
-          std::cout << "\n===== ACTOR FACADE BENCHMARK RESULTS (N=" << matrix_size << ") =====\n";
-          std::cout << "create_in_arg A: " << ms(t_a_inarg_end - t_a_inarg_start).count() << " ms\n";
-          std::cout << "create_in_arg B: " << ms(t_b_inarg_end - t_b_inarg_start).count() << " ms\n";
-          std::cout << "request \xE2\x86\x92 response latency (includes transfers & exec): "
-                    << ms(t_response_received - t_request_start).count() << " ms\n";
-          std::cout << "TOTAL end-to-end: " << ms(t_total_end - t_total_start).count() << " ms\n";
-          std::cout << "=============================================\n";
-
-          self_actor->send_exit(gpuActor, exit_reason::user_shutdown);
-          self_actor->quit();
-        });
-  });
+struct latency_test_state {
+    std::chrono::steady_clock::time_point start_time;
+    int N;
+    std::vector<int> h_a;
+    std::vector<int> h_b;
+    in<int> arg1;
+    in<int> arg2;
+    out<int> arg3;
+    in<int> arg4;
+};
 
-  sys.await_all_actors_done();
+caf::behavior latency_manager(caf::stateful_actor<latency_test_state>* self, 
+                              caf::actor facade, int N) {
+    auto& st = self->state();
+    st.N = N;
+    
+    // Initialize data and reuseable kernel arguments in state
+    st.h_a.assign(N * N, 2);
+    st.h_b.assign(N * N, 3);
+    st.arg1 = caf::cuda::create_in_arg(st.h_a);
+    st.arg2 = caf::cuda::create_in_arg(st.h_b);
+    st.arg3 = caf::cuda::create_out_arg_with_size<int>(N * N);
+    st.arg4 = caf::cuda::create_in_arg(N);
+
+    // Only copy back index 2 (Matrix C)
+    std::vector<int> output_indices = {2};
+    st.start_time = std::chrono::steady_clock::now();
+    
+    // Launch the work with selective index-based copy-back
+    self->mail(output_indices, st.arg1, st.arg2, st.arg3, st.arg4).send(facade);
+
+    return {
+        [=](int r_id, int index, std::vector<int> data) {
+            if (index == 2) { // Completion signal for Matrix C
+                auto end_time = std::chrono::steady_clock::now();
+                auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
+                    end_time - self->state().start_time).count();
+
+                std::cout << "[LATENCY TEST] matrix_size=" << self->state().N 
+                          << ", time=" << elapsed << " ms" << std::endl;
+
+                self->send_exit(facade, exit_reason::user_shutdown);
+                self->quit();
+            }
+        }
+    };
 }
 
-class config : public actor_system_config {
-public:
-  config() {
-      set("caf.scheduler.max-threads", 1u);
-  }
-};
+void run_latency_test(caf::actor_system& sys, int matrix_size) {
+    caf::cuda::manager::init(sys);
+    auto& mgr = caf::cuda::manager::get();
 
-void caf_main(caf::actor_system& sys, const config& cfg) {
-  caf::cuda::manager::init(sys);  // F5: init once before all sizes
+    int THREADS = 32;
+    int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-  // F2: warmup run to prime CUDA context, JIT, and CAF infrastructure
-  std::cout << "--- warmup starting ---\n";
-  run_mmul_test(sys, 64);
-  std::cout << "--- warmup complete ---\n";
+    // Spawn facade
+    auto facade = mgr.spawnFromCUBIN(
+        "../mmul.cubin", "matrixMul", dims,
+        in<int>{}, in<int>{}, out<int>{}, in<int>{});
 
-  // F1: unified sizes matching cuda_native: {1000, 2000, 4000, 8000, 16000}
-  run_mmul_test(sys, 1000);
-  run_mmul_test(sys, 2000);
-  run_mmul_test(sys, 4000);
-  run_mmul_test(sys, 8000);
-  run_mmul_test(sys, 16000);
+    sys.spawn(latency_manager, facade, matrix_size);
+    sys.await_all_actors_done();
+    caf::cuda::manager::shutdown();
+}
 
-  caf::cuda::manager::shutdown();  // F5: shutdown once after all sizes
+void caf_main(caf::actor_system& sys) {
+    std::vector<int> sizes = {1000, 4000, 8000, 12000};
+    for (int size : sizes) {
+        run_latency_test(sys, size);
+    }
 }
 
-CAF_MAIN()
+int main(int argc, char** argv) {
+    core::init_global_meta_objects();
+    actor_system_config cfg;
+    cfg.set("caf.scheduler.max-threads", 1);
+    cfg.set("caf.scheduler.policy", "sharing");
+    
+    auto err = cfg.parse(argc, argv);
+    if (err) return EXIT_FAILURE;
+    if (cfg.helptext_printed()) return 0;
+
+    actor_system sys{cfg};
+    caf_main(sys);
+    
+    return 0;
+}
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
index 3f40cc7dce..5896806acf 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
@@ -1,144 +1,108 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
 #include <iostream>
-#include <iomanip>
 #include <vector>
 #include <chrono>
 
 using namespace caf;
 using namespace std::chrono_literals;
 
-struct bench_state {
-  int completed = 0;
-  int total = 0;
-  bool is_warmup = false;
-  std::chrono::steady_clock::time_point start_time;
-  caf::actor gpuActor;
-  int matrix_size = 0;
-  std::vector<int> h_a;
-  std::vector<int> h_b;
-  std::vector<int> h_c;
+struct mapping_throughput_state {
+    int total_expected = 0;
+    int results_received = 0;
+    std::chrono::steady_clock::time_point start_time;
+    int N;
+    std::vector<int> h_a;
+    std::vector<int> h_b;
+    std::vector<int> h_c_global; // The persistent buffer
+    in<int> arg1;
+    in<int> arg2;
+    out<int> arg3;
+    in<int> arg4;
 };
 
-// Forward declaration
-void send_next_request(caf::stateful_actor<bench_state>* self);
-
-// S3 fix: back-pressure — sends exactly one request and re-chains on response,
-// bounding in-flight memory to a single 8 MB pair of in-args at any time.
-void send_next_request(caf::stateful_actor<bench_state>* self) {
-  auto& st = self->state();
-  auto arg1 = caf::cuda::create_in_arg(st.h_a);
-  auto arg2 = caf::cuda::create_in_arg(st.h_b);
-  auto arg3 = caf::cuda::create_out_arg(st.h_c);
-  auto arg4 = caf::cuda::create_in_arg(st.matrix_size);
-
-  self->mail(st.gpuActor, arg1, arg2, arg3, arg4)
-    .request(st.gpuActor, infinite)
-    .then(
-      [=](const std::vector<output_buffer>& /*outputs*/) {
-        auto& st2 = self->state();
-        st2.completed++;
-
-        // S6 fix: milestone reporting every 1000 completions
-        if (!st2.is_warmup && st2.completed % 1000 == 0) {
-          auto now = std::chrono::steady_clock::now();
-          using ms = std::chrono::duration<double, std::milli>;
-          double elapsed = ms(now - st2.start_time).count();
-          std::cout << "[MILESTONE] " << st2.completed << " / " << st2.total
-                    << " iterations, elapsed = " << elapsed << " ms\n";
-        }
+caf::behavior throughput_mapping_manager(caf::stateful_actor<mapping_throughput_state>* self, 
+                                         caf::actor facade, int N, int iterations) {
+    auto& st = self->state();
+    st.total_expected = iterations;
+    st.N = N;
+
+    // Initialize data and reuseable kernel arguments
+    st.h_a.assign(N * N, 2);
+    st.h_b.assign(N * N, 3);
+    st.h_c_global.assign(N * N, 0); // Pre-allocate the global destination
+    
+    st.arg1 = caf::cuda::create_in_arg(st.h_a);
+    st.arg2 = caf::cuda::create_in_arg(st.h_b);
+    st.arg3 = caf::cuda::create_out_arg_with_size<int>(N * N);
+    st.arg4 = caf::cuda::create_in_arg(N);
+
+    // Define the mapping once
+    output_mapping mapping{2, st.h_c_global.data(), st.h_c_global.size()};
+    std::vector<output_mapping> mappings = {mapping};
+
+    st.start_time = std::chrono::steady_clock::now();
+
+    for (int i = 0; i < iterations; ++i) {
+        // Send using the mappings overload
+        self->mail(mappings, st.arg1, st.arg2, st.arg3, st.arg4).send(facade);
+    }
 
-        if (st2.completed == st2.total) {
-          if (!st2.is_warmup) {
-            auto end_time = std::chrono::steady_clock::now();
-            using ms = std::chrono::duration<double, std::milli>;
-            double duration_ms = ms(end_time - st2.start_time).count();
-            std::cout << "[SERIES RESULT] Matrix " << st2.matrix_size << "x" << st2.matrix_size
-                      << ", iterations = " << st2.total
-                      << ", total CPU/Actor time = " << duration_ms << " ms\n";
-          }
-          self->send_exit(st2.gpuActor, exit_reason::user_shutdown);
-          self->quit();
-        } else {
-          send_next_request(self);
-        }
-      },
-      [=](const error& err) {
-        if (!self->state().is_warmup)
-          std::cout << "Error in iteration: " << to_string(err) << std::endl;
-        auto& st2 = self->state();
-        st2.completed++;
-        if (st2.completed == st2.total) {
-          self->send_exit(st2.gpuActor, exit_reason::user_shutdown);
-          self->quit();
-        } else {
-          send_next_request(self);
+    return {
+        [=](int r_id, int index) {
+            if (index == 2) { // Received notification for index 2 (Matrix C)
+                if (++self->state().results_received == self->state().total_expected) {
+                    auto end_time = std::chrono::steady_clock::now();
+                    auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
+                        end_time - self->state().start_time).count();
+                    
+                    std::cout << "[MAPPING THROUGHPUT TEST] matrix_size=" << self->state().N
+                              << " iterations=" << self->state().total_expected
+                              << ", total_time=" << elapsed << " ms" << std::endl;
+                    
+                    self->send_exit(facade, exit_reason::user_shutdown);
+                    self->quit();
+                }
+            }
         }
-      }
-    );
-}
-
-caf::behavior bench_coordinator(caf::stateful_actor<bench_state>* self,
-                                 caf::actor gpuActor,
-                                 int matrix_size,
-                                 int iterations,
-                                 bool is_warmup) {
-  auto& st = self->state();
-  st.completed = 0;
-  st.total = iterations;
-  st.is_warmup = is_warmup;
-  st.gpuActor = gpuActor;
-  st.matrix_size = matrix_size;
-  st.h_a.assign(matrix_size * matrix_size, 2);
-  st.h_b.assign(matrix_size * matrix_size, 3);
-  st.h_c.assign(matrix_size * matrix_size, 0);
-
-  self->monitor(gpuActor, [self](const error& err) {
-    if (!self->state().is_warmup)
-      std::cout << "GPU Actor died unexpectedly! " << to_string(err) << std::endl;
-    self->quit();
-  });
-
-  return {
-    [=](int) {
-      self->state().start_time = std::chrono::steady_clock::now();
-      // S3 fix: fire one request; subsequent requests are chained via back-pressure
-      send_next_request(self);
-    }
-  };
+    };
 }
 
-// S4 fix: manager::init/shutdown moved to caf_main — called once for all series
-void run_series(caf::actor_system& sys, int matrix_size, int iterations,
-                bool is_warmup = false) {
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
+void run_mapping_throughput_test(caf::actor_system& sys, int matrix_size, int iterations) {
+    caf::cuda::manager::init(sys);
+    auto& mgr = caf::cuda::manager::get();
 
-  int THREADS = 32;
-  int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dim(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
+    int THREADS = 32;
+    int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
+    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
-  auto gpuActor = mgr.spawnFromCUBIN("mmul.cubin", "matrixMul", dim,
-                                     in<int>{}, in<int>{}, out<int>{}, in<int>{});
+    auto facade = mgr.spawnFromCUBIN(
+        "../mmul.cubin", "matrixMul", dims,
+        in<int>{}, in<int>{}, out<int>{}, in<int>{});
 
-  auto coordinator = sys.spawn(bench_coordinator, gpuActor, matrix_size, iterations, is_warmup);
-  anon_mail(1).send(coordinator);
-  sys.await_all_actors_done();
+    sys.spawn(throughput_mapping_manager, facade, matrix_size, iterations);
+    sys.await_all_actors_done();
+    caf::cuda::manager::shutdown();
 }
 
 void caf_main(caf::actor_system& sys) {
-  constexpr int matrix_size = 1000;
-  constexpr int total_iterations = 10000;
-
-  caf::cuda::manager::init(sys);  // S4 fix: init once before all series
-
-  // S2 fix: warmup run to prime CUDA context and CAF infrastructure
-  std::cout << "--- warmup starting ---\n";
-  run_series(sys, matrix_size, 10, /*is_warmup=*/true);
-  std::cout << "--- warmup complete ---\n";
+    int fixed_size = 1000;
+    for (int i = 1000; i <= 10000; i += 1000) {
+        run_mapping_throughput_test(sys, fixed_size, i);
+    }
+}
 
-  run_series(sys, matrix_size, total_iterations);
+int main(int argc, char** argv) {
+    core::init_global_meta_objects();
+    actor_system_config cfg;
+    cfg.set("caf.scheduler.max-threads", 1);
+    cfg.set("caf.scheduler.policy", "sharing");
 
-  caf::cuda::manager::shutdown();  // S4 fix: shutdown once after all series
-}
+    auto err = cfg.parse(argc, argv);
+    if (err) return EXIT_FAILURE;
+    if (cfg.helptext_printed()) return 0;
 
-CAF_MAIN()
+    actor_system sys{cfg};
+    caf_main(sys);
+    return 0;
+}
\ No newline at end of file

From 2ede92d81af8698df4d3f2f3b6461dde1e36642a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 11:21:02 -0600
Subject: [PATCH 0865/1000] updated scripts to include new actor facade

---
 .../analyze_mmul_benchmarks.py                | 219 ++++++++++--------
 .../generate_graphs.py                        | 137 ++++++-----
 2 files changed, 213 insertions(+), 143 deletions(-)

diff --git a/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py b/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
index 773b058a6a..7a0a73e684 100644
--- a/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
+++ b/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
@@ -1,98 +1,135 @@
 #!/usr/bin/env python3
 
+import glob
 import re
+import numpy as np
 import matplotlib.pyplot as plt
-from matplotlib.ticker import ScalarFormatter
+from collections import defaultdict
 from pathlib import Path
 
-# Path Configuration
-# This script is located in sc26/scripts/Runtime-Overhead/
-# We expect results in sc26/Runtime-Overhead/results/
+# -----------------------------
+# Path configuration (MATCHING STYLE OF FIRST SCRIPT)
+# -----------------------------
 SCRIPT_DIR = Path(__file__).resolve().parent
-RESULTS_FILE = SCRIPT_DIR.parent.parent / "Runtime-Overhead" / "results" / "benchmark_results.txt"
-OUTPUT_PLOT = SCRIPT_DIR.parent.parent / "Runtime-Overhead" / "results" / "runtime_overhead_comparison.png"
-
-def parse_comparison_table():
-    """Parses the 'CROSS-IMPLEMENTATION COMPARISON' table from the results file."""
-    if not RESULTS_FILE.exists():
-        print(f"Error: Results file not found at {RESULTS_FILE}")
-        return None
-
-    with open(RESULTS_FILE, 'r', encoding='utf-8') as f:
-        content = f.read()
-
-    # Locate the comparison section. It uses a specific header format.
-    # We capture the data rows between the dashes and the next double newline.
-    table_pattern = re.compile(
-        r"CROSS-IMPLEMENTATION COMPARISON.*?N\s+cuda_native\s+actor_facade\s+command_runner.*?\n-+\n(.*?)(?=\n\n|\Z)",
-        re.DOTALL
-    )
-    
-    match = table_pattern.search(content)
-    if not match:
-        print("Error: Could not find the comparison table in the benchmark results.")
-        return None
-
-    data = {
-        "N": [],
-        "cuda_native": [],
-        "actor_facade": [],
-        "command_runner": []
-    }
-
-    rows = match.group(1).strip().splitlines()
-    for row in rows:
-        parts = row.split()
-        # Expected format: N, cuda_native_mean, actor_facade_mean, command_runner_mean, ...
-        if len(parts) >= 4:
-            try:
-                data["N"].append(int(parts[0]))
-                data["cuda_native"].append(float(parts[1]))
-                data["actor_facade"].append(float(parts[2]))
-                data["command_runner"].append(float(parts[3]))
-            except ValueError:
-                continue # Skip header/footer noise if any
-
-    return data
-
-def generate_plot(data):
-    """Generates a PNG graph comparing mean execution times."""
-    if not data or not data["N"]:
-        return
-
-    plt.figure(figsize=(11, 7))
-
-    # Plotting each implementation
-    plt.plot(data["N"], data["cuda_native"], marker='o', linestyle='-', label='CUDA Native (Baseline)')
-    plt.plot(data["N"], data["actor_facade"], marker='s', linestyle='--', label='CAF Actor Facade')
-    plt.plot(data["N"], data["command_runner"], marker='^', linestyle=':', label='CAF Command Runner')
-
-    plt.title('Matrix Multiplication Performance by Size', fontsize=14, fontweight='bold')
-    plt.xlabel('Matrix Size N', fontsize=12)
-    plt.ylabel('Mean Execution Time (ms)', fontsize=12)
-    
-    plt.grid(True, which="both", linestyle='--', alpha=0.6)
-    plt.legend(fontsize=10)
-    
-    # Apply log scale for better visualization if the N range is large
-    if max(data["N"]) / min(data["N"]) > 10:
-        plt.xscale('log')
-        plt.yscale('log')
-
-    # Set X-axis ticks to exactly the N values tested to avoid abbreviation
-    plt.xticks(data["N"], data["N"])
-
-    # Force scalar formatting for Y axis to avoid scientific notation (e.g., 10^x)
-    ax = plt.gca()
-    formatter = ScalarFormatter()
-    formatter.set_scientific(False)
-    ax.yaxis.set_major_formatter(formatter)
-
-    plt.tight_layout()
-    plt.savefig(OUTPUT_PLOT, dpi=300)
-    print(f"Successfully generated comparison graph: {OUTPUT_PLOT}")
-
-if __name__ == "__main__":
-    results = parse_comparison_table()
-    if results:
-        generate_plot(results)
+
+DATA_DIR = SCRIPT_DIR.parent.parent / "Runtime-Overhead" / "results"
+
+driver_files = glob.glob(f"{DATA_DIR}/matrix_mul_driver_run*.txt")
+actor_files = glob.glob(f"{DATA_DIR}/test_run*.txt")
+latency_files = glob.glob(f"{DATA_DIR}/latency_bench_test_run*.txt")
+
+OUTPUT_PLOT = DATA_DIR / "mmul_benchmark_plot.png"
+
+# -----------------------------
+# Regex
+# -----------------------------
+size_pattern = re.compile(r"N=(\d+)")
+driver_total_pattern = re.compile(r"TOTAL:\s+([\d.]+)")
+actor_total_pattern = re.compile(r"TOTAL end-to-end:\s+([\d.]+)")
+
+latency_pattern = re.compile(
+    r"\[LATENCY TEST\]\s+matrix_size=(\d+),\s*time=(\d+)\s*ms"
+)
+
+# -----------------------------
+# Data containers
+# -----------------------------
+driver_data = defaultdict(list)
+actor_data = defaultdict(list)
+latency_data = defaultdict(list)
+
+# -----------------------------
+# Parsers
+# -----------------------------
+def parse_driver(file):
+    with open(file) as f:
+        current_size = None
+        for line in f:
+            size_match = size_pattern.search(line)
+            if size_match:
+                current_size = int(size_match.group(1))
+
+            total_match = driver_total_pattern.search(line)
+            if total_match and current_size:
+                driver_data[current_size].append(float(total_match.group(1)))
+
+
+def parse_actor(file):
+    with open(file) as f:
+        current_size = None
+        for line in f:
+            size_match = size_pattern.search(line)
+            if size_match:
+                current_size = int(size_match.group(1))
+
+            total_match = actor_total_pattern.search(line)
+            if total_match and current_size:
+                actor_data[current_size].append(float(total_match.group(1)))
+
+
+def parse_latency(file):
+    with open(file) as f:
+        for line in f:
+            m = latency_pattern.search(line)
+            if m:
+                size = int(m.group(1))
+                time = float(m.group(2))
+                latency_data[size].append(time)
+
+# -----------------------------
+# Parse files
+# -----------------------------
+for f in driver_files:
+    parse_driver(f)
+
+for f in actor_files:
+    parse_actor(f)
+
+for f in latency_files:
+    parse_latency(f)
+
+# -----------------------------
+# Aggregate
+# -----------------------------
+sizes = sorted(driver_data.keys())
+
+driver_means = [np.mean(driver_data[s]) for s in sizes]
+actor_means = [np.mean(actor_data[s]) for s in sizes]
+latency_means = [np.mean(latency_data[s]) for s in sizes]
+
+# -----------------------------
+# Metrics
+# -----------------------------
+abs_diff = [a - d for d, a in zip(driver_means, actor_means)]
+speedup = [d / a if a != 0 else float('inf') for d, a in zip(driver_means, actor_means)]
+percent_diff = [((a - d) / d) * 100 if d != 0 else 0 for d, a in zip(driver_means, actor_means)]
+
+# -----------------------------
+# Print
+# -----------------------------
+print("===== MEAN RESULTS =====")
+print("N     | CUDA (ms) | Actors (ms) | Latency Facade (ms) | % Actor Diff | Speedup")
+print("-" * 100)
+
+for s, d, a, l, pct, sp in zip(
+    sizes, driver_means, actor_means, latency_means, percent_diff, speedup
+):
+    print(f"N={s:5d} | {d:10.3f} | {a:11.3f} | {l:18.3f} | {pct:11.2f}% | {sp:8.3f}")
+
+# -----------------------------
+# Plot
+# -----------------------------
+plt.figure()
+
+plt.plot(sizes, driver_means, marker='o', label="CUDA (Driver)")
+plt.plot(sizes, actor_means, marker='s', label="CUDA Actors")
+plt.plot(sizes, latency_means, marker='^', label="Actor-Facade")
+
+plt.xlabel("Matrix Size (N)")
+plt.ylabel("Mean Execution Time (ms)")
+plt.title("CUDA vs Actors vs Actor-Facade")
+plt.legend()
+plt.grid(True)
+
+plt.savefig(OUTPUT_PLOT, dpi=300)
+plt.show()
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py b/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py
index 6a27697b17..c9f5db4693 100644
--- a/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py
+++ b/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py
@@ -1,74 +1,107 @@
 #!/usr/bin/env python3
 
+import glob
 import re
 import numpy as np
 import matplotlib.pyplot as plt
 from pathlib import Path
+from collections import defaultdict
 
 # -----------------------------
-# Path Configuration
+# Path Configuration (same style as first script)
 # -----------------------------
 SCRIPT_DIR = Path(__file__).resolve().parent
-RESULTS_DIR = SCRIPT_DIR.parent.parent / "Sequence-Independent-Tasks" / "results"
-RESULTS_FILE = RESULTS_DIR / "benchmark_results.txt"
-OUTPUT_PLOT = RESULTS_DIR / "mmul_comparison.png"
-
-# -----------------------------
-# Parse Benchmark Results
-# -----------------------------
-def parse_results():
-    if not RESULTS_FILE.exists():
-        print(f"Error: Results file not found at {RESULTS_FILE}")
-        return None
-
-    with open(RESULTS_FILE, 'r') as f:
-        content = f.read()
-
-    # Extract the comparison table data
-    # Row format: series milestone main_cuda_native main_actor_facade main_command_runner ...
-    table_pattern = re.compile(
-        r"CROSS-IMPLEMENTATION COMPARISON.*?series\s+milestone.*?\n\s*-+\n(.*?)(?=\n\n|\Z)",
-        re.DOTALL
-    )
-    
-    match = table_pattern.search(content)
-    if not match:
-        print("Error: Could not find comparison table in results.")
-        return None
-
-    data = {"it": [], "cuda": [], "facade": [], "runner": []}
-    for line in match.group(1).strip().splitlines():
-        parts = line.split()
-        if len(parts) >= 5:
-            data["it"].append(int(parts[1]))      # milestone
-            data["cuda"].append(float(parts[2]))    # main_cuda_native
-            data["facade"].append(float(parts[3]))  # main_actor_facade
-            data["runner"].append(float(parts[4]))  # main_command_runner
-    return data
-
-data = parse_results()
-if not data:
-    exit(1)
-
-# -----------------------------
-# CLI Output
+
+BASE_DIR = SCRIPT_DIR.parent.parent / "Sequence-Independent-Tasks" / "results"
+
+driver_files = glob.glob(str(BASE_DIR / "matrix_mul_driver_run*.txt"))
+actor_files = glob.glob(str(BASE_DIR / "test_run*.txt"))
+runner_files = glob.glob(str(BASE_DIR / "throughput_mapping_bench_test_run*.txt"))
+
+OUTPUT_PLOT = BASE_DIR / "mmul_comparison.png"
+
+# -----------------------------
+# Data containers
+# -----------------------------
+cuda_data = defaultdict(list)
+actor_facade_data = defaultdict(list)
+command_runner_data = defaultdict(list)
+
+# -----------------------------
+# Regex
+# -----------------------------
+cuda_pattern = re.compile(r"iterations\s*=\s*(\d+),\s*total GPU time\s*=\s*([0-9.]+)")
+actor_pattern = re.compile(r"iterations\s*=\s*(\d+),\s*time\s*=\s*([0-9.]+)")
+runner_pattern = re.compile(r"iterations\s*=\s*(\d+).*total_time\s*=\s*([0-9.]+)\s*ms")
+
+# -----------------------------
+# Parsers
+# -----------------------------
+def parse_cuda(file):
+    with open(file) as f:
+        for line in f:
+            m = cuda_pattern.search(line)
+            if m:
+                it = int(m.group(1))
+                cuda_data[it].append(float(m.group(2)))
+
+def parse_actor(file):
+    with open(file) as f:
+        for line in f:
+            m = actor_pattern.search(line)
+            if m:
+                it = int(m.group(1))
+                actor_facade_data[it].append(float(m.group(2)))
+
+def parse_runner(file):
+    with open(file) as f:
+        for line in f:
+            m = runner_pattern.search(line)
+            if m:
+                it = int(m.group(1))
+                command_runner_data[it].append(float(m.group(2)))
+
+# -----------------------------
+# Parse all files
+# -----------------------------
+for f in driver_files:
+    parse_cuda(f)
+
+for f in actor_files:
+    parse_actor(f)
+
+for f in runner_files:
+    parse_runner(f)
+
+# -----------------------------
+# Aggregate
+# -----------------------------
+iterations = sorted(cuda_data.keys())
+
+cuda_mean = [np.mean(cuda_data[i]) for i in iterations]
+actor_mean = [np.mean(actor_facade_data[i]) for i in iterations]
+runner_mean = [np.mean(command_runner_data[i]) for i in iterations]
+
+# -----------------------------
+# Print
 # -----------------------------
 print("\nMean Performance Comparison\n")
 print(f"{'Iterations':>10} {'CUDA(ms)':>12} {'Facade(ms)':>12} {'Runner(ms)':>12} {'Facade Ovhd %':>15}")
 
-for i in range(len(data["it"])):
-    pct = ((data["facade"][i] - data["cuda"][i]) / data["cuda"][i]) * 100
-    print(f"{data['it'][i]:>10} {data['cuda'][i]:>12.2f} {data['facade'][i]:>12.2f} {data['runner'][i]:>12.2f} {pct:>14.2f}%")
+for i, it in enumerate(iterations):
+    pct = ((actor_mean[i] - cuda_mean[i]) / cuda_mean[i]) * 100
+    print(f"{it:>10} {cuda_mean[i]:>12.2f} {actor_mean[i]:>12.2f} {runner_mean[i]:>12.2f} {pct:>14.2f}%")
 
 # -----------------------------
 # Plot
 # -----------------------------
 plt.figure(figsize=(8,6))
-plt.plot(data["it"], data["cuda"], marker='o', label="CUDA Native")
-plt.plot(data["it"], data["facade"], marker='s', label="CAF Actor Facade")
-plt.plot(data["it"], data["runner"], marker='^', label="CAF Command Runner")
 
-plt.xlabel("Iterations (Milestone)")
+plt.plot(iterations, cuda_mean, marker='o', label="CUDA Native")
+plt.plot(iterations, actor_mean, marker='s', label="Actor Facade")
+plt.plot(iterations, runner_mean, marker='^', label="Command Runner")
+
+plt.xlabel("Iterations")
 plt.ylabel("Time (ms)")
 plt.title("Matrix Multiplication Performance")
 plt.legend()
@@ -76,4 +109,4 @@ def parse_results():
 plt.tight_layout()
 
 plt.savefig(OUTPUT_PLOT)
-plt.show()
+plt.show()
\ No newline at end of file

From 119933ff1ec6e69167f34ac06dc40262e1b38de2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 11:35:42 -0600
Subject: [PATCH 0866/1000] updated title

---
 .../sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py b/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
index 7a0a73e684..8ba65d8965 100644
--- a/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
+++ b/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
@@ -122,7 +122,7 @@ def parse_latency(file):
 plt.figure()
 
 plt.plot(sizes, driver_means, marker='o', label="CUDA (Driver)")
-plt.plot(sizes, actor_means, marker='s', label="CUDA Actors")
+plt.plot(sizes, actor_means, marker='s', label="Command-Runner")
 plt.plot(sizes, latency_means, marker='^', label="Actor-Facade")
 
 plt.xlabel("Matrix Size (N)")

From 79b3d12892355947cd8a04d97cea7cdf9cdeb5dc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 12:14:36 -0600
Subject: [PATCH 0867/1000] updated

---
 .../tests/workload-test/hot-potatoe.cpp       | 652 +++++++-----------
 1 file changed, 235 insertions(+), 417 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
index e9d9ebfb40..c0b1ce9a70 100644
--- a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
+++ b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
@@ -1,417 +1,235 @@
-// #include <caf/all.hpp>
-// #include <caf/cuda/all.hpp>
-// #include <iostream>
-// #include <deque>
-// #include <vector>
-// #include <random>
-// #include <chrono>
-// #include <filesystem>
-// #include <memory>
-
-// #include "caf/actorSOLVE/actorSOLVE.hpp"
-// #include "sparse_utils.hpp"
-
-// using namespace caf;
-// using namespace caf::cuda;
-// namespace fs = std::filesystem;
-
-// // ============================================================
-// // TYPE BLOCK (unchanged + extended)
-// // ============================================================
-
-// CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
-
-//     CAF_ADD_ATOM(workload_test, get_work_atom)
-//     CAF_ADD_ATOM(workload_test, request_work_atom)
-//     CAF_ADD_ATOM(workload_test, worker_done_atom)
-//     CAF_ADD_ATOM(workload_test, work_tick_atom)
-
-//     CAF_ADD_TYPE_ID(workload_test, (SolverType))
-//     CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
-//     CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
-//     CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
-
-// CAF_END_TYPE_ID_BLOCK(workload_test)
-
-// CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
-// CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
-// CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
-
-// // ============================================================
-// // WORK TOKEN SYSTEM
-// // ============================================================
-
-// enum class Direction : uint8_t {
-//     L2R = 0,
-//     R2L = 1
-// };
-
-// inline Direction flip(Direction d) {
-//     return d == Direction::L2R ? Direction::R2L : Direction::L2R;
-// }
-
-// struct WorkToken {
-//     int device;
-//     int stream;
-//     size_t partition;
-//     Direction dir;
-// };
-
-// template <class Inspector>
-// bool inspect(Inspector& f, WorkToken& x) {
-//     return f.object(x).fields(
-//         f.field("device", x.device),
-//         f.field("stream", x.stream),
-//         f.field("partition", x.partition),
-//         f.field("dir", x.dir)
-//     );
-// }
-
-// // ============================================================
-// // PARTITION STATE (supervisor owns)
-// // ============================================================
-
-// struct Partition {
-//     size_t begin;
-//     size_t end;
-
-//     size_t active_tokens = 0;
-//     bool completed = false;
-// };
-
-// // ============================================================
-// // TASK ACTOR STATE
-// // ============================================================
-
-// struct task_state {
-//     MatrixTask task;
-
-//     actor supervisor;
-//     actor left;
-//     actor right;
-
-//     bool seen_l2r = false;
-//     bool seen_r2l = false;
-
-//     bool solve_started = false;
-
-//     std::optional<WorkToken> pending;
-
-//     caf::actor cg_facade;
-
-//     std::chrono::steady_clock::time_point start;
-// };
-
-// // ============================================================
-// // TASK ACTOR
-// // ============================================================
-
-// behavior task_actor(stateful_actor<task_state>* self) {
-
-//     return {
-
-//         // ----------------------------------------------------
-//         // INIT
-//         // ----------------------------------------------------
-//         [=](MatrixTask t,
-//             actor supervisor,
-//             actor left,
-//             actor right) {
-
-//             auto& st = self->state();
-
-//             st.task = std::move(t);
-//             st.supervisor = supervisor;
-//             st.left = left;
-//             st.right = right;
-
-//             st.cg_facade =
-//                 self->spawn<sparse_cg_facade<float>, linked>(0);
-//         },
-
-//         // ----------------------------------------------------
-//         // TOKEN PROPAGATION
-//         // ----------------------------------------------------
-//         [=](const WorkToken& tok) {
-
-//             auto& st = self->state();
-
-//             bool& seen_this =
-//                 tok.dir == Direction::L2R
-//                     ? st.seen_l2r
-//                     : st.seen_r2l;
-
-//             bool& seen_other =
-//                 tok.dir == Direction::L2R
-//                     ? st.seen_r2l
-//                     : st.seen_l2r;
-
-//             // ------------------------------------------------
-//             // COLLISION => PARTITION COMPLETED SIGNAL
-//             // ------------------------------------------------
-//             if (seen_other) {
-//                 self->send(
-//                     st.supervisor,
-//                     worker_done_atom_v,
-//                     tok.partition,
-//                     tok.device,
-//                     tok.stream
-//                 );
-//                 return;
-//             }
-
-//             // already visited this direction → just forward
-//             if (seen_this) {
-//                 actor next =
-//                     tok.dir == Direction::L2R
-//                         ? st.right
-//                         : st.left;
-
-//                 if (next)
-//                     self->send(next, tok);
-
-//                 return;
-//             }
-
-//             seen_this = true;
-
-//             // ------------------------------------------------
-//             // FIRST VISIT → LAUNCH SOLVER
-//             // ------------------------------------------------
-//             if (!st.solve_started) {
-//                 st.solve_started = true;
-//                 st.pending = tok;
-//                 st.start = std::chrono::steady_clock::now();
-
-//                 auto& d = *st.task.data;
-
-//                 self->mail(
-//                     create_in_arg(d.row_ptr),
-//                     create_in_arg(d.col_indices),
-//                     create_in_arg(d.values),
-//                     create_in_arg(d.b),
-//                     create_in_out_arg(d.x_guess),
-//                     matrix_format::csr,
-//                     (int)d.row_ptr.size() - 1,
-//                     (int)d.values.size(),
-//                     1e-5f,
-//                     2000,
-//                     tok.device,
-//                     tok.stream
-//                 ).send(st.cg_facade);
-
-//                 return;
-//             }
-
-//             // ------------------------------------------------
-//             // NORMAL FORWARD
-//             // ------------------------------------------------
-//             actor next =
-//                 tok.dir == Direction::L2R
-//                     ? st.right
-//                     : st.left;
-
-//             if (next)
-//                 self->send(next, tok);
-//         },
-
-//         // ----------------------------------------------------
-//         // SOLVER DONE
-//         // ----------------------------------------------------
-//         [=](uint32_t,
-//             int,
-//             std::vector<float>&,
-//             solver_result_meta meta) {
-
-//             auto& st = self->state();
-
-//             auto tok = *st.pending;
-
-//             actor next =
-//                 tok.dir == Direction::L2R
-//                     ? st.right
-//                     : st.left;
-
-//             if (next) {
-//                 self->send(next, tok);
-//             } else {
-//                 self->send(
-//                     st.supervisor,
-//                     worker_done_atom_v,
-//                     tok.partition,
-//                     tok.device,
-//                     tok.stream
-//                 );
-//             }
-//         }
-//     };
-// }
-
-// // ============================================================
-// // SUPERVISOR STATE
-// // ============================================================
-
-// struct supervisor_state {
-//     std::vector<MatrixTask> batch;
-//     std::vector<actor> actors;
-//     std::vector<Partition> partitions;
-
-//     size_t next_partition = 0;
-//     size_t completed = 0;
-
-//     int num_streams = 0;
-// };
-
-// // ============================================================
-// // SUPERVISOR
-// // ============================================================
-
-// behavior supervisor_actor(stateful_actor<supervisor_state>* self,
-//                           std::vector<MatrixTask> batch,
-//                           int num_gpus,
-//                           int streams_per_gpu) {
-
-//     auto& st = self->state();
-//     st.batch = std::move(batch);
-
-//     size_t num_parts = num_gpus * streams_per_gpu;
-
-//     st.partitions.resize(num_parts);
-
-//     for (size_t i = 0; i < num_parts; ++i) {
-//         size_t begin = (i * st.batch.size()) / num_parts;
-//         size_t end   = ((i + 1) * st.batch.size()) / num_parts;
-
-//         st.partitions[i] = {begin, end, 0, false};
-//     }
-
-//     // spawn actors
-//     int total = st.batch.size();
-
-//     for (int i = 0; i < total; ++i)
-//         st.actors.push_back(self->spawn(task_actor));
-
-//     // link neighbors
-//     for (size_t i = 0; i < st.actors.size(); ++i) {
-//         actor left  = (i == 0) ? actor{} : st.actors[i - 1];
-//         actor right = (i + 1 < st.actors.size()) ? st.actors[i + 1] : actor{};
-
-//         self->send(st.actors[i],
-//                    st.batch[i],
-//                    actor_cast<actor>(self),
-//                    left,
-//                    right);
-//     }
-
-//     // inject first wave
-//     auto& p = st.partitions[0];
-
-//     p.active_tokens = 1;
-
-//     self->send(
-//         st.actors[p.begin],
-//         WorkToken{
-//             0,
-//             0,
-//             0,
-//             Direction::L2R
-//         }
-//     );
-
-//     return {
-
-//         // ------------------------------------------------
-//         // TOKEN COMPLETION EVENT
-//         // ------------------------------------------------
-//         [=](worker_done_atom,
-//             size_t partition,
-//             int device,
-//             int stream) {
-
-//             auto& st = self->state();
-
-//             auto& p = st.partitions[partition];
-
-//             if (p.completed)
-//                 return;
-
-//             if (p.active_tokens > 0)
-//                 p.active_tokens--;
-
-//             if (p.active_tokens == 0) {
-//                 p.completed = true;
-//                 st.completed++;
-
-//                 std::cout << "[DONE] partition "
-//                           << partition << "\n";
-//             }
-
-//             // find next unfinished
-//             for (size_t i = 0; i < st.partitions.size(); ++i) {
-//                 size_t idx = (partition + i + 1)
-//                            % st.partitions.size();
-
-//                 if (!st.partitions[idx].completed) {
-
-//                     auto& np = st.partitions[idx];
-//                     np.active_tokens++;
-
-//                     Direction dir =
-//                         (i % 2 == 0)
-//                             ? Direction::L2R
-//                             : Direction::R2L;
-
-//                     size_t start =
-//                         (dir == Direction::L2R)
-//                             ? np.begin
-//                             : np.end - 1;
-
-//                     self->send(
-//                         st.actors[start],
-//                         WorkToken{
-//                             device,
-//                             stream,
-//                             idx,
-//                             dir
-//                         });
-
-//                     break;
-//                 }
-//             }
-//         }
-//     };
-// }
-
-// // ============================================================
-// // MAIN
-// // ============================================================
-
-// void caf_main(actor_system& sys) {
-
-//     manager::init(sys, manager_config(true, true));
-
-//     int streams = 8;
-//     int batches = 25;
-//     int batch_size = 100;
-
-//     auto tasks = scan_for_matrices(
-//         "/scratch/nqr159/matrix-collection",
-//         CGS_SOLVER
-//     );
-
-//     auto batch = generate_batch(tasks, std::mt19937{42}, batch_size);
-
-//     manager& mgr = manager::get();
-//     int gpus = mgr.get_num_devices();
-
-//     sys.spawn(supervisor_actor,
-//               batch,
-//               gpus,
-//               streams);
-
-//     sys.await_all_actors_done();
-
-//     manager::shutdown();
-// }
-
-// CAF_MAIN(id_block::cuda, workload_test)
\ No newline at end of file
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <deque>
+#include <vector>
+#include <random>
+#include <chrono>
+#include <filesystem>
+#include <memory>
+
+#include "caf/actorSOLVE/actorSOLVE.hpp"
+#include "sparse_utils.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+namespace fs = std::filesystem;
+
+// ============================================================
+// TYPE BLOCK (unchanged + extended)
+// ============================================================
+
+CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
+
+    CAF_ADD_ATOM(workload_test, get_work_atom)
+    CAF_ADD_ATOM(workload_test, start_atom)
+    CAF_ADD_ATOM(workload_test, started_atom)
+    CAF_ADD_ATOM(workload_test, neighbor_atom)
+    CAF_ADD_ATOM(workload_test, request_work_atom)
+    CAF_ADD_ATOM(workload_test, worker_done_atom)
+    CAF_ADD_ATOM(workload_test, work_tick_atom)
+
+    CAF_ADD_TYPE_ID(workload_test, (SolverType))
+    CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
+    CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
+    CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
+
+CAF_END_TYPE_ID_BLOCK(workload_test)
+
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
+
+// ============================================================
+// PARTITION STATE (supervisor owns)
+// ============================================================
+
+struct Partition {
+    size_t begin;
+    size_t end;
+
+    size_t active_tokens = 0;
+    bool completed = false;
+};
+
+// ============================================================
+// TASK ACTOR STATE
+// ============================================================
+
+struct worker_state {
+    MatrixTask task;
+    actor supervisor;
+    actor neighbor;
+
+    int device_id;
+    int stream_id;
+
+    caf::actor cg_facade;
+    
+    bool solve_done = false;
+    bool neighbor_received = false;
+};
+
+// ============================================================
+// TASK ACTOR
+// ============================================================
+
+behavior worker_actor(stateful_actor<worker_state>* self, MatrixTask t, actor supervisor) {
+    auto& st = self->state();
+    st.task = std::move(t);
+    st.supervisor = supervisor;
+    st.cg_facade = self->spawn<sparse_cg_facade<float>, linked>(0);
+    
+    return {
+        [=](start_atom, int dev, int stream) {
+            auto& s = self->state();
+            s.device_id = dev;
+            s.stream_id = stream;
+
+            // Alert master that we have begun work
+            self->send(s.supervisor, started_atom_v);
+
+            // Execute solver
+            auto& d = *s.task.data;
+            self->mail(
+                create_in_arg(d.row_ptr),
+                create_in_arg(d.col_indices),
+                create_in_arg(d.values),
+                create_in_arg(d.b),
+                create_in_out_arg(d.x_guess),
+                matrix_format::csr,
+                (int)d.row_ptr.size() - 1,
+                (int)d.values.size(),
+                1e-5f,
+                2000,
+                dev,
+                stream
+            ).send(s.cg_facade);
+        },
+
+        [=](neighbor_atom, actor n) {
+            auto& s = self->state();
+            s.neighbor = n;
+            s.neighbor_received = true;
+            
+            if (s.solve_done) {
+                if (s.neighbor)
+                    self->send(s.neighbor, start_atom_v, s.device_id, s.stream_id);
+                self->quit();
+            }
+        },
+
+        [=](uint32_t, int, std::vector<float>&, solver_result_meta) {
+            auto& s = self->state();
+            s.solve_done = true;
+            self->send(s.supervisor, worker_done_atom_v);
+
+            if (s.neighbor_received) {
+                if (s.neighbor)
+                    self->send(s.neighbor, start_atom_v, s.device_id, s.stream_id);
+                self->quit();
+            }
+        }
+    };
+}
+
+// ============================================================
+// SUPERVISOR STATE
+// ============================================================
+
+struct supervisor_state {
+    std::vector<MatrixTask> batch;
+
+    size_t next_task_idx = 0;
+    size_t completed = 0;
+
+    int num_streams = 0;
+};
+
+// ============================================================
+// SUPERVISOR
+// ============================================================
+
+behavior supervisor_actor(stateful_actor<supervisor_state>* self,
+                          std::vector<MatrixTask> batch,
+                          int num_gpus,
+                          int streams_per_gpu) {
+    auto& st = self->state();
+    st.batch = std::move(batch);
+
+    size_t num_parts = num_gpus * streams_per_gpu;
+
+    // Decision Logic: Initially create and start one worker per stream slot
+    for (size_t i = 0; i < num_parts; ++i) {
+        if (st.next_task_idx < st.batch.size()) {
+            int dev = i / streams_per_gpu;
+            int stream = i % streams_per_gpu;
+            auto w = self->spawn(worker_actor, st.batch[st.next_task_idx++], actor_cast<actor>(self));
+            self->send(w, start_atom_v, dev, stream);
+        }
+    }
+
+    return {
+        [=](started_atom) {
+            auto& s = self->state();
+            // Master Decision: Assign the next available task as a neighbor to the worker that just started
+            if (s.next_task_idx < s.batch.size()) {
+                auto next_worker = self->spawn(worker_actor, s.batch[s.next_task_idx++], actor_cast<actor>(self));
+                self->send(self->current_sender(), neighbor_atom_v, next_worker);
+            } else {
+                // No more tasks to assign
+                self->send(self->current_sender(), neighbor_atom_v, actor{});
+            }
+        },
+
+        [=](worker_done_atom) {
+            auto& s = self->state();
+            s.completed++;
+            
+            if (s.completed % 10 == 0 || s.completed == s.batch.size()) {
+                std::cout << "[PROGRESS] Completed " 
+                          << s.completed << "/" 
+                          << s.batch.size() << " tasks\n";
+            }
+
+            if (s.completed >= s.batch.size()) {
+                std::cout << "[INFO] All tasks completed. Shutting down supervisor.\n";
+                self->quit();
+            }
+        }
+    };
+}
+
+// ============================================================
+// MAIN
+// ============================================================
+
+void caf_main(actor_system& sys) {
+
+    manager::init(sys, manager_config(true, true));
+
+    int streams = 8;
+    int batches = 25;
+    int batch_size = 100;
+
+    auto tasks = scan_for_matrices(
+        "/scratch/nqr159/matrix-collection",
+        CGS_SOLVER
+    );
+
+    auto batch = generate_batch(tasks, std::mt19937{42}, batch_size);
+
+    manager& mgr = manager::get();
+    int gpus = mgr.get_num_devices();
+
+    sys.spawn(supervisor_actor,
+              batch,
+              gpus,
+              streams);
+
+    sys.await_all_actors_done();
+
+    manager::shutdown();
+}
+
+CAF_MAIN(id_block::cuda, workload_test)
\ No newline at end of file

From fb1eb14b858e1bc5357e41b64d4b64f1ae089905 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 12:19:55 -0600
Subject: [PATCH 0868/1000] updated to be better

---
 .../tests/workload-test/hot-potatoe.cpp       | 62 ++++++-------------
 1 file changed, 19 insertions(+), 43 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
index c0b1ce9a70..42845ad708 100644
--- a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
+++ b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
@@ -40,18 +40,6 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
 
-// ============================================================
-// PARTITION STATE (supervisor owns)
-// ============================================================
-
-struct Partition {
-    size_t begin;
-    size_t end;
-
-    size_t active_tokens = 0;
-    bool completed = false;
-};
-
 // ============================================================
 // TASK ACTOR STATE
 // ============================================================
@@ -111,10 +99,9 @@ behavior worker_actor(stateful_actor<worker_state>* self, MatrixTask t, actor su
             auto& s = self->state();
             s.neighbor = n;
             s.neighbor_received = true;
-            
-            if (s.solve_done) {
-                if (s.neighbor)
-                    self->send(s.neighbor, start_atom_v, s.device_id, s.stream_id);
+
+            if (s.solve_done && s.neighbor) {
+                self->send(s.neighbor, start_atom_v, s.device_id, s.stream_id);
                 self->quit();
             }
         },
@@ -122,11 +109,8 @@ behavior worker_actor(stateful_actor<worker_state>* self, MatrixTask t, actor su
         [=](uint32_t, int, std::vector<float>&, solver_result_meta) {
             auto& s = self->state();
             s.solve_done = true;
-            self->send(s.supervisor, worker_done_atom_v);
-
-            if (s.neighbor_received) {
-                if (s.neighbor)
-                    self->send(s.neighbor, start_atom_v, s.device_id, s.stream_id);
+            if (s.neighbor_received && s.neighbor) {
+                self->send(s.neighbor, start_atom_v, s.device_id, s.stream_id);
                 self->quit();
             }
         }
@@ -139,11 +123,9 @@ behavior worker_actor(stateful_actor<worker_state>* self, MatrixTask t, actor su
 
 struct supervisor_state {
     std::vector<MatrixTask> batch;
-
     size_t next_task_idx = 0;
-    size_t completed = 0;
-
-    int num_streams = 0;
+    size_t returned_potatoes = 0;
+    size_t active_slots = 0;
 };
 
 // ============================================================
@@ -157,15 +139,16 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
     auto& st = self->state();
     st.batch = std::move(batch);
 
-    size_t num_parts = num_gpus * streams_per_gpu;
+    size_t total_slots = static_cast<size_t>(num_gpus * streams_per_gpu);
 
-    // Decision Logic: Initially create and start one worker per stream slot
-    for (size_t i = 0; i < num_parts; ++i) {
+    // Dynamically spawn only the first set of workers to fill the GPU slots
+    for (size_t i = 0; i < total_slots; ++i) {
         if (st.next_task_idx < st.batch.size()) {
-            int dev = i / streams_per_gpu;
-            int stream = i % streams_per_gpu;
+            int dev = static_cast<int>(i / streams_per_gpu);
+            int stream = static_cast<int>(i % streams_per_gpu);
             auto w = self->spawn(worker_actor, st.batch[st.next_task_idx++], actor_cast<actor>(self));
             self->send(w, start_atom_v, dev, stream);
+            st.active_slots++;
         }
     }
 
@@ -177,23 +160,16 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
                 auto next_worker = self->spawn(worker_actor, s.batch[s.next_task_idx++], actor_cast<actor>(self));
                 self->send(self->current_sender(), neighbor_atom_v, next_worker);
             } else {
-                // No more tasks to assign
-                self->send(self->current_sender(), neighbor_atom_v, actor{});
+                // No more tasks to assign: the Supervisor becomes the neighbor (to reclaim resources)
+                self->send(self->current_sender(), neighbor_atom_v, actor_cast<actor>(self));
             }
         },
 
-        [=](worker_done_atom) {
+        [=](start_atom, int, int) {
             auto& s = self->state();
-            s.completed++;
-            
-            if (s.completed % 10 == 0 || s.completed == s.batch.size()) {
-                std::cout << "[PROGRESS] Completed " 
-                          << s.completed << "/" 
-                          << s.batch.size() << " tasks\n";
-            }
-
-            if (s.completed >= s.batch.size()) {
-                std::cout << "[INFO] All tasks completed. Shutting down supervisor.\n";
+            // Potato returned! One processing chain has finished.
+            if (++s.returned_potatoes == s.active_slots) {
+                self->println("[INFO] All {} GPU streams returned. Shutting down.", s.active_slots);
                 self->quit();
             }
         }

From 163e8b62212512d6d1b7f36d5733ded2651318fb Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 12:26:40 -0600
Subject: [PATCH 0869/1000] added producer actor

---
 .../tests/workload-test/hot-potatoe.cpp       | 155 ++++++++++++++----
 1 file changed, 121 insertions(+), 34 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
index 42845ad708..f43b20b576 100644
--- a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
+++ b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
@@ -28,11 +28,13 @@ CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
     CAF_ADD_ATOM(workload_test, request_work_atom)
     CAF_ADD_ATOM(workload_test, worker_done_atom)
     CAF_ADD_ATOM(workload_test, work_tick_atom)
+    CAF_ADD_ATOM(workload_test, add_work_atom)
 
     CAF_ADD_TYPE_ID(workload_test, (SolverType))
     CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
     CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
     CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
+    CAF_ADD_TYPE_ID(workload_test, (std::deque<MatrixTask>))
 
 CAF_END_TYPE_ID_BLOCK(workload_test)
 
@@ -117,15 +119,65 @@ behavior worker_actor(stateful_actor<worker_state>* self, MatrixTask t, actor su
     };
 }
 
+// ============================================================
+// PRODUCER ACTOR
+// ============================================================
+
+struct producer_state {
+    std::vector<MatrixTask> matrix_pool;
+    int num_batches;
+    int batch_size;
+    double mean_arrival_ms;
+    actor supervisor;
+    std::mt19937 rng;
+    int batches_sent = 0;
+};
+
+behavior producer_actor(stateful_actor<producer_state>* self,
+                       std::vector<MatrixTask> pool,
+                       int num_batches, int batch_size, 
+                       double mean_ms, actor supervisor) {
+    auto& st = self->state();
+    st.matrix_pool = std::move(pool);
+    st.num_batches = num_batches;
+    st.batch_size = batch_size;
+    st.mean_arrival_ms = mean_ms;
+    st.supervisor = std::move(supervisor);
+    st.rng.seed(42);
+
+    return {
+        [=](work_tick_atom) {
+            auto& s = self->state();
+            if (s.batches_sent < s.num_batches) {
+                auto batch = generate_batch(s.matrix_pool, s.rng, s.batch_size);
+                self->send(s.supervisor, add_work_atom_v, std::move(batch));
+                s.batches_sent++;
+
+                if (s.batches_sent < s.num_batches) {
+                    auto delay = generate_random_interval(s.rng, s.mean_arrival_ms);
+                    self->delayed_send(self, delay, work_tick_atom_v);
+                }
+            }
+        }
+    };
+}
+
 // ============================================================
 // SUPERVISOR STATE
 // ============================================================
 
+struct resource_slot {
+    int device_id;
+    int stream_id;
+};
+
 struct supervisor_state {
-    std::vector<MatrixTask> batch;
-    size_t next_task_idx = 0;
-    size_t returned_potatoes = 0;
-    size_t active_slots = 0;
+    std::deque<MatrixTask> pending_queue;
+    std::deque<resource_slot> available_slots;
+    
+    size_t total_expected;
+    size_t completed = 0;
+    bool initialized = false;
 };
 
 // ============================================================
@@ -133,43 +185,74 @@ struct supervisor_state {
 // ============================================================
 
 behavior supervisor_actor(stateful_actor<supervisor_state>* self,
-                          std::vector<MatrixTask> batch,
+                          size_t total_tasks,
                           int num_gpus,
                           int streams_per_gpu) {
-    auto& st = self->state();
-    st.batch = std::move(batch);
-
-    size_t total_slots = static_cast<size_t>(num_gpus * streams_per_gpu);
-
-    // Dynamically spawn only the first set of workers to fill the GPU slots
-    for (size_t i = 0; i < total_slots; ++i) {
-        if (st.next_task_idx < st.batch.size()) {
-            int dev = static_cast<int>(i / streams_per_gpu);
-            int stream = static_cast<int>(i % streams_per_gpu);
-            auto w = self->spawn(worker_actor, st.batch[st.next_task_idx++], actor_cast<actor>(self));
-            self->send(w, start_atom_v, dev, stream);
-            st.active_slots++;
-        }
-    }
+    self->state().total_expected = total_tasks;
 
     return {
+        [=](add_work_atom, std::vector<MatrixTask>& batch) {
+            auto& s = self->state();
+            for (auto& t : batch)
+                s.pending_queue.push_back(std::move(t));
+
+            // Bootstrapping: fill GPU slots with the first available tasks
+            if (!s.initialized) {
+                s.initialized = true;
+                size_t total_slots = static_cast<size_t>(num_gpus * streams_per_gpu);
+                for (size_t i = 0; i < total_slots; ++i) {
+                    int dev = static_cast<int>(i / streams_per_gpu);
+                    int stream = static_cast<int>(i % streams_per_gpu);
+                    
+                    if (!s.pending_queue.empty()) {
+                        auto w = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
+                        s.pending_queue.pop_front();
+                        self->send(w, start_atom_v, dev, stream);
+                    } else {
+                        s.available_slots.push_back({dev, stream});
+                    }
+                }
+            } else {
+                // New work arrived, check if we have idle GPU slots to fill
+                while (!s.available_slots.empty() && !s.pending_queue.empty()) {
+                    auto slot = s.available_slots.front();
+                    s.available_slots.pop_front();
+                    auto w = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
+                    s.pending_queue.pop_front();
+                    self->send(w, start_atom_v, slot.device_id, slot.stream_id);
+                }
+            }
+        },
+
         [=](started_atom) {
             auto& s = self->state();
-            // Master Decision: Assign the next available task as a neighbor to the worker that just started
-            if (s.next_task_idx < s.batch.size()) {
-                auto next_worker = self->spawn(worker_actor, s.batch[s.next_task_idx++], actor_cast<actor>(self));
+            if (!s.pending_queue.empty()) {
+                auto next_worker = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
+                s.pending_queue.pop_front();
                 self->send(self->current_sender(), neighbor_atom_v, next_worker);
             } else {
-                // No more tasks to assign: the Supervisor becomes the neighbor (to reclaim resources)
                 self->send(self->current_sender(), neighbor_atom_v, actor_cast<actor>(self));
             }
         },
 
-        [=](start_atom, int, int) {
+        [=](start_atom, int dev, int stream) {
             auto& s = self->state();
-            // Potato returned! One processing chain has finished.
-            if (++s.returned_potatoes == s.active_slots) {
-                self->println("[INFO] All {} GPU streams returned. Shutting down.", s.active_slots);
+            s.completed++;
+
+            if (s.completed % 25 == 0 || s.completed == s.total_expected) {
+                self->println("[PROGRESS] Completed {}/{} tasks", s.completed, s.total_expected);
+            }
+
+            if (!s.pending_queue.empty()) {
+                auto w = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
+                s.pending_queue.pop_front();
+                self->send(w, start_atom_v, dev, stream);
+            } else {
+                s.available_slots.push_back({dev, stream});
+            }
+
+            if (s.completed >= s.total_expected) {
+                self->println("[INFO] All {} tasks completed. Shutting down.", s.total_expected);
                 self->quit();
             }
         }
@@ -187,24 +270,28 @@ void caf_main(actor_system& sys) {
     int streams = 8;
     int batches = 25;
     int batch_size = 100;
+    double mean_arrival = 1000.0;
 
     auto tasks = scan_for_matrices(
-        "/scratch/nqr159/matrix-collection",
+        "/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd",
         CGS_SOLVER
     );
 
-    auto batch = generate_batch(tasks, std::mt19937{42}, batch_size);
-
     manager& mgr = manager::get();
     int gpus = mgr.get_num_devices();
 
-    sys.spawn(supervisor_actor,
-              batch,
+    auto supervisor = sys.spawn(supervisor_actor,
+              static_cast<size_t>(batches * batch_size),
               gpus,
               streams);
 
-    sys.await_all_actors_done();
+    auto producer = sys.spawn(producer_actor,
+                             std::move(tasks), batches, batch_size, mean_arrival,
+                             supervisor);
 
+    anon_send(producer, work_tick_atom_v);
+
+    sys.await_all_actors_done();
     manager::shutdown();
 }
 

From c1cb8d70fd38677d14cd336ece829fdb789fc004 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 12:34:37 -0600
Subject: [PATCH 0870/1000] fixed issues

---
 .../tests/workload-test/CMakeLists.txt        | 28 +++++++-------
 .../tests/workload-test/hot-potatoe.cpp       | 38 ++++++++++++-------
 .../tests/workload-test/main.native.cpp       | 18 ++++-----
 libcaf_cuda/tests/workload-test/main.test.cpp |  3 +-
 4 files changed, 48 insertions(+), 39 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/CMakeLists.txt b/libcaf_cuda/tests/workload-test/CMakeLists.txt
index dcf78fb02c..d7ca8d87fa 100644
--- a/libcaf_cuda/tests/workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/workload-test/CMakeLists.txt
@@ -45,20 +45,20 @@ target_link_libraries(test
 )
 
 
-# # 5) Declare your executable
-# add_executable(hot-potatoe hot-potatoe.cpp sparse_utils.cpp)
-
-# target_compile_definitions(hot-potatoe PRIVATE CAF_ENABLE_LOGGING)
-
-# target_link_libraries(hot-potatoe
-#   PRIVATE
-#     "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-#     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-#     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-#     CUDA::nvrtc
-#     CUDA::cublas
-#     CUDA::cusparse
-# )
+# 5) Declare your executable
+add_executable(hot-potatoe hot-potatoe.cpp sparse_utils.cpp)
+
+target_compile_definitions(hot-potatoe PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(hot-potatoe
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
+)
 
 
 # FindThreads is required for std::thread in the native version
diff --git a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
index f43b20b576..c83247df57 100644
--- a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
+++ b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
@@ -15,6 +15,17 @@ using namespace caf;
 using namespace caf::cuda;
 namespace fs = std::filesystem;
 
+template <class Inspector>
+bool inspect(Inspector& f, SolverType& x) {
+    auto val = static_cast<int>(x);
+    if (f.apply(val)) {
+        if constexpr (Inspector::is_loading)
+            x = static_cast<SolverType>(val);
+        return true;
+    }
+    return false;
+}
+
 // ============================================================
 // TYPE BLOCK (unchanged + extended)
 // ============================================================
@@ -22,7 +33,6 @@ namespace fs = std::filesystem;
 CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
 
     CAF_ADD_ATOM(workload_test, get_work_atom)
-    CAF_ADD_ATOM(workload_test, start_atom)
     CAF_ADD_ATOM(workload_test, started_atom)
     CAF_ADD_ATOM(workload_test, neighbor_atom)
     CAF_ADD_ATOM(workload_test, request_work_atom)
@@ -77,7 +87,7 @@ behavior worker_actor(stateful_actor<worker_state>* self, MatrixTask t, actor su
             s.stream_id = stream;
 
             // Alert master that we have begun work
-            self->send(s.supervisor, started_atom_v);
+            self->mail(started_atom_v).send(s.supervisor);
 
             // Execute solver
             auto& d = *s.task.data;
@@ -103,7 +113,7 @@ behavior worker_actor(stateful_actor<worker_state>* self, MatrixTask t, actor su
             s.neighbor_received = true;
 
             if (s.solve_done && s.neighbor) {
-                self->send(s.neighbor, start_atom_v, s.device_id, s.stream_id);
+                self->mail(start_atom_v, s.device_id, s.stream_id).send(s.neighbor);
                 self->quit();
             }
         },
@@ -112,7 +122,7 @@ behavior worker_actor(stateful_actor<worker_state>* self, MatrixTask t, actor su
             auto& s = self->state();
             s.solve_done = true;
             if (s.neighbor_received && s.neighbor) {
-                self->send(s.neighbor, start_atom_v, s.device_id, s.stream_id);
+                self->mail(start_atom_v, s.device_id, s.stream_id).send(s.neighbor);
                 self->quit();
             }
         }
@@ -150,12 +160,12 @@ behavior producer_actor(stateful_actor<producer_state>* self,
             auto& s = self->state();
             if (s.batches_sent < s.num_batches) {
                 auto batch = generate_batch(s.matrix_pool, s.rng, s.batch_size);
-                self->send(s.supervisor, add_work_atom_v, std::move(batch));
+                self->mail(add_work_atom_v, std::move(batch)).send(s.supervisor);
                 s.batches_sent++;
 
                 if (s.batches_sent < s.num_batches) {
                     auto delay = generate_random_interval(s.rng, s.mean_arrival_ms);
-                    self->delayed_send(self, delay, work_tick_atom_v);
+                    self->mail(work_tick_atom_v).delay(delay).send(self);
                 }
             }
         }
@@ -207,7 +217,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
                     if (!s.pending_queue.empty()) {
                         auto w = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
                         s.pending_queue.pop_front();
-                        self->send(w, start_atom_v, dev, stream);
+                        self->mail(start_atom_v, dev, stream).send(w);
                     } else {
                         s.available_slots.push_back({dev, stream});
                     }
@@ -219,7 +229,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
                     s.available_slots.pop_front();
                     auto w = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
                     s.pending_queue.pop_front();
-                    self->send(w, start_atom_v, slot.device_id, slot.stream_id);
+                    self->mail(start_atom_v, slot.device_id, slot.stream_id).send(w);
                 }
             }
         },
@@ -229,9 +239,9 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
             if (!s.pending_queue.empty()) {
                 auto next_worker = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
                 s.pending_queue.pop_front();
-                self->send(self->current_sender(), neighbor_atom_v, next_worker);
+                self->mail(neighbor_atom_v, next_worker).send(actor_cast<actor>(self->current_sender()));
             } else {
-                self->send(self->current_sender(), neighbor_atom_v, actor_cast<actor>(self));
+                self->mail(neighbor_atom_v, actor_cast<actor>(self)).send(actor_cast<actor>(self->current_sender()));
             }
         },
 
@@ -246,7 +256,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
             if (!s.pending_queue.empty()) {
                 auto w = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
                 s.pending_queue.pop_front();
-                self->send(w, start_atom_v, dev, stream);
+                self->mail(start_atom_v, dev, stream).send(w);
             } else {
                 s.available_slots.push_back({dev, stream});
             }
@@ -267,7 +277,7 @@ void caf_main(actor_system& sys) {
 
     manager::init(sys, manager_config(true, true));
 
-    int streams = 8;
+    int streams = 4;
     int batches = 25;
     int batch_size = 100;
     double mean_arrival = 1000.0;
@@ -289,10 +299,10 @@ void caf_main(actor_system& sys) {
                              std::move(tasks), batches, batch_size, mean_arrival,
                              supervisor);
 
-    anon_send(producer, work_tick_atom_v);
+    anon_mail(work_tick_atom_v).send(producer);
 
     sys.await_all_actors_done();
     manager::shutdown();
 }
 
-CAF_MAIN(id_block::cuda, workload_test)
\ No newline at end of file
+CAF_MAIN(id_block::cuda, id_block::workload_test)
\ No newline at end of file
diff --git a/libcaf_cuda/tests/workload-test/main.native.cpp b/libcaf_cuda/tests/workload-test/main.native.cpp
index 9c7a48b6c4..cc96cac393 100644
--- a/libcaf_cuda/tests/workload-test/main.native.cpp
+++ b/libcaf_cuda/tests/workload-test/main.native.cpp
@@ -326,22 +326,22 @@ void gpu_stream_worker(
 
 int main(int argc, char** argv)
 {
-    int num_streams = 8;
+    int num_streams = 4;
     int num_batches = 25;
     int batch_size = 100;
     double mean_arrival_ms = 1000.0;
 
-    if (argc > 1)
-        num_streams = std::max(1, std::atoi(argv[1]));
+    // if (argc > 1)
+    //     num_streams = std::max(1, std::atoi(argv[1]));
 
-    if (argc > 2)
-        num_batches = std::max(1, std::atoi(argv[2]));
+    // if (argc > 2)
+    //     num_batches = std::max(1, std::atoi(argv[2]));
 
-    if (argc > 3)
-        batch_size = std::max(1, std::atoi(argv[3]));
+    // if (argc > 3)
+    //     batch_size = std::max(1, std::atoi(argv[3]));
 
-    if (argc > 4)
-        mean_arrival_ms = std::atof(argv[4]);
+    // if (argc > 4)
+    //     mean_arrival_ms = std::atof(argv[4]);
 
     std::cout << "[INFO] Loading matrices...\n";
 
diff --git a/libcaf_cuda/tests/workload-test/main.test.cpp b/libcaf_cuda/tests/workload-test/main.test.cpp
index 1f183a9973..1c238ead11 100644
--- a/libcaf_cuda/tests/workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/workload-test/main.test.cpp
@@ -43,7 +43,6 @@ CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
     CAF_ADD_TYPE_ID(workload_test, (SolverType))
     CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
     CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
-    CAF_ADD_TYPE_ID(workload_test, (std::vector<caf::actor>))
     CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
 CAF_END_TYPE_ID_BLOCK(workload_test)
 
@@ -267,7 +266,7 @@ behavior supervisor_actor_fun(stateful_actor<supervisor_state>* self,
 void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
     
-    int num_streams = 8;
+    int num_streams = 4;
     int num_batches = 25;
     int batch_size = 100;
     double mean_arrival_ms = 1000.0;

From 2e2add8043fa9b717df987b53a4b8a478656272a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 12:42:30 -0600
Subject: [PATCH 0871/1000] added timing logic

---
 .../tests/workload-test/hot-potatoe.cpp       | 48 ++++++++++++++++---
 1 file changed, 42 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
index c83247df57..88abc24e86 100644
--- a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
+++ b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
@@ -15,6 +15,8 @@ using namespace caf;
 using namespace caf::cuda;
 namespace fs = std::filesystem;
 
+constexpr uint32_t WORKLOAD_SEED = 42;
+
 template <class Inspector>
 bool inspect(Inspector& f, SolverType& x) {
     auto val = static_cast<int>(x);
@@ -153,7 +155,7 @@ behavior producer_actor(stateful_actor<producer_state>* self,
     st.batch_size = batch_size;
     st.mean_arrival_ms = mean_ms;
     st.supervisor = std::move(supervisor);
-    st.rng.seed(42);
+    st.rng.seed(WORKLOAD_SEED);
 
     return {
         [=](work_tick_atom) {
@@ -188,6 +190,7 @@ struct supervisor_state {
     size_t total_expected;
     size_t completed = 0;
     bool initialized = false;
+    actor parent;
 };
 
 // ============================================================
@@ -197,8 +200,10 @@ struct supervisor_state {
 behavior supervisor_actor(stateful_actor<supervisor_state>* self,
                           size_t total_tasks,
                           int num_gpus,
-                          int streams_per_gpu) {
+                          int streams_per_gpu,
+                          actor parent) {
     self->state().total_expected = total_tasks;
+    self->state().parent = std::move(parent);
 
     return {
         [=](add_work_atom, std::vector<MatrixTask>& batch) {
@@ -263,6 +268,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
 
             if (s.completed >= s.total_expected) {
                 self->println("[INFO] All {} tasks completed. Shutting down.", s.total_expected);
+                self->mail(worker_done_atom_v).send(s.parent);
                 self->quit();
             }
         }
@@ -290,18 +296,48 @@ void caf_main(actor_system& sys) {
     manager& mgr = manager::get();
     int gpus = mgr.get_num_devices();
 
-    auto supervisor = sys.spawn(supervisor_actor,
+    std::cout << "[INFO] Streams/GPU:      " << streams << "\n";
+    std::cout << "[INFO] Batches:          " << batches << "\n";
+    std::cout << "[INFO] Batch size:       " << batch_size << "\n";
+    std::cout << "[INFO] Mean arrival:     " << mean_arrival << " ms\n";
+
+    auto start = std::chrono::steady_clock::now();
+    scoped_actor self{sys};
+
+    auto supervisor = self->spawn(supervisor_actor,
               static_cast<size_t>(batches * batch_size),
               gpus,
-              streams);
+              streams,
+              actor_cast<actor>(self));
 
-    auto producer = sys.spawn(producer_actor,
+    auto producer = self->spawn(producer_actor,
                              std::move(tasks), batches, batch_size, mean_arrival,
                              supervisor);
 
     anon_mail(work_tick_atom_v).send(producer);
 
-    sys.await_all_actors_done();
+    self->receive(
+        [&](worker_done_atom) {
+            std::cout << "[INFO] Supervisor signaled completion. Shutting down...\n";
+        }
+    );
+
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+
+    int total_tasks = batches * batch_size;
+
+    std::cout << "\n===== BENCHMARK COMPLETE =====\n";
+    std::cout << "Seed:               " << WORKLOAD_SEED << "\n";
+    std::cout << "Streams per GPU:    " << streams << "\n";
+    std::cout << "Batches:            " << batches << "\n";
+    std::cout << "Batch Size:         " << batch_size << "\n";
+    std::cout << "Mean Arrival (ms):  " << mean_arrival << "\n";
+    std::cout << "Tasks Processed:    " << total_tasks << "\n";
+    std::cout << "Total Runtime:      " << elapsed.count() << " s\n";
+    std::cout << "Throughput:         " << total_tasks / elapsed.count() << " tasks/s\n";
+    std::cout << "==============================\n";
+
     manager::shutdown();
 }
 

From 5775bffcf13c3220921fb1e2bf9642972b7bfc50 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 13:06:45 -0600
Subject: [PATCH 0872/1000] fixed exit deadlock

---
 .../tests/workload-test/hot-potatoe.cpp       | 34 ++++++++++++++-----
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
index 88abc24e86..787ac28dad 100644
--- a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
+++ b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
@@ -168,7 +168,11 @@ behavior producer_actor(stateful_actor<producer_state>* self,
                 if (s.batches_sent < s.num_batches) {
                     auto delay = generate_random_interval(s.rng, s.mean_arrival_ms);
                     self->mail(work_tick_atom_v).delay(delay).send(self);
+                } else {
+                    self->quit();
                 }
+            } else {
+                self->quit();
             }
         }
     };
@@ -189,6 +193,8 @@ struct supervisor_state {
     
     size_t total_expected;
     size_t completed = 0;
+    int num_gpus;
+    int streams_per_gpu;
     bool initialized = false;
     actor parent;
 };
@@ -204,6 +210,14 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
                           actor parent) {
     self->state().total_expected = total_tasks;
     self->state().parent = std::move(parent);
+    self->state().num_gpus = num_gpus;
+    self->state().streams_per_gpu = streams_per_gpu;
+
+    if (total_tasks == 0) {
+        self->mail(worker_done_atom_v).send(self->state().parent);
+        self->quit();
+        return {};
+    }
 
     return {
         [=](add_work_atom, std::vector<MatrixTask>& batch) {
@@ -222,6 +236,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
                     if (!s.pending_queue.empty()) {
                         auto w = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
                         s.pending_queue.pop_front();
+                        s.completed++;
                         self->mail(start_atom_v, dev, stream).send(w);
                     } else {
                         s.available_slots.push_back({dev, stream});
@@ -233,6 +248,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
                     auto slot = s.available_slots.front();
                     s.available_slots.pop_front();
                     auto w = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
+                    s.completed++;
                     s.pending_queue.pop_front();
                     self->mail(start_atom_v, slot.device_id, slot.stream_id).send(w);
                 }
@@ -244,6 +260,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
             if (!s.pending_queue.empty()) {
                 auto next_worker = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
                 s.pending_queue.pop_front();
+                s.completed++;
                 self->mail(neighbor_atom_v, next_worker).send(actor_cast<actor>(self->current_sender()));
             } else {
                 self->mail(neighbor_atom_v, actor_cast<actor>(self)).send(actor_cast<actor>(self->current_sender()));
@@ -252,22 +269,18 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
 
         [=](start_atom, int dev, int stream) {
             auto& s = self->state();
-            s.completed++;
-
-            if (s.completed % 25 == 0 || s.completed == s.total_expected) {
-                self->println("[PROGRESS] Completed {}/{} tasks", s.completed, s.total_expected);
-            }
-
             if (!s.pending_queue.empty()) {
                 auto w = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
                 s.pending_queue.pop_front();
+                s.completed++;
                 self->mail(start_atom_v, dev, stream).send(w);
             } else {
                 s.available_slots.push_back({dev, stream});
             }
 
-            if (s.completed >= s.total_expected) {
-                self->println("[INFO] All {} tasks completed. Shutting down.", s.total_expected);
+            size_t total_slots = static_cast<size_t>(s.num_gpus * s.streams_per_gpu);
+            if (s.completed == s.total_expected && s.available_slots.size() == total_slots) {
+                self->println("[INFO] All {} tasks finished. Shutting down.", s.total_expected);
                 self->mail(worker_done_atom_v).send(s.parent);
                 self->quit();
             }
@@ -293,6 +306,11 @@ void caf_main(actor_system& sys) {
         CGS_SOLVER
     );
 
+    //  auto tasks = scan_for_matrices(
+    //     "/scratch/nqr159/matrix-collection/matrices/spd",
+    //     CGS_SOLVER
+    // );
+
     manager& mgr = manager::get();
     int gpus = mgr.get_num_devices();
 

From 34be706e191f6b95fb2c14bdfd92a730f6cbfa89 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 14:08:52 -0600
Subject: [PATCH 0873/1000] updated facade to take in mem_ptrs

---
 .../sparse-CGS-actor/sparse-CGS-actor.hpp     | 124 ++++++++++++++++++
 1 file changed, 124 insertions(+)

diff --git a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
index b7320202f8..a90bae8d52 100644
--- a/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
+++ b/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp
@@ -266,6 +266,18 @@ class sparse_cg_facade : public event_based_actor {
                                     device_num, stream_id);
 
         dispatch_result(actor_cast<caf::actor>(this->current_sender()), {}, std::move(x), n, meta);
+      },
+
+      // Mode 4: mem_ptr variant (assumes data is already on the GPU)
+      [this](mem_ptr<int> rp, mem_ptr<int> ci, mem_ptr<T> val,
+             mem_ptr<T> b_in, mem_ptr<T> x_in,
+             matrix_format fmt, int n, int nnz,
+             T tol, int max_iter,
+             int device_num, int stream_id) {
+        auto [x, meta] = solve_core_mem_ptr(rp, ci, val, b_in, x_in,
+                                            fmt, n, nnz, tol, max_iter,
+                                            device_num, stream_id);
+        dispatch_result(actor_cast<caf::actor>(this->current_sender()), {}, std::move(x), n, meta);
       }
     };
   }
@@ -389,6 +401,118 @@ class sparse_cg_facade : public event_based_actor {
     return {x, solver_result_meta(device_num, stream_id, iterations, rho_val <= threshold)};
   }
 
+  // New solve_core overload that accepts mem_ptr directly.
+  // This bypasses the initial transfer_memory call as the data is assumed
+  // to already be on the device and managed by the provided mem_ptrs.
+  virtual std::pair<mem_ptr<T>, solver_result_meta> solve_core_mem_ptr(mem_ptr<int> A_row_ptr, mem_ptr<int> A_col_ind, mem_ptr<T> A_values, mem_ptr<T> b,
+                                                                   mem_ptr<T> x,
+                                                                   matrix_format fmt, int n, int nnz,
+                                                                   T tol, int max_iter,
+                                                                   int device_num, int stream_id) {
+
+    command_runner<T> runner;
+
+    auto d_ptr = platform::create()->schedule(stream_id, device_num);
+
+    command_runner<out<T>> work_runner;
+    auto r     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto p     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto w     = work_runner.transfer_memory(device_num, stream_id, out<T>(n));
+    auto y_tmp = work_runner.transfer_memory(device_num, stream_id, create_out_arg_with_size<T>(1));
+
+    mem_ptr<char> spmv_workspace;
+    size_t ws_size = 0;
+
+    if (fmt == matrix_format::csr)
+      ws_size = d_ptr->spmv_csr_buffer_size(stream_id, n, n, nnz,
+                                            A_row_ptr, A_col_ind, A_values, x, w);
+    else if (fmt == matrix_format::csc)
+      ws_size = d_ptr->spmv_csc_buffer_size(stream_id, n, n, nnz,
+                                            A_row_ptr, A_col_ind, A_values, x, w);
+    else if (fmt == matrix_format::coo)
+      ws_size = d_ptr->spmv_coo_buffer_size(stream_id, n, n, nnz,
+                                            A_row_ptr, A_col_ind, A_values, x, w);
+
+    if (ws_size > 0) {
+      command_runner<out<char>> ws_runner;
+      spmv_workspace =
+        ws_runner.transfer_memory(device_num, stream_id,
+                                  out<char>(static_cast<int>(ws_size)));
+    }
+
+    auto execute_spmv = [&](mem_ptr<T> input_v, mem_ptr<T> output_v) {
+      switch (fmt) {
+        case matrix_format::csr:
+          d_ptr->spmv_csr(stream_id, n, n, nnz, T{1},
+                          A_row_ptr, A_col_ind, A_values,
+                          input_v, T{0}, output_v, spmv_workspace);
+          break;
+
+        case matrix_format::csc:
+          d_ptr->spmv_csc(stream_id, n, n, nnz, T{1},
+                          A_row_ptr, A_col_ind, A_values,
+                          input_v, T{0}, output_v, spmv_workspace);
+          break;
+
+        case matrix_format::coo:
+          d_ptr->spmv_coo(stream_id, n, n, nnz, T{1},
+                          A_row_ptr, A_col_ind, A_values,
+                          input_v, T{0}, output_v, spmv_workspace);
+          break;
+
+        default:
+          break;
+      }
+    };
+    auto execute_copy = [&](mem_ptr<T> src, mem_ptr<T> dst) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->dcopy(stream_id, n, src, dst); else d_ptr->scopy(stream_id, n, src, dst);
+    };
+    auto execute_axpy = [&](T alpha, mem_ptr<T> xv, mem_ptr<T> yv) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->daxpy(stream_id, n, alpha, xv, yv); else d_ptr->saxpy(stream_id, n, static_cast<float>(alpha), xv, yv);
+    };
+    auto execute_dot = [&](mem_ptr<T> xv, mem_ptr<T> yv, mem_ptr<T> rv) {
+      if constexpr (std::is_same_v<T, double>) d_ptr->ddot(stream_id, n, xv, yv, rv); else d_ptr->sdot(stream_id, n, xv, yv, rv);
+    };
+
+    execute_spmv(x, w);
+    execute_copy(b, r);
+    execute_axpy(T{-1}, w, r);
+    execute_dot(r, r, y_tmp);
+
+    T rho_val = runner.copy_to_host(y_tmp)[0];
+    T old_rho_val = T{0};
+    int iterations = 0;
+    T threshold = tol * tol;
+
+    while (rho_val > threshold && iterations < max_iter) {
+      iterations++;
+
+      if (iterations > 1) {
+        T beta_val = rho_val / old_rho_val;
+        execute_copy(r, w);
+        execute_axpy(beta_val, p, w);
+        execute_copy(w, p);
+      } else {
+       execute_copy(r, p);
+      }
+
+      execute_spmv(p, w);
+      execute_dot(p, w, y_tmp);
+
+      T alpha_val =
+        rho_val / runner.copy_to_host(y_tmp)[0];
+
+      execute_axpy(alpha_val, p, x);
+      execute_axpy(-alpha_val, w, r);
+
+      old_rho_val = rho_val;
+      execute_dot(r, r, y_tmp);
+      rho_val = runner.copy_to_host(y_tmp)[0];
+    }
+
+    return {x, solver_result_meta(device_num, stream_id, iterations, rho_val <= threshold)};
+  }
+
   void dispatch_result(caf::actor target,
                        std::vector<output_mapping> mappings,
                        mem_ptr<T> x,

From 7050482a680b367a5b2af311ee4e636cdfec8c2f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 14:09:15 -0600
Subject: [PATCH 0874/1000] implemented prefetching

---
 .../tests/workload-test/hot-potatoe.cpp       | 206 +++++++++++++++---
 1 file changed, 177 insertions(+), 29 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
index 787ac28dad..ee10c213d6 100644
--- a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
+++ b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
@@ -7,6 +7,7 @@
 #include <chrono>
 #include <filesystem>
 #include <memory>
+#include <cstdint>
 
 #include "caf/actorSOLVE/actorSOLVE.hpp"
 #include "sparse_utils.hpp"
@@ -28,6 +29,18 @@ bool inspect(Inspector& f, SolverType& x) {
     return false;
 }
 
+struct PrefetchedTask {
+    MatrixTask task;
+    int device_id;
+    size_t memory_usage;
+    mem_ptr<int32_t> row_ptr;
+    mem_ptr<int32_t> col_indices;
+    mem_ptr<float> values;
+    mem_ptr<float> b;
+    mem_ptr<float> x_guess;
+    bool is_valid = false;
+};
+
 // ============================================================
 // TYPE BLOCK (unchanged + extended)
 // ============================================================
@@ -41,18 +54,23 @@ CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
     CAF_ADD_ATOM(workload_test, worker_done_atom)
     CAF_ADD_ATOM(workload_test, work_tick_atom)
     CAF_ADD_ATOM(workload_test, add_work_atom)
+    CAF_ADD_ATOM(workload_test, memory_report_atom)
+    CAF_ADD_ATOM(workload_test, prefetch_done_atom)
+    CAF_ADD_ATOM(workload_test, shutdown_atom)
 
     CAF_ADD_TYPE_ID(workload_test, (SolverType))
     CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
     CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
     CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
     CAF_ADD_TYPE_ID(workload_test, (std::deque<MatrixTask>))
+    CAF_ADD_TYPE_ID(workload_test, (PrefetchedTask))
 
 CAF_END_TYPE_ID_BLOCK(workload_test)
 
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(PrefetchedTask)
 
 // ============================================================
 // TASK ACTOR STATE
@@ -66,21 +84,30 @@ struct worker_state {
     int device_id;
     int stream_id;
 
-    caf::actor cg_facade;
-    
+    actor cg_facade;
+    PrefetchedTask pref_task;
     bool solve_done = false;
     bool neighbor_received = false;
+    size_t memory_usage = 0;
 };
 
 // ============================================================
 // TASK ACTOR
 // ============================================================
 
+size_t get_task_memory(const MatrixTask& t) {
+    if (!t.data) return 0;
+    auto& d = *t.data;
+    return (d.row_ptr.size() + d.col_indices.size()) * sizeof(int32_t) 
+           + (d.values.size() + d.b.size() + d.x_guess.size()) * sizeof(float);
+}
+
 behavior worker_actor(stateful_actor<worker_state>* self, MatrixTask t, actor supervisor) {
     auto& st = self->state();
     st.task = std::move(t);
     st.supervisor = supervisor;
     st.cg_facade = self->spawn<sparse_cg_facade<float>, linked>(0);
+    st.memory_usage = get_task_memory(st.task);
     
     return {
         [=](start_atom, int dev, int stream) {
@@ -88,25 +115,44 @@ behavior worker_actor(stateful_actor<worker_state>* self, MatrixTask t, actor su
             s.device_id = dev;
             s.stream_id = stream;
 
-            // Alert master that we have begun work
-            self->mail(started_atom_v).send(s.supervisor);
+            // Alert master that we have begun work so it can find us a neighbor
+            self->mail(started_atom_v, dev).send(s.supervisor);
 
             // Execute solver
-            auto& d = *s.task.data;
-            self->mail(
-                create_in_arg(d.row_ptr),
-                create_in_arg(d.col_indices),
-                create_in_arg(d.values),
-                create_in_arg(d.b),
-                create_in_out_arg(d.x_guess),
-                matrix_format::csr,
-                (int)d.row_ptr.size() - 1,
-                (int)d.values.size(),
-                1e-5f,
-                2000,
-                dev,
-                stream
-            ).send(s.cg_facade);
+            if (s.pref_task.is_valid) {
+                auto& pt = s.pref_task;
+                self->mail(
+                    pt.row_ptr,
+                    pt.col_indices,
+                    pt.values,
+                    pt.b,
+                    pt.x_guess,
+                    matrix_format::csr,
+                    (int)pt.task.data->row_ptr.size() - 1,
+                    (int)pt.task.data->values.size(),
+                    1e-5f, 2000, dev, stream
+                ).send(s.cg_facade);
+            } else {
+                auto& d = *s.task.data;
+                self->mail(
+                    create_in_arg(d.row_ptr),
+                    create_in_arg(d.col_indices),
+                    create_in_arg(d.values),
+                    create_in_arg(d.b),
+                    create_in_out_arg(d.x_guess),
+                    matrix_format::csr,
+                    (int)d.row_ptr.size() - 1,
+                    (int)d.values.size(),
+                    1e-5f, 2000, dev, stream
+                ).send(s.cg_facade);
+            }
+        },
+
+        [=](PrefetchedTask& pt) {
+            auto& s = self->state();
+            s.pref_task = std::move(pt);
+            s.task = s.pref_task.task;
+            s.memory_usage = s.pref_task.memory_usage;
         },
 
         [=](neighbor_atom, actor n) {
@@ -116,6 +162,7 @@ behavior worker_actor(stateful_actor<worker_state>* self, MatrixTask t, actor su
 
             if (s.solve_done && s.neighbor) {
                 self->mail(start_atom_v, s.device_id, s.stream_id).send(s.neighbor);
+                self->mail(memory_report_atom_v, s.device_id, static_cast<int64_t>(s.memory_usage)).send(s.supervisor);
                 self->quit();
             }
         },
@@ -125,12 +172,43 @@ behavior worker_actor(stateful_actor<worker_state>* self, MatrixTask t, actor su
             s.solve_done = true;
             if (s.neighbor_received && s.neighbor) {
                 self->mail(start_atom_v, s.device_id, s.stream_id).send(s.neighbor);
+                self->mail(memory_report_atom_v, s.device_id, static_cast<int64_t>(s.memory_usage)).send(s.supervisor);
                 self->quit();
             }
         }
     };
 }
 
+// ============================================================
+// PREFETCHER ACTOR
+// ============================================================
+
+behavior prefetcher_actor(stateful_actor<bool>* self, MatrixTask task, int dev, actor supervisor) {
+    auto runner = std::make_shared<command_runner<>>();
+    auto& d = *task.data;
+    int prefetch_stream = 0; 
+
+    auto rp = runner->transfer_memory(dev, prefetch_stream, create_in_arg(d.row_ptr));
+    auto ci = runner->transfer_memory(dev, prefetch_stream, create_in_arg(d.col_indices));
+    auto val = runner->transfer_memory(dev, prefetch_stream, create_in_arg(d.values));
+    auto b = runner->transfer_memory(dev, prefetch_stream, create_in_arg(d.b));
+    auto x = runner->transfer_memory(dev, prefetch_stream, create_in_out_arg(d.x_guess));
+
+    size_t mem = get_task_memory(task);
+    PrefetchedTask pt{std::move(task), dev, mem, rp, ci, val, b, x, true};
+
+    runner->add_callback(prefetch_stream, dev, [pt_moved = std::move(pt), supervisor, self_handle = actor_cast<actor>(self)]() mutable {
+        anon_mail(prefetch_done_atom_v, std::move(pt_moved)).send(supervisor);
+        anon_mail(shutdown_atom_v).send(self_handle);
+    });
+
+    return {
+        [=](shutdown_atom) {
+            self->quit();
+        }
+    };
+}
+
 // ============================================================
 // PRODUCER ACTOR
 // ============================================================
@@ -190,6 +268,8 @@ struct resource_slot {
 struct supervisor_state {
     std::deque<MatrixTask> pending_queue;
     std::deque<resource_slot> available_slots;
+    std::vector<std::deque<actor>> warm_workers;
+    std::vector<size_t> gpu_free_mem;
     
     size_t total_expected;
     size_t completed = 0;
@@ -212,6 +292,12 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
     self->state().parent = std::move(parent);
     self->state().num_gpus = num_gpus;
     self->state().streams_per_gpu = streams_per_gpu;
+    self->state().warm_workers.resize(num_gpus);
+
+    for (int i = 0; i < num_gpus; ++i) {
+        auto dev = manager::get().find_device(i);
+        self->state().gpu_free_mem.push_back(dev ? dev->total_memory_bytes() : 0);
+    }
 
     if (total_tasks == 0) {
         self->mail(worker_done_atom_v).send(self->state().parent);
@@ -219,13 +305,33 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
         return {};
     }
 
+    auto check_prefetch = [=]() {
+        auto& s = self->state();
+        // Only prefetch if we don't have idle slots (saturated)
+        if (!s.available_slots.empty())
+            return;
+
+        for (int i = 0; i < s.num_gpus; ++i) {
+            while (s.gpu_free_mem[i] > 1024 * 1024 * 1024 && !s.pending_queue.empty()) {
+                auto task = std::move(s.pending_queue.front());
+                s.pending_queue.pop_front();
+                
+                size_t cost = get_task_memory(task);
+                s.gpu_free_mem[i] -= cost; 
+                s.completed++;
+
+                self->spawn(prefetcher_actor, std::move(task), i, actor_cast<actor>(self));
+            }
+        }
+    };
+
     return {
         [=](add_work_atom, std::vector<MatrixTask>& batch) {
             auto& s = self->state();
             for (auto& t : batch)
                 s.pending_queue.push_back(std::move(t));
 
-            // Bootstrapping: fill GPU slots with the first available tasks
+            // Bootstrapping: fill available GPU slots first (cold start)
             if (!s.initialized) {
                 s.initialized = true;
                 size_t total_slots = static_cast<size_t>(num_gpus * streams_per_gpu);
@@ -234,45 +340,69 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
                     int stream = static_cast<int>(i % streams_per_gpu);
                     
                     if (!s.pending_queue.empty()) {
-                        auto w = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
+                        auto t = std::move(s.pending_queue.front());
                         s.pending_queue.pop_front();
+                        s.gpu_free_mem[dev] -= get_task_memory(t);
                         s.completed++;
+
+                        auto w = self->spawn(worker_actor, std::move(t), actor_cast<actor>(self));
                         self->mail(start_atom_v, dev, stream).send(w);
                     } else {
                         s.available_slots.push_back({dev, stream});
                     }
                 }
+                check_prefetch();
             } else {
                 // New work arrived, check if we have idle GPU slots to fill
                 while (!s.available_slots.empty() && !s.pending_queue.empty()) {
                     auto slot = s.available_slots.front();
                     s.available_slots.pop_front();
-                    auto w = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
-                    s.completed++;
+                    
+                    auto t = std::move(s.pending_queue.front());
                     s.pending_queue.pop_front();
+                    s.gpu_free_mem[slot.device_id] -= get_task_memory(t);
+                    s.completed++;
+
+                    auto w = self->spawn(worker_actor, std::move(t), actor_cast<actor>(self));
                     self->mail(start_atom_v, slot.device_id, slot.stream_id).send(w);
                 }
+                check_prefetch();
             }
         },
 
-        [=](started_atom) {
+        [=](started_atom, int dev) {
             auto& s = self->state();
-            if (!s.pending_queue.empty()) {
-                auto next_worker = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
+            if (!s.warm_workers[dev].empty()) {
+                auto next_worker = s.warm_workers[dev].front();
+                s.warm_workers[dev].pop_front();
+                self->mail(neighbor_atom_v, next_worker).send(actor_cast<actor>(self->current_sender()));
+            } else if (!s.pending_queue.empty()) {
+                auto t = std::move(s.pending_queue.front());
                 s.pending_queue.pop_front();
+                s.gpu_free_mem[dev] -= get_task_memory(t);
                 s.completed++;
+
+                auto next_worker = self->spawn(worker_actor, std::move(t), actor_cast<actor>(self));
                 self->mail(neighbor_atom_v, next_worker).send(actor_cast<actor>(self->current_sender()));
             } else {
                 self->mail(neighbor_atom_v, actor_cast<actor>(self)).send(actor_cast<actor>(self->current_sender()));
             }
+            check_prefetch();
         },
 
         [=](start_atom, int dev, int stream) {
             auto& s = self->state();
-            if (!s.pending_queue.empty()) {
-                auto w = self->spawn(worker_actor, std::move(s.pending_queue.front()), actor_cast<actor>(self));
+            if (!s.warm_workers[dev].empty()) {
+                auto w = s.warm_workers[dev].front();
+                s.warm_workers[dev].pop_front();
+                self->mail(start_atom_v, dev, stream).send(w);
+            } else if (!s.pending_queue.empty()) {
+                auto t = std::move(s.pending_queue.front());
                 s.pending_queue.pop_front();
+                s.gpu_free_mem[dev] -= get_task_memory(t);
                 s.completed++;
+
+                auto w = self->spawn(worker_actor, std::move(t), actor_cast<actor>(self));
                 self->mail(start_atom_v, dev, stream).send(w);
             } else {
                 s.available_slots.push_back({dev, stream});
@@ -284,6 +414,24 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
                 self->mail(worker_done_atom_v).send(s.parent);
                 self->quit();
             }
+            check_prefetch();
+        },
+
+        [=](memory_report_atom, int dev, int64_t delta) {
+            auto& s = self->state();
+            if (dev >= 0 && dev < static_cast<int>(s.gpu_free_mem.size())) {
+                s.gpu_free_mem[dev] += static_cast<size_t>(delta);
+            }
+            check_prefetch();
+        },
+
+        [=](prefetch_done_atom, PrefetchedTask& pt) {
+            auto& s = self->state();
+            int dev = pt.device_id;
+            // Create the actor but don't solve yet
+            auto w = self->spawn(worker_actor, MatrixTask{}, actor_cast<actor>(self));
+            self->mail(std::move(pt)).send(w);
+            s.warm_workers[dev].push_back(w);
         }
     };
 }
@@ -359,4 +507,4 @@ void caf_main(actor_system& sys) {
     manager::shutdown();
 }
 
-CAF_MAIN(id_block::cuda, id_block::workload_test)
\ No newline at end of file
+CAF_MAIN(id_block::cuda, id_block::workload_test)

From 1f4559f80c4bc6084d211c3dc74d11ff74d7ff09 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 14:18:24 -0600
Subject: [PATCH 0875/1000] fixed oom issue

---
 libcaf_cuda/tests/workload-test/hot-potatoe.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
index ee10c213d6..6f9718a7ac 100644
--- a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
+++ b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
@@ -312,7 +312,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
             return;
 
         for (int i = 0; i < s.num_gpus; ++i) {
-            while (s.gpu_free_mem[i] > 1024 * 1024 * 1024 && !s.pending_queue.empty()) {
+            while (s.gpu_free_mem[i] > 2ULL * 1024 * 1024 * 1024 && !s.pending_queue.empty()) {
                 auto task = std::move(s.pending_queue.front());
                 s.pending_queue.pop_front();
                 

From 0afb549b6653bcc55e7ec614d9cb30980e170ac3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 8 Jun 2026 14:38:33 -0600
Subject: [PATCH 0876/1000] really fixed oom this time

---
 libcaf_cuda/tests/workload-test/hot-potatoe.cpp | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
index 6f9718a7ac..8e3de6ecac 100644
--- a/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
+++ b/libcaf_cuda/tests/workload-test/hot-potatoe.cpp
@@ -8,6 +8,7 @@
 #include <filesystem>
 #include <memory>
 #include <cstdint>
+#include <atomic>
 
 #include "caf/actorSOLVE/actorSOLVE.hpp"
 #include "sparse_utils.hpp"
@@ -182,11 +183,9 @@ behavior worker_actor(stateful_actor<worker_state>* self, MatrixTask t, actor su
 // ============================================================
 // PREFETCHER ACTOR
 // ============================================================
-
-behavior prefetcher_actor(stateful_actor<bool>* self, MatrixTask task, int dev, actor supervisor) {
+behavior prefetcher_actor(stateful_actor<bool>* self, MatrixTask task, int dev, actor supervisor, int prefetch_stream) {
     auto runner = std::make_shared<command_runner<>>();
     auto& d = *task.data;
-    int prefetch_stream = 0; 
 
     auto rp = runner->transfer_memory(dev, prefetch_stream, create_in_arg(d.row_ptr));
     auto ci = runner->transfer_memory(dev, prefetch_stream, create_in_arg(d.col_indices));
@@ -276,6 +275,7 @@ struct supervisor_state {
     int num_gpus;
     int streams_per_gpu;
     bool initialized = false;
+    int prefetch_stream_counter = 0;
     actor parent;
 };
 
@@ -312,15 +312,16 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self,
             return;
 
         for (int i = 0; i < s.num_gpus; ++i) {
-            while (s.gpu_free_mem[i] > 2ULL * 1024 * 1024 * 1024 && !s.pending_queue.empty()) {
+            while (s.gpu_free_mem[i] > 3ULL * 1024 * 1024 * 1024 && !s.pending_queue.empty()) {
                 auto task = std::move(s.pending_queue.front());
                 s.pending_queue.pop_front();
                 
                 size_t cost = get_task_memory(task);
                 s.gpu_free_mem[i] -= cost; 
                 s.completed++;
-
-                self->spawn(prefetcher_actor, std::move(task), i, actor_cast<actor>(self));
+                
+                int p_stream = -1 - (s.prefetch_stream_counter++ % 5);
+                self->spawn(prefetcher_actor, std::move(task), i, actor_cast<actor>(self), p_stream);
             }
         }
     };

From 1196608909ec2f8ef270cd68b1d338c741edd1f0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 07:53:27 -0600
Subject: [PATCH 0877/1000] Updated makefile to link with cublas and cusparse.

---
 libcaf_cuda/sc26/Runtime-Overhead/Makefile | 55 ++++++++++++----------
 1 file changed, 31 insertions(+), 24 deletions(-)

diff --git a/libcaf_cuda/sc26/Runtime-Overhead/Makefile b/libcaf_cuda/sc26/Runtime-Overhead/Makefile
index 3b096cd8f6..f55d417138 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/Makefile
+++ b/libcaf_cuda/sc26/Runtime-Overhead/Makefile
@@ -1,46 +1,53 @@
 NVCC ?= nvcc
 CXX ?= g++
+
 CXXFLAGS = -std=c++20 -include cstddef -include cerrno -include climits
 
 CUDA_INCLUDE_DIR ?=
-CUDA_LIB_DIR ?= 
+CUDA_LIB_DIR ?=
 
 CAF_INCLUDE_DIR ?=
 CAF_LIB_DIR ?=
 
-LIBS = -L$(CAF_LIB_DIR) \
- 			-lcaf_core \
-			-lcaf_io \
-			-lcaf_net \
-			-lcaf_cuda \
-			-L$(CUDA_LIB_DIR) \
-			-lcuda
-
-INCLUDES = -I$(CAF_INCLUDE_DIR) \
-		   -I$(CUDA_INCLUDE_DIR)
+INCLUDES = \
+	-I$(CAF_INCLUDE_DIR) \
+	-I$(CUDA_INCLUDE_DIR)
+
+LIBS = \
+	-L$(CAF_LIB_DIR) \
+	-lcaf_core \
+	-lcaf_io \
+	-lcaf_net \
+	-lcaf_cuda \
+	-L$(CUDA_LIB_DIR) \
+	-lcuda \
+	-lcublas \
+	-lcusparse
+
+LDFLAGS = \
+	-Wl,-rpath,$(CAF_LIB_DIR) \
+	-Wl,-rpath,$(CUDA_LIB_DIR)
 
-LDFLAGS=-Wl,-rpath,$(CAF_LIB_DIR) \
-		 		-Wl,-rpath,$(CUDA_LIB_DIR)
 NVCC_ARCH ?= sm_$(shell nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.')
 
-all: mmul_kernel cuda_native command_runner actor_facade
+TARGETS = cuda_native command_runner actor_facade
+
+all: mmul_kernel $(TARGETS)
 
 mmul_kernel: mmul.cu
-	$(NVCC) -arch=$(NVCC_ARCH) --cubin mmul.cu -o mmul.cubin
+	$(NVCC) -arch=$(NVCC_ARCH) --cubin $< -o mmul.cubin
 
 cuda_native: cuda_native.cpp
-	$(CXX) $(CXXFLAGS) $(INCLUDES) -o cuda_native cuda_native.cpp $(LIBS) $(LDFLAGS)
-
-actor_facade: actor_facade.cpp
-	$(CXX) $(CXXFLAGS) $(INCLUDES) -o actor_facade actor_facade.cpp $(LIBS) $(LDFLAGS)
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o $@ $< $(LIBS) $(LDFLAGS)
 
 command_runner: command_runner.cpp
-	$(CXX) $(CXXFLAGS) $(INCLUDES) -o command_runner command_runner.cpp $(LIBS) $(LDFLAGS)
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o $@ $< $(LIBS) $(LDFLAGS)
+
+actor_facade: actor_facade.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o $@ $< $(LIBS) $(LDFLAGS)
 
 clean:
 	rm -f mmul.cubin
-	rm -f cuda_native
-	rm -f main_actor_facade
-	rm -f command_runner
+	rm -f $(TARGETS)
 
-.PHONY: all clean
\ No newline at end of file
+.PHONY: all clean

From 4f8f870e5e131338cc73e33b2246438ac4e01fe5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 07:56:36 -0600
Subject: [PATCH 0878/1000] Updated filepaths to mmul.cubin to fix
 filenotfounderror.

---
 libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp   | 2 +-
 libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp b/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
index 1f0b19622d..a8383d4e81 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/actor_facade.cpp
@@ -65,7 +65,7 @@ void run_latency_test(caf::actor_system& sys, int matrix_size) {
 
     // Spawn facade
     auto facade = mgr.spawnFromCUBIN(
-        "../mmul.cubin", "matrixMul", dims,
+        "mmul.cubin", "matrixMul", dims,
         in<int>{}, in<int>{}, out<int>{}, in<int>{});
 
     sys.spawn(latency_manager, facade, matrix_size);
diff --git a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
index 31e65b18ae..82ddad6e09 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/command_runner.cpp
@@ -71,7 +71,7 @@ caf::behavior mmul_actor_fun_2(caf::stateful_actor<mmul_state>* self) {
       int stream = 1;
 
       auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
 
       auto t_total_start = clock::now();
       // -------------------------
@@ -280,4 +280,4 @@ int main(int argc, char** argv) {
   caf_main(sys);
   
   return 0;
-}
\ No newline at end of file
+}

From 99b06e93047c427ef93ad632d995db79c0f98c44 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 08:03:10 -0600
Subject: [PATCH 0879/1000] updated cuda to contain the proper experiment and
 updated the filepath of the cubin to fix filenotfound error

---
 .../sc26/Runtime-Overhead/cuda_native.cpp     | 242 +++++++++++++-----
 1 file changed, 175 insertions(+), 67 deletions(-)

diff --git a/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp b/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
index 629577c08c..19ba316789 100644
--- a/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
+++ b/libcaf_cuda/sc26/Runtime-Overhead/cuda_native.cpp
@@ -1,3 +1,4 @@
+// matrix_mul_driver.cpp
 #include <cuda.h>
 #include <iostream>
 #include <vector>
@@ -5,6 +6,20 @@
 #include <fstream>
 #include <sstream>
 #include <stdexcept>
+#include <cmath>
+#include <atomic>
+#include <thread>
+
+struct TimingState {
+    std::chrono::steady_clock::time_point end_time;
+    std::atomic<bool> ready{false};
+};
+
+void completion_callback(void* userData) {
+    auto* state = static_cast<TimingState*>(userData);
+    state->end_time = std::chrono::steady_clock::now();
+    state->ready = true;
+}
 
 static void checkCU(CUresult r, const char* where) {
     if (r != CUDA_SUCCESS) {
@@ -24,9 +39,60 @@ std::string readFile(const std::string &path) {
     return ss.str();
 }
 
-// Launch kernel once
-void launchKernel(CUfunction kernel, CUstream stream,
-                  CUdeviceptr d_a, CUdeviceptr d_b, CUdeviceptr d_c, int N) {
+std::vector<int> h_c; // Declared globally
+
+void runMatrixMul(CUmodule module, CUfunction kernel, int N, CUstream stream) {
+    using clock = std::chrono::steady_clock;
+    using ms = std::chrono::duration<double, std::milli>;
+
+    std::cout << "\n===== DRIVER BENCHMARK (N=" << N << ") =====\n";
+
+    size_t elements = (size_t)N * (size_t)N;
+    size_t bytes = elements * sizeof(int);
+
+    std::vector<int> h_a(elements, 1); // These remain local as they are initialized with N
+    std::vector<int> h_b(elements, 1); // These remain local as they are initialized with N
+    h_c.resize(elements); // Resize the global h_c for the current N
+
+    CUdeviceptr d_a, d_b, d_c;
+
+    auto t_total_start = clock::now();
+
+    // ----------------------------------
+    // Device Allocation
+    // ----------------------------------
+    auto t_alloc_start = clock::now();
+
+    checkCU(cuMemAllocAsync(&d_a, bytes, stream), "cuMemAllocAsync d_a");
+    checkCU(cuMemAllocAsync(&d_b, bytes, stream), "cuMemAllocAsync d_b");
+    checkCU(cuMemAllocAsync(&d_c, bytes, stream), "cuMemAllocAsync d_c");
+
+    auto t_alloc_end = clock::now();
+
+    // ----------------------------------
+    // H2D copy A
+    // ----------------------------------
+    auto t_h2d_a_start = clock::now();
+
+    checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), bytes, stream), "cuMemcpyHtoDAsync A");
+    std::cout << "  (Transfer size: " << bytes << " bytes)\n";
+
+    auto t_h2d_a_end = clock::now();
+
+    // ----------------------------------
+    // H2D copy B
+    // ----------------------------------
+    auto t_h2d_b_start = clock::now();
+
+    checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), bytes, stream), "cuMemcpyHtoDAsync B");
+    //checkCU(cuStreamSynchronize(stream), "sync B");
+    std::cout << "  (Transfer size: " << bytes << " bytes)\n";
+
+    auto t_h2d_b_end = clock::now();
+
+    // ----------------------------------
+    // Kernel launch + execution
+    // ----------------------------------
     const unsigned int blockX = 32;
     const unsigned int blockY = 32;
     unsigned int gridX = (N + blockX - 1) / blockX;
@@ -34,6 +100,8 @@ void launchKernel(CUfunction kernel, CUstream stream,
 
     void* kernelParams[] = { &d_a, &d_b, &d_c, &N };
 
+    auto t_kernel_start = clock::now();
+
     checkCU(cuLaunchKernel(kernel,
                            gridX, gridY, 1,
                            blockX, blockY, 1,
@@ -42,12 +110,91 @@ void launchKernel(CUfunction kernel, CUstream stream,
                            kernelParams,
                            nullptr),
             "cuLaunchKernel");
+
+   // checkCU(cuStreamSynchronize(stream), "kernel sync");
+
+    auto t_kernel_end = clock::now();
+
+    // ----------------------------------
+    // D2H copy
+    // ----------------------------------
+    auto t_d2h_start = clock::now();
+    TimingState t_state;
+
+    cuMemcpyDtoHAsync(h_c.data(), d_c, bytes, stream);
+
+    // Enqueue the host function to capture timing when the copy finishes
+    checkCU(cuLaunchHostFunc(stream, completion_callback, &t_state), "cuLaunchHostFunc");
+
+    // In a real actor, we would not wait here. 
+    // For this benchmark driver, we wait for the callback to fire.
+    while (!t_state.ready) {
+        std::this_thread::yield();
+    }
+
+    std::cout << "  (Transfer size: " << bytes << " bytes)\n";
+    
+    auto t_d2h_end = t_state.end_time;
+    auto t_total_end = t_state.end_time;
+
+    
+    // ----------------------------------
+    // Free device memory
+    // ----------------------------------
+    auto t_free_start = clock::now();
+
+    checkCU(cuMemFreeAsync(d_a, stream), "cuMemFreeAsync A");
+    checkCU(cuMemFreeAsync(d_b, stream), "cuMemFreeAsync B");
+    checkCU(cuMemFreeAsync(d_c, stream), "cuMemFreeAsync C");
+
+    auto t_free_end = clock::now();
+
+
+
+
+    // ----------------------------------
+    // Print Results
+    // ----------------------------------
+
+    std::cout << "Device allocation: "
+              << ms(t_alloc_end - t_alloc_start).count()
+              << " ms\n";
+
+    std::cout << "H2D copy A: "
+              << ms(t_h2d_a_end - t_h2d_a_start).count()
+              << " ms\n";
+
+    std::cout << "H2D copy B: "
+              << ms(t_h2d_b_end - t_h2d_b_start).count()
+              << " ms\n";
+
+    std::cout << "Kernel execution: "
+              << ms(t_kernel_end - t_kernel_start).count()
+              << " ms\n";
+
+    std::cout << "D2H copy: "
+              << ms(t_d2h_end - t_d2h_start).count()
+              << " ms\n";
+
+    std::cout << "Device free: "
+              << ms(t_free_end - t_free_start).count()
+              << " ms\n";
+
+    std::cout << "TOTAL: "
+              << ms(t_total_end - t_total_start).count()
+              << " ms\n";
+
+    std::cout << "=============================================\n";
 }
 
-int main() {
-    const int N = 1000;
-    std::vector<int> iteration_series = {1000, 2000, 3000, 4000, 5000,
-                                         6000, 7000, 8000, 9000, 10000};
+int main(int argc, char** argv) {
+  std::vector<int> sizes = {1000, 4000, 8000, 12000};
+    
+//   std::vector<int> sizes = {12000};	
+   if (argc > 1) {
+        sizes.clear();
+        for (int i = 1; i < argc; ++i) sizes.push_back(std::stoi(argv[i]));
+    }
 
     checkCU(cuInit(0), "cuInit");
 
@@ -55,79 +202,40 @@ int main() {
     checkCU(cuDeviceGet(&dev, 0), "cuDeviceGet(0)");
 
     CUcontext ctx;
-    checkCU(cuCtxCreate(&ctx, 0, dev), "cuCtxCreate");
+    checkCU(cuCtxCreate(&ctx, CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, dev), "cuCtxCreate");
 
-    std::string ptx = readFile("mmul.ptx");
+    const std::string cubinPath = "mmul.cubin";
+    std::string cubin;
+
+    try {
+        cubin = readFile(cubinPath);
+    } catch (const std::exception &e) {
+        std::cerr << "Failed to read CUBIN file '" << cubinPath << "': " << e.what() << "\n";
+        return EXIT_FAILURE;
+    }
 
     CUmodule module;
-    checkCU(cuModuleLoadDataEx(&module, ptx.c_str(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
+    checkCU(cuModuleLoadDataEx(&module, cubin.data(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
 
     CUfunction kernel;
     checkCU(cuModuleGetFunction(&kernel, module, "matrixMul"), "cuModuleGetFunction matrixMul");
 
-    // ----------------------------------
-    // Persistent host buffers
-    // ----------------------------------
-    size_t elements = (size_t)N * N;
-    std::vector<int> h_a(elements, 1);
-    std::vector<int> h_b(elements, 1);
-    std::vector<int> h_c(elements, 0);
-
     CUstream stream;
     checkCU(cuStreamCreate(&stream, CU_STREAM_DEFAULT), "cuStreamCreate");
 
-    using clock = std::chrono::steady_clock;
-
-    for (int iterations : iteration_series) {
-        auto start = clock::now();
-
-        for (int i = 0; i < iterations; ++i) {
-            // ----------------------------------
-            // Allocate device memory each iteration
-            // ----------------------------------
-            CUdeviceptr d_a, d_b, d_c;
-            checkCU(cuMemAllocAsync(&d_a, elements * sizeof(int), stream), "cuMemAllocAsync d_a");
-            checkCU(cuMemAllocAsync(&d_b, elements * sizeof(int), stream), "cuMemAllocAsync d_b");
-            checkCU(cuMemAllocAsync(&d_c, elements * sizeof(int), stream), "cuMemAllocAsync d_c");
-
-            // ----------------------------------
-            // Copy persistent host buffers to device
-            // ----------------------------------
-            checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), elements * sizeof(int), stream), "H2D d_a");
-            checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), elements * sizeof(int), stream), "H2D d_b");
-
-            // ----------------------------------
-            // Launch kernel
-            // ----------------------------------
-            launchKernel(kernel, stream, d_a, d_b, d_c, N);
-
-            // ----------------------------------
-            // Copy result back
-            // ----------------------------------
-            checkCU(cuMemcpyDtoHAsync(h_c.data(), d_c, elements * sizeof(int), stream), "D2H d_c");
-
-            // ----------------------------------
-            // Free device memory
-            // ----------------------------------
-            checkCU(cuMemFreeAsync(d_a, stream), "cuMemFreeAsync d_a");
-            checkCU(cuMemFreeAsync(d_b, stream), "cuMemFreeAsync d_b");
-            checkCU(cuMemFreeAsync(d_c, stream), "cuMemFreeAsync d_c");
+    for (int N : sizes) {
+        try {
+            runMatrixMul(module, kernel, N, stream);
+            
+            // Ensure stream is completely empty before starting the next size
+            checkCU(cuStreamSynchronize(stream), "cuStreamSynchronize between sizes");
+            
+        } catch (const std::exception &e) {
+            std::cerr << "Exception while running N=" << N << ": " << e.what() << "\n";
         }
-
-        // Synchronize stream after series
-        checkCU(cuStreamSynchronize(stream), "stream sync after series");
-
-        auto end = clock::now();
-        double total_ms = std::chrono::duration<double, std::milli>(end - start).count();
-
-        std::cout << "[SERIES RESULT] Matrix " << N << "x" << N
-                  << ", iterations = " << iterations
-                  << ", total GPU time = " << total_ms << " ms\n";
+        std::cout << "----------------------------------------\n";
     }
 
-    // ----------------------------------
-    // Cleanup
-    // ----------------------------------
     checkCU(cuStreamDestroy(stream), "cuStreamDestroy");
     checkCU(cuModuleUnload(module), "cuModuleUnload");
     checkCU(cuCtxDestroy(ctx), "cuCtxDestroy");

From a4dd0f6c7eee3c5a2c4281ab530e2b48b6d518ab Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 08:40:22 -0600
Subject: [PATCH 0880/1000] updated script to parse output file

---
 .../analyze_mmul_benchmarks.py                | 250 +++++++++++-------
 1 file changed, 157 insertions(+), 93 deletions(-)

diff --git a/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py b/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
index 8ba65d8965..3d25250eb1 100644
--- a/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
+++ b/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
@@ -1,135 +1,199 @@
 #!/usr/bin/env python3
 
-import glob
 import re
 import numpy as np
 import matplotlib.pyplot as plt
-from collections import defaultdict
 from pathlib import Path
 
 # -----------------------------
-# Path configuration (MATCHING STYLE OF FIRST SCRIPT)
+# Path configuration
 # -----------------------------
 SCRIPT_DIR = Path(__file__).resolve().parent
 
 DATA_DIR = SCRIPT_DIR.parent.parent / "Runtime-Overhead" / "results"
-
-driver_files = glob.glob(f"{DATA_DIR}/matrix_mul_driver_run*.txt")
-actor_files = glob.glob(f"{DATA_DIR}/test_run*.txt")
-latency_files = glob.glob(f"{DATA_DIR}/latency_bench_test_run*.txt")
+INPUT_FILE = DATA_DIR / "benchmark_results.txt"
 
 OUTPUT_PLOT = DATA_DIR / "mmul_benchmark_plot.png"
 
-# -----------------------------
-# Regex
-# -----------------------------
-size_pattern = re.compile(r"N=(\d+)")
-driver_total_pattern = re.compile(r"TOTAL:\s+([\d.]+)")
-actor_total_pattern = re.compile(r"TOTAL end-to-end:\s+([\d.]+)")
-
-latency_pattern = re.compile(
-    r"\[LATENCY TEST\]\s+matrix_size=(\d+),\s*time=(\d+)\s*ms"
-)
-
 # -----------------------------
 # Data containers
 # -----------------------------
-driver_data = defaultdict(list)
-actor_data = defaultdict(list)
-latency_data = defaultdict(list)
+driver_means = {}
+actor_means = {}
+latency_data = {}
 
 # -----------------------------
-# Parsers
+# Read benchmark file
 # -----------------------------
-def parse_driver(file):
-    with open(file) as f:
-        current_size = None
-        for line in f:
-            size_match = size_pattern.search(line)
-            if size_match:
-                current_size = int(size_match.group(1))
-
-            total_match = driver_total_pattern.search(line)
-            if total_match and current_size:
-                driver_data[current_size].append(float(total_match.group(1)))
-
-
-def parse_actor(file):
-    with open(file) as f:
-        current_size = None
-        for line in f:
-            size_match = size_pattern.search(line)
-            if size_match:
-                current_size = int(size_match.group(1))
-
-            total_match = actor_total_pattern.search(line)
-            if total_match and current_size:
-                actor_data[current_size].append(float(total_match.group(1)))
-
-
-def parse_latency(file):
-    with open(file) as f:
-        for line in f:
-            m = latency_pattern.search(line)
-            if m:
-                size = int(m.group(1))
-                time = float(m.group(2))
-                latency_data[size].append(time)
+with open(INPUT_FILE, "r") as f:
+    text = f.read()
 
-# -----------------------------
-# Parse files
-# -----------------------------
-for f in driver_files:
-    parse_driver(f)
+# --------------------------------------------------
+# CUDA Native Statistics Section
+# --------------------------------------------------
+cuda_section = re.search(
+    r"Implementation:\s*cuda_native(.*?)Implementation:\s*actor_facade",
+    text,
+    re.DOTALL,
+)
 
-for f in actor_files:
-    parse_actor(f)
+if cuda_section:
+    stats = re.findall(
+        r"^\s*(\d+)\s+([\d.]+)\s+[\d.]+\s+[\d.]+\s+[\d.]+\s+\d+",
+        cuda_section.group(1),
+        re.MULTILINE,
+    )
+
+    for n, mean in stats:
+        n = int(n)
+
+        # Skip raw run-number rows
+        if n < 1000:
+            continue
+
+        driver_means[n] = float(mean)
+
+# --------------------------------------------------
+# Command Runner Statistics Section
+# --------------------------------------------------
+runner_section = re.search(
+    r"Implementation:\s*command_runner(.*?)={10,}",
+    text,
+    re.DOTALL,
+)
 
-for f in latency_files:
-    parse_latency(f)
+if runner_section:
+    stats = re.findall(
+        r"^\s*(\d+)\s+([\d.]+)\s+[\d.]+\s+[\d.]+\s+[\d.]+\s+\d+",
+        runner_section.group(1),
+        re.MULTILINE,
+    )
 
-# -----------------------------
-# Aggregate
-# -----------------------------
-sizes = sorted(driver_data.keys())
+    for n, mean in stats:
+        n = int(n)
 
-driver_means = [np.mean(driver_data[s]) for s in sizes]
-actor_means = [np.mean(actor_data[s]) for s in sizes]
-latency_means = [np.mean(latency_data[s]) for s in sizes]
+        # Skip raw run-number rows
+        if n < 1000:
+            continue
 
-# -----------------------------
-# Metrics
-# -----------------------------
-abs_diff = [a - d for d, a in zip(driver_means, actor_means)]
-speedup = [d / a if a != 0 else float('inf') for d, a in zip(driver_means, actor_means)]
-percent_diff = [((a - d) / d) * 100 if d != 0 else 0 for d, a in zip(driver_means, actor_means)]
+        actor_means[n] = float(mean)
 
-# -----------------------------
-# Print
-# -----------------------------
+# --------------------------------------------------
+# Actor Facade Latency Data
+# --------------------------------------------------
+latency_matches = re.findall(
+    r"\[LATENCY TEST\]\s+matrix_size=(\d+),\s*time=(\d+)\s*ms",
+    text,
+)
+
+for n, t in latency_matches:
+    n = int(n)
+    t = float(t)
+
+    latency_data.setdefault(n, []).append(t)
+
+latency_means = {
+    n: np.mean(times)
+    for n, times in latency_data.items()
+}
+
+# --------------------------------------------------
+# Sizes
+# --------------------------------------------------
+sizes = sorted(
+    set(driver_means.keys())
+    & set(actor_means.keys())
+    & set(latency_means.keys())
+)
+
+# --------------------------------------------------
+# Extract ordered values
+# --------------------------------------------------
+driver_vals = [driver_means[s] for s in sizes]
+actor_vals = [actor_means[s] for s in sizes]
+latency_vals = [latency_means[s] for s in sizes]
+
+# --------------------------------------------------
+# Metrics
+# --------------------------------------------------
+abs_diff = [
+    a - d
+    for d, a in zip(driver_vals, actor_vals)
+]
+
+speedup = [
+    d / a if a != 0 else float("inf")
+    for d, a in zip(driver_vals, actor_vals)
+]
+
+percent_diff = [
+    ((a - d) / d) * 100
+    if d != 0 else 0
+    for d, a in zip(driver_vals, actor_vals)
+]
+
+# --------------------------------------------------
+# Print results
+# --------------------------------------------------
 print("===== MEAN RESULTS =====")
-print("N     | CUDA (ms) | Actors (ms) | Latency Facade (ms) | % Actor Diff | Speedup")
+print(
+    "N     | CUDA (ms) | Command-Runner (ms) | "
+    "Actor-Facade (ms) | % Diff | Speedup"
+)
 print("-" * 100)
 
 for s, d, a, l, pct, sp in zip(
-    sizes, driver_means, actor_means, latency_means, percent_diff, speedup
+    sizes,
+    driver_vals,
+    actor_vals,
+    latency_vals,
+    percent_diff,
+    speedup,
 ):
-    print(f"N={s:5d} | {d:10.3f} | {a:11.3f} | {l:18.3f} | {pct:11.2f}% | {sp:8.3f}")
-
-# -----------------------------
+    print(
+        f"N={s:5d} | "
+        f"{d:10.3f} | "
+        f"{a:19.3f} | "
+        f"{l:17.3f} | "
+        f"{pct:8.2f}% | "
+        f"{sp:8.3f}"
+    )
+
+# --------------------------------------------------
 # Plot
-# -----------------------------
-plt.figure()
+# --------------------------------------------------
+plt.figure(figsize=(8, 5))
+
+plt.plot(
+    sizes,
+    driver_vals,
+    marker="o",
+    label="CUDA (Driver)"
+)
 
-plt.plot(sizes, driver_means, marker='o', label="CUDA (Driver)")
-plt.plot(sizes, actor_means, marker='s', label="Command-Runner")
-plt.plot(sizes, latency_means, marker='^', label="Actor-Facade")
+plt.plot(
+    sizes,
+    actor_vals,
+    marker="s",
+    label="Command-Runner"
+)
+
+plt.plot(
+    sizes,
+    latency_vals,
+    marker="^",
+    label="Actor-Facade"
+)
 
 plt.xlabel("Matrix Size (N)")
 plt.ylabel("Mean Execution Time (ms)")
-plt.title("CUDA vs Actors vs Actor-Facade")
-plt.legend()
+plt.title("CUDA vs Command-Runner vs Actor-Facade")
 plt.grid(True)
+plt.legend()
 
+plt.tight_layout()
 plt.savefig(OUTPUT_PLOT, dpi=300)
+
+print(f"\nSaved graph: {OUTPUT_PLOT}")
+
 plt.show()
\ No newline at end of file

From ffe6d96895240388289a0d9851a7e5a824cf8365 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 08:47:33 -0600
Subject: [PATCH 0881/1000] updated script to draw cuda native plot

---
 .../analyze_mmul_benchmarks.py                | 202 +++++++-----------
 1 file changed, 78 insertions(+), 124 deletions(-)

diff --git a/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py b/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
index 3d25250eb1..40039df258 100644
--- a/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
+++ b/libcaf_cuda/sc26/scripts/Runtime-Overhead/analyze_mmul_benchmarks.py
@@ -6,194 +6,148 @@
 from pathlib import Path
 
 # -----------------------------
-# Path configuration
+# Paths
 # -----------------------------
 SCRIPT_DIR = Path(__file__).resolve().parent
 
 DATA_DIR = SCRIPT_DIR.parent.parent / "Runtime-Overhead" / "results"
 INPUT_FILE = DATA_DIR / "benchmark_results.txt"
-
 OUTPUT_PLOT = DATA_DIR / "mmul_benchmark_plot.png"
 
 # -----------------------------
-# Data containers
-# -----------------------------
-driver_means = {}
-actor_means = {}
-latency_data = {}
-
-# -----------------------------
-# Read benchmark file
+# Load file
 # -----------------------------
 with open(INPUT_FILE, "r") as f:
     text = f.read()
 
-# --------------------------------------------------
-# CUDA Native Statistics Section
-# --------------------------------------------------
-cuda_section = re.search(
-    r"Implementation:\s*cuda_native(.*?)Implementation:\s*actor_facade",
-    text,
-    re.DOTALL,
-)
 
-if cuda_section:
-    stats = re.findall(
-        r"^\s*(\d+)\s+([\d.]+)\s+[\d.]+\s+[\d.]+\s+[\d.]+\s+\d+",
-        cuda_section.group(1),
-        re.MULTILINE,
-    )
+# ----------------------------------------------------------
+# Extract statistics table
+# ----------------------------------------------------------
+def extract_stats(text, implementation):
 
-    for n, mean in stats:
-        n = int(n)
+    pattern = (
+        rf"Implementation:\s*{implementation}"
+        rf".*?--- Statistics ---"
+        rf".*?^-+\n"
+        rf"(.*?)\n\n"
+    )
 
-        # Skip raw run-number rows
-        if n < 1000:
-            continue
+    m = re.search(pattern, text, re.DOTALL | re.MULTILINE)
+    if not m:
+        raise RuntimeError(f"Could not locate statistics for {implementation}")
 
-        driver_means[n] = float(mean)
+    stats_text = m.group(1)
 
-# --------------------------------------------------
-# Command Runner Statistics Section
-# --------------------------------------------------
-runner_section = re.search(
-    r"Implementation:\s*command_runner(.*?)={10,}",
-    text,
-    re.DOTALL,
-)
+    result = {}
 
-if runner_section:
-    stats = re.findall(
-        r"^\s*(\d+)\s+([\d.]+)\s+[\d.]+\s+[\d.]+\s+[\d.]+\s+\d+",
-        runner_section.group(1),
-        re.MULTILINE,
-    )
+    for line in stats_text.splitlines():
+        row = re.match(r"\s*(\d+)\s+([\d.]+)", line)
+        if not row:
+            continue
 
-    for n, mean in stats:
-        n = int(n)
+        n = int(row.group(1))
 
-        # Skip raw run-number rows
+        # filter out run indices / noise
         if n < 1000:
             continue
 
-        actor_means[n] = float(mean)
+        result[n] = float(row.group(2))
 
-# --------------------------------------------------
-# Actor Facade Latency Data
-# --------------------------------------------------
-latency_matches = re.findall(
-    r"\[LATENCY TEST\]\s+matrix_size=(\d+),\s*time=(\d+)\s*ms",
-    text,
-)
+    return result
 
-for n, t in latency_matches:
-    n = int(n)
-    t = float(t)
 
-    latency_data.setdefault(n, []).append(t)
+# ----------------------------------------------------------
+# Parse benchmark data
+# ----------------------------------------------------------
+driver_means = extract_stats(text, "cuda_native")
+actor_means = extract_stats(text, "command_runner")
+
+latency_data = {}
+
+for n, t in re.findall(
+    r"\[LATENCY TEST\]\s+matrix_size=(\d+),\s*time=(\d+)\s*ms",
+    text,
+):
+    latency_data.setdefault(int(n), []).append(float(t))
 
 latency_means = {
-    n: np.mean(times)
-    for n, times in latency_data.items()
+    n: np.mean(v)
+    for n, v in latency_data.items()
 }
 
-# --------------------------------------------------
-# Sizes
-# --------------------------------------------------
+
+# ----------------------------------------------------------
+# Align sizes
+# ----------------------------------------------------------
 sizes = sorted(
     set(driver_means.keys())
     & set(actor_means.keys())
     & set(latency_means.keys())
 )
 
-# --------------------------------------------------
-# Extract ordered values
-# --------------------------------------------------
 driver_vals = [driver_means[s] for s in sizes]
 actor_vals = [actor_means[s] for s in sizes]
 latency_vals = [latency_means[s] for s in sizes]
 
-# --------------------------------------------------
-# Metrics
-# --------------------------------------------------
-abs_diff = [
-    a - d
-    for d, a in zip(driver_vals, actor_vals)
-]
-
-speedup = [
-    d / a if a != 0 else float("inf")
-    for d, a in zip(driver_vals, actor_vals)
-]
-
-percent_diff = [
-    ((a - d) / d) * 100
-    if d != 0 else 0
-    for d, a in zip(driver_vals, actor_vals)
-]
-
-# --------------------------------------------------
-# Print results
-# --------------------------------------------------
-print("===== MEAN RESULTS =====")
-print(
-    "N     | CUDA (ms) | Command-Runner (ms) | "
-    "Actor-Facade (ms) | % Diff | Speedup"
-)
-print("-" * 100)
 
-for s, d, a, l, pct, sp in zip(
-    sizes,
-    driver_vals,
-    actor_vals,
-    latency_vals,
-    percent_diff,
-    speedup,
-):
-    print(
-        f"N={s:5d} | "
-        f"{d:10.3f} | "
-        f"{a:19.3f} | "
-        f"{l:17.3f} | "
-        f"{pct:8.2f}% | "
-        f"{sp:8.3f}"
-    )
+# ----------------------------------------------------------
+# Debug print (optional but useful)
+# ----------------------------------------------------------
+print("sizes       =", sizes)
+print("cuda        =", driver_vals)
+print("runner      =", actor_vals)
+print("facade      =", latency_vals)
+
 
-# --------------------------------------------------
+# ----------------------------------------------------------
 # Plot
-# --------------------------------------------------
-plt.figure(figsize=(8, 5))
+# ----------------------------------------------------------
+plt.figure(figsize=(10, 6))
 
 plt.plot(
     sizes,
     driver_vals,
-    marker="o",
-    label="CUDA (Driver)"
+    marker='o',
+    markersize=8,
+    linewidth=2.5,
+    linestyle='-',
+    label='CUDA Native'
 )
 
 plt.plot(
     sizes,
     actor_vals,
-    marker="s",
-    label="Command-Runner"
+    marker='s',
+    markersize=8,
+    linewidth=2.5,
+    linestyle='--',
+    label='Command Runner'
 )
 
 plt.plot(
     sizes,
     latency_vals,
-    marker="^",
-    label="Actor-Facade"
+    marker='^',
+    markersize=8,
+    linewidth=2.5,
+    linestyle=':',
+    label='Actor Facade'
 )
 
+# Log scale helps since you span ~11 ms → ~1885 ms
+plt.yscale("log")
+
 plt.xlabel("Matrix Size (N)")
-plt.ylabel("Mean Execution Time (ms)")
-plt.title("CUDA vs Command-Runner vs Actor-Facade")
-plt.grid(True)
+plt.ylabel("Mean Runtime (ms)")
+plt.title("Runtime Overhead Comparison")
+
+plt.grid(True, which="both", linestyle="--", alpha=0.4)
 plt.legend()
 
 plt.tight_layout()
 plt.savefig(OUTPUT_PLOT, dpi=300)
 
-print(f"\nSaved graph: {OUTPUT_PLOT}")
+print(f"\nSaved graph to: {OUTPUT_PLOT}")
 
 plt.show()
\ No newline at end of file

From 791e01eaec40f586e2d04e68a5db7e52e9c35b5f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 09:29:55 -0600
Subject: [PATCH 0882/1000] Updated makefile to build correctly.

---
 .../sc26/Sequence-Independent-Tasks/Makefile  | 41 +++++++++++--------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/Makefile b/libcaf_cuda/sc26/Sequence-Independent-Tasks/Makefile
index a43dcbdc26..482519b2ac 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/Makefile
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/Makefile
@@ -1,26 +1,35 @@
 NVCC ?= nvcc
 CXX ?= g++
-CXXFLAGS = -std=c++20 -include cstddef -include cerrno -include climits
 
-CUDA_INCLUDE_DIR ?=
-CUDA_LIB_DIR ?= 
+CXXFLAGS = -std=c++20 -include cstddef -include cerrno -include climits
 
-CAF_INCLUDE_DIR ?=
-CAF_LIB_DIR ?=
+CUDA_INCLUDE_DIR ?= /usr/local/cuda/include
+CUDA_LIB_DIR ?= /usr/local/cuda/lib64
 
-LIBS = -L$(CAF_LIB_DIR) \
- 			-lcaf_core \
-			-lcaf_io \
-			-lcaf_net \
-			-lcaf_cuda \
-			-L$(CUDA_LIB_DIR) \
-			-lcuda
+CAF_INCLUDE_DIR ?= /student/nqr159/gpu-actors/actor-framework/include
+CAF_LIB_DIR ?= /student/nqr159/gpu-actors/actor-framework/build/libcaf_core
 
 INCLUDES = -I$(CAF_INCLUDE_DIR) \
-		  	 	 -I$(CUDA_INCLUDE_DIR)
+           -I$(CUDA_INCLUDE_DIR) \
+           -I/student/nqr159/gpu-actors/actor-framework/libcaf_core \
+           -I/student/nqr159/gpu-actors/actor-framework/libcaf_io \
+           -I/student/nqr159/gpu-actors/actor-framework/libcaf_net \
+           -I/student/nqr159/gpu-actors/actor-framework/libcaf_cuda
+
+LIBS = -L$(CAF_LIB_DIR) \
+       -lcaf_core \
+       -lcaf_io \
+       -lcaf_net \
+       -lcaf_cuda \
+       -L$(CUDA_LIB_DIR) \
+       -lcublas \
+       -lcusparse \
+       -lcudart \
+       -lcuda
+
+LDFLAGS = -Wl,-rpath,$(CAF_LIB_DIR) \
+          -Wl,-rpath,$(CUDA_LIB_DIR)
 
-LDFLAGS=-Wl,-rpath,$(CAF_LIB_DIR) \
-		 		-Wl,-rpath,$(CUDA_LIB_DIR)
 NVCC_ARCH ?= sm_$(shell nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.')
 
 all: mmul_kernel base_cuda_program actor_facade actor_command_runner
@@ -41,4 +50,4 @@ clean:
 	rm -f mmul.cubin
 	rm -f main_cuda_native
 	rm -f main_actor_facade
-	rm -f main_command_runner
\ No newline at end of file
+	rm -f main_command_runner

From 0601dbb11cd48234379d64e04db70d1011636ac5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 09:34:36 -0600
Subject: [PATCH 0883/1000] Updated file paths.

---
 libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp  | 4 ++--
 .../sc26/Sequence-Independent-Tasks/command_runner.cpp        | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
index 5896806acf..f8ac8d32ec 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/actor_facade.cpp
@@ -77,7 +77,7 @@ void run_mapping_throughput_test(caf::actor_system& sys, int matrix_size, int it
     caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
 
     auto facade = mgr.spawnFromCUBIN(
-        "../mmul.cubin", "matrixMul", dims,
+        "mmul.cubin", "matrixMul", dims,
         in<int>{}, in<int>{}, out<int>{}, in<int>{});
 
     sys.spawn(throughput_mapping_manager, facade, matrix_size, iterations);
@@ -105,4 +105,4 @@ int main(int argc, char** argv) {
     actor_system sys{cfg};
     caf_main(sys);
     return 0;
-}
\ No newline at end of file
+}
diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
index 17758da4d5..916480d448 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/command_runner.cpp
@@ -163,7 +163,7 @@ void run_mmul_test(caf::actor_system& sys, int matrix_size,int iterations) {
   auto& mgr = caf::cuda::manager::get();
   
   auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
 
 
   using clock = std::chrono::steady_clock;

From 816459d05622bceb3f8b493ffd0fd4c848c6f09a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 09:37:29 -0600
Subject: [PATCH 0884/1000] updated test and filepaths

---
 .../cuda_native.cpp                           | 111 +++++++-----------
 1 file changed, 43 insertions(+), 68 deletions(-)

diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp
index fdbd3e5218..629577c08c 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp
@@ -46,8 +46,8 @@ void launchKernel(CUfunction kernel, CUstream stream,
 
 int main() {
     const int N = 1000;
-    const int total_iterations = 10000;
-    const int milestone_interval = 1000;
+    std::vector<int> iteration_series = {1000, 2000, 3000, 4000, 5000,
+                                         6000, 7000, 8000, 9000, 10000};
 
     checkCU(cuInit(0), "cuInit");
 
@@ -57,10 +57,10 @@ int main() {
     CUcontext ctx;
     checkCU(cuCtxCreate(&ctx, 0, dev), "cuCtxCreate");
 
-    std::string cubin = readFile("mmul.cubin");
+    std::string ptx = readFile("mmul.ptx");
 
     CUmodule module;
-    checkCU(cuModuleLoadDataEx(&module, cubin.data(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
+    checkCU(cuModuleLoadDataEx(&module, ptx.c_str(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
 
     CUfunction kernel;
     checkCU(cuModuleGetFunction(&kernel, module, "matrixMul"), "cuModuleGetFunction matrixMul");
@@ -78,77 +78,52 @@ int main() {
 
     using clock = std::chrono::steady_clock;
 
-    // Warmup: prime CUDA context and allocator before timed series
-    {
-        const int warmup_iters = 10;
-        for (int i = 0; i < warmup_iters; ++i) {
+    for (int iterations : iteration_series) {
+        auto start = clock::now();
+
+        for (int i = 0; i < iterations; ++i) {
+            // ----------------------------------
+            // Allocate device memory each iteration
+            // ----------------------------------
             CUdeviceptr d_a, d_b, d_c;
-            checkCU(cuMemAlloc(&d_a, elements * sizeof(int)), "warmup alloc d_a");
-            checkCU(cuMemAlloc(&d_b, elements * sizeof(int)), "warmup alloc d_b");
-            checkCU(cuMemAlloc(&d_c, elements * sizeof(int)), "warmup alloc d_c");
-            checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), elements * sizeof(int), stream), "warmup H2D d_a");
-            checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), elements * sizeof(int), stream), "warmup H2D d_b");
+            checkCU(cuMemAllocAsync(&d_a, elements * sizeof(int), stream), "cuMemAllocAsync d_a");
+            checkCU(cuMemAllocAsync(&d_b, elements * sizeof(int), stream), "cuMemAllocAsync d_b");
+            checkCU(cuMemAllocAsync(&d_c, elements * sizeof(int), stream), "cuMemAllocAsync d_c");
+
+            // ----------------------------------
+            // Copy persistent host buffers to device
+            // ----------------------------------
+            checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), elements * sizeof(int), stream), "H2D d_a");
+            checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), elements * sizeof(int), stream), "H2D d_b");
+
+            // ----------------------------------
+            // Launch kernel
+            // ----------------------------------
             launchKernel(kernel, stream, d_a, d_b, d_c, N);
-            checkCU(cuMemcpyDtoHAsync(h_c.data(), d_c, elements * sizeof(int), stream), "warmup D2H d_c");
-            checkCU(cuMemFree(d_a), "warmup free d_a");
-            checkCU(cuMemFree(d_b), "warmup free d_b");
-            checkCU(cuMemFree(d_c), "warmup free d_c");
-        }
-        checkCU(cuStreamSynchronize(stream), "warmup sync");
-        std::cout << "--- warmup complete ---\n";
-    }
 
-    auto start = clock::now();
-
-    for (int i = 0; i < total_iterations; ++i) {
-        // ----------------------------------
-        // Allocate device memory each iteration
-        // ----------------------------------
-        CUdeviceptr d_a, d_b, d_c;
-        checkCU(cuMemAlloc(&d_a, elements * sizeof(int)), "cuMemAlloc d_a");
-        checkCU(cuMemAlloc(&d_b, elements * sizeof(int)), "cuMemAlloc d_b");
-        checkCU(cuMemAlloc(&d_c, elements * sizeof(int)), "cuMemAlloc d_c");
-
-        // ----------------------------------
-        // Copy persistent host buffers to device
-        // ----------------------------------
-        checkCU(cuMemcpyHtoDAsync(d_a, h_a.data(), elements * sizeof(int), stream), "H2D d_a");
-        checkCU(cuMemcpyHtoDAsync(d_b, h_b.data(), elements * sizeof(int), stream), "H2D d_b");
-
-        // ----------------------------------
-        // Launch kernel
-        // ----------------------------------
-        launchKernel(kernel, stream, d_a, d_b, d_c, N);
-
-        // ----------------------------------
-        // Copy result back
-        // ----------------------------------
-        checkCU(cuMemcpyDtoHAsync(h_c.data(), d_c, elements * sizeof(int), stream), "D2H d_c");
-
-        // ----------------------------------
-        // Free device memory
-        // ----------------------------------
-        checkCU(cuMemFree(d_a), "cuMemFree d_a");
-        checkCU(cuMemFree(d_b), "cuMemFree d_b");
-        checkCU(cuMemFree(d_c), "cuMemFree d_c");
-
-        if ((i + 1) % milestone_interval == 0) {
-            checkCU(cuStreamSynchronize(stream), "stream sync at milestone");
-            auto now = clock::now();
-            double elapsed_ms = std::chrono::duration<double, std::milli>(now - start).count();
-            std::cout << "[MILESTONE] " << (i + 1) << " / " << total_iterations
-                      << " iterations, elapsed = " << elapsed_ms << " ms\n";
+            // ----------------------------------
+            // Copy result back
+            // ----------------------------------
+            checkCU(cuMemcpyDtoHAsync(h_c.data(), d_c, elements * sizeof(int), stream), "D2H d_c");
+
+            // ----------------------------------
+            // Free device memory
+            // ----------------------------------
+            checkCU(cuMemFreeAsync(d_a, stream), "cuMemFreeAsync d_a");
+            checkCU(cuMemFreeAsync(d_b, stream), "cuMemFreeAsync d_b");
+            checkCU(cuMemFreeAsync(d_c, stream), "cuMemFreeAsync d_c");
         }
-    }
 
-    checkCU(cuStreamSynchronize(stream), "stream sync after series");
+        // Synchronize stream after series
+        checkCU(cuStreamSynchronize(stream), "stream sync after series");
 
-    auto end = clock::now();
-    double total_ms = std::chrono::duration<double, std::milli>(end - start).count();
+        auto end = clock::now();
+        double total_ms = std::chrono::duration<double, std::milli>(end - start).count();
 
-    std::cout << "[SERIES RESULT] Matrix " << N << "x" << N
-              << ", iterations = " << total_iterations
-              << ", total GPU time = " << total_ms << " ms\n";
+        std::cout << "[SERIES RESULT] Matrix " << N << "x" << N
+                  << ", iterations = " << iterations
+                  << ", total GPU time = " << total_ms << " ms\n";
+    }
 
     // ----------------------------------
     // Cleanup

From 74575cae051b30fe6fbd3e055bf62d84e79773db Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 09:37:42 -0600
Subject: [PATCH 0885/1000] updated tests and filepaths

---
 libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp b/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp
index 629577c08c..db076f8860 100644
--- a/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp
+++ b/libcaf_cuda/sc26/Sequence-Independent-Tasks/cuda_native.cpp
@@ -57,7 +57,7 @@ int main() {
     CUcontext ctx;
     checkCU(cuCtxCreate(&ctx, 0, dev), "cuCtxCreate");
 
-    std::string ptx = readFile("mmul.ptx");
+    std::string ptx = readFile("mmul.cubin");
 
     CUmodule module;
     checkCU(cuModuleLoadDataEx(&module, ptx.c_str(), 0, nullptr, nullptr), "cuModuleLoadDataEx");

From 96b3eb60c35e081f235526f9401dc18843d64ecb Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 10:45:02 -0600
Subject: [PATCH 0886/1000] refactored scheduler actor to have  a cleaner
 interface

---
 .../cuda/control-layer/scheduler_actor.hpp    |  43 ++--
 .../control-layer/scheduler_actor_state.hpp   |   4 +-
 .../src/control-layer/scheduler_actor.cpp     | 193 +++++++++---------
 3 files changed, 129 insertions(+), 111 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
index 80c4e8b8dd..52d939b202 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
@@ -1,18 +1,37 @@
 #pragma once
 #include <caf/all.hpp>
-#include "caf/cuda/control-layer/launch_token.hpp"
-#include "caf/cuda/control-layer/launch_response_token.hpp"
-#include "caf/cuda/control-layer/behavior.hpp"
-#include "caf/cuda/control-layer/behavior_table.hpp"
-#include "caf/cuda/control-layer/behavior_token.hpp"
+#include "caf/cuda/control-layer/token.hpp"
 #include <queue>
 #include "caf/cuda/global_export.hpp"
 
-/*
- * The scheduler actor 
- * meant to make scheduling decisions using s/r/r ipc
- */
-
 namespace caf::cuda {
-caf::behavior CAF_CUDA_EXPORT scheduler_actor(caf::stateful_actor<scheduler_actor_state> * self,int device_number,bool multi_gpu);
-}//namespace caf::cuda
+
+class CAF_CUDA_EXPORT scheduler_actor : public caf::event_based_actor {
+public:
+    scheduler_actor(caf::actor_config& cfg, int device_number, int num_streams, int stream_depth, bool multi_gpu);
+    
+    caf::behavior make_behavior() override;
+
+    // Message Handler Methods
+    virtual void on_receive(const token_ptr& tok);
+    virtual void on_receive_batch(std::vector<token_ptr> tokens);
+    virtual void on_reclaim(int val, int mem, int time, int dep);
+    virtual void on_set_neighbors(std::vector<caf::actor> neighbors);
+    virtual std::vector<token_ptr> on_steal_request(int requesting_device);
+
+protected:
+    // Scheduling logic
+    virtual void schedule_work();
+
+    // State Attributes
+    int device_number_;
+    int num_streams_;
+    int stream_depth_;
+    bool multi_gpu_;
+    
+    int in_flight_ = 0;
+    std::queue<token_ptr> queue_;
+    std::vector<caf::actor> schedulers_;
+};
+
+} // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
index fff453484a..f13acee568 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
@@ -17,7 +17,9 @@ struct scheduler_actor_state {
     std::queue<token_ptr> queue; // here for legacy prototype schedulers
     std::vector<kernel_graph> operations; //more modern dependency based data structure 
     int device_number;
-    int num_streams = 500; // number of streams that can be used by the scheduler
+    int in_flight = 0; // current number of tasks active on the GPU
+    int num_streams = 500;
+    int stream_depth = 1; // maximum number of in-flight tasks per stream
     std::vector<caf::actor> schedulers; //the other scheduler actors in the system, an actor for a 
 			     //specific GPU can be accessed via there corrosponding 
 			     //device number in the std::vector
diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 626d73d7a1..3a61e48dbf 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -1,119 +1,116 @@
 #include "caf/cuda/control-layer/all-control-layer.hpp"
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
-#include "caf/cuda/control-layer/green_light_behavior.hpp"
-#include "caf/cuda/control-layer/red_light_behavior.hpp"
-#include "caf/cuda/control-layer/core_usage_behavior.hpp"
-#include "caf/cuda/control-layer/single_usage_behavior.hpp"
-#include "caf/cuda/control-layer/multilevel_usage_behavior.hpp"
-#include "caf/cuda/control-layer/pressure_scheduler.hpp"
-#include "caf/cuda/control-layer/kernel_graph.hpp"
-#include <string>
 #include <iostream>
 
-/*
- * This class is meant to handle actor GPU scheduling via s/r/r IPC
- * it has nothing to do with the scheduler class, that is kernel layer
- */
 namespace caf::cuda {
 
-caf::behavior scheduler_actor(caf::stateful_actor<scheduler_actor_state>* self, int device_number,bool multi_gpu) {
-    auto& state = self->state();
-
-    // add self reference
-    state.self = self;
-
-    // set device number
-    state.device_number = device_number;
-
-    //check if multiple gpus
-    state.multiple_gpus = multi_gpu;
-
-   // default behavior
-    state.table = std::make_unique<behavior_table>(state);
-    state.current_behavior = state.table -> get(behavior_token("single_usage"));
-    state.current_behavior->on_enter();
-
-
+scheduler_actor::scheduler_actor(caf::actor_config& cfg, 
+                                 int device_number, int num_streams, 
+                                 int stream_depth, bool multi_gpu)
+    : caf::event_based_actor(cfg),
+      device_number_(device_number),
+      num_streams_(num_streams),
+      stream_depth_(stream_depth),
+      multi_gpu_(multi_gpu) {
+    // Constructor logic if needed
+}
 
+caf::behavior scheduler_actor::make_behavior() {
     return {
-        [&](const token_ptr& tok) {
-            // std::cout << "Received token\n";
-            state.current_behavior->receive(tok);
+        [this](const token_ptr& tok) {
+            on_receive(tok);
         },
-        
-   [&state](const caf::cuda::behavior_token_ptr& tok) -> bool {
-    auto* next = state.table -> get(*tok);
-    if (next) {
-        if (next != state.current_behavior) {
-            state.current_behavior->on_exit();   // cleanup current behavior
-            state.current_behavior = next;       // swap behavior
-            state.current_behavior->on_enter();  // init new behavior
-            std::cout << "[INFO] Behavior changed to: " << state.current_behavior->name() << "\n";
-	    return true; // behavior changed
-        } else {
-            std::cout << "[INFO] Behavior already active: " << state.current_behavior->name() << "\n";
-            return false; // behavior was already current
-        }
-    } else {
-        std::cout << "[WARN] No behavior found for token: " << tok->name() << "\n";
-        return false; // no change
-    }
-	}
-	,
-        [&](std::vector<token_ptr> tokens) {
-            for (size_t i = 0; i < tokens.size(); ++i) {
-                state.current_behavior->receive(tokens[i]);
-            }
+        [this](std::vector<token_ptr> tokens) {
+            on_receive_batch(std::move(tokens));
         },
-
-	//can send the scheduler a message if you want 
-	//it is more than happy to print it out for you 
-        [=](std::string word) {
-             std::cout << "Received message " << word << "\n";
+        [this](int val, int mem, int time, int dep) {
+            on_reclaim(val, mem, time, dep);
         },
-	
-	//message handler for reclaim
-	[&](int value, int memory,int runtime,int dependency) {	
-
-                //std::cout << "Received reclaim request\n";
-		state.current_behavior->reclaim(value,memory,runtime,dependency);
-	},
-
-
-	//message handler for reclaim
-	[&](ack payload) {	
-		state.current_behavior->reclaim(payload);
-	},
-
+        [this](std::vector<caf::actor> neighbors) {
+            on_set_neighbors(std::move(neighbors));
+        },
+        [this](int requesting_device) -> std::vector<token_ptr> {
+            return on_steal_request(requesting_device);
+        },
+        [this](std::string word) {
+             std::cout << "Scheduler " << device_number_ << " received: " << word << "\n";
+        }
+    };
+}
 
+void scheduler_actor::on_receive(const token_ptr& tok) {
+    if (tok->getType() == LAUNCH) {
+        queue_.push(tok);
+        schedule_work();
+    } else if (tok->getType() == MEMORY) {
+        const auto& mem = static_cast<const memory_transfer_token&>(*tok);
+        auto response = make_memory_response_token(this, mem, device_number_, 0);
+        this->mail(response).send(mem.getReplyActor());
+    }
+}
 
+void scheduler_actor::on_receive_batch(std::vector<token_ptr> tokens) {
+    for (auto& tok : tokens) {
+        queue_.push(std::move(tok));
+    }
+    schedule_work();
+}
 
-	//handler sent to set the scheduler actors 
-	//do not send a message more than once
-	//or else undefined behavior
-	[&](std::vector<caf::actor> s) {
-		state.schedulers = s;
-	},
+void scheduler_actor::on_reclaim(int /*val*/, int /*mem*/, int /*time*/, int /*dep*/) {
+    if (in_flight_ > 0) {
+        in_flight_--;
+    }
+    schedule_work();
+}
 
+void scheduler_actor::on_set_neighbors(std::vector<caf::actor> neighbors) {
+    schedulers_ = std::move(neighbors);
+}
 
+std::vector<token_ptr> scheduler_actor::on_steal_request(int requesting_device) {
+    if (queue_.size() > 1) {
+        size_t to_give = queue_.size() / 2;
+        std::vector<token_ptr> batch;
+        for (size_t i = 0; i < to_give; ++i) {
+            batch.push_back(queue_.front());
+            queue_.pop();
+        }
+        std::cout << "[INFO] Scheduler " << device_number_ 
+                  << " sharing " << batch.size() 
+                  << " tasks with device " << requesting_device << "\n";
+        return batch;
+    }
+    return {};
+}
 
-	//message handler for a request for work from another scheduler actor
-	[&](int device_number) {
-		state.current_behavior -> handle_load_balance_request(device_number);
-	},
+void scheduler_actor::schedule_work() {
+    int capacity = num_streams_ * stream_depth_;
+
+    // Dispatch queued tasks while we have capacity
+    while (!queue_.empty() && in_flight_ < capacity) {
+        auto tok = queue_.front();
+        queue_.pop();
+
+        if (tok->getType() == LAUNCH) {
+            in_flight_++;
+            const auto& launch = static_cast<const launch_token&>(*tok);
+            
+            // Distribute tasks across streams in round-robin fashion
+            int stream_id = in_flight_ % num_streams_;
+            
+            auto response = make_launch_response_token(this, launch, device_number_, stream_id);
+            this->mail(response).send(launch.getReplyActor());
+        }
+    }
 
-	//message handler for work being transfered over from another scheduler actor
-	[&](std::vector<kernel_graph> work_graphs) {
-		state.current_behavior -> receive_work(work_graphs);
-	},
-	
-	//TEMPORARY FIX SINCE CAF TYPE ID IS STATIC SO POLYMORPHISM WONT WORK HERE
-	//TODO FIGURE OUT A WAY FOR ACK AND ITS CHILDREN TO BE 1 SINGLE CLASS AND
-	//DOWNCASTED EASILY
-	[&](transfer_ack payload) {
-		state.current_behavior->reclaim(static_cast<ack&>(payload));
-	}
-    };
+    // Work Stealing logic
+    if (multi_gpu_ && queue_.empty() && in_flight_ < (capacity / 2)) {
+        for (auto& neighbor : schedulers_) {
+            if (neighbor != this) {
+                this->mail(device_number_).send(neighbor);
+            }
+        }
+    }
 }
 
 } // namespace caf::cuda

From c4d14a680a13422ccbe163ce644dcbc6e7759040 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 10:47:44 -0600
Subject: [PATCH 0887/1000] updated stream management to manage depth of stream

---
 .../control-layer/launch_response_token.hpp   |  3 ++-
 .../cuda/control-layer/scheduler_actor.hpp    |  3 ++-
 .../src/control-layer/scheduler_actor.cpp     | 23 ++++++++++++-------
 3 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
index 8554c941be..985c232a0a 100644
--- a/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/launch_response_token.hpp
@@ -97,7 +97,8 @@ class CAF_CUDA_EXPORT launch_response_token : public response_token {
                 reclaim_value_,
                 reclaim_memory_returned_,
                 reclaim_runtime_,
-                reclaim_dependency_
+                reclaim_dependency_,
+                stream_id_
             ).urgent().send(receiver_);
         } catch (...) {
             // swallow exceptions — destructor safe
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
index 52d939b202..4806c350b3 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
@@ -15,7 +15,7 @@ class CAF_CUDA_EXPORT scheduler_actor : public caf::event_based_actor {
     // Message Handler Methods
     virtual void on_receive(const token_ptr& tok);
     virtual void on_receive_batch(std::vector<token_ptr> tokens);
-    virtual void on_reclaim(int val, int mem, int time, int dep);
+    virtual void on_reclaim(int val, int mem, int time, int dep, int stream_id);
     virtual void on_set_neighbors(std::vector<caf::actor> neighbors);
     virtual std::vector<token_ptr> on_steal_request(int requesting_device);
 
@@ -32,6 +32,7 @@ class CAF_CUDA_EXPORT scheduler_actor : public caf::event_based_actor {
     int in_flight_ = 0;
     std::queue<token_ptr> queue_;
     std::vector<caf::actor> schedulers_;
+    std::queue<int> available_streams_;
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 3a61e48dbf..8c77dbe051 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -12,7 +12,12 @@ scheduler_actor::scheduler_actor(caf::actor_config& cfg,
       num_streams_(num_streams),
       stream_depth_(stream_depth),
       multi_gpu_(multi_gpu) {
-    // Constructor logic if needed
+    // Initialize the pool of available execution slots
+    for (int depth = 0; depth < stream_depth_; ++depth) {
+        for (int s = 0; s < num_streams_; ++s) {
+            available_streams_.push(s);
+        }
+    }
 }
 
 caf::behavior scheduler_actor::make_behavior() {
@@ -23,8 +28,8 @@ caf::behavior scheduler_actor::make_behavior() {
         [this](std::vector<token_ptr> tokens) {
             on_receive_batch(std::move(tokens));
         },
-        [this](int val, int mem, int time, int dep) {
-            on_reclaim(val, mem, time, dep);
+        [this](int val, int mem, int time, int dep, int stream_id) {
+            on_reclaim(val, mem, time, dep, stream_id);
         },
         [this](std::vector<caf::actor> neighbors) {
             on_set_neighbors(std::move(neighbors));
@@ -56,10 +61,11 @@ void scheduler_actor::on_receive_batch(std::vector<token_ptr> tokens) {
     schedule_work();
 }
 
-void scheduler_actor::on_reclaim(int /*val*/, int /*mem*/, int /*time*/, int /*dep*/) {
+void scheduler_actor::on_reclaim(int /*val*/, int /*mem*/, int /*time*/, int /*dep*/, int stream_id) {
     if (in_flight_ > 0) {
         in_flight_--;
     }
+    available_streams_.push(stream_id);
     schedule_work();
 }
 
@@ -87,16 +93,17 @@ void scheduler_actor::schedule_work() {
     int capacity = num_streams_ * stream_depth_;
 
     // Dispatch queued tasks while we have capacity
-    while (!queue_.empty() && in_flight_ < capacity) {
+    while (!queue_.empty() && !available_streams_.empty()) {
         auto tok = queue_.front();
         queue_.pop();
 
         if (tok->getType() == LAUNCH) {
             in_flight_++;
             const auto& launch = static_cast<const launch_token&>(*tok);
-            
-            // Distribute tasks across streams in round-robin fashion
-            int stream_id = in_flight_ % num_streams_;
+
+            // Take an available stream ID from the pool
+            int stream_id = available_streams_.front();
+            available_streams_.pop();
             
             auto response = make_launch_response_token(this, launch, device_number_, stream_id);
             this->mail(response).send(launch.getReplyActor());

From 41458ee396d88eb0c0fd4271d79bf2444e43d3cc Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 11:23:49 -0600
Subject: [PATCH 0888/1000] updated manager to create the new scheduler actor

---
 libcaf_cuda/caf/cuda/manager.hpp        |  25 +++++-
 libcaf_cuda/caf/cuda/manager_config.hpp |  11 ++-
 libcaf_cuda/src/manager.cpp             | 104 ++++++++++++++++++++++++
 3 files changed, 136 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index eb8a6b9f7e..b9ccb3582a 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -23,7 +23,7 @@
 #include "caf/cuda/actor_facade.hpp"
 #include "caf/cuda/platform.hpp"
 #include "caf/cuda/manager_config.hpp"
-#include "caf/cuda/control-layer/token.hpp"
+#include "caf/cuda/control-layer/token.hpp" // For send_scheduler_actor_message
 #include "caf/cuda/control-layer/behavior_token.hpp"
 
 
@@ -186,6 +186,27 @@ class CAF_CUDA_EXPORT manager {
 
   caf::actor spawn_exit_actor(int num_actors);
 
+  // Toggles the scheduler actors on. If called multiple times, it will only spawn them once.
+  void toggle_scheduler_actor(int num_streams, int stream_depth);
+
+  // Enables cuBLAS support on all detected devices.
+  void enable_blas_actors();
+
+  // Enables cuSparse support on all detected devices.
+  void enable_sparse_actors();
+
+  // Sends a batch of tokens to the scheduler actors, distributing them statically.
+  void send_scheduler_actor_message(std::vector<token_ptr> tokens);
+
+  // Returns the first scheduler actor. Useful for single-GPU setups or general dispatch.
+  caf::actor get_scheduler_actor();
+
+  // Returns a specific scheduler actor by device number.
+  caf::actor get_scheduler_actor(int device_number);
+
+  // Sends a behavior change message to a specific scheduler actor.
+  void send_scheduler_actor_message(const std::string& behavior_name, int device_number);
+
 private:
   explicit manager(caf::actor_system& sys)
     : system_(sys), platform_(platform::create()) {
@@ -203,6 +224,8 @@ class CAF_CUDA_EXPORT manager {
 
   mutable std::shared_mutex programs_mutex_;
   std::unordered_map<size_t, program_ptr> programs_;
+  std::vector<caf::actor> scheduler_actors_; // Stores handles to spawned scheduler actors
+  bool scheduler_actors_spawned_ = false; // Flag to ensure idempotency
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/manager_config.hpp b/libcaf_cuda/caf/cuda/manager_config.hpp
index 5adca8a716..f20e99d396 100644
--- a/libcaf_cuda/caf/cuda/manager_config.hpp
+++ b/libcaf_cuda/caf/cuda/manager_config.hpp
@@ -8,16 +8,21 @@ namespace caf::cuda {
 
 class manager_config {
 public:
-    manager_config() : actorBLAS(false), actorSparse(false) {}
-    manager_config(bool blas) : actorBLAS(blas), actorSparse(false) {}
-    manager_config(bool blas, bool sparse) : actorBLAS(blas), actorSparse(sparse) {}
+    manager_config() : actorBLAS(false), actorSparse(false), num_scheduler_streams(500), scheduler_stream_depth(1) {}
+    manager_config(bool blas) : actorBLAS(blas), actorSparse(false), num_scheduler_streams(500), scheduler_stream_depth(1) {}
+    manager_config(bool blas, bool sparse) : actorBLAS(blas), actorSparse(false), num_scheduler_streams(500), scheduler_stream_depth(1) {}
+    manager_config(bool blas, bool sparse, int num_streams, int stream_depth) : actorBLAS(blas), actorSparse(sparse), num_scheduler_streams(num_streams), scheduler_stream_depth(stream_depth) {}
     
     bool getActorBLAS() const { return actorBLAS; }
     bool getActorSparse() const { return actorSparse; }
+    int getNumSchedulerStreams() const { return num_scheduler_streams; }
+    int getSchedulerStreamDepth() const { return scheduler_stream_depth; }
 
 private:
     bool actorBLAS;
     bool actorSparse = false;
+    int num_scheduler_streams;
+    int scheduler_stream_depth;
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 14af914d38..6871b8725d 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -1,8 +1,10 @@
+#include <algorithm>
 #include "caf/cuda/manager.hpp"
 #include <stdexcept>
 #include <mutex>
 #include <map>
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
+#include "caf/cuda/control-layer/token_factory.hpp" // For make_behavior_token
 #include "caf/cuda/manager_config.hpp"
 #include "caf/cuda/control-layer/all-control-layer.hpp"
 
@@ -31,6 +33,7 @@ void manager::init(caf::actor_system& sys) {
     cuCtxGetCurrent(&ctx);
 
     instance_ = new manager(sys);
+    caf::init_global_meta_objects<caf::id_block::cuda_control>(); // Ensure control-layer types are registered
 
     caf::init_global_meta_objects<caf::id_block::cuda>();
 }
@@ -64,6 +67,8 @@ void manager::init(caf::actor_system& sys, manager_config config) {
         for (auto& dev : instance_->platform_->devices())
             dev->enable_cusparse();
     }
+
+  
 }
 
 int manager::get_num_devices() {return platform_ -> get_num_devices();}
@@ -92,6 +97,14 @@ void manager::shutdown() {
     if (!instance_)
         return;
 
+
+    if (instance_->scheduler_actors_spawned_) {
+
+    // Send exit message to all scheduler actors
+    for (const auto& actor : instance_->scheduler_actors_) {
+        caf::anon_mail(caf::exit_reason::user_shutdown).send(actor);
+    }
+  }
     delete instance_;
     instance_ = nullptr;
 }
@@ -312,5 +325,96 @@ caf::actor manager::spawn_exit_actor(int num_actors) {
 
 }
 
+void manager::toggle_scheduler_actor(int num_streams, int stream_depth) {
+    if (scheduler_actors_spawned_)
+        return;
+
+    const auto& devices = platform_->devices();
+    bool multi_gpu = devices.size() > 1;
+
+    for (const auto& dev : devices) {
+        auto hdl = system_.spawn<scheduler_actor>(
+            dev->getId(), 
+            num_streams, 
+            stream_depth, 
+            multi_gpu
+        );
+        scheduler_actors_.push_back(hdl);
+    }
+
+    // Link neighbors for work-stealing
+    if (multi_gpu) {
+        for (const auto& actor : scheduler_actors_) {
+            caf::anon_mail(scheduler_actors_).send(actor);
+        }
+    }
+
+    scheduler_actors_spawned_ = true;
+}
+
+void manager::enable_blas_actors() {
+    for (auto& dev : platform_->devices())
+        dev->enable_cublas();
+}
+
+void manager::enable_sparse_actors() {
+    for (auto& dev : platform_->devices())
+        dev->enable_cusparse();
+}
+
+void manager::send_scheduler_actor_message(std::vector<token_ptr> tokens) {
+    if (scheduler_actors_.empty())
+        return;
+
+    size_t num_schedulers = scheduler_actors_.size();
+    size_t total_tokens = tokens.size();
+    size_t base_chunk = total_tokens / num_schedulers;
+    size_t remainder = total_tokens % num_schedulers;
+
+    auto it = tokens.begin();
+    for (size_t i = 0; i < num_schedulers; ++i) {
+        size_t count = base_chunk + (i < remainder ? 1 : 0);
+        if (count == 0) continue;
+
+        std::vector<token_ptr> chunk;
+        chunk.reserve(count);
+        std::move(it, it + count, std::back_inserter(chunk));
+        it += count;
+
+        caf::anon_mail(std::move(chunk)).send(scheduler_actors_[i]);
+    }
+}
+
+caf::actor manager::get_scheduler_actor() {
+    if (scheduler_actors_.empty()) {
+        throw std::runtime_error(
+            "Scheduler actors not spawned. Call manager::toggle_scheduler_actor() first."
+        );
+    }
+    return scheduler_actors_[0];
+}
+
+caf::actor manager::get_scheduler_actor(int device_number) {
+    if (scheduler_actors_.empty()) {
+        throw std::runtime_error(
+            "Scheduler actors not spawned. Call manager::toggle_scheduler_actor() first."
+        );
+    }
+    
+    if (device_number >= 0 && static_cast<size_t>(device_number) < scheduler_actors_.size()) {
+        return scheduler_actors_[device_number];
+    }
+    
+    // Modulo logic to handle arbitrary device numbers as per documentation
+    int idx = device_number % static_cast<int>(scheduler_actors_.size());
+    return scheduler_actors_[idx];
+}
+
+void manager::send_scheduler_actor_message(const std::string& behavior_name, int device_number) {
+    auto target = get_scheduler_actor(device_number);
+    if (target) {
+        caf::anon_mail(make_behavior_token(behavior_name)).send(target);
+    }
+}
 
 } // namespace caf::cuda

From 703f8e0dc38f58071b1e3e2d0655f89033ad96b1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 11:27:44 -0600
Subject: [PATCH 0889/1000] updated script to parse data

---
 .../generate_graphs.py                        | 99 +++++++++----------
 1 file changed, 48 insertions(+), 51 deletions(-)

diff --git a/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py b/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py
index c9f5db4693..45640b150b 100644
--- a/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py
+++ b/libcaf_cuda/sc26/scripts/Sequence-Independent-Tasks/generate_graphs.py
@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import glob
 import re
 import numpy as np
 import matplotlib.pyplot as plt
@@ -8,17 +7,19 @@
 from collections import defaultdict
 
 # -----------------------------
-# Path Configuration (same style as first script)
+# Path Configuration
 # -----------------------------
 SCRIPT_DIR = Path(__file__).resolve().parent
 
-BASE_DIR = SCRIPT_DIR.parent.parent / "Sequence-Independent-Tasks" / "results"
+# Check if benchmark_results.txt is in the same directory as the script
+INPUT_FILE = SCRIPT_DIR / "benchmark_results.txt"
+OUTPUT_PLOT = SCRIPT_DIR / "mmul_comparison.png"
 
-driver_files = glob.glob(str(BASE_DIR / "matrix_mul_driver_run*.txt"))
-actor_files = glob.glob(str(BASE_DIR / "test_run*.txt"))
-runner_files = glob.glob(str(BASE_DIR / "throughput_mapping_bench_test_run*.txt"))
-
-OUTPUT_PLOT = BASE_DIR / "mmul_comparison.png"
+# Fallback to the original parent-directory structure if not found locally
+if not INPUT_FILE.exists():
+    BASE_DIR = SCRIPT_DIR.parent.parent / "Sequence-Independent-Tasks" / "results"
+    INPUT_FILE = BASE_DIR / "benchmark_results.txt"
+    OUTPUT_PLOT = BASE_DIR / "mmul_comparison.png"
 
 # -----------------------------
 # Data containers
@@ -28,53 +29,49 @@
 command_runner_data = defaultdict(list)
 
 # -----------------------------
-# Regex
+# Regex Patterns
 # -----------------------------
 cuda_pattern = re.compile(r"iterations\s*=\s*(\d+),\s*total GPU time\s*=\s*([0-9.]+)")
-actor_pattern = re.compile(r"iterations\s*=\s*(\d+),\s*time\s*=\s*([0-9.]+)")
-runner_pattern = re.compile(r"iterations\s*=\s*(\d+).*total_time\s*=\s*([0-9.]+)\s*ms")
-
-# -----------------------------
-# Parsers
-# -----------------------------
-def parse_cuda(file):
-    with open(file) as f:
-        for line in f:
+actor_pattern = re.compile(r"iterations\s*=\s*(\d+),\s*total_time\s*=\s*([0-9.]+)")
+runner_pattern = re.compile(r"iterations\s*=\s*(\d+),\s*time\s*=\s*([0-9.]+)")
+
+# -----------------------------
+# Parse benchmark_results.txt
+# -----------------------------
+current_impl = None
+
+with open(INPUT_FILE, "r") as f:
+    for line in f:
+        # Detect block switches
+        if "========== main_cuda_native ==========" in line:
+            current_impl = "cuda"
+            continue
+        elif "========== main_actor_facade ==========" in line:
+            current_impl = "facade"
+            continue
+        elif "========== main_command_runner ==========" in line:
+            current_impl = "runner"
+            continue
+            
+        # Parse data depending on the active block section
+        if current_impl == "cuda":
             m = cuda_pattern.search(line)
             if m:
                 it = int(m.group(1))
                 cuda_data[it].append(float(m.group(2)))
-
-def parse_actor(file):
-    with open(file) as f:
-        for line in f:
+        elif current_impl == "facade":
             m = actor_pattern.search(line)
             if m:
                 it = int(m.group(1))
                 actor_facade_data[it].append(float(m.group(2)))
-
-def parse_runner(file):
-    with open(file) as f:
-        for line in f:
+        elif current_impl == "runner":
             m = runner_pattern.search(line)
             if m:
                 it = int(m.group(1))
                 command_runner_data[it].append(float(m.group(2)))
 
 # -----------------------------
-# Parse all files
-# -----------------------------
-for f in driver_files:
-    parse_cuda(f)
-
-for f in actor_files:
-    parse_actor(f)
-
-for f in runner_files:
-    parse_runner(f)
-
-# -----------------------------
-# Aggregate
+# Aggregate and Compute Means
 # -----------------------------
 iterations = sorted(cuda_data.keys())
 
@@ -83,7 +80,7 @@ def parse_runner(file):
 runner_mean = [np.mean(command_runner_data[i]) for i in iterations]
 
 # -----------------------------
-# Print
+# Print Performance Table
 # -----------------------------
 print("\nMean Performance Comparison\n")
 print(f"{'Iterations':>10} {'CUDA(ms)':>12} {'Facade(ms)':>12} {'Runner(ms)':>12} {'Facade Ovhd %':>15}")
@@ -93,20 +90,20 @@ def parse_runner(file):
     print(f"{it:>10} {cuda_mean[i]:>12.2f} {actor_mean[i]:>12.2f} {runner_mean[i]:>12.2f} {pct:>14.2f}%")
 
 # -----------------------------
-# Plot
+# Plot and Save Chart
 # -----------------------------
-plt.figure(figsize=(8,6))
+fig, ax = plt.subplots(figsize=(8, 6))
 
-plt.plot(iterations, cuda_mean, marker='o', label="CUDA Native")
-plt.plot(iterations, actor_mean, marker='s', label="Actor Facade")
-plt.plot(iterations, runner_mean, marker='^', label="Command Runner")
+ax.plot(iterations, cuda_mean, marker='o', label="CUDA Native")
+ax.plot(iterations, actor_mean, marker='s', label="Actor Facade")
+ax.plot(iterations, runner_mean, marker='^', label="Command Runner")
 
-plt.xlabel("Iterations")
-plt.ylabel("Time (ms)")
-plt.title("Matrix Multiplication Performance")
-plt.legend()
-plt.grid(True)
+ax.set_xlabel("Iterations")
+ax.set_ylabel("Time (ms)")
+ax.set_title("Matrix Multiplication Performance")
+ax.legend()
+ax.grid(True)
 plt.tight_layout()
 
 plt.savefig(OUTPUT_PLOT)
-plt.show()
\ No newline at end of file
+# plt.show() # Uncomment if running in an interactive graphical interface
\ No newline at end of file

From 5c001f43ee4b0e292517da517f0a4f6d056bed04 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 11:50:28 -0600
Subject: [PATCH 0890/1000] inital commit

---
 .../tests/scheduler_integration_test.cpp      | 163 ++++++++++++++++++
 1 file changed, 163 insertions(+)
 create mode 100644 libcaf_cuda/tests/scheduler_integration_test.cpp

diff --git a/libcaf_cuda/tests/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler_integration_test.cpp
new file mode 100644
index 0000000000..2cfc9a3903
--- /dev/null
+++ b/libcaf_cuda/tests/scheduler_integration_test.cpp
@@ -0,0 +1,163 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
+#include <iostream>
+#include <vector>
+#include <string>
+#include <random>
+#include <unordered_map>
+#include <unordered_set>
+#include <algorithm>
+
+using namespace caf;
+using namespace caf::cuda;
+
+// A generic command runner to provide access to CUDA stream callbacks
+static command_runner<> runner;
+
+struct task_actor_state {
+    std::vector<int> h_a;
+    std::vector<int> h_b;
+    std::vector<int> h_c;
+    int N_val; // Store N for this specific task
+};
+
+// MatrixPool structure from mmul-random-batch-benchmark
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
+};
+
+// create_matrix_pool_random function from mmul-random-batch-benchmark
+MatrixPool create_matrix_pool_random(
+    int num_sizes,
+    int min_N,
+    int max_N,
+    unsigned int seed
+) {
+    MatrixPool pool;
+    std::mt19937 rng(seed);
+    std::uniform_int_distribution<int> dist(min_N, max_N);
+    std::unordered_set<int> used;
+    while (used.size() < static_cast<size_t>(num_sizes)) {
+        int N = dist(rng);
+        if (used.insert(N).second) {
+            pool.A[N] = std::vector<int>(N * N, 1);
+            pool.B[N] = std::vector<int>(N * N, 2); // Changed to 2 for distinct input
+        }
+    }
+    return pool;
+}
+
+// This actor represents a single task that requests permission from the scheduler.
+behavior task_actor_fun(stateful_actor<task_actor_state>* self) {
+    auto& st = self->state();
+
+    return {
+        [=](response_token_ptr res) mutable {
+            if (res->getType() == LAUNCH_RESPONSE) {
+                auto& st_inner = self->state();
+                
+                // 1. Setup GPU arguments.
+                auto in_a = create_in_arg(st_inner.h_a);
+                auto in_b = create_in_arg(st_inner.h_b);
+                auto out_c = create_out_arg_with_size<int>(st_inner.N_val * st_inner.N_val);
+                auto in_n = create_in_arg(st_inner.N_val);
+
+                // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
+                auto result_tuple = runner.run_async(res, in_a, in_b, out_c, in_n);
+                auto d_c = std::get<2>(result_tuple);
+
+                // 3. Asynchronous Copyback.
+                // The launch_response_token is released inside the callback 
+                // to signal to the scheduler that the resource is free. (No serial verification here)
+                runner.copy_to_host_async(d_c, st_inner.h_c.data(), st_inner.N_val * st_inner.N_val, [res, self](int* /*ptr*/, size_t /*sz*/) {
+                    std::cout << "[TASK] " << res->name() << " finished. Releasing token." << std::endl;
+                    res->release();
+                });
+            }
+        }
+    };
+}
+
+// Helper function to initialize task_actor_state
+behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, int N_val, const std::vector<int>& h_a_data, const std::vector<int>& h_b_data) {
+    self->state().N_val = N_val;
+    self->state().h_a = h_a_data;
+    self->state().h_b = h_b_data;
+    self->state().h_c.resize(N_val * N_val, 0);
+    return task_actor_fun(self);
+}
+
+template <class Fn>
+double time_run(Fn&& fn) {
+    auto start = std::chrono::steady_clock::now();
+    fn();
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+    return elapsed.count();
+}
+
+void run_scheduler_integration_scaling_test(actor_system& sys) {
+    const int min_N = 32;
+    const int max_N = 1024;
+    const int num_distinct_sizes = 10;
+    const std::vector<int> actor_counts = {60, 120};
+
+    // Generate deterministic random pool once
+    MatrixPool pool = create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42);
+    
+    std::vector<int> available_Ns;
+    for (const auto& pair : pool.A) available_Ns.push_back(pair.first);
+
+    for (int num_tasks : actor_counts) {
+        manager_config config;
+        manager::init(sys, config);
+        auto& mgr = manager::get();
+
+        // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
+        mgr.toggle_scheduler_actor(4, 2);
+
+        auto program = mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
+        const int THREADS = 32;
+
+        std::vector<token_ptr> tokens;
+        std::mt19937 rng(42);
+        std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
+
+        std::cout << "=====================================\n";
+        std::cout << "Scheduler Test | tasks=" << num_tasks << "\n";
+
+        // Spawn task actors and prepare tokens
+        for (int i = 0; i < num_tasks; ++i) {
+            int current_N = available_Ns[dist_N_idx(rng)];
+            nd_range range((current_N + THREADS - 1) / THREADS, 
+                           (current_N + THREADS - 1) / THREADS, 1, 
+                           THREADS, THREADS, 1);
+            
+            auto worker = sys.spawn(make_task_actor_behavior, 
+                                    current_N, 
+                                    pool.A.at(current_N), 
+                                    pool.B.at(current_N));
+
+            tokens.push_back(make_launch_token(program, range, 0, 
+                             "task_" + std::to_string(i), worker));
+        }
+
+        double elapsed = time_run([&]() {
+            std::cout << "[MAIN] Dispatching batch to scheduler..." << std::endl;
+            mgr.send_scheduler_actor_message(std::move(tokens));
+            sys.await_all_actors_done();
+        });
+
+        std::cout << "Run complete. Time: " << elapsed << " s\n";
+        manager::shutdown();
+    }
+}
+
+void caf_main(actor_system& sys) {
+    run_scheduler_integration_scaling_test(sys);
+    std::cout << "[MAIN] Integration test complete." << std::endl;
+}
+
+CAF_MAIN(id_block::cuda_control)
\ No newline at end of file

From 33cdb3454e207f8885810fabc25729e94dbba8a5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 11:52:11 -0600
Subject: [PATCH 0891/1000] moved file to right location

---
 .../scheduler_integration_test.cpp            | 163 ++++++++++++++++++
 1 file changed, 163 insertions(+)
 create mode 100644 libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
new file mode 100644
index 0000000000..2cfc9a3903
--- /dev/null
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
@@ -0,0 +1,163 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
+#include <iostream>
+#include <vector>
+#include <string>
+#include <random>
+#include <unordered_map>
+#include <unordered_set>
+#include <algorithm>
+
+using namespace caf;
+using namespace caf::cuda;
+
+// A generic command runner to provide access to CUDA stream callbacks
+static command_runner<> runner;
+
+struct task_actor_state {
+    std::vector<int> h_a;
+    std::vector<int> h_b;
+    std::vector<int> h_c;
+    int N_val; // Store N for this specific task
+};
+
+// MatrixPool structure from mmul-random-batch-benchmark
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
+};
+
+// create_matrix_pool_random function from mmul-random-batch-benchmark
+MatrixPool create_matrix_pool_random(
+    int num_sizes,
+    int min_N,
+    int max_N,
+    unsigned int seed
+) {
+    MatrixPool pool;
+    std::mt19937 rng(seed);
+    std::uniform_int_distribution<int> dist(min_N, max_N);
+    std::unordered_set<int> used;
+    while (used.size() < static_cast<size_t>(num_sizes)) {
+        int N = dist(rng);
+        if (used.insert(N).second) {
+            pool.A[N] = std::vector<int>(N * N, 1);
+            pool.B[N] = std::vector<int>(N * N, 2); // Changed to 2 for distinct input
+        }
+    }
+    return pool;
+}
+
+// This actor represents a single task that requests permission from the scheduler.
+behavior task_actor_fun(stateful_actor<task_actor_state>* self) {
+    auto& st = self->state();
+
+    return {
+        [=](response_token_ptr res) mutable {
+            if (res->getType() == LAUNCH_RESPONSE) {
+                auto& st_inner = self->state();
+                
+                // 1. Setup GPU arguments.
+                auto in_a = create_in_arg(st_inner.h_a);
+                auto in_b = create_in_arg(st_inner.h_b);
+                auto out_c = create_out_arg_with_size<int>(st_inner.N_val * st_inner.N_val);
+                auto in_n = create_in_arg(st_inner.N_val);
+
+                // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
+                auto result_tuple = runner.run_async(res, in_a, in_b, out_c, in_n);
+                auto d_c = std::get<2>(result_tuple);
+
+                // 3. Asynchronous Copyback.
+                // The launch_response_token is released inside the callback 
+                // to signal to the scheduler that the resource is free. (No serial verification here)
+                runner.copy_to_host_async(d_c, st_inner.h_c.data(), st_inner.N_val * st_inner.N_val, [res, self](int* /*ptr*/, size_t /*sz*/) {
+                    std::cout << "[TASK] " << res->name() << " finished. Releasing token." << std::endl;
+                    res->release();
+                });
+            }
+        }
+    };
+}
+
+// Helper function to initialize task_actor_state
+behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, int N_val, const std::vector<int>& h_a_data, const std::vector<int>& h_b_data) {
+    self->state().N_val = N_val;
+    self->state().h_a = h_a_data;
+    self->state().h_b = h_b_data;
+    self->state().h_c.resize(N_val * N_val, 0);
+    return task_actor_fun(self);
+}
+
+template <class Fn>
+double time_run(Fn&& fn) {
+    auto start = std::chrono::steady_clock::now();
+    fn();
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+    return elapsed.count();
+}
+
+void run_scheduler_integration_scaling_test(actor_system& sys) {
+    const int min_N = 32;
+    const int max_N = 1024;
+    const int num_distinct_sizes = 10;
+    const std::vector<int> actor_counts = {60, 120};
+
+    // Generate deterministic random pool once
+    MatrixPool pool = create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42);
+    
+    std::vector<int> available_Ns;
+    for (const auto& pair : pool.A) available_Ns.push_back(pair.first);
+
+    for (int num_tasks : actor_counts) {
+        manager_config config;
+        manager::init(sys, config);
+        auto& mgr = manager::get();
+
+        // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
+        mgr.toggle_scheduler_actor(4, 2);
+
+        auto program = mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
+        const int THREADS = 32;
+
+        std::vector<token_ptr> tokens;
+        std::mt19937 rng(42);
+        std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
+
+        std::cout << "=====================================\n";
+        std::cout << "Scheduler Test | tasks=" << num_tasks << "\n";
+
+        // Spawn task actors and prepare tokens
+        for (int i = 0; i < num_tasks; ++i) {
+            int current_N = available_Ns[dist_N_idx(rng)];
+            nd_range range((current_N + THREADS - 1) / THREADS, 
+                           (current_N + THREADS - 1) / THREADS, 1, 
+                           THREADS, THREADS, 1);
+            
+            auto worker = sys.spawn(make_task_actor_behavior, 
+                                    current_N, 
+                                    pool.A.at(current_N), 
+                                    pool.B.at(current_N));
+
+            tokens.push_back(make_launch_token(program, range, 0, 
+                             "task_" + std::to_string(i), worker));
+        }
+
+        double elapsed = time_run([&]() {
+            std::cout << "[MAIN] Dispatching batch to scheduler..." << std::endl;
+            mgr.send_scheduler_actor_message(std::move(tokens));
+            sys.await_all_actors_done();
+        });
+
+        std::cout << "Run complete. Time: " << elapsed << " s\n";
+        manager::shutdown();
+    }
+}
+
+void caf_main(actor_system& sys) {
+    run_scheduler_integration_scaling_test(sys);
+    std::cout << "[MAIN] Integration test complete." << std::endl;
+}
+
+CAF_MAIN(id_block::cuda_control)
\ No newline at end of file

From fa1009b87397e048b8170c362d85692f84784197 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 11:56:13 -0600
Subject: [PATCH 0892/1000] updated tests to be more alike

---
 .../mmul-randonom-batch-benchmark/main.test.cpp           | 4 ++--
 .../scheduler_integration_test.cpp                        | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index bb2438e795..066811c37f 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -390,8 +390,8 @@ void run_mmul_random_scaling_tests(caf::actor_system& sys,
     const int max_N = 2048;
     const int num_sizes = 10;
 
-    const int workers_per_gpu = 8; // Admission control: only 16 concurrent tasks per GPU
-    const int max_in_flight_tasks_per_worker = 3; // Each worker keeps 2 tasks in flight
+    const int workers_per_gpu = 4; // Admission control: only 16 concurrent tasks per GPU
+    const int max_in_flight_tasks_per_worker = 2; // Each worker keeps 2 tasks in flight
 
     const std::vector<int> actor_counts = {
 	  1,30000,40000,50000
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
index 2cfc9a3903..1d418b829e 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
@@ -43,7 +43,7 @@ MatrixPool create_matrix_pool_random(
         int N = dist(rng);
         if (used.insert(N).second) {
             pool.A[N] = std::vector<int>(N * N, 1);
-            pool.B[N] = std::vector<int>(N * N, 2); // Changed to 2 for distinct input
+            pool.B[N] = std::vector<int>(N * N, 1); 
         }
     }
     return pool;
@@ -100,9 +100,9 @@ double time_run(Fn&& fn) {
 
 void run_scheduler_integration_scaling_test(actor_system& sys) {
     const int min_N = 32;
-    const int max_N = 1024;
+    const int max_N = 2048;
     const int num_distinct_sizes = 10;
-    const std::vector<int> actor_counts = {60, 120};
+    const std::vector<int> actor_counts = {50000};
 
     // Generate deterministic random pool once
     MatrixPool pool = create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42);
@@ -118,7 +118,7 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
         mgr.toggle_scheduler_actor(4, 2);
 
-        auto program = mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
+        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
         const int THREADS = 32;
 
         std::vector<token_ptr> tokens;

From b11827bdef069a24b21db5354a8ba1e54b8f008d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 12:00:28 -0600
Subject: [PATCH 0893/1000] updated cmakelists

---
 .../mmul-randonom-batch-benchmark/CMakeLists.txt   | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
index 9e2014d33b..22963221d9 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/CMakeLists.txt
@@ -32,6 +32,8 @@ include_directories(
 # 5) Declare your executables
 add_executable(test main.test.cpp)
 target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+add_executable(scheduler_test scheduler_integration_test.cpp)
+target_compile_definitions(scheduler_test PRIVATE CAF_ENABLE_LOGGING)
 
 add_executable(work-stealing work-stealing.cpp)
 target_compile_definitions(work-stealing PRIVATE CAF_ENABLE_LOGGING)
@@ -49,6 +51,18 @@ target_link_libraries(test
     CUDA::cusparse
 )
 
+
+
+target_link_libraries(scheduler_test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc 
+    CUDA::cublas
+    CUDA::cusparse
+)
+
 target_link_libraries(work-stealing
   PRIVATE
     "${CAF_BUILD}/libcaf_core/libcaf_core.so"

From 1e6371dec903ef3bc9b772db6baac137a6cc59e2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 12:07:55 -0600
Subject: [PATCH 0894/1000] fixed compiler errors

---
 .../scheduler_integration_test.cpp            | 22 ++++++++++++-------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
index 1d418b829e..07ea34f7a5 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
@@ -13,12 +13,13 @@ using namespace caf;
 using namespace caf::cuda;
 
 // A generic command runner to provide access to CUDA stream callbacks
-static command_runner<> runner;
+static command_runner<in<int>, in<int>, out<int>, in<int>> runner;
 
 struct task_actor_state {
     std::vector<int> h_a;
     std::vector<int> h_b;
     std::vector<int> h_c;
+    program_ptr prog; // Store the program handle in the actor state
     int N_val; // Store N for this specific task
 };
 
@@ -43,7 +44,7 @@ MatrixPool create_matrix_pool_random(
         int N = dist(rng);
         if (used.insert(N).second) {
             pool.A[N] = std::vector<int>(N * N, 1);
-            pool.B[N] = std::vector<int>(N * N, 1); 
+            pool.B[N] = std::vector<int>(N * N, 2); // Changed to 2 for distinct input
         }
     }
     return pool;
@@ -58,6 +59,9 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self) {
             if (res->getType() == LAUNCH_RESPONSE) {
                 auto& st_inner = self->state();
                 
+                // We need to cast the base response_token to access the specific nd_range stored in it.
+                auto launch_res = static_cast<launch_response_token*>(res.get());
+
                 // 1. Setup GPU arguments.
                 auto in_a = create_in_arg(st_inner.h_a);
                 auto in_b = create_in_arg(st_inner.h_b);
@@ -65,7 +69,7 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self) {
                 auto in_n = create_in_arg(st_inner.N_val);
 
                 // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
-                auto result_tuple = runner.run_async(res, in_a, in_b, out_c, in_n);
+                auto result_tuple = runner.run_async(st_inner.prog, launch_res->getRange(), res, in_a, in_b, out_c, in_n);
                 auto d_c = std::get<2>(result_tuple);
 
                 // 3. Asynchronous Copyback.
@@ -81,7 +85,8 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self) {
 }
 
 // Helper function to initialize task_actor_state
-behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, int N_val, const std::vector<int>& h_a_data, const std::vector<int>& h_b_data) {
+behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, const std::vector<int>& h_a_data, const std::vector<int>& h_b_data) {
+    self->state().prog = std::move(prog);
     self->state().N_val = N_val;
     self->state().h_a = h_a_data;
     self->state().h_b = h_b_data;
@@ -100,9 +105,9 @@ double time_run(Fn&& fn) {
 
 void run_scheduler_integration_scaling_test(actor_system& sys) {
     const int min_N = 32;
-    const int max_N = 2048;
+    const int max_N = 1024;
     const int num_distinct_sizes = 10;
-    const std::vector<int> actor_counts = {50000};
+    const std::vector<int> actor_counts = {60, 120};
 
     // Generate deterministic random pool once
     MatrixPool pool = create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42);
@@ -118,7 +123,7 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
         mgr.toggle_scheduler_actor(4, 2);
 
-        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        auto program = mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
         const int THREADS = 32;
 
         std::vector<token_ptr> tokens;
@@ -136,6 +141,7 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
                            THREADS, THREADS, 1);
             
             auto worker = sys.spawn(make_task_actor_behavior, 
+                                    program,
                                     current_N, 
                                     pool.A.at(current_N), 
                                     pool.B.at(current_N));
@@ -160,4 +166,4 @@ void caf_main(actor_system& sys) {
     std::cout << "[MAIN] Integration test complete." << std::endl;
 }
 
-CAF_MAIN(id_block::cuda_control)
\ No newline at end of file
+CAF_MAIN(id_block::cuda_control)

From 6f78d6cd26cfdbfb78bf02df199d9e1362b477a0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 12:18:10 -0600
Subject: [PATCH 0895/1000] updated test

---
 .../scheduler_integration_test.cpp            | 22 +++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
index 07ea34f7a5..8ee4bf75b7 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
@@ -51,7 +51,7 @@ MatrixPool create_matrix_pool_random(
 }
 
 // This actor represents a single task that requests permission from the scheduler.
-behavior task_actor_fun(stateful_actor<task_actor_state>* self) {
+behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_actor) {
     auto& st = self->state();
 
     return {
@@ -75,9 +75,10 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self) {
                 // 3. Asynchronous Copyback.
                 // The launch_response_token is released inside the callback 
                 // to signal to the scheduler that the resource is free. (No serial verification here)
-                runner.copy_to_host_async(d_c, st_inner.h_c.data(), st_inner.N_val * st_inner.N_val, [res, self](int* /*ptr*/, size_t /*sz*/) {
+                runner.copy_to_host_async(d_c, st_inner.h_c.data(), st_inner.N_val * st_inner.N_val, [res, self, exit_actor = exit_actor](int* /*ptr*/, size_t /*sz*/) {
                     std::cout << "[TASK] " << res->name() << " finished. Releasing token." << std::endl;
                     res->release();
+                    anon_mail(1).send(exit_actor);
                 });
             }
         }
@@ -85,13 +86,13 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self) {
 }
 
 // Helper function to initialize task_actor_state
-behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, const std::vector<int>& h_a_data, const std::vector<int>& h_b_data) {
+behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, const std::vector<int>& h_a_data, const std::vector<int>& h_b_data, caf::actor exit_actor) {
     self->state().prog = std::move(prog);
     self->state().N_val = N_val;
     self->state().h_a = h_a_data;
     self->state().h_b = h_b_data;
     self->state().h_c.resize(N_val * N_val, 0);
-    return task_actor_fun(self);
+    return task_actor_fun(self, exit_actor); // Pass exit_actor to task_actor_fun
 }
 
 template <class Fn>
@@ -123,7 +124,7 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
         mgr.toggle_scheduler_actor(4, 2);
 
-        auto program = mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
+        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
         const int THREADS = 32;
 
         std::vector<token_ptr> tokens;
@@ -133,6 +134,9 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         std::cout << "=====================================\n";
         std::cout << "Scheduler Test | tasks=" << num_tasks << "\n";
 
+        // Spawn the exit actor for this specific test run (moved inside the loop)
+        auto exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_tasks);
+
         // Spawn task actors and prepare tokens
         for (int i = 0; i < num_tasks; ++i) {
             int current_N = available_Ns[dist_N_idx(rng)];
@@ -140,11 +144,12 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
                            (current_N + THREADS - 1) / THREADS, 1, 
                            THREADS, THREADS, 1);
             
-            auto worker = sys.spawn(make_task_actor_behavior, 
+            auto worker = sys.spawn(make_task_actor_behavior,
                                     program,
                                     current_N, 
                                     pool.A.at(current_N), 
-                                    pool.B.at(current_N));
+                                    pool.B.at(current_N),
+                                    exit_actor); // Pass exit_actor to task actors
 
             tokens.push_back(make_launch_token(program, range, 0, 
                              "task_" + std::to_string(i), worker));
@@ -153,12 +158,11 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         double elapsed = time_run([&]() {
             std::cout << "[MAIN] Dispatching batch to scheduler..." << std::endl;
             mgr.send_scheduler_actor_message(std::move(tokens));
-            sys.await_all_actors_done();
         });
 
         std::cout << "Run complete. Time: " << elapsed << " s\n";
-        manager::shutdown();
     }
+    sys.await_all_actors_done(); // Wait for all actors, including exit_actor, to finish
 }
 
 void caf_main(actor_system& sys) {

From d34b6086d73b1955bb8066711d0ba58067000859 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 12:20:24 -0600
Subject: [PATCH 0896/1000] exporting to make visibile

---
 libcaf_cuda/caf/cuda/control-layer/exit_actor/exit_actor.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/exit_actor/exit_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/exit_actor/exit_actor.hpp
index 15ff353a38..aba59fbc50 100644
--- a/libcaf_cuda/caf/cuda/control-layer/exit_actor/exit_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/exit_actor/exit_actor.hpp
@@ -6,5 +6,5 @@ struct exit_actor_state {
         int completed = 0;
 };
 
-caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit); 
+caf::behavior CAF_CUDA_EXPORT  exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit); 
 } //namespace caf::cuda

From d9787090540606b8e800fe7d97c56a40a4a90116 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 9 Jun 2026 12:22:43 -0600
Subject: [PATCH 0897/1000] fixed test

---
 .../scheduler_integration_test.cpp                            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
index 8ee4bf75b7..8eabaa8fdc 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
@@ -106,9 +106,9 @@ double time_run(Fn&& fn) {
 
 void run_scheduler_integration_scaling_test(actor_system& sys) {
     const int min_N = 32;
-    const int max_N = 1024;
+    const int max_N = 2048;
     const int num_distinct_sizes = 10;
-    const std::vector<int> actor_counts = {60, 120};
+    const std::vector<int> actor_counts = {50000};
 
     // Generate deterministic random pool once
     MatrixPool pool = create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42);

From 7352b970c04b0954087478d911bc633b6cbf9d67 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 07:56:22 -0600
Subject: [PATCH 0898/1000] fixed oom error

---
 .../scheduler_integration_test.cpp            | 54 +++++++++----------
 1 file changed, 26 insertions(+), 28 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
index 8eabaa8fdc..89b87b510c 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
@@ -15,20 +15,18 @@ using namespace caf::cuda;
 // A generic command runner to provide access to CUDA stream callbacks
 static command_runner<in<int>, in<int>, out<int>, in<int>> runner;
 
-struct task_actor_state {
-    std::vector<int> h_a;
-    std::vector<int> h_b;
-    std::vector<int> h_c;
-    program_ptr prog; // Store the program handle in the actor state
-    int N_val; // Store N for this specific task
-};
-
 // MatrixPool structure from mmul-random-batch-benchmark
 struct MatrixPool {
     std::unordered_map<int, std::vector<int>> A;
     std::unordered_map<int, std::vector<int>> B;
 };
 
+struct task_actor_state {
+    program_ptr prog;
+    int N_val;
+    std::shared_ptr<MatrixPool> pool;
+};
+
 // create_matrix_pool_random function from mmul-random-batch-benchmark
 MatrixPool create_matrix_pool_random(
     int num_sizes,
@@ -52,31 +50,32 @@ MatrixPool create_matrix_pool_random(
 
 // This actor represents a single task that requests permission from the scheduler.
 behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_actor) {
-    auto& st = self->state();
-
     return {
         [=](response_token_ptr res) mutable {
             if (res->getType() == LAUNCH_RESPONSE) {
-                auto& st_inner = self->state();
+                auto& st = self->state();
                 
                 // We need to cast the base response_token to access the specific nd_range stored in it.
                 auto launch_res = static_cast<launch_response_token*>(res.get());
+                int N = st.N_val;
 
                 // 1. Setup GPU arguments.
-                auto in_a = create_in_arg(st_inner.h_a);
-                auto in_b = create_in_arg(st_inner.h_b);
-                auto out_c = create_out_arg_with_size<int>(st_inner.N_val * st_inner.N_val);
-                auto in_n = create_in_arg(st_inner.N_val);
+                // Fetch data from the shared pool only when scheduled to save RAM
+                auto in_a = create_in_arg(st.pool->A.at(N));
+                auto in_b = create_in_arg(st.pool->B.at(N));
+                auto out_c = create_out_arg_with_size<int>(N * N);
+                auto in_n = create_in_arg(N);
 
                 // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
-                auto result_tuple = runner.run_async(st_inner.prog, launch_res->getRange(), res, in_a, in_b, out_c, in_n);
+                auto result_tuple = runner.run_async(st.prog, launch_res->getRange(), res, in_a, in_b, out_c, in_n);
                 auto d_c = std::get<2>(result_tuple);
 
                 // 3. Asynchronous Copyback.
+                // Allocate a local buffer for the result to keep the total system memory low.
+                auto h_c = std::make_shared<std::vector<int>>(N * N);
                 // The launch_response_token is released inside the callback 
                 // to signal to the scheduler that the resource is free. (No serial verification here)
-                runner.copy_to_host_async(d_c, st_inner.h_c.data(), st_inner.N_val * st_inner.N_val, [res, self, exit_actor = exit_actor](int* /*ptr*/, size_t /*sz*/) {
-                    std::cout << "[TASK] " << res->name() << " finished. Releasing token." << std::endl;
+                runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [res, exit_actor, h_c](int* /*ptr*/, size_t /*sz*/) {
                     res->release();
                     anon_mail(1).send(exit_actor);
                 });
@@ -86,12 +85,11 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_
 }
 
 // Helper function to initialize task_actor_state
-behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, const std::vector<int>& h_a_data, const std::vector<int>& h_b_data, caf::actor exit_actor) {
-    self->state().prog = std::move(prog);
-    self->state().N_val = N_val;
-    self->state().h_a = h_a_data;
-    self->state().h_b = h_b_data;
-    self->state().h_c.resize(N_val * N_val, 0);
+behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, std::shared_ptr<MatrixPool> pool, caf::actor exit_actor) {
+    auto& st = self->state();
+    st.prog = std::move(prog);
+    st.N_val = N_val;
+    st.pool = std::move(pool);
     return task_actor_fun(self, exit_actor); // Pass exit_actor to task_actor_fun
 }
 
@@ -111,10 +109,11 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
     const std::vector<int> actor_counts = {50000};
 
     // Generate deterministic random pool once
-    MatrixPool pool = create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42);
+    auto pool_ptr = std::make_shared<MatrixPool>(
+        create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42));
     
     std::vector<int> available_Ns;
-    for (const auto& pair : pool.A) available_Ns.push_back(pair.first);
+    for (const auto& pair : pool_ptr->A) available_Ns.push_back(pair.first);
 
     for (int num_tasks : actor_counts) {
         manager_config config;
@@ -147,8 +146,7 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
             auto worker = sys.spawn(make_task_actor_behavior,
                                     program,
                                     current_N, 
-                                    pool.A.at(current_N), 
-                                    pool.B.at(current_N),
+                                    pool_ptr,
                                     exit_actor); // Pass exit_actor to task actors
 
             tokens.push_back(make_launch_token(program, range, 0, 

From 6fd59e199d6686190705897ce95499bffc2190a3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 07:59:11 -0600
Subject: [PATCH 0899/1000] fixed errors with the timer

---
 .../scheduler_integration_test.cpp                   | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
index 89b87b510c..69d21a59fb 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
@@ -108,6 +108,11 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
     const int num_distinct_sizes = 10;
     const std::vector<int> actor_counts = {50000};
 
+
+    //  const std::vector<int> actor_counts = {5};
+
+
+
     // Generate deterministic random pool once
     auto pool_ptr = std::make_shared<MatrixPool>(
         create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42));
@@ -156,11 +161,16 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         double elapsed = time_run([&]() {
             std::cout << "[MAIN] Dispatching batch to scheduler..." << std::endl;
             mgr.send_scheduler_actor_message(std::move(tokens));
+
+            // The dispatch is asynchronous. To get an accurate measurement, we must
+            // block until the exit_actor terminates (signaling all 50k tasks are done).
+            scoped_actor self{sys};
+            self->wait_for(exit_actor);
         });
 
         std::cout << "Run complete. Time: " << elapsed << " s\n";
+        manager::shutdown(); // Reset manager state for the next potential iteration
     }
-    sys.await_all_actors_done(); // Wait for all actors, including exit_actor, to finish
 }
 
 void caf_main(actor_system& sys) {

From 9218736067289c64322e9810a4f635bdb8258bc6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 09:49:12 -0600
Subject: [PATCH 0900/1000] split actors into seperate files

---
 .../CMakeLists.txt                            |   7 +-
 .../fault-tolerance-workload-test/atoms.hpp   |  51 ++
 .../ft_cg_actor.hpp                           | 171 +++++++
 .../main.test.cpp                             | 473 +-----------------
 .../supervisor_actor.hpp                      | 157 ++++++
 5 files changed, 389 insertions(+), 470 deletions(-)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/atoms.hpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/supervisor_actor.hpp

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
index a492765224..242169debe 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
@@ -52,8 +52,13 @@ add_custom_command(
 add_custom_target(stability_kernels_cubin ALL DEPENDS ${STABILITY_KERNEL_CUBIN})
 
 
-# 5) Declare your executable
+# 5) Declare your executable and its source files
 add_executable(test main.test.cpp sparse_utils.cpp)
+target_sources(test PRIVATE
+  atoms.hpp
+  ft_cg_actor.hpp
+  supervisor_actor.hpp
+)
 
 target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
 
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/atoms.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/atoms.hpp
new file mode 100644
index 0000000000..ab2a741b6c
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/atoms.hpp
@@ -0,0 +1,51 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include "sparse_utils.hpp"
+
+constexpr int MAX_ITERATIONS = 16000;
+
+CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
+    CAF_ADD_ATOM(workload_test, get_work_atom)
+    CAF_ADD_ATOM(workload_test, release_memory_atom)
+    CAF_ADD_ATOM(workload_test, request_work_atom)
+    CAF_ADD_ATOM(workload_test, worker_done_atom)
+    CAF_ADD_ATOM(workload_test, work_tick_atom)
+    CAF_ADD_ATOM(workload_test, add_work_atom)
+    CAF_ADD_ATOM(workload_test, steal_work_atom)
+    CAF_ADD_ATOM(workload_test, update_stream_atom)
+    CAF_ADD_ATOM(workload_test, shutdown_atom)
+    CAF_ADD_TYPE_ID(workload_test, (SolverType))
+    CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
+    CAF_ADD_TYPE_ID(workload_test, (MatrixData))
+    CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
+    CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
+CAF_END_TYPE_ID_BLOCK(workload_test)
+
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixData)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(SolverType)
+
+enum cg_error_type : int {
+  CG_SUCCESS = 0,
+  CG_MAX_ITER = 1,
+  CG_NAN_INF = 2,
+  CG_STAGNATION = 3,
+  CG_BREAKDOWN = 4,
+  CG_RESIDUAL_FACTOR_FAIL = 5
+};
+
+inline std::string to_string(cg_error_type err) {
+  switch (err) {
+    case CG_SUCCESS: return "Success";
+    case CG_MAX_ITER: return "Maximum Iterations Reached";
+    case CG_NAN_INF: return "Stability Check Failed (NaN/Inf Detected)";
+    case CG_STAGNATION: return "Stagnation Detected (Residual stopped changing)";
+    case CG_BREAKDOWN: return "Solver Breakdown (Division by zero/near-zero)";
+    case CG_RESIDUAL_FACTOR_FAIL: return "Residual Factor Check Failed";
+    default: return "Unknown Error (" + std::to_string(static_cast<int>(err)) + ")";
+  }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp
new file mode 100644
index 0000000000..86d7a4fe97
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp
@@ -0,0 +1,171 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <string>
+#include <vector>
+#include <memory>
+#include "atoms.hpp"
+#include "sparse_utils.hpp"
+#include "caf/actorSOLVE/actorSOLVE.hpp"
+
+template <class T = float>
+struct ft_cg_state {
+  // Host Data
+  std::string path;
+  in<int> h_row_ptr, h_col_ind;
+  in<T> h_values, h_b;
+  in_out<T> h_x;
+
+  // GPU Buffers
+  caf::cuda::mem_ptr<int> A_rp, A_ci, d_err;
+  caf::cuda::mem_ptr<T> A_val, b, x, r, p, w, y_tmp;
+  caf::cuda::mem_ptr<char> spmv_ws;
+
+  // Config
+  int n, nnz, max_iter;
+  int iterations = 0;
+  int strikes = 0;
+  T tol;
+  int device_id, stream_id;
+
+  // Supervision & Monitoring
+  caf::actor supervisor;
+  caf::cuda::device_ptr d_ptr;
+  caf::cuda::program_ptr stab_prog;
+
+  T initial_rho = 0;
+  T current_rho = 0;
+  T old_rho = 0;
+  bool initialized = false;
+  std::shared_ptr<MatrixData> pinned_data;
+};
+
+template <class T = float>
+caf::behavior fault_tolerant_cg_actor(caf::stateful_actor<ft_cg_state<T>>* self,
+                                 std::string path,
+                                 std::shared_ptr<MatrixData> data,
+                                 in<int> rp, in<int> ci, 
+                                 in<T> val, in<T> b_in, 
+                                 in_out<T> x_in,
+                                 int n, int nnz, T tol, int max_iter,
+                                 int dev_num, int stream, caf::actor supervisor) {
+  auto& s = self->state();
+  s.pinned_data = std::move(data);
+  s.path = std::move(path);
+  s.h_row_ptr = std::move(rp); s.h_col_ind = std::move(ci);
+  s.h_values = std::move(val); s.h_b = std::move(b_in); s.h_x = std::move(x_in);
+  s.n = n; s.nnz = nnz; s.tol = tol; s.max_iter = max_iter;
+  s.device_id = dev_num; s.stream_id = stream; s.supervisor = supervisor;
+
+  return {
+    [=](start_atom) { // This was already correct
+      auto& st = self->state();
+      if (st.initialized) return;
+
+      caf::cuda::command_runner<> runner;
+      st.A_rp = runner.transfer_memory(st.device_id, st.stream_id, st.h_row_ptr);
+      st.A_ci = runner.transfer_memory(st.device_id, st.stream_id, st.h_col_ind);
+      st.A_val = runner.transfer_memory(st.device_id, st.stream_id, st.h_values);
+      st.b = runner.transfer_memory(st.device_id, st.stream_id, st.h_b);
+      st.x = runner.transfer_memory(st.device_id, st.stream_id, st.h_x);
+
+      st.d_ptr = caf::cuda::platform::create()->schedule(st.stream_id, st.device_id);
+      st.d_ptr->enable_cublas(); st.d_ptr->enable_cusparse();
+
+      auto& mgr = caf::cuda::manager::get();
+      st.stab_prog = mgr.create_program_from_cubin("stability_kernels.cubin", "check_stability", st.d_ptr);
+
+      st.r = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.p = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.w = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.y_tmp = runner.transfer_memory(st.device_id, st.stream_id, out<T>(1));
+      st.d_err = runner.transfer_memory(st.device_id, st.stream_id, out<int>(1));
+      
+      size_t ws_sz = st.d_ptr->spmv_csr_buffer_size(st.stream_id, st.n, st.n, st.nnz, st.A_rp, st.A_ci, st.A_val, st.x, st.w);
+      if (ws_sz > 0) st.spmv_ws = caf::cuda::command_runner<out<char>>{}.transfer_memory(st.device_id, st.stream_id, out<char>{static_cast<int>(ws_sz)});
+
+      st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.x, T{0}, st.w, st.spmv_ws);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.b, st.r);
+      else st.d_ptr->scopy(st.stream_id, st.n, st.b, st.r);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->daxpy(st.stream_id, st.n, -1.0, st.w, st.r);
+      else st.d_ptr->saxpy(st.stream_id, st.n, -1.0f, st.w, st.r);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+      else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+
+      st.initial_rho = runner.copy_to_host(st.y_tmp)[0];
+      st.current_rho = st.initial_rho;
+      st.initialized = true;
+
+      caf::cuda::solver_result_meta meta(st.device_id, st.stream_id, 0, false, CG_SUCCESS);
+      self->mail(gpu_done_atom, st.path, caf::actor_cast<caf::actor>(self), std::vector<T>{}, meta).send(st.supervisor);
+    },
+
+    [=](cg_next_step_atom, int num_iters) { // Changed to cg_next_step_atom_v
+      auto& st = self->state();
+      caf::cuda::command_runner<T> runner;
+      T threshold = st.tol * st.tol;
+      int code = CG_SUCCESS;
+      int step_count = 0;
+
+      while (step_count < num_iters && st.iterations < st.max_iter && st.current_rho > threshold) {
+        st.iterations++;
+        step_count++;
+
+        if (st.iterations > 1) {
+          T beta = st.current_rho / st.old_rho;
+          if constexpr (std::is_same_v<T, double>) {
+            st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.w);
+            st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
+            st.d_ptr->dcopy(st.stream_id, st.n, st.w, st.p);
+          } else {
+            st.d_ptr->scopy(st.stream_id, st.n, st.r, st.w);
+            st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(beta), st.p, st.w);
+            st.d_ptr->scopy(st.stream_id, st.n, st.w, st.p);
+          }
+        } else {
+          if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.p);
+          else st.d_ptr->scopy(st.stream_id, st.n, st.r, st.p);
+        }
+
+        st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.p, T{0}, st.w, st.spmv_ws);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
+        else st.d_ptr->sdot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
+        
+        T dot_pw = runner.copy_to_host(st.y_tmp)[0];
+        if (std::abs(dot_pw) < 1e-25) { code = CG_BREAKDOWN; break; }
+
+        T alpha = st.current_rho / dot_pw;
+        if constexpr (std::is_same_v<T, double>) {
+          st.d_ptr->daxpy(st.stream_id, st.n, alpha, st.p, st.x);
+          st.d_ptr->daxpy(st.stream_id, st.n, -alpha, st.w, st.r);
+        } else {
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(alpha), st.p, st.x);
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(-alpha), st.w, st.r);
+        }
+        st.old_rho = st.current_rho;
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+        else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+        st.current_rho = runner.copy_to_host(st.y_tmp)[0];
+      }
+
+      bool converged = (st.current_rho <= threshold);
+      if (code == CG_SUCCESS && !converged && std::abs(st.old_rho - st.current_rho) < 1e-12) code = CG_STAGNATION;
+      if (code == CG_SUCCESS) {
+        if (!converged && st.iterations >= st.max_iter) code = CG_MAX_ITER;
+        if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
+        if (code == CG_SUCCESS) st.strikes = 0;
+      }
+      if (code != CG_SUCCESS && code != CG_NAN_INF && code != CG_BREAKDOWN) {
+        if (++st.strikes >= 3) { /* fatal */ } else { code = CG_SUCCESS; }
+      }
+      caf::cuda::solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
+      runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor, path = st.path, self_h = caf::actor_cast<caf::actor>(self)](std::vector<T> sol) {
+        caf::anon_mail(gpu_done_atom, path, self_h, std::move(sol), meta).send(supervisor);
+        if (converged || code != CG_SUCCESS) self->quit();
+      });
+    },
+    [=](update_stream_atom, int new_stream) { self->state().stream_id = new_stream; },
+    [=](shutdown_atom) { self->quit(); }
+  };
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
index 3a025a94ea..d3f0fd4cd4 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/main.test.cpp
@@ -11,481 +11,16 @@
 #include <algorithm>
 #include <filesystem>
 #include <memory>
-#include "caf/actorSOLVE/actorSOLVE.hpp"
 #include "sparse_utils.hpp"
+#include "atoms.hpp"
+#include "ft_cg_actor.hpp"
+#include "supervisor_actor.hpp"
 
 using namespace caf;
 using namespace caf::cuda;
-namespace fs = std::filesystem;
 
-constexpr uint32_t WORKLOAD_SEED = 42;
-constexpr int MAX_ITERATIONS = 16000;
-
-
-
-CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
-    CAF_ADD_ATOM(workload_test, get_work_atom)
-    CAF_ADD_ATOM(workload_test, release_memory_atom)
-    CAF_ADD_ATOM(workload_test, request_work_atom)
-    CAF_ADD_ATOM(workload_test, worker_done_atom)
-    CAF_ADD_ATOM(workload_test, work_tick_atom)
-    CAF_ADD_ATOM(workload_test, add_work_atom)
-    CAF_ADD_ATOM(workload_test, steal_work_atom)
-    CAF_ADD_ATOM(workload_test, update_stream_atom)
-    CAF_ADD_ATOM(workload_test, shutdown_atom)
-    CAF_ADD_TYPE_ID(workload_test, (SolverType))
-    CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
-    CAF_ADD_TYPE_ID(workload_test, (MatrixData))
-    CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
-    CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
-CAF_END_TYPE_ID_BLOCK(workload_test)
-
-
-
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixData)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(SolverType)
-
-
-/**
- * Error codes for solver_result_meta.error_code
- */
-enum cg_error_type : int {
-  CG_SUCCESS = 0,
-  CG_MAX_ITER = 1,
-  CG_NAN_INF = 2,
-  CG_STAGNATION = 3,
-  CG_BREAKDOWN = 4,
-  CG_RESIDUAL_FACTOR_FAIL = 5
-};
-
-std::string to_string(cg_error_type err) {
-  switch (err) {
-    case CG_SUCCESS: return "Success";
-    case CG_MAX_ITER: return "Maximum Iterations Reached";
-    case CG_NAN_INF: return "Stability Check Failed (NaN/Inf Detected)";
-    case CG_STAGNATION: return "Stagnation Detected (Residual stopped changing)";
-    case CG_BREAKDOWN: return "Solver Breakdown (Division by zero/near-zero)";
-    case CG_RESIDUAL_FACTOR_FAIL: return "Residual Factor Check Failed";
-    default: return "Unknown Error (" + std::to_string(static_cast<int>(err)) + ")";
-  }
-}
-
-// ---------------------------- FAULT TOLERANT SOLVER ----------------------------
-
-template <class T = float>
-struct ft_cg_state {
-  // Host Data
-  std::string path;
-  in<int> h_row_ptr, h_col_ind;
-  in<T> h_values, h_b;
-  in_out<T> h_x;
-
-  // GPU Buffers
-  mem_ptr<int> A_rp, A_ci, d_err;
-  mem_ptr<T> A_val, b, x, r, p, w, y_tmp;
-  mem_ptr<char> spmv_ws;
-
-  // Config
-  int n, nnz, max_iter;
-  int iterations = 0;
-  int strikes = 0;
-  T tol;
-  int device_id, stream_id;
-
-  // Supervision & Monitoring
-  caf::actor supervisor;
-  device_ptr d_ptr;
-  program_ptr stab_prog;
-
-  T initial_rho = 0;
-  T current_rho = 0;
-  T old_rho = 0;
-  bool initialized = false;
-  std::shared_ptr<MatrixData> pinned_data;
-};
-
-template <class T = float>
-behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
-                                 std::string path,
-                                 std::shared_ptr<MatrixData> data,
-                                 in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
-                                 int n, int nnz, T tol, int max_iter,
-                                 int dev_num, int stream, caf::actor supervisor) {
-  auto& s = self->state();
-  s.pinned_data = std::move(data);
-  s.path = std::move(path);
-  s.h_row_ptr = std::move(rp); s.h_col_ind = std::move(ci);
-  s.h_values = std::move(val); s.h_b = std::move(b_in); s.h_x = std::move(x_in);
-  s.n = n; s.nnz = nnz; s.tol = tol; s.max_iter = max_iter;
-  s.device_id = dev_num; s.stream_id = stream; s.supervisor = supervisor;
-
-  return {
-    [=](start_atom) {
-      auto& st = self->state();
-      if (st.initialized) return;
-
-      command_runner<> runner;
-      st.A_rp = runner.transfer_memory(st.device_id, st.stream_id, st.h_row_ptr);
-      st.A_ci = runner.transfer_memory(st.device_id, st.stream_id, st.h_col_ind);
-      st.A_val = runner.transfer_memory(st.device_id, st.stream_id, st.h_values);
-      st.b = runner.transfer_memory(st.device_id, st.stream_id, st.h_b);
-      st.x = runner.transfer_memory(st.device_id, st.stream_id, st.h_x);
-
-      st.d_ptr = platform::create()->schedule(st.stream_id, st.device_id);
-      st.d_ptr->enable_cublas(); st.d_ptr->enable_cusparse();
-
-      auto& mgr = manager::get();
-      // Load stability kernel from file as requested
-      st.stab_prog = mgr.create_program_from_cubin("stability_kernels.cubin", "check_stability", st.d_ptr);
-
-      st.r = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
-      st.p = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
-      st.w = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
-      st.y_tmp = runner.transfer_memory(st.device_id, st.stream_id, out<T>(1));
-      st.d_err = runner.transfer_memory(st.device_id, st.stream_id, out<int>(1));
-      
-      size_t ws_sz = st.d_ptr->spmv_csr_buffer_size(st.stream_id, st.n, st.n, st.nnz, st.A_rp, st.A_ci, st.A_val, st.x, st.w);
-      if (ws_sz > 0) st.spmv_ws = command_runner<out<char>>{}.transfer_memory(st.device_id, st.stream_id, out<char>{static_cast<int>(ws_sz)});
-
-      st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.x, T{0}, st.w, st.spmv_ws);
-      if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.b, st.r);
-      else st.d_ptr->scopy(st.stream_id, st.n, st.b, st.r);
-      if constexpr (std::is_same_v<T, double>) st.d_ptr->daxpy(st.stream_id, st.n, -1.0, st.w, st.r);
-      else st.d_ptr->saxpy(st.stream_id, st.n, -1.0f, st.w, st.r);
-      if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
-      else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
-
-      st.initial_rho = runner.copy_to_host(st.y_tmp)[0];
-      st.current_rho = st.initial_rho;
-      st.initialized = true;
-
-      // Notify supervisor that setup is complete and report initial status
-      solver_result_meta meta(st.device_id, st.stream_id, 0, false, CG_SUCCESS);
-      self->mail(gpu_done_atom_v, st.path, actor_cast<actor>(self), std::vector<T>{}, meta).send(st.supervisor);
-    },
-
-    [=](cg_next_step_atom, int num_iters) {
-      auto& st = self->state();
-      command_runner<T> runner;
-      T threshold = st.tol * st.tol;
-      int code = CG_SUCCESS;
-      int step_count = 0;
-
-      // Execute exactly the number of iterations requested by the supervisor
-      while (step_count < num_iters && st.iterations < st.max_iter && st.current_rho > threshold) {
-        // std::cout << "iterations = " << st.iterations << ", current_rho = " << st.current_rho << std::endl;
-        st.iterations++;
-        step_count++;
-
-        if (st.iterations > 1) {
-          T beta = st.current_rho / st.old_rho;
-          if constexpr (std::is_same_v<T, double>) {
-            st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.w);
-            st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
-            st.d_ptr->dcopy(st.stream_id, st.n, st.w, st.p);
-          } else {
-            st.d_ptr->scopy(st.stream_id, st.n, st.r, st.w);
-            st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(beta), st.p, st.w);
-            st.d_ptr->scopy(st.stream_id, st.n, st.w, st.p);
-          }
-        } else {
-          if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.p);
-          else st.d_ptr->scopy(st.stream_id, st.n, st.r, st.p);
-        }
-
-        st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.p, T{0}, st.w, st.spmv_ws);
-        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
-        else st.d_ptr->sdot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
-        
-        T dot_pw = runner.copy_to_host(st.y_tmp)[0];
-        if (std::abs(dot_pw) < 1e-25) { 
-          code = CG_BREAKDOWN; 
-          break; 
-        }
-
-        T alpha = st.current_rho / dot_pw;
-        if constexpr (std::is_same_v<T, double>) {
-          st.d_ptr->daxpy(st.stream_id, st.n, alpha, st.p, st.x);
-          st.d_ptr->daxpy(st.stream_id, st.n, -alpha, st.w, st.r);
-        } else {
-          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(alpha), st.p, st.x);
-          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(-alpha), st.w, st.r);
-        }
-
-        st.old_rho = st.current_rho;
-        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
-        else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
-        st.current_rho = runner.copy_to_host(st.y_tmp)[0];
-
-      }
-
-      // Run error checks and prepare progress report
-      bool converged = (st.current_rho <= threshold);
-
-      // Check for non-fatal errors that can be retried (Stagnation, Max Iter, Residual Factor)
-      if (code == CG_SUCCESS && !converged && std::abs(st.old_rho - st.current_rho) < 1e-12)
-        code = CG_STAGNATION;
-
-      if (code == CG_SUCCESS) {
-        if (!converged && st.iterations >= st.max_iter) code = CG_MAX_ITER;
-        // Residual decrease check: treat as non-fatal strike if residual didn't decrease significantly
-        if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
-        
-        // If we reached here with CG_SUCCESS, reset strikes as this was a productive batch
-        if (code == CG_SUCCESS) st.strikes = 0;
-      }
-
-      // Reset and launch NaN/Inf stability kernel from file
-      nd_range range(static_cast<int>((st.n + 255) / 256), 1, 1, 256, 1, 1);
-      CHECK_CUDA(cuMemsetD32Async(st.d_err->mem(), 0, 1, st.d_ptr->get_stream_for_actor(st.stream_id)));
-      st.d_ptr->launch_kernel_mem_ref(st.stab_prog->get_kernel(st.d_ptr->getId()), range,
-                                      std::make_tuple(in<int>(st.n), st.x, st.r, st.d_err), st.stream_id);
-
-      int err_flag = runner.copy_to_host(st.d_err)[0];
-      if (err_flag != 0 || std::isnan(st.current_rho) || std::isinf(st.current_rho)) code = CG_NAN_INF;
-
-      // Three strikes policy for non-fatal errors (Stagnation, Max Iterations, Residual Factor Failure)
-      bool is_fatal = (code == CG_NAN_INF || code == CG_BREAKDOWN);
-      if (code != CG_SUCCESS && !is_fatal) {
-        st.strikes++;
-        if (st.strikes < 3) {
-          code = CG_SUCCESS; // Reset code to SUCCESS to allow the supervisor to retry/suspend
-        }
-      }
-
-      if (code != CG_SUCCESS) converged = false;
-
-      solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
-      
-      // Report current solution and metadata to the supervisor
-      runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor, path = st.path, self_h = actor_cast<actor>(self)](std::vector<T> sol) {
-        anon_mail(gpu_done_atom_v, path, self_h, std::move(sol), meta).send(supervisor);
-        if (converged || code != CG_SUCCESS)
-          self->quit();
-      });
-    },
-    [=](update_stream_atom, int new_stream) {
-      self->state().stream_id = new_stream;
-    },
-    [=](shutdown_atom) {
-      self->quit();
-    }
-  };
-}
-
-// ---------------------------- SUPERVISOR ACTOR ----------------------------
-
-struct suspended_task {
-  caf::actor solver;
-  std::string path;
-  int last_batch_size;
-  int device_id;
-  int stream_id;
-};
-
-struct resource_slot {
-  int device_id;
-  int stream_id;
-};
-
-struct supervisor_state {
-  std::deque<MatrixTask> queue;
-  std::deque<suspended_task> suspended_queue;
-  std::vector<caf::actor> active_solvers;
-  std::unordered_map<std::string, std::chrono::steady_clock::time_point> start_times;
-  std::unordered_map<std::string, resource_slot> task_resources;
-  std::unordered_map<std::string, std::chrono::steady_clock::time_point> enqueue_times;
-  std::unordered_map<std::string, std::chrono::steady_clock::time_point> pick_times;
-  std::unordered_map<std::string, double> cumulative_active_ms;
-  std::unordered_map<caf::actor, int> actor_batch_sizes;
-  std::deque<resource_slot> available_slots;
-  int max_active = 1; // Admission control limit to be mindful of GPU memory.
-  int num_iterations = MAX_ITERATIONS / 2;
-  int num_gpus = 0;
-  int tasks_succeeded = 0;
-  int tasks_failed = 0;
-  std::chrono::steady_clock::time_point benchmark_start;
-};
-
-behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks, int initial_max_active, std::chrono::steady_clock::time_point start_time) {
-    auto& st = self->state();
-    st.queue.insert(st.queue.end(), std::make_move_iterator(tasks.begin()), std::make_move_iterator(tasks.end()));
-    st.max_active = initial_max_active;
-    st.benchmark_start = start_time;
-    st.num_gpus = manager::get().get_num_devices();
-
-    // Initialize the pool with streams interleaved across all available GPUs
-    for (int s = 0; s < 32; ++s) {
-      for (int g = 0; g < st.num_gpus; ++g) {
-        st.available_slots.push_back({g, s});
-      }
-    }
-
-    auto spawn_next = [self]() {
-        auto& s = self->state();
-        while (s.active_solvers.size() < static_cast<size_t>(s.max_active) && !s.available_slots.empty()) {
-          if (!s.queue.empty()) {
-            auto task = std::move(s.queue.front());
-            s.queue.pop_front();
-            std::string path = task.path;
-
-            s.cumulative_active_ms[path] = 0;
-            s.enqueue_times[path] = task.enqueue_time;
-            s.pick_times[path] = std::chrono::steady_clock::now();
-
-            resource_slot slot = s.available_slots.front();
-            s.available_slots.pop_front();
-
-            self->println("[INFO] Starting solver for: {} (Device: {}, Stream: {})", 
-                          path, slot.device_id, slot.stream_id);
-            auto solver = self->spawn(fault_tolerant_cg_actor<float>,
-                                    path,
-                                    task.data,
-                                    create_in_arg(task.data->row_ptr),
-                                    create_in_arg(task.data->col_indices),
-                                    create_in_arg(task.data->values),
-                                    create_in_arg(task.data->b),
-                                    create_in_out_arg(task.data->x_guess),
-                                    (int)task.data->row_ptr.size() - 1,
-                                    (int)task.data->values.size(),
-                                    1e-5f, MAX_ITERATIONS, slot.device_id, slot.stream_id, actor_cast<actor>(self));
-
-            s.start_times[path] = std::chrono::steady_clock::now();
-            s.active_solvers.push_back(solver);
-            s.task_resources[path] = slot;
-            s.actor_batch_sizes[solver] = s.num_iterations;
-            self->mail(start_atom_v).send(solver);
-          } else {
-            bool resumed = false;
-            for (auto it = s.suspended_queue.begin(); it != s.suspended_queue.end(); ++it) {
-              auto slot_it = std::find_if(s.available_slots.begin(), s.available_slots.end(), [&](const resource_slot& slot) {
-                return slot.device_id == it->device_id && slot.stream_id == it->stream_id;
-              });
-              if (slot_it != s.available_slots.end()) {
-                auto suspended = std::move(*it);
-                s.suspended_queue.erase(it);
-                resource_slot slot = *slot_it;
-                s.available_slots.erase(slot_it);
-
-                s.pick_times[suspended.path] = std::chrono::steady_clock::now();
-
-                // Cap the minimum batch size to MAX_ITERATIONS / 8
-                int next_batch = std::max(MAX_ITERATIONS / 8, suspended.last_batch_size / 2);
-                self->println("[INFO] Resuming solver for: {} (Device: {}, Stream: {}, Batch: {})", 
-                              suspended.path, slot.device_id, slot.stream_id, next_batch);
-
-                // Resume on the same stream ID. No update_stream_atom_v needed.
-                self->mail(cg_next_step_atom_v, next_batch).send(suspended.solver);
-
-                s.active_solvers.push_back(suspended.solver);
-                s.task_resources[suspended.path] = slot;
-                s.actor_batch_sizes[suspended.solver] = next_batch;
-                resumed = true;
-                break;
-              }
-            }
-            if (!resumed) break;
-          }
-        }
-    };
-
-    spawn_next();
-    
-    return {
-        [=](gpu_done_atom, const std::string& task_name, caf::actor solver, std::vector<float>& solution, solver_result_meta meta) {
-            auto& s = self->state();
-
-            if (meta.converged || meta.error_code != CG_SUCCESS) {
-                auto end_time = std::chrono::steady_clock::now();
-                auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
-                                  end_time - s.start_times[task_name]).count();
-
-                if (meta.converged && meta.iterations < MAX_ITERATIONS) {
-                    s.tasks_succeeded++;
-                    if (meta.iterations == 0)
-                        self->println("[DONE] {}: Initial guess satisfied tolerance ({} ms).", task_name, duration);
-                    else
-                        self->println("[DONE] {}: Converged in {} iterations ({} ms).", task_name, meta.iterations, duration);
-                } else {
-                    s.tasks_failed++;
-                    std::string reason = (meta.error_code == CG_SUCCESS)
-                                         ? "Maximum Iterations Reached"
-                                         : to_string(static_cast<cg_error_type>(meta.error_code));
-                    self->println("[FAIL] {}: {} (after {} iterations, {} ms).", 
-                                  task_name, reason,
-                                  meta.iterations,
-                                  duration);
-                }
-
-                // Final active slice
-                auto active_slice = std::chrono::duration<double, std::milli>(end_time - s.pick_times[task_name]).count();
-                s.cumulative_active_ms[task_name] += active_slice;
-
-                // We override pick_time in record_job to simulate a single continuous run that equals 
-                // the actual time spent on the GPU.
-                auto simulated_pick = end_time - std::chrono::duration_cast<std::chrono::steady_clock::duration>(
-                                                std::chrono::duration<double, std::milli>(s.cumulative_active_ms[task_name]));
-
-                record_job(task_name, s.enqueue_times[task_name], simulated_pick, end_time, meta.iterations, meta.converged);
-                s.enqueue_times.erase(task_name);
-                s.pick_times.erase(task_name);
-                s.cumulative_active_ms.erase(task_name);
-
-                auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
-                if (it != s.active_solvers.end())
-                    s.active_solvers.erase(it);
-                s.start_times.erase(task_name);
-                s.actor_batch_sizes.erase(solver);
-
-                // Reclaim the device/stream slot and put it back in the pool
-                auto res_it = s.task_resources.find(task_name);
-                if (res_it != s.task_resources.end()) {
-                    s.available_slots.push_back(res_it->second);
-                    s.task_resources.erase(res_it);
-                }
-
-                spawn_next();
-
-                if (s.active_solvers.empty() && s.queue.empty() && s.suspended_queue.empty()) {
-                    self->println("All tasks in the pool have been processed.");
-
-                    report_workload_stats();
-                    self->quit();
-                }
-            } else if (meta.iterations == 0) {
-                // Just finished initialization: trigger the first iteration batch immediately.
-                self->mail(cg_next_step_atom_v, s.num_iterations).send(solver);
-            } else {
-                // Not done: Suspend the actor to allow others to use the stream
-                auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
-                
-                // Record the time spent in this active slice before suspending
-                auto active_slice = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - s.pick_times[task_name]).count();
-                s.cumulative_active_ms[task_name] += active_slice;
-
-                if (it != s.active_solvers.end())
-                    s.active_solvers.erase(it);
-
-                resource_slot slot = s.task_resources[task_name];
-                s.available_slots.push_back(slot);
-                s.task_resources.erase(task_name);
-
-                int last_batch = s.actor_batch_sizes[solver];
-                s.suspended_queue.push_back({solver, task_name, last_batch, slot.device_id, slot.stream_id});
-
-                self->println("[INFO] Suspending solver for: {} (Reclaimed Device: {}, Stream: {})", 
-                              task_name, slot.device_id, slot.stream_id);
-
-                spawn_next();
-            }
-        }
-       
-    };
-}
 
+constexpr uint32_t WORKLOAD_SEED = 42;
 void caf_main(actor_system& sys) {
     manager::init(sys, manager_config(true, true));
     std::cout << "[INFO] Loading matrices...\n";
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/supervisor_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/supervisor_actor.hpp
new file mode 100644
index 0000000000..d7595d929c
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/supervisor_actor.hpp
@@ -0,0 +1,157 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <deque>
+#include <unordered_map>
+#include <vector>
+#include <chrono>
+#include "atoms.hpp"
+#include "ft_cg_actor.hpp"
+#include "sparse_utils.hpp"
+
+struct suspended_task {
+  caf::actor solver;
+  std::string path;
+  int last_batch_size;
+  int device_id;
+  int stream_id;
+};
+
+struct resource_slot {
+  int device_id;
+  int stream_id;
+};
+
+struct supervisor_state {
+  std::deque<MatrixTask> queue;
+  std::deque<suspended_task> suspended_queue;
+  std::vector<caf::actor> active_solvers;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> start_times;
+  std::unordered_map<std::string, resource_slot> task_resources;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> enqueue_times;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> pick_times;
+  std::unordered_map<std::string, double> cumulative_active_ms;
+  std::unordered_map<caf::actor, int> actor_batch_sizes;
+  std::deque<resource_slot> available_slots;
+  int max_active = 1;
+  int num_iterations = MAX_ITERATIONS / 2;
+  int num_gpus = 0;
+  int tasks_succeeded = 0;
+  int tasks_failed = 0;
+  std::chrono::steady_clock::time_point benchmark_start;
+};
+
+inline caf::behavior supervisor_actor(caf::stateful_actor<supervisor_state>* self, 
+                                      std::vector<MatrixTask> tasks, 
+                                      int initial_max_active, 
+                                      std::chrono::steady_clock::time_point start_time) {
+    auto& st = self->state();
+    st.queue.insert(st.queue.end(), std::make_move_iterator(tasks.begin()), std::make_move_iterator(tasks.end()));
+    st.max_active = initial_max_active;
+    st.benchmark_start = start_time;
+    st.num_gpus = caf::cuda::manager::get().get_num_devices();
+
+    for (int s = 0; s < 32; ++s) {
+      for (int g = 0; g < st.num_gpus; ++g) {
+        st.available_slots.push_back({g, s});
+      }
+    }
+
+    auto spawn_next = [self]() {
+        auto& s = self->state();
+        while (s.active_solvers.size() < static_cast<size_t>(s.max_active) && !s.available_slots.empty()) {
+          if (!s.queue.empty()) {
+            auto task = std::move(s.queue.front());
+            s.queue.pop_front();
+            std::string path = task.path;
+            s.cumulative_active_ms[path] = 0;
+            s.enqueue_times[path] = task.enqueue_time;
+            s.pick_times[path] = std::chrono::steady_clock::now();
+            resource_slot slot = s.available_slots.front();
+            s.available_slots.pop_front();
+
+            auto solver = self->spawn(fault_tolerant_cg_actor<float>,
+                                    path, task.data,
+                                    caf::cuda::create_in_arg(task.data->row_ptr),
+                                    caf::cuda::create_in_arg(task.data->col_indices),
+                                    caf::cuda::create_in_arg(task.data->values),
+                                    caf::cuda::create_in_arg(task.data->b),
+                                    caf::cuda::create_in_out_arg(task.data->x_guess),
+                                    (int)task.data->row_ptr.size() - 1,
+                                    (int)task.data->values.size(),
+                                    1e-5f, MAX_ITERATIONS, slot.device_id, slot.stream_id, caf::actor_cast<caf::actor>(self));
+
+            s.start_times[path] = std::chrono::steady_clock::now();
+            s.active_solvers.push_back(solver);
+            s.task_resources[path] = slot;
+            s.actor_batch_sizes[solver] = s.num_iterations;
+            self->mail(start_atom_v).send(solver);
+          } else {
+            bool resumed = false;
+            for (auto it = s.suspended_queue.begin(); it != s.suspended_queue.end(); ++it) {
+              auto slot_it = std::find_if(s.available_slots.begin(), s.available_slots.end(), [&](const resource_slot& slot) {
+                return slot.device_id == it->device_id && slot.stream_id == it->stream_id;
+              });
+              if (slot_it != s.available_slots.end()) {
+                auto suspended = std::move(*it);
+                s.suspended_queue.erase(it);
+                resource_slot slot = *slot_it;
+                s.available_slots.erase(slot_it);
+                s.pick_times[suspended.path] = std::chrono::steady_clock::now();
+                int next_batch = std::max(MAX_ITERATIONS / 8, suspended.last_batch_size / 2);
+                self->mail(cg_next_step_atom, next_batch).send(suspended.solver);
+                s.active_solvers.push_back(suspended.solver);
+                s.task_resources[suspended.path] = slot;
+                s.actor_batch_sizes[suspended.solver] = next_batch;
+                resumed = true; break;
+              }
+            }
+            if (!resumed) break;
+          }
+        }
+    };
+
+    spawn_next();
+    
+    return {
+        [=](gpu_done_atom, const std::string& task_name, caf::actor solver, std::vector<float>& solution, caf::cuda::solver_result_meta meta) {
+            auto& s = self->state();
+            if (meta.converged || meta.error_code != CG_SUCCESS) {
+                auto end_time = std::chrono::steady_clock::now();
+                if (meta.converged && meta.iterations < MAX_ITERATIONS) s.tasks_succeeded++;
+                else s.tasks_failed++;
+
+                auto active_slice = std::chrono::duration<double, std::milli>(end_time - s.pick_times[task_name]).count();
+                s.cumulative_active_ms[task_name] += active_slice;
+                auto simulated_pick = end_time - std::chrono::duration_cast<std::chrono::steady_clock::duration>(
+                                                std::chrono::duration<double, std::milli>(s.cumulative_active_ms[task_name]));
+
+                record_job(task_name, s.enqueue_times[task_name], simulated_pick, end_time, meta.iterations, meta.converged);
+                auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
+                if (it != s.active_solvers.end()) s.active_solvers.erase(it);
+                auto res_it = s.task_resources.find(task_name);
+                if (res_it != s.task_resources.end()) {
+                    s.available_slots.push_back(res_it->second);
+                    s.task_resources.erase(res_it);
+                }
+                spawn_next();
+                if (s.active_solvers.empty() && s.queue.empty() && s.suspended_queue.empty()) {
+                    report_workload_stats();
+                    self->quit();
+                }
+            } else if (meta.iterations == 0) {
+                self->mail(cg_next_step_atom, s.num_iterations).send(solver);
+            } else {
+                auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
+                auto active_slice = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - s.pick_times[task_name]).count();
+                s.cumulative_active_ms[task_name] += active_slice;
+                if (it != s.active_solvers.end()) s.active_solvers.erase(it);
+                resource_slot slot = s.task_resources[task_name];
+                s.available_slots.push_back(slot); s.task_resources.erase(task_name);
+                s.suspended_queue.push_back({solver, task_name, s.actor_batch_sizes[solver], slot.device_id, slot.stream_id});
+                spawn_next();
+            }
+        }
+    };
+}
\ No newline at end of file

From da501e587153125efd3be997780b916f25bbdf60 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 09:59:15 -0600
Subject: [PATCH 0901/1000] fixed compiler errors

---
 .../tests/fault-tolerance-workload-test/ft_cg_actor.hpp     | 6 +++---
 .../fault-tolerance-workload-test/supervisor_actor.hpp      | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp
index 86d7a4fe97..0c9ab2a6ed 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp
@@ -98,10 +98,10 @@ caf::behavior fault_tolerant_cg_actor(caf::stateful_actor<ft_cg_state<T>>* self,
       st.initialized = true;
 
       caf::cuda::solver_result_meta meta(st.device_id, st.stream_id, 0, false, CG_SUCCESS);
-      self->mail(gpu_done_atom, st.path, caf::actor_cast<caf::actor>(self), std::vector<T>{}, meta).send(st.supervisor);
+      self->mail(gpu_done_atom_v, st.path, caf::actor_cast<caf::actor>(self), std::vector<T>{}, meta).send(st.supervisor);
     },
 
-    [=](cg_next_step_atom, int num_iters) { // Changed to cg_next_step_atom_v
+    [=](cg_next_step_atom, int num_iters) {
       auto& st = self->state();
       caf::cuda::command_runner<T> runner;
       T threshold = st.tol * st.tol;
@@ -161,7 +161,7 @@ caf::behavior fault_tolerant_cg_actor(caf::stateful_actor<ft_cg_state<T>>* self,
       }
       caf::cuda::solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
       runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor, path = st.path, self_h = caf::actor_cast<caf::actor>(self)](std::vector<T> sol) {
-        caf::anon_mail(gpu_done_atom, path, self_h, std::move(sol), meta).send(supervisor);
+        caf::anon_mail(gpu_done_atom_v, path, self_h, std::move(sol), meta).send(supervisor);
         if (converged || code != CG_SUCCESS) self->quit();
       });
     },
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/supervisor_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/supervisor_actor.hpp
index d7595d929c..059cfee98f 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/supervisor_actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/supervisor_actor.hpp
@@ -100,7 +100,7 @@ inline caf::behavior supervisor_actor(caf::stateful_actor<supervisor_state>* sel
                 s.available_slots.erase(slot_it);
                 s.pick_times[suspended.path] = std::chrono::steady_clock::now();
                 int next_batch = std::max(MAX_ITERATIONS / 8, suspended.last_batch_size / 2);
-                self->mail(cg_next_step_atom, next_batch).send(suspended.solver);
+                self->mail(cg_next_step_atom_v, next_batch).send(suspended.solver);
                 s.active_solvers.push_back(suspended.solver);
                 s.task_resources[suspended.path] = slot;
                 s.actor_batch_sizes[suspended.solver] = next_batch;
@@ -141,7 +141,7 @@ inline caf::behavior supervisor_actor(caf::stateful_actor<supervisor_state>* sel
                     self->quit();
                 }
             } else if (meta.iterations == 0) {
-                self->mail(cg_next_step_atom, s.num_iterations).send(solver);
+                self->mail(cg_next_step_atom_v, s.num_iterations).send(solver);
             } else {
                 auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
                 auto active_slice = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - s.pick_times[task_name]).count();

From c37be3150fd7744dd709916b0b45c3a834d64ff1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 10:19:01 -0600
Subject: [PATCH 0902/1000] fixed errors

---
 .../ft_cg_actor.hpp                           |  99 +++++++++++-----
 .../supervisor_actor.hpp                      | 111 ++++++++++++++----
 2 files changed, 161 insertions(+), 49 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp
index 0c9ab2a6ed..82ee13b769 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp
@@ -9,6 +9,11 @@
 #include "sparse_utils.hpp"
 #include "caf/actorSOLVE/actorSOLVE.hpp"
 
+using namespace caf;
+using namespace caf::cuda;
+
+// ---------------------------- FAULT TOLERANT SOLVER ----------------------------
+
 template <class T = float>
 struct ft_cg_state {
   // Host Data
@@ -18,9 +23,9 @@ struct ft_cg_state {
   in_out<T> h_x;
 
   // GPU Buffers
-  caf::cuda::mem_ptr<int> A_rp, A_ci, d_err;
-  caf::cuda::mem_ptr<T> A_val, b, x, r, p, w, y_tmp;
-  caf::cuda::mem_ptr<char> spmv_ws;
+  mem_ptr<int> A_rp, A_ci, d_err;
+  mem_ptr<T> A_val, b, x, r, p, w, y_tmp;
+  mem_ptr<char> spmv_ws;
 
   // Config
   int n, nnz, max_iter;
@@ -31,8 +36,8 @@ struct ft_cg_state {
 
   // Supervision & Monitoring
   caf::actor supervisor;
-  caf::cuda::device_ptr d_ptr;
-  caf::cuda::program_ptr stab_prog;
+  device_ptr d_ptr;
+  program_ptr stab_prog;
 
   T initial_rho = 0;
   T current_rho = 0;
@@ -42,12 +47,10 @@ struct ft_cg_state {
 };
 
 template <class T = float>
-caf::behavior fault_tolerant_cg_actor(caf::stateful_actor<ft_cg_state<T>>* self,
+behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
                                  std::string path,
                                  std::shared_ptr<MatrixData> data,
-                                 in<int> rp, in<int> ci, 
-                                 in<T> val, in<T> b_in, 
-                                 in_out<T> x_in,
+                                 in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
                                  int n, int nnz, T tol, int max_iter,
                                  int dev_num, int stream, caf::actor supervisor) {
   auto& s = self->state();
@@ -59,21 +62,22 @@ caf::behavior fault_tolerant_cg_actor(caf::stateful_actor<ft_cg_state<T>>* self,
   s.device_id = dev_num; s.stream_id = stream; s.supervisor = supervisor;
 
   return {
-    [=](start_atom) { // This was already correct
+    [=](start_atom) {
       auto& st = self->state();
       if (st.initialized) return;
 
-      caf::cuda::command_runner<> runner;
+      command_runner<> runner;
       st.A_rp = runner.transfer_memory(st.device_id, st.stream_id, st.h_row_ptr);
       st.A_ci = runner.transfer_memory(st.device_id, st.stream_id, st.h_col_ind);
       st.A_val = runner.transfer_memory(st.device_id, st.stream_id, st.h_values);
       st.b = runner.transfer_memory(st.device_id, st.stream_id, st.h_b);
       st.x = runner.transfer_memory(st.device_id, st.stream_id, st.h_x);
 
-      st.d_ptr = caf::cuda::platform::create()->schedule(st.stream_id, st.device_id);
+      st.d_ptr = platform::create()->schedule(st.stream_id, st.device_id);
       st.d_ptr->enable_cublas(); st.d_ptr->enable_cusparse();
 
-      auto& mgr = caf::cuda::manager::get();
+      auto& mgr = manager::get();
+      // Load stability kernel from file as requested
       st.stab_prog = mgr.create_program_from_cubin("stability_kernels.cubin", "check_stability", st.d_ptr);
 
       st.r = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
@@ -83,7 +87,7 @@ caf::behavior fault_tolerant_cg_actor(caf::stateful_actor<ft_cg_state<T>>* self,
       st.d_err = runner.transfer_memory(st.device_id, st.stream_id, out<int>(1));
       
       size_t ws_sz = st.d_ptr->spmv_csr_buffer_size(st.stream_id, st.n, st.n, st.nnz, st.A_rp, st.A_ci, st.A_val, st.x, st.w);
-      if (ws_sz > 0) st.spmv_ws = caf::cuda::command_runner<out<char>>{}.transfer_memory(st.device_id, st.stream_id, out<char>{static_cast<int>(ws_sz)});
+      if (ws_sz > 0) st.spmv_ws = command_runner<out<char>>{}.transfer_memory(st.device_id, st.stream_id, out<char>{static_cast<int>(ws_sz)});
 
       st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.x, T{0}, st.w, st.spmv_ws);
       if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.b, st.r);
@@ -97,18 +101,21 @@ caf::behavior fault_tolerant_cg_actor(caf::stateful_actor<ft_cg_state<T>>* self,
       st.current_rho = st.initial_rho;
       st.initialized = true;
 
-      caf::cuda::solver_result_meta meta(st.device_id, st.stream_id, 0, false, CG_SUCCESS);
-      self->mail(gpu_done_atom_v, st.path, caf::actor_cast<caf::actor>(self), std::vector<T>{}, meta).send(st.supervisor);
+      // Notify supervisor that setup is complete and report initial status
+      solver_result_meta meta(st.device_id, st.stream_id, 0, false, CG_SUCCESS);
+      self->mail(gpu_done_atom_v, st.path, actor_cast<actor>(self), std::vector<T>{}, meta).send(st.supervisor);
     },
 
     [=](cg_next_step_atom, int num_iters) {
       auto& st = self->state();
-      caf::cuda::command_runner<T> runner;
+      command_runner<T> runner;
       T threshold = st.tol * st.tol;
       int code = CG_SUCCESS;
       int step_count = 0;
 
+      // Execute exactly the number of iterations requested by the supervisor
       while (step_count < num_iters && st.iterations < st.max_iter && st.current_rho > threshold) {
+        // std::cout << "iterations = " << st.iterations << ", current_rho = " << st.current_rho << std::endl;
         st.iterations++;
         step_count++;
 
@@ -133,7 +140,10 @@ caf::behavior fault_tolerant_cg_actor(caf::stateful_actor<ft_cg_state<T>>* self,
         else st.d_ptr->sdot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
         
         T dot_pw = runner.copy_to_host(st.y_tmp)[0];
-        if (std::abs(dot_pw) < 1e-25) { code = CG_BREAKDOWN; break; }
+        if (std::abs(dot_pw) < 1e-25) { 
+          code = CG_BREAKDOWN; 
+          break; 
+        }
 
         T alpha = st.current_rho / dot_pw;
         if constexpr (std::is_same_v<T, double>) {
@@ -143,29 +153,64 @@ caf::behavior fault_tolerant_cg_actor(caf::stateful_actor<ft_cg_state<T>>* self,
           st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(alpha), st.p, st.x);
           st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(-alpha), st.w, st.r);
         }
+
         st.old_rho = st.current_rho;
         if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
         else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
         st.current_rho = runner.copy_to_host(st.y_tmp)[0];
+
       }
 
+      // Run error checks and prepare progress report
       bool converged = (st.current_rho <= threshold);
-      if (code == CG_SUCCESS && !converged && std::abs(st.old_rho - st.current_rho) < 1e-12) code = CG_STAGNATION;
+
+      // Check for non-fatal errors that can be retried (Stagnation, Max Iter, Residual Factor)
+      if (code == CG_SUCCESS && !converged && std::abs(st.old_rho - st.current_rho) < 1e-12)
+        code = CG_STAGNATION;
+
       if (code == CG_SUCCESS) {
         if (!converged && st.iterations >= st.max_iter) code = CG_MAX_ITER;
+        // Residual decrease check: treat as non-fatal strike if residual didn't decrease significantly
         if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
+        
+        // If we reached here with CG_SUCCESS, reset strikes as this was a productive batch
         if (code == CG_SUCCESS) st.strikes = 0;
       }
-      if (code != CG_SUCCESS && code != CG_NAN_INF && code != CG_BREAKDOWN) {
-        if (++st.strikes >= 3) { /* fatal */ } else { code = CG_SUCCESS; }
+
+      // Reset and launch NaN/Inf stability kernel from file
+      nd_range range(static_cast<int>((st.n + 255) / 256), 1, 1, 256, 1, 1);
+      CHECK_CUDA(cuMemsetD32Async(st.d_err->mem(), 0, 1, st.d_ptr->get_stream_for_actor(st.stream_id)));
+      st.d_ptr->launch_kernel_mem_ref(st.stab_prog->get_kernel(st.d_ptr->getId()), range,
+                                      std::make_tuple(in<int>(st.n), st.x, st.r, st.d_err), st.stream_id);
+
+      int err_flag = runner.copy_to_host(st.d_err)[0];
+      if (err_flag != 0 || std::isnan(st.current_rho) || std::isinf(st.current_rho)) code = CG_NAN_INF;
+
+      // Three strikes policy for non-fatal errors (Stagnation, Max Iterations, Residual Factor Failure)
+      bool is_fatal = (code == CG_NAN_INF || code == CG_BREAKDOWN);
+      if (code != CG_SUCCESS && !is_fatal) {
+        st.strikes++;
+        if (st.strikes < 3) {
+          code = CG_SUCCESS; // Reset code to SUCCESS to allow the supervisor to retry/suspend
+        }
       }
-      caf::cuda::solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
-      runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor, path = st.path, self_h = caf::actor_cast<caf::actor>(self)](std::vector<T> sol) {
-        caf::anon_mail(gpu_done_atom_v, path, self_h, std::move(sol), meta).send(supervisor);
-        if (converged || code != CG_SUCCESS) self->quit();
+
+      if (code != CG_SUCCESS) converged = false;
+
+      solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
+      
+      // Report current solution and metadata to the supervisor
+      runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor, path = st.path, self_h = actor_cast<actor>(self)](std::vector<T> sol) {
+        anon_mail(gpu_done_atom_v, path, self_h, std::move(sol), meta).send(supervisor);
+        if (converged || code != CG_SUCCESS)
+          self->quit();
       });
     },
-    [=](update_stream_atom, int new_stream) { self->state().stream_id = new_stream; },
-    [=](shutdown_atom) { self->quit(); }
+    [=](update_stream_atom, int new_stream) {
+      self->state().stream_id = new_stream;
+    },
+    [=](shutdown_atom) {
+      self->quit();
+    }
   };
 }
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/supervisor_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/supervisor_actor.hpp
index 059cfee98f..dc68991c36 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/supervisor_actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/supervisor_actor.hpp
@@ -10,6 +10,11 @@
 #include "ft_cg_actor.hpp"
 #include "sparse_utils.hpp"
 
+
+using namespace caf;
+using namespace caf::cuda;
+// ---------------------------- SUPERVISOR ACTOR ----------------------------
+
 struct suspended_task {
   caf::actor solver;
   std::string path;
@@ -34,7 +39,7 @@ struct supervisor_state {
   std::unordered_map<std::string, double> cumulative_active_ms;
   std::unordered_map<caf::actor, int> actor_batch_sizes;
   std::deque<resource_slot> available_slots;
-  int max_active = 1;
+  int max_active = 1; // Admission control limit to be mindful of GPU memory.
   int num_iterations = MAX_ITERATIONS / 2;
   int num_gpus = 0;
   int tasks_succeeded = 0;
@@ -42,16 +47,14 @@ struct supervisor_state {
   std::chrono::steady_clock::time_point benchmark_start;
 };
 
-inline caf::behavior supervisor_actor(caf::stateful_actor<supervisor_state>* self, 
-                                      std::vector<MatrixTask> tasks, 
-                                      int initial_max_active, 
-                                      std::chrono::steady_clock::time_point start_time) {
+behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks, int initial_max_active, std::chrono::steady_clock::time_point start_time) {
     auto& st = self->state();
     st.queue.insert(st.queue.end(), std::make_move_iterator(tasks.begin()), std::make_move_iterator(tasks.end()));
     st.max_active = initial_max_active;
     st.benchmark_start = start_time;
-    st.num_gpus = caf::cuda::manager::get().get_num_devices();
+    st.num_gpus = manager::get().get_num_devices();
 
+    // Initialize the pool with streams interleaved across all available GPUs
     for (int s = 0; s < 32; ++s) {
       for (int g = 0; g < st.num_gpus; ++g) {
         st.available_slots.push_back({g, s});
@@ -65,22 +68,27 @@ inline caf::behavior supervisor_actor(caf::stateful_actor<supervisor_state>* sel
             auto task = std::move(s.queue.front());
             s.queue.pop_front();
             std::string path = task.path;
+
             s.cumulative_active_ms[path] = 0;
             s.enqueue_times[path] = task.enqueue_time;
             s.pick_times[path] = std::chrono::steady_clock::now();
+
             resource_slot slot = s.available_slots.front();
             s.available_slots.pop_front();
 
+            self->println("[INFO] Starting solver for: {} (Device: {}, Stream: {})", 
+                          path, slot.device_id, slot.stream_id);
             auto solver = self->spawn(fault_tolerant_cg_actor<float>,
-                                    path, task.data,
-                                    caf::cuda::create_in_arg(task.data->row_ptr),
-                                    caf::cuda::create_in_arg(task.data->col_indices),
-                                    caf::cuda::create_in_arg(task.data->values),
-                                    caf::cuda::create_in_arg(task.data->b),
-                                    caf::cuda::create_in_out_arg(task.data->x_guess),
+                                    path,
+                                    task.data,
+                                    create_in_arg(task.data->row_ptr),
+                                    create_in_arg(task.data->col_indices),
+                                    create_in_arg(task.data->values),
+                                    create_in_arg(task.data->b),
+                                    create_in_out_arg(task.data->x_guess),
                                     (int)task.data->row_ptr.size() - 1,
                                     (int)task.data->values.size(),
-                                    1e-5f, MAX_ITERATIONS, slot.device_id, slot.stream_id, caf::actor_cast<caf::actor>(self));
+                                    1e-5f, MAX_ITERATIONS, slot.device_id, slot.stream_id, actor_cast<actor>(self));
 
             s.start_times[path] = std::chrono::steady_clock::now();
             s.active_solvers.push_back(solver);
@@ -98,13 +106,22 @@ inline caf::behavior supervisor_actor(caf::stateful_actor<supervisor_state>* sel
                 s.suspended_queue.erase(it);
                 resource_slot slot = *slot_it;
                 s.available_slots.erase(slot_it);
+
                 s.pick_times[suspended.path] = std::chrono::steady_clock::now();
+
+                // Cap the minimum batch size to MAX_ITERATIONS / 8
                 int next_batch = std::max(MAX_ITERATIONS / 8, suspended.last_batch_size / 2);
+                self->println("[INFO] Resuming solver for: {} (Device: {}, Stream: {}, Batch: {})", 
+                              suspended.path, slot.device_id, slot.stream_id, next_batch);
+
+                // Resume on the same stream ID. No update_stream_atom_v needed.
                 self->mail(cg_next_step_atom_v, next_batch).send(suspended.solver);
+
                 s.active_solvers.push_back(suspended.solver);
                 s.task_resources[suspended.path] = slot;
                 s.actor_batch_sizes[suspended.solver] = next_batch;
-                resumed = true; break;
+                resumed = true;
+                break;
               }
             }
             if (!resumed) break;
@@ -115,43 +132,93 @@ inline caf::behavior supervisor_actor(caf::stateful_actor<supervisor_state>* sel
     spawn_next();
     
     return {
-        [=](gpu_done_atom, const std::string& task_name, caf::actor solver, std::vector<float>& solution, caf::cuda::solver_result_meta meta) {
+        [=](gpu_done_atom, const std::string& task_name, caf::actor solver, std::vector<float>& solution, solver_result_meta meta) {
             auto& s = self->state();
+
             if (meta.converged || meta.error_code != CG_SUCCESS) {
                 auto end_time = std::chrono::steady_clock::now();
-                if (meta.converged && meta.iterations < MAX_ITERATIONS) s.tasks_succeeded++;
-                else s.tasks_failed++;
+                auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
+                                  end_time - s.start_times[task_name]).count();
+
+                if (meta.converged && meta.iterations < MAX_ITERATIONS) {
+                    s.tasks_succeeded++;
+                    if (meta.iterations == 0)
+                        self->println("[DONE] {}: Initial guess satisfied tolerance ({} ms).", task_name, duration);
+                    else
+                        self->println("[DONE] {}: Converged in {} iterations ({} ms).", task_name, meta.iterations, duration);
+                } else {
+                    s.tasks_failed++;
+                    std::string reason = (meta.error_code == CG_SUCCESS)
+                                         ? "Maximum Iterations Reached"
+                                         : to_string(static_cast<cg_error_type>(meta.error_code));
+                    self->println("[FAIL] {}: {} (after {} iterations, {} ms).", 
+                                  task_name, reason,
+                                  meta.iterations,
+                                  duration);
+                }
 
+                // Final active slice
                 auto active_slice = std::chrono::duration<double, std::milli>(end_time - s.pick_times[task_name]).count();
                 s.cumulative_active_ms[task_name] += active_slice;
+
+                // We override pick_time in record_job to simulate a single continuous run that equals 
+                // the actual time spent on the GPU.
                 auto simulated_pick = end_time - std::chrono::duration_cast<std::chrono::steady_clock::duration>(
                                                 std::chrono::duration<double, std::milli>(s.cumulative_active_ms[task_name]));
 
                 record_job(task_name, s.enqueue_times[task_name], simulated_pick, end_time, meta.iterations, meta.converged);
+                s.enqueue_times.erase(task_name);
+                s.pick_times.erase(task_name);
+                s.cumulative_active_ms.erase(task_name);
+
                 auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
-                if (it != s.active_solvers.end()) s.active_solvers.erase(it);
+                if (it != s.active_solvers.end())
+                    s.active_solvers.erase(it);
+                s.start_times.erase(task_name);
+                s.actor_batch_sizes.erase(solver);
+
+                // Reclaim the device/stream slot and put it back in the pool
                 auto res_it = s.task_resources.find(task_name);
                 if (res_it != s.task_resources.end()) {
                     s.available_slots.push_back(res_it->second);
                     s.task_resources.erase(res_it);
                 }
+
                 spawn_next();
+
                 if (s.active_solvers.empty() && s.queue.empty() && s.suspended_queue.empty()) {
+                    self->println("All tasks in the pool have been processed.");
+
                     report_workload_stats();
                     self->quit();
                 }
             } else if (meta.iterations == 0) {
+                // Just finished initialization: trigger the first iteration batch immediately.
                 self->mail(cg_next_step_atom_v, s.num_iterations).send(solver);
             } else {
+                // Not done: Suspend the actor to allow others to use the stream
                 auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
+                
+                // Record the time spent in this active slice before suspending
                 auto active_slice = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - s.pick_times[task_name]).count();
                 s.cumulative_active_ms[task_name] += active_slice;
-                if (it != s.active_solvers.end()) s.active_solvers.erase(it);
+
+                if (it != s.active_solvers.end())
+                    s.active_solvers.erase(it);
+
                 resource_slot slot = s.task_resources[task_name];
-                s.available_slots.push_back(slot); s.task_resources.erase(task_name);
-                s.suspended_queue.push_back({solver, task_name, s.actor_batch_sizes[solver], slot.device_id, slot.stream_id});
+                s.available_slots.push_back(slot);
+                s.task_resources.erase(task_name);
+
+                int last_batch = s.actor_batch_sizes[solver];
+                s.suspended_queue.push_back({solver, task_name, last_batch, slot.device_id, slot.stream_id});
+
+                self->println("[INFO] Suspending solver for: {} (Reclaimed Device: {}, Stream: {})", 
+                              task_name, slot.device_id, slot.stream_id);
+
                 spawn_next();
             }
         }
+       
     };
-}
\ No newline at end of file
+}

From 38794cc93ca7633f243ad00a3f43e7f626cda6b2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 10:21:37 -0600
Subject: [PATCH 0903/1000] fixed illegal operation error

---
 libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp
index 82ee13b769..4fd5335cd8 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_actor.hpp
@@ -203,7 +203,7 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
       runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor, path = st.path, self_h = actor_cast<actor>(self)](std::vector<T> sol) {
         anon_mail(gpu_done_atom_v, path, self_h, std::move(sol), meta).send(supervisor);
         if (converged || code != CG_SUCCESS)
-          self->quit();
+          anon_mail(shutdown_atom_v).send(self_h);
       });
     },
     [=](update_stream_atom, int new_stream) {

From 6059c2d3bfaf87f1baa7d30850cd11f9945daff2 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 10:45:00 -0600
Subject: [PATCH 0904/1000] Intial commit.

---
 .../ft_cg_jacobi_actor.hpp                    | 191 ++++++++++++++++++
 1 file changed, 191 insertions(+)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_jacobi_actor.hpp

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_jacobi_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_jacobi_actor.hpp
new file mode 100644
index 0000000000..7e42785bff
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/ft_cg_jacobi_actor.hpp
@@ -0,0 +1,191 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <string>
+#include <vector>
+#include <memory>
+#include "atoms.hpp"
+#include "sparse_utils.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+template <class T = float>
+struct ft_cg_jacobi_state {
+  // Host Data
+  std::string path;
+  in<int> h_row_ptr, h_col_ind;
+  in<T> h_values, h_b;
+  in_out<T> h_x;
+
+  // GPU Buffers
+  mem_ptr<int> A_rp, A_ci, d_err;
+  mem_ptr<T> A_val, b, x, r, p, w, z, D_inv, y_tmp;
+  mem_ptr<char> spmv_ws;
+
+  // Config
+  int n, nnz, max_iter;
+  int iterations = 0;
+  int strikes = 0;
+  T tol;
+  int device_id, stream_id;
+
+  // Supervision & Monitoring
+  caf::actor supervisor;
+  device_ptr d_ptr;
+  program_ptr stab_prog;
+  program_ptr diag_prog;
+
+  T initial_rho = 0;
+  T current_rho = 0;
+  T old_rho = 0;
+  bool initialized = false;
+  std::shared_ptr<MatrixData> pinned_data;
+};
+
+template <class T = float>
+behavior fault_tolerant_cg_jacobi_actor(stateful_actor<ft_cg_jacobi_state<T>>* self,
+                                        std::string path,
+                                        std::shared_ptr<MatrixData> data,
+                                        in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+                                        int n, int nnz, T tol, int max_iter,
+                                        int dev_num, int stream, caf::actor supervisor) {
+  auto& s = self->state();
+  s.pinned_data = std::move(data);
+  s.path = std::move(path);
+  s.h_row_ptr = std::move(rp); s.h_col_ind = std::move(ci);
+  s.h_values = std::move(val); s.h_b = std::move(b_in); s.h_x = std::move(x_in);
+  s.n = n; s.nnz = nnz; s.tol = tol; s.max_iter = max_iter;
+  s.device_id = dev_num; s.stream_id = stream; s.supervisor = supervisor;
+
+  return {
+    [=](start_atom) {
+      auto& st = self->state();
+      if (st.initialized) return;
+
+      command_runner<> runner;
+      st.A_rp = runner.transfer_memory(st.device_id, st.stream_id, st.h_row_ptr);
+      st.A_ci = runner.transfer_memory(st.device_id, st.stream_id, st.h_col_ind);
+      st.A_val = runner.transfer_memory(st.device_id, st.stream_id, st.h_values);
+      st.b = runner.transfer_memory(st.device_id, st.stream_id, st.h_b);
+      st.x = runner.transfer_memory(st.device_id, st.stream_id, st.h_x);
+
+      st.d_ptr = platform::create()->schedule(st.stream_id, st.device_id);
+      st.d_ptr->enable_cublas(); st.d_ptr->enable_cusparse();
+
+      auto& mgr = manager::get();
+      st.stab_prog = mgr.create_program_from_cubin("stability_kernels.cubin", "check_stability", st.d_ptr);
+      st.diag_prog = mgr.create_program_from_cubin("jacobi_kernels.cubin", "extract_diag_inv", st.d_ptr);
+
+      st.r = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.p = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.w = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.z = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.D_inv = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.y_tmp = runner.transfer_memory(st.device_id, st.stream_id, out<T>(1));
+      st.d_err = runner.transfer_memory(st.device_id, st.stream_id, out<int>(1));
+      
+      size_t ws_sz = st.d_ptr->spmv_csr_buffer_size(st.stream_id, st.n, st.n, st.nnz, st.A_rp, st.A_ci, st.A_val, st.x, st.w);
+      if (ws_sz > 0) st.spmv_ws = command_runner<out<char>>{}.transfer_memory(st.device_id, st.stream_id, out<char>{static_cast<int>(ws_sz)});
+
+      // Jacobi setup: Extract D_inv
+      int threads = 256;
+      nd_range range((st.n + threads - 1) / threads, 1, 1, threads, 1, 1);
+      st.d_ptr->launch_kernel_mem_ref(st.diag_prog->get_kernel(st.d_ptr->getId()), range,
+                                      std::make_tuple(in<int>(st.n), st.A_rp, st.A_ci, st.A_val, st.D_inv), st.stream_id);
+
+      // Initial r = b - Ax
+      st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.x, T{0}, st.w, st.spmv_ws);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.b, st.r); else st.d_ptr->scopy(st.stream_id, st.n, st.b, st.r);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->daxpy(st.stream_id, st.n, -1.0, st.w, st.r); else st.d_ptr->saxpy(st.stream_id, st.n, -1.0f, st.w, st.r);
+
+      // Initial z = D_inv * r
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->d_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+      else st.d_ptr->s_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+
+      // Initial rho = r * z
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.z, st.y_tmp); else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+
+      st.initial_rho = runner.copy_to_host(st.y_tmp)[0];
+      st.current_rho = st.initial_rho;
+      st.initialized = true;
+
+      solver_result_meta meta(st.device_id, st.stream_id, 0, false, CG_SUCCESS);
+      self->mail(gpu_done_atom_v, st.path, actor_cast<actor>(self), std::vector<T>{}, meta).send(st.supervisor);
+    },
+
+    [=](cg_next_step_atom, int num_iters) {
+      auto& st = self->state();
+      command_runner<T> runner;
+      T threshold = st.tol * st.tol;
+      int code = CG_SUCCESS;
+      int step_count = 0;
+
+      while (step_count < num_iters && st.iterations < st.max_iter && st.current_rho > threshold) {
+        st.iterations++;
+        step_count++;
+
+        if (st.iterations > 1) {
+          T beta = st.current_rho / st.old_rho;
+          if constexpr (std::is_same_v<T, double>) {
+            st.d_ptr->dcopy(st.stream_id, st.n, st.z, st.w);
+            st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
+            st.d_ptr->dcopy(st.stream_id, st.n, st.w, st.p);
+          } else {
+            st.d_ptr->scopy(st.stream_id, st.n, st.z, st.w);
+            st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(beta), st.p, st.w);
+            st.d_ptr->scopy(st.stream_id, st.n, st.w, st.p);
+          }
+        } else {
+          if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.z, st.p); else st.d_ptr->scopy(st.stream_id, st.n, st.z, st.p);
+        }
+
+        st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.p, T{0}, st.w, st.spmv_ws);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.p, st.w, st.y_tmp); else st.d_ptr->sdot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
+        
+        T dot_pw = runner.copy_to_host(st.y_tmp)[0];
+        if (std::abs(dot_pw) < 1e-25) { code = CG_BREAKDOWN; break; }
+
+        T alpha = st.current_rho / dot_pw;
+        if constexpr (std::is_same_v<T, double>) {
+          st.d_ptr->daxpy(st.stream_id, st.n, alpha, st.p, st.x);
+          st.d_ptr->daxpy(st.stream_id, st.n, -alpha, st.w, st.r);
+        } else {
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(alpha), st.p, st.x);
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(-alpha), st.w, st.r);
+        }
+
+        st.old_rho = st.current_rho;
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->d_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+        else st.d_ptr->s_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.z, st.y_tmp); else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+        st.current_rho = runner.copy_to_host(st.y_tmp)[0];
+      }
+
+      bool converged = (st.current_rho <= threshold);
+      if (code == CG_SUCCESS && !converged && std::abs(st.old_rho - st.current_rho) < 1e-12) code = CG_STAGNATION;
+      if (code == CG_SUCCESS) {
+        if (!converged && st.iterations >= st.max_iter) code = CG_MAX_ITER;
+        if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
+        if (code == CG_SUCCESS) st.strikes = 0;
+      }
+      nd_range range(static_cast<int>((st.n + 255) / 256), 1, 1, 256, 1, 1);
+      CHECK_CUDA(cuMemsetD32Async(st.d_err->mem(), 0, 1, st.d_ptr->get_stream_for_actor(st.stream_id)));
+      st.d_ptr->launch_kernel_mem_ref(st.stab_prog->get_kernel(st.d_ptr->getId()), range,
+                                      std::make_tuple(in<int>(st.n), st.x, st.r, st.d_err), st.stream_id);
+      int err_flag = runner.copy_to_host(st.d_err)[0];
+      if (err_flag != 0 || std::isnan(st.current_rho) || std::isinf(st.current_rho)) code = CG_NAN_INF;
+      bool is_fatal = (code == CG_NAN_INF || code == CG_BREAKDOWN);
+      if (code != CG_SUCCESS && !is_fatal) { st.strikes++; if (st.strikes < 3) code = CG_SUCCESS; }
+      if (code != CG_SUCCESS) converged = false;
+      solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
+      runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor, path = st.path, self_h = actor_cast<actor>(self)](std::vector<T> sol) {
+        anon_mail(gpu_done_atom_v, path, self_h, std::move(sol), meta).send(supervisor);
+        if (converged || code != CG_SUCCESS) anon_mail(shutdown_atom_v).send(self_h);
+      });
+    },
+    [=](update_stream_atom, int new_stream) { self->state().stream_id = new_stream; },
+    [=](shutdown_atom) { self->quit(); }
+  };
+}

From 0144d8fcb5ad2c17d0c9f7d92cf84a04457d9d2f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 10:49:13 -0600
Subject: [PATCH 0905/1000] updated cmakelists to build with changes

---
 .../CMakeLists.txt                              | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
index 242169debe..ed8e26b561 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/CMakeLists.txt
@@ -49,7 +49,23 @@ add_custom_command(
   VERBATIM
 )
 
+set(JACOBI_KERNEL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/jacobi_kernels.cu")
+set(JACOBI_KERNEL_CUBIN "${CMAKE_CURRENT_BINARY_DIR}/jacobi_kernels.cubin")
+
+add_custom_command(
+  OUTPUT ${JACOBI_KERNEL_CUBIN}
+  COMMAND ${CUDAToolkit_NVCC_EXECUTABLE}
+          -cubin
+          -arch=${CUDA_ARCH}
+          -o ${JACOBI_KERNEL_CUBIN}
+          ${JACOBI_KERNEL_SRC}
+  DEPENDS ${JACOBI_KERNEL_SRC}
+  COMMENT "Compiling CUDA kernel ${JACOBI_KERNEL_SRC} for architecture: ${CUDA_ARCH}"
+  VERBATIM
+)
+
 add_custom_target(stability_kernels_cubin ALL DEPENDS ${STABILITY_KERNEL_CUBIN})
+add_custom_target(jacobi_kernels_cubin ALL DEPENDS ${JACOBI_KERNEL_CUBIN})
 
 
 # 5) Declare your executable and its source files
@@ -57,6 +73,7 @@ add_executable(test main.test.cpp sparse_utils.cpp)
 target_sources(test PRIVATE
   atoms.hpp
   ft_cg_actor.hpp
+  ft_cg_jacobi_actor.hpp
   supervisor_actor.hpp
 )
 

From b32434887db82b9683c310b4b07c2ab62d4cdf02 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 10:51:11 -0600
Subject: [PATCH 0906/1000] Saving.

---
 .../jacobi_kernels.cu                             | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/jacobi_kernels.cu

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/jacobi_kernels.cu b/libcaf_cuda/tests/fault-tolerance-workload-test/jacobi_kernels.cu
new file mode 100644
index 0000000000..16c17e7352
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/jacobi_kernels.cu
@@ -0,0 +1,15 @@
+extern "C" __global__
+void extract_diag_inv(int n, const int* row_ptr, const int* col_ind, const float* val, float* d_inv) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        float d = 0.0f;
+        // Search for the diagonal element (A[i][i]) in the sparse row
+        for (int j = row_ptr[i]; j < row_ptr[i+1]; j++) {
+            if (col_ind[j] == i) {
+                d = val[j];
+                break;
+            }
+        }
+        d_inv[i] = (d != 0.0f) ? 1.0f / d : 1.0f;
+    }
+}
\ No newline at end of file

From 442942debd6fd63f6ee243bd39b03bc5262160f5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 11:09:53 -0600
Subject: [PATCH 0907/1000] Initial commit.

---
 .../workload-A/CMakeLists.txt                 | 132 +++++++++
 .../workload-A/atoms.hpp                      |  51 ++++
 .../workload-A/ft_cg_actor.hpp                | 216 ++++++++++++++
 .../workload-A/ft_cg_jacobi_actor.hpp         | 191 +++++++++++++
 .../workload-A/jacobi_kernels.cu              |  15 +
 .../workload-A/main.native.cpp                |  60 ++++
 .../workload-A/main.native_sorted.cpp         |  63 +++++
 .../workload-A/main.test.cpp                  |  61 ++++
 .../workload-A/native_utils.cpp               |   0
 .../workload-A/native_utils.cu                | 248 ++++++++++++++++
 .../workload-A/native_utils.hpp               |  95 +++++++
 .../workload-A/sparse_utils.cpp               | 267 ++++++++++++++++++
 .../workload-A/sparse_utils.hpp               |  94 ++++++
 .../workload-A/stability_kernels.cu           |  10 +
 .../workload-A/supervisor_actor.hpp           | 224 +++++++++++++++
 15 files changed, 1727 insertions(+)
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/CMakeLists.txt
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/atoms.hpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_jacobi_actor.hpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/jacobi_kernels.cu
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native.cpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native_sorted.cpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.test.cpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.hpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/sparse_utils.cpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/sparse_utils.hpp
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/stability_kernels.cu
 create mode 100644 libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/CMakeLists.txt b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/CMakeLists.txt
new file mode 100644
index 0000000000..199d000211
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/CMakeLists.txt
@@ -0,0 +1,132 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+# Enable CUDA as a first-class language for the project
+project(CUDA_ACTORS LANGUAGES CXX CUDA)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+# --- CUDA Kernel Compilation ---
+set(STABILITY_KERNEL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/stability_kernels.cu")
+set(STABILITY_KERNEL_CUBIN "${CMAKE_CURRENT_BINARY_DIR}/stability_kernels.cubin")
+
+# Target CUDA architecture. 'native' targets the current machine's GPU (requires CUDA 11.6+).
+# You can override this with -DCUDA_ARCH=sm_XX if needed.
+set(CUDA_ARCH "native" CACHE STRING "Target CUDA architecture (e.g., native, sm_70, sm_75, sm_80, sm_86)")
+
+add_custom_command(
+  OUTPUT ${STABILITY_KERNEL_CUBIN}
+  COMMAND ${CUDAToolkit_NVCC_EXECUTABLE}
+          -cubin
+          -arch=${CUDA_ARCH}
+          -o ${STABILITY_KERNEL_CUBIN}
+          ${STABILITY_KERNEL_SRC}
+  DEPENDS ${STABILITY_KERNEL_SRC}
+  COMMENT "Compiling CUDA kernel ${STABILITY_KERNEL_SRC} for architecture: ${CUDA_ARCH}"
+  VERBATIM
+)
+
+set(JACOBI_KERNEL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/jacobi_kernels.cu")
+set(JACOBI_KERNEL_CUBIN "${CMAKE_CURRENT_BINARY_DIR}/jacobi_kernels.cubin")
+
+add_custom_command(
+  OUTPUT ${JACOBI_KERNEL_CUBIN}
+  COMMAND ${CUDAToolkit_NVCC_EXECUTABLE}
+          -cubin
+          -arch=${CUDA_ARCH}
+          -o ${JACOBI_KERNEL_CUBIN}
+          ${JACOBI_KERNEL_SRC}
+  DEPENDS ${JACOBI_KERNEL_SRC}
+  COMMENT "Compiling CUDA kernel ${JACOBI_KERNEL_SRC} for architecture: ${CUDA_ARCH}"
+  VERBATIM
+)
+
+add_custom_target(stability_kernels_cubin ALL DEPENDS ${STABILITY_KERNEL_CUBIN})
+add_custom_target(jacobi_kernels_cubin ALL DEPENDS ${JACOBI_KERNEL_CUBIN})
+
+
+# 5) Declare your executable and its source files
+add_executable(test main.test.cpp sparse_utils.cpp)
+target_sources(test PRIVATE
+  atoms.hpp
+  ft_cg_actor.hpp
+  ft_cg_jacobi_actor.hpp
+  supervisor_actor.hpp
+)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
+)
+
+
+# # 5) Declare your executable
+# add_executable(hot-potatoe hot-potatoe.cpp sparse_utils.cpp)
+
+# target_compile_definitions(hot-potatoe PRIVATE CAF_ENABLE_LOGGING)
+
+# target_link_libraries(hot-potatoe
+#   PRIVATE
+#     "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+#     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+#     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+#     CUDA::nvrtc
+#     CUDA::cublas
+#     CUDA::cusparse
+# )
+
+
+# FindThreads is required for std::thread in the native version
+find_package(Threads REQUIRED)
+
+# 6) Declare the native benchmark executable (raw CUDA/cuBLAS/cuSPARSE)
+add_executable(workload-native main.native.cpp sparse_utils.cpp native_utils.cu)
+
+target_link_libraries(workload-native
+  PRIVATE
+    CUDA::cudart
+    CUDA::cublas
+    CUDA::cusparse
+    Threads::Threads
+)
+
+# 7) Declare the native sorted benchmark executable
+add_executable(workload-native-sorted main.native_sorted.cpp sparse_utils.cpp native_utils.cu)
+
+target_link_libraries(workload-native-sorted
+  PRIVATE
+    CUDA::cudart
+    CUDA::cublas
+    CUDA::cusparse
+    Threads::Threads
+)
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/atoms.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/atoms.hpp
new file mode 100644
index 0000000000..ab2a741b6c
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/atoms.hpp
@@ -0,0 +1,51 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include "sparse_utils.hpp"
+
+constexpr int MAX_ITERATIONS = 16000;
+
+CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
+    CAF_ADD_ATOM(workload_test, get_work_atom)
+    CAF_ADD_ATOM(workload_test, release_memory_atom)
+    CAF_ADD_ATOM(workload_test, request_work_atom)
+    CAF_ADD_ATOM(workload_test, worker_done_atom)
+    CAF_ADD_ATOM(workload_test, work_tick_atom)
+    CAF_ADD_ATOM(workload_test, add_work_atom)
+    CAF_ADD_ATOM(workload_test, steal_work_atom)
+    CAF_ADD_ATOM(workload_test, update_stream_atom)
+    CAF_ADD_ATOM(workload_test, shutdown_atom)
+    CAF_ADD_TYPE_ID(workload_test, (SolverType))
+    CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
+    CAF_ADD_TYPE_ID(workload_test, (MatrixData))
+    CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
+    CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
+CAF_END_TYPE_ID_BLOCK(workload_test)
+
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixData)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(SolverType)
+
+enum cg_error_type : int {
+  CG_SUCCESS = 0,
+  CG_MAX_ITER = 1,
+  CG_NAN_INF = 2,
+  CG_STAGNATION = 3,
+  CG_BREAKDOWN = 4,
+  CG_RESIDUAL_FACTOR_FAIL = 5
+};
+
+inline std::string to_string(cg_error_type err) {
+  switch (err) {
+    case CG_SUCCESS: return "Success";
+    case CG_MAX_ITER: return "Maximum Iterations Reached";
+    case CG_NAN_INF: return "Stability Check Failed (NaN/Inf Detected)";
+    case CG_STAGNATION: return "Stagnation Detected (Residual stopped changing)";
+    case CG_BREAKDOWN: return "Solver Breakdown (Division by zero/near-zero)";
+    case CG_RESIDUAL_FACTOR_FAIL: return "Residual Factor Check Failed";
+    default: return "Unknown Error (" + std::to_string(static_cast<int>(err)) + ")";
+  }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp
new file mode 100644
index 0000000000..4fd5335cd8
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp
@@ -0,0 +1,216 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <string>
+#include <vector>
+#include <memory>
+#include "atoms.hpp"
+#include "sparse_utils.hpp"
+#include "caf/actorSOLVE/actorSOLVE.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+// ---------------------------- FAULT TOLERANT SOLVER ----------------------------
+
+template <class T = float>
+struct ft_cg_state {
+  // Host Data
+  std::string path;
+  in<int> h_row_ptr, h_col_ind;
+  in<T> h_values, h_b;
+  in_out<T> h_x;
+
+  // GPU Buffers
+  mem_ptr<int> A_rp, A_ci, d_err;
+  mem_ptr<T> A_val, b, x, r, p, w, y_tmp;
+  mem_ptr<char> spmv_ws;
+
+  // Config
+  int n, nnz, max_iter;
+  int iterations = 0;
+  int strikes = 0;
+  T tol;
+  int device_id, stream_id;
+
+  // Supervision & Monitoring
+  caf::actor supervisor;
+  device_ptr d_ptr;
+  program_ptr stab_prog;
+
+  T initial_rho = 0;
+  T current_rho = 0;
+  T old_rho = 0;
+  bool initialized = false;
+  std::shared_ptr<MatrixData> pinned_data;
+};
+
+template <class T = float>
+behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
+                                 std::string path,
+                                 std::shared_ptr<MatrixData> data,
+                                 in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+                                 int n, int nnz, T tol, int max_iter,
+                                 int dev_num, int stream, caf::actor supervisor) {
+  auto& s = self->state();
+  s.pinned_data = std::move(data);
+  s.path = std::move(path);
+  s.h_row_ptr = std::move(rp); s.h_col_ind = std::move(ci);
+  s.h_values = std::move(val); s.h_b = std::move(b_in); s.h_x = std::move(x_in);
+  s.n = n; s.nnz = nnz; s.tol = tol; s.max_iter = max_iter;
+  s.device_id = dev_num; s.stream_id = stream; s.supervisor = supervisor;
+
+  return {
+    [=](start_atom) {
+      auto& st = self->state();
+      if (st.initialized) return;
+
+      command_runner<> runner;
+      st.A_rp = runner.transfer_memory(st.device_id, st.stream_id, st.h_row_ptr);
+      st.A_ci = runner.transfer_memory(st.device_id, st.stream_id, st.h_col_ind);
+      st.A_val = runner.transfer_memory(st.device_id, st.stream_id, st.h_values);
+      st.b = runner.transfer_memory(st.device_id, st.stream_id, st.h_b);
+      st.x = runner.transfer_memory(st.device_id, st.stream_id, st.h_x);
+
+      st.d_ptr = platform::create()->schedule(st.stream_id, st.device_id);
+      st.d_ptr->enable_cublas(); st.d_ptr->enable_cusparse();
+
+      auto& mgr = manager::get();
+      // Load stability kernel from file as requested
+      st.stab_prog = mgr.create_program_from_cubin("stability_kernels.cubin", "check_stability", st.d_ptr);
+
+      st.r = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.p = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.w = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.y_tmp = runner.transfer_memory(st.device_id, st.stream_id, out<T>(1));
+      st.d_err = runner.transfer_memory(st.device_id, st.stream_id, out<int>(1));
+      
+      size_t ws_sz = st.d_ptr->spmv_csr_buffer_size(st.stream_id, st.n, st.n, st.nnz, st.A_rp, st.A_ci, st.A_val, st.x, st.w);
+      if (ws_sz > 0) st.spmv_ws = command_runner<out<char>>{}.transfer_memory(st.device_id, st.stream_id, out<char>{static_cast<int>(ws_sz)});
+
+      st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.x, T{0}, st.w, st.spmv_ws);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.b, st.r);
+      else st.d_ptr->scopy(st.stream_id, st.n, st.b, st.r);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->daxpy(st.stream_id, st.n, -1.0, st.w, st.r);
+      else st.d_ptr->saxpy(st.stream_id, st.n, -1.0f, st.w, st.r);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+      else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+
+      st.initial_rho = runner.copy_to_host(st.y_tmp)[0];
+      st.current_rho = st.initial_rho;
+      st.initialized = true;
+
+      // Notify supervisor that setup is complete and report initial status
+      solver_result_meta meta(st.device_id, st.stream_id, 0, false, CG_SUCCESS);
+      self->mail(gpu_done_atom_v, st.path, actor_cast<actor>(self), std::vector<T>{}, meta).send(st.supervisor);
+    },
+
+    [=](cg_next_step_atom, int num_iters) {
+      auto& st = self->state();
+      command_runner<T> runner;
+      T threshold = st.tol * st.tol;
+      int code = CG_SUCCESS;
+      int step_count = 0;
+
+      // Execute exactly the number of iterations requested by the supervisor
+      while (step_count < num_iters && st.iterations < st.max_iter && st.current_rho > threshold) {
+        // std::cout << "iterations = " << st.iterations << ", current_rho = " << st.current_rho << std::endl;
+        st.iterations++;
+        step_count++;
+
+        if (st.iterations > 1) {
+          T beta = st.current_rho / st.old_rho;
+          if constexpr (std::is_same_v<T, double>) {
+            st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.w);
+            st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
+            st.d_ptr->dcopy(st.stream_id, st.n, st.w, st.p);
+          } else {
+            st.d_ptr->scopy(st.stream_id, st.n, st.r, st.w);
+            st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(beta), st.p, st.w);
+            st.d_ptr->scopy(st.stream_id, st.n, st.w, st.p);
+          }
+        } else {
+          if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.p);
+          else st.d_ptr->scopy(st.stream_id, st.n, st.r, st.p);
+        }
+
+        st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.p, T{0}, st.w, st.spmv_ws);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
+        else st.d_ptr->sdot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
+        
+        T dot_pw = runner.copy_to_host(st.y_tmp)[0];
+        if (std::abs(dot_pw) < 1e-25) { 
+          code = CG_BREAKDOWN; 
+          break; 
+        }
+
+        T alpha = st.current_rho / dot_pw;
+        if constexpr (std::is_same_v<T, double>) {
+          st.d_ptr->daxpy(st.stream_id, st.n, alpha, st.p, st.x);
+          st.d_ptr->daxpy(st.stream_id, st.n, -alpha, st.w, st.r);
+        } else {
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(alpha), st.p, st.x);
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(-alpha), st.w, st.r);
+        }
+
+        st.old_rho = st.current_rho;
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+        else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+        st.current_rho = runner.copy_to_host(st.y_tmp)[0];
+
+      }
+
+      // Run error checks and prepare progress report
+      bool converged = (st.current_rho <= threshold);
+
+      // Check for non-fatal errors that can be retried (Stagnation, Max Iter, Residual Factor)
+      if (code == CG_SUCCESS && !converged && std::abs(st.old_rho - st.current_rho) < 1e-12)
+        code = CG_STAGNATION;
+
+      if (code == CG_SUCCESS) {
+        if (!converged && st.iterations >= st.max_iter) code = CG_MAX_ITER;
+        // Residual decrease check: treat as non-fatal strike if residual didn't decrease significantly
+        if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
+        
+        // If we reached here with CG_SUCCESS, reset strikes as this was a productive batch
+        if (code == CG_SUCCESS) st.strikes = 0;
+      }
+
+      // Reset and launch NaN/Inf stability kernel from file
+      nd_range range(static_cast<int>((st.n + 255) / 256), 1, 1, 256, 1, 1);
+      CHECK_CUDA(cuMemsetD32Async(st.d_err->mem(), 0, 1, st.d_ptr->get_stream_for_actor(st.stream_id)));
+      st.d_ptr->launch_kernel_mem_ref(st.stab_prog->get_kernel(st.d_ptr->getId()), range,
+                                      std::make_tuple(in<int>(st.n), st.x, st.r, st.d_err), st.stream_id);
+
+      int err_flag = runner.copy_to_host(st.d_err)[0];
+      if (err_flag != 0 || std::isnan(st.current_rho) || std::isinf(st.current_rho)) code = CG_NAN_INF;
+
+      // Three strikes policy for non-fatal errors (Stagnation, Max Iterations, Residual Factor Failure)
+      bool is_fatal = (code == CG_NAN_INF || code == CG_BREAKDOWN);
+      if (code != CG_SUCCESS && !is_fatal) {
+        st.strikes++;
+        if (st.strikes < 3) {
+          code = CG_SUCCESS; // Reset code to SUCCESS to allow the supervisor to retry/suspend
+        }
+      }
+
+      if (code != CG_SUCCESS) converged = false;
+
+      solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
+      
+      // Report current solution and metadata to the supervisor
+      runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor, path = st.path, self_h = actor_cast<actor>(self)](std::vector<T> sol) {
+        anon_mail(gpu_done_atom_v, path, self_h, std::move(sol), meta).send(supervisor);
+        if (converged || code != CG_SUCCESS)
+          anon_mail(shutdown_atom_v).send(self_h);
+      });
+    },
+    [=](update_stream_atom, int new_stream) {
+      self->state().stream_id = new_stream;
+    },
+    [=](shutdown_atom) {
+      self->quit();
+    }
+  };
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_jacobi_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_jacobi_actor.hpp
new file mode 100644
index 0000000000..7e42785bff
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_jacobi_actor.hpp
@@ -0,0 +1,191 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <string>
+#include <vector>
+#include <memory>
+#include "atoms.hpp"
+#include "sparse_utils.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+template <class T = float>
+struct ft_cg_jacobi_state {
+  // Host Data
+  std::string path;
+  in<int> h_row_ptr, h_col_ind;
+  in<T> h_values, h_b;
+  in_out<T> h_x;
+
+  // GPU Buffers
+  mem_ptr<int> A_rp, A_ci, d_err;
+  mem_ptr<T> A_val, b, x, r, p, w, z, D_inv, y_tmp;
+  mem_ptr<char> spmv_ws;
+
+  // Config
+  int n, nnz, max_iter;
+  int iterations = 0;
+  int strikes = 0;
+  T tol;
+  int device_id, stream_id;
+
+  // Supervision & Monitoring
+  caf::actor supervisor;
+  device_ptr d_ptr;
+  program_ptr stab_prog;
+  program_ptr diag_prog;
+
+  T initial_rho = 0;
+  T current_rho = 0;
+  T old_rho = 0;
+  bool initialized = false;
+  std::shared_ptr<MatrixData> pinned_data;
+};
+
+template <class T = float>
+behavior fault_tolerant_cg_jacobi_actor(stateful_actor<ft_cg_jacobi_state<T>>* self,
+                                        std::string path,
+                                        std::shared_ptr<MatrixData> data,
+                                        in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+                                        int n, int nnz, T tol, int max_iter,
+                                        int dev_num, int stream, caf::actor supervisor) {
+  auto& s = self->state();
+  s.pinned_data = std::move(data);
+  s.path = std::move(path);
+  s.h_row_ptr = std::move(rp); s.h_col_ind = std::move(ci);
+  s.h_values = std::move(val); s.h_b = std::move(b_in); s.h_x = std::move(x_in);
+  s.n = n; s.nnz = nnz; s.tol = tol; s.max_iter = max_iter;
+  s.device_id = dev_num; s.stream_id = stream; s.supervisor = supervisor;
+
+  return {
+    [=](start_atom) {
+      auto& st = self->state();
+      if (st.initialized) return;
+
+      command_runner<> runner;
+      st.A_rp = runner.transfer_memory(st.device_id, st.stream_id, st.h_row_ptr);
+      st.A_ci = runner.transfer_memory(st.device_id, st.stream_id, st.h_col_ind);
+      st.A_val = runner.transfer_memory(st.device_id, st.stream_id, st.h_values);
+      st.b = runner.transfer_memory(st.device_id, st.stream_id, st.h_b);
+      st.x = runner.transfer_memory(st.device_id, st.stream_id, st.h_x);
+
+      st.d_ptr = platform::create()->schedule(st.stream_id, st.device_id);
+      st.d_ptr->enable_cublas(); st.d_ptr->enable_cusparse();
+
+      auto& mgr = manager::get();
+      st.stab_prog = mgr.create_program_from_cubin("stability_kernels.cubin", "check_stability", st.d_ptr);
+      st.diag_prog = mgr.create_program_from_cubin("jacobi_kernels.cubin", "extract_diag_inv", st.d_ptr);
+
+      st.r = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.p = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.w = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.z = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.D_inv = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.y_tmp = runner.transfer_memory(st.device_id, st.stream_id, out<T>(1));
+      st.d_err = runner.transfer_memory(st.device_id, st.stream_id, out<int>(1));
+      
+      size_t ws_sz = st.d_ptr->spmv_csr_buffer_size(st.stream_id, st.n, st.n, st.nnz, st.A_rp, st.A_ci, st.A_val, st.x, st.w);
+      if (ws_sz > 0) st.spmv_ws = command_runner<out<char>>{}.transfer_memory(st.device_id, st.stream_id, out<char>{static_cast<int>(ws_sz)});
+
+      // Jacobi setup: Extract D_inv
+      int threads = 256;
+      nd_range range((st.n + threads - 1) / threads, 1, 1, threads, 1, 1);
+      st.d_ptr->launch_kernel_mem_ref(st.diag_prog->get_kernel(st.d_ptr->getId()), range,
+                                      std::make_tuple(in<int>(st.n), st.A_rp, st.A_ci, st.A_val, st.D_inv), st.stream_id);
+
+      // Initial r = b - Ax
+      st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.x, T{0}, st.w, st.spmv_ws);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.b, st.r); else st.d_ptr->scopy(st.stream_id, st.n, st.b, st.r);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->daxpy(st.stream_id, st.n, -1.0, st.w, st.r); else st.d_ptr->saxpy(st.stream_id, st.n, -1.0f, st.w, st.r);
+
+      // Initial z = D_inv * r
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->d_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+      else st.d_ptr->s_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+
+      // Initial rho = r * z
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.z, st.y_tmp); else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+
+      st.initial_rho = runner.copy_to_host(st.y_tmp)[0];
+      st.current_rho = st.initial_rho;
+      st.initialized = true;
+
+      solver_result_meta meta(st.device_id, st.stream_id, 0, false, CG_SUCCESS);
+      self->mail(gpu_done_atom_v, st.path, actor_cast<actor>(self), std::vector<T>{}, meta).send(st.supervisor);
+    },
+
+    [=](cg_next_step_atom, int num_iters) {
+      auto& st = self->state();
+      command_runner<T> runner;
+      T threshold = st.tol * st.tol;
+      int code = CG_SUCCESS;
+      int step_count = 0;
+
+      while (step_count < num_iters && st.iterations < st.max_iter && st.current_rho > threshold) {
+        st.iterations++;
+        step_count++;
+
+        if (st.iterations > 1) {
+          T beta = st.current_rho / st.old_rho;
+          if constexpr (std::is_same_v<T, double>) {
+            st.d_ptr->dcopy(st.stream_id, st.n, st.z, st.w);
+            st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
+            st.d_ptr->dcopy(st.stream_id, st.n, st.w, st.p);
+          } else {
+            st.d_ptr->scopy(st.stream_id, st.n, st.z, st.w);
+            st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(beta), st.p, st.w);
+            st.d_ptr->scopy(st.stream_id, st.n, st.w, st.p);
+          }
+        } else {
+          if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.z, st.p); else st.d_ptr->scopy(st.stream_id, st.n, st.z, st.p);
+        }
+
+        st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.p, T{0}, st.w, st.spmv_ws);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.p, st.w, st.y_tmp); else st.d_ptr->sdot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
+        
+        T dot_pw = runner.copy_to_host(st.y_tmp)[0];
+        if (std::abs(dot_pw) < 1e-25) { code = CG_BREAKDOWN; break; }
+
+        T alpha = st.current_rho / dot_pw;
+        if constexpr (std::is_same_v<T, double>) {
+          st.d_ptr->daxpy(st.stream_id, st.n, alpha, st.p, st.x);
+          st.d_ptr->daxpy(st.stream_id, st.n, -alpha, st.w, st.r);
+        } else {
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(alpha), st.p, st.x);
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(-alpha), st.w, st.r);
+        }
+
+        st.old_rho = st.current_rho;
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->d_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+        else st.d_ptr->s_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.z, st.y_tmp); else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+        st.current_rho = runner.copy_to_host(st.y_tmp)[0];
+      }
+
+      bool converged = (st.current_rho <= threshold);
+      if (code == CG_SUCCESS && !converged && std::abs(st.old_rho - st.current_rho) < 1e-12) code = CG_STAGNATION;
+      if (code == CG_SUCCESS) {
+        if (!converged && st.iterations >= st.max_iter) code = CG_MAX_ITER;
+        if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
+        if (code == CG_SUCCESS) st.strikes = 0;
+      }
+      nd_range range(static_cast<int>((st.n + 255) / 256), 1, 1, 256, 1, 1);
+      CHECK_CUDA(cuMemsetD32Async(st.d_err->mem(), 0, 1, st.d_ptr->get_stream_for_actor(st.stream_id)));
+      st.d_ptr->launch_kernel_mem_ref(st.stab_prog->get_kernel(st.d_ptr->getId()), range,
+                                      std::make_tuple(in<int>(st.n), st.x, st.r, st.d_err), st.stream_id);
+      int err_flag = runner.copy_to_host(st.d_err)[0];
+      if (err_flag != 0 || std::isnan(st.current_rho) || std::isinf(st.current_rho)) code = CG_NAN_INF;
+      bool is_fatal = (code == CG_NAN_INF || code == CG_BREAKDOWN);
+      if (code != CG_SUCCESS && !is_fatal) { st.strikes++; if (st.strikes < 3) code = CG_SUCCESS; }
+      if (code != CG_SUCCESS) converged = false;
+      solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
+      runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor, path = st.path, self_h = actor_cast<actor>(self)](std::vector<T> sol) {
+        anon_mail(gpu_done_atom_v, path, self_h, std::move(sol), meta).send(supervisor);
+        if (converged || code != CG_SUCCESS) anon_mail(shutdown_atom_v).send(self_h);
+      });
+    },
+    [=](update_stream_atom, int new_stream) { self->state().stream_id = new_stream; },
+    [=](shutdown_atom) { self->quit(); }
+  };
+}
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/jacobi_kernels.cu b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/jacobi_kernels.cu
new file mode 100644
index 0000000000..16c17e7352
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/jacobi_kernels.cu
@@ -0,0 +1,15 @@
+extern "C" __global__
+void extract_diag_inv(int n, const int* row_ptr, const int* col_ind, const float* val, float* d_inv) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        float d = 0.0f;
+        // Search for the diagonal element (A[i][i]) in the sparse row
+        for (int j = row_ptr[i]; j < row_ptr[i+1]; j++) {
+            if (col_ind[j] == i) {
+                d = val[j];
+                break;
+            }
+        }
+        d_inv[i] = (d != 0.0f) ? 1.0f / d : 1.0f;
+    }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native.cpp
new file mode 100644
index 0000000000..ea334586ac
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native.cpp
@@ -0,0 +1,60 @@
+#include <iostream>
+#include <vector>
+#include <string>
+#include <chrono>
+#include <thread>
+#include <filesystem>
+#include <algorithm>
+#include <atomic>
+#include "native_utils.hpp"
+
+void producer(ThreadSafeQueue<MatrixTask>& queue, std::vector<MatrixTask> matrix_pool) {
+    for (auto& task : matrix_pool) {
+        task.enqueue_time = std::chrono::steady_clock::now();
+        queue.push(task);
+    }
+    queue.signal_shutdown();
+}
+
+int main(int argc, char** argv)
+{
+    constexpr uint32_t WORKLOAD_SEED = 42;
+    int num_streams = 4;
+    // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
+
+    std::cout << "[INFO] Loading matrices...\n";
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+
+    if (matrix_pool.empty()) {
+        std::cerr << "No matrices found.\n";
+        return 1;
+    }
+
+    int num_gpus = 0;
+    CHECK_CUDA(cudaGetDeviceCount(&num_gpus));
+    
+    std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
+    std::cout << "[INFO] Matrix pool size: " << matrix_pool.size() << "\n";
+    std::cout << "[INFO] Streams/GPU: " << num_streams << "\n";
+
+    std::atomic<int> tasks_succeeded{0};
+    std::atomic<int> tasks_failed{0};
+    ThreadSafeQueue<MatrixTask> work_queue;
+    
+    init_benchmark_timer();
+    std::thread producer_thread(producer, std::ref(work_queue), matrix_pool);
+    
+    std::vector<std::thread> workers;
+    for (int gpu = 0; gpu < num_gpus; ++gpu) {
+        for (int stream = 0; stream < num_streams; ++stream) {
+            workers.emplace_back(gpu_stream_worker, gpu, gpu * num_streams + stream, std::ref(work_queue), std::ref(tasks_succeeded), std::ref(tasks_failed));
+        }
+    }
+
+    producer_thread.join();
+    for (auto& worker : workers) worker.join();
+
+    report_workload_stats();
+
+    return 0;
+}
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native_sorted.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native_sorted.cpp
new file mode 100644
index 0000000000..85aa4d301d
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native_sorted.cpp
@@ -0,0 +1,63 @@
+#include <iostream>
+#include <vector>
+#include <string>
+#include <chrono>
+#include <thread>
+#include <filesystem>
+#include <algorithm>
+#include <atomic>
+#include "native_utils.hpp"
+
+void producer(ThreadSafeQueue<MatrixTask>& queue, std::vector<MatrixTask> matrix_pool) {
+    // Order tasks from lowest NNZ to highest NNZ
+    std::sort(matrix_pool.begin(), matrix_pool.end(), [](const MatrixTask& a, const MatrixTask& b) {
+        return a.data->nnz < b.data->nnz;
+    });
+
+    for (auto& task : matrix_pool) {
+        task.enqueue_time = std::chrono::steady_clock::now();
+        queue.push(task);
+    }
+    queue.signal_shutdown();
+}
+
+int main(int argc, char** argv) {
+    constexpr uint32_t WORKLOAD_SEED = 42;
+    int num_streams = 4;
+    // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
+
+    std::cout << "[INFO] Loading matrices...\n";
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+
+    if (matrix_pool.empty()) {
+        std::cerr << "No matrices found.\n";
+        return 1;
+    }
+
+    int num_gpus = 0;
+    CHECK_CUDA(cudaGetDeviceCount(&num_gpus));
+
+    std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
+    std::cout << "[INFO] Matrix pool size: " << matrix_pool.size() << "\n";
+    std::cout << "[INFO] Streams/GPU: " << num_streams << "\n";
+
+    std::atomic<int> tasks_succeeded{0};
+    std::atomic<int> tasks_failed{0};
+    ThreadSafeQueue<MatrixTask> work_queue;
+    
+    init_benchmark_timer();
+    std::thread producer_thread(producer, std::ref(work_queue), matrix_pool);
+    
+    std::vector<std::thread> workers;
+    for (int gpu = 0; gpu < num_gpus; ++gpu) {
+        for (int stream = 0; stream < num_streams; ++stream) {
+            workers.emplace_back(gpu_stream_worker, gpu, gpu * num_streams + stream, std::ref(work_queue), std::ref(tasks_succeeded), std::ref(tasks_failed));
+        }
+    }
+    producer_thread.join();
+    for (auto& worker : workers) worker.join();
+
+    report_workload_stats();
+
+    return 0;
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.test.cpp
new file mode 100644
index 0000000000..d3f0fd4cd4
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.test.cpp
@@ -0,0 +1,61 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <deque>
+#include <string>
+#include <chrono>
+#include <numeric>
+#include <cmath>
+#include <algorithm>
+#include <filesystem>
+#include <memory>
+#include "sparse_utils.hpp"
+#include "atoms.hpp"
+#include "ft_cg_actor.hpp"
+#include "supervisor_actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+
+constexpr uint32_t WORKLOAD_SEED = 42;
+void caf_main(actor_system& sys) {
+    manager::init(sys, manager_config(true, true));
+    std::cout << "[INFO] Loading matrices...\n";
+    {
+         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+        //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/unsymmetric", CGS_SOLVER);
+         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric", CGS_SOLVER);
+         auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+
+        int num_gpus = manager::get().get_num_devices();
+        std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
+        std::cout << "[INFO] Matrix pool size: " << tasks_vec.size() << "\n";
+
+        if (tasks_vec.empty()) {
+            std::cerr << "No matrices found. Running dummy test task." << std::endl;
+            auto data = std::make_shared<MatrixData>();
+            data->row_ptr = {0, 1, 2};
+            data->col_indices = {0, 1};
+            data->values = {10.0f, 10.0f};
+            data->b = {100.0f, 100.0f};
+            data->x_guess = {0.0f, 0.0f};
+            tasks_vec.push_back({"dummy_task", CGS_SOLVER, data});
+        }
+
+        init_benchmark_timer();
+        for (auto& task : tasks_vec) {
+            task.enqueue_time = std::chrono::steady_clock::now();
+        }
+
+        auto benchmark_start = std::chrono::steady_clock::now();
+        int admission_control_limit = 4 * num_gpus; // 4 concurrent tasks per GPU
+
+        sys.spawn(supervisor_actor, std::move(tasks_vec), admission_control_limit, benchmark_start);
+        sys.await_all_actors_done();
+    }
+    manager::shutdown();
+}
+CAF_MAIN(id_block::cuda, id_block::workload_test)
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cpp
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
new file mode 100644
index 0000000000..22a426f494
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
@@ -0,0 +1,248 @@
+#include "native_utils.hpp"
+#include <cmath>
+#include <chrono>
+
+// ============================================================
+// PCG Kernels
+// ============================================================
+
+__global__ void extract_diag_inv_kernel(int n, const int* row_ptr, const int* col_ind, const float* values, float* d_inv) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        float diag = 1.0f;
+        for (int j = row_ptr[i]; j < row_ptr[i + 1]; j++) {
+            if (col_ind[j] == i) {
+                diag = values[j];
+                break;
+            }
+        }
+        d_inv[i] = (fabsf(diag) > 1e-20f) ? 1.0f / diag : 1.0f;
+    }
+}
+
+__global__ void elementwise_mul_kernel(int n, const float* a, const float* b, float* c) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) c[i] = a[i] * b[i];
+}
+
+int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
+                           const MatrixTask& task, cudaStream_t stream) {
+    int n = (int)task.data->row_ptr.size() - 1;
+    float alpha = 1.0f, beta = 0.0f, rho = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
+    float tolerance = 1e-5f;
+    int max_iters = 16000;
+
+    float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b, *d_z, *d_Dinv;
+    int *d_row_ptr, *d_col_ind;
+
+    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_z, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_Dinv, n * sizeof(float), stream));
+
+    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
+
+    CHECK_CUBLAS(cublasSetStream(cublas, stream));
+    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
+
+    // Extract Diagonal Inverse
+    int threads = 256;
+    int blocks = (n + threads - 1) / threads;
+    extract_diag_inv_kernel<<<blocks, threads, 0, stream>>>(n, d_row_ptr, d_col_ind, d_val, d_Dinv);
+
+    cusparseSpMatDescr_t matA;
+    cusparseDnVecDescr_t vecP, vecAp;
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val,
+                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                     CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecAp, n, d_Ap, CUDA_R_32F));
+
+    size_t bufferSize = 0;
+    void* d_buffer = nullptr;
+    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, vecP, &beta, vecAp,
+                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
+
+    // r = b (assuming x=0), z = M^-1 * r, p = z
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
+    elementwise_mul_kernel<<<blocks, threads, 0, stream>>>(n, d_Dinv, d_r, d_z);
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_z, 1, d_p, 1));
+    CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_z, 1, &rho));
+
+    int k = 0;
+    float r_norm_sq = 0.0f;
+    while (k < max_iters) {
+        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                    &alpha, matA, vecP, &beta, vecAp,
+                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
+        float pAp, old_rho;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
+        a = rho / pAp;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
+        na = -a;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &na, d_Ap, 1, d_r, 1));
+        
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r_norm_sq));
+        if (std::sqrt(r_norm_sq) < tolerance) break;
+
+        elementwise_mul_kernel<<<blocks, threads, 0, stream>>>(n, d_Dinv, d_r, d_z);
+        old_rho = rho;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_z, 1, &rho));
+        b = rho / old_rho;
+        
+        // p = z + beta * p
+        CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_z, 1, d_p, 1));
+        k++;
+    }
+
+    CHECK_CUSPARSE(cusparseDestroySpMat(matA));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
+    CHECK_CUDA(cudaFreeAsync(d_val, stream));
+    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
+    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
+    CHECK_CUDA(cudaFreeAsync(d_x, stream));
+    CHECK_CUDA(cudaFreeAsync(d_r, stream));
+    CHECK_CUDA(cudaFreeAsync(d_p, stream));
+    CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
+    CHECK_CUDA(cudaFreeAsync(d_b, stream));
+    CHECK_CUDA(cudaFreeAsync(d_z, stream));
+    CHECK_CUDA(cudaFreeAsync(d_Dinv, stream));
+    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
+    return k;
+}
+
+int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
+                    const MatrixTask& task, cudaStream_t stream) {
+    int n = (int)task.data->row_ptr.size() - 1;
+    float alpha = 1.0f, beta = 0.0f, r1 = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
+    float tolerance = 1e-5f;
+    int max_iters = 16000;
+
+    float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b;
+    int *d_row_ptr, *d_col_ind;
+
+    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
+
+    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
+
+    CHECK_CUBLAS(cublasSetStream(cublas, stream));
+    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
+
+    cusparseSpMatDescr_t matA;
+    cusparseDnVecDescr_t vecX, vecP, vecAp;
+
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val,
+                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                     CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecAp, n, d_Ap, CUDA_R_32F));
+
+    size_t bufferSize = 0;
+    void* d_buffer = nullptr;
+    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, vecP, &beta, vecAp,
+                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
+
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_p, 1));
+    CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
+
+    int k = 0;
+    while (k < max_iters) {
+        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                    &alpha, matA, vecP, &beta, vecAp,
+                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
+        float pAp, r0;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
+        a = r1 / pAp;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
+        na = -a;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &na, d_Ap, 1, d_r, 1));
+        r0 = r1;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
+        if (std::sqrt(r1) < tolerance) break;
+        b = r1 / r0;
+        CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_r, 1, d_p, 1));
+        k++;
+    }
+
+    CHECK_CUSPARSE(cusparseDestroySpMat(matA));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecX));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
+    CHECK_CUDA(cudaFreeAsync(d_val, stream));
+    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
+    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
+    CHECK_CUDA(cudaFreeAsync(d_x, stream));
+    CHECK_CUDA(cudaFreeAsync(d_r, stream));
+    CHECK_CUDA(cudaFreeAsync(d_p, stream));
+    CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
+    CHECK_CUDA(cudaFreeAsync(d_b, stream));
+    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
+    return k;
+}
+
+void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue,
+                       std::atomic<int>& succeeded, std::atomic<int>& failed) {
+    CHECK_CUDA(cudaSetDevice(device_id));
+    cudaStream_t stream;
+    cublasHandle_t cublas;
+    cusparseHandle_t cusparse;
+    CHECK_CUDA(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
+    CHECK_CUBLAS(cublasCreate(&cublas));
+    CHECK_CUSPARSE(cusparseCreate(&cusparse));
+
+    MatrixTask task;
+    while (queue.wait_pop(task)) {
+        auto pick_time = std::chrono::steady_clock::now();
+        std::cout << "[WORKER " << worker_id << "] Starting: " << task.path << " (NNZ: " << task.data->nnz << ")" << std::endl;
+
+        int iterations = solve_cg_async(cublas, cusparse, task, stream);
+        CHECK_CUDA(cudaStreamSynchronize(stream));
+
+        auto finish_time = std::chrono::steady_clock::now();
+        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish_time - pick_time).count();
+
+        bool success = (iterations >= 0 && iterations < MAX_ITERATIONS);
+        if (success) {
+            succeeded++;
+        } else {
+            failed++;
+        }
+
+        record_job(task.path, task.enqueue_time, pick_time, finish_time, iterations, success);
+
+        std::cout << "[WORKER " << worker_id << "] Done: " << task.path << " (" << iterations << " iters, " << duration << " ms)." << std::endl;
+    }
+    CHECK_CUBLAS(cublasDestroy(cublas));
+    CHECK_CUSPARSE(cusparseDestroy(cusparse));
+    CHECK_CUDA(cudaStreamDestroy(stream));
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.hpp
new file mode 100644
index 0000000000..b9de530986
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.hpp
@@ -0,0 +1,95 @@
+#pragma once
+
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+#include <cusparse.h>
+#include <iostream>
+#include <queue>
+#include <mutex>
+#include <condition_variable>
+#include <atomic>
+#include "sparse_utils.hpp"
+
+constexpr int MAX_ITERATIONS = 16000;
+
+// ============================================================
+// Error Checking Macros
+// ============================================================
+
+#define CHECK_CUDA(call)                                                   \
+    do {                                                                   \
+        cudaError_t status = call;                                         \
+        if (status != cudaSuccess) {                                       \
+            std::cerr << "CUDA Error: " << cudaGetErrorString(status)      \
+                      << " at " << __FILE__ << ":" << __LINE__ << std::endl; \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+#define CHECK_CUBLAS(call)                                                 \
+    do {                                                                   \
+        cublasStatus_t status = call;                                      \
+        if (status != CUBLAS_STATUS_SUCCESS) {                             \
+            std::cerr << "cuBLAS Error at "                               \
+                      << __FILE__ << ":" << __LINE__ << std::endl;         \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+#define CHECK_CUSPARSE(call)                                               \
+    do {                                                                   \
+        cusparseStatus_t status = call;                                    \
+        if (status != CUSPARSE_STATUS_SUCCESS) {                           \
+            std::cerr << "cuSPARSE Error at "                             \
+                      << __FILE__ << ":" << __LINE__ << std::endl;         \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+// ============================================================
+// Thread Safe Queue
+// ============================================================
+
+template<typename T>
+class ThreadSafeQueue {
+public:
+    void push(T item) {
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            queue_.push(std::move(item));
+        }
+        cv_.notify_one();
+    }
+
+    bool wait_pop(T& item) {
+        std::unique_lock<std::mutex> lock(mutex_);
+        cv_.wait(lock, [&] {
+            return shutdown_ || !queue_.empty();
+        });
+
+        if (!queue_.empty()) {
+            item = std::move(queue_.front());
+            queue_.pop();
+            return true;
+        }
+        return false;
+    }
+
+    void signal_shutdown() {
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            shutdown_ = true;
+        }
+        cv_.notify_all();
+    }
+
+private:
+    std::queue<T> queue_;
+    std::mutex mutex_;
+    std::condition_variable cv_;
+    bool shutdown_ = false;
+};
+
+int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream);
+int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream);
+void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue, std::atomic<int>& succeeded, std::atomic<int>& failed);
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/sparse_utils.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/sparse_utils.cpp
new file mode 100644
index 0000000000..dd09e63fc3
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/sparse_utils.cpp
@@ -0,0 +1,267 @@
+#include "sparse_utils.hpp"
+#include <fstream>
+#include <stdexcept>
+#include <filesystem>
+#include <random>
+#include <mutex>
+#include <algorithm>
+#include <iomanip>
+#include <iostream>
+#include <numeric>
+
+namespace fs = std::filesystem;
+
+SparseMatrixCOO load_binary_coo(const std::string& filepath) {
+    std::ifstream file(filepath, std::ios::binary);
+    if (!file) {
+        throw std::runtime_error("Failed to open matrix file: " + filepath);
+    }
+
+    SparseMatrixCOO coo;
+    
+    file.read(reinterpret_cast<char*>(&coo.rows), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&coo.cols), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&coo.nnz), sizeof(int32_t));
+
+    coo.row_indices.resize(coo.nnz);
+    coo.col_indices.resize(coo.nnz);
+    coo.values.resize(coo.nnz);
+
+    file.read(reinterpret_cast<char*>(coo.row_indices.data()), coo.nnz * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(coo.col_indices.data()), coo.nnz * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(coo.values.data()), coo.nnz * sizeof(float));
+
+    return coo;
+}
+
+SparseMatrixCSR convert_coo_to_csr(const SparseMatrixCOO& coo) {
+    SparseMatrixCSR csr;
+    csr.rows = coo.rows;
+    csr.cols = coo.cols;
+    csr.nnz = coo.nnz;
+    
+    csr.row_ptr.assign(csr.rows + 1, 0);
+    csr.col_indices.resize(csr.nnz);
+    csr.values.resize(csr.nnz);
+
+    for (int32_t i = 0; i < coo.nnz; ++i) {
+        csr.row_ptr[coo.row_indices[i] + 1]++;
+    }
+
+    for (int32_t i = 0; i < csr.rows; ++i) {
+        csr.row_ptr[i + 1] += csr.row_ptr[i];
+    }
+
+    std::vector<int32_t> current_row_pos = csr.row_ptr;
+
+    for (int32_t i = 0; i < coo.nnz; ++i) {
+        int32_t row = coo.row_indices[i];
+        int32_t dest_pos = current_row_pos[row]++;
+        csr.col_indices[dest_pos] = coo.col_indices[i];
+        csr.values[dest_pos] = coo.values[i];
+    }
+
+    return csr;
+}
+
+std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x) {
+    std::vector<float> b(A.rows, 0.0f);
+    
+    for (int32_t i = 0; i < A.rows; ++i) {
+        float sum = 0.0f;
+        int32_t row_start = A.row_ptr[i];
+        int32_t row_end = A.row_ptr[i + 1];
+        
+        for (int32_t j = row_start; j < row_end; ++j) {
+            sum += A.values[j] * x[A.col_indices[j]];
+        }
+        b[i] = sum;
+    }
+    return b;
+}
+
+std::vector<MatrixTask> scan_for_matrices(const std::string& dir, SolverType type) {
+    std::vector<MatrixTask> tasks;
+    if (!fs::exists(dir)) return tasks;
+    for (const auto& entry : fs::directory_iterator(dir)) {
+        if (entry.path().extension() == ".bin") {
+            auto coo = load_binary_coo(entry.path().string());
+            auto csr = convert_coo_to_csr(coo);
+            auto data = std::make_shared<MatrixData>();
+            data->rows = csr.rows;
+            data->cols = csr.cols;
+            data->nnz = csr.nnz;
+            data->b = compute_rhs_spmv(csr, std::vector<float>(csr.cols, 1.0f));
+            data->row_ptr = std::move(csr.row_ptr);
+            data->col_indices = std::move(csr.col_indices);
+            data->values = std::move(csr.values);
+            data->x_guess.assign(data->cols, 0.0f);
+            
+            tasks.push_back({entry.path().string(), type, data});
+        }
+    }
+    return tasks;
+}
+
+
+std::vector<Partition>
+make_contiguous_partitions(size_t num_tasks, size_t rows, size_t cols)
+{
+    size_t num_parts = rows * cols;
+
+    std::vector<Partition> parts;
+    parts.reserve(num_parts);
+
+    size_t base = num_tasks / num_parts;
+    size_t rem  = num_tasks % num_parts;
+
+    size_t current = 0;
+
+    for (size_t p = 0; p < num_parts; ++p) {
+        size_t size = base + (p < rem ? 1 : 0);
+
+        parts.push_back({
+            current,
+            current + size
+        });
+
+        current += size;
+    }
+
+    return parts;
+}
+
+
+int generate_random_sleep_ms(int min_ms, int max_ms) {
+    static std::random_device rd;
+    static std::mt19937 gen(rd());
+    std::uniform_int_distribution<> dis(min_ms, max_ms);
+    return dis(gen);
+}
+
+std::chrono::milliseconds generate_random_interval(
+    std::mt19937& rng,
+    double mean_ms)
+{
+    std::exponential_distribution<double> dist(
+        1.0 / mean_ms);
+
+    return std::chrono::milliseconds(
+        static_cast<int>(dist(rng)));
+}
+
+std::vector<MatrixTask> generate_batch(
+    const std::vector<MatrixTask>& matrix_pool,
+    std::mt19937& rng,
+    size_t batch_size)
+{
+    std::vector<MatrixTask> batch;
+    batch.reserve(batch_size);
+
+    std::uniform_int_distribution<size_t> dist(
+        0,
+        matrix_pool.size() - 1);
+
+    for (size_t i = 0; i < batch_size; ++i) {
+        batch.push_back(matrix_pool[dist(rng)]);
+    }
+
+    return batch;
+}
+
+static std::vector<JobStats> global_stats;
+static std::mutex stats_mutex;
+static std::chrono::steady_clock::time_point benchmark_start_tp;
+
+void init_benchmark_timer() {
+    benchmark_start_tp = std::chrono::steady_clock::now();
+}
+
+void record_job(const std::string& name,
+                std::chrono::steady_clock::time_point enqueue_time,
+                std::chrono::steady_clock::time_point pick_time,
+                std::chrono::steady_clock::time_point finish_time,
+                int iterations, bool success) {
+    std::lock_guard<std::mutex> lock(stats_mutex);
+    
+    auto wait = std::chrono::duration<double, std::milli>(pick_time - enqueue_time).count();
+    auto total = std::chrono::duration<double, std::milli>(finish_time - enqueue_time).count();
+    auto finish_rel = std::chrono::duration<double, std::milli>(finish_time - benchmark_start_tp).count();
+
+    global_stats.push_back({name, wait, total, iterations, success, finish_rel});
+}
+
+void report_workload_stats() {
+    std::lock_guard<std::mutex> lock(stats_mutex);
+    if (global_stats.empty()) {
+        std::cout << "No job statistics recorded.\n";
+        return;
+    }
+
+    int total_iters = 0;
+    int success_count = 0;
+    double wasted_gpu_time_ms = 0;
+    int failed_count = 0;
+    std::vector<double> completions;
+    completions.reserve(global_stats.size());
+
+    // Ensure total_jobs is not zero to avoid division by zero
+    size_t total_jobs = global_stats.size();
+
+    for (const auto& s : global_stats) {
+        total_iters += s.iterations;
+        if (s.success) success_count++;
+        else wasted_gpu_time_ms += (s.completion_time_ms - s.wait_time_ms);
+        
+        completions.push_back(s.completion_time_ms);
+    }
+
+    std::sort(completions.begin(), completions.end());
+    failed_count = total_jobs - success_count;
+    double success_percentage = (total_jobs > 0) ? (100.0 * success_count / total_jobs) : 0.0;
+    double failed_percentage = (total_jobs > 0) ? (100.0 * failed_count / total_jobs) : 0.0;
+    double mean = (total_jobs > 0) ? std::accumulate(completions.begin(), completions.end(), 0.0) / total_jobs : 0.0;
+    double median = (total_jobs > 0) ? completions[total_jobs / 2] : 0.0;
+    double p95 = completions[static_cast<size_t>(completions.size() * 0.95)];
+
+    auto max_finish = std::max_element(global_stats.begin(), global_stats.end(), [](const JobStats& a, const JobStats& b) {
+        return a.finish_relative_ms < b.finish_relative_ms;
+    });
+
+    std::cout << "\n" << std::string(45, '=') << "\n";
+    std::cout << "          WORKLOAD PERFORMANCE REPORT\n";
+    std::cout << std::string(45, '=') << "\n";
+    std::cout << std::left << std::setw(25) << "Total Jobs:" << total_jobs << "\n";
+    std::cout << std::left << std::setw(25) << "Succeeded Jobs:" << success_count << " (" << std::fixed << std::setprecision(2) << success_percentage << "%)\n";
+    std::cout << std::left << std::setw(25) << "Failed Jobs:" << failed_count << " (" << std::fixed << std::setprecision(2) << failed_percentage << "%)\n";
+    std::cout << std::left << std::setw(25) << "Total Iterations:" << total_iters << "\n";
+    std::cout << std::left << std::setw(25) << "Cumul. Wasted GPU Time:" << wasted_gpu_time_ms / 1000.0 << " s (all streams)\n";
+    std::cout << std::left << std::setw(25) << "Makespan:" << max_finish->finish_relative_ms / 1000.0 << " s\n";
+    std::cout << std::left << std::setw(25) << "Mean Completion:" << mean << " ms\n";
+    std::cout << std::left << std::setw(25) << "Median Completion:" << median << " ms\n";
+    std::cout << std::left << std::setw(25) << "95th Percentile:" << p95 << " ms\n";
+    
+    std::cout << "\nThroughput Timeline (Job Completion & Success vs Wall-clock):\n";
+    std::sort(global_stats.begin(), global_stats.end(), [](const JobStats& a, const JobStats& b) {
+        return a.finish_relative_ms < b.finish_relative_ms;
+    });
+    
+    double total_time = max_finish->finish_relative_ms;
+    for (int i = 1; i <= 10; ++i) {
+        double threshold = (total_time / 10.0) * i;
+        auto it = std::upper_bound(global_stats.begin(), global_stats.end(), threshold, 
+            [](double val, const JobStats& s) {
+                return val < s.finish_relative_ms;
+            });
+        size_t total_at_t = std::distance(global_stats.begin(), it);
+        size_t success_at_t = 0;
+        for (auto s_it = global_stats.begin(); s_it != it; ++s_it) {
+            if (s_it->success) success_at_t++;
+        }
+
+        std::cout << "  T + " << std::setw(5) << std::fixed << std::setprecision(0) << threshold 
+                  << " ms | Total Progress: " << std::setw(3) << (100 * total_at_t / total_jobs) 
+                  << "% | Successful solves: " << std::setw(3) << (100 * success_at_t / total_jobs) << "%\n";
+    }
+    std::cout << std::string(45, '=') << "\n\n";
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/sparse_utils.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/sparse_utils.hpp
new file mode 100644
index 0000000000..eef98a0456
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/sparse_utils.hpp
@@ -0,0 +1,94 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <cstdint>
+#include <memory>
+#include <random>
+#include <chrono>
+
+enum SolverType { CGS_SOLVER, BICSTAB_SOLVER };
+
+// Structure to hold raw data from the binary file
+struct SparseMatrixCOO {
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+    std::vector<int32_t> row_indices;
+    std::vector<int32_t> col_indices;
+    std::vector<float> values;
+};
+
+// Structure optimized for high-performance solvers
+struct SparseMatrixCSR {
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+    std::vector<int32_t> row_ptr;    // Size: rows + 1
+    std::vector<int32_t> col_indices;// Size: nnz
+    std::vector<float> values;       // Size: nnz
+};
+
+struct MatrixData {
+    std::vector<int32_t> row_ptr;
+    std::vector<int32_t> col_indices;
+    std::vector<float> values;
+    std::vector<float> b;
+    std::vector<float> x_guess;
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+};
+
+struct MatrixTask {
+    std::string path;
+    SolverType type;
+    std::shared_ptr<MatrixData> data;
+    std::chrono::steady_clock::time_point enqueue_time;
+};
+
+struct JobStats {
+    std::string task_name;
+    double wait_time_ms;       // Time spent in queue
+    double completion_time_ms; // Total turnaround time (enqueue to finish)
+    int iterations;
+    bool success;
+    double finish_relative_ms; // Wall-clock timestamp relative to benchmark start
+};
+
+void init_benchmark_timer();
+void record_job(const std::string& name,
+                std::chrono::steady_clock::time_point enqueue_time,
+                std::chrono::steady_clock::time_point pick_time,
+                std::chrono::steady_clock::time_point finish_time,
+                int iterations, bool success);
+void report_workload_stats();
+
+struct Partition {
+    size_t begin;
+    size_t end;
+    std::vector<int> devices;
+    std::vector<int> streams;
+};
+
+// Function to slurp the binary data into memory
+SparseMatrixCOO load_binary_coo(const std::string& filepath);
+
+// Converts COO to CSR format for solver compatibility
+SparseMatrixCSR convert_coo_to_csr(const SparseMatrixCOO& coo);
+
+// Compute b = A * x using CSR layout (Sparse Matrix-Vector Multiplication)
+std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x);
+
+std::vector<MatrixTask> scan_for_matrices(const std::string& dir, SolverType type);
+int generate_random_sleep_ms(int min_ms, int max_ms);
+
+// Workload generation helpers
+std::chrono::milliseconds generate_random_interval(
+    std::mt19937& rng,
+    double mean_ms);
+
+std::vector<MatrixTask> generate_batch(
+    const std::vector<MatrixTask>& matrix_pool,
+    std::mt19937& rng,
+    size_t batch_size);
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/stability_kernels.cu b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/stability_kernels.cu
new file mode 100644
index 0000000000..56d38d3694
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/stability_kernels.cu
@@ -0,0 +1,10 @@
+extern "C" __global__
+void check_stability(int n, const float* x, const float* r, int* err) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx < n) {
+        // If any value in the solution or residual is invalid, set the error flag
+        if (isnan(x[idx]) || isinf(x[idx]) || isnan(r[idx]) || isinf(r[idx])) {
+            atomicExch(err, 1);
+        }
+    }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp
new file mode 100644
index 0000000000..dc68991c36
--- /dev/null
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp
@@ -0,0 +1,224 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <deque>
+#include <unordered_map>
+#include <vector>
+#include <chrono>
+#include "atoms.hpp"
+#include "ft_cg_actor.hpp"
+#include "sparse_utils.hpp"
+
+
+using namespace caf;
+using namespace caf::cuda;
+// ---------------------------- SUPERVISOR ACTOR ----------------------------
+
+struct suspended_task {
+  caf::actor solver;
+  std::string path;
+  int last_batch_size;
+  int device_id;
+  int stream_id;
+};
+
+struct resource_slot {
+  int device_id;
+  int stream_id;
+};
+
+struct supervisor_state {
+  std::deque<MatrixTask> queue;
+  std::deque<suspended_task> suspended_queue;
+  std::vector<caf::actor> active_solvers;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> start_times;
+  std::unordered_map<std::string, resource_slot> task_resources;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> enqueue_times;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> pick_times;
+  std::unordered_map<std::string, double> cumulative_active_ms;
+  std::unordered_map<caf::actor, int> actor_batch_sizes;
+  std::deque<resource_slot> available_slots;
+  int max_active = 1; // Admission control limit to be mindful of GPU memory.
+  int num_iterations = MAX_ITERATIONS / 2;
+  int num_gpus = 0;
+  int tasks_succeeded = 0;
+  int tasks_failed = 0;
+  std::chrono::steady_clock::time_point benchmark_start;
+};
+
+behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks, int initial_max_active, std::chrono::steady_clock::time_point start_time) {
+    auto& st = self->state();
+    st.queue.insert(st.queue.end(), std::make_move_iterator(tasks.begin()), std::make_move_iterator(tasks.end()));
+    st.max_active = initial_max_active;
+    st.benchmark_start = start_time;
+    st.num_gpus = manager::get().get_num_devices();
+
+    // Initialize the pool with streams interleaved across all available GPUs
+    for (int s = 0; s < 32; ++s) {
+      for (int g = 0; g < st.num_gpus; ++g) {
+        st.available_slots.push_back({g, s});
+      }
+    }
+
+    auto spawn_next = [self]() {
+        auto& s = self->state();
+        while (s.active_solvers.size() < static_cast<size_t>(s.max_active) && !s.available_slots.empty()) {
+          if (!s.queue.empty()) {
+            auto task = std::move(s.queue.front());
+            s.queue.pop_front();
+            std::string path = task.path;
+
+            s.cumulative_active_ms[path] = 0;
+            s.enqueue_times[path] = task.enqueue_time;
+            s.pick_times[path] = std::chrono::steady_clock::now();
+
+            resource_slot slot = s.available_slots.front();
+            s.available_slots.pop_front();
+
+            self->println("[INFO] Starting solver for: {} (Device: {}, Stream: {})", 
+                          path, slot.device_id, slot.stream_id);
+            auto solver = self->spawn(fault_tolerant_cg_actor<float>,
+                                    path,
+                                    task.data,
+                                    create_in_arg(task.data->row_ptr),
+                                    create_in_arg(task.data->col_indices),
+                                    create_in_arg(task.data->values),
+                                    create_in_arg(task.data->b),
+                                    create_in_out_arg(task.data->x_guess),
+                                    (int)task.data->row_ptr.size() - 1,
+                                    (int)task.data->values.size(),
+                                    1e-5f, MAX_ITERATIONS, slot.device_id, slot.stream_id, actor_cast<actor>(self));
+
+            s.start_times[path] = std::chrono::steady_clock::now();
+            s.active_solvers.push_back(solver);
+            s.task_resources[path] = slot;
+            s.actor_batch_sizes[solver] = s.num_iterations;
+            self->mail(start_atom_v).send(solver);
+          } else {
+            bool resumed = false;
+            for (auto it = s.suspended_queue.begin(); it != s.suspended_queue.end(); ++it) {
+              auto slot_it = std::find_if(s.available_slots.begin(), s.available_slots.end(), [&](const resource_slot& slot) {
+                return slot.device_id == it->device_id && slot.stream_id == it->stream_id;
+              });
+              if (slot_it != s.available_slots.end()) {
+                auto suspended = std::move(*it);
+                s.suspended_queue.erase(it);
+                resource_slot slot = *slot_it;
+                s.available_slots.erase(slot_it);
+
+                s.pick_times[suspended.path] = std::chrono::steady_clock::now();
+
+                // Cap the minimum batch size to MAX_ITERATIONS / 8
+                int next_batch = std::max(MAX_ITERATIONS / 8, suspended.last_batch_size / 2);
+                self->println("[INFO] Resuming solver for: {} (Device: {}, Stream: {}, Batch: {})", 
+                              suspended.path, slot.device_id, slot.stream_id, next_batch);
+
+                // Resume on the same stream ID. No update_stream_atom_v needed.
+                self->mail(cg_next_step_atom_v, next_batch).send(suspended.solver);
+
+                s.active_solvers.push_back(suspended.solver);
+                s.task_resources[suspended.path] = slot;
+                s.actor_batch_sizes[suspended.solver] = next_batch;
+                resumed = true;
+                break;
+              }
+            }
+            if (!resumed) break;
+          }
+        }
+    };
+
+    spawn_next();
+    
+    return {
+        [=](gpu_done_atom, const std::string& task_name, caf::actor solver, std::vector<float>& solution, solver_result_meta meta) {
+            auto& s = self->state();
+
+            if (meta.converged || meta.error_code != CG_SUCCESS) {
+                auto end_time = std::chrono::steady_clock::now();
+                auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
+                                  end_time - s.start_times[task_name]).count();
+
+                if (meta.converged && meta.iterations < MAX_ITERATIONS) {
+                    s.tasks_succeeded++;
+                    if (meta.iterations == 0)
+                        self->println("[DONE] {}: Initial guess satisfied tolerance ({} ms).", task_name, duration);
+                    else
+                        self->println("[DONE] {}: Converged in {} iterations ({} ms).", task_name, meta.iterations, duration);
+                } else {
+                    s.tasks_failed++;
+                    std::string reason = (meta.error_code == CG_SUCCESS)
+                                         ? "Maximum Iterations Reached"
+                                         : to_string(static_cast<cg_error_type>(meta.error_code));
+                    self->println("[FAIL] {}: {} (after {} iterations, {} ms).", 
+                                  task_name, reason,
+                                  meta.iterations,
+                                  duration);
+                }
+
+                // Final active slice
+                auto active_slice = std::chrono::duration<double, std::milli>(end_time - s.pick_times[task_name]).count();
+                s.cumulative_active_ms[task_name] += active_slice;
+
+                // We override pick_time in record_job to simulate a single continuous run that equals 
+                // the actual time spent on the GPU.
+                auto simulated_pick = end_time - std::chrono::duration_cast<std::chrono::steady_clock::duration>(
+                                                std::chrono::duration<double, std::milli>(s.cumulative_active_ms[task_name]));
+
+                record_job(task_name, s.enqueue_times[task_name], simulated_pick, end_time, meta.iterations, meta.converged);
+                s.enqueue_times.erase(task_name);
+                s.pick_times.erase(task_name);
+                s.cumulative_active_ms.erase(task_name);
+
+                auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
+                if (it != s.active_solvers.end())
+                    s.active_solvers.erase(it);
+                s.start_times.erase(task_name);
+                s.actor_batch_sizes.erase(solver);
+
+                // Reclaim the device/stream slot and put it back in the pool
+                auto res_it = s.task_resources.find(task_name);
+                if (res_it != s.task_resources.end()) {
+                    s.available_slots.push_back(res_it->second);
+                    s.task_resources.erase(res_it);
+                }
+
+                spawn_next();
+
+                if (s.active_solvers.empty() && s.queue.empty() && s.suspended_queue.empty()) {
+                    self->println("All tasks in the pool have been processed.");
+
+                    report_workload_stats();
+                    self->quit();
+                }
+            } else if (meta.iterations == 0) {
+                // Just finished initialization: trigger the first iteration batch immediately.
+                self->mail(cg_next_step_atom_v, s.num_iterations).send(solver);
+            } else {
+                // Not done: Suspend the actor to allow others to use the stream
+                auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
+                
+                // Record the time spent in this active slice before suspending
+                auto active_slice = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - s.pick_times[task_name]).count();
+                s.cumulative_active_ms[task_name] += active_slice;
+
+                if (it != s.active_solvers.end())
+                    s.active_solvers.erase(it);
+
+                resource_slot slot = s.task_resources[task_name];
+                s.available_slots.push_back(slot);
+                s.task_resources.erase(task_name);
+
+                int last_batch = s.actor_batch_sizes[solver];
+                s.suspended_queue.push_back({solver, task_name, last_batch, slot.device_id, slot.stream_id});
+
+                self->println("[INFO] Suspending solver for: {} (Reclaimed Device: {}, Stream: {})", 
+                              task_name, slot.device_id, slot.stream_id);
+
+                spawn_next();
+            }
+        }
+       
+    };
+}

From 88000c14dc0516cb65c60f23e21b92f2a2ba576d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 11:15:17 -0600
Subject: [PATCH 0908/1000] Added jacobi fallback.

---
 .../workload-A/ft_cg_actor.hpp                | 76 +++++++++++++++----
 1 file changed, 63 insertions(+), 13 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp
index 4fd5335cd8..60f3ebd23a 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp
@@ -24,7 +24,7 @@ struct ft_cg_state {
 
   // GPU Buffers
   mem_ptr<int> A_rp, A_ci, d_err;
-  mem_ptr<T> A_val, b, x, r, p, w, y_tmp;
+  mem_ptr<T> A_val, b, x, r, p, w, z, D_inv, y_tmp;
   mem_ptr<char> spmv_ws;
 
   // Config
@@ -37,11 +37,12 @@ struct ft_cg_state {
   // Supervision & Monitoring
   caf::actor supervisor;
   device_ptr d_ptr;
-  program_ptr stab_prog;
+  program_ptr stab_prog, diag_prog;
 
   T initial_rho = 0;
   T current_rho = 0;
   T old_rho = 0;
+  bool is_jacobi = false;
   bool initialized = false;
   std::shared_ptr<MatrixData> pinned_data;
 };
@@ -79,10 +80,13 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
       auto& mgr = manager::get();
       // Load stability kernel from file as requested
       st.stab_prog = mgr.create_program_from_cubin("stability_kernels.cubin", "check_stability", st.d_ptr);
+      st.diag_prog = mgr.create_program_from_cubin("jacobi_kernels.cubin", "extract_diag_inv", st.d_ptr);
 
       st.r = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
       st.p = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
       st.w = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.z = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.D_inv = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
       st.y_tmp = runner.transfer_memory(st.device_id, st.stream_id, out<T>(1));
       st.d_err = runner.transfer_memory(st.device_id, st.stream_id, out<int>(1));
       
@@ -121,18 +125,35 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
 
         if (st.iterations > 1) {
           T beta = st.current_rho / st.old_rho;
-          if constexpr (std::is_same_v<T, double>) {
-            st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.w);
-            st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
-            st.d_ptr->dcopy(st.stream_id, st.n, st.w, st.p);
+          if (st.is_jacobi) {
+            if constexpr (std::is_same_v<T, double>) {
+              st.d_ptr->dcopy(st.stream_id, st.n, st.z, st.w);
+              st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
+              st.d_ptr->dcopy(st.stream_id, st.n, st.w, st.p);
+            } else {
+              st.d_ptr->scopy(st.stream_id, st.n, st.z, st.w);
+              st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(beta), st.p, st.w);
+              st.d_ptr->scopy(st.stream_id, st.n, st.w, st.p);
+            }
           } else {
-            st.d_ptr->scopy(st.stream_id, st.n, st.r, st.w);
-            st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(beta), st.p, st.w);
-            st.d_ptr->scopy(st.stream_id, st.n, st.w, st.p);
+            if constexpr (std::is_same_v<T, double>) {
+              st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.w);
+              st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
+              st.d_ptr->dcopy(st.stream_id, st.n, st.w, st.p);
+            } else {
+              st.d_ptr->scopy(st.stream_id, st.n, st.r, st.w);
+              st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(beta), st.p, st.w);
+              st.d_ptr->scopy(st.stream_id, st.n, st.w, st.p);
+            }
           }
         } else {
-          if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.p);
-          else st.d_ptr->scopy(st.stream_id, st.n, st.r, st.p);
+          if (st.is_jacobi) {
+            if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.z, st.p);
+            else st.d_ptr->scopy(st.stream_id, st.n, st.z, st.p);
+          } else {
+            if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.p);
+            else st.d_ptr->scopy(st.stream_id, st.n, st.r, st.p);
+          }
         }
 
         st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.p, T{0}, st.w, st.spmv_ws);
@@ -155,8 +176,15 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
         }
 
         st.old_rho = st.current_rho;
-        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
-        else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+        if (st.is_jacobi) {
+          if constexpr (std::is_same_v<T, double>) st.d_ptr->d_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+          else st.d_ptr->s_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+          if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+          else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+        } else {
+          if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+          else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+        }
         st.current_rho = runner.copy_to_host(st.y_tmp)[0];
 
       }
@@ -197,6 +225,28 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
 
       if (code != CG_SUCCESS) converged = false;
 
+      // Fallback: if CG failed and we haven't tried Jacobi, transition and retry.
+      if (code != CG_SUCCESS && !st.is_jacobi) {
+        self->println("[INFO] Solver failed in standard mode ({}). Falling back to Jacobi for: {}", to_string(static_cast<cg_error_type>(code)), st.path);
+        st.is_jacobi = true;
+        st.iterations = 0;
+        st.strikes = 0;
+
+        // Setup Jacobi Preconditioning
+        nd_range range_diag((st.n + 255) / 256, 1, 1, 256, 1, 1);
+        st.d_ptr->launch_kernel_mem_ref(st.diag_prog->get_kernel(st.d_ptr->getId()), range_diag,
+                                        std::make_tuple(in<int>(st.n), st.A_rp, st.A_ci, st.A_val, st.D_inv), st.stream_id);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->d_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+        else st.d_ptr->s_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+        else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+        
+        st.current_rho = runner.copy_to_host(st.y_tmp)[0];
+        st.initial_rho = st.current_rho;
+        code = CG_SUCCESS;
+        converged = false;
+      }
+
       solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
       
       // Report current solution and metadata to the supervisor

From 2a7ea73f68a32ea97f17337866c61269f8c374d9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 11:39:53 -0600
Subject: [PATCH 0909/1000] made some changes to that if the actor decides to
 go jacobi, its iteration count will be reset

---
 .../workload-A/atoms.hpp                      |  4 +++-
 .../workload-A/ft_cg_actor.hpp                |  2 +-
 .../workload-A/supervisor_actor.hpp           | 20 ++++++++++++++++++-
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/atoms.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/atoms.hpp
index ab2a741b6c..4b6ccd5e2b 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/atoms.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/atoms.hpp
@@ -35,7 +35,8 @@ enum cg_error_type : int {
   CG_NAN_INF = 2,
   CG_STAGNATION = 3,
   CG_BREAKDOWN = 4,
-  CG_RESIDUAL_FACTOR_FAIL = 5
+  CG_RESIDUAL_FACTOR_FAIL = 5,
+  CG_JACOBI_RETRY = 6
 };
 
 inline std::string to_string(cg_error_type err) {
@@ -46,6 +47,7 @@ inline std::string to_string(cg_error_type err) {
     case CG_STAGNATION: return "Stagnation Detected (Residual stopped changing)";
     case CG_BREAKDOWN: return "Solver Breakdown (Division by zero/near-zero)";
     case CG_RESIDUAL_FACTOR_FAIL: return "Residual Factor Check Failed";
+    case CG_JACOBI_RETRY: return "Falling back to Jacobi Preconditioner";
     default: return "Unknown Error (" + std::to_string(static_cast<int>(err)) + ")";
   }
 }
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp
index 60f3ebd23a..c2b4eee1bd 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp
@@ -243,7 +243,7 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
         
         st.current_rho = runner.copy_to_host(st.y_tmp)[0];
         st.initial_rho = st.current_rho;
-        code = CG_SUCCESS;
+        code = CG_JACOBI_RETRY;
         converged = false;
       }
 
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp
index dc68991c36..3f80ba9ac8 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp
@@ -114,6 +114,12 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 self->println("[INFO] Resuming solver for: {} (Device: {}, Stream: {}, Batch: {})", 
                               suspended.path, slot.device_id, slot.stream_id, next_batch);
 
+                
+                // Resume on the same stream ID. No update_stream_atom_v needed.
+                self->mail(cg_next_step_atom_v, next_batch).send(suspended.solver);
+
+               
+
                 // Resume on the same stream ID. No update_stream_atom_v needed.
                 self->mail(cg_next_step_atom_v, next_batch).send(suspended.solver);
 
@@ -135,7 +141,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
         [=](gpu_done_atom, const std::string& task_name, caf::actor solver, std::vector<float>& solution, solver_result_meta meta) {
             auto& s = self->state();
 
-            if (meta.converged || meta.error_code != CG_SUCCESS) {
+            if (meta.converged || (meta.error_code != CG_SUCCESS && meta.error_code != CG_JACOBI_RETRY)) {
                 auto end_time = std::chrono::steady_clock::now();
                 auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
                                   end_time - s.start_times[task_name]).count();
@@ -211,6 +217,12 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 s.task_resources.erase(task_name);
 
                 int last_batch = s.actor_batch_sizes[solver];
+                
+                // If switching to Jacobi, set last_batch such that resume uses MAX_ITERATIONS
+                if (meta.error_code == CG_JACOBI_RETRY) {
+                    last_batch = MAX_ITERATIONS;
+                }
+
                 s.suspended_queue.push_back({solver, task_name, last_batch, slot.device_id, slot.stream_id});
 
                 self->println("[INFO] Suspending solver for: {} (Reclaimed Device: {}, Stream: {})", 
@@ -219,6 +231,12 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 spawn_next();
             }
         }
+        ,
+        [=](jacobi_fallback_atom, const std::string& task_name, caf::actor solver_actor) {
+            auto& s = self->state();
+            self->println("[INFO] Solver for {} (Actor: {}) fell back to Jacobi. Adjusting next batch size to initial value.", task_name, to_string(solver_actor));
+            s.actor_batch_sizes[solver_actor] = s.num_iterations; // Reset to initial batch size (MAX_ITERATIONS )
+        }
        
     };
 }

From 702b9a2444ab63163d10e6fbb4fdf1189ffaea2d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 11:50:14 -0600
Subject: [PATCH 0910/1000] Reverting changes.

---
 .../workload-A/ft_cg_actor.hpp                |  2 +-
 .../workload-A/supervisor_actor.hpp           | 20 +------------------
 2 files changed, 2 insertions(+), 20 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp
index c2b4eee1bd..60f3ebd23a 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/ft_cg_actor.hpp
@@ -243,7 +243,7 @@ behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
         
         st.current_rho = runner.copy_to_host(st.y_tmp)[0];
         st.initial_rho = st.current_rho;
-        code = CG_JACOBI_RETRY;
+        code = CG_SUCCESS;
         converged = false;
       }
 
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp
index 3f80ba9ac8..dc68991c36 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp
@@ -114,12 +114,6 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 self->println("[INFO] Resuming solver for: {} (Device: {}, Stream: {}, Batch: {})", 
                               suspended.path, slot.device_id, slot.stream_id, next_batch);
 
-                
-                // Resume on the same stream ID. No update_stream_atom_v needed.
-                self->mail(cg_next_step_atom_v, next_batch).send(suspended.solver);
-
-               
-
                 // Resume on the same stream ID. No update_stream_atom_v needed.
                 self->mail(cg_next_step_atom_v, next_batch).send(suspended.solver);
 
@@ -141,7 +135,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
         [=](gpu_done_atom, const std::string& task_name, caf::actor solver, std::vector<float>& solution, solver_result_meta meta) {
             auto& s = self->state();
 
-            if (meta.converged || (meta.error_code != CG_SUCCESS && meta.error_code != CG_JACOBI_RETRY)) {
+            if (meta.converged || meta.error_code != CG_SUCCESS) {
                 auto end_time = std::chrono::steady_clock::now();
                 auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
                                   end_time - s.start_times[task_name]).count();
@@ -217,12 +211,6 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 s.task_resources.erase(task_name);
 
                 int last_batch = s.actor_batch_sizes[solver];
-                
-                // If switching to Jacobi, set last_batch such that resume uses MAX_ITERATIONS
-                if (meta.error_code == CG_JACOBI_RETRY) {
-                    last_batch = MAX_ITERATIONS;
-                }
-
                 s.suspended_queue.push_back({solver, task_name, last_batch, slot.device_id, slot.stream_id});
 
                 self->println("[INFO] Suspending solver for: {} (Reclaimed Device: {}, Stream: {})", 
@@ -231,12 +219,6 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
                 spawn_next();
             }
         }
-        ,
-        [=](jacobi_fallback_atom, const std::string& task_name, caf::actor solver_actor) {
-            auto& s = self->state();
-            self->println("[INFO] Solver for {} (Actor: {}) fell back to Jacobi. Adjusting next batch size to initial value.", task_name, to_string(solver_actor));
-            s.actor_batch_sizes[solver_actor] = s.num_iterations; // Reset to initial batch size (MAX_ITERATIONS )
-        }
        
     };
 }

From 150a01775ed9c4c43247f58c1d79874e603492ed Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 12:00:39 -0600
Subject: [PATCH 0911/1000] added jacobi fallback mechanism

---
 .../workload-A/atoms.hpp                             |  4 +---
 .../workload-A/native_utils.cu                       | 12 +++++++++++-
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/atoms.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/atoms.hpp
index 4b6ccd5e2b..ab2a741b6c 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/atoms.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/atoms.hpp
@@ -35,8 +35,7 @@ enum cg_error_type : int {
   CG_NAN_INF = 2,
   CG_STAGNATION = 3,
   CG_BREAKDOWN = 4,
-  CG_RESIDUAL_FACTOR_FAIL = 5,
-  CG_JACOBI_RETRY = 6
+  CG_RESIDUAL_FACTOR_FAIL = 5
 };
 
 inline std::string to_string(cg_error_type err) {
@@ -47,7 +46,6 @@ inline std::string to_string(cg_error_type err) {
     case CG_STAGNATION: return "Stagnation Detected (Residual stopped changing)";
     case CG_BREAKDOWN: return "Solver Breakdown (Division by zero/near-zero)";
     case CG_RESIDUAL_FACTOR_FAIL: return "Residual Factor Check Failed";
-    case CG_JACOBI_RETRY: return "Falling back to Jacobi Preconditioner";
     default: return "Unknown Error (" + std::to_string(static_cast<int>(err)) + ")";
   }
 }
\ No newline at end of file
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
index 22a426f494..5e1beba8f1 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
@@ -228,10 +228,20 @@ void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>
         int iterations = solve_cg_async(cublas, cusparse, task, stream);
         CHECK_CUDA(cudaStreamSynchronize(stream));
 
+        bool success = (iterations >= 0 && iterations < MAX_ITERATIONS);
+
+        // Fallback mechanism: If standard CG fails to converge, retry using the Jacobi Preconditioner
+        if (!success) {
+            std::cout << "[WORKER " << worker_id << "] CG reached max iterations (" << iterations 
+                      << "). Falling back to Jacobi solver for: " << task.path << std::endl;
+            iterations = solve_pcg_jacobi_async(cublas, cusparse, task, stream);
+            CHECK_CUDA(cudaStreamSynchronize(stream));
+            success = (iterations >= 0 && iterations < MAX_ITERATIONS);
+        }
+
         auto finish_time = std::chrono::steady_clock::now();
         auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish_time - pick_time).count();
 
-        bool success = (iterations >= 0 && iterations < MAX_ITERATIONS);
         if (success) {
             succeeded++;
         } else {

From 114f26e82f01aab8dc94e06da4685915df951aab Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 12:18:31 -0600
Subject: [PATCH 0912/1000] updated worker to keep memory on device if it uses
 jacobi fallback

---
 .../workload-A/native_utils.cu                | 104 ++++++++----------
 1 file changed, 44 insertions(+), 60 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
index 5e1beba8f1..72f8f4b695 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
@@ -26,30 +26,13 @@ __global__ void elementwise_mul_kernel(int n, const float* a, const float* b, fl
 }
 
 int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
-                           const MatrixTask& task, cudaStream_t stream) {
-    int n = (int)task.data->row_ptr.size() - 1;
+                           int n, int nnz, float* d_val, int* d_row_ptr, int* d_col_ind,
+                           float* d_b, float* d_x, float* d_r, float* d_p, float* d_Ap,
+                           float* d_z, float* d_Dinv, cudaStream_t stream) {
     float alpha = 1.0f, beta = 0.0f, rho = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
     float tolerance = 1e-5f;
     int max_iters = 16000;
 
-    float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b, *d_z, *d_Dinv;
-    int *d_row_ptr, *d_col_ind;
-
-    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_z, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_Dinv, n * sizeof(float), stream));
-
-    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
     CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
 
     CHECK_CUBLAS(cublasSetStream(cublas, stream));
@@ -62,7 +45,7 @@ int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
 
     cusparseSpMatDescr_t matA;
     cusparseDnVecDescr_t vecP, vecAp;
-    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val,
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, nnz, d_row_ptr, d_col_ind, d_val,
                                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
                                      CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
     CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
@@ -111,43 +94,17 @@ int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
     CHECK_CUSPARSE(cusparseDestroySpMat(matA));
     CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
     CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
-    CHECK_CUDA(cudaFreeAsync(d_val, stream));
-    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
-    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
-    CHECK_CUDA(cudaFreeAsync(d_x, stream));
-    CHECK_CUDA(cudaFreeAsync(d_r, stream));
-    CHECK_CUDA(cudaFreeAsync(d_p, stream));
-    CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
-    CHECK_CUDA(cudaFreeAsync(d_b, stream));
-    CHECK_CUDA(cudaFreeAsync(d_z, stream));
-    CHECK_CUDA(cudaFreeAsync(d_Dinv, stream));
     CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
     return k;
 }
 
 int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
-                    const MatrixTask& task, cudaStream_t stream) {
-    int n = (int)task.data->row_ptr.size() - 1;
+                    int n, int nnz, float* d_val, int* d_row_ptr, int* d_col_ind,
+                    float* d_b, float* d_x, float* d_r, float* d_p, float* d_Ap, cudaStream_t stream) {
     float alpha = 1.0f, beta = 0.0f, r1 = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
     float tolerance = 1e-5f;
     int max_iters = 16000;
 
-    float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b;
-    int *d_row_ptr, *d_col_ind;
-
-    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
-    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
-
-    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
-    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
     CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
 
     CHECK_CUBLAS(cublasSetStream(cublas, stream));
@@ -156,7 +113,7 @@ int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
     cusparseSpMatDescr_t matA;
     cusparseDnVecDescr_t vecX, vecP, vecAp;
 
-    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val,
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, nnz, d_row_ptr, d_col_ind, d_val,
                                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
                                      CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
     CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
@@ -198,14 +155,6 @@ int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
     CHECK_CUSPARSE(cusparseDestroyDnVec(vecX));
     CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
     CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
-    CHECK_CUDA(cudaFreeAsync(d_val, stream));
-    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
-    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
-    CHECK_CUDA(cudaFreeAsync(d_x, stream));
-    CHECK_CUDA(cudaFreeAsync(d_r, stream));
-    CHECK_CUDA(cudaFreeAsync(d_p, stream));
-    CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
-    CHECK_CUDA(cudaFreeAsync(d_b, stream));
     CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
     return k;
 }
@@ -225,7 +174,30 @@ void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>
         auto pick_time = std::chrono::steady_clock::now();
         std::cout << "[WORKER " << worker_id << "] Starting: " << task.path << " (NNZ: " << task.data->nnz << ")" << std::endl;
 
-        int iterations = solve_cg_async(cublas, cusparse, task, stream);
+        int n = (int)task.data->row_ptr.size() - 1;
+        int nnz = task.data->nnz;
+        float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b, *d_z, *d_Dinv;
+        int *d_row_ptr, *d_col_ind;
+
+        // Allocate GPU memory once per task
+        CHECK_CUDA(cudaMallocAsync(&d_val, nnz * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_col_ind, nnz * sizeof(int), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_z, n * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_Dinv, n * sizeof(float), stream));
+
+        // Initial Transfer
+        CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
+        CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
+        CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
+        CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
+
+        int iterations = solve_cg_async(cublas, cusparse, n, nnz, d_val, d_row_ptr, d_col_ind, d_b, d_x, d_r, d_p, d_Ap, stream);
         CHECK_CUDA(cudaStreamSynchronize(stream));
 
         bool success = (iterations >= 0 && iterations < MAX_ITERATIONS);
@@ -234,11 +206,23 @@ void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>
         if (!success) {
             std::cout << "[WORKER " << worker_id << "] CG reached max iterations (" << iterations 
                       << "). Falling back to Jacobi solver for: " << task.path << std::endl;
-            iterations = solve_pcg_jacobi_async(cublas, cusparse, task, stream);
+            iterations = solve_pcg_jacobi_async(cublas, cusparse, n, nnz, d_val, d_row_ptr, d_col_ind, d_b, d_x, d_r, d_p, d_Ap, d_z, d_Dinv, stream);
             CHECK_CUDA(cudaStreamSynchronize(stream));
             success = (iterations >= 0 && iterations < MAX_ITERATIONS);
         }
 
+        // Clean up task memory
+        CHECK_CUDA(cudaFreeAsync(d_val, stream));
+        CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
+        CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
+        CHECK_CUDA(cudaFreeAsync(d_x, stream));
+        CHECK_CUDA(cudaFreeAsync(d_r, stream));
+        CHECK_CUDA(cudaFreeAsync(d_p, stream));
+        CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
+        CHECK_CUDA(cudaFreeAsync(d_b, stream));
+        CHECK_CUDA(cudaFreeAsync(d_z, stream));
+        CHECK_CUDA(cudaFreeAsync(d_Dinv, stream));
+
         auto finish_time = std::chrono::steady_clock::now();
         auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish_time - pick_time).count();
 

From c5e06ab8b5a8adcf7cdbdcea80ca178b222a0426 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 13:02:03 -0600
Subject: [PATCH 0913/1000] made changes to work stealing policy in an attempt
 to reduce thrashing

---
 .../cuda/control-layer/scheduler_actor.hpp    |  4 ++++
 .../src/control-layer/scheduler_actor.cpp     | 24 ++++++++++++++-----
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
index 4806c350b3..7749fa4570 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
@@ -2,6 +2,8 @@
 #include <caf/all.hpp>
 #include "caf/cuda/control-layer/token.hpp"
 #include <queue>
+#include <vector>
+#include <random>
 #include "caf/cuda/global_export.hpp"
 
 namespace caf::cuda {
@@ -32,6 +34,8 @@ class CAF_CUDA_EXPORT scheduler_actor : public caf::event_based_actor {
     int in_flight_ = 0;
     std::queue<token_ptr> queue_;
     std::vector<caf::actor> schedulers_;
+    std::vector<caf::actor> victims_;
+    std::mt19937 rng_;
     std::queue<int> available_streams_;
 };
 
diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 8c77dbe051..563e761309 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -2,6 +2,8 @@
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
 #include <iostream>
 
+#include <random> // Required for random number generation
+#include <algorithm> // Required for std::remove_if if filtering self from neighbors
 namespace caf::cuda {
 
 scheduler_actor::scheduler_actor(caf::actor_config& cfg, 
@@ -11,7 +13,8 @@ scheduler_actor::scheduler_actor(caf::actor_config& cfg,
       device_number_(device_number),
       num_streams_(num_streams),
       stream_depth_(stream_depth),
-      multi_gpu_(multi_gpu) {
+      multi_gpu_(multi_gpu),
+      rng_(std::random_device{}()) {
     // Initialize the pool of available execution slots
     for (int depth = 0; depth < stream_depth_; ++depth) {
         for (int s = 0; s < num_streams_; ++s) {
@@ -71,10 +74,18 @@ void scheduler_actor::on_reclaim(int /*val*/, int /*mem*/, int /*time*/, int /*d
 
 void scheduler_actor::on_set_neighbors(std::vector<caf::actor> neighbors) {
     schedulers_ = std::move(neighbors);
+    victims_.clear();
+    auto self_handle = neighbors[device_number_];
+    for (const auto& neighbor : schedulers_) {
+        if (neighbor != self_handle) {
+            victims_.push_back(neighbor);
+        }
+    }
 }
 
 std::vector<token_ptr> scheduler_actor::on_steal_request(int requesting_device) {
-    if (queue_.size() > 1) {
+    size_t min_giveaway_threshold = static_cast<size_t>(num_streams_) * stream_depth_ * 2;
+    if (queue_.size() > min_giveaway_threshold) {
         size_t to_give = queue_.size() / 2;
         std::vector<token_ptr> batch;
         for (size_t i = 0; i < to_give; ++i) {
@@ -112,10 +123,11 @@ void scheduler_actor::schedule_work() {
 
     // Work Stealing logic
     if (multi_gpu_ && queue_.empty() && in_flight_ < (capacity / 2)) {
-        for (auto& neighbor : schedulers_) {
-            if (neighbor != this) {
-                this->mail(device_number_).send(neighbor);
-            }
+        // Randomly select a neighbor to request work from using the pre-generated victims list
+        if (!victims_.empty()) {
+            std::uniform_int_distribution<size_t> distrib(0, victims_.size() - 1);
+            size_t idx = distrib(rng_);
+            this->mail(device_number_).send(victims_[idx]);
         }
     }
 }

From 2ed6371880ee53f101055b324fae4077f0358c4a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 13:07:53 -0600
Subject: [PATCH 0914/1000] fixed invalid memory access

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 563e761309..0f698eb018 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -75,7 +75,7 @@ void scheduler_actor::on_reclaim(int /*val*/, int /*mem*/, int /*time*/, int /*d
 void scheduler_actor::on_set_neighbors(std::vector<caf::actor> neighbors) {
     schedulers_ = std::move(neighbors);
     victims_.clear();
-    auto self_handle = neighbors[device_number_];
+    auto self_handle = caf::actor_cast<caf::actor>(this);
     for (const auto& neighbor : schedulers_) {
         if (neighbor != self_handle) {
             victims_.push_back(neighbor);

From 09f8b9ab41858462085c970247c96a7aa3b3e0e3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 13:23:09 -0600
Subject: [PATCH 0915/1000] updated giveaway threshold

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 0f698eb018..9009d14aef 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -84,7 +84,9 @@ void scheduler_actor::on_set_neighbors(std::vector<caf::actor> neighbors) {
 }
 
 std::vector<token_ptr> scheduler_actor::on_steal_request(int requesting_device) {
-    size_t min_giveaway_threshold = static_cast<size_t>(num_streams_) * stream_depth_ * 2;
+    // size_t min_giveaway_threshold = static_cast<size_t>(num_streams_) * stream_depth_ * 2;
+    size_t min_giveaway_threshold = static_cast<size_t>(num_streams_) * stream_depth_;
+
     if (queue_.size() > min_giveaway_threshold) {
         size_t to_give = queue_.size() / 2;
         std::vector<token_ptr> batch;

From 99370b543a077d408455e0920abc7d045bbdc679 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 10 Jun 2026 13:38:11 -0600
Subject: [PATCH 0916/1000] updated work stealing thresholds

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 9009d14aef..af41d22b10 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -124,7 +124,9 @@ void scheduler_actor::schedule_work() {
     }
 
     // Work Stealing logic
-    if (multi_gpu_ && queue_.empty() && in_flight_ < (capacity / 2)) {
+    // if (multi_gpu_ && queue_.empty() && in_flight_ < (capacity / 2)) {
+    if (multi_gpu_ && queue_.empty() && in_flight_ < (capacity )) {
+
         // Randomly select a neighbor to request work from using the pre-generated victims list
         if (!victims_.empty()) {
             std::uniform_int_distribution<size_t> distrib(0, victims_.size() - 1);

From 220439c8e59a4c84625e4d21c6e354d884803235 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Jun 2026 08:48:51 -0600
Subject: [PATCH 0917/1000] moved capacity back to what it was

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index af41d22b10..4dbee27f91 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -124,8 +124,8 @@ void scheduler_actor::schedule_work() {
     }
 
     // Work Stealing logic
-    // if (multi_gpu_ && queue_.empty() && in_flight_ < (capacity / 2)) {
-    if (multi_gpu_ && queue_.empty() && in_flight_ < (capacity )) {
+     if (multi_gpu_ && queue_.empty() && in_flight_ < (capacity / 2)) {
+    // if (multi_gpu_ && queue_.empty() && in_flight_ < (capacity )) {
 
         // Randomly select a neighbor to request work from using the pre-generated victims list
         if (!victims_.empty()) {

From c9fe798a20f78b43298f13b59eb9d114253b8ed1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Jun 2026 08:58:36 -0600
Subject: [PATCH 0918/1000] added error codes

---
 .../workload-A/native_utils.cu                | 138 ++++++++++++++++--
 1 file changed, 124 insertions(+), 14 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
index 72f8f4b695..1e10bfcc96 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
@@ -6,6 +6,45 @@
 // PCG Kernels
 // ============================================================
 
+enum cg_error_type {
+    CG_SUCCESS = 0,
+    CG_BREAKDOWN = 1,
+    CG_STAGNATION = 2,
+    CG_MAX_ITER = 3,
+    CG_RESIDUAL_FACTOR_FAIL = 4,
+    CG_NAN_INF = 5,
+    CG_JACOBI_RETRY = 6
+};
+
+const char* get_cg_error_string(int code) {
+    switch (code) {
+        case CG_SUCCESS:
+            return "CG_SUCCESS";
+        case CG_BREAKDOWN:
+            return "CG_BREAKDOWN";
+        case CG_STAGNATION:
+            return "CG_STAGNATION";
+        case CG_MAX_ITER:
+            return "CG_MAX_ITER";
+        case CG_RESIDUAL_FACTOR_FAIL:
+            return "CG_RESIDUAL_FACTOR_FAIL";
+        case CG_NAN_INF:
+            return "CG_NAN_INF";
+        case CG_JACOBI_RETRY:
+            return "CG_JACOBI_RETRY";
+        default:
+            return "UNKNOWN_ERROR";
+    }
+}
+__global__ void check_stability_kernel(int n, const float* x, const float* r, int* d_err) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        if (isnan(x[i]) || isinf(x[i]) || isnan(r[i]) || isinf(r[i])) {
+            atomicExch(d_err, 1);
+        }
+    }
+}
+
 __global__ void extract_diag_inv_kernel(int n, const int* row_ptr, const int* col_ind, const float* values, float* d_inv) {
     int i = blockIdx.x * blockDim.x + threadIdx.x;
     if (i < n) {
@@ -28,10 +67,11 @@ __global__ void elementwise_mul_kernel(int n, const float* a, const float* b, fl
 int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
                            int n, int nnz, float* d_val, int* d_row_ptr, int* d_col_ind,
                            float* d_b, float* d_x, float* d_r, float* d_p, float* d_Ap,
-                           float* d_z, float* d_Dinv, cudaStream_t stream) {
+                           float* d_z, float* d_Dinv, int* d_err, cudaStream_t stream, int& out_code) {
     float alpha = 1.0f, beta = 0.0f, rho = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
     float tolerance = 1e-5f;
     int max_iters = 16000;
+    out_code = CG_SUCCESS;
 
     CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
 
@@ -64,7 +104,8 @@ int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
     CHECK_CUBLAS(cublasScopy(cublas, n, d_z, 1, d_p, 1));
     CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_z, 1, &rho));
 
-    int k = 0;
+    float initial_rho = rho;
+    int k = 0; 
     float r_norm_sq = 0.0f;
     while (k < max_iters) {
         CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
@@ -72,6 +113,12 @@ int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
                                     CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
         float pAp, old_rho;
         CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
+        
+        if (std::abs(pAp) < 1e-25f) {
+            out_code = CG_BREAKDOWN;
+            break;
+        }
+
         a = rho / pAp;
         CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
         na = -a;
@@ -81,8 +128,9 @@ int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
         if (std::sqrt(r_norm_sq) < tolerance) break;
 
         elementwise_mul_kernel<<<blocks, threads, 0, stream>>>(n, d_Dinv, d_r, d_z);
-        old_rho = rho;
+        old_rho = rho; 
         CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_z, 1, &rho));
+
         b = rho / old_rho;
         
         // p = z + beta * p
@@ -91,6 +139,26 @@ int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
         k++;
     }
 
+    bool converged = (std::sqrt(r_norm_sq) < tolerance);
+
+    if (out_code == CG_SUCCESS) {
+        if (!converged) {
+            if (k >= max_iters) out_code = CG_MAX_ITER;
+            // Stagnation check: did the total progress over the entire solve stall?
+            else if (std::abs(initial_rho - rho) < 1e-14f) out_code = CG_STAGNATION;
+            // Residual decrease check: did it fail to drop by at least 0.01%?
+            else if (initial_rho > 0 && (rho / initial_rho) > 0.9999f) out_code = CG_RESIDUAL_FACTOR_FAIL;
+        }
+    }
+
+    // Stability check
+    CHECK_CUDA(cudaMemsetAsync(d_err, 0, sizeof(int), stream));
+    check_stability_kernel<<<blocks, threads, 0, stream>>>(n, d_x, d_r, d_err);
+    int h_err = 0;
+    CHECK_CUDA(cudaMemcpyAsync(&h_err, d_err, sizeof(int), cudaMemcpyDeviceToHost, stream));
+    CHECK_CUDA(cudaStreamSynchronize(stream));
+    if (h_err != 0 || std::isnan(rho) || std::isinf(rho)) out_code = CG_NAN_INF;
+
     CHECK_CUSPARSE(cusparseDestroySpMat(matA));
     CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
     CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
@@ -100,10 +168,12 @@ int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
 
 int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
                     int n, int nnz, float* d_val, int* d_row_ptr, int* d_col_ind,
-                    float* d_b, float* d_x, float* d_r, float* d_p, float* d_Ap, cudaStream_t stream) {
+                    float* d_b, float* d_x, float* d_r, float* d_p, float* d_Ap, 
+                    int* d_err, cudaStream_t stream, int& out_code) {
     float alpha = 1.0f, beta = 0.0f, r1 = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
     float tolerance = 1e-5f;
     int max_iters = 16000;
+    out_code = CG_SUCCESS;
 
     CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
 
@@ -131,13 +201,20 @@ int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
     CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_p, 1));
     CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
 
-    int k = 0;
+    float initial_rho = r1;
+    int k = 0; 
     while (k < max_iters) {
         CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
                                     &alpha, matA, vecP, &beta, vecAp,
                                     CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
         float pAp, r0;
         CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
+        
+        if (std::abs(pAp) < 1e-25f) {
+            out_code = CG_BREAKDOWN;
+            break;
+        }
+
         a = r1 / pAp;
         CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
         na = -a;
@@ -145,12 +222,32 @@ int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
         r0 = r1;
         CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
         if (std::sqrt(r1) < tolerance) break;
+        
         b = r1 / r0;
         CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
         CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_r, 1, d_p, 1));
         k++;
     }
 
+    bool converged = (std::sqrt(r1) < tolerance);
+
+    if (out_code == CG_SUCCESS) {
+        if (!converged) {
+            if (k >= max_iters) out_code = CG_MAX_ITER;
+            else if (std::abs(initial_rho - r1) < 1e-14f) out_code = CG_STAGNATION;
+            else if (initial_rho > 0 && (r1 / initial_rho) > 0.9999f) out_code = CG_RESIDUAL_FACTOR_FAIL;
+        }
+    }
+
+    // Stability check
+    CHECK_CUDA(cudaMemsetAsync(d_err, 0, sizeof(int), stream));
+    int threads = 256;
+    check_stability_kernel<<<(n + threads - 1) / threads, threads, 0, stream>>>(n, d_x, d_r, d_err);
+    int h_err = 0;
+    CHECK_CUDA(cudaMemcpyAsync(&h_err, d_err, sizeof(int), cudaMemcpyDeviceToHost, stream));
+    CHECK_CUDA(cudaStreamSynchronize(stream));
+    if (h_err != 0 || std::isnan(r1) || std::isinf(r1)) out_code = CG_NAN_INF;
+
     CHECK_CUSPARSE(cusparseDestroySpMat(matA));
     CHECK_CUSPARSE(cusparseDestroyDnVec(vecX));
     CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
@@ -177,7 +274,7 @@ void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>
         int n = (int)task.data->row_ptr.size() - 1;
         int nnz = task.data->nnz;
         float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b, *d_z, *d_Dinv;
-        int *d_row_ptr, *d_col_ind;
+        int *d_row_ptr, *d_col_ind, *d_err;
 
         // Allocate GPU memory once per task
         CHECK_CUDA(cudaMallocAsync(&d_val, nnz * sizeof(float), stream));
@@ -190,6 +287,7 @@ void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>
         CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
         CHECK_CUDA(cudaMallocAsync(&d_z, n * sizeof(float), stream));
         CHECK_CUDA(cudaMallocAsync(&d_Dinv, n * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_err, sizeof(int), stream));
 
         // Initial Transfer
         CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
@@ -197,18 +295,29 @@ void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>
         CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
         CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
 
-        int iterations = solve_cg_async(cublas, cusparse, n, nnz, d_val, d_row_ptr, d_col_ind, d_b, d_x, d_r, d_p, d_Ap, stream);
-        CHECK_CUDA(cudaStreamSynchronize(stream));
+        int iterations = 0;
+        int code = CG_SUCCESS;
+        int strikes = 0;
+
+        // Try standard CG with the 3-strikes policy for non-fatal errors
+        do {
+            iterations = solve_cg_async(cublas, cusparse, n, nnz, d_val, d_row_ptr, d_col_ind, d_b, d_x, d_r, d_p, d_Ap, d_err, stream, code);
+            if (code == CG_SUCCESS) break;
+            
+            bool is_fatal = (code == CG_NAN_INF || code == CG_BREAKDOWN);
+            if (is_fatal) break;
+
+            strikes++;
+        } while (strikes < 3);
 
-        bool success = (iterations >= 0 && iterations < MAX_ITERATIONS);
+        bool success = (code == CG_SUCCESS);
 
         // Fallback mechanism: If standard CG fails to converge, retry using the Jacobi Preconditioner
         if (!success) {
-            std::cout << "[WORKER " << worker_id << "] CG reached max iterations (" << iterations 
-                      << "). Falling back to Jacobi solver for: " << task.path << std::endl;
-            iterations = solve_pcg_jacobi_async(cublas, cusparse, n, nnz, d_val, d_row_ptr, d_col_ind, d_b, d_x, d_r, d_p, d_Ap, d_z, d_Dinv, stream);
-            CHECK_CUDA(cudaStreamSynchronize(stream));
-            success = (iterations >= 0 && iterations < MAX_ITERATIONS);
+            std::cout << "[WORKER " << worker_id << "] CG failed with error: " << get_cg_error_string(code)
+                      << ". Falling back to Jacobi solver for: " << task.path << std::endl;
+            iterations = solve_pcg_jacobi_async(cublas, cusparse, n, nnz, d_val, d_row_ptr, d_col_ind, d_b, d_x, d_r, d_p, d_Ap, d_z, d_Dinv, d_err, stream, code);
+            success = (code == CG_SUCCESS);
         }
 
         // Clean up task memory
@@ -222,6 +331,7 @@ void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>
         CHECK_CUDA(cudaFreeAsync(d_b, stream));
         CHECK_CUDA(cudaFreeAsync(d_z, stream));
         CHECK_CUDA(cudaFreeAsync(d_Dinv, stream));
+        CHECK_CUDA(cudaFreeAsync(d_err, stream));
 
         auto finish_time = std::chrono::steady_clock::now();
         auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish_time - pick_time).count();

From 18323e3c0ee84904c23e2096b963d57b9e59426c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Jun 2026 09:28:34 -0600
Subject: [PATCH 0919/1000] removed while loop that dictated that solve cg
 async must be performed three times to reduce redundant computation

---
 .../workload-A/native_utils.cu                            | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
index 1e10bfcc96..4020b6eeea 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
@@ -300,15 +300,15 @@ void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>
         int strikes = 0;
 
         // Try standard CG with the 3-strikes policy for non-fatal errors
-        do {
+        // do {
             iterations = solve_cg_async(cublas, cusparse, n, nnz, d_val, d_row_ptr, d_col_ind, d_b, d_x, d_r, d_p, d_Ap, d_err, stream, code);
-            if (code == CG_SUCCESS) break;
+            // if (code == CG_SUCCESS) break;
             
             bool is_fatal = (code == CG_NAN_INF || code == CG_BREAKDOWN);
-            if (is_fatal) break;
+            // if (is_fatal) break;
 
             strikes++;
-        } while (strikes < 3);
+        // } while (strikes < 3);
 
         bool success = (code == CG_SUCCESS);
 

From dac8b0c31fbae806e1fe70462f39eeb317d00db5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Jun 2026 09:32:49 -0600
Subject: [PATCH 0920/1000] updated some values

---
 .../workload-A/supervisor_actor.hpp                        | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp
index dc68991c36..4053847e0b 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp
@@ -40,14 +40,15 @@ struct supervisor_state {
   std::unordered_map<caf::actor, int> actor_batch_sizes;
   std::deque<resource_slot> available_slots;
   int max_active = 1; // Admission control limit to be mindful of GPU memory.
-  int num_iterations = MAX_ITERATIONS / 2;
+  int num_iterations = MAX_ITERATIONS;
   int num_gpus = 0;
   int tasks_succeeded = 0;
   int tasks_failed = 0;
   std::chrono::steady_clock::time_point benchmark_start;
 };
 
-behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks, int initial_max_active, std::chrono::steady_clock::time_point start_time) {
+behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks, 
+  int initial_max_active, std::chrono::steady_clock::time_point start_time) {
     auto& st = self->state();
     st.queue.insert(st.queue.end(), std::make_move_iterator(tasks.begin()), std::make_move_iterator(tasks.end()));
     st.max_active = initial_max_active;
@@ -55,7 +56,7 @@ behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<Ma
     st.num_gpus = manager::get().get_num_devices();
 
     // Initialize the pool with streams interleaved across all available GPUs
-    for (int s = 0; s < 32; ++s) {
+    for (int s = 0; s < 4; ++s) {
       for (int g = 0; g < st.num_gpus; ++g) {
         st.available_slots.push_back({g, s});
       }

From 154b27dec1b45cf0e3725adc7468834d597e60f9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Jun 2026 10:15:49 -0600
Subject: [PATCH 0921/1000] updated logging and filepaths

---
 .../workload-A/main.native.cpp                              | 6 ++++--
 .../fault-tolerance-workload-test/workload-A/main.test.cpp  | 5 ++++-
 .../workload-A/native_utils.cu                              | 5 +++--
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native.cpp
index ea334586ac..b6486ee944 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native.cpp
@@ -23,8 +23,10 @@ int main(int argc, char** argv)
     // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
 
     std::cout << "[INFO] Loading matrices...\n";
-    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
-
+    //std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrices/workloadA", CGS_SOLVER);
+  
+    
     if (matrix_pool.empty()) {
         std::cerr << "No matrices found.\n";
         return 1;
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.test.cpp
index d3f0fd4cd4..4fd671a148 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.test.cpp
@@ -28,7 +28,10 @@ void caf_main(actor_system& sys) {
          //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/unsymmetric", CGS_SOLVER);
          //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric", CGS_SOLVER);
-         auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+
+         auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrices/workloadA", CGS_SOLVER);
+
 
         int num_gpus = manager::get().get_num_devices();
         std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
index 4020b6eeea..c3a110bed6 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/native_utils.cu
@@ -234,7 +234,7 @@ int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
     if (out_code == CG_SUCCESS) {
         if (!converged) {
             if (k >= max_iters) out_code = CG_MAX_ITER;
-            else if (std::abs(initial_rho - r1) < 1e-14f) out_code = CG_STAGNATION;
+            else if (std::abs(initial_rho - r1) < 1e-12f) out_code = CG_STAGNATION;
             else if (initial_rho > 0 && (r1 / initial_rho) > 0.9999f) out_code = CG_RESIDUAL_FACTOR_FAIL;
         }
     }
@@ -344,7 +344,8 @@ void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>
 
         record_job(task.path, task.enqueue_time, pick_time, finish_time, iterations, success);
 
-        std::cout << "[WORKER " << worker_id << "] Done: " << task.path << " (" << iterations << " iters, " << duration << " ms)." << std::endl;
+        std::cout << "[WORKER " << worker_id << "] Done: " << task.path << " (" << iterations << " iters, " << duration << " ms) ["
+                  << (success ? "SUCCESS" : "FAILED") << "]." << std::endl;
     }
     CHECK_CUBLAS(cublasDestroy(cublas));
     CHECK_CUSPARSE(cusparseDestroy(cusparse));

From 729a07bb1e27efa5c1d5b9aae3b2b3e9a6b2bf96 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Jun 2026 10:27:59 -0600
Subject: [PATCH 0922/1000] updated filepath

---
 .../workload-A/main.native_sorted.cpp                          | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native_sorted.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native_sorted.cpp
index 85aa4d301d..ca1ff57196 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native_sorted.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.native_sorted.cpp
@@ -27,7 +27,8 @@ int main(int argc, char** argv) {
     // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
 
     std::cout << "[INFO] Loading matrices...\n";
-    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrices/workloadA", CGS_SOLVER);
+
 
     if (matrix_pool.empty()) {
         std::cerr << "No matrices found.\n";

From ff5fc5e37751e88b793c283f4f65a080d97c7cc1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Jun 2026 11:01:41 -0600
Subject: [PATCH 0923/1000] added more logs

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 4dbee27f91..81ed6ffb60 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -99,6 +99,9 @@ std::vector<token_ptr> scheduler_actor::on_steal_request(int requesting_device)
                   << " tasks with device " << requesting_device << "\n";
         return batch;
     }
+     std::cout << "[INFO] Scheduler " << device_number_ 
+                  << " sharing no "
+                  << " tasks with device " << requesting_device << "\n";
     return {};
 }
 

From 1d618ac59aac33ea541c493fa44658266a1a98f6 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Jun 2026 11:12:08 -0600
Subject: [PATCH 0924/1000] updated min giveaway threshold

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 81ed6ffb60..f8dc7b7270 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -85,7 +85,11 @@ void scheduler_actor::on_set_neighbors(std::vector<caf::actor> neighbors) {
 
 std::vector<token_ptr> scheduler_actor::on_steal_request(int requesting_device) {
     // size_t min_giveaway_threshold = static_cast<size_t>(num_streams_) * stream_depth_ * 2;
-    size_t min_giveaway_threshold = static_cast<size_t>(num_streams_) * stream_depth_;
+    // size_t min_giveaway_threshold = static_cast<size_t>(num_streams_) * stream_depth_;
+
+
+    size_t min_giveaway_threshold = 1;
+
 
     if (queue_.size() > min_giveaway_threshold) {
         size_t to_give = queue_.size() / 2;

From 2a9cd4f4d882c3ac8c6a189eac6168d5680612c7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Jun 2026 11:28:45 -0600
Subject: [PATCH 0925/1000] updated giveawaythreshold

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index f8dc7b7270..e3aa896210 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -88,7 +88,7 @@ std::vector<token_ptr> scheduler_actor::on_steal_request(int requesting_device)
     // size_t min_giveaway_threshold = static_cast<size_t>(num_streams_) * stream_depth_;
 
 
-    size_t min_giveaway_threshold = 1;
+    size_t min_giveaway_threshold = 0;
 
 
     if (queue_.size() > min_giveaway_threshold) {

From 3df8808b6e31905d37a97670eeb795134fb53c6e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Jun 2026 11:36:17 -0600
Subject: [PATCH 0926/1000] implemented an exponential backoff polocy to try
 and reduce overhead from failed steals

---
 .../cuda/control-layer/scheduler_actor.hpp    |  2 ++
 .../src/control-layer/scheduler_actor.cpp     | 36 ++++++++++++++++---
 2 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
index 7749fa4570..61b799c3fd 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
@@ -37,6 +37,8 @@ class CAF_CUDA_EXPORT scheduler_actor : public caf::event_based_actor {
     std::vector<caf::actor> victims_;
     std::mt19937 rng_;
     std::queue<int> available_streams_;
+    int current_backoff_ = 0;
+    bool awaiting_steal_ = false;
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index e3aa896210..c8e76d2a97 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -41,12 +41,18 @@ caf::behavior scheduler_actor::make_behavior() {
             return on_steal_request(requesting_device);
         },
         [this](std::string word) {
-             std::cout << "Scheduler " << device_number_ << " received: " << word << "\n";
+            if (word == "retry_steal") {
+                current_backoff_ = 0;
+                schedule_work();
+            } else {
+                std::cout << "Scheduler " << device_number_ << " received: " << word << "\n";
+            }
         }
     };
 }
 
 void scheduler_actor::on_receive(const token_ptr& tok) {
+    current_backoff_ = 0; // Reset backoff when new work arrives locally
     if (tok->getType() == LAUNCH) {
         queue_.push(tok);
         schedule_work();
@@ -58,6 +64,9 @@ void scheduler_actor::on_receive(const token_ptr& tok) {
 }
 
 void scheduler_actor::on_receive_batch(std::vector<token_ptr> tokens) {
+    if (!tokens.empty()) {
+        current_backoff_ = 0; // Reset backoff when work is successfully received
+    }
     for (auto& tok : tokens) {
         queue_.push(std::move(tok));
     }
@@ -131,14 +140,31 @@ void scheduler_actor::schedule_work() {
     }
 
     // Work Stealing logic
-     if (multi_gpu_ && queue_.empty() && in_flight_ < (capacity / 2)) {
-    // if (multi_gpu_ && queue_.empty() && in_flight_ < (capacity )) {
-
+    if (multi_gpu_ && queue_.empty() && in_flight_ < (capacity / 2) && !awaiting_steal_ && current_backoff_ == 0) {
         // Randomly select a neighbor to request work from using the pre-generated victims list
         if (!victims_.empty()) {
             std::uniform_int_distribution<size_t> distrib(0, victims_.size() - 1);
             size_t idx = distrib(rng_);
-            this->mail(device_number_).send(victims_[idx]);
+            
+            awaiting_steal_ = true;
+            this->mail(device_number_).request(victims_[idx], infinite).then(
+                [this](std::vector<token_ptr>& stolen) {
+                    awaiting_steal_ = false;
+                    if (!stolen.empty()) {
+                        on_receive_batch(std::move(stolen));
+                    } else {
+                        // Work stealing failed: increase backoff and schedule a retry
+                        current_backoff_ = (current_backoff_ == 0) ? 1 : std::min(100, current_backoff_ * 2);
+                        this->mail("retry_steal").urgent().delay(std::chrono::milliseconds(current_backoff_)).send(this);
+                    }
+                },
+                [this](error& err) {
+                    awaiting_steal_ = false;
+                    // On network error or timeout, also backoff to avoid hammerring a dead/slow neighbor
+                    current_backoff_ = (current_backoff_ == 0) ? 1 : std::min(100, current_backoff_ * 2);
+                    this->mail("retry_steal").urgent().delay(std::chrono::milliseconds(current_backoff_)).send(this);
+                }
+            );
         }
     }
 }

From 1b796ca2aec4b59b45502c09e9b5fc3f36e8f5dd Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Jun 2026 11:47:38 -0600
Subject: [PATCH 0927/1000] updated timer values

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index c8e76d2a97..0e1ee5052f 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -161,7 +161,7 @@ void scheduler_actor::schedule_work() {
                 [this](error& err) {
                     awaiting_steal_ = false;
                     // On network error or timeout, also backoff to avoid hammerring a dead/slow neighbor
-                    current_backoff_ = (current_backoff_ == 0) ? 1 : std::min(100, current_backoff_ * 2);
+                    current_backoff_ = (current_backoff_ == 0) ? 50 : std::min(500, current_backoff_ * 2);
                     this->mail("retry_steal").urgent().delay(std::chrono::milliseconds(current_backoff_)).send(this);
                 }
             );

From ad7ef458a4406b85d2a5639e4cf614162d41bbd0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 11 Jun 2026 11:54:02 -0600
Subject: [PATCH 0928/1000] updated log messages

---
 libcaf_cuda/src/control-layer/scheduler_actor.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index 0e1ee5052f..fb0f088233 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -112,9 +112,9 @@ std::vector<token_ptr> scheduler_actor::on_steal_request(int requesting_device)
                   << " tasks with device " << requesting_device << "\n";
         return batch;
     }
-     std::cout << "[INFO] Scheduler " << device_number_ 
-                  << " sharing no "
-                  << " tasks with device " << requesting_device << "\n";
+    //  std::cout << "[INFO] Scheduler " << device_number_ 
+    //               << " sharing no "
+    //               << " tasks with device " << requesting_device << "\n";
     return {};
 }
 

From 1cb4e2c3a568fee093d6e4eb362e889edf0954a1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 12 Jun 2026 08:29:21 -0600
Subject: [PATCH 0929/1000] made changes to limit the number of actors within
 the system

---
 .../scheduler_integration_test.cpp            | 32 +++++++++++--------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
index 69d21a59fb..f9d823981f 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
@@ -23,7 +23,6 @@ struct MatrixPool {
 
 struct task_actor_state {
     program_ptr prog;
-    int N_val;
     std::shared_ptr<MatrixPool> pool;
 };
 
@@ -55,9 +54,14 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_
             if (res->getType() == LAUNCH_RESPONSE) {
                 auto& st = self->state();
                 
-                // We need to cast the base response_token to access the specific nd_range stored in it.
+                // Extract N from the token ID and compute kernel dimensions dynamically.
                 auto launch_res = static_cast<launch_response_token*>(res.get());
-                int N = st.N_val;
+                int N = std::stoi(launch_res->getId());
+
+                const int THREADS = 32;
+                nd_range range((N + THREADS - 1) / THREADS, 
+                               (N + THREADS - 1) / THREADS, 1, 
+                               THREADS, THREADS, 1);
 
                 // 1. Setup GPU arguments.
                 // Fetch data from the shared pool only when scheduled to save RAM
@@ -67,7 +71,7 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_
                 auto in_n = create_in_arg(N);
 
                 // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
-                auto result_tuple = runner.run_async(st.prog, launch_res->getRange(), res, in_a, in_b, out_c, in_n);
+                auto result_tuple = runner.run_async(st.prog, range, res, in_a, in_b, out_c, in_n);
                 auto d_c = std::get<2>(result_tuple);
 
                 // 3. Asynchronous Copyback.
@@ -85,10 +89,9 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_
 }
 
 // Helper function to initialize task_actor_state
-behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, std::shared_ptr<MatrixPool> pool, caf::actor exit_actor) {
+behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, std::shared_ptr<MatrixPool> pool, caf::actor exit_actor) {
     auto& st = self->state();
     st.prog = std::move(prog);
-    st.N_val = N_val;
     st.pool = std::move(pool);
     return task_actor_fun(self, exit_actor); // Pass exit_actor to task_actor_fun
 }
@@ -141,21 +144,24 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         // Spawn the exit actor for this specific test run (moved inside the loop)
         auto exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_tasks);
 
-        // Spawn task actors and prepare tokens
+        // 1. Spawn a fixed pool of 100 workers to reuse across all tasks.
+        const int max_workers = 100;
+        std::vector<caf::actor> workers;
+        for (int i = 0; i < max_workers; ++i) {
+            workers.push_back(sys.spawn(make_task_actor_behavior, program, pool_ptr, exit_actor));
+        }
+
+        // 2. Prepare tokens and assign to workers in round-robin fashion.
         for (int i = 0; i < num_tasks; ++i) {
             int current_N = available_Ns[dist_N_idx(rng)];
             nd_range range((current_N + THREADS - 1) / THREADS, 
                            (current_N + THREADS - 1) / THREADS, 1, 
                            THREADS, THREADS, 1);
             
-            auto worker = sys.spawn(make_task_actor_behavior,
-                                    program,
-                                    current_N, 
-                                    pool_ptr,
-                                    exit_actor); // Pass exit_actor to task actors
+            auto worker = workers[i % max_workers];
 
             tokens.push_back(make_launch_token(program, range, 0, 
-                             "task_" + std::to_string(i), worker));
+                             std::to_string(current_N), worker));
         }
 
         double elapsed = time_run([&]() {

From 850709fc72c388201d77c0f7f37e255beb082633 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 12 Jun 2026 08:54:38 -0600
Subject: [PATCH 0930/1000] reverting back since the worker actors will act as
 bottleneck

---
 .../scheduler_integration_test.cpp            | 34 ++++++++-----------
 1 file changed, 14 insertions(+), 20 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
index f9d823981f..ff92ec5fca 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/scheduler_integration_test.cpp
@@ -23,6 +23,7 @@ struct MatrixPool {
 
 struct task_actor_state {
     program_ptr prog;
+    int N_val;
     std::shared_ptr<MatrixPool> pool;
 };
 
@@ -54,14 +55,9 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_
             if (res->getType() == LAUNCH_RESPONSE) {
                 auto& st = self->state();
                 
-                // Extract N from the token ID and compute kernel dimensions dynamically.
+                // We need to cast the base response_token to access the specific nd_range stored in it.
                 auto launch_res = static_cast<launch_response_token*>(res.get());
-                int N = std::stoi(launch_res->getId());
-
-                const int THREADS = 32;
-                nd_range range((N + THREADS - 1) / THREADS, 
-                               (N + THREADS - 1) / THREADS, 1, 
-                               THREADS, THREADS, 1);
+                int N = st.N_val;
 
                 // 1. Setup GPU arguments.
                 // Fetch data from the shared pool only when scheduled to save RAM
@@ -71,7 +67,7 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_
                 auto in_n = create_in_arg(N);
 
                 // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
-                auto result_tuple = runner.run_async(st.prog, range, res, in_a, in_b, out_c, in_n);
+                auto result_tuple = runner.run_async(st.prog, launch_res->getRange(), res, in_a, in_b, out_c, in_n);
                 auto d_c = std::get<2>(result_tuple);
 
                 // 3. Asynchronous Copyback.
@@ -89,9 +85,10 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_
 }
 
 // Helper function to initialize task_actor_state
-behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, std::shared_ptr<MatrixPool> pool, caf::actor exit_actor) {
+behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, std::shared_ptr<MatrixPool> pool, caf::actor exit_actor) {
     auto& st = self->state();
     st.prog = std::move(prog);
+    st.N_val = N_val;
     st.pool = std::move(pool);
     return task_actor_fun(self, exit_actor); // Pass exit_actor to task_actor_fun
 }
@@ -129,7 +126,7 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         auto& mgr = manager::get();
 
         // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
-        mgr.toggle_scheduler_actor(4, 2);
+        mgr.toggle_scheduler_actor(8, 1);
 
         auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
         const int THREADS = 32;
@@ -144,24 +141,21 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         // Spawn the exit actor for this specific test run (moved inside the loop)
         auto exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_tasks);
 
-        // 1. Spawn a fixed pool of 100 workers to reuse across all tasks.
-        const int max_workers = 100;
-        std::vector<caf::actor> workers;
-        for (int i = 0; i < max_workers; ++i) {
-            workers.push_back(sys.spawn(make_task_actor_behavior, program, pool_ptr, exit_actor));
-        }
-
-        // 2. Prepare tokens and assign to workers in round-robin fashion.
+        // Spawn task actors and prepare tokens
         for (int i = 0; i < num_tasks; ++i) {
             int current_N = available_Ns[dist_N_idx(rng)];
             nd_range range((current_N + THREADS - 1) / THREADS, 
                            (current_N + THREADS - 1) / THREADS, 1, 
                            THREADS, THREADS, 1);
             
-            auto worker = workers[i % max_workers];
+            auto worker = sys.spawn(make_task_actor_behavior,
+                                    program,
+                                    current_N, 
+                                    pool_ptr,
+                                    exit_actor); // Pass exit_actor to task actors
 
             tokens.push_back(make_launch_token(program, range, 0, 
-                             std::to_string(current_N), worker));
+                             "task_" + std::to_string(i), worker));
         }
 
         double elapsed = time_run([&]() {

From cad01704a34baf769190d9fde0721caee324ed84 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 12 Jun 2026 09:27:51 -0600
Subject: [PATCH 0931/1000] updated test to partiton the worklaod based on
 number of GPUs

---
 .../workload-A/main.test.cpp                  | 21 +++++++++++++++++--
 .../workload-A/supervisor_actor.hpp           | 10 ++++-----
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.test.cpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.test.cpp
index 4fd671a148..5fd9f46a83 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.test.cpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/main.test.cpp
@@ -53,10 +53,27 @@ void caf_main(actor_system& sys) {
             task.enqueue_time = std::chrono::steady_clock::now();
         }
 
+        // Workload partitioning logic (Timed)
+        auto part_start = std::chrono::steady_clock::now();
+        auto partitions = make_contiguous_partitions(tasks_vec.size(), num_gpus, 1);
+        
+        std::vector<std::vector<MatrixTask>> partitioned_workloads(num_gpus);
+        for (int i = 0; i < num_gpus; ++i) {
+            auto& p = partitions[i];
+            partitioned_workloads[i].reserve(p.end - p.begin);
+            for (size_t j = p.begin; j < p.end; ++j) {
+                partitioned_workloads[i].push_back(std::move(tasks_vec[j]));
+            }
+        }
+        auto part_end = std::chrono::steady_clock::now();
+        auto part_ms = std::chrono::duration_cast<std::chrono::milliseconds>(part_end - part_start).count();
+        std::cout << "[INFO] Workload partitioning completed in " << part_ms << " ms\n";
+
         auto benchmark_start = std::chrono::steady_clock::now();
-        int admission_control_limit = 4 * num_gpus; // 4 concurrent tasks per GPU
+        for (int i = 0; i < num_gpus; ++i) {
+            sys.spawn(supervisor_actor, std::move(partitioned_workloads[i]), 4, benchmark_start, i);
+        }
 
-        sys.spawn(supervisor_actor, std::move(tasks_vec), admission_control_limit, benchmark_start);
         sys.await_all_actors_done();
     }
     manager::shutdown();
diff --git a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp
index 4053847e0b..88be59d8f3 100644
--- a/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp
+++ b/libcaf_cuda/tests/fault-tolerance-workload-test/workload-A/supervisor_actor.hpp
@@ -48,18 +48,16 @@ struct supervisor_state {
 };
 
 behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks, 
-  int initial_max_active, std::chrono::steady_clock::time_point start_time) {
+  int initial_max_active, std::chrono::steady_clock::time_point start_time, int target_device) {
     auto& st = self->state();
     st.queue.insert(st.queue.end(), std::make_move_iterator(tasks.begin()), std::make_move_iterator(tasks.end()));
     st.max_active = initial_max_active;
     st.benchmark_start = start_time;
-    st.num_gpus = manager::get().get_num_devices();
+    st.num_gpus = 1; 
 
-    // Initialize the pool with streams interleaved across all available GPUs
+    // Initialize the pool with streams for the assigned GPU
     for (int s = 0; s < 4; ++s) {
-      for (int g = 0; g < st.num_gpus; ++g) {
-        st.available_slots.push_back({g, s});
-      }
+      st.available_slots.push_back({target_device, s});
     }
 
     auto spawn_next = [self]() {

From a18ece3e816f49e74d21322200bfec7beac820f4 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 16 Jun 2026 10:05:53 -0600
Subject: [PATCH 0932/1000] added method to send scheduler actor a single
 message

---
 libcaf_cuda/caf/cuda/manager.hpp |  3 +++
 libcaf_cuda/src/manager.cpp      | 12 ++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index b9ccb3582a..52de5138c8 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -198,6 +198,9 @@ class CAF_CUDA_EXPORT manager {
   // Sends a batch of tokens to the scheduler actors, distributing them statically.
   void send_scheduler_actor_message(std::vector<token_ptr> tokens);
 
+  // Sends a single token to a randomly selected scheduler actor.
+  void send_scheduler_actor_message(token_ptr token);
+
   // Returns the first scheduler actor. Useful for single-GPU setups or general dispatch.
   caf::actor get_scheduler_actor();
 
diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 6871b8725d..7dbfac5c75 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -3,6 +3,7 @@
 #include <stdexcept>
 #include <mutex>
 #include <map>
+#include <random>
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
 #include "caf/cuda/control-layer/token_factory.hpp" // For make_behavior_token
 #include "caf/cuda/manager_config.hpp"
@@ -385,6 +386,17 @@ void manager::send_scheduler_actor_message(std::vector<token_ptr> tokens) {
     }
 }
 
+void manager::send_scheduler_actor_message(token_ptr token) {
+    if (scheduler_actors_.empty())
+        return;
+
+    static thread_local std::mt19937 generator(std::random_device{}());
+    std::uniform_int_distribution<size_t> distribution(0, scheduler_actors_.size() - 1);
+    size_t idx = distribution(generator);
+
+    caf::anon_mail(std::move(token)).send(scheduler_actors_[idx]);
+}
+
 caf::actor manager::get_scheduler_actor() {
     if (scheduler_actors_.empty()) {
         throw std::runtime_error(

From 76229085a3f3dda007801c08634ac1b55c992f15 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 16 Jun 2026 12:22:40 -0600
Subject: [PATCH 0933/1000] updated methosd to return custreams and cucontexts
 if the user wants

---
 libcaf_cuda/caf/cuda/command_runner.hpp   | 18 +++++++++
 libcaf_cuda/caf/cuda/manager.hpp          |  6 +++
 libcaf_cuda/src/manager.cpp               | 16 ++++++++
 libcaf_cuda/tests/unit-test/main.test.cpp | 45 ++++++++++++++++++++---
 4 files changed, 79 insertions(+), 6 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/command_runner.hpp b/libcaf_cuda/caf/cuda/command_runner.hpp
index 613b485598..1def06d3f8 100644
--- a/libcaf_cuda/caf/cuda/command_runner.hpp
+++ b/libcaf_cuda/caf/cuda/command_runner.hpp
@@ -359,6 +359,24 @@ class command_runner {
     dev->synchronize_event(std::move(e));
   }
 
+  // -------------------------------------------------------------------------
+  // CUDA Context and Stream Retrieval
+  // -------------------------------------------------------------------------
+
+  /// Returns the CUDA context associated with the given device number.
+  CUcontext get_context(int device_number) {
+      auto plat = platform::create();
+      auto dev = plat->getDevice(device_number);
+      return dev->getContext();
+  }
+
+  /// Returns the CUDA stream associated with the given stream number (actor ID) and device number.
+  CUstream get_stream(int stream_number, int device_number) {
+      auto plat = platform::create();
+      auto dev = plat->schedule(stream_number, device_number);
+      return dev->get_stream_for_actor(stream_number);
+  }
+
 };
 
 } // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/manager.hpp b/libcaf_cuda/caf/cuda/manager.hpp
index 52de5138c8..cf4cded82d 100644
--- a/libcaf_cuda/caf/cuda/manager.hpp
+++ b/libcaf_cuda/caf/cuda/manager.hpp
@@ -210,6 +210,12 @@ class CAF_CUDA_EXPORT manager {
   // Sends a behavior change message to a specific scheduler actor.
   void send_scheduler_actor_message(const std::string& behavior_name, int device_number);
 
+  /// Returns the CUDA context associated with the given device number.
+  CUcontext get_context(int device_number);
+
+  /// Returns the CUDA stream associated with the given stream number (actor ID) and device number.
+  CUstream get_stream(int stream_number, int device_number);
+
 private:
   explicit manager(caf::actor_system& sys)
     : system_(sys), platform_(platform::create()) {
diff --git a/libcaf_cuda/src/manager.cpp b/libcaf_cuda/src/manager.cpp
index 7dbfac5c75..ef144ea835 100644
--- a/libcaf_cuda/src/manager.cpp
+++ b/libcaf_cuda/src/manager.cpp
@@ -429,4 +429,20 @@ void manager::send_scheduler_actor_message(const std::string& behavior_name, int
     }
 }
 
+CUcontext manager::get_context(int device_number) {
+    auto dev = find_device(device_number);
+    if (!dev) {
+        throw std::runtime_error("Invalid device number: " + std::to_string(device_number));
+    }
+    return dev->getContext();
+}
+
+CUstream manager::get_stream(int stream_number, int device_number) {
+    auto dev = find_device(device_number);
+    if (!dev) {
+        throw std::runtime_error("Invalid device number: " + std::to_string(device_number));
+    }
+    return dev->get_stream_for_actor(stream_number);
+}
+
 } // namespace caf::cuda
diff --git a/libcaf_cuda/tests/unit-test/main.test.cpp b/libcaf_cuda/tests/unit-test/main.test.cpp
index 785c17bb1d..beabae053d 100644
--- a/libcaf_cuda/tests/unit-test/main.test.cpp
+++ b/libcaf_cuda/tests/unit-test/main.test.cpp
@@ -757,11 +757,43 @@ void test_command_runner_memory_transfer([[maybe_unused]] caf::actor_system& sys
 }
 
 
-
-
-
-
-
+// Test for context and stream retrieval helper methods
+void test_context_stream_retrieval([[maybe_unused]] caf::actor_system& sys) {
+  using namespace caf::cuda;
+  auto& mgr = manager::get();
+  
+  // Test manager context and stream retrieval
+  CUcontext m_ctx = mgr.get_context(0);
+  assert(m_ctx != nullptr);
+  
+  CUstream m_stream = mgr.get_stream(1, 0);
+  assert(m_stream != nullptr);
+  
+  // Test command_runner context and stream retrieval
+  command_runner<> runner;
+  CUcontext cr_ctx = runner.get_context(0);
+  assert(cr_ctx == m_ctx);
+  
+  CUstream cr_stream = runner.get_stream(1, 0);
+  assert(cr_stream == m_stream);
+  
+  // Test invalid device exception throwing
+  bool manager_threw = false;
+  try {
+    mgr.get_context(9999);
+  } catch (const std::exception&) {
+    manager_threw = true;
+  }
+  assert(manager_threw);
+  
+  bool runner_threw = false;
+  try {
+    runner.get_context(9999);
+  } catch (const std::exception&) {
+    runner_threw = true;
+  }
+  assert(runner_threw);
+}
 
 
 // Structure to hold test information
@@ -790,7 +822,8 @@ const std::vector<Test> tests = {
     {"test_invalid_kernel_params", test_invalid_kernel_params},
     {"test_stream_async_execution", test_stream_async_execution},
     {"test_compare_strings", test_compare_strings},
-    {"test_command_runner_memory_transfer", test_command_runner_memory_transfer}
+    {"test_command_runner_memory_transfer", test_command_runner_memory_transfer},
+    {"test_context_stream_retrieval", test_context_stream_retrieval}
 };
 
 // Function to run a single test and report its result

From c43c2e8910eb65edc9700efe3c73993723ef0e2b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 16 Jun 2026 12:39:01 -0600
Subject: [PATCH 0934/1000] updated mmul actor to be more efficient with
 CPU-GPU synchronization

---
 .../baseline-comparison/actors/main.test.cpp  | 28 ++++++++++++-------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
index 79d34ad42a..88e28c7388 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
@@ -132,18 +132,26 @@ return {
 
 	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);
 
-            if (++self->state().results_received == self->state().total_expected) {
-                auto self_hdl = caf::actor_cast<caf::actor>(self);
-                mmul_command.copy_to_host_async(dC, matrixC.data(), N*N, [self_hdl](int*, size_t) {
-                    caf::anon_mail(kernel_done_atom_v).send(self_hdl);
-                });
-            } else {
-                // Fire-and-forget: No host callback, no synchronization
-                mmul_command.copy_to_host_async(dC, matrixC.data(), N*N);
-            }
+        auto self_hdl = caf::actor_cast<caf::actor>(self);
+      
+        
+        if (++self->state().results_received == self->state().total_expected) {
+           mmul_command.copy_to_host_async(dC, matrixC.data(), N*N, [self_hdl](int*, size_t) {
+                caf::anon_mail(kernel_done_atom_v).send(self_hdl);
+            });
+        }
+        else {
+           mmul_command.copy_to_host_async(dC, matrixC.data(), N*N, [self_hdl](int*, size_t) {
+            //do nothing there is nothing to do
+          });
+
+        }
+        
     },
     [=](kernel_done_atom) {
-        self->quit();
+        if (++self->state().results_received == self->state().total_expected) {
+            self->quit();
+        }
     }
   };
 }

From 795d711623c278539d4f31851a5c4bcffee02269 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 16 Jun 2026 12:44:14 -0600
Subject: [PATCH 0935/1000] updated kerenl done atom to fix deadlock occurance

---
 .../baseline-comparison/actors/main.test.cpp                    | 2 --
 1 file changed, 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
index 88e28c7388..c6b1b2b99e 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
@@ -149,9 +149,7 @@ return {
         
     },
     [=](kernel_done_atom) {
-        if (++self->state().results_received == self->state().total_expected) {
             self->quit();
-        }
     }
   };
 }

From 1537d9ba12df3a2c64f435a17c4f7ef5a6b42747 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 16 Jun 2026 12:51:12 -0600
Subject: [PATCH 0936/1000] made copying to host more efficient

---
 .../baseline-comparison/actors/main.test.cpp              | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
index c6b1b2b99e..4246671214 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/actors/main.test.cpp
@@ -127,7 +127,7 @@ return {
       auto output = command.run_async(
 		      self->state().program,
 		      dims,
-		      1, 0, device,
+		      stream, 0, device,
 		      arg1,arg2,out<int>{N*N},in<int>{N});
 
 	    caf::cuda::mem_ptr<int> dC = std::get<2>(output);
@@ -141,10 +141,8 @@ return {
             });
         }
         else {
-           mmul_command.copy_to_host_async(dC, matrixC.data(), N*N, [self_hdl](int*, size_t) {
-            //do nothing there is nothing to do
-          });
-
+           mmul_command.copy_to_host_async(dC, matrixC.data(), N*N );
+          
         }
         
     },

From 2254715aeb884540e010b2754707780ec1516e24 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 08:22:56 -0600
Subject: [PATCH 0937/1000] Initial commit.

---
 .../CMakeLists.txt                            |  74 +++++++
 .../compile_kernels.sh                        |  14 ++
 .../scheduler-fault-tolerance-test/mmul.cu    |  16 ++
 .../scheduler_integration_test.cpp            | 181 ++++++++++++++++++
 4 files changed, 285 insertions(+)
 create mode 100644 libcaf_cuda/tests/scheduler-fault-tolerance-test/CMakeLists.txt
 create mode 100755 libcaf_cuda/tests/scheduler-fault-tolerance-test/compile_kernels.sh
 create mode 100644 libcaf_cuda/tests/scheduler-fault-tolerance-test/mmul.cu
 create mode 100644 libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/CMakeLists.txt b/libcaf_cuda/tests/scheduler-fault-tolerance-test/CMakeLists.txt
new file mode 100644
index 0000000000..22963221d9
--- /dev/null
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/CMakeLists.txt
@@ -0,0 +1,74 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+project(CUDA_ACTORS)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+
+# 5) Declare your executables
+add_executable(test main.test.cpp)
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+add_executable(scheduler_test scheduler_integration_test.cpp)
+target_compile_definitions(scheduler_test PRIVATE CAF_ENABLE_LOGGING)
+
+add_executable(work-stealing work-stealing.cpp)
+target_compile_definitions(work-stealing PRIVATE CAF_ENABLE_LOGGING)
+
+add_executable(cuda-baseline cuda-baseline.cpp)
+target_link_libraries(cuda-baseline PRIVATE CUDA::cuda_driver)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc 
+    CUDA::cublas
+    CUDA::cusparse
+)
+
+
+
+target_link_libraries(scheduler_test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc 
+    CUDA::cublas
+    CUDA::cusparse
+)
+
+target_link_libraries(work-stealing
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
+)
diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/compile_kernels.sh b/libcaf_cuda/tests/scheduler-fault-tolerance-test/compile_kernels.sh
new file mode 100755
index 0000000000..c0bebf8833
--- /dev/null
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/compile_kernels.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+# Detect first GPU compute capability to ensure binary compatibility
+ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
+SM_ARCH="sm_${ARCH/./}"
+echo "Using NVCC arch flag: $SM_ARCH"
+
+# Compile kernels to cubin for Driver API loading
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
+nvcc -arch=$SM_ARCH -cubin vector_add.cu -o vector_add.cubin
+nvcc -arch=$SM_ARCH -cubin conv1d.cu -o conv1d.cubin
+
+echo "Kernels compiled successfully for $SM_ARCH."
\ No newline at end of file
diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/mmul.cu b/libcaf_cuda/tests/scheduler-fault-tolerance-test/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+
diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
new file mode 100644
index 0000000000..ff92ec5fca
--- /dev/null
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -0,0 +1,181 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
+#include <iostream>
+#include <vector>
+#include <string>
+#include <random>
+#include <unordered_map>
+#include <unordered_set>
+#include <algorithm>
+
+using namespace caf;
+using namespace caf::cuda;
+
+// A generic command runner to provide access to CUDA stream callbacks
+static command_runner<in<int>, in<int>, out<int>, in<int>> runner;
+
+// MatrixPool structure from mmul-random-batch-benchmark
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
+};
+
+struct task_actor_state {
+    program_ptr prog;
+    int N_val;
+    std::shared_ptr<MatrixPool> pool;
+};
+
+// create_matrix_pool_random function from mmul-random-batch-benchmark
+MatrixPool create_matrix_pool_random(
+    int num_sizes,
+    int min_N,
+    int max_N,
+    unsigned int seed
+) {
+    MatrixPool pool;
+    std::mt19937 rng(seed);
+    std::uniform_int_distribution<int> dist(min_N, max_N);
+    std::unordered_set<int> used;
+    while (used.size() < static_cast<size_t>(num_sizes)) {
+        int N = dist(rng);
+        if (used.insert(N).second) {
+            pool.A[N] = std::vector<int>(N * N, 1);
+            pool.B[N] = std::vector<int>(N * N, 2); // Changed to 2 for distinct input
+        }
+    }
+    return pool;
+}
+
+// This actor represents a single task that requests permission from the scheduler.
+behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_actor) {
+    return {
+        [=](response_token_ptr res) mutable {
+            if (res->getType() == LAUNCH_RESPONSE) {
+                auto& st = self->state();
+                
+                // We need to cast the base response_token to access the specific nd_range stored in it.
+                auto launch_res = static_cast<launch_response_token*>(res.get());
+                int N = st.N_val;
+
+                // 1. Setup GPU arguments.
+                // Fetch data from the shared pool only when scheduled to save RAM
+                auto in_a = create_in_arg(st.pool->A.at(N));
+                auto in_b = create_in_arg(st.pool->B.at(N));
+                auto out_c = create_out_arg_with_size<int>(N * N);
+                auto in_n = create_in_arg(N);
+
+                // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
+                auto result_tuple = runner.run_async(st.prog, launch_res->getRange(), res, in_a, in_b, out_c, in_n);
+                auto d_c = std::get<2>(result_tuple);
+
+                // 3. Asynchronous Copyback.
+                // Allocate a local buffer for the result to keep the total system memory low.
+                auto h_c = std::make_shared<std::vector<int>>(N * N);
+                // The launch_response_token is released inside the callback 
+                // to signal to the scheduler that the resource is free. (No serial verification here)
+                runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [res, exit_actor, h_c](int* /*ptr*/, size_t /*sz*/) {
+                    res->release();
+                    anon_mail(1).send(exit_actor);
+                });
+            }
+        }
+    };
+}
+
+// Helper function to initialize task_actor_state
+behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, std::shared_ptr<MatrixPool> pool, caf::actor exit_actor) {
+    auto& st = self->state();
+    st.prog = std::move(prog);
+    st.N_val = N_val;
+    st.pool = std::move(pool);
+    return task_actor_fun(self, exit_actor); // Pass exit_actor to task_actor_fun
+}
+
+template <class Fn>
+double time_run(Fn&& fn) {
+    auto start = std::chrono::steady_clock::now();
+    fn();
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+    return elapsed.count();
+}
+
+void run_scheduler_integration_scaling_test(actor_system& sys) {
+    const int min_N = 32;
+    const int max_N = 2048;
+    const int num_distinct_sizes = 10;
+    const std::vector<int> actor_counts = {50000};
+
+
+    //  const std::vector<int> actor_counts = {5};
+
+
+
+    // Generate deterministic random pool once
+    auto pool_ptr = std::make_shared<MatrixPool>(
+        create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42));
+    
+    std::vector<int> available_Ns;
+    for (const auto& pair : pool_ptr->A) available_Ns.push_back(pair.first);
+
+    for (int num_tasks : actor_counts) {
+        manager_config config;
+        manager::init(sys, config);
+        auto& mgr = manager::get();
+
+        // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
+        mgr.toggle_scheduler_actor(8, 1);
+
+        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        const int THREADS = 32;
+
+        std::vector<token_ptr> tokens;
+        std::mt19937 rng(42);
+        std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
+
+        std::cout << "=====================================\n";
+        std::cout << "Scheduler Test | tasks=" << num_tasks << "\n";
+
+        // Spawn the exit actor for this specific test run (moved inside the loop)
+        auto exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_tasks);
+
+        // Spawn task actors and prepare tokens
+        for (int i = 0; i < num_tasks; ++i) {
+            int current_N = available_Ns[dist_N_idx(rng)];
+            nd_range range((current_N + THREADS - 1) / THREADS, 
+                           (current_N + THREADS - 1) / THREADS, 1, 
+                           THREADS, THREADS, 1);
+            
+            auto worker = sys.spawn(make_task_actor_behavior,
+                                    program,
+                                    current_N, 
+                                    pool_ptr,
+                                    exit_actor); // Pass exit_actor to task actors
+
+            tokens.push_back(make_launch_token(program, range, 0, 
+                             "task_" + std::to_string(i), worker));
+        }
+
+        double elapsed = time_run([&]() {
+            std::cout << "[MAIN] Dispatching batch to scheduler..." << std::endl;
+            mgr.send_scheduler_actor_message(std::move(tokens));
+
+            // The dispatch is asynchronous. To get an accurate measurement, we must
+            // block until the exit_actor terminates (signaling all 50k tasks are done).
+            scoped_actor self{sys};
+            self->wait_for(exit_actor);
+        });
+
+        std::cout << "Run complete. Time: " << elapsed << " s\n";
+        manager::shutdown(); // Reset manager state for the next potential iteration
+    }
+}
+
+void caf_main(actor_system& sys) {
+    run_scheduler_integration_scaling_test(sys);
+    std::cout << "[MAIN] Integration test complete." << std::endl;
+}
+
+CAF_MAIN(id_block::cuda_control)

From 77c39ddc45a47fb9393f11d972d3828a4598c0f0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 08:24:22 -0600
Subject: [PATCH 0938/1000] Updated cmakelists to build properly

---
 .../CMakeLists.txt                            | 50 +++++++++----------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/CMakeLists.txt b/libcaf_cuda/tests/scheduler-fault-tolerance-test/CMakeLists.txt
index 22963221d9..d1a345a58d 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/CMakeLists.txt
@@ -6,7 +6,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS      OFF)
 
 # 2) Set CAF source and build directories
-set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../actor-framework")
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
 set(CAF_BUILD "${CAF_SRC}/build")
 
 
@@ -30,26 +30,26 @@ include_directories(
 
 
 # 5) Declare your executables
-add_executable(test main.test.cpp)
-target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+# add_executable(test main.test.cpp)
+# target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
 add_executable(scheduler_test scheduler_integration_test.cpp)
 target_compile_definitions(scheduler_test PRIVATE CAF_ENABLE_LOGGING)
 
-add_executable(work-stealing work-stealing.cpp)
-target_compile_definitions(work-stealing PRIVATE CAF_ENABLE_LOGGING)
+# add_executable(work-stealing work-stealing.cpp)
+# target_compile_definitions(work-stealing PRIVATE CAF_ENABLE_LOGGING)
 
-add_executable(cuda-baseline cuda-baseline.cpp)
-target_link_libraries(cuda-baseline PRIVATE CUDA::cuda_driver)
+# add_executable(cuda-baseline cuda-baseline.cpp)
+# target_link_libraries(cuda-baseline PRIVATE CUDA::cuda_driver)
 
-target_link_libraries(test
-  PRIVATE
-    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-    CUDA::nvrtc 
-    CUDA::cublas
-    CUDA::cusparse
-)
+# target_link_libraries(test
+#   PRIVATE
+#     "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+#     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+#     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+#     CUDA::nvrtc 
+#     CUDA::cublas
+#     CUDA::cusparse
+# )
 
 
 
@@ -63,12 +63,12 @@ target_link_libraries(scheduler_test
     CUDA::cusparse
 )
 
-target_link_libraries(work-stealing
-  PRIVATE
-    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-    CUDA::nvrtc
-    CUDA::cublas
-    CUDA::cusparse
-)
+# target_link_libraries(work-stealing
+#   PRIVATE
+#     "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+#     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+#     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+#     CUDA::nvrtc
+#     CUDA::cublas
+#     CUDA::cusparse
+# )

From 4ae890256be10a1f7adbdecbf4129db06ff16b9e Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 08:25:03 -0600
Subject: [PATCH 0939/1000] Updated script

---
 .../tests/scheduler-fault-tolerance-test/compile_kernels.sh   | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/compile_kernels.sh b/libcaf_cuda/tests/scheduler-fault-tolerance-test/compile_kernels.sh
index c0bebf8833..9ff3ab7ab2 100755
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/compile_kernels.sh
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/compile_kernels.sh
@@ -8,7 +8,5 @@ echo "Using NVCC arch flag: $SM_ARCH"
 
 # Compile kernels to cubin for Driver API loading
 nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
-nvcc -arch=$SM_ARCH -cubin vector_add.cu -o vector_add.cubin
-nvcc -arch=$SM_ARCH -cubin conv1d.cu -o conv1d.cubin
 
-echo "Kernels compiled successfully for $SM_ARCH."
\ No newline at end of file
+echo "Kernels compiled successfully for $SM_ARCH."

From d6fede9d93db7b57d49d063c70220cd0b1915c82 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 08:37:44 -0600
Subject: [PATCH 0940/1000] Updated test to almost work correctly.

---
 .../scheduler_integration_test.cpp            | 136 +++++++++++++++---
 1 file changed, 119 insertions(+), 17 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index ff92ec5fca..4299f9952f 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -27,6 +27,30 @@ struct task_actor_state {
     std::shared_ptr<MatrixPool> pool;
 };
 
+struct supervisor_state {
+    program_ptr prog;
+    int N_val;
+    std::shared_ptr<MatrixPool> pool;
+    actor exit_actor;
+    actor stats_actor;
+    response_token_ptr res;
+};
+
+struct stats_actor_state {
+    int completed = 0;
+};
+
+behavior stats_actor_fun(stateful_actor<stats_actor_state>* self) {
+    return {
+        [=](int count) {
+            self->state().completed += count;
+            if (self->state().completed % 1000 == 0) {
+                std::cout << "[STATS] Jobs completed: " << self->state().completed << std::endl;
+            }
+        }
+    };
+}
+
 // create_matrix_pool_random function from mmul-random-batch-benchmark
 MatrixPool create_matrix_pool_random(
     int num_sizes,
@@ -49,10 +73,10 @@ MatrixPool create_matrix_pool_random(
 }
 
 // This actor represents a single task that requests permission from the scheduler.
-behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_actor) {
+behavior task_worker_fun(stateful_actor<task_actor_state>* self, caf::actor stats_actor, caf::actor exit_actor) {
     return {
         [=](response_token_ptr res) mutable {
-            if (res->getType() == LAUNCH_RESPONSE) {
+            try {
                 auto& st = self->state();
                 
                 // We need to cast the base response_token to access the specific nd_range stored in it.
@@ -73,24 +97,83 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_
                 // 3. Asynchronous Copyback.
                 // Allocate a local buffer for the result to keep the total system memory low.
                 auto h_c = std::make_shared<std::vector<int>>(N * N);
+                auto self_hdl = actor_cast<actor>(self);
+
                 // The launch_response_token is released inside the callback 
                 // to signal to the scheduler that the resource is free. (No serial verification here)
-                runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [res, exit_actor, h_c](int* /*ptr*/, size_t /*sz*/) {
+                runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [res, stats_actor, exit_actor, h_c, self_hdl](int* /*ptr*/, size_t /*sz*/) {
                     res->release();
+                    anon_mail(1).send(stats_actor);
                     anon_mail(1).send(exit_actor);
+                    anon_mail(0).send(self_hdl); // Signal normal completion
                 });
+            } catch (const std::exception& e) {
+                self->quit(sec::runtime_error);
             }
+        },
+        [=](int signal) {
+            if (signal == 0)
+                self->quit();
         }
     };
 }
 
 // Helper function to initialize task_actor_state
-behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, std::shared_ptr<MatrixPool> pool, caf::actor exit_actor) {
+behavior make_task_worker_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, std::shared_ptr<MatrixPool> pool, caf::actor stats_actor, caf::actor exit_actor) {
+    auto& st = self->state();
+    st.prog = std::move(prog);
+    st.N_val = N_val;
+    st.pool = std::move(pool);
+    return task_worker_fun(self, stats_actor, exit_actor);
+}
+
+behavior task_supervisor_fun(stateful_actor<supervisor_state>* self) {
+    self->set_down_handler([=](down_msg& msg) {
+        if (msg.reason != exit_reason::normal) {
+            std::cout << "[SUPERVISOR] Task worker failed (reason: " << to_string(msg.reason) 
+                      << "). Restarting N=" << self->state().N_val << std::endl;
+            
+            auto w = self->spawn<monitored>(make_task_worker_behavior, 
+                                          self->state().prog, 
+                                          self->state().N_val, 
+                                          self->state().pool, 
+                                          self->state().stats_actor, 
+                                          self->state().exit_actor);
+            self->mail(self->state().res).send(w);
+        } else {
+            self->quit();
+        }
+    });
+
+    return {
+        [=](response_token_ptr res) {
+            if (res->getType() == LAUNCH_RESPONSE) {
+                self->state().res = res;
+                auto w = self->spawn<monitored>(make_task_worker_behavior, 
+                                              self->state().prog, 
+                                              self->state().N_val, 
+                                              self->state().pool, 
+                                              self->state().stats_actor, 
+                                              self->state().exit_actor);
+                self->mail(res).send(w);
+            }
+        }
+    };
+}
+
+behavior make_task_supervisor_behavior(stateful_actor<supervisor_state>* self, 
+                                      program_ptr prog, 
+                                      int N_val, 
+                                      std::shared_ptr<MatrixPool> pool, 
+                                      actor exit_actor, 
+                                      actor stats_actor) {
     auto& st = self->state();
     st.prog = std::move(prog);
     st.N_val = N_val;
     st.pool = std::move(pool);
-    return task_actor_fun(self, exit_actor); // Pass exit_actor to task_actor_fun
+    st.exit_actor = exit_actor;
+    st.stats_actor = stats_actor;
+    return task_supervisor_fun(self);
 }
 
 template <class Fn>
@@ -102,16 +185,29 @@ double time_run(Fn&& fn) {
     return elapsed.count();
 }
 
+
+static std::vector<caf::cuda::mem_ptr<char>> pressure_holder;
+
+void apply_memory_pressure(int device_id, size_t target_free_bytes) {
+    auto dev = manager::get().find_device(device_id);
+    size_t total = dev->total_memory_bytes();
+    if (total > target_free_bytes) {
+        size_t to_allocate = total - target_free_bytes;
+        auto arg = create_out_arg_with_size<char>(to_allocate);
+        static command_runner<out<char>> p_runner;
+        pressure_holder.push_back(p_runner.transfer_memory(device_id, 0, arg));
+        std::cout << "[MAIN] Device " << device_id << " memory pressure: allocated " 
+                  << to_allocate / (1024*1024) << " MB, " 
+                  << target_free_bytes / (1024*1024) << " MB left free." << std::endl;
+    }
+}
+
 void run_scheduler_integration_scaling_test(actor_system& sys) {
     const int min_N = 32;
     const int max_N = 2048;
     const int num_distinct_sizes = 10;
-    const std::vector<int> actor_counts = {50000};
-
-
-    //  const std::vector<int> actor_counts = {5};
-
-
+    // const std::vector<int> actor_counts = {50000};
+    const std::vector<int> actor_counts = {2000};
 
     // Generate deterministic random pool once
     auto pool_ptr = std::make_shared<MatrixPool>(
@@ -125,6 +221,9 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         manager::init(sys, config);
         auto& mgr = manager::get();
 
+        // Occupy GPU memory such that only 1 GB remains
+        apply_memory_pressure(0, 1024ULL * 1024ULL * 1024ULL);
+
         // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
         mgr.toggle_scheduler_actor(8, 1);
 
@@ -141,6 +240,8 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         // Spawn the exit actor for this specific test run (moved inside the loop)
         auto exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_tasks);
 
+        auto stats_actor = sys.spawn(stats_actor_fun);
+
         // Spawn task actors and prepare tokens
         for (int i = 0; i < num_tasks; ++i) {
             int current_N = available_Ns[dist_N_idx(rng)];
@@ -148,14 +249,15 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
                            (current_N + THREADS - 1) / THREADS, 1, 
                            THREADS, THREADS, 1);
             
-            auto worker = sys.spawn(make_task_actor_behavior,
-                                    program,
-                                    current_N, 
-                                    pool_ptr,
-                                    exit_actor); // Pass exit_actor to task actors
+            auto supervisor = sys.spawn(make_task_supervisor_behavior,
+                                       program,
+                                       current_N, 
+                                       pool_ptr,
+                                       exit_actor,
+                                       stats_actor);
 
             tokens.push_back(make_launch_token(program, range, 0, 
-                             "task_" + std::to_string(i), worker));
+                             "task_" + std::to_string(i), supervisor));
         }
 
         double elapsed = time_run([&]() {

From e6ab2d2d215489d4447927e2d1fe6ce039ce2ca7 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 08:54:57 -0600
Subject: [PATCH 0941/1000] Made changes to fix intial OOM errors and attempt
 to add more memory pressure.

---
 .../scheduler_integration_test.cpp            | 75 +++++++++++--------
 1 file changed, 42 insertions(+), 33 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index 4299f9952f..5563c1d019 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -77,6 +77,7 @@ behavior task_worker_fun(stateful_actor<task_actor_state>* self, caf::actor stat
     return {
         [=](response_token_ptr res) mutable {
             try {
+                // std::cout << "[WORKER] Starting task N=" << self->state().N_val << std::endl;
                 auto& st = self->state();
                 
                 // We need to cast the base response_token to access the specific nd_range stored in it.
@@ -128,33 +129,36 @@ behavior make_task_worker_behavior(stateful_actor<task_actor_state>* self, progr
 }
 
 behavior task_supervisor_fun(stateful_actor<supervisor_state>* self) {
-    self->set_down_handler([=](down_msg& msg) {
-        if (msg.reason != exit_reason::normal) {
-            std::cout << "[SUPERVISOR] Task worker failed (reason: " << to_string(msg.reason) 
-                      << "). Restarting N=" << self->state().N_val << std::endl;
-            
-            auto w = self->spawn<monitored>(make_task_worker_behavior, 
-                                          self->state().prog, 
-                                          self->state().N_val, 
-                                          self->state().pool, 
-                                          self->state().stats_actor, 
-                                          self->state().exit_actor);
-            self->mail(self->state().res).send(w);
-        } else {
-            self->quit();
-        }
-    });
-
     return {
+        [=](down_msg& msg) {
+            if (msg.reason != exit_reason::normal) {
+                std::cout << "[SUPERVISOR] Task worker failed (reason: " << to_string(msg.reason) 
+                          << "). Restarting N=" << self->state().N_val << std::endl;
+                
+                // self->state().stats_actor->send(0); // Optional: could track restarts in stats
+                
+                auto w = self->spawn(make_task_worker_behavior, 
+                                              self->state().prog, 
+                                              self->state().N_val, 
+                                              self->state().pool, 
+                                              self->state().stats_actor, 
+                                              self->state().exit_actor);
+                self->monitor(w);
+                self->mail(self->state().res).send(w);
+            } else {
+                self->quit();
+            }
+        },
         [=](response_token_ptr res) {
             if (res->getType() == LAUNCH_RESPONSE) {
                 self->state().res = res;
-                auto w = self->spawn<monitored>(make_task_worker_behavior, 
+                auto w = self->spawn(make_task_worker_behavior, 
                                               self->state().prog, 
                                               self->state().N_val, 
                                               self->state().pool, 
                                               self->state().stats_actor, 
                                               self->state().exit_actor);
+                self->monitor(w);
                 self->mail(res).send(w);
             }
         }
@@ -190,24 +194,28 @@ static std::vector<caf::cuda::mem_ptr<char>> pressure_holder;
 
 void apply_memory_pressure(int device_id, size_t target_free_bytes) {
     auto dev = manager::get().find_device(device_id);
-    size_t total = dev->total_memory_bytes();
-    if (total > target_free_bytes) {
-        size_t to_allocate = total - target_free_bytes;
-        auto arg = create_out_arg_with_size<char>(to_allocate);
-        static command_runner<out<char>> p_runner;
-        pressure_holder.push_back(p_runner.transfer_memory(device_id, 0, arg));
-        std::cout << "[MAIN] Device " << device_id << " memory pressure: allocated " 
-                  << to_allocate / (1024*1024) << " MB, " 
-                  << target_free_bytes / (1024*1024) << " MB left free." << std::endl;
+    size_t available = dev->available_memory_bytes();
+    if (available > target_free_bytes) {
+        size_t to_allocate = available - target_free_bytes;
+        try {
+            auto arg = create_out_arg_with_size<char>(to_allocate);
+            static command_runner<out<char>> p_runner;
+            pressure_holder.push_back(p_runner.transfer_memory(device_id, 0, arg));
+            std::cout << "[MAIN] Device " << device_id << " memory pressure: allocated " 
+                      << to_allocate / (1024*1024) << " MB from available, " 
+                      << target_free_bytes / (1024*1024) << " MB left free." << std::endl;
+        } catch (const std::exception& e) {
+            std::cerr << "[MAIN] Warning: Initial memory pressure failed: " << e.what() << std::endl;
+        }
     }
 }
 
 void run_scheduler_integration_scaling_test(actor_system& sys) {
-    const int min_N = 32;
-    const int max_N = 2048;
+    const int min_N = 2048;
+    const int max_N = 4096;
     const int num_distinct_sizes = 10;
     // const std::vector<int> actor_counts = {50000};
-    const std::vector<int> actor_counts = {2000};
+    const std::vector<int> actor_counts = {3000};
 
     // Generate deterministic random pool once
     auto pool_ptr = std::make_shared<MatrixPool>(
@@ -221,11 +229,11 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         manager::init(sys, config);
         auto& mgr = manager::get();
 
-        // Occupy GPU memory such that only 1 GB remains
-        apply_memory_pressure(0, 1024ULL * 1024ULL * 1024ULL);
+        // Occupy GPU memory such that only 300 MB remains to force OOMs during task runs
+        apply_memory_pressure(0, 1500ULL * 1024ULL * 1024ULL);
 
         // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
-        mgr.toggle_scheduler_actor(8, 1);
+        mgr.toggle_scheduler_actor(14, 1);
 
         auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
         const int THREADS = 32;
@@ -271,6 +279,7 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         });
 
         std::cout << "Run complete. Time: " << elapsed << " s\n";
+        pressure_holder.clear();
         manager::shutdown(); // Reset manager state for the next potential iteration
     }
 }

From 9170d303fd0bda22fea96e07d52d00a13adec377 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 09:01:23 -0600
Subject: [PATCH 0942/1000] made out size be sizet type to prevent integer
 overflow errors

---
 libcaf_cuda/caf/cuda/helpers.hpp | 2 +-
 libcaf_cuda/caf/cuda/types.hpp   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/helpers.hpp b/libcaf_cuda/caf/cuda/helpers.hpp
index 80be75ec4a..7de8446f30 100644
--- a/libcaf_cuda/caf/cuda/helpers.hpp
+++ b/libcaf_cuda/caf/cuda/helpers.hpp
@@ -90,7 +90,7 @@ out<T> create_out_arg(const std::vector<T>& buffer) {
 
 // Create `out<T>` from scalar or vector
 template <typename T>
-out<T> create_out_arg_with_size(int size) {
+out<T> create_out_arg_with_size(size_t size) {
   return out<T>{size};
 }
 
diff --git a/libcaf_cuda/caf/cuda/types.hpp b/libcaf_cuda/caf/cuda/types.hpp
index dab8945880..469c711779 100644
--- a/libcaf_cuda/caf/cuda/types.hpp
+++ b/libcaf_cuda/caf/cuda/types.hpp
@@ -162,7 +162,7 @@ template <typename T>
 class out_impl {
 private:
   T scalar_;
-  int size_ = 1;
+  size_t size_ = 1;
   bool is_scalar_ = true;
   bool moved_from_ = false;
 
@@ -177,7 +177,7 @@ class out_impl {
   out_impl() : scalar_{}, size_(1), is_scalar_(true) {}
 
   // allocate GPU buffer of given size
-  explicit out_impl(int size)
+  explicit out_impl(size_t size)
     : scalar_{}, size_(size), is_scalar_(false) {}
 
   explicit out_impl(const std::vector<T>& buf)

From 8bb6477ea9665aaf18a535344007fad1703742d5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 09:01:56 -0600
Subject: [PATCH 0943/1000] Updated test to have memory checks.

---
 .../scheduler_integration_test.cpp            | 20 ++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index 5563c1d019..5a37912e27 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -201,9 +201,23 @@ void apply_memory_pressure(int device_id, size_t target_free_bytes) {
             auto arg = create_out_arg_with_size<char>(to_allocate);
             static command_runner<out<char>> p_runner;
             pressure_holder.push_back(p_runner.transfer_memory(device_id, 0, arg));
-            std::cout << "[MAIN] Device " << device_id << " memory pressure: allocated " 
-                      << to_allocate / (1024*1024) << " MB from available, " 
-                      << target_free_bytes / (1024*1024) << " MB left free." << std::endl;
+
+            // Synchronize stream 0 to ensure the allocation is complete before checking.
+            auto stream = p_runner.get_stream(0, device_id);
+            CHECK_CUDA(cuStreamSynchronize(stream));
+
+            size_t post_available = dev->available_memory_bytes();
+            std::cout << "[MAIN] Device " << device_id << " memory pressure: allocated "
+                      << to_allocate / (1024 * 1024) << " MB. Actual free: "
+                      << post_available / (1024 * 1024) << " MB." << std::endl;
+
+            // Check if the pressure "stuck". We allow a 50MB tolerance for driver overhead.
+            if (post_available > target_free_bytes + (50ULL * 1024 * 1024)) {
+                std::cerr << "[WARNING] Memory pressure check failed! Expected ~"
+                          << target_free_bytes / (1024 * 1024) << " MB free, but found "
+                          << post_available / (1024 * 1024) << " MB. "
+                          << "The allocation might have been freed prematurely." << std::endl;
+            }
         } catch (const std::exception& e) {
             std::cerr << "[MAIN] Warning: Initial memory pressure failed: " << e.what() << std::endl;
         }

From b62580034b15e43e70a20e512fdfb4bce05baba3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 09:36:53 -0600
Subject: [PATCH 0944/1000] Saving current test. In theory recovery is possible
 but in practice. there needs to be some sort of cooridination mmechanism
 since they are doing partial allocations and crashing each other.

---
 .../scheduler_integration_test.cpp            | 75 ++++++++++++-------
 1 file changed, 46 insertions(+), 29 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index 5a37912e27..150f575c5f 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -74,7 +74,18 @@ MatrixPool create_matrix_pool_random(
 
 // This actor represents a single task that requests permission from the scheduler.
 behavior task_worker_fun(stateful_actor<task_actor_state>* self, caf::actor stats_actor, caf::actor exit_actor) {
+   
+   
+    //   self->attach_functor([](const caf::error& reason) {
+    //     std::cout << "[worker EXIT] "
+    //               << to_string(reason)
+    //               << std::endl;
+    // });
+   
+   
+   
     return {
+        
         [=](response_token_ptr res) mutable {
             try {
                 // std::cout << "[WORKER] Starting task N=" << self->state().N_val << std::endl;
@@ -109,6 +120,7 @@ behavior task_worker_fun(stateful_actor<task_actor_state>* self, caf::actor stat
                     anon_mail(0).send(self_hdl); // Signal normal completion
                 });
             } catch (const std::exception& e) {
+                // std::cerr << "[WORKER] Exception caught: " << e.what() << std::endl;
                 self->quit(sec::runtime_error);
             }
         },
@@ -129,38 +141,41 @@ behavior make_task_worker_behavior(stateful_actor<task_actor_state>* self, progr
 }
 
 behavior task_supervisor_fun(stateful_actor<supervisor_state>* self) {
+   
+   
+    //   self->attach_functor([](const caf::error& reason) {
+    //     std::cout << "[SUPERVISOR EXIT] "
+    //               << to_string(reason)
+    //               << std::endl;
+    // });
+   
+   
     return {
-        [=](down_msg& msg) {
-            if (msg.reason != exit_reason::normal) {
-                std::cout << "[SUPERVISOR] Task worker failed (reason: " << to_string(msg.reason) 
-                          << "). Restarting N=" << self->state().N_val << std::endl;
-                
-                // self->state().stats_actor->send(0); // Optional: could track restarts in stats
-                
-                auto w = self->spawn(make_task_worker_behavior, 
-                                              self->state().prog, 
-                                              self->state().N_val, 
-                                              self->state().pool, 
-                                              self->state().stats_actor, 
-                                              self->state().exit_actor);
-                self->monitor(w);
-                self->mail(self->state().res).send(w);
-            } else {
-                self->quit();
-            }
-        },
         [=](response_token_ptr res) {
             if (res->getType() == LAUNCH_RESPONSE) {
                 self->state().res = res;
-                auto w = self->spawn(make_task_worker_behavior, 
-                                              self->state().prog, 
-                                              self->state().N_val, 
-                                              self->state().pool, 
-                                              self->state().stats_actor, 
-                                              self->state().exit_actor);
-                self->monitor(w);
+                auto w = self->spawn(make_task_worker_behavior,
+                                     self->state().prog,
+                                     self->state().N_val,
+                                     self->state().pool,
+                                     self->state().stats_actor,
+                                     self->state().exit_actor);
+
+                self->monitor(w, [self, res](const error& err) mutable {
+                    if (err) {
+                        std::cout << "[SUPERVISOR] Task worker failed (" << to_string(err)
+                                  << "). Restarting N=" << self->state().N_val << std::endl;
+                        self->mail(res).send(self); // Trigger restart logic by re-sending token to self
+                    } else {
+                        self->quit();
+                    }
+                });
+                
                 self->mail(res).send(w);
+            
+            
             }
+            
         }
     };
 }
@@ -225,8 +240,8 @@ void apply_memory_pressure(int device_id, size_t target_free_bytes) {
 }
 
 void run_scheduler_integration_scaling_test(actor_system& sys) {
-    const int min_N = 2048;
-    const int max_N = 4096;
+    const int min_N = 1024;
+    const int max_N = 3000;
     const int num_distinct_sizes = 10;
     // const std::vector<int> actor_counts = {50000};
     const std::vector<int> actor_counts = {3000};
@@ -247,7 +262,7 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         apply_memory_pressure(0, 1500ULL * 1024ULL * 1024ULL);
 
         // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
-        mgr.toggle_scheduler_actor(14, 1);
+        mgr.toggle_scheduler_actor(32, 1);
 
         auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
         const int THREADS = 32;
@@ -264,6 +279,7 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
 
         auto stats_actor = sys.spawn(stats_actor_fun);
 
+        std::vector<caf::actor> sups;
         // Spawn task actors and prepare tokens
         for (int i = 0; i < num_tasks; ++i) {
             int current_N = available_Ns[dist_N_idx(rng)];
@@ -277,6 +293,7 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
                                        pool_ptr,
                                        exit_actor,
                                        stats_actor);
+            sups.push_back(supervisor);
 
             tokens.push_back(make_launch_token(program, range, 0, 
                              "task_" + std::to_string(i), supervisor));

From c76ffef271124ed65a63c0a93a38d389e1d3027d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 09:44:56 -0600
Subject: [PATCH 0945/1000] Added random exponential backoff as a means of
 coordinating retries.

---
 .../scheduler_integration_test.cpp               | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index 150f575c5f..227c6bfea7 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -34,6 +34,7 @@ struct supervisor_state {
     actor exit_actor;
     actor stats_actor;
     response_token_ptr res;
+    std::mt19937 rng;
 };
 
 struct stats_actor_state {
@@ -120,7 +121,7 @@ behavior task_worker_fun(stateful_actor<task_actor_state>* self, caf::actor stat
                     anon_mail(0).send(self_hdl); // Signal normal completion
                 });
             } catch (const std::exception& e) {
-                // std::cerr << "[WORKER] Exception caught: " << e.what() << std::endl;
+                std::cerr << "[WORKER] Exception caught: " << e.what() << std::endl;
                 self->quit(sec::runtime_error);
             }
         },
@@ -163,9 +164,13 @@ behavior task_supervisor_fun(stateful_actor<supervisor_state>* self) {
 
                 self->monitor(w, [self, res](const error& err) mutable {
                     if (err) {
+                        std::uniform_int_distribution<> dis(100, 1000);
+                        auto backoff = std::chrono::milliseconds(dis(self->state().rng));
+
                         std::cout << "[SUPERVISOR] Task worker failed (" << to_string(err)
-                                  << "). Restarting N=" << self->state().N_val << std::endl;
-                        self->mail(res).send(self); // Trigger restart logic by re-sending token to self
+                                  << "). Restarting N=" << self->state().N_val 
+                                  << " after " << backoff.count() << "ms backoff." << std::endl;
+                        self->mail(res).delay(backoff).send(self); // Trigger restart logic with delay
                     } else {
                         self->quit();
                     }
@@ -192,6 +197,7 @@ behavior make_task_supervisor_behavior(stateful_actor<supervisor_state>* self,
     st.pool = std::move(pool);
     st.exit_actor = exit_actor;
     st.stats_actor = stats_actor;
+    st.rng.seed(std::random_device{}());
     return task_supervisor_fun(self);
 }
 
@@ -240,8 +246,8 @@ void apply_memory_pressure(int device_id, size_t target_free_bytes) {
 }
 
 void run_scheduler_integration_scaling_test(actor_system& sys) {
-    const int min_N = 1024;
-    const int max_N = 3000;
+    const int min_N = 2000;
+    const int max_N = 2048;
     const int num_distinct_sizes = 10;
     // const std::vector<int> actor_counts = {50000};
     const std::vector<int> actor_counts = {3000};

From 21b38b5fac1ce331bc5676fcc7cc2470b0686d20 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 09:55:16 -0600
Subject: [PATCH 0946/1000] Added restart limits and statistics tracking to bad
 jobs.

---
 .../scheduler_integration_test.cpp            | 60 ++++++++++++++-----
 1 file changed, 46 insertions(+), 14 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index 227c6bfea7..d987d7ef3f 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -8,6 +8,7 @@
 #include <unordered_map>
 #include <unordered_set>
 #include <algorithm>
+#include <iomanip>
 
 using namespace caf;
 using namespace caf::cuda;
@@ -35,18 +36,36 @@ struct supervisor_state {
     actor stats_actor;
     response_token_ptr res;
     std::mt19937 rng;
+    int retries = 0;
 };
 
 struct stats_actor_state {
-    int completed = 0;
+    int succeeded = 0;
+    int failed = 0;
+
+    ~stats_actor_state() {
+        int total = succeeded + failed;
+        std::cout << "\n=====================================\n"
+                  << "[STATS REPORT] Iteration Complete\n"
+                  << "  Succeeded: " << succeeded << "\n"
+                  << "  Failed:    " << failed << "\n";
+        if (total > 0) {
+            double pct = (static_cast<double>(succeeded) / total) * 100.0;
+            std::cout << "  Success %: " << std::fixed << std::setprecision(2) << pct << "%\n";
+        }
+        std::cout << "=====================================\n" << std::endl;
+    }
 };
 
 behavior stats_actor_fun(stateful_actor<stats_actor_state>* self) {
     return {
-        [=](int count) {
-            self->state().completed += count;
-            if (self->state().completed % 1000 == 0) {
-                std::cout << "[STATS] Jobs completed: " << self->state().completed << std::endl;
+        [=](bool success) {
+            if (success) self->state().succeeded++;
+            else self->state().failed++;
+
+            int total = self->state().succeeded + self->state().failed;
+            if (total % 1000 == 0) {
+                std::cout << "[STATS] Progress: " << total << " jobs processed." << std::endl;
             }
         }
     };
@@ -114,9 +133,9 @@ behavior task_worker_fun(stateful_actor<task_actor_state>* self, caf::actor stat
 
                 // The launch_response_token is released inside the callback 
                 // to signal to the scheduler that the resource is free. (No serial verification here)
-                runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [res, stats_actor, exit_actor, h_c, self_hdl](int* /*ptr*/, size_t /*sz*/) {
+                runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [res, stats_actor, exit_actor, h_c, self_hdl](int* /*ptr*/, size_t /*sz*/) mutable {
                     res->release();
-                    anon_mail(1).send(stats_actor);
+                    anon_mail(true).send(stats_actor);
                     anon_mail(1).send(exit_actor);
                     anon_mail(0).send(self_hdl); // Signal normal completion
                 });
@@ -164,13 +183,25 @@ behavior task_supervisor_fun(stateful_actor<supervisor_state>* self) {
 
                 self->monitor(w, [self, res](const error& err) mutable {
                     if (err) {
-                        std::uniform_int_distribution<> dis(100, 1000);
-                        auto backoff = std::chrono::milliseconds(dis(self->state().rng));
-
-                        std::cout << "[SUPERVISOR] Task worker failed (" << to_string(err)
-                                  << "). Restarting N=" << self->state().N_val 
-                                  << " after " << backoff.count() << "ms backoff." << std::endl;
-                        self->mail(res).delay(backoff).send(self); // Trigger restart logic with delay
+                        self->state().retries++;
+                        if (self->state().retries > 5) {
+                            std::cout << "[SUPERVISOR] Task N=" << self->state().N_val 
+                                      << " failed permanently after 5 retries. Giving up." << std::endl;
+                            
+                            self->mail(false).send(self->state().stats_actor);
+                            self->mail(1).send(self->state().exit_actor);
+                            res->release();
+                            self->quit();
+                        } else {
+                            std::uniform_int_distribution<> dis(100, 1000);
+                            auto backoff = std::chrono::milliseconds(dis(self->state().rng));
+
+                            std::cout << "[SUPERVISOR] Task worker failed (" << to_string(err)
+                                      << "). Restarting N=" << self->state().N_val 
+                                      << " after " << backoff.count() << "ms backoff (Retry " 
+                                      << self->state().retries << "/5)." << std::endl;
+                            self->mail(res).delay(backoff).send(self); // Trigger restart logic with delay
+                        }
                     } else {
                         self->quit();
                     }
@@ -313,6 +344,7 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
             // block until the exit_actor terminates (signaling all 50k tasks are done).
             scoped_actor self{sys};
             self->wait_for(exit_actor);
+            anon_send_exit(stats_actor, exit_reason::user_shutdown);
         });
 
         std::cout << "Run complete. Time: " << elapsed << " s\n";

From 8d1ac0f1b6a95b020b4a0a42f5b8c1891d35cbce Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 10:08:30 -0600
Subject: [PATCH 0947/1000] Moved data reporting to stats actor and put memory
 pressure to all GPUs instead of just one.

---
 .../scheduler_integration_test.cpp            | 22 ++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index d987d7ef3f..9f2cd01417 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -42,6 +42,7 @@ struct supervisor_state {
 struct stats_actor_state {
     int succeeded = 0;
     int failed = 0;
+    double runtime = 0.0;
 
     ~stats_actor_state() {
         int total = succeeded + failed;
@@ -53,6 +54,9 @@ struct stats_actor_state {
             double pct = (static_cast<double>(succeeded) / total) * 100.0;
             std::cout << "  Success %: " << std::fixed << std::setprecision(2) << pct << "%\n";
         }
+        if (runtime > 0.0) {
+            std::cout << "  Runtime:   " << std::fixed << std::setprecision(2) << runtime << " s\n";
+        }
         std::cout << "=====================================\n" << std::endl;
     }
 };
@@ -67,6 +71,9 @@ behavior stats_actor_fun(stateful_actor<stats_actor_state>* self) {
             if (total % 1000 == 0) {
                 std::cout << "[STATS] Progress: " << total << " jobs processed." << std::endl;
             }
+        },
+        [=](double runtime) {
+            self->state().runtime = runtime;
         }
     };
 }
@@ -295,8 +302,12 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         manager::init(sys, config);
         auto& mgr = manager::get();
 
-        // Occupy GPU memory such that only 300 MB remains to force OOMs during task runs
-        apply_memory_pressure(0, 1500ULL * 1024ULL * 1024ULL);
+        
+        
+        // Occupy GPU memory such that only 1500 MB remains to force OOMs during task runs
+        for (int i=0; i < mgr.get_num_devices(); i++) {
+            apply_memory_pressure(i, 1500ULL * 1024ULL * 1024ULL);
+        }
 
         // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
         mgr.toggle_scheduler_actor(32, 1);
@@ -336,18 +347,19 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
                              "task_" + std::to_string(i), supervisor));
         }
 
+        scoped_actor self{sys};
         double elapsed = time_run([&]() {
             std::cout << "[MAIN] Dispatching batch to scheduler..." << std::endl;
             mgr.send_scheduler_actor_message(std::move(tokens));
 
             // The dispatch is asynchronous. To get an accurate measurement, we must
             // block until the exit_actor terminates (signaling all 50k tasks are done).
-            scoped_actor self{sys};
             self->wait_for(exit_actor);
-            anon_send_exit(stats_actor, exit_reason::user_shutdown);
         });
 
-        std::cout << "Run complete. Time: " << elapsed << " s\n";
+        self->mail(elapsed).send(stats_actor);
+        self->send_exit(stats_actor, exit_reason::user_shutdown);
+        self->wait_for(stats_actor);
         pressure_holder.clear();
         manager::shutdown(); // Reset manager state for the next potential iteration
     }

From 4b44295dc9027a3c7650e8822d74807af7a2c2ac Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 10:22:41 -0600
Subject: [PATCH 0948/1000] Added job retry stats tracking.

---
 .../scheduler_integration_test.cpp            | 59 +++++++++++++++----
 1 file changed, 46 insertions(+), 13 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index 9f2cd01417..12f190697f 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -40,19 +40,34 @@ struct supervisor_state {
 };
 
 struct stats_actor_state {
-    int succeeded = 0;
+    std::map<int, int> succeeded_by_retries; // retry_count -> num_jobs
     int failed = 0;
     double runtime = 0.0;
 
     ~stats_actor_state() {
-        int total = succeeded + failed;
+        int total_succeeded = 0;
+        for (auto const& [retries, count] : succeeded_by_retries) {
+            total_succeeded += count;
+        }
+        int total = total_succeeded + failed;
+
         std::cout << "\n=====================================\n"
                   << "[STATS REPORT] Iteration Complete\n"
-                  << "  Succeeded: " << succeeded << "\n"
-                  << "  Failed:    " << failed << "\n";
+                  << "  Total Processed: " << total << "\n"
+                  << "  Total Succeeded: " << total_succeeded << "\n"
+                  << "  Total Failed:    " << failed << "\n";
+
+        if (total_succeeded > 0) {
+            std::cout << "  Succeeded by Retries:\n";
+            for (auto const& [retries, count] : succeeded_by_retries) {
+                double pct = (static_cast<double>(count) / total_succeeded) * 100.0;
+                std::cout << "    " << retries << " retries: " << count << " jobs ("
+                          << std::fixed << std::setprecision(2) << pct << "%)\n";
+            }
+        }
         if (total > 0) {
-            double pct = (static_cast<double>(succeeded) / total) * 100.0;
-            std::cout << "  Success %: " << std::fixed << std::setprecision(2) << pct << "%\n";
+            double overall_pct = (static_cast<double>(total_succeeded) / total) * 100.0;
+            std::cout << "  Overall Success %: " << std::fixed << std::setprecision(2) << overall_pct << "%\n";
         }
         if (runtime > 0.0) {
             std::cout << "  Runtime:   " << std::fixed << std::setprecision(2) << runtime << " s\n";
@@ -63,11 +78,19 @@ struct stats_actor_state {
 
 behavior stats_actor_fun(stateful_actor<stats_actor_state>* self) {
     return {
-        [=](bool success) {
-            if (success) self->state().succeeded++;
-            else self->state().failed++;
+        [=](bool success, int retries_taken) { // Changed signature
+            if (success) {
+                self->state().succeeded_by_retries[retries_taken]++;
+            } else {
+                self->state().failed++;
+            }
+
+            int total_succeeded = 0;
+            for (auto const& [retries, count] : self->state().succeeded_by_retries) {
+                total_succeeded += count;
+            }
+            int total = total_succeeded + self->state().failed;
 
-            int total = self->state().succeeded + self->state().failed;
             if (total % 1000 == 0) {
                 std::cout << "[STATS] Progress: " << total << " jobs processed." << std::endl;
             }
@@ -141,8 +164,9 @@ behavior task_worker_fun(stateful_actor<task_actor_state>* self, caf::actor stat
                 // The launch_response_token is released inside the callback 
                 // to signal to the scheduler that the resource is free. (No serial verification here)
                 runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [res, stats_actor, exit_actor, h_c, self_hdl](int* /*ptr*/, size_t /*sz*/) mutable {
-                    res->release();
-                    anon_mail(true).send(stats_actor);
+                    res->release(); // Release the token
+                    // The worker no longer sends directly to stats_actor.
+                    // It signals normal completion to itself, which causes its supervisor to be notified.
                     anon_mail(1).send(exit_actor);
                     anon_mail(0).send(self_hdl); // Signal normal completion
                 });
@@ -192,10 +216,13 @@ behavior task_supervisor_fun(stateful_actor<supervisor_state>* self) {
                     if (err) {
                         self->state().retries++;
                         if (self->state().retries > 5) {
+                            // Max retries reached, permanent failure
                             std::cout << "[SUPERVISOR] Task N=" << self->state().N_val 
                                       << " failed permanently after 5 retries. Giving up." << std::endl;
                             
-                            self->mail(false).send(self->state().stats_actor);
+                            // Send failure to stats_actor with the retry count (5)
+                            self->mail(false, self->state().retries).send(self->state().stats_actor);
+                            
                             self->mail(1).send(self->state().exit_actor);
                             res->release();
                             self->quit();
@@ -210,6 +237,9 @@ behavior task_supervisor_fun(stateful_actor<supervisor_state>* self) {
                             self->mail(res).delay(backoff).send(self); // Trigger restart logic with delay
                         }
                     } else {
+                        // Worker exited normally (success)
+                        // Send success to stats_actor with the retry count
+                        self->mail(true, self->state().retries).send(self->state().stats_actor);
                         self->quit();
                     }
                 });
@@ -354,9 +384,12 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
 
             // The dispatch is asynchronous. To get an accurate measurement, we must
             // block until the exit_actor terminates (signaling all 50k tasks are done).
+            scoped_actor self{sys};
             self->wait_for(exit_actor);
+            anon_send_exit(stats_actor, exit_reason::user_shutdown);
         });
 
+        std::cout << "Run complete. Time: " << elapsed << " s\n";
         self->mail(elapsed).send(stats_actor);
         self->send_exit(stats_actor, exit_reason::user_shutdown);
         self->wait_for(stats_actor);

From 6573f6357a5a4ed84b2f84cffcb0c193792c2e0b Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 10:52:25 -0600
Subject: [PATCH 0949/1000] Updated test to track stats better.

---
 .../scheduler_integration_test.cpp            | 117 +++++++++++-------
 1 file changed, 74 insertions(+), 43 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index 12f190697f..e7c0eb9731 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -40,63 +40,93 @@ struct supervisor_state {
 };
 
 struct stats_actor_state {
-    std::map<int, int> succeeded_by_retries; // retry_count -> num_jobs
+    std::map<int, int> succeeded_by_retries;
     int failed = 0;
     double runtime = 0.0;
 
-    ~stats_actor_state() {
-        int total_succeeded = 0;
-        for (auto const& [retries, count] : succeeded_by_retries) {
-            total_succeeded += count;
-        }
-        int total = total_succeeded + failed;
+    // prevents double-finalize
+    bool finalized = false;
+};
 
-        std::cout << "\n=====================================\n"
-                  << "[STATS REPORT] Iteration Complete\n"
-                  << "  Total Processed: " << total << "\n"
-                  << "  Total Succeeded: " << total_succeeded << "\n"
-                  << "  Total Failed:    " << failed << "\n";
 
-        if (total_succeeded > 0) {
-            std::cout << "  Succeeded by Retries:\n";
-            for (auto const& [retries, count] : succeeded_by_retries) {
-                double pct = (static_cast<double>(count) / total_succeeded) * 100.0;
-                std::cout << "    " << retries << " retries: " << count << " jobs ("
-                          << std::fixed << std::setprecision(2) << pct << "%)\n";
-            }
-        }
-        if (total > 0) {
-            double overall_pct = (static_cast<double>(total_succeeded) / total) * 100.0;
-            std::cout << "  Overall Success %: " << std::fixed << std::setprecision(2) << overall_pct << "%\n";
-        }
-        if (runtime > 0.0) {
-            std::cout << "  Runtime:   " << std::fixed << std::setprecision(2) << runtime << " s\n";
-        }
-        std::cout << "=====================================\n" << std::endl;
-    }
-};
 
 behavior stats_actor_fun(stateful_actor<stats_actor_state>* self) {
     return {
-        [=](bool success, int retries_taken) { // Changed signature
+
+        // SUCCESS / FAILURE REPORTING
+        [=](bool success, int retries_taken) {
+            auto& st = self->state();
+
             if (success) {
-                self->state().succeeded_by_retries[retries_taken]++;
+                st.succeeded_by_retries[retries_taken]++;
             } else {
-                self->state().failed++;
+                st.failed++;
             }
 
-            int total_succeeded = 0;
-            for (auto const& [retries, count] : self->state().succeeded_by_retries) {
-                total_succeeded += count;
-            }
-            int total = total_succeeded + self->state().failed;
+            int total = 0;
+            for (auto const& [r, c] : st.succeeded_by_retries)
+                total += c;
+            total += st.failed;
 
-            if (total % 1000 == 0) {
-                std::cout << "[STATS] Progress: " << total << " jobs processed." << std::endl;
+            if (total % 1000 == 0 && total > 0) {
+                std::cout << "[STATS] Progress: "
+                          << total << " jobs processed\n";
             }
         },
+
+        // RUNTIME UPDATE
         [=](double runtime) {
             self->state().runtime = runtime;
+        },
+
+        // FINAL REPORT (IMPORTANT FIX)
+        [=](char finalize_stats) {
+            auto& st = self->state();
+
+            if (st.finalized)
+                return;
+
+            st.finalized = true;
+
+            int total_succeeded = 0;
+            for (auto const& [r, c] : st.succeeded_by_retries)
+                total_succeeded += c;
+
+            int total = total_succeeded + st.failed;
+
+            double success_pct = (total > 0)
+                ? (100.0 * total_succeeded / total)
+                : 0.0;
+
+            std::cout << "\n=====================================\n";
+            std::cout << "[STATS REPORT] Iteration Complete\n";
+            std::cout << "  Total Processed: " << total << "\n";
+            std::cout << "  Total Succeeded: " << total_succeeded << "\n";
+            std::cout << "  Total Failed:    " << st.failed << "\n";
+
+            std::cout << "  Succeeded by Retries:\n";
+            for (auto const& [r, c] : st.succeeded_by_retries) {
+                double pct = (total_succeeded > 0)
+                    ? (100.0 * c / total_succeeded)
+                    : 0.0;
+
+                std::cout << "    " << r << " retries: "
+                          << c << " jobs ("
+                          << std::fixed << std::setprecision(2)
+                          << pct << "%)\n";
+            }
+
+            std::cout << "  Overall Success %: "
+                      << std::fixed << std::setprecision(2)
+                      << success_pct << "%\n";
+
+            if (st.runtime > 0.0) {
+                std::cout << "  Runtime: "
+                          << std::fixed << std::setprecision(3)
+                          << st.runtime << " s\n";
+            }
+
+            std::cout << "=====================================\n";
         }
     };
 }
@@ -386,12 +416,13 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
             // block until the exit_actor terminates (signaling all 50k tasks are done).
             scoped_actor self{sys};
             self->wait_for(exit_actor);
-            anon_send_exit(stats_actor, exit_reason::user_shutdown);
+            // anon_send_exit(stats_actor, exit_reason::user_shutdown);
         });
 
         std::cout << "Run complete. Time: " << elapsed << " s\n";
-        self->mail(elapsed).send(stats_actor);
-        self->send_exit(stats_actor, exit_reason::user_shutdown);
+       // self->mail(elapsed).send(stats_actor);
+        self->mail('c').send(stats_actor);   
+        anon_send_exit(stats_actor, exit_reason::user_shutdown);    
         self->wait_for(stats_actor);
         pressure_holder.clear();
         manager::shutdown(); // Reset manager state for the next potential iteration

From caa38677d9f1c5aa2c75d5978ddafd7a562ce136 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 11:10:30 -0600
Subject: [PATCH 0950/1000] update matrix sizes to introduce more variability

---
 .../scheduler_integration_test.cpp                            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index e7c0eb9731..87a0446001 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -344,8 +344,8 @@ void apply_memory_pressure(int device_id, size_t target_free_bytes) {
 }
 
 void run_scheduler_integration_scaling_test(actor_system& sys) {
-    const int min_N = 2000;
-    const int max_N = 2048;
+    const int min_N = 2048;
+    const int max_N = 4096;
     const int num_distinct_sizes = 10;
     // const std::vector<int> actor_counts = {50000};
     const std::vector<int> actor_counts = {3000};

From 63506b4c868cec9116ddb8333390fe1ec09cf6c1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 13:17:11 -0600
Subject: [PATCH 0951/1000] updated documentation to include info about the
 scheduler actor

---
 libcaf_cuda/documentation.txt | 79 +++++++++++++++++++++++++++++++++++
 1 file changed, 79 insertions(+)

diff --git a/libcaf_cuda/documentation.txt b/libcaf_cuda/documentation.txt
index 1f3541c7d6..8b2fe577fd 100644
--- a/libcaf_cuda/documentation.txt
+++ b/libcaf_cuda/documentation.txt
@@ -258,5 +258,84 @@ The command runner does support kernels with shared memory (actor facade does no
                           );
 }}}
 
+== Multi GPU Scheduling 
+We use actors as a way to provide single node Multi GPU task scheduling. To accomplish this the runtime uses Send Receive Reply IPC as a means
+Of GPU scheduling.
+What this means is that if you want the scheduler to load balance tasks across multiple devices for you, actors cannot directly submit GPU work
+they must instead seek permission from the scheduler actor to do so and only when the scheduler actor tells the actor that its ok it may then
+do GPU work.
+
+The scheduler actor uses tokens as a way to track tasks. A token contains metadata about a task such as what device and stream to execute on and is given 
+to the GPU actor requesting to do work granting it permission to do work.
+Tokens are C++ smart pointers that embedd a message signalling to the scheduler actor that created it that a task is completed in its deconstructor.
+Meaning that the scheduler actor expects that the issued token object lifetime matches that of the GPU task that is being completed. Destroying it 
+too early will make the scheduler actor behave as if the task has been completed. 
+
+Tokens can either be sent to scheduler actor one by one or in batches. Submitting one by one looks like 
+
+{{{
+///sending a single token
+caf::cuda::launch_token token = make_launch_token(program, range, 0, 
+                             "task_name", reply_actor);
+
+mgr.send_scheduler_actor_message(token);
+
+
+
+
+//sending a batch of tokens
+  std::vector<token_ptr> tokens;
+  for (int i = 0; i < num_tasks; ++i) {
+            
+            
+            caf::actor supervisor = sys.spawn(supervisor_behavior);
+
+            tokens.push_back(make_launch_token(program, range, 0, 
+                             "task_" + std::to_string(i), supervisor));
+        }
+
+mgr.send_scheduler_actor_message(tokens);
+
+
+}}}
+
+a request token contains a program_ptr, kernel dimensions, memory being used, a task identifer (string) and
+a handle to an actor that will receive the response token. The scheduler actor does not look at program, range, memory size or task identifers,
+it only cares about the handle to the actor.
+
+
+ {{{
+  //receiving a response token
+
+
+    [=](response_token_ptr res) mutable {
+       if (res->getType() == LAUNCH_RESPONSE) {
+              auto& st = self->state();
+                
+               
+                int N = st.N_val;
+
+                // 1. Setup GPU arguments
+
+                // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
+                auto result_tuple = runner.run_async(st.prog, dims, res, in_a, in_b, out_c, in_n);
+                //alternatively you can use res->getDeviceNumber() and res->getStreamId() to access the values assigned to the actor via the scheduler
+                auto d_c = std::get<2>(result_tuple);
+
+                // 3. Asynchronous Copyback.
+                // Allocate a local buffer for the result to keep the total system memory low.
+                auto h_c = std::make_shared<std::vector<int>>(N * N);
+                // The launch_response_token is released inside the callback 
+                // to signal to the scheduler that the resource is free. 
+                runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [res, exit_actor, h_c](int* /*ptr*/, size_t /*sz*/) {
+                    res->release();//marks the tasks lifetime as complete, alternatively could have just dropped it out of scope.
+                });
+            }
+
+
+ }}}
+
+to send scheduler actor exit messages invoke caf::cuda::manager::shutdown() and this will send exit messages to the scheduler actor
+
 
 For more details, see the example code found at https://github.com/uofs-simlab/actor-framework/tree/main/libcaf_cuda/examples

From 639834faebfa0ca0f2410cd5d6c34b81b20d3b41 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 17 Jun 2026 13:38:59 -0600
Subject: [PATCH 0952/1000] added compiler flags to remove compiler warnings

---
 .../caf/cuda/control-layer/behavior.hpp       |   9 +-
 .../control-layer/core_usage_behavior.hpp     |   3 +-
 .../control-layer/green_light_behavior.hpp    |   2 +-
 .../mmul_batch_scheduler_behavior.hpp         |   7 +-
 .../multilevel_usage_behavior.hpp             |   7 +-
 .../cuda/control-layer/pressure_scheduler.hpp |   3 +-
 .../cuda/control-layer/red_light_behavior.hpp |   4 +-
 .../cuda/control-layer/scheduler_actor.hpp    |   4 +-
 .../control-layer/single_usage_behavior.hpp   |   6 +-
 libcaf_cuda/caf/cuda/control-layer/token.hpp  |   2 +-
 .../caf/cuda/control-layer/transfer_token.hpp |   3 +-
 libcaf_cuda/caf/cuda/global.hpp               |   2 +-
 libcaf_cuda/caf/cuda/manager_config.hpp       |   2 +-
 libcaf_cuda/caf/cuda/program.hpp              |   3 +-
 .../src/control-layer/core_usage_behavior.cpp |   4 +-
 .../mmul_batch_scheduler_behavior.cpp         |  11 +-
 .../multilevel_usage_behavior.cpp             |   3 +-
 .../src/control-layer/red_light_behavior.cpp  |   4 +-
 .../src/control-layer/scheduler_actor.cpp     |   2 +-
 .../control-layer/single_usage_behavior.cpp   |   6 +-
 libcaf_cuda/src/platform.cpp                  |  22 +--
 .../tests/scheduler_integration_test.cpp      | 163 ------------------
 22 files changed, 49 insertions(+), 223 deletions(-)
 delete mode 100644 libcaf_cuda/tests/scheduler_integration_test.cpp

diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
index 57ed52f8ce..521abdee33 100644
--- a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
@@ -33,7 +33,7 @@ class scheduler_actor_behavior {
     virtual void schedule() = 0;
     virtual void receive(const token_ptr& tok) = 0;
 
-    virtual void reclaim(int value, int memory_returned,int runtime,int dependency) {
+    virtual void reclaim([[maybe_unused]] int value, [[maybe_unused]] int memory_returned, [[maybe_unused]] int runtime, [[maybe_unused]] int dependency) {
 	    //default implementation is to do nothing, this should be overidden
 	    //by children classes 
     }
@@ -42,7 +42,7 @@ class scheduler_actor_behavior {
 
     //this is here to ensure that payloads on return can conform to an interface
     //rather than changing the interface to accomidate every scheduling need
-    virtual void reclaim(ack& payload) {
+    virtual void reclaim([[maybe_unused]] ack& payload) {
 	    //default implementation is to do nothing, this should be overidden
 	    //by children classes 
     }
@@ -52,7 +52,7 @@ class scheduler_actor_behavior {
 
     //this method is meant to be a handler for when
     //another scheduler actor queries for more work
-    virtual void handle_load_balance_request(int device_number) {
+    virtual void handle_load_balance_request([[maybe_unused]] int device_number) {
     
 	    //default action is to do nothing and not particpate in load balancing
 	    //whether of not a scheduler wants to participate in load balancing 
@@ -61,7 +61,7 @@ class scheduler_actor_behavior {
 
 
     //method is meant to handle work being sent over from another scheduler actor
-   virtual void receive_work(std::vector<kernel_graph> work_graphs) {
+   virtual void receive_work([[maybe_unused]] std::vector<kernel_graph> work_graphs) {
      //ideally this should not default to do nothing 
     //however I do not have the time implement this on every existing behavior 
     //as of right now
@@ -84,4 +84,3 @@ class scheduler_actor_behavior {
 };
 
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
index 68598606a6..8c20051681 100644
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
@@ -15,7 +15,7 @@ class core_usage_behavior : public scheduler_actor_behavior {
     void on_enter() override;
     void schedule() override;
     void receive(const token_ptr& tok) override;
-    void reclaim(int value /*blocks consumed*/,int memory_returned,int time,int dependency) override; 
+    void reclaim([[maybe_unused]] int value /*blocks consumed*/, [[maybe_unused]] int memory_returned, [[maybe_unused]] int time, [[maybe_unused]] int dependency) override; 
     ~core_usage_behavior() override;  
     std::string name() const override {return "core_usage\n";}
 
@@ -57,4 +57,3 @@ class core_usage_behavior : public scheduler_actor_behavior {
 };
 
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
index eb14717c47..25c2c04a5d 100644
--- a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
@@ -8,7 +8,7 @@ class green_light_behavior : public scheduler_actor_behavior {
     explicit green_light_behavior(scheduler_actor_state& state);
     void on_enter() override;
     void schedule() override;
-    void receive(const token_ptr& tok) override;
+    void receive([[maybe_unused]] const token_ptr& tok) override;
     std::string name() const override {return "green\n";}
 };
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp
index 1bf2894a86..a0740d4524 100644
--- a/libcaf_cuda/caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp
@@ -23,13 +23,13 @@ class mmul_batch_scheduler_behavior : public scheduler_actor_behavior {
     void schedule() override;
     void receive(const token_ptr& tok) override;
 
-    void reclaim(int blocks_consumed, int memory_returned, int time, int dependency_number) override;
+    void reclaim([[maybe_unused]] int blocks_consumed, [[maybe_unused]] int memory_returned, [[maybe_unused]] int time, [[maybe_unused]] int dependency_number) override;
     void reclaim(ack& return_msg) override;
 
     std::string name() const override { return "mmul_batch_scheduler"; }
 
 protected:
-    void process_launch_token(const token_ptr& tok, int stream_id, int assigned_queue);
+    void process_launch_token(const token_ptr& tok, int stream_id, [[maybe_unused]] int assigned_queue);
 
 private:
     enum queue_type { LOW = 0, MED = 1, HIGH = 2 };
@@ -80,6 +80,3 @@ class mmul_batch_scheduler_behavior : public scheduler_actor_behavior {
 };
 
 } // namespace caf::cuda
-
-
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
index 72fba04272..325912907d 100644
--- a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
@@ -29,7 +29,7 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
 
     // reclaim: called when resources are returned; dependency_number
     // allows this behavior to find the graph that might now be ready
-    void reclaim(int blocks_consumed, int memory_returned, int time, int dependency_number) override;
+    void reclaim(int blocks_consumed, int memory_returned, [[maybe_unused]] int time, int dependency_number) override;
 
     //more improved version of reclaim, meant for when we need to dispatch transfer
     //tokens
@@ -42,8 +42,8 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
     //multi GPU load balancing methods
     //by default this scheduler behavior will try to load balance
     //across all gpus 
-    void handle_load_balance_request(int device_number) override;
-    void receive_work(std::vector<kernel_graph> work_graphs) override;
+    void handle_load_balance_request([[maybe_unused]] int device_number) override;
+    void receive_work([[maybe_unused]] std::vector<kernel_graph> work_graphs) override;
     void request_load_balance();
 
 
@@ -105,4 +105,3 @@ class multilevel_usage_behavior : public scheduler_actor_behavior {
 
 
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp b/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
index 1407b209b4..2e6533914e 100644
--- a/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
@@ -35,7 +35,7 @@ class pressure_scheduler : public scheduler_actor_behavior {
 
     // reclaim: called when resources are returned; dependency_number
     // allows this behavior to find the graph that might now be ready
-    void reclaim(int resources_consumed, int memory_returned, int time, int dependency_number) override;
+    void reclaim([[maybe_unused]] int resources_consumed, [[maybe_unused]] int memory_returned, [[maybe_unused]] int time, [[maybe_unused]] int dependency_number) override;
 
     std::string name() const override { return "pressure_scheduler\n"; }
 
@@ -128,4 +128,3 @@ class pressure_scheduler : public scheduler_actor_behavior {
 
 
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
index c5c00057c1..c5f5929553 100644
--- a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
@@ -12,8 +12,8 @@ class red_light_behavior : public scheduler_actor_behavior {
     ~red_light_behavior() noexcept override;
 
 protected:
-    virtual void process_launch_token(const token_ptr& tok, int stream_id);
-    virtual void process_memory_transfer_token(const token_ptr& tok, int stream_id);
+    virtual void process_launch_token([[maybe_unused]] const token_ptr& tok, [[maybe_unused]] int stream_id);
+    virtual void process_memory_transfer_token([[maybe_unused]] const token_ptr& tok, [[maybe_unused]] int stream_id);
 
 };
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
index 61b799c3fd..a4907a3388 100644
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor.hpp
@@ -17,9 +17,9 @@ class CAF_CUDA_EXPORT scheduler_actor : public caf::event_based_actor {
     // Message Handler Methods
     virtual void on_receive(const token_ptr& tok);
     virtual void on_receive_batch(std::vector<token_ptr> tokens);
-    virtual void on_reclaim(int val, int mem, int time, int dep, int stream_id);
+    virtual void on_reclaim([[maybe_unused]] int val, [[maybe_unused]] int mem, [[maybe_unused]] int time, [[maybe_unused]] int dep, [[maybe_unused]] int stream_id);
     virtual void on_set_neighbors(std::vector<caf::actor> neighbors);
-    virtual std::vector<token_ptr> on_steal_request(int requesting_device);
+    virtual std::vector<token_ptr> on_steal_request([[maybe_unused]] int requesting_device);
 
 protected:
     // Scheduling logic
diff --git a/libcaf_cuda/caf/cuda/control-layer/single_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/single_usage_behavior.hpp
index 58b55f78c8..46a8218214 100644
--- a/libcaf_cuda/caf/cuda/control-layer/single_usage_behavior.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/single_usage_behavior.hpp
@@ -19,10 +19,10 @@ class single_usage_behavior : public scheduler_actor_behavior {
     void on_enter() override;
     void schedule() override;
     void receive(const token_ptr& tok) override;
-    void reclaim(int blocks_consumed,
+    void reclaim([[maybe_unused]] int blocks_consumed,
                  int memory_returned,
-                 int time,
-                 int dependency_number) override;
+                 [[maybe_unused]] int time,
+                 [[maybe_unused]] int dependency_number) override;
 
     std::string name() const override { return "single_usage"; }
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/token.hpp b/libcaf_cuda/caf/cuda/control-layer/token.hpp
index 43ef888416..49d49da905 100644
--- a/libcaf_cuda/caf/cuda/control-layer/token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/token.hpp
@@ -23,7 +23,7 @@ class CAF_CUDA_EXPORT token : public caf::ref_counted {
 public:
    virtual ~token() {
     // Print ref count when destructor runs
-    size_t count = ref_count_.load(std::memory_order_acquire);
+    [[maybe_unused]] size_t count = ref_count_.load(std::memory_order_acquire);
     //std::cout << "token object getting deleted, ref_count = " << count << "\n";
 }
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp b/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp
index 9c57875a1e..b9cb019e7a 100644
--- a/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp
@@ -70,7 +70,7 @@ class CAF_CUDA_EXPORT transfer_token : public response_token {
             transfer_ack ack_obj{dependency_number_};
 
             // Upcast explicitly to ack reference before sending
-            const ack& base_ack = ack_obj;
+            [[maybe_unused]] const ack& base_ack = ack_obj;
 	    caf::anon_mail(std::move(ack_obj)).urgent().send(receiver_);
 
 
@@ -87,4 +87,3 @@ class CAF_CUDA_EXPORT transfer_token : public response_token {
 };
 
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index 30f766476f..f276edb38b 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -17,7 +17,7 @@
 #include <caf/anon_mail.hpp>
 #include "caf/cuda/global_export.hpp"
 
-
+ 
 
 
 //helper function to check errors
diff --git a/libcaf_cuda/caf/cuda/manager_config.hpp b/libcaf_cuda/caf/cuda/manager_config.hpp
index f20e99d396..4edf3149d7 100644
--- a/libcaf_cuda/caf/cuda/manager_config.hpp
+++ b/libcaf_cuda/caf/cuda/manager_config.hpp
@@ -10,7 +10,7 @@ class manager_config {
 public:
     manager_config() : actorBLAS(false), actorSparse(false), num_scheduler_streams(500), scheduler_stream_depth(1) {}
     manager_config(bool blas) : actorBLAS(blas), actorSparse(false), num_scheduler_streams(500), scheduler_stream_depth(1) {}
-    manager_config(bool blas, bool sparse) : actorBLAS(blas), actorSparse(false), num_scheduler_streams(500), scheduler_stream_depth(1) {}
+    manager_config(bool blas, [[maybe_unused]] bool sparse) : actorBLAS(blas), actorSparse(false), num_scheduler_streams(500), scheduler_stream_depth(1) {}
     manager_config(bool blas, bool sparse, int num_streams, int stream_depth) : actorBLAS(blas), actorSparse(sparse), num_scheduler_streams(num_streams), scheduler_stream_depth(stream_depth) {}
     
     bool getActorBLAS() const { return actorBLAS; }
diff --git a/libcaf_cuda/caf/cuda/program.hpp b/libcaf_cuda/caf/cuda/program.hpp
index 18f075533b..3ab35e1e3b 100644
--- a/libcaf_cuda/caf/cuda/program.hpp
+++ b/libcaf_cuda/caf/cuda/program.hpp
@@ -27,7 +27,7 @@ class CAF_CUDA_EXPORT program : public caf::ref_counted {
   /// @throws std::runtime_error if the kernel was not loaded for the device.
   CUfunction get_kernel(int device_id);
 
-    friend void intrusive_ptr_add_ref(const program* p) noexcept {
+    friend void intrusive_ptr_add_ref([[maybe_unused]] const program* p) noexcept {
         //p->ref_count_.fetch_add(1, std::memory_order_relaxed);
     }
     friend void intrusive_ptr_release(const program* p) noexcept {
@@ -62,4 +62,3 @@ class CAF_CUDA_EXPORT program : public caf::ref_counted {
 using program_ptr = caf::intrusive_ptr<program>;
 
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
index 411d434e52..605d6dc5c3 100644
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
@@ -31,8 +31,8 @@ void core_usage_behavior::on_enter() {
 
 void core_usage_behavior::reclaim(int blocks_consumed,
 	       	int memory_returned,
-	       	int time,
-	       	int dependency_number) {
+	       	[[maybe_unused]] int time,
+	       	[[maybe_unused]] int dependency_number) {
 
 	//std::cout << "blocks is " <<  blocks_consumed << "\n";
 	available_SM += blocks_consumed;
diff --git a/libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp b/libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp
index 0a685c05e7..9ee0a11720 100644
--- a/libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp
@@ -79,7 +79,7 @@ void mmul_batch_scheduler_behavior::receive(const token_ptr& tok) {
             kernel_graph new_graph(state_.device_number, assigned_stream);
             new_graph.add_operation(tok);
             independent_graphs.push_back(std::move(new_graph));
-            graph_ref ref{graph_ref::kind_t::independent, -1, static_cast<int>(independent_graphs.size() - 1)};
+            graph_ref ref{graph_ref::kind_t::independent, -1, independent_graphs.size() - 1};
             enqueue_graph_by_blocks(ref, qt);
             schedule();
             return;
@@ -220,7 +220,7 @@ void mmul_batch_scheduler_behavior::schedule() {
     try_dispatch_queue(high_queue, HIGH);
 }
 
-void mmul_batch_scheduler_behavior::process_launch_token(const token_ptr& tok, int stream_id, int assigned_queue) {
+void mmul_batch_scheduler_behavior::process_launch_token(const token_ptr& tok, int stream_id, [[maybe_unused]] int assigned_queue) {
     // Create a launch response token and send it (same pattern as multilevel)
     const auto& launch = static_cast<const launch_token&>(*tok);
     auto response = make_launch_response_token(state_.self, launch, state_.device_number, stream_id, /*reclaim_value*/ 0, /*reclaim_runtime*/ 0);
@@ -230,9 +230,9 @@ void mmul_batch_scheduler_behavior::process_launch_token(const token_ptr& tok, i
     // when launch_response_token::release() runs on the device side. That triggers reclaim(...) in this actor.
 }
 
-void mmul_batch_scheduler_behavior::reclaim(int blocks_consumed,
-                                            int memory_returned,
-                                            int time,
+void mmul_batch_scheduler_behavior::reclaim([[maybe_unused]] int blocks_consumed,
+                                            [[maybe_unused]] int memory_returned,
+                                            [[maybe_unused]] int time,
                                             int dependency_number) {
     // This reclaim() is called when the device (or the launch_response_token destructor)
     // sends the 4-tuple (reclaim_value, reclaim_memory, reclaim_runtime, reclaim_dependency).
@@ -298,4 +298,3 @@ kernel_graph* mmul_batch_scheduler_behavior::resolve(const graph_ref& ref) {
 }
 
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
index 713e9f9367..41eb12a2ab 100644
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
@@ -183,7 +183,7 @@ void multilevel_usage_behavior::schedule() {
 
 void multilevel_usage_behavior::reclaim(int blocks_consumed,
                                        int memory_returned,
-                                       int time,
+                                       [[maybe_unused]] int time,
                                        int dependency_number) {
     // update available resources
     available_SM += blocks_consumed;
@@ -447,4 +447,3 @@ int multilevel_usage_behavior::get_device_for_dependency(int dependency_number)
 }
 
 } // namespace caf::cuda
-
diff --git a/libcaf_cuda/src/control-layer/red_light_behavior.cpp b/libcaf_cuda/src/control-layer/red_light_behavior.cpp
index bc4dbe9e2c..662be3b850 100644
--- a/libcaf_cuda/src/control-layer/red_light_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/red_light_behavior.cpp
@@ -25,11 +25,11 @@ void red_light_behavior::on_enter() {
             .send(state_.self);
 }
 
-void red_light_behavior::process_launch_token(const token_ptr& tok, int stream_id) {
+void red_light_behavior::process_launch_token([[maybe_unused]] const token_ptr& tok, [[maybe_unused]] int stream_id) {
     // Override to do nothing, as red light should not process
 }
 
-void red_light_behavior::process_memory_transfer_token(const token_ptr& tok, int stream_id) {
+void red_light_behavior::process_memory_transfer_token([[maybe_unused]] const token_ptr& tok, [[maybe_unused]] int stream_id) {
     // Override to do nothing
 }
 
diff --git a/libcaf_cuda/src/control-layer/scheduler_actor.cpp b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
index fb0f088233..9e92c73745 100644
--- a/libcaf_cuda/src/control-layer/scheduler_actor.cpp
+++ b/libcaf_cuda/src/control-layer/scheduler_actor.cpp
@@ -158,7 +158,7 @@ void scheduler_actor::schedule_work() {
                         this->mail("retry_steal").urgent().delay(std::chrono::milliseconds(current_backoff_)).send(this);
                     }
                 },
-                [this](error& err) {
+                [this]([[maybe_unused]] error& err) {
                     awaiting_steal_ = false;
                     // On network error or timeout, also backoff to avoid hammerring a dead/slow neighbor
                     current_backoff_ = (current_backoff_ == 0) ? 50 : std::min(500, current_backoff_ * 2);
diff --git a/libcaf_cuda/src/control-layer/single_usage_behavior.cpp b/libcaf_cuda/src/control-layer/single_usage_behavior.cpp
index d1a98b560d..3965170d42 100644
--- a/libcaf_cuda/src/control-layer/single_usage_behavior.cpp
+++ b/libcaf_cuda/src/control-layer/single_usage_behavior.cpp
@@ -28,10 +28,10 @@ void single_usage_behavior::on_enter() {
     schedule();  // try to launch something right away if tokens already waiting
 }
 
-void single_usage_behavior::reclaim(int blocks_consumed,
+void single_usage_behavior::reclaim([[maybe_unused]] int blocks_consumed,
                                     int memory_returned,
-                                    int time,
-                                    int dependency_number) {
+                                    [[maybe_unused]] int time,
+                                    [[maybe_unused]] int dependency_number) {
    
     std::cout << "reclaiming\n";
     // GPU is now free again
diff --git a/libcaf_cuda/src/platform.cpp b/libcaf_cuda/src/platform.cpp
index ff63e42420..1c6a43f86e 100644
--- a/libcaf_cuda/src/platform.cpp
+++ b/libcaf_cuda/src/platform.cpp
@@ -14,7 +14,7 @@ platform_ptr platform::create() {
 //constructor
 platform::platform() {
   int device_count = 0;
-  check(cuDeviceGetCount(&device_count), "cuDeviceGetCount");
+  CHECK_CUDA(cuDeviceGetCount(&device_count));
   devices_.resize(device_count);
   contexts_.resize(device_count);
 
@@ -23,25 +23,25 @@ platform::platform() {
 
   for (int i = 0; i < device_count; ++i) {
     CUdevice cuda_device;
-    check(cuDeviceGet(&cuda_device, i), "cuDeviceGet");
+    CHECK_CUDA(cuDeviceGet(&cuda_device, i));
 
     char name[256];
-    check(cuDeviceGetName(name, sizeof(name), cuda_device), "cuDeviceGetName");
+    CHECK_CUDA(cuDeviceGetName(name, sizeof(name), cuda_device));
     device_names[i] = name;
 
 
 #if CUDA_VERSION >= 13000
     {
       CUctxCreateParams ctx_params = {};
-      check(cuCtxCreate(&contexts_[i], 
-                        &ctx_params, 
-                        CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, 
-                        cuda_device),"creating context");
+      CHECK_CUDA(cuCtxCreate(&contexts_[i], 
+                             &ctx_params, 
+                             CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, 
+                             cuda_device));
     }
 #else
-    check(cuCtxCreate(&contexts_[i], 
-                      CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, 
-                      cuda_device),"creating context");
+    CHECK_CUDA(cuCtxCreate(&contexts_[i], 
+                           CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, 
+                           cuda_device));
 #endif
     devices_[i] = make_counted<device>(cuda_device, contexts_[i], name, i);
  
@@ -65,7 +65,7 @@ platform::platform() {
   scheduler_->set_devices(devices_);
 
   if (device_count > 0) {
-    check(cuCtxSetCurrent(contexts_[0]), "cuCtxSetCurrent");
+    CHECK_CUDA(cuCtxSetCurrent(contexts_[0]));
   }
 }
 
diff --git a/libcaf_cuda/tests/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler_integration_test.cpp
deleted file mode 100644
index 2cfc9a3903..0000000000
--- a/libcaf_cuda/tests/scheduler_integration_test.cpp
+++ /dev/null
@@ -1,163 +0,0 @@
-#include <caf/all.hpp>
-#include <caf/cuda/all.hpp>
-#include <caf/cuda/control-layer/all-control-layer.hpp>
-#include <iostream>
-#include <vector>
-#include <string>
-#include <random>
-#include <unordered_map>
-#include <unordered_set>
-#include <algorithm>
-
-using namespace caf;
-using namespace caf::cuda;
-
-// A generic command runner to provide access to CUDA stream callbacks
-static command_runner<> runner;
-
-struct task_actor_state {
-    std::vector<int> h_a;
-    std::vector<int> h_b;
-    std::vector<int> h_c;
-    int N_val; // Store N for this specific task
-};
-
-// MatrixPool structure from mmul-random-batch-benchmark
-struct MatrixPool {
-    std::unordered_map<int, std::vector<int>> A;
-    std::unordered_map<int, std::vector<int>> B;
-};
-
-// create_matrix_pool_random function from mmul-random-batch-benchmark
-MatrixPool create_matrix_pool_random(
-    int num_sizes,
-    int min_N,
-    int max_N,
-    unsigned int seed
-) {
-    MatrixPool pool;
-    std::mt19937 rng(seed);
-    std::uniform_int_distribution<int> dist(min_N, max_N);
-    std::unordered_set<int> used;
-    while (used.size() < static_cast<size_t>(num_sizes)) {
-        int N = dist(rng);
-        if (used.insert(N).second) {
-            pool.A[N] = std::vector<int>(N * N, 1);
-            pool.B[N] = std::vector<int>(N * N, 2); // Changed to 2 for distinct input
-        }
-    }
-    return pool;
-}
-
-// This actor represents a single task that requests permission from the scheduler.
-behavior task_actor_fun(stateful_actor<task_actor_state>* self) {
-    auto& st = self->state();
-
-    return {
-        [=](response_token_ptr res) mutable {
-            if (res->getType() == LAUNCH_RESPONSE) {
-                auto& st_inner = self->state();
-                
-                // 1. Setup GPU arguments.
-                auto in_a = create_in_arg(st_inner.h_a);
-                auto in_b = create_in_arg(st_inner.h_b);
-                auto out_c = create_out_arg_with_size<int>(st_inner.N_val * st_inner.N_val);
-                auto in_n = create_in_arg(st_inner.N_val);
-
-                // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
-                auto result_tuple = runner.run_async(res, in_a, in_b, out_c, in_n);
-                auto d_c = std::get<2>(result_tuple);
-
-                // 3. Asynchronous Copyback.
-                // The launch_response_token is released inside the callback 
-                // to signal to the scheduler that the resource is free. (No serial verification here)
-                runner.copy_to_host_async(d_c, st_inner.h_c.data(), st_inner.N_val * st_inner.N_val, [res, self](int* /*ptr*/, size_t /*sz*/) {
-                    std::cout << "[TASK] " << res->name() << " finished. Releasing token." << std::endl;
-                    res->release();
-                });
-            }
-        }
-    };
-}
-
-// Helper function to initialize task_actor_state
-behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, int N_val, const std::vector<int>& h_a_data, const std::vector<int>& h_b_data) {
-    self->state().N_val = N_val;
-    self->state().h_a = h_a_data;
-    self->state().h_b = h_b_data;
-    self->state().h_c.resize(N_val * N_val, 0);
-    return task_actor_fun(self);
-}
-
-template <class Fn>
-double time_run(Fn&& fn) {
-    auto start = std::chrono::steady_clock::now();
-    fn();
-    auto end = std::chrono::steady_clock::now();
-    std::chrono::duration<double> elapsed = end - start;
-    return elapsed.count();
-}
-
-void run_scheduler_integration_scaling_test(actor_system& sys) {
-    const int min_N = 32;
-    const int max_N = 1024;
-    const int num_distinct_sizes = 10;
-    const std::vector<int> actor_counts = {60, 120};
-
-    // Generate deterministic random pool once
-    MatrixPool pool = create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42);
-    
-    std::vector<int> available_Ns;
-    for (const auto& pair : pool.A) available_Ns.push_back(pair.first);
-
-    for (int num_tasks : actor_counts) {
-        manager_config config;
-        manager::init(sys, config);
-        auto& mgr = manager::get();
-
-        // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
-        mgr.toggle_scheduler_actor(4, 2);
-
-        auto program = mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
-        const int THREADS = 32;
-
-        std::vector<token_ptr> tokens;
-        std::mt19937 rng(42);
-        std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
-
-        std::cout << "=====================================\n";
-        std::cout << "Scheduler Test | tasks=" << num_tasks << "\n";
-
-        // Spawn task actors and prepare tokens
-        for (int i = 0; i < num_tasks; ++i) {
-            int current_N = available_Ns[dist_N_idx(rng)];
-            nd_range range((current_N + THREADS - 1) / THREADS, 
-                           (current_N + THREADS - 1) / THREADS, 1, 
-                           THREADS, THREADS, 1);
-            
-            auto worker = sys.spawn(make_task_actor_behavior, 
-                                    current_N, 
-                                    pool.A.at(current_N), 
-                                    pool.B.at(current_N));
-
-            tokens.push_back(make_launch_token(program, range, 0, 
-                             "task_" + std::to_string(i), worker));
-        }
-
-        double elapsed = time_run([&]() {
-            std::cout << "[MAIN] Dispatching batch to scheduler..." << std::endl;
-            mgr.send_scheduler_actor_message(std::move(tokens));
-            sys.await_all_actors_done();
-        });
-
-        std::cout << "Run complete. Time: " << elapsed << " s\n";
-        manager::shutdown();
-    }
-}
-
-void caf_main(actor_system& sys) {
-    run_scheduler_integration_scaling_test(sys);
-    std::cout << "[MAIN] Integration test complete." << std::endl;
-}
-
-CAF_MAIN(id_block::cuda_control)
\ No newline at end of file

From b6114212a1f62b71511b50e4501843eac272ad22 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 18 Jun 2026 08:53:06 -0600
Subject: [PATCH 0953/1000] updated tests to  align more with the new scheduler
 actors functionality

---
 .../load-balancing-test/CMakeLists.txt        |    2 +
 .../load-balancing-test/main.test.cpp         | 1173 ++---------------
 2 files changed, 147 insertions(+), 1028 deletions(-)

diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/CMakeLists.txt b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/CMakeLists.txt
index 89bcf5ba7c..59d7388f4a 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/CMakeLists.txt
@@ -39,6 +39,8 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
 )
 
 
diff --git a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
index 3dafb5e6c6..ff92ec5fca 100644
--- a/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/load-balancing-test/main.test.cpp
@@ -2,1063 +2,180 @@
 #include <caf/cuda/all.hpp>
 #include <caf/cuda/control-layer/all-control-layer.hpp>
 #include <iostream>
-#include <iomanip>
 #include <vector>
-#include <chrono>
-#include <thread>
-#include <algorithm>
-#include <numeric>
+#include <string>
 #include <random>
-#include <unistd.h>
-#include "caf/actor_registry.hpp"
-#include <chrono>
-#include <iostream>
-//#include <caf/atoms.hpp>
-
-
-
+#include <unordered_map>
+#include <unordered_set>
+#include <algorithm>
 
 using namespace caf;
-using namespace std::chrono_literals;
-
-
+using namespace caf::cuda;
 
+// A generic command runner to provide access to CUDA stream callbacks
+static command_runner<in<int>, in<int>, out<int>, in<int>> runner;
 
-
-struct mmul_actor_state {
-  static inline const char* name = "my_actor";
-  int last_N = 0; // example state variable
-  int id = rand(); // an actor id
-  int times = 0;
+// MatrixPool structure from mmul-random-batch-benchmark
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
 };
 
-
-
-
-//commands classes used to launch kernels
-using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
-using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
-
-using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
-
-mmulCommand mmul;
-matrixGenCommand randomMatrix;
-mmulAsyncCommand mmulAsync;
-
-
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
-    }
-  }
-}
-
-
-
-
-
-
-struct exit_actor_state {
-	int completed = 0;
+struct task_actor_state {
+    program_ptr prog;
+    int N_val;
+    std::shared_ptr<MatrixPool> pool;
 };
 
-
-caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
-
-
-	return {
-		[=](int num_completed) {
-			self->state().completed += num_completed;
-			
-//			std::cout << "Actors finished is " << self->state().completed << "\n";
-			if (self->state().completed >= limit) {
-			
-				caf::cuda::manager::shutdown();
-				self->quit();
-			}
-		}
-	};
-
-
-}
-
-
-
-
-
-
-// Stateful actor behavior
-caf::behavior mmul_actor_fun(
-    caf::stateful_actor<mmul_actor_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims)
-{
-
-
-	caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-	caf::actor scheduler = mgr.get_scheduler_actor();
-
-	//send a launch token
-	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-			program,
-			dims,
-			0,
-			"hello",
-			self
-			);
-	self -> mail(launch_token).send(scheduler);
-
-	return {
-
-		[=] (caf::cuda::response_token_ptr res_token) {
-
-			if (res_token -> getType() == LAUNCH_RESPONSE) {
-				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-				//assume N = 1024
-				std::vector<int> matrix1(N*N);
-				matrix1.reserve(N);
-				std::vector<int> matrix2(N*N);
-				matrix2.reserve(N);
-
-				//std::cout << "GPU ACTOR sending data to compute\n";
-				self -> mail(matrix1,matrix2,res_token,N).send(self);
-
-			}
-			else {
-				std::cout << "Got a memory response token\n";
-			}
-			//token should drop out of scope now, triggering a response 
-		},
-
-			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-			[=](const std::vector<int>& matrixA,
-					const std::vector<int>& matrixB,
-					const caf::cuda::response_token_ptr& res_token, int N) {
-
-
-				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
-
-				//caf::cuda::launch_response_token& kt =
-				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
-
-				//std::cout << "GPU ACTOR  computing\n";
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-				const int THREADS = 32;
-				const int BLOCKS = (N + THREADS - 1) / THREADS;
-				caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrixA);
-				auto arg2 = caf::cuda::create_in_arg(matrixB);
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
-
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				//std::cout << "GPU ACTOR done  computing\n";
-				//verify its own result 
-				self -> mail(matrixA,matrixB,matrixC,N).send(self);
-
-			},
-
-					// 3rd handler: CPU atom + matrices + N
-					[=](const std::vector<int>& matrixA,
-							const std::vector<int>& matrixB,
-							const std::vector<int>& matrixC,
-							int N) {
-
-						using clock = std::chrono::high_resolution_clock;
-
-						auto start = clock::now();
-
-						//std::cout << "GPU ACTOR verifying\n";
-
-						std::vector<int> result(N * N);
-
-						serial_matrix_multiply(matrixA, matrixB, result, N);
-
-						if (result == matrixC) {
-							std::cout << "actor with id " << self->state().id
-								<< " references match\n";
-						} else {
-							std::cout << "actor with id " << self->state().id
-								<< " references did not match\n";
-						}
-
-						auto end = clock::now();
-
-						auto ms =
-							std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-						std::cout << "[TIMING] verification took "
-							<< ms << " ms (actor id "
-							<< self->state().id << ")\n";
-
-						// signal exit actor and quit
-						self->mail(1).send(exit_actor);
-						self->quit();
-
-					}
-	};
-}
-
-
-
-
-
-// this actor will not verify its results
-// great for performance analysis 
-caf::behavior mmul_actor_fun_no_verify(
-    caf::stateful_actor<mmul_actor_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims,
-    bool request
-    )
-{
-
-	//set the value of N correctly to overide the base option.	
-
-	caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-	caf::actor scheduler = mgr.get_scheduler_actor();
-
-	if (request) {
-		//send a launch token
-		caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-				program,
-				dims,
-				0,
-				"hello",
-				self
-				);
-		self -> mail(launch_token).send(scheduler);
-	}
-	return {
-
-		[=] (caf::cuda::response_token_ptr res_token) {
-
-			if (res_token -> getType() == LAUNCH_RESPONSE) {
-				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-				//assume N = 1024
-				std::vector<int> matrix1(N*N);
-				matrix1.reserve(N);
-				std::vector<int> matrix2(N*N);
-				matrix2.reserve(N);
-
-
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrix1);
-				auto arg2 = caf::cuda::create_in_arg(matrix2);
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
-
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-			
-			
-				//mask the transfer back to the cpu for scheduler
-				res_token -> release();
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				//std::cout << "GPU ACTOR done  computing\n";
-				// signal exit actor and quit
-				self->mail(1).send(exit_actor);
-				self->quit();
-
-				//std::cout << "GPU ACTOR sending data to compute\n";
-			//	self -> mail(matrix1,matrix2,res_token,N).send(self);
-
-			}
-			else {
-				std::cout << "Got a memory response token\n";
-			}
-			//token should drop out of scope now, triggering a response 
-		},
-
-			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-			[=](const std::vector<int>& matrixA,
-					const std::vector<int>& matrixB,
-					const caf::cuda::response_token_ptr& res_token, int N) {
-
-
-				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
-
-				//caf::cuda::launch_response_token& kt =
-				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
-
-				//std::cout << "GPU ACTOR  computing\n";
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrixA);
-				auto arg2 = caf::cuda::create_in_arg(matrixB);
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
-
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-			
-			
-				//mask the transfer back to the cpu for scheduler
-				res_token -> release();
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				//std::cout << "GPU ACTOR done  computing\n";
-				// signal exit actor and quit
-				self->mail(1).send(exit_actor);
-				self->quit();
-
-			}
-
-				
-	};
-}
-
-
-
-
-
-template <class Fn>
-double time_run(Fn&& fn) {
-  auto start = std::chrono::steady_clock::now();
-  fn();
-  auto end = std::chrono::steady_clock::now();
-  std::chrono::duration<double> elapsed = end - start;
-  return elapsed.count();
-}
-
-
-//this test is meant to demonstrate the fact that scheduler actors can 
-//migrate work to correct load imbalance
-//the sizes should be large enough such that the tests exceed 4-5 seconds in total
-//otherwise the schedulers wont care to do this fast enough
-void run_load_balance_test(
-    caf::actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors,
-    bool randomize = false)
-{
-    caf::cuda::manager& mgr = caf::cuda::manager::get(); 
-
-
-    //set the behaviors of each scheduler actor
-    for (int i = 0; i < mgr.get_num_devices();i++) {
-	    mgr.send_scheduler_actor_message("multilevel",i);
+// create_matrix_pool_random function from mmul-random-batch-benchmark
+MatrixPool create_matrix_pool_random(
+    int num_sizes,
+    int min_N,
+    int max_N,
+    unsigned int seed
+) {
+    MatrixPool pool;
+    std::mt19937 rng(seed);
+    std::uniform_int_distribution<int> dist(min_N, max_N);
+    std::unordered_set<int> used;
+    while (used.size() < static_cast<size_t>(num_sizes)) {
+        int N = dist(rng);
+        if (used.insert(N).second) {
+            pool.A[N] = std::vector<int>(N * N, 1);
+            pool.B[N] = std::vector<int>(N * N, 2); // Changed to 2 for distinct input
+        }
     }
-
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    std::vector<caf::cuda::token_ptr> tokens(num_actors);
-
-    std::mt19937 rng(123456);
-    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
-
-    const int THREADS = 32;
-
-    auto t_start = std::chrono::steady_clock::now();
-
-for (int i = 0; i < num_actors; ++i) {
-    int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
-    int BLOCKS = (N + THREADS - 1) / THREADS;
-
-    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1,
-                             THREADS, THREADS, 1);
-
-    caf::actor a = sys.spawn(
-        mmul_actor_fun_no_verify,
-        exit_actor,
-        N,
-        program,
-        dims,
-        false
-    );
-
-    tokens[i] = caf::cuda::make_launch_token(
-        program,
-        dims,
-        0, // memory usage is zero for now, we still do not track it at all
-        "hello",
-        a
-    );
+    return pool;
 }
 
-auto t_end = std::chrono::steady_clock::now();
-
-auto us = std::chrono::duration_cast<std::chrono::microseconds>(
-              t_end - t_start
-          ).count();
-
-std::cout << "Actor spawn + token creation loop took "
-          << us << " us\n";
-
-   
-	  //send the tokens to only 1 GPU and let them
-	  //figure out that there is a load imbalance 
-	  mgr.send_scheduler_actor_message(tokens);
-          sys.await_all_actors_done();
-}
-
-
-//-------------------------------------try load balancing with actors with dependencies
-
-
-using namespace caf;
-using namespace std::chrono_literals;
-
-
-// --- command runner types (put near top of file) -------------------------
-using initCommand =
-  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, in<int>, in<unsigned long long>>;
-
-using divCommand  =
-  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, in<int>>;
-
-using sumCommand  =
-  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, in<int>>;
-
-// single instances (can be file-global)
-static initCommand init_cmd;
-static divCommand  div_cmd;
-static sumCommand  sum_cmd;
-
-// --- pipeline actor state (device buffers persist here) ------------------
-struct pipeline_actor_state {
-    int id = rand();
-
-    int finished_stage = 0;
-
-    // device-side buffers that must persist across stages:
-    caf::cuda::mem_ptr<float> d_denoms;
-    caf::cuda::mem_ptr<float> d_results;
-    caf::cuda::mem_ptr<float> d_sum;
-    
-};
-
-// --- corrected pipeline_actor -------------------------------------------
-behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
-                        actor supervisor,
-                        caf::cuda::program_ptr p1,
-                        caf::cuda::program_ptr p2,
-                        caf::cuda::program_ptr p3,
-                        int n)
-{
-    // host-side scratch (only used for post-stage2 NaN/Inf detection)
-    std::vector<float> h_results;
-
-
-    // nd_range used for all stages (adapt to your kernels as needed)
-    caf::cuda::nd_range range{
-        {(n + 255) / 256, 1, 1},
-        {256, 1, 1}
-    };
-
-    // helper to create and send a launch token
-    auto launch = [&](caf::cuda::program_ptr prog, const std::string& stage) {
-        auto tok = make_launch_token(
-            prog,
-            range,
-            /*memory_usage=*/static_cast<int>(sizeof(float) * n),
-            stage,
-            self,
-            self->state().id  // dependency/demo id
-        );
-
-	//do not specifiy a device number to send it to, let it figure it out
-	//caf::cuda::manager::get().send_scheduler_actor_message(tok);
-    
-	//forcefully send to the first scheduler actor
-	caf::actor scheduler = caf::cuda::manager::get().get_scheduler_actor();
-	anon_mail(tok).send(scheduler);
-
-    };
-
-    // fire all three tokens (scheduler will reply with response_token on grants)
-    launch(p1, "stage1");
-    launch(p2, "stage2");
-    launch(p3, "stage3");
-
+// This actor represents a single task that requests permission from the scheduler.
+behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_actor) {
     return {
-
-        // handle response tokens by name — opaque to reclaim payload
-        [=](caf::cuda::response_token_ptr res_token) mutable {
-
-            const auto& stage = res_token->name();
-
-	    if (res_token->getType() == LAUNCH_RESPONSE) {
-
-		    // --------------------- Stage 1: init_denominators ---------------------
-		    if (stage == "stage1") {
-			    // allocate device buffer for denominators (persist in state)
-
-			    //std::cout << "Starting stage 1\n";
-			    unsigned long long seed = static_cast<unsigned long long>(
-					    std::chrono::high_resolution_clock::now().time_since_epoch().count()
-					    );
-
-
-
-			    out<float> buffer = caf::cuda::create_out_arg_with_size<float>(n);
-			    self->state().d_denoms = init_cmd.transfer_memory(res_token,buffer);
-
-			    // run kernel on the stream/device from res_token
-			    // kernel signature: (float* denominators, int n, unsigned long long seed)
-			    init_cmd.run_async(
-					    p1,
-					    range,
-					    res_token,                            // uses token's stream/device
-					    self->state().d_denoms,               // device buffer
-					    caf::cuda::create_in_arg(n),          // n
-					    caf::cuda::create_in_arg(seed)     // seed
-					    );
-
-			    //std::cout << "Finished stage 1\n";
-			    // stage1 intentionally no checks — data may contain zeros
-			    
-			    self->state().finished_stage++;
-			    return;
-		    }
-
-		    // --------------------- Stage 2: perform_division ---------------------
-		    if (stage == "stage2") {
-			    // allocate device buffer for results (persist in state)
-			    std::vector<float> buffer1(n);
-
-			    //std::cout << "Starting stage 2\n";
-			    self->state().d_results = div_cmd.transfer_memory(res_token,out<float>{buffer1});
-
-			    // create a host numerators vector (all ones)
-			    std::vector<float> h_nums(n, 1.0f);
-
-			    // transfer numerators to device on the token's stream/device
-			    // transfer_memory returns a caf::cuda::mem_ptr<float>
-			    auto d_nums = div_cmd.transfer_memory(res_token, in_out<float>{h_nums});
-
-			    // run division kernel on the token's stream/device:
-			    // kernel signature: (float* numerators, float* denominators, float* results, int n)
-			    
-			    if (self->state().d_denoms == nullptr) {
-			    
-				    std::cout << "Error with pipeline actor d_denoms is nullptr\n";
-			    
-			    }
-
-
-			    if (d_nums == nullptr) {
-			    
-				    std::cout << "Error with pipeline actor d_denoms is nullptr\n";
-			    
-			    }
-
-			    
-			    div_cmd.run(
-					    p2,
-					    range,
-					    res_token,
-					    d_nums,
-					    self->state().d_denoms,
-					    self->state().d_results,
-					    caf::cuda::create_in_arg(n)
-				       );
-
-
-			    //there could be a division by zero in here 
-			    //but this is a load balancing test
-			    //not a fault test,
-			    //go see the fault tolerance test to see how thats handled	
-
-			    /*
-			    // extract the device results back to host for verification.
-			    // extract_vector will synchronize as needed.
-			    h_results = self->state().d_results -> copy_to_host();
-
-			    // check for NaN/Inf AFTER the kernel finished
-			    bool fault = false;
-			    for (float v : h_results) {
-				    if (!std::isfinite(v)) {
-					    fault = true;
-					    break;
-				    }
-			    }
-
-			    if (fault) {
-				    // inform supervisor and exit;
-				    anon_mail(std::string("crash")).send(supervisor);
-				    self->quit();
-				    return;
-			    }
-
-			    // stage2 passed — keep d_results in state for stage3
-			    */
-
-			    self->state().finished_stage++;
-			    return;
-		    }
-
-		    // --------------------- Stage 3: sum_results --------------------------
-		    if (stage == "stage3") {
-			    // allocate device scalar for sum result
-
-			    //std::cout << "Starting stage 3\n";
-			    std::vector<float> buffer1(1);
-
-			    self->state().d_sum = div_cmd.transfer_memory(res_token,out<float>{buffer1});
-
-			    // run reduction on the token's stream/device:
-			    // kernel signature: (float* results, float* final_sum, int n)
-			    sum_cmd.run(
-					    p3,
-					    range,
-					    res_token,
-					    self->state().d_results,
-					    self->state().d_sum,
-					    caf::cuda::create_in_arg(n)
-				       );
-
-			    // extract final scalar
-
-			    std::vector<float> buf = self->state().d_sum -> copy_to_host();
-			    float final_sum = buf[0];
-			    //std::cout << "[pipeline] completed, sum = " << final_sum << "\n";
-
-			    anon_mail(1).send(supervisor);
-
-			    // quit the pipeline actor
-			    self->state().finished_stage++;
-			    self->quit();
-			    return;
-		    }
-	    }
-
-	    else if (res_token->getType() == TRANSFER) {
-	   
-		    std::cout << "Got a transfer token\n";
-
-		   if (stage == "stage1") {
-			   res_token->release(); // no dependencies at this point clear to continue
-		   	   return;
-		   } 
-
-		   else if (stage == "stage2") {
-		  
-			   //at this point the d_results needs to be transfer over to the other device
-			   
-		           std::cout << "Transfering at stage 2\n";
-			      if (self->state().d_denoms == nullptr) {
-			    
-				    std::cout << "Error with pipeline actor d_denoms is nullptr during transfer\n";
-				    std::cout << "Completed stage is " << self->state().finished_stage << "\n";
-			    
-			    }
-			   
-			   
-			   
-			   //TODO FIX SEGFAULT TRIGGERED BY THIS LINE 
-			    in_out<float> temp_buffer{self->state().d_denoms -> copy_to_host()};
-
-			    self->state().d_denoms = div_cmd.transfer_memory(res_token,temp_buffer);
-			    //all done
-			    res_token->release();
-		   	    return;
-		   }
-
-		   else if (stage == "stage3") {
-		   
-			   //at this point d_results needs to be copied over to the new GPU   
-			    self->state().d_results = div_cmd.transfer_memory(res_token,in_out<float>{self->state().d_results->copy_to_host()});
-			    //all done
-			    res_token->release();
-			    return;
-		   }
-
-		   else { 
-			   std::cout << "Error unrecognized transfer token\n";
-		   }
-		    
-	    
-	    }
-
-            // unknown stage: ignore or log
-            std::cerr << "[pipeline] received unknown response token: " << stage << "\n";
-	}
+        [=](response_token_ptr res) mutable {
+            if (res->getType() == LAUNCH_RESPONSE) {
+                auto& st = self->state();
+                
+                // We need to cast the base response_token to access the specific nd_range stored in it.
+                auto launch_res = static_cast<launch_response_token*>(res.get());
+                int N = st.N_val;
+
+                // 1. Setup GPU arguments.
+                // Fetch data from the shared pool only when scheduled to save RAM
+                auto in_a = create_in_arg(st.pool->A.at(N));
+                auto in_b = create_in_arg(st.pool->B.at(N));
+                auto out_c = create_out_arg_with_size<int>(N * N);
+                auto in_n = create_in_arg(N);
+
+                // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
+                auto result_tuple = runner.run_async(st.prog, launch_res->getRange(), res, in_a, in_b, out_c, in_n);
+                auto d_c = std::get<2>(result_tuple);
+
+                // 3. Asynchronous Copyback.
+                // Allocate a local buffer for the result to keep the total system memory low.
+                auto h_c = std::make_shared<std::vector<int>>(N * N);
+                // The launch_response_token is released inside the callback 
+                // to signal to the scheduler that the resource is free. (No serial verification here)
+                runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [res, exit_actor, h_c](int* /*ptr*/, size_t /*sz*/) {
+                    res->release();
+                    anon_mail(1).send(exit_actor);
+                });
+            }
+        }
     };
 }
 
-
-
-
-
-
-
-
-
-//this test is meant to demonstrate the fact that scheduler actors can 
-//migrate work to correct load imbalance
-//the sizes should be large enough such that the tests exceed 4-5 seconds in total
-//otherwise the schedulers wont care to do this fast enough
-//As it turns out pipeline actor does not do enough work in order to convince the GPUs
-//that it should even attempt to migrate it
-void run_load_balance_test_with_dependencies(
-    caf::actor_system& sys,
-    const int n,
-    int num_actors,
-    bool randomize = false)
-{
-    caf::cuda::manager& mgr = caf::cuda::manager::get(); 
-
-
-    //set the behaviors of each scheduler actor
-    for (int i = 0; i < mgr.get_num_devices();i++) {
-	    mgr.send_scheduler_actor_message("multilevel",i);
-    }
-
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-    caf::cuda::program_ptr p1 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","init_denominators");
-    caf::cuda::program_ptr p2 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","perform_division");
-    caf::cuda::program_ptr p3 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","sum_results");
-
-
-
-
-
-
-    auto t_start = std::chrono::steady_clock::now();
-
-for (int i = 0; i < num_actors; ++i) {
-	sys.spawn(pipeline_actor, exit_actor, p1, p2, p3, n);	
-}
-  
-	  //this time the gpu actors can figure out how to send tokens to the correct GPU scheduler
-          sys.await_all_actors_done();
+// Helper function to initialize task_actor_state
+behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, std::shared_ptr<MatrixPool> pool, caf::actor exit_actor) {
+    auto& st = self->state();
+    st.prog = std::move(prog);
+    st.N_val = N_val;
+    st.pool = std::move(pool);
+    return task_actor_fun(self, exit_actor); // Pass exit_actor to task_actor_fun
 }
 
-
-
-struct mmul_async_actor_state {
-  static inline const char* name = "mmul_actor";
-
-  int N = 0;
-  int id = rand();
-
-  // timing / bookkeeping only
-  std::chrono::high_resolution_clock::time_point start_time;
-  int times = 0;
-
-  // --- mmul_async state (added) -------------------------------------------
-  caf::cuda::mem_ptr<int> d_genA;                // device buffer for generated A
-  caf::cuda::mem_ptr<int> d_genB;                // device buffer for generated B
-  bool have_genA = false;
-  bool have_genB = false;
-};
-
-
-
-//we intentionally send to only 1 actor to force load balancing and also
-//see what happens if an actor gets a request that it is not responsible for
-caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_async_actor_state>* self,
-		caf::actor exit_actor) {
-  return {
-
-    // ------------------------------------------------------------------
-    // 1) Initial request: generate two matrices
-    // ------------------------------------------------------------------
-    [=](int N) {
-      caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-      self->state().N = N;
-      self->state().have_genA = false;
-      self->state().have_genB = false;
-      self->state().d_genA = nullptr;
-      self->state().d_genB = nullptr;
-
-      // Explicit generator launch configuration
-      const int THREADS = 256;
-      const int BLOCKS  = (N * N + THREADS - 1) / THREADS;
-      caf::cuda::nd_range gen_range(BLOCKS, 1, 1,
-                                    THREADS, 1, 1);
-
-      auto gen_program =
-        mgr.create_program_from_fatbin(
-          "../generate_random_matrix.fatbin",
-          "generate_random_matrix");
-
-      auto send_launch = [&](const std::string& name) {
-        auto tok = caf::cuda::make_launch_token(
-                     gen_program,
-                     gen_range,
-                     sizeof(int) * N * N,
-                     name,
-                     self,
-		     self->state().id //use this for dependency number
-				      //required for scheduler actor internal bookeeping
-                   );
-        anon_mail(tok).send(mgr.get_scheduler_actor());
-      };
-
-      send_launch("genA");
-      send_launch("genB");
-    },
-
-    // ------------------------------------------------------------------
-    // 2) Handle scheduler response tokens
-    // ------------------------------------------------------------------
-[=](caf::cuda::response_token_ptr res_token) mutable {
-  try {
-    const auto type = res_token->getType();
-    const auto name = res_token->name();
-    int N = self->state().N;
-
-    // ----------------------------
-    // TRANSFER handling
-    // ----------------------------
-if (type == TRANSFER) {
-
-  //std::cout << "TRANSFER for " << name << std::endl;
-
-  // If mmul is moving, move BOTH matrices
-  if (name == "mmul") {
-
-    if (self->state().d_genA) {
-      auto host_copyA = self->state().d_genA->copy_to_host();
-      self->state().d_genA =
-        randomMatrix.transfer_memory(res_token, in_out<int>{host_copyA});
-    }
-
-    if (self->state().d_genB) {
-      auto host_copyB = self->state().d_genB->copy_to_host();
-      self->state().d_genB =
-        randomMatrix.transfer_memory(res_token, in_out<int>{host_copyB});
-    }
-
-    if (res_token->getDeviceNumber() == 1) {
-    //std::cout << "Moved genA and genB for mmul\n";
-  } 
-  }
-
-  res_token->release();
-  return;
+template <class Fn>
+double time_run(Fn&& fn) {
+    auto start = std::chrono::steady_clock::now();
+    fn();
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+    return elapsed.count();
 }
 
+void run_scheduler_integration_scaling_test(actor_system& sys) {
+    const int min_N = 32;
+    const int max_N = 2048;
+    const int num_distinct_sizes = 10;
+    const std::vector<int> actor_counts = {50000};
 
-    // ----------------------------
-    // LAUNCH_RESPONSE handling
-    // ----------------------------
-    if (type != LAUNCH_RESPONSE)
-      return;
-
-    // Generator completion (genA / genB)
-    if (name == "genA" || name == "genB") {
-      auto out_arg  = caf::cuda::create_out_arg(N * N);
-      auto size_arg = caf::cuda::create_in_arg(N * N);
-      auto seed_arg = caf::cuda::create_in_arg(rand());
-      auto maxval_arg = caf::cuda::create_in_arg(9999);
-
-      const int THREADS = 256;
-      const int BLOCKS  = (N * N + THREADS - 1) / THREADS;
-
-      caf::cuda::nd_range gen_range(BLOCKS,1,1, THREADS,1,1);
-
-      auto gen_program =
-        caf::cuda::manager::get().create_program_from_fatbin(
-          "../generate_random_matrix.fatbin",
-          "generate_random_matrix");
-
-      auto result = randomMatrix.run_async(
-        gen_program, gen_range, res_token,
-        out_arg, size_arg, seed_arg, maxval_arg);
-
-      auto device_buffer = std::get<0>(result);
-
-      if (name == "genA") {
-	      self->state().d_genA = device_buffer;
-      	      self->state().have_genA = true;
-      
-      }
-      else {
-	      self->state().d_genB = device_buffer;
-      	      self->state().have_genB = true;
-      }
-
-      // After handling genA / genB completion
-if (self->state().have_genA && self->state().have_genB) {
-    const int THREADS_M = 32;
-    int BLOCKS_M = (N + THREADS_M - 1) / THREADS_M;
-
-    caf::cuda::nd_range mmul_range(
-      BLOCKS_M, BLOCKS_M, 1,
-      THREADS_M, THREADS_M, 1
-    );
-
-    auto mmul_program =
-      caf::cuda::manager::get().create_program_from_cubin(
-        "../mmul.cubin",
-        "matrixMul"
-      );
-
-    // Create a launch token for mmul
-    auto mmul_token = caf::cuda::make_launch_token(
-      mmul_program,
-      mmul_range,
-      sizeof(int) * N * N,
-      "mmul",
-      self,
-      self ->state().id //needed to help track depedencies
-    );
-
-    // Send the launch token to the scheduler actor
-    anon_mail(mmul_token)
-      .send(caf::cuda::manager::get().get_scheduler_actor());
-  }
-
-
-      res_token->release();
-      return;
-    }
-
-    // ----------------------------
-    // mmul completion / kernel launch
-    // ----------------------------
-    if (name == "mmul") {
-      const int THREADS_M = 32;
-      int BLOCKS_M = (N + THREADS_M - 1) / THREADS_M;
-
-      caf::cuda::nd_range mmul_range(BLOCKS_M, BLOCKS_M, 1,
-                                    THREADS_M, THREADS_M, 1);
-
-      auto mmul_program =
-        caf::cuda::manager::get().create_program_from_cubin(
-          "../mmul.cubin",
-          "matrixMul");
-
-      auto outC = caf::cuda::create_out_arg(N * N);
-      auto inN  = caf::cuda::create_in_arg(N);
 
-      auto result = mmulAsync.run(
-        mmul_program,
-        mmul_range,
-        res_token,
-        self->state().d_genA,
-        self->state().d_genB,
-        outC,
-        inN);
+    //  const std::vector<int> actor_counts = {5};
 
-      std::vector<int> matrixC = caf::cuda::extract_vector<int>(result, 2);
 
-      self->state().have_genA = false;
-      self->state().have_genB = false;
-      self->state().d_genA.reset();
-      self->state().d_genB.reset();
 
-      res_token->release();
-      self->mail(1).send(exit_actor);
-    }
-
-  } catch (std::exception& e) {
-    std::cerr << "*** Caught exception: " << e.what() << "\n";
-
-    if (self->state().d_genA)
-      std::cerr << "d_genA deviceID: " << self->state().d_genA->deviceID() << "\n";
-    if (self->state().d_genB)
-      std::cerr << "d_genB deviceID: " << self->state().d_genB->deviceID() << "\n";
-    if (res_token) {
-      std::cerr << "res_token deviceID: " << res_token->getDeviceNumber() << "\n";
-      std::cerr << "res_token name: " << res_token->name() << "\n";
-    
-    }
+    // Generate deterministic random pool once
+    auto pool_ptr = std::make_shared<MatrixPool>(
+        create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42));
     
+    std::vector<int> available_Ns;
+    for (const auto& pair : pool_ptr->A) available_Ns.push_back(pair.first);
+
+    for (int num_tasks : actor_counts) {
+        manager_config config;
+        manager::init(sys, config);
+        auto& mgr = manager::get();
+
+        // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
+        mgr.toggle_scheduler_actor(8, 1);
+
+        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        const int THREADS = 32;
+
+        std::vector<token_ptr> tokens;
+        std::mt19937 rng(42);
+        std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
+
+        std::cout << "=====================================\n";
+        std::cout << "Scheduler Test | tasks=" << num_tasks << "\n";
+
+        // Spawn the exit actor for this specific test run (moved inside the loop)
+        auto exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_tasks);
+
+        // Spawn task actors and prepare tokens
+        for (int i = 0; i < num_tasks; ++i) {
+            int current_N = available_Ns[dist_N_idx(rng)];
+            nd_range range((current_N + THREADS - 1) / THREADS, 
+                           (current_N + THREADS - 1) / THREADS, 1, 
+                           THREADS, THREADS, 1);
+            
+            auto worker = sys.spawn(make_task_actor_behavior,
+                                    program,
+                                    current_N, 
+                                    pool_ptr,
+                                    exit_actor); // Pass exit_actor to task actors
+
+            tokens.push_back(make_launch_token(program, range, 0, 
+                             "task_" + std::to_string(i), worker));
+        }
+
+        double elapsed = time_run([&]() {
+            std::cout << "[MAIN] Dispatching batch to scheduler..." << std::endl;
+            mgr.send_scheduler_actor_message(std::move(tokens));
+
+            // The dispatch is asynchronous. To get an accurate measurement, we must
+            // block until the exit_actor terminates (signaling all 50k tasks are done).
+            scoped_actor self{sys};
+            self->wait_for(exit_actor);
+        });
+
+        std::cout << "Run complete. Time: " << elapsed << " s\n";
+        manager::shutdown(); // Reset manager state for the next potential iteration
     }
-
-
-}
-
-  };
-}
-
-
-
-
-//this test is meant to demonstrate the fact that scheduler actors can 
-//migrate work to correct load imbalance
-//the sizes should be large enough such that the tests exceed 4-5 seconds in total
-//otherwise the schedulers wont care to do this fast enough
-void run_load_balance_test_with_large_dependencies(
-    caf::actor_system& sys,
-    const int n,
-    int num_actors,
-    bool randomize = false)
-{
-    caf::cuda::manager& mgr = caf::cuda::manager::get(); 
-
-
-    //set the behaviors of each scheduler actor
-    for (int i = 0; i < mgr.get_num_devices();i++) {
-	    mgr.send_scheduler_actor_message("multilevel",i);
-    }
-
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-
-
-
-
-
-    auto t_start = std::chrono::steady_clock::now();
-
-for (int i = 0; i < num_actors; ++i) {
-	caf::actor a = sys.spawn(mmul_async_actor_fun, exit_actor);
-	anon_mail(n).send(a);	
-}
-  
-	  //this time the gpu actors can figure out how to send tokens to the correct GPU scheduler
-          sys.await_all_actors_done();
 }
 
-
-
-void caf_main(caf::actor_system& sys) {
-  
-	caf::cuda::manager_config man_config(true); //turns the scheduler on
-	caf::cuda::manager::init(sys,man_config);
-	
-	
-	//no dependencies
-//	std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048,4096};
-//	const int num_actors = 2000;
-//	run_load_balance_test(sys,sizes,num_actors);
-
-
-	//dependencies
-	run_load_balance_test_with_large_dependencies(sys,1024,2000);
-
-
+void caf_main(actor_system& sys) {
+    run_scheduler_integration_scaling_test(sys);
+    std::cout << "[MAIN] Integration test complete." << std::endl;
 }
 
-
-
-
-CAF_MAIN()
+CAF_MAIN(id_block::cuda_control)

From 241b222476a798e036e6b44e9fbf2e3a93819a3d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 18 Jun 2026 09:08:49 -0600
Subject: [PATCH 0954/1000] updated test to align with new scheduler actor
 changes

---
 .../fault-tolerance-test/CMakeLists.txt       |   2 +
 .../fault-tolerance-test/compile_kernels.sh   |   9 +-
 .../fault-tolerance-test/main.test.cpp        | 448 ++++++++----------
 .../fault-tolerance-test/mmul.cu              |  16 +
 4 files changed, 228 insertions(+), 247 deletions(-)
 create mode 100644 libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/mmul.cu

diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/CMakeLists.txt b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/CMakeLists.txt
index 89bcf5ba7c..59d7388f4a 100644
--- a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/CMakeLists.txt
@@ -39,6 +39,8 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
 )
 
 
diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/compile_kernels.sh b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/compile_kernels.sh
index cf7385147a..1661ecf24e 100755
--- a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/compile_kernels.sh
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/compile_kernels.sh
@@ -1,4 +1,3 @@
-#!/bin/bash
 set -e
 
 # Detect first GPU compute capability
@@ -7,12 +6,8 @@ SM_ARCH="sm_${ARCH/./}"
 echo "Using NVCC arch flag: $SM_ARCH"
 
 # Compile mmul.cu to cubin in current directory
-nvcc -arch=$SM_ARCH -cubin faulty_kernels.cu -o fault.cubin
+nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
 echo "Generated mmul.cubin"
 
-# Compile genMatrix.cu to fatbin in current directory
-#nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
-#echo "Generated generate_random_matrix.fatbin"
-
-echo "All kernels compiled successfully!"
 
+echo "All kernels compiled successfully!"
\ No newline at end of file
diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
index 70a4323957..e1e6d21c08 100644
--- a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/main.test.cpp
@@ -2,296 +2,264 @@
 #include <caf/cuda/all.hpp>
 #include <caf/cuda/control-layer/all-control-layer.hpp>
 #include <iostream>
-#include <iomanip>
 #include <vector>
-#include <chrono>
-#include <thread>
-#include <algorithm>
-#include <numeric>
+#include <string>
 #include <random>
-#include <unistd.h>
-#include "caf/actor_registry.hpp"
-#include <chrono>
-#include <iostream>
-//#include <caf/atoms.hpp>
-
-
-
+#include <unordered_map>
+#include <unordered_set>
+#include <algorithm>
 
 using namespace caf;
-using namespace std::chrono_literals;
+using namespace caf::cuda;
 
+// A generic command runner to provide access to CUDA stream callbacks
+static command_runner<in<int>, in<int>, out<int>, in<int>> runner;
 
-struct exit_actor_state {
-	int completed = 0;
+// MatrixPool structure from mmul-random-batch-benchmark
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
 };
 
-
-// --- command runner types (put near top of file) -------------------------
-using initCommand =
-  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, in<int>, in<unsigned long long>>;
-
-using divCommand  =
-  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, in<int>>;
-
-using sumCommand  =
-  caf::cuda::command_runner<caf::cuda::mem_ptr<float>, caf::cuda::mem_ptr<float>, in<int>>;
-
-// single instances (can be file-global)
-static initCommand init_cmd;
-static divCommand  div_cmd;
-static sumCommand  sum_cmd;
-
-// --- pipeline actor state (device buffers persist here) ------------------
-struct pipeline_actor_state {
-    int id = rand();
-
-    // device-side buffers that must persist across stages:
-    caf::cuda::mem_ptr<float> d_denoms;
-    caf::cuda::mem_ptr<float> d_results;
-    caf::cuda::mem_ptr<float> d_sum;
+struct task_actor_state {
+    program_ptr prog;
+    int N_val;
+    std::shared_ptr<MatrixPool> pool;
 };
 
-// --- corrected pipeline_actor -------------------------------------------
-behavior pipeline_actor(caf::stateful_actor<pipeline_actor_state>* self,
-                        actor supervisor,
-                        caf::cuda::program_ptr p1,
-                        caf::cuda::program_ptr p2,
-                        caf::cuda::program_ptr p3,
-                        int n)
-{
-    // host-side scratch (only used for post-stage2 NaN/Inf detection)
-    std::vector<float> h_results;
-
-    // scheduler from manager
-    caf::actor scheduler = caf::cuda::manager::get().get_scheduler_actor();
-
-    // nd_range used for all stages (adapt to your kernels as needed)
-    caf::cuda::nd_range range{
-        {(n + 255) / 256, 1, 1},
-        {256, 1, 1}
-    };
-
-    // helper to create and send a launch token
-    auto launch = [&](caf::cuda::program_ptr prog, const std::string& stage) {
-        auto tok = make_launch_token(
-            prog,
-            range,
-            /*memory_usage=*/static_cast<int>(sizeof(float) * n),
-            stage,
-            self,
-            self->state().id  // dependency/demo id
-        );
-        anon_mail(tok).send(scheduler);
-    };
-
-    // fire all three tokens (scheduler will reply with response_token on grants)
-    launch(p1, "stage1");
-    launch(p2, "stage2");
-    launch(p3, "stage3");
+// create_matrix_pool_random function from mmul-random-batch-benchmark
+MatrixPool create_matrix_pool_random(
+    int num_sizes,
+    int min_N,
+    int max_N,
+    unsigned int seed
+) {
+    MatrixPool pool;
+    std::mt19937 rng(seed);
+    std::uniform_int_distribution<int> dist(min_N, max_N);
+    std::unordered_set<int> used;
+    while (used.size() < static_cast<size_t>(num_sizes)) {
+        int N = dist(rng);
+        if (used.insert(N).second) {
+            pool.A[N] = std::vector<int>(N * N, 1);
+            pool.B[N] = std::vector<int>(N * N, 2); // Changed to 2 for distinct input
+        }
+    }
+    return pool;
+}
 
+// This actor represents a single task that requests permission from the scheduler.
+//this actor is mean to represent the case of when it sends out a request but crashes 
+//before it can receive a response, showing the scheduler actor will just move on
+behavior bad_task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_actor) {
+   
+   
+        //lets crash the actor before it can receive a response
+        
+        auto token = make_launch_token(self->state().prog,caf::cuda::nd_range(1,1,1,1,1,1),0,"hello",self);
+        caf::cuda::manager::get().send_scheduler_actor_message(token);
+        self->quit();
+   
+   
     return {
-
-        // handle response tokens by name — opaque to reclaim payload
-        [=](caf::cuda::response_token_ptr res_token) mutable {
-
-            const auto& stage = res_token->name();
-
-            // --------------------- Stage 1: init_denominators ---------------------
-            if (stage == "stage1") {
-                // allocate device buffer for denominators (persist in state)
-
-     	           std::cout << "Starting stage 1\n";		    
-		    unsigned long long seed = static_cast<unsigned long long>(
-    std::chrono::high_resolution_clock::now().time_since_epoch().count()
-);
-
+        [=](response_token_ptr res) mutable {
+            if (res->getType() == LAUNCH_RESPONSE) {
+                auto& st = self->state();
+                
+                // We need to cast the base response_token to access the specific nd_range stored in it.
+                auto launch_res = static_cast<launch_response_token*>(res.get());
+                int N = st.N_val;
+
+                // 1. Setup GPU arguments.
+                // Fetch data from the shared pool only when scheduled to save RAM
+                auto in_a = create_in_arg(st.pool->A.at(N));
+                auto in_b = create_in_arg(st.pool->B.at(N));
+                auto out_c = create_out_arg_with_size<int>(N * N);
+                auto in_n = create_in_arg(N);
+
+                int threads = 32;
+                int blocks = (N + threads - 1) / threads;
+                caf::cuda::nd_range dims = caf::cuda::nd_range(blocks,blocks,1,
+                                                               threads,threads,1);
+
+
+                // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
+                auto result_tuple = runner.run_async(st.prog, dims , res, in_a, in_b, out_c, in_n);
+                auto d_c = std::get<2>(result_tuple);
+
+                // 3. Asynchronous Copyback.
+                // Allocate a local buffer for the result to keep the total system memory low.
+                auto h_c = std::make_shared<std::vector<int>>(N * N);
+                // The launch_response_token is released inside the callback 
+                // to signal to the scheduler that the resource is free. (No serial verification here)
+                runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [res, exit_actor, h_c](int* /*ptr*/, size_t /*sz*/) {
+                    res->release();
+                    anon_mail(1).send(exit_actor);
+                });
+            }
+        }
+    };
+}
 
 
-		out<float> buffer = caf::cuda::create_out_arg_with_size<float>(n);
-                self->state().d_denoms = init_cmd.transfer_memory(res_token,buffer);
 
-                // run kernel on the stream/device from res_token
-                // kernel signature: (float* denominators, int n, unsigned long long seed)
-                init_cmd.run_async(
-                    p1,
-                    range,
-                    res_token,                            // uses token's stream/device
-                    self->state().d_denoms,               // device buffer
-                    caf::cuda::create_in_arg(n),          // n
-                    caf::cuda::create_in_arg(seed)     // seed
-                );
 
-		std::cout << "Finished stage 1\n";
-                // stage1 intentionally no checks — data may contain zeros
-                return;
-            }
+// This actor represents a single task that requests permission from the scheduler.
+behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_actor) {
+   
+   
+        
+        auto token = make_launch_token(self->state().prog,caf::cuda::nd_range(1,1,1,1,1,1),0,"hello",self);
+        caf::cuda::manager::get().send_scheduler_actor_message(token);
 
-            // --------------------- Stage 2: perform_division ---------------------
-            if (stage == "stage2") {
-                // allocate device buffer for results (persist in state)
-		    std::vector<float> buffer1(n); 
-
-     	           std::cout << "Starting stage 2\n";		    
-		self->state().d_results = div_cmd.transfer_memory(res_token,out<float>{buffer1});
-
-                // create a host numerators vector (all ones)
-                std::vector<float> h_nums(n, 1.0f);
-
-                // transfer numerators to device on the token's stream/device
-                // transfer_memory returns a caf::cuda::mem_ptr<float>
-                auto d_nums = div_cmd.transfer_memory(res_token, in_out<float>{h_nums});
-
-                // run division kernel on the token's stream/device:
-                // kernel signature: (float* numerators, float* denominators, float* results, int n)
-                div_cmd.run(
-                    p2,
-                    range,
-                    res_token,
-                    d_nums,
-                    self->state().d_denoms,
-                    self->state().d_results,
-                    caf::cuda::create_in_arg(n)
-                );
-
-                // extract the device results back to host for verification.
-                // extract_vector will synchronize as needed.
-                h_results = self->state().d_results -> copy_to_host();
-
-                // check for NaN/Inf AFTER the kernel finished
-                bool fault = false;
-                for (float v : h_results) {
-                    if (!std::isfinite(v)) {
-                        fault = true;
-                        break;
-                    }
-                }
-
-                if (fault) {
-                    // inform supervisor and exit;
-                    anon_mail(std::string("crash")).send(supervisor);
-                    self->quit();
-                    return;
-                }
-
-                // stage2 passed — keep d_results in state for stage3
-                return;
-            }
-
-            // --------------------- Stage 3: sum_results --------------------------
-            if (stage == "stage3") {
-                // allocate device scalar for sum result
+      
+   
+   
+    return {
+        [=](response_token_ptr res) mutable {
+            if (res->getType() == LAUNCH_RESPONSE) {
+                auto& st = self->state();
                 
-     	           std::cout << "Starting stage 3\n";		    
-		    std::vector<float> buffer1(1); 
-
-		self->state().d_sum = div_cmd.transfer_memory(res_token,out<float>{buffer1});
-		    
-                // run reduction on the token's stream/device:
-                // kernel signature: (float* results, float* final_sum, int n)
-                sum_cmd.run(
-                    p3,
-                    range,
-                    res_token,
-                    self->state().d_results,
-                    self->state().d_sum,
-                    caf::cuda::create_in_arg(n)
-                );
-
-                // extract final scalar
-		
-		std::vector<float> buf = self->state().d_sum -> copy_to_host();
-                float final_sum = buf[0];
-                std::cout << "[pipeline] completed, sum = " << final_sum << "\n";
-
-                // successful completion -> tell supervisor to tear everything down
-                anon_mail(std::string("done")).send(supervisor);
-
-                // quit the pipeline actor
-                self->quit();
-                return;
+                // We need to cast the base response_token to access the specific nd_range stored in it.
+                auto launch_res = static_cast<launch_response_token*>(res.get());
+                int N = st.N_val;
+
+                // 1. Setup GPU arguments.
+                // Fetch data from the shared pool only when scheduled to save RAM
+                auto in_a = create_in_arg(st.pool->A.at(N));
+                auto in_b = create_in_arg(st.pool->B.at(N));
+                auto out_c = create_out_arg_with_size<int>(N * N);
+                auto in_n = create_in_arg(N);
+
+                // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
+                auto result_tuple = runner.run_async(st.prog, launch_res->getRange(), res, in_a, in_b, out_c, in_n);
+                auto d_c = std::get<2>(result_tuple);
+
+                // 3. Asynchronous Copyback.
+                // Allocate a local buffer for the result to keep the total system memory low.
+                auto h_c = std::make_shared<std::vector<int>>(N * N);
+                // The launch_response_token is released inside the callback 
+                // to signal to the scheduler that the resource is free. (No serial verification here)
+                runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [res, exit_actor, h_c](int* /*ptr*/, size_t /*sz*/) {
+                    res->release();
+                    anon_mail(1).send(exit_actor);
+                });
             }
-
-            // unknown stage: ignore or log
-            std::cerr << "[pipeline] received unknown response token: " << stage << "\n";
         }
     };
 }
 
 
 
-void supervisor_handle_msg(event_based_actor* self,
-                           caf::cuda::program_ptr p1,
-                           caf::cuda::program_ptr p2,
-                           caf::cuda::program_ptr p3,
-                           int n,
-                           const std::string& msg) {
-    if (msg == "crash") {
-        std::cout << "[supervisor] Pipeline crashed — restarting\n";
-        self->system().spawn(pipeline_actor, self, p1, p2, p3, n);
-    } else if (msg == "done") {
-        std::cout << "[supervisor] Pipeline completed — shutting down\n";
-        caf::cuda::manager::shutdown();
-        self->quit();
-    } else {
-        std::cerr << "[supervisor] Unknown message: " << msg << "\n";
-    }
+// Helper function to initialize task_actor_state
+behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, std::shared_ptr<MatrixPool> pool, caf::actor exit_actor) {
+    auto& st = self->state();
+    st.prog = std::move(prog);
+    st.N_val = N_val;
+    st.pool = std::move(pool);
+    return task_actor_fun(self, exit_actor); // Pass exit_actor to task_actor_fun
 }
 
 
 
-
-behavior supervisor_actor(event_based_actor* self,
-                          caf::cuda::program_ptr p1,
-                          caf::cuda::program_ptr p2,
-                          caf::cuda::program_ptr p3,
-                          int n) {
-    // Spawn first pipeline safely
-    self->system().spawn(pipeline_actor, self, p1, p2, p3, n);
-
-    // Behavior: just route string messages to the helper
-    return {
-        [=](const std::string& msg) {
-            supervisor_handle_msg(self, p1, p2, p3, n, msg);
-        }
-    };
+// Helper function to initialize bad+task_actor_state
+behavior make_bad_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, std::shared_ptr<MatrixPool> pool, caf::actor exit_actor) {
+    auto& st = self->state();
+    st.prog = std::move(prog);
+    st.N_val = N_val;
+    st.pool = std::move(pool);
+    return bad_task_actor_fun(self, exit_actor); // Pass exit_actor to task_actor_fun
 }
 
+template <class Fn>
+double time_run(Fn&& fn) {
+    auto start = std::chrono::steady_clock::now();
+    fn();
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+    return elapsed.count();
+}
 
+void run_scheduler_integration_scaling_test(actor_system& sys) {
+    const int min_N = 1024;
+    const int max_N = 2048;
+    const int num_distinct_sizes = 10;
+    const std::vector<int> actor_counts = {1000};
 
 
-void caf_main(caf::actor_system& sys) {
-
-
+    //  const std::vector<int> actor_counts = {5};
 
-        caf::cuda::manager_config man_config(true); //turns the scheduler on
-        caf::cuda::manager::init(sys,man_config);
 
-	//change the scheduler to core_usage
-	anon_mail(
-        caf::cuda::make_behavior_token("single_usage")
-        ).send(caf::cuda::manager::get().get_scheduler_actor());
 
+    // Generate deterministic random pool once
+    auto pool_ptr = std::make_shared<MatrixPool>(
+        create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42));
+    
+    std::vector<int> available_Ns;
+    for (const auto& pair : pool_ptr->A) available_Ns.push_back(pair.first);
 
-	caf::cuda::program_ptr p1 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","init_denominators");
-	caf::cuda::program_ptr p2 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","perform_division");
-	caf::cuda::program_ptr p3 = caf::cuda::manager::get().create_program_from_cubin("../fault.cubin","sum_results");
+    for (int num_tasks : actor_counts) {
+        manager_config config;
+        manager::init(sys, config);
+        auto& mgr = manager::get();
 
-	sys.spawn(supervisor_actor,p1,p2,p3,1);
-	sys.await_all_actors_done();
+        // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
+        mgr.toggle_scheduler_actor(8, 1);
 
+        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        const int THREADS = 32;
 
-}
+        std::vector<token_ptr> tokens;
+        std::mt19937 rng(42);
+        std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
 
+        std::cout << "=====================================\n";
+        std::cout << "Scheduler Test | tasks=" << num_tasks << "\n";
 
+        // Spawn the exit actor for this specific test run (moved inside the loop)
+        auto exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_tasks);
 
+        // Spawn task actors and prepare tokens
+        for (int i = 0; i < num_tasks; ++i) {
+            int current_N = available_Ns[dist_N_idx(rng)];
+          
 
-CAF_MAIN()
 
+             auto worker = sys.spawn(make_bad_task_actor_behavior,
+                                    program,
+                                    current_N, 
+                                    pool_ptr,
+                                    exit_actor); // Pass exit_actor to task actors
+            
+            auto worker2 = sys.spawn(make_task_actor_behavior,
+                                    program,
+                                    current_N, 
+                                    pool_ptr,
+                                    exit_actor); // Pass exit_actor to task actors
 
 
+    
+        }
 
+        double elapsed = time_run([&]() {
+            std::cout << "[MAIN] Dispatching batch to scheduler..." << std::endl;
+            // mgr.send_scheduler_actor_message(std::move(tokens));
 
+            // The dispatch is asynchronous. To get an accurate measurement, we must
+            // block until the exit_actor terminates (signaling all 50k tasks are done).
+            // anon_mail(num_tasks).delay(std::chrono::seconds(5)).send(exit_actor);
+            scoped_actor self{sys};
+            self->wait_for(exit_actor);
+        });
 
+        std::cout << "Run complete. Time: " << elapsed << " s\n";
+        manager::shutdown(); // Reset manager state for the next potential iteration
+    }
+}
 
+void caf_main(actor_system& sys) {
+    run_scheduler_integration_scaling_test(sys);
+    std::cout << "[MAIN] Integration test complete." << std::endl;
+}
 
+CAF_MAIN(id_block::cuda_control)
diff --git a/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/mmul.cu b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/mmul.cu
new file mode 100644
index 0000000000..c87a1cada8
--- /dev/null
+++ b/libcaf_cuda/tests/control-layer-tests/fault-tolerance-test/mmul.cu
@@ -0,0 +1,16 @@
+// mmul.cu
+extern "C" __global__
+void matrixMul(const int* a, const int* b, int* c, int N) {
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    //printf("%d\n",N);
+    if (row < N && col < N) {
+        int temp = 0;
+        for (int k = 0; k < N; ++k) {
+            temp += a[row * N + k] * b[k * N + col];
+        }
+        c[row * N + col] = temp;
+    }
+}
+
+

From f158bc4d6ca34d300813530f370f99a22153ee44 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Thu, 18 Jun 2026 09:16:24 -0600
Subject: [PATCH 0955/1000] Removed legacy tests.

---
 .../bulk-token-test/CMakeLists.txt            |   44 -
 .../bulk-token-test/compile_kernels.sh        |   18 -
 .../bulk-token-test/main.test.cpp             |  423 -------
 .../bulk-token-test/mmul.cu                   |   16 -
 .../bulk-token-test/mmul.cubin                |  Bin 5608 -> 0 bytes
 .../CMakeLists.txt                            |   44 -
 .../compile_kernels.sh                        |   18 -
 .../main.test.cpp                             |  418 -------
 .../command-runner-integration-test/mmul.cu   |   16 -
 .../core_usage_behavior_tests/CMakeLists.txt  |   44 -
 .../compile_kernels.sh                        |   18 -
 .../core_usage_behavior_tests/main.test.cpp   | 1029 -----------------
 .../core_usage_behavior_tests/mmul.cu         |   16 -
 13 files changed, 2104 deletions(-)
 delete mode 100644 libcaf_cuda/tests/control-layer-tests/bulk-token-test/CMakeLists.txt
 delete mode 100755 libcaf_cuda/tests/control-layer-tests/bulk-token-test/compile_kernels.sh
 delete mode 100644 libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
 delete mode 100644 libcaf_cuda/tests/control-layer-tests/bulk-token-test/mmul.cu
 delete mode 100644 libcaf_cuda/tests/control-layer-tests/bulk-token-test/mmul.cubin
 delete mode 100644 libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/CMakeLists.txt
 delete mode 100755 libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/compile_kernels.sh
 delete mode 100644 libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
 delete mode 100644 libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/mmul.cu
 delete mode 100644 libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/CMakeLists.txt
 delete mode 100755 libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/compile_kernels.sh
 delete mode 100644 libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
 delete mode 100644 libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/mmul.cu

diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/CMakeLists.txt b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/CMakeLists.txt
deleted file mode 100644
index 89bcf5ba7c..0000000000
--- a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/CMakeLists.txt
+++ /dev/null
@@ -1,44 +0,0 @@
-cmake_minimum_required(VERSION 3.16.3)
-
-# 1) Enforce C++20 (needs modern C++ features)
-set(CMAKE_CXX_STANDARD        20)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_EXTENSIONS      OFF)
-
-# 2) Set CAF source and build directories
-set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
-set(CAF_BUILD "${CAF_SRC}/build")
-
-
-#required since cmake cant seem to find the compiler 
-set(CMAKE_CXX_COMPILER "/usr/bin/g++")
-set(CMAKE_C_COMPILER "/usr/bin/gcc")
-
-project(CUDA_ACTORS)
-
-find_package(CUDA REQUIRED)
-find_package(CUDAToolkit REQUIRED)
-
-include_directories(
-  "${CAF_SRC}/include"
-  "${CAF_SRC}/libcaf_io"
-  "${CAF_SRC}/libcaf_core"
-  "${CAF_SRC}/libcaf_opencl"
-  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
-  "${CAF_SRC}/libcaf_cuda"
-)
-
-
-# 5) Declare your executable
-add_executable(test main.test.cpp)
-target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
-
-target_link_libraries(test
-  PRIVATE
-    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-    CUDA::nvrtc
-)
-
-
diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/compile_kernels.sh b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/compile_kernels.sh
deleted file mode 100755
index 586196454e..0000000000
--- a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/compile_kernels.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-set -e
-
-# Detect first GPU compute capability
-ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
-SM_ARCH="sm_${ARCH/./}"
-echo "Using NVCC arch flag: $SM_ARCH"
-
-# Compile mmul.cu to cubin in current directory
-nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
-echo "Generated mmul.cubin"
-
-# Compile genMatrix.cu to fatbin in current directory
-#nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
-#echo "Generated generate_random_matrix.fatbin"
-
-echo "All kernels compiled successfully!"
-
diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
deleted file mode 100644
index 5ebdae1ec0..0000000000
--- a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/main.test.cpp
+++ /dev/null
@@ -1,423 +0,0 @@
-#include <caf/all.hpp>
-#include <caf/cuda/all.hpp>
-#include <caf/cuda/control-layer/all-control-layer.hpp>
-#include <iostream>
-#include <iomanip>
-#include <vector>
-#include <chrono>
-#include <thread>
-#include <algorithm>
-#include <numeric>
-#include <random>
-#include <unistd.h>
-#include "caf/actor_registry.hpp"
-#include <chrono>
-#include <iostream>
-//#include <caf/atoms.hpp>
-
-
-
-using namespace caf;
-using namespace std::chrono_literals;
-
-
-struct exit_actor_state {
-	int completed = 0;
-    std::chrono::steady_clock::time_point start_time;
-
-};
-
-
-
-
-
-// Define a custom type ID block for custom actors
-CAF_ADD_ATOM(cuda,shared_mem)
-
-
-
-
-
-// Extend your actor state to keep the start time
-struct mmul_actor_state {
-  static inline const char* name = "my_actor";
-  int N = 1024; // example state variable
-  int id = rand(); // an actor id
-  // per-actor timing start
-  int times = 0;
-};
-
-
-
-
-//commands classes used to launch kernels 
-using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
-using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
-
-using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
-
-mmulCommand mmul;
-matrixGenCommand randomMatrix;
-mmulAsyncCommand mmulAsync;
-
-
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-  
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
-    }
-  }
-}
-
-
-
-
-// Stateful actor behavior
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N, const std::vector<int>& matrix1, const std::vector<int>& matrix2) {
- 
-
-	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
-
-  	return {
-
-	  [=] (caf::cuda::response_token_ptr launch_response_token) {
-	 
-		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-		  //assume N = 1024
-		  int N = self -> state().N;
-
-		  //std::cout << "GPU ACTOR sending data to compute\n";
-		  self -> mail(matrix1,matrix2,N).send(self);
-	 
-		 //token should drop out of scope now, triggering a response 
-	  
-	  },
-
-    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB, int N) {
- 
-
-		  //std::cout << "GPU ACTOR  computing\n";
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //create program and dims   
-  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    //create args
-    auto arg1 = caf::cuda::create_in_arg(matrixA);
-    auto arg2 = caf::cuda::create_in_arg(matrixB);
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
-
-    auto tempC = mmul.run(program,dims,self -> state().id,arg1,arg2,arg3,arg4);
-    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-		  //std::cout << "GPU ACTOR done  computing\n";
-  
-    //do not verify result just exit 
-    self->mail(1).send(exit_actor);
-    self->quit();
- // print timestamp in milliseconds
-   // auto now = std::chrono::system_clock::now();
-   // auto ms_since_epoch = std::chrono::duration_cast<std::chrono::milliseconds>(
-    //                          now.time_since_epoch())
-      //                        .count();
-
-//    std::cout << "[GPU ACTOR] actor " << self->state().id
-  //            << " calling self->quit() at "
-    //          << ms_since_epoch << " ms since epoch\n";
-    },
-
-    // 3rd handler: CPU atom + matrices + N
-    [=](const std::vector<int>& matrixA,
-    const std::vector<int>& matrixB,
-    const std::vector<int>& matrixC,
-    int N) {
-
-  //using clock = std::chrono::high_resolution_clock;
-
- // auto start = clock::now();
-
-  //std::cout << "GPU ACTOR verifying\n";
-
-  std::vector<int> result(N * N);
-
-  serial_matrix_multiply(matrixA, matrixB, result, N);
-
-  if (result == matrixC) {
-    std::cout << "actor with id " << self->state().id
-              << " references match\n";
-  } else {
-    std::cout << "actor with id " << self->state().id
-              << " references did not match\n";
-  }
-
- // auto end = clock::now();
-
-  //auto ms =
-    //std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-  //std::cout << "[TIMING] verification took "
-    //        << ms << " ms (actor id "
-      //      << self->state().id << ")\n";
-
-
-
-  /*
-   // print timestamp in milliseconds
-    auto now = std::chrono::system_clock::now();
-    auto ms_since_epoch = std::chrono::duration_cast<std::chrono::milliseconds>(
-                              now.time_since_epoch())
-                              .count();
-
-    std::cout << "[GPU ACTOR] actor " << self->state().id
-              << " calling self->quit() at "
-              << ms_since_epoch << " ms since epoch\n";
-
-	      */
-
-  // signal exit actor and quit
-  self->mail(1).send(exit_actor);
-  self->quit();
-
-    }
-  };
-}
-
-
-caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,
-                             int limit,
-                             int matrix_size) {
-
-    // ------------------------------------
-    // Record exit actor start time
-    // ------------------------------------
-    
-    int N = matrix_size;
-			  std::vector<int> matrix1(N*N);
-		  std::vector<int> matrix2(N*N);
-
-
-	
-   self->state().start_time = std::chrono::steady_clock::now();
-
-    caf::cuda::program_ptr program = caf::cuda::manager::get()
-        .create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    int THREADS = 32;
-    int BLOCKS = (N + THREADS - 1) / THREADS;
-    caf::cuda::nd_range dims(
-        BLOCKS, BLOCKS, 1,
-        THREADS, THREADS, THREADS);
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-    caf::actor scheduler = mgr.get_scheduler_actor();
-    caf::actor exit_actor = self;
-
-    int num_actors = limit;
-    std::vector<caf::actor> actors;
-    actors.reserve(num_actors);
-
-    std::vector<caf::cuda::token_ptr> tokens;
-    tokens.reserve(num_actors);
-
-    // ------------------------------------
-    // Timing accumulators
-    // ------------------------------------
-    auto t_start_all = std::chrono::steady_clock::now();
-
-    long long total_spawn_us = 0;
-    long long total_token_us = 0;
-
-    // ------------------------------------
-    // Spawn actors + create launch tokens
-    // ------------------------------------
-    for (int j = 0; j < num_actors; ++j) {
-
-        auto t_spawn_start = std::chrono::steady_clock::now();
-        caf::actor a = self->spawn(mmul_actor_fun, exit_actor, matrix_size,matrix1,matrix2);
-        actors.push_back(a);
-        auto t_spawn_end = std::chrono::steady_clock::now();
-
-        total_spawn_us += std::chrono::duration_cast<
-            std::chrono::microseconds>(t_spawn_end - t_spawn_start).count();
-
-        auto t_token_start = std::chrono::steady_clock::now();
-        caf::cuda::token_ptr launch_token =
-            caf::cuda::make_launch_token(program, dims, 0, "hello", a);
-        tokens.emplace_back(std::move(launch_token));
-        auto t_token_end = std::chrono::steady_clock::now();
-
-        total_token_us += std::chrono::duration_cast<
-            std::chrono::microseconds>(t_token_end - t_token_start).count();
-    }
-
-    // ------------------------------------
-    // Send tokens to scheduler
-    // ------------------------------------
-    auto t_send_start = std::chrono::steady_clock::now();
-    self->mail(tokens).send(scheduler);
-    auto t_send_end = std::chrono::steady_clock::now();
-
-    auto send_ms = std::chrono::duration_cast<
-        std::chrono::milliseconds>(t_send_end - t_send_start).count();
-
-    auto t_end_all = std::chrono::steady_clock::now();
-    auto total_ms = std::chrono::duration_cast<
-        std::chrono::milliseconds>(t_end_all - t_start_all).count();
-
-    // ------------------------------------
-    // Print setup timings
-    // ------------------------------------
-    std::cout << "[EXIT] total spawn time: "
-              << total_spawn_us / 1000.0 << " ms\n";
-
-    std::cout << "[EXIT] total launch token creation time: "
-              << total_token_us / 1000.0 << " ms\n";
-
-    std::cout << "[EXIT] sending tokens took: "
-              << send_ms << " ms\n";
-
-    std::cout << "[EXIT] total elapsed time (spawn + token + send): "
-              << total_ms << " ms for "
-              << num_actors << " actors\n";
-
-    // ------------------------------------
-    // Exit actor behavior
-    // ------------------------------------
-    return {
-        [=](int num_completed) {
-            self->state().completed += num_completed;
-
-            if (self->state().completed >= limit) {
-
-                auto end_time = std::chrono::steady_clock::now();
-                auto lifetime_ms = std::chrono::duration_cast<
-                    std::chrono::milliseconds>(
-                        end_time - self->state().start_time).count();
-
-                std::cout << "[EXIT] exit actor lifetime: "
-                          << lifetime_ms << " ms\n";
-
-                caf::cuda::manager::shutdown();
-                self->quit();
-            }
-        }
-    };
-}
-
-
-
-
-
-
-
-
-
-#include <chrono>
-#include <iostream>
-
-void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  std::cout << "Starting run mmul test with matrix_size: "
-            << matrix_size << " and num_actors " << num_actors << "\n";
-
-  int limit = 1;
-
-  // ------------------------------------
-  // Start timing
-  // ------------------------------------
-  auto start = std::chrono::steady_clock::now();
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors,matrix_size);
-
-  // Wait for all actors to finish
-  sys.await_all_actors_done();
-
-  // ------------------------------------
-  // Stop timing
-  // ------------------------------------
-  auto end = std::chrono::steady_clock::now();
-  auto duration_ms =
-      std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-  std::cout << "[MMUL TEST] matrix_size=" << matrix_size
-            << ", actors=" << num_actors
-            << ", iterations=" << limit
-            << ", time=" << duration_ms << " ms\n";
-}
-
-
-void run_mmul_scaling_tests(caf::actor_system& sys,caf::cuda::manager_config man_config) {
-  const int min_size   = 10;
-  const int max_size   = 1024;
-  const int min_actors = 1;
-  const int max_actors = 1024;
-
-  // Matrix sizes: 10, 32, 64, 128, ..., 1024
-  std::vector<int> matrix_sizes = {10};
-  for (int s = 32; s <= max_size; s *= 2)
-    matrix_sizes.push_back(s);
-
-  // Actor counts: 1, 2, 4, 8, ..., 1024
-  std::vector<int> actor_counts;
-  for (int a = min_actors; a <= max_actors; a *= 2)
-    actor_counts.push_back(a);
-
-  std::cout << "=== MMUL Scaling Tests ===\n";
-
-  for (int size : matrix_sizes) {
-    for (int actors : actor_counts) {
-      std::cout << "\n[RUN] matrix_size=" << size
-                << ", actors=" << actors << "\n";
-
-      run_mmul_test(sys, size, actors);
-	caf::cuda::manager::init(sys,man_config);
-    }
-  }
-
-  std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
-}
-
-
-
-void caf_main(caf::actor_system& sys) {
-  
-	
-
-	caf::cuda::manager_config man_config(true); //turns the scheduler on
-	caf::cuda::manager::init(sys,man_config);
-       	run_mmul_test(sys,512,512);
-	
-	//tests will delete the old manager so will have to reinit if you do this 
-	//in conjunction with each other	
-//	caf::cuda::manager::init(sys,man_config);
-//	run_red_light_green_light_test(sys,10,1000);
-	
-	//run_mmul_scaling_tests(sys,man_config);
-
-
-}
-
-
-
-
-CAF_MAIN()
diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/mmul.cu b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/mmul.cu
deleted file mode 100644
index c87a1cada8..0000000000
--- a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/mmul.cu
+++ /dev/null
@@ -1,16 +0,0 @@
-// mmul.cu
-extern "C" __global__
-void matrixMul(const int* a, const int* b, int* c, int N) {
-    int row = blockIdx.y * blockDim.y + threadIdx.y;
-    int col = blockIdx.x * blockDim.x + threadIdx.x;
-    //printf("%d\n",N);
-    if (row < N && col < N) {
-        int temp = 0;
-        for (int k = 0; k < N; ++k) {
-            temp += a[row * N + k] * b[k * N + col];
-        }
-        c[row * N + col] = temp;
-    }
-}
-
-
diff --git a/libcaf_cuda/tests/control-layer-tests/bulk-token-test/mmul.cubin b/libcaf_cuda/tests/control-layer-tests/bulk-token-test/mmul.cubin
deleted file mode 100644
index 901c492f65180ca6d4013bb2a448df2f41f8fb45..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 5608
zcmeHL&1+oO6+iDoZ${(MdrxCo9$8W)JF=NTkeoEMS!7{MV=o$`&_xI(#Fk>I;}Xlm
zn~8ZAfs9%xWK##a%C30jW$7OfQfiFwDugb)RD=jk$ii(X1QIg&{m#AbV@A#*Z7E&k
zg5JI7o{xLJ?;XAU!iyJN$N6zZ%y8v>xoOv&&K)yAzhUMJV;AI{JSQH$N5m0X?zX$V
zq_?(?RW{$cZeX1+ciUT6Q(4}+xqNMF^M)+<Hqzem^|fAdExq*S8#!WRtJ_{nHm;6r
zzJ6n?+gsb}oh|QQ-B^F~SC==Fwd)(QoNT<YJi4(qyaPo@wf1`N+KsJ$!+?_N|7BAy
zch+Eq46FP7Ei#dM{P%fmdr~`u?CJjaq5VJTzJF-{uezUQO@V#}%SK$t@7E1(fU#uB
zxaRQ0e~WWe-|6>TQXTB{TUA|(aR1pt?9eKc=~FGgg^v;7f9U3P{z_HG04?Irf5NPZ
z^c$u1WNG~qpV=(_PkqgQzg87WyW-4=H!~;xabt)lO_4{FIqXUd_+$<{QXPVMCh^|v
zK>VkDQj0%Vl|OR(h{(?|>u}v_Tw6JXH3V=}?Ht>=Y3D^dx9oh@&d=!#8|-ZO9Wnfl
z7=A|#zaxg<5yS6@;fKx2@7nWWpM?#-Yx!Nv?^=G>@<Z{dr||I*6k72!elX}u#^Z<a
z|K$O98yUlLjqrCbl6Ss9;bA$DP-H)nnOj!h)0kT!U>n7g8H`ACoWWn4ct#q7nY<p#
z(zZe}=Kj7S4;G|O_%)Hv`Xi6>m$p5Rh36`Mg?LZ#n;nsdkzC4KQs0x$B5}7NAN;Cs
z#t(Qve!y<hgzgr95J^217vjx%kazJNkw0mBGV<cociC$OdxMG`M1CV3b7|4w1IRP{
zKH<(4=nL3Q{2}>6=Y448zoa@_`P-E->O1&Y<%^q6iFQ-^oAMK>P~XmZ$iE-S>6=wg
zUO^Gjo?XIIBEPgS?2FS-3wyBlh2rBR8t}Nm0i*y=`(DzvjPpNN`ztKO&iQ|;{&Oh+
z!jEv?-$4JKNM98)`sVnLfxj2YPIm?hV0`)`kvk%%(}qiO)erjabZen}iIgG!ZU}fj
zl&>xPcfu#k%YpcP7yQ%8-=KZUKZ!)M^3TBDw66GR;t`LX?vYSC3HjSRFP_fnPkEr9
zQu?~eTL8Xp8a3%R9pI~lzWbCn8wxGT<xLg#wLS1pgc=~?*B8KV?5(Q&72r)@HT(|X
zW1)6TrKkR*_63FgDt|oGOTfxsO6F#mj!l)n3VF4{KJ|CA@CWh%^e5&)Kd1h=4f+!y
zuO!t+{XI{}!23nKRQ~P?`Qu5T4S(dEPGgm?@j(6{-k~g7{uce&DD-d9-X@CL;!mkQ
z^@&X0{m9+v9u0*TK<+;i&sgoF{Fqmu!Jl%V&-4_Zm{y<Rc%nXbx^tn-S$gEhQc|tr
z+7!(X)wdMyb>@Tdf03W1_^y*b2v{!~Psq>mqmYMuY}xbLw12vYhw{(XW`PdW|Eu7y
zssB`->Jx-=*7DyK@Q>^Lss2!S!aR=XpF7LYXLkMs$Y<3*HlEK~e$-2xRw|FO{)G6W
zpAFS>oL0kgARp}g3;k~a@)`}gOXclG&QdZH&j8l=YW~#gOpqC#w;Q=jNqvg>p!H+$
z(?>HH_OQRORSgc_c4U7gk57&K#HYT`fZG0v{%Fb}@6Smv9%S-%SD{~Xg7&Ka_HWBY
zDEIDHH1N0D-)SHFi}ow_YxJipX!+<;ibp((dR2cSUx)h7@uL6ihq=Bc9~e*6KL`D<
z{if8<wBMkAQ32xH%)g*GPwUCVyXaq~`Z&Y<GyO*Wqw-Ln%z#{8t%(0R`;(~;(_eUA
zslRM8KTUmVd{iFlfgLWFSI&RZn|^5OjrNx6P3=K_GGNMo5_t4iT1XiBpdP20FGW9U
z7WUt9fe(V?z-zti-X`40>p}BvzQ+Ef@zwmAuQJ~AdA<bUBH+Q^KeYcN|LV-w9AD%c
z?hE!`V~@rU`8RYw-Df@5<Dq;-KGNM;KC*t09@+h_`N@1N^0V1wKT-eQTP^P6mpl6Y
z;C<TX1H$HC7BAID<AHh~%D-bJe=+||KA8K7@tr8{r%#9<?H_TGzuM1B{i8+x(tW9L
zqm-}g&qqu7#rmvNa2*@_cT4Y$Uzqm{>zC=S{z5$?p2z4Pb6>Dtg9)ZfR<Cgyho;`~
z{s4UzAI*2j2mLtviJ_02IGMEl1Md^`TlNR?tN#xc07pXlQaq0G!m<4BMKtUFH#ETZ
zheXEoy+r-(dv1^XHa>9@ctiGV!@Spkw)y9#<3hZi_W=6e3<AEl#!Wxu`^)EhOJTg{
zP#;VuQy<!Id`%y%7t}+Xj*U?}^`G%?p`ZH4#NXTx3!o1IqaWXIE9kEkJPhG~`+h@z
zJVLzs|G}!z{*E4C?2i*q^nGgHt9VbkBlh6=`l@|TL;v4(#nlJFr|>%Txpa%haF~92
z5wPAb^zR3eG;=&i{s*wC@*jLyn)VmSkd$*4`peL+=x_Y;7Hj`;{EpVV$VX*jtnGHb
z%ig2cQ@&#s4=v%lcAqr-<z$ix@wNNE!8|IHsJ0c_v2@uqO8cZijFQPi%YW7~W;SZ>
zXTmU_{|<iq3el08Nqxk|GRpsF?7@eJ&PR6X*fUf9&Fs(S+rS{j4~;AKpx9z4j$0XD
zw*PDF(I?hitjy?G{<lgFdz}8yfSe?zs4X)myOI5`iRUAJi+$osgJg_r_J7h6N|$%-
bMmA^q^Gxz(dRnr4!Y!a9n@9P+?|I(>-yt?^

diff --git a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/CMakeLists.txt b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/CMakeLists.txt
deleted file mode 100644
index 89bcf5ba7c..0000000000
--- a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/CMakeLists.txt
+++ /dev/null
@@ -1,44 +0,0 @@
-cmake_minimum_required(VERSION 3.16.3)
-
-# 1) Enforce C++20 (needs modern C++ features)
-set(CMAKE_CXX_STANDARD        20)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_EXTENSIONS      OFF)
-
-# 2) Set CAF source and build directories
-set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
-set(CAF_BUILD "${CAF_SRC}/build")
-
-
-#required since cmake cant seem to find the compiler 
-set(CMAKE_CXX_COMPILER "/usr/bin/g++")
-set(CMAKE_C_COMPILER "/usr/bin/gcc")
-
-project(CUDA_ACTORS)
-
-find_package(CUDA REQUIRED)
-find_package(CUDAToolkit REQUIRED)
-
-include_directories(
-  "${CAF_SRC}/include"
-  "${CAF_SRC}/libcaf_io"
-  "${CAF_SRC}/libcaf_core"
-  "${CAF_SRC}/libcaf_opencl"
-  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
-  "${CAF_SRC}/libcaf_cuda"
-)
-
-
-# 5) Declare your executable
-add_executable(test main.test.cpp)
-target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
-
-target_link_libraries(test
-  PRIVATE
-    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-    CUDA::nvrtc
-)
-
-
diff --git a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/compile_kernels.sh b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/compile_kernels.sh
deleted file mode 100755
index 586196454e..0000000000
--- a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/compile_kernels.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-set -e
-
-# Detect first GPU compute capability
-ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
-SM_ARCH="sm_${ARCH/./}"
-echo "Using NVCC arch flag: $SM_ARCH"
-
-# Compile mmul.cu to cubin in current directory
-nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
-echo "Generated mmul.cubin"
-
-# Compile genMatrix.cu to fatbin in current directory
-#nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
-#echo "Generated generate_random_matrix.fatbin"
-
-echo "All kernels compiled successfully!"
-
diff --git a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
deleted file mode 100644
index f5de194eb7..0000000000
--- a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/main.test.cpp
+++ /dev/null
@@ -1,418 +0,0 @@
-#include <caf/all.hpp>
-#include <caf/cuda/all.hpp>
-#include <caf/cuda/control-layer/all-control-layer.hpp>
-#include <iostream>
-#include <iomanip>
-#include <vector>
-#include <chrono>
-#include <thread>
-#include <algorithm>
-#include <numeric>
-#include <random>
-#include <unistd.h>
-#include "caf/actor_registry.hpp"
-#include <chrono>
-#include <iostream>
-//#include <caf/atoms.hpp>
-
-
-
-using namespace caf;
-using namespace std::chrono_literals;
-
-
-struct exit_actor_state {
-	int completed = 0;
-};
-
-
-caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
-
-
-	return {
-		[=](int num_completed) {
-			self->state().completed += num_completed;
-			
-			std::cout << "Actors finished is " << self->state().completed << "\n";
-			if (self->state().completed >= limit) {
-			
-				caf::cuda::manager::shutdown();
-				self->quit();
-			}
-		}
-	};
-
-
-}
-
-
-
-
-
-
-// Define a custom type ID block for custom actors
-CAF_ADD_ATOM(cuda,shared_mem)
-
-
-
-
-
-// Extend your actor state to keep the start time
-struct mmul_actor_state {
-  static inline const char* name = "my_actor";
-  int N = 1024; // example state variable
-  int id = rand(); // an actor id
-  // per-actor timing start
-  std::chrono::high_resolution_clock::time_point start_time;
-  int times = 0;
-  caf::cuda::program_ptr program = caf::cuda::manager::get().create_program_from_cubin("../mmul.cubin","matrixMul");
-  int THREADS = 32;
-  int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims = caf::cuda::nd_range(BLOCKS,BLOCKS,1,THREADS,THREADS,THREADS);
-};
-
-
-
-
-//commands classes used to launch kernels 
-using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
-using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
-
-using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
-
-mmulCommand mmul;
-matrixGenCommand randomMatrix;
-mmulAsyncCommand mmulAsync;
-
-
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-  
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
-    }
-  }
-}
-
-
-
-
-// Stateful actor behavior
-caf::behavior mmul_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
- 
-
-	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-	
-  	caf::actor scheduler = mgr.get_scheduler_actor();
-	
-	//send a launch token
-	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
-			self -> state().dims,
-			0,
-			"hello",
-			self
-			);
-	self -> mail(launch_token).send(scheduler);
-
-	return {
-
-	  [=] (caf::cuda::response_token_ptr res_token) {
-	
-		 if (res_token -> getType() == LAUNCH_RESPONSE) {
-		  //std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-		  //assume N = 1024
-		  int N = self -> state().N;
-		  std::vector<int> matrix1(N*N);
-		  matrix1.reserve(N);
-		  std::vector<int> matrix2(N*N);
-		  matrix2.reserve(N);
-
-		  //std::cout << "GPU ACTOR sending data to compute\n";
-		  self -> mail(matrix1,matrix2,res_token,N).send(self);
-	 
-		 }
-		 else {
-			 std::cout << "Got a memory response token\n";
-		 }
-		 //token should drop out of scope now, triggering a response 
-	  },
-
-    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const std::vector<int>& matrixA,
-        const std::vector<int>& matrixB,
-	const caf::cuda::response_token_ptr& res_token, int N) {
- 
-
-	    //caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
-
-	    //caf::cuda::launch_response_token& kt =
-	//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
-
-		  //std::cout << "GPU ACTOR  computing\n";
-	    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //create program and dims   
-  auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-  const int THREADS = 32;
-  const int BLOCKS = (N + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    //create args
-    auto arg1 = caf::cuda::create_in_arg(matrixA);
-    auto arg2 = caf::cuda::create_in_arg(matrixB);
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
-
-    auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-    std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-		  //std::cout << "GPU ACTOR done  computing\n";
-    //verify its own result 
-    self -> mail(matrixA,matrixB,matrixC,N).send(self);
-
-    },
-
-    // 3rd handler: CPU atom + matrices + N
-    [=](const std::vector<int>& matrixA,
-    const std::vector<int>& matrixB,
-    const std::vector<int>& matrixC,
-    int N) {
-
-  using clock = std::chrono::high_resolution_clock;
-
-  auto start = clock::now();
-
-  //std::cout << "GPU ACTOR verifying\n";
-
-  std::vector<int> result(N * N);
-
-  serial_matrix_multiply(matrixA, matrixB, result, N);
-
-  if (result == matrixC) {
-    std::cout << "actor with id " << self->state().id
-              << " references match\n";
-  } else {
-    std::cout << "actor with id " << self->state().id
-              << " references did not match\n";
-  }
-
-  auto end = clock::now();
-
-  auto ms =
-    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-  std::cout << "[TIMING] verification took "
-            << ms << " ms (actor id "
-            << self->state().id << ")\n";
-
-  // signal exit actor and quit
-  self->mail(1).send(exit_actor);
-  self->quit();
-
-    }
-  };
-}
-
-
-
-void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  int limit = 1;
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
-
-  for (int i = 0; i < limit; i++) {
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_actor_fun,exit_actor,matrix_size));
-  }
-
-
- // std::cout << actors.size() << "\n";
-
-  }
-
-   sys.await_all_actors_done();
-}
-
-// Stateful actor behavior
-caf::behavior mmul_async_actor_fun(caf::stateful_actor<mmul_actor_state>* self,caf::actor exit_actor,int N) {
- 
-
-	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-	
-  	caf::actor scheduler = mgr.get_scheduler_actor();
-	
-	//send a launch token
-	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(self ->state().program,
-			self -> state().dims,
-			0,
-			"hello",
-			self
-			);
-	self -> mail(launch_token).send(scheduler);
-
-	return {
-
-	  [=] (caf::cuda::response_token_ptr res_token) {
-	
-		 if (res_token -> getType() == LAUNCH_RESPONSE) {
-		  int N = self -> state().N;
-
-		  caf::cuda::command_runner<in_out<int>> mem_transfer_command;
-
-
-		  std::vector<int> matrix1(N*N);
-		  std::vector<int> matrix2(N*N);
-
-		  caf::cuda::mem_ptr<int> matrixA = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix1});
-		  caf::cuda::mem_ptr<int> matrixB = mem_transfer_command.transfer_memory(res_token,in_out<int>{matrix2});
-
-		  //std::cout << "GPU ACTOR sending data to compute\n";
-		  self -> mail(matrixA,matrixB,res_token,N).send(self);
-	 
-		 }
-		 else {
-			 //std::cout << "Got a memory response token\n";
-		 }
-		 //token should drop out of scope now, triggering a response 
-	  },
-
-    // 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-    [=](const caf::cuda::mem_ptr<int>& matrixA,
-        const caf::cuda::mem_ptr<int>& matrixB,
-	const caf::cuda::response_token_ptr& res_token, int N) {
- 
-	caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-	//create program and dims   
- 	auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-  	const int THREADS = 32;
-  	const int BLOCKS = (N + THREADS - 1) / THREADS;
-  	caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    //create args
-    auto arg3 = caf::cuda::create_out_arg(N*N);
-    auto arg4 = caf::cuda::create_in_arg(N);
-
-    auto temp = mmulAsync.run_async(program,dims,res_token,matrixA,matrixB,arg3,arg4);
-    caf::cuda::mem_ptr<int> matrixC = std::get<2>(temp);
-
-    std::vector<int> matrix1 = matrixA -> copy_to_host();
-    std::vector<int> matrix2 = matrixB -> copy_to_host();
-    std::vector<int> matrix3 = matrixC -> copy_to_host();
-
-		  //std::cout << "GPU ACTOR done  computing\n";
-    //verify its own result 
-    self -> mail(matrix1,matrix2,matrix3,N).send(self);
-
-    },
-
-    // 3rd handler: CPU atom + matrices + N
-    [=](const std::vector<int>& matrixA,
-    const std::vector<int>& matrixB,
-    const std::vector<int>& matrixC,
-    int N) {
-
-  using clock = std::chrono::high_resolution_clock;
-
-  auto start = clock::now();
-
-  //std::cout << "GPU ACTOR verifying\n";
-
-  std::vector<int> result(N * N);
-
-  serial_matrix_multiply(matrixA, matrixB, result, N);
-
-  if (result == matrixC) {
-    std::cout << "actor with id " << self->state().id
-              << " references match\n";
-  } else {
-    std::cout << "actor with id " << self->state().id
-              << " references did not match\n";
-  }
-
-  auto end = clock::now();
-
-  auto ms =
-    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-  std::cout << "[TIMING] verification took "
-            << ms << " ms (actor id "
-            << self->state().id << ")\n";
-
-  // signal exit actor and quit
-  self->mail(1).send(exit_actor);
-  self->quit();
-
-    }
-  };
-}
-
-
-
-void run_async_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  int limit = 1;
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun,num_actors);
-
-  for (int i = 0; i < limit; i++) {
-  // Spawn num_actors actors running the mmul behavior
-  std::vector<caf::actor> actors;
-  actors.reserve(num_actors);
-  for (int i = 0; i < num_actors; ++i) {
-    actors.push_back(sys.spawn(mmul_async_actor_fun,exit_actor,matrix_size));
-  }
-
-
- // std::cout << actors.size() << "\n";
-
-  }
-
-   sys.await_all_actors_done();
-}
-
-
-
-void caf_main(caf::actor_system& sys) {
-  
-	
-
-	caf::cuda::manager_config man_config(true); //turns the scheduler on
-	caf::cuda::manager::init(sys,man_config);
-       	run_async_mmul_test(sys,10,500);
-	
-	//tests will delete the old manager so will have to reinit if you do this 
-	//in conjunction with each other	
-	//caf::cuda::manager::init(sys,man_config);
-}
-
-
-
-
-CAF_MAIN()
diff --git a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/mmul.cu b/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/mmul.cu
deleted file mode 100644
index c87a1cada8..0000000000
--- a/libcaf_cuda/tests/control-layer-tests/command-runner-integration-test/mmul.cu
+++ /dev/null
@@ -1,16 +0,0 @@
-// mmul.cu
-extern "C" __global__
-void matrixMul(const int* a, const int* b, int* c, int N) {
-    int row = blockIdx.y * blockDim.y + threadIdx.y;
-    int col = blockIdx.x * blockDim.x + threadIdx.x;
-    //printf("%d\n",N);
-    if (row < N && col < N) {
-        int temp = 0;
-        for (int k = 0; k < N; ++k) {
-            temp += a[row * N + k] * b[k * N + col];
-        }
-        c[row * N + col] = temp;
-    }
-}
-
-
diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/CMakeLists.txt b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/CMakeLists.txt
deleted file mode 100644
index 89bcf5ba7c..0000000000
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/CMakeLists.txt
+++ /dev/null
@@ -1,44 +0,0 @@
-cmake_minimum_required(VERSION 3.16.3)
-
-# 1) Enforce C++20 (needs modern C++ features)
-set(CMAKE_CXX_STANDARD        20)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_EXTENSIONS      OFF)
-
-# 2) Set CAF source and build directories
-set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
-set(CAF_BUILD "${CAF_SRC}/build")
-
-
-#required since cmake cant seem to find the compiler 
-set(CMAKE_CXX_COMPILER "/usr/bin/g++")
-set(CMAKE_C_COMPILER "/usr/bin/gcc")
-
-project(CUDA_ACTORS)
-
-find_package(CUDA REQUIRED)
-find_package(CUDAToolkit REQUIRED)
-
-include_directories(
-  "${CAF_SRC}/include"
-  "${CAF_SRC}/libcaf_io"
-  "${CAF_SRC}/libcaf_core"
-  "${CAF_SRC}/libcaf_opencl"
-  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
-  "${CAF_SRC}/libcaf_cuda"
-)
-
-
-# 5) Declare your executable
-add_executable(test main.test.cpp)
-target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
-
-target_link_libraries(test
-  PRIVATE
-    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-    CUDA::nvrtc
-)
-
-
diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/compile_kernels.sh b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/compile_kernels.sh
deleted file mode 100755
index 586196454e..0000000000
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/compile_kernels.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-set -e
-
-# Detect first GPU compute capability
-ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -n1)
-SM_ARCH="sm_${ARCH/./}"
-echo "Using NVCC arch flag: $SM_ARCH"
-
-# Compile mmul.cu to cubin in current directory
-nvcc -arch=$SM_ARCH -cubin mmul.cu -o mmul.cubin
-echo "Generated mmul.cubin"
-
-# Compile genMatrix.cu to fatbin in current directory
-#nvcc -arch=$SM_ARCH --fatbin genMatrix.cu -o generate_random_matrix.fatbin -lcudadevrt -lcurand
-#echo "Generated generate_random_matrix.fatbin"
-
-echo "All kernels compiled successfully!"
-
diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
deleted file mode 100644
index eec64bab2e..0000000000
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/main.test.cpp
+++ /dev/null
@@ -1,1029 +0,0 @@
-#include <caf/all.hpp>
-#include <caf/cuda/all.hpp>
-#include <caf/cuda/control-layer/all-control-layer.hpp>
-#include <iostream>
-#include <iomanip>
-#include <vector>
-#include <chrono>
-#include <thread>
-#include <algorithm>
-#include <numeric>
-#include <random>
-#include <unistd.h>
-#include "caf/actor_registry.hpp"
-#include <chrono>
-#include <iostream>
-//#include <caf/atoms.hpp>
-
-
-
-
-using namespace caf;
-using namespace std::chrono_literals;
-
-
-struct exit_actor_state {
-	int completed = 0;
-};
-
-
-caf::behavior exit_actor_fun(caf::stateful_actor<exit_actor_state>* self,int limit) {
-
-
-	return {
-		[=](int num_completed) {
-			self->state().completed += num_completed;
-			
-			//std::cout << "Actors finished is " << self->state().completed << "\n";
-			if (self->state().completed >= limit) {
-			
-				caf::cuda::manager::shutdown();
-				self->quit();
-			}
-		}
-	};
-
-
-}
-
-
-
-
-
-
-// Define a custom type ID block for custom actors
-CAF_ADD_ATOM(cuda,shared_mem)
-
-
-
-
-
-// Extend your actor state to keep the start time
-struct mmul_actor_state {
-  static inline const char* name = "mmul_actor";
-
-  int N = 0;
-  int id = rand();
-
-  // timing / bookkeeping only
-  std::chrono::high_resolution_clock::time_point start_time;
-  int times = 0;
-};
-
-
-
-
-
-//commands classes used to launch kernels 
-using mmulCommand = caf::cuda::command_runner<in<int>,in<int>,out<int>,in<int>>;
-using matrixGenCommand = caf::cuda::command_runner<out<int>,in<int>,in<int>,in<int>>;
-
-using mmulAsyncCommand = caf::cuda::command_runner<caf::cuda::mem_ptr<int>,caf::cuda::mem_ptr<int>,out<int>,in<int>>;
-
-mmulCommand mmul;
-matrixGenCommand randomMatrix;
-mmulAsyncCommand mmulAsync;
-
-
-void serial_matrix_multiply(const std::vector<int>& a,
-                            const std::vector<int>& b,
-                            std::vector<int>& c,
-                            int N) {
-  
-
- for (int i = 0; i < N; ++i) {
-    for (int j = 0; j < N; ++j) {
-      int sum = 0;
-      for (int k = 0; k < N; ++k) {
-        sum += a[i * N + k] * b[k * N + j];
-      }
-      c[i * N + j] = sum;
-    }
-  }
-}
-
-
-
-
-// Stateful actor behavior
-caf::behavior mmul_actor_fun(
-    caf::stateful_actor<mmul_actor_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims)
-{
-
-	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
-
-	caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-	caf::actor scheduler = mgr.get_scheduler_actor();
-
-	//send a launch token
-	caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-			program,
-			dims,
-			0,
-			"hello",
-			self
-			);
-	self -> mail(launch_token).send(scheduler);
-
-	return {
-
-		[=] (caf::cuda::response_token_ptr res_token) {
-
-			if (res_token -> getType() == LAUNCH_RESPONSE) {
-				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-				//assume N = 1024
-				int N = self -> state().N;
-				std::vector<int> matrix1(N*N);
-				matrix1.reserve(N);
-				std::vector<int> matrix2(N*N);
-				matrix2.reserve(N);
-
-				//std::cout << "GPU ACTOR sending data to compute\n";
-				self -> mail(matrix1,matrix2,res_token,N).send(self);
-
-			}
-			else {
-				std::cout << "Got a memory response token\n";
-			}
-			//token should drop out of scope now, triggering a response 
-		},
-
-			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-			[=](const std::vector<int>& matrixA,
-					const std::vector<int>& matrixB,
-					const caf::cuda::response_token_ptr& res_token, int N) {
-
-
-				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
-
-				//caf::cuda::launch_response_token& kt =
-				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
-
-				//std::cout << "GPU ACTOR  computing\n";
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				auto program = mgr.create_program_from_cubin("../mmul.cubin","matrixMul");
-				const int THREADS = 32;
-				const int BLOCKS = (N + THREADS - 1) / THREADS;
-				caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(matrixA);
-				auto arg2 = caf::cuda::create_in_arg(matrixB);
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
-
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				//std::cout << "GPU ACTOR done  computing\n";
-				//verify its own result 
-				self -> mail(matrixA,matrixB,matrixC,N).send(self);
-
-			},
-
-					// 3rd handler: CPU atom + matrices + N
-					[=](const std::vector<int>& matrixA,
-							const std::vector<int>& matrixB,
-							const std::vector<int>& matrixC,
-							int N) {
-
-						using clock = std::chrono::high_resolution_clock;
-
-						auto start = clock::now();
-
-						//std::cout << "GPU ACTOR verifying\n";
-
-						std::vector<int> result(N * N);
-
-						serial_matrix_multiply(matrixA, matrixB, result, N);
-
-						if (result == matrixC) {
-							std::cout << "actor with id " << self->state().id
-								<< " references match\n";
-						} else {
-							std::cout << "actor with id " << self->state().id
-								<< " references did not match\n";
-						}
-
-						auto end = clock::now();
-
-						auto ms =
-							std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-
-						std::cout << "[TIMING] verification took "
-							<< ms << " ms (actor id "
-							<< self->state().id << ")\n";
-
-						// signal exit actor and quit
-						self->mail(1).send(exit_actor);
-						self->quit();
-
-					}
-	};
-}
-
-
-
-
-
-// this actor will not verify its results
-// great for performance analysis 
-caf::behavior mmul_actor_fun_no_verify(
-    caf::stateful_actor<mmul_actor_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims,
-    bool request
-    )
-{
-
-	//set the value of N correctly to overide the base option.	
-	self->state().N = N;
-
-	caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-	caf::actor scheduler = mgr.get_scheduler_actor();
-
-	if (request) {
-		//send a launch token
-		caf::cuda::token_ptr launch_token = caf::cuda::make_launch_token(
-				program,
-				dims,
-				0,
-				"hello",
-				self
-				);
-		self -> mail(launch_token).send(scheduler);
-	}
-	return {
-
-		[=] (caf::cuda::response_token_ptr res_token) {
-
-			if (res_token -> getType() == LAUNCH_RESPONSE) {
-				//std::cout << "GPU ACTOR RECEIVED PERMISSION TO LAUNCH\n"; 
-				//assume N = 1024
-				int N = self -> state().N;
-				std::vector<int> matrix1(N*N);
-				matrix1.reserve(N);
-				std::vector<int> matrix2(N*N);
-				matrix2.reserve(N);
-
-
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(std::move(matrix1));
-				auto arg2 = caf::cuda::create_in_arg(std::move(matrix2));
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
-
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-			
-			
-				//mask the transfer back to the cpu for scheduler
-				res_token -> release();
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				//std::cout << "GPU ACTOR done  computing\n";
-				// signal exit actor and quit
-				self->mail(1).send(exit_actor);
-				self->quit();
-
-				//std::cout << "GPU ACTOR sending data to compute\n";
-			//	self -> mail(matrix1,matrix2,res_token,N).send(self);
-
-			}
-			else {
-				std::cout << "Got a memory response token\n";
-			}
-			//token should drop out of scope now, triggering a response 
-		},
-
-			// 2nd handler: GPU atom + matrices + N, launches a kenrel and sends its result to itself for verification
-			[=](const std::vector<int>& matrixA,
-					const std::vector<int>& matrixB,
-					const caf::cuda::response_token_ptr& res_token, int N) {
-
-
-				//caf::cuda::kernel_launch_token kernelToken = caf::intrusive_ptr_cast<caf::cuda::token_ptr>(kToken);
-
-				//caf::cuda::launch_response_token& kt =
-				//	    static_cast<caf::cuda::launch_response_token&>(*kToken);
-
-				//std::cout << "GPU ACTOR  computing\n";
-				caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-				//create program and dims   
-				//create args
-				auto arg1 = caf::cuda::create_in_arg(std::move(matrixA));
-				auto arg2 = caf::cuda::create_in_arg(std::move(matrixB));
-				auto arg3 = caf::cuda::create_out_arg(N*N);
-				auto arg4 = caf::cuda::create_in_arg(N);
-
-				auto tempC = mmul.run(program,dims,res_token,arg1,arg2,arg3,arg4);
-			
-			
-				//mask the transfer back to the cpu for scheduler
-				res_token -> release();
-				std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-				//std::cout << "GPU ACTOR done  computing\n";
-				// signal exit actor and quit
-				self->mail(1).send(exit_actor);
-				self->quit();
-
-			}
-
-				
-	};
-}
-
-
-
-// Stateful actor behavior
-// this actor does not invoke the scheduler at all 
-caf::behavior mmul_actor_fun_no_schedule(
-    caf::stateful_actor<mmul_actor_state>* self,
-    caf::actor exit_actor,
-    int N,
-    caf::cuda::program_ptr program,
-    caf::cuda::nd_range dims) {
-
-    self->state().N = N;
-
-    std::vector<int> matrix1(N * N);
-    std::vector<int> matrix2(N * N);
-
-    // send initial mail to self
-    self->mail(matrix1, matrix2, N).send(self);
-
-    return {
-        // GPU atom + matrices + N
-        [=](const std::vector<int>& matrixA,
-            const std::vector<int>& matrixB,
-            int N_local) {   // avoid shadowing outer N
-            
-	    
-    	    //std::cout << "Hello\n";
-	    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-            auto arg1 = caf::cuda::create_in_arg(std::move(matrixA));
-            auto arg2 = caf::cuda::create_in_arg(std::move(matrixB));
-            auto arg3 = caf::cuda::create_out_arg(N_local * N_local);
-            auto arg4 = caf::cuda::create_in_arg(N_local);
-
-            auto tempC = mmul.run(program, dims, self->state().id, arg1, arg2, arg3, arg4);
-            std::vector<int> matrixC = caf::cuda::extract_vector<int>(tempC);
-
-	    self->mail(1).send(exit_actor);
-            self->quit();
-        },
-
-        // CPU verification
-        [=](const std::vector<int>& matrixA,
-            const std::vector<int>& matrixB,
-            const std::vector<int>& matrixC,
-            int N_local) {
-
-            std::vector<int> result(N_local * N_local);
-            serial_matrix_multiply(matrixA, matrixB, result, N_local);
-
-            if (result == matrixC) {
-                std::cout << "actor with id " << self->state().id << " references match\n";
-            } else {
-                std::cout << "actor with id " << self->state().id << " references did not match\n";
-            }
-
-            self->quit();
-        }
-    };
-}
-
-
-
-
-
-
-
-
-
-
-void run_mmul_test(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  //change the scheduler to mulitlevle_usage
-  anon_mail(
-	caf::cuda::make_behavior_token("multilevel")
-	).send(mgr.get_scheduler_actor());
-
-  // CREATE ONCE
-  auto program =
-      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-  const int THREADS = 32;
-  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(
-      BLOCKS, BLOCKS, 1,
-      THREADS, THREADS, 1);
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-  for (int i = 0; i < num_actors; ++i) {
-    /*
-    sys.spawn(
-        mmul_actor_fun,
-        exit_actor,
-        matrix_size,
-        program,
-        dims);
-    */
-      sys.spawn(
-        mmul_actor_fun_no_verify,
-        exit_actor,
-        matrix_size,
-        program,
-        dims,
-	true);
-    
-  }
-
-  sys.await_all_actors_done();
-}
-
-
-void run_mmul_test_no_scheduler(caf::actor_system& sys, int matrix_size, int num_actors) {
-  if (num_actors < 1) {
-    std::cerr << "[ERROR] Number of actors must be >= 1\n";
-    return;
-  }
-
-  caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-  /*
-  //change the scheduler to core_usage
-  anon_mail(
-	caf::cuda::make_behavior_token("core_usage")
-	).send(mgr.get_scheduler_actor());
-
-  */
-
-    mgr.send_scheduler_actor_message("green",0);
-
-  // CREATE ONCE
-  auto program =
-      mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-  const int THREADS = 32;
-  const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-  caf::cuda::nd_range dims(
-      BLOCKS, BLOCKS, 1,
-      THREADS, THREADS, 1);
-
-  caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-  for (int i = 0; i < num_actors; ++i) {
-    
-	sys.spawn(
-        mmul_actor_fun_no_verify,
-        exit_actor,
-        matrix_size,
-        program,
-        dims,
-	true);
-    
-  
-	/*  
-sys.spawn(
-        mmul_actor_fun,
-        exit_actor,
-        matrix_size,
-        program,
-        dims);
-  */
-  }
-
-  sys.await_all_actors_done();
-}
-
-void run_mmul_test_no_scheduler_actor(caf::actor_system& sys, int matrix_size, int num_actors) {
-    if (num_actors < 1) {
-        std::cerr << "[ERROR] Number of actors must be >= 1\n";
-        return;
-    }
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    const int THREADS = 32;
-    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    for (int i = 0; i < num_actors; ++i) {
-        sys.spawn(
-            mmul_actor_fun_no_schedule,
-            exit_actor,
-            matrix_size,
-            program,
-            dims
-        );
-    }
-
-    sys.await_all_actors_done();
-}
-
-
-template <class Fn>
-double time_run(Fn&& fn) {
-  auto start = std::chrono::steady_clock::now();
-  fn();
-  auto end = std::chrono::steady_clock::now();
-  std::chrono::duration<double> elapsed = end - start;
-  return elapsed.count();
-}
-void run_mmul_scaling_tests(caf::actor_system& sys,
-                            caf::cuda::manager_config man_config) {
-    const int max_size   = 2048;
-    const int min_actors = 1;
-    const int max_actors = 1024;
-
-    std::vector<int> matrix_sizes = {10};
-    for (int s = 32; s <= max_size; s *= 2)
-        matrix_sizes.push_back(s);
-
-    std::vector<int> actor_counts;
-    for (int a = min_actors; a <= max_actors; a *= 2)
-        actor_counts.push_back(a);
-
-    std::cout << "=== MMUL Scaling Tests ===\n";
-    std::cout << "Format:\n";
-    std::cout << "scheduler matrix_size actors time_seconds\n";
-
-    for (int size : matrix_sizes) {
-        for (int actors : actor_counts) {
-
-            /* ================= Scheduler-enabled (core_usage) ================= */
-            caf::cuda::manager::init(sys, man_config); // scheduler enabled
-            std::cout << "\n[RUN] scheduler=multilevel_usage "
-                      << "matrix_size=" << size
-                      << " actors=" << actors << "\n";
-
-            double core_usage_time = time_run([&] {
-                run_mmul_test(sys, size, actors); // uses mmul_actor_fun_no_verify
-            });
-
-            std::cout << std::fixed << std::setprecision(6)
-                      << "RESULT core_usage "
-                      << size << " "
-                      << actors << " "
-                      << core_usage_time << "\n";
-
-            caf::cuda::manager::shutdown(); // make sure manager is cleaned up
-
-            /* ================= Scheduler-disabled actor ( uses green-light) ================= */
-
-	    caf::cuda::manager::init(sys, man_config); // init with scheduler
-            std::cout << "\n[RUN] scheduler=green_light_only "
-                      << "matrix_size=" << size
-                      << " actors=" << actors << "\n";
-
-            double green_light_time = time_run([&] {
-                run_mmul_test_no_scheduler(sys, size, actors); // your previous "no scheduler" actor
-            });
-
-            std::cout << std::fixed << std::setprecision(6)
-                      << "RESULT green_light_only "
-                      << size << " "
-                      << actors << " "
-                      << green_light_time << "\n";
-
-            caf::cuda::manager::shutdown();
-
-            /* ================= No scheduler at all actor ================= */
-            caf::cuda::manager_config no_sched_config(false); // disable scheduler
-            caf::cuda::manager::init(sys, no_sched_config);
-            std::cout << "\n[RUN] scheduler=none "
-                      << "matrix_size=" << size
-                      << " actors=" << actors << "\n";
-
-            double no_scheduler_time = time_run([&] {
-                run_mmul_test_no_scheduler_actor(sys, size, actors); // mmul_actor_fun_no_schedule
-            });
-
-            std::cout << std::fixed << std::setprecision(6)
-                      << "RESULT none "
-                      << size << " "
-                      << actors << " "
-                      << no_scheduler_time << "\n";
-
-            caf::cuda::manager::shutdown();
-        }
-    }
-
-    std::cout << "\n=== MMUL Scaling Tests Complete ===\n";
-}
-void run_mmul_mixed_batch_one_mode(
-    caf::actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors,
-    bool use_scheduler_actor,
-    bool use_core_usage_behavior,
-    bool randomize = false)
-{
-    caf::cuda::manager& mgr = caf::cuda::manager::get(); // SAFE NOW
-
-    if (use_scheduler_actor && use_core_usage_behavior) {
-        anon_mail(caf::cuda::make_behavior_token("core_usage"))
-            .send(mgr.get_scheduler_actor());
-    }
-
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    std::mt19937 rng(123456);
-    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
-
-    const int THREADS = 32;
-
-    for (int i = 0; i < num_actors; ++i) {
-        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
-        int BLOCKS = (N + THREADS - 1) / THREADS;
-
-        caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-        if (use_scheduler_actor) {
-            sys.spawn(
-                mmul_actor_fun_no_verify,
-                exit_actor,
-                N,
-                program,
-                dims,
-		true);
-        } else {
-            sys.spawn(
-                mmul_actor_fun_no_schedule,
-                exit_actor,
-                N,
-                program,
-                dims);
-        }
-    }
-
-    sys.await_all_actors_done();
-}
-
-
-
-
-void run_mmul_mixed_batch_one_mode_bulk(
-    caf::actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors,
-    bool randomize = false)
-{
-    caf::cuda::manager& mgr = caf::cuda::manager::get(); // SAFE NOW
-
-        anon_mail(caf::cuda::make_behavior_token("core_usage"))
-            .send(mgr.get_scheduler_actor());
-
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    std::vector<caf::cuda::token_ptr> tokens(num_actors);
-
-    std::mt19937 rng(123456);
-    std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
-
-    const int THREADS = 32;
-
-    for (int i = 0; i < num_actors; ++i) {
-        int N = randomize ? sizes[dist(rng)] : sizes[i % sizes.size()];
-        int BLOCKS = (N + THREADS - 1) / THREADS;
-
-	caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-	caf::actor a = sys.spawn(
-			mmul_actor_fun_no_verify,
-			exit_actor,
-			N,
-			program,
-			dims,
-			false);
-
-	tokens[i] = caf::cuda::make_launch_token(
-			program,
-			dims,
-			0 /*this should not be 0 but its fine for now*/,
-			"hello",
-			a);    
-    }
-
-    
-        anon_mail(tokens)
-            .send(mgr.get_scheduler_actor());
-
-
-    sys.await_all_actors_done();
-}
-
-
-
-
-
-
-
-
-
-
-void run_mmul_mixed_batch_comparison(
-    caf::actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors)
-{
-    std::cout << "\n=== MMUL Mixed-Size Batch Comparison ===\n";
-    std::cout << "scheduler actors sizes time_seconds\n\n";
-
-    /* ================= core_usage ================= */
-    {
-        caf::cuda::manager_config cfg(true);
-        caf::cuda::manager::init(sys, cfg);
-
-	
-        //double t = time_run([&] {
-          //  run_mmul_mixed_batch_one_mode(
-            //    sys, sizes, num_actors,
-              //  /*use_scheduler_actor=*/true,
-               // /*use_core_usage_behavior=*/true);
-       // });
-
-	double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode_bulk(
-                sys, sizes, num_actors);
-        });
-
-
-
-
-        std::cout << "RESULT core_usage "
-                  << num_actors << " "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    /* ================= green-light only ================= */
-   
-    /*
-    {
-	    std::cout << "Starting green_light tests\n";
-        caf::cuda::manager_config cfg(true);
-        caf::cuda::manager::init(sys, cfg);
-
-        double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode(
-                sys, sizes, num_actors,
-               true,
-                false);
-        });
-
-        std::cout << "RESULT green_light_only "
-                  << num_actors << " "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    */
-	
-    /* ================= no scheduler ================= */
-    {
-        caf::cuda::manager_config cfg(false);
-        caf::cuda::manager::init(sys, cfg);
-
-        double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode(
-                sys, sizes, num_actors,
-                /*use_scheduler_actor=*/false,
-                /*use_core_usage_behavior=*/false);
-        });
-
-        std::cout << "RESULT none "
-                  << num_actors << " "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    std::cout << "\n=== Comparison Complete ===\n";
-}
-
-
-
-void test_core_usage_uniform_mmul(
-    caf::actor_system& sys,
-    int matrix_size,
-    int num_actors,
-    std::string scheduler_behavior)
-{
-    std::cout << "\n[TEST] core_usage uniform matrix size\n";
-    std::cout << "N=" << matrix_size
-              << " actors=" << num_actors << "\n";
-
-    caf::cuda::manager_config cfg(true);
-    caf::cuda::manager::init(sys, cfg);
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-    // force core_usage behavior
-    anon_mail(caf::cuda::make_behavior_token(scheduler_behavior))
-        .send(mgr.get_scheduler_actor());
-
-    auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    const int THREADS = 32;
-    const int BLOCKS = (matrix_size + THREADS - 1) / THREADS;
-    caf::cuda::nd_range dims(
-        BLOCKS, BLOCKS, 1,
-        THREADS, THREADS, 1);
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    for (int i = 0; i < num_actors; ++i) {
-        sys.spawn(
-            mmul_actor_fun_no_verify,
-            exit_actor,
-            matrix_size,
-            program,
-            dims,
-	    true);
-    }
-
-    sys.await_all_actors_done();
-    caf::cuda::manager::shutdown();
-
-    std::cout << "[PASS] core_usage uniform test complete\n";
-}
-
-
-
-void test_core_usage_mixed_mmul(
-    caf::actor_system& sys,
-    const std::vector<int>& sizes,
-    int num_actors,
-    std::string scheduler_behavior)
-{
-    std::cout << "\n[TEST] core_usage mixed matrix sizes\n";
-    std::cout << "actors=" << num_actors << "\n";
-
-    caf::cuda::manager_config cfg(true);
-    caf::cuda::manager::init(sys, cfg);
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-
-    anon_mail(caf::cuda::make_behavior_token(scheduler_behavior))
-        .send(mgr.get_scheduler_actor());
-
-    auto program =
-        mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    caf::actor exit_actor = sys.spawn(exit_actor_fun, num_actors);
-
-    const int THREADS = 32;
-
-    for (int i = 0; i < num_actors; ++i) {
-        int N = sizes[i % sizes.size()];
-
-        int BLOCKS = (N + THREADS - 1) / THREADS;
-        caf::cuda::nd_range dims(
-            BLOCKS, BLOCKS, 1,
-            THREADS, THREADS, 1);
-
-        sys.spawn(
-            mmul_actor_fun_no_verify,
-            exit_actor,
-            N,
-            program,
-            dims,
-	    true);
-    }
-
-    sys.await_all_actors_done();
-    caf::cuda::manager::shutdown();
-
-    std::cout << "[PASS] core_usage mixed-size test complete\n";
-}
-
-
-void run_mmul_fixed_256_batch_comparison(
-    caf::actor_system& sys,
-    int num_actors)
-{
-    // All actors run the same matrix size: 256
-    std::vector<int> sizes(num_actors, 256);
-
-    std::cout << "\n=== MMUL Fixed-Size (256) Batch Comparison ===\n";
-    std::cout << "scheduler actors size time_seconds\n\n";
-
-    /* ================= core_usage ================= */
-    {
-        caf::cuda::manager_config cfg(true);
-        caf::cuda::manager::init(sys, cfg);
-
-        double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode(
-                sys,
-                sizes,
-                num_actors,
-                /*use_scheduler_actor=*/true,
-                /*use_core_usage_behavior=*/true);
-        });
-
-        std::cout << "RESULT core_usage "
-                  << num_actors << " 256 "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    /* ================= no scheduler ================= */
-    {
-        caf::cuda::manager_config cfg(false);
-        caf::cuda::manager::init(sys, cfg);
-
-        double t = time_run([&] {
-            run_mmul_mixed_batch_one_mode(
-                sys,
-                sizes,
-                num_actors,
-                /*use_scheduler_actor=*/false,
-                /*use_core_usage_behavior=*/false);
-        });
-
-        std::cout << "RESULT none "
-                  << num_actors << " 256 "
-                  << t << "\n";
-
-        caf::cuda::manager::shutdown();
-    }
-
-    std::cout << "\n=== Fixed-256 Comparison Complete ===\n";
-}
-
-
-
-
-
-void caf_main(caf::actor_system& sys) {
-  
-	caf::cuda::manager_config man_config(true); //turns the scheduler on
-	//caf::cuda::manager::init(sys,man_config);
-        // run_mmul_test(sys,10,64);	
-	run_mmul_scaling_tests(sys,man_config);
-
-	/*
-   std::vector<int> sizes = {32, 64, 128, 256, 512, 1024,2048,4096};
-    const int num_actors = 1000;
-    run_mmul_mixed_batch_comparison(sys, sizes, num_actors);    
-
-    */
-
-    //run_mmul_mixed_batch_one_mode_bulk(sys,sizes,num_actors);
-    //run_mmul_fixed_256_batch_comparison(sys, /*num_actors=*/200);
-
-
-	// test_core_usage_uniform_mmul(sys, 256, 1000,"multilevel");
-	 //test_core_usage_mixed_mmul(sys, 256, 1000,"multilevel");
-
-  //std::vector<int> sizes = {32, 64, 128, 256, 512, 1024};
-    //test_core_usage_mixed_mmul(sys, sizes, 200);
-
-
-
-//tests will delete the old manager so will have to reinit if you do this 
-	//in conjunction with each other	
-	//caf::cuda::manager::init(sys,man_config);
-}
-
-
-
-
-CAF_MAIN()
diff --git a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/mmul.cu b/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/mmul.cu
deleted file mode 100644
index c87a1cada8..0000000000
--- a/libcaf_cuda/tests/control-layer-tests/core_usage_behavior_tests/mmul.cu
+++ /dev/null
@@ -1,16 +0,0 @@
-// mmul.cu
-extern "C" __global__
-void matrixMul(const int* a, const int* b, int* c, int N) {
-    int row = blockIdx.y * blockDim.y + threadIdx.y;
-    int col = blockIdx.x * blockDim.x + threadIdx.x;
-    //printf("%d\n",N);
-    if (row < N && col < N) {
-        int temp = 0;
-        for (int k = 0; k < N; ++k) {
-            temp += a[row * N + k] * b[k * N + col];
-        }
-        c[row * N + col] = temp;
-    }
-}
-
-

From 7b5e0179c32929fc4f0f95eb47f963a35cd1402c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 19 Jun 2026 09:49:02 -0600
Subject: [PATCH 0956/1000] updated docs to contain section with Native CUDA
 compatability

---
 libcaf_cuda/documentation.txt | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/libcaf_cuda/documentation.txt b/libcaf_cuda/documentation.txt
index 8b2fe577fd..723f465403 100644
--- a/libcaf_cuda/documentation.txt
+++ b/libcaf_cuda/documentation.txt
@@ -339,3 +339,21 @@ to send scheduler actor exit messages invoke caf::cuda::manager::shutdown() and
 
 
 For more details, see the example code found at https://github.com/uofs-simlab/actor-framework/tree/main/libcaf_cuda/examples
+
+
+== Native CUDA compatability 
+If the Abstractions above are not compatabile with a codebase you wish to integrate, you may use 
+command_runner get_context and get_stream to acquire CUContext and CUStream accordingly to integrate into
+existing code bases you may also create mem_ptr objects by doing the following 
+
+{{{
+
+mem_ptr<T> dptr =  caf::intrusive_ptr<mem_ref<T>>(
+       new mem_ref<T>(size, dev_ptr, access, device number, 0, CUContext, CUstream)));
+  }
+
+}}}
+
+where size is size of the array in elements (not in bytes). dev_ptr is CUdevice_ptr of memory, 
+access is indicator of access, 0 is readonly, 1 is readwrite and 2 is writeonly, the last two parameters are CUContext and
+CUstream belonging to the memory.

From 0def0c536ba7aeb69ebb8c5ff28d6d60078e1ea0 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 19 Jun 2026 10:14:42 -0600
Subject: [PATCH 0957/1000] added cuda variant to this for comparisions

---
 .../CMakeLists.txt                            |  49 +---
 .../main.native.cpp                           | 237 ++++++++++++++++++
 2 files changed, 248 insertions(+), 38 deletions(-)
 create mode 100644 libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/CMakeLists.txt b/libcaf_cuda/tests/scheduler-fault-tolerance-test/CMakeLists.txt
index d1a345a58d..add6c4ece7 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/CMakeLists.txt
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/CMakeLists.txt
@@ -1,21 +1,19 @@
-cmake_minimum_required(VERSION 3.16.3)
+cmake_minimum_required(VERSION 3.16) # Back to your original minimum
 
-# 1) Enforce C++20 (needs modern C++ features)
+# Enforce C++20
 set(CMAKE_CXX_STANDARD        20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS      OFF)
 
-# 2) Set CAF source and build directories
 set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
 set(CAF_BUILD "${CAF_SRC}/build")
 
-
-#required since cmake cant seem to find the compiler 
 set(CMAKE_CXX_COMPILER "/usr/bin/g++")
 set(CMAKE_C_COMPILER "/usr/bin/gcc")
 
 project(CUDA_ACTORS)
 
+# Use the old reliable finder to discover where the driver library is hidden
 find_package(CUDA REQUIRED)
 find_package(CUDAToolkit REQUIRED)
 
@@ -24,34 +22,19 @@ include_directories(
   "${CAF_SRC}/libcaf_io"
   "${CAF_SRC}/libcaf_core"
   "${CAF_SRC}/libcaf_opencl"
-  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_BUILD}/libcaf_core"  
   "${CAF_SRC}/libcaf_cuda"
 )
 
-
-# 5) Declare your executables
-# add_executable(test main.test.cpp)
-# target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
 add_executable(scheduler_test scheduler_integration_test.cpp)
 target_compile_definitions(scheduler_test PRIVATE CAF_ENABLE_LOGGING)
 
-# add_executable(work-stealing work-stealing.cpp)
-# target_compile_definitions(work-stealing PRIVATE CAF_ENABLE_LOGGING)
-
-# add_executable(cuda-baseline cuda-baseline.cpp)
-# target_link_libraries(cuda-baseline PRIVATE CUDA::cuda_driver)
-
-# target_link_libraries(test
-#   PRIVATE
-#     "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-#     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-#     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-#     CUDA::nvrtc 
-#     CUDA::cublas
-#     CUDA::cusparse
-# )
-
-
+# --- THE FIX FOR BASELINE ---
+add_executable(cuda-baseline main.native.cpp)
+target_link_libraries(cuda-baseline PRIVATE 
+    CUDA::cudart           # Fixes Runtime API (cudaMalloc, etc.)
+    CUDA::cuda_driver   # Fixes Driver API (cuInit, cuModuleLoad) via raw path variable
+)
 
 target_link_libraries(scheduler_test
   PRIVATE
@@ -61,14 +44,4 @@ target_link_libraries(scheduler_test
     CUDA::nvrtc 
     CUDA::cublas
     CUDA::cusparse
-)
-
-# target_link_libraries(work-stealing
-#   PRIVATE
-#     "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-#     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-#     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-#     CUDA::nvrtc
-#     CUDA::cublas
-#     CUDA::cusparse
-# )
+)
\ No newline at end of file
diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp
new file mode 100644
index 0000000000..613726d951
--- /dev/null
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp
@@ -0,0 +1,237 @@
+#include <iostream>
+#include <vector>
+#include <unordered_map>
+#include <unordered_set>
+#include <random>
+#include <chrono>
+#include <thread>
+#include <mutex>
+#include <atomic>
+#include <queue>
+#include <cuda_runtime.h>
+#include <cuda.h> // Include CUDA Driver API header
+
+// Macro for safe CUDA Runtime error checking
+#define CHECK_CUDA_THROW(call) \
+    do { \
+        cudaError_t err = call; \
+        if (err != cudaSuccess) { \
+            throw std::runtime_error(cudaGetErrorString(err)); \
+        } \
+    } while (0)
+
+// Macro for safe CUDA Driver error checking
+#define CHECK_CUDA_DRV_THROW(call) \
+    do { \
+        CUresult res = call; \
+        if (res != CUDA_SUCCESS) { \
+            const char* errStr = nullptr; \
+            cuGetErrorString(res, &errStr); \
+            throw std::runtime_error(errStr ? errStr : "Unknown Driver Error"); \
+        } \
+    } while (0)
+
+struct MatrixPool {
+    std::unordered_map<int, std::vector<int>> A;
+    std::unordered_map<int, std::vector<int>> B;
+};
+
+struct TaskQueue {
+    std::queue<int> tasks;
+    std::mutex mtx;
+
+    void push(int N) {
+        std::lock_guard<std::mutex> lock(mtx);
+        tasks.push(N);
+    }
+
+    bool pop(int& N) {
+        std::lock_guard<std::mutex> lock(mtx);
+        if (tasks.empty()) return false;
+        N = tasks.front();
+        tasks.pop();
+        return true;
+    }
+};
+
+struct Stats {
+    std::atomic<int> succeeded{0};
+    std::atomic<int> failed{0};
+};
+
+MatrixPool create_matrix_pool_random(int num_sizes, int min_N, int max_N, unsigned int seed) {
+    MatrixPool pool;
+    std::mt19937 rng(seed);
+    std::uniform_int_distribution<int> dist(min_N, max_N);
+    std::unordered_set<int> used;
+
+    while (used.size() < static_cast<size_t>(num_sizes)) {
+        int N = dist(rng);
+        if (used.insert(N).second) {
+            pool.A[N] = std::vector<int>(N * N, 1);
+            pool.B[N] = std::vector<int>(N * N, 2);
+        }
+    }
+    return pool;
+}
+
+void apply_memory_pressure(size_t target_free_bytes) {
+    size_t free_mem = 0;
+    size_t total_mem = 0;
+    cudaMemGetInfo(&free_mem, &total_mem);
+
+    if (free_mem > target_free_bytes) {
+        size_t to_allocate = free_mem - target_free_bytes;
+        void* d_pressure = nullptr;
+        
+        if (cudaMalloc(&d_pressure, to_allocate) == cudaSuccess) {
+            std::cout << "[MAIN] Memory pressure applied. Allocated " 
+                      << to_allocate / (1024 * 1024) << " MB. Roughly "
+                      << target_free_bytes / (1024 * 1024) << " MB left free.\n";
+        } else {
+            std::cerr << "[WARNING] Failed to apply initial memory pressure.\n";
+        }
+    }
+}
+
+// Worker thread logic
+void worker_thread_fun(int thread_id, int device_id, TaskQueue& queue, const MatrixPool& pool, Stats& stats, CUmodule module) {
+    cudaSetDevice(device_id);
+
+    // Fetch kernel function handle from the preloaded module
+    CUfunction matrixMulKernel;
+    if (cuModuleGetFunction(&matrixMulKernel, module, "matrixMul") != CUDA_SUCCESS) {
+        std::cerr << "[THREAD " << thread_id << "] Failed to find 'matrixMul' kernel symbol.\n";
+        return;
+    }
+
+    cudaStream_t stream;
+    if (cudaStreamCreate(&stream) != cudaSuccess) {
+        std::cerr << "[THREAD " << thread_id << "] Failed to create stream.\n";
+        return;
+    }
+
+    int N = 0;
+    while (queue.pop(N)) {
+        int* d_A = nullptr;
+        int* d_B = nullptr;
+        int* d_C = nullptr;
+        size_t size_bytes = N * N * sizeof(int);
+
+        try {
+            // 1. Allocations
+            CHECK_CUDA_THROW(cudaMalloc(&d_A, size_bytes));
+            CHECK_CUDA_THROW(cudaMalloc(&d_B, size_bytes));
+            CHECK_CUDA_THROW(cudaMalloc(&d_C, size_bytes));
+
+            // 2. Upload host data
+            CHECK_CUDA_THROW(cudaMemcpyAsync(d_A, pool.A.at(N).data(), size_bytes, cudaMemcpyHostToDevice, stream));
+            CHECK_CUDA_THROW(cudaMemcpyAsync(d_B, pool.B.at(N).data(), size_bytes, cudaMemcpyHostToDevice, stream));
+
+            // 3. Launch Kernel via Driver API
+            unsigned int gridX = (N + 31) / 32;
+            unsigned int gridY = (N + 31) / 32;
+
+            // Pack arguments exactly as expected by the kernel parameters layout
+            void* args[] = { &d_A, &d_B, &d_C, &N };
+
+            CHECK_CUDA_DRV_THROW(
+                cuLaunchKernel(matrixMulKernel,
+                               gridX, gridY, 1,       // Grid dims
+                               32, 32, 1,             // Block dims
+                               0,                     // Shared memory bytes
+                               stream,                // Casts cleanly to CUstream
+                               args,                  // Kernel arguments
+                               nullptr)               // Extra arguments
+            );
+
+            // 4. Synchronize stream to catch runtime/OOM issues execution pipeline errors
+            CHECK_CUDA_THROW(cudaStreamSynchronize(stream));
+            stats.succeeded++;
+
+        } catch (const std::exception& e) {
+            stats.failed++;
+        }
+
+        // Clean up resources for this job loop
+        if (d_A) cudaFree(d_A);
+        if (d_B) cudaFree(d_B);
+        if (d_C) cudaFree(d_C);
+    }
+
+    cudaStreamDestroy(stream);
+}
+
+int main() {
+    // Initialize CUDA Driver API
+    if (cuInit(0) != CUDA_SUCCESS) {
+        std::cerr << "Failed to initialize CUDA Driver API.\n";
+        return -1;
+    }
+
+    const int min_N = 2048;
+    const int max_N = 4096;
+    const int num_distinct_sizes = 10;
+    const int num_tasks = 3000;
+    const int num_worker_threads = 16; 
+    const int target_device = 0;
+
+    MatrixPool pool = create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42);
+    std::vector<int> available_Ns;
+    for (const auto& pair : pool.A) {
+        available_Ns.push_back(pair.first);
+    }
+
+    TaskQueue queue;
+    std::mt19937 rng(42);
+    std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
+    for (int i = 0; i < num_tasks; ++i) {
+        queue.push(available_Ns[dist_N_idx(rng)]);
+    }
+
+    cudaSetDevice(target_device);
+
+    // Load the CUBIN module once globally before processing worker loops
+    CUmodule module;
+    if (cuModuleLoad(&module, "../mmul.cubin") != CUDA_SUCCESS) {
+        std::cerr << "CRITICAL: Failed to load cubin file from '../mmul.cubin'\n";
+        return -1;
+    }
+
+    apply_memory_pressure(1500ULL * 1024ULL * 1024ULL); 
+
+    Stats stats;
+    std::cout << "Starting benchmark execution with " << num_tasks << " matrix jobs across " << num_worker_threads << " streams...\n";
+
+    auto start_time = std::chrono::steady_clock::now();
+
+    std::vector<std::thread> workers;
+    for (int i = 0; i < num_worker_threads; ++i) {
+        // Pass the module handle down to the thread execution context
+        workers.emplace_back(worker_thread_fun, i, target_device, std::ref(queue), std::ref(pool), std::ref(stats), module);
+    }
+
+    for (auto& worker : workers) {
+        if (worker.joinable()) worker.join();
+    }
+
+    auto end_time = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end_time - start_time;
+
+    // Output Stats Report
+    int total_processed = stats.succeeded + stats.failed;
+    double success_ratio = total_processed > 0 ? (100.0 * stats.succeeded / total_processed) : 0.0;
+
+    std::cout << "\n=====================================\n";
+    std::cout << "[STATS REPORT] Run Complete\n";
+    std::cout << "  Total Processed: " << total_processed << "\n";
+    std::cout << "  Total Succeeded: " << stats.succeeded.load() << "\n";
+    std::cout << "  Total Failed:    " << stats.failed.load() << "\n";
+    std::cout << "  Success Ratio:   " << success_ratio << "%\n";
+    std::cout << "  Total Makespan:  " << elapsed.count() << " seconds\n";
+    std::cout << "=====================================\n";
+
+    // Unload the CUBIN before termination
+    cuModuleUnload(module);
+    return 0;
+}
\ No newline at end of file

From 0937ba36c85c3d37f1a8834100f92920d9c01f8a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 19 Jun 2026 10:16:45 -0600
Subject: [PATCH 0958/1000] updated the number of streams

---
 .../tests/scheduler-fault-tolerance-test/main.native.cpp        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp
index 613726d951..e16b718f96 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp
@@ -173,7 +173,7 @@ int main() {
     const int max_N = 4096;
     const int num_distinct_sizes = 10;
     const int num_tasks = 3000;
-    const int num_worker_threads = 16; 
+    const int num_worker_threads = 32; 
     const int target_device = 0;
 
     MatrixPool pool = create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42);

From 7a7f523462e6d35249c4885c71c90127eec013de Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 19 Jun 2026 10:29:29 -0600
Subject: [PATCH 0959/1000] updated test to increase GPU performance by making
 supervisor actors let go of their tokens while they sleep for a fixed
 interval

---
 .../scheduler_integration_test.cpp                | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index 87a0446001..966267046c 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -181,9 +181,17 @@ behavior task_worker_fun(stateful_actor<task_actor_state>* self, caf::actor stat
                 auto in_b = create_in_arg(st.pool->B.at(N));
                 auto out_c = create_out_arg_with_size<int>(N * N);
                 auto in_n = create_in_arg(N);
+                int THREADS= 32;
+
+                  nd_range range((N + THREADS - 1) / THREADS, 
+                           (N + THREADS - 1) / THREADS, 1, 
+                           THREADS, THREADS, 1);
+
+
+
 
                 // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
-                auto result_tuple = runner.run_async(st.prog, launch_res->getRange(), res, in_a, in_b, out_c, in_n);
+                auto result_tuple = runner.run_async(st.prog, range, res, in_a, in_b, out_c, in_n);
                 auto d_c = std::get<2>(result_tuple);
 
                 // 3. Asynchronous Copyback.
@@ -257,6 +265,7 @@ behavior task_supervisor_fun(stateful_actor<supervisor_state>* self) {
                             res->release();
                             self->quit();
                         } else {
+                            res -> release();
                             std::uniform_int_distribution<> dis(100, 1000);
                             auto backoff = std::chrono::milliseconds(dis(self->state().rng));
 
@@ -264,6 +273,10 @@ behavior task_supervisor_fun(stateful_actor<supervisor_state>* self) {
                                       << "). Restarting N=" << self->state().N_val 
                                       << " after " << backoff.count() << "ms backoff (Retry " 
                                       << self->state().retries << "/5)." << std::endl;
+
+
+                            auto token = make_launch_token(self->state().prog, nd_range(1,1,1,1,1,1), 0, 
+                             res -> name(), self);
                             self->mail(res).delay(backoff).send(self); // Trigger restart logic with delay
                         }
                     } else {

From 0cc86998e73ec2eaf7db26244c1c6725413db1e9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 19 Jun 2026 10:34:28 -0600
Subject: [PATCH 0960/1000] updated test to be multidevice compatablie

---
 .../main.native.cpp                           | 46 +++++++++++++++++--
 1 file changed, 42 insertions(+), 4 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp
index e16b718f96..f46b5f66fb 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp
@@ -200,17 +200,55 @@ int main() {
 
     apply_memory_pressure(1500ULL * 1024ULL * 1024ULL); 
 
+  // 1. Fetch total number of available CUDA devices
+    int num_devices = 0;
+    CHECK_CUDA_THROW(cudaGetDeviceCount(&num_devices));
+    if (num_devices == 0) {
+        std::cerr << "CRITICAL: No CUDA-capable devices found.\n";
+        return -1;
+    }
+
+    // 2. Enforce exactly 32 streams/threads per device
+    const int STREAMS_PER_DEVICE = 32;
+    const int total_worker_threads = num_devices * STREAMS_PER_DEVICE;
+
+    // 3. Apply memory pressure to ALL devices uniformly
+    for (int d = 0; d < num_devices; ++d) {
+        cudaSetDevice(d);
+        apply_memory_pressure(1500ULL * 1024ULL * 1024ULL); // Keep ~1500 MB free per GPU
+    }
+
     Stats stats;
-    std::cout << "Starting benchmark execution with " << num_tasks << " matrix jobs across " << num_worker_threads << " streams...\n";
+    std::cout << "Starting benchmark execution with " << num_tasks << " matrix jobs\n"
+              << "  Devices found:       " << num_devices << "\n"
+              << "  Streams per device:  " << STREAMS_PER_DEVICE << "\n"
+              << "  Total async workers: " << total_worker_threads << "\n"
+              << "=========================================================\n";
 
     auto start_time = std::chrono::steady_clock::now();
 
+    // 4. Spawn threads grouped by device to ensure precise distribution
     std::vector<std::thread> workers;
-    for (int i = 0; i < num_worker_threads; ++i) {
-        // Pass the module handle down to the thread execution context
-        workers.emplace_back(worker_thread_fun, i, target_device, std::ref(queue), std::ref(pool), std::ref(stats), module);
+    workers.reserve(total_worker_threads);
+
+    for (int d = 0; d < num_devices; ++d) {
+        for (int s = 0; s < STREAMS_PER_DEVICE; ++s) {
+            // Unique thread ID calculation for logging/tracking if needed
+            int global_thread_id = (d * STREAMS_PER_DEVICE) + s;
+
+            workers.emplace_back(
+                worker_thread_fun, 
+                global_thread_id, 
+                d, // Explicit target device ID
+                std::ref(queue), 
+                std::ref(pool), 
+                std::ref(stats), 
+                module
+            );
+        }
     }
 
+    // 5. Await processing completion across all GPU lanes
     for (auto& worker : workers) {
         if (worker.joinable()) worker.join();
     }

From 9cb6ebfc0a2a41e47483cf6c45703b0a38de254c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 19 Jun 2026 10:40:44 -0600
Subject: [PATCH 0961/1000] fixed issue in scheduling logic where supervisor
 would release their token but fail to ask for another one

---
 .../scheduler_integration_test.cpp                          | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index 966267046c..a41d48fa18 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -277,7 +277,7 @@ behavior task_supervisor_fun(stateful_actor<supervisor_state>* self) {
 
                             auto token = make_launch_token(self->state().prog, nd_range(1,1,1,1,1,1), 0, 
                              res -> name(), self);
-                            self->mail(res).delay(backoff).send(self); // Trigger restart logic with delay
+                            self->mail(token).delay(backoff).send(self); // Trigger restart logic with delay
                         }
                     } else {
                         // Worker exited normally (success)
@@ -292,7 +292,11 @@ behavior task_supervisor_fun(stateful_actor<supervisor_state>* self) {
             
             }
             
+        },
+        [=](token_ptr token) {
+                caf::cuda::manager::get().send_scheduler_actor_message(token);
         }
+
     };
 }
 

From f63c371fa7db418580a43ae3cb82923d5e45de83 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 19 Jun 2026 10:55:19 -0600
Subject: [PATCH 0962/1000] updated code to be more  multi device friendly

---
 .../main.native.cpp                           | 136 +++++++++++-------
 1 file changed, 86 insertions(+), 50 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp
index f46b5f66fb..a635f9317e 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp
@@ -74,26 +74,47 @@ MatrixPool create_matrix_pool_random(int num_sizes, int min_N, int max_N, unsign
     }
     return pool;
 }
+#include <vector>
+
+// Global container to hold onto the allocated pressure pointers so they stick around
+// and don't get cleaned up until the program completely finishes.
+static std::vector<void*> global_pressure_allocations;
+static std::mutex pressure_mutex;
+
+void apply_memory_pressure(int device_id, size_t target_free_bytes) {
+    // 1. Explicitly switch the current host thread context to the target GPU
+    CHECK_CUDA_THROW(cudaSetDevice(device_id));
 
-void apply_memory_pressure(size_t target_free_bytes) {
     size_t free_mem = 0;
     size_t total_mem = 0;
-    cudaMemGetInfo(&free_mem, &total_mem);
+    
+    // 2. Query free/total memory specifically for the newly active device
+    CHECK_CUDA_THROW(cudaMemGetInfo(&free_mem, &total_mem));
 
     if (free_mem > target_free_bytes) {
         size_t to_allocate = free_mem - target_free_bytes;
         void* d_pressure = nullptr;
         
+        // 3. Allocate the pressure block on the selected device's VRAM
         if (cudaMalloc(&d_pressure, to_allocate) == cudaSuccess) {
-            std::cout << "[MAIN] Memory pressure applied. Allocated " 
+            std::cout << "[MAIN] Device " << device_id << " memory pressure applied. Allocated " 
                       << to_allocate / (1024 * 1024) << " MB. Roughly "
                       << target_free_bytes / (1024 * 1024) << " MB left free.\n";
+            
+            // Track the allocation pointer so it stays locked in memory
+            std::lock_guard<std::mutex> lock(pressure_mutex);
+            global_pressure_allocations.push_back(d_pressure);
         } else {
-            std::cerr << "[WARNING] Failed to apply initial memory pressure.\n";
+            std::cerr << "[WARNING] Failed to apply memory pressure on Device " << device_id << ".\n";
         }
+    } else {
+        std::cout << "[MAIN] Device " << device_id << " already has less free memory ("
+                  << free_mem / (1024 * 1024) << " MB) than target ("
+                  << target_free_bytes / (1024 * 1024) << " MB). Skipping.\n";
     }
 }
 
+
 // Worker thread logic
 void worker_thread_fun(int thread_id, int device_id, TaskQueue& queue, const MatrixPool& pool, Stats& stats, CUmodule module) {
     cudaSetDevice(device_id);
@@ -120,9 +141,9 @@ void worker_thread_fun(int thread_id, int device_id, TaskQueue& queue, const Mat
 
         try {
             // 1. Allocations
-            CHECK_CUDA_THROW(cudaMalloc(&d_A, size_bytes));
-            CHECK_CUDA_THROW(cudaMalloc(&d_B, size_bytes));
-            CHECK_CUDA_THROW(cudaMalloc(&d_C, size_bytes));
+            CHECK_CUDA_THROW(cudaMallocAsync(&d_A, size_bytes,stream));
+            CHECK_CUDA_THROW(cudaMallocAsync(&d_B, size_bytes,stream));
+            CHECK_CUDA_THROW(cudaMallocAsync(&d_C, size_bytes,stream));
 
             // 2. Upload host data
             CHECK_CUDA_THROW(cudaMemcpyAsync(d_A, pool.A.at(N).data(), size_bytes, cudaMemcpyHostToDevice, stream));
@@ -161,21 +182,30 @@ void worker_thread_fun(int thread_id, int device_id, TaskQueue& queue, const Mat
 
     cudaStreamDestroy(stream);
 }
-
 int main() {
-    // Initialize CUDA Driver API
+    // 1. Initialize CUDA Driver API
     if (cuInit(0) != CUDA_SUCCESS) {
         std::cerr << "Failed to initialize CUDA Driver API.\n";
         return -1;
     }
 
+    // 2. Discover total available physical GPUs
+    int num_devices = 0;
+    CHECK_CUDA_THROW(cudaGetDeviceCount(&num_devices));
+    if (num_devices == 0) {
+        std::cerr << "CRITICAL: No CUDA-capable devices found.\n";
+        return -1;
+    }
+
     const int min_N = 2048;
     const int max_N = 4096;
     const int num_distinct_sizes = 10;
     const int num_tasks = 3000;
-    const int num_worker_threads = 32; 
-    const int target_device = 0;
+    
+    const int STREAMS_PER_DEVICE = 32;
+    const int total_worker_threads = num_devices * STREAMS_PER_DEVICE;
 
+    // 3. Prepare task datasets
     MatrixPool pool = create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42);
     std::vector<int> available_Ns;
     for (const auto& pair : pool.A) {
@@ -189,66 +219,53 @@ int main() {
         queue.push(available_Ns[dist_N_idx(rng)]);
     }
 
-    cudaSetDevice(target_device);
-
-    // Load the CUBIN module once globally before processing worker loops
-    CUmodule module;
-    if (cuModuleLoad(&module, "../mmul.cubin") != CUDA_SUCCESS) {
-        std::cerr << "CRITICAL: Failed to load cubin file from '../mmul.cubin'\n";
-        return -1;
-    }
-
-    apply_memory_pressure(1500ULL * 1024ULL * 1024ULL); 
-
-  // 1. Fetch total number of available CUDA devices
-    int num_devices = 0;
-    CHECK_CUDA_THROW(cudaGetDeviceCount(&num_devices));
-    if (num_devices == 0) {
-        std::cerr << "CRITICAL: No CUDA-capable devices found.\n";
-        return -1;
-    }
-
-    // 2. Enforce exactly 32 streams/threads per device
-    const int STREAMS_PER_DEVICE = 32;
-    const int total_worker_threads = num_devices * STREAMS_PER_DEVICE;
+    // Allocate an isolated module handle tracker array for each device
+    std::vector<CUmodule> modules(num_devices);
 
-    // 3. Apply memory pressure to ALL devices uniformly
+    // 4. Localize configurations per device card completely
     for (int d = 0; d < num_devices; ++d) {
-        cudaSetDevice(d);
-        apply_memory_pressure(1500ULL * 1024ULL * 1024ULL); // Keep ~1500 MB free per GPU
+        // Apply Virtual VRAM pressure boundaries to device 'd'
+        apply_memory_pressure(d, 1500ULL * 1024ULL * 1024ULL); 
+
+        // CRITICAL FIX: Explicitly enter context 'd' to load its local code copy
+        CHECK_CUDA_THROW(cudaSetDevice(d));
+        if (cuModuleLoad(&modules[d], "../mmul.cubin") != CUDA_SUCCESS) {
+            std::cerr << "CRITICAL: Failed to load cubin module on Device " << d << "\n";
+            return -1;
+        }
     }
 
     Stats stats;
-    std::cout << "Starting benchmark execution with " << num_tasks << " matrix jobs\n"
-              << "  Devices found:       " << num_devices << "\n"
-              << "  Streams per device:  " << STREAMS_PER_DEVICE << "\n"
-              << "  Total async workers: " << total_worker_threads << "\n"
-              << "=========================================================\n";
+    std::cout << "\n=========================================================\n";
+    std::cout << "Starting benchmark execution with " << num_tasks << " matrix jobs\n";
+    std::cout << "  Devices found:       " << num_devices << "\n";
+    std::cout << "  Streams per device:  " << STREAMS_PER_DEVICE << "\n";
+    std::cout << "  Total async workers: " << total_worker_threads << "\n";
+    std::cout << "=========================================================\n\n";
 
     auto start_time = std::chrono::steady_clock::now();
 
-    // 4. Spawn threads grouped by device to ensure precise distribution
+    // 5. Spawn worker threads passing the matching targeted hardware context module copy
     std::vector<std::thread> workers;
     workers.reserve(total_worker_threads);
 
     for (int d = 0; d < num_devices; ++d) {
         for (int s = 0; s < STREAMS_PER_DEVICE; ++s) {
-            // Unique thread ID calculation for logging/tracking if needed
             int global_thread_id = (d * STREAMS_PER_DEVICE) + s;
 
             workers.emplace_back(
                 worker_thread_fun, 
                 global_thread_id, 
-                d, // Explicit target device ID
+                d, // Target device binding
                 std::ref(queue), 
                 std::ref(pool), 
                 std::ref(stats), 
-                module
+                modules[d] // Pass the distinct module loaded for this specific GPU context
             );
         }
     }
 
-    // 5. Await processing completion across all GPU lanes
+    // 6. Await execution complete
     for (auto& worker : workers) {
         if (worker.joinable()) worker.join();
     }
@@ -256,7 +273,7 @@ int main() {
     auto end_time = std::chrono::steady_clock::now();
     std::chrono::duration<double> elapsed = end_time - start_time;
 
-    // Output Stats Report
+    // 7. Output Final Metrics Reports
     int total_processed = stats.succeeded + stats.failed;
     double success_ratio = total_processed > 0 ? (100.0 * stats.succeeded / total_processed) : 0.0;
 
@@ -269,7 +286,26 @@ int main() {
     std::cout << "  Total Makespan:  " << elapsed.count() << " seconds\n";
     std::cout << "=====================================\n";
 
-    // Unload the CUBIN before termination
-    cuModuleUnload(module);
+    // 8. Resource Deallocation Sweep
+    {
+        std::lock_guard<std::mutex> lock(pressure_mutex);
+        for (size_t i = 0; i < global_pressure_allocations.size(); ++i) {
+            if (global_pressure_allocations[i]) {
+                int assigned_device = static_cast<int>(i); 
+                if (assigned_device < num_devices) {
+                    cudaSetDevice(assigned_device);
+                }
+                cudaFree(global_pressure_allocations[i]);
+            }
+        }
+        global_pressure_allocations.clear();
+    }
+
+    // Unload each distinct module handle in its corresponding context
+    for (int d = 0; d < num_devices; ++d) {
+        cudaSetDevice(d);
+        cuModuleUnload(modules[d]);
+    }
+
     return 0;
-}
\ No newline at end of file
+}

From c4c1a203f48a5ae59a85ab1e511fe18c26ec3664 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 19 Jun 2026 11:05:14 -0600
Subject: [PATCH 0963/1000] updated worker to use driver api

---
 .../main.native.cpp                           | 141 ++++++++++++------
 1 file changed, 99 insertions(+), 42 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp
index a635f9317e..9e3a4d0829 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/main.native.cpp
@@ -11,26 +11,52 @@
 #include <cuda_runtime.h>
 #include <cuda.h> // Include CUDA Driver API header
 
-// Macro for safe CUDA Runtime error checking
-#define CHECK_CUDA_THROW(call) \
-    do { \
-        cudaError_t err = call; \
-        if (err != cudaSuccess) { \
-            throw std::runtime_error(cudaGetErrorString(err)); \
-        } \
+#include <iostream>
+#include <string>
+#include <stdexcept>
+
+// Clean Runtime API Check Macro
+#define CHECK_CUDA_THROW(call)                                                 \
+    do {                                                                       \
+        cudaError_t err = call;                                                \
+        if (err != cudaSuccess) {                                              \
+            std::string error_msg =                                            \
+                std::string("[CUDA RUNTIME ERROR] API Call '") + #call +       \
+                "' failed.\n  Error Name:        " +                           \
+                cudaGetErrorName(err) +                                        \
+                "\n  Error Description: " + cudaGetErrorString(err) +          \
+                "\n  Location:          " + __FILE__ + ":" +                   \
+                std::to_string(__LINE__) + "\n";                               \
+            std::cerr << error_msg << std::endl;                               \
+            throw std::runtime_error(error_msg);                               \
+        }                                                                      \
     } while (0)
 
-// Macro for safe CUDA Driver error checking
-#define CHECK_CUDA_DRV_THROW(call) \
-    do { \
-        CUresult res = call; \
-        if (res != CUDA_SUCCESS) { \
-            const char* errStr = nullptr; \
-            cuGetErrorString(res, &errStr); \
-            throw std::runtime_error(errStr ? errStr : "Unknown Driver Error"); \
-        } \
+// Clean Driver API Check Macro
+#define CHECK_CUDA_DRV_THROW(call)                                             \
+    do {                                                                       \
+        CUresult res = call;                                                   \
+        if (res != CUDA_SUCCESS) {                                             \
+            const char* errorName = nullptr;                                   \
+            const char* errorStr = nullptr;                                    \
+            cuGetErrorName(res, &errorName);                                   \
+            cuGetErrorString(res, &errorStr);                                  \
+            std::string error_msg =                                            \
+                std::string("[CUDA DRIVER ERROR] API Call '") + #call +        \
+                "' failed.\n  Error Code:        " +                           \
+                std::to_string(static_cast<int>(res)) +                        \
+                "\n  Error Name:        " +                                    \
+                (errorName ? errorName : "UNKNOWN") +                          \
+                "\n  Error Description: " +                                    \
+                (errorStr ? errorStr : "UNKNOWN") +                            \
+                "\n  Location:          " + __FILE__ + ":" +                   \
+                std::to_string(__LINE__) + "\n";                               \
+            std::cerr << error_msg << std::endl;                               \
+            throw std::runtime_error(error_msg);                               \
+        }                                                                      \
     } while (0)
 
+
 struct MatrixPool {
     std::unordered_map<int, std::vector<int>> A;
     std::unordered_map<int, std::vector<int>> B;
@@ -117,43 +143,61 @@ void apply_memory_pressure(int device_id, size_t target_free_bytes) {
 
 // Worker thread logic
 void worker_thread_fun(int thread_id, int device_id, TaskQueue& queue, const MatrixPool& pool, Stats& stats, CUmodule module) {
-    cudaSetDevice(device_id);
+    // 1. In the Driver API, we must explicitly fetch or activate the primary context for this device
+    CUdevice device;
+    CUcontext context;
+    if (cuDeviceGet(&device, device_id) != CUDA_SUCCESS || 
+        cuDevicePrimaryCtxRetain(&context, device) != CUDA_SUCCESS) {
+        std::cerr << "[THREAD " << thread_id << "] Failed to retain primary context.\n";
+        return;
+    }
+
+    // Bind this thread to the device's context
+    if (cuCtxSetCurrent(context) != CUDA_SUCCESS) {
+        std::cerr << "[THREAD " << thread_id << "] Failed to set context.\n";
+        cuDevicePrimaryCtxRelease(device);
+        return;
+    }
 
-    // Fetch kernel function handle from the preloaded module
+    // 2. Fetch kernel function handle
     CUfunction matrixMulKernel;
     if (cuModuleGetFunction(&matrixMulKernel, module, "matrixMul") != CUDA_SUCCESS) {
         std::cerr << "[THREAD " << thread_id << "] Failed to find 'matrixMul' kernel symbol.\n";
+        cuDevicePrimaryCtxRelease(device);
         return;
     }
 
-    cudaStream_t stream;
-    if (cudaStreamCreate(&stream) != cudaSuccess) {
+    // 3. Create an asynchronous stream using Driver API (CUstream)
+    CUstream stream;
+    if (cuStreamCreate(&stream, CU_STREAM_DEFAULT) != CUDA_SUCCESS) {
         std::cerr << "[THREAD " << thread_id << "] Failed to create stream.\n";
+        cuDevicePrimaryCtxRelease(device);
         return;
     }
 
     int N = 0;
     while (queue.pop(N)) {
-        int* d_A = nullptr;
-        int* d_B = nullptr;
-        int* d_C = nullptr;
+        // Driver API pointers are defined as CUdeviceptr (which is an unsigned long long / uintptr_t)
+        CUdeviceptr d_A = 0;
+        CUdeviceptr d_B = 0;
+        CUdeviceptr d_C = 0;
         size_t size_bytes = N * N * sizeof(int);
 
         try {
-            // 1. Allocations
-            CHECK_CUDA_THROW(cudaMallocAsync(&d_A, size_bytes,stream));
-            CHECK_CUDA_THROW(cudaMallocAsync(&d_B, size_bytes,stream));
-            CHECK_CUDA_THROW(cudaMallocAsync(&d_C, size_bytes,stream));
+            // 4. Stream-ordered allocations using Driver API
+            CHECK_CUDA_DRV_THROW(cuMemAllocAsync(&d_A, size_bytes, stream));
+            CHECK_CUDA_DRV_THROW(cuMemAllocAsync(&d_B, size_bytes, stream));
+            CHECK_CUDA_DRV_THROW(cuMemAllocAsync(&d_C, size_bytes, stream));
 
-            // 2. Upload host data
-            CHECK_CUDA_THROW(cudaMemcpyAsync(d_A, pool.A.at(N).data(), size_bytes, cudaMemcpyHostToDevice, stream));
-            CHECK_CUDA_THROW(cudaMemcpyAsync(d_B, pool.B.at(N).data(), size_bytes, cudaMemcpyHostToDevice, stream));
+            // 5. Asynchronous host-to-device memory copies
+            CHECK_CUDA_DRV_THROW(cuMemcpyHtoDAsync(d_A, pool.A.at(N).data(), size_bytes, stream));
+            CHECK_CUDA_DRV_THROW(cuMemcpyHtoDAsync(d_B, pool.B.at(N).data(), size_bytes, stream));
 
-            // 3. Launch Kernel via Driver API
+            // 6. Launch Kernel
             unsigned int gridX = (N + 31) / 32;
             unsigned int gridY = (N + 31) / 32;
 
-            // Pack arguments exactly as expected by the kernel parameters layout
+            // Pack arguments (passing pointers to the CUdeviceptr handles)
             void* args[] = { &d_A, &d_B, &d_C, &N };
 
             CHECK_CUDA_DRV_THROW(
@@ -161,27 +205,40 @@ void worker_thread_fun(int thread_id, int device_id, TaskQueue& queue, const Mat
                                gridX, gridY, 1,       // Grid dims
                                32, 32, 1,             // Block dims
                                0,                     // Shared memory bytes
-                               stream,                // Casts cleanly to CUstream
-                               args,                  // Kernel arguments
-                               nullptr)               // Extra arguments
+                               stream,                // Stream handle
+                               args,                  // Arguments
+                               nullptr)               // Extra parameters
             );
 
-            // 4. Synchronize stream to catch runtime/OOM issues execution pipeline errors
-            CHECK_CUDA_THROW(cudaStreamSynchronize(stream));
+            // 7. Synchronize the stream via Driver API to catch any execution anomalies
+            CHECK_CUDA_DRV_THROW(cuStreamSynchronize(stream));
             stats.succeeded++;
 
         } catch (const std::exception& e) {
+            // Because we used pure Driver API, an exception caught here does NOT poison 
+            // the whole global runtime state. The thread can safely loop and request the next task.
             stats.failed++;
         }
 
-        // Clean up resources for this job loop
-        if (d_A) cudaFree(d_A);
-        if (d_B) cudaFree(d_B);
-        if (d_C) cudaFree(d_C);
+        // 8. Stream-ordered deallocations using Driver API
+        if (d_A) cuMemFreeAsync(d_A, stream);
+        if (d_B) cuMemFreeAsync(d_B, stream);
+        if (d_C) cuMemFreeAsync(d_C, stream);
+        
+        // // Ensure frees are complete before this specific worker thread loops back
+        // cuStreamSynchronize(stream);
     }
 
-    cudaStreamDestroy(stream);
+    // Clean up local thread environment variables cleanly
+    cuStreamDestroy(stream);
+    cuDevicePrimaryCtxRelease(device);
 }
+
+
+
+
+
+
 int main() {
     // 1. Initialize CUDA Driver API
     if (cuInit(0) != CUDA_SUCCESS) {

From 2e1b53d287a1cf3c796ff81adfcef27c180a07ec Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 19 Jun 2026 13:19:06 -0600
Subject: [PATCH 0964/1000] updated type id registration to reduce possibility
 of collisions

---
 libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp | 6 +++++-
 libcaf_cuda/caf/cuda/global.hpp                          | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index 41977ebf0f..aab5ebb0f0 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -2,6 +2,10 @@
 
 #include "caf/cuda/all.hpp"
 #include "caf/cuda/control-layer/return_payloads/all_return_payloads.hpp"
+#include "caf/cuda/global.hpp"
+
+
+
 
 // Control-layer object types
 #include "caf/cuda/control-layer/token.hpp"
@@ -36,7 +40,7 @@
 // Type IDs (required for typed behaviors)
 // -----------------------------------------------------------------------------
 
-CAF_BEGIN_TYPE_ID_BLOCK(cuda_control, caf::first_custom_type_id + 200)
+CAF_BEGIN_TYPE_ID_BLOCK(cuda_control, caf::first_custom_type_id + 12000)
 
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::token>))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::behavior_token>))
diff --git a/libcaf_cuda/caf/cuda/global.hpp b/libcaf_cuda/caf/cuda/global.hpp
index f276edb38b..9fca74fef9 100644
--- a/libcaf_cuda/caf/cuda/global.hpp
+++ b/libcaf_cuda/caf/cuda/global.hpp
@@ -234,7 +234,7 @@ bool inspect(Inspector& f, matrix_format& x) {
 
 // Define a custom type ID block for CUDA types
 // TODO should this become a macro???
-CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id)
+CAF_BEGIN_TYPE_ID_BLOCK(cuda, caf::first_custom_type_id+ 10000)
 
   // Your type IDs
   CAF_ADD_TYPE_ID(cuda, (std::vector<char>))

From 025e7dbbd27dff4ae643bc2dab5a580454d6c564 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Fri, 19 Jun 2026 13:32:46 -0600
Subject: [PATCH 0965/1000] Deleted redundant files and modified existing files
 to no longer reference them via headers or using their functionality. Change
 is being made since these files did not contribute to the CAF CUDA runtime
 environment functionality whatsoever

---
 .../cuda/control-layer/all-control-layer.hpp  |  16 +-
 .../caf/cuda/control-layer/behavior.hpp       |  86 ----
 .../caf/cuda/control-layer/behavior_table.hpp |  34 --
 .../control-layer/core_usage_behavior.hpp     |  59 ---
 .../control-layer/green_light_behavior.hpp    |  15 -
 .../caf/cuda/control-layer/kernel_graph.hpp   | 125 -----
 .../control-layer/memory_actor/mem_token.hpp  |   2 +-
 .../mmul_batch_scheduler_behavior.hpp         |  82 ----
 .../multilevel_usage_behavior.hpp             | 107 -----
 .../cuda/control-layer/pressure_scheduler.hpp | 130 -----
 .../cuda/control-layer/red_light_behavior.hpp |  20 -
 .../control-layer/return_payloads/ack.hpp     |  33 --
 .../return_payloads/all_return_payloads.hpp   |   3 -
 .../return_payloads/transfer_ack.hpp          |  24 -
 .../core_heuristic_function.hpp               |  65 ---
 .../heuristic_function.hpp                    |  53 ---
 .../scheduler-functions/profiler.hpp          |  25 -
 .../sm_usage_heuristic.hpp                    |  71 ---
 .../control-layer/scheduler_actor_state.hpp   |  30 --
 .../caf/cuda/control-layer/transfer_token.hpp |  12 +-
 libcaf_cuda/src/control-layer/behavior.cpp    |  40 --
 .../src/control-layer/behavior_table.cpp      |  43 --
 .../src/control-layer/core_usage_behavior.cpp | 255 ----------
 .../control-layer/green_light_behavior.cpp    |  59 ---
 .../memory_actor/memory_actor.cpp             |   1 -
 .../mmul_batch_scheduler_behavior.cpp         | 300 ------------
 .../multilevel_usage_behavior.cpp             | 449 ------------------
 .../src/control-layer/pressure_scheduler.cpp  | 278 -----------
 .../src/control-layer/red_light_behavior.cpp  |  36 --
 .../control-layer/single_usage_behavior.cpp   | 149 ------
 .../src/control-layer/token_factory.cpp       |  15 -
 .../workflow-tests/mcts/CMakeLists.txt        |  52 --
 .../workflow-tests/mcts/main.cpp              | 186 --------
 .../workflow-tests/mcts/simulation_kernel.cu  |  73 ---
 34 files changed, 8 insertions(+), 2920 deletions(-)
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/behavior.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/return_payloads/all_return_payloads.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/return_payloads/transfer_ack.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/scheduler-functions/profiler.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
 delete mode 100644 libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
 delete mode 100644 libcaf_cuda/src/control-layer/behavior.cpp
 delete mode 100644 libcaf_cuda/src/control-layer/behavior_table.cpp
 delete mode 100644 libcaf_cuda/src/control-layer/core_usage_behavior.cpp
 delete mode 100644 libcaf_cuda/src/control-layer/green_light_behavior.cpp
 delete mode 100644 libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp
 delete mode 100644 libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
 delete mode 100644 libcaf_cuda/src/control-layer/pressure_scheduler.cpp
 delete mode 100644 libcaf_cuda/src/control-layer/red_light_behavior.cpp
 delete mode 100644 libcaf_cuda/src/control-layer/single_usage_behavior.cpp
 delete mode 100644 libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/CMakeLists.txt
 delete mode 100644 libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/main.cpp
 delete mode 100644 libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/simulation_kernel.cu

diff --git a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
index aab5ebb0f0..a75180da10 100644
--- a/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/all-control-layer.hpp
@@ -1,7 +1,6 @@
 #pragma once
 
 #include "caf/cuda/all.hpp"
-#include "caf/cuda/control-layer/return_payloads/all_return_payloads.hpp"
 #include "caf/cuda/global.hpp"
 
 
@@ -18,14 +17,8 @@
 #include "caf/cuda/control-layer/transfer_token.hpp"
 
 //scheduler actor includes
-#include "caf/cuda/control-layer/behavior.hpp"
 #include "caf/cuda/control-layer/scheduler_actor.hpp"
 #include "caf/cuda/control-layer/token_factory.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp"
-#include "caf/cuda/control-layer/kernel_graph.hpp"
-#include "caf/cuda/control-layer/core_usage_behavior.hpp"
 
 //memory actor includes
 #include "caf/cuda/control-layer/memory_actor/memory_actor.hpp"
@@ -53,10 +46,6 @@ CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::memory_request_token))
 CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::mem_token))
 CAF_ADD_TYPE_ID(cuda_control, (caf::intrusive_ptr<caf::cuda::transfer_token>))
 CAF_ADD_TYPE_ID(cuda_control, (std::vector<caf::intrusive_ptr<caf::cuda::token>>))
-CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::kernel_graph))
-CAF_ADD_TYPE_ID(cuda_control, (std::vector<caf::cuda::kernel_graph>))
-CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::ack))
-CAF_ADD_TYPE_ID(cuda_control, (caf::cuda::transfer_ack))
 
 CAF_END_TYPE_ID_BLOCK(cuda_control)
 
@@ -83,8 +72,5 @@ CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::transfer_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::memory_transfer_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::intrusive_ptr<caf::cuda::memory_response_token>)
 CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<caf::intrusive_ptr<caf::cuda::token>>)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::kernel_graph)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<caf::cuda::kernel_graph>)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::ack)
-CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::cuda::transfer_ack)
+
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
deleted file mode 100644
index 521abdee33..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/behavior.hpp
+++ /dev/null
@@ -1,86 +0,0 @@
-#pragma once
-#include "caf/cuda/control-layer/launch_token.hpp"
-#include "caf/cuda/control-layer/launch_response_token.hpp"
-#include "caf/cuda/control-layer/memory_transfer_token.hpp"
-#include "caf/cuda/control-layer/transfer_token.hpp"
-#include "caf/cuda/control-layer/scheduler_actor_state.hpp"
-#include "caf/cuda/control-layer/token.hpp"
-#include "caf/cuda/control-layer/return_payloads/all_return_payloads.hpp"
-
-#include <string>
-
-
-
-//this class is meant to provide an interface so that 
-//the actor can change behavior at runtime
-//normally I would say use become 
-//however that is too much boiler plate 
-//so instead we can just build a behavior
-//abstract class
-//and create a dispatch table
-
-namespace caf::cuda {
-
-class scheduler_actor_behavior {
-public:
-    explicit scheduler_actor_behavior(scheduler_actor_state& state)
-        : state_(state) {}
-    virtual ~scheduler_actor_behavior() = default;
-
-    virtual void on_enter() {}
-    virtual void on_exit() {}
-
-    virtual void schedule() = 0;
-    virtual void receive(const token_ptr& tok) = 0;
-
-    virtual void reclaim([[maybe_unused]] int value, [[maybe_unused]] int memory_returned, [[maybe_unused]] int runtime, [[maybe_unused]] int dependency) {
-	    //default implementation is to do nothing, this should be overidden
-	    //by children classes 
-    }
-
-
-
-    //this is here to ensure that payloads on return can conform to an interface
-    //rather than changing the interface to accomidate every scheduling need
-    virtual void reclaim([[maybe_unused]] ack& payload) {
-	    //default implementation is to do nothing, this should be overidden
-	    //by children classes 
-    }
-
-
-    virtual std::string name() const {return "No name\n";}
-
-    //this method is meant to be a handler for when
-    //another scheduler actor queries for more work
-    virtual void handle_load_balance_request([[maybe_unused]] int device_number) {
-    
-	    //default action is to do nothing and not particpate in load balancing
-	    //whether of not a scheduler wants to participate in load balancing 
-	    //and what actions it should take is a policy decision
-    }
-
-
-    //method is meant to handle work being sent over from another scheduler actor
-   virtual void receive_work([[maybe_unused]] std::vector<kernel_graph> work_graphs) {
-     //ideally this should not default to do nothing 
-    //however I do not have the time implement this on every existing behavior 
-    //as of right now
-    //so be warned if you do not implement an override and request work to do be done
-    //this will end in a deadlock  
-    
-    
-    }
-
-
-
-
-protected:
-    scheduler_actor_state& state_;
-
-    // Default implementation (immediate response) – takes token_ptr and casts internally
-    virtual void process_launch_token(const token_ptr& tok, int stream_id);
-    virtual void process_memory_transfer_token(const token_ptr& tok, int stream_id);
-    virtual void dispatch_transfer_token(const token_ptr& tok, int stream_id);
-};
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp b/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
deleted file mode 100644
index d6c993fad1..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/behavior_table.hpp
+++ /dev/null
@@ -1,34 +0,0 @@
-#pragma once
-#include <unordered_map>
-#include <string>
-
-namespace caf::cuda {
-
-class scheduler_actor_behavior; // Forward declaration is fine here
-class behavior_token;
-class scheduler_actor_state;
-
-class behavior_table {
-public:
-       	behavior_table() = default;
-      // Constructor that creates all default behaviors for a given actor state
-    explicit behavior_table(scheduler_actor_state& state);
-    
-
- 
-    ~behavior_table();
-        
-
-    void add(const std::string& name, scheduler_actor_behavior* beh) {
-        table_[name] = beh;
-    }
-
-    scheduler_actor_behavior* get(const behavior_token& tok) const;
-    auto& all_behaviors() { return table_; }
-    const auto& all_behaviors() const { return table_; }
-
-private:
-    std::unordered_map<std::string, scheduler_actor_behavior*> table_;
-};
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
deleted file mode 100644
index 8c20051681..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/core_usage_behavior.hpp
+++ /dev/null
@@ -1,59 +0,0 @@
-#pragma once
-#include "caf/cuda/control-layer/behavior.hpp"
-#include "caf/cuda/control-layer/kernel_graph.hpp"
-//#include "caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp"
-#include "caf/cuda/device.hpp"
-#include <unordered_map>
-#include <optional>
-
-namespace caf::cuda {
-
-class core_usage_behavior : public scheduler_actor_behavior {
-public:
-    explicit core_usage_behavior(scheduler_actor_state& state);
-    void on_enter() override;
-    void schedule() override;
-    void receive(const token_ptr& tok) override;
-    void reclaim([[maybe_unused]] int value /*blocks consumed*/, [[maybe_unused]] int memory_returned, [[maybe_unused]] int time, [[maybe_unused]] int dependency) override; 
-    ~core_usage_behavior() override;  
-    std::string name() const override {return "core_usage\n";}
-
-protected:
-   void process_launch_token(const token_ptr& tok, int stream_id) override;
-
-private:
-   device_ptr device_;
-   //core_heuristic_function heuristic;
-  
-   std::optional<sm_usage_heuristic> heuristic;
-
-
-
-   //tracking the resources of the device 
-   int total_SM;
-   int available_SM; 
-   int available_memory; //in bytes 
-   int num_tokens = 0;
-   
-   int num_streams = 0;
-   int current_stream = 0;
-
-
-   //data structures to manage dependencies
-   std::unordered_map<int,kernel_graph> graphs;
-   std::vector<kernel_graph> independent_graphs;
-   std::vector<graph_ref> best_graphs; //should contain top 5-10 best selections ideally or something along the lines
-
-   void init_state();
-   void create_new_graph(const token_ptr& token); //this should either add to indepedent or graphs data structure
-			    //note to self use std::move for cheap copies
-   
-   void rank(std::size_t); //this should rank the graphs (high to low) for best canidates
-
-   int get_next_stream(); // this should return the next stream based on some decisions
-   void dummy_schedule(); //just a test scheduler algorthim do not use seriously
-   kernel_graph* resolve(const graph_ref& ref);
-};
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
deleted file mode 100644
index 25c2c04a5d..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/green_light_behavior.hpp
+++ /dev/null
@@ -1,15 +0,0 @@
-#pragma once
-#include "caf/cuda/control-layer/behavior.hpp"
-
-namespace caf::cuda {
-
-class green_light_behavior : public scheduler_actor_behavior {
-public:
-    explicit green_light_behavior(scheduler_actor_state& state);
-    void on_enter() override;
-    void schedule() override;
-    void receive([[maybe_unused]] const token_ptr& tok) override;
-    std::string name() const override {return "green\n";}
-};
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp b/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
deleted file mode 100644
index 4f78b99002..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/kernel_graph.hpp
+++ /dev/null
@@ -1,125 +0,0 @@
-#pragma once
-#include <vector>
-#include "caf/cuda/control-layer/token.hpp"
-#include <chrono>
-
-
-/* Meant to represent a directed acylic graph for kernel operations
- * operations or token_ptrs are meant to come in order and declared with a dependency number to indicate dependency 
- */
-
-// status codes
-#define WAITING 0
-#define READY 1
-#define ERROR 2
-
-namespace caf::cuda {
-
-class kernel_graph {
-public:
-    using clock_t = std::chrono::steady_clock;
-
-    // for caf's messaging system do not use
-    kernel_graph() = default;
-
-    kernel_graph(int device_number,
-                 int stream_id,
-                 int dependency_number)
-        : device_number_(device_number),
-          stream_id_(stream_id),
-          dependency_number_(dependency_number),
-          last_move_(clock_t::now() - std::chrono::seconds{10})	{}
-
-
-        // Convenience constructor for independent graphs
-    kernel_graph(int device_number, int stream_id)
-        : device_number_(device_number),
-          stream_id_(stream_id),
-          dependency_number_(INDEPENDENT),
-          last_move_(clock_t::now() - std::chrono::seconds{10})	{}
-
-
-    // returns the next operation/token_ptr that can be dequeued
-    token_ptr peek() const {
-        if (operations.empty())
-            return nullptr;
-        return operations.front();
-    }
-
-    void add_operation(token_ptr operation) {
-        operations.push_back(operation);
-    }
-
-    // removes the operation and returns it
-    token_ptr getOperation() {
-        if (operations.empty())
-            return nullptr;
-
-        token_ptr op = operations.front();
-        operations.erase(operations.begin());
-        return op;
-    }
-
-     // Returns true if enough time has passed to allow movement
-    bool canMove(std::chrono::seconds min_interval = std::chrono::seconds{2}) const noexcept {
-        return (clock_t::now() - last_move_) >= min_interval;
-    }
-
-    // Call when the graph is actually moved / rescheduled
-    void markMoved() noexcept {
-        last_move_ = clock_t::now();
-    }
-
-    // Optional: force-disable movement (useful for debugging)
-    void disableMove() noexcept {
-        last_move_ = clock_t::now() + std::chrono::hours{24};
-    }
-
-
-    bool empty() const {
-        return operations.empty();
-    }
-
-    int stream_id() const noexcept { return stream_id_; }
-    void set_status(int s) noexcept {status = s;}
-    int get_status() const noexcept {return status;}
-
-     bool is_independent() const noexcept {
-        return dependency_number_ == INDEPENDENT;
-    }
-
-    int get_dependency_number() const noexcept {
-        return dependency_number_;
-    }
-
-
-private:
-    int device_number_;
-    int stream_id_;
-    int dependency_number_;
-    int status = READY;
-    std::vector<token_ptr> operations;
-    clock_t::time_point last_move_;
-};
-
-
-
-struct graph_ref {
-    enum class kind_t {
-        dependent,
-        independent
-    };
-
-    kind_t kind;
-
-    // Only valid if kind == dependent
-    int dependency = -1;
-
-    // Only valid if kind == independent
-    std::size_t index = 0;
-};
-
-
-
-} // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp b/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp
index 694a0b1322..84aee26bdc 100644
--- a/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/memory_actor/mem_token.hpp
@@ -1,6 +1,6 @@
 #pragma once
 #include <caf/actor.hpp>
-#include "caf/cuda/control-layer/return_payloads/ack.hpp"
+// #include "caf/cuda/control-layer/return_payloads/ack.hpp"
 #include "caf/cuda/global_export.hpp"
 #include <utility>
 
diff --git a/libcaf_cuda/caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp
deleted file mode 100644
index a0740d4524..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp
+++ /dev/null
@@ -1,82 +0,0 @@
-// ========================= mmul_batch_scheduler_behavior.hpp =========================
-#pragma once
-
-#include "caf/cuda/control-layer/behavior.hpp"
-#include "caf/cuda/control-layer/kernel_graph.hpp"
-#include "caf/cuda/device.hpp"
-#include "caf/cuda/manager.hpp"
-#include "caf/cuda/control-layer/launch_token.hpp"
-
-#include <deque>
-#include <unordered_map>
-#include <vector>
-#include <atomic>
-
-namespace caf::cuda {
-
-class mmul_batch_scheduler_behavior : public scheduler_actor_behavior {
-public:
-    explicit mmul_batch_scheduler_behavior(scheduler_actor_state& state);
-    ~mmul_batch_scheduler_behavior() override;
-
-    void on_enter() override;
-    void schedule() override;
-    void receive(const token_ptr& tok) override;
-
-    void reclaim([[maybe_unused]] int blocks_consumed, [[maybe_unused]] int memory_returned, [[maybe_unused]] int time, [[maybe_unused]] int dependency_number) override;
-    void reclaim(ack& return_msg) override;
-
-    std::string name() const override { return "mmul_batch_scheduler"; }
-
-protected:
-    void process_launch_token(const token_ptr& tok, int stream_id, [[maybe_unused]] int assigned_queue);
-
-private:
-    enum queue_type { LOW = 0, MED = 1, HIGH = 2 };
-
-    int small_block_threshold = 64;
-    int medium_block_threshold = 1024;
-
-    int max_concurrent_low = 8;
-    int max_concurrent_med = 4;
-    int max_concurrent_high = 1;
-
-    int low_stream_begin = 0;
-    int low_stream_end = 0;
-    int med_stream_begin = 0;
-    int med_stream_end = 0;
-    int high_stream_begin = 0;
-    int high_stream_end = 0;
-
-    std::atomic<int> low_stream_counter{0};
-    std::atomic<int> med_stream_counter{0};
-    std::atomic<int> high_stream_counter{0};
-
-    std::atomic<int> active_low{0};
-    std::atomic<int> active_med{0};
-    std::atomic<int> active_high{0};
-
-    std::unordered_map<int, queue_type> dispatched_dependency_queue;
-
-    std::deque<graph_ref> low_queue;
-    std::deque<graph_ref> med_queue;
-    std::deque<graph_ref> high_queue;
-
-    device_ptr device_;
-    int num_streams = 0;
-
-    std::unordered_map<int,kernel_graph> graphs;
-    std::vector<kernel_graph> independent_graphs;
-
-    std::unordered_map<int,int> dependency_device_map;
-
-    void init_state();
-    void enqueue_graph_by_blocks(const graph_ref& ref, queue_type forced_qt = LOW);
-    void try_dispatch_queue(std::deque<graph_ref>& q, queue_type qt);
-    kernel_graph* resolve(const graph_ref& ref);
-
-    int get_stream_for_queue(queue_type qt);
-    queue_type classify_blocks(int blocks) const;
-};
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
deleted file mode 100644
index 325912907d..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/multilevel_usage_behavior.hpp
+++ /dev/null
@@ -1,107 +0,0 @@
-#pragma once
-
-#include "caf/cuda/control-layer/behavior.hpp"
-#include "caf/cuda/control-layer/kernel_graph.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp"
-#include "caf/cuda/device.hpp"
-#include "caf/cuda/manager.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/profiler.hpp"
-
-#include <deque>
-#include <optional>
-#include <algorithm>
-#include <unordered_map>
-#include <unordered_set>
-
-namespace caf::cuda {
-
-// Multilevel queue scheduling behavior (low / medium / high)
-// - Graphs are classified by the *next operation's* cost relative to total_SM
-
-class multilevel_usage_behavior : public scheduler_actor_behavior {
-public:
-    explicit multilevel_usage_behavior(scheduler_actor_state& state);
-    ~multilevel_usage_behavior() override;
-
-    void on_enter() override;
-    void schedule() override;
-    void receive(const token_ptr& tok) override;
-
-    // reclaim: called when resources are returned; dependency_number
-    // allows this behavior to find the graph that might now be ready
-    void reclaim(int blocks_consumed, int memory_returned, [[maybe_unused]] int time, int dependency_number) override;
-
-    //more improved version of reclaim, meant for when we need to dispatch transfer
-    //tokens
-    void reclaim(ack& return_msg) override;
-    void process_transfer_ack(ack& msg); 
-
-    std::string name() const override { return "multilevel_usage\n"; }
-
-
-    //multi GPU load balancing methods
-    //by default this scheduler behavior will try to load balance
-    //across all gpus 
-    void handle_load_balance_request([[maybe_unused]] int device_number) override;
-    void receive_work([[maybe_unused]] std::vector<kernel_graph> work_graphs) override;
-    void request_load_balance();
-
-
-protected:
-    int num_devices;
-    void process_launch_token(const token_ptr& tok, int stream_id) override;
-       
-
-private:
-    device_ptr device_;
-    std::optional<sm_usage_heuristic> heuristic; //we also clamp results since
-						 //it leads to more concurrent work
-
-
-    int total_SM = 0; //this is not really total_SM anymore, more like a threshold 
-    int available_SM = 0;
-    int available_memory = 0; // bytes
-    int num_streams = 0;
-    int current_stream = 0;
-    int low_threshold = 0; //this is used to check if we should request more work
-			   //or not
-
-    int transfer_threshold =0; //check if we should transfer work or not
-    // dependency -> device mapping
-    std::unordered_map<int, int> dependency_device_map;
-
-
-    //tracking graphs
-    std::unordered_map<int,kernel_graph> graphs;
-    std::vector<kernel_graph> independent_graphs;
-
-    // multilevel queues of graph_refs
-    std::deque<graph_ref> low_queue;
-    std::deque<graph_ref> med_queue;
-    std::deque<graph_ref> high_queue;
-
-    void init_state();
-    void create_new_graph(const token_ptr& token);
-
-    int get_next_stream();
-
-    // classify & enqueue a graph reference based on its next op cost
-    void enqueue_graph_by_cost(const graph_ref& ref);
-
-    // attempt to dispatch as many graphs from q as possible (front-first)
-    void try_dispatch_queue(std::deque<graph_ref>& q);
-
-    kernel_graph* resolve(const graph_ref& ref);
-
-    void send_timed_msg();
-
-    void add_dependency_to_device(int dependency_number, int device_number);
-    void remove_dependency(int dependency_number);
-    int get_device_for_dependency(int dependency_number) const;
-
-
-};
-
-
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp b/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
deleted file mode 100644
index 2e6533914e..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/pressure_scheduler.hpp
+++ /dev/null
@@ -1,130 +0,0 @@
-#pragma once
-
-#include "caf/cuda/control-layer/behavior.hpp"
-#include "caf/cuda/control-layer/kernel_graph.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp"
-#include "caf/cuda/device.hpp"
-#include "caf/cuda/manager.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/profiler.hpp"
-
-#include <deque>
-#include <optional>
-#include <algorithm>
-#include <unordered_map>
-
-
-#define LOW 14
-#define MEDIUM 15
-#define HIGH 16
-
-
-namespace caf::cuda {
-
-// - When a graph has work dispatched it is removed from the queues and
-//   not re-inserted. reclaim(...) can push graphs back into queues by
-//   looking up the dependency number and re-evaluating the front op.
-
-class pressure_scheduler : public scheduler_actor_behavior {
-public:
-    explicit pressure_scheduler(scheduler_actor_state& state);
-    ~pressure_scheduler() override;
-
-    void on_enter() override;
-    void schedule() override;
-    void receive(const token_ptr& tok) override;
-
-    // reclaim: called when resources are returned; dependency_number
-    // allows this behavior to find the graph that might now be ready
-    void reclaim([[maybe_unused]] int resources_consumed, [[maybe_unused]] int memory_returned, [[maybe_unused]] int time, [[maybe_unused]] int dependency_number) override;
-
-    std::string name() const override { return "pressure_scheduler\n"; }
-
-protected:
-    void process_launch_token(const token_ptr& tok, int stream_id) override;
-
-private:
-    device_ptr device_;
-    std::optional<sm_usage_heuristic> heuristic;
-    void init_state();
-
-
-    // resource values of the GPU
-    int total_SM = 0; //can be used for proportional resource consumption
-    int available_memory = 0; // bytes
-    int num_streams = 0;
-    int current_stream = 0;
-
-
-    //threshold values
-    //should inited with int_state method
-    int resource_threshold; //if exceeded do not dispatch kernel
-    int resource_pressure; // tracks resources in use, if low we should dispatch heavy kernels, if high dispatch light kernels 
-    
-    double low_concurreny_threshold; //if we under this immediately accept any work
-				  //of if multiple gpus, seek out work
-
-    double high_concurrency_threshold; //if we are above this, enqueue any work
-				    //since could flood GPU with requests
-
-    double current_concurreny; //number to assign how much kernels on the GPU
-			    //values should be in proportion to how much 
-			    //resources a kernel intends to consume 
-
-    double current_sm_pressure;
-    
-    int compute_bound_pressure; //determines if we should favor compute or memory bound kernels when seeking work to dispatch 
-
-
-    //Methods and data structures that organize and dispatch kernels
- 
-
-    //tracking graphs
-    std::unordered_map<int,kernel_graph> graphs;
-    std::vector<kernel_graph> independent_graphs;
-
-    // multilevel queues of graph_refs of 
-    // graphs whose next kernel is compute bound 
-    std::deque<graph_ref> low_compute_queue;
-    std::deque<graph_ref> med_compute_queue;
-    std::deque<graph_ref> high_compute_queue;
-
-
-    // multilevel queues of graph_refs of 
-    // graphs whose next kernel is memory bound 
-    std::deque<graph_ref> low_memory_queue;
-    std::deque<graph_ref> med_memory_queue;
-    std::deque<graph_ref> high_memory_queue;
-
-    // classify & enqueue a graph reference based on its next op cost
-    void enqueue_graph_by_cost(const graph_ref& ref);
-
-    // attempt to dispatch as many graphs from q as possible (front-first)
-    void try_dispatch_queue(std::deque<graph_ref>& q);
-
-    void dispatch_prefer_compute();
-    void dispatch_prefer_memory();
-
-    kernel_graph* resolve(const graph_ref& ref);
-
-    void create_new_graph(const token_ptr& token);
-
-    int get_next_stream();
-
-
-    //methods that return some value of resources consumed 
-
-    //returns integer code signalling low medium or high
-    //should be used in combination of thresholds to decide how much 
-    //pressure it puts on a dimension of the GPU 
-    //(concurreny,memory vs compute bound, resource)
-    int get_resource_pressure(int blocks_consumed);
-    int get_concurrency_pressure(int);
-    int get_bucket_level(double); 
-
-    double clamp_sm_ratio(double) const;
-
-};
-
-
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp b/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
deleted file mode 100644
index c5f5929553..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/red_light_behavior.hpp
+++ /dev/null
@@ -1,20 +0,0 @@
-#pragma once
-#include "caf/cuda/control-layer/behavior.hpp"
-
-namespace caf::cuda {
-
-class red_light_behavior : public scheduler_actor_behavior {
-public:
-    explicit red_light_behavior(scheduler_actor_state& state);
-    void on_enter() override;
-    void schedule() override;
-    void receive(const token_ptr& tok) override;
-    ~red_light_behavior() noexcept override;
-
-protected:
-    virtual void process_launch_token([[maybe_unused]] const token_ptr& tok, [[maybe_unused]] int stream_id);
-    virtual void process_memory_transfer_token([[maybe_unused]] const token_ptr& tok, [[maybe_unused]] int stream_id);
-
-};
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp b/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
deleted file mode 100644
index ac861bf9de..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/return_payloads/ack.hpp
+++ /dev/null
@@ -1,33 +0,0 @@
-#pragma once
-#include "caf/cuda/global_export.hpp"
-
-namespace caf::cuda {
-
-// -----------------------------------------------------------------------------
-// ACK type codes (stable ABI, no enum churn)
-// -----------------------------------------------------------------------------
-#define CAF_CUDA_ACK_TRANSFER  1
-#define CAF_CUDA_ACK_LAUNCH    2
-#define CAF_CUDA_ACK_MEMORY    3
-#define TIMER 4
-
-
-// -----------------------------------------------------------------------------
-// Base ACK payload
-// -----------------------------------------------------------------------------
-class CAF_CUDA_EXPORT ack {
-public:
-   
-   //for caf messaging system do not use	
-    ack() = default;
-    explicit ack(int type) : type_(type) {}
-    virtual ~ack() = default;
-
-    int getType() const { return type_; }
-
-private:
-    int type_;
-};
-
-} // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/return_payloads/all_return_payloads.hpp b/libcaf_cuda/caf/cuda/control-layer/return_payloads/all_return_payloads.hpp
deleted file mode 100644
index 1c97dfc8fc..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/return_payloads/all_return_payloads.hpp
+++ /dev/null
@@ -1,3 +0,0 @@
-#pragma once
-#include "caf/cuda/control-layer/return_payloads/ack.hpp"
-#include "caf/cuda/control-layer/return_payloads/transfer_ack.hpp"
diff --git a/libcaf_cuda/caf/cuda/control-layer/return_payloads/transfer_ack.hpp b/libcaf_cuda/caf/cuda/control-layer/return_payloads/transfer_ack.hpp
deleted file mode 100644
index 107ed8c6d6..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/return_payloads/transfer_ack.hpp
+++ /dev/null
@@ -1,24 +0,0 @@
-#pragma once
-#include "caf/cuda/control-layer/return_payloads/ack.hpp"
-
-namespace caf::cuda {
-
-class CAF_CUDA_EXPORT transfer_ack final : public ack {
-public:
-    
-     //for caf's messaging system
-     transfer_ack() = default;
-	
-    explicit transfer_ack(int dependency)
-      : ack(CAF_CUDA_ACK_TRANSFER),
-        dependency_(dependency) {}
-
-    int dependency() const { return dependency_; }
-
-private:
-    int dependency_;
-};
-
-} //namespace caf::cuda
-
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
deleted file mode 100644
index b4535eb635..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/core_heuristic_function.hpp
+++ /dev/null
@@ -1,65 +0,0 @@
-#pragma once
-
-#include <string>
-#include <exception>
-
-#include "caf/cuda/device.hpp"
-#include "caf/cuda/nd_range.hpp"
-#include "caf/cuda/program.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp"
-/*
- * Was originally supposed to estimate core usage 
- * but it just tells how much blocks can fit into 1 SM
- */
-
-
-namespace caf::cuda {
-
-class core_heuristic_function : public heuristic_function {
-public:
-  explicit core_heuristic_function(device_ptr dev)
-    : dev_(dev) {}
-
-  // Copy from ANY heuristic_function
-  core_heuristic_function(device_ptr dev,
-                          const heuristic_function& other)
-    : heuristic_function(other),
-      dev_(dev) {}
-
-
-  //mostly here to please the c++ compiler
-  int getCost(const token_ptr& tok) override {
-    return heuristic_function::getCost(tok); // call base
-  }
-
-
-
-  int getCost(const program_ptr& prog,
-            const nd_range& range) override {
-  try {
-    // Use integer hashes instead of concatenated strings
-    int key = prog->getHash() ^ static_cast<int>(range.getHash());
-
-    auto it = values_.find(key);
-    if (it != values_.end())
-      return it->second;
-
-    int cost = dev_->max_active_blocks_per_sm(prog, range);
-    values_[key] = cost;
-    return cost;
-  }
-  catch (const std::exception&) {
-    return ERROR_CODE;
-  }
-  catch (...) {
-    return ERROR_CODE;
-  }
-}
-
-
-private:
-  device_ptr dev_;
-};
-
-} // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
deleted file mode 100644
index c2d684de20..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp
+++ /dev/null
@@ -1,53 +0,0 @@
-#pragma once
-
-#include "caf/cuda/program.hpp"
-#include "caf/cuda/nd_range.hpp"
-
-#include <unordered_map>
-#include <string>
-
-
-#define ERROR_CODE -999999
-
-
-namespace caf::cuda {
-
-// Abstract heuristic function object.
-// Subclasses estimate the "cost" of launching a kernel based on
-// the program and execution configuration.
-class heuristic_function {
-public:
-  /// Default constructor
-  heuristic_function() = default;
-
-  /// Copy constructor from another heuristic_function
-  /// Copies all internal heuristic values 
-  heuristic_function(const heuristic_function& other)
-    : values_(other.values_) {}
-
-  /// Virtual destructor (required for polymorphic base classes)
-  virtual ~heuristic_function() = default;
-
-  /// Abstract cost function
-  /// @param prog Kernel program
-  /// @param range Kernel execution configuration
-  /// @return Cost metric (interpretation left to implementation)
-  virtual int getCost(const program_ptr& prog,
-		  const nd_range& range) = 0;
-
-
-  /// Token-based cost function (default implementation)
-  virtual int getCost(const token_ptr& tok) {
-	  // Assume type checking is done elsewhere
-	  const auto& launch = static_cast<const launch_token&>(*tok);
-	  return getCost(launch.getProgram(), launch.getRange());
-  }
-
-
-protected:
-  /// Heuristic-specific values
-  std::unordered_map<int, int> values_;
-};
-
-} // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/profiler.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/profiler.hpp
deleted file mode 100644
index def0278f46..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/profiler.hpp
+++ /dev/null
@@ -1,25 +0,0 @@
-#pragma once
-#include <chrono>
-#include <iostream>
-
-
-namespace caf::cuda {
-struct scoped_timer {
-    const char* name;
-    std::chrono::steady_clock::time_point start;
-
-    explicit scoped_timer(const char* n)
-        : name(n), start(std::chrono::steady_clock::now()) {}
-
-    ~scoped_timer() {
-        auto end = std::chrono::steady_clock::now();
-        auto us =
-            std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
-
-        std::cout << "[PROFILE] " << name << " took "
-                  << us << " us\n";
-    }
-};
-
-} //namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
deleted file mode 100644
index 319c459335..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler-functions/sm_usage_heuristic.hpp
+++ /dev/null
@@ -1,71 +0,0 @@
-#pragma once
-
-#include <cmath>
-#include "caf/cuda/device.hpp"
-#include "caf/cuda/nd_range.hpp"
-#include "caf/cuda/program.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/heuristic_function.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/profiler.hpp"
-
-/*
- * will return a value indicating 
- * how much blocks a kernel with dimensions will consume
- */
-
-namespace caf::cuda {
-
-class sm_usage_heuristic : public heuristic_function {
-public:
-  explicit sm_usage_heuristic(device_ptr dev,bool ceil = true)
-    : dev_(dev),ceil_(ceil) {}
-
-int getCost(const program_ptr& prog,
-            const nd_range& range) override {
-
-  try {
-    int key = prog->getHash() ^ static_cast<int>(range.getHash());
-
-    auto it = values_.find(key);
-    if (it != values_.end())
-      return it->second;
-
-    int total_blocks = static_cast<int>(range.get_num_blocks());
-    int blocks_per_sm = dev_->max_active_blocks_per_sm(prog, range);
-
-    if (blocks_per_sm <= 0) {
-      std::cout << "blocks_per_sm = " << blocks_per_sm << "\n"; 
-      std::cout << "blocks is less than zero\n";
-      return ERROR_CODE;
-    }
-
-    int sms_needed = (total_blocks + blocks_per_sm - 1) / blocks_per_sm; // ceil
-    int sms_used = sms_needed;
-    if (ceil_) {
-	    sms_used = std::min(dev_->num_sms(), sms_needed);
-    }
-
-
-    values_[key] = sms_used;
-    return sms_used;
-  }
-  catch (const std::exception& e) {
-    std::cerr << "Caught std::exception: " << e.what() << "\n";
-    return ERROR_CODE;
-  }
-  catch (...) {
-    std::cerr << "Caught unknown exception!\n";
-    return ERROR_CODE;
-  }
-}
-
-  int getCost(const token_ptr& tok) override {
-    return heuristic_function::getCost(tok);
-  }
-
-private:
-  device_ptr dev_;
-  bool ceil_ = true;
-};
-
-} // namespace caf::cuda
-
diff --git a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp b/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
deleted file mode 100644
index f13acee568..0000000000
--- a/libcaf_cuda/caf/cuda/control-layer/scheduler_actor_state.hpp
+++ /dev/null
@@ -1,30 +0,0 @@
-#pragma once
-#include <queue>
-#include <vector>
-#include "caf/cuda/control-layer/behavior_table.hpp"
-#include "caf/cuda/control-layer/behavior_token.hpp" 
-#include "caf/cuda/control-layer/token.hpp"
-#include "caf/cuda/control-layer/kernel_graph.hpp"
-
-namespace caf::cuda {
-class scheduler_actor_behavior; 
-
-struct scheduler_actor_state {
-    caf::actor  self;
-    scheduler_actor_behavior* current_behavior = nullptr;
-    std::unique_ptr<behavior_table> table;
-    //behavior_table table;
-    std::queue<token_ptr> queue; // here for legacy prototype schedulers
-    std::vector<kernel_graph> operations; //more modern dependency based data structure 
-    int device_number;
-    int in_flight = 0; // current number of tasks active on the GPU
-    int num_streams = 500;
-    int stream_depth = 1; // maximum number of in-flight tasks per stream
-    std::vector<caf::actor> schedulers; //the other scheduler actors in the system, an actor for a 
-			     //specific GPU can be accessed via there corrosponding 
-			     //device number in the std::vector
-   bool multiple_gpus = false; //flag that will indicate to the actor whether or not there is
-			       //multiple GPUs 
-
-};
-} // namespace caf::cuda
diff --git a/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp b/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp
index b9cb019e7a..092924b6d3 100644
--- a/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp
+++ b/libcaf_cuda/caf/cuda/control-layer/transfer_token.hpp
@@ -1,7 +1,7 @@
 #pragma once
 #include "caf/cuda/control-layer/response_token.hpp"
 #include "caf/cuda/control-layer/launch_token.hpp"
-#include "caf/cuda/control-layer/return_payloads/transfer_ack.hpp"
+// #include "caf/cuda/control-layer/return_payloads/transfer_ack.hpp"
 #include "caf/cuda/global_export.hpp"
 
 #include <caf/actor.hpp>
@@ -66,12 +66,12 @@ class CAF_CUDA_EXPORT transfer_token : public response_token {
             return;
 
         try {
-            // Create concrete transfer_ack
-            transfer_ack ack_obj{dependency_number_};
+        //     // Create concrete transfer_ack
+        //     transfer_ack ack_obj{dependency_number_};
 
-            // Upcast explicitly to ack reference before sending
-            [[maybe_unused]] const ack& base_ack = ack_obj;
-	    caf::anon_mail(std::move(ack_obj)).urgent().send(receiver_);
+        //     // Upcast explicitly to ack reference before sending
+        //     [[maybe_unused]] const ack& base_ack = ack_obj;
+	    // caf::anon_mail(std::move(ack_obj)).urgent().send(receiver_);
 
 
         } catch (...) {
diff --git a/libcaf_cuda/src/control-layer/behavior.cpp b/libcaf_cuda/src/control-layer/behavior.cpp
deleted file mode 100644
index bdb0f72246..0000000000
--- a/libcaf_cuda/src/control-layer/behavior.cpp
+++ /dev/null
@@ -1,40 +0,0 @@
-#include "caf/cuda/control-layer/all-control-layer.hpp"
-#include "caf/cuda/control-layer/behavior.hpp"
-#include "caf/cuda/control-layer/return_payloads/all_return_payloads.hpp"
-#include "caf/all.hpp"
-
-namespace caf::cuda {
-
-void scheduler_actor_behavior::process_launch_token(const token_ptr& tok, int stream_id) {
-    const auto& launch = static_cast<const launch_token&>(*tok);
-    auto response = make_launch_response_token(state_.self, launch, state_.device_number, stream_id);
-    anon_mail(response).send(launch.getReplyActor());
-}
-
-
-void scheduler_actor_behavior::process_memory_transfer_token(const token_ptr& tok, int stream_id) {
-    const auto& mem = static_cast<const memory_transfer_token&>(*tok);
-    auto response = make_memory_response_token(state_.self, mem, state_.device_number, stream_id);
-    anon_mail(response).send(mem.getReplyActor());
-}
-
-
-void scheduler_actor_behavior::dispatch_transfer_token(const token_ptr& tok, int stream_id) {
-    
-   //for right now only assumes launch tokens
-   //anything else is undefined behavior 
-    const auto& launch = static_cast<const launch_token&>(*tok);
-    
-     response_token_ptr transfer = make_transfer_token(
-            state_.self,                    // ack should return to this scheduler
-            launch,                    // original launch token
-            state_.device_number,           // new device
-            stream_id        // stream we stored it under
-        );
-    
-    anon_mail(transfer).send(launch.getReplyActor());
-}
-
-
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/behavior_table.cpp b/libcaf_cuda/src/control-layer/behavior_table.cpp
deleted file mode 100644
index 9866bda5d3..0000000000
--- a/libcaf_cuda/src/control-layer/behavior_table.cpp
+++ /dev/null
@@ -1,43 +0,0 @@
-#include "caf/cuda/control-layer/behavior_table.hpp"
-
-#include "caf/cuda/control-layer/behavior.hpp"  
-#include "caf/cuda/control-layer/behavior_token.hpp"  
-
-// If your derived classes have important cleanup, also include them if needed
-#include "caf/cuda/control-layer/green_light_behavior.hpp"
-#include "caf/cuda/control-layer/red_light_behavior.hpp"
-#include "caf/cuda/control-layer/core_usage_behavior.hpp"
-#include "caf/cuda/control-layer/single_usage_behavior.hpp"
-#include "caf/cuda/control-layer/multilevel_usage_behavior.hpp"
-#include "caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp"
-#include "caf/cuda/control-layer/pressure_scheduler.hpp"
-#include "caf/cuda/control-layer/all-control-layer.hpp"
-
-namespace caf::cuda {
-
- // Constructor that creates all default behaviors for a given actor state
-    behavior_table::behavior_table(scheduler_actor_state& state) {
-        // dynamically allocate behaviors and add to table
-        add("red",          new red_light_behavior(state));
-        add("green",        new green_light_behavior(state));
-        add("core_usage",   new core_usage_behavior(state));
-        add("single_usage", new single_usage_behavior(state));
-        add("multilevel",   new multilevel_usage_behavior(state));
-        add("pressure",     new pressure_scheduler(state));
-        add("mmul",     new mmul_batch_scheduler_behavior(state));
-    }
-
-
-    behavior_table::~behavior_table() {
-        for (auto& [name, beh] : table_)
-            delete beh;  // clean up all behaviors on destruction
-    }
-
-
-scheduler_actor_behavior* behavior_table::get(const behavior_token& tok) const {
-    auto it = table_.find(tok.name());
-    return it != table_.end() ? it->second : nullptr;
-}
-
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp b/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
deleted file mode 100644
index 605d6dc5c3..0000000000
--- a/libcaf_cuda/src/control-layer/core_usage_behavior.cpp
+++ /dev/null
@@ -1,255 +0,0 @@
-#include "caf/cuda/control-layer/all-control-layer.hpp"
-#include "caf/cuda/control-layer/core_usage_behavior.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/profiler.hpp"
-#include "caf/cuda/manager.hpp"
-#include "caf/cuda/device.hpp"
-
-namespace caf::cuda {
-
-core_usage_behavior::core_usage_behavior(scheduler_actor_state& state)
-    : scheduler_actor_behavior(state) {
-    init_state();
-}
-
-core_usage_behavior::~core_usage_behavior(){
-
-}
-
-void core_usage_behavior::init_state() {
-    device_ = manager::get().find_device(state_.device_number);
-    heuristic.emplace(device_);
-    total_SM = device_->num_sms() * 16;
-    available_SM = total_SM;
-    available_memory = static_cast<int>(device_->total_memory_bytes());
-    num_streams = state_.num_streams;
-}
-
-void core_usage_behavior::on_enter() {
-    // TODO implement
-	//std::cout << "Hello\n";
-}
-
-void core_usage_behavior::reclaim(int blocks_consumed,
-	       	int memory_returned,
-	       	[[maybe_unused]] int time,
-	       	[[maybe_unused]] int dependency_number) {
-
-	//std::cout << "blocks is " <<  blocks_consumed << "\n";
-	available_SM += blocks_consumed;
-	available_memory+= memory_returned;
-	//will eventually do something with the dependency number and stalling or maybe not
-	schedule();
-
-}
-
-
-
-
-void core_usage_behavior::process_launch_token(const token_ptr& tok,int stream_id )  {
-
-    
-	scoped_timer timer("core_usage_behavior::process_launch_token");
-	
-	int cost = heuristic -> getCost(tok);
-
-	const auto& launch = static_cast<const launch_token&>(*tok);
-	auto response = make_launch_response_token(state_.self, launch, state_.device_number, stream_id,cost);
-	anon_mail(response).send(launch.getReplyActor());
-	available_SM -= cost;
-}
-
-
-void core_usage_behavior::receive(const token_ptr& tok) {
-
-    scoped_timer timer("core_usage_behavior::receive");
-    
-	if (tok->getType() == LAUNCH) {
-        create_new_graph(tok);
-	
-	//if we have the resources to dispatch, just do it right away 
-	if (available_SM -  heuristic->getCost(tok) >= 0)
-	 {
-		if (tok->isIndependent()) 
-		{
-			process_launch_token(tok,get_next_stream());
-			return;
-		}
-
-		int stream = graphs[tok->getDependency()].stream_id();
-		process_launch_token(tok,stream);
-	}
-
-    } else if (tok->getType() == MEMORY) {
-        process_memory_transfer_token(tok, 0);
-    }
-}
-
-int core_usage_behavior::get_next_stream() {
-    return current_stream++ % num_streams;
-}
-
-void core_usage_behavior::create_new_graph(const token_ptr& tok) {
-    if (tok->isIndependent()) {
-        kernel_graph new_graph(state_.device_number, get_next_stream());
-        new_graph.add_operation(tok);
-        independent_graphs.push_back(std::move(new_graph));
-        return;
-    }
-    else if (graphs.contains(tok->getDependency())) {
-        graphs[tok->getDependency()].add_operation(tok);
-    }
-    else {
-        kernel_graph new_graph(state_.device_number, get_next_stream());
-        new_graph.add_operation(tok);
-        graphs[tok->getDependency()] = std::move(new_graph);
-    }
-}
-
-void core_usage_behavior::dummy_schedule() {
-    // Drain independent graphs fully
-    for (auto it = independent_graphs.begin(); it != independent_graphs.end(); ) {
-        kernel_graph& graph = *it;
-        while (!graph.empty()) {
-            token_ptr tok = graph.getOperation();
-            if (!tok) break;
-            if (tok->getType() == LAUNCH)
-                process_launch_token(tok, graph.stream_id());
-        }
-        it = independent_graphs.erase(it);
-    }
-
-    // Drain dependency graphs, do not delete
-    for (auto& [dep, graph] : graphs) {
-        while (!graph.empty()) {
-            token_ptr tok = graph.getOperation();
-            if (!tok) break;
-            if (tok->getType() == LAUNCH)
-                process_launch_token(tok, graph.stream_id());
-        }
-    }
-}
-
-void core_usage_behavior::rank(std::size_t max_best=5) {
-    scoped_timer timer("core_usage_behavior::rank");
-
-    best_graphs.clear();
-
-    struct candidate {
-        int cost;
-        graph_ref ref;
-    };
-    std::vector<candidate> candidates;
-
-    // Dependent graphs
-    for (auto& [dep, graph] : graphs) {
-        if (graph.empty()) continue;
-
-        token_ptr tok = graph.peek();
-        if (!tok || tok->getType() != LAUNCH) continue;
-
-        int cost = heuristic->getCost(tok);
-        if (cost == ERROR_CODE) continue;
-
-        candidates.push_back({cost,
-            graph_ref{graph_ref::kind_t::dependent, dep}});
-    }
-
-    // Independent graphs
-    for (std::size_t i = 0; i < independent_graphs.size(); ++i) {
-        auto& graph = independent_graphs[i];
-        if (graph.empty()) continue;
-
-        token_ptr tok = graph.peek();
-        if (!tok || tok->getType() != LAUNCH) continue;
-
-        int cost = heuristic->getCost(tok);
-        if (cost == ERROR_CODE) continue;
-
-        candidates.push_back({cost,
-            graph_ref{graph_ref::kind_t::independent, -1, i}});
-    }
-
-    if (candidates.empty()) return;
-
-    std::sort(candidates.begin(), candidates.end(),
-              [](const candidate& a, const candidate& b) {
-                  return a.cost < b.cost;
-              });
-
-    const auto limit = std::min(max_best, candidates.size());
-    for (std::size_t i = 0; i < limit; ++i) {
-        best_graphs.push_back(candidates[i].ref);
-    }
-}
-
-
-
-kernel_graph* core_usage_behavior::resolve(const graph_ref& ref) {
-    //scoped_timer timer("core_usage_behavior::resolve");
-
-    switch (ref.kind) {
-        case graph_ref::kind_t::dependent: {
-            auto it = graphs.find(ref.dependency);
-            if (it == graphs.end()) return nullptr;
-            return &it->second;
-        }
-        case graph_ref::kind_t::independent: {
-            if (ref.index >= independent_graphs.size()) return nullptr;
-            return &independent_graphs[ref.index];
-        }
-    }
-    return nullptr;
-}
-
-
-
-
-
-void core_usage_behavior::schedule() {
-    scoped_timer timer("core_usage_behavior::schedule");
-
-    if (best_graphs.empty()) {
-        rank(20); // profiled independently
-        if (best_graphs.empty()) {
-            return;
-        }
-    }
-
-   for (std::size_t i = 0; i < best_graphs.size(); ) {
-    kernel_graph* graph = resolve(best_graphs[i]);
-    if (!graph || graph->empty()) {
-        best_graphs.erase(best_graphs.begin() + i);
-        continue; // stay at same index
-    }
-
-    token_ptr tok = graph->peek();
-    if (!tok || tok->getType() != LAUNCH) {
-        ++i;
-        continue;
-    }
-
-    int cost = heuristic->getCost(tok);
-    if (cost == ERROR_CODE) {
-        ++i;
-        continue;
-    }
-
-    if (available_SM >= cost) {
-        tok = graph->getOperation();
-        process_launch_token(tok, graph->stream_id());
-
-        best_graphs.erase(best_graphs.begin() + i);
-        continue; // stay at same index
-    }
-
-    ++i;
-
-   }
-
-}
-
-
-
-
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/green_light_behavior.cpp b/libcaf_cuda/src/control-layer/green_light_behavior.cpp
deleted file mode 100644
index 4d2e3d0ae8..0000000000
--- a/libcaf_cuda/src/control-layer/green_light_behavior.cpp
+++ /dev/null
@@ -1,59 +0,0 @@
-#include "caf/cuda/control-layer/all-control-layer.hpp"
-#include "caf/cuda/control-layer/green_light_behavior.hpp"
-#include <iostream>
-
-namespace caf::cuda {
-
-green_light_behavior::green_light_behavior(scheduler_actor_state& state)
-    : scheduler_actor_behavior(state) {}
-
-void green_light_behavior::on_enter() {
-    //std::cout << "GREEN LIGHT\n";
-
-    behavior_token_ptr red_light = make_behavior_token("red");
-    //send a request to change behavior to red light after 5 seconds
-    /*
-    anon_mail(red_light)
-            .delay(std::chrono::seconds(5))
-            .send(state_.self);
-    */
-}
-
-void green_light_behavior::schedule() {
-    //TODO IMPLEMENT
-}
-
-void green_light_behavior::receive(const token_ptr& tok) {
-
-//  std::cout << "Green light receive\n";
-
-    if (tok->getType() == LAUNCH) {
-
-        const auto& launch = static_cast<const launch_token&>(*tok);
-
-        int stream_id = rand() % state_.num_streams;
-
-        // Manually construct launch_response_token with anon_mail disabled
-        response_token_ptr response(
-            new launch_response_token(
-                state_.self,
-                launch,
-                state_.device_number,
-                stream_id,
-                0,      // reclaim_value
-                0,      // reclaim_runtime
-                false   // send_mail disabled
-            )
-        );
-
-        anon_mail(response).send(launch.getReplyActor());
-
-        //(void)response; // suppress unused warning
-    }
-    else if (tok->getType() == MEMORY) {
-        process_memory_transfer_token(tok, 0);
-    }
-}
-
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
index bdd7ff80c3..a93dc7e2a5 100644
--- a/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
+++ b/libcaf_cuda/src/control-layer/memory_actor/memory_actor.cpp
@@ -1,6 +1,5 @@
 #include "caf/cuda/control-layer/all-control-layer.hpp"
 #include "caf/cuda/manager.hpp"
-#include "caf/cuda/control-layer/return_payloads/ack.hpp"
 #include "caf/cuda/control-layer/memory_actor/memory_request_token.hpp"
 #include "caf/cuda/control-layer/memory_actor/mem_token.hpp"
 #include <iostream>
diff --git a/libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp b/libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp
deleted file mode 100644
index 9ee0a11720..0000000000
--- a/libcaf_cuda/src/control-layer/mmul_batch_scheduler_behavior.cpp
+++ /dev/null
@@ -1,300 +0,0 @@
-
-// ========================= mmul_batch_scheduler_behavior.cpp =========================
-
-#include "caf/cuda/control-layer/behavior.hpp"
-#include "caf/cuda/control-layer/mmul_batch_scheduler_behavior.hpp"
-#include "caf/cuda/control-layer/kernel_graph.hpp"
-#include "caf/cuda/device.hpp"
-#include "caf/cuda/manager.hpp"
-#include "caf/cuda/control-layer/launch_token.hpp"
-#include "caf/cuda/control-layer/all-control-layer.hpp"
-
-#include <deque>
-#include <optional>
-#include <algorithm>
-#include <unordered_map>
-#include <vector>
-#include <atomic>
-
-namespace caf::cuda {
-
-// A scheduler tuned for batched matrix-multiply kernels (mmul).
-// Classification is based on the kernel grid block count instead of SM occupancy.
-// Streams are partitioned per-class and each class has a configurable maximum
-// concurrent kernels.
-
-// -------------------- implementation --------------------
-
-mmul_batch_scheduler_behavior::mmul_batch_scheduler_behavior(scheduler_actor_state& state)
-    : scheduler_actor_behavior(state) {
-    init_state();
-}
-
-mmul_batch_scheduler_behavior::~mmul_batch_scheduler_behavior() {}
-
-void mmul_batch_scheduler_behavior::init_state() {
-    device_ = manager::get().find_device(state_.device_number);
-    num_streams = 128;
-
-    // default partitioning: LOW:50% of streams, MED:25%, HIGH:25%
-    low_stream_begin = 0;
-    low_stream_end = std::max(1, num_streams / 2);
-
-    med_stream_begin = low_stream_end;
-    med_stream_end = med_stream_begin + std::max(1, num_streams / 4);
-
-    high_stream_begin = med_stream_end;
-    high_stream_end = std::max(high_stream_begin + 1, num_streams);
-
-    // clamp ends to num_streams
-    if (low_stream_end > num_streams) low_stream_end = num_streams;
-    if (med_stream_end > num_streams) med_stream_end = num_streams;
-    if (high_stream_end > num_streams) high_stream_end = num_streams;
-
-    // thresholds and max concurrents can be tuned by editing these members
-    small_block_threshold = 128;
-    medium_block_threshold = 512;
-
-    max_concurrent_low = 20000;
-    max_concurrent_med = 16000;
-    max_concurrent_high = 60000;
-}
-
-void mmul_batch_scheduler_behavior::on_enter() {
-    // nothing special for now
-}
-
-void mmul_batch_scheduler_behavior::receive(const token_ptr& tok) {
-    if (!tok) return;
-
-    if (tok->getType() == LAUNCH) {
-        // Inspect block count now to choose queue + stream before creating the graph.
-        const auto& launch = static_cast<const launch_token&>(*tok);
-        int blocks = launch.getBlocks();
-        queue_type qt = classify_blocks(blocks);
-        int assigned_stream = get_stream_for_queue(qt);
-
-        if (tok->isIndependent()) {
-            // create independent graph with preselected stream
-            kernel_graph new_graph(state_.device_number, assigned_stream);
-            new_graph.add_operation(tok);
-            independent_graphs.push_back(std::move(new_graph));
-            graph_ref ref{graph_ref::kind_t::independent, -1, independent_graphs.size() - 1};
-            enqueue_graph_by_blocks(ref, qt);
-            schedule();
-            return;
-        }
-
-        int dep = tok->getDependency();
-
-        // If multiple GPUs are in play, forward to the owning device if found
-        if (state_.multiple_gpus) {
-            int dev_num = -1;
-            auto it = dependency_device_map.find(dep);
-            if (it != dependency_device_map.end()) dev_num = it->second;
-
-            if (dev_num != state_.device_number && dev_num != -1) {
-                // preserve stream choice on target side by forwarding token (target will reclassify)
-                anon_mail(tok).send(state_.schedulers[dev_num]);
-                return;
-            }
-        }
-
-        if (graphs.contains(dep)) {
-            // append operation to existing graph; keep the stream already assigned to that graph
-            graphs[dep].add_operation(tok);
-            graph_ref ref{graph_ref::kind_t::dependent, dep};
-            enqueue_graph_by_blocks(ref, qt);
-        } else {
-            // create a new dependent graph, assign the stream we selected earlier
-            kernel_graph new_graph(state_.device_number, assigned_stream);
-            new_graph.add_operation(tok);
-            graphs[dep] = std::move(new_graph);
-            graph_ref ref{graph_ref::kind_t::dependent, dep};
-            // record that we own this dependency on this device
-            dependency_device_map[dep] = state_.device_number;
-            enqueue_graph_by_blocks(ref, qt);
-        }
-
-        schedule();
-
-    } else if (tok->getType() == MEMORY) {
-        process_memory_transfer_token(tok, 0);
-    }
-}
-
-mmul_batch_scheduler_behavior::queue_type mmul_batch_scheduler_behavior::classify_blocks(int blocks) const {
-    if (blocks <= small_block_threshold) return LOW;
-    if (blocks <= medium_block_threshold) return MED;
-    return HIGH;
-}
-
-void mmul_batch_scheduler_behavior::enqueue_graph_by_blocks(const graph_ref& ref, queue_type forced_qt) {
-    kernel_graph* g = resolve(ref);
-    if (!g || g->empty()) return;
-
-    // If caller provided forced_qt, prefer it. Otherwise classify by peeking the front op.
-    queue_type qt = forced_qt;
-
-    // push into the appropriate queue
-    switch (qt) {
-        case LOW:  low_queue.push_back(ref); break;
-        case MED:  med_queue.push_back(ref); break;
-        case HIGH: high_queue.push_back(ref); break;
-    }
-}
-
-int mmul_batch_scheduler_behavior::get_stream_for_queue(queue_type qt) {
-    if (qt == LOW) {
-        int range = std::max(1, low_stream_end - low_stream_begin);
-        int idx = low_stream_counter++ % range;
-        return low_stream_begin + idx;
-    }
-    if (qt == MED) {
-        int range = std::max(1, med_stream_end - med_stream_begin);
-        int idx = med_stream_counter++ % range;
-        return med_stream_begin + idx;
-    }
-    // HIGH
-    int range = std::max(1, high_stream_end - high_stream_begin);
-    int idx = high_stream_counter++ % range;
-    return high_stream_begin + idx;
-}
-
-void mmul_batch_scheduler_behavior::try_dispatch_queue(std::deque<graph_ref>& q, queue_type qt) {
-    while (!q.empty()) {
-        // check concurrency cap for this queue
-        if (qt == LOW && active_low.load() >= max_concurrent_low) break;
-        if (qt == MED && active_med.load() >= max_concurrent_med) break;
-        if (qt == HIGH && active_high.load() >= max_concurrent_high) break;
-
-        graph_ref ref = q.front();
-        kernel_graph* g = resolve(ref);
-        if (!g || g->empty()) {
-            q.pop_front();
-            continue;
-        }
-
-        token_ptr tok = g->peek();
-        if (!tok || tok->getType() != LAUNCH) {
-            q.pop_front();
-            continue;
-        }
-
-        // Use the stream pre-assigned to the graph at creation time
-        int stream = g->stream_id();
-
-        // pop from queue before launching
-        q.pop_front();
-
-        // take the operation from the graph
-        token_ptr op = g->getOperation();
-        if (!op) continue;
-
-        int dep = op->getDependency();
-
-        // increment active counter and record mapping by dependency (if dependent)
-        switch (qt) {
-            case LOW:  active_low.fetch_add(1); break;
-            case MED:  active_med.fetch_add(1); break;
-            case HIGH: active_high.fetch_add(1); break;
-        }
-
-        if (dep != INDEPENDENT) {
-            dispatched_dependency_queue[dep] = qt;
-        } else {
-            // Independent graphs: we don't have a unique dependency id to map on reclaim.
-            // Best-effort: nothing to record. If independent graphs are common, consider
-            // generating a unique id per-independent-graph and setting it in the
-            // launch_response_token's reclaim_dependency_ so reclaim(...) can map it back.
-        }
-
-        process_launch_token(op, stream, static_cast<int>(qt));
-    }
-}
-
-void mmul_batch_scheduler_behavior::schedule() {
-    // Favor small kernels first to keep latency low and allow concurrency
-    try_dispatch_queue(low_queue, LOW);
-    try_dispatch_queue(med_queue, MED);
-    try_dispatch_queue(high_queue, HIGH);
-}
-
-void mmul_batch_scheduler_behavior::process_launch_token(const token_ptr& tok, int stream_id, [[maybe_unused]] int assigned_queue) {
-    // Create a launch response token and send it (same pattern as multilevel)
-    const auto& launch = static_cast<const launch_token&>(*tok);
-    auto response = make_launch_response_token(state_.self, launch, state_.device_number, stream_id, /*reclaim_value*/ 0, /*reclaim_runtime*/ 0);
-    anon_mail(response).send(launch.getReplyActor());
-
-    // Note: actual reclaim will arrive via anon_mail(reclaim_value, reclaim_memory, reclaim_runtime, reclaim_dependency)
-    // when launch_response_token::release() runs on the device side. That triggers reclaim(...) in this actor.
-}
-
-void mmul_batch_scheduler_behavior::reclaim([[maybe_unused]] int blocks_consumed,
-                                            [[maybe_unused]] int memory_returned,
-                                            [[maybe_unused]] int time,
-                                            int dependency_number) {
-    // This reclaim() is called when the device (or the launch_response_token destructor)
-    // sends the 4-tuple (reclaim_value, reclaim_memory, reclaim_runtime, reclaim_dependency).
-
-    // resource tracking (if other components use available_SM/available_memory)
-    //available_SM += blocks_consumed;
-    //available_memory += memory_returned;
-
-    // If reclaim references a dependent graph, decrement the active counter we recorded
-    if (dependency_number != INDEPENDENT) {
-        auto it = dispatched_dependency_queue.find(dependency_number);
-        if (it != dispatched_dependency_queue.end()) {
-            queue_type qt = it->second;
-            dispatched_dependency_queue.erase(it);
-            switch (qt) {
-                case LOW:  active_low.fetch_sub(1); break;
-                case MED:  active_med.fetch_sub(1); break;
-                case HIGH: active_high.fetch_sub(1); break;
-            }
-        }
-
-        // re-enqueue the graph if there are queued operations on that dependency
-        if (graphs.contains(dependency_number)) {
-            graph_ref ref{graph_ref::kind_t::dependent, dependency_number};
-            enqueue_graph_by_blocks(ref);
-        }
-    } else {
-        // Independent graphs
-        // The reclaim reports INDEPENDENT as the dependency. Without a unique ID we can't map
-        // this reclaim back to a specific independent graph's active counter. If independent
-        // graphs are common, add a unique graph id into the reclaim fields in
-        // launch_response_token so we can correctly decrement the active counter here.
-    }
-
-    // attempt to schedule immediately after resources returned
-    schedule();
-}
-
-void mmul_batch_scheduler_behavior::reclaim(ack& return_msg) {
-    // The actor-level ack path: if the device or other parts send specialized acks
-    // (e.g. TIMER, CAF_CUDA_ACK_TRANSFER), handle them here.
-    if (return_msg.getType() == TIMER) {
-        // no-op for now
-        return;
-    } else if (return_msg.getType() == CAF_CUDA_ACK_TRANSFER) {
-        //process_transfer_ack(return_msg);
-    }
-}
-
-kernel_graph* mmul_batch_scheduler_behavior::resolve(const graph_ref& ref) {
-    switch (ref.kind) {
-        case graph_ref::kind_t::dependent: {
-            auto it = graphs.find(ref.dependency);
-            if (it == graphs.end()) return nullptr;
-            return &it->second;
-        }
-        case graph_ref::kind_t::independent: {
-            if (ref.index >= independent_graphs.size()) return nullptr;
-            return &independent_graphs[ref.index];
-        }
-    }
-    return nullptr;
-}
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp b/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
deleted file mode 100644
index 41eb12a2ab..0000000000
--- a/libcaf_cuda/src/control-layer/multilevel_usage_behavior.cpp
+++ /dev/null
@@ -1,449 +0,0 @@
-#include "caf/cuda/control-layer/multilevel_usage_behavior.hpp"
-#include "caf/cuda/control-layer/all-control-layer.hpp"
-
-namespace caf::cuda {
-
-
-multilevel_usage_behavior::multilevel_usage_behavior(scheduler_actor_state& state)
-    : scheduler_actor_behavior(state) {
-    init_state();
-}
-
-multilevel_usage_behavior::~multilevel_usage_behavior() {}
-
-void multilevel_usage_behavior::init_state() {
-    device_ = manager::get().find_device(state_.device_number);
-    heuristic.emplace(device_);
-    total_SM = device_->num_sms() * 16;
-    available_SM = total_SM;
-    available_memory = static_cast<int>(device_->total_memory_bytes());
-    num_streams = state_.num_streams;
-    low_threshold = total_SM / 6;
-    transfer_threshold = total_SM / 2;
-    num_devices = manager::get().get_num_devices();
-}
-
-void multilevel_usage_behavior::on_enter() {
-
-	//std::cout << "scheduler actor with device number " << state_.device_number << " Says hello\n";
-	//trigger load balancing mechanisms
-	if (state_.multiple_gpus) {	
-		send_timed_msg();
-	}
-
-}
-
-void multilevel_usage_behavior::send_timed_msg() {
-
-
-	anon_mail(ack(TIMER)).delay(std::chrono::seconds(2)).send(state_.self);
-
-
-}
-
-void multilevel_usage_behavior::process_launch_token(const token_ptr& tok, int stream_id) {
-    //scoped_timer timer("multilevel_usage_behavior::process_launch_token");
-
-    int cost = heuristic->getCost(tok);
-
-    const auto& launch = static_cast<const launch_token&>(*tok);
-    auto response = make_launch_response_token(state_.self, launch, state_.device_number, stream_id, cost);
-    anon_mail(response).send(launch.getReplyActor());
-    available_SM -= cost;
-}
-
-void multilevel_usage_behavior::receive(const token_ptr& tok) {
-   // scoped_timer timer("multilevel_usage_behavior::receive");
-
-    if (tok->getType() == LAUNCH) {
-        create_new_graph(tok);
-	schedule();
-    } else if (tok->getType() == MEMORY) {
-        process_memory_transfer_token(tok, 0);
-    }
-}
-
-int multilevel_usage_behavior::get_next_stream() {
-    return current_stream++ % num_streams;
-}
-
-void multilevel_usage_behavior::create_new_graph(const token_ptr& tok) {
-    // create graph similar to core_usage_behavior but also enqueue by cost
-    if (tok->isIndependent()) {
-        kernel_graph new_graph(state_.device_number, get_next_stream());
-        new_graph.add_operation(tok);
-        independent_graphs.push_back(std::move(new_graph));
-        // reference to the newly added independent graph
-        graph_ref ref{graph_ref::kind_t::independent, -1, independent_graphs.size() - 1};
-        enqueue_graph_by_cost(ref);
-        return;
-    }
-
-    int dep = tok->getDependency();
-    
-    //check to ensure that the token is not found on another device
-    if (state_.multiple_gpus) {
-	    int dev_num = get_device_for_dependency(dep);
-
-	    //found elsewhere
-	    if (dev_num != state_.device_number && dev_num != -1) {
-	    
-		    anon_mail(tok).send(state_.schedulers[dev_num]);
-		    return;
-	    }
-    }
-    
-    
-    
-    if (graphs.contains(dep)) {
-        graphs[dep].add_operation(tok);
-        // If graph already existed, ensure it's enqueued only if not currently in any queue
-        // For simplicity we enqueue it — caller reclaim/schedule will ensure duplicates don't cause re-dispatch
-        //this may lead in an error where dependencies are triggered before they are ready
-	//however since each graph gets a designated stream for now 
-	//this will not until that happens 
-	graph_ref ref{graph_ref::kind_t::dependent, dep};
-        enqueue_graph_by_cost(ref);
-    } else {
-        kernel_graph new_graph(state_.device_number, get_next_stream());
-        new_graph.add_operation(tok);
-        graphs[dep] = std::move(new_graph);
-        graph_ref ref{graph_ref::kind_t::dependent, dep};
-        enqueue_graph_by_cost(ref);
-    }
-}
-
-void multilevel_usage_behavior::enqueue_graph_by_cost(const graph_ref& ref) {
-    kernel_graph* g = resolve(ref);
-    if (!g || g->empty()) return;
-    token_ptr tok = g->peek();
-    if (!tok || tok->getType() != LAUNCH) return;
-
-    int cost = heuristic->getCost(tok);
-    if (cost == ERROR_CODE) return;
-
-    // classification thresholds
-    const long long medium_threshold = 16LL * static_cast<long long>(total_SM);
-
-    if (cost <= total_SM) {
-        low_queue.push_back(ref);
-    } else if (cost <= medium_threshold) {
-        med_queue.push_back(ref);
-    } else {
-        high_queue.push_back(ref);
-    }
-}
-
-void multilevel_usage_behavior::try_dispatch_queue(std::deque<graph_ref>& q) {
-    // Dispatch front-first while resources allow
-    while (!q.empty()) {
-        graph_ref ref = q.front();
-        kernel_graph* g = resolve(ref);
-        if (!g || g->empty()) {
-            q.pop_front();
-            continue;
-        }
-
-        token_ptr tok = g->peek();
-        if (!tok || tok->getType() != LAUNCH) {
-            // not a launch op at front => remove and continue
-            q.pop_front();
-            continue;
-        }
-
-        int cost = heuristic->getCost(tok);
-        if (cost == ERROR_CODE) {
-            q.pop_front();
-            continue;
-        }
-
-        if (available_SM >= cost) {
-            // we have enough resources, dispatch
-            q.pop_front();
-            tok = g->getOperation();
-            process_launch_token(tok, g->stream_id());
-
-            // do not reinsert; when that graph becomes ready again it will be
-            // re-enqueued by reclaim(...)
-        } else {
-            // not enough resources for this graph; stop trying this queue
-            break;
-        }
-    }
-}
-
-void multilevel_usage_behavior::schedule() {
-   // scoped_timer timer("multilevel_usage_behavior::schedule");
-    
-    // Prioritize low, then medium, then high
-    try_dispatch_queue(high_queue);
-    try_dispatch_queue(med_queue);
-    try_dispatch_queue(low_queue);
-}
-
-void multilevel_usage_behavior::reclaim(int blocks_consumed,
-                                       int memory_returned,
-                                       [[maybe_unused]] int time,
-                                       int dependency_number) {
-    // update available resources
-    available_SM += blocks_consumed;
-    available_memory += memory_returned;
-
-    // If this reclaim call references a dependent graph, re-enqueue its graph
-    if (dependency_number != INDEPENDENT) {
-        if (graphs.contains(dependency_number)) {
-            graph_ref ref{graph_ref::kind_t::dependent, dependency_number};
-            enqueue_graph_by_cost(ref);
-        }
-    } else {
-        // dependency_number < 0: we don't have a direct index for independent graphs here.
-        // As a best-effort, enqueue any independent graphs that are non-empty
-        //for (std::size_t i = 0; i < independent_graphs.size(); ++i) {
-          ///  if (!independent_graphs[i].empty()) {
-             //   graph_ref ref{graph_ref::kind_t::independent, -1, i};
-               // enqueue_graph_by_cost(ref);
-            //}
-       // }
-    }
-
-    // After re-enqueue, attempt to schedule immediately
-    schedule();
-}
-
-void multilevel_usage_behavior::reclaim(ack& return_msg) {
-
-	//TODO IMPLEMENT TIMER ACK AND TRANSFER ACK 
-
-	//std::cout << "scheduler actor with device number " << state_.device_number << " got an ack\n";
-	if (return_msg.getType() == TIMER) {
-
-	//std::cout << "scheduler actor with device number " << state_.device_number << " got a timer ack\n";
-	
-		request_load_balance();	
-		send_timed_msg();	
-	}
-
-	else if (return_msg.getType() == CAF_CUDA_ACK_TRANSFER) {
-		process_transfer_ack(return_msg);
-	}
-
-}
-
-void multilevel_usage_behavior::process_transfer_ack(ack& msg) {
-    // We already verified type before calling this
-    auto& transfer = static_cast<transfer_ack&>(msg);
-
-    int dep = transfer.dependency();
-
-    // Check if we still own this dependency
-    auto it = graphs.find(dep);
-    if (it == graphs.end())
-        return;
-
-    graph_ref ref;
-    ref.kind = graph_ref::kind_t::dependent;
-    ref.dependency = dep;
-
-    enqueue_graph_by_cost(ref);
-
-    schedule();
-}
-
-
-void multilevel_usage_behavior::request_load_balance() {
-	//std::cout << "Scheduler with device number " << state_.device_number << "is requesting load balance\n";  
-  if (!state_.multiple_gpus)  {
-
-        return;
-  }
-    // Only request work if we're underutilized
-
-    int busy_SM = total_SM - available_SM;
-    if (busy_SM > low_threshold)
-    {
-	 // std::cout << "Returning from since too busy\n";
-        return;
-    }
-    //std::cout << "Hello from request load_balance\n";
-    int my_device = state_.device_number;
-
-    for (int i = 0; i < num_devices; ++i) {
-        // Skip sending to self
-        if (i == my_device)
-            continue;
-
-        // Send our device number to other scheduler actors
-        anon_mail(my_device).urgent().send(state_.schedulers[i]);
-    }
-}
-
-
-
-kernel_graph* multilevel_usage_behavior::resolve(const graph_ref& ref) {
-	switch (ref.kind) {
-		case graph_ref::kind_t::dependent: {
-							   auto it = graphs.find(ref.dependency);
-							   if (it == graphs.end()) return nullptr;
-							   return &it->second;
-						   }
-		case graph_ref::kind_t::independent: {
-							     if (ref.index >= independent_graphs.size()) return nullptr;
-							     return &independent_graphs[ref.index];
-						     }
-	}
-	return nullptr;
-}
-
-
-//multi GPU load balancing methods
-void multilevel_usage_behavior::handle_load_balance_request(int device_number) {
-    
-    // Only transfer work if we are busy
-
-    int free_SM = available_SM;
-    if (free_SM > transfer_threshold) {
-        return; // GPU not busy enough, do nothing
-    }
-
-    std::vector<kernel_graph> work_to_transfer;
-
-    // ---- Helper lambda to collect transferable graphs from a queue ----
-    auto collect_graphs_from_queue = [&](std::deque<graph_ref>& q, std::size_t max_count) {
-        std::size_t collected = 0;
-
-        for (auto it = q.begin(); it != q.end() && collected < max_count;) {
-            graph_ref ref = *it;
-            kernel_graph* g = resolve(ref);
-
-            if (!g || g->empty() || !g->canMove()) {
-                ++it;
-                continue; // skip invalid or non-movable graphs
-            }
-
-            // Move the graph into the transfer vector
-            work_to_transfer.push_back(std::move(*g));
-            ++collected;
-
-            // Clean up local structures
-            if (ref.kind == graph_ref::kind_t::dependent) {
-                remove_dependency(ref.dependency);
-                graphs.erase(ref.dependency);
-		add_dependency_to_device(ref.dependency,device_number);
-            } else { // independent
-                if (ref.index < independent_graphs.size()) {
-		    //TODO come up with a better way to clean this up
-                    independent_graphs[ref.index] = kernel_graph(); // reset empty
-                }
-            }
-
-            // Remove from queue
-            it = q.erase(it);
-        }
-    };
-
-    // ---- Step 1: transfer independent graphs first ----
-    std::size_t max_independent = independent_graphs.size() / 2;
-    std::size_t transferred_independent = 0;
-
-    for (std::size_t i = 0; i < independent_graphs.size() && transferred_independent < max_independent; ++i) {
-        kernel_graph& g = independent_graphs[i];
-        if (g.empty() || !g.canMove()) continue;
-
-        work_to_transfer.push_back(std::move(g));
-        independent_graphs[i] = kernel_graph(); // reset
-        ++transferred_independent;
-    }
-
-    // ---- Step 2: transfer from high and medium queues ----
-    std::size_t mid_high = high_queue.size() / 2;
-    std::size_t mid_med = med_queue.size() / 2;
-    std::size_t mid_low = low_queue.size() / 2;
-    collect_graphs_from_queue(high_queue, mid_high);
-    collect_graphs_from_queue(med_queue, mid_med);
-    collect_graphs_from_queue(low_queue, mid_low);
-
-    // ---- Step 3: send if we have anything ----
-    if (!work_to_transfer.empty()) {
-        anon_mail(work_to_transfer).send(state_.schedulers[device_number]);
-    }
-}
-
-
-void multilevel_usage_behavior::receive_work(std::vector<kernel_graph> work_graphs) {
-    for (auto& g : work_graphs) {
-
-        if (g.empty() || !g.canMove())
-            continue;
-
-        token_ptr tok = g.peek();
-        if (!tok)
-            continue;
-
-        int dep = tok->getDependency();
-
-        // ============================
-        // Independent Graph
-        // ============================
-        if (dep == INDEPENDENT) {
-
-            g.markMoved();
-
-            independent_graphs.push_back(std::move(g));
-
-            graph_ref ref;
-            ref.kind = graph_ref::kind_t::independent;
-            ref.index = independent_graphs.size() - 1;
-
-            enqueue_graph_by_cost(ref);
-            continue;
-        }
-
-        // ============================
-        // Dependent Graph
-        // ============================
-
-        // We are taking ownership of this dependency
-        add_dependency_to_device(dep, state_.device_number);
-
-        g.markMoved();
-
-        // Store or merge into graphs map
-        auto it = graphs.find(dep);
-        if (it == graphs.end()) {
-            graphs.emplace(dep, std::move(g));
-        } else {
-            // Merge operations into existing graph
-            while (!g.empty()) {
-                token_ptr op = g.getOperation();
-                if (!op) break;
-                it->second.add_operation(op);
-            }
-        }
-
-        // Send transfer token to actor so it migrates dependencies
-	dispatch_transfer_token(tok,g.stream_id());
-   }
-
-    // Only independent graphs were enqueued here
-    schedule();
-}
-
-
-void multilevel_usage_behavior::add_dependency_to_device(int dependency_number, int device_number) {
-	// Always override existing value
-	dependency_device_map[dependency_number] = device_number;
-}
-
-void multilevel_usage_behavior::remove_dependency(int dependency_number) {
-	dependency_device_map.erase(dependency_number);
-}
-
-int multilevel_usage_behavior::get_device_for_dependency(int dependency_number) const {
-	auto it = dependency_device_map.find(dependency_number);
-	if (it != dependency_device_map.end()) {
-		return it->second;
-	}
-	return -1; // not found
-}
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/pressure_scheduler.cpp b/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
deleted file mode 100644
index 3509d36bd5..0000000000
--- a/libcaf_cuda/src/control-layer/pressure_scheduler.cpp
+++ /dev/null
@@ -1,278 +0,0 @@
-#include "caf/cuda/control-layer/pressure_scheduler.hpp"
-#include "caf/cuda/control-layer/all-control-layer.hpp"
-#include <iostream>
-#include <algorithm>
-
-namespace caf::cuda {
-
-pressure_scheduler::pressure_scheduler(scheduler_actor_state& state)
-    : scheduler_actor_behavior(state) {
-    init_state();
-}
-
-pressure_scheduler::~pressure_scheduler() = default;
-
-void pressure_scheduler::init_state() {
-    device_ = manager::get().find_device(state_.device_number);
-    heuristic.emplace(device_, false); // turn off ceiling functionality
-
-    total_SM = device_->num_sms();
-    available_memory = static_cast<int>(device_->total_memory_bytes());
-    num_streams = state_.num_streams;
-
-    // policy thresholds (tunable)
-    resource_threshold = 1.0;
-
-    // concurrency pressure thresholds (absolute units)
-    low_concurreny_threshold  = 25.0;
-    high_concurrency_threshold = 75.0;
-
-    // initial accounting
-    current_sm_pressure = 0.0;
-}
-
-void pressure_scheduler::on_enter() {
-    // nothing special at enter
-}
-
-void pressure_scheduler::receive(const token_ptr& tok) {
-    if (tok->getType() == LAUNCH) {
-        create_new_graph(tok);
-        schedule();
-    } else if (tok->getType() == MEMORY) {
-        process_memory_transfer_token(tok, 0);
-    } else {
-        create_new_graph(tok);
-    }
-}
-
-/* -------------------------------------------------------------------------- */
-/* Helper utilities                                                            */
-/* -------------------------------------------------------------------------- */
-
-double pressure_scheduler::clamp_sm_ratio(double raw_ratio) const {
-    const double max_ratio = 0.20 * high_concurrency_threshold;
-    return std::min(raw_ratio, max_ratio);
-}
-
-int pressure_scheduler::get_bucket_level(double sm_ratio) {
-    // sm_ratio here is already clamped
-    if (sm_ratio >= 10.0) return HIGH;    // wide kernel
-    if (sm_ratio >=  6.0) return MEDIUM;  // mid-sized
-    return LOW;                           // narrow
-}
-
-/* -------------------------------------------------------------------------- */
-/* Scheduling                                                                  */
-/* -------------------------------------------------------------------------- */
-
-void pressure_scheduler::schedule() {
-    // Hard cutoff: if concurrency pressure is too high, wait
-    if (current_sm_pressure >= high_concurrency_threshold) {
-        return;
-    }
-
-    // Phase 1: everything treated as memory-bound for now
-    dispatch_prefer_memory();
-}
-
-void pressure_scheduler::process_launch_token(const token_ptr& tok, int stream_id) {
-    int sm_used = heuristic->getCost(tok);
-    if (sm_used == ERROR_CODE) return;
-
-    double raw_ratio = static_cast<double>(sm_used) / total_SM;
-    double sm_ratio  = clamp_sm_ratio(raw_ratio);
-
-    // Update pressure accounting
-    current_sm_pressure += sm_ratio;
-
-    if (tok->getType() == LAUNCH) {
-        const auto& launch = static_cast<const launch_token&>(*tok);
-        auto response = make_launch_response_token(
-            state_.self,
-            launch,
-            state_.device_number,
-            stream_id,
-            sm_used
-        );
-        anon_mail(response).send(launch.getReplyActor());
-    } else {
-        std::cerr << "[pressure_scheduler] dispatched non-launch token on stream "
-                  << stream_id << "\n";
-    }
-}
-
-void pressure_scheduler::reclaim(int sm_used,
-                                 int memory_returned,
-                                 int /*time*/,
-                                 int dependency_number) {
-    double raw_ratio = static_cast<double>(sm_used) / total_SM;
-    double sm_ratio  = clamp_sm_ratio(raw_ratio);
-
-    current_sm_pressure = std::max(0.0, current_sm_pressure - sm_ratio);
-    available_memory = std::min(
-        available_memory + memory_returned,
-        static_cast<int>(device_->total_memory_bytes())
-    );
-
-    if (dependency_number != INDEPENDENT) {
-        if (graphs.contains(dependency_number)) {
-            graph_ref ref{graph_ref::kind_t::dependent, dependency_number, 0};
-            enqueue_graph_by_cost(ref);
-        }
-    }
-
-    schedule();
-}
-
-/* -------------------------------------------------------------------------- */
-/* Queueing                                                                    */
-/* -------------------------------------------------------------------------- */
-
-void pressure_scheduler::enqueue_graph_by_cost(const graph_ref& ref) {
-    kernel_graph* g = resolve(ref);
-    if (!g || g->empty()) return;
-
-    token_ptr tok = g->peek();
-    if (!tok) return;
-
-    int sm_used = 1;
-    if (tok->getType() == LAUNCH) {
-        sm_used = heuristic->getCost(tok);
-        if (sm_used == ERROR_CODE) return;
-    }
-
-    double raw_ratio = static_cast<double>(sm_used) / total_SM;
-    double sm_ratio  = clamp_sm_ratio(raw_ratio);
-    int level = get_bucket_level(sm_ratio);
-
-    switch (level) {
-        case LOW:    low_memory_queue.push_back(ref); break;
-        case MEDIUM: med_memory_queue.push_back(ref); break;
-        case HIGH:   high_memory_queue.push_back(ref); break;
-    }
-}
-
-void pressure_scheduler::try_dispatch_queue(std::deque<graph_ref>& q) {
-    while (!q.empty()) {
-        graph_ref ref = q.front();
-        kernel_graph* g = resolve(ref);
-        if (!g || g->empty()) {
-            q.pop_front();
-            continue;
-        }
-
-        token_ptr tok = g->peek();
-        if (!tok) {
-            q.pop_front();
-            continue;
-        }
-
-        int sm_used = (tok->getType() == LAUNCH)
-                        ? heuristic->getCost(tok)
-                        : 1;
-
-        if (sm_used == ERROR_CODE) {
-            q.pop_front();
-            continue;
-        }
-
-        double raw_ratio = static_cast<double>(sm_used) / total_SM;
-        double sm_ratio  = clamp_sm_ratio(raw_ratio);
-
-        if (current_sm_pressure + sm_ratio > high_concurrency_threshold)
-            break;
-
-        q.pop_front();
-        token_ptr op = g->getOperation();
-        if (!op) continue;
-
-        int stream = get_next_stream();
-        process_launch_token(op, stream);
-
-        if (!g->empty()) {
-            enqueue_graph_by_cost(ref);
-        }
-    }
-}
-
-void pressure_scheduler::dispatch_prefer_compute() {
-    try_dispatch_queue(low_compute_queue);
-    try_dispatch_queue(med_compute_queue);
-    try_dispatch_queue(high_compute_queue);
-}
-
-void pressure_scheduler::dispatch_prefer_memory() {
-    try_dispatch_queue(high_memory_queue);
-    try_dispatch_queue(med_memory_queue);
-    try_dispatch_queue(low_memory_queue);
-}
-
-/* -------------------------------------------------------------------------- */
-/* Graph management                                                            */
-/* -------------------------------------------------------------------------- */
-
-kernel_graph* pressure_scheduler::resolve(const graph_ref& ref) {
-    if (ref.kind == graph_ref::kind_t::independent) {
-        if (ref.index < independent_graphs.size())
-            return &independent_graphs[ref.index];
-        return nullptr;
-    }
-
-    auto it = graphs.find(ref.dependency);
-    if (it == graphs.end()) return nullptr;
-    return &it->second;
-}
-
-void pressure_scheduler::create_new_graph(const token_ptr& token) {
-    if (token->isIndependent()) {
-        kernel_graph g(state_.device_number, get_next_stream());
-        g.add_operation(token);
-        independent_graphs.push_back(std::move(g));
-
-        graph_ref ref{
-            graph_ref::kind_t::independent,
-            -1,
-            independent_graphs.size() - 1
-        };
-        enqueue_graph_by_cost(ref);
-        return;
-    }
-
-    int dep = token->getDependency();
-    if (graphs.contains(dep)) {
-        graphs[dep].add_operation(token);
-    } else {
-        kernel_graph new_graph(state_.device_number, get_next_stream());
-        new_graph.add_operation(token);
-        graphs[dep] = std::move(new_graph);
-    }
-
-    graph_ref ref{graph_ref::kind_t::dependent, dep, 0};
-    enqueue_graph_by_cost(ref);
-}
-
-int pressure_scheduler::get_next_stream() {
-    return current_stream++ % std::max(1, num_streams);
-}
-
-/* -------------------------------------------------------------------------- */
-/* Legacy / unused hooks                                                       */
-/* -------------------------------------------------------------------------- */
-
-int pressure_scheduler::get_resource_pressure(int blocks_consumed) {
-    return blocks_consumed > 0 ? blocks_consumed : 1;
-}
-
-int pressure_scheduler::get_concurrency_pressure(int sm_used) {
-    double ratio = double(sm_used) / double(total_SM);
-
-    if (ratio < 0.10) return 1;
-    if (ratio < 0.25) return 2;
-    if (ratio < 0.50) return 4;
-    if (ratio < 0.75) return 8;
-    return 16;
-}
-
-} // namespace caf::cuda
-
diff --git a/libcaf_cuda/src/control-layer/red_light_behavior.cpp b/libcaf_cuda/src/control-layer/red_light_behavior.cpp
deleted file mode 100644
index 662be3b850..0000000000
--- a/libcaf_cuda/src/control-layer/red_light_behavior.cpp
+++ /dev/null
@@ -1,36 +0,0 @@
-#include "caf/cuda/control-layer/all-control-layer.hpp"
-#include "caf/cuda/control-layer/red_light_behavior.hpp"
-#include "caf/cuda/control-layer/token_factory.hpp"
-#include <iostream>
-
-namespace caf::cuda {
-
-red_light_behavior::red_light_behavior(scheduler_actor_state& state)
-    : scheduler_actor_behavior(state) {}
-
-void red_light_behavior::schedule() {}
-
-void red_light_behavior::receive(const token_ptr& tok) {
-    state_.queue.push(tok); // enqueue everything
-}
-
-red_light_behavior::~red_light_behavior() noexcept = default;
-
-void red_light_behavior::on_enter() {
-    std::cout << "RED LIGHT\n";
-    behavior_token_ptr green_light = make_behavior_token("green");
-    //send a request to change behavior to green light after 5 seconds
-    anon_mail(green_light)
-            .delay(std::chrono::seconds(5))
-            .send(state_.self);
-}
-
-void red_light_behavior::process_launch_token([[maybe_unused]] const token_ptr& tok, [[maybe_unused]] int stream_id) {
-    // Override to do nothing, as red light should not process
-}
-
-void red_light_behavior::process_memory_transfer_token([[maybe_unused]] const token_ptr& tok, [[maybe_unused]] int stream_id) {
-    // Override to do nothing
-}
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/single_usage_behavior.cpp b/libcaf_cuda/src/control-layer/single_usage_behavior.cpp
deleted file mode 100644
index 3965170d42..0000000000
--- a/libcaf_cuda/src/control-layer/single_usage_behavior.cpp
+++ /dev/null
@@ -1,149 +0,0 @@
-#include "caf/cuda/control-layer/all-control-layer.hpp"
-#include "caf/cuda/control-layer/single_usage_behavior.hpp"
-#include "caf/cuda/control-layer/scheduler-functions/profiler.hpp"
-#include "caf/cuda/manager.hpp"
-#include "caf/cuda/device.hpp"
-
-namespace caf::cuda {
-
-single_usage_behavior::single_usage_behavior(scheduler_actor_state& state)
-    : scheduler_actor_behavior(state) {
-    init_state();
-}
-
-single_usage_behavior::~single_usage_behavior() = default;
-
-void single_usage_behavior::init_state() {
-    device_ = manager::get().find_device(state_.device_number);
-    // We can still keep the heuristic if you want to log real SM usage later,
-    // but we won't use it for scheduling decisions
-    heuristic.emplace(device_);
-
-    // For single-usage we treat the whole GPU as occupied or free
-    gpu_available = true;           // initially free
-    available_memory = static_cast<int64_t>(device_->total_memory_bytes());
-}
-
-void single_usage_behavior::on_enter() {
-    schedule();  // try to launch something right away if tokens already waiting
-}
-
-void single_usage_behavior::reclaim([[maybe_unused]] int blocks_consumed,
-                                    int memory_returned,
-                                    [[maybe_unused]] int time,
-                                    [[maybe_unused]] int dependency_number) {
-   
-    std::cout << "reclaiming\n";
-    // GPU is now free again
-    gpu_available = true;
-    available_memory += memory_returned;
-
-    // Optional: could log real usage
-    // std::cout << "Reclaimed: " << blocks_consumed << " SMs, " << time << " μs\n";
-
-    schedule();  // try to launch the next one immediately
-}
-
-void single_usage_behavior::process_launch_token(const token_ptr& tok, int stream_id) {
-    //scoped_timer timer("single_usage_behavior::process_launch_token");
-
-    // For pure single-usage mode we usually don't care about the heuristic cost
-    // but we can still compute it for logging / debugging
-    int reported_cost = heuristic->getCost(tok);
-
-    const auto& launch = static_cast<const launch_token&>(*tok);
-    auto response = make_launch_response_token(
-        state_.self,
-        launch,
-        state_.device_number,
-        stream_id,
-        reported_cost  // report real cost even if we don't use it for decision
-    );
-
-    anon_mail(response).send(launch.getReplyActor());
-
-    // Mark GPU as busy
-    gpu_available = false;
-}
-
-void single_usage_behavior::receive(const token_ptr& tok) {
-
-    if (tok->getType() == LAUNCH) {
-        create_new_graph(tok);
-
-        // Try to dispatch immediately if GPU is currently free
-        if (gpu_available) {
-            schedule();
-        }
-    }
-    else if (tok->getType() == MEMORY) {
-        // For pure kernel serialisation testing you can often just forward memory ops
-        // without blocking — or implement strict ordering if needed
-        process_memory_transfer_token(tok, 0);
-    }
-}
-
-void single_usage_behavior::create_new_graph(const token_ptr& tok) {
-    if (tok->isIndependent()) {
-        kernel_graph g(state_.device_number, get_next_stream());
-        g.add_operation(tok);
-        independent_graphs.push_back(std::move(g));
-    }
-    else {
-        int dep = tok->getDependency();
-        if (!graphs.contains(dep)) {
-            kernel_graph g(state_.device_number, get_next_stream());
-            graphs[dep] = std::move(g);
-        }
-        graphs[dep].add_operation(tok);
-    }
-}
-
-int single_usage_behavior::get_next_stream() {
-    return current_stream++ % num_streams;
-}
-
-// ────────────────────────────────────────────────
-// The only real scheduling logic — find and launch ONE kernel if possible
-// ────────────────────────────────────────────────
-void single_usage_behavior::schedule() {
-    if (!gpu_available) {
-        return;  // GPU still busy → do nothing
-    }
-
-    token_ptr next = nullptr;
-    int stream_id = -1;
-
-    // 1. Prefer independent kernels (they have no dependencies → lowest risk)
-    for (auto it = independent_graphs.begin(); it != independent_graphs.end(); ++it) {
-        if (!it->empty()) {
-            next = it->getOperation();
-            stream_id = it->stream_id();
-            if (next && next->getType() == LAUNCH) {
-                // found one → launch and remove empty graph if needed
-                if (it->empty()) {
-                    independent_graphs.erase(it);
-                }
-                goto launch;
-            }
-        }
-    }
-
-    // 2. Otherwise take the first non-empty dependent graph (FIFO-ish)
-    for (auto& [dep, graph] : graphs) {
-        if (!graph.empty()) {
-            next = graph.getOperation();
-            stream_id = graph.stream_id();
-            if (next && next->getType() == LAUNCH) {
-                goto launch;
-            }
-        }
-    }
-
-    return;  // nothing ready to run
-
-launch:
-    process_launch_token(next, stream_id);
-}
-
-} // namespace caf::cuda
diff --git a/libcaf_cuda/src/control-layer/token_factory.cpp b/libcaf_cuda/src/control-layer/token_factory.cpp
index e28a8c37ae..f001807532 100644
--- a/libcaf_cuda/src/control-layer/token_factory.cpp
+++ b/libcaf_cuda/src/control-layer/token_factory.cpp
@@ -78,20 +78,5 @@ response_token_ptr make_transfer_token(caf::actor receiver,
 
 
 
-
-/// Factory function: create a mem_ptr<int> with fake data
- mem_ptr<int> make_mem_ptr(size_t num_elements = 16) {
-    if (num_elements == 1) {
-        // scalar test
-        return mem_ptr<int>(new mem_ref<int>(42, /*access=*/0));
-    } else {
-        // create a device-like mem_ref with scalar backing for simplicity
-        // normally this would allocate GPU memory, here just fake values
-        auto ptr = mem_ptr<int>(new mem_ref<int>(0, /*CUdeviceptr*/0, /*access=*/0));
-        return ptr;
-    }
-}
-
-
 } // namespace caf::cuda
 
diff --git a/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/CMakeLists.txt b/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/CMakeLists.txt
deleted file mode 100644
index cda77250ef..0000000000
--- a/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/CMakeLists.txt
+++ /dev/null
@@ -1,52 +0,0 @@
-cmake_minimum_required(VERSION 3.16.3)
-
-# 1) Enforce C++20 (needs modern C++ features)
-set(CMAKE_CXX_STANDARD        20)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_EXTENSIONS      OFF)
-
-# 2) Set CAF source and build directories
-set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../actor-framework")
-set(CAF_BUILD "${CAF_SRC}/build")
-
-
-#required since cmake cant seem to find the compiler 
-set(CMAKE_CXX_COMPILER "/usr/bin/g++")
-set(CMAKE_C_COMPILER "/usr/bin/gcc")
-
-project(CUDA_ACTORS)
-
-find_package(CUDA REQUIRED)
-find_package(CUDAToolkit REQUIRED)
-
-include_directories(
-  "${CAF_SRC}/include"
-  "${CAF_SRC}/libcaf_io"
-  "${CAF_SRC}/libcaf_core"
-  "${CAF_SRC}/libcaf_opencl"
-  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
-  "${CAF_SRC}/libcaf_cuda"
-)
-
-
-# 5) Declare your executables
-add_executable(test main.cpp)
-target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
-
-
-
-target_link_libraries(test
-  PRIVATE
-    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-    CUDA::nvrtc
-)
-
-target_link_libraries(work-stealing
-  PRIVATE
-    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-    CUDA::nvrtc
-)
diff --git a/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/main.cpp b/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/main.cpp
deleted file mode 100644
index 80699a5865..0000000000
--- a/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/main.cpp
+++ /dev/null
@@ -1,186 +0,0 @@
-#include <caf/all.hpp>
-#include <caf/cuda/all.hpp>
-#include <iostream>
-#include <vector>
-#include <cmath>
-#include <algorithm>
-#include <memory>
-
-using namespace caf;
-using namespace std::chrono_literals;
-
-// Atoms for MCTS operations
-using search_atom = atom_constant<uint32_t, 1>;
-using simulate_atom = atom_constant<uint32_t, 2>;
-using update_atom = atom_constant<uint32_t, 3>;
-using expand_atom = atom_constant<uint32_t, 4>;
-
-// Mock Game State
-struct game_state {
-    int board[9]; // e.g., Tic-Tac-Toe
-    int turn;     // 1 or 2
-
-    template <class Inspector>
-    friend bool inspect(Inspector& f, game_state& x) {
-        return f.object(x).fields(f.field("board", x.board), f.field("turn", x.turn));
-    }
-};
-
-// Command runner for the GPU Simulation kernel
-// The kernel takes the current game state and returns a win probability (float)
-using simulation_command = caf::cuda::command_runner<
-    caf::cuda::in<game_state>, // Input game state
-    caf::cuda::out<float>,     // Output score
-    caf::cuda::in<int>         // Seed offset
->;
-
-struct mcts_node_state {
-    game_state state;
-    caf::actor parent;
-    std::vector<caf::actor> children;
-    int stream_id = -1;
-    
-    int visits = 0;
-    float total_value = 0.0f;
-    bool is_expanded = false;
-
-    caf::cuda::program_ptr sim_program;
-
-    float ucb1(int parent_visits) const {
-        if (visits == 0) return 1e9f; // Priority for unvisited nodes
-        return (total_value / visits) + 1.41f * std::sqrt(std::log(parent_visits) / visits);
-    }
-};
-
-// The MCTS Node Actor
-caf::behavior mcts_node_fun(caf::stateful_actor<mcts_node_state>* self, 
-                            caf::actor parent, 
-                            game_state state,
-                            caf::cuda::program_ptr sim_prog,
-                            int stream_id) {
-    self->state().parent = parent;
-    self->state().state = state;
-    self->state().sim_program = sim_prog;
-    self->state().stream_id = (stream_id == -1) ? static_cast<int>(self->id()) : stream_id;
-
-    return {
-        // Traversal / Selection Phase
-        [=](search_atom) {
-            if (!self->state().is_expanded) {
-                // Leaf node reached: Trigger Simulation on GPU
-                self->mail(simulate_atom_v).send(self);
-            } else if (self->state().children.empty()) {
-                // Terminal node or no moves possible
-                self->mail(update_atom_v, 0.5f).send(self);
-            } else {
-                // Select best child using UCB1
-                auto it = std::max_element(self->state().children.begin(), 
-                                         self->state().children.end(),
-                                         [=](const caf::actor& a, const caf::actor& b) {
-                                             // Note: In a real app, you'd request stats from children 
-                                             // or cache them in the parent state for speed.
-                                             return true; 
-                                         });
-                self->mail(search_atom_v).send(*it);
-            }
-        },
-
-        // Simulation Phase: Launching CUDA Kernel
-        [=](simulate_atom) {
-            simulation_command cmd;
-            int device = 0;
-            int stream_id = self->state().stream_id;
-            int node_id = static_cast<int>(self->id()); // Unique ID for RNG seed
-
-            // Prepare GPU arguments
-            auto in_state = caf::cuda::create_in_arg(self->state().state);
-            auto out_score = caf::cuda::create_out_arg_with_size<float>(1);
-            auto seed_arg = caf::cuda::create_in_arg(node_id);
-
-            // Configure Kernel Dims (1 block, 1 thread for a single simulation rollout)
-            // In a real scenario, you'd run many rollouts in parallel on the GPU.
-            caf::cuda::nd_range dims(1, 1, 1, 1, 1, 1);
-
-            // Use run_async to avoid blocking the actor thread
-            auto results = cmd.run_async(self->state().sim_program, dims, stream_id, in_state, out_score, seed_arg);
-            auto score_ptr = std::get<1>(results);
-            auto self_hdl = caf::actor_cast<caf::actor>(self);
-            cmd.copy_to_host_async(score_ptr, [self_hdl](std::vector<float> win_rates) {
-                caf::anon_mail(expand_atom_v).send(self_hdl);
-                caf::anon_mail(update_atom_v, win_rates[0]).send(self_hdl);
-            });
-        },
-
-        // Expansion Phase: Spawning child actors for new moves
-        [=](expand_atom) {
-            if (self->state().is_expanded) return;
-
-            // Mock expansion: spawn 3 child actors representing possible moves
-            for (int i = 0; i < 3; ++i) {
-                game_state next_state = self->state().state;
-                next_state.turn = (next_state.turn == 1) ? 2 : 1;
-                
-                auto child = self->spawn(mcts_node_fun, 
-                                       caf::actor_cast<caf::actor>(self), 
-                                       next_state,
-                                       self->state().sim_program,
-                                       self->state().stream_id);
-                self->state().children.push_back(child);
-            }
-            self->state().is_expanded = true;
-        },
-
-        // Backpropagation Phase
-        [=](update_atom, float result) {
-            self->state().visits++;
-            self->state().total_value += result;
-
-            if (self->state().parent) {
-                // Pass result up the tree
-                self->mail(update_atom_v, result).send(self->state().parent);
-            } else {
-                std::cout << "[Root] Search iteration complete. Root visits: " 
-                          << self->state().visits << std::endl;
-            }
-        }
-    };
-}
-
-void run_mcts_demo(caf::actor_system& sys) {
-    caf::cuda::manager::init(sys);
-    auto& mgr = caf::cuda::manager::get();
-
-    // Load the simulation kernel
-    // This kernel would perform random rollouts from the given state
-    auto program = mgr.create_program_from_cubin("simulation_kernel.cubin", "evaluate_state");
-
-    game_state initial_state;
-    initial_state.turn = 1;
-    for(int& i : initial_state.board) i = 0;
-
-    // Spawn the Root actor
-    auto root = sys.spawn(mcts_node_fun, nullptr, initial_state, program, -1);
-
-    std::cout << "Starting MCTS iterations..." << std::endl;
-
-    // Run 100 search iterations
-    for (int i = 0; i < 100; ++i) {
-        caf::anon_mail(search_atom_v).send(root);
-    }
-
-    // Wait for some results
-    std::this_thread::sleep_for(2s);
-
-    // Demonstration of pruning: kill the root to stop the whole tree
-    caf::anon_mail(exit_msg{root, exit_reason::user_shutdown}).send(root);
-    
-    sys.await_all_actors_done();
-    caf::cuda::manager::shutdown();
-    std::cout << "MCTS Demo finished." << std::endl;
-}
-
-void caf_main(caf::actor_system& sys) {
-    run_mcts_demo(sys);
-}
-
-CAF_MAIN(caf::cuda::id_block)
\ No newline at end of file
diff --git a/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/simulation_kernel.cu b/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/simulation_kernel.cu
deleted file mode 100644
index 1130865f4f..0000000000
--- a/libcaf_cuda/tests/benchmark-tests/workflow-tests/mcts/simulation_kernel.cu
+++ /dev/null
@@ -1,73 +0,0 @@
-#include <cuda_runtime.h>
-#include <curand_kernel.h>
-
-// Must match the C++ definition exactly
-struct game_state {
-    int board[9]; 
-    int turn;     
-};
-
-__device__ int check_winner(const int* board) {
-    // Rows
-    for (int i = 0; i < 9; i += 3)
-        if (board[i] != 0 && board[i] == board[i+1] && board[i] == board[i+2]) return board[i];
-    // Cols
-    for (int i = 0; i < 3; ++i)
-        if (board[i] != 0 && board[i] == board[i+3] && board[i] == board[i+6]) return board[i];
-    // Diagonals
-    if (board[0] != 0 && board[0] == board[4] && board[8]) return board[0];
-    if (board[2] != 0 && board[2] == board[4] && board[6]) return board[2];
-
-    // Check for draw
-    bool full = true;
-    for (int i = 0; i < 9; ++i) if (board[i] == 0) full = false;
-    if (full) return 3; // 3 represents Draw
-
-    return 0; // Ongoing
-}
-
-__device__ float perform_rollout(game_state state, curandState* local_state) {
-    int current_turn = state.turn;
-    int winner = 0;
-
-    // Play randomly until terminal state
-    for (int move = 0; move < 9; ++move) {
-        winner = check_winner(state.board);
-        if (winner != 0) break;
-
-        // Find available moves
-        int available[9];
-        int count = 0;
-        for (int i = 0; i < 9; ++i) {
-            if (state.board[i] == 0) available[count++] = i;
-        }
-
-        if (count == 0) break;
-
-        // Pick a random move
-        int pick = curand(local_state) % count;
-        state.board[available[pick]] = current_turn;
-        current_turn = (current_turn == 1) ? 2 : 1;
-    }
-
-    if (winner == 3) return 0.5f; // Draw
-    if (winner == 1) return 1.0f; // Player 1 wins
-    return 0.0f;                 // Player 2 wins (or loss for P1)
-}
-
-extern "C" {
-
-__global__ void evaluate_state(const game_state* initial_state, float* score_out, int seed_offset) {
-    int idx = blockIdx.x * blockDim.x + threadIdx.x;
-    
-    // Initialize random number generator
-    curandState local_state;
-    curand_init(1234ULL, idx + seed_offset, 0, &local_state);
-
-    // In this simple example, one thread does one rollout.
-    // In a high-perf MCTS, one thread block might cooperate 
-    // to do hundreds of rollouts for the same state.
-    score_out[idx] = perform_rollout(*initial_state, &local_state);
-}
-
-} // extern "C"

From 6d677a5fbce5b757cbb9d757db78e6b9558c230a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 07:54:00 -0600
Subject: [PATCH 0966/1000] updated experiment to use the new scheduler actor

---
 .../Batched-Matrix-Multiply/main.test.cpp     | 465 +++++-------------
 1 file changed, 114 insertions(+), 351 deletions(-)

diff --git a/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp b/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp
index 271464aff7..ff92ec5fca 100644
--- a/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp
+++ b/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp
@@ -1,47 +1,33 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
-#include <caf/component-actors/all-component-actors.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
 #include <iostream>
-#include <iomanip>
 #include <vector>
-#include <chrono>
-#include <thread>
-#include <algorithm>
-#include <numeric>
+#include <string>
 #include <random>
-#include "caf/actor_registry.hpp"
-#include <deque>
 #include <unordered_map>
 #include <unordered_set>
-//#include <caf/atoms.hpp>
+#include <algorithm>
 
 using namespace caf;
-using namespace std::chrono_literals;
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Atoms
-// ─────────────────────────────────────────────────────────────────────────────
-CAF_BEGIN_TYPE_ID_BLOCK(mmul_benchmark, caf::id_block::cuda::end)
-    CAF_ADD_ATOM(mmul_benchmark, get_work_atom)
-    CAF_ADD_ATOM(mmul_benchmark, task_done_atom)
-    CAF_ADD_ATOM(mmul_benchmark, release_memory_atom)
-    CAF_ADD_ATOM(mmul_benchmark, request_work_atom)
-    CAF_ADD_ATOM(mmul_benchmark, worker_done_atom)
-    CAF_ADD_ATOM(mmul_benchmark, refill_buffer_atom)
-CAF_END_TYPE_ID_BLOCK(mmul_benchmark)
+using namespace caf::cuda;
 
-// Command runners for GPU operations
-caf::cuda::command_runner<> mmul_command;
-using mmul_kernel_t = caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>, out<int>, in<int>>;
-mmul_kernel_t mmul_kernel;
+// A generic command runner to provide access to CUDA stream callbacks
+static command_runner<in<int>, in<int>, out<int>, in<int>> runner;
 
+// MatrixPool structure from mmul-random-batch-benchmark
 struct MatrixPool {
     std::unordered_map<int, std::vector<int>> A;
     std::unordered_map<int, std::vector<int>> B;
 };
 
+struct task_actor_state {
+    program_ptr prog;
+    int N_val;
+    std::shared_ptr<MatrixPool> pool;
+};
 
-
+// create_matrix_pool_random function from mmul-random-batch-benchmark
 MatrixPool create_matrix_pool_random(
     int num_sizes,
     int min_N,
@@ -49,370 +35,147 @@ MatrixPool create_matrix_pool_random(
     unsigned int seed
 ) {
     MatrixPool pool;
-
     std::mt19937 rng(seed);
     std::uniform_int_distribution<int> dist(min_N, max_N);
-
     std::unordered_set<int> used;
-
     while (used.size() < static_cast<size_t>(num_sizes)) {
         int N = dist(rng);
         if (used.insert(N).second) {
             pool.A[N] = std::vector<int>(N * N, 1);
-            pool.B[N] = std::vector<int>(N * N, 1);
+            pool.B[N] = std::vector<int>(N * N, 2); // Changed to 2 for distinct input
         }
     }
-
     return pool;
 }
 
-// ---------------------------- GLOBAL TASK POOL ----------------------------
-// The central source of truth for work. Implements a pull-based model.
-struct task_pool_state {
-    std::vector<int> tasks;
-    size_t next_task_idx = 0;
-};
-
-caf::behavior global_task_pool(caf::stateful_actor<task_pool_state>* self, std::vector<int> tasks) {
-    self->state().tasks = std::move(tasks);
+// This actor represents a single task that requests permission from the scheduler.
+behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_actor) {
     return {
-        [=](get_work_atom, size_t batch_size) -> result<std::vector<int>> {
-            auto& st = self->state();
-            if (st.next_task_idx >= st.tasks.size())
-                return sec::end_of_stream;
-            size_t count = std::min(batch_size, st.tasks.size() - st.next_task_idx);
-            std::vector<int> batch(st.tasks.begin() + st.next_task_idx, 
-                                   st.tasks.begin() + st.next_task_idx + count);
-            st.next_task_idx += count;
-            return batch;
-        }
-    };
-}
-
-// ---------------------------- DEVICE/GPU ACTOR ----------------------------
-// Manages memory for a specific GPU and steals (pulls) work from the Global Pool.
-struct device_actor_state {
-    MatrixPool pool;
-    caf::actor global_pool;
-    std::deque<int> local_tasks; // Local buffer to keep GPU busy
-    size_t total_device_memory_bytes = 0;
-    size_t current_allocated_memory_bytes = 0;
-    int active_workers = 0;
-    int device_id = -1;
-    size_t batch_size = 0;
-    size_t low_water_mark = 0;
-    bool fetching = false;
-};
-
-caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
-                               MatrixPool pool, caf::actor global_pool, int num_workers, int dev_id, int max_in_flight) {
-    self->state().pool = std::move(pool);
-    self->state().global_pool = global_pool;
-    self->state().device_id = dev_id;
-    self->state().active_workers = num_workers;
-
-    // Dynamically calculate prefetch markers based on the total pipeline capacity
-    self->state().low_water_mark = static_cast<size_t>(num_workers * max_in_flight);
-    self->state().batch_size = self->state().low_water_mark * 2;
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-    caf::cuda::device_ptr dev_obj = mgr.find_device(dev_id);
-    if (dev_obj) {
-        self->state().total_device_memory_bytes = dev_obj->total_memory_bytes();
-    }
-
-    // Helper to refill the local task buffer from the global pool
-    auto refill = [=]() {
-        auto& st = self->state();
-        if (st.fetching || st.local_tasks.size() >= st.low_water_mark + st.batch_size)
-            return;
-
-        st.fetching = true;
-        self->mail(get_work_atom_v, (size_t)st.batch_size).request(st.global_pool, infinite).then(
-            [=](std::vector<int>& batch) {
-                auto& st_inner = self->state();
-                for (int N : batch)
-                    st_inner.local_tasks.push_back(N);
-                st_inner.fetching = false;
-                if (st_inner.local_tasks.size() < st_inner.low_water_mark)
-                    self->mail(refill_buffer_atom_v).send(self);
-            },
-            [=](error& err) {
-                self->state().fetching = false;
-            }
-        );
-    };
-
-    return {
-        [=](refill_buffer_atom) {
-            refill();
-        },
-        [=](get_work_atom) -> caf::result<int, in<int>, in<int>> {
-            auto& st = self->state();
-
-            // If we have tasks locally, satisfy the request immediately
-            if (!st.local_tasks.empty()) {
-                int N = st.local_tasks.front();
-                size_t memory_needed = (size_t)N * N * sizeof(int) * 3;
-                if (st.current_allocated_memory_bytes + memory_needed > st.total_device_memory_bytes)
-                    return make_error(sec::runtime_error, "Device Actor: Not enough memory");
+        [=](response_token_ptr res) mutable {
+            if (res->getType() == LAUNCH_RESPONSE) {
+                auto& st = self->state();
                 
-                st.local_tasks.pop_front();
-                st.current_allocated_memory_bytes += memory_needed;
-                
-                // Proactively steal more work if the buffer is getting low
-                if (st.local_tasks.size() < st.low_water_mark)
-                    refill();
-
-                return {N, caf::cuda::create_in_arg(st.pool.A[N]), 
-                           caf::cuda::create_in_arg(st.pool.B[N])};
-            }
-
-            // Buffer empty: must fetch from global pool reactively
-            auto promise = self->make_response_promise<int, in<int>, in<int>>();
-            self->mail(get_work_atom_v, (size_t)st.batch_size).request(st.global_pool, infinite).then(
-                [=](std::vector<int>& batch) mutable {
-                    auto& st_inner = self->state();
-                    int N = batch.front();
-                    for(size_t i = 1; i < batch.size(); ++i) st_inner.local_tasks.push_back(batch[i]);
-                    
-                    size_t needed = (size_t)N * N * sizeof(int) * 3;
-                    st_inner.current_allocated_memory_bytes += needed;
-                    promise.deliver(N, caf::cuda::create_in_arg(st_inner.pool.A[N]), 
-                                       caf::cuda::create_in_arg(st_inner.pool.B[N]));
-                },
-                [=](error& err) mutable { promise.deliver(err); }
-            );
-            return promise;
-        },
-        [=](release_memory_atom, int N_completed) {
-            auto& st = self->state();
-            size_t memory_released = (size_t)N_completed * N_completed * sizeof(int) * 3;
-            st.current_allocated_memory_bytes -= memory_released;
-            refill(); // Try to get more work now that memory is free
-        },
-        [=](worker_done_atom) {
-            auto& st = self->state();
-            if (--st.active_workers <= 0) {
-                self->quit();
+                // We need to cast the base response_token to access the specific nd_range stored in it.
+                auto launch_res = static_cast<launch_response_token*>(res.get());
+                int N = st.N_val;
+
+                // 1. Setup GPU arguments.
+                // Fetch data from the shared pool only when scheduled to save RAM
+                auto in_a = create_in_arg(st.pool->A.at(N));
+                auto in_b = create_in_arg(st.pool->B.at(N));
+                auto out_c = create_out_arg_with_size<int>(N * N);
+                auto in_n = create_in_arg(N);
+
+                // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
+                auto result_tuple = runner.run_async(st.prog, launch_res->getRange(), res, in_a, in_b, out_c, in_n);
+                auto d_c = std::get<2>(result_tuple);
+
+                // 3. Asynchronous Copyback.
+                // Allocate a local buffer for the result to keep the total system memory low.
+                auto h_c = std::make_shared<std::vector<int>>(N * N);
+                // The launch_response_token is released inside the callback 
+                // to signal to the scheduler that the resource is free. (No serial verification here)
+                runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [res, exit_actor, h_c](int* /*ptr*/, size_t /*sz*/) {
+                    res->release();
+                    anon_mail(1).send(exit_actor);
+                });
             }
         }
     };
 }
 
-// ---------------------------- WORKER ACTOR ----------------------------
-// Manages 1 stream and pulls work from the Device Actor.
-struct worker_state {
-    int device_id;
-    int stream_id;
-    caf::cuda::program_ptr program;
-    caf::actor device_actor;
-    caf::actor supervisor;
-    int max_in_flight_tasks;
-    int in_flight_tasks_count = 0;
-    bool draining = false;
-};
-
-caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
-                              caf::actor supervisor, caf::actor device_actor, caf::cuda::program_ptr program,
-                              int dev_id, int stream_id, int max_in_flight_tasks) {
-    self->state().supervisor = supervisor;
-    self->state().device_actor = device_actor;
-    self->state().program = program;
-    self->state().device_id = dev_id;
-    self->state().stream_id = stream_id;
-    self->state().max_in_flight_tasks = max_in_flight_tasks;
-
-    // Trigger initial work requests up to max_in_flight_tasks
-    for (int i = 0; i < max_in_flight_tasks; ++i) {
-        self->mail(request_work_atom_v).send(self);
-    }
-
-    return {
-        [=](request_work_atom) {
-            auto& st = self->state();
-            if (st.in_flight_tasks_count >= st.max_in_flight_tasks || st.draining) {
-                return; // Already at max capacity, don't request more yet
-            }
-
-            st.in_flight_tasks_count++; // Mark as pending immediately
-            self->mail(get_work_atom_v).request(st.device_actor, infinite).then(
-                [=](int N, in<int> matrixA, in<int> matrixB) {
-                    // GPU Pipeline: Transfer -> Kernel -> Copyback
-                    auto arg1 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixA));
-                    auto arg2 = mmul_command.transfer_memory(st.device_id, st.stream_id, std::move(matrixB));
-
-                    const int THREADS = 32;
-                    const int BLOCKS = (N + THREADS - 1) / THREADS;
-                    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-                    auto result = mmul_kernel.run_async(st.program, dims, st.stream_id, 0, st.device_id,
-                                                       arg1, arg2,
-                                                       caf::cuda::create_out_arg<int>(N * N),
-                                                       caf::cuda::create_in_arg<int>(N));
-
-                    auto bufferC = std::get<2>(result);
-                    auto self_hdl = caf::actor_cast<caf::actor>(self);
-
-                    mmul_command.copy_to_host_async(bufferC, [self_hdl, N_task = N](std::vector<int>&&) {
-                        caf::anon_mail(task_done_atom_v, N_task).send(self_hdl); // Pass N back to self
-                    });
-                },
-                [=](error& err) {
-                    auto& st = self->state();
-                    st.in_flight_tasks_count--; // Revert pending status on failure
-                    if (err == sec::runtime_error) {
-                        // Not enough memory, retry after a delay
-                        self->println("Worker {}: Not enough memory, retrying for work...", st.stream_id);
-                        self->delayed_anon_send(self, 100ms, request_work_atom_v);
-                    } else if (err == sec::end_of_stream) {
-                        st.draining = true; // Mark as draining, let in-flight finish
-                        if (st.in_flight_tasks_count == 0) {
-                            self->mail(worker_done_atom_v).send(st.device_actor);
-                            mmul_command.release_stream_for_actor(st.stream_id);
-                            self->quit();
-                        }
-                    }
-                }
-            );
-        },
-        [=](task_done_atom, int N_completed) {
-            auto& st = self->state();
-            st.in_flight_tasks_count--; // Decrement count
-            self->mail(1).send(st.supervisor); // Notify supervisor
-            self->mail(release_memory_atom_v, N_completed).send(st.device_actor); // Release memory
-            
-            if (st.draining && st.in_flight_tasks_count == 0) {
-                self->mail(worker_done_atom_v).send(st.device_actor);
-                mmul_command.release_stream_for_actor(st.stream_id);
-                self->quit();
-            } else if (!st.draining) {
-                self->mail(request_work_atom_v).send(self); // Request next task if capacity allows
-            }
-        }
-    };
-}
-
-// ---------------------------- SUPERVISOR ACTOR ----------------------------
-struct supervisor_actor_state {
-    int total_tasks;
-    int completed = 0;
-    std::chrono::steady_clock::time_point start_time;
-};
-
-caf::behavior supervisor_actor_fun(
-    caf::stateful_actor<supervisor_actor_state>* self,
-    int total_tasks,
-    int workers_per_gpu,
-    int max_in_flight_tasks_per_worker,
-    MatrixPool pool,
-    std::vector<int> Ns
-    ) {
-    self->state().total_tasks = total_tasks;
-    self->state().start_time = std::chrono::steady_clock::now();
-
-    auto pool_actor = self->spawn(global_task_pool, std::move(Ns));
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-    int num_gpus = mgr.get_num_devices();
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-
-    for (int i = 0; i < num_gpus; ++i) {
-        auto broker = self->spawn(gpu_device_actor, pool, pool_actor, workers_per_gpu, i, max_in_flight_tasks_per_worker);
-        
-        for (int j = 0; j < workers_per_gpu; ++j)
-            self->spawn(mmul_worker_fun, self, broker, program, i, (i * 1000) + j, max_in_flight_tasks_per_worker);
-    }
-
-    return {
-        [=](int done) {
-            self->state().completed += done;
-            if (self->state().completed >= self->state().total_tasks) {
-                auto end_time = std::chrono::steady_clock::now();
-                std::chrono::duration<double> total_time = end_time - self->state().start_time;
-
-                std::cout << "\n===== BENCHMARK COMPLETE =====\n";
-                std::cout << "Tasks: " << self->state().total_tasks << "\n";
-                std::cout << "Runtime: " << total_time.count() << " s\n";
-                
-                caf::cuda::manager::shutdown();
-                self->quit();
-            }
-        }
-    };
+// Helper function to initialize task_actor_state
+behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, std::shared_ptr<MatrixPool> pool, caf::actor exit_actor) {
+    auto& st = self->state();
+    st.prog = std::move(prog);
+    st.N_val = N_val;
+    st.pool = std::move(pool);
+    return task_actor_fun(self, exit_actor); // Pass exit_actor to task_actor_fun
 }
 
 template <class Fn>
 double time_run(Fn&& fn) {
-  auto start = std::chrono::steady_clock::now();
-  fn();
-  auto end = std::chrono::steady_clock::now();
-  std::chrono::duration<double> elapsed = end - start;
-  return elapsed.count();
+    auto start = std::chrono::steady_clock::now();
+    fn();
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+    return elapsed.count();
 }
 
-void run_mmul_random_scaling_tests(caf::actor_system& sys,
-                                  caf::cuda::manager_config man_config) {
-
+void run_scheduler_integration_scaling_test(actor_system& sys) {
     const int min_N = 32;
     const int max_N = 2048;
-    const int num_sizes = 10;
+    const int num_distinct_sizes = 10;
+    const std::vector<int> actor_counts = {50000};
+
+
+    //  const std::vector<int> actor_counts = {5};
 
-    const int workers_per_gpu = 8; // Admission control: only 16 concurrent tasks per GPU
-    const int max_in_flight_tasks_per_worker = 3; // Each worker keeps 2 tasks in flight
 
-    const std::vector<int> actor_counts = {
-	  1,30000,40000,50000
-    };
 
     // Generate deterministic random pool once
-    MatrixPool pool = create_matrix_pool_random(
-        num_sizes,
-        min_N,
-        max_N,
-        42  // fixed seed
-    );
+    auto pool_ptr = std::make_shared<MatrixPool>(
+        create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42));
+    
+    std::vector<int> available_Ns;
+    for (const auto& pair : pool_ptr->A) available_Ns.push_back(pair.first);
 
-    //scheduler
-    caf::cuda::manager_config scheduler_off(false);
-    for (int num_tasks_for_this_run : actor_counts) {
-	    // Initialize CUDA manager
-	    caf::cuda::manager::init(sys, scheduler_off);
-        std::cout << "=====================================\n";
-        std::cout << "Random Scaling | actors=" << num_tasks_for_this_run << "\n";
+    for (int num_tasks : actor_counts) {
+        manager_config config;
+        manager::init(sys, config);
+        auto& mgr = manager::get();
 
-        // Precompute all task Ns for this run
-        std::vector<int> sizes;
-        for (const auto& [N, _] : pool.A) sizes.push_back(N);
+        // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
+        mgr.toggle_scheduler_actor(8, 1);
 
-        std::vector<int> Ns_for_this_run;
-        Ns_for_this_run.reserve(num_tasks_for_this_run);
+        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        const int THREADS = 32;
 
+        std::vector<token_ptr> tokens;
         std::mt19937 rng(42);
-        std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
-        for (int i = 0; i < num_tasks_for_this_run; ++i)
-	        Ns_for_this_run.push_back(sizes[dist(rng)]);
+        std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
 
-        // Execute the supervisor which manages the asynchronous workload
-        double elapsed = time_run([&]() {
+        std::cout << "=====================================\n";
+        std::cout << "Scheduler Test | tasks=" << num_tasks << "\n";
 
-            auto sup = sys.spawn(
-                supervisor_actor_fun,
-                (int)Ns_for_this_run.size(), // total_tasks
-                workers_per_gpu,
-                max_in_flight_tasks_per_worker,
-                pool,
-		Ns_for_this_run
-            );
+        // Spawn the exit actor for this specific test run (moved inside the loop)
+        auto exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_tasks);
 
-	      sys.await_all_actors_done();
-	    });
+        // Spawn task actors and prepare tokens
+        for (int i = 0; i < num_tasks; ++i) {
+            int current_N = available_Ns[dist_N_idx(rng)];
+            nd_range range((current_N + THREADS - 1) / THREADS, 
+                           (current_N + THREADS - 1) / THREADS, 1, 
+                           THREADS, THREADS, 1);
+            
+            auto worker = sys.spawn(make_task_actor_behavior,
+                                    program,
+                                    current_N, 
+                                    pool_ptr,
+                                    exit_actor); // Pass exit_actor to task actors
+
+            tokens.push_back(make_launch_token(program, range, 0, 
+                             "task_" + std::to_string(i), worker));
+        }
 
-        caf::cuda::manager::shutdown();
+        double elapsed = time_run([&]() {
+            std::cout << "[MAIN] Dispatching batch to scheduler..." << std::endl;
+            mgr.send_scheduler_actor_message(std::move(tokens));
+
+            // The dispatch is asynchronous. To get an accurate measurement, we must
+            // block until the exit_actor terminates (signaling all 50k tasks are done).
+            scoped_actor self{sys};
+            self->wait_for(exit_actor);
+        });
+
+        std::cout << "Run complete. Time: " << elapsed << " s\n";
+        manager::shutdown(); // Reset manager state for the next potential iteration
     }
 }
-void caf_main(caf::actor_system& sys) {
-  caf::cuda::manager_config man_config(false);
-  run_mmul_random_scaling_tests(sys, man_config);
+
+void caf_main(actor_system& sys) {
+    run_scheduler_integration_scaling_test(sys);
+    std::cout << "[MAIN] Integration test complete." << std::endl;
 }
-CAF_MAIN(id_block::mmul_benchmark)
+
+CAF_MAIN(id_block::cuda_control)

From 46c7bd7c9ce2cd7c008eb61f0a3a89988aeb7f20 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 08:02:38 -0600
Subject: [PATCH 0967/1000] updated scripts

---
 .../parse_gpu_scaling.py                      | 219 ++++++++----------
 .../Multi-GPU-Scaling-test/run_test.sh        |  30 +--
 2 files changed, 112 insertions(+), 137 deletions(-)

diff --git a/libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/parse_gpu_scaling.py b/libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/parse_gpu_scaling.py
index 077beec867..068ef150a6 100644
--- a/libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/parse_gpu_scaling.py
+++ b/libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/parse_gpu_scaling.py
@@ -1,128 +1,103 @@
-#!/usr/bin/env python3
+import os
+import glob
 import re
-import argparse
-from pathlib import Path
-from collections import defaultdict
-from statistics import mean
-
-import matplotlib
-matplotlib.use("Agg")
+import numpy as np
 import matplotlib.pyplot as plt
 
-
-# Only match NO scheduler sections
-NO_SCHED_PATTERN = re.compile(
-    r"Random Scaling NO scheduler \| actors=(\d+)\s+"
-    r"===== SUPERVISOR TOTAL TIME spawn =====.*?"
-    r"===== SUPERVISOR TOTAL TIME =====\s*"
-    r"Total runtime:\s*([0-9]*\.?[0-9]+)\s*s",
-    re.DOTALL,
-)
-
-
-def parse_file(path: Path):
-    text = path.read_text(errors="replace")
-    results = []
-    for actors_str, runtime_str in NO_SCHED_PATTERN.findall(text):
-        results.append((int(actors_str), float(runtime_str)))
-    return results
-
-
-def collect_data(base_dir: Path):
-    """
-    data[actors][gpu_count] = [runtime1, runtime2, ...]
-    """
-    data = defaultdict(lambda: defaultdict(list))
-
-    for gpu_dir in sorted(base_dir.glob("gpus_*")):
-        if not gpu_dir.is_dir():
-            continue
-
-        m = re.match(r"gpus_(\d+)$", gpu_dir.name)
-        if not m:
-            continue
-        gpu_count = int(m.group(1))
-
-        for txt_file in sorted(gpu_dir.glob("*.txt")):
-            for actors, runtime in parse_file(txt_file):
-                data[actors][gpu_count].append(runtime)
-
-    return data
-
-
-def make_plots(data, out_dir: Path):
-    out_dir.mkdir(parents=True, exist_ok=True)
-
-    for actors in sorted(data.keys()):
-        gpu_map = data[actors]
-        gpu_counts = sorted(gpu_map.keys())
-        if not gpu_counts:
+def parse_runtime_data(base_dir):
+    # Dictionary to store list of runtimes for each GPU count
+    # e.g., {1: [750.088, ...], 2: [386.738, ...]}
+    gpu_data = {}
+    
+    # Pattern to match directories like 'gpus_1', 'gpus_2', 'gpus_4', 'gpus_7'
+    dir_pattern = os.path.join(base_dir, 'gpus_*')
+    
+    # Regex to find the runtime value in the text files
+    time_regex = re.compile(r'Run complete\.\s+Time:\s+([\d.]+)\s+s')
+
+    for gpu_dir in glob.glob(dir_pattern):
+        # Extract the number of GPUs from the directory name
+        dir_name = os.path.basename(gpu_dir)
+        try:
+            num_gpus = int(dir_name.split('_')[1])
+        except (IndexError, ValueError):
             continue
-
-        means = [mean(gpu_map[g]) for g in gpu_counts]
-
-        # 🔥 FIX: use categorical spacing instead of numeric spacing
-        x_pos = list(range(len(gpu_counts)))
-
-        fig, ax = plt.subplots(figsize=(8, 5))
-        bars = ax.bar(x_pos, means)
-
-        # Add value labels on top of bars
-        for bar, value in zip(bars, means):
-            ax.text(
-                bar.get_x() + bar.get_width() / 2,
-                bar.get_height(),
-                f"{value:.2f}",
-                ha="center",
-                va="bottom",
-                fontsize=9,
-            )
-
-        fig.suptitle("multi-gpu scaling test on gpufarm7")
-        ax.set_title(f"actors = {actors}")
-        ax.set_xlabel("Number of GPUs")
-        ax.set_ylabel("Runtime (s)")
-
-        # Show actual GPU counts as labels (1,2,4,7) but evenly spaced
-        ax.set_xticks(x_pos)
-        ax.set_xticklabels(gpu_counts)
-
-        ax.grid(True, axis="y", alpha=0.3)
-
-        fig.tight_layout(rect=[0, 0, 1, 0.95])
-
-        out_file = out_dir / f"multi_gpu_scaling_actors_{actors}.png"
-        fig.savefig(out_file, dpi=200)
-        plt.close(fig)
-
-        print(f"Saved {out_file}")
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Generate GPU scaling bar charts (NO scheduler only)"
-    )
-    parser.add_argument(
-        "--base-dir",
-        type=Path,
-        default=Path("."),
-        help="Directory containing gpus_* folders",
-    )
-    parser.add_argument(
-        "--out-dir",
-        type=Path,
-        default=Path("gpu_scaling_plots"),
-        help="Output directory for plots",
-    )
-    args = parser.parse_args()
-
-    data = collect_data(args.base_dir)
-
-    if not data:
-        raise SystemExit("No data found.")
-
-    make_plots(data, args.out_dir)
-
+            
+        gpu_data[num_gpus] = []
+        
+        # Read all output_*.txt files in this directory
+        file_pattern = os.path.join(gpu_dir, 'output_*.txt')
+        for file_path in glob.glob(file_pattern):
+            try:
+                with open(file_path, 'r') as f:
+                    content = f.read()
+                    match = time_regex.search(content)
+                    if match:
+                        runtime = float(match.group(1))
+                        gpu_data[num_gpus].append(runtime)
+            except Exception as e:
+                print(f"Error reading file {file_path}: {e}")
+                
+    return gpu_data
+
+def calculate_metrics(gpu_data):
+    # Calculate means
+    mean_runtimes = {gpus: np.mean(times) for gpus, times in gpu_data.items() if times}
+    
+    # Sort by number of GPUs to keep things in order
+    sorted_gpus = sorted(mean_runtimes.keys())
+    
+    if 1 not in mean_runtimes:
+        raise ValueError("Error: Could not find 1 GPU data to calculate the scaling factor.")
+        
+    base_runtime = mean_runtimes[1]
+    
+    # Calculate scaling factors: (Runtime of 1 GPU) / (Runtime of N GPUs)
+    scaling_factors = {gpus: base_runtime / mean_runtimes[gpus] for gpus in sorted_gpus}
+    
+    return sorted_gpus, mean_runtimes, scaling_factors
+
+def plot_and_report(sorted_gpus, mean_runtimes, scaling_factors):
+    # Print the text report
+    print(f"{'GPUs':<10}{'Mean Runtime (s)':<20}{'Scaling Factor':<15}")
+    print("-" * 45)
+    for gpus in sorted_gpus:
+        print(f"{gpus:<10}{mean_runtimes[gpus]:<20.3f}{scaling_factors[gpus]:<15.2f}x")
+    
+    # Generate the bar chart
+    runtimes = [mean_runtimes[gpus] for gpus in sorted_gpus]
+    labels = [f"{gpus} GPU(s)" for gpus in sorted_gpus]
+    
+    plt.figure(figsize=(8, 6))
+    bars = plt.bar(labels, runtimes, color='lightgreen', edgecolor='black', width=0.6)
+    
+    # Add values on top of the bars
+    for bar in bars:
+        height = bar.get_height()
+        plt.text(bar.get_x() + bar.get_width()/2., height + max(runtimes)*0.01,
+                 f'{height:.2f}s', ha='center', va='bottom', fontsize=10)
+                 
+    plt.title('Mean Runtime vs. Number of GPUs', fontsize=14, fontweight='bold')
+    plt.xlabel('GPU Configuration', fontsize=12)
+    plt.ylabel('Mean Runtime (seconds)', fontsize=12)
+    plt.grid(axis='y', linestyle='--', alpha=0.7)
+    plt.tight_layout()
+    
+    # Save chart image and show
+    plt.savefig('gpu_scaling_chart.png', dpi=300)
+    print("\n[INFO] Bar chart saved as 'gpu_scaling_chart.png'")
+    plt.show()
 
 if __name__ == "__main__":
-    main()
+    # Use current directory '.' if script is placed inside 'scheduler/' 
+    # Otherwise replace with your absolute path: '/home/nqr159/data/scheduler-test/...'
+    BASE_DIR = '.' 
+    
+    print("Parsing dataset...")
+    raw_data = parse_runtime_data(BASE_DIR)
+    
+    if not raw_data:
+        print("No data found. Ensure you are running the script in the correct folder.")
+    else:
+        gpus, means, scaling = calculate_metrics(raw_data)
+        plot_and_report(gpus, means, scaling)
diff --git a/libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/run_test.sh b/libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/run_test.sh
index 9cc3b17b2a..15d3db4d06 100755
--- a/libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/run_test.sh
+++ b/libcaf_cuda/sc26/scripts/Multi-GPU-Scaling-test/run_test.sh
@@ -1,31 +1,31 @@
 #!/bin/bash
 
-BASE_DIR=/student/nqr159/data/scheduler-test/hetergenous-workloads/gpu_scaling_tests
-BIN=/student/nqr159/gpufarm5/actor-framework/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/build/test
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+BASE_DIR=/student/nqr159/data/scheduler-test/hetergenous-workloads/gpu_scaling_tests/scheduler
+BIN="$SCRIPT_DIR/scheduler_test"
 
 mkdir -p "$BASE_DIR"
 
-# Define GPU configurations
 declare -A GPU_CONFIGS
 GPU_CONFIGS[1]="0"
 GPU_CONFIGS[2]="0,1"
 GPU_CONFIGS[4]="0,1,2,3"
 GPU_CONFIGS[7]="0,1,2,3,4,5,6"
-
 for gpus in 1 2 4 7; do
-  echo "=============================="
-  echo "Testing with $gpus GPU(s)"
-  echo "=============================="
+    echo "=============================="
+    echo "Testing with $gpus GPU(s)"
+    echo "=============================="
 
-  OUT_DIR="$BASE_DIR/gpus_${gpus}"
-  mkdir -p "$OUT_DIR"
+    OUT_DIR="$BASE_DIR/gpus_${gpus}"
+    mkdir -p "$OUT_DIR"
 
-  CUDA_DEVICES=${GPU_CONFIGS[$gpus]}
+    CUDA_DEVICES=${GPU_CONFIGS[$gpus]}
 
-  for i in {1..10}; do
-    echo "Running iteration $i with $gpus GPU(s)..."
+    for i in {1..10}; do
+        echo "Running iteration $i with $gpus GPU(s)..."
 
-    CUDA_VISIBLE_DEVICES=$CUDA_DEVICES \
-    "$BIN" > "$OUT_DIR/output_${i}.txt"
-  done
+        CUDA_VISIBLE_DEVICES=$CUDA_DEVICES \
+        "$BIN" > "$OUT_DIR/output_${i}.txt"
+    done
 done

From 3c775e475d33e043efb78aa7bffc32a5c16a4962 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 10:49:00 -0600
Subject: [PATCH 0968/1000] Moved files from workloadA over here.

---
 .../workloadA/CMakeLists.txt                  | 132 +++++++
 .../Irregular-Workload/workloadA/atoms.hpp    |  51 +++
 .../workloadA/ft_cg_actor.hpp                 | 266 +++++++++++++
 .../workloadA/ft_cg_jacobi_actor.hpp          | 191 ++++++++++
 .../workloadA/jacobi_kernels.cu               |  15 +
 .../workloadA/main.native.cpp                 |  62 +++
 .../workloadA/main.native_sorted.cpp          |  64 ++++
 .../workloadA/main.test.cpp                   |  81 ++++
 .../workloadA/native_utils.cpp                |   0
 .../workloadA/native_utils.cu                 | 353 ++++++++++++++++++
 .../workloadA/native_utils.hpp                |  95 +++++
 .../workloadA/sparse_utils.cpp                | 267 +++++++++++++
 .../workloadA/sparse_utils.hpp                |  94 +++++
 .../workloadA/stability_kernels.cu            |  10 +
 .../workloadA/supervisor_actor.hpp            | 223 +++++++++++
 15 files changed, 1904 insertions(+)
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/CMakeLists.txt
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/atoms.hpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/ft_cg_actor.hpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/ft_cg_jacobi_actor.hpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/jacobi_kernels.cu
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native.cpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native_sorted.cpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/main.test.cpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/native_utils.cpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/native_utils.cu
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/native_utils.hpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/sparse_utils.cpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/sparse_utils.hpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/stability_kernels.cu
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadA/supervisor_actor.hpp

diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/CMakeLists.txt b/libcaf_cuda/sc26/Irregular-Workload/workloadA/CMakeLists.txt
new file mode 100644
index 0000000000..ed8e26b561
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/CMakeLists.txt
@@ -0,0 +1,132 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+# Enable CUDA as a first-class language for the project
+project(CUDA_ACTORS LANGUAGES CXX CUDA)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+# --- CUDA Kernel Compilation ---
+set(STABILITY_KERNEL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/stability_kernels.cu")
+set(STABILITY_KERNEL_CUBIN "${CMAKE_CURRENT_BINARY_DIR}/stability_kernels.cubin")
+
+# Target CUDA architecture. 'native' targets the current machine's GPU (requires CUDA 11.6+).
+# You can override this with -DCUDA_ARCH=sm_XX if needed.
+set(CUDA_ARCH "native" CACHE STRING "Target CUDA architecture (e.g., native, sm_70, sm_75, sm_80, sm_86)")
+
+add_custom_command(
+  OUTPUT ${STABILITY_KERNEL_CUBIN}
+  COMMAND ${CUDAToolkit_NVCC_EXECUTABLE}
+          -cubin
+          -arch=${CUDA_ARCH}
+          -o ${STABILITY_KERNEL_CUBIN}
+          ${STABILITY_KERNEL_SRC}
+  DEPENDS ${STABILITY_KERNEL_SRC}
+  COMMENT "Compiling CUDA kernel ${STABILITY_KERNEL_SRC} for architecture: ${CUDA_ARCH}"
+  VERBATIM
+)
+
+set(JACOBI_KERNEL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/jacobi_kernels.cu")
+set(JACOBI_KERNEL_CUBIN "${CMAKE_CURRENT_BINARY_DIR}/jacobi_kernels.cubin")
+
+add_custom_command(
+  OUTPUT ${JACOBI_KERNEL_CUBIN}
+  COMMAND ${CUDAToolkit_NVCC_EXECUTABLE}
+          -cubin
+          -arch=${CUDA_ARCH}
+          -o ${JACOBI_KERNEL_CUBIN}
+          ${JACOBI_KERNEL_SRC}
+  DEPENDS ${JACOBI_KERNEL_SRC}
+  COMMENT "Compiling CUDA kernel ${JACOBI_KERNEL_SRC} for architecture: ${CUDA_ARCH}"
+  VERBATIM
+)
+
+add_custom_target(stability_kernels_cubin ALL DEPENDS ${STABILITY_KERNEL_CUBIN})
+add_custom_target(jacobi_kernels_cubin ALL DEPENDS ${JACOBI_KERNEL_CUBIN})
+
+
+# 5) Declare your executable and its source files
+add_executable(test main.test.cpp sparse_utils.cpp)
+target_sources(test PRIVATE
+  atoms.hpp
+  ft_cg_actor.hpp
+  ft_cg_jacobi_actor.hpp
+  supervisor_actor.hpp
+)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
+)
+
+
+# # 5) Declare your executable
+# add_executable(hot-potatoe hot-potatoe.cpp sparse_utils.cpp)
+
+# target_compile_definitions(hot-potatoe PRIVATE CAF_ENABLE_LOGGING)
+
+# target_link_libraries(hot-potatoe
+#   PRIVATE
+#     "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+#     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+#     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+#     CUDA::nvrtc
+#     CUDA::cublas
+#     CUDA::cusparse
+# )
+
+
+# FindThreads is required for std::thread in the native version
+find_package(Threads REQUIRED)
+
+# 6) Declare the native benchmark executable (raw CUDA/cuBLAS/cuSPARSE)
+add_executable(workload-native main.native.cpp sparse_utils.cpp native_utils.cu)
+
+target_link_libraries(workload-native
+  PRIVATE
+    CUDA::cudart
+    CUDA::cublas
+    CUDA::cusparse
+    Threads::Threads
+)
+
+# 7) Declare the native sorted benchmark executable
+add_executable(workload-native-sorted main.native_sorted.cpp sparse_utils.cpp native_utils.cu)
+
+target_link_libraries(workload-native-sorted
+  PRIVATE
+    CUDA::cudart
+    CUDA::cublas
+    CUDA::cusparse
+    Threads::Threads
+)
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/atoms.hpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/atoms.hpp
new file mode 100644
index 0000000000..ab2a741b6c
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/atoms.hpp
@@ -0,0 +1,51 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include "sparse_utils.hpp"
+
+constexpr int MAX_ITERATIONS = 16000;
+
+CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
+    CAF_ADD_ATOM(workload_test, get_work_atom)
+    CAF_ADD_ATOM(workload_test, release_memory_atom)
+    CAF_ADD_ATOM(workload_test, request_work_atom)
+    CAF_ADD_ATOM(workload_test, worker_done_atom)
+    CAF_ADD_ATOM(workload_test, work_tick_atom)
+    CAF_ADD_ATOM(workload_test, add_work_atom)
+    CAF_ADD_ATOM(workload_test, steal_work_atom)
+    CAF_ADD_ATOM(workload_test, update_stream_atom)
+    CAF_ADD_ATOM(workload_test, shutdown_atom)
+    CAF_ADD_TYPE_ID(workload_test, (SolverType))
+    CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
+    CAF_ADD_TYPE_ID(workload_test, (MatrixData))
+    CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
+    CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
+CAF_END_TYPE_ID_BLOCK(workload_test)
+
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixData)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(SolverType)
+
+enum cg_error_type : int {
+  CG_SUCCESS = 0,
+  CG_MAX_ITER = 1,
+  CG_NAN_INF = 2,
+  CG_STAGNATION = 3,
+  CG_BREAKDOWN = 4,
+  CG_RESIDUAL_FACTOR_FAIL = 5
+};
+
+inline std::string to_string(cg_error_type err) {
+  switch (err) {
+    case CG_SUCCESS: return "Success";
+    case CG_MAX_ITER: return "Maximum Iterations Reached";
+    case CG_NAN_INF: return "Stability Check Failed (NaN/Inf Detected)";
+    case CG_STAGNATION: return "Stagnation Detected (Residual stopped changing)";
+    case CG_BREAKDOWN: return "Solver Breakdown (Division by zero/near-zero)";
+    case CG_RESIDUAL_FACTOR_FAIL: return "Residual Factor Check Failed";
+    default: return "Unknown Error (" + std::to_string(static_cast<int>(err)) + ")";
+  }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/ft_cg_actor.hpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/ft_cg_actor.hpp
new file mode 100644
index 0000000000..60f3ebd23a
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/ft_cg_actor.hpp
@@ -0,0 +1,266 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <string>
+#include <vector>
+#include <memory>
+#include "atoms.hpp"
+#include "sparse_utils.hpp"
+#include "caf/actorSOLVE/actorSOLVE.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+// ---------------------------- FAULT TOLERANT SOLVER ----------------------------
+
+template <class T = float>
+struct ft_cg_state {
+  // Host Data
+  std::string path;
+  in<int> h_row_ptr, h_col_ind;
+  in<T> h_values, h_b;
+  in_out<T> h_x;
+
+  // GPU Buffers
+  mem_ptr<int> A_rp, A_ci, d_err;
+  mem_ptr<T> A_val, b, x, r, p, w, z, D_inv, y_tmp;
+  mem_ptr<char> spmv_ws;
+
+  // Config
+  int n, nnz, max_iter;
+  int iterations = 0;
+  int strikes = 0;
+  T tol;
+  int device_id, stream_id;
+
+  // Supervision & Monitoring
+  caf::actor supervisor;
+  device_ptr d_ptr;
+  program_ptr stab_prog, diag_prog;
+
+  T initial_rho = 0;
+  T current_rho = 0;
+  T old_rho = 0;
+  bool is_jacobi = false;
+  bool initialized = false;
+  std::shared_ptr<MatrixData> pinned_data;
+};
+
+template <class T = float>
+behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
+                                 std::string path,
+                                 std::shared_ptr<MatrixData> data,
+                                 in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+                                 int n, int nnz, T tol, int max_iter,
+                                 int dev_num, int stream, caf::actor supervisor) {
+  auto& s = self->state();
+  s.pinned_data = std::move(data);
+  s.path = std::move(path);
+  s.h_row_ptr = std::move(rp); s.h_col_ind = std::move(ci);
+  s.h_values = std::move(val); s.h_b = std::move(b_in); s.h_x = std::move(x_in);
+  s.n = n; s.nnz = nnz; s.tol = tol; s.max_iter = max_iter;
+  s.device_id = dev_num; s.stream_id = stream; s.supervisor = supervisor;
+
+  return {
+    [=](start_atom) {
+      auto& st = self->state();
+      if (st.initialized) return;
+
+      command_runner<> runner;
+      st.A_rp = runner.transfer_memory(st.device_id, st.stream_id, st.h_row_ptr);
+      st.A_ci = runner.transfer_memory(st.device_id, st.stream_id, st.h_col_ind);
+      st.A_val = runner.transfer_memory(st.device_id, st.stream_id, st.h_values);
+      st.b = runner.transfer_memory(st.device_id, st.stream_id, st.h_b);
+      st.x = runner.transfer_memory(st.device_id, st.stream_id, st.h_x);
+
+      st.d_ptr = platform::create()->schedule(st.stream_id, st.device_id);
+      st.d_ptr->enable_cublas(); st.d_ptr->enable_cusparse();
+
+      auto& mgr = manager::get();
+      // Load stability kernel from file as requested
+      st.stab_prog = mgr.create_program_from_cubin("stability_kernels.cubin", "check_stability", st.d_ptr);
+      st.diag_prog = mgr.create_program_from_cubin("jacobi_kernels.cubin", "extract_diag_inv", st.d_ptr);
+
+      st.r = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.p = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.w = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.z = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.D_inv = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.y_tmp = runner.transfer_memory(st.device_id, st.stream_id, out<T>(1));
+      st.d_err = runner.transfer_memory(st.device_id, st.stream_id, out<int>(1));
+      
+      size_t ws_sz = st.d_ptr->spmv_csr_buffer_size(st.stream_id, st.n, st.n, st.nnz, st.A_rp, st.A_ci, st.A_val, st.x, st.w);
+      if (ws_sz > 0) st.spmv_ws = command_runner<out<char>>{}.transfer_memory(st.device_id, st.stream_id, out<char>{static_cast<int>(ws_sz)});
+
+      st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.x, T{0}, st.w, st.spmv_ws);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.b, st.r);
+      else st.d_ptr->scopy(st.stream_id, st.n, st.b, st.r);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->daxpy(st.stream_id, st.n, -1.0, st.w, st.r);
+      else st.d_ptr->saxpy(st.stream_id, st.n, -1.0f, st.w, st.r);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+      else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+
+      st.initial_rho = runner.copy_to_host(st.y_tmp)[0];
+      st.current_rho = st.initial_rho;
+      st.initialized = true;
+
+      // Notify supervisor that setup is complete and report initial status
+      solver_result_meta meta(st.device_id, st.stream_id, 0, false, CG_SUCCESS);
+      self->mail(gpu_done_atom_v, st.path, actor_cast<actor>(self), std::vector<T>{}, meta).send(st.supervisor);
+    },
+
+    [=](cg_next_step_atom, int num_iters) {
+      auto& st = self->state();
+      command_runner<T> runner;
+      T threshold = st.tol * st.tol;
+      int code = CG_SUCCESS;
+      int step_count = 0;
+
+      // Execute exactly the number of iterations requested by the supervisor
+      while (step_count < num_iters && st.iterations < st.max_iter && st.current_rho > threshold) {
+        // std::cout << "iterations = " << st.iterations << ", current_rho = " << st.current_rho << std::endl;
+        st.iterations++;
+        step_count++;
+
+        if (st.iterations > 1) {
+          T beta = st.current_rho / st.old_rho;
+          if (st.is_jacobi) {
+            if constexpr (std::is_same_v<T, double>) {
+              st.d_ptr->dcopy(st.stream_id, st.n, st.z, st.w);
+              st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
+              st.d_ptr->dcopy(st.stream_id, st.n, st.w, st.p);
+            } else {
+              st.d_ptr->scopy(st.stream_id, st.n, st.z, st.w);
+              st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(beta), st.p, st.w);
+              st.d_ptr->scopy(st.stream_id, st.n, st.w, st.p);
+            }
+          } else {
+            if constexpr (std::is_same_v<T, double>) {
+              st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.w);
+              st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
+              st.d_ptr->dcopy(st.stream_id, st.n, st.w, st.p);
+            } else {
+              st.d_ptr->scopy(st.stream_id, st.n, st.r, st.w);
+              st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(beta), st.p, st.w);
+              st.d_ptr->scopy(st.stream_id, st.n, st.w, st.p);
+            }
+          }
+        } else {
+          if (st.is_jacobi) {
+            if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.z, st.p);
+            else st.d_ptr->scopy(st.stream_id, st.n, st.z, st.p);
+          } else {
+            if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.p);
+            else st.d_ptr->scopy(st.stream_id, st.n, st.r, st.p);
+          }
+        }
+
+        st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.p, T{0}, st.w, st.spmv_ws);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
+        else st.d_ptr->sdot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
+        
+        T dot_pw = runner.copy_to_host(st.y_tmp)[0];
+        if (std::abs(dot_pw) < 1e-25) { 
+          code = CG_BREAKDOWN; 
+          break; 
+        }
+
+        T alpha = st.current_rho / dot_pw;
+        if constexpr (std::is_same_v<T, double>) {
+          st.d_ptr->daxpy(st.stream_id, st.n, alpha, st.p, st.x);
+          st.d_ptr->daxpy(st.stream_id, st.n, -alpha, st.w, st.r);
+        } else {
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(alpha), st.p, st.x);
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(-alpha), st.w, st.r);
+        }
+
+        st.old_rho = st.current_rho;
+        if (st.is_jacobi) {
+          if constexpr (std::is_same_v<T, double>) st.d_ptr->d_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+          else st.d_ptr->s_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+          if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+          else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+        } else {
+          if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+          else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+        }
+        st.current_rho = runner.copy_to_host(st.y_tmp)[0];
+
+      }
+
+      // Run error checks and prepare progress report
+      bool converged = (st.current_rho <= threshold);
+
+      // Check for non-fatal errors that can be retried (Stagnation, Max Iter, Residual Factor)
+      if (code == CG_SUCCESS && !converged && std::abs(st.old_rho - st.current_rho) < 1e-12)
+        code = CG_STAGNATION;
+
+      if (code == CG_SUCCESS) {
+        if (!converged && st.iterations >= st.max_iter) code = CG_MAX_ITER;
+        // Residual decrease check: treat as non-fatal strike if residual didn't decrease significantly
+        if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
+        
+        // If we reached here with CG_SUCCESS, reset strikes as this was a productive batch
+        if (code == CG_SUCCESS) st.strikes = 0;
+      }
+
+      // Reset and launch NaN/Inf stability kernel from file
+      nd_range range(static_cast<int>((st.n + 255) / 256), 1, 1, 256, 1, 1);
+      CHECK_CUDA(cuMemsetD32Async(st.d_err->mem(), 0, 1, st.d_ptr->get_stream_for_actor(st.stream_id)));
+      st.d_ptr->launch_kernel_mem_ref(st.stab_prog->get_kernel(st.d_ptr->getId()), range,
+                                      std::make_tuple(in<int>(st.n), st.x, st.r, st.d_err), st.stream_id);
+
+      int err_flag = runner.copy_to_host(st.d_err)[0];
+      if (err_flag != 0 || std::isnan(st.current_rho) || std::isinf(st.current_rho)) code = CG_NAN_INF;
+
+      // Three strikes policy for non-fatal errors (Stagnation, Max Iterations, Residual Factor Failure)
+      bool is_fatal = (code == CG_NAN_INF || code == CG_BREAKDOWN);
+      if (code != CG_SUCCESS && !is_fatal) {
+        st.strikes++;
+        if (st.strikes < 3) {
+          code = CG_SUCCESS; // Reset code to SUCCESS to allow the supervisor to retry/suspend
+        }
+      }
+
+      if (code != CG_SUCCESS) converged = false;
+
+      // Fallback: if CG failed and we haven't tried Jacobi, transition and retry.
+      if (code != CG_SUCCESS && !st.is_jacobi) {
+        self->println("[INFO] Solver failed in standard mode ({}). Falling back to Jacobi for: {}", to_string(static_cast<cg_error_type>(code)), st.path);
+        st.is_jacobi = true;
+        st.iterations = 0;
+        st.strikes = 0;
+
+        // Setup Jacobi Preconditioning
+        nd_range range_diag((st.n + 255) / 256, 1, 1, 256, 1, 1);
+        st.d_ptr->launch_kernel_mem_ref(st.diag_prog->get_kernel(st.d_ptr->getId()), range_diag,
+                                        std::make_tuple(in<int>(st.n), st.A_rp, st.A_ci, st.A_val, st.D_inv), st.stream_id);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->d_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+        else st.d_ptr->s_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+        else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+        
+        st.current_rho = runner.copy_to_host(st.y_tmp)[0];
+        st.initial_rho = st.current_rho;
+        code = CG_SUCCESS;
+        converged = false;
+      }
+
+      solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
+      
+      // Report current solution and metadata to the supervisor
+      runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor, path = st.path, self_h = actor_cast<actor>(self)](std::vector<T> sol) {
+        anon_mail(gpu_done_atom_v, path, self_h, std::move(sol), meta).send(supervisor);
+        if (converged || code != CG_SUCCESS)
+          anon_mail(shutdown_atom_v).send(self_h);
+      });
+    },
+    [=](update_stream_atom, int new_stream) {
+      self->state().stream_id = new_stream;
+    },
+    [=](shutdown_atom) {
+      self->quit();
+    }
+  };
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/ft_cg_jacobi_actor.hpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/ft_cg_jacobi_actor.hpp
new file mode 100644
index 0000000000..7e42785bff
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/ft_cg_jacobi_actor.hpp
@@ -0,0 +1,191 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <string>
+#include <vector>
+#include <memory>
+#include "atoms.hpp"
+#include "sparse_utils.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+template <class T = float>
+struct ft_cg_jacobi_state {
+  // Host Data
+  std::string path;
+  in<int> h_row_ptr, h_col_ind;
+  in<T> h_values, h_b;
+  in_out<T> h_x;
+
+  // GPU Buffers
+  mem_ptr<int> A_rp, A_ci, d_err;
+  mem_ptr<T> A_val, b, x, r, p, w, z, D_inv, y_tmp;
+  mem_ptr<char> spmv_ws;
+
+  // Config
+  int n, nnz, max_iter;
+  int iterations = 0;
+  int strikes = 0;
+  T tol;
+  int device_id, stream_id;
+
+  // Supervision & Monitoring
+  caf::actor supervisor;
+  device_ptr d_ptr;
+  program_ptr stab_prog;
+  program_ptr diag_prog;
+
+  T initial_rho = 0;
+  T current_rho = 0;
+  T old_rho = 0;
+  bool initialized = false;
+  std::shared_ptr<MatrixData> pinned_data;
+};
+
+template <class T = float>
+behavior fault_tolerant_cg_jacobi_actor(stateful_actor<ft_cg_jacobi_state<T>>* self,
+                                        std::string path,
+                                        std::shared_ptr<MatrixData> data,
+                                        in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+                                        int n, int nnz, T tol, int max_iter,
+                                        int dev_num, int stream, caf::actor supervisor) {
+  auto& s = self->state();
+  s.pinned_data = std::move(data);
+  s.path = std::move(path);
+  s.h_row_ptr = std::move(rp); s.h_col_ind = std::move(ci);
+  s.h_values = std::move(val); s.h_b = std::move(b_in); s.h_x = std::move(x_in);
+  s.n = n; s.nnz = nnz; s.tol = tol; s.max_iter = max_iter;
+  s.device_id = dev_num; s.stream_id = stream; s.supervisor = supervisor;
+
+  return {
+    [=](start_atom) {
+      auto& st = self->state();
+      if (st.initialized) return;
+
+      command_runner<> runner;
+      st.A_rp = runner.transfer_memory(st.device_id, st.stream_id, st.h_row_ptr);
+      st.A_ci = runner.transfer_memory(st.device_id, st.stream_id, st.h_col_ind);
+      st.A_val = runner.transfer_memory(st.device_id, st.stream_id, st.h_values);
+      st.b = runner.transfer_memory(st.device_id, st.stream_id, st.h_b);
+      st.x = runner.transfer_memory(st.device_id, st.stream_id, st.h_x);
+
+      st.d_ptr = platform::create()->schedule(st.stream_id, st.device_id);
+      st.d_ptr->enable_cublas(); st.d_ptr->enable_cusparse();
+
+      auto& mgr = manager::get();
+      st.stab_prog = mgr.create_program_from_cubin("stability_kernels.cubin", "check_stability", st.d_ptr);
+      st.diag_prog = mgr.create_program_from_cubin("jacobi_kernels.cubin", "extract_diag_inv", st.d_ptr);
+
+      st.r = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.p = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.w = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.z = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.D_inv = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.y_tmp = runner.transfer_memory(st.device_id, st.stream_id, out<T>(1));
+      st.d_err = runner.transfer_memory(st.device_id, st.stream_id, out<int>(1));
+      
+      size_t ws_sz = st.d_ptr->spmv_csr_buffer_size(st.stream_id, st.n, st.n, st.nnz, st.A_rp, st.A_ci, st.A_val, st.x, st.w);
+      if (ws_sz > 0) st.spmv_ws = command_runner<out<char>>{}.transfer_memory(st.device_id, st.stream_id, out<char>{static_cast<int>(ws_sz)});
+
+      // Jacobi setup: Extract D_inv
+      int threads = 256;
+      nd_range range((st.n + threads - 1) / threads, 1, 1, threads, 1, 1);
+      st.d_ptr->launch_kernel_mem_ref(st.diag_prog->get_kernel(st.d_ptr->getId()), range,
+                                      std::make_tuple(in<int>(st.n), st.A_rp, st.A_ci, st.A_val, st.D_inv), st.stream_id);
+
+      // Initial r = b - Ax
+      st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.x, T{0}, st.w, st.spmv_ws);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.b, st.r); else st.d_ptr->scopy(st.stream_id, st.n, st.b, st.r);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->daxpy(st.stream_id, st.n, -1.0, st.w, st.r); else st.d_ptr->saxpy(st.stream_id, st.n, -1.0f, st.w, st.r);
+
+      // Initial z = D_inv * r
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->d_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+      else st.d_ptr->s_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+
+      // Initial rho = r * z
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.z, st.y_tmp); else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+
+      st.initial_rho = runner.copy_to_host(st.y_tmp)[0];
+      st.current_rho = st.initial_rho;
+      st.initialized = true;
+
+      solver_result_meta meta(st.device_id, st.stream_id, 0, false, CG_SUCCESS);
+      self->mail(gpu_done_atom_v, st.path, actor_cast<actor>(self), std::vector<T>{}, meta).send(st.supervisor);
+    },
+
+    [=](cg_next_step_atom, int num_iters) {
+      auto& st = self->state();
+      command_runner<T> runner;
+      T threshold = st.tol * st.tol;
+      int code = CG_SUCCESS;
+      int step_count = 0;
+
+      while (step_count < num_iters && st.iterations < st.max_iter && st.current_rho > threshold) {
+        st.iterations++;
+        step_count++;
+
+        if (st.iterations > 1) {
+          T beta = st.current_rho / st.old_rho;
+          if constexpr (std::is_same_v<T, double>) {
+            st.d_ptr->dcopy(st.stream_id, st.n, st.z, st.w);
+            st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
+            st.d_ptr->dcopy(st.stream_id, st.n, st.w, st.p);
+          } else {
+            st.d_ptr->scopy(st.stream_id, st.n, st.z, st.w);
+            st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(beta), st.p, st.w);
+            st.d_ptr->scopy(st.stream_id, st.n, st.w, st.p);
+          }
+        } else {
+          if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.z, st.p); else st.d_ptr->scopy(st.stream_id, st.n, st.z, st.p);
+        }
+
+        st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.p, T{0}, st.w, st.spmv_ws);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.p, st.w, st.y_tmp); else st.d_ptr->sdot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
+        
+        T dot_pw = runner.copy_to_host(st.y_tmp)[0];
+        if (std::abs(dot_pw) < 1e-25) { code = CG_BREAKDOWN; break; }
+
+        T alpha = st.current_rho / dot_pw;
+        if constexpr (std::is_same_v<T, double>) {
+          st.d_ptr->daxpy(st.stream_id, st.n, alpha, st.p, st.x);
+          st.d_ptr->daxpy(st.stream_id, st.n, -alpha, st.w, st.r);
+        } else {
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(alpha), st.p, st.x);
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(-alpha), st.w, st.r);
+        }
+
+        st.old_rho = st.current_rho;
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->d_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+        else st.d_ptr->s_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.z, st.y_tmp); else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+        st.current_rho = runner.copy_to_host(st.y_tmp)[0];
+      }
+
+      bool converged = (st.current_rho <= threshold);
+      if (code == CG_SUCCESS && !converged && std::abs(st.old_rho - st.current_rho) < 1e-12) code = CG_STAGNATION;
+      if (code == CG_SUCCESS) {
+        if (!converged && st.iterations >= st.max_iter) code = CG_MAX_ITER;
+        if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
+        if (code == CG_SUCCESS) st.strikes = 0;
+      }
+      nd_range range(static_cast<int>((st.n + 255) / 256), 1, 1, 256, 1, 1);
+      CHECK_CUDA(cuMemsetD32Async(st.d_err->mem(), 0, 1, st.d_ptr->get_stream_for_actor(st.stream_id)));
+      st.d_ptr->launch_kernel_mem_ref(st.stab_prog->get_kernel(st.d_ptr->getId()), range,
+                                      std::make_tuple(in<int>(st.n), st.x, st.r, st.d_err), st.stream_id);
+      int err_flag = runner.copy_to_host(st.d_err)[0];
+      if (err_flag != 0 || std::isnan(st.current_rho) || std::isinf(st.current_rho)) code = CG_NAN_INF;
+      bool is_fatal = (code == CG_NAN_INF || code == CG_BREAKDOWN);
+      if (code != CG_SUCCESS && !is_fatal) { st.strikes++; if (st.strikes < 3) code = CG_SUCCESS; }
+      if (code != CG_SUCCESS) converged = false;
+      solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
+      runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor, path = st.path, self_h = actor_cast<actor>(self)](std::vector<T> sol) {
+        anon_mail(gpu_done_atom_v, path, self_h, std::move(sol), meta).send(supervisor);
+        if (converged || code != CG_SUCCESS) anon_mail(shutdown_atom_v).send(self_h);
+      });
+    },
+    [=](update_stream_atom, int new_stream) { self->state().stream_id = new_stream; },
+    [=](shutdown_atom) { self->quit(); }
+  };
+}
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/jacobi_kernels.cu b/libcaf_cuda/sc26/Irregular-Workload/workloadA/jacobi_kernels.cu
new file mode 100644
index 0000000000..16c17e7352
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/jacobi_kernels.cu
@@ -0,0 +1,15 @@
+extern "C" __global__
+void extract_diag_inv(int n, const int* row_ptr, const int* col_ind, const float* val, float* d_inv) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        float d = 0.0f;
+        // Search for the diagonal element (A[i][i]) in the sparse row
+        for (int j = row_ptr[i]; j < row_ptr[i+1]; j++) {
+            if (col_ind[j] == i) {
+                d = val[j];
+                break;
+            }
+        }
+        d_inv[i] = (d != 0.0f) ? 1.0f / d : 1.0f;
+    }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native.cpp
new file mode 100644
index 0000000000..b6486ee944
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native.cpp
@@ -0,0 +1,62 @@
+#include <iostream>
+#include <vector>
+#include <string>
+#include <chrono>
+#include <thread>
+#include <filesystem>
+#include <algorithm>
+#include <atomic>
+#include "native_utils.hpp"
+
+void producer(ThreadSafeQueue<MatrixTask>& queue, std::vector<MatrixTask> matrix_pool) {
+    for (auto& task : matrix_pool) {
+        task.enqueue_time = std::chrono::steady_clock::now();
+        queue.push(task);
+    }
+    queue.signal_shutdown();
+}
+
+int main(int argc, char** argv)
+{
+    constexpr uint32_t WORKLOAD_SEED = 42;
+    int num_streams = 4;
+    // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
+
+    std::cout << "[INFO] Loading matrices...\n";
+    //std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrices/workloadA", CGS_SOLVER);
+  
+    
+    if (matrix_pool.empty()) {
+        std::cerr << "No matrices found.\n";
+        return 1;
+    }
+
+    int num_gpus = 0;
+    CHECK_CUDA(cudaGetDeviceCount(&num_gpus));
+    
+    std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
+    std::cout << "[INFO] Matrix pool size: " << matrix_pool.size() << "\n";
+    std::cout << "[INFO] Streams/GPU: " << num_streams << "\n";
+
+    std::atomic<int> tasks_succeeded{0};
+    std::atomic<int> tasks_failed{0};
+    ThreadSafeQueue<MatrixTask> work_queue;
+    
+    init_benchmark_timer();
+    std::thread producer_thread(producer, std::ref(work_queue), matrix_pool);
+    
+    std::vector<std::thread> workers;
+    for (int gpu = 0; gpu < num_gpus; ++gpu) {
+        for (int stream = 0; stream < num_streams; ++stream) {
+            workers.emplace_back(gpu_stream_worker, gpu, gpu * num_streams + stream, std::ref(work_queue), std::ref(tasks_succeeded), std::ref(tasks_failed));
+        }
+    }
+
+    producer_thread.join();
+    for (auto& worker : workers) worker.join();
+
+    report_workload_stats();
+
+    return 0;
+}
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native_sorted.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native_sorted.cpp
new file mode 100644
index 0000000000..ca1ff57196
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native_sorted.cpp
@@ -0,0 +1,64 @@
+#include <iostream>
+#include <vector>
+#include <string>
+#include <chrono>
+#include <thread>
+#include <filesystem>
+#include <algorithm>
+#include <atomic>
+#include "native_utils.hpp"
+
+void producer(ThreadSafeQueue<MatrixTask>& queue, std::vector<MatrixTask> matrix_pool) {
+    // Order tasks from lowest NNZ to highest NNZ
+    std::sort(matrix_pool.begin(), matrix_pool.end(), [](const MatrixTask& a, const MatrixTask& b) {
+        return a.data->nnz < b.data->nnz;
+    });
+
+    for (auto& task : matrix_pool) {
+        task.enqueue_time = std::chrono::steady_clock::now();
+        queue.push(task);
+    }
+    queue.signal_shutdown();
+}
+
+int main(int argc, char** argv) {
+    constexpr uint32_t WORKLOAD_SEED = 42;
+    int num_streams = 4;
+    // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
+
+    std::cout << "[INFO] Loading matrices...\n";
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrices/workloadA", CGS_SOLVER);
+
+
+    if (matrix_pool.empty()) {
+        std::cerr << "No matrices found.\n";
+        return 1;
+    }
+
+    int num_gpus = 0;
+    CHECK_CUDA(cudaGetDeviceCount(&num_gpus));
+
+    std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
+    std::cout << "[INFO] Matrix pool size: " << matrix_pool.size() << "\n";
+    std::cout << "[INFO] Streams/GPU: " << num_streams << "\n";
+
+    std::atomic<int> tasks_succeeded{0};
+    std::atomic<int> tasks_failed{0};
+    ThreadSafeQueue<MatrixTask> work_queue;
+    
+    init_benchmark_timer();
+    std::thread producer_thread(producer, std::ref(work_queue), matrix_pool);
+    
+    std::vector<std::thread> workers;
+    for (int gpu = 0; gpu < num_gpus; ++gpu) {
+        for (int stream = 0; stream < num_streams; ++stream) {
+            workers.emplace_back(gpu_stream_worker, gpu, gpu * num_streams + stream, std::ref(work_queue), std::ref(tasks_succeeded), std::ref(tasks_failed));
+        }
+    }
+    producer_thread.join();
+    for (auto& worker : workers) worker.join();
+
+    report_workload_stats();
+
+    return 0;
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.test.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.test.cpp
new file mode 100644
index 0000000000..5fd9f46a83
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.test.cpp
@@ -0,0 +1,81 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <deque>
+#include <string>
+#include <chrono>
+#include <numeric>
+#include <cmath>
+#include <algorithm>
+#include <filesystem>
+#include <memory>
+#include "sparse_utils.hpp"
+#include "atoms.hpp"
+#include "ft_cg_actor.hpp"
+#include "supervisor_actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+
+constexpr uint32_t WORKLOAD_SEED = 42;
+void caf_main(actor_system& sys) {
+    manager::init(sys, manager_config(true, true));
+    std::cout << "[INFO] Loading matrices...\n";
+    {
+         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+        //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/unsymmetric", CGS_SOLVER);
+         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric", CGS_SOLVER);
+         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+
+         auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrices/workloadA", CGS_SOLVER);
+
+
+        int num_gpus = manager::get().get_num_devices();
+        std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
+        std::cout << "[INFO] Matrix pool size: " << tasks_vec.size() << "\n";
+
+        if (tasks_vec.empty()) {
+            std::cerr << "No matrices found. Running dummy test task." << std::endl;
+            auto data = std::make_shared<MatrixData>();
+            data->row_ptr = {0, 1, 2};
+            data->col_indices = {0, 1};
+            data->values = {10.0f, 10.0f};
+            data->b = {100.0f, 100.0f};
+            data->x_guess = {0.0f, 0.0f};
+            tasks_vec.push_back({"dummy_task", CGS_SOLVER, data});
+        }
+
+        init_benchmark_timer();
+        for (auto& task : tasks_vec) {
+            task.enqueue_time = std::chrono::steady_clock::now();
+        }
+
+        // Workload partitioning logic (Timed)
+        auto part_start = std::chrono::steady_clock::now();
+        auto partitions = make_contiguous_partitions(tasks_vec.size(), num_gpus, 1);
+        
+        std::vector<std::vector<MatrixTask>> partitioned_workloads(num_gpus);
+        for (int i = 0; i < num_gpus; ++i) {
+            auto& p = partitions[i];
+            partitioned_workloads[i].reserve(p.end - p.begin);
+            for (size_t j = p.begin; j < p.end; ++j) {
+                partitioned_workloads[i].push_back(std::move(tasks_vec[j]));
+            }
+        }
+        auto part_end = std::chrono::steady_clock::now();
+        auto part_ms = std::chrono::duration_cast<std::chrono::milliseconds>(part_end - part_start).count();
+        std::cout << "[INFO] Workload partitioning completed in " << part_ms << " ms\n";
+
+        auto benchmark_start = std::chrono::steady_clock::now();
+        for (int i = 0; i < num_gpus; ++i) {
+            sys.spawn(supervisor_actor, std::move(partitioned_workloads[i]), 4, benchmark_start, i);
+        }
+
+        sys.await_all_actors_done();
+    }
+    manager::shutdown();
+}
+CAF_MAIN(id_block::cuda, id_block::workload_test)
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/native_utils.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/native_utils.cpp
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/native_utils.cu b/libcaf_cuda/sc26/Irregular-Workload/workloadA/native_utils.cu
new file mode 100644
index 0000000000..c3a110bed6
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/native_utils.cu
@@ -0,0 +1,353 @@
+#include "native_utils.hpp"
+#include <cmath>
+#include <chrono>
+
+// ============================================================
+// PCG Kernels
+// ============================================================
+
+enum cg_error_type {
+    CG_SUCCESS = 0,
+    CG_BREAKDOWN = 1,
+    CG_STAGNATION = 2,
+    CG_MAX_ITER = 3,
+    CG_RESIDUAL_FACTOR_FAIL = 4,
+    CG_NAN_INF = 5,
+    CG_JACOBI_RETRY = 6
+};
+
+const char* get_cg_error_string(int code) {
+    switch (code) {
+        case CG_SUCCESS:
+            return "CG_SUCCESS";
+        case CG_BREAKDOWN:
+            return "CG_BREAKDOWN";
+        case CG_STAGNATION:
+            return "CG_STAGNATION";
+        case CG_MAX_ITER:
+            return "CG_MAX_ITER";
+        case CG_RESIDUAL_FACTOR_FAIL:
+            return "CG_RESIDUAL_FACTOR_FAIL";
+        case CG_NAN_INF:
+            return "CG_NAN_INF";
+        case CG_JACOBI_RETRY:
+            return "CG_JACOBI_RETRY";
+        default:
+            return "UNKNOWN_ERROR";
+    }
+}
+__global__ void check_stability_kernel(int n, const float* x, const float* r, int* d_err) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        if (isnan(x[i]) || isinf(x[i]) || isnan(r[i]) || isinf(r[i])) {
+            atomicExch(d_err, 1);
+        }
+    }
+}
+
+__global__ void extract_diag_inv_kernel(int n, const int* row_ptr, const int* col_ind, const float* values, float* d_inv) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        float diag = 1.0f;
+        for (int j = row_ptr[i]; j < row_ptr[i + 1]; j++) {
+            if (col_ind[j] == i) {
+                diag = values[j];
+                break;
+            }
+        }
+        d_inv[i] = (fabsf(diag) > 1e-20f) ? 1.0f / diag : 1.0f;
+    }
+}
+
+__global__ void elementwise_mul_kernel(int n, const float* a, const float* b, float* c) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) c[i] = a[i] * b[i];
+}
+
+int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
+                           int n, int nnz, float* d_val, int* d_row_ptr, int* d_col_ind,
+                           float* d_b, float* d_x, float* d_r, float* d_p, float* d_Ap,
+                           float* d_z, float* d_Dinv, int* d_err, cudaStream_t stream, int& out_code) {
+    float alpha = 1.0f, beta = 0.0f, rho = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
+    float tolerance = 1e-5f;
+    int max_iters = 16000;
+    out_code = CG_SUCCESS;
+
+    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
+
+    CHECK_CUBLAS(cublasSetStream(cublas, stream));
+    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
+
+    // Extract Diagonal Inverse
+    int threads = 256;
+    int blocks = (n + threads - 1) / threads;
+    extract_diag_inv_kernel<<<blocks, threads, 0, stream>>>(n, d_row_ptr, d_col_ind, d_val, d_Dinv);
+
+    cusparseSpMatDescr_t matA;
+    cusparseDnVecDescr_t vecP, vecAp;
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, nnz, d_row_ptr, d_col_ind, d_val,
+                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                     CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecAp, n, d_Ap, CUDA_R_32F));
+
+    size_t bufferSize = 0;
+    void* d_buffer = nullptr;
+    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, vecP, &beta, vecAp,
+                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
+
+    // r = b (assuming x=0), z = M^-1 * r, p = z
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
+    elementwise_mul_kernel<<<blocks, threads, 0, stream>>>(n, d_Dinv, d_r, d_z);
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_z, 1, d_p, 1));
+    CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_z, 1, &rho));
+
+    float initial_rho = rho;
+    int k = 0; 
+    float r_norm_sq = 0.0f;
+    while (k < max_iters) {
+        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                    &alpha, matA, vecP, &beta, vecAp,
+                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
+        float pAp, old_rho;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
+        
+        if (std::abs(pAp) < 1e-25f) {
+            out_code = CG_BREAKDOWN;
+            break;
+        }
+
+        a = rho / pAp;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
+        na = -a;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &na, d_Ap, 1, d_r, 1));
+        
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r_norm_sq));
+        if (std::sqrt(r_norm_sq) < tolerance) break;
+
+        elementwise_mul_kernel<<<blocks, threads, 0, stream>>>(n, d_Dinv, d_r, d_z);
+        old_rho = rho; 
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_z, 1, &rho));
+
+        b = rho / old_rho;
+        
+        // p = z + beta * p
+        CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_z, 1, d_p, 1));
+        k++;
+    }
+
+    bool converged = (std::sqrt(r_norm_sq) < tolerance);
+
+    if (out_code == CG_SUCCESS) {
+        if (!converged) {
+            if (k >= max_iters) out_code = CG_MAX_ITER;
+            // Stagnation check: did the total progress over the entire solve stall?
+            else if (std::abs(initial_rho - rho) < 1e-14f) out_code = CG_STAGNATION;
+            // Residual decrease check: did it fail to drop by at least 0.01%?
+            else if (initial_rho > 0 && (rho / initial_rho) > 0.9999f) out_code = CG_RESIDUAL_FACTOR_FAIL;
+        }
+    }
+
+    // Stability check
+    CHECK_CUDA(cudaMemsetAsync(d_err, 0, sizeof(int), stream));
+    check_stability_kernel<<<blocks, threads, 0, stream>>>(n, d_x, d_r, d_err);
+    int h_err = 0;
+    CHECK_CUDA(cudaMemcpyAsync(&h_err, d_err, sizeof(int), cudaMemcpyDeviceToHost, stream));
+    CHECK_CUDA(cudaStreamSynchronize(stream));
+    if (h_err != 0 || std::isnan(rho) || std::isinf(rho)) out_code = CG_NAN_INF;
+
+    CHECK_CUSPARSE(cusparseDestroySpMat(matA));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
+    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
+    return k;
+}
+
+int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
+                    int n, int nnz, float* d_val, int* d_row_ptr, int* d_col_ind,
+                    float* d_b, float* d_x, float* d_r, float* d_p, float* d_Ap, 
+                    int* d_err, cudaStream_t stream, int& out_code) {
+    float alpha = 1.0f, beta = 0.0f, r1 = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
+    float tolerance = 1e-5f;
+    int max_iters = 16000;
+    out_code = CG_SUCCESS;
+
+    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
+
+    CHECK_CUBLAS(cublasSetStream(cublas, stream));
+    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
+
+    cusparseSpMatDescr_t matA;
+    cusparseDnVecDescr_t vecX, vecP, vecAp;
+
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, nnz, d_row_ptr, d_col_ind, d_val,
+                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                     CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecAp, n, d_Ap, CUDA_R_32F));
+
+    size_t bufferSize = 0;
+    void* d_buffer = nullptr;
+    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, vecP, &beta, vecAp,
+                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
+
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_p, 1));
+    CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
+
+    float initial_rho = r1;
+    int k = 0; 
+    while (k < max_iters) {
+        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                    &alpha, matA, vecP, &beta, vecAp,
+                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
+        float pAp, r0;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
+        
+        if (std::abs(pAp) < 1e-25f) {
+            out_code = CG_BREAKDOWN;
+            break;
+        }
+
+        a = r1 / pAp;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
+        na = -a;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &na, d_Ap, 1, d_r, 1));
+        r0 = r1;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
+        if (std::sqrt(r1) < tolerance) break;
+        
+        b = r1 / r0;
+        CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_r, 1, d_p, 1));
+        k++;
+    }
+
+    bool converged = (std::sqrt(r1) < tolerance);
+
+    if (out_code == CG_SUCCESS) {
+        if (!converged) {
+            if (k >= max_iters) out_code = CG_MAX_ITER;
+            else if (std::abs(initial_rho - r1) < 1e-12f) out_code = CG_STAGNATION;
+            else if (initial_rho > 0 && (r1 / initial_rho) > 0.9999f) out_code = CG_RESIDUAL_FACTOR_FAIL;
+        }
+    }
+
+    // Stability check
+    CHECK_CUDA(cudaMemsetAsync(d_err, 0, sizeof(int), stream));
+    int threads = 256;
+    check_stability_kernel<<<(n + threads - 1) / threads, threads, 0, stream>>>(n, d_x, d_r, d_err);
+    int h_err = 0;
+    CHECK_CUDA(cudaMemcpyAsync(&h_err, d_err, sizeof(int), cudaMemcpyDeviceToHost, stream));
+    CHECK_CUDA(cudaStreamSynchronize(stream));
+    if (h_err != 0 || std::isnan(r1) || std::isinf(r1)) out_code = CG_NAN_INF;
+
+    CHECK_CUSPARSE(cusparseDestroySpMat(matA));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecX));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
+    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
+    return k;
+}
+
+void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue,
+                       std::atomic<int>& succeeded, std::atomic<int>& failed) {
+    CHECK_CUDA(cudaSetDevice(device_id));
+    cudaStream_t stream;
+    cublasHandle_t cublas;
+    cusparseHandle_t cusparse;
+    CHECK_CUDA(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
+    CHECK_CUBLAS(cublasCreate(&cublas));
+    CHECK_CUSPARSE(cusparseCreate(&cusparse));
+
+    MatrixTask task;
+    while (queue.wait_pop(task)) {
+        auto pick_time = std::chrono::steady_clock::now();
+        std::cout << "[WORKER " << worker_id << "] Starting: " << task.path << " (NNZ: " << task.data->nnz << ")" << std::endl;
+
+        int n = (int)task.data->row_ptr.size() - 1;
+        int nnz = task.data->nnz;
+        float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b, *d_z, *d_Dinv;
+        int *d_row_ptr, *d_col_ind, *d_err;
+
+        // Allocate GPU memory once per task
+        CHECK_CUDA(cudaMallocAsync(&d_val, nnz * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_col_ind, nnz * sizeof(int), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_z, n * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_Dinv, n * sizeof(float), stream));
+        CHECK_CUDA(cudaMallocAsync(&d_err, sizeof(int), stream));
+
+        // Initial Transfer
+        CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
+        CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
+        CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
+        CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
+
+        int iterations = 0;
+        int code = CG_SUCCESS;
+        int strikes = 0;
+
+        // Try standard CG with the 3-strikes policy for non-fatal errors
+        // do {
+            iterations = solve_cg_async(cublas, cusparse, n, nnz, d_val, d_row_ptr, d_col_ind, d_b, d_x, d_r, d_p, d_Ap, d_err, stream, code);
+            // if (code == CG_SUCCESS) break;
+            
+            bool is_fatal = (code == CG_NAN_INF || code == CG_BREAKDOWN);
+            // if (is_fatal) break;
+
+            strikes++;
+        // } while (strikes < 3);
+
+        bool success = (code == CG_SUCCESS);
+
+        // Fallback mechanism: If standard CG fails to converge, retry using the Jacobi Preconditioner
+        if (!success) {
+            std::cout << "[WORKER " << worker_id << "] CG failed with error: " << get_cg_error_string(code)
+                      << ". Falling back to Jacobi solver for: " << task.path << std::endl;
+            iterations = solve_pcg_jacobi_async(cublas, cusparse, n, nnz, d_val, d_row_ptr, d_col_ind, d_b, d_x, d_r, d_p, d_Ap, d_z, d_Dinv, d_err, stream, code);
+            success = (code == CG_SUCCESS);
+        }
+
+        // Clean up task memory
+        CHECK_CUDA(cudaFreeAsync(d_val, stream));
+        CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
+        CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
+        CHECK_CUDA(cudaFreeAsync(d_x, stream));
+        CHECK_CUDA(cudaFreeAsync(d_r, stream));
+        CHECK_CUDA(cudaFreeAsync(d_p, stream));
+        CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
+        CHECK_CUDA(cudaFreeAsync(d_b, stream));
+        CHECK_CUDA(cudaFreeAsync(d_z, stream));
+        CHECK_CUDA(cudaFreeAsync(d_Dinv, stream));
+        CHECK_CUDA(cudaFreeAsync(d_err, stream));
+
+        auto finish_time = std::chrono::steady_clock::now();
+        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish_time - pick_time).count();
+
+        if (success) {
+            succeeded++;
+        } else {
+            failed++;
+        }
+
+        record_job(task.path, task.enqueue_time, pick_time, finish_time, iterations, success);
+
+        std::cout << "[WORKER " << worker_id << "] Done: " << task.path << " (" << iterations << " iters, " << duration << " ms) ["
+                  << (success ? "SUCCESS" : "FAILED") << "]." << std::endl;
+    }
+    CHECK_CUBLAS(cublasDestroy(cublas));
+    CHECK_CUSPARSE(cusparseDestroy(cusparse));
+    CHECK_CUDA(cudaStreamDestroy(stream));
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/native_utils.hpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/native_utils.hpp
new file mode 100644
index 0000000000..b9de530986
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/native_utils.hpp
@@ -0,0 +1,95 @@
+#pragma once
+
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+#include <cusparse.h>
+#include <iostream>
+#include <queue>
+#include <mutex>
+#include <condition_variable>
+#include <atomic>
+#include "sparse_utils.hpp"
+
+constexpr int MAX_ITERATIONS = 16000;
+
+// ============================================================
+// Error Checking Macros
+// ============================================================
+
+#define CHECK_CUDA(call)                                                   \
+    do {                                                                   \
+        cudaError_t status = call;                                         \
+        if (status != cudaSuccess) {                                       \
+            std::cerr << "CUDA Error: " << cudaGetErrorString(status)      \
+                      << " at " << __FILE__ << ":" << __LINE__ << std::endl; \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+#define CHECK_CUBLAS(call)                                                 \
+    do {                                                                   \
+        cublasStatus_t status = call;                                      \
+        if (status != CUBLAS_STATUS_SUCCESS) {                             \
+            std::cerr << "cuBLAS Error at "                               \
+                      << __FILE__ << ":" << __LINE__ << std::endl;         \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+#define CHECK_CUSPARSE(call)                                               \
+    do {                                                                   \
+        cusparseStatus_t status = call;                                    \
+        if (status != CUSPARSE_STATUS_SUCCESS) {                           \
+            std::cerr << "cuSPARSE Error at "                             \
+                      << __FILE__ << ":" << __LINE__ << std::endl;         \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+// ============================================================
+// Thread Safe Queue
+// ============================================================
+
+template<typename T>
+class ThreadSafeQueue {
+public:
+    void push(T item) {
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            queue_.push(std::move(item));
+        }
+        cv_.notify_one();
+    }
+
+    bool wait_pop(T& item) {
+        std::unique_lock<std::mutex> lock(mutex_);
+        cv_.wait(lock, [&] {
+            return shutdown_ || !queue_.empty();
+        });
+
+        if (!queue_.empty()) {
+            item = std::move(queue_.front());
+            queue_.pop();
+            return true;
+        }
+        return false;
+    }
+
+    void signal_shutdown() {
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            shutdown_ = true;
+        }
+        cv_.notify_all();
+    }
+
+private:
+    std::queue<T> queue_;
+    std::mutex mutex_;
+    std::condition_variable cv_;
+    bool shutdown_ = false;
+};
+
+int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream);
+int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream);
+void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue, std::atomic<int>& succeeded, std::atomic<int>& failed);
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/sparse_utils.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/sparse_utils.cpp
new file mode 100644
index 0000000000..dd09e63fc3
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/sparse_utils.cpp
@@ -0,0 +1,267 @@
+#include "sparse_utils.hpp"
+#include <fstream>
+#include <stdexcept>
+#include <filesystem>
+#include <random>
+#include <mutex>
+#include <algorithm>
+#include <iomanip>
+#include <iostream>
+#include <numeric>
+
+namespace fs = std::filesystem;
+
+SparseMatrixCOO load_binary_coo(const std::string& filepath) {
+    std::ifstream file(filepath, std::ios::binary);
+    if (!file) {
+        throw std::runtime_error("Failed to open matrix file: " + filepath);
+    }
+
+    SparseMatrixCOO coo;
+    
+    file.read(reinterpret_cast<char*>(&coo.rows), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&coo.cols), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&coo.nnz), sizeof(int32_t));
+
+    coo.row_indices.resize(coo.nnz);
+    coo.col_indices.resize(coo.nnz);
+    coo.values.resize(coo.nnz);
+
+    file.read(reinterpret_cast<char*>(coo.row_indices.data()), coo.nnz * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(coo.col_indices.data()), coo.nnz * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(coo.values.data()), coo.nnz * sizeof(float));
+
+    return coo;
+}
+
+SparseMatrixCSR convert_coo_to_csr(const SparseMatrixCOO& coo) {
+    SparseMatrixCSR csr;
+    csr.rows = coo.rows;
+    csr.cols = coo.cols;
+    csr.nnz = coo.nnz;
+    
+    csr.row_ptr.assign(csr.rows + 1, 0);
+    csr.col_indices.resize(csr.nnz);
+    csr.values.resize(csr.nnz);
+
+    for (int32_t i = 0; i < coo.nnz; ++i) {
+        csr.row_ptr[coo.row_indices[i] + 1]++;
+    }
+
+    for (int32_t i = 0; i < csr.rows; ++i) {
+        csr.row_ptr[i + 1] += csr.row_ptr[i];
+    }
+
+    std::vector<int32_t> current_row_pos = csr.row_ptr;
+
+    for (int32_t i = 0; i < coo.nnz; ++i) {
+        int32_t row = coo.row_indices[i];
+        int32_t dest_pos = current_row_pos[row]++;
+        csr.col_indices[dest_pos] = coo.col_indices[i];
+        csr.values[dest_pos] = coo.values[i];
+    }
+
+    return csr;
+}
+
+std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x) {
+    std::vector<float> b(A.rows, 0.0f);
+    
+    for (int32_t i = 0; i < A.rows; ++i) {
+        float sum = 0.0f;
+        int32_t row_start = A.row_ptr[i];
+        int32_t row_end = A.row_ptr[i + 1];
+        
+        for (int32_t j = row_start; j < row_end; ++j) {
+            sum += A.values[j] * x[A.col_indices[j]];
+        }
+        b[i] = sum;
+    }
+    return b;
+}
+
+std::vector<MatrixTask> scan_for_matrices(const std::string& dir, SolverType type) {
+    std::vector<MatrixTask> tasks;
+    if (!fs::exists(dir)) return tasks;
+    for (const auto& entry : fs::directory_iterator(dir)) {
+        if (entry.path().extension() == ".bin") {
+            auto coo = load_binary_coo(entry.path().string());
+            auto csr = convert_coo_to_csr(coo);
+            auto data = std::make_shared<MatrixData>();
+            data->rows = csr.rows;
+            data->cols = csr.cols;
+            data->nnz = csr.nnz;
+            data->b = compute_rhs_spmv(csr, std::vector<float>(csr.cols, 1.0f));
+            data->row_ptr = std::move(csr.row_ptr);
+            data->col_indices = std::move(csr.col_indices);
+            data->values = std::move(csr.values);
+            data->x_guess.assign(data->cols, 0.0f);
+            
+            tasks.push_back({entry.path().string(), type, data});
+        }
+    }
+    return tasks;
+}
+
+
+std::vector<Partition>
+make_contiguous_partitions(size_t num_tasks, size_t rows, size_t cols)
+{
+    size_t num_parts = rows * cols;
+
+    std::vector<Partition> parts;
+    parts.reserve(num_parts);
+
+    size_t base = num_tasks / num_parts;
+    size_t rem  = num_tasks % num_parts;
+
+    size_t current = 0;
+
+    for (size_t p = 0; p < num_parts; ++p) {
+        size_t size = base + (p < rem ? 1 : 0);
+
+        parts.push_back({
+            current,
+            current + size
+        });
+
+        current += size;
+    }
+
+    return parts;
+}
+
+
+int generate_random_sleep_ms(int min_ms, int max_ms) {
+    static std::random_device rd;
+    static std::mt19937 gen(rd());
+    std::uniform_int_distribution<> dis(min_ms, max_ms);
+    return dis(gen);
+}
+
+std::chrono::milliseconds generate_random_interval(
+    std::mt19937& rng,
+    double mean_ms)
+{
+    std::exponential_distribution<double> dist(
+        1.0 / mean_ms);
+
+    return std::chrono::milliseconds(
+        static_cast<int>(dist(rng)));
+}
+
+std::vector<MatrixTask> generate_batch(
+    const std::vector<MatrixTask>& matrix_pool,
+    std::mt19937& rng,
+    size_t batch_size)
+{
+    std::vector<MatrixTask> batch;
+    batch.reserve(batch_size);
+
+    std::uniform_int_distribution<size_t> dist(
+        0,
+        matrix_pool.size() - 1);
+
+    for (size_t i = 0; i < batch_size; ++i) {
+        batch.push_back(matrix_pool[dist(rng)]);
+    }
+
+    return batch;
+}
+
+static std::vector<JobStats> global_stats;
+static std::mutex stats_mutex;
+static std::chrono::steady_clock::time_point benchmark_start_tp;
+
+void init_benchmark_timer() {
+    benchmark_start_tp = std::chrono::steady_clock::now();
+}
+
+void record_job(const std::string& name,
+                std::chrono::steady_clock::time_point enqueue_time,
+                std::chrono::steady_clock::time_point pick_time,
+                std::chrono::steady_clock::time_point finish_time,
+                int iterations, bool success) {
+    std::lock_guard<std::mutex> lock(stats_mutex);
+    
+    auto wait = std::chrono::duration<double, std::milli>(pick_time - enqueue_time).count();
+    auto total = std::chrono::duration<double, std::milli>(finish_time - enqueue_time).count();
+    auto finish_rel = std::chrono::duration<double, std::milli>(finish_time - benchmark_start_tp).count();
+
+    global_stats.push_back({name, wait, total, iterations, success, finish_rel});
+}
+
+void report_workload_stats() {
+    std::lock_guard<std::mutex> lock(stats_mutex);
+    if (global_stats.empty()) {
+        std::cout << "No job statistics recorded.\n";
+        return;
+    }
+
+    int total_iters = 0;
+    int success_count = 0;
+    double wasted_gpu_time_ms = 0;
+    int failed_count = 0;
+    std::vector<double> completions;
+    completions.reserve(global_stats.size());
+
+    // Ensure total_jobs is not zero to avoid division by zero
+    size_t total_jobs = global_stats.size();
+
+    for (const auto& s : global_stats) {
+        total_iters += s.iterations;
+        if (s.success) success_count++;
+        else wasted_gpu_time_ms += (s.completion_time_ms - s.wait_time_ms);
+        
+        completions.push_back(s.completion_time_ms);
+    }
+
+    std::sort(completions.begin(), completions.end());
+    failed_count = total_jobs - success_count;
+    double success_percentage = (total_jobs > 0) ? (100.0 * success_count / total_jobs) : 0.0;
+    double failed_percentage = (total_jobs > 0) ? (100.0 * failed_count / total_jobs) : 0.0;
+    double mean = (total_jobs > 0) ? std::accumulate(completions.begin(), completions.end(), 0.0) / total_jobs : 0.0;
+    double median = (total_jobs > 0) ? completions[total_jobs / 2] : 0.0;
+    double p95 = completions[static_cast<size_t>(completions.size() * 0.95)];
+
+    auto max_finish = std::max_element(global_stats.begin(), global_stats.end(), [](const JobStats& a, const JobStats& b) {
+        return a.finish_relative_ms < b.finish_relative_ms;
+    });
+
+    std::cout << "\n" << std::string(45, '=') << "\n";
+    std::cout << "          WORKLOAD PERFORMANCE REPORT\n";
+    std::cout << std::string(45, '=') << "\n";
+    std::cout << std::left << std::setw(25) << "Total Jobs:" << total_jobs << "\n";
+    std::cout << std::left << std::setw(25) << "Succeeded Jobs:" << success_count << " (" << std::fixed << std::setprecision(2) << success_percentage << "%)\n";
+    std::cout << std::left << std::setw(25) << "Failed Jobs:" << failed_count << " (" << std::fixed << std::setprecision(2) << failed_percentage << "%)\n";
+    std::cout << std::left << std::setw(25) << "Total Iterations:" << total_iters << "\n";
+    std::cout << std::left << std::setw(25) << "Cumul. Wasted GPU Time:" << wasted_gpu_time_ms / 1000.0 << " s (all streams)\n";
+    std::cout << std::left << std::setw(25) << "Makespan:" << max_finish->finish_relative_ms / 1000.0 << " s\n";
+    std::cout << std::left << std::setw(25) << "Mean Completion:" << mean << " ms\n";
+    std::cout << std::left << std::setw(25) << "Median Completion:" << median << " ms\n";
+    std::cout << std::left << std::setw(25) << "95th Percentile:" << p95 << " ms\n";
+    
+    std::cout << "\nThroughput Timeline (Job Completion & Success vs Wall-clock):\n";
+    std::sort(global_stats.begin(), global_stats.end(), [](const JobStats& a, const JobStats& b) {
+        return a.finish_relative_ms < b.finish_relative_ms;
+    });
+    
+    double total_time = max_finish->finish_relative_ms;
+    for (int i = 1; i <= 10; ++i) {
+        double threshold = (total_time / 10.0) * i;
+        auto it = std::upper_bound(global_stats.begin(), global_stats.end(), threshold, 
+            [](double val, const JobStats& s) {
+                return val < s.finish_relative_ms;
+            });
+        size_t total_at_t = std::distance(global_stats.begin(), it);
+        size_t success_at_t = 0;
+        for (auto s_it = global_stats.begin(); s_it != it; ++s_it) {
+            if (s_it->success) success_at_t++;
+        }
+
+        std::cout << "  T + " << std::setw(5) << std::fixed << std::setprecision(0) << threshold 
+                  << " ms | Total Progress: " << std::setw(3) << (100 * total_at_t / total_jobs) 
+                  << "% | Successful solves: " << std::setw(3) << (100 * success_at_t / total_jobs) << "%\n";
+    }
+    std::cout << std::string(45, '=') << "\n\n";
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/sparse_utils.hpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/sparse_utils.hpp
new file mode 100644
index 0000000000..eef98a0456
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/sparse_utils.hpp
@@ -0,0 +1,94 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <cstdint>
+#include <memory>
+#include <random>
+#include <chrono>
+
+enum SolverType { CGS_SOLVER, BICSTAB_SOLVER };
+
+// Structure to hold raw data from the binary file
+struct SparseMatrixCOO {
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+    std::vector<int32_t> row_indices;
+    std::vector<int32_t> col_indices;
+    std::vector<float> values;
+};
+
+// Structure optimized for high-performance solvers
+struct SparseMatrixCSR {
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+    std::vector<int32_t> row_ptr;    // Size: rows + 1
+    std::vector<int32_t> col_indices;// Size: nnz
+    std::vector<float> values;       // Size: nnz
+};
+
+struct MatrixData {
+    std::vector<int32_t> row_ptr;
+    std::vector<int32_t> col_indices;
+    std::vector<float> values;
+    std::vector<float> b;
+    std::vector<float> x_guess;
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+};
+
+struct MatrixTask {
+    std::string path;
+    SolverType type;
+    std::shared_ptr<MatrixData> data;
+    std::chrono::steady_clock::time_point enqueue_time;
+};
+
+struct JobStats {
+    std::string task_name;
+    double wait_time_ms;       // Time spent in queue
+    double completion_time_ms; // Total turnaround time (enqueue to finish)
+    int iterations;
+    bool success;
+    double finish_relative_ms; // Wall-clock timestamp relative to benchmark start
+};
+
+void init_benchmark_timer();
+void record_job(const std::string& name,
+                std::chrono::steady_clock::time_point enqueue_time,
+                std::chrono::steady_clock::time_point pick_time,
+                std::chrono::steady_clock::time_point finish_time,
+                int iterations, bool success);
+void report_workload_stats();
+
+struct Partition {
+    size_t begin;
+    size_t end;
+    std::vector<int> devices;
+    std::vector<int> streams;
+};
+
+// Function to slurp the binary data into memory
+SparseMatrixCOO load_binary_coo(const std::string& filepath);
+
+// Converts COO to CSR format for solver compatibility
+SparseMatrixCSR convert_coo_to_csr(const SparseMatrixCOO& coo);
+
+// Compute b = A * x using CSR layout (Sparse Matrix-Vector Multiplication)
+std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x);
+
+std::vector<MatrixTask> scan_for_matrices(const std::string& dir, SolverType type);
+int generate_random_sleep_ms(int min_ms, int max_ms);
+
+// Workload generation helpers
+std::chrono::milliseconds generate_random_interval(
+    std::mt19937& rng,
+    double mean_ms);
+
+std::vector<MatrixTask> generate_batch(
+    const std::vector<MatrixTask>& matrix_pool,
+    std::mt19937& rng,
+    size_t batch_size);
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/stability_kernels.cu b/libcaf_cuda/sc26/Irregular-Workload/workloadA/stability_kernels.cu
new file mode 100644
index 0000000000..56d38d3694
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/stability_kernels.cu
@@ -0,0 +1,10 @@
+extern "C" __global__
+void check_stability(int n, const float* x, const float* r, int* err) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx < n) {
+        // If any value in the solution or residual is invalid, set the error flag
+        if (isnan(x[idx]) || isinf(x[idx]) || isnan(r[idx]) || isinf(r[idx])) {
+            atomicExch(err, 1);
+        }
+    }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/supervisor_actor.hpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/supervisor_actor.hpp
new file mode 100644
index 0000000000..88be59d8f3
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/supervisor_actor.hpp
@@ -0,0 +1,223 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <deque>
+#include <unordered_map>
+#include <vector>
+#include <chrono>
+#include "atoms.hpp"
+#include "ft_cg_actor.hpp"
+#include "sparse_utils.hpp"
+
+
+using namespace caf;
+using namespace caf::cuda;
+// ---------------------------- SUPERVISOR ACTOR ----------------------------
+
+struct suspended_task {
+  caf::actor solver;
+  std::string path;
+  int last_batch_size;
+  int device_id;
+  int stream_id;
+};
+
+struct resource_slot {
+  int device_id;
+  int stream_id;
+};
+
+struct supervisor_state {
+  std::deque<MatrixTask> queue;
+  std::deque<suspended_task> suspended_queue;
+  std::vector<caf::actor> active_solvers;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> start_times;
+  std::unordered_map<std::string, resource_slot> task_resources;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> enqueue_times;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> pick_times;
+  std::unordered_map<std::string, double> cumulative_active_ms;
+  std::unordered_map<caf::actor, int> actor_batch_sizes;
+  std::deque<resource_slot> available_slots;
+  int max_active = 1; // Admission control limit to be mindful of GPU memory.
+  int num_iterations = MAX_ITERATIONS;
+  int num_gpus = 0;
+  int tasks_succeeded = 0;
+  int tasks_failed = 0;
+  std::chrono::steady_clock::time_point benchmark_start;
+};
+
+behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks, 
+  int initial_max_active, std::chrono::steady_clock::time_point start_time, int target_device) {
+    auto& st = self->state();
+    st.queue.insert(st.queue.end(), std::make_move_iterator(tasks.begin()), std::make_move_iterator(tasks.end()));
+    st.max_active = initial_max_active;
+    st.benchmark_start = start_time;
+    st.num_gpus = 1; 
+
+    // Initialize the pool with streams for the assigned GPU
+    for (int s = 0; s < 4; ++s) {
+      st.available_slots.push_back({target_device, s});
+    }
+
+    auto spawn_next = [self]() {
+        auto& s = self->state();
+        while (s.active_solvers.size() < static_cast<size_t>(s.max_active) && !s.available_slots.empty()) {
+          if (!s.queue.empty()) {
+            auto task = std::move(s.queue.front());
+            s.queue.pop_front();
+            std::string path = task.path;
+
+            s.cumulative_active_ms[path] = 0;
+            s.enqueue_times[path] = task.enqueue_time;
+            s.pick_times[path] = std::chrono::steady_clock::now();
+
+            resource_slot slot = s.available_slots.front();
+            s.available_slots.pop_front();
+
+            self->println("[INFO] Starting solver for: {} (Device: {}, Stream: {})", 
+                          path, slot.device_id, slot.stream_id);
+            auto solver = self->spawn(fault_tolerant_cg_actor<float>,
+                                    path,
+                                    task.data,
+                                    create_in_arg(task.data->row_ptr),
+                                    create_in_arg(task.data->col_indices),
+                                    create_in_arg(task.data->values),
+                                    create_in_arg(task.data->b),
+                                    create_in_out_arg(task.data->x_guess),
+                                    (int)task.data->row_ptr.size() - 1,
+                                    (int)task.data->values.size(),
+                                    1e-5f, MAX_ITERATIONS, slot.device_id, slot.stream_id, actor_cast<actor>(self));
+
+            s.start_times[path] = std::chrono::steady_clock::now();
+            s.active_solvers.push_back(solver);
+            s.task_resources[path] = slot;
+            s.actor_batch_sizes[solver] = s.num_iterations;
+            self->mail(start_atom_v).send(solver);
+          } else {
+            bool resumed = false;
+            for (auto it = s.suspended_queue.begin(); it != s.suspended_queue.end(); ++it) {
+              auto slot_it = std::find_if(s.available_slots.begin(), s.available_slots.end(), [&](const resource_slot& slot) {
+                return slot.device_id == it->device_id && slot.stream_id == it->stream_id;
+              });
+              if (slot_it != s.available_slots.end()) {
+                auto suspended = std::move(*it);
+                s.suspended_queue.erase(it);
+                resource_slot slot = *slot_it;
+                s.available_slots.erase(slot_it);
+
+                s.pick_times[suspended.path] = std::chrono::steady_clock::now();
+
+                // Cap the minimum batch size to MAX_ITERATIONS / 8
+                int next_batch = std::max(MAX_ITERATIONS / 8, suspended.last_batch_size / 2);
+                self->println("[INFO] Resuming solver for: {} (Device: {}, Stream: {}, Batch: {})", 
+                              suspended.path, slot.device_id, slot.stream_id, next_batch);
+
+                // Resume on the same stream ID. No update_stream_atom_v needed.
+                self->mail(cg_next_step_atom_v, next_batch).send(suspended.solver);
+
+                s.active_solvers.push_back(suspended.solver);
+                s.task_resources[suspended.path] = slot;
+                s.actor_batch_sizes[suspended.solver] = next_batch;
+                resumed = true;
+                break;
+              }
+            }
+            if (!resumed) break;
+          }
+        }
+    };
+
+    spawn_next();
+    
+    return {
+        [=](gpu_done_atom, const std::string& task_name, caf::actor solver, std::vector<float>& solution, solver_result_meta meta) {
+            auto& s = self->state();
+
+            if (meta.converged || meta.error_code != CG_SUCCESS) {
+                auto end_time = std::chrono::steady_clock::now();
+                auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
+                                  end_time - s.start_times[task_name]).count();
+
+                if (meta.converged && meta.iterations < MAX_ITERATIONS) {
+                    s.tasks_succeeded++;
+                    if (meta.iterations == 0)
+                        self->println("[DONE] {}: Initial guess satisfied tolerance ({} ms).", task_name, duration);
+                    else
+                        self->println("[DONE] {}: Converged in {} iterations ({} ms).", task_name, meta.iterations, duration);
+                } else {
+                    s.tasks_failed++;
+                    std::string reason = (meta.error_code == CG_SUCCESS)
+                                         ? "Maximum Iterations Reached"
+                                         : to_string(static_cast<cg_error_type>(meta.error_code));
+                    self->println("[FAIL] {}: {} (after {} iterations, {} ms).", 
+                                  task_name, reason,
+                                  meta.iterations,
+                                  duration);
+                }
+
+                // Final active slice
+                auto active_slice = std::chrono::duration<double, std::milli>(end_time - s.pick_times[task_name]).count();
+                s.cumulative_active_ms[task_name] += active_slice;
+
+                // We override pick_time in record_job to simulate a single continuous run that equals 
+                // the actual time spent on the GPU.
+                auto simulated_pick = end_time - std::chrono::duration_cast<std::chrono::steady_clock::duration>(
+                                                std::chrono::duration<double, std::milli>(s.cumulative_active_ms[task_name]));
+
+                record_job(task_name, s.enqueue_times[task_name], simulated_pick, end_time, meta.iterations, meta.converged);
+                s.enqueue_times.erase(task_name);
+                s.pick_times.erase(task_name);
+                s.cumulative_active_ms.erase(task_name);
+
+                auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
+                if (it != s.active_solvers.end())
+                    s.active_solvers.erase(it);
+                s.start_times.erase(task_name);
+                s.actor_batch_sizes.erase(solver);
+
+                // Reclaim the device/stream slot and put it back in the pool
+                auto res_it = s.task_resources.find(task_name);
+                if (res_it != s.task_resources.end()) {
+                    s.available_slots.push_back(res_it->second);
+                    s.task_resources.erase(res_it);
+                }
+
+                spawn_next();
+
+                if (s.active_solvers.empty() && s.queue.empty() && s.suspended_queue.empty()) {
+                    self->println("All tasks in the pool have been processed.");
+
+                    report_workload_stats();
+                    self->quit();
+                }
+            } else if (meta.iterations == 0) {
+                // Just finished initialization: trigger the first iteration batch immediately.
+                self->mail(cg_next_step_atom_v, s.num_iterations).send(solver);
+            } else {
+                // Not done: Suspend the actor to allow others to use the stream
+                auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
+                
+                // Record the time spent in this active slice before suspending
+                auto active_slice = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - s.pick_times[task_name]).count();
+                s.cumulative_active_ms[task_name] += active_slice;
+
+                if (it != s.active_solvers.end())
+                    s.active_solvers.erase(it);
+
+                resource_slot slot = s.task_resources[task_name];
+                s.available_slots.push_back(slot);
+                s.task_resources.erase(task_name);
+
+                int last_batch = s.actor_batch_sizes[solver];
+                s.suspended_queue.push_back({solver, task_name, last_batch, slot.device_id, slot.stream_id});
+
+                self->println("[INFO] Suspending solver for: {} (Reclaimed Device: {}, Stream: {})", 
+                              task_name, slot.device_id, slot.stream_id);
+
+                spawn_next();
+            }
+        }
+       
+    };
+}

From 023c7c99dd2a21ca409a73c8df32cf1561b1348a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 10:50:46 -0600
Subject: [PATCH 0969/1000] Moved files into this directory.

---
 .../workloadB/CMakeLists.txt                  | 132 +++++++++
 .../Irregular-Workload/workloadB/atoms.hpp    |  51 ++++
 .../workloadB/ft_cg_actor.hpp                 | 216 ++++++++++++++
 .../workloadB/ft_cg_jacobi_actor.hpp          | 191 +++++++++++++
 .../workloadB/jacobi_kernels.cu               |  15 +
 .../workloadB/main.native.cpp                 |  60 ++++
 .../workloadB/main.native_sorted.cpp          |  63 +++++
 .../workloadB/main.test.cpp                   |  61 ++++
 .../workloadB/native_utils.cpp                |   0
 .../workloadB/native_utils.cu                 | 248 ++++++++++++++++
 .../workloadB/native_utils.hpp                |  95 +++++++
 .../workloadB/sparse_utils.cpp                | 267 ++++++++++++++++++
 .../workloadB/sparse_utils.hpp                |  94 ++++++
 .../workloadB/stability_kernels.cu            |  10 +
 .../workloadB/supervisor_actor.hpp            | 224 +++++++++++++++
 15 files changed, 1727 insertions(+)
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/CMakeLists.txt
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/atoms.hpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/ft_cg_actor.hpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/ft_cg_jacobi_actor.hpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/jacobi_kernels.cu
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native.cpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native_sorted.cpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/main.test.cpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/native_utils.cpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/native_utils.cu
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/native_utils.hpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/sparse_utils.cpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/sparse_utils.hpp
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/stability_kernels.cu
 create mode 100644 libcaf_cuda/sc26/Irregular-Workload/workloadB/supervisor_actor.hpp

diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/CMakeLists.txt b/libcaf_cuda/sc26/Irregular-Workload/workloadB/CMakeLists.txt
new file mode 100644
index 0000000000..ed8e26b561
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/CMakeLists.txt
@@ -0,0 +1,132 @@
+cmake_minimum_required(VERSION 3.16.3)
+
+# 1) Enforce C++20 (needs modern C++ features)
+set(CMAKE_CXX_STANDARD        20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS      OFF)
+
+# 2) Set CAF source and build directories
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
+set(CAF_BUILD "${CAF_SRC}/build")
+
+
+#required since cmake cant seem to find the compiler 
+set(CMAKE_CXX_COMPILER "/usr/bin/g++")
+set(CMAKE_C_COMPILER "/usr/bin/gcc")
+
+# Enable CUDA as a first-class language for the project
+project(CUDA_ACTORS LANGUAGES CXX CUDA)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit REQUIRED)
+
+include_directories(
+  "${CAF_SRC}/include"
+  "${CAF_SRC}/libcaf_io"
+  "${CAF_SRC}/libcaf_core"
+  "${CAF_SRC}/libcaf_opencl"
+  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
+  "${CAF_SRC}/libcaf_cuda"
+)
+
+# --- CUDA Kernel Compilation ---
+set(STABILITY_KERNEL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/stability_kernels.cu")
+set(STABILITY_KERNEL_CUBIN "${CMAKE_CURRENT_BINARY_DIR}/stability_kernels.cubin")
+
+# Target CUDA architecture. 'native' targets the current machine's GPU (requires CUDA 11.6+).
+# You can override this with -DCUDA_ARCH=sm_XX if needed.
+set(CUDA_ARCH "native" CACHE STRING "Target CUDA architecture (e.g., native, sm_70, sm_75, sm_80, sm_86)")
+
+add_custom_command(
+  OUTPUT ${STABILITY_KERNEL_CUBIN}
+  COMMAND ${CUDAToolkit_NVCC_EXECUTABLE}
+          -cubin
+          -arch=${CUDA_ARCH}
+          -o ${STABILITY_KERNEL_CUBIN}
+          ${STABILITY_KERNEL_SRC}
+  DEPENDS ${STABILITY_KERNEL_SRC}
+  COMMENT "Compiling CUDA kernel ${STABILITY_KERNEL_SRC} for architecture: ${CUDA_ARCH}"
+  VERBATIM
+)
+
+set(JACOBI_KERNEL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/jacobi_kernels.cu")
+set(JACOBI_KERNEL_CUBIN "${CMAKE_CURRENT_BINARY_DIR}/jacobi_kernels.cubin")
+
+add_custom_command(
+  OUTPUT ${JACOBI_KERNEL_CUBIN}
+  COMMAND ${CUDAToolkit_NVCC_EXECUTABLE}
+          -cubin
+          -arch=${CUDA_ARCH}
+          -o ${JACOBI_KERNEL_CUBIN}
+          ${JACOBI_KERNEL_SRC}
+  DEPENDS ${JACOBI_KERNEL_SRC}
+  COMMENT "Compiling CUDA kernel ${JACOBI_KERNEL_SRC} for architecture: ${CUDA_ARCH}"
+  VERBATIM
+)
+
+add_custom_target(stability_kernels_cubin ALL DEPENDS ${STABILITY_KERNEL_CUBIN})
+add_custom_target(jacobi_kernels_cubin ALL DEPENDS ${JACOBI_KERNEL_CUBIN})
+
+
+# 5) Declare your executable and its source files
+add_executable(test main.test.cpp sparse_utils.cpp)
+target_sources(test PRIVATE
+  atoms.hpp
+  ft_cg_actor.hpp
+  ft_cg_jacobi_actor.hpp
+  supervisor_actor.hpp
+)
+
+target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
+
+target_link_libraries(test
+  PRIVATE
+    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+    CUDA::nvrtc
+    CUDA::cublas
+    CUDA::cusparse
+)
+
+
+# # 5) Declare your executable
+# add_executable(hot-potatoe hot-potatoe.cpp sparse_utils.cpp)
+
+# target_compile_definitions(hot-potatoe PRIVATE CAF_ENABLE_LOGGING)
+
+# target_link_libraries(hot-potatoe
+#   PRIVATE
+#     "${CAF_BUILD}/libcaf_core/libcaf_core.so"
+#     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
+#     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
+#     CUDA::nvrtc
+#     CUDA::cublas
+#     CUDA::cusparse
+# )
+
+
+# FindThreads is required for std::thread in the native version
+find_package(Threads REQUIRED)
+
+# 6) Declare the native benchmark executable (raw CUDA/cuBLAS/cuSPARSE)
+add_executable(workload-native main.native.cpp sparse_utils.cpp native_utils.cu)
+
+target_link_libraries(workload-native
+  PRIVATE
+    CUDA::cudart
+    CUDA::cublas
+    CUDA::cusparse
+    Threads::Threads
+)
+
+# 7) Declare the native sorted benchmark executable
+add_executable(workload-native-sorted main.native_sorted.cpp sparse_utils.cpp native_utils.cu)
+
+target_link_libraries(workload-native-sorted
+  PRIVATE
+    CUDA::cudart
+    CUDA::cublas
+    CUDA::cusparse
+    Threads::Threads
+)
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/atoms.hpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/atoms.hpp
new file mode 100644
index 0000000000..ab2a741b6c
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/atoms.hpp
@@ -0,0 +1,51 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include "sparse_utils.hpp"
+
+constexpr int MAX_ITERATIONS = 16000;
+
+CAF_BEGIN_TYPE_ID_BLOCK(workload_test, caf::id_block::cuda::end)
+    CAF_ADD_ATOM(workload_test, get_work_atom)
+    CAF_ADD_ATOM(workload_test, release_memory_atom)
+    CAF_ADD_ATOM(workload_test, request_work_atom)
+    CAF_ADD_ATOM(workload_test, worker_done_atom)
+    CAF_ADD_ATOM(workload_test, work_tick_atom)
+    CAF_ADD_ATOM(workload_test, add_work_atom)
+    CAF_ADD_ATOM(workload_test, steal_work_atom)
+    CAF_ADD_ATOM(workload_test, update_stream_atom)
+    CAF_ADD_ATOM(workload_test, shutdown_atom)
+    CAF_ADD_TYPE_ID(workload_test, (SolverType))
+    CAF_ADD_TYPE_ID(workload_test, (MatrixTask))
+    CAF_ADD_TYPE_ID(workload_test, (MatrixData))
+    CAF_ADD_TYPE_ID(workload_test, (std::vector<MatrixTask>))
+    CAF_ADD_TYPE_ID(workload_test, (std::shared_ptr<MatrixData>))
+CAF_END_TYPE_ID_BLOCK(workload_test)
+
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixData)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::shared_ptr<MatrixData>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(MatrixTask)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(std::vector<MatrixTask>)
+CAF_ALLOW_UNSAFE_MESSAGE_TYPE(SolverType)
+
+enum cg_error_type : int {
+  CG_SUCCESS = 0,
+  CG_MAX_ITER = 1,
+  CG_NAN_INF = 2,
+  CG_STAGNATION = 3,
+  CG_BREAKDOWN = 4,
+  CG_RESIDUAL_FACTOR_FAIL = 5
+};
+
+inline std::string to_string(cg_error_type err) {
+  switch (err) {
+    case CG_SUCCESS: return "Success";
+    case CG_MAX_ITER: return "Maximum Iterations Reached";
+    case CG_NAN_INF: return "Stability Check Failed (NaN/Inf Detected)";
+    case CG_STAGNATION: return "Stagnation Detected (Residual stopped changing)";
+    case CG_BREAKDOWN: return "Solver Breakdown (Division by zero/near-zero)";
+    case CG_RESIDUAL_FACTOR_FAIL: return "Residual Factor Check Failed";
+    default: return "Unknown Error (" + std::to_string(static_cast<int>(err)) + ")";
+  }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/ft_cg_actor.hpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/ft_cg_actor.hpp
new file mode 100644
index 0000000000..4fd5335cd8
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/ft_cg_actor.hpp
@@ -0,0 +1,216 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <string>
+#include <vector>
+#include <memory>
+#include "atoms.hpp"
+#include "sparse_utils.hpp"
+#include "caf/actorSOLVE/actorSOLVE.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+// ---------------------------- FAULT TOLERANT SOLVER ----------------------------
+
+template <class T = float>
+struct ft_cg_state {
+  // Host Data
+  std::string path;
+  in<int> h_row_ptr, h_col_ind;
+  in<T> h_values, h_b;
+  in_out<T> h_x;
+
+  // GPU Buffers
+  mem_ptr<int> A_rp, A_ci, d_err;
+  mem_ptr<T> A_val, b, x, r, p, w, y_tmp;
+  mem_ptr<char> spmv_ws;
+
+  // Config
+  int n, nnz, max_iter;
+  int iterations = 0;
+  int strikes = 0;
+  T tol;
+  int device_id, stream_id;
+
+  // Supervision & Monitoring
+  caf::actor supervisor;
+  device_ptr d_ptr;
+  program_ptr stab_prog;
+
+  T initial_rho = 0;
+  T current_rho = 0;
+  T old_rho = 0;
+  bool initialized = false;
+  std::shared_ptr<MatrixData> pinned_data;
+};
+
+template <class T = float>
+behavior fault_tolerant_cg_actor(stateful_actor<ft_cg_state<T>>* self,
+                                 std::string path,
+                                 std::shared_ptr<MatrixData> data,
+                                 in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+                                 int n, int nnz, T tol, int max_iter,
+                                 int dev_num, int stream, caf::actor supervisor) {
+  auto& s = self->state();
+  s.pinned_data = std::move(data);
+  s.path = std::move(path);
+  s.h_row_ptr = std::move(rp); s.h_col_ind = std::move(ci);
+  s.h_values = std::move(val); s.h_b = std::move(b_in); s.h_x = std::move(x_in);
+  s.n = n; s.nnz = nnz; s.tol = tol; s.max_iter = max_iter;
+  s.device_id = dev_num; s.stream_id = stream; s.supervisor = supervisor;
+
+  return {
+    [=](start_atom) {
+      auto& st = self->state();
+      if (st.initialized) return;
+
+      command_runner<> runner;
+      st.A_rp = runner.transfer_memory(st.device_id, st.stream_id, st.h_row_ptr);
+      st.A_ci = runner.transfer_memory(st.device_id, st.stream_id, st.h_col_ind);
+      st.A_val = runner.transfer_memory(st.device_id, st.stream_id, st.h_values);
+      st.b = runner.transfer_memory(st.device_id, st.stream_id, st.h_b);
+      st.x = runner.transfer_memory(st.device_id, st.stream_id, st.h_x);
+
+      st.d_ptr = platform::create()->schedule(st.stream_id, st.device_id);
+      st.d_ptr->enable_cublas(); st.d_ptr->enable_cusparse();
+
+      auto& mgr = manager::get();
+      // Load stability kernel from file as requested
+      st.stab_prog = mgr.create_program_from_cubin("stability_kernels.cubin", "check_stability", st.d_ptr);
+
+      st.r = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.p = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.w = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.y_tmp = runner.transfer_memory(st.device_id, st.stream_id, out<T>(1));
+      st.d_err = runner.transfer_memory(st.device_id, st.stream_id, out<int>(1));
+      
+      size_t ws_sz = st.d_ptr->spmv_csr_buffer_size(st.stream_id, st.n, st.n, st.nnz, st.A_rp, st.A_ci, st.A_val, st.x, st.w);
+      if (ws_sz > 0) st.spmv_ws = command_runner<out<char>>{}.transfer_memory(st.device_id, st.stream_id, out<char>{static_cast<int>(ws_sz)});
+
+      st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.x, T{0}, st.w, st.spmv_ws);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.b, st.r);
+      else st.d_ptr->scopy(st.stream_id, st.n, st.b, st.r);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->daxpy(st.stream_id, st.n, -1.0, st.w, st.r);
+      else st.d_ptr->saxpy(st.stream_id, st.n, -1.0f, st.w, st.r);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+      else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+
+      st.initial_rho = runner.copy_to_host(st.y_tmp)[0];
+      st.current_rho = st.initial_rho;
+      st.initialized = true;
+
+      // Notify supervisor that setup is complete and report initial status
+      solver_result_meta meta(st.device_id, st.stream_id, 0, false, CG_SUCCESS);
+      self->mail(gpu_done_atom_v, st.path, actor_cast<actor>(self), std::vector<T>{}, meta).send(st.supervisor);
+    },
+
+    [=](cg_next_step_atom, int num_iters) {
+      auto& st = self->state();
+      command_runner<T> runner;
+      T threshold = st.tol * st.tol;
+      int code = CG_SUCCESS;
+      int step_count = 0;
+
+      // Execute exactly the number of iterations requested by the supervisor
+      while (step_count < num_iters && st.iterations < st.max_iter && st.current_rho > threshold) {
+        // std::cout << "iterations = " << st.iterations << ", current_rho = " << st.current_rho << std::endl;
+        st.iterations++;
+        step_count++;
+
+        if (st.iterations > 1) {
+          T beta = st.current_rho / st.old_rho;
+          if constexpr (std::is_same_v<T, double>) {
+            st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.w);
+            st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
+            st.d_ptr->dcopy(st.stream_id, st.n, st.w, st.p);
+          } else {
+            st.d_ptr->scopy(st.stream_id, st.n, st.r, st.w);
+            st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(beta), st.p, st.w);
+            st.d_ptr->scopy(st.stream_id, st.n, st.w, st.p);
+          }
+        } else {
+          if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.r, st.p);
+          else st.d_ptr->scopy(st.stream_id, st.n, st.r, st.p);
+        }
+
+        st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.p, T{0}, st.w, st.spmv_ws);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
+        else st.d_ptr->sdot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
+        
+        T dot_pw = runner.copy_to_host(st.y_tmp)[0];
+        if (std::abs(dot_pw) < 1e-25) { 
+          code = CG_BREAKDOWN; 
+          break; 
+        }
+
+        T alpha = st.current_rho / dot_pw;
+        if constexpr (std::is_same_v<T, double>) {
+          st.d_ptr->daxpy(st.stream_id, st.n, alpha, st.p, st.x);
+          st.d_ptr->daxpy(st.stream_id, st.n, -alpha, st.w, st.r);
+        } else {
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(alpha), st.p, st.x);
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(-alpha), st.w, st.r);
+        }
+
+        st.old_rho = st.current_rho;
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+        else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.r, st.y_tmp);
+        st.current_rho = runner.copy_to_host(st.y_tmp)[0];
+
+      }
+
+      // Run error checks and prepare progress report
+      bool converged = (st.current_rho <= threshold);
+
+      // Check for non-fatal errors that can be retried (Stagnation, Max Iter, Residual Factor)
+      if (code == CG_SUCCESS && !converged && std::abs(st.old_rho - st.current_rho) < 1e-12)
+        code = CG_STAGNATION;
+
+      if (code == CG_SUCCESS) {
+        if (!converged && st.iterations >= st.max_iter) code = CG_MAX_ITER;
+        // Residual decrease check: treat as non-fatal strike if residual didn't decrease significantly
+        if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
+        
+        // If we reached here with CG_SUCCESS, reset strikes as this was a productive batch
+        if (code == CG_SUCCESS) st.strikes = 0;
+      }
+
+      // Reset and launch NaN/Inf stability kernel from file
+      nd_range range(static_cast<int>((st.n + 255) / 256), 1, 1, 256, 1, 1);
+      CHECK_CUDA(cuMemsetD32Async(st.d_err->mem(), 0, 1, st.d_ptr->get_stream_for_actor(st.stream_id)));
+      st.d_ptr->launch_kernel_mem_ref(st.stab_prog->get_kernel(st.d_ptr->getId()), range,
+                                      std::make_tuple(in<int>(st.n), st.x, st.r, st.d_err), st.stream_id);
+
+      int err_flag = runner.copy_to_host(st.d_err)[0];
+      if (err_flag != 0 || std::isnan(st.current_rho) || std::isinf(st.current_rho)) code = CG_NAN_INF;
+
+      // Three strikes policy for non-fatal errors (Stagnation, Max Iterations, Residual Factor Failure)
+      bool is_fatal = (code == CG_NAN_INF || code == CG_BREAKDOWN);
+      if (code != CG_SUCCESS && !is_fatal) {
+        st.strikes++;
+        if (st.strikes < 3) {
+          code = CG_SUCCESS; // Reset code to SUCCESS to allow the supervisor to retry/suspend
+        }
+      }
+
+      if (code != CG_SUCCESS) converged = false;
+
+      solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
+      
+      // Report current solution and metadata to the supervisor
+      runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor, path = st.path, self_h = actor_cast<actor>(self)](std::vector<T> sol) {
+        anon_mail(gpu_done_atom_v, path, self_h, std::move(sol), meta).send(supervisor);
+        if (converged || code != CG_SUCCESS)
+          anon_mail(shutdown_atom_v).send(self_h);
+      });
+    },
+    [=](update_stream_atom, int new_stream) {
+      self->state().stream_id = new_stream;
+    },
+    [=](shutdown_atom) {
+      self->quit();
+    }
+  };
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/ft_cg_jacobi_actor.hpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/ft_cg_jacobi_actor.hpp
new file mode 100644
index 0000000000..7e42785bff
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/ft_cg_jacobi_actor.hpp
@@ -0,0 +1,191 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <string>
+#include <vector>
+#include <memory>
+#include "atoms.hpp"
+#include "sparse_utils.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+template <class T = float>
+struct ft_cg_jacobi_state {
+  // Host Data
+  std::string path;
+  in<int> h_row_ptr, h_col_ind;
+  in<T> h_values, h_b;
+  in_out<T> h_x;
+
+  // GPU Buffers
+  mem_ptr<int> A_rp, A_ci, d_err;
+  mem_ptr<T> A_val, b, x, r, p, w, z, D_inv, y_tmp;
+  mem_ptr<char> spmv_ws;
+
+  // Config
+  int n, nnz, max_iter;
+  int iterations = 0;
+  int strikes = 0;
+  T tol;
+  int device_id, stream_id;
+
+  // Supervision & Monitoring
+  caf::actor supervisor;
+  device_ptr d_ptr;
+  program_ptr stab_prog;
+  program_ptr diag_prog;
+
+  T initial_rho = 0;
+  T current_rho = 0;
+  T old_rho = 0;
+  bool initialized = false;
+  std::shared_ptr<MatrixData> pinned_data;
+};
+
+template <class T = float>
+behavior fault_tolerant_cg_jacobi_actor(stateful_actor<ft_cg_jacobi_state<T>>* self,
+                                        std::string path,
+                                        std::shared_ptr<MatrixData> data,
+                                        in<int> rp, in<int> ci, in<T> val, in<T> b_in, in_out<T> x_in,
+                                        int n, int nnz, T tol, int max_iter,
+                                        int dev_num, int stream, caf::actor supervisor) {
+  auto& s = self->state();
+  s.pinned_data = std::move(data);
+  s.path = std::move(path);
+  s.h_row_ptr = std::move(rp); s.h_col_ind = std::move(ci);
+  s.h_values = std::move(val); s.h_b = std::move(b_in); s.h_x = std::move(x_in);
+  s.n = n; s.nnz = nnz; s.tol = tol; s.max_iter = max_iter;
+  s.device_id = dev_num; s.stream_id = stream; s.supervisor = supervisor;
+
+  return {
+    [=](start_atom) {
+      auto& st = self->state();
+      if (st.initialized) return;
+
+      command_runner<> runner;
+      st.A_rp = runner.transfer_memory(st.device_id, st.stream_id, st.h_row_ptr);
+      st.A_ci = runner.transfer_memory(st.device_id, st.stream_id, st.h_col_ind);
+      st.A_val = runner.transfer_memory(st.device_id, st.stream_id, st.h_values);
+      st.b = runner.transfer_memory(st.device_id, st.stream_id, st.h_b);
+      st.x = runner.transfer_memory(st.device_id, st.stream_id, st.h_x);
+
+      st.d_ptr = platform::create()->schedule(st.stream_id, st.device_id);
+      st.d_ptr->enable_cublas(); st.d_ptr->enable_cusparse();
+
+      auto& mgr = manager::get();
+      st.stab_prog = mgr.create_program_from_cubin("stability_kernels.cubin", "check_stability", st.d_ptr);
+      st.diag_prog = mgr.create_program_from_cubin("jacobi_kernels.cubin", "extract_diag_inv", st.d_ptr);
+
+      st.r = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.p = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.w = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.z = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.D_inv = runner.transfer_memory(st.device_id, st.stream_id, out<T>(st.n));
+      st.y_tmp = runner.transfer_memory(st.device_id, st.stream_id, out<T>(1));
+      st.d_err = runner.transfer_memory(st.device_id, st.stream_id, out<int>(1));
+      
+      size_t ws_sz = st.d_ptr->spmv_csr_buffer_size(st.stream_id, st.n, st.n, st.nnz, st.A_rp, st.A_ci, st.A_val, st.x, st.w);
+      if (ws_sz > 0) st.spmv_ws = command_runner<out<char>>{}.transfer_memory(st.device_id, st.stream_id, out<char>{static_cast<int>(ws_sz)});
+
+      // Jacobi setup: Extract D_inv
+      int threads = 256;
+      nd_range range((st.n + threads - 1) / threads, 1, 1, threads, 1, 1);
+      st.d_ptr->launch_kernel_mem_ref(st.diag_prog->get_kernel(st.d_ptr->getId()), range,
+                                      std::make_tuple(in<int>(st.n), st.A_rp, st.A_ci, st.A_val, st.D_inv), st.stream_id);
+
+      // Initial r = b - Ax
+      st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.x, T{0}, st.w, st.spmv_ws);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.b, st.r); else st.d_ptr->scopy(st.stream_id, st.n, st.b, st.r);
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->daxpy(st.stream_id, st.n, -1.0, st.w, st.r); else st.d_ptr->saxpy(st.stream_id, st.n, -1.0f, st.w, st.r);
+
+      // Initial z = D_inv * r
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->d_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+      else st.d_ptr->s_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+
+      // Initial rho = r * z
+      if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.z, st.y_tmp); else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+
+      st.initial_rho = runner.copy_to_host(st.y_tmp)[0];
+      st.current_rho = st.initial_rho;
+      st.initialized = true;
+
+      solver_result_meta meta(st.device_id, st.stream_id, 0, false, CG_SUCCESS);
+      self->mail(gpu_done_atom_v, st.path, actor_cast<actor>(self), std::vector<T>{}, meta).send(st.supervisor);
+    },
+
+    [=](cg_next_step_atom, int num_iters) {
+      auto& st = self->state();
+      command_runner<T> runner;
+      T threshold = st.tol * st.tol;
+      int code = CG_SUCCESS;
+      int step_count = 0;
+
+      while (step_count < num_iters && st.iterations < st.max_iter && st.current_rho > threshold) {
+        st.iterations++;
+        step_count++;
+
+        if (st.iterations > 1) {
+          T beta = st.current_rho / st.old_rho;
+          if constexpr (std::is_same_v<T, double>) {
+            st.d_ptr->dcopy(st.stream_id, st.n, st.z, st.w);
+            st.d_ptr->daxpy(st.stream_id, st.n, beta, st.p, st.w);
+            st.d_ptr->dcopy(st.stream_id, st.n, st.w, st.p);
+          } else {
+            st.d_ptr->scopy(st.stream_id, st.n, st.z, st.w);
+            st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(beta), st.p, st.w);
+            st.d_ptr->scopy(st.stream_id, st.n, st.w, st.p);
+          }
+        } else {
+          if constexpr (std::is_same_v<T, double>) st.d_ptr->dcopy(st.stream_id, st.n, st.z, st.p); else st.d_ptr->scopy(st.stream_id, st.n, st.z, st.p);
+        }
+
+        st.d_ptr->spmv_csr(st.stream_id, st.n, st.n, st.nnz, T{1}, st.A_rp, st.A_ci, st.A_val, st.p, T{0}, st.w, st.spmv_ws);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.p, st.w, st.y_tmp); else st.d_ptr->sdot(st.stream_id, st.n, st.p, st.w, st.y_tmp);
+        
+        T dot_pw = runner.copy_to_host(st.y_tmp)[0];
+        if (std::abs(dot_pw) < 1e-25) { code = CG_BREAKDOWN; break; }
+
+        T alpha = st.current_rho / dot_pw;
+        if constexpr (std::is_same_v<T, double>) {
+          st.d_ptr->daxpy(st.stream_id, st.n, alpha, st.p, st.x);
+          st.d_ptr->daxpy(st.stream_id, st.n, -alpha, st.w, st.r);
+        } else {
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(alpha), st.p, st.x);
+          st.d_ptr->saxpy(st.stream_id, st.n, static_cast<float>(-alpha), st.w, st.r);
+        }
+
+        st.old_rho = st.current_rho;
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->d_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+        else st.d_ptr->s_elementwise_multiply(st.stream_id, st.n, st.D_inv, st.r, st.z);
+        if constexpr (std::is_same_v<T, double>) st.d_ptr->ddot(st.stream_id, st.n, st.r, st.z, st.y_tmp); else st.d_ptr->sdot(st.stream_id, st.n, st.r, st.z, st.y_tmp);
+        st.current_rho = runner.copy_to_host(st.y_tmp)[0];
+      }
+
+      bool converged = (st.current_rho <= threshold);
+      if (code == CG_SUCCESS && !converged && std::abs(st.old_rho - st.current_rho) < 1e-12) code = CG_STAGNATION;
+      if (code == CG_SUCCESS) {
+        if (!converged && st.iterations >= st.max_iter) code = CG_MAX_ITER;
+        if (st.initial_rho > 0 && (st.current_rho / st.initial_rho) > 0.999) code = CG_RESIDUAL_FACTOR_FAIL;
+        if (code == CG_SUCCESS) st.strikes = 0;
+      }
+      nd_range range(static_cast<int>((st.n + 255) / 256), 1, 1, 256, 1, 1);
+      CHECK_CUDA(cuMemsetD32Async(st.d_err->mem(), 0, 1, st.d_ptr->get_stream_for_actor(st.stream_id)));
+      st.d_ptr->launch_kernel_mem_ref(st.stab_prog->get_kernel(st.d_ptr->getId()), range,
+                                      std::make_tuple(in<int>(st.n), st.x, st.r, st.d_err), st.stream_id);
+      int err_flag = runner.copy_to_host(st.d_err)[0];
+      if (err_flag != 0 || std::isnan(st.current_rho) || std::isinf(st.current_rho)) code = CG_NAN_INF;
+      bool is_fatal = (code == CG_NAN_INF || code == CG_BREAKDOWN);
+      if (code != CG_SUCCESS && !is_fatal) { st.strikes++; if (st.strikes < 3) code = CG_SUCCESS; }
+      if (code != CG_SUCCESS) converged = false;
+      solver_result_meta meta(st.device_id, st.stream_id, st.iterations, converged, code);
+      runner.copy_to_host_async(st.x, [=, supervisor = st.supervisor, path = st.path, self_h = actor_cast<actor>(self)](std::vector<T> sol) {
+        anon_mail(gpu_done_atom_v, path, self_h, std::move(sol), meta).send(supervisor);
+        if (converged || code != CG_SUCCESS) anon_mail(shutdown_atom_v).send(self_h);
+      });
+    },
+    [=](update_stream_atom, int new_stream) { self->state().stream_id = new_stream; },
+    [=](shutdown_atom) { self->quit(); }
+  };
+}
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/jacobi_kernels.cu b/libcaf_cuda/sc26/Irregular-Workload/workloadB/jacobi_kernels.cu
new file mode 100644
index 0000000000..16c17e7352
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/jacobi_kernels.cu
@@ -0,0 +1,15 @@
+extern "C" __global__
+void extract_diag_inv(int n, const int* row_ptr, const int* col_ind, const float* val, float* d_inv) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        float d = 0.0f;
+        // Search for the diagonal element (A[i][i]) in the sparse row
+        for (int j = row_ptr[i]; j < row_ptr[i+1]; j++) {
+            if (col_ind[j] == i) {
+                d = val[j];
+                break;
+            }
+        }
+        d_inv[i] = (d != 0.0f) ? 1.0f / d : 1.0f;
+    }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native.cpp
new file mode 100644
index 0000000000..ea334586ac
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native.cpp
@@ -0,0 +1,60 @@
+#include <iostream>
+#include <vector>
+#include <string>
+#include <chrono>
+#include <thread>
+#include <filesystem>
+#include <algorithm>
+#include <atomic>
+#include "native_utils.hpp"
+
+void producer(ThreadSafeQueue<MatrixTask>& queue, std::vector<MatrixTask> matrix_pool) {
+    for (auto& task : matrix_pool) {
+        task.enqueue_time = std::chrono::steady_clock::now();
+        queue.push(task);
+    }
+    queue.signal_shutdown();
+}
+
+int main(int argc, char** argv)
+{
+    constexpr uint32_t WORKLOAD_SEED = 42;
+    int num_streams = 4;
+    // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
+
+    std::cout << "[INFO] Loading matrices...\n";
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+
+    if (matrix_pool.empty()) {
+        std::cerr << "No matrices found.\n";
+        return 1;
+    }
+
+    int num_gpus = 0;
+    CHECK_CUDA(cudaGetDeviceCount(&num_gpus));
+    
+    std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
+    std::cout << "[INFO] Matrix pool size: " << matrix_pool.size() << "\n";
+    std::cout << "[INFO] Streams/GPU: " << num_streams << "\n";
+
+    std::atomic<int> tasks_succeeded{0};
+    std::atomic<int> tasks_failed{0};
+    ThreadSafeQueue<MatrixTask> work_queue;
+    
+    init_benchmark_timer();
+    std::thread producer_thread(producer, std::ref(work_queue), matrix_pool);
+    
+    std::vector<std::thread> workers;
+    for (int gpu = 0; gpu < num_gpus; ++gpu) {
+        for (int stream = 0; stream < num_streams; ++stream) {
+            workers.emplace_back(gpu_stream_worker, gpu, gpu * num_streams + stream, std::ref(work_queue), std::ref(tasks_succeeded), std::ref(tasks_failed));
+        }
+    }
+
+    producer_thread.join();
+    for (auto& worker : workers) worker.join();
+
+    report_workload_stats();
+
+    return 0;
+}
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native_sorted.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native_sorted.cpp
new file mode 100644
index 0000000000..85aa4d301d
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native_sorted.cpp
@@ -0,0 +1,63 @@
+#include <iostream>
+#include <vector>
+#include <string>
+#include <chrono>
+#include <thread>
+#include <filesystem>
+#include <algorithm>
+#include <atomic>
+#include "native_utils.hpp"
+
+void producer(ThreadSafeQueue<MatrixTask>& queue, std::vector<MatrixTask> matrix_pool) {
+    // Order tasks from lowest NNZ to highest NNZ
+    std::sort(matrix_pool.begin(), matrix_pool.end(), [](const MatrixTask& a, const MatrixTask& b) {
+        return a.data->nnz < b.data->nnz;
+    });
+
+    for (auto& task : matrix_pool) {
+        task.enqueue_time = std::chrono::steady_clock::now();
+        queue.push(task);
+    }
+    queue.signal_shutdown();
+}
+
+int main(int argc, char** argv) {
+    constexpr uint32_t WORKLOAD_SEED = 42;
+    int num_streams = 4;
+    // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
+
+    std::cout << "[INFO] Loading matrices...\n";
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+
+    if (matrix_pool.empty()) {
+        std::cerr << "No matrices found.\n";
+        return 1;
+    }
+
+    int num_gpus = 0;
+    CHECK_CUDA(cudaGetDeviceCount(&num_gpus));
+
+    std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
+    std::cout << "[INFO] Matrix pool size: " << matrix_pool.size() << "\n";
+    std::cout << "[INFO] Streams/GPU: " << num_streams << "\n";
+
+    std::atomic<int> tasks_succeeded{0};
+    std::atomic<int> tasks_failed{0};
+    ThreadSafeQueue<MatrixTask> work_queue;
+    
+    init_benchmark_timer();
+    std::thread producer_thread(producer, std::ref(work_queue), matrix_pool);
+    
+    std::vector<std::thread> workers;
+    for (int gpu = 0; gpu < num_gpus; ++gpu) {
+        for (int stream = 0; stream < num_streams; ++stream) {
+            workers.emplace_back(gpu_stream_worker, gpu, gpu * num_streams + stream, std::ref(work_queue), std::ref(tasks_succeeded), std::ref(tasks_failed));
+        }
+    }
+    producer_thread.join();
+    for (auto& worker : workers) worker.join();
+
+    report_workload_stats();
+
+    return 0;
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.test.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.test.cpp
new file mode 100644
index 0000000000..d3f0fd4cd4
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.test.cpp
@@ -0,0 +1,61 @@
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <deque>
+#include <string>
+#include <chrono>
+#include <numeric>
+#include <cmath>
+#include <algorithm>
+#include <filesystem>
+#include <memory>
+#include "sparse_utils.hpp"
+#include "atoms.hpp"
+#include "ft_cg_actor.hpp"
+#include "supervisor_actor.hpp"
+
+using namespace caf;
+using namespace caf::cuda;
+
+
+constexpr uint32_t WORKLOAD_SEED = 42;
+void caf_main(actor_system& sys) {
+    manager::init(sys, manager_config(true, true));
+    std::cout << "[INFO] Loading matrices...\n";
+    {
+         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
+        //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/unsymmetric", CGS_SOLVER);
+         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric", CGS_SOLVER);
+         auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+
+        int num_gpus = manager::get().get_num_devices();
+        std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
+        std::cout << "[INFO] Matrix pool size: " << tasks_vec.size() << "\n";
+
+        if (tasks_vec.empty()) {
+            std::cerr << "No matrices found. Running dummy test task." << std::endl;
+            auto data = std::make_shared<MatrixData>();
+            data->row_ptr = {0, 1, 2};
+            data->col_indices = {0, 1};
+            data->values = {10.0f, 10.0f};
+            data->b = {100.0f, 100.0f};
+            data->x_guess = {0.0f, 0.0f};
+            tasks_vec.push_back({"dummy_task", CGS_SOLVER, data});
+        }
+
+        init_benchmark_timer();
+        for (auto& task : tasks_vec) {
+            task.enqueue_time = std::chrono::steady_clock::now();
+        }
+
+        auto benchmark_start = std::chrono::steady_clock::now();
+        int admission_control_limit = 4 * num_gpus; // 4 concurrent tasks per GPU
+
+        sys.spawn(supervisor_actor, std::move(tasks_vec), admission_control_limit, benchmark_start);
+        sys.await_all_actors_done();
+    }
+    manager::shutdown();
+}
+CAF_MAIN(id_block::cuda, id_block::workload_test)
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/native_utils.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/native_utils.cpp
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/native_utils.cu b/libcaf_cuda/sc26/Irregular-Workload/workloadB/native_utils.cu
new file mode 100644
index 0000000000..22a426f494
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/native_utils.cu
@@ -0,0 +1,248 @@
+#include "native_utils.hpp"
+#include <cmath>
+#include <chrono>
+
+// ============================================================
+// PCG Kernels
+// ============================================================
+
+__global__ void extract_diag_inv_kernel(int n, const int* row_ptr, const int* col_ind, const float* values, float* d_inv) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) {
+        float diag = 1.0f;
+        for (int j = row_ptr[i]; j < row_ptr[i + 1]; j++) {
+            if (col_ind[j] == i) {
+                diag = values[j];
+                break;
+            }
+        }
+        d_inv[i] = (fabsf(diag) > 1e-20f) ? 1.0f / diag : 1.0f;
+    }
+}
+
+__global__ void elementwise_mul_kernel(int n, const float* a, const float* b, float* c) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n) c[i] = a[i] * b[i];
+}
+
+int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
+                           const MatrixTask& task, cudaStream_t stream) {
+    int n = (int)task.data->row_ptr.size() - 1;
+    float alpha = 1.0f, beta = 0.0f, rho = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
+    float tolerance = 1e-5f;
+    int max_iters = 16000;
+
+    float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b, *d_z, *d_Dinv;
+    int *d_row_ptr, *d_col_ind;
+
+    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_z, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_Dinv, n * sizeof(float), stream));
+
+    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
+
+    CHECK_CUBLAS(cublasSetStream(cublas, stream));
+    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
+
+    // Extract Diagonal Inverse
+    int threads = 256;
+    int blocks = (n + threads - 1) / threads;
+    extract_diag_inv_kernel<<<blocks, threads, 0, stream>>>(n, d_row_ptr, d_col_ind, d_val, d_Dinv);
+
+    cusparseSpMatDescr_t matA;
+    cusparseDnVecDescr_t vecP, vecAp;
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val,
+                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                     CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecAp, n, d_Ap, CUDA_R_32F));
+
+    size_t bufferSize = 0;
+    void* d_buffer = nullptr;
+    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, vecP, &beta, vecAp,
+                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
+
+    // r = b (assuming x=0), z = M^-1 * r, p = z
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
+    elementwise_mul_kernel<<<blocks, threads, 0, stream>>>(n, d_Dinv, d_r, d_z);
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_z, 1, d_p, 1));
+    CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_z, 1, &rho));
+
+    int k = 0;
+    float r_norm_sq = 0.0f;
+    while (k < max_iters) {
+        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                    &alpha, matA, vecP, &beta, vecAp,
+                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
+        float pAp, old_rho;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
+        a = rho / pAp;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
+        na = -a;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &na, d_Ap, 1, d_r, 1));
+        
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r_norm_sq));
+        if (std::sqrt(r_norm_sq) < tolerance) break;
+
+        elementwise_mul_kernel<<<blocks, threads, 0, stream>>>(n, d_Dinv, d_r, d_z);
+        old_rho = rho;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_z, 1, &rho));
+        b = rho / old_rho;
+        
+        // p = z + beta * p
+        CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_z, 1, d_p, 1));
+        k++;
+    }
+
+    CHECK_CUSPARSE(cusparseDestroySpMat(matA));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
+    CHECK_CUDA(cudaFreeAsync(d_val, stream));
+    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
+    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
+    CHECK_CUDA(cudaFreeAsync(d_x, stream));
+    CHECK_CUDA(cudaFreeAsync(d_r, stream));
+    CHECK_CUDA(cudaFreeAsync(d_p, stream));
+    CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
+    CHECK_CUDA(cudaFreeAsync(d_b, stream));
+    CHECK_CUDA(cudaFreeAsync(d_z, stream));
+    CHECK_CUDA(cudaFreeAsync(d_Dinv, stream));
+    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
+    return k;
+}
+
+int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse,
+                    const MatrixTask& task, cudaStream_t stream) {
+    int n = (int)task.data->row_ptr.size() - 1;
+    float alpha = 1.0f, beta = 0.0f, r1 = 0.0f, a = 0.0f, na = 0.0f, b = 0.0f;
+    float tolerance = 1e-5f;
+    int max_iters = 16000;
+
+    float *d_val, *d_x, *d_r, *d_p, *d_Ap, *d_b;
+    int *d_row_ptr, *d_col_ind;
+
+    CHECK_CUDA(cudaMallocAsync(&d_val, task.data->nnz * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_row_ptr, (n + 1) * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_col_ind, task.data->nnz * sizeof(int), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_x, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_r, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_p, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_Ap, n * sizeof(float), stream));
+    CHECK_CUDA(cudaMallocAsync(&d_b, n * sizeof(float), stream));
+
+    CHECK_CUDA(cudaMemcpyAsync(d_val, task.data->values.data(), task.data->nnz * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_row_ptr, task.data->row_ptr.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_col_ind, task.data->col_indices.data(), task.data->nnz * sizeof(int), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemcpyAsync(d_b, task.data->b.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK_CUDA(cudaMemsetAsync(d_x, 0, n * sizeof(float), stream));
+
+    CHECK_CUBLAS(cublasSetStream(cublas, stream));
+    CHECK_CUSPARSE(cusparseSetStream(cusparse, stream));
+
+    cusparseSpMatDescr_t matA;
+    cusparseDnVecDescr_t vecX, vecP, vecAp;
+
+    CHECK_CUSPARSE(cusparseCreateCsr(&matA, n, n, task.data->nnz, d_row_ptr, d_col_ind, d_val,
+                                     CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                     CUSPARSE_INDEX_BASE_ZERO, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecX, n, d_x, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecP, n, d_p, CUDA_R_32F));
+    CHECK_CUSPARSE(cusparseCreateDnVec(&vecAp, n, d_Ap, CUDA_R_32F));
+
+    size_t bufferSize = 0;
+    void* d_buffer = nullptr;
+    CHECK_CUSPARSE(cusparseSpMV_bufferSize(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                           &alpha, matA, vecP, &beta, vecAp,
+                                           CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    CHECK_CUDA(cudaMallocAsync(&d_buffer, bufferSize, stream));
+
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_b, 1, d_r, 1));
+    CHECK_CUBLAS(cublasScopy(cublas, n, d_r, 1, d_p, 1));
+    CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
+
+    int k = 0;
+    while (k < max_iters) {
+        CHECK_CUSPARSE(cusparseSpMV(cusparse, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                                    &alpha, matA, vecP, &beta, vecAp,
+                                    CUDA_R_32F, CUSPARSE_SPMV_ALG_DEFAULT, d_buffer));
+        float pAp, r0;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_p, 1, d_Ap, 1, &pAp));
+        a = r1 / pAp;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &a, d_p, 1, d_x, 1));
+        na = -a;
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &na, d_Ap, 1, d_r, 1));
+        r0 = r1;
+        CHECK_CUBLAS(cublasSdot(cublas, n, d_r, 1, d_r, 1, &r1));
+        if (std::sqrt(r1) < tolerance) break;
+        b = r1 / r0;
+        CHECK_CUBLAS(cublasSscal(cublas, n, &b, d_p, 1));
+        CHECK_CUBLAS(cublasSaxpy(cublas, n, &alpha, d_r, 1, d_p, 1));
+        k++;
+    }
+
+    CHECK_CUSPARSE(cusparseDestroySpMat(matA));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecX));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecP));
+    CHECK_CUSPARSE(cusparseDestroyDnVec(vecAp));
+    CHECK_CUDA(cudaFreeAsync(d_val, stream));
+    CHECK_CUDA(cudaFreeAsync(d_row_ptr, stream));
+    CHECK_CUDA(cudaFreeAsync(d_col_ind, stream));
+    CHECK_CUDA(cudaFreeAsync(d_x, stream));
+    CHECK_CUDA(cudaFreeAsync(d_r, stream));
+    CHECK_CUDA(cudaFreeAsync(d_p, stream));
+    CHECK_CUDA(cudaFreeAsync(d_Ap, stream));
+    CHECK_CUDA(cudaFreeAsync(d_b, stream));
+    CHECK_CUDA(cudaFreeAsync(d_buffer, stream));
+    return k;
+}
+
+void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue,
+                       std::atomic<int>& succeeded, std::atomic<int>& failed) {
+    CHECK_CUDA(cudaSetDevice(device_id));
+    cudaStream_t stream;
+    cublasHandle_t cublas;
+    cusparseHandle_t cusparse;
+    CHECK_CUDA(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
+    CHECK_CUBLAS(cublasCreate(&cublas));
+    CHECK_CUSPARSE(cusparseCreate(&cusparse));
+
+    MatrixTask task;
+    while (queue.wait_pop(task)) {
+        auto pick_time = std::chrono::steady_clock::now();
+        std::cout << "[WORKER " << worker_id << "] Starting: " << task.path << " (NNZ: " << task.data->nnz << ")" << std::endl;
+
+        int iterations = solve_cg_async(cublas, cusparse, task, stream);
+        CHECK_CUDA(cudaStreamSynchronize(stream));
+
+        auto finish_time = std::chrono::steady_clock::now();
+        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish_time - pick_time).count();
+
+        bool success = (iterations >= 0 && iterations < MAX_ITERATIONS);
+        if (success) {
+            succeeded++;
+        } else {
+            failed++;
+        }
+
+        record_job(task.path, task.enqueue_time, pick_time, finish_time, iterations, success);
+
+        std::cout << "[WORKER " << worker_id << "] Done: " << task.path << " (" << iterations << " iters, " << duration << " ms)." << std::endl;
+    }
+    CHECK_CUBLAS(cublasDestroy(cublas));
+    CHECK_CUSPARSE(cusparseDestroy(cusparse));
+    CHECK_CUDA(cudaStreamDestroy(stream));
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/native_utils.hpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/native_utils.hpp
new file mode 100644
index 0000000000..b9de530986
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/native_utils.hpp
@@ -0,0 +1,95 @@
+#pragma once
+
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+#include <cusparse.h>
+#include <iostream>
+#include <queue>
+#include <mutex>
+#include <condition_variable>
+#include <atomic>
+#include "sparse_utils.hpp"
+
+constexpr int MAX_ITERATIONS = 16000;
+
+// ============================================================
+// Error Checking Macros
+// ============================================================
+
+#define CHECK_CUDA(call)                                                   \
+    do {                                                                   \
+        cudaError_t status = call;                                         \
+        if (status != cudaSuccess) {                                       \
+            std::cerr << "CUDA Error: " << cudaGetErrorString(status)      \
+                      << " at " << __FILE__ << ":" << __LINE__ << std::endl; \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+#define CHECK_CUBLAS(call)                                                 \
+    do {                                                                   \
+        cublasStatus_t status = call;                                      \
+        if (status != CUBLAS_STATUS_SUCCESS) {                             \
+            std::cerr << "cuBLAS Error at "                               \
+                      << __FILE__ << ":" << __LINE__ << std::endl;         \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+#define CHECK_CUSPARSE(call)                                               \
+    do {                                                                   \
+        cusparseStatus_t status = call;                                    \
+        if (status != CUSPARSE_STATUS_SUCCESS) {                           \
+            std::cerr << "cuSPARSE Error at "                             \
+                      << __FILE__ << ":" << __LINE__ << std::endl;         \
+            std::exit(EXIT_FAILURE);                                       \
+        }                                                                  \
+    } while (0)
+
+// ============================================================
+// Thread Safe Queue
+// ============================================================
+
+template<typename T>
+class ThreadSafeQueue {
+public:
+    void push(T item) {
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            queue_.push(std::move(item));
+        }
+        cv_.notify_one();
+    }
+
+    bool wait_pop(T& item) {
+        std::unique_lock<std::mutex> lock(mutex_);
+        cv_.wait(lock, [&] {
+            return shutdown_ || !queue_.empty();
+        });
+
+        if (!queue_.empty()) {
+            item = std::move(queue_.front());
+            queue_.pop();
+            return true;
+        }
+        return false;
+    }
+
+    void signal_shutdown() {
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            shutdown_ = true;
+        }
+        cv_.notify_all();
+    }
+
+private:
+    std::queue<T> queue_;
+    std::mutex mutex_;
+    std::condition_variable cv_;
+    bool shutdown_ = false;
+};
+
+int solve_cg_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream);
+int solve_pcg_jacobi_async(cublasHandle_t cublas, cusparseHandle_t cusparse, const MatrixTask& task, cudaStream_t stream);
+void gpu_stream_worker(int device_id, int worker_id, ThreadSafeQueue<MatrixTask>& queue, std::atomic<int>& succeeded, std::atomic<int>& failed);
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/sparse_utils.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/sparse_utils.cpp
new file mode 100644
index 0000000000..dd09e63fc3
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/sparse_utils.cpp
@@ -0,0 +1,267 @@
+#include "sparse_utils.hpp"
+#include <fstream>
+#include <stdexcept>
+#include <filesystem>
+#include <random>
+#include <mutex>
+#include <algorithm>
+#include <iomanip>
+#include <iostream>
+#include <numeric>
+
+namespace fs = std::filesystem;
+
+SparseMatrixCOO load_binary_coo(const std::string& filepath) {
+    std::ifstream file(filepath, std::ios::binary);
+    if (!file) {
+        throw std::runtime_error("Failed to open matrix file: " + filepath);
+    }
+
+    SparseMatrixCOO coo;
+    
+    file.read(reinterpret_cast<char*>(&coo.rows), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&coo.cols), sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(&coo.nnz), sizeof(int32_t));
+
+    coo.row_indices.resize(coo.nnz);
+    coo.col_indices.resize(coo.nnz);
+    coo.values.resize(coo.nnz);
+
+    file.read(reinterpret_cast<char*>(coo.row_indices.data()), coo.nnz * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(coo.col_indices.data()), coo.nnz * sizeof(int32_t));
+    file.read(reinterpret_cast<char*>(coo.values.data()), coo.nnz * sizeof(float));
+
+    return coo;
+}
+
+SparseMatrixCSR convert_coo_to_csr(const SparseMatrixCOO& coo) {
+    SparseMatrixCSR csr;
+    csr.rows = coo.rows;
+    csr.cols = coo.cols;
+    csr.nnz = coo.nnz;
+    
+    csr.row_ptr.assign(csr.rows + 1, 0);
+    csr.col_indices.resize(csr.nnz);
+    csr.values.resize(csr.nnz);
+
+    for (int32_t i = 0; i < coo.nnz; ++i) {
+        csr.row_ptr[coo.row_indices[i] + 1]++;
+    }
+
+    for (int32_t i = 0; i < csr.rows; ++i) {
+        csr.row_ptr[i + 1] += csr.row_ptr[i];
+    }
+
+    std::vector<int32_t> current_row_pos = csr.row_ptr;
+
+    for (int32_t i = 0; i < coo.nnz; ++i) {
+        int32_t row = coo.row_indices[i];
+        int32_t dest_pos = current_row_pos[row]++;
+        csr.col_indices[dest_pos] = coo.col_indices[i];
+        csr.values[dest_pos] = coo.values[i];
+    }
+
+    return csr;
+}
+
+std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x) {
+    std::vector<float> b(A.rows, 0.0f);
+    
+    for (int32_t i = 0; i < A.rows; ++i) {
+        float sum = 0.0f;
+        int32_t row_start = A.row_ptr[i];
+        int32_t row_end = A.row_ptr[i + 1];
+        
+        for (int32_t j = row_start; j < row_end; ++j) {
+            sum += A.values[j] * x[A.col_indices[j]];
+        }
+        b[i] = sum;
+    }
+    return b;
+}
+
+std::vector<MatrixTask> scan_for_matrices(const std::string& dir, SolverType type) {
+    std::vector<MatrixTask> tasks;
+    if (!fs::exists(dir)) return tasks;
+    for (const auto& entry : fs::directory_iterator(dir)) {
+        if (entry.path().extension() == ".bin") {
+            auto coo = load_binary_coo(entry.path().string());
+            auto csr = convert_coo_to_csr(coo);
+            auto data = std::make_shared<MatrixData>();
+            data->rows = csr.rows;
+            data->cols = csr.cols;
+            data->nnz = csr.nnz;
+            data->b = compute_rhs_spmv(csr, std::vector<float>(csr.cols, 1.0f));
+            data->row_ptr = std::move(csr.row_ptr);
+            data->col_indices = std::move(csr.col_indices);
+            data->values = std::move(csr.values);
+            data->x_guess.assign(data->cols, 0.0f);
+            
+            tasks.push_back({entry.path().string(), type, data});
+        }
+    }
+    return tasks;
+}
+
+
+std::vector<Partition>
+make_contiguous_partitions(size_t num_tasks, size_t rows, size_t cols)
+{
+    size_t num_parts = rows * cols;
+
+    std::vector<Partition> parts;
+    parts.reserve(num_parts);
+
+    size_t base = num_tasks / num_parts;
+    size_t rem  = num_tasks % num_parts;
+
+    size_t current = 0;
+
+    for (size_t p = 0; p < num_parts; ++p) {
+        size_t size = base + (p < rem ? 1 : 0);
+
+        parts.push_back({
+            current,
+            current + size
+        });
+
+        current += size;
+    }
+
+    return parts;
+}
+
+
+int generate_random_sleep_ms(int min_ms, int max_ms) {
+    static std::random_device rd;
+    static std::mt19937 gen(rd());
+    std::uniform_int_distribution<> dis(min_ms, max_ms);
+    return dis(gen);
+}
+
+std::chrono::milliseconds generate_random_interval(
+    std::mt19937& rng,
+    double mean_ms)
+{
+    std::exponential_distribution<double> dist(
+        1.0 / mean_ms);
+
+    return std::chrono::milliseconds(
+        static_cast<int>(dist(rng)));
+}
+
+std::vector<MatrixTask> generate_batch(
+    const std::vector<MatrixTask>& matrix_pool,
+    std::mt19937& rng,
+    size_t batch_size)
+{
+    std::vector<MatrixTask> batch;
+    batch.reserve(batch_size);
+
+    std::uniform_int_distribution<size_t> dist(
+        0,
+        matrix_pool.size() - 1);
+
+    for (size_t i = 0; i < batch_size; ++i) {
+        batch.push_back(matrix_pool[dist(rng)]);
+    }
+
+    return batch;
+}
+
+static std::vector<JobStats> global_stats;
+static std::mutex stats_mutex;
+static std::chrono::steady_clock::time_point benchmark_start_tp;
+
+void init_benchmark_timer() {
+    benchmark_start_tp = std::chrono::steady_clock::now();
+}
+
+void record_job(const std::string& name,
+                std::chrono::steady_clock::time_point enqueue_time,
+                std::chrono::steady_clock::time_point pick_time,
+                std::chrono::steady_clock::time_point finish_time,
+                int iterations, bool success) {
+    std::lock_guard<std::mutex> lock(stats_mutex);
+    
+    auto wait = std::chrono::duration<double, std::milli>(pick_time - enqueue_time).count();
+    auto total = std::chrono::duration<double, std::milli>(finish_time - enqueue_time).count();
+    auto finish_rel = std::chrono::duration<double, std::milli>(finish_time - benchmark_start_tp).count();
+
+    global_stats.push_back({name, wait, total, iterations, success, finish_rel});
+}
+
+void report_workload_stats() {
+    std::lock_guard<std::mutex> lock(stats_mutex);
+    if (global_stats.empty()) {
+        std::cout << "No job statistics recorded.\n";
+        return;
+    }
+
+    int total_iters = 0;
+    int success_count = 0;
+    double wasted_gpu_time_ms = 0;
+    int failed_count = 0;
+    std::vector<double> completions;
+    completions.reserve(global_stats.size());
+
+    // Ensure total_jobs is not zero to avoid division by zero
+    size_t total_jobs = global_stats.size();
+
+    for (const auto& s : global_stats) {
+        total_iters += s.iterations;
+        if (s.success) success_count++;
+        else wasted_gpu_time_ms += (s.completion_time_ms - s.wait_time_ms);
+        
+        completions.push_back(s.completion_time_ms);
+    }
+
+    std::sort(completions.begin(), completions.end());
+    failed_count = total_jobs - success_count;
+    double success_percentage = (total_jobs > 0) ? (100.0 * success_count / total_jobs) : 0.0;
+    double failed_percentage = (total_jobs > 0) ? (100.0 * failed_count / total_jobs) : 0.0;
+    double mean = (total_jobs > 0) ? std::accumulate(completions.begin(), completions.end(), 0.0) / total_jobs : 0.0;
+    double median = (total_jobs > 0) ? completions[total_jobs / 2] : 0.0;
+    double p95 = completions[static_cast<size_t>(completions.size() * 0.95)];
+
+    auto max_finish = std::max_element(global_stats.begin(), global_stats.end(), [](const JobStats& a, const JobStats& b) {
+        return a.finish_relative_ms < b.finish_relative_ms;
+    });
+
+    std::cout << "\n" << std::string(45, '=') << "\n";
+    std::cout << "          WORKLOAD PERFORMANCE REPORT\n";
+    std::cout << std::string(45, '=') << "\n";
+    std::cout << std::left << std::setw(25) << "Total Jobs:" << total_jobs << "\n";
+    std::cout << std::left << std::setw(25) << "Succeeded Jobs:" << success_count << " (" << std::fixed << std::setprecision(2) << success_percentage << "%)\n";
+    std::cout << std::left << std::setw(25) << "Failed Jobs:" << failed_count << " (" << std::fixed << std::setprecision(2) << failed_percentage << "%)\n";
+    std::cout << std::left << std::setw(25) << "Total Iterations:" << total_iters << "\n";
+    std::cout << std::left << std::setw(25) << "Cumul. Wasted GPU Time:" << wasted_gpu_time_ms / 1000.0 << " s (all streams)\n";
+    std::cout << std::left << std::setw(25) << "Makespan:" << max_finish->finish_relative_ms / 1000.0 << " s\n";
+    std::cout << std::left << std::setw(25) << "Mean Completion:" << mean << " ms\n";
+    std::cout << std::left << std::setw(25) << "Median Completion:" << median << " ms\n";
+    std::cout << std::left << std::setw(25) << "95th Percentile:" << p95 << " ms\n";
+    
+    std::cout << "\nThroughput Timeline (Job Completion & Success vs Wall-clock):\n";
+    std::sort(global_stats.begin(), global_stats.end(), [](const JobStats& a, const JobStats& b) {
+        return a.finish_relative_ms < b.finish_relative_ms;
+    });
+    
+    double total_time = max_finish->finish_relative_ms;
+    for (int i = 1; i <= 10; ++i) {
+        double threshold = (total_time / 10.0) * i;
+        auto it = std::upper_bound(global_stats.begin(), global_stats.end(), threshold, 
+            [](double val, const JobStats& s) {
+                return val < s.finish_relative_ms;
+            });
+        size_t total_at_t = std::distance(global_stats.begin(), it);
+        size_t success_at_t = 0;
+        for (auto s_it = global_stats.begin(); s_it != it; ++s_it) {
+            if (s_it->success) success_at_t++;
+        }
+
+        std::cout << "  T + " << std::setw(5) << std::fixed << std::setprecision(0) << threshold 
+                  << " ms | Total Progress: " << std::setw(3) << (100 * total_at_t / total_jobs) 
+                  << "% | Successful solves: " << std::setw(3) << (100 * success_at_t / total_jobs) << "%\n";
+    }
+    std::cout << std::string(45, '=') << "\n\n";
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/sparse_utils.hpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/sparse_utils.hpp
new file mode 100644
index 0000000000..eef98a0456
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/sparse_utils.hpp
@@ -0,0 +1,94 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <cstdint>
+#include <memory>
+#include <random>
+#include <chrono>
+
+enum SolverType { CGS_SOLVER, BICSTAB_SOLVER };
+
+// Structure to hold raw data from the binary file
+struct SparseMatrixCOO {
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+    std::vector<int32_t> row_indices;
+    std::vector<int32_t> col_indices;
+    std::vector<float> values;
+};
+
+// Structure optimized for high-performance solvers
+struct SparseMatrixCSR {
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+    std::vector<int32_t> row_ptr;    // Size: rows + 1
+    std::vector<int32_t> col_indices;// Size: nnz
+    std::vector<float> values;       // Size: nnz
+};
+
+struct MatrixData {
+    std::vector<int32_t> row_ptr;
+    std::vector<int32_t> col_indices;
+    std::vector<float> values;
+    std::vector<float> b;
+    std::vector<float> x_guess;
+    int32_t rows;
+    int32_t cols;
+    int32_t nnz;
+};
+
+struct MatrixTask {
+    std::string path;
+    SolverType type;
+    std::shared_ptr<MatrixData> data;
+    std::chrono::steady_clock::time_point enqueue_time;
+};
+
+struct JobStats {
+    std::string task_name;
+    double wait_time_ms;       // Time spent in queue
+    double completion_time_ms; // Total turnaround time (enqueue to finish)
+    int iterations;
+    bool success;
+    double finish_relative_ms; // Wall-clock timestamp relative to benchmark start
+};
+
+void init_benchmark_timer();
+void record_job(const std::string& name,
+                std::chrono::steady_clock::time_point enqueue_time,
+                std::chrono::steady_clock::time_point pick_time,
+                std::chrono::steady_clock::time_point finish_time,
+                int iterations, bool success);
+void report_workload_stats();
+
+struct Partition {
+    size_t begin;
+    size_t end;
+    std::vector<int> devices;
+    std::vector<int> streams;
+};
+
+// Function to slurp the binary data into memory
+SparseMatrixCOO load_binary_coo(const std::string& filepath);
+
+// Converts COO to CSR format for solver compatibility
+SparseMatrixCSR convert_coo_to_csr(const SparseMatrixCOO& coo);
+
+// Compute b = A * x using CSR layout (Sparse Matrix-Vector Multiplication)
+std::vector<float> compute_rhs_spmv(const SparseMatrixCSR& A, const std::vector<float>& x);
+
+std::vector<MatrixTask> scan_for_matrices(const std::string& dir, SolverType type);
+int generate_random_sleep_ms(int min_ms, int max_ms);
+
+// Workload generation helpers
+std::chrono::milliseconds generate_random_interval(
+    std::mt19937& rng,
+    double mean_ms);
+
+std::vector<MatrixTask> generate_batch(
+    const std::vector<MatrixTask>& matrix_pool,
+    std::mt19937& rng,
+    size_t batch_size);
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/stability_kernels.cu b/libcaf_cuda/sc26/Irregular-Workload/workloadB/stability_kernels.cu
new file mode 100644
index 0000000000..56d38d3694
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/stability_kernels.cu
@@ -0,0 +1,10 @@
+extern "C" __global__
+void check_stability(int n, const float* x, const float* r, int* err) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx < n) {
+        // If any value in the solution or residual is invalid, set the error flag
+        if (isnan(x[idx]) || isinf(x[idx]) || isnan(r[idx]) || isinf(r[idx])) {
+            atomicExch(err, 1);
+        }
+    }
+}
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/supervisor_actor.hpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/supervisor_actor.hpp
new file mode 100644
index 0000000000..dc68991c36
--- /dev/null
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/supervisor_actor.hpp
@@ -0,0 +1,224 @@
+#pragma once
+
+#include <caf/all.hpp>
+#include <caf/cuda/all.hpp>
+#include <deque>
+#include <unordered_map>
+#include <vector>
+#include <chrono>
+#include "atoms.hpp"
+#include "ft_cg_actor.hpp"
+#include "sparse_utils.hpp"
+
+
+using namespace caf;
+using namespace caf::cuda;
+// ---------------------------- SUPERVISOR ACTOR ----------------------------
+
+struct suspended_task {
+  caf::actor solver;
+  std::string path;
+  int last_batch_size;
+  int device_id;
+  int stream_id;
+};
+
+struct resource_slot {
+  int device_id;
+  int stream_id;
+};
+
+struct supervisor_state {
+  std::deque<MatrixTask> queue;
+  std::deque<suspended_task> suspended_queue;
+  std::vector<caf::actor> active_solvers;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> start_times;
+  std::unordered_map<std::string, resource_slot> task_resources;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> enqueue_times;
+  std::unordered_map<std::string, std::chrono::steady_clock::time_point> pick_times;
+  std::unordered_map<std::string, double> cumulative_active_ms;
+  std::unordered_map<caf::actor, int> actor_batch_sizes;
+  std::deque<resource_slot> available_slots;
+  int max_active = 1; // Admission control limit to be mindful of GPU memory.
+  int num_iterations = MAX_ITERATIONS / 2;
+  int num_gpus = 0;
+  int tasks_succeeded = 0;
+  int tasks_failed = 0;
+  std::chrono::steady_clock::time_point benchmark_start;
+};
+
+behavior supervisor_actor(stateful_actor<supervisor_state>* self, std::vector<MatrixTask> tasks, int initial_max_active, std::chrono::steady_clock::time_point start_time) {
+    auto& st = self->state();
+    st.queue.insert(st.queue.end(), std::make_move_iterator(tasks.begin()), std::make_move_iterator(tasks.end()));
+    st.max_active = initial_max_active;
+    st.benchmark_start = start_time;
+    st.num_gpus = manager::get().get_num_devices();
+
+    // Initialize the pool with streams interleaved across all available GPUs
+    for (int s = 0; s < 32; ++s) {
+      for (int g = 0; g < st.num_gpus; ++g) {
+        st.available_slots.push_back({g, s});
+      }
+    }
+
+    auto spawn_next = [self]() {
+        auto& s = self->state();
+        while (s.active_solvers.size() < static_cast<size_t>(s.max_active) && !s.available_slots.empty()) {
+          if (!s.queue.empty()) {
+            auto task = std::move(s.queue.front());
+            s.queue.pop_front();
+            std::string path = task.path;
+
+            s.cumulative_active_ms[path] = 0;
+            s.enqueue_times[path] = task.enqueue_time;
+            s.pick_times[path] = std::chrono::steady_clock::now();
+
+            resource_slot slot = s.available_slots.front();
+            s.available_slots.pop_front();
+
+            self->println("[INFO] Starting solver for: {} (Device: {}, Stream: {})", 
+                          path, slot.device_id, slot.stream_id);
+            auto solver = self->spawn(fault_tolerant_cg_actor<float>,
+                                    path,
+                                    task.data,
+                                    create_in_arg(task.data->row_ptr),
+                                    create_in_arg(task.data->col_indices),
+                                    create_in_arg(task.data->values),
+                                    create_in_arg(task.data->b),
+                                    create_in_out_arg(task.data->x_guess),
+                                    (int)task.data->row_ptr.size() - 1,
+                                    (int)task.data->values.size(),
+                                    1e-5f, MAX_ITERATIONS, slot.device_id, slot.stream_id, actor_cast<actor>(self));
+
+            s.start_times[path] = std::chrono::steady_clock::now();
+            s.active_solvers.push_back(solver);
+            s.task_resources[path] = slot;
+            s.actor_batch_sizes[solver] = s.num_iterations;
+            self->mail(start_atom_v).send(solver);
+          } else {
+            bool resumed = false;
+            for (auto it = s.suspended_queue.begin(); it != s.suspended_queue.end(); ++it) {
+              auto slot_it = std::find_if(s.available_slots.begin(), s.available_slots.end(), [&](const resource_slot& slot) {
+                return slot.device_id == it->device_id && slot.stream_id == it->stream_id;
+              });
+              if (slot_it != s.available_slots.end()) {
+                auto suspended = std::move(*it);
+                s.suspended_queue.erase(it);
+                resource_slot slot = *slot_it;
+                s.available_slots.erase(slot_it);
+
+                s.pick_times[suspended.path] = std::chrono::steady_clock::now();
+
+                // Cap the minimum batch size to MAX_ITERATIONS / 8
+                int next_batch = std::max(MAX_ITERATIONS / 8, suspended.last_batch_size / 2);
+                self->println("[INFO] Resuming solver for: {} (Device: {}, Stream: {}, Batch: {})", 
+                              suspended.path, slot.device_id, slot.stream_id, next_batch);
+
+                // Resume on the same stream ID. No update_stream_atom_v needed.
+                self->mail(cg_next_step_atom_v, next_batch).send(suspended.solver);
+
+                s.active_solvers.push_back(suspended.solver);
+                s.task_resources[suspended.path] = slot;
+                s.actor_batch_sizes[suspended.solver] = next_batch;
+                resumed = true;
+                break;
+              }
+            }
+            if (!resumed) break;
+          }
+        }
+    };
+
+    spawn_next();
+    
+    return {
+        [=](gpu_done_atom, const std::string& task_name, caf::actor solver, std::vector<float>& solution, solver_result_meta meta) {
+            auto& s = self->state();
+
+            if (meta.converged || meta.error_code != CG_SUCCESS) {
+                auto end_time = std::chrono::steady_clock::now();
+                auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
+                                  end_time - s.start_times[task_name]).count();
+
+                if (meta.converged && meta.iterations < MAX_ITERATIONS) {
+                    s.tasks_succeeded++;
+                    if (meta.iterations == 0)
+                        self->println("[DONE] {}: Initial guess satisfied tolerance ({} ms).", task_name, duration);
+                    else
+                        self->println("[DONE] {}: Converged in {} iterations ({} ms).", task_name, meta.iterations, duration);
+                } else {
+                    s.tasks_failed++;
+                    std::string reason = (meta.error_code == CG_SUCCESS)
+                                         ? "Maximum Iterations Reached"
+                                         : to_string(static_cast<cg_error_type>(meta.error_code));
+                    self->println("[FAIL] {}: {} (after {} iterations, {} ms).", 
+                                  task_name, reason,
+                                  meta.iterations,
+                                  duration);
+                }
+
+                // Final active slice
+                auto active_slice = std::chrono::duration<double, std::milli>(end_time - s.pick_times[task_name]).count();
+                s.cumulative_active_ms[task_name] += active_slice;
+
+                // We override pick_time in record_job to simulate a single continuous run that equals 
+                // the actual time spent on the GPU.
+                auto simulated_pick = end_time - std::chrono::duration_cast<std::chrono::steady_clock::duration>(
+                                                std::chrono::duration<double, std::milli>(s.cumulative_active_ms[task_name]));
+
+                record_job(task_name, s.enqueue_times[task_name], simulated_pick, end_time, meta.iterations, meta.converged);
+                s.enqueue_times.erase(task_name);
+                s.pick_times.erase(task_name);
+                s.cumulative_active_ms.erase(task_name);
+
+                auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
+                if (it != s.active_solvers.end())
+                    s.active_solvers.erase(it);
+                s.start_times.erase(task_name);
+                s.actor_batch_sizes.erase(solver);
+
+                // Reclaim the device/stream slot and put it back in the pool
+                auto res_it = s.task_resources.find(task_name);
+                if (res_it != s.task_resources.end()) {
+                    s.available_slots.push_back(res_it->second);
+                    s.task_resources.erase(res_it);
+                }
+
+                spawn_next();
+
+                if (s.active_solvers.empty() && s.queue.empty() && s.suspended_queue.empty()) {
+                    self->println("All tasks in the pool have been processed.");
+
+                    report_workload_stats();
+                    self->quit();
+                }
+            } else if (meta.iterations == 0) {
+                // Just finished initialization: trigger the first iteration batch immediately.
+                self->mail(cg_next_step_atom_v, s.num_iterations).send(solver);
+            } else {
+                // Not done: Suspend the actor to allow others to use the stream
+                auto it = std::find(s.active_solvers.begin(), s.active_solvers.end(), solver);
+                
+                // Record the time spent in this active slice before suspending
+                auto active_slice = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - s.pick_times[task_name]).count();
+                s.cumulative_active_ms[task_name] += active_slice;
+
+                if (it != s.active_solvers.end())
+                    s.active_solvers.erase(it);
+
+                resource_slot slot = s.task_resources[task_name];
+                s.available_slots.push_back(slot);
+                s.task_resources.erase(task_name);
+
+                int last_batch = s.actor_batch_sizes[solver];
+                s.suspended_queue.push_back({solver, task_name, last_batch, slot.device_id, slot.stream_id});
+
+                self->println("[INFO] Suspending solver for: {} (Reclaimed Device: {}, Stream: {})", 
+                              task_name, slot.device_id, slot.stream_id);
+
+                spawn_next();
+            }
+        }
+       
+    };
+}

From 3215412bb54b12347e1e4e2a9fcc42f3811d54da Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 11:12:53 -0600
Subject: [PATCH 0970/1000] added script to download matrix from suitesparse
 collection

---
 .../workloadA/download-matrices.py            | 161 ++++++++++++++++++
 .../workloadA/valid_matrix_subset.txt         | 143 ++++++++++++++++
 2 files changed, 304 insertions(+)
 create mode 100644 libcaf_cuda/sc26/scripts/Irregular-Workload/workloadA/download-matrices.py
 create mode 100644 libcaf_cuda/sc26/scripts/Irregular-Workload/workloadA/valid_matrix_subset.txt

diff --git a/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadA/download-matrices.py b/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadA/download-matrices.py
new file mode 100644
index 0000000000..3b3a8e9aa0
--- /dev/null
+++ b/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadA/download-matrices.py
@@ -0,0 +1,161 @@
+import os
+import sys
+import json
+import struct
+import shutil
+import numpy as np
+import ssgetpy
+
+from scipy.io import mmread
+from scipy.sparse import issparse
+
+# ============================================================
+# CONFIGURATION
+# ============================================================
+INPUT_FILE = "valid_matrix_subset.txt"
+OUTPUT_DIR = "./downloaded_matrices"
+
+MATRIX_DIR = os.path.join(OUTPUT_DIR, "matrices", "spd")
+METADATA_DIR = os.path.join(OUTPUT_DIR, "metadata", "spd")
+INDEX_PATH = os.path.join(OUTPUT_DIR, "spd_index.json")
+
+# ============================================================
+# SETUP & PARSING
+# ============================================================
+def setup_directories():
+    os.makedirs(MATRIX_DIR, exist_ok=True)
+    os.makedirs(METADATA_DIR, exist_ok=True)
+
+def parse_matrix_names(file_path):
+    """Extracts matrix names from the file paths (e.g., '1138_bus' from '.../1138_bus.bin')"""
+    if not os.path.exists(file_path):
+        print(f"[ERROR] Input file {file_path} not found.")
+        sys.exit(1)
+        
+    matrix_names = []
+    with open(file_path, "r") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                # Get the filename (e.g., 1138_bus.bin) and strip extension
+                base = os.path.basename(line)
+                name, _ = os.path.splitext(base)
+                matrix_names.append(name)
+    return matrix_names
+
+# ============================================================
+# UTILITIES
+# ============================================================
+def find_mtx(meta_name):
+    for root, dirs, files in os.walk(os.path.expanduser("~/.ssgetpy")):
+        for file in files:
+            if file.lower() == f"{meta_name.lower()}.mtx":
+                return os.path.join(root, file)
+    return None
+
+def export_binary(csr, output_path):
+    coo = csr.tocoo()
+    rows = int(coo.shape[0])
+    cols = int(coo.shape[1])
+    nnz = int(coo.nnz)
+
+    row_indices = coo.row.astype(np.int32)
+    col_indices = coo.col.astype(np.int32)
+    values = coo.data.astype(np.float32)
+
+    with open(output_path, "wb") as f:
+        f.write(struct.pack("iii", rows, cols, nnz))
+        f.write(row_indices.tobytes())
+        f.write(col_indices.tobytes())
+        f.write(values.tobytes())
+
+# ============================================================
+# MAIN EXECUTION
+# ============================================================
+if __name__ == "__main__":
+    setup_directories()
+    matrix_names = parse_matrix_names(INPUT_FILE)
+    
+    print(f"[INFO] Found {len(matrix_names)} matrices to process from {INPUT_FILE}")
+    index_entries = []
+
+    for idx, name in enumerate(matrix_names, 1):
+        print(f"\n[INFO] Processing [{idx}/{len(matrix_names)}]: {name}")
+        
+        # Search for exact name match in SuiteSparse
+        results = ssgetpy.search(name=name)
+        if not results:
+            print(f"[WARN] Matrix '{name}' not found in SuiteSparse Collection. Skipping.")
+            continue
+            
+        # Grab the first match (ssgetpy ranks exact matches highly)
+        meta = results[0]
+        
+        try:
+            matrices = ssgetpy.fetch(meta.id)
+            if matrices:
+                matrices.download(extract=True)
+            else:
+                print(f"[SKIP] Fetch failed for {meta.name}")
+                continue
+        except Exception as e:
+            print(f"[WARN] Download failed for {meta.name}: {e}")
+            continue
+
+        # Locate downloaded .mtx file
+        mtx_file = find_mtx(meta.name)
+        if mtx_file is None:
+            print(f"[SKIP] no .mtx file found locally for: {meta.name}")
+            continue
+
+        # Process and write file
+        try:
+            mat = mmread(mtx_file)
+            if not issparse(mat):
+                print(f"[SKIP] Not a sparse matrix: {meta.name}")
+                continue
+
+            csr = mat.tocsr()
+            
+            # Paths configuration
+            bin_path = os.path.join(MATRIX_DIR, f"{name}.bin")
+            meta_path = os.path.join(METADATA_DIR, f"{name}.json")
+
+            # Export custom binary format
+            export_binary(csr, bin_path)
+
+            # Build metadata object
+            metadata = {
+                "id": meta.id,
+                "name": meta.name,
+                "group": meta.group,
+                "rows": int(csr.shape[0]),
+                "cols": int(csr.shape[1]),
+                "nnz": int(csr.nnz),
+                "paths": {
+                    "binary": bin_path
+                }
+            }
+
+            with open(meta_path, "w") as f:
+                json.dump(metadata, f, indent=2)
+
+            index_entries.append({
+                "name": meta.name,
+                "binary": bin_path,
+                "metadata": meta_path
+            })
+
+            print(f"[PASS] Successfully processed and saved {meta.name}")
+
+        except Exception as e:
+            print(f"[WARN] Failed processing data for {meta.name}: {e}")
+
+        # Clean cache to prevent disk explosion
+        shutil.rmtree(os.path.expanduser("~/.ssgetpy"), ignore_errors=True)
+
+    # Save the execution index mapping
+    with open(INDEX_PATH, "w") as f:
+        json.dump(index_entries, f, indent=2)
+
+    print(f"\n[DONE] Pipeline complete. Saved execution index map to {INDEX_PATH}")
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadA/valid_matrix_subset.txt b/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadA/valid_matrix_subset.txt
new file mode 100644
index 0000000000..e628361812
--- /dev/null
+++ b/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadA/valid_matrix_subset.txt
@@ -0,0 +1,143 @@
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/1138_bus.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/2cubes_sphere.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/662_bus.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/685_bus.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Andrews.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Chem97ZtZ.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Dubcova1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Dubcova2.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Dubcova3.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/G2_circuit.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/G3_circuit.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Journals.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Kuu.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Muu.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Pres_Poisson.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Spielman_k100.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Trefethen_150.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Trefethen_200.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Trefethen_2000.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Trefethen_20000.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Trefethen_20000b.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Trefethen_200b.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Trefethen_300.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Trefethen_500.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/Trefethen_700.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/af_shell3.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/af_shell4.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/af_shell7.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/af_shell8.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/aft01.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/apache1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/apache2.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk02.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk04.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk05.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk06.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk07.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk08.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk09.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk10.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk11.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk12.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk13.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk14.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk15.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk16.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk17.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk18.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk21.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk26.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk27.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk28.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstk34.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstm07.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstm12.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstm21.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstm23.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstm24.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstm25.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bcsstm39.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bloweybq.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bmwcra_1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bodyy4.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bodyy5.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/bodyy6.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/boneS01.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/cant.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/cbuckle.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/cfd1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/cfd2.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/consph.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/crankseg_1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/crankseg_2.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/crystm01.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/crystm02.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/crystm03.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/cvxbqp1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/denormal.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/ecology2.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/finan512.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/fv1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/fv2.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/fv3.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/gr_30_30.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/gridgena.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/gyro_m.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/hood.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/jnlbrng1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/lund_a.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/lund_b.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/mesh2e1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/mesh2em5.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/mhd3200b.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/mhd4800b.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/mhdb416.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/minsurfo.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/msc00726.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/msc01050.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/msc01440.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/msc04515.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/msc10848.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/nasa1824.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/nasa2146.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/nasa2910.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/nasa4704.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/nd12k.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/nd3k.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/nd6k.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/nos3.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/nos5.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/nos6.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/nos7.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/obstclae.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/parabolic_fem.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/plat1919.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/plat362.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/plbuckle.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/qa8fm.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/s1rmq4m1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/s1rmt3m1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/s2rmq4m1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/s2rmt3m1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/shallow_water1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/shallow_water2.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/smt.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/sts4098.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/t2dah_e.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/t2dal_e.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/t3dl_e.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/ted_B.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/ted_B_unscaled.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/thermal1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/thermal2.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/thermomech_TC.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/thermomech_TK.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/thermomech_dM.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/tmt_sym.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/torsion1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/uni_chimera_i1.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/uni_chimera_i2.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/uni_chimera_i5.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/wathen100.bin
+/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/spd/wathen120.bin

From 4ec637dbf4d8578e35373d49ef3f9bc6f9688e42 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 11:16:52 -0600
Subject: [PATCH 0971/1000] add scripts to download workloadB matrices from
 suitesparse collection

---
 .../workloadB/download-matrices.py            | 161 ++++++++++++++++++
 .../workloadB/valid_matrix_subset.txt         |  40 +++++
 2 files changed, 201 insertions(+)
 create mode 100644 libcaf_cuda/sc26/scripts/Irregular-Workload/workloadB/download-matrices.py
 create mode 100644 libcaf_cuda/sc26/scripts/Irregular-Workload/workloadB/valid_matrix_subset.txt

diff --git a/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadB/download-matrices.py b/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadB/download-matrices.py
new file mode 100644
index 0000000000..3b3a8e9aa0
--- /dev/null
+++ b/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadB/download-matrices.py
@@ -0,0 +1,161 @@
+import os
+import sys
+import json
+import struct
+import shutil
+import numpy as np
+import ssgetpy
+
+from scipy.io import mmread
+from scipy.sparse import issparse
+
+# ============================================================
+# CONFIGURATION
+# ============================================================
+INPUT_FILE = "valid_matrix_subset.txt"
+OUTPUT_DIR = "./downloaded_matrices"
+
+MATRIX_DIR = os.path.join(OUTPUT_DIR, "matrices", "spd")
+METADATA_DIR = os.path.join(OUTPUT_DIR, "metadata", "spd")
+INDEX_PATH = os.path.join(OUTPUT_DIR, "spd_index.json")
+
+# ============================================================
+# SETUP & PARSING
+# ============================================================
+def setup_directories():
+    os.makedirs(MATRIX_DIR, exist_ok=True)
+    os.makedirs(METADATA_DIR, exist_ok=True)
+
+def parse_matrix_names(file_path):
+    """Extracts matrix names from the file paths (e.g., '1138_bus' from '.../1138_bus.bin')"""
+    if not os.path.exists(file_path):
+        print(f"[ERROR] Input file {file_path} not found.")
+        sys.exit(1)
+        
+    matrix_names = []
+    with open(file_path, "r") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                # Get the filename (e.g., 1138_bus.bin) and strip extension
+                base = os.path.basename(line)
+                name, _ = os.path.splitext(base)
+                matrix_names.append(name)
+    return matrix_names
+
+# ============================================================
+# UTILITIES
+# ============================================================
+def find_mtx(meta_name):
+    for root, dirs, files in os.walk(os.path.expanduser("~/.ssgetpy")):
+        for file in files:
+            if file.lower() == f"{meta_name.lower()}.mtx":
+                return os.path.join(root, file)
+    return None
+
+def export_binary(csr, output_path):
+    coo = csr.tocoo()
+    rows = int(coo.shape[0])
+    cols = int(coo.shape[1])
+    nnz = int(coo.nnz)
+
+    row_indices = coo.row.astype(np.int32)
+    col_indices = coo.col.astype(np.int32)
+    values = coo.data.astype(np.float32)
+
+    with open(output_path, "wb") as f:
+        f.write(struct.pack("iii", rows, cols, nnz))
+        f.write(row_indices.tobytes())
+        f.write(col_indices.tobytes())
+        f.write(values.tobytes())
+
+# ============================================================
+# MAIN EXECUTION
+# ============================================================
+if __name__ == "__main__":
+    setup_directories()
+    matrix_names = parse_matrix_names(INPUT_FILE)
+    
+    print(f"[INFO] Found {len(matrix_names)} matrices to process from {INPUT_FILE}")
+    index_entries = []
+
+    for idx, name in enumerate(matrix_names, 1):
+        print(f"\n[INFO] Processing [{idx}/{len(matrix_names)}]: {name}")
+        
+        # Search for exact name match in SuiteSparse
+        results = ssgetpy.search(name=name)
+        if not results:
+            print(f"[WARN] Matrix '{name}' not found in SuiteSparse Collection. Skipping.")
+            continue
+            
+        # Grab the first match (ssgetpy ranks exact matches highly)
+        meta = results[0]
+        
+        try:
+            matrices = ssgetpy.fetch(meta.id)
+            if matrices:
+                matrices.download(extract=True)
+            else:
+                print(f"[SKIP] Fetch failed for {meta.name}")
+                continue
+        except Exception as e:
+            print(f"[WARN] Download failed for {meta.name}: {e}")
+            continue
+
+        # Locate downloaded .mtx file
+        mtx_file = find_mtx(meta.name)
+        if mtx_file is None:
+            print(f"[SKIP] no .mtx file found locally for: {meta.name}")
+            continue
+
+        # Process and write file
+        try:
+            mat = mmread(mtx_file)
+            if not issparse(mat):
+                print(f"[SKIP] Not a sparse matrix: {meta.name}")
+                continue
+
+            csr = mat.tocsr()
+            
+            # Paths configuration
+            bin_path = os.path.join(MATRIX_DIR, f"{name}.bin")
+            meta_path = os.path.join(METADATA_DIR, f"{name}.json")
+
+            # Export custom binary format
+            export_binary(csr, bin_path)
+
+            # Build metadata object
+            metadata = {
+                "id": meta.id,
+                "name": meta.name,
+                "group": meta.group,
+                "rows": int(csr.shape[0]),
+                "cols": int(csr.shape[1]),
+                "nnz": int(csr.nnz),
+                "paths": {
+                    "binary": bin_path
+                }
+            }
+
+            with open(meta_path, "w") as f:
+                json.dump(metadata, f, indent=2)
+
+            index_entries.append({
+                "name": meta.name,
+                "binary": bin_path,
+                "metadata": meta_path
+            })
+
+            print(f"[PASS] Successfully processed and saved {meta.name}")
+
+        except Exception as e:
+            print(f"[WARN] Failed processing data for {meta.name}: {e}")
+
+        # Clean cache to prevent disk explosion
+        shutil.rmtree(os.path.expanduser("~/.ssgetpy"), ignore_errors=True)
+
+    # Save the execution index mapping
+    with open(INDEX_PATH, "w") as f:
+        json.dump(index_entries, f, indent=2)
+
+    print(f"\n[DONE] Pipeline complete. Saved execution index map to {INDEX_PATH}")
\ No newline at end of file
diff --git a/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadB/valid_matrix_subset.txt b/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadB/valid_matrix_subset.txt
new file mode 100644
index 0000000000..b2259ade4c
--- /dev/null
+++ b/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadB/valid_matrix_subset.txt
@@ -0,0 +1,40 @@
+/scratch/nqr159/matrix-collection/matrices/mixed/1138_bus.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/662_bus.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/685_bus.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk02.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk04.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk05.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk06.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk07.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk08.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk09.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk10.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk11.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk12.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk13.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk14.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk15.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk16.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk17.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk18.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstk19.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstm10.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstm13.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bcsstm27.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/beacxc.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/beaflw.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/beause.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bp_0.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bp_1000.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bp_1200.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bp_1400.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bp_1600.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bp_200.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bp_400.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bp_600.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/bp_800.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/fs_541_1.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/fs_541_2.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/fs_541_3.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/fs_541_4.bin
+/scratch/nqr159/matrix-collection/matrices/mixed/fs_680_1.bin

From 4aa7e8b1e232de51cf5321c31eff1daeae74207c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 11:24:32 -0600
Subject: [PATCH 0972/1000] Updated scripts.

---
 .../workloadA/download-matrices.py            | 25 +++++++------------
 .../workloadB/download-matrices.py            | 25 +++++++------------
 2 files changed, 18 insertions(+), 32 deletions(-)

diff --git a/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadA/download-matrices.py b/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadA/download-matrices.py
index 3b3a8e9aa0..838e1c7039 100644
--- a/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadA/download-matrices.py
+++ b/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadA/download-matrices.py
@@ -12,12 +12,13 @@
 # ============================================================
 # CONFIGURATION
 # ============================================================
-INPUT_FILE = "valid_matrix_subset.txt"
+INPUT_FILE = "valid_matrix_subset.txt"  # Your generated text file
 OUTPUT_DIR = "./downloaded_matrices"
 
-MATRIX_DIR = os.path.join(OUTPUT_DIR, "matrices", "spd")
-METADATA_DIR = os.path.join(OUTPUT_DIR, "metadata", "spd")
-INDEX_PATH = os.path.join(OUTPUT_DIR, "spd_index.json")
+# No more "spd" subdirectories; everything goes directly into these folders
+MATRIX_DIR = os.path.join(OUTPUT_DIR, "matrices")
+METADATA_DIR = os.path.join(OUTPUT_DIR, "metadata")
+INDEX_PATH = os.path.join(OUTPUT_DIR, "collection_index.json")
 
 # ============================================================
 # SETUP & PARSING
@@ -27,7 +28,7 @@ def setup_directories():
     os.makedirs(METADATA_DIR, exist_ok=True)
 
 def parse_matrix_names(file_path):
-    """Extracts matrix names from the file paths (e.g., '1138_bus' from '.../1138_bus.bin')"""
+    """Extracts matrix names from the file paths or raw lists."""
     if not os.path.exists(file_path):
         print(f"[ERROR] Input file {file_path} not found.")
         sys.exit(1)
@@ -37,7 +38,6 @@ def parse_matrix_names(file_path):
         for line in f:
             line = line.strip()
             if line:
-                # Get the filename (e.g., 1138_bus.bin) and strip extension
                 base = os.path.basename(line)
                 name, _ = os.path.splitext(base)
                 matrix_names.append(name)
@@ -82,13 +82,11 @@ def export_binary(csr, output_path):
     for idx, name in enumerate(matrix_names, 1):
         print(f"\n[INFO] Processing [{idx}/{len(matrix_names)}]: {name}")
         
-        # Search for exact name match in SuiteSparse
         results = ssgetpy.search(name=name)
         if not results:
             print(f"[WARN] Matrix '{name}' not found in SuiteSparse Collection. Skipping.")
             continue
             
-        # Grab the first match (ssgetpy ranks exact matches highly)
         meta = results[0]
         
         try:
@@ -102,13 +100,12 @@ def export_binary(csr, output_path):
             print(f"[WARN] Download failed for {meta.name}: {e}")
             continue
 
-        # Locate downloaded .mtx file
         mtx_file = find_mtx(meta.name)
         if mtx_file is None:
             print(f"[SKIP] no .mtx file found locally for: {meta.name}")
             continue
 
-        # Process and write file
+        # --- FIXED TRY/EXCEPT BLOCK INDENTATION HERE ---
         try:
             mat = mmread(mtx_file)
             if not issparse(mat):
@@ -117,14 +114,12 @@ def export_binary(csr, output_path):
 
             csr = mat.tocsr()
             
-            # Paths configuration
+            # Simplified flat paths
             bin_path = os.path.join(MATRIX_DIR, f"{name}.bin")
             meta_path = os.path.join(METADATA_DIR, f"{name}.json")
 
-            # Export custom binary format
             export_binary(csr, bin_path)
 
-            # Build metadata object
             metadata = {
                 "id": meta.id,
                 "name": meta.name,
@@ -151,11 +146,9 @@ def export_binary(csr, output_path):
         except Exception as e:
             print(f"[WARN] Failed processing data for {meta.name}: {e}")
 
-        # Clean cache to prevent disk explosion
         shutil.rmtree(os.path.expanduser("~/.ssgetpy"), ignore_errors=True)
 
-    # Save the execution index mapping
     with open(INDEX_PATH, "w") as f:
         json.dump(index_entries, f, indent=2)
 
-    print(f"\n[DONE] Pipeline complete. Saved execution index map to {INDEX_PATH}")
\ No newline at end of file
+    print(f"\n[DONE] Pipeline complete. Saved master index map to {INDEX_PATH}")
diff --git a/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadB/download-matrices.py b/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadB/download-matrices.py
index 3b3a8e9aa0..838e1c7039 100644
--- a/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadB/download-matrices.py
+++ b/libcaf_cuda/sc26/scripts/Irregular-Workload/workloadB/download-matrices.py
@@ -12,12 +12,13 @@
 # ============================================================
 # CONFIGURATION
 # ============================================================
-INPUT_FILE = "valid_matrix_subset.txt"
+INPUT_FILE = "valid_matrix_subset.txt"  # Your generated text file
 OUTPUT_DIR = "./downloaded_matrices"
 
-MATRIX_DIR = os.path.join(OUTPUT_DIR, "matrices", "spd")
-METADATA_DIR = os.path.join(OUTPUT_DIR, "metadata", "spd")
-INDEX_PATH = os.path.join(OUTPUT_DIR, "spd_index.json")
+# No more "spd" subdirectories; everything goes directly into these folders
+MATRIX_DIR = os.path.join(OUTPUT_DIR, "matrices")
+METADATA_DIR = os.path.join(OUTPUT_DIR, "metadata")
+INDEX_PATH = os.path.join(OUTPUT_DIR, "collection_index.json")
 
 # ============================================================
 # SETUP & PARSING
@@ -27,7 +28,7 @@ def setup_directories():
     os.makedirs(METADATA_DIR, exist_ok=True)
 
 def parse_matrix_names(file_path):
-    """Extracts matrix names from the file paths (e.g., '1138_bus' from '.../1138_bus.bin')"""
+    """Extracts matrix names from the file paths or raw lists."""
     if not os.path.exists(file_path):
         print(f"[ERROR] Input file {file_path} not found.")
         sys.exit(1)
@@ -37,7 +38,6 @@ def parse_matrix_names(file_path):
         for line in f:
             line = line.strip()
             if line:
-                # Get the filename (e.g., 1138_bus.bin) and strip extension
                 base = os.path.basename(line)
                 name, _ = os.path.splitext(base)
                 matrix_names.append(name)
@@ -82,13 +82,11 @@ def export_binary(csr, output_path):
     for idx, name in enumerate(matrix_names, 1):
         print(f"\n[INFO] Processing [{idx}/{len(matrix_names)}]: {name}")
         
-        # Search for exact name match in SuiteSparse
         results = ssgetpy.search(name=name)
         if not results:
             print(f"[WARN] Matrix '{name}' not found in SuiteSparse Collection. Skipping.")
             continue
             
-        # Grab the first match (ssgetpy ranks exact matches highly)
         meta = results[0]
         
         try:
@@ -102,13 +100,12 @@ def export_binary(csr, output_path):
             print(f"[WARN] Download failed for {meta.name}: {e}")
             continue
 
-        # Locate downloaded .mtx file
         mtx_file = find_mtx(meta.name)
         if mtx_file is None:
             print(f"[SKIP] no .mtx file found locally for: {meta.name}")
             continue
 
-        # Process and write file
+        # --- FIXED TRY/EXCEPT BLOCK INDENTATION HERE ---
         try:
             mat = mmread(mtx_file)
             if not issparse(mat):
@@ -117,14 +114,12 @@ def export_binary(csr, output_path):
 
             csr = mat.tocsr()
             
-            # Paths configuration
+            # Simplified flat paths
             bin_path = os.path.join(MATRIX_DIR, f"{name}.bin")
             meta_path = os.path.join(METADATA_DIR, f"{name}.json")
 
-            # Export custom binary format
             export_binary(csr, bin_path)
 
-            # Build metadata object
             metadata = {
                 "id": meta.id,
                 "name": meta.name,
@@ -151,11 +146,9 @@ def export_binary(csr, output_path):
         except Exception as e:
             print(f"[WARN] Failed processing data for {meta.name}: {e}")
 
-        # Clean cache to prevent disk explosion
         shutil.rmtree(os.path.expanduser("~/.ssgetpy"), ignore_errors=True)
 
-    # Save the execution index mapping
     with open(INDEX_PATH, "w") as f:
         json.dump(index_entries, f, indent=2)
 
-    print(f"\n[DONE] Pipeline complete. Saved execution index map to {INDEX_PATH}")
\ No newline at end of file
+    print(f"\n[DONE] Pipeline complete. Saved master index map to {INDEX_PATH}")

From 3eb82c7747393ca62101d26215b05e6bd343b919 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 11:48:02 -0600
Subject: [PATCH 0973/1000] Fixed build issues.

---
 libcaf_cuda/sc26/Irregular-Workload/workloadA/CMakeLists.txt  | 2 +-
 .../sc26/Irregular-Workload/workloadA/sparse_utils.hpp        | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/CMakeLists.txt b/libcaf_cuda/sc26/Irregular-Workload/workloadA/CMakeLists.txt
index ed8e26b561..199d000211 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadA/CMakeLists.txt
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/CMakeLists.txt
@@ -6,7 +6,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS      OFF)
 
 # 2) Set CAF source and build directories
-set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
 set(CAF_BUILD "${CAF_SRC}/build")
 
 
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/sparse_utils.hpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/sparse_utils.hpp
index eef98a0456..020b649ba6 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadA/sparse_utils.hpp
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/sparse_utils.hpp
@@ -91,4 +91,6 @@ std::chrono::milliseconds generate_random_interval(
 std::vector<MatrixTask> generate_batch(
     const std::vector<MatrixTask>& matrix_pool,
     std::mt19937& rng,
-    size_t batch_size);
\ No newline at end of file
+    size_t batch_size);
+
+std::vector<Partition> make_contiguous_partitions(size_t num_tasks, size_t rows, size_t cols);
\ No newline at end of file

From c12bf184c2867de668882cf818f8249c04eba35f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 11:49:49 -0600
Subject: [PATCH 0974/1000] fixed build issues

---
 libcaf_cuda/sc26/Irregular-Workload/workloadB/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/CMakeLists.txt b/libcaf_cuda/sc26/Irregular-Workload/workloadB/CMakeLists.txt
index ed8e26b561..199d000211 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadB/CMakeLists.txt
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/CMakeLists.txt
@@ -6,7 +6,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS      OFF)
 
 # 2) Set CAF source and build directories
-set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
+set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../actor-framework")
 set(CAF_BUILD "${CAF_SRC}/build")
 
 

From fd5190eea457384bd155dbad63fc9fe9f4394d44 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 11:55:13 -0600
Subject: [PATCH 0975/1000] updated filepaths

---
 libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native.cpp | 2 +-
 .../sc26/Irregular-Workload/workloadA/main.native_sorted.cpp  | 2 +-
 libcaf_cuda/sc26/Irregular-Workload/workloadA/main.test.cpp   | 4 +++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native.cpp
index b6486ee944..a394eafa4a 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native.cpp
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native.cpp
@@ -24,7 +24,7 @@ int main(int argc, char** argv)
 
     std::cout << "[INFO] Loading matrices...\n";
     //std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
-    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrices/workloadA", CGS_SOLVER);
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("../../../scripts/Irregular-Workload/workloadA/downloaded_matrices/matrices", CGS_SOLVER);
   
     
     if (matrix_pool.empty()) {
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native_sorted.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native_sorted.cpp
index ca1ff57196..c1167267e2 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native_sorted.cpp
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native_sorted.cpp
@@ -27,7 +27,7 @@ int main(int argc, char** argv) {
     // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
 
     std::cout << "[INFO] Loading matrices...\n";
-    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrices/workloadA", CGS_SOLVER);
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("../../../scripts/Irregular-Workload/workloadA/downloaded_matrices/matrices", CGS_SOLVER);
 
 
     if (matrix_pool.empty()) {
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.test.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.test.cpp
index 5fd9f46a83..4d67c5db90 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.test.cpp
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.test.cpp
@@ -30,7 +30,9 @@ void caf_main(actor_system& sys) {
          //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric", CGS_SOLVER);
          //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
 
-         auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrices/workloadA", CGS_SOLVER);
+        //  auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrices/workloadA", CGS_SOLVER);
+        auto tasks_vec = scan_for_matrices("../../../scripts/Irregular-Workload/workloadA/downloaded_matrices/matrices", CGS_SOLVER);
+
 
 
         int num_gpus = manager::get().get_num_devices();

From d227bdc2755f02d989805a500a3b2602cdfca8a1 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 11:58:19 -0600
Subject: [PATCH 0976/1000] updated file paths

---
 libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native.cpp   | 2 +-
 .../sc26/Irregular-Workload/workloadB/main.native_sorted.cpp    | 2 +-
 libcaf_cuda/sc26/Irregular-Workload/workloadB/main.test.cpp     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native.cpp
index ea334586ac..a7fdc75afa 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native.cpp
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native.cpp
@@ -23,7 +23,7 @@ int main(int argc, char** argv)
     // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
 
     std::cout << "[INFO] Loading matrices...\n";
-    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("../../../scripts/Irregular-Workload/workloadB/downloaded_matrices/matrices", CGS_SOLVER);
 
     if (matrix_pool.empty()) {
         std::cerr << "No matrices found.\n";
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native_sorted.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native_sorted.cpp
index 85aa4d301d..f488f2d3f5 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native_sorted.cpp
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native_sorted.cpp
@@ -27,7 +27,7 @@ int main(int argc, char** argv) {
     // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
 
     std::cout << "[INFO] Loading matrices...\n";
-    std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("../../../scripts/Irregular-Workload/workloadB/downloaded_matrices/matrices", CGS_SOLVER);
 
     if (matrix_pool.empty()) {
         std::cerr << "No matrices found.\n";
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.test.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.test.cpp
index d3f0fd4cd4..2a1103702b 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.test.cpp
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.test.cpp
@@ -28,7 +28,7 @@ void caf_main(actor_system& sys) {
          //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/unsymmetric", CGS_SOLVER);
          //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric", CGS_SOLVER);
-         auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
+         auto tasks_vec = scan_for_matrices("../../../scripts/Irregular-Workload/workloadB/downloaded_matrices/matrices", CGS_SOLVER);
 
         int num_gpus = manager::get().get_num_devices();
         std::cout << "[INFO] Found " << num_gpus << " GPUs\n";

From ebcc3ea176155014e87f8a38bf33ceed19e8a417 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 14:53:38 -0600
Subject: [PATCH 0977/1000] implemented different levels of memory pressure
 testing

---
 .../scheduler_integration_test.cpp            | 305 ++++++++++++------
 1 file changed, 211 insertions(+), 94 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index a41d48fa18..e9c42f2398 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -201,13 +201,24 @@ behavior task_worker_fun(stateful_actor<task_actor_state>* self, caf::actor stat
 
                 // The launch_response_token is released inside the callback 
                 // to signal to the scheduler that the resource is free. (No serial verification here)
-                runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [res, stats_actor, exit_actor, h_c, self_hdl](int* /*ptr*/, size_t /*sz*/) mutable {
+                // runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [res, stats_actor, exit_actor, h_c, self_hdl](int* /*ptr*/, size_t /*sz*/) mutable {
+                //     res->release(); // Release the token
+                //     // The worker no longer sends directly to stats_actor.
+                //     // It signals normal completion to itself, which causes its supervisor to be notified.
+                //     anon_mail(1).send(exit_actor);
+                //     anon_mail(0).send(self_hdl); // Signal normal completion
+                // });
+
+
+                runner.add_callback(res->getStreamId(),res->getDeviceNumber(),[res,stats_actor,exit_actor,self_hdl](){
                     res->release(); // Release the token
                     // The worker no longer sends directly to stats_actor.
                     // It signals normal completion to itself, which causes its supervisor to be notified.
                     anon_mail(1).send(exit_actor);
                     anon_mail(0).send(self_hdl); // Signal normal completion
+
                 });
+
             } catch (const std::exception& e) {
                 std::cerr << "[WORKER] Exception caught: " << e.what() << std::endl;
                 self->quit(sec::runtime_error);
@@ -253,7 +264,7 @@ behavior task_supervisor_fun(stateful_actor<supervisor_state>* self) {
                 self->monitor(w, [self, res](const error& err) mutable {
                     if (err) {
                         self->state().retries++;
-                        if (self->state().retries > 5) {
+                        if (self->state().retries > 0) {
                             // Max retries reached, permanent failure
                             std::cout << "[SUPERVISOR] Task N=" << self->state().N_val 
                                       << " failed permanently after 5 retries. Giving up." << std::endl;
@@ -266,7 +277,7 @@ behavior task_supervisor_fun(stateful_actor<supervisor_state>* self) {
                             self->quit();
                         } else {
                             res -> release();
-                            std::uniform_int_distribution<> dis(100, 1000);
+                            std::uniform_int_distribution<> dis(10, 100);
                             auto backoff = std::chrono::milliseconds(dis(self->state().rng));
 
                             std::cout << "[SUPERVISOR] Task worker failed (" << to_string(err)
@@ -326,128 +337,234 @@ double time_run(Fn&& fn) {
 }
 
 
+
+enum class memory_pressure_level {
+    none,
+    low,
+    medium,
+    high
+};
+
+inline const char* to_string(memory_pressure_level lvl) {
+    switch (lvl) {
+        case memory_pressure_level::none:   return "NONE";
+        case memory_pressure_level::low:    return "LOW";
+        case memory_pressure_level::medium: return "MEDIUM";
+        case memory_pressure_level::high:   return "HIGH";
+    }
+    return "UNKNOWN";
+}
+
+
+struct pressure_profile {
+    size_t target_free_bytes;
+};
+
+inline pressure_profile get_profile(memory_pressure_level lvl, size_t total_mem) {
+    switch (lvl) {
+        case memory_pressure_level::none:
+            return { total_mem };              // no allocation pressure
+
+        case memory_pressure_level::low:
+            return { static_cast<size_t>(total_mem * 0.40) }; // 40% free
+
+        case memory_pressure_level::medium:
+            return { static_cast<size_t>(total_mem * 0.20) }; // 20% free
+
+        case memory_pressure_level::high:
+            return { static_cast<size_t>(total_mem * 0.05) }; // 5% free
+    }
+
+    return { static_cast<size_t>(total_mem * 0.20) };
+}
+
+
+
 static std::vector<caf::cuda::mem_ptr<char>> pressure_holder;
 
-void apply_memory_pressure(int device_id, size_t target_free_bytes) {
+
+void apply_memory_pressure(int device_id,
+                           size_t total_mem_bytes,
+                           memory_pressure_level level)
+{
     auto dev = manager::get().find_device(device_id);
+
+    size_t target_free_bytes = get_profile(level, total_mem_bytes).target_free_bytes;
     size_t available = dev->available_memory_bytes();
-    if (available > target_free_bytes) {
-        size_t to_allocate = available - target_free_bytes;
-        try {
-            auto arg = create_out_arg_with_size<char>(to_allocate);
-            static command_runner<out<char>> p_runner;
-            pressure_holder.push_back(p_runner.transfer_memory(device_id, 0, arg));
-
-            // Synchronize stream 0 to ensure the allocation is complete before checking.
-            auto stream = p_runner.get_stream(0, device_id);
-            CHECK_CUDA(cuStreamSynchronize(stream));
-
-            size_t post_available = dev->available_memory_bytes();
-            std::cout << "[MAIN] Device " << device_id << " memory pressure: allocated "
-                      << to_allocate / (1024 * 1024) << " MB. Actual free: "
-                      << post_available / (1024 * 1024) << " MB." << std::endl;
-
-            // Check if the pressure "stuck". We allow a 50MB tolerance for driver overhead.
-            if (post_available > target_free_bytes + (50ULL * 1024 * 1024)) {
-                std::cerr << "[WARNING] Memory pressure check failed! Expected ~"
-                          << target_free_bytes / (1024 * 1024) << " MB free, but found "
-                          << post_available / (1024 * 1024) << " MB. "
-                          << "The allocation might have been freed prematurely." << std::endl;
-            }
-        } catch (const std::exception& e) {
-            std::cerr << "[MAIN] Warning: Initial memory pressure failed: " << e.what() << std::endl;
+
+    std::cout << "\n[PRESSURE] ================================\n";
+    std::cout << "[PRESSURE] Device " << device_id
+              << " | Level: " << to_string(level) << "\n";
+    std::cout << "[PRESSURE] Available: " << available / (1024 * 1024) << " MB\n";
+    std::cout << "[PRESSURE] Target free: " << target_free_bytes / (1024 * 1024) << " MB\n";
+
+    // NONE mode → explicitly do nothing
+    if (level == memory_pressure_level::none) {
+        std::cout << "[PRESSURE] NONE selected → no allocations performed\n";
+        return;
+    }
+
+    if (available <= target_free_bytes) {
+        std::cout << "[PRESSURE] Already under target pressure\n";
+        return;
+    }
+
+    size_t to_allocate = available - target_free_bytes;
+
+    try {
+        auto arg = create_out_arg_with_size<char>(to_allocate);
+
+        static command_runner<out<char>> p_runner;
+
+        pressure_holder.push_back(
+            p_runner.transfer_memory(device_id, 0, arg)
+        );
+
+        auto stream = p_runner.get_stream(0, device_id);
+        CHECK_CUDA(cuStreamSynchronize(stream));
+
+        size_t post_available = dev->available_memory_bytes();
+
+        std::cout << "[PRESSURE] Allocated: "
+                  << to_allocate / (1024 * 1024) << " MB\n";
+
+        std::cout << "[PRESSURE] Post-free: "
+                  << post_available / (1024 * 1024) << " MB\n";
+
+        if (post_available > target_free_bytes + (50ULL * 1024 * 1024)) {
+            std::cerr << "[WARNING] Pressure mismatch at level "
+                      << to_string(level) << "\n";
         }
+
+    } catch (const std::exception& e) {
+        std::cerr << "[PRESSURE] FAILED (" << to_string(level)
+                  << "): " << e.what() << "\n";
     }
 }
 
-void run_scheduler_integration_scaling_test(actor_system& sys) {
+
+
+
+void run_single_pressure_test(actor_system& sys,
+                              memory_pressure_level level,
+                              int num_tasks,
+                              int streams)
+{
+    caf::cuda::manager::init(sys);
+    auto& mgr = manager::get();
+
+    std::cout << "\n\n====================================================\n";
+    std::cout << "[TEST] Scheduler run under pressure: "
+              << to_string(level) << "\n";
+    std::cout << "====================================================\n";
+
+    // Apply pressure BEFORE launching workload
+    for (int i = 0; i < mgr.get_num_devices(); i++) {
+        auto dev = mgr.find_device(i);
+        size_t total_mem = dev->total_memory_bytes();
+
+        apply_memory_pressure(i, total_mem, level);
+    }
+
+    mgr.toggle_scheduler_actor(streams, 1);
+
+    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+
+    const int THREADS = 32;
+
+    std::vector<token_ptr> tokens;
+    std::mt19937 rng(42);
+ 
     const int min_N = 2048;
     const int max_N = 4096;
-    const int num_distinct_sizes = 10;
+    const int num_distinct_sizes = 20;
     // const std::vector<int> actor_counts = {50000};
-    const std::vector<int> actor_counts = {3000};
+    const std::vector<int> actor_counts = {200};
+    // int streams = 8;
 
     // Generate deterministic random pool once
     auto pool_ptr = std::make_shared<MatrixPool>(
         create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42));
     
     std::vector<int> available_Ns;
-    for (const auto& pair : pool_ptr->A) available_Ns.push_back(pair.first);
+    for (auto& p : pool_ptr->A)
+        available_Ns.push_back(p.first);
 
-    for (int num_tasks : actor_counts) {
-        manager_config config;
-        manager::init(sys, config);
-        auto& mgr = manager::get();
+    std::uniform_int_distribution<size_t> dist(0, available_Ns.size() - 1);
 
-        
-        
-        // Occupy GPU memory such that only 1500 MB remains to force OOMs during task runs
-        for (int i=0; i < mgr.get_num_devices(); i++) {
-            apply_memory_pressure(i, 1500ULL * 1024ULL * 1024ULL);
-        }
+    auto exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_tasks);
+    auto stats_actor = sys.spawn(stats_actor_fun);
 
-        // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
-        mgr.toggle_scheduler_actor(32, 1);
+    std::vector<caf::actor> sups;
 
-        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-        const int THREADS = 32;
+    for (int i = 0; i < num_tasks; ++i) {
+        int N = available_Ns[dist(rng)];
 
-        std::vector<token_ptr> tokens;
-        std::mt19937 rng(42);
-        std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
+        nd_range range((N + THREADS - 1) / THREADS,
+                       (N + THREADS - 1) / THREADS,
+                       1,
+                       THREADS,
+                       THREADS,
+                       1);
 
-        std::cout << "=====================================\n";
-        std::cout << "Scheduler Test | tasks=" << num_tasks << "\n";
+        auto supervisor = sys.spawn(make_task_supervisor_behavior,
+                                    program,
+                                    N,
+                                    pool_ptr,
+                                    exit_actor,
+                                    stats_actor);
 
-        // Spawn the exit actor for this specific test run (moved inside the loop)
-        auto exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_tasks);
+        sups.push_back(supervisor);
 
-        auto stats_actor = sys.spawn(stats_actor_fun);
+        tokens.push_back(
+            make_launch_token(program, range, 0,
+                              "task_" + std::to_string(i),
+                              supervisor));
+    }
 
-        std::vector<caf::actor> sups;
-        // Spawn task actors and prepare tokens
-        for (int i = 0; i < num_tasks; ++i) {
-            int current_N = available_Ns[dist_N_idx(rng)];
-            nd_range range((current_N + THREADS - 1) / THREADS, 
-                           (current_N + THREADS - 1) / THREADS, 1, 
-                           THREADS, THREADS, 1);
-            
-            auto supervisor = sys.spawn(make_task_supervisor_behavior,
-                                       program,
-                                       current_N, 
-                                       pool_ptr,
-                                       exit_actor,
-                                       stats_actor);
-            sups.push_back(supervisor);
-
-            tokens.push_back(make_launch_token(program, range, 0, 
-                             "task_" + std::to_string(i), supervisor));
-        }
+    scoped_actor self{sys};
+
+    double elapsed = time_run([&]() {
+        std::cout << "[TEST] Dispatching workload...\n";
+
+        mgr.send_scheduler_actor_message(std::move(tokens));
+
+        self->wait_for(exit_actor);
+    });
+
+    std::cout << "[RESULT] Pressure=" << to_string(level)
+              << " | time=" << elapsed << " s\n";
+
+    self->mail('c').send(stats_actor);
+    anon_send_exit(stats_actor, exit_reason::user_shutdown);
+    self->wait_for(stats_actor);
 
-        scoped_actor self{sys};
-        double elapsed = time_run([&]() {
-            std::cout << "[MAIN] Dispatching batch to scheduler..." << std::endl;
-            mgr.send_scheduler_actor_message(std::move(tokens));
-
-            // The dispatch is asynchronous. To get an accurate measurement, we must
-            // block until the exit_actor terminates (signaling all 50k tasks are done).
-            scoped_actor self{sys};
-            self->wait_for(exit_actor);
-            // anon_send_exit(stats_actor, exit_reason::user_shutdown);
-        });
-
-        std::cout << "Run complete. Time: " << elapsed << " s\n";
-       // self->mail(elapsed).send(stats_actor);
-        self->mail('c').send(stats_actor);   
-        anon_send_exit(stats_actor, exit_reason::user_shutdown);    
-        self->wait_for(stats_actor);
-        pressure_holder.clear();
-        manager::shutdown(); // Reset manager state for the next potential iteration
+    pressure_holder.clear();
+    manager::shutdown();
+}
+
+void run_pressure_benchmark(actor_system& sys)
+{
+    std::vector<memory_pressure_level> levels = {
+        memory_pressure_level::none,
+        memory_pressure_level::low,
+        memory_pressure_level::medium,
+        memory_pressure_level::high
+    };
+
+    const int num_tasks = 200;
+    const int streams = 8;
+
+    for (auto level : levels) {
+        run_single_pressure_test(sys, level, num_tasks, streams);
     }
 }
 
+
+
+
 void caf_main(actor_system& sys) {
-    run_scheduler_integration_scaling_test(sys);
+    run_pressure_benchmark(sys);
     std::cout << "[MAIN] Integration test complete." << std::endl;
 }
 

From 59daabf3938eb2bffa170d56371325bb212afc38 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 14:56:34 -0600
Subject: [PATCH 0978/1000] fixed synchronization error between runs where
 memory would still be on device for some reason resulting in a lower
 allocation then required

---
 .../scheduler_integration_test.cpp                          | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index e9c42f2398..ad34dc3262 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -387,7 +387,11 @@ void apply_memory_pressure(int device_id,
                            size_t total_mem_bytes,
                            memory_pressure_level level)
 {
-    auto dev = manager::get().find_device(device_id);
+  auto dev = manager::get().find_device(device_id);
+CUcontext ctx = dev->getContext();
+
+CHECK_CUDA(cuCtxSetCurrent(ctx));
+CHECK_CUDA(cuCtxSynchronize());
 
     size_t target_free_bytes = get_profile(level, total_mem_bytes).target_free_bytes;
     size_t available = dev->available_memory_bytes();

From 73812df33f3275707ab4247216ee2a176f0c05a9 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 14:58:27 -0600
Subject: [PATCH 0979/1000] updated pressure levels to trigger more faults

---
 .../scheduler_integration_test.cpp                            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index ad34dc3262..df3d359185 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -366,10 +366,10 @@ inline pressure_profile get_profile(memory_pressure_level lvl, size_t total_mem)
             return { total_mem };              // no allocation pressure
 
         case memory_pressure_level::low:
-            return { static_cast<size_t>(total_mem * 0.40) }; // 40% free
+            return { static_cast<size_t>(total_mem * 0.20) }; // 40% free
 
         case memory_pressure_level::medium:
-            return { static_cast<size_t>(total_mem * 0.20) }; // 20% free
+            return { static_cast<size_t>(total_mem * 0.10) }; // 20% free
 
         case memory_pressure_level::high:
             return { static_cast<size_t>(total_mem * 0.05) }; // 5% free

From 9587a73d2e362e904cc34bb2ee3a86f36ec0a60d Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Mon, 22 Jun 2026 15:03:05 -0600
Subject: [PATCH 0980/1000] added throughput counter

---
 .../scheduler_integration_test.cpp             | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
index df3d359185..bb50fdf3b4 100644
--- a/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
+++ b/libcaf_cuda/tests/scheduler-fault-tolerance-test/scheduler_integration_test.cpp
@@ -81,7 +81,7 @@ behavior stats_actor_fun(stateful_actor<stats_actor_state>* self) {
 
         // FINAL REPORT (IMPORTANT FIX)
         [=](char finalize_stats) {
-            auto& st = self->state();
+             auto& st = self->state();
 
             if (st.finalized)
                 return;
@@ -98,6 +98,11 @@ behavior stats_actor_fun(stateful_actor<stats_actor_state>* self) {
                 ? (100.0 * total_succeeded / total)
                 : 0.0;
 
+            double throughput = 0.0;
+            if (st.runtime > 0.0) {
+                throughput = total / st.runtime;
+            }
+
             std::cout << "\n=====================================\n";
             std::cout << "[STATS REPORT] Iteration Complete\n";
             std::cout << "  Total Processed: " << total << "\n";
@@ -120,10 +125,16 @@ behavior stats_actor_fun(stateful_actor<stats_actor_state>* self) {
                       << std::fixed << std::setprecision(2)
                       << success_pct << "%\n";
 
+            std::cout << "  Runtime: ";
             if (st.runtime > 0.0) {
-                std::cout << "  Runtime: "
-                          << std::fixed << std::setprecision(3)
+                std::cout << std::fixed << std::setprecision(3)
                           << st.runtime << " s\n";
+
+                std::cout << "  Throughput: "
+                          << std::fixed << std::setprecision(2)
+                          << throughput << " tasks/sec\n";
+            } else {
+                std::cout << "N/A\n";
             }
 
             std::cout << "=====================================\n";
@@ -539,6 +550,7 @@ void run_single_pressure_test(actor_system& sys,
     std::cout << "[RESULT] Pressure=" << to_string(level)
               << " | time=" << elapsed << " s\n";
 
+    self->mail(elapsed).send(stats_actor);
     self->mail('c').send(stats_actor);
     anon_send_exit(stats_actor, exit_reason::user_shutdown);
     self->wait_for(stats_actor);

From c32f3bdc65e542fec5be3fa111d1644ac518d6ac Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 08:12:13 -0600
Subject: [PATCH 0981/1000] Updated cuda baseline to be like static round robin
 partioning of 50000 tasks so that it can be more like the scheduler.

---
 .../cuda-baseline.cpp                         | 232 ++++++------------
 1 file changed, 69 insertions(+), 163 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index 3ba1b7f5e3..717c1aee60 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -3,13 +3,27 @@
 #include <vector>
 #include <chrono>
 #include <random>
+#include <cstdlib>
 #include <thread>
 #include <algorithm>
 #include <functional>
 #include <stdexcept> // For runtime_error
 #include <unordered_map> // For MatrixPool
+#include <deque>
 #include <unordered_set> // For create_matrix_pool_random
 
+#define CUDA_CHECK(expr)                                                       \
+  do {                                                                         \
+    CUresult _res = (expr);                                                    \
+    if (_res != CUDA_SUCCESS) {                                                \
+      const char* _err_str;                                                    \
+      cuGetErrorString(_res, &_err_str);                                       \
+      std::cerr << "CUDA Error in " << #expr << " at line " << __LINE__        \
+                << ": " << _err_str << std::endl;                              \
+      std::exit(EXIT_FAILURE);                                                 \
+    }                                                                          \
+  } while (0)
+
 enum TaskType { MMUL = 0, VADD = 1, CONV = 2 };
 
 struct Task {
@@ -29,19 +43,12 @@ struct MatrixPool {
 // Per-GPU execution logic
 void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_gpu, CUcontext ctx, CUfunction mmul_func,
      CUfunction vadd_func, CUfunction conv_func, const MatrixPool& pool, int* shared_dtoh_buffer) {
-    // Set the CUDA context for this thread
-    CUresult err = cuCtxSetCurrent(ctx);
-    if (err != CUDA_SUCCESS) {
-        const char* err_str;
-        cuGetErrorString(err, &err_str);
-        std::cerr << "Error setting context for device " << device_id << ": " << err_str << std::endl;
-        return;
-    }
+    CUDA_CHECK(cuCtxSetCurrent(ctx));
 
     // Prepare Streams
     std::vector<CUstream> streams(streams_per_gpu);
     for (int i = 0; i < streams_per_gpu; ++i) {
-        cuStreamCreate(&streams[i], CU_STREAM_NON_BLOCKING);
+        CUDA_CHECK(cuStreamCreate(&streams[i], CU_STREAM_NON_BLOCKING));
     }
 
     // Use the first stream for initial allocations and cleanup
@@ -52,62 +59,48 @@ void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_g
         int N = tasks[i].N;
         CUstream stream = streams[i % streams_per_gpu];
         TaskType type = tasks[i].type;
-        size_t bytes_a = (type == MMUL) ? (size_t)N * N * sizeof(int) : (size_t)N * sizeof(int);
-        size_t bytes_b = (type == CONV) ? 5 * sizeof(int) : bytes_a;
-        size_t bytes_out = (type == MMUL) ? (size_t)N * N * sizeof(int) : (size_t)N * sizeof(int);
+        size_t bytes_a = (size_t)N * N * sizeof(int);
+        size_t bytes_b = bytes_a;
+        size_t bytes_out = (size_t)N * N * sizeof(int);
 
         CUdeviceptr d_a, d_b, d_c;
 
-        cuMemAllocAsync(&d_a, bytes_a, stream);
-        cuMemAllocAsync(&d_b, bytes_b, stream);
-        cuMemAllocAsync(&d_c, bytes_out, stream);
+        CUDA_CHECK(cuMemAllocAsync(&d_a, bytes_a, stream));
+        CUDA_CHECK(cuMemAllocAsync(&d_b, bytes_b, stream));
+        CUDA_CHECK(cuMemAllocAsync(&d_c, bytes_out, stream));
 
         // Perform Host-to-Device transfer
-        const std::vector<int>& h_a = (type == MMUL) ? pool.A.at(N) : (type == VADD ? pool.vec_A.at(N) : pool.conv_A.at(N));
-        const std::vector<int>& h_b = (type == MMUL) ? pool.B.at(N) : (type == VADD ? pool.vec_B.at(N) : pool.conv_K.at(N));
+        const std::vector<int>& h_a = pool.A.at(N);
+        const std::vector<int>& h_b = pool.B.at(N);
 
-        cuMemcpyHtoDAsync(d_a, h_a.data(), bytes_a, stream);
-        cuMemcpyHtoDAsync(d_b, h_b.data(), bytes_b, stream);
+        CUDA_CHECK(cuMemcpyHtoDAsync(d_a, h_a.data(), bytes_a, stream));
+        CUDA_CHECK(cuMemcpyHtoDAsync(d_b, h_b.data(), bytes_b, stream));
 
         // Kernel arguments for cuLaunchKernel
         void *kernel_args[] = { &d_a, &d_b, &d_c, &N };
 
-        if (type == MMUL) {
-            unsigned int block_dim = 32;
-            unsigned int grid_dim = (N + block_dim - 1) / block_dim;
-            cuLaunchKernel(mmul_func, grid_dim, grid_dim, 1,
-                           block_dim, block_dim, 1,
-                           0, stream, kernel_args, nullptr);
-        } else if (type == VADD) {
-            unsigned int block_dim = 256;
-            unsigned int grid_dim = (N + block_dim - 1) / block_dim;
-            cuLaunchKernel(vadd_func, grid_dim, 1, 1,
-                           block_dim, 1, 1,
-                           0, stream, kernel_args, nullptr);
-        } else {
-            unsigned int block_dim = 256;
-            unsigned int grid_dim = (N + block_dim - 1) / block_dim;
-            cuLaunchKernel(conv_func, grid_dim, 1, 1,
-                           block_dim, 1, 1,
-                           0, stream, kernel_args, nullptr);
-        }
-        
-        // Simulating the result retrieval (Copy back)
-        size_t res_count = (type == MMUL) ? (size_t)N * N : (size_t)N;
-        cuMemcpyDtoHAsync(shared_dtoh_buffer, d_c, bytes_out, stream);
-
-        // Free GPU memory for this task
-        cuMemFreeAsync(d_a, stream);
-        cuMemFreeAsync(d_b, stream);
-        cuMemFreeAsync(d_c, stream);
+        unsigned int block_dim = 32;
+        unsigned int grid_dim = (N + block_dim - 1) / block_dim;
+        CUDA_CHECK(cuLaunchKernel(mmul_func, grid_dim, grid_dim, 1,
+                                      block_dim, block_dim, 1,
+                                      0, stream, kernel_args, nullptr));
+      
+
+
+    auto h_c = std::make_shared<std::vector<int>>(N * N);
+    CUDA_CHECK(cuMemcpyDtoHAsync(h_c->data(), d_c, bytes_out, stream));
+
+    CUDA_CHECK(cuMemFreeAsync(d_a, stream));
+    CUDA_CHECK(cuMemFreeAsync(d_b, stream));
+    CUDA_CHECK(cuMemFreeAsync(d_c, stream));
     }
 
     // Synchronize this GPU context
-    cuCtxSynchronize();
+    CUDA_CHECK(cuCtxSynchronize());
 
     // Cleanup
     for (auto s : streams) {
-        cuStreamDestroy(s);
+        CUDA_CHECK(cuStreamDestroy(s));
     }
 }
 
@@ -127,38 +120,26 @@ MatrixPool create_matrix_pool_random(
         if (used_Ns.insert(N_val).second) {
             pool.A[N_val] = std::vector<int>(N_val * N_val, 1);
             pool.B[N_val] = std::vector<int>(N_val * N_val, 1);
-            pool.vec_A[N_val] = std::vector<int>(N_val, 1);
-            pool.vec_B[N_val] = std::vector<int>(N_val, 1);
-            pool.conv_A[N_val] = std::vector<int>(N_val, 1);
-            pool.conv_K[N_val] = std::vector<int>(5, 1);
         }
     }
     return pool;
 }
 
 int main() {
-    CUresult err;
-
     // Initialize the CUDA Driver API
-    err = cuInit(0);
-    if (err != CUDA_SUCCESS) {
-        const char* err_str;
-        cuGetErrorString(err, &err_str);
-        std::cerr << "Error initializing CUDA Driver API: " << err_str << std::endl;
-        return 1;
-    }
+    CUDA_CHECK(cuInit(0));
 
     const int streams_per_gpu = 8;
 
     int num_gpus;
-    cuDeviceGetCount(&num_gpus);
+    CUDA_CHECK(cuDeviceGetCount(&num_gpus));
     if (num_gpus == 0) {
         std::cerr << "No CUDA devices found." << std::endl;
         return 1;
     }
 
     // Define parameters for irregular workload
-    const int num_matrix_sizes = 60; // Number of distinct N values
+    const int num_matrix_sizes = 10; // Number of distinct N values
     const int min_N_val = 32;
     const int max_N_val = 2048;
     const unsigned int pool_seed = 42; // Fixed seed for deterministic pool generation
@@ -191,110 +172,29 @@ int main() {
     // Create a CUDA context for each device
     for (int i = 0; i < num_gpus; ++i) {
         CUdevice dev;
-        cuDeviceGet(&dev, i);
-        err = cuCtxCreate(&contexts[i], 0, dev); // Flag 0 for default context creation
-        if (err != CUDA_SUCCESS) {
-            const char* err_str;
-            cuGetErrorString(err, &err_str);
-            std::cerr << "Error creating context for device " << i << ": " << err_str << std::endl;
-            // Clean up already created contexts
-            for (int j = 0; j < i; ++j) cuCtxDestroy(contexts[j]);
-            return 1;
-        }
+        CUDA_CHECK(cuDeviceGet(&dev, i));
+        CUDA_CHECK(cuCtxCreate(&contexts[i], 0, dev));
     }
 
     // Make the context for device 0 current on the main thread before loading the module
-    err = cuCtxPushCurrent(contexts[0]);
-    if (err != CUDA_SUCCESS) {
-        const char* err_str;
-        cuGetErrorString(err, &err_str);
-        std::cerr << "Error pushing context for device 0: " << err_str << std::endl;
-        for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
-        return 1;
-    }
+    CUDA_CHECK(cuCtxPushCurrent(contexts[0]));
 
     // Load the cubin module (assuming mmul.cu is compiled to mmul.cubin)
-    err = cuModuleLoad(&mmul_mod, "../mmul.cubin");
-    if (err != CUDA_SUCCESS) {
-        const char* err_str;
-        cuGetErrorString(err, &err_str);
-        std::cerr << "Error loading module ../mmul.cubin: " << err_str << std::endl;
-        cuCtxPopCurrent(nullptr); // Pop context on error
-        return 1;
-    }
-
-    err = cuModuleLoad(&vadd_mod, "../vector_add.cubin");
-    if (err != CUDA_SUCCESS) {
-        const char* err_str;
-        cuGetErrorString(err, &err_str);
-        std::cerr << "Error loading module ../vector_add.cubin: " << err_str << std::endl;
-        cuCtxPopCurrent(nullptr);
-        return 1;
-    }
-
-    err = cuModuleLoad(&conv_mod, "../conv1d.cubin");
-    if (err != CUDA_SUCCESS) {
-        const char* err_str;
-        cuGetErrorString(err, &err_str);
-        std::cerr << "Error loading module ../conv1d.cubin: " << err_str << std::endl;
-        cuCtxPopCurrent(nullptr);
-        return 1;
-    }
-
+    CUDA_CHECK(cuModuleLoad(&mmul_mod, "../mmul.cubin"));
     // Get function handles
-    err = cuModuleGetFunction(&mmul_funcs[0], mmul_mod, "matrixMul");
-    if (err != CUDA_SUCCESS) {
-        const char* err_str;
-        cuGetErrorString(err, &err_str);
-        std::cerr << "Error getting function matrixMul: " << err_str << std::endl;
-        cuCtxPopCurrent(nullptr); // Pop context on error
-        cuModuleUnload(mmul_mod);
-        cuModuleUnload(vadd_mod);
-        for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
-        return 1;
-    }
-
-    err = cuModuleGetFunction(&vadd_funcs[0], vadd_mod, "vectorAdd");
-    if (err != CUDA_SUCCESS) {
-        const char* err_str;
-        cuGetErrorString(err, &err_str);
-        std::cerr << "Error getting function vectorAdd: " << err_str << std::endl;
-        cuCtxPopCurrent(nullptr);
-        return 1;
-    }
-
-    err = cuModuleGetFunction(&conv_funcs[0], conv_mod, "conv1d");
-    if (err != CUDA_SUCCESS) {
-        const char* err_str;
-        cuGetErrorString(err, &err_str);
-        std::cerr << "Error getting function conv1d: " << err_str << std::endl;
-        cuCtxPopCurrent(nullptr);
-        return 1;
-    }
+    CUDA_CHECK(cuModuleGetFunction(&mmul_funcs[0], mmul_mod, "matrixMul"));
 
     // Pop the context from the main thread
-    err = cuCtxPopCurrent(nullptr);
-    if (err != CUDA_SUCCESS) {
-        const char* err_str;
-        cuGetErrorString(err, &err_str);
-        std::cerr << "Error popping context from main thread: " << err_str << std::endl;
-        cuModuleUnload(mmul_mod);
-        cuModuleUnload(vadd_mod);
-        cuModuleUnload(conv_mod);
-        for (int i = 0; i < num_gpus; ++i) cuCtxDestroy(contexts[i]);
-        return 1;
-    }
+    CUDA_CHECK(cuCtxPopCurrent(nullptr));
 
     // Assuming all GPUs can use the same function handle from the same module.
     for (int i = 1; i < num_gpus; ++i) {
         mmul_funcs[i] = mmul_funcs[0];
-        vadd_funcs[i] = vadd_funcs[0];
-        conv_funcs[i] = conv_funcs[0];
     }
 
     std::vector<int> shared_dtoh_buffer((size_t)max_N_val * max_N_val);
 
-    std::vector<int> batch_configs = {5, 10};
+    std::vector<int> batch_configs = {1};
     for (int num_batches : batch_configs) {
         std::cout << "=====================================" << std::endl;
         std::cout << "Starting Run with " << num_batches << " batches" << std::endl;
@@ -302,9 +202,12 @@ int main() {
         std::mt19937 rng_prod(42); 
         std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
         std::uniform_int_distribution<int> dist_type(0, 2);
-        std::uniform_int_distribution<int> dist_batch_size(5000, 15000);
+        std::uniform_int_distribution<int> dist_batch_size(50000, 50000);
         std::uniform_int_distribution<int> dist_sleep(500, 2000);
 
+        std::vector<std::thread> all_threads;
+        std::deque<std::vector<std::vector<Task>>> run_partitions_storage;
+
         auto start = std::chrono::steady_clock::now();
 
         for (int b = 0; b < num_batches; ++b) {
@@ -330,15 +233,18 @@ int main() {
                 partitions[i % num_gpus].push_back(batch_tasks[i]);
             }
 
-            std::vector<std::thread> threads;
+            run_partitions_storage.push_back(std::move(partitions));
+            auto& saved_partitions = run_partitions_storage.back();
+
             for (int i = 0; i < num_gpus; ++i) {
-                threads.emplace_back(gpu_worker, i, std::ref(partitions[i]), streams_per_gpu, contexts[i], mmul_funcs[i], vadd_funcs[i], conv_funcs[i],
-                 std::ref(global_host_matrix_pool), shared_dtoh_buffer.data());
+                all_threads.emplace_back(gpu_worker, i, std::ref(saved_partitions[i]), streams_per_gpu,
+                                            contexts[i], mmul_funcs[i], vadd_funcs[i], conv_funcs[i],
+                                         std::ref(global_host_matrix_pool), shared_dtoh_buffer.data());
             }
+        }
 
-            for (auto& t : threads) {
-                t.join();
-            }
+        for (auto& t : all_threads) {
+            t.join();
         }
 
         auto end = std::chrono::steady_clock::now();
@@ -348,11 +254,11 @@ int main() {
 
     // Cleanup contexts and module
     for (int i = 0; i < num_gpus; ++i) {
-        cuCtxDestroy(contexts[i]);
+        CUDA_CHECK(cuCtxDestroy(contexts[i]));
     }
-    cuModuleUnload(mmul_mod);
-    cuModuleUnload(vadd_mod);
-    cuModuleUnload(conv_mod);
+    CUDA_CHECK(cuModuleUnload(mmul_mod));
+    CUDA_CHECK(cuModuleUnload(vadd_mod));
+    CUDA_CHECK(cuModuleUnload(conv_mod));
 
     return 0;
 }

From 429d2315af0ccc221c3d583c569154fb2dd93f30 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 08:13:54 -0600
Subject: [PATCH 0982/1000] Fixed cuda error.

---
 .../mmul-randonom-batch-benchmark/cuda-baseline.cpp         | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index 717c1aee60..bbff941f86 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -256,9 +256,9 @@ int main() {
     for (int i = 0; i < num_gpus; ++i) {
         CUDA_CHECK(cuCtxDestroy(contexts[i]));
     }
-    CUDA_CHECK(cuModuleUnload(mmul_mod));
-    CUDA_CHECK(cuModuleUnload(vadd_mod));
-    CUDA_CHECK(cuModuleUnload(conv_mod));
+    // CUDA_CHECK(cuModuleUnload(mmul_mod));
+    // CUDA_CHECK(cuModuleUnload(vadd_mod));
+    // CUDA_CHECK(cuModuleUnload(conv_mod));
 
     return 0;
 }

From b33a443a65ded66f6770ed2271a692236c982d17 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 08:18:54 -0600
Subject: [PATCH 0983/1000] Fixing invalid moduel error.

---
 .../mmul-randonom-batch-benchmark/cuda-baseline.cpp            | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index bbff941f86..560bc51622 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -187,9 +187,8 @@ int main() {
     // Pop the context from the main thread
     CUDA_CHECK(cuCtxPopCurrent(nullptr));
 
-    // Assuming all GPUs can use the same function handle from the same module.
     for (int i = 1; i < num_gpus; ++i) {
-        mmul_funcs[i] = mmul_funcs[0];
+        CUDA_CHECK(cuModuleGetFunction(&mmul_funcs[i], mmul_mod, "matrixMul"));
     }
 
     std::vector<int> shared_dtoh_buffer((size_t)max_N_val * max_N_val);

From b9a709751e0a333be67c77a86c893bab3e4ba451 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 08:21:17 -0600
Subject: [PATCH 0984/1000] Fixing invalid moduel error.

---
 .../mmul-randonom-batch-benchmark/cuda-baseline.cpp             | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index 560bc51622..1f2f0a7fef 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -188,7 +188,9 @@ int main() {
     CUDA_CHECK(cuCtxPopCurrent(nullptr));
 
     for (int i = 1; i < num_gpus; ++i) {
+        CUDA_CHECK(cuCtxPushCurrent(contexts[i]));
         CUDA_CHECK(cuModuleGetFunction(&mmul_funcs[i], mmul_mod, "matrixMul"));
+        CUDA_CHECK(cuCtxPopCurrent(nullptr));
     }
 
     std::vector<int> shared_dtoh_buffer((size_t)max_N_val * max_N_val);

From 8e088fb6371c538e0a8c763d4f0fa54d0bc1b6b3 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 08:26:55 -0600
Subject: [PATCH 0985/1000] Fixing invalid moduel error.

---
 .../cuda-baseline.cpp                         | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index 1f2f0a7fef..bfd5407f9c 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -176,19 +176,20 @@ int main() {
         CUDA_CHECK(cuCtxCreate(&contexts[i], 0, dev));
     }
 
-    // Make the context for device 0 current on the main thread before loading the module
-    CUDA_CHECK(cuCtxPushCurrent(contexts[0]));
+    // // Make the context for device 0 current on the main thread before loading the module
+    // CUDA_CHECK(cuCtxPushCurrent(contexts[0]));
 
-    // Load the cubin module (assuming mmul.cu is compiled to mmul.cubin)
-    CUDA_CHECK(cuModuleLoad(&mmul_mod, "../mmul.cubin"));
-    // Get function handles
-    CUDA_CHECK(cuModuleGetFunction(&mmul_funcs[0], mmul_mod, "matrixMul"));
+    // // Load the cubin module (assuming mmul.cu is compiled to mmul.cubin)
+    // CUDA_CHECK(cuModuleLoad(&mmul_mod, "../mmul.cubin"));
+    // // Get function handles
+    // // CUDA_CHECK(cuModuleGetFunction(&mmul_funcs[0], mmul_mod, "matrixMul"));
 
-    // Pop the context from the main thread
-    CUDA_CHECK(cuCtxPopCurrent(nullptr));
+    // // Pop the context from the main thread
+    // CUDA_CHECK(cuCtxPopCurrent(nullptr));
 
-    for (int i = 1; i < num_gpus; ++i) {
+    for (int i = 0; i < num_gpus; ++i) {
         CUDA_CHECK(cuCtxPushCurrent(contexts[i]));
+        CUDA_CHECK(cuModuleLoad(&mmul_mod, "../mmul.cubin"));
         CUDA_CHECK(cuModuleGetFunction(&mmul_funcs[i], mmul_mod, "matrixMul"));
         CUDA_CHECK(cuCtxPopCurrent(nullptr));
     }

From 11dce8f02acb5b26f5bc0e67e35b132c089e1585 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 09:39:21 -0600
Subject: [PATCH 0986/1000] Changed test to do static round robin partitioning
 of the GPUs.

---
 .../main.test.cpp                             | 512 +++++-------------
 1 file changed, 128 insertions(+), 384 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 066811c37f..007de9beb4 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -1,47 +1,35 @@
 #include <caf/all.hpp>
 #include <caf/cuda/all.hpp>
-#include <caf/component-actors/all-component-actors.hpp>
+#include <caf/cuda/control-layer/all-control-layer.hpp>
 #include <iostream>
-#include <iomanip>
 #include <vector>
-#include <chrono>
-#include <thread>
-#include <algorithm>
-#include <numeric>
+#include <string>
 #include <random>
-#include "caf/actor_registry.hpp"
-#include <deque>
 #include <unordered_map>
 #include <unordered_set>
-//#include <caf/atoms.hpp>
+#include <algorithm>
 
 using namespace caf;
-using namespace std::chrono_literals;
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Atoms
-// ─────────────────────────────────────────────────────────────────────────────
-CAF_BEGIN_TYPE_ID_BLOCK(mmul_benchmark, caf::id_block::cuda::end)
-    CAF_ADD_ATOM(mmul_benchmark, get_work_atom)
-    CAF_ADD_ATOM(mmul_benchmark, task_done_atom)
-    CAF_ADD_ATOM(mmul_benchmark, release_memory_atom)
-    CAF_ADD_ATOM(mmul_benchmark, request_work_atom)
-    CAF_ADD_ATOM(mmul_benchmark, worker_done_atom)
-    CAF_ADD_ATOM(mmul_benchmark, refill_buffer_atom)
-CAF_END_TYPE_ID_BLOCK(mmul_benchmark)
+using namespace caf::cuda;
 
-// Command runners for GPU operations
-caf::cuda::command_runner<> mmul_command;
-using mmul_kernel_t = caf::cuda::command_runner<caf::cuda::mem_ptr<int>, caf::cuda::mem_ptr<int>, out<int>, in<int>>;
-mmul_kernel_t mmul_kernel;
+// A generic command runner to provide access to CUDA stream callbacks
+static command_runner<in<int>, in<int>, out<int>, in<int>> runner;
 
+// MatrixPool structure from mmul-random-batch-benchmark
 struct MatrixPool {
     std::unordered_map<int, std::vector<int>> A;
     std::unordered_map<int, std::vector<int>> B;
 };
 
+struct task_actor_state {
+    program_ptr prog;
+    int N_val;
+    std::shared_ptr<MatrixPool> pool;
+    int device;
+    int stream;
+};
 
-
+// create_matrix_pool_random function from mmul-random-batch-benchmark
 MatrixPool create_matrix_pool_random(
     int num_sizes,
     int min_N,
@@ -49,402 +37,158 @@ MatrixPool create_matrix_pool_random(
     unsigned int seed
 ) {
     MatrixPool pool;
-
     std::mt19937 rng(seed);
     std::uniform_int_distribution<int> dist(min_N, max_N);
-
     std::unordered_set<int> used;
-
     while (used.size() < static_cast<size_t>(num_sizes)) {
         int N = dist(rng);
         if (used.insert(N).second) {
             pool.A[N] = std::vector<int>(N * N, 1);
-            pool.B[N] = std::vector<int>(N * N, 1);
+            pool.B[N] = std::vector<int>(N * N, 2); // Changed to 2 for distinct input
         }
     }
-
     return pool;
 }
 
-// ---------------------------- GLOBAL TASK POOL ----------------------------
-// The central source of truth for work. Implements a pull-based model.
-struct task_pool_state {
-    std::vector<int> tasks;
-    size_t next_task_idx = 0;
-};
+// This actor represents a single task that requests permission from the scheduler.
+behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_actor) {
+    self->mail("hello").send(self);
 
-caf::behavior global_task_pool(caf::stateful_actor<task_pool_state>* self, std::vector<int> tasks) {
-    self->state().tasks = std::move(tasks);
     return {
-        [=](get_work_atom, size_t batch_size) -> result<std::vector<int>> {
-            auto& st = self->state();
-            if (st.next_task_idx >= st.tasks.size())
-                return sec::end_of_stream;
-            size_t count = std::min(batch_size, st.tasks.size() - st.next_task_idx);
-            std::vector<int> batch(st.tasks.begin() + st.next_task_idx, 
-                                   st.tasks.begin() + st.next_task_idx + count);
-            st.next_task_idx += count;
-            return batch;
-        }
-    };
-}
-
-// ---------------------------- DEVICE/GPU ACTOR ----------------------------
-// Manages memory for a specific GPU and steals (pulls) work from the Global Pool.
-struct device_actor_state {
-    MatrixPool pool;
-    caf::actor global_pool;
-    std::deque<int> local_tasks; // Local buffer to keep GPU busy
-    size_t total_device_memory_bytes = 0;
-    size_t current_allocated_memory_bytes = 0;
-    int active_workers = 0;
-    int device_id = -1;
-    size_t batch_size = 0;
-    size_t low_water_mark = 0;
-    bool fetching = false;
-};
-
-caf::behavior gpu_device_actor(caf::stateful_actor<device_actor_state>* self,
-                               MatrixPool pool, caf::actor global_pool, int num_workers, int dev_id, int max_in_flight) {
-    self->state().pool = std::move(pool);
-    self->state().global_pool = global_pool;
-    self->state().device_id = dev_id;
-    self->state().active_workers = num_workers;
-
-    // Dynamically calculate prefetch markers based on the total pipeline capacity
-    self->state().low_water_mark = static_cast<size_t>(num_workers * max_in_flight);
-    self->state().batch_size = self->state().low_water_mark * 2;
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-    caf::cuda::device_ptr dev_obj = mgr.find_device(dev_id);
-    if (dev_obj) {
-        self->state().total_device_memory_bytes = dev_obj->total_memory_bytes();
-    }
-
-    // Helper to refill the local task buffer from the global pool
-    auto refill = [=]() {
-        auto& st = self->state();
-        if (st.fetching || st.local_tasks.size() >= st.low_water_mark + st.batch_size)
-            return;
-
-        st.fetching = true;
-        self->mail(get_work_atom_v, (size_t)st.batch_size).request(st.global_pool, infinite).then(
-            [=](std::vector<int>& batch) {
-                auto& st_inner = self->state();
-                for (int N : batch)
-                    st_inner.local_tasks.push_back(N);
-                st_inner.fetching = false;
-                if (st_inner.local_tasks.size() < st_inner.low_water_mark)
-                    self->mail(refill_buffer_atom_v).send(self);
-            },
-            [=](error& err) {
-                self->state().fetching = false;
-            }
-        );
-    };
-
-    return {
-        [=](refill_buffer_atom) {
-            refill();
-        },
-        [=](get_work_atom) -> caf::result<int, in<int>, in<int>> {
-            auto& st = self->state();
-
-            // If we have tasks locally, satisfy the request immediately
-            if (!st.local_tasks.empty()) {
-                int N = st.local_tasks.front();
-                size_t memory_needed = (size_t)N * N * sizeof(int) * 3;
-                if (st.current_allocated_memory_bytes + memory_needed > st.total_device_memory_bytes)
-                    return make_error(sec::runtime_error, "Device Actor: Not enough memory");
-                
-                st.local_tasks.pop_front();
-                st.current_allocated_memory_bytes += memory_needed;
+        [=](std::string start_tag) mutable {
+                auto& st = self->state();
                 
-                // Proactively steal more work if the buffer is getting low
-                if (st.local_tasks.size() < st.low_water_mark)
-                    refill();
-
-                return {N, caf::cuda::create_in_arg(st.pool.A[N]), 
-                           caf::cuda::create_in_arg(st.pool.B[N])};
-            }
-
-            // Buffer empty: must fetch from global pool reactively
-            auto promise = self->make_response_promise<int, in<int>, in<int>>();
-            self->mail(get_work_atom_v, (size_t)st.batch_size).request(st.global_pool, infinite).then(
-                [=](std::vector<int>& batch) mutable {
-                    auto& st_inner = self->state();
-                    int N = batch.front();
-                    for(size_t i = 1; i < batch.size(); ++i) st_inner.local_tasks.push_back(batch[i]);
-                    
-                    size_t needed = (size_t)N * N * sizeof(int) * 3;
-                    st_inner.current_allocated_memory_bytes += needed;
-                    promise.deliver(N, caf::cuda::create_in_arg(st_inner.pool.A[N]), 
-                                       caf::cuda::create_in_arg(st_inner.pool.B[N]));
-                },
-                [=](error& err) mutable { promise.deliver(err); }
-            );
-            return promise;
-        },
-        [=](release_memory_atom, int N_completed) {
-            auto& st = self->state();
-            size_t memory_released = (size_t)N_completed * N_completed * sizeof(int) * 3;
-            st.current_allocated_memory_bytes -= memory_released;
-            refill(); // Try to get more work now that memory is free
-        },
-        [=](worker_done_atom) {
-            auto& st = self->state();
-            if (--st.active_workers <= 0) {
-                self->quit();
-            }
-        }
-    };
-}
-
-// ---------------------------- WORKER ACTOR ----------------------------
-// Manages 1 stream and pulls work from the Device Actor.
-struct worker_state {
-    int device_id;
-    std::vector<int> stream_ids;
-    caf::cuda::program_ptr program;
-    caf::actor device_actor;
-    caf::actor supervisor;
-    int max_in_flight_tasks;
-    int in_flight_tasks_count = 0;
-    bool draining = false;
-};
-
-caf::behavior mmul_worker_fun(caf::stateful_actor<worker_state>* self,
-                              caf::actor supervisor, caf::actor device_actor, caf::cuda::program_ptr program,
-                              int dev_id, std::vector<int> stream_ids, int max_in_flight_tasks) {
-    self->state().supervisor = supervisor;
-    self->state().device_actor = device_actor;
-    self->state().program = program;
-    self->state().device_id = dev_id;
-    self->state().stream_ids = std::move(stream_ids);
-    self->state().max_in_flight_tasks = max_in_flight_tasks;
-
-    // Trigger initial work requests up to max_in_flight_tasks
-    for (int i = 0; i < max_in_flight_tasks; ++i) {
-        self->mail(request_work_atom_v).send(self);
-    }
-
-    return {
-        [=](request_work_atom) {
-            auto& st = self->state();
-            if (st.in_flight_tasks_count >= st.max_in_flight_tasks || st.draining) {
-                return; // Already at max capacity, don't request more yet
-            }
-
-            st.in_flight_tasks_count++; // Mark as pending immediately
-            self->mail(get_work_atom_v).request(st.device_actor, infinite).then(
-                [=](int N, in<int> matrixA, in<int> matrixB) mutable {
-                    auto& st = self->state(); // Access state via self
-                    int s_h2d = st.stream_ids[0];
-                    int s_ker = st.stream_ids[1];
-                    int s_d2h = st.stream_ids[2];
-
-                    // Create fresh events for this specific task pipeline
-                    auto h2d_done = mmul_command.create_event(st.device_id);
-                    auto kernel_done = mmul_command.create_event(st.device_id);
-
-                    // Stage 1: H2D Transfer
-                    auto arg1 = mmul_command.transfer_memory(st.device_id, s_h2d, std::move(matrixA));
-                    auto arg2 = mmul_command.transfer_memory(st.device_id, s_h2d, std::move(matrixB));
-                    mmul_command.record_event(h2d_done, s_h2d, st.device_id);
-                  
-                    const int THREADS = 32;
-                    const int BLOCKS = (N + THREADS - 1) / THREADS;
-                    caf::cuda::nd_range dims(BLOCKS, BLOCKS, 1, THREADS, THREADS, 1);
-
-                    // Stage 2: Kernel Execution (Wait for H2D to finish)
-                    mmul_command.wait_event(h2d_done, s_ker, st.device_id);
-                    auto result = mmul_kernel.run_async(st.program, dims, s_ker, 0, st.device_id,
-                                                       arg1, arg2,
-                                                       caf::cuda::create_out_arg<int>(N * N),
-                                                       caf::cuda::create_in_arg<int>(N));
-                    mmul_command.record_event(kernel_done, s_ker, st.device_id);
-                      mmul_command.add_callback(s_ker, st.device_id, [arg1, arg2, self]() {
-                        anon_mail(arg1,arg2).send(self);
-                    });
-
-
-                    // Stage 3: D2H Copyback (Wait for Kernel to finish)
-                    mmul_command.wait_event(kernel_done, s_d2h, st.device_id);
-                    auto bufferC = std::get<2>(result);
-                    auto self_hdl = caf::actor_cast<caf::actor>(self);
-
-                  
-                    mmul_command.copy_to_host_async(bufferC, s_d2h,
-                      [self_hdl, N_task = N](std::vector<int>&&) {
-                        caf::anon_mail(task_done_atom_v, N_task).send(self_hdl); // Pass N back to self
-                    });
-                },
-                [=](error& err) {
-                    auto& st = self->state();
-                    st.in_flight_tasks_count--; // Revert pending status on failure
-                    if (err == sec::runtime_error) {
-                        // Not enough memory, retry after a delay
-                        self->println("Worker (dev:{}): Not enough memory, retrying for work...", st.device_id);
-                        self->delayed_anon_send(self, 100ms, request_work_atom_v);
-                    } else if (err == sec::end_of_stream) {
-                        st.draining = true; // Mark as draining, let in-flight finish
-                        if (st.in_flight_tasks_count == 0) {
-                            self->mail(worker_done_atom_v).send(st.device_actor);
-                            for (auto id : st.stream_ids)
-                                mmul_command.release_stream_for_actor(id);
-                            self->quit();
-                        }
-                    }
-                }
-            );
-        },
-        [=](caf::cuda::mem_ptr<int> matrixA, caf::cuda::mem_ptr<int> matrixB) {
-            mmul_command.free_memory(matrixA, stream_ids[1]);
-            mmul_command.free_memory(matrixB, stream_ids[1]);
-        },
-        [=](task_done_atom, int N_completed) {
-            auto& st = self->state();
-            st.in_flight_tasks_count--; // Decrement count
-            self->mail(1).send(st.supervisor); // Notify supervisor
-            self->mail(release_memory_atom_v, N_completed).send(st.device_actor); // Release memory
-            
-            if (st.draining && st.in_flight_tasks_count == 0) {
-                self->mail(worker_done_atom_v).send(st.device_actor);
-                for (auto id : st.stream_ids)
-                    mmul_command.release_stream_for_actor(id);
-                self->quit();
-            } else if (!st.draining) {
-                self->mail(request_work_atom_v).send(self); // Request next task if capacity allows
+                int N = st.N_val;
+
+                // 1. Setup GPU arguments.
+                // Fetch data from the shared pool only when scheduled to save RAM
+                auto in_a = create_in_arg(st.pool->A.at(N));
+                auto in_b = create_in_arg(st.pool->B.at(N));
+                auto out_c = create_out_arg_with_size<int>(N * N);
+                auto in_n = create_in_arg(N);
+
+                const int THREADS = 32;
+                int current_N = N;
+                nd_range range((current_N + THREADS - 1) / THREADS, 
+                           (current_N + THREADS - 1) / THREADS, 1, 
+                           THREADS, THREADS, 1);
+
+                // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
+                auto result_tuple = runner.run_async(st.prog, range,st.stream,0,st.device ,in_a, in_b, out_c, in_n);
+                auto d_c = std::get<2>(result_tuple);
+
+                // 3. Asynchronous Copyback.
+                // Allocate a local buffer for the result to keep the total system memory low.
+                auto h_c = std::make_shared<std::vector<int>>(N * N);
+                // The launch_response_token is released inside the callback 
+                // to signal to the scheduler that the resource is free. (No serial verification here)
+                runner.copy_to_host_async(d_c, h_c->data(), h_c->size(), [exit_actor, h_c](int* /*ptr*/, size_t /*sz*/) {
+                    anon_mail(1).send(exit_actor);
+                });
             }
-        }
+        
     };
 }
 
-// ---------------------------- SUPERVISOR ACTOR ----------------------------
-struct supervisor_actor_state {
-    int total_tasks;
-    int completed = 0;
-    std::chrono::steady_clock::time_point start_time;
-};
-
-caf::behavior supervisor_actor_fun(
-    caf::stateful_actor<supervisor_actor_state>* self,
-    int total_tasks,
-    int workers_per_gpu,
-    int max_in_flight_tasks_per_worker,
-    MatrixPool pool,
-    std::vector<int> Ns
-    ) {
-    self->state().total_tasks = total_tasks;
-    self->state().start_time = std::chrono::steady_clock::now();
-
-    auto pool_actor = self->spawn(global_task_pool, std::move(Ns));
-
-    caf::cuda::manager& mgr = caf::cuda::manager::get();
-    int num_gpus = mgr.get_num_devices();
-    auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
-    
-    int next_stream_base = 0;
-
-    for (int i = 0; i < num_gpus; ++i) {
-        auto broker = self->spawn(gpu_device_actor, pool, pool_actor, workers_per_gpu, i, max_in_flight_tasks_per_worker);
-        
-        for (int j = 0; j < workers_per_gpu; ++j) {
-            std::vector<int> streams = {next_stream_base, next_stream_base + 1, next_stream_base + 2};
-            self->spawn(mmul_worker_fun, self, broker, program, i, streams, max_in_flight_tasks_per_worker);
-            next_stream_base += 3;
-        }
-    }
-
-    return {
-        [=](int done) {
-            self->state().completed += done;
-            if (self->state().completed >= self->state().total_tasks) {
-                auto end_time = std::chrono::steady_clock::now();
-                std::chrono::duration<double> total_time = end_time - self->state().start_time;
-
-                std::cout << "\n===== BENCHMARK COMPLETE =====\n";
-                std::cout << "Tasks: " << self->state().total_tasks << "\n";
-                std::cout << "Runtime: " << total_time.count() << " s\n";
-                
-                caf::cuda::manager::shutdown();
-                self->quit();
-            }
-        }
-    };
+// Helper function to initialize task_actor_state
+behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, std::shared_ptr<MatrixPool> pool, 
+                                caf::actor exit_actor, int device, int stream) {
+    auto& st = self->state();
+    st.prog = std::move(prog);
+    st.N_val = N_val;
+    st.pool = std::move(pool);
+    st.device = device;
+    st.stream = stream;
+    return task_actor_fun(self, exit_actor); // Pass exit_actor to task_actor_fun
 }
 
 template <class Fn>
 double time_run(Fn&& fn) {
-  auto start = std::chrono::steady_clock::now();
-  fn();
-  auto end = std::chrono::steady_clock::now();
-  std::chrono::duration<double> elapsed = end - start;
-  return elapsed.count();
+    auto start = std::chrono::steady_clock::now();
+    fn();
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> elapsed = end - start;
+    return elapsed.count();
 }
 
-void run_mmul_random_scaling_tests(caf::actor_system& sys,
-                                  caf::cuda::manager_config man_config) {
-
+void run_scheduler_integration_scaling_test(actor_system& sys) {
     const int min_N = 32;
     const int max_N = 2048;
-    const int num_sizes = 10;
+    const int num_distinct_sizes = 10;
+    const std::vector<int> actor_counts = {50000};
+    int streams = 8;
 
-    const int workers_per_gpu = 4; // Admission control: only 16 concurrent tasks per GPU
-    const int max_in_flight_tasks_per_worker = 2; // Each worker keeps 2 tasks in flight
 
-    const std::vector<int> actor_counts = {
-	  1,30000,40000,50000
-    };
+    //  const std::vector<int> actor_counts = {5};
 
-    // Generate deterministic random pool once
-    MatrixPool pool = create_matrix_pool_random(
-        num_sizes,
-        min_N,
-        max_N,
-        42  // fixed seed
-    );
 
-    //scheduler
-    caf::cuda::manager_config scheduler_off(false);
-    for (int num_tasks_for_this_run : actor_counts) {
-	    // Initialize CUDA manager
-	    caf::cuda::manager::init(sys, scheduler_off);
-        std::cout << "=====================================\n";
-        std::cout << "Random Scaling | actors=" << num_tasks_for_this_run << "\n";
 
-        // Precompute all task Ns for this run
-        std::vector<int> sizes;
-        for (const auto& [N, _] : pool.A) sizes.push_back(N);
+    // Generate deterministic random pool once
+    auto pool_ptr = std::make_shared<MatrixPool>(
+        create_matrix_pool_random(num_distinct_sizes, min_N, max_N, 42));
+    
+    std::vector<int> available_Ns;
+    for (const auto& pair : pool_ptr->A) available_Ns.push_back(pair.first);
+
+    for (int num_tasks : actor_counts) {
+        manager_config config;
+        manager::init(sys, config);
+        auto& mgr = manager::get();
+        int devices = mgr.get_num_devices();
+        int stream_id = 0;
+        int device_id = 0;
+        
 
-        std::vector<int> Ns_for_this_run;
-        Ns_for_this_run.reserve(num_tasks_for_this_run);
+      
+
+        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        const int THREADS = 32;
 
         std::mt19937 rng(42);
-        std::uniform_int_distribution<size_t> dist(0, sizes.size() - 1);
-        for (int i = 0; i < num_tasks_for_this_run; ++i)
-	        Ns_for_this_run.push_back(sizes[dist(rng)]);
+        std::uniform_int_distribution<size_t> dist_N_idx(0, available_Ns.size() - 1);
 
-        // Execute the supervisor which manages the asynchronous workload
-        double elapsed = time_run([&]() {
+        std::cout << "=====================================\n";
+        std::cout << "Scheduler Test | tasks=" << num_tasks << "\n";
+
+        // Spawn the exit actor for this specific test run (moved inside the loop)
+        auto exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_tasks);
+
+        std::vector<actor> workers;
+        // Spawn task actors and prepare tokens
+        for (int i = 0; i < num_tasks; ++i) {
+            int current_N = available_Ns[dist_N_idx(rng)];
+           
+            
+            auto worker = sys.spawn(make_task_actor_behavior,
+                                    program,
+                                    current_N, 
+                                    pool_ptr,
+                                    exit_actor, 
+                                    device_id, 
+                                    stream_id); // Pass exit_actor to task actors
+            workers.push_back(worker);
+            stream_id = (stream_id + 1) % streams;
+            device_id = (device_id + 1) % devices;
+        }
 
-            auto sup = sys.spawn(
-                supervisor_actor_fun,
-                (int)Ns_for_this_run.size(), // total_tasks
-                workers_per_gpu,
-                max_in_flight_tasks_per_worker,
-                pool,
-		Ns_for_this_run
-            );
+        double elapsed = time_run([&]() {
+            // std::cout << "[MAIN] Dispatching batch to scheduler..." << std::endl;
+            // mgr.send_scheduler_actor_message(std::move(tokens));
 
-	      sys.await_all_actors_done();
-	    });
+            // The dispatch is asynchronous. To get an accurate measurement, we must
+            // block until the exit_actor terminates (signaling all 50k tasks are done).
+            scoped_actor self{sys};
+            self->wait_for(exit_actor);
+        });
 
-        caf::cuda::manager::shutdown();
+        std::cout << "Run complete. Time: " << elapsed << " s\n";
+        manager::shutdown(); // Reset manager state for the next potential iteration
     }
 }
-void caf_main(caf::actor_system& sys) {
-  caf::cuda::manager_config man_config(false);
-  run_mmul_random_scaling_tests(sys, man_config);
+
+void caf_main(actor_system& sys) {
+    run_scheduler_integration_scaling_test(sys);
+    std::cout << "[MAIN] Integration test complete." << std::endl;
 }
-CAF_MAIN(id_block::mmul_benchmark)
+
+CAF_MAIN(id_block::cuda_control)

From 996a270f7e1bbf477441434acbebe81292cf6413 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 10:02:24 -0600
Subject: [PATCH 0987/1000] Fixed rng issue.

---
 .../mmul-randonom-batch-benchmark/cuda-baseline.cpp           | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index bfd5407f9c..90f69a56f2 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -222,8 +222,8 @@ int main() {
             std::vector<Task> batch_tasks;
             for (int i = 0; i < current_batch_size; ++i) {
                 int N_for_task = available_Ns[dist_N_idx(rng_prod)];
-                TaskType t_type = static_cast<TaskType>(dist_type(rng_prod));
-                batch_tasks.push_back({N_for_task, t_type});
+                // TaskType t_type = static_cast<TaskType>(dist_type(rng_prod));
+                batch_tasks.push_back({N_for_task, 0});
             }
 
             std::cout << "Producer: Dispatching Batch " << b + 1 << "/" << num_batches 

From 99c6137ccd0436e9ebaa20bf9b3aaf7eb721d075 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 10:03:10 -0600
Subject: [PATCH 0988/1000] Fixed rng issue.

---
 .../mmul-randonom-batch-benchmark/cuda-baseline.cpp           | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index 90f69a56f2..e081c6f8b4 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -222,8 +222,8 @@ int main() {
             std::vector<Task> batch_tasks;
             for (int i = 0; i < current_batch_size; ++i) {
                 int N_for_task = available_Ns[dist_N_idx(rng_prod)];
-                // TaskType t_type = static_cast<TaskType>(dist_type(rng_prod));
-                batch_tasks.push_back({N_for_task, 0});
+                TaskType t_type = static_cast<TaskType>(0);
+                batch_tasks.push_back({N_for_task, t_type});
             }
 
             std::cout << "Producer: Dispatching Batch " << b + 1 << "/" << num_batches 

From 0be82c921e0ef97b7edae453f0cc1e34b7f89420 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 10:04:34 -0600
Subject: [PATCH 0989/1000] Fixed rng issue.

---
 .../mmul-randonom-batch-benchmark/cuda-baseline.cpp           | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index e081c6f8b4..a6249c3f16 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -213,10 +213,6 @@ int main() {
         auto start = std::chrono::steady_clock::now();
 
         for (int b = 0; b < num_batches; ++b) {
-            // Sleep timer (random time)
-            int sleep_ms = dist_sleep(rng_prod);
-            std::this_thread::sleep_for(std::chrono::milliseconds(sleep_ms));
-
             // Generate random sized partition (batch)
             int current_batch_size = dist_batch_size(rng_prod);
             std::vector<Task> batch_tasks;

From 62c1e79a0b976c628978eda4295e66bc48ae918c Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 10:14:43 -0600
Subject: [PATCH 0990/1000] Fixed tests to make more rigourous.

---
 .../cuda-baseline.cpp                         | 19 +++++++++++++------
 .../main.test.cpp                             |  8 +++++---
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index a6249c3f16..c8e3521648 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -210,9 +210,9 @@ int main() {
         std::vector<std::thread> all_threads;
         std::deque<std::vector<std::vector<Task>>> run_partitions_storage;
 
-        auto start = std::chrono::steady_clock::now();
 
-        for (int b = 0; b < num_batches; ++b) {
+         for (int b = 0; b < num_batches; ++b) {
+            
             // Generate random sized partition (batch)
             int current_batch_size = dist_batch_size(rng_prod);
             std::vector<Task> batch_tasks;
@@ -222,6 +222,9 @@ int main() {
                 batch_tasks.push_back({N_for_task, t_type});
             }
 
+            
+            auto start = std::chrono::steady_clock::now();
+
             std::cout << "Producer: Dispatching Batch " << b + 1 << "/" << num_batches 
                       << " with " << current_batch_size << " tasks..." << std::endl;
 
@@ -239,15 +242,19 @@ int main() {
                                             contexts[i], mmul_funcs[i], vadd_funcs[i], conv_funcs[i],
                                          std::ref(global_host_matrix_pool), shared_dtoh_buffer.data());
             }
-        }
+        
 
         for (auto& t : all_threads) {
             t.join();
         }
 
-        auto end = std::chrono::steady_clock::now();
-        std::chrono::duration<double> elapsed = end - start;
-        std::cout << "Total Makespan: " << elapsed.count() << "s" << std::endl;
+          auto end = std::chrono::steady_clock::now();
+          std::chrono::duration<double> elapsed = end - start;
+          std::cout << "Total Makespan: " << elapsed.count() << "s" << std::endl;
+    
+        }   
+      
+
     }
 
     // Cleanup contexts and module
diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 007de9beb4..14c991744e 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -153,6 +153,9 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         // Spawn the exit actor for this specific test run (moved inside the loop)
         auto exit_actor = sys.spawn(caf::cuda::exit_actor_fun, num_tasks);
 
+
+
+        double elapsed = time_run([&]() {
         std::vector<actor> workers;
         // Spawn task actors and prepare tokens
         for (int i = 0; i < num_tasks; ++i) {
@@ -171,9 +174,8 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
             device_id = (device_id + 1) % devices;
         }
 
-        double elapsed = time_run([&]() {
-            // std::cout << "[MAIN] Dispatching batch to scheduler..." << std::endl;
-            // mgr.send_scheduler_actor_message(std::move(tokens));
+       
+          
 
             // The dispatch is asynchronous. To get an accurate measurement, we must
             // block until the exit_actor terminates (signaling all 50k tasks are done).

From a524cbd878081ee86afa57477a656807e7028d57 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 10:25:24 -0600
Subject: [PATCH 0991/1000] Fixed error in stream and device allocation.

---
 .../mmul-randonom-batch-benchmark/main.test.cpp               | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 14c991744e..dcc37a7cca 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -171,7 +171,9 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
                                     stream_id); // Pass exit_actor to task actors
             workers.push_back(worker);
             stream_id = (stream_id + 1) % streams;
-            device_id = (device_id + 1) % devices;
+            if (stream_id == 0) {
+                device_id = (device_id + 1) % devices;
+            }
         }
 
        

From d49e28fb07edeb19fb0267d57467508c19ab866a Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 11:09:47 -0600
Subject: [PATCH 0992/1000] CHANGED stream settings to match caf cuda.

---
 .../mmul-randonom-batch-benchmark/cuda-baseline.cpp             | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
index c8e3521648..5836ee4d5d 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/cuda-baseline.cpp
@@ -48,7 +48,7 @@ void gpu_worker(int device_id, const std::vector<Task>& tasks, int streams_per_g
     // Prepare Streams
     std::vector<CUstream> streams(streams_per_gpu);
     for (int i = 0; i < streams_per_gpu; ++i) {
-        CUDA_CHECK(cuStreamCreate(&streams[i], CU_STREAM_NON_BLOCKING));
+        CUDA_CHECK(cuStreamCreate(&streams[i], CU_STREAM_DEFAULT));
     }
 
     // Use the first stream for initial allocations and cleanup

From 9813f34be3879b3b091fda7a5b47b5c87f167642 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 13:55:59 -0600
Subject: [PATCH 0993/1000] updated to make setting more identical to caf cuda

---
 .../baseline-comparison/cuda/matrix_mul_driver.cpp   | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
index 629577c08c..310a79d0f3 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
@@ -55,12 +55,20 @@ int main() {
     checkCU(cuDeviceGet(&dev, 0), "cuDeviceGet(0)");
 
     CUcontext ctx;
-    checkCU(cuCtxCreate(&ctx, 0, dev), "cuCtxCreate");
+  
+#if CUDA_VERSION >= 13000
+    {
+      CUctxCreateParams ctx_params = {};
+      checkCU(cuCtxCreate(&ctx, &ctx_params, CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, dev),"create context");
+    }
+#else
+      checkCU(cuCtxCreate(&ctx, CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, dev),"create context");
+#endif
 
     std::string ptx = readFile("mmul.ptx");
 
     CUmodule module;
-    checkCU(cuModuleLoadDataEx(&module, ptx.c_str(), 0, nullptr, nullptr), "cuModuleLoadDataEx");
+    checkCU(cuModuleLoadData(&module, ptx.c_str()), "cuModuleLoadData");
 
     CUfunction kernel;
     checkCU(cuModuleGetFunction(&kernel, module, "matrixMul"), "cuModuleGetFunction matrixMul");

From ad0ab669dbe5ef1933ddb1f783e250d4c655b15f Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 14:45:13 -0600
Subject: [PATCH 0994/1000] made changes to hopefully alleviate CPU bottleneck

---
 .../main.test.cpp                             | 34 +++++++++----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index dcc37a7cca..6ddb905f87 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -52,13 +52,13 @@ MatrixPool create_matrix_pool_random(
 
 // This actor represents a single task that requests permission from the scheduler.
 behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_actor) {
-    self->mail("hello").send(self);
+   
 
     return {
-        [=](std::string start_tag) mutable {
+        [=](int device,int stream,int current_N) mutable {
                 auto& st = self->state();
-                
-                int N = st.N_val;
+
+                int N = current_N;
 
                 // 1. Setup GPU arguments.
                 // Fetch data from the shared pool only when scheduled to save RAM
@@ -68,7 +68,6 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_
                 auto in_n = create_in_arg(N);
 
                 const int THREADS = 32;
-                int current_N = N;
                 nd_range range((current_N + THREADS - 1) / THREADS, 
                            (current_N + THREADS - 1) / THREADS, 1, 
                            THREADS, THREADS, 1);
@@ -92,13 +91,11 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_
 
 // Helper function to initialize task_actor_state
 behavior make_task_actor_behavior(stateful_actor<task_actor_state>* self, program_ptr prog, int N_val, std::shared_ptr<MatrixPool> pool, 
-                                caf::actor exit_actor, int device, int stream) {
+                                caf::actor exit_actor) {
     auto& st = self->state();
     st.prog = std::move(prog);
     st.N_val = N_val;
     st.pool = std::move(pool);
-    st.device = device;
-    st.stream = stream;
     return task_actor_fun(self, exit_actor); // Pass exit_actor to task_actor_fun
 }
 
@@ -157,23 +154,26 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
 
         double elapsed = time_run([&]() {
         std::vector<actor> workers;
-        // Spawn task actors and prepare tokens
-        for (int i = 0; i < num_tasks; ++i) {
-            int current_N = available_Ns[dist_N_idx(rng)];
-           
-            
-            auto worker = sys.spawn(make_task_actor_behavior,
+        for (int i =0; i < 200; i++) {
+            int current_N = 1;
+               auto worker = sys.spawn(make_task_actor_behavior,
                                     program,
                                     current_N, 
                                     pool_ptr,
-                                    exit_actor, 
-                                    device_id, 
-                                    stream_id); // Pass exit_actor to task actors
+                                    exit_actor); // Pass exit_actor to task actors
             workers.push_back(worker);
+           
+        }
+        // Spawn task actors and prepare tokens
+        for (int i = 0; i < num_tasks; ++i) {
+            int current_N = available_Ns[dist_N_idx(rng)];
+           
+            anon_mail(device_id,stream_id,current_N).send(workers[i%workers.size()]);
             stream_id = (stream_id + 1) % streams;
             if (stream_id == 0) {
                 device_id = (device_id + 1) % devices;
             }
+           
         }
 
        

From bbdee31805d629032ba3a8aa0a9fe2a06da118f5 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Tue, 23 Jun 2026 15:44:19 -0600
Subject: [PATCH 0995/1000] Fixed errors with device and steram selection.

---
 .../mmul-randonom-batch-benchmark/main.test.cpp                 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
index 6ddb905f87..f022666dc6 100644
--- a/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/scheduler-benchmark-tests/mmul-randonom-batch-benchmark/main.test.cpp
@@ -73,7 +73,7 @@ behavior task_actor_fun(stateful_actor<task_actor_state>* self, caf::actor exit_
                            THREADS, THREADS, 1);
 
                 // 2. Launch Work asynchronously using the stream and device assigned by the scheduler.
-                auto result_tuple = runner.run_async(st.prog, range,st.stream,0,st.device ,in_a, in_b, out_c, in_n);
+                auto result_tuple = runner.run_async(st.prog, range,stream,0,device ,in_a, in_b, out_c, in_n);
                 auto d_c = std::get<2>(result_tuple);
 
                 // 3. Asynchronous Copyback.

From c8a9e7866b049960252ddcca81e7965d968fa924 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 24 Jun 2026 08:25:48 -0600
Subject: [PATCH 0996/1000] corrected filepath

---
 .../baseline-comparison/cuda/matrix_mul_driver.cpp              | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
index 310a79d0f3..77c5ae300b 100644
--- a/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
+++ b/libcaf_cuda/tests/benchmark-tests/mmul-actor-benchmarking/baseline-comparison/cuda/matrix_mul_driver.cpp
@@ -65,7 +65,7 @@ int main() {
       checkCU(cuCtxCreate(&ctx, CU_CTX_SCHED_AUTO | CU_CTX_MAP_HOST, dev),"create context");
 #endif
 
-    std::string ptx = readFile("mmul.ptx");
+    std::string ptx = readFile("../mmul.cubin");
 
     CUmodule module;
     checkCU(cuModuleLoadData(&module, ptx.c_str()), "cuModuleLoadData");

From 235db4b1c47f745c1b40860ea9ed141b3eab6f13 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 24 Jun 2026 10:49:01 -0600
Subject: [PATCH 0997/1000] Updated stuff to actually compile in accordance to
 AD.

---
 libcaf_cuda/sc26/Batched-Matrix-Multiply/CMakeLists.txt | 3 +++
 libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp  | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/libcaf_cuda/sc26/Batched-Matrix-Multiply/CMakeLists.txt b/libcaf_cuda/sc26/Batched-Matrix-Multiply/CMakeLists.txt
index 12886f5340..f5e0fd114c 100644
--- a/libcaf_cuda/sc26/Batched-Matrix-Multiply/CMakeLists.txt
+++ b/libcaf_cuda/sc26/Batched-Matrix-Multiply/CMakeLists.txt
@@ -39,6 +39,9 @@ target_link_libraries(test
     "${CAF_BUILD}/libcaf_io/libcaf_io.so"
     "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
     CUDA::nvrtc
+    CUDA::cuda_driver
+    CUDA::cusparse
+    CUDA::cublas
 )
 
 
diff --git a/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp b/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp
index ff92ec5fca..628adc38dc 100644
--- a/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp
+++ b/libcaf_cuda/sc26/Batched-Matrix-Multiply/main.test.cpp
@@ -128,7 +128,7 @@ void run_scheduler_integration_scaling_test(actor_system& sys) {
         // Start scheduler with 4 streams and depth 2 (8 slots) to force queuing
         mgr.toggle_scheduler_actor(8, 1);
 
-        auto program = mgr.create_program_from_cubin("../mmul.cubin", "matrixMul");
+        auto program = mgr.create_program_from_cubin("mmul.cubin", "matrixMul");
         const int THREADS = 32;
 
         std::vector<token_ptr> tokens;

From 839e7477314d9ad49bd606432743e72a84f345ed Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 24 Jun 2026 10:52:05 -0600
Subject: [PATCH 0998/1000] Removed directory since not being used.

---
 .../sc26/Fault-Tolerance-2/CMakeLists.txt     |  50 ----
 .../sc26/Fault-Tolerance-2/main.test.cpp      | 222 ------------------
 2 files changed, 272 deletions(-)
 delete mode 100644 libcaf_cuda/sc26/Fault-Tolerance-2/CMakeLists.txt
 delete mode 100644 libcaf_cuda/sc26/Fault-Tolerance-2/main.test.cpp

diff --git a/libcaf_cuda/sc26/Fault-Tolerance-2/CMakeLists.txt b/libcaf_cuda/sc26/Fault-Tolerance-2/CMakeLists.txt
deleted file mode 100644
index e64bc364c2..0000000000
--- a/libcaf_cuda/sc26/Fault-Tolerance-2/CMakeLists.txt
+++ /dev/null
@@ -1,50 +0,0 @@
-cmake_minimum_required(VERSION 3.16.3)
-
-# 1) Enforce C++20 (needs modern C++ features)
-set(CMAKE_CXX_STANDARD        20)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_EXTENSIONS      OFF)
-
-# 2) Set CAF source and build directories
-set(CAF_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../../../actor-framework")
-set(CAF_BUILD "${CAF_SRC}/build")
-
-
-#required since cmake cant seem to find the compiler 
-set(CMAKE_CXX_COMPILER "/usr/bin/g++")
-set(CMAKE_C_COMPILER "/usr/bin/gcc")
-
-project(CUDA_ACTORS)
-
-find_package(CUDA REQUIRED)
-find_package(CUDAToolkit REQUIRED)
-
-include_directories(
-  "${CAF_SRC}/include"
-  "${CAF_SRC}/libcaf_io"
-  "${CAF_SRC}/libcaf_core"
-  "${CAF_SRC}/libcaf_opencl"
-  "${CAF_BUILD}/libcaf_core"  # for generated build_config.hpp  
-  "${CAF_SRC}/libcaf_cuda"
-  "${CAF_SRC}/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor"
-  "${CAF_SRC}/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor"
-  "${CAF_SRC}/libcaf_cuda/caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor"
-)
-
-
-# 5) Declare your executables
-
-add_executable(test main.test.cpp)
-target_compile_definitions(test PRIVATE CAF_ENABLE_LOGGING)
-
-
-target_link_libraries(test
-  PRIVATE
-    "${CAF_BUILD}/libcaf_core/libcaf_core.so"
-    "${CAF_BUILD}/libcaf_io/libcaf_io.so"
-    "${CAF_BUILD}/libcaf_cuda/libcaf_cuda.so"
-    CUDA::nvrtc
-    CUDA::cublas
-    CUDA::cusparse
-)
-
diff --git a/libcaf_cuda/sc26/Fault-Tolerance-2/main.test.cpp b/libcaf_cuda/sc26/Fault-Tolerance-2/main.test.cpp
deleted file mode 100644
index e80ca34cef..0000000000
--- a/libcaf_cuda/sc26/Fault-Tolerance-2/main.test.cpp
+++ /dev/null
@@ -1,222 +0,0 @@
-#include <caf/all.hpp>
-#include <caf/cuda/all.hpp>
-#include <iostream>
-#include <vector>
-#include <fstream>
-#include <cmath>
-#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-CGS-actor/sparse-CGS-actor.hpp"
-#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-BiCGSTAB-actor/sparse-BiCGSTAB-actor.hpp"
-#include "caf/actorSOLVE/sparse-matrix-solvers/sparse-GMRES-actor/sparse-GMRES-actor.hpp"
-
-using namespace caf;
-using namespace caf::cuda;
-
-// --- Matrix Utilities ---
-
-template <class T = float>
-struct LocalCSR {
-    int rows, cols, nnz;
-    std::vector<int> row_ptr;
-    std::vector<int> col_ind;
-    std::vector<T> values;
-};
-
-template <class T = float>
-LocalCSR<T> load_binary_matrix(const std::string& path) {
-    std::ifstream file(path, std::ios::binary);
-    if (!file) throw std::runtime_error("Could not open " + path);
-    int32_t r, c, n;
-    file.read(reinterpret_cast<char*>(&r), sizeof(int32_t));
-    file.read(reinterpret_cast<char*>(&c), sizeof(int32_t));
-    file.read(reinterpret_cast<char*>(&n), sizeof(int32_t));
-    std::vector<int32_t> rows_coo(n), cols_coo(n);
-    std::vector<float> vals_coo(n);
-    file.read(reinterpret_cast<char*>(rows_coo.data()), n * sizeof(int32_t));
-    file.read(reinterpret_cast<char*>(cols_coo.data()), n * sizeof(int32_t));
-    file.read(reinterpret_cast<char*>(vals_coo.data()), n * sizeof(float));
-    int max_idx = 0;
-    for(auto v : rows_coo) if(v > max_idx) max_idx = v;
-    if (max_idx == r || max_idx == c) {
-        for(auto& v : rows_coo) v--;
-        for(auto& v : cols_coo) v--;
-    }
-    LocalCSR<T> csr;
-    csr.rows = r; csr.cols = c; csr.nnz = n;
-    csr.row_ptr.assign(r + 1, 0);
-    csr.col_ind.resize(n);
-    csr.values.resize(n);
-    for (int i = 0; i < n; ++i) csr.row_ptr[rows_coo[i] + 1]++;
-    for (int i = 0; i < r; ++i) csr.row_ptr[i + 1] += csr.row_ptr[i];
-    std::vector<int> current_pos = csr.row_ptr;
-    for (int i = 0; i < n; ++i) {
-        int row = rows_coo[i];
-        int dest = current_pos[row]++;
-        csr.col_ind[dest] = cols_coo[i];
-        csr.values[dest] = static_cast<T>(vals_coo[i]);
-    }
-
-    std::cout << "rows=" << csr.rows
-              << " cols=" << csr.cols
-              << " nnz=" << csr.nnz
-              << " row_ptr.back()=" << csr.row_ptr.back()
-              << std::endl;
-
-    for (int i = 0; i < csr.nnz; ++i) {
-        if (csr.col_ind[i] < 0 || csr.col_ind[i] >= csr.cols)
-            throw std::runtime_error("bad col index");
-    }
-
-    return csr;
-}
-
-template <class T = float>
-std::vector<T> compute_rhs(const LocalCSR<T>& A, const std::vector<T>& x) {
-    std::vector<T> b(A.rows, T{0});
-    for (int i = 0; i < A.rows; ++i) {
-        T sum = T{0};
-        for (int j = A.row_ptr[i]; j < A.row_ptr[i+1]; ++j)
-            sum += A.values[j] * x[A.col_ind[j]];
-        b[i] = sum;
-    }
-    return b;
-}
-
-bool is_valid(const std::vector<double>& x) {
-    for (double val : x) {
-        if (std::isnan(val) || std::isinf(val)) return false;
-    }
-    return true;
-}
-
-// --- Robust Solver Actor (Facade Orchestrator) ---
-
-enum class solver_strategy { cgs, bicgstab, gmres };
-
-struct robust_solver_state {
-    LocalCSR<double> A;
-    std::vector<double> b;
-    std::vector<double> x;
-    double tol;
-    int max_iter;
-    std::vector<caf::actor> facades;
-    solver_strategy current_strategy = solver_strategy::cgs;
-    caf::actor requester;
-};
-
-behavior robust_solver(stateful_actor<robust_solver_state>* self, 
-                       LocalCSR<double> A, std::vector<double> b, double tol, int max_iter) {
-    self->state().A = std::move(A);
-    self->state().b = std::move(b);
-    self->state().tol = tol;
-    self->state().x.assign(self->state().A.rows, 0.0);
-    self->state().max_iter = max_iter;
-
-    self->state().facades.push_back(self->spawn<sparse_cg_facade<double>>(100));
-    self->state().facades.push_back(self->spawn<sparse_bicgstab_facade<double>>(100));
-    self->state().facades.push_back(self->spawn<sparse_gmres_facade<double>>(100));
-
-    auto get_method_name = [](solver_strategy s) {
-        switch (s) {
-            case solver_strategy::cgs:      return "CGS";
-            case solver_strategy::bicgstab: return "BiCGSTAB";
-            case solver_strategy::gmres:    return "GMRES";
-            default:                        return "Unknown";
-        }
-    };
-
-    auto start_cgs = [=] {
-        auto& s = self->state();
-        s.current_strategy = solver_strategy::cgs;
-        self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
-                   create_in_arg(s.b), create_in_out_arg(s.x),
-                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 0, 0).send(s.facades[0]);
-    };
-
-    auto start_bicgstab = [=] {
-        auto& s = self->state();
-        s.current_strategy = solver_strategy::bicgstab;
-        self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
-                   create_in_arg(s.b), create_in_out_arg(s.x),
-                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 0, 0).send(s.facades[1]);
-    };
-
-    auto start_gmres = [=] {
-        auto& s = self->state();
-        s.current_strategy = solver_strategy::gmres;
-        self->mail(create_in_arg(s.A.row_ptr), create_in_arg(s.A.col_ind), create_in_arg(s.A.values),
-                   create_in_arg(s.b), create_in_out_arg(s.x),
-                   matrix_format::csr, s.A.rows, s.A.nnz, s.tol, s.max_iter, 30, 0, 0).send(s.facades[2]);
-    };
-
-    return {
-        [=](start_atom) {
-            self->state().requester = actor_cast<caf::actor>(self->current_sender());
-            std::cout << "[INFO] Attempting solve with CGS..." << std::endl;
-            start_cgs();
-        },
-        [=](uint32_t /*id*/, int /*idx*/, const std::vector<double>& result, solver_result_meta meta) {
-            auto& s = self->state();
-            const char* method_name = get_method_name(s.current_strategy);
-            if (meta.converged && is_valid(result)) {
-                std::cout << "[SUCCESS] " << method_name << " converged. Matrix good." << std::endl;
-                if (s.requester)
-                    self->mail(true).send(s.requester);
-                self->quit();
-            } else {
-                std::cout << "[WARNING] " << method_name;
-                if (!meta.converged) {
-                    std::cout << " failed to converge.";
-                }
-                if (!is_valid(result)) {
-                    if (!meta.converged) {
-                        std::cout << " and";
-                    }
-                    std::cout << " produced NaN/Inf values.";
-                }
-                std::cout << " Retrying..." << std::endl;
-                switch (s.current_strategy) {
-                    case solver_strategy::cgs:
-                        start_bicgstab();
-                        break;
-                    case solver_strategy::bicgstab:
-                        start_gmres();
-                        break;
-                    default:
-                        std::cout << "[ERROR] All facade solvers failed." << std::endl;
-                        if (s.requester)
-                            self->mail(std::string("All facade solvers failed")).send(s.requester);
-                        self->quit();
-                        break;
-                }
-            }
-        }
-    };
-}
-
-void caf_main(actor_system& sys) {
-    manager::init(sys, manager_config(true, true)); 
-    scoped_actor self{sys};
-
-    std::string path = "/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric/jpwh_991.bin";
-    std::cout << "[INFO] Loading real-world matrix: " << path << std::endl;
-
-    try {
-        LocalCSR<double> A = load_binary_matrix<double>(path);
-        std::vector<double> x_target(A.rows, 1.0);
-        std::vector<double> b = compute_rhs(A, x_target);
-
-        auto robust = sys.spawn(robust_solver, std::move(A), std::move(b), 1e-10, 5000);
-        self->mail(start_atom_v).send(robust);
-        
-        self->receive(
-            [](bool) { std::cout << "[INFO] Robust solver converged successfully." << std::endl; },
-            [](std::string err) { std::cout << "[INFO] Robust solver aborted: " << err << std::endl; }
-        );
-    } catch (const std::exception& e) {
-        std::cerr << "[ERROR] " << e.what() << std::endl;
-    }
-
-    manager::shutdown();
-}
-
-CAF_MAIN(id_block::cuda)
\ No newline at end of file

From 62238a42fa8e6db19d84034c1dc8a4955f50c5ed Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 24 Jun 2026 11:40:37 -0600
Subject: [PATCH 0999/1000] updated paths to work with AD build process.

---
 libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native.cpp | 2 +-
 .../sc26/Irregular-Workload/workloadA/main.native_sorted.cpp  | 4 ++--
 libcaf_cuda/sc26/Irregular-Workload/workloadA/main.test.cpp   | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native.cpp
index a394eafa4a..2186aa4e93 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native.cpp
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native.cpp
@@ -24,7 +24,7 @@ int main(int argc, char** argv)
 
     std::cout << "[INFO] Loading matrices...\n";
     //std::vector<MatrixTask> matrix_pool = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
-    std::vector<MatrixTask> matrix_pool = scan_for_matrices("../../../scripts/Irregular-Workload/workloadA/downloaded_matrices/matrices", CGS_SOLVER);
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("../../scripts/Irregular-Workload/workloadA/downloaded_matrices/matrices", CGS_SOLVER);
   
     
     if (matrix_pool.empty()) {
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native_sorted.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native_sorted.cpp
index c1167267e2..e28fbcd7a5 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native_sorted.cpp
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.native_sorted.cpp
@@ -27,7 +27,7 @@ int main(int argc, char** argv) {
     // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
 
     std::cout << "[INFO] Loading matrices...\n";
-    std::vector<MatrixTask> matrix_pool = scan_for_matrices("../../../scripts/Irregular-Workload/workloadA/downloaded_matrices/matrices", CGS_SOLVER);
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("../../scripts/Irregular-Workload/workloadA/downloaded_matrices/matrices", CGS_SOLVER);
 
 
     if (matrix_pool.empty()) {
@@ -61,4 +61,4 @@ int main(int argc, char** argv) {
     report_workload_stats();
 
     return 0;
-}
\ No newline at end of file
+}
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.test.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.test.cpp
index 4d67c5db90..da20c1e9c5 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.test.cpp
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadA/main.test.cpp
@@ -31,7 +31,7 @@ void caf_main(actor_system& sys) {
          //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/mixed", CGS_SOLVER);
 
         //  auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrices/workloadA", CGS_SOLVER);
-        auto tasks_vec = scan_for_matrices("../../../scripts/Irregular-Workload/workloadA/downloaded_matrices/matrices", CGS_SOLVER);
+        auto tasks_vec = scan_for_matrices("../../scripts/Irregular-Workload/workloadA/downloaded_matrices/matrices", CGS_SOLVER);
 
 
 
@@ -80,4 +80,4 @@ void caf_main(actor_system& sys) {
     }
     manager::shutdown();
 }
-CAF_MAIN(id_block::cuda, id_block::workload_test)
\ No newline at end of file
+CAF_MAIN(id_block::cuda, id_block::workload_test)

From 8d811fa9c6b7f6fb51747ee18647b1deac49fe94 Mon Sep 17 00:00:00 2001
From: "Dalton Beauchesne (nqr159)" <nqr159@mail.usask.ca>
Date: Wed, 24 Jun 2026 11:45:43 -0600
Subject: [PATCH 1000/1000] Updated filepaths.

---
 libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native.cpp | 2 +-
 .../sc26/Irregular-Workload/workloadB/main.native_sorted.cpp  | 4 ++--
 libcaf_cuda/sc26/Irregular-Workload/workloadB/main.test.cpp   | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native.cpp
index a7fdc75afa..d3e1dafba9 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native.cpp
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native.cpp
@@ -23,7 +23,7 @@ int main(int argc, char** argv)
     // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
 
     std::cout << "[INFO] Loading matrices...\n";
-    std::vector<MatrixTask> matrix_pool = scan_for_matrices("../../../scripts/Irregular-Workload/workloadB/downloaded_matrices/matrices", CGS_SOLVER);
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("../../scripts/Irregular-Workload/workloadB/downloaded_matrices/matrices", CGS_SOLVER);
 
     if (matrix_pool.empty()) {
         std::cerr << "No matrices found.\n";
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native_sorted.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native_sorted.cpp
index f488f2d3f5..7b53620ee1 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native_sorted.cpp
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.native_sorted.cpp
@@ -27,7 +27,7 @@ int main(int argc, char** argv) {
     // if (argc > 1) num_streams = std::max(num_streams, std::atoi(argv[1]));
 
     std::cout << "[INFO] Loading matrices...\n";
-    std::vector<MatrixTask> matrix_pool = scan_for_matrices("../../../scripts/Irregular-Workload/workloadB/downloaded_matrices/matrices", CGS_SOLVER);
+    std::vector<MatrixTask> matrix_pool = scan_for_matrices("../../scripts/Irregular-Workload/workloadB/downloaded_matrices/matrices", CGS_SOLVER);
 
     if (matrix_pool.empty()) {
         std::cerr << "No matrices found.\n";
@@ -60,4 +60,4 @@ int main(int argc, char** argv) {
     report_workload_stats();
 
     return 0;
-}
\ No newline at end of file
+}
diff --git a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.test.cpp b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.test.cpp
index 2a1103702b..54687dc1f7 100644
--- a/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.test.cpp
+++ b/libcaf_cuda/sc26/Irregular-Workload/workloadB/main.test.cpp
@@ -28,7 +28,7 @@ void caf_main(actor_system& sys) {
          //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/spd", CGS_SOLVER);
         //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrices/unsymmetric", CGS_SOLVER);
          //auto tasks_vec = scan_for_matrices("/scratch/nqr159/matrix-collection/matrix_corpus_v2/matrices/unsymmetric", CGS_SOLVER);
-         auto tasks_vec = scan_for_matrices("../../../scripts/Irregular-Workload/workloadB/downloaded_matrices/matrices", CGS_SOLVER);
+         auto tasks_vec = scan_for_matrices("../../scripts/Irregular-Workload/workloadB/downloaded_matrices/matrices", CGS_SOLVER);
 
         int num_gpus = manager::get().get_num_devices();
         std::cout << "[INFO] Found " << num_gpus << " GPUs\n";
@@ -58,4 +58,4 @@ void caf_main(actor_system& sys) {
     }
     manager::shutdown();
 }
-CAF_MAIN(id_block::cuda, id_block::workload_test)
\ No newline at end of file
+CAF_MAIN(id_block::cuda, id_block::workload_test)