From 2c4da709ae3f04bae84743d6a05148fc1acce63e Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Tue, 16 Jun 2026 16:57:31 +0200 Subject: [PATCH 1/6] [UR][L0v2] Fix out-of-order queue fork-join graph capture routing Route operations waiting on an event from a capturing queue to the capture command list instead of an arbitrary one, so the fork is recorded. --- .../v2/queue_immediate_out_of_order.hpp | 136 ++++++++++++------ .../urQueueIsGraphCaptureEnabledExp.cpp | 4 - 2 files changed, 96 insertions(+), 44 deletions(-) diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp index 525a3b6802523..39d4ffa4f38ea 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp @@ -48,16 +48,37 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { std::array barrierEvents; - uint32_t getNextCommandListId() { + uint32_t getNextCommandListId(const ur_event_handle_t *phWaitEvents = nullptr, + uint32_t numWaitEvents = 0) { bool captureActive; auto &cmdListManager = (*commandListManagers.get_no_lock())[captureCmdListManagerIdx]; cmdListManager.queryGraphCaptureActive(&captureActive); - return captureActive - ? captureCmdListManagerIdx - : commandListIndex.fetch_add(1, std::memory_order_relaxed) % - numCommandLists; + if (captureActive) { + return captureCmdListManagerIdx; + } + + // Fork-join: if any wait event was produced by a queue that is currently + // recording a graph, this operation joins that capture and must be appended + // on the dedicated capture command list. The L0 driver then automatically + // enters capture mode on that command list, so subsequent operations and + // capture queries observe the active capture as well. + for (uint32_t i = 0; i < numWaitEvents; i++) { + auto *srcQueue = phWaitEvents[i]->getQueue(); + if (!srcQueue) { + continue; + } + bool srcCaptureActive = false; + if (srcQueue->queueIsGraphCapteEnabledExp(&srcCaptureActive) == + UR_RESULT_SUCCESS && + srcCaptureActive) { + return captureCmdListManagerIdx; + } + } + + return commandListIndex.fetch_add(1, std::memory_order_relaxed) % + numCommandLists; } public: @@ -87,7 +108,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendEventsWait( waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); } @@ -106,7 +128,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferRead( hBuffer, blockingRead, offset, size, pDst, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -121,7 +144,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferWrite( hBuffer, blockingWrite, offset, size, pSrc, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -137,7 +161,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferReadRect( hBuffer, blockingRead, bufferOrigin, hostOrigin, region, bufferRowPitch, bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst, waitListView, @@ -154,7 +179,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferWriteRect( hBuffer, blockingWrite, bufferOrigin, hostOrigin, region, bufferRowPitch, bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc, @@ -170,7 +196,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferCopy( hBufferSrc, hBufferDst, srcOffset, dstOffset, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -186,7 +213,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferCopyRect( hBufferSrc, hBufferDst, srcOrigin, dstOrigin, region, srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, waitListView, @@ -202,7 +230,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferFill( hBuffer, pPattern, patternSize, offset, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -218,7 +247,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemImageRead( hImage, blockingRead, origin, region, rowPitch, slicePitch, pDst, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -234,7 +264,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemImageWrite( hImage, blockingWrite, origin, region, rowPitch, slicePitch, pSrc, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -249,7 +280,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemImageCopy( hImageSrc, hImageDst, srcOrigin, dstOrigin, region, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -264,7 +296,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferMap( hBuffer, blockingMap, mapFlags, offset, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this), ppRetMap); @@ -277,7 +310,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemUnmap( hMem, pMappedPtr, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -291,7 +325,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMFill( pMem, patternSize, pPattern, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -304,7 +339,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMMemcpy( blocking, pDst, pSrc, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -318,7 +354,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMFill2D( pMem, pitch, patternSize, pPattern, width, height, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -333,7 +370,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMMemcpy2D( blocking, pDst, dstPitch, pSrc, srcPitch, width, height, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -347,7 +385,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMPrefetch( pMem, size, flags, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -372,7 +411,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .appendDeviceGlobalVariableWrite( hProgram, name, blockingWrite, count, offset, pSrc, waitListView, @@ -387,7 +427,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .appendDeviceGlobalVariableRead( hProgram, name, blockingRead, count, offset, pDst, waitListView, @@ -403,7 +444,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendReadHostPipe( hProgram, pipe_symbol, blocking, pDst, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -418,7 +460,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendWriteHostPipe( hProgram, pipe_symbol, blocking, pSrc, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -432,7 +475,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMAllocHelper( this, pPool, size, pProperties, waitListView, ppMem, createEventIfRequested(eventPool.get(), phEvent, this), @@ -447,7 +491,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMAllocHelper( this, pPool, size, pProperties, waitListView, ppMem, createEventIfRequested(eventPool.get(), phEvent, this), @@ -463,7 +508,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMAllocHelper( this, pPool, size, pProperties, waitListView, ppMem, createEventIfRequested(eventPool.get(), phEvent, this), @@ -477,7 +523,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMFreeExp( this, pPool, pMem, waitListView, createEvent(eventPool.get(), phEvent, this)); @@ -496,7 +543,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].bindlessImagesImageCopyExp( pSrc, pDst, pSrcImageDesc, pDstImageDesc, pSrcImageFormat, pDstImageFormat, pCopyRegion, imageCopyFlags, imageCopyInputTypes, @@ -511,7 +559,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .bindlessImagesWaitExternalSemaphoreExp( hSemaphore, hasWaitValue, waitValue, waitListView, @@ -526,7 +575,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .bindlessImagesSignalExternalSemaphoreExp( hSemaphore, hasSignalValue, signalValue, waitListView, @@ -540,7 +590,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .appendTimestampRecordingExp( blocking, waitListView, @@ -555,7 +606,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendCommandBufferExp( hCommandBuffer, waitListView, createEventAndRetain(eventPool.get(), phEvent, this)); @@ -570,7 +622,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendNativeCommandExp( pfnNativeEnqueue, data, numMemsInMemList, phMemList, pProperties, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -587,7 +640,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .appendKernelLaunchWithArgsExp( hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, @@ -618,7 +672,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendGraph( hGraph, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -643,7 +698,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendHostTaskExp( pfnHostTask, data, pProperties, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); diff --git a/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp b/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp index 95838078737ce..8858891436516 100644 --- a/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp +++ b/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp @@ -67,10 +67,6 @@ struct urQueueIsGraphCaptureEnabledExpMultiQueueTest : uur::urGraphSupportedExpMultiQueueTest { void SetUp() override { UUR_RETURN_ON_FATAL_FAILURE(urGraphSupportedExpMultiQueueTest::SetUp()); - // Fork-join with out-of-order queue broken due to multi command list capture bug - if (getQueueFlag() & UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE) { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); - } } void TearDown() override { bool isCaptureEnabled = false; From bd43e5c984c6f94193037d6e190d80a016227cc3 Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Tue, 16 Jun 2026 16:57:31 +0200 Subject: [PATCH 2/6] [UR][L0v2] Apply barrier on capture list during graph capture Avoids forking the other command lists, which would never join back. --- .../level_zero/v2/queue_immediate_out_of_order.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp index a63232829f79b..7659856a0af1a 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp @@ -171,6 +171,14 @@ ur_result_t ur_queue_immediate_out_of_order_t::enqueueEventsWaitWithBarrierExt( : &ur_command_list_manager::appendEventsWait; auto commandListManagersLocked = commandListManagers.lock(); + bool captureActive = false; + commandListManagersLocked[captureCmdListManagerIdx].queryGraphCaptureActive( + &captureActive); + if (captureActive) { + return std::invoke( + barrierFn, commandListManagersLocked[captureCmdListManagerIdx], + waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); + } // Enqueue wait for the user-provider events on the first command list. UR_CALL(commandListManagersLocked[0].appendEventsWait(waitListView, From 79d17f913677f2890cfa9ed8da657ce0bdd13e44 Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Tue, 23 Jun 2026 15:06:18 +0200 Subject: [PATCH 3/6] [UR][L0v2] Add tests and temporary entering the recording on secondary queue --- .../v2/queue_immediate_out_of_order.hpp | 62 ++++++++++++++++--- .../urQueueIsGraphCaptureEnabledExp.cpp | 52 ++++++++++++++++ 2 files changed, 106 insertions(+), 8 deletions(-) diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp index 39d4ffa4f38ea..67d0da2f2b464 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp @@ -21,6 +21,8 @@ #include "lockable.hpp" #include "ur/ur.hpp" +#include + namespace ur::level_zero::v2 { struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { @@ -48,6 +50,14 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { std::array barrierEvents; + // The primary queue this out-of-order queue joined during a fork-join graph + // capture, or nullptr when not part of a fork-join. While set, all operations + // are routed to the dedicated capture command list (see + // getNextCommandListId). The Level Zero record-replay driver forks a command + // list into a capturing graph exactly once (a list cannot be forked by more + // than one fork event), so a single primary queue is sufficient to track. + std::atomic forkJoinPrimaryQueue = nullptr; + uint32_t getNextCommandListId(const ur_event_handle_t *phWaitEvents = nullptr, uint32_t numWaitEvents = 0) { bool captureActive; @@ -59,28 +69,58 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { return captureCmdListManagerIdx; } - // Fork-join: if any wait event was produced by a queue that is currently - // recording a graph, this operation joins that capture and must be appended - // on the dedicated capture command list. The L0 driver then automatically - // enters capture mode on that command list, so subsequent operations and - // capture queries observe the active capture as well. + // Fork-join: if any wait event was produced by another queue that is + // currently recording a graph, this operation joins that capture and must + // be appended on the dedicated capture command list. Remember the + // originating ("primary") queue so that subsequent operations - even those + // without an explicit dependency on the primary queue - keep being routed + // onto the capture command list. The L0 driver then automatically enters + // capture mode on that command list, putting it into a temporary recording + // state that lasts until the primary queue stops recording. for (uint32_t i = 0; i < numWaitEvents; i++) { auto *srcQueue = phWaitEvents[i]->getQueue(); - if (!srcQueue) { + if (!srcQueue || srcQueue == this) { continue; } bool srcCaptureActive = false; if (srcQueue->queueIsGraphCapteEnabledExp(&srcCaptureActive) == UR_RESULT_SUCCESS && srcCaptureActive) { + forkJoinPrimaryQueue.store(srcQueue, std::memory_order_relaxed); return captureCmdListManagerIdx; } } + // Still part of a fork started by an earlier operation: keep routing onto + // the capture command list until the primary queue finishes recording the + // graph. Without this, operations submitted to this queue without an + // explicit dependency on the primary queue would escape the capture. + if (isForkJoinCaptureActive()) { + return captureCmdListManagerIdx; + } + return commandListIndex.fetch_add(1, std::memory_order_relaxed) % numCommandLists; } + // Returns true while this queue is temporarily recording as part of a + // fork-join capture started by another (primary) queue. When the primary + // queue is no longer recording, the temporary state is cleared. + bool isForkJoinCaptureActive() { + auto *primaryQueue = forkJoinPrimaryQueue.load(std::memory_order_relaxed); + if (!primaryQueue) { + return false; + } + bool primaryCaptureActive = false; + if (primaryQueue->queueIsGraphCapteEnabledExp(&primaryCaptureActive) == + UR_RESULT_SUCCESS && + primaryCaptureActive) { + return true; + } + forkJoinPrimaryQueue.store(nullptr, std::memory_order_relaxed); + return false; + } + public: ur_queue_immediate_out_of_order_t(ur_context_handle_t, ur_device_handle_t, uint32_t ordinal, @@ -680,8 +720,14 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { } ur_result_t queueIsGraphCapteEnabledExp(bool *pResult) override { - return commandListManagers.lock()[captureCmdListManagerIdx] - .queryGraphCaptureActive(pResult); + UR_CALL(commandListManagers.lock()[captureCmdListManagerIdx] + .queryGraphCaptureActive(pResult)); + // Treat fork-join recording on another queue as active for chained joins + // and capture queries. + if (!*pResult) { + *pResult = isForkJoinCaptureActive(); + } + return UR_RESULT_SUCCESS; } ur_result_t queueGetGraphExp(ur_exp_graph_handle_t *phGraph) override { diff --git a/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp b/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp index 8858891436516..20393f202cac9 100644 --- a/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp +++ b/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp @@ -129,3 +129,55 @@ TEST_P(urQueueIsGraphCaptureEnabledExpMultiQueueTest, ForkJoinPattern) { ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); ASSERT_FALSE(isEnabled); } + +// Tests that, once an out-of-order queue has joined a capture via the fork-join +// pattern, operations subsequently submitted to it *without* an explicit +// dependency on the recording queue are still recorded. The secondary queue +// stays in the temporary recording state until the primary queue ends the +// capture. With the L0v2 out-of-order queue this would otherwise round-robin +// the dependency-less operation onto a non-capture command list, escaping the +// capture. +TEST_P(urQueueIsGraphCaptureEnabledExpMultiQueueTest, + ForkJoinSubsequentOpsWithoutDependency) { + bool isEnabled = false; + + // Advance the out-of-order queue's command list selection so the next + // operation would not land on the dedicated capture command list by chance. + uur::raii::Event preEvent = nullptr; + ASSERT_SUCCESS(urEnqueueEventsWait(queue2, 0, nullptr, preEvent.ptr())); + ASSERT_SUCCESS(urEventWait(1, preEvent.ptr())); + + ASSERT_SUCCESS(urQueueBeginGraphCaptureExp(queue1)); + + // Fork: queue1 produces an event that queue2 waits on, pulling queue2 into + // the capture. + uur::raii::Event forkEvent = nullptr; + ASSERT_SUCCESS(urEnqueueEventsWait(queue1, 0, nullptr, forkEvent.ptr())); + + uur::raii::Event joinEvent = nullptr; + ASSERT_SUCCESS( + urEnqueueEventsWait(queue2, 1, forkEvent.ptr(), joinEvent.ptr())); + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); + ASSERT_TRUE(isEnabled); + + // Subsequent operation on queue2 without any dependency on queue1. It must + // remain part of the capture: queue2 should still report recording enabled. + uur::raii::Event noDepEvent = nullptr; + ASSERT_SUCCESS(urEnqueueEventsWait(queue2, 0, nullptr, noDepEvent.ptr())); + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); + ASSERT_TRUE(isEnabled); + + // Join both queue2 operations back to queue1 and finish recording. + ur_event_handle_t joinEvents[] = {joinEvent.get(), noDepEvent.get()}; + ASSERT_SUCCESS(urEnqueueEventsWait(queue1, 2, joinEvents, nullptr)); + + ASSERT_SUCCESS(urQueueEndGraphCaptureExp(queue1, &graph)); + + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue1, &isEnabled)); + ASSERT_FALSE(isEnabled); + + // The primary queue stopped recording, so queue2 must leave the temporary + // recording state as well. + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); + ASSERT_FALSE(isEnabled); +} From 4174499ad40fdd2dec835c949b159d6725cfee89 Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Thu, 6 Aug 2026 09:54:26 +0200 Subject: [PATCH 4/6] Remove: Fork-join with out-of-order queue broken due to multi command list capture bug --- .../test/conformance/exp_graph/urQueueGetGraphExp.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/unified-runtime/test/conformance/exp_graph/urQueueGetGraphExp.cpp b/unified-runtime/test/conformance/exp_graph/urQueueGetGraphExp.cpp index 1d9cbb72bd38a..a65fba13ac4e1 100644 --- a/unified-runtime/test/conformance/exp_graph/urQueueGetGraphExp.cpp +++ b/unified-runtime/test/conformance/exp_graph/urQueueGetGraphExp.cpp @@ -96,11 +96,6 @@ struct urQueueGetGraphExpMultiQueueTest // Fork-join was initially broken with zeCommandListGetGraph std::tuple minL0DriverVersion = {1, 15, 38146}; SKIP_IF_DRIVER_TOO_OLD("Level-Zero", minL0DriverVersion, platform, device); - - // Fork-join with out-of-order queue broken due to multi command list capture bug - if (getQueueFlag() & UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE) { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); - } } void TearDown() override { bool isCaptureEnabled = false; From 7b8a4d8a16086f5000293a65dcc23886baf09b7b Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Thu, 6 Aug 2026 15:35:37 +0200 Subject: [PATCH 5/6] Add data copy to verify that copy is just recorded --- .../urQueueIsGraphCaptureEnabledExp.cpp | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp b/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp index 20393f202cac9..27c98aa3e3619 100644 --- a/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp +++ b/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp @@ -4,6 +4,8 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "fixtures.h" +#include "unified-runtime/ur_api.h" +#include "uur/raii.h" struct urQueueIsGraphCaptureEnabledExpTest : uur::urGraphSupportedExpTest { void SetUp() override { @@ -163,10 +165,32 @@ TEST_P(urQueueIsGraphCaptureEnabledExpMultiQueueTest, // Subsequent operation on queue2 without any dependency on queue1. It must // remain part of the capture: queue2 should still report recording enabled. uur::raii::Event noDepEvent = nullptr; - ASSERT_SUCCESS(urEnqueueEventsWait(queue2, 0, nullptr, noDepEvent.ptr())); + size_t size = 1024; + void *ptr1 = nullptr; + void *ptr2 = nullptr; + ASSERT_SUCCESS(urUSMHostAlloc(context, nullptr, nullptr, size, &ptr1)); + ASSERT_SUCCESS(urUSMHostAlloc(context, nullptr, nullptr, size, &ptr2)); + + // Fill ptr1 with a pattern and clear ptr2 so we can tell whether the copy + // was recorded or executed. + uint32_t *ptr1_data = static_cast(ptr1); + uint32_t *ptr2_data = static_cast(ptr2); + *ptr1_data = 0xdeadbeefU; + *ptr2_data = 0U; + + // Submit memcpy operation on queue2 without dependency on queue1. This + // operation should be recorded in the capture. It must be non-blocking: a + // blocking copy would host-synchronize the capturing command list, which + // has nothing to wait for because the copy is only recorded, and the driver + // rejects the synchronization. + ASSERT_SUCCESS(urEnqueueUSMMemcpy(queue2, false, ptr2, ptr1, size, 0, nullptr, + noDepEvent.ptr())); ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); ASSERT_TRUE(isEnabled); + // Verify that the copy is not performed yet (it's recorded but not executed) + ASSERT_EQ(*ptr2_data, 0U); + // Join both queue2 operations back to queue1 and finish recording. ur_event_handle_t joinEvents[] = {joinEvent.get(), noDepEvent.get()}; ASSERT_SUCCESS(urEnqueueEventsWait(queue1, 2, joinEvents, nullptr)); @@ -180,4 +204,7 @@ TEST_P(urQueueIsGraphCaptureEnabledExpMultiQueueTest, // recording state as well. ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); ASSERT_FALSE(isEnabled); + + ASSERT_SUCCESS(urUSMFree(context, ptr1)); + ASSERT_SUCCESS(urUSMFree(context, ptr2)); } From 785c0553031381b8666de0edc8ccac1d3270e8a1 Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Thu, 6 Aug 2026 14:58:57 +0000 Subject: [PATCH 6/6] Add non-blocking memcopy --- .../exp_graph/urQueueIsGraphCaptureEnabledExp.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp b/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp index 27c98aa3e3619..534c8e508da75 100644 --- a/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp +++ b/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp @@ -205,6 +205,17 @@ TEST_P(urQueueIsGraphCaptureEnabledExpMultiQueueTest, ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); ASSERT_FALSE(isEnabled); + // Replaying the captured graph must execute the copy that was recorded on + // the forked queue, proving it really became part of the graph. + ur_exp_executable_graph_handle_t exGraph = nullptr; + ASSERT_SUCCESS(urGraphInstantiateGraphExp(graph, &exGraph)); + + ASSERT_SUCCESS(urEnqueueGraphExp(queue1, exGraph, 0, nullptr, nullptr)); + ASSERT_SUCCESS(urQueueFinish(queue1)); + + EXPECT_EQ(*ptr2_data, 0xdeadbeefU); + + EXPECT_SUCCESS(urGraphExecutableGraphDestroyExp(exGraph)); ASSERT_SUCCESS(urUSMFree(context, ptr1)); ASSERT_SUCCESS(urUSMFree(context, ptr2)); }