diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp index a63232829f79b..7659856a0af1a 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp @@ -171,6 +171,14 @@ ur_result_t ur_queue_immediate_out_of_order_t::enqueueEventsWaitWithBarrierExt( : &ur_command_list_manager::appendEventsWait; auto commandListManagersLocked = commandListManagers.lock(); + bool captureActive = false; + commandListManagersLocked[captureCmdListManagerIdx].queryGraphCaptureActive( + &captureActive); + if (captureActive) { + return std::invoke( + barrierFn, commandListManagersLocked[captureCmdListManagerIdx], + waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); + } // Enqueue wait for the user-provider events on the first command list. UR_CALL(commandListManagersLocked[0].appendEventsWait(waitListView, diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp index 525a3b6802523..67d0da2f2b464 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp @@ -21,6 +21,8 @@ #include "lockable.hpp" #include "ur/ur.hpp" +#include + namespace ur::level_zero::v2 { struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { @@ -48,16 +50,75 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { std::array barrierEvents; - uint32_t getNextCommandListId() { + // The primary queue this out-of-order queue joined during a fork-join graph + // capture, or nullptr when not part of a fork-join. While set, all operations + // are routed to the dedicated capture command list (see + // getNextCommandListId). The Level Zero record-replay driver forks a command + // list into a capturing graph exactly once (a list cannot be forked by more + // than one fork event), so a single primary queue is sufficient to track. + std::atomic forkJoinPrimaryQueue = nullptr; + + uint32_t getNextCommandListId(const ur_event_handle_t *phWaitEvents = nullptr, + uint32_t numWaitEvents = 0) { bool captureActive; auto &cmdListManager = (*commandListManagers.get_no_lock())[captureCmdListManagerIdx]; cmdListManager.queryGraphCaptureActive(&captureActive); - return captureActive - ? captureCmdListManagerIdx - : commandListIndex.fetch_add(1, std::memory_order_relaxed) % - numCommandLists; + if (captureActive) { + return captureCmdListManagerIdx; + } + + // Fork-join: if any wait event was produced by another queue that is + // currently recording a graph, this operation joins that capture and must + // be appended on the dedicated capture command list. Remember the + // originating ("primary") queue so that subsequent operations - even those + // without an explicit dependency on the primary queue - keep being routed + // onto the capture command list. The L0 driver then automatically enters + // capture mode on that command list, putting it into a temporary recording + // state that lasts until the primary queue stops recording. + for (uint32_t i = 0; i < numWaitEvents; i++) { + auto *srcQueue = phWaitEvents[i]->getQueue(); + if (!srcQueue || srcQueue == this) { + continue; + } + bool srcCaptureActive = false; + if (srcQueue->queueIsGraphCapteEnabledExp(&srcCaptureActive) == + UR_RESULT_SUCCESS && + srcCaptureActive) { + forkJoinPrimaryQueue.store(srcQueue, std::memory_order_relaxed); + return captureCmdListManagerIdx; + } + } + + // Still part of a fork started by an earlier operation: keep routing onto + // the capture command list until the primary queue finishes recording the + // graph. Without this, operations submitted to this queue without an + // explicit dependency on the primary queue would escape the capture. + if (isForkJoinCaptureActive()) { + return captureCmdListManagerIdx; + } + + return commandListIndex.fetch_add(1, std::memory_order_relaxed) % + numCommandLists; + } + + // Returns true while this queue is temporarily recording as part of a + // fork-join capture started by another (primary) queue. When the primary + // queue is no longer recording, the temporary state is cleared. + bool isForkJoinCaptureActive() { + auto *primaryQueue = forkJoinPrimaryQueue.load(std::memory_order_relaxed); + if (!primaryQueue) { + return false; + } + bool primaryCaptureActive = false; + if (primaryQueue->queueIsGraphCapteEnabledExp(&primaryCaptureActive) == + UR_RESULT_SUCCESS && + primaryCaptureActive) { + return true; + } + forkJoinPrimaryQueue.store(nullptr, std::memory_order_relaxed); + return false; } public: @@ -87,7 +148,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendEventsWait( waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); } @@ -106,7 +168,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferRead( hBuffer, blockingRead, offset, size, pDst, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -121,7 +184,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferWrite( hBuffer, blockingWrite, offset, size, pSrc, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -137,7 +201,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferReadRect( hBuffer, blockingRead, bufferOrigin, hostOrigin, region, bufferRowPitch, bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst, waitListView, @@ -154,7 +219,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferWriteRect( hBuffer, blockingWrite, bufferOrigin, hostOrigin, region, bufferRowPitch, bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc, @@ -170,7 +236,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferCopy( hBufferSrc, hBufferDst, srcOffset, dstOffset, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -186,7 +253,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferCopyRect( hBufferSrc, hBufferDst, srcOrigin, dstOrigin, region, srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, waitListView, @@ -202,7 +270,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferFill( hBuffer, pPattern, patternSize, offset, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -218,7 +287,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemImageRead( hImage, blockingRead, origin, region, rowPitch, slicePitch, pDst, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -234,7 +304,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemImageWrite( hImage, blockingWrite, origin, region, rowPitch, slicePitch, pSrc, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -249,7 +320,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemImageCopy( hImageSrc, hImageDst, srcOrigin, dstOrigin, region, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -264,7 +336,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferMap( hBuffer, blockingMap, mapFlags, offset, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this), ppRetMap); @@ -277,7 +350,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemUnmap( hMem, pMappedPtr, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -291,7 +365,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMFill( pMem, patternSize, pPattern, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -304,7 +379,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMMemcpy( blocking, pDst, pSrc, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -318,7 +394,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMFill2D( pMem, pitch, patternSize, pPattern, width, height, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -333,7 +410,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMMemcpy2D( blocking, pDst, dstPitch, pSrc, srcPitch, width, height, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -347,7 +425,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMPrefetch( pMem, size, flags, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -372,7 +451,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .appendDeviceGlobalVariableWrite( hProgram, name, blockingWrite, count, offset, pSrc, waitListView, @@ -387,7 +467,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .appendDeviceGlobalVariableRead( hProgram, name, blockingRead, count, offset, pDst, waitListView, @@ -403,7 +484,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendReadHostPipe( hProgram, pipe_symbol, blocking, pDst, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -418,7 +500,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendWriteHostPipe( hProgram, pipe_symbol, blocking, pSrc, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -432,7 +515,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMAllocHelper( this, pPool, size, pProperties, waitListView, ppMem, createEventIfRequested(eventPool.get(), phEvent, this), @@ -447,7 +531,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMAllocHelper( this, pPool, size, pProperties, waitListView, ppMem, createEventIfRequested(eventPool.get(), phEvent, this), @@ -463,7 +548,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMAllocHelper( this, pPool, size, pProperties, waitListView, ppMem, createEventIfRequested(eventPool.get(), phEvent, this), @@ -477,7 +563,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMFreeExp( this, pPool, pMem, waitListView, createEvent(eventPool.get(), phEvent, this)); @@ -496,7 +583,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].bindlessImagesImageCopyExp( pSrc, pDst, pSrcImageDesc, pDstImageDesc, pSrcImageFormat, pDstImageFormat, pCopyRegion, imageCopyFlags, imageCopyInputTypes, @@ -511,7 +599,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .bindlessImagesWaitExternalSemaphoreExp( hSemaphore, hasWaitValue, waitValue, waitListView, @@ -526,7 +615,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .bindlessImagesSignalExternalSemaphoreExp( hSemaphore, hasSignalValue, signalValue, waitListView, @@ -540,7 +630,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .appendTimestampRecordingExp( blocking, waitListView, @@ -555,7 +646,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendCommandBufferExp( hCommandBuffer, waitListView, createEventAndRetain(eventPool.get(), phEvent, this)); @@ -570,7 +662,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendNativeCommandExp( pfnNativeEnqueue, data, numMemsInMemList, phMemList, pProperties, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -587,7 +680,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .appendKernelLaunchWithArgsExp( hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, @@ -618,15 +712,22 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendGraph( hGraph, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); } ur_result_t queueIsGraphCapteEnabledExp(bool *pResult) override { - return commandListManagers.lock()[captureCmdListManagerIdx] - .queryGraphCaptureActive(pResult); + UR_CALL(commandListManagers.lock()[captureCmdListManagerIdx] + .queryGraphCaptureActive(pResult)); + // Treat fork-join recording on another queue as active for chained joins + // and capture queries. + if (!*pResult) { + *pResult = isForkJoinCaptureActive(); + } + return UR_RESULT_SUCCESS; } ur_result_t queueGetGraphExp(ur_exp_graph_handle_t *phGraph) override { @@ -643,7 +744,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendHostTaskExp( pfnHostTask, data, pProperties, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); diff --git a/unified-runtime/test/conformance/exp_graph/urQueueGetGraphExp.cpp b/unified-runtime/test/conformance/exp_graph/urQueueGetGraphExp.cpp index 1d9cbb72bd38a..a65fba13ac4e1 100644 --- a/unified-runtime/test/conformance/exp_graph/urQueueGetGraphExp.cpp +++ b/unified-runtime/test/conformance/exp_graph/urQueueGetGraphExp.cpp @@ -96,11 +96,6 @@ struct urQueueGetGraphExpMultiQueueTest // Fork-join was initially broken with zeCommandListGetGraph std::tuple minL0DriverVersion = {1, 15, 38146}; SKIP_IF_DRIVER_TOO_OLD("Level-Zero", minL0DriverVersion, platform, device); - - // Fork-join with out-of-order queue broken due to multi command list capture bug - if (getQueueFlag() & UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE) { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); - } } void TearDown() override { bool isCaptureEnabled = false; diff --git a/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp b/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp index 95838078737ce..534c8e508da75 100644 --- a/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp +++ b/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp @@ -4,6 +4,8 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "fixtures.h" +#include "unified-runtime/ur_api.h" +#include "uur/raii.h" struct urQueueIsGraphCaptureEnabledExpTest : uur::urGraphSupportedExpTest { void SetUp() override { @@ -67,10 +69,6 @@ struct urQueueIsGraphCaptureEnabledExpMultiQueueTest : uur::urGraphSupportedExpMultiQueueTest { void SetUp() override { UUR_RETURN_ON_FATAL_FAILURE(urGraphSupportedExpMultiQueueTest::SetUp()); - // Fork-join with out-of-order queue broken due to multi command list capture bug - if (getQueueFlag() & UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE) { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); - } } void TearDown() override { bool isCaptureEnabled = false; @@ -133,3 +131,91 @@ TEST_P(urQueueIsGraphCaptureEnabledExpMultiQueueTest, ForkJoinPattern) { ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); ASSERT_FALSE(isEnabled); } + +// Tests that, once an out-of-order queue has joined a capture via the fork-join +// pattern, operations subsequently submitted to it *without* an explicit +// dependency on the recording queue are still recorded. The secondary queue +// stays in the temporary recording state until the primary queue ends the +// capture. With the L0v2 out-of-order queue this would otherwise round-robin +// the dependency-less operation onto a non-capture command list, escaping the +// capture. +TEST_P(urQueueIsGraphCaptureEnabledExpMultiQueueTest, + ForkJoinSubsequentOpsWithoutDependency) { + bool isEnabled = false; + + // Advance the out-of-order queue's command list selection so the next + // operation would not land on the dedicated capture command list by chance. + uur::raii::Event preEvent = nullptr; + ASSERT_SUCCESS(urEnqueueEventsWait(queue2, 0, nullptr, preEvent.ptr())); + ASSERT_SUCCESS(urEventWait(1, preEvent.ptr())); + + ASSERT_SUCCESS(urQueueBeginGraphCaptureExp(queue1)); + + // Fork: queue1 produces an event that queue2 waits on, pulling queue2 into + // the capture. + uur::raii::Event forkEvent = nullptr; + ASSERT_SUCCESS(urEnqueueEventsWait(queue1, 0, nullptr, forkEvent.ptr())); + + uur::raii::Event joinEvent = nullptr; + ASSERT_SUCCESS( + urEnqueueEventsWait(queue2, 1, forkEvent.ptr(), joinEvent.ptr())); + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); + ASSERT_TRUE(isEnabled); + + // Subsequent operation on queue2 without any dependency on queue1. It must + // remain part of the capture: queue2 should still report recording enabled. + uur::raii::Event noDepEvent = nullptr; + size_t size = 1024; + void *ptr1 = nullptr; + void *ptr2 = nullptr; + ASSERT_SUCCESS(urUSMHostAlloc(context, nullptr, nullptr, size, &ptr1)); + ASSERT_SUCCESS(urUSMHostAlloc(context, nullptr, nullptr, size, &ptr2)); + + // Fill ptr1 with a pattern and clear ptr2 so we can tell whether the copy + // was recorded or executed. + uint32_t *ptr1_data = static_cast(ptr1); + uint32_t *ptr2_data = static_cast(ptr2); + *ptr1_data = 0xdeadbeefU; + *ptr2_data = 0U; + + // Submit memcpy operation on queue2 without dependency on queue1. This + // operation should be recorded in the capture. It must be non-blocking: a + // blocking copy would host-synchronize the capturing command list, which + // has nothing to wait for because the copy is only recorded, and the driver + // rejects the synchronization. + ASSERT_SUCCESS(urEnqueueUSMMemcpy(queue2, false, ptr2, ptr1, size, 0, nullptr, + noDepEvent.ptr())); + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); + ASSERT_TRUE(isEnabled); + + // Verify that the copy is not performed yet (it's recorded but not executed) + ASSERT_EQ(*ptr2_data, 0U); + + // Join both queue2 operations back to queue1 and finish recording. + ur_event_handle_t joinEvents[] = {joinEvent.get(), noDepEvent.get()}; + ASSERT_SUCCESS(urEnqueueEventsWait(queue1, 2, joinEvents, nullptr)); + + ASSERT_SUCCESS(urQueueEndGraphCaptureExp(queue1, &graph)); + + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue1, &isEnabled)); + ASSERT_FALSE(isEnabled); + + // The primary queue stopped recording, so queue2 must leave the temporary + // recording state as well. + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); + ASSERT_FALSE(isEnabled); + + // Replaying the captured graph must execute the copy that was recorded on + // the forked queue, proving it really became part of the graph. + ur_exp_executable_graph_handle_t exGraph = nullptr; + ASSERT_SUCCESS(urGraphInstantiateGraphExp(graph, &exGraph)); + + ASSERT_SUCCESS(urEnqueueGraphExp(queue1, exGraph, 0, nullptr, nullptr)); + ASSERT_SUCCESS(urQueueFinish(queue1)); + + EXPECT_EQ(*ptr2_data, 0xdeadbeefU); + + EXPECT_SUCCESS(urGraphExecutableGraphDestroyExp(exGraph)); + ASSERT_SUCCESS(urUSMFree(context, ptr1)); + ASSERT_SUCCESS(urUSMFree(context, ptr2)); +}