From 6528728161e0743ea1919a78bc5440d0ab685f37 Mon Sep 17 00:00:00 2001 From: Kevin B Smith Date: Fri, 25 Jun 2021 14:25:14 -0700 Subject: [PATCH 1/2] [SYCL][PI][L0] Force reset of memcpy command-list. This change set forces a host syncronization on a mempcy's command list fence, and the reset of that commandlist. Resetting the command list has the effect of causing Level Zero to free any resources the command list may be using, and by doing this as early as possible helps remove any bookkeeping that level zero may be doing on memory regions. --- sycl/plugins/level_zero/pi_level_zero.cpp | 28 +++++++++++++++++------ 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/sycl/plugins/level_zero/pi_level_zero.cpp b/sycl/plugins/level_zero/pi_level_zero.cpp index bf1743d80fddd..ef4adb9856659 100644 --- a/sycl/plugins/level_zero/pi_level_zero.cpp +++ b/sycl/plugins/level_zero/pi_level_zero.cpp @@ -185,10 +185,10 @@ template <> ze_result_t zeHostSynchronize(ze_event_handle_t Handle) { template <> ze_result_t zeHostSynchronize(ze_command_queue_handle_t Handle) { return zeHostSynchronizeImpl(zeCommandQueueSynchronize, Handle); } -// template <> -// ze_result_t zeHostSynchronize(ze_fence_handle_t Handle) { -// return zeHostSynchronizeImpl(zeFenceHostSynchronize, Handle); -// } +template <> +ze_result_t zeHostSynchronize(ze_fence_handle_t Handle) { + return zeHostSynchronizeImpl(zeFenceHostSynchronize, Handle); +} template pi_result getInfoImpl(size_t param_value_size, void *param_value, @@ -4231,11 +4231,25 @@ static pi_result cleanupAfterEvent(pi_event Event) { } // It is possible that the fence was already noted as signalled and - // reset. In that case the InUse flag will be false, and there is - // no need to query the fence's status or try to reset it. + // reset. In that case the InUse flag will be false, and + // we shouldn't query it, synchronize on it, or try to reset it. if (it->second.InUse) { + // Workaround for VM_BIND mode. + // Make sure that the command-list doing memcpy is reset before + // non-USM host memory potentially involved in the memcpy is freed. + // + // NOTE: it is valid to wait for the fence here as long as we aren't + // doing batching on the involved command-list. Today memcpy goes by + // itself in a command list. + // + // TODO: this will unnecessarily(?) wait for non-USM memory buffers + // too, so we might need to add a new command type to differentiate. + // ze_result_t ZeResult = - ZE_CALL_NOCHECK(zeFenceQueryStatus, (it->second.ZeFence)); + (Event->CommandType == PI_COMMAND_TYPE_MEM_BUFFER_COPY) + ? ZE_CALL_NOCHECK(zeHostSynchronize, (it->second.ZeFence)) + : ZE_CALL_NOCHECK(zeFenceQueryStatus, (it->second.ZeFence)); + if (ZeResult == ZE_RESULT_SUCCESS) { Queue->resetCommandListFenceEntry(*it, true); Event->ZeCommandList = nullptr; From 2d7ee9b1b480f3e9f94c16802c27c42a03c9c6f0 Mon Sep 17 00:00:00 2001 From: Kevin B Smith Date: Fri, 25 Jun 2021 14:46:19 -0700 Subject: [PATCH 2/2] Fixes clang-format problem in previous commit. --- sycl/plugins/level_zero/pi_level_zero.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sycl/plugins/level_zero/pi_level_zero.cpp b/sycl/plugins/level_zero/pi_level_zero.cpp index ef4adb9856659..e175ccdf61002 100644 --- a/sycl/plugins/level_zero/pi_level_zero.cpp +++ b/sycl/plugins/level_zero/pi_level_zero.cpp @@ -185,8 +185,7 @@ template <> ze_result_t zeHostSynchronize(ze_event_handle_t Handle) { template <> ze_result_t zeHostSynchronize(ze_command_queue_handle_t Handle) { return zeHostSynchronizeImpl(zeCommandQueueSynchronize, Handle); } -template <> -ze_result_t zeHostSynchronize(ze_fence_handle_t Handle) { +template <> ze_result_t zeHostSynchronize(ze_fence_handle_t Handle) { return zeHostSynchronizeImpl(zeFenceHostSynchronize, Handle); }