Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[L0] Interrupt-based event implementation #2334

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
+263 −109
Open

[L0] Interrupt-based event implementation #2334

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ad277f0
[L0] Interrupt-based event implementation
winstonzhang-intel Nov 19, 2024
933760b
[L0] Fix urEnqueueEventsWaitWithBarrier option1
winstonzhang-intel Nov 21, 2024
28e7d38
[L0] Cleaned up urEnqueueEventsWaitWithBarrier(Ext) with helper option
winstonzhang-intel Nov 22, 2024
d9576ca
[L0] Rebased against top of main
winstonzhang-intel Nov 22, 2024
0856313
[L0] Interrupt-based event implementation
winstonzhang-intel Nov 26, 2024
f7fa435
[L0] Interrupt-based event implementation
winstonzhang-intel Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions source/adapters/level_zero/command_buffer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,8 @@ ur_result_t createSyncPointAndGetZeEvents(
UR_CALL(EventCreate(CommandBuffer->Context, nullptr /*Queue*/,
false /*IsMultiDevice*/, HostVisible, &LaunchEvent,
false /*CounterBasedEventEnabled*/,
!CommandBuffer->IsProfilingEnabled));
!CommandBuffer->IsProfilingEnabled,
false /*InterruptBasedEventEnabled*/));
LaunchEvent->CommandType = CommandType;
ZeLaunchEvent = LaunchEvent->ZeEvent;

Expand Down Expand Up @@ -662,13 +663,15 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device,
if (Device->hasMainCopyEngine()) {
UR_CALL(EventCreate(Context, nullptr /*Queue*/, false, false,
&CopyFinishedEvent, UseCounterBasedEvents,
!EnableProfiling));
!EnableProfiling,
false /*InterruptBasedEventEnabled*/));
}

if (EnableProfiling) {
UR_CALL(EventCreate(Context, nullptr /*Queue*/, false /*IsMultiDevice*/,
false /*HostVisible*/, &ComputeFinishedEvent,
UseCounterBasedEvents, !EnableProfiling));
UseCounterBasedEvents, !EnableProfiling,
false /*InterruptBasedEventEnabled*/));
}
}

Expand All @@ -677,7 +680,8 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device,
if (WaitEventPath) {
UR_CALL(EventCreate(Context, nullptr /*Queue*/, false /*IsMultiDevice*/,
false /*HostVisible*/, &WaitEvent,
false /*CounterBasedEventEnabled*/, !EnableProfiling));
false /*CounterBasedEventEnabled*/, !EnableProfiling,
false /*InterruptBasedEventEnabled*/));
}

// Create ZeCommandListResetEvents only if counter-based events are not being
Expand All @@ -689,15 +693,17 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device,
if (!UseCounterBasedEvents) {
UR_CALL(EventCreate(Context, nullptr /*Queue*/, false /*IsMultiDevice*/,
false /*HostVisible*/, &AllResetEvent,
false /*CounterBasedEventEnabled*/, !EnableProfiling));
false /*CounterBasedEventEnabled*/, !EnableProfiling,
false /*InterruptBasedEventEnabled*/));

UR_CALL(createMainCommandList(Context, Device, false, false, false,
ZeCommandListResetEvents));

// The ExecutionFinishedEvent is only waited on by ZeCommandListResetEvents.
UR_CALL(EventCreate(Context, nullptr /*Queue*/, false /*IsMultiDevice*/,
false /*HostVisible*/, &ExecutionFinishedEvent,
false /*CounterBasedEventEnabled*/, !EnableProfiling));
false /*CounterBasedEventEnabled*/, !EnableProfiling,
false /*InterruptBased*/));
}

try {
Expand Down
25 changes: 19 additions & 6 deletions source/adapters/level_zero/context.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -478,7 +478,8 @@ static const uint32_t MaxNumEventsPerPool = [] {
ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
ze_event_pool_handle_t &Pool, size_t &Index, bool HostVisible,
bool ProfilingEnabled, ur_device_handle_t Device,
bool CounterBasedEventEnabled, bool UsingImmCmdList) {
bool CounterBasedEventEnabled, bool UsingImmCmdList,
bool InterruptBasedEventEnabled) {
// Lock while updating event pool machinery.
std::scoped_lock<ur_mutex> Lock(ZeEventPoolCacheMutex);

Expand All @@ -487,9 +488,9 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
if (Device) {
ZeDevice = Device->ZeDevice;
}
std::list<ze_event_pool_handle_t> *ZePoolCache =
getZeEventPoolCache(HostVisible, ProfilingEnabled,
CounterBasedEventEnabled, UsingImmCmdList, ZeDevice);
std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
HostVisible, ProfilingEnabled, CounterBasedEventEnabled, UsingImmCmdList,
InterruptBasedEventEnabled, ZeDevice);

if (!ZePoolCache->empty()) {
if (NumEventsAvailableInEventPool[ZePoolCache->front()] == 0) {
Expand Down Expand Up @@ -537,6 +538,14 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
counterBasedExt.flags);
ZeEventPoolDesc.pNext = &counterBasedExt;
}
if (InterruptBasedEventEnabled) {
ze_intel_event_sync_mode_exp_desc_t eventSyncMode = {
ZE_INTEL_STRUCTURE_TYPE_EVENT_SYNC_MODE_EXP_DESC, nullptr, 0};
eventSyncMode.syncModeFlags =
ZE_INTEL_EVENT_SYNC_MODE_EXP_FLAG_LOW_POWER_WAIT |
ZE_INTEL_EVENT_SYNC_MODE_EXP_FLAG_SIGNAL_INTERRUPT;
ZeEventPoolDesc.pNext = &eventSyncMode;
}

std::vector<ze_device_handle_t> ZeDevices;
if (ZeDevice) {
Expand All @@ -563,7 +572,7 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(

ur_event_handle_t ur_context_handle_t_::getEventFromContextCache(
bool HostVisible, bool WithProfiling, ur_device_handle_t Device,
bool CounterBasedEventEnabled) {
bool CounterBasedEventEnabled, bool InterruptBasedEventEnabled) {
std::scoped_lock<ur_mutex> Lock(EventCacheMutex);
auto Cache = getEventCache(HostVisible, WithProfiling, Device);
if (Cache->empty())
Expand All @@ -574,6 +583,9 @@ ur_event_handle_t ur_context_handle_t_::getEventFromContextCache(
if (Event->CounterBasedEventsEnabled != CounterBasedEventEnabled) {
return nullptr;
}
if (Event->InterruptBasedEventsEnabled != InterruptBasedEventEnabled) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will not work unless the interrupt based events are somehow first on the Cache container. The same is true of the counter-based events. There needs to be separate pool for interrupt-driven events (or maybe they shouldn't be cached at all?).

return nullptr;
}
Cache->erase(It);
// We have to reset event before using it.
Event->reset();
Expand Down Expand Up @@ -614,7 +626,8 @@ ur_context_handle_t_::decrementUnreleasedEventsInPool(ur_event_handle_t Event) {

std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
Event->isHostVisible(), Event->isProfilingEnabled(),
Event->CounterBasedEventsEnabled, UsingImmediateCommandlists, ZeDevice);
Event->CounterBasedEventsEnabled, UsingImmediateCommandlists,
Event->InterruptBasedEventsEnabled, ZeDevice);

// Put the empty pool to the cache of the pools.
if (NumEventsUnreleasedInEventPool[Event->ZeEventPool] == 0)
Expand Down
106 changes: 86 additions & 20 deletions source/adapters/level_zero/context.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,24 @@ struct l0_command_list_cache_info {
bool IsImmediate = false;
};

typedef uint32_t ze_intel_event_sync_mode_exp_flags_t;
typedef enum _ze_intel_event_sync_mode_exp_flag_t {
ZE_INTEL_EVENT_SYNC_MODE_EXP_FLAG_LOW_POWER_WAIT = ZE_BIT(0),
ZE_INTEL_EVENT_SYNC_MODE_EXP_FLAG_SIGNAL_INTERRUPT = ZE_BIT(1),
ZE_INTEL_EVENT_SYNC_MODE_EXP_EXP_FLAG_FORCE_UINT32 = 0x7fffffff

} ze_intel_event_sync_mode_exp_flag_t;

#define ZE_INTEL_STRUCTURE_TYPE_EVENT_SYNC_MODE_EXP_DESC \
(ze_structure_type_t)0x00030016

typedef struct _ze_intel_event_sync_mode_exp_desc_t {
ze_structure_type_t stype;
const void *pNext;

ze_intel_event_sync_mode_exp_flags_t syncModeFlags;
} ze_intel_event_sync_mode_exp_desc_t;

struct ur_context_handle_t_ : _ur_object {
ur_context_handle_t_(ze_context_handle_t ZeContext, uint32_t NumDevices,
const ur_device_handle_t *Devs, bool OwnZeContext)
Expand Down Expand Up @@ -150,9 +168,9 @@ struct ur_context_handle_t_ : _ur_object {
// head.
//
// Cache of event pools to which host-visible events are added to.
std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{12};
std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{30};
std::vector<std::unordered_map<ze_device_handle_t, size_t>>
ZeEventPoolCacheDeviceMap{12};
ZeEventPoolCacheDeviceMap{30};

// This map will be used to determine if a pool is full or not
// by storing number of empty slots available in the pool.
Expand Down Expand Up @@ -208,13 +226,15 @@ struct ur_context_handle_t_ : _ur_object {
bool ProfilingEnabled,
ur_device_handle_t Device,
bool CounterBasedEventEnabled,
bool UsingImmCmdList);
bool UsingImmCmdList,
bool InterruptBasedEventEnabled);

// Get ur_event_handle_t from cache.
ur_event_handle_t getEventFromContextCache(bool HostVisible,
bool WithProfiling,
ur_device_handle_t Device,
bool CounterBasedEventEnabled);
bool CounterBasedEventEnabled,
bool InterruptBasedEventEnabled);

// Add ur_event_handle_t to cache.
void addEventToContextCache(ur_event_handle_t);
Expand All @@ -225,17 +245,29 @@ struct ur_context_handle_t_ : _ur_object {
HostVisibleCounterBasedRegularCacheType,
HostInvisibleCounterBasedRegularCacheType,
HostVisibleCounterBasedImmediateCacheType,
HostInvisibleCounterBasedImmediateCacheType
HostInvisibleCounterBasedImmediateCacheType,

HostVisibleInterruptBasedRegularCacheType,
HostInvisibleInterruptBasedRegularCacheType,
HostVisibleInterruptBasedImmediateCacheType,
HostInvisibleInterruptBasedImmediateCacheType,

HostVisibleInterruptAndCounterBasedRegularCacheType,
HostInvisibleInterruptAndCounterBasedRegularCacheType,
HostVisibleInterruptAndCounterBasedImmediateCacheType,
HostInvisibleInterruptAndCounterBasedImmediateCacheType
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is exactly why I've asked you to do the refactor. This will grow exponentially with new event parameters.

};

std::list<ze_event_pool_handle_t> *
getZeEventPoolCache(bool HostVisible, bool WithProfiling,
bool CounterBasedEventEnabled, bool UsingImmediateCmdList,
bool InterruptBasedEventEnabled,
ze_device_handle_t ZeDevice) {
EventPoolCacheType CacheType;

calculateCacheIndex(HostVisible, CounterBasedEventEnabled,
UsingImmediateCmdList, CacheType);
UsingImmediateCmdList, InterruptBasedEventEnabled,
CacheType);
if (ZeDevice) {
auto ZeEventPoolCacheMap =
WithProfiling ? &ZeEventPoolCacheDeviceMap[CacheType * 2]
Expand All @@ -255,23 +287,57 @@ struct ur_context_handle_t_ : _ur_object {
ur_result_t calculateCacheIndex(bool HostVisible,
bool CounterBasedEventEnabled,
bool UsingImmediateCmdList,
bool InterruptBasedEventEnabled,
EventPoolCacheType &CacheType) {
if (CounterBasedEventEnabled && HostVisible && !UsingImmediateCmdList) {
CacheType = HostVisibleCounterBasedRegularCacheType;
} else if (CounterBasedEventEnabled && !HostVisible &&
!UsingImmediateCmdList) {
CacheType = HostInvisibleCounterBasedRegularCacheType;
} else if (CounterBasedEventEnabled && HostVisible &&
UsingImmediateCmdList) {
CacheType = HostVisibleCounterBasedImmediateCacheType;
} else if (CounterBasedEventEnabled && !HostVisible &&
UsingImmediateCmdList) {
CacheType = HostInvisibleCounterBasedImmediateCacheType;
} else if (!CounterBasedEventEnabled && HostVisible) {
CacheType = HostVisibleCacheType;
if (InterruptBasedEventEnabled) {
if (CounterBasedEventEnabled) {
if (HostVisible) {
if (UsingImmediateCmdList) {
CacheType = HostVisibleInterruptAndCounterBasedImmediateCacheType;
} else {
CacheType = HostVisibleInterruptAndCounterBasedRegularCacheType;
}
} else {
if (UsingImmediateCmdList) {
CacheType = HostInvisibleInterruptAndCounterBasedImmediateCacheType;
} else {
CacheType = HostInvisibleInterruptAndCounterBasedRegularCacheType;
}
}
} else {
if (HostVisible) {
if (UsingImmediateCmdList) {
CacheType = HostVisibleInterruptBasedImmediateCacheType;
} else {
CacheType = HostVisibleInterruptBasedRegularCacheType;
}
} else {
if (UsingImmediateCmdList) {
CacheType = HostInvisibleInterruptBasedImmediateCacheType;
} else {
CacheType = HostInvisibleInterruptBasedRegularCacheType;
}
}
}
} else {
CacheType = HostInvisibleCacheType;
if (CounterBasedEventEnabled && HostVisible && !UsingImmediateCmdList) {
CacheType = HostVisibleCounterBasedRegularCacheType;
} else if (CounterBasedEventEnabled && !HostVisible &&
!UsingImmediateCmdList) {
CacheType = HostInvisibleCounterBasedRegularCacheType;
} else if (CounterBasedEventEnabled && HostVisible &&
UsingImmediateCmdList) {
CacheType = HostVisibleCounterBasedImmediateCacheType;
} else if (CounterBasedEventEnabled && !HostVisible &&
UsingImmediateCmdList) {
CacheType = HostInvisibleCounterBasedImmediateCacheType;
} else if (!CounterBasedEventEnabled && HostVisible) {
CacheType = HostVisibleCacheType;
} else {
CacheType = HostInvisibleCacheType;
}
}

return UR_RESULT_SUCCESS;
}

Expand Down
4 changes: 2 additions & 2 deletions source/adapters/level_zero/device.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,8 @@ ur_result_t urDeviceGetInfo(
case UR_DEVICE_INFO_BUILT_IN_KERNELS:
// TODO: To find out correct value
return ReturnValue("");
case UR_DEVICE_INFO_LOW_POWER_EVENTS_EXP:
return ReturnValue(static_cast<ur_bool_t>(true));
case UR_DEVICE_INFO_QUEUE_PROPERTIES:
return ReturnValue(
ur_queue_flag_t(UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE |
Expand Down Expand Up @@ -1151,8 +1153,6 @@ ur_result_t urDeviceGetInfo(
return ReturnValue(true);
case UR_DEVICE_INFO_USM_POOL_SUPPORT:
return ReturnValue(true);
case UR_DEVICE_INFO_LOW_POWER_EVENTS_EXP:
return ReturnValue(false);
default:
logger::error("Unsupported ParamName in urGetDeviceInfo");
logger::error("ParamNameParamName={}(0x{})", ParamName,
Expand Down
Loading
Loading