[alpaka] Add support for the SYCL back-end

Makefile

@@ -132,38 +132,52 @@ OCLOC_IDS := pvc    # tgllp acm_g10 pvc
 
 ifdef SYCL_USE_INTEL_ONEAPI
   ONEAPI_BASE := /opt/intel/oneapi
+endif
 
-  ifeq ($(wildcard $(ONEAPI_BASE)),)
-    $(warning Cannot find an Intel oneAPI installation at $(ONEAPI_BASE))
-  endif
+ifeq ($(wildcard $(ONEAPI_BASE)),)
+  # Intel oneAPI not available
+  $(warning Cannot find an Intel oneAPI installation at $(ONEAPI_BASE))
+  SYCL_USE_INTEL_ONEAPI :=
+  ONEAPI_BASE :=
+endif
+
+INTELOCLCPU_BASE := $(BASE_DIR)/external/oclcpuexp
+INTELOCLCPU_ICD := $(INTELOCLCPU_BASE)/x64/libintelocl.so
 
+ifdef SYCL_USE_INTEL_ONEAPI
   # Intel oneTBB
-  TBB_BASE      := $(ONEAPI_BASE)/tbb/latest
-  TBB_LIBDIR    := $(TBB_BASE)/lib/intel64/gcc4.8
+  TBB_BASE    := $(ONEAPI_BASE)/tbb/latest
+  TBB_LIBDIR  := $(TBB_BASE)/lib/intel64/gcc4.8
+
+  # Intel debugger
+  GDB_ONEAPI_BASE := $(ONEAPI_BASE)/debugger/latest
 
   # use Intel oneAPI DPC++/C++ Compiler
-  SYCL_BASE     := $(ONEAPI_BASE)/compiler/latest/linux
-  SYCL_PATH     := $(SYCL_BASE)/bin:$(SYCL_BASE)/bin-llvm
-  SYCL_LDPATH   := $(SYCL_BASE)/lib:$(SYCL_BASE)/lib/x64:$(SYCL_BASE)/compiler/lib/intel64_lin
-  SYCL_LIBDIR   := $(SYCL_BASE)/lib
+  SYCL_BASE   := $(ONEAPI_BASE)/compiler/latest/linux
+  SYCL_PATH   := $(SYCL_BASE)/bin:$(SYCL_BASE)/bin-llvm
+  SYCL_LDPATH := $(SYCL_BASE)/lib:$(SYCL_BASE)/lib/x64:$(SYCL_BASE)/compiler/lib/intel64_lin
+  SYCL_LIBDIR := $(SYCL_BASE)/lib
   # use ICPX:       $(SYCL_BASE)/bin/icpx
   # use clang++:    $(SYCL_BASE)/bin-llvm/clang++
-  SYCL_CXX      := $(SYCL_BASE)/bin/icpx
+  SYCL_CXX    := $(SYCL_BASE)/bin/icpx
 
   # use the oneAPI CPU OpenCL runtime
-  export OCL_ICD_FILENAMES := $(SYCL_BASE)/lib/x64/libintelocl.so
+  INTELOCLCPU_ICD := $(SYCL_BASE)/lib/x64/libintelocl.so
+
+  # override the CPU OpenCL runtime with the last known working version (2022.14.8.0.04)
+  INTELOCLCPU_ICD := $(INTELOCLCPU_BASE)/x64/libintelocl.so
 else
   # use clang++
   # latest release: /cvmfs/patatrack.cern.ch/externals/x86_64/rhel8/intel/sycl/release/2022-12
   # latest nightly: /cvmfs/patatrack.cern.ch/externals/x86_64/rhel8/intel/sycl/nightly/20230708
-  SYCL_BASE     := /cvmfs/patatrack.cern.ch/externals/x86_64/rhel8/intel/sycl/nightly/20230805_160000
-  SYCL_PATH     := $(SYCL_BASE)/bin
-  SYCL_LDPATH   := $(SYCL_BASE)/lib:$(SYCL_BASE)/lib64
-  SYCL_LIBDIR   := $(SYCL_BASE)/lib
-  SYCL_CXX      := $(SYCL_BASE)/bin/clang++
+  SYCL_BASE   := /cvmfs/patatrack.cern.ch/externals/x86_64/rhel8/intel/sycl/nightly/20230805_160000
+  SYCL_PATH   := $(SYCL_BASE)/bin
+  SYCL_LDPATH := $(SYCL_BASE)/lib:$(SYCL_BASE)/lib64
+  SYCL_LIBDIR := $(SYCL_BASE)/lib
+  SYCL_CXX    := $(SYCL_BASE)/bin/clang++
 
-  # use the latest CPU OpenCL runtime (2023.16.6.0.28)
-  export OCL_ICD_FILENAMES := /cvmfs/patatrack.cern.ch/externals/x86_64/rhel8/intel/sycl/runtime/intel/oclcpuexp_2023.16.6.0.28/x64/libintelocl.so
+  # use the last known working CPU OpenCL runtime (2022.14.8.0.04)
+  INTELOCLCPU_ICD := /cvmfs/patatrack.cern.ch/externals/x86_64/rhel8/intel/sycl/runtime/intel/oclcpuexp_2022.14.8.0.04/x64/libintelocl.so
 endif
 
 ifneq ($(wildcard $(SYCL_BASE)),)
@@ -189,6 +203,8 @@ ifneq ($(wildcard $(SYCL_BASE)),)
   SYCL_FLAGS        := -fsycl -fsycl-targets=$(SYCL_TARGETS)
   SYCL_LDFLAGS      := -fsycl-fp32-prec-sqrt -fsycl-link-huge-device-code $(JIT_FLAGS) $(AOT_CPU_FLAGS) $(AOT_INTEL_FLAGS) $(AOT_CUDA_FLAGS) $(AOT_ROCM_FLAGS)
 
+  export OCL_ICD_FILENAMES := $(INTELOCLCPU_ICD)
+
   # other SYCL options
   #  -fsycl-device-code-split=per_kernel
   #    build one binary image per kernel (very slow), to allow kernel built for subgroup sizes not supported by all devices;
@@ -234,6 +250,15 @@ ifneq ($(wildcard $(SYCL_BASE)),)
   export SYCL_CXXFLAGS := $(filter-out $(LLVM_UNSUPPORTED_CXXFLAGS),$(CXXFLAGS)) $(SYCL_FLAGS) $(USER_SYCLFLAGS)
   export SYCL_LDFLAGS
 
+  # alpaka SYCL targets$(ALPAKA_SYCL_GPU_TARGETS)
+  export SYCL_BASE
+  export ALPAKA_SYCL_CXXFLAGS    := -fsycl $(filter-out $(LLVM_UNSUPPORTED_CXXFLAGS),$(CXXFLAGS)) $(USER_SYCLFLAGS) -Wno-unused-const-variable -Wno-constant-conversion -Wno-tautological-constant-compare
+  export ALPAKA_SYCL_LDFLAGS     := -fsycl-fp32-prec-sqrt -fsycl-link-huge-device-code -fsycl-max-parallel-link-jobs=8
+  export ALPAKA_SYCL_CPU_TARGETS := -fsycl-targets=spir64_x86_64
+  export ALPAKA_SYCL_CPU_FLAGS   :=
+  export ALPAKA_SYCL_GPU_TARGETS := -fsycl-targets=$(foreach ARCH,$(OCLOC_IDS),intel_gpu_$(ARCH))
+  export ALPAKA_SYCL_GPU_FLAGS   := -Xsycl-target-backend=intel_gpu_pvc '-q -options -ze-intel-enable-auto-large-GRF-mode'
+
   # add the SYCL paths to the PATH and LD_LIBRARY_PATH
   export PATH := $(SYCL_PATH):$(PATH)
   export LD_LIBRARY_PATH := $(SYCL_LDPATH):$(TBB_LIBDIR):$(LD_LIBRARY_PATH)
@@ -253,6 +278,10 @@ DATA_TAR_GZ := $(DATA_BASE)/data.tar.gz
 # External definitions
 EXTERNAL_BASE := $(BASE_DIR)/external
 
+export INTELOCLCPU_DEPS := $(INTELOCLCPU_ICD)
+INTELOCLCPU_CXXFLAGS :=
+INTELOCLCPU_LDFLAGS :=
+
 HWLOC_BASE := $(EXTERNAL_BASE)/hwloc
 export HWLOC_DEPS := $(HWLOC_BASE)
 HWLOC_CXXFLAGS := -isystem $(HWLOC_BASE)/include
@@ -320,7 +349,7 @@ export BACKTRACE_SYCL_CXXFLAGS :=
 
 ALPAKA_BASE := $(EXTERNAL_BASE)/alpaka
 export ALPAKA_DEPS := $(ALPAKA_BASE)
-export ALPAKA_CXXFLAGS := -isystem $(ALPAKA_BASE)/include
+export ALPAKA_CXXFLAGS := -isystem $(ALPAKA_BASE)/include -DALPAKA_DISABLE_VENDOR_RNG
 
 KOKKOS_BASE := $(EXTERNAL_BASE)/kokkos
 KOKKOS_SRC := $(KOKKOS_BASE)/source
@@ -526,7 +555,8 @@ test_auto: $(TEST_AUTO_TARGETS)
 .PHONY: test_auto $(TEST_AUTO_TARGETS)
 .PHONY: format $(patsubst %,format_%,$(TARGETS_ALL))
 .PHONY: environment print_targets clean distclean dataclean
-.PHONY: external_tbb external_cub external_eigen external_kokkos external_kokkos_clean
+.PHONY: external_oclcpu external_tbb external_eigen external_boost external_libbacktrace external_hwloc external_alpaka external_kokkos external_kokkos_clean
+
 
 environment: env.sh
 env.sh: Makefile
@@ -556,6 +586,9 @@ endif
 	@echo -n '$(KOKKOS_LIBDIR):'                                            >> $@
 ifneq ($(SYCL_BASE),)
 	@echo -n '$(SYCL_LDPATH):'                                              >> $@
+ifneq ($(SYCL_USE_INTEL_ONEAPI),)
+	@echo -n '$(GDB_ONEAPI_BASE)/gdb/intel64/lib:'                          >> $@
+endif
 endif
 	@echo '$$LD_LIBRARY_PATH'                                               >> $@
 	@# set the PATH
@@ -568,9 +601,15 @@ ifdef ROCM_BASE
 endif
 ifneq ($(SYCL_BASE),)
 	@echo -n '$(SYCL_PATH):'                                                >> $@
+ifneq ($(SYCL_USE_INTEL_ONEAPI),)
+	@echo -n '$(GDB_ONEAPI_BASE)/gdb/intel64/bin:'                          >> $@
+endif
 endif
 	@echo '$$PATH'                                                          >> $@
 ifneq ($(SYCL_BASE),)
+ifneq ($(SYCL_USE_INTEL_ONEAPI),)
+	@echo 'export INTEL_PYTHONHOME=$(GDB_ONEAPI_BASE)/dep'                  >> $@
+endif
 	@# load the CPU OpenCL runtime
 	@echo 'export OCL_ICD_FILENAMES=$(OCL_ICD_FILENAMES)'                   >> $@
 	@# enable double precision floating point emulation for Intel GPUs
@@ -667,6 +706,15 @@ $(DATA_TAR_GZ): | $(DATA_BASE)/url.txt
 $(EXTERNAL_BASE):
 	mkdir -p $@
 
+# OpenCL CPU runtime
+external_oclcpu: $(INTELOCLCPU_ICD)
+ifneq ($(findstring $(EXTERNAL_BASE),$(INTELOCLCPU_ICD)),)
+$(INTELOCLCPU_ICD):
+	mkdir -p $(INTELOCLCPU_BASE)
+	curl -L -s -S https://github.com/intel/llvm/releases/download/2022-WW33/oclcpuexp-2022.14.8.0.04_rel.tar.gz | tar xz -C $(INTELOCLCPU_BASE)
+	chmod -R +rwX $(INTELOCLCPU_BASE)
+endif
+
 # TBB
 external_tbb: $(TBB_LIB)
 
@@ -748,7 +796,7 @@ external_alpaka: $(ALPAKA_BASE)
 
 $(ALPAKA_BASE):
 	git clone https://github.com/alpaka-group/alpaka.git -b develop $@
-	cd $@ && git checkout bb74c9129e8761cb74b9733b034eec62f7c0f600
+	cd $@ && git checkout 819974ddc5b2eb4b33e709bd317701793cdb7d15
 
 # Kokkos
 external_kokkos: $(KOKKOS_LIB)

src/alpaka/AlpakaCore/AllocatorPolicy.h

@@ -48,6 +48,26 @@ namespace cms::alpakatools {
 #endif
 #endif  // ALPAKA_ACC_GPU_HIP_ENABLED
 
+#if defined ALPAKA_SYCL_ONEAPI_CPU
+  template <>
+  constexpr inline AllocatorPolicy allocator_policy<alpaka::DevCpuSycl> =
+#if !defined ALPAKA_DISABLE_CACHING_ALLOCATOR
+      AllocatorPolicy::Caching;
+#else
+      AllocatorPolicy::Synchronous;
+#endif
+#endif  // ALPAKA_SYCL_ONEAPI_CPU
+
+#if defined ALPAKA_SYCL_ONEAPI_GPU
+  template <>
+  constexpr inline AllocatorPolicy allocator_policy<alpaka::DevGpuSyclIntel> =
+#if !defined ALPAKA_DISABLE_CACHING_ALLOCATOR
+      AllocatorPolicy::Caching;
+#else
+      AllocatorPolicy::Synchronous;
+#endif
+#endif  // ALPAKA_SYCL_ONEAPI_GPU
+
 }  // namespace cms::alpakatools
 
 #endif  // AlpakaCore_AllocatorPolicy_h

src/alpaka/AlpakaCore/CachedBufAlloc.h

@@ -134,6 +134,111 @@ namespace cms::alpakatools {
 
 #endif  // ALPAKA_ACC_GPU_HIP_ENABLED
 
+#ifdef ALPAKA_SYCL_ONEAPI_CPU
+
+    //! The caching memory allocator implementation for the pinned host memory
+    template <typename TElem, typename TDim, typename TIdx>
+    struct CachedBufAlloc<TElem, TDim, TIdx, alpaka::DevCpu, alpaka::QueueCpuSyclNonBlocking, void> {
+      template <typename TExtent>
+      ALPAKA_FN_HOST static auto allocCachedBuf(alpaka::DevCpu const& dev,
+                                                alpaka::QueueCpuSyclNonBlocking queue,
+                                                TExtent const& extent) -> alpaka::BufCpu<TElem, TDim, TIdx> {
+        ALPAKA_DEBUG_MINIMAL_LOG_SCOPE;
+
+        auto& allocator = getHostCachingAllocator<alpaka::QueueCpuSyclNonBlocking>();
+
+        // FIXME the BufCpu does not support a pitch ?
+        size_t size = alpaka::getExtentProduct(extent);
+        size_t sizeBytes = size * sizeof(TElem);
+        void* memPtr = allocator.allocate(sizeBytes, queue);
+
+        // use a custom deleter to return the buffer to the CachingAllocator
+        auto deleter = [alloc = &allocator](TElem* ptr) { alloc->free(ptr); };
+
+        return alpaka::BufCpu<TElem, TDim, TIdx>(dev, reinterpret_cast<TElem*>(memPtr), std::move(deleter), extent);
+      }
+    };
+
+    //! The caching memory allocator implementation for the SYCL CPU device
+    template <typename TElem, typename TDim, typename TIdx, typename TQueue>
+    struct CachedBufAlloc<TElem, TDim, TIdx, alpaka::DevCpuSycl, TQueue, void> {
+      template <typename TExtent>
+      ALPAKA_FN_HOST static auto allocCachedBuf(alpaka::DevCpuSycl const& dev, TQueue queue, TExtent const& extent)
+          -> alpaka::BufCpuSycl<TElem, TDim, TIdx> {
+        ALPAKA_DEBUG_MINIMAL_LOG_SCOPE;
+
+        auto& allocator = getDeviceCachingAllocator<alpaka::DevCpuSycl, TQueue>(dev);
+
+        // size_t width = alpaka::getWidth(extent);
+        // size_t widthBytes = width * static_cast<TIdx>(sizeof(TElem));
+        // TODO implement pitch in SYCL
+        // size_t pitchBytes = widthBytes;
+        size_t size = alpaka::getExtentProduct(extent);
+        size_t sizeBytes = size * sizeof(TElem);
+        void* memPtr = allocator.allocate(sizeBytes, queue);
+
+        // use a custom deleter to return the buffer to the CachingAllocator
+        auto deleter = [alloc = &allocator](TElem* ptr) { alloc->free(ptr); };
+
+        return alpaka::BufCpuSycl<TElem, TDim, TIdx>(dev, reinterpret_cast<TElem*>(memPtr), std::move(deleter), extent);
+      }
+    };
+
+#endif  // ALPAKA_SYCL_ONEAPI_CPU
+
+#ifdef ALPAKA_SYCL_ONEAPI_GPU
+
+    //! The caching memory allocator implementation for the pinned host memory
+    template <typename TElem, typename TDim, typename TIdx>
+    struct CachedBufAlloc<TElem, TDim, TIdx, alpaka::DevCpu, alpaka::QueueGpuSyclIntelNonBlocking, void> {
+      template <typename TExtent>
+      ALPAKA_FN_HOST static auto allocCachedBuf(alpaka::DevCpu const& dev,
+                                                alpaka::QueueGpuSyclIntelNonBlocking queue,
+                                                TExtent const& extent) -> alpaka::BufCpu<TElem, TDim, TIdx> {
+        ALPAKA_DEBUG_MINIMAL_LOG_SCOPE;
+
+        auto& allocator = getHostCachingAllocator<alpaka::QueueGpuSyclIntelNonBlocking>();
+
+        // FIXME the BufCpu does not support a pitch ?
+        size_t size = alpaka::getExtentProduct(extent);
+        size_t sizeBytes = size * sizeof(TElem);
+        void* memPtr = allocator.allocate(sizeBytes, queue);
+
+        // use a custom deleter to return the buffer to the CachingAllocator
+        auto deleter = [alloc = &allocator](TElem* ptr) { alloc->free(ptr); };
+
+        return alpaka::BufCpu<TElem, TDim, TIdx>(dev, reinterpret_cast<TElem*>(memPtr), std::move(deleter), extent);
+      }
+    };
+
+    //! The caching memory allocator implementation for the SYCL GPU device
+    template <typename TElem, typename TDim, typename TIdx, typename TQueue>
+    struct CachedBufAlloc<TElem, TDim, TIdx, alpaka::DevGpuSyclIntel, TQueue, void> {
+      template <typename TExtent>
+      ALPAKA_FN_HOST static auto allocCachedBuf(alpaka::DevGpuSyclIntel const& dev, TQueue queue, TExtent const& extent)
+          -> alpaka::BufGpuSyclIntel<TElem, TDim, TIdx> {
+        ALPAKA_DEBUG_MINIMAL_LOG_SCOPE;
+
+        auto& allocator = getDeviceCachingAllocator<alpaka::DevGpuSyclIntel, TQueue>(dev);
+
+        // size_t width = alpaka::getWidth(extent);
+        // size_t widthBytes = width * static_cast<TIdx>(sizeof(TElem));
+        // TODO implement pitch in SYCL
+        // size_t pitchBytes = widthBytes;
+        size_t size = alpaka::getExtentProduct(extent);
+        size_t sizeBytes = size * sizeof(TElem);
+        void* memPtr = allocator.allocate(sizeBytes, queue);
+
+        // use a custom deleter to return the buffer to the CachingAllocator
+        auto deleter = [alloc = &allocator](TElem* ptr) { alloc->free(ptr); };
+
+        return alpaka::BufGpuSyclIntel<TElem, TDim, TIdx>(
+            dev, reinterpret_cast<TElem*>(memPtr), std::move(deleter), extent);
+      }
+    };
+
+#endif  // ALPAKA_SYCL_ONEAPI_GPU
+
   }  // namespace traits
 
   template <typename TElem, typename TIdx, typename TExtent, typename TQueue, typename TDev>

src/alpaka/AlpakaCore/CachingAllocator.h

@@ -201,9 +201,13 @@ namespace cms::alpakatools {
           std::ostringstream out;
           out << "\t" << deviceType_ << " " << alpaka::getName(device_) << " returned " << block.bytes << " bytes at "
               << ptr << " from associated queue " << block.queue->m_spQueueImpl.get() << " , event "
-              << block.event->m_spEventImpl.get() << " .\n\t\t " << cachedBlocks_.size() << " available blocks cached ("
-              << cachedBytes_.free << " bytes), " << liveBlocks_.size() << " live blocks (" << cachedBytes_.live
-              << " bytes) outstanding." << std::endl;
+#if ALPAKA_ACC_SYCL_ENABLED
+              << " is not a shared pointer in SYCL"
+#else
+              << block.event->m_spEventImpl.get()
+#endif
+              << " .\n\t\t " << cachedBlocks_.size() << " available blocks cached (" << cachedBytes_.free << " bytes), "
+              << liveBlocks_.size() << " live blocks (" << cachedBytes_.live << " bytes) outstanding." << std::endl;
           std::cout << out.str() << std::endl;
         }
       } else {
@@ -212,9 +216,13 @@ namespace cms::alpakatools {
           std::ostringstream out;
           out << "\t" << deviceType_ << " " << alpaka::getName(device_) << " freed " << block.bytes << " bytes at "
               << ptr << " from associated queue " << block.queue->m_spQueueImpl.get() << ", event "
-              << block.event->m_spEventImpl.get() << " .\n\t\t " << cachedBlocks_.size() << " available blocks cached ("
-              << cachedBytes_.free << " bytes), " << liveBlocks_.size() << " live blocks (" << cachedBytes_.live
-              << " bytes) outstanding." << std::endl;
+#if ALPAKA_ACC_SYCL_ENABLED
+              << " is not a shared pointer in SYCL"
+#else
+              << block.event->m_spEventImpl.get()
+#endif
+              << " .\n\t\t " << cachedBlocks_.size() << " available blocks cached (" << cachedBytes_.free << " bytes), "
+              << liveBlocks_.size() << " live blocks (" << cachedBytes_.live << " bytes) outstanding." << std::endl;
           std::cout << out.str() << std::endl;
         }
       }
@@ -301,9 +309,18 @@ namespace cms::alpakatools {
             std::ostringstream out;
             out << "\t" << deviceType_ << " " << alpaka::getName(device_) << " reused cached block at "
                 << block.buffer->data() << " (" << block.bytes << " bytes) for queue "
-                << block.queue->m_spQueueImpl.get() << ", event " << block.event->m_spEventImpl.get()
+                << block.queue->m_spQueueImpl.get() << ", event "
+#if ALPAKA_ACC_SYCL_ENABLED
+                << " is not a shared pointer in SYCL"
+#else
+                << block.event->m_spEventImpl.get()
+#endif
                 << " (previously associated with stream " << iBlock->second.queue->m_spQueueImpl.get() << " , event "
+#if ALPAKA_ACC_SYCL_ENABLED
+                << " is not a shared pointer in SYCL)." << std::endl;
+#else
                 << iBlock->second.event->m_spEventImpl.get() << ")." << std::endl;
+#endif
             std::cout << out.str() << std::endl;
           }
 
@@ -366,7 +383,13 @@ namespace cms::alpakatools {
         std::ostringstream out;
         out << "\t" << deviceType_ << " " << alpaka::getName(device_) << " allocated new block at "
             << block.buffer->data() << " (" << block.bytes << " bytes associated with queue "
-            << block.queue->m_spQueueImpl.get() << ", event " << block.event->m_spEventImpl.get() << "." << std::endl;
+            << block.queue->m_spQueueImpl.get() << ", event "
+#if ALPAKA_ACC_SYCL_ENABLED
+            << " is not a shared pointer in SYCL"
+#else
+            << block.event->m_spEventImpl.get()
+#endif
+            << "." << std::endl;
         std::cout << out.str() << std::endl;
       }
     }

src/alpaka/AlpakaCore/ESProduct.h

@@ -26,7 +26,12 @@ namespace cms::alpakatools {
 
     ESProduct() : gpuDataPerDevice_(devices<Platform>().size()) {
       for (size_t i = 0; i < gpuDataPerDevice_.size(); ++i) {
-        gpuDataPerDevice_[i].m_event = getEventCache<Event>().get(devices<Platform>()[i]);
+        gpuDataPerDevice_[i].m_event =
+#if !defined(ALPAKA_ACC_SYCL_ENABLED)
+            getEventCache<Event>().get(devices<Platform>()[i]);
+#else
+            std::make_shared<Event>(devices<Platform>()[i]);
+#endif
       }
     }