Merge pull request #6 from PhilipFackler/count-intersections

Count intersections

scripts/config_defiant.sh

@@ -10,10 +10,15 @@ echo $MV_DIR
 module purge
 
 # load required modules
-module load PrgEnv-cray-amd
-module load cray-mpich
+module load craype-accel-amd-gfx908
+module load PrgEnv-cray
+module load amd
 module load julia
 
+export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0
+export MPICH_GPU_SUPPORT_ENABLED=1
+export MPICH_SMP_SINGLE_COPY_MODE=CMA
+
 # remove existing generated Manifest.toml
 rm -f $MV_DIR/Manifest.toml
 rm -f $MV_DIR/LocalPreferences.toml
@@ -23,31 +28,33 @@ export JULIA_AMDGPU_DISABLE_ARTIFACTS=1
 
 julia --project=$MV_DIR -e 'using Pkg; Pkg.instantiate()'
 
-# cray-mpich
+# MPI + AMDGPU
 julia --project=$MV_DIR -e ' \
     using Pkg; \
-    Pkg.add("MPIPreferences"); \
-    '
-julia --project=$MV_DIR -e ' \
+    if !haskey(Pkg.project().dependencies, "MPIPreferences"); \
+        Pkg.add("MPIPreferences"); \
+    end; \
     using MPIPreferences; \
     MPIPreferences.use_system_binary(mpiexec="srun", vendor="cray"); \
-    '
-
-# amdgpu
-julia --project=$MV_DIR -e ' \
-    using Pkg; \
-    Pkg.add(; name="AMDGPU", version = "v0.8.11"); \
+    if !haskey(Pkg.project().dependencies, "AMDGPU"); \
+        Pkg.add(; name="AMDGPU", version = "v0.8.11"); \
+    end; \
     '
 
 # JACC
 julia --project=$MV_DIR -e ' \
     using Pkg; \
-    Pkg.add(; name="JACC", rev = "main"); \
-    '
-julia --project=$MV_DIR -e ' \
+    jaccInfo = Pkg.dependencies()[Pkg.project().dependencies["JACC"]]; \
+    if jaccInfo.git_revision != "main"; \
+        Pkg.add(; name="JACC", rev = "main"); \
+    end; \
     using JACC; \
     JACC.JACCPreferences.set_backend("amdgpu"); \
     '
 
 # Verify the packages are installed correctly
 julia --project=$MV_DIR -e 'using Pkg; Pkg.instantiate()'
+julia --project=$MV_DIR -e ' \
+    using Pkg; \
+    Pkg.status(); \
+    '

scripts/config_milan0.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+MV_DIR=/home/4pf/MiniVATES.jl
+
+module load nvhpc
+
+#export CUDA_HOME=/sw/wombat/Nvidia_HPC_SDK/Linux_aarch64/24.5/cuda/12.4
+
+# rm -f $MV_DIR/Manifest.toml
+# rm -f $MV_DIR/LocalPreferences.toml
+
+julia --project=$MV_DIR -e 'using Pkg; Pkg.instantiate()'
+
+# MPI + CUDA
+julia --project=$MV_DIR -e ' \
+    using Pkg; \
+    if !haskey(Pkg.project().dependencies, "MPIPreferences"); \
+        Pkg.add("MPIPreferences"); \
+    end; \
+    using MPIPreferences; \
+    MPIPreferences.use_jll_binary(); \
+    using MPI; \
+    MPI.install_mpiexecjl(force = true); \
+    if !haskey(Pkg.project().dependencies, "CUDA"); \
+        Pkg.add("CUDA"); \
+    end; \
+    using CUDA; \
+    '
+
+# JACC
+julia --project=$MV_DIR -e ' \
+    using Pkg; \
+    jaccInfo = Pkg.dependencies()[Pkg.project().dependencies["JACC"]]; \
+    if jaccInfo.git_revision != "fix-cuda-thread-counts-2d"; \
+        Pkg.add(; name="JACC", url = "https://github.com/PhilipFackler/JACC.jl.git", rev = "fix-cuda-thread-counts-2d"); \
+    end; \
+    using JACC; \
+    JACC.JACCPreferences.set_backend("cuda"); \
+    '
+
+# Verify the packages are installed correctly
+julia --project=$MV_DIR -e 'using Pkg; Pkg.instantiate()'
+julia --project=$MV_DIR -e ' \
+    using Pkg; \
+    Pkg.status(); \
+    using CUDA; \
+    CUDA.versioninfo(); \
+    '

scripts/config_wombat.sh

@@ -36,7 +36,6 @@ julia --project=$MV_DIR -e ' \
 julia --project=$MV_DIR -e ' \
     using Pkg; \
     jaccInfo = Pkg.dependencies()[Pkg.project().dependencies["JACC"]]; \
-    @show jaccInfo.git_revision; \
     if jaccInfo.git_revision != "main"; \
         Pkg.add(; name="JACC", rev = "main"); \
     end; \

src/BinSeries.jl

@@ -20,7 +20,7 @@ using Printf
     return (start + 1, stop + 1)
 end
 
-tmfmt(tm::AbstractFloat) = @sprintf("%3.6f", tm)
+tmfmt(tm::AbstractFloat) = @sprintf("%3.6fs", tm)
 
 @inline function binSeries!(
     signal::Hist3,
@@ -55,6 +55,10 @@ tmfmt(tm::AbstractFloat) = @sprintf("%3.6f", tm)
         @show rank, start, stop
     end
 
+    updAvgJ = 0.0
+    mdnAvgJ = 0.0
+    binAvgJ = 0.0
+
     updAvg = 0.0
     mdnAvg = 0.0
     binAvg = 0.0
@@ -92,37 +96,57 @@ tmfmt(tm::AbstractFloat) = @sprintf("%3.6f", tm)
                     lpad(fi, 3),
                     "; updateEvents: ",
                     tmfmt(updateEventsTime),
-                    "s, mdNorm: ",
+                    ", mdNorm: ",
                     tmfmt(mdNormTime),
-                    "s, binEvents: ",
+                    ", binEvents: ",
                     tmfmt(binEventsTime),
-                    "s",
                 )
             end
             MPI.Barrier(comm)
         end
+
+        if fi == start
+            updAvgJ = updAvg
+            mdnAvgJ = mdnAvg
+            binAvgJ = binAvg
+            updAvg = 0.0
+            mdnAvg = 0.0
+            binAvg = 0.0
+        end
     end
-    sum = MPI.Reduce((updAvg, mdnAvg, binAvg), .+, comm)
+    sumJ = [updAvgJ, mdnAvgJ, binAvgJ]
+    MPI.Reduce!(sumJ, MPI.SUM, 0, comm)
+    sum = [updAvg, mdnAvg, binAvg]
+    MPI.Reduce!(sum, MPI.SUM, 0, comm)
     if rank == 0
+        avgJ = sumJ ./ commSize
         avg = sum ./ nFiles
         println("Averages:")
-        println("    updateEvents: ", tmfmt(avg[1]), "s")
-        println("    mdNorm:       ", tmfmt(avg[2]), "s")
-        println("    binEvents:    ", tmfmt(avg[3]), "s")
-        println()
+        println("    updateEvents (JIT): ", tmfmt(avgJ[1]))
+        println("    updateEvents:       ", tmfmt(avg[1]))
+        println("    mdNorm (JIT):       ", tmfmt(avgJ[2]))
+        println("    mdNorm:             ", tmfmt(avg[2]))
+        println("    binEvents (JIT):    ", tmfmt(avgJ[3]))
+        println("    binEvents:          ", tmfmt(avg[3]))
     end
 
     return (signal, eventsHist)
 end
 
-function mergeHistogramToRootProcess(hist::Hist3)
-    weights = MPI.Reduce(Core.Array(binweights(hist)), .+, MPI.COMM_WORLD)
-    x, y, z = edges(hist)
-    return Hist3(
-        (Core.Array(x), Core.Array(y), Core.Array(z)),
-        nbins(hist),
-        origin(hist),
-        boxLength(hist),
-        weights,
-    )
+function mergeHistogramToRootProcess!(hist::Hist3)
+    dur = @elapsed begin
+        weights = Core.Array(binweights(hist))
+        MPI.Reduce!(weights, MPI.SUM, 0, MPI.COMM_WORLD)
+        ret = Hist3(
+            edges(hist),
+            nbins(hist),
+            origin(hist),
+            boxLength(hist),
+            adapt_structure(JACC.Array, weights),
+        )
+    end
+    if MPI.Comm_rank(MPI.COMM_WORLD) == 0
+        println("Reduce: ", tmfmt(dur))
+    end
+    return ret
 end

src/Hist.jl

@@ -2,6 +2,13 @@ import Atomix
 import Adapt
 import Base: @propagate_inbounds
 
+@inline function _maxIntersections(hX, kX, lX)
+    hNPts = length(hX)
+    kNPts = length(kX)
+    lNPts = length(lX)
+    return hNPts + kNPts + lNPts + 2
+end
+
 struct Hist3{TArray1,TArray3}
     edges::NTuple{3,TArray1}
     nbins::NTuple{3,SizeType}
@@ -23,6 +30,10 @@ end
 
 Adapt.@adapt_structure Hist3
 
+@inline function maxIntersections(h::Hist3)
+    return _maxIntersections(h.edges[1], h.edges[2], h.edges[3])
+end
+
 @inline edges(h::Hist3) = h.edges
 
 @inline nbins(h::Hist3) = h.nbins

src/Load.jl

@@ -335,15 +335,9 @@ mutable struct EventData
 end
 
 @inline function updateEvents!(data::EventData, ws::EventWorkspace)
-    ds = _getEventsDataset(ws)
-    dims, _ = HDF5.get_extent_dims(ds)
-    eventsMat = parent(data.events)
-    if dims[2] > size(eventsMat)[2]
-        eventsMat = Array2c(undef, dims)
-    end
-    events = view(eventsMat, :, 1:dims[2])
-    copyto!(events, _getEventsDataset(ws))
-    data.events = adapt_structure(JACC.Array, events)
+    unsafe_free!(parent(data.events))
+    data.events = getEvents(ws)
+    return nothing
 end
 
 function loadEventData(event_nxs_file::AbstractString)