Skip to content

Commit

Permalink
parallelize zdc and insert simulations
Browse files Browse the repository at this point in the history
  • Loading branch information
veprbl committed Sep 30, 2024
1 parent 58f4b0a commit 8edecb0
Show file tree
Hide file tree
Showing 18 changed files with 172 additions and 201 deletions.
17 changes: 10 additions & 7 deletions benchmarks/insert_muon/Snakefile
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,14 @@ rule insert_muon_simulate:
params:
PHYSICS_LIST="FTFP_BERT"
output:
SIM_FILE="sim_output/insert_muon/{DETECTOR_CONFIG}_sim_mu-_{P}GeV.edm4hep.root"
SIM_FILE="sim_output/insert_muon/{DETECTOR_CONFIG}_sim_mu-_{P}GeV_{INDEX}.edm4hep.root"
shell:
"""
NEVENTS_SIM=5000
NEVENTS_SIM=1000
# Running simulation
npsim \
--compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \
--skipNEvents $(( $NEVENTS_SIM * {wildcards.INDEX} )) \
--numberOfEvents $NEVENTS_SIM \
--physicsList {params.PHYSICS_LIST} \
--inputFiles {input.GEN_FILE} \
Expand All @@ -33,20 +34,22 @@ npsim \

rule insert_muon_recon:
input:
SIM_FILE="sim_output/insert_muon/{DETECTOR_CONFIG}_sim_mu-_{P}GeV.edm4hep.root"
SIM_FILE="sim_output/insert_muon/{DETECTOR_CONFIG}_sim_mu-_{P}GeV_{INDEX}.edm4hep.root"
output:
REC_FILE="sim_output/insert_muon/{DETECTOR_CONFIG}_rec_mu-_{P}GeV.edm4hep.root"
REC_FILE="sim_output/insert_muon/{DETECTOR_CONFIG}_rec_mu-_{P}GeV_{INDEX}.edm4hep.root"
shell:
"""
NEVENTS_REC=5000
NEVENTS_REC=1000
eicrecon {input.SIM_FILE} -Ppodio:output_file={output.REC_FILE} -Pdd4hep:xml_files=$DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml -Ppodio:output_collections=MCParticles,HcalEndcapPInsertRecHits,HcalEndcapPInsertClusters,HcalEndcapPInsertSubcellHits,EcalEndcapPInsertRecHits,EcalEndcapPInsertClusters -Pjana:nevents=$NEVENTS_REC
"""

rule insert_muon_analysis:
input:
expand("sim_output/insert_muon/{DETECTOR_CONFIG}_sim_mu-_{P}GeV.edm4hep.root",
expand("sim_output/insert_muon/{DETECTOR_CONFIG}_sim_mu-_{P}GeV_{INDEX}.edm4hep.root",
P=[50],
DETECTOR_CONFIG=["{DETECTOR_CONFIG}"]),
DETECTOR_CONFIG=["{DETECTOR_CONFIG}"],
INDEX=range(5),
),
script="benchmarks/insert_muon/analysis/muon_plots.py",
output:
results_dir=directory("results/{DETECTOR_CONFIG}/insert_muon"),
Expand Down
169 changes: 83 additions & 86 deletions benchmarks/insert_muon/analysis/muon_plots.py
Original file line number Diff line number Diff line change
@@ -1,86 +1,83 @@
import numpy as np, pandas as pd, matplotlib.pyplot as plt, matplotlib as mpl, awkward as ak, sys
import mplhep as hep
hep.style.use("CMS")

plt.rcParams['figure.facecolor']='white'
plt.rcParams['savefig.facecolor']='white'
plt.rcParams['savefig.bbox']='tight'

plt.rcParams["figure.figsize"] = (7, 7)

config=sys.argv[1].split("/")[1] #results/{config}/{benchmark_name}
outdir=sys.argv[1]+"/"
try:
import os
os.mkdir(outdir[:-1])
except:
pass

def Landau(x, normalization,location,stdev):
#print(type(x))
u=(x-location)*3.591/stdev/2.355
renormalization = 1.64872*normalization
return renormalization * np.exp(-u/2 - np.exp(-u)/2)

import uproot as ur
arrays_sim={}
momenta=50,
for p in momenta:
filename=f'sim_output/insert_muon/{config}_sim_mu-_{p}GeV.edm4hep.root'
print("opening file", filename)
events = ur.open(filename+':events')
arrays_sim[p] = events.arrays()
import gc
gc.collect()
print("read", filename)

for array in arrays_sim.values():
tilt=-0.025
px=array['MCParticles.momentum.x'][:,2]
py=array['MCParticles.momentum.y'][:,2]
pz=array['MCParticles.momentum.z'][:,2]
p=np.sqrt(px**2+py**2+pz**2)

pxp=px*np.cos(tilt)-pz*np.sin(tilt)
pyp=py
pzp=pz*np.cos(tilt)+px*np.sin(tilt)

array['eta_truth']=1/2*np.log((p+pzp)/(p-pzp))
array['phi_truth']=np.arctan2(pyp,pxp)

for p in 50,:
E=arrays_sim[p]["HcalEndcapPInsertHits.energy"]
y, x,_=plt.hist(1e3*ak.flatten(E),bins=100, range=(0, 1.2), histtype='step')
bc=(x[1:]+x[:-1])/2
from scipy.optimize import curve_fit
slc=abs(bc-.48)<.15
fnc=Landau
p0=[100, .5, .05]
#print(list(y), list(x))
coeff, var_matrix = curve_fit(fnc, list(bc[slc]), list(y[slc]), p0=p0,
sigma=list(np.sqrt(y[slc])+(y[slc]==0)))
print(coeff)
xx=np.linspace(0,.7, 100)
MIP=coeff[1]/1000
plt.plot(xx, fnc(xx,*coeff), label=f'Landau fit:\nMIP={coeff[1]*1e3:.0f}$\\pm${1e3*np.sqrt(var_matrix[1][1]):.0f} keV')
plt.xlabel("hit energy [MeV]")
plt.ylabel("hits")
plt.title(f"$E_{{\\mu^-}}=${p} GeV")
plt.legend(fontsize=20)
plt.savefig(outdir+"/MIP.pdf")

plt.figure(figsize=(10,7))
array=arrays_sim[p]
bins=30; r=((-np.pi, np.pi),(2.8, 4.2))
selection=np.sum(array["HcalEndcapPInsertHits.energy"]>0.5*MIP,axis=-1)>0
h1, xedges, yedges = np.histogram2d(list(array[selection]['phi_truth']),list(array[selection]['eta_truth']), bins=bins, range=r)
h2, xedges, yedges = np.histogram2d(list(array['phi_truth']),list(array['eta_truth']), bins=bins, range=r)

h = h1 / h2
pc=plt.pcolor(xedges, yedges, h.T,linewidth=0)
plt.xlabel("$\\phi^*$ [rad]")
plt.ylabel("$\\eta^*$")
cb = plt.colorbar(pc)
cb.set_label("acceptance")
plt.title(f"$E_{{\\mu^-}}=${p} GeV")
plt.savefig(outdir+"/acceptance.pdf")
import numpy as np, pandas as pd, matplotlib.pyplot as plt, matplotlib as mpl, awkward as ak, sys
import mplhep as hep
hep.style.use("CMS")

plt.rcParams['figure.facecolor']='white'
plt.rcParams['savefig.facecolor']='white'
plt.rcParams['savefig.bbox']='tight'

plt.rcParams["figure.figsize"] = (7, 7)

config=sys.argv[1].split("/")[1] #results/{config}/{benchmark_name}
outdir=sys.argv[1]+"/"
try:
import os
os.mkdir(outdir[:-1])
except:
pass

def Landau(x, normalization,location,stdev):
#print(type(x))
u=(x-location)*3.591/stdev/2.355
renormalization = 1.64872*normalization
return renormalization * np.exp(-u/2 - np.exp(-u)/2)

import uproot as ur
arrays_sim={}
momenta=50,
for p in momenta:
arrays_sim[p] = ur.concatenate({
f'sim_output/insert_muon/{config}_rec_mu-_{p}GeV_{index}.edm4eic.root': 'events'
for index in range(5)
})

for array in arrays_sim.values():
tilt=-0.025
px=array['MCParticles.momentum.x'][:,2]
py=array['MCParticles.momentum.y'][:,2]
pz=array['MCParticles.momentum.z'][:,2]
p=np.sqrt(px**2+py**2+pz**2)

pxp=px*np.cos(tilt)-pz*np.sin(tilt)
pyp=py
pzp=pz*np.cos(tilt)+px*np.sin(tilt)

array['eta_truth']=1/2*np.log((p+pzp)/(p-pzp))
array['phi_truth']=np.arctan2(pyp,pxp)

for p in 50,:
E=arrays_sim[p]["HcalEndcapPInsertHits.energy"]
y, x,_=plt.hist(1e3*ak.flatten(E),bins=100, range=(0, 1.2), histtype='step')
bc=(x[1:]+x[:-1])/2
from scipy.optimize import curve_fit
slc=abs(bc-.48)<.15
fnc=Landau
p0=[100, .5, .05]
#print(list(y), list(x))
coeff, var_matrix = curve_fit(fnc, list(bc[slc]), list(y[slc]), p0=p0,
sigma=list(np.sqrt(y[slc])+(y[slc]==0)))
print(coeff)
xx=np.linspace(0,.7, 100)
MIP=coeff[1]/1000
plt.plot(xx, fnc(xx,*coeff), label=f'Landau fit:\nMIP={coeff[1]*1e3:.0f}$\\pm${1e3*np.sqrt(var_matrix[1][1]):.0f} keV')
plt.xlabel("hit energy [MeV]")
plt.ylabel("hits")
plt.title(f"$E_{{\\mu^-}}=${p} GeV")
plt.legend(fontsize=20)
plt.savefig(outdir+"/MIP.pdf")

plt.figure(figsize=(10,7))
array=arrays_sim[p]
bins=30; r=((-np.pi, np.pi),(2.8, 4.2))
selection=np.sum(array["HcalEndcapPInsertHits.energy"]>0.5*MIP,axis=-1)>0
h1, xedges, yedges = np.histogram2d(list(array[selection]['phi_truth']),list(array[selection]['eta_truth']), bins=bins, range=r)
h2, xedges, yedges = np.histogram2d(list(array['phi_truth']),list(array['eta_truth']), bins=bins, range=r)

h = h1 / h2
pc=plt.pcolor(xedges, yedges, h.T,linewidth=0)
plt.xlabel("$\\phi^*$ [rad]")
plt.ylabel("$\\eta^*$")
cb = plt.colorbar(pc)
cb.set_label("acceptance")
plt.title(f"$E_{{\\mu^-}}=${p} GeV")
plt.savefig(outdir+"/acceptance.pdf")
3 changes: 1 addition & 2 deletions benchmarks/insert_muon/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,8 @@ sim:insert_muon:
parallel:
matrix:
- P: 50
timeout: 1 hours
script:
- snakemake --cores 1 sim_output/insert_muon/epic_craterlake_sim_mu-_${P}GeV.edm4hep.root
- snakemake --cores 5 sim_output/insert_muon/epic_craterlake_sim_mu-_${P}GeV.edm4hep.root
retry:
max: 2
when:
Expand Down
17 changes: 10 additions & 7 deletions benchmarks/insert_neutron/Snakefile
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,14 @@ rule insert_neutron_simulate:
params:
PHYSICS_LIST="FTFP_BERT"
output:
SIM_FILE="sim_output/insert_neutron/{DETECTOR_CONFIG}_sim_neutron_{P}GeV.edm4hep.root"
SIM_FILE="sim_output/insert_neutron/{DETECTOR_CONFIG}_sim_neutron_{P}GeV_{INDEX}.edm4hep.root"
shell:
"""
NEVENTS_SIM=1000
NEVENTS_SIM=200
# Running simulation
npsim \
--compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \
--skipNEvents $(( $NEVENTS_SIM * {wildcards.INDEX} )) \
--numberOfEvents $NEVENTS_SIM \
--physicsList {params.PHYSICS_LIST} \
--inputFiles {input.GEN_FILE} \
Expand All @@ -34,20 +35,22 @@ npsim \

rule insert_neutron_recon:
input:
SIM_FILE="sim_output/insert_neutron/{DETECTOR_CONFIG}_sim_neutron_{P}GeV.edm4hep.root"
SIM_FILE="sim_output/insert_neutron/{DETECTOR_CONFIG}_sim_neutron_{P}GeV_{INDEX}.edm4hep.root"
output:
REC_FILE="sim_output/insert_neutron/{DETECTOR_CONFIG}_rec_neutron_{P}GeV.edm4eic.root"
REC_FILE="sim_output/insert_neutron/{DETECTOR_CONFIG}_rec_neutron_{P}GeV_{INDEX}.edm4eic.root"
shell:
"""
NEVENTS_REC=1000
NEVENTS_REC=200
eicrecon {input.SIM_FILE} -Ppodio:output_file={output.REC_FILE} -Pdd4hep:xml_files=$DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml -Ppodio:output_collections=MCParticles,HcalEndcapPInsertRecHits,HcalEndcapPInsertClusters,HcalEndcapPInsertSubcellHits,EcalEndcapPInsertRecHits,EcalEndcapPInsertClusters -Pjana:nevents=$NEVENTS_REC
"""

rule insert_neutron_analysis:
input:
expand("sim_output/insert_neutron/{DETECTOR_CONFIG}_rec_neutron_{P}GeV.edm4eic.root",
expand("sim_output/insert_neutron/{DETECTOR_CONFIG}_rec_neutron_{P}GeV_{INDEX}.edm4eic.root",
P=[20, 30, 40, 50, 60, 70, 80],
DETECTOR_CONFIG=["{DETECTOR_CONFIG}"]),
DETECTOR_CONFIG=["{DETECTOR_CONFIG}"],
INDEX=range(5),
),
script="benchmarks/insert_neutron/analysis/neutron_plots.py",
output:
results_dir=directory("results/{DETECTOR_CONFIG}/insert_neutron"),
Expand Down
6 changes: 4 additions & 2 deletions benchmarks/insert_neutron/analysis/neutron_plots.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@
#read files
arrays_sim={}
for p in 20,30,40,50,60,70,80:
arrays_sim[p] = ur.open(f'sim_output/insert_neutron/{config}_rec_neutron_{p}GeV.edm4eic.root:events')\
.arrays()
arrays_sim[p] = ur.concatenate({
f'sim_output/insert_neutron/{config}_rec_neutron_{p}GeV_{index}.edm4eic.root': 'events'
for index in range(5)
})

def gauss(x, A,mu, sigma):
return A * np.exp(-(x-mu)**2/(2*sigma**2))
Expand Down
3 changes: 1 addition & 2 deletions benchmarks/insert_neutron/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,8 @@ sim:insert_neutron:
- P: 60
- P: 70
- P: 80
timeout: 1 hours
script:
- snakemake --cores 1 sim_output/insert_neutron/epic_craterlake_rec_neutron_${P}GeV.edm4eic.root
- snakemake --cores 5 sim_output/insert_neutron/epic_craterlake_rec_neutron_${P}GeV.edm4eic.root
retry:
max: 2
when:
Expand Down
27 changes: 11 additions & 16 deletions benchmarks/zdc_lambda/Snakefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ rule zdc_lambda_generate:
input:
script="benchmarks/zdc_lambda/analysis/gen_lambda_decay.cxx",
params:
NEVENTS_GEN=100000,
NEVENTS_GEN=1000,
output:
GEN_FILE="sim_output/zdc_lambda/lambda_decay_{P}GeV.hepmc"
shell:
Expand All @@ -16,17 +16,14 @@ rule zdc_lambda_simulate:
params:
PHYSICS_LIST="FTFP_BERT"
output:
SIM_FILE="sim_output/zdc_lambda/{DETECTOR_CONFIG}_sim_lambda_dec_{P}GeV.edm4hep.root"
SIM_FILE="sim_output/zdc_lambda/{DETECTOR_CONFIG}_sim_lambda_dec_{P}GeV_{INDEX}.edm4hep.root"
shell:
"""
if [[ {wildcards.P} -gt 220 ]]; then
NEVENTS_SIM=500
else
NEVENTS_SIM=1000
fi
NEVENTS_SIM=200
# Running simulation
npsim \
--compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \
--skipNEvents $(( $NEVENTS_SIM * {wildcards.INDEX} )) \
--numberOfEvents $NEVENTS_SIM \
--physicsList {params.PHYSICS_LIST} \
--inputFiles {input.GEN_FILE} \
Expand All @@ -35,24 +32,22 @@ npsim \

rule zdc_lambda_recon:
input:
SIM_FILE="sim_output/zdc_lambda/{DETECTOR_CONFIG}_sim_lambda_dec_{P}GeV.edm4hep.root"
SIM_FILE="sim_output/zdc_lambda/{DETECTOR_CONFIG}_sim_lambda_dec_{P}GeV_{INDEX}.edm4hep.root"
output:
REC_FILE="sim_output/zdc_lambda/{DETECTOR_CONFIG}_rec_lambda_dec_{P}GeV.edm4eic.root"
REC_FILE="sim_output/zdc_lambda/{DETECTOR_CONFIG}_rec_lambda_dec_{P}GeV_{INDEX}.edm4eic.root"
shell:
"""
if [[ {wildcards.P} -gt 220 ]]; then
NEVENTS_REC=500
else
NEVENTS_REC=1000
fi
NEVENTS_REC=200
eicrecon {input.SIM_FILE} -Ppodio:output_file={output.REC_FILE} -Pdd4hep:xml_files=$DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml -Ppodio:output_collections=MCParticles,HcalFarForwardZDCClusters,HcalFarForwardZDCRecHits,HcalFarForwardZDCSubcellHits -Pjana:nevents=$NEVENTS_REC
"""

rule zdc_lambda_analysis:
input:
expand("sim_output/zdc_lambda/{DETECTOR_CONFIG}_rec_lambda_dec_{P}GeV.edm4eic.root",
expand("sim_output/zdc_lambda/{DETECTOR_CONFIG}_rec_lambda_dec_{P}GeV_{INDEX}.edm4eic.root",
P=[100, 125, 150,175, 200, 225, 250, 275],
DETECTOR_CONFIG=["{DETECTOR_CONFIG}"]),
DETECTOR_CONFIG=["{DETECTOR_CONFIG}"],
INDEX=range(5),
),
script="benchmarks/zdc_lambda/analysis/lambda_plots.py",
output:
results_dir=directory("results/{DETECTOR_CONFIG}/zdc_lambda"),
Expand Down
11 changes: 4 additions & 7 deletions benchmarks/zdc_lambda/analysis/lambda_plots.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,10 @@
arrays_sim={}
momenta=100, 125, 150, 175,200,225,250,275
for p in momenta:
filename=f'sim_output/zdc_lambda/{config}_rec_lambda_dec_{p}GeV.edm4eic.root'
print("opening file", filename)
events = ur.open(filename+':events')
arrays_sim[p] = events.arrays()[:-1] #remove last event, which for some reason is blank
import gc
gc.collect()
print("read", filename)
arrays_sim[p] = ur.concatenate({
f'sim_output/zdc_lambda/{config}_rec_lambda_dec_{p}GeV_{index}.edm4eic.root': 'events'
for index in range(5)
})

def gauss(x, A,mu, sigma):
return A * np.exp(-(x-mu)**2/(2*sigma**2))
Expand Down
3 changes: 1 addition & 2 deletions benchmarks/zdc_lambda/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,8 @@ sim:zdc_lambda:
- P: 225
- P: 250
- P: 275
timeout: 1 hours
script:
- snakemake --cores 1 sim_output/zdc_lambda/epic_zdc_sipm_on_tile_only_rec_lambda_dec_${P}GeV.edm4eic.root
- snakemake --cores 5 sim_output/zdc_lambda/epic_zdc_sipm_on_tile_only_rec_lambda_dec_${P}GeV.edm4eic.root
retry:
max: 2
when:
Expand Down
Loading

0 comments on commit 8edecb0

Please sign in to comment.