-
Notifications
You must be signed in to change notification settings - Fork 0
/
generateAE_init.sh
136 lines (118 loc) · 3.9 KB
/
generateAE_init.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
#!/bin/sh
# This file is called . generateAE_init.sh
JobName="generateAE_serial_job_test" # for slurm
output="/sps/cms/chiron/TEMP/generateAE_%j.log" # for slurm
timeFolder="$(date +"%Y%m%d-%H%M%S")"
declare -a readarray
aa=$PWD
echo "actual path : $aa"
STR=$aa
Choice='Local'
for SUB in 'llr' 'pbs' 'cern'
do
if [[ "$STR" == *"$SUB"* ]]; then
echo "It's $SUB there.";
Choice=${SUB^^};
fi
done
echo "Choice is : $Choice"
FileName="paths$Choice.py"
echo $FileName
readarray toto -t array < CommonFiles/$FileName
N=${#toto[@]}
echo "N= $N"
LOG_SOURCE="$aa/${toto[15]}"
LOG_SOURCE=${LOG_SOURCE//LOG_SOURCE=}
LOG_SOURCE=${LOG_SOURCE//\"}
#LOG_OUTPUT="${toto[16]}"
#LOG_OUTPUT=${LOG_OUTPUT//LOG_OUTPUT=}
#LOG_OUTPUT=${LOG_OUTPUT//\"}
#RESULTFOLDER="${toto[15]}"
#RESULTFOLDER=${RESULTFOLDER//RESULTFOLDER=}
#RESULTFOLDER=${RESULTFOLDER//\"}
#LOG_KS_SOURCE="$aa/${toto[18]}"
#LOG_KS_SOURCE=${LOG_KS_SOURCE//LOG_KS_SOURCE=}
#LOG_KS_SOURCE=${LOG_KS_SOURCE//\"}
#LIB_SOURCE="${toto[19]}"
#LIB_SOURCE=${LIB_SOURCE//LIB_SOURCE=}
#LIB_SOURCE=${LIB_SOURCE//\"}
COMMON_SOURCE="$aa/${toto[20]}"
COMMON_SOURCE=${COMMON_SOURCE//COMMON_SOURCE=}
COMMON_SOURCE=${COMMON_SOURCE//\"}
LOG_AE_SOURCE="$aa/${toto[23]}"
LOG_AE_SOURCE=${LOG_AE_SOURCE//LOG_AE_SOURCE=}
LOG_AE_SOURCE=${LOG_AE_SOURCE//\"}
echo "LOG_SOURCE : $LOG_SOURCE"
#echo "LOG_OUTPUT : $LOG_OUTPUT"
#echo "RESULTFOLDER : $RESULTFOLDER"
#echo "LOG_KS_SOURCE : $LOG_KS_SOURCE"
echo "LOG_AE_SOURCE : $LOG_AE_SOURCE"
#echo "LIB_SOURCE : $LIB_SOURCE"
echo "COMMON_SOURCE : $COMMON_SOURCE"
readarray datasets -t array < ChiLib/HistosConfigFiles/ElectronMcSignalHistos.txt # $Chilib_path
N2=${#datasets[@]}
echo "nb lines in datasets= $N2"
var=0
for line in "${datasets[@]}"
do
if [[ $line == *"ElectronMcSignalValidator/"* ]]; then
#echo $line
arrLine=(${line//dator/ })
#echo "${arrLine[1]} - $var"
let "var++"
fi
done
echo "nb datasets in datasets= $var"
let "var--"
timeFolder2="$(date +"%Y%m%d-%H%M%S")_V2"
if [[ "$Choice" == "LLR" ]]
then
echo "LLR"
source /opt/exp_soft/llr/root/v6.24.04-el7-gcc9xx-py370/etc/init.sh
module purge
module use /opt/exp_soft/vo.gridcl.fr/software/modules/
module use /opt/exp_soft/vo.llr.in2p3.fr/modulefiles_el7
module use /opt/exp_soft/vo.llr.in2p3.fr/modulefiles
module load torch/1.5.0-py37-nocuda
module load python/3.7.0
cd $LOG_SOURCE
options="-reserv" # -short -long -reserv
# initialization of the temps folders
/opt/exp_soft/cms/t3/t3submit -8c $options initAE.sh $LOG_AE_SOURCE $COMMON_SOURCE $FileName # -mail [email protected]
#sleep 30
process_id=$!
echo "PID: $process_id"
wait $process_id
for line in "${datasets[@]}"
do
if [[ $line == *"ElectronMcSignalValidator/"* ]]; then
#echo $line
arrLine=(${line//dator/ })
echo "${arrLine[1]}"
#/opt/exp_soft/cms/t3/t3submit -8c $options generateAE.sh $LOG_AE_SOURCE $COMMON_SOURCE AEGeneration_V2.py $FileName $var ${arrLine[1]} 'cpu' $timeFolder # $LOG_SOURCE # -mail [email protected]
fi
done
elif [[ "$Choice" == "PBS" ]]
then
echo "PBS"
module load Programming_Languages/python/3.9.1
source /pbs/home/c/chiron/private/ValidationsTools/ValidationsTools/bin/activate
cd $LOG_SOURCE
# initialization of the temps folders
sbatch -L sps -n 4 --mem=16000 -t 0-0:20:0 -J $JobName -o $output initAE.sh $LOG_AE_SOURCE $COMMON_SOURCE $FileName
#sleep 30
process_id=$!
echo "PID: $process_id"
wait $process_id
for line in "${datasets[@]}"
do
if [[ $line == *"ElectronMcSignalValidator/"* ]]; then
#echo $line
arrLine=(${line//dator/ })
echo "${arrLine[1]}"
sbatch -L sps -n 4 --mem=16000 -t 0-3:0:0 -J $JobName -o $output generateAE.sh $LOG_AE_SOURCE $COMMON_SOURCE AEGeneration_V2.py $FileName $var ${arrLine[1]} 'cpu' $timeFolder # $LOG_SOURCE
fi
done
deactivate
fi
echo "END"