From a3f2d3eaf865312d584f90f678e4ce79a7caa78d Mon Sep 17 00:00:00 2001 From: mmusich Date: Sat, 16 Nov 2024 23:07:10 +0100 Subject: [PATCH 1/3] improve submitPVResolutionJobs.py --- .../scripts/submitPVResolutionJobs.py | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/Alignment/OfflineValidation/scripts/submitPVResolutionJobs.py b/Alignment/OfflineValidation/scripts/submitPVResolutionJobs.py index a9e9c26373a2a..07933cf678435 100755 --- a/Alignment/OfflineValidation/scripts/submitPVResolutionJobs.py +++ b/Alignment/OfflineValidation/scripts/submitPVResolutionJobs.py @@ -302,11 +302,11 @@ def main(): runs.sort() print("\n\n Will run on the following runs: \n",runs) - if(not os.path.exists("cfg")): - os.system("mkdir cfg") - os.system("mkdir BASH") - os.system("mkdir harvest") - os.system("mkdir out") + # List of directories to create + directories = ["cfg", "BASH", "harvest", "out"] + + for directory in directories: + os.makedirs(directory, exist_ok=True) cwd = os.getcwd() bashdir = os.path.join(cwd,"BASH") @@ -412,10 +412,25 @@ def main(): key = key.split(":", 1)[1] print("dealing with",key) - os.system("cp "+input_CMSSW_BASE+"/src/Alignment/OfflineValidation/test/PrimaryVertexResolution_templ_cfg.py ./cfg/PrimaryVertexResolution_"+key+"_"+run+"_cfg.py") - os.system("sed -i 's|XXX_FILES_XXX|"+listOfFiles+"|g' "+cwd+"/cfg/PrimaryVertexResolution_"+key+"_"+run+"_cfg.py") - os.system("sed -i 's|XXX_RUN_XXX|"+run+"|g' "+cwd+"/cfg/PrimaryVertexResolution_"+key+"_"+run+"_cfg.py") - os.system("sed -i 's|YYY_KEY_YYY|"+key+"|g' "+cwd+"/cfg/PrimaryVertexResolution_"+key+"_"+run+"_cfg.py") + # Paths and variables + template_file = os.path.join(input_CMSSW_BASE, "src/Alignment/OfflineValidation/test/PrimaryVertexResolution_templ_cfg.py") + output_file = f"./cfg/PrimaryVertexResolution_{key}_{run}_cfg.py" + + # Copy the template file to the destination + shutil.copy(template_file, output_file) + + # Read and replace placeholders in the copied file + with open(output_file, 'r') as file: + content = file.read() + + # Replace placeholders with actual values + content = content.replace("XXX_FILES_XXX", listOfFiles) + content = content.replace("XXX_RUN_XXX", run) + content = content.replace("YYY_KEY_YYY", key) + + # Write the modified content back to the file + with open(output_file, 'w') as file: + file.write(content) scriptFileName = os.path.join(bashdir,"batchHarvester_"+key+"_"+str(count-1)+".sh") scriptFile = open(scriptFileName,'w') From ebed28f64ccbc362e65de656fd3145e25f551ce6 Mon Sep 17 00:00:00 2001 From: mmusich Date: Sun, 17 Nov 2024 15:55:22 +0100 Subject: [PATCH 2/3] changes in submitPVResolutionJobs.py and test_unitSubmitPVsplit.sh to avoid memory leaks in ASAN_X --- .../scripts/submitPVResolutionJobs.py | 17 ++++++++++++----- .../testingScripts/test_unitSubmitPVsplit.sh | 11 ++++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/Alignment/OfflineValidation/scripts/submitPVResolutionJobs.py b/Alignment/OfflineValidation/scripts/submitPVResolutionJobs.py index 07933cf678435..e5b8653d5c3f0 100755 --- a/Alignment/OfflineValidation/scripts/submitPVResolutionJobs.py +++ b/Alignment/OfflineValidation/scripts/submitPVResolutionJobs.py @@ -190,16 +190,23 @@ def batchScriptCERN(theCMSSW_BASE, cfgdir, runindex, eosdir, lumiToRun, key, con ####################################################### '''prepare the batch script, to run on HTCondor''' script = """#!/bin/bash -#source /afs/cern.ch/cms/caf/setup.sh CMSSW_DIR={CMSSW_BASE_DIR}/src/Alignment/OfflineValidation/test -echo "the mother directory is $CMSSW_DIR" +echo "The mother directory is $CMSSW_DIR" export X509_USER_PROXY=$CMSSW_DIR/.user_proxy #OUT_DIR=$CMSSW_DIR/harvest ## for local storage OUT_DIR={MYDIR} LOG_DIR=$CMSSW_DIR/out -LXBATCH_DIR=`pwd` -cd $CMSSW_DIR -eval `scram runtime -sh` +LXBATCH_DIR=$PWD +# Check if CMSSW environment is set by checking CMSSW_BASE or other variables +if [[ -z "$CMSSW_BASE" || -z "$CMSSW_VERSION" || -z "$SCRAM_ARCH" ]]; then + echo "CMSSW environment not detected. Sourcing scramv1 runtime..." + cd $CMSSW_DIR + # Assuming you have a valid CMSSW release environment to source + source /cvmfs/cms.cern.ch/cmsset_default.sh + eval $(scramv1 runtime -sh) # This sets the CMSSW environment +else + echo "CMSSW environment is already set. Continuing..." +fi cd $LXBATCH_DIR cp -pr {CFGDIR}/PrimaryVertexResolution_{KEY}_{runindex}_cfg.py . cmsRun PrimaryVertexResolution_{KEY}_{runindex}_cfg.py TrackCollection={TRKS} GlobalTag={GT} lumi={LUMITORUN} {REC} {EXT} >& log_{KEY}_run{runindex}.out diff --git a/Alignment/OfflineValidation/test/testingScripts/test_unitSubmitPVsplit.sh b/Alignment/OfflineValidation/test/testingScripts/test_unitSubmitPVsplit.sh index e797ed9a9d15d..aaced2fc5aac2 100755 --- a/Alignment/OfflineValidation/test/testingScripts/test_unitSubmitPVsplit.sh +++ b/Alignment/OfflineValidation/test/testingScripts/test_unitSubmitPVsplit.sh @@ -11,12 +11,13 @@ echo -e "\n\n TESTING Primary Vertex Split script execution ..." scriptName="batchHarvester_Prompt_0.sh" # Create directory if it doesn't exist -mkdir -p "./testExecution" +testdir=$PWD +mkdir ${testdir}/"testExecution" # Check if the script exists and is a regular file -if [ -f "./BASH/${scriptName}" ]; then +if [ -f "${testdir}/BASH/${scriptName}" ]; then # Copy script to the test execution directory - cp -pr "./BASH/${scriptName}" "./testExecution/" + cp "${testdir}/BASH/${scriptName}" "${testdir}/testExecution/" else # Emit a warning if the script doesn't exist or is not a regular file echo "Warning: Script '${scriptName}' not found or is not a regular file. Skipping excution of further tests." @@ -24,10 +25,10 @@ else fi # Change directory to the test execution directory -cd "./testExecution" || exit 1 +cd "${testdir}/testExecution" || exit 1 # Execute the script and handle errors -./"${scriptName}" || die "Failure running PVSplit script" $? +$PWD/"${scriptName}" || die "Failure running PVSplit script" $? # Dump to screen the content of the log file cat log*.out From db24ba3854ce41ee78b0fcf07a35cc5f7b265583 Mon Sep 17 00:00:00 2001 From: mmusich Date: Sun, 17 Nov 2024 17:10:18 +0100 Subject: [PATCH 3/3] clearer printout of the outpus in test_unitSubmitPVsplit --- .../scripts/submitPVResolutionJobs.py | 4 +++- .../testingScripts/test_unitSubmitPVsplit.sh | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/Alignment/OfflineValidation/scripts/submitPVResolutionJobs.py b/Alignment/OfflineValidation/scripts/submitPVResolutionJobs.py index e5b8653d5c3f0..34e6565964bba 100755 --- a/Alignment/OfflineValidation/scripts/submitPVResolutionJobs.py +++ b/Alignment/OfflineValidation/scripts/submitPVResolutionJobs.py @@ -210,7 +210,9 @@ def batchScriptCERN(theCMSSW_BASE, cfgdir, runindex, eosdir, lumiToRun, key, con cd $LXBATCH_DIR cp -pr {CFGDIR}/PrimaryVertexResolution_{KEY}_{runindex}_cfg.py . cmsRun PrimaryVertexResolution_{KEY}_{runindex}_cfg.py TrackCollection={TRKS} GlobalTag={GT} lumi={LUMITORUN} {REC} {EXT} >& log_{KEY}_run{runindex}.out -ls -lh . +# Print the contents of the current directory using $PWD and echo +echo "Contents of the current directory ($PWD):" +echo "$(ls -lh "$PWD")" """.format(CMSSW_BASE_DIR=theCMSSW_BASE, CFGDIR=cfgdir, runindex=runindex, diff --git a/Alignment/OfflineValidation/test/testingScripts/test_unitSubmitPVsplit.sh b/Alignment/OfflineValidation/test/testingScripts/test_unitSubmitPVsplit.sh index aaced2fc5aac2..405a21cad34b1 100755 --- a/Alignment/OfflineValidation/test/testingScripts/test_unitSubmitPVsplit.sh +++ b/Alignment/OfflineValidation/test/testingScripts/test_unitSubmitPVsplit.sh @@ -30,5 +30,17 @@ cd "${testdir}/testExecution" || exit 1 # Execute the script and handle errors $PWD/"${scriptName}" || die "Failure running PVSplit script" $? -# Dump to screen the content of the log file -cat log*.out +# Dump to screen the content of the log file(s) with clear headers +log_files=(log*.out) +if [[ ${#log_files[@]} -gt 0 ]]; then + echo "Displaying content of log files:" + for log_file in "${log_files[@]}"; do + echo "========================================" + echo "Content of $log_file:" + echo "========================================" + cat "$log_file" + echo # Add an extra blank line for separation + done +else + echo "No log files found matching 'log*.out'." +fi