From 4a414a4459456662584b7db06ff959d54dad4008 Mon Sep 17 00:00:00 2001 From: Ed Safford <62339196+EdwardSafford-NOAA@users.noreply.github.com> Date: Wed, 26 Jun 2024 16:28:10 -0400 Subject: [PATCH] Port obs-monitor to wcoss2 (#34) This release allows obs-monitor to run on wcoss2. The strategy is to split the model level yaml plot file into lots of pieces and submitted as a single serial job with one node per piece (plot job). Satellite plots (rad and ozn) are broken down by instrument, for iasi and cris down to instrument/plot_type. The minimization and conventional plots are submitted as two plot jobs without breaking them down further. There are some caveats: - Surface pressure plots are not working on either wcoss2 or hera. This is a problem in eva (`mon_data_space.py`), and issue [#194](https://github.com/JCSDA-internal/eva/issues/194) has been opened to address it. - `Logos `and `tight layout` options have been temporarily commented out of the yaml plot tempates. These features fail on wcoss2 and seem to be issues in emcpy. Issue [#142](https://github.com/NOAA-EMC/emcpy/issues/142) has been opened to address the problem. - Some of the plots in `parm/gfs/gfs_plots.yaml` have been commented out owing to what seems to be file corruption issues in my test data suite (probably hera stmp space issues). Testing has been conducted on wcoss2 and hera, and all plots work on both machines within the stated caveats. --- driver/runObsMon.sh | 14 ++-- jobs/JMON_PLOT_OBS | 4 +- parm/OM_config | 4 +- parm/gfs/conTime.yaml | 20 +++--- parm/gfs/conVert.yaml | 10 +-- parm/gfs/gfs_plots.yaml | 4 +- parm/gfs/minGnormFourCycle.yaml | 8 +-- parm/gfs/minGnormOneCycle.yaml | 16 ++--- parm/gfs/minSummary.yaml | 10 +-- parm/gfs/oznSummary.yaml | 10 +-- parm/gfs/oznTime.yaml | 16 ++--- parm/gfs/radAngle.yaml | 12 ++-- parm/gfs/radBcoef.yaml | 8 +-- parm/gfs/radTime.yaml | 26 ++++---- scripts/exobsmon_plot.sh | 110 ++++++++++++-------------------- ush/plotObsMon.py | 6 +- ush/setup_wcoss2.sh | 18 ++++++ ush/splitPlotYaml.py | 11 ++-- 18 files changed, 151 insertions(+), 156 deletions(-) mode change 100644 => 100755 jobs/JMON_PLOT_OBS create mode 100644 ush/setup_wcoss2.sh diff --git a/driver/runObsMon.sh b/driver/runObsMon.sh index 4267482..3e349ef 100755 --- a/driver/runObsMon.sh +++ b/driver/runObsMon.sh @@ -86,13 +86,13 @@ export COMOUT=${COMOUT}/${NET} #------------------------- # Set up & submit j-job # -jobname="PlotObsMon" +jobname="PlotObsMon_setup" jobfile="${JOBSobsmon}/JMON_PLOT_OBS" logdir="${OM_LOGS}/${MODEL}" if [[ ! -d ${logdir} ]]; then mkdir -p ${logdir}; fi -logfile="${OM_LOGS}/${MODEL}/OM_log" +logfile="${logdir}/OM_setup.log" if [[ -e ${logfile} ]]; then rm ${logfile}; fi case ${MACHINE_ID} in @@ -101,9 +101,13 @@ case ${MACHINE_ID} in -J ${jobname} --partition service -o ${logfile} ${jobfile} ;; - wcoss2) # NOTE: this has not been tested; eva doesn't yet run on wcoss2 - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${logfile} \ - -V -l select=1:mem=500M -l walltime=0:05:00 -N ${jobname} ${jobfile} + wcoss2) + $SUB -q ${JOB_QUEUE} -A ${ACCOUNT} -o ${logfile} -e ${logfile} \ + -v "PYTHONPATH=${PYTHONPATH}, PATH=${PATH}, HOMEobsmon=${HOMEobsmon}, COMOUT=${COMOUT}, \ + MODEL=${MODEL}, PDY=${PDY}, cyc=${cyc}, DATAROOT=${DATAROOT}, APRUN_PY=${APRUN_PY}, \ + MACHINE_ID=${MACHINE_ID}, ACCOUNT=${ACCOUNT}, JOB_QUEUE=${JOB_QUEUE}, SUB=${SUB}, + OM_LOGS=${OM_LOGS}" \ + -l select=1:mem=500mb -l walltime=0:05:00 -N ${jobname} ${jobfile} ;; esac diff --git a/jobs/JMON_PLOT_OBS b/jobs/JMON_PLOT_OBS old mode 100644 new mode 100755 index 84f2c6f..d2450d2 --- a/jobs/JMON_PLOT_OBS +++ b/jobs/JMON_PLOT_OBS @@ -26,7 +26,7 @@ mkdir -m 775 -p $COMOUTplots # Create $DATA for temp workspace ################################### export DATA=${DATA:-${DATAROOT}/${NET}/${MODEL}/Plot} -if [[ -d ${DATA} ]]; then rm -rf ${DATA}; fi +if [[ -d ${DATAROOT}/${NET} ]]; then rm -rf ${DATAROOT}/${NET}; fi mkdir -p ${DATA} cd ${DATA} @@ -51,5 +51,3 @@ fi KEEPDATA=${KEEPDATA:-"NO"} cd ${DATAROOT} [[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} - -report-mem diff --git a/parm/OM_config b/parm/OM_config index 7c891dd..7cdc7a0 100644 --- a/parm/OM_config +++ b/parm/OM_config @@ -7,7 +7,7 @@ export JOBSobsmon=${om_dir_root}/jobs # Export job submission vars per MACHINE_ID # source ${om_dir_root}/ush/detect_machine.sh -MACHINE_ID=`echo "${MACHINE_ID%%.*}"` +export MACHINE_ID=`echo "${MACHINE_ID%%.*}"` case ${MACHINE_ID} in @@ -31,7 +31,7 @@ case ${MACHINE_ID} in queue="dev" project="GDAS-DEV" account="GFS-DEV" - aprun_py="python" + aprun_py=`which python` ;; esac diff --git a/parm/gfs/conTime.yaml b/parm/gfs/conTime.yaml index 736543c..aa367ab 100644 --- a/parm/gfs/conTime.yaml +++ b/parm/gfs/conTime.yaml @@ -460,13 +460,13 @@ graphics: figure: layout: [4,1] figure size: [20,18] - tight layout: +# tight layout: title: "Valid: {{ PDATE | to_YMDH }} \n ${datatype}, Global, All Levels" output name: line_plots/con/time/${datatype}_count.{{ PDATE | to_YMDH }}.png - plot logo: - which: 'noaa/nws' - loc: 'upper left' - subplot_orientation: 'last' +# plot logo: +# which: 'noaa/nws' +# loc: 'upper left' +# subplot_orientation: 'last' plots: - add_xlabel: 'Assimilated' @@ -584,13 +584,13 @@ graphics: figure: layout: [6,1] figure size: [20,18] - tight layout: +# tight layout: title: "Valid: {{ PDATE | to_YMDH }}\n ${datatype}, Global, All Levels" output name: line_plots/con/time/${datatype}_bias.{{ PDATE | to_YMDH }}.png - plot logo: - which: 'noaa/nws' - loc: 'upper left' - subplot_orientation: 'last' +# plot logo: +# which: 'noaa/nws' +# loc: 'upper left' +# subplot_orientation: 'last' plots: - add_xlabel: 'Obs-Ges Assimilated' diff --git a/parm/gfs/conVert.yaml b/parm/gfs/conVert.yaml index 0a18486..3c9c737 100644 --- a/parm/gfs/conVert.yaml +++ b/parm/gfs/conVert.yaml @@ -67,13 +67,13 @@ graphics: figure: layout: [2,2] figure size: [20,18] - tight layout: +# tight layout: title: "Valid: {{ PDATE | to_YMDH }} \n ${datatype}, Global" output name: line_plots/con/vert/${datatype}.vert.png - plot logo: - which: 'noaa/nws' - loc: 'upper left' - subplot_orientation: 'first' +# plot logo: +# which: 'noaa/nws' +# loc: 'upper left' +# subplot_orientation: 'first' plots: - add_xlabel: 'Assimilated' diff --git a/parm/gfs/gfs_plots.yaml b/parm/gfs/gfs_plots.yaml index c79f7dc..9282523 100644 --- a/parm/gfs/gfs_plots.yaml +++ b/parm/gfs/gfs_plots.yaml @@ -174,7 +174,7 @@ satellites: times: 121 channels: 'all' component: ges -# file corruption here? +# test data file corruption here? # - plot: rad angle # times: 121 # channels: 'all' @@ -321,7 +321,7 @@ satellites: channels: 'all' component: ges -# file corruption? +# test data file corruption here # - name: n19 # instruments: # - name: amsua diff --git a/parm/gfs/minGnormFourCycle.yaml b/parm/gfs/minGnormFourCycle.yaml index 0084bdc..a1c0e91 100644 --- a/parm/gfs/minGnormFourCycle.yaml +++ b/parm/gfs/minGnormFourCycle.yaml @@ -94,12 +94,12 @@ graphics: - figure: layout: [1,1] figure size: [20,18] - tight layout: +# tight layout: title: "Valid: {{ PDATE | to_YMDH }}" output name: line_plots/min/{{MODEL}}_{{RUN}}.4cycle.gnorms.png - plot logo: - which: 'noaa/nws' - loc: 'upper center' + # plot logo: + # which: 'noaa/nws' + # loc: 'upper center' plots: - add_xlabel: 'Iteration Number' diff --git a/parm/gfs/minGnormOneCycle.yaml b/parm/gfs/minGnormOneCycle.yaml index 25e9591..5b9583f 100644 --- a/parm/gfs/minGnormOneCycle.yaml +++ b/parm/gfs/minGnormOneCycle.yaml @@ -68,12 +68,12 @@ graphics: - figure: layout: [1,1] figure size: [20,18] - tight layout: +# tight layout: title: "Valid: {{ PDATE | to_YMDH }}" output name: line_plots/min/{{MODEL}}_{{RUN}}.{{ PDATE | to_YMDH }}.gnorms.png - plot logo: - which: 'noaa/nws' - loc: 'upper right' + # plot logo: + # which: 'noaa/nws' + # loc: 'upper right' plots: - add_xlabel: 'Iteration Number' @@ -106,12 +106,12 @@ graphics: - figure: layout: [1,1] figure size: [20,18] - tight layout: +# tight layout: title: "Valid: {{ PDATE | to_YMDH }}" output name: line_plots/min/{{MODEL}}_{{RUN}}.{{ PDATE | to_YMDH }}.reduction.png - plot logo: - which: 'noaa/nws' - loc: 'upper right' +# plot logo: +# which: 'noaa/nws' +# loc: 'upper right' plots: - add_xlabel: 'Iteration Number' diff --git a/parm/gfs/minSummary.yaml b/parm/gfs/minSummary.yaml index b698f64..bacfec4 100644 --- a/parm/gfs/minSummary.yaml +++ b/parm/gfs/minSummary.yaml @@ -37,13 +37,13 @@ graphics: - figure: layout: [3,1] figure size: [20,18] - tight layout: + # tight layout: title: "Valid: {{ PDATE | to_YMDH }}" output name: line_plots/min/{{MODEL}}_{{RUN}}.summary.gnorms.png - plot logo: - which: 'noaa/nws' - loc: 'upper left' - subplot_orientation: 'first' + # plot logo: + #which: 'noaa/nws' + #loc: 'upper left' + #subplot_orientation: 'first' plots: - add_xlabel: 'Cycle Time' diff --git a/parm/gfs/oznSummary.yaml b/parm/gfs/oznSummary.yaml index 7729b76..72de5f4 100644 --- a/parm/gfs/oznSummary.yaml +++ b/parm/gfs/oznSummary.yaml @@ -295,13 +295,13 @@ graphics: - figure: layout: [3,1] figure size: [20,18] - tight layout: +# tight layout: title: "Summary Plot {{SENSOR}}_{{SAT}} \n Valid: {{ PDATE | to_YMDH }}" output name: line_plots/ozn/summary/{{SENSOR}}_{{SAT}}.summary.png - plot logo: - which: 'noaa/nws' - loc: 'upper left' - subplot_orientation: 'first' +# plot logo: +# which: 'noaa/nws' +# loc: 'upper left' +# subplot_orientation: 'first' plots: # Obs counts, last 4 cycles # ------------------------- diff --git a/parm/gfs/oznTime.yaml b/parm/gfs/oznTime.yaml index 49660b3..386e6d6 100644 --- a/parm/gfs/oznTime.yaml +++ b/parm/gfs/oznTime.yaml @@ -164,12 +164,12 @@ graphics: figure: layout: [1,1] figure size: [20,18] - tight layout: +# tight layout: title: "${variable}, {{SENSOR}}_{{SAT}} level ${level} \n Valid: {{ PDATE | to_YMDH }}" output name: line_plots/ozn/time/{{SENSOR}}_{{SAT}}.${level}.${variable}.png - plot logo: - which: 'noaa/nws' - loc: 'upper right' +# plot logo: +# which: 'noaa/nws' +# loc: 'upper right' plots: - add_xlabel: 'Cycle' @@ -198,12 +198,12 @@ graphics: figure: layout: [2,1] figure size: [20,18] - tight layout: +# tight layout: title: "obs-{{COMPONENT}} {{SENSOR}}_{{SAT}} level ${level} \n Valid: {{ PDATE | to_YMDH }}" output name: line_plots/ozn/time/{{SENSOR}}_{{SAT}}.${level}.omg.png - plot logo: - which: 'noaa/nws' - loc: 'upper right' +# plot logo: +# which: 'noaa/nws' +# loc: 'upper right' plots: - add_xlabel: 'Cycle' diff --git a/parm/gfs/radAngle.yaml b/parm/gfs/radAngle.yaml index cab545c..77f5d61 100644 --- a/parm/gfs/radAngle.yaml +++ b/parm/gfs/radAngle.yaml @@ -255,9 +255,9 @@ graphics: figure size: [20,18] title: "${variable}, {{SENSOR}}_{{SAT}} channel ${channel} \n Valid: {{ PDATE | to_YMDH }}" output name: line_plots/rad/angle/angle.{{SENSOR}}_{{SAT}}.${channel}.${variable}.png - plot logo: - which: 'noaa/nws' - loc: 'upper left' +# plot logo: +# which: 'noaa/nws' +# loc: 'upper left' plots: @@ -317,9 +317,9 @@ graphics: figure size: [20,18] title: "${variable}, {{SENSOR}}_{{SAT}} channel ${channel} \n Valid: {{ PDATE | to_YMDH }}" output name: line_plots/rad/angle/angle.{{SENSOR}}_{{SAT}}.${channel}.${variable}.png - plot logo: - which: 'noaa/nws' - loc: 'upper left' +# plot logo: +# which: 'noaa/nws' +# loc: 'upper left' plots: - add_xlabel: 'Scan Position' diff --git a/parm/gfs/radBcoef.yaml b/parm/gfs/radBcoef.yaml index 8ea5cd2..3c654e4 100644 --- a/parm/gfs/radBcoef.yaml +++ b/parm/gfs/radBcoef.yaml @@ -177,12 +177,12 @@ graphics: figure: layout: [1,1] figure size: [20,18] - tight layout: +# tight layout: title: "${variable}, {{SENSOR}}_{{SAT}} channel ${channel} \n Valid: {{ PDATE | to_YMDH }}" output name: line_plots/rad/bcoef/bcoef.{{SENSOR}}_{{SAT}}.${channel}.${variable}.png - plot logo: - which: 'noaa/nws' - loc: 'upper right' +# plot logo: +# which: 'noaa/nws' +# loc: 'upper right' plots: - add_xlabel: 'Cycle' diff --git a/parm/gfs/radTime.yaml b/parm/gfs/radTime.yaml index 39b8b6a..07a5d14 100644 --- a/parm/gfs/radTime.yaml +++ b/parm/gfs/radTime.yaml @@ -328,12 +328,12 @@ graphics: figure: layout: [1,1] figure size: [20,18] - tight layout: +# tight layout: title: "${variable}, {{SENSOR}}_{{SAT}} channel ${channel} \n Valid: {{ PDATE | to_YMDH }}" output name: line_plots/rad/time/time.{{SENSOR}}_{{SAT}}.${channel}.${variable}.png - plot logo: - which: 'noaa/nws' - loc: 'upper right' +# plot logo: +# which: 'noaa/nws' +# loc: 'upper right' plots: - add_xlabel: 'Cycle' @@ -356,12 +356,12 @@ graphics: figure: layout: [1,1] figure size: [20,18] - tight layout: +# tight layout: title: "${variable}, {{SENSOR}}_{{SAT}} channel ${channel} \n Valid: {{ PDATE | to_YMDH }}" output name: line_plots/rad/time/time.{{SENSOR}}_{{SAT}}.${channel}.${variable}.png - plot logo: - which: 'noaa/nws' - loc: 'upper right' +# plot logo: +# which: 'noaa/nws' +# loc: 'upper right' plots: - add_xlabel: 'Cycle' @@ -378,13 +378,13 @@ graphics: - figure: layout: [4,1] figure size: [20,18] - tight layout: +# tight layout: title: "Summary Plot {{SENSOR}}_{{SAT}} \n Valid: {{ PDATE | to_YMDH }}" output name: line_plots/rad/summary/summary.{{SENSOR}}_{{SAT}}.png - plot logo: - which: 'noaa/nws' - loc: 'upper center' - subplot_orientation: 'first' +# plot logo: +# which: 'noaa/nws' +# loc: 'upper center' +# subplot_orientation: 'first' plots: # Obs counts, last 4 cycles # ------------------------- diff --git a/scripts/exobsmon_plot.sh b/scripts/exobsmon_plot.sh index 078974a..41c5ae1 100755 --- a/scripts/exobsmon_plot.sh +++ b/scripts/exobsmon_plot.sh @@ -7,8 +7,8 @@ #------------------------------------------------------------- # locate $model_plots.yaml and instrument_channels.yaml files #------------------------------------------------------------- -plot_yaml=${plot_yaml:-${PARMobsmon}/${MODEL}/${MODEL}_plots.yaml} +plot_yaml=${plot_yaml:-${PARMobsmon}/${MODEL}/${MODEL}_plots.yaml} if [[ ! -e ${plot_yaml} ]]; then echo "ERROR: yaml plot file ${plot_yaml} NOT FOUND" exit 1 @@ -20,87 +20,61 @@ if [[ ! -e ${chan_yaml} ]]; then exit 2 fi -#----------------------------------------------------------- -# split $plot_yaml into sat/instr[/plot], minimization, obs +#--------------------------------------------------------------- +# split $plot_yaml into sat/instr[/plot], minimization, and obs +# in order to reduce the plot jobs to a more managable size # ${APRUN_PY} ${USHobsmon}/splitPlotYaml.py -i ${plot_yaml} -c ${chan_yaml} - #-------------------------------------------------------------- -# Submit OM_sat_plots job if split yields any sat_*.yaml files -# -if compgen -G "${DATA}/sat_*.yaml" > /dev/null; then - - jobname="OM_sat_plots" - logfile="${OM_LOGS}/${MODEL}/OM_sat_plot.log" - if [[ -e ${logfile} ]]; then rm ${logfile}; fi - - cmdfile="OM_sat_jobscript" - >$cmdfile - ctr=0 - - for yaml in ${DATA}/sat_*.yaml; do - echo "${ctr} $yaml" - echo "${ctr} ${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${yaml} -p ${PDATE}" >> $cmdfile - ((ctr+=1)) - done - chmod 755 $cmdfile - - echo "ctr: $ctr" - echo "submitting job ${jobname}" - - if [[ ${ctr} > 0 ]]; then - $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ - --mem=80000M --wrap "srun -l --multi-prog ${cmdfile}" - fi - -fi - -#------------------------------------------------------------------ -# Submit OM_min_plots job if split yields a minimization.yaml file +# Submit OM_plots job if split yields any *.yaml files # -if compgen -G "${DATA}/minimization.yaml" > /dev/null; then +if compgen -G "${DATA}/OM_PLOT*.yaml" > /dev/null; then - jobname="OM_min_plots" - logfile="${OM_LOGS}/${MODEL}/OM_min_plot.log" + jobname="OM_plots" + export logfile="${OM_LOGS}/${MODEL}/OM_plot.log" if [[ -e ${logfile} ]]; then rm ${logfile}; fi - cmdfile="OM_min_jobscript" - echo "0 ${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${DATA}/minimization.yaml -p ${PDATE}" > $cmdfile - chmod 755 $cmdfile - - echo "submitting job ${jobname}" + cmdfile="OM_jobscript" + >${cmdfile} - $SUB --account ${ACCOUNT} -n 1 -o ${logfile} -D . -J ${jobname} --time=0:05:00 \ - --mem=80000M --wrap "srun -l --multi-prog ${cmdfile}" -fi - -#------------------------------------------------------------------ -# Submit OM_con_plots job if split yields any obs_*.yaml files -# -if compgen -G "${DATA}/obs*.yaml" > /dev/null; then - echo "have OBS plots" - jobname="OM_obs_plots" - logfile="${OM_LOGS}/${MODEL}/OM_obs_plot.log" - if [[ -e ${logfile} ]]; then rm ${logfile}; fi - - cmdfile="OM_obs_jobscript" - >$cmdfile ctr=0 - - for yaml in ${DATA}/obs*.yaml; do - echo "${ctr} $yaml" - echo "${ctr} ${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${yaml} -p ${PDATE}" >> $cmdfile + for yaml in ${DATA}/OM_PLOT*.yaml; do + echo "processing yaml: $ctr $yaml" + case ${MACHINE_ID} in + hera) + echo "${ctr} ${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${yaml} -p ${PDATE}" >> ${cmdfile} + ;; + wcoss2) + echo "${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${yaml} -p ${PDATE}" >> ${cmdfile} + ;; + esac ((ctr+=1)) done - chmod 755 $cmdfile - echo "ctr: $ctr" - echo "submitting job ${jobname}" - if [[ ${ctr} > 0 ]]; then - $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ - --mem=80000M --wrap "srun -l --multi-prog ${cmdfile}" + if (( ${ctr} > 0 )); then + case ${MACHINE_ID} in + hera) + ${SUB} --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ + --mem=80000M --wrap "srun -l --multi-prog ${cmdfile}" + ;; + + wcoss2) + # prepend setup script to $cmdfile + echo -e ". ${USHobsmon}/setup_wcoss2.sh\n $(cat ${cmdfile})" > ${cmdfile} + chmod 775 ${cmdfile} + + mem=$((4*${ctr})) + echo "submitting ${jobname} on wcoss2, ctr = $ctr, mem = $mem, cmdfile = ${cmdfile}" + + ${SUB} -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${logfile} \ + -v "PYTHONPATH=${PYTHONPATH}, PATH=${PATH}, HOMEobsmon=${HOMEobsmon}, MODEL=${MODEL}, \ + CNTRLobsmon=${CNTRLobsmon}, PARMobsmon=${PARMobsmon}, DATA=${DATA}, \ + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}, cmdfile=${cmdfile}, ncpus=${ctr}" \ + -l place=vscatter,select=1:ncpus=${ctr}:mem=${mem}gb,walltime=1:30:00 -N ${jobname} ${cmdfile} + ;; + esac fi fi diff --git a/ush/plotObsMon.py b/ush/plotObsMon.py index 5b2d166..4c0ef64 100755 --- a/ush/plotObsMon.py +++ b/ush/plotObsMon.py @@ -86,7 +86,7 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval, times = int(plot.get('times')) if plot.get('times') else None if times is not None: for x in range(1, times+1): - date_str = f"PDATEm{x*cycle_interval}" + date_str = f'PDATEm{x*cycle_interval}' config[date_str] = add_to_datetime(cycle_tm, to_timedelta(f"-{cycle_interval*x}H")) if config['CHANNELS'] is not None: @@ -138,6 +138,9 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval, args = parser.parse_args() cycle_tm = to_datetime(args.pdate) + data = os.environ.get('DATA', '.') + os.chdir(data) + try: mon_sources = args.input with open(mon_sources, 'r') as mon_sources_opened: @@ -185,7 +188,6 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval, plot_template = os.path.join(parm_location, plot_template) genYaml(plot_template, plot_yaml, config) - eva(plot_yaml) os.remove(plot_yaml) diff --git a/ush/setup_wcoss2.sh b/ush/setup_wcoss2.sh new file mode 100644 index 0000000..6becb3e --- /dev/null +++ b/ush/setup_wcoss2.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +module reset + +module load libfabric/1.11.0.0. +module load PrgEnv-intel/8.3.3 +module load craype +module load cray-pals +module load git/2.29.0 +module load intel/19.1.3.304 +module load python/3.10.4 +module load ve/evs/1.0 + +export PATH=${PATH}:/lfs/h2/emc/da/noscrub/edward.safford/eva/opt/bin +export PYTHONPATH=${PYTHONPATH}:/lfs/h2/emc/da/noscrub/edward.safford/eva/opt/ + +if [ ! -z ${PBS_O_WORKDIR} ]; then cd ${PBS_O_WORKDIR}; fi + diff --git a/ush/splitPlotYaml.py b/ush/splitPlotYaml.py index 8d4ccb9..7ea6322 100644 --- a/ush/splitPlotYaml.py +++ b/ush/splitPlotYaml.py @@ -48,8 +48,8 @@ def removeKey(d, keys): parser.add_argument('-i', '--input', type=str, help='Input YAML plot file', required=True) parser.add_argument('-c', '--chan', type=str, help='Input YAML instrument channel file', required=True) - args = parser.parse_args() + try: mon_sources = args.input with open(mon_sources, 'r') as mon_sources_opened: @@ -63,7 +63,7 @@ def removeKey(d, keys): with open(chan_data, 'r') as chan_data_opened: chan_dict = yaml.safe_load(chan_data_opened) except Exception as e: - logger.abort('plotObsMon is expecting a valid satellite channel file, but it encountered ' + + logger.abort('splitPlotYaml is expecting a valid satellite channel file, but encountered ' + f'errors when attempting to load: {chan_data}, error: {e}') model = mon_dict.get('model') @@ -106,22 +106,21 @@ def removeKey(d, keys): pd['satellites'] = [{'name': satname, 'instruments': [{'name': iname, 'plot_list': plist}]}] - fname = f'sat_{satname}_{iname}.yaml' + fname = f'OM_PLOT_sat_{satname}_{iname}.yaml' file = open(fname, "w") yaml.dump(pd, file) file.close() if 'minimization' in mon_dict.keys(): md = removeKey(mon_dict, ['satellites', 'observations']) - fname = f'minimization.yaml' + fname = f'OM_PLOT_minimization.yaml' file = open(fname, "w") yaml.dump(md, file) file.close() if 'observations' in mon_dict.keys(): - logger.info(f'HAVE OBS') od = removeKey(mon_dict, ['satellites', 'minimization']) - fname = f'observations.yaml' + fname = f'OM_PLOT_observations.yaml' file = open(fname, "w") yaml.dump(od, file) file.close()