From 92520faef3942a5d0e63a6487c98d7906cced9d1 Mon Sep 17 00:00:00 2001 From: Ed Safford <62339196+EdwardSafford-NOAA@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:55:35 -0500 Subject: [PATCH] Avoid attempting to make plots which have an incomplete specification (#50) Avoid attempting to make any plots which have an incomplete specification in the top-level yaml plot file. This is in response to problems Andrew found in testing. Absent these changes such incomplete specifications cause a catastrophic failure of all remaining plots. With these changes incomplete specifications are skipped over and an identifying warning message is produced. Closes #49 --- ush/plotObsMon.py | 19 +++++++++--- ush/splitPlotYaml.py | 72 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/ush/plotObsMon.py b/ush/plotObsMon.py index d806837..ecfb612 100755 --- a/ush/plotObsMon.py +++ b/ush/plotObsMon.py @@ -9,11 +9,15 @@ from re import sub import yaml from om_data import OM_data +import datetime as dt from wxflow import parse_j2yaml, save_as_yaml from wxflow import add_to_datetime, to_timedelta, to_datetime from eva.eva_driver import eva from eva.utilities.logger import Logger +from datetime import timedelta + +# -------------------------------------------------------------------------------------------- def genYaml(input_yaml, output_yaml, config): @@ -181,7 +185,6 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval, for plot in inst.get('plot_list'): config = loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval, data_location, model, chan_dict) - plot_template = f"{config['PLOT_TEMPLATE']}.yaml" plot_yaml = f"{config['SENSOR']}_{config['SAT']}_{plot_template}" @@ -195,11 +198,17 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval, os.chdir(plot_dir) config['DATA'] = plot_dir - genYaml(plot_template, plot_yaml, config) + try: + genYaml(plot_template, plot_yaml, config) + + plotData = OM_data(data_location, config, plot_yaml, logger) + eva(plot_yaml) + os.remove(plot_yaml) - plotData = OM_data(data_location, config, plot_yaml, logger) - eva(plot_yaml) - os.remove(plot_yaml) + except Exception as e: + logger.info(f'Warning: unable to run genYaml() with plot_yaml file ' + + f'{plot_yaml} error: {e}') + continue if 'minimization' in mon_dict.keys(): satname = None diff --git a/ush/splitPlotYaml.py b/ush/splitPlotYaml.py index 67fe847..e507259 100644 --- a/ush/splitPlotYaml.py +++ b/ush/splitPlotYaml.py @@ -29,6 +29,61 @@ def removeKey(d, keys): return r +def check_plotlist(type, logger, plotlist, sat=None, instr=None, obstype=None): + """ + Parse plotlist for missing required elements. + + Parameters: + type (str): type of plot + logger (Logger): logger for output warnings + plotlist (list): list of requested plot specifications + sat (str): satellite name (default None) + instr (str); instrument name (default None) + obstype (str): observation type namea (default None) + + Return: + True if no required elements are missing from plotlist, else False + """ + + sat_reqs = {'rad': ['plot', 'times', 'channels', 'component', 'run'], + 'ozn': ['plot', 'times', 'levels', 'component', 'run']} + min_reqs = ['plot', 'times', 'run'] + obs_reqs = ['plot', 'times', 'datatypes', 'run'] + + rtn_val = True + missing = [] + match type: + case 'sat': + if 'rad' in plotlist.get('plot'): + reqs = sat_reqs.get('rad') + else: + reqs = sat_reqs.get('ozn') + + for val in reqs: + if val not in plotlist: + missing.append(val) + plot_info = (f" {instr} {sat} {plotlist.get('plot')}") + + case 'min': + for val in min_reqs: + if val not in plotlist: + missing.append(val) + plot_info = (f" {plotlist.get('plot')}") + + case 'obs': + for val in obs_reqs: + if val not in plotlist: + missing.append(val) + plot_info = (f" {obstype}, {plotlist.get('plot')}") + + if len(missing) > 0: + logger.info(f'WARNING: YAML for plot {plot_info} is missing {missing}') + logger.info(f'WARNING: Requested plot will be SKIPPED.') + rtn_val = False + + return (rtn_val) + + if __name__ == "__main__": """ splitPlotYaml @@ -79,6 +134,9 @@ def removeKey(d, keys): for inst in sat.get('instruments'): iname = inst.get('name') plist = inst.get('plot_list') + for pl in inst.get('plot_list'): + if not check_plotlist('sat', logger, pl, sat=satname, instr=iname): + continue # -------------------------------------------------------------------- # For instruments with a large number of channels split the plot_list @@ -114,12 +172,26 @@ def removeKey(d, keys): if 'minimization' in mon_dict.keys(): md = removeKey(mon_dict, ['satellites', 'observations']) fname = f'OM_PLOT_minimization.yaml' + mm = md.get('minimization') + + for pl in mm[0]['plot_list']: + if not check_plotlist('min', logger, pl): + mm[0]['plot_list'].remove(pl) + file = open(fname, "w") yaml.dump(md, file) file.close() if 'observations' in mon_dict.keys(): od = removeKey(mon_dict, ['satellites', 'minimization']) + obs = od.get('observations') + + for ob in obs: + obstype = ob.get('obstype') + for pl in ob.get('plot_list'): + if not check_plotlist('obs', logger, pl, obstype=obstype): + ob.get('plot_list').remove(pl) + fname = f'OM_PLOT_observations.yaml' file = open(fname, "w") yaml.dump(od, file)