From dafc0de91513640ecf708088bf54672a093bfb50 Mon Sep 17 00:00:00 2001 From: Ed Safford <62339196+EdwardSafford-NOAA@users.noreply.github.com> Date: Tue, 26 Dec 2023 14:45:56 -0500 Subject: [PATCH] Add MinMon templates (#12) Modify `plotObsMon.py `to support minmon plots and add templates to produce: - minmon summary plot - minmon 4 cycle gnorm plot - minmon single cycle gnorm plot Please note this PR includes the same changes as #11 . I messed up and started from it rather than from develop. So I'll make this a draft and convert it to release if/when #11 is merged. Resolves #8. --- parm/gfs/gfs_plots.yaml | 14 +++ parm/gfs/minGnormFourCycle.yaml | 149 ++++++++++++++++++++++++++++++++ parm/gfs/minGnormOneCycle.yaml | 142 ++++++++++++++++++++++++++++++ parm/gfs/minSummary.yaml | 110 +++++++++++++++++++++++ ush/plotObsMon.py | 57 ++++++++---- 5 files changed, 454 insertions(+), 18 deletions(-) create mode 100644 parm/gfs/minGnormFourCycle.yaml create mode 100644 parm/gfs/minGnormOneCycle.yaml create mode 100644 parm/gfs/minSummary.yaml diff --git a/parm/gfs/gfs_plots.yaml b/parm/gfs/gfs_plots.yaml index 83117da..6dceccf 100644 --- a/parm/gfs/gfs_plots.yaml +++ b/parm/gfs/gfs_plots.yaml @@ -50,3 +50,17 @@ satellites: times: 121 channels: '12,13,14,15,16' run: ges + +minimization: + - net: gfs + plot_list: + - plot: min summary + run: gdas + + - plot: min gnorm four cycle + run: gdas + times: 28 + + - plot: min gnorm one cycle + run: gdas + times: 28 diff --git a/parm/gfs/minGnormFourCycle.yaml b/parm/gfs/minGnormFourCycle.yaml new file mode 100644 index 0000000..8a3458e --- /dev/null +++ b/parm/gfs/minGnormFourCycle.yaml @@ -0,0 +1,149 @@ +# +# Gnorm 4 cycle plots for MinMon gfs +# +# Generate one plot: +# - gnorm values (all iterations) for last 4 cycles & 7 day avg + + +# Data read +# --------- +datasets: + - name: gnorm + type: MonDataSpace + control_file: + - {{DATA}}/min_data/glb_allgnorm.ctl + filenames: + - {{DATA}}/min_data/{{ PDATE | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm6 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm12 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm18 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm24 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm30 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm36 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm42 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm48 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm54 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm60 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm66 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm72 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm78 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm84 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm90 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm96 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm102 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm108 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm114 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm120 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm126 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm132 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm138 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm144 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm150 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm156 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm162 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm168 | to_YMDH }}.gnorms.ieee_d + + groups: + - name: GsiIeee + variables: &variables ['gnorm'] + +transforms: + - transform: select time + new name: gnorm::GsiIeee::log_gnorm_cyc1 + starting field: gnorm::GsiIeee::log_gnorm + cycle: {{ PDATE | to_YMDH }} + for: + variable: [none] + + - transform: select time + new name: gnorm::GsiIeee::log_gnorm_cyc2 + starting field: gnorm::GsiIeee::log_gnorm + cycle: {{ PDATEm6 | to_YMDH }} + for: + variable: [none] + + - transform: select time + new name: gnorm::GsiIeee::log_gnorm_cyc3 + starting field: gnorm::GsiIeee::log_gnorm + cycle: {{ PDATEm12 | to_YMDH }} + for: + variable: [none] + + - transform: select time + new name: gnorm::GsiIeee::log_gnorm_cyc4 + starting field: gnorm::GsiIeee::log_gnorm + cycle: {{ PDATEm18 | to_YMDH }} + for: + variable: [none] + + - transform: select time + new name: gnorm::GsiIeee::log_gnorm_7d + starting field: gnorm::GsiIeee::log_gnorm + start cycle: {{ PDATE | to_YMDH }} + end cycle: {{ PDATEm162 | to_YMDH }} + for: + variable: [none] + +graphics: + + plotting_backend: Emcpy + figure_list: + + # Gnorm single cycle & 7 day hourly mean + # -------------------------------------- + - figure: + layout: [1,1] + figure size: [20,18] + tight layout: + title: "Valid: {{ PDATE | to_YMDH }}" + output name: line_plots/minimization/{{NET}}_{{RUN}}.4cycle.gnorms.png + plot logo: + which: 'noaa/nws' + loc: 'upper center' + + plots: + - add_xlabel: 'Iteration Number' + add_ylabel: 'log( gnorm )' + add_grid: + axis: 'both' + linestyle: 'dotted' + linewidth: 0.5 + color: 'black' + add_legend: + loc: 'upper right' + layers: + - type: LinePlot + x: + variable: gnorm::GsiIeee::iteration + y: + variable: gnorm::GsiIeee::log_gnorm_cyc1 + color: 'blue' + label: '{{ PDATE | to_YMDH }}' + - type: LinePlot + x: + variable: gnorm::GsiIeee::iteration + y: + variable: gnorm::GsiIeee::log_gnorm_cyc2 + color: 'red' + label: '{{ PDATEm6 | to_YMDH }}' + - type: LinePlot + x: + variable: gnorm::GsiIeee::iteration + y: + variable: gnorm::GsiIeee::log_gnorm_cyc3 + color: 'green' + label: '{{ PDATEm12 | to_YMDH }}' + - type: LinePlot + x: + variable: gnorm::GsiIeee::iteration + y: + variable: gnorm::GsiIeee::log_gnorm_cyc4 + color: 'orange' + label: '{{ PDATEm18 | to_YMDH }}' + - type: LinePlot + x: + variable: gnorm::GsiIeee::iteration + y: + variable: gnorm::GsiIeee::log_gnorm_7d + color: 'black' + label: '7 day mean' diff --git a/parm/gfs/minGnormOneCycle.yaml b/parm/gfs/minGnormOneCycle.yaml new file mode 100644 index 0000000..60e8b02 --- /dev/null +++ b/parm/gfs/minGnormOneCycle.yaml @@ -0,0 +1,142 @@ +# +# Gnorm plots for MinMon gfs +# +# Generate two plots: +# - gnorm values (all iterations) for last 4 cycles & 7 day avg +# - single cycle gnorm values (all iterations) & 7 day avg + + +# Data read +# --------- +datasets: + - name: gnorm + type: MonDataSpace + control_file: + - {{DATA}}/min_data/glb_allgnorm.ctl + filenames: + - {{DATA}}/min_data/{{ PDATE | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm24 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm48 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm72 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm96 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm120 | to_YMDH }}.gnorms.ieee_d + - {{DATA}}/min_data/{{ PDATEm144 | to_YMDH }}.gnorms.ieee_d + + groups: + - name: GsiIeee + variables: &variables ['gnorm'] + +transforms: + - transform: select time + new name: gnorm::GsiIeee::log_gnorm_1cyc + starting field: gnorm::GsiIeee::log_gnorm + cycle: {{ PDATE | to_YMDH }} + for: + variable: [none] + + - transform: select time + new name: gnorm::GsiIeee::allgnorm_1cyc + starting field: gnorm::GsiIeee::allgnorm + cycle: {{ PDATE | to_YMDH }} + for: + variable: [none] + + - transform: select time + new name: gnorm::GsiIeee::log_gnorm_7d + starting field: gnorm::GsiIeee::log_gnorm + start cycle: {{ PDATE | to_YMDH }} + end cycle: {{ PDATEm144 | to_YMDH }} + for: + variable: [none] + + - transform: select time + new name: gnorm::GsiIeee::allgnorm_7d + starting field: gnorm::GsiIeee::allgnorm + start cycle: {{ PDATE | to_YMDH }} + end cycle: {{ PDATEm144 | to_YMDH }} + for: + variable: [none] + + +graphics: + + plotting_backend: Emcpy + figure_list: + + # Gnorm single cycle & 7 day hourly mean + # -------------------------------------- + - figure: + layout: [1,1] + figure size: [20,18] + tight layout: + title: "Valid: {{ PDATE | to_YMDH }}" + output name: line_plots/minimization/{{NET}}_{{RUN}}.{{ PDATE | to_YMDH }}.gnorms.png + plot logo: + which: 'noaa/nws' + loc: 'upper right' + + plots: + - add_xlabel: 'Iteration Number' + add_ylabel: 'log( gnorm )' + add_grid: + axis: 'both' + linestyle: 'dotted' + linewidth: 0.5 + color: 'black' + add_legend: + loc: 'center right' + layers: + - type: LinePlot + x: + variable: gnorm::GsiIeee::iteration + y: + variable: gnorm::GsiIeee::log_gnorm_1cyc + color: 'blue' + label: '{{ PDATE | to_YMDH }}' + - type: LinePlot + x: + variable: gnorm::GsiIeee::iteration + y: + variable: gnorm::GsiIeee::log_gnorm_7d + color: 'gray' + label: '7 day mean of 06z cycles' + + # Reduction single cycle & 7 day hourly mean + # ------------------------------------------ + - figure: + layout: [1,1] + figure size: [20,18] + tight layout: + title: "Valid: {{ PDATE | to_YMDH }}" + output name: line_plots/minimization/{{NET}}_{{RUN}}.{{ PDATE | to_YMDH }}.reduction.png + plot logo: + which: 'noaa/nws' + loc: 'upper right' + + plots: + - add_xlabel: 'Iteration Number' + add_ylabel: 'Reduction Value' + add_grid: + axis: 'both' + linestyle: 'dotted' + linewidth: 0.5 + color: 'black' + add_legend: + loc: 'center right' + + layers: + - type: LinePlot + x: + variable: gnorm::GsiIeee::iteration + y: + variable: gnorm::GsiIeee::allgnorm_1cyc + label: '{{ PDATE | to_YMDH }}' + color: 'blue' + - type: LinePlot + x: + variable: gnorm::GsiIeee::iteration + y: + variable: gnorm::GsiIeee::allgnorm_7d + color: 'gray' + label: '7 day mean of 06z cycles' # NEED {{HH}} value for 06 + diff --git a/parm/gfs/minSummary.yaml b/parm/gfs/minSummary.yaml new file mode 100644 index 0000000..122b014 --- /dev/null +++ b/parm/gfs/minSummary.yaml @@ -0,0 +1,110 @@ +# +# MinMon summary plots for gfs +# +# Generate three, 120 cycle (30 day) time series plots: +# - initial gradients +# - final gnorm values +# - min/max gnorm values for final 10 iterations + + +# Data read +# --------- +datasets: + - name: gnorm + type: CsvSpace + filenames: + - {{DATA}}/min_data/gnorm_data.txt + groups: + - name: GsiIeee + variables: + igrad: 4 + final_gnorm: 5 + min_gnorm: 6 + max_gnorm: 7 + date: + year: 0 + month: 1 + day: 2 + hour: 3 + +graphics: + + plotting_backend: Emcpy + figure_list: + + # Gnorm single cycle & 7 day hourly mean + # -------------------------------------- + - figure: + layout: [3,1] + figure size: [20,18] + tight layout: + title: "Valid: {{ PDATE | to_YMDH }}" + output name: line_plots/minimization/{{NET}}_{{RUN}}.summary.gnorms.png + plot logo: + which: 'noaa/nws' + loc: 'upper left' + subplot_orientation: 'first' + + plots: + - add_xlabel: 'Cycle Time' + add_ylabel: 'Initial Gradient' + add_grid: + axis: 'both' + linestyle: 'dotted' + linewidth: 0.5 + color: 'black' + add_legend: + loc: 'upper right' + layers: + - type: LinePlot + x: + variable: gnorm::GsiIeee::datetime + y: + variable: gnorm::GsiIeee::igrad + color: 'blue' + label: 'Initial Gradient' + + - add_xlabel: 'Cycle Time' + add_ylabel: 'Final Gnorm' + set_yscale: 'log' + add_grid: + axis: 'both' + linestyle: 'dotted' + linewidth: 0.5 + color: 'black' + add_legend: + loc: 'upper right' + layers: + - type: LinePlot + x: + variable: gnorm::GsiIeee::datetime + y: + variable: gnorm::GsiIeee::final_gnorm + color: 'blue' + label: 'Final Gnorm' + + - add_xlabel: 'Cycle Time' + add_ylabel: 'Min/Max Gnorm, final 10 iterations' + set_yscale: 'log' + add_grid: + axis: 'both' + linestyle: 'dotted' + linewidth: 0.5 + color: 'black' + add_legend: + loc: 'upper right' + layers: + - type: LinePlot + x: + variable: gnorm::GsiIeee::datetime + y: + variable: gnorm::GsiIeee::min_gnorm + color: 'blue' + label: 'Minimum Gnorm' + - type: LinePlot + x: + variable: gnorm::GsiIeee::datetime + y: + variable: gnorm::GsiIeee::max_gnorm + color: 'red' + label: 'Maximum Gnorm' diff --git a/ush/plotObsMon.py b/ush/plotObsMon.py index f21a787..7a8bca9 100755 --- a/ush/plotObsMon.py +++ b/ush/plotObsMon.py @@ -48,7 +48,7 @@ def camelCase(s): # -------------------------------------------------------------------------------------------- -def loadConfig(satname, instrument, plot, cycle_tm, cycle_interval, data_location): +def loadConfig(satname, instrument, plot, cycle_tm, cycle_interval, data_location, net=None): """ Load configuration dictionary. @@ -59,7 +59,7 @@ def loadConfig(satname, instrument, plot, cycle_tm, cycle_interval, data_locatio cycle_tm (datetime): Cycle time of plot cycle_interval (int): number of hours between cycles data_location (str): path to directory containing data files - + net (str): model run NET value Return: config(dict): Dictionary containing configuration information """ @@ -68,16 +68,18 @@ def loadConfig(satname, instrument, plot, cycle_tm, cycle_interval, data_locatio 'SENSOR': instrument, 'LEVELS': plot.get('levels'), 'CHANNELS': plot.get('channels'), + 'NET': net, 'RUN': plot.get('run'), 'PDATE': cycle_tm, 'PLOT_TEMPLATE': camelCase(plot.get('plot')), 'DATA': data_location } - times = int(plot.get('times')) - for x in range(1, times+1): - date_str = f"PDATEm{x*6}" - config[date_str] = add_to_datetime(cycle_tm, to_timedelta(f"-{cycle_interval*x}H")) + times = int(plot.get('times')) if plot.get('times') else None + if times is not None: + for x in range(1, times+1): + date_str = f"PDATEm{x*6}" + config[date_str] = add_to_datetime(cycle_tm, to_timedelta(f"-{cycle_interval*x}H")) # Some plots with channels require a configuration value of XTICKS (tick marks on the # plotted x axis). The x axis tick marks indicate the actual channel number, which can @@ -116,7 +118,6 @@ def loadConfig(satname, instrument, plot, cycle_tm, cycle_interval, data_locatio parser.add_argument('-i', '--input', type=str, help='Input YAML plot file', required=True) parser.add_argument('-p', '--pdate', type=str, help='Plot time YYYYMMDDHH', required=True) args = parser.parse_args() - cycle_tm = to_datetime(args.pdate) try: @@ -130,21 +131,41 @@ def loadConfig(satname, instrument, plot, cycle_tm, cycle_interval, data_locatio cycle_interval = mon_dict.get('cycle_interval') data_location = mon_dict.get('data') - sats = mon_dict.get('satellites') - for sat in sats: - satname = sat.get('name') + # if specified, generate template YAMLS and figures for satellite based obs + if 'satellites' in mon_dict.keys(): + for sat in mon_dict.get('satellites'): + satname = sat.get('name') - for inst in sat.get('instruments'): - instrument = inst.get('name') + for inst in sat.get('instruments'): + instrument = inst.get('name') - for plot in inst.get('plot_list'): - config = loadConfig(satname, instrument, plot, cycle_tm, - cycle_interval, data_location) - plot_template = f"{config['PLOT_TEMPLATE']}.yaml" - plot_yaml = f"{config['SENSOR']}_{config['SAT']}_{plot_template}" + for plot in inst.get('plot_list'): + config = loadConfig(satname, instrument, plot, cycle_tm, + cycle_interval, data_location) + plot_template = f"{config['PLOT_TEMPLATE']}.yaml" + plot_yaml = f"{config['SENSOR']}_{config['SAT']}_{plot_template}" + + plot_template = os.path.join('../parm/gfs/', plot_template) + genYaml(plot_template, plot_yaml, config) + + eva(plot_yaml) + os.remove(plot_yaml) + # if specified, generate template YAMLs and figures for minimization + if 'minimization' in mon_dict.keys(): + satname = None + instrument = None + for min in mon_dict.get('minimization'): + net = min.get('net') + + for plot in min.get('plot_list'): + config = loadConfig(satname, instrument, plot, cycle_tm, cycle_interval, + data_location, net) + + plot_template = f"{config['PLOT_TEMPLATE']}.yaml" + plot_yaml = f"{config['NET']}_{config['RUN']}_{plot_template}" plot_template = os.path.join('../parm/gfs/', plot_template) - genYaml(plot_template, plot_yaml, config) + genYaml(plot_template, plot_yaml, config) eva(plot_yaml) os.remove(plot_yaml)