diff --git a/.github/workflows/gcc.yml b/.github/workflows/gcc.yml index bd74aa65..644a7ed3 100644 --- a/.github/workflows/gcc.yml +++ b/.github/workflows/gcc.yml @@ -30,7 +30,7 @@ env: jobs: setup: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: # Cache spack, compiler and dependencies diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index c19a5ab0..0ff5c739 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -9,7 +9,7 @@ defaults: # Set I_MPI_CC/F90 so Intel MPI wrapper uses icc/ifort instead of gcc/gfortran env: - cache_key: intel3 # The number (#) following the cache_key "intel" is to flush Action cache. + cache_key: intel4 # The number (#) following the cache_key "intel" is to flush Action cache. CC: icc FC: ifort CXX: icpc @@ -33,7 +33,7 @@ env: jobs: setup: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: # Cache spack, compiler and dependencies @@ -79,7 +79,7 @@ jobs: build: needs: setup - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: checkout-gsi-monitor diff --git a/src/Minimization_Monitor/data_xtrct/ush/MinMon_CP.sh b/src/Minimization_Monitor/data_xtrct/ush/MinMon_CP.sh index 712ae2e5..b7b86e5e 100755 --- a/src/Minimization_Monitor/data_xtrct/ush/MinMon_CP.sh +++ b/src/Minimization_Monitor/data_xtrct/ush/MinMon_CP.sh @@ -39,6 +39,8 @@ fi # Process command line arguments # pdate="" +run="" + while [[ $# -ge 1 ]] do key="$1" @@ -123,7 +125,7 @@ cyc=`echo $pdate|cut -c9-10` # Verify the data files are available for this cycle # data_dir="" -data_dir=`$MON_USH/get_stats_path.sh --run $RUN --pdate ${pdate} --net ${MINMON_SUFFIX} --tank ${data} --mon minmon` +data_dir=`$MON_USH/get_stats_path.sh --run $run --pdate ${pdate} --net ${MINMON_SUFFIX} --tank ${data} --mon minmon` echo data_dir = $data_dir if [[ ! -d ${data_dir} ]]; then diff --git a/src/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh b/src/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh index 52d294f4..0853d1f1 100755 --- a/src/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh +++ b/src/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh @@ -53,6 +53,8 @@ export RAD_AREA=glb # Set default values and process command line arguments. # run=gdas +pdate="" +data_file_loc="" while [[ $# -ge 1 ]] do @@ -145,7 +147,8 @@ fi # $TANKverf and increment 6 hours. #--------------------------------------------------------------- if [[ $pdate = "" ]]; then - ldate=`${DE_SCRIPTS}/nu_find_cycle.pl --run $RUN --cyc 1 --dir ${TANKverf}` + ldate=`${MON_USH}/find_last_cycle.sh --net ${RADMON_SUFFIX} \ + --run ${RUN} --mon radmon --tank ${TANKDIR}` pdate=`${NDATE} +06 ${ldate}` fi echo "pdate = $pdate" diff --git a/src/Radiance_Monitor/data_extract/ush/radmon_copy.sh b/src/Radiance_Monitor/data_extract/ush/radmon_copy.sh index 912f3d1d..4195ac33 100755 --- a/src/Radiance_Monitor/data_extract/ush/radmon_copy.sh +++ b/src/Radiance_Monitor/data_extract/ush/radmon_copy.sh @@ -61,9 +61,15 @@ fi # 'drop out' plots. # nfile_src=`ls -l ${DATA_LOCATION}/*${PDATE}*ieee_d* | egrep -c '^-'` +if [[ ${nfile_src} -le 0 ]]; then + nfile_src=`ls -l ${DATA_LOCATION}/radmon_*tar* | egrep -c '^-'` +fi echo "nfile_src = ${nfile_src}" nfile_thirty=`find ${DATA_LOCATION}/*${PDATE}*ieee_d* -maxdepth 0 -mmin -30` +if [[ ${nfile_thirty} -le 0 ]]; then + nfile_thirty=`find ${DATA_LOCATION}/*radmon_*tar* -maxdepth 0 -mmin -30` +fi echo "nfile_thirty = ${nfile_thirty}" if [[ ${nfile_src} -le 0 ]]; then @@ -82,7 +88,7 @@ if [[ ${exit_value} -eq 0 ]]; then for type in ${type_list}; do - file_list=`ls ${DATA_LOCATION}/${type}.*${PDATE}*ieee_d* ${DATA_LOCATION}/${type}*tar* ` + file_list=`ls ${DATA_LOCATION}/${type}.*${PDATE}*ieee_d* ${DATA_LOCATION}/radmon_${type}*tar* ` for file in ${file_list}; do bfile=`basename ${file}` @@ -188,7 +194,7 @@ if [[ $exit_value == 0 ]]; then ${DE_SCRIPTS}/radmon_diag_ck.sh --rad ${radstat} --sat ${satype_file} --out ${diag_out} if [[ -e ${diag_out} ]]; then - $NCP ./${diag_out} ${TANKverf}/${RUN}.${day}/${cyc}/radmon/. + $NCP ./${diag_out} ${TANKverf}/${RUN}.${PDY}/${CYC}/radmon/. fi fi diff --git a/src/Radiance_Monitor/image_gen/html/Install_html.sh b/src/Radiance_Monitor/image_gen/html/Install_html.sh index 0d968be9..5e144ac3 100755 --- a/src/Radiance_Monitor/image_gen/html/Install_html.sh +++ b/src/Radiance_Monitor/image_gen/html/Install_html.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash #-------------------------------------------------------------------- #-------------------------------------------------------------------- @@ -10,25 +10,51 @@ #-------------------------------------------------------------------- #-------------------------------------------------------------------- +#-------------------------------------------------------------------- +# usage +#-------------------------------------------------------------------- function usage { - echo "Usage: Install_html.sh suffix area" + echo "Usage: Install_html.sh suffix [-t|--tank]" echo " Suffix is data source identifier that matches data in " echo " the $TANKDIR/stats directory." - echo " area is either 'glb' or 'rgn' (global or regional)" + echo " -t | --tank parent directory to the adnmon data file location. This" + echo " will be extended by \$RADMON_SUFFIX, \$RUN, and \$PDATE to locate the" + echo " extracted radmon data." + echo "" } echo "BEGIN Install_html.sh" echo "" nargs=$# -if [[ $nargs -ne 2 ]]; then + +if [[ $nargs -lt 1 || $nargs -gt 3 ]]; then usage exit 2 fi -export RADMON_SUFFIX=$1 -echo RADMON_SUFFIX = $RADMON_SUFFIX -export RAD_AREA=$2 +#----------------------------------------------------------- +# Set default values and process command line arguments. +# +#run=gdas +tank="" +area="" + +while [[ $# -ge 1 ]]; do + key="$1" + + case $key in + -t|--tank) + tank="$2" + shift # past argument + ;; + *) + #any unspecified key is RADMON_SUFFIX + export RADMON_SUFFIX=$key + ;; + esac + shift +done this_file=`basename $0` this_dir=`dirname $0` @@ -60,14 +86,15 @@ if [[ $? -ne 0 ]]; then exit $? fi -#-------------------------------------------------------------- -# call the appropriate child script for glb or rgn -# -if [[ $RAD_AREA == "glb" ]]; then - ${RADMON_IMAGE_GEN}/html/install_glb.sh -else - ${RADMON_IMAGE_GEN}/html/install_rgn.sh + +if [[ ${#tank} -le 0 ]]; then + tank=${TANKDIR} fi +export R_TANKDIR=${tank} +echo R_TANKDIR = $R_TANKDIR + + +${RADMON_IMAGE_GEN}/html/install_glb.sh echo "END Install_html.sh" diff --git a/src/Radiance_Monitor/image_gen/html/bcoef.html.rgn b/src/Radiance_Monitor/image_gen/html/bcoef.html.rgn deleted file mode 100644 index b0f920d8..00000000 --- a/src/Radiance_Monitor/image_gen/html/bcoef.html.rgn +++ /dev/null @@ -1,275 +0,0 @@ - - - - - - Radiance Monitoring - - - - - - - - - - -

Bias correction coefficients

-
- - - - - - - - - - - - - - -
Select Platform:
-

-

Bias correction term:
-

-

Channel groupings:
- -
- Click button to...
-
-
- - -
- - - diff --git a/src/Radiance_Monitor/image_gen/html/bcor.html.rgn b/src/Radiance_Monitor/image_gen/html/bcor.html.rgn deleted file mode 100644 index 3fb23142..00000000 --- a/src/Radiance_Monitor/image_gen/html/bcor.html.rgn +++ /dev/null @@ -1,269 +0,0 @@ - - - - - Radiance Monitoring - - - - - - - - - - -

Bias correction terms

- -
- - - - - - - - - - - - - - -
Select Platform:
- -
Term:
- -
Channel groupings:
- -
- Click button to...
-
-
- -
- - diff --git a/src/Radiance_Monitor/image_gen/html/bcor_angle.html.rgn b/src/Radiance_Monitor/image_gen/html/bcor_angle.html.rgn deleted file mode 100644 index f5108f67..00000000 --- a/src/Radiance_Monitor/image_gen/html/bcor_angle.html.rgn +++ /dev/null @@ -1,292 +0,0 @@ - - - - - Radiance Monitoring - - - - - - - - -

Bias correction terms

-
- - - - - - - - - - - - - - -
Select Platform:
-

-

Term:
- -
Channel groupings:
-

-

- Click button to...
-
-
- Click button to...
-
-
- -
- - - - - diff --git a/src/Radiance_Monitor/image_gen/html/horiz.html.rgn b/src/Radiance_Monitor/image_gen/html/horiz.html.rgn deleted file mode 100644 index fc4cf8e2..00000000 --- a/src/Radiance_Monitor/image_gen/html/horiz.html.rgn +++ /dev/null @@ -1,729 +0,0 @@ - - - - Radiance Monitoring - - - - - - - - - - -

Horizontal maps

- -
- - - - - - - - - - - - - - -
Select Platform:
- -
Field:
- -
Channel:
- -
- Click button to...
-
- -
- -
- - - diff --git a/src/Radiance_Monitor/image_gen/html/index.html.rgn b/src/Radiance_Monitor/image_gen/html/index.html.rgn deleted file mode 100644 index 1ee84c96..00000000 --- a/src/Radiance_Monitor/image_gen/html/index.html.rgn +++ /dev/null @@ -1,331 +0,0 @@ - - - - - - INSERT_SUFFIX -- Experimental Radiance Monitoring plots - - - - - - - - - - - -
- - -
- - -
- -
- -
- - - -
- - - diff --git a/src/Radiance_Monitor/image_gen/html/install_glb.sh b/src/Radiance_Monitor/image_gen/html/install_glb.sh index c813d994..260f688c 100755 --- a/src/Radiance_Monitor/image_gen/html/install_glb.sh +++ b/src/Radiance_Monitor/image_gen/html/install_glb.sh @@ -69,14 +69,18 @@ cd $workdir #------------------------------------------------------------- # Assemble the SATYPE list from available data files in # $TANKverf using angle.* files. -#------------------------------------------------------------- - -#----------------------------------------------------------- +# # Find the first date with data. Start at today and work -# backwards. If not found stop after 90 days and exit. +# backwards. If not found stop after 5 days and exit. # -PDATE=`${IG_SCRIPTS}/nu_find_cycle.pl --dir ${TANKverf} --cyc 1` -limit=`$NDATE -2160 $PDATE` # 90 days + +RUN=gdas + +PDATE=`${MON_USH}/find_last_cycle.sh --net ${RADMON_SUFFIX} \ + --run ${RUN} --mon radmon --tank ${TANKDIR}` +echo PDATE=$PDATE + +limit=`$NDATE -120 $PDATE` # 5 days #----------------------------------------------------------- # Build test_list which will contain all data files for @@ -87,39 +91,32 @@ while [[ data_found -eq 0 && $PDATE -ge $limit ]]; do PDY=`echo $PDATE|cut -c1-8` CYC=`echo $PDATE|cut -c9-10` - test_dir=${TANKverf}/${RUN}.${PDY}/${CYC}/${MONITOR} - if [[ ! -d ${test_dir} ]]; then - test_dir=${TANKverf}/${RUN}.${PDY}/${MONITOR} - fi - if [[ ! -d ${test_dir} ]]; then - test_dir=${TANKverf}/${RUN}.${PDY} - fi - - if [[ -d ${test_dir} ]]; then - echo " test_dir is GO " - - if [[ -e ${test_dir}/radmon_angle.tar || -e ${test_dir}/radmon_angle.tar.gz ]]; then - gzipped=0 - if [[ -e ${test_dir}/radmon_angle.tar.gz ]]; then - gunzip ${test_dir}/radmon_angle.tar.gz - gzipped=1 + ieee_src=`${MON_USH}/get_stats_path.sh --run $RUN --pdate ${PDATE} \ + --net ${RADMON_SUFFIX} --tank ${TANKDIR} --mon radmon` + echo "ieee_src = $ieee_src" + + if [[ -d ${ieee_src} ]]; then + using_tar=0 + echo " ieee_src is GO " + + if [[ -e ${ieee_src}/radmon_angle.tar || -e ${ieee_src}/radmon_angle.tar.gz ]]; then + if [[ -e ${ieee_src}/radmon_angle.tar.gz ]]; then + test_list=`tar -tf ${ieee_src}/radmon_angle.tar.gz` + else + test_list=`tar -tf ${ieee_src}/radmon_angle.tar` fi - test_list=`tar -tf ${test_dir}/radmon_angle.tar` data_found=1 - if [[ $gzipped -eq 1 ]]; then - gzip ${test_dir}/radmon_angle.tar - fi else - test=`ls ${test_dir}/angle.*${PDATE}*.ieee_d* | wc -l` + test=`ls ${ieee_src}/angle.*${PDATE}*.ieee_d* | wc -l` if [[ $test -gt 0 ]]; then - test_list=`ls ${test_dir}/angle.*${PDATE}*.ieee_d*` + test_list=`ls ${ieee_src}/angle.*${PDATE}*.ieee_d*` data_found=1 fi fi else - echo "test_dir is NOGO" + echo "ieee_src is NOGO" fi if [[ data_found -eq 0 ]]; then @@ -127,9 +124,8 @@ while [[ data_found -eq 0 && $PDATE -ge $limit ]]; do fi done - if [[ $data_found -eq 0 ]]; then - echo Unable to locate any data files in the past 90 days for $SUFFIX + echo Unable to locate any data files in the past 5 days for $SUFFIX echo in $TANKverf/angle. exit fi @@ -173,6 +169,7 @@ UNSORTED_LIST=./unsorted.txt export SORTED_LIST=./sorted.txt >$SORTED_LIST +echo "SATYPE = $SATYPE" for satype in $SATYPE; do ins=${satype%_*} @@ -224,6 +221,7 @@ done # Sort the list by Satellite # `sort -d -u $UNSORTED_LIST > $SORTED_LIST` +echo SORTED_LIST = $SORTED_LIST #-------------------------------------------------------------- # Read the sorted list and create the platform table @@ -253,7 +251,6 @@ while read line; do done < "$SORTED_LIST" -imgndir=`dirname ${IMGNDIR}` #-------------------------------------------------------------- # Edit the html files to add the platform table to each. # @@ -354,6 +351,7 @@ fi rm ./${index_file} +echo workdir = $workdir #-------------------------------------------------------------- # Make starting directory in $imgndir and copy over html, diff --git a/src/Radiance_Monitor/image_gen/html/install_rgn.sh b/src/Radiance_Monitor/image_gen/html/install_rgn.sh deleted file mode 100755 index 4a1a1666..00000000 --- a/src/Radiance_Monitor/image_gen/html/install_rgn.sh +++ /dev/null @@ -1,413 +0,0 @@ -#!/bin/sh - -#-------------------------------------------------------------------- -#-------------------------------------------------------------------- -# install_rgn.sh -# -# Given a suffix and a global/regional flag as inputs, build the -# html necessary for a radiance monitor web site and tranfer it to -# the server (rgn only, regional is handled by Install_html.sh). -#-------------------------------------------------------------------- -#-------------------------------------------------------------------- -echo "BEGIN install_rgn.sh" -echo "" - -SUFFIX=$RADMON_SUFFIX -RAD_AREA="rgn" - -this_file=`basename $0` -this_dir=`dirname $0` - - -#-------------------------------------------------------------- -# Create a temporary working directory. -# -workdir=$STMP_USER/${SUFFIX}_html -if [[ -d ${workdir} ]]; then - rm -rf $workdir -fi -mkdir $workdir -cd $workdir - - -#------------------------------------------------------------- -# Assemble the SATYPE list from available data files in -# $TANKverf using angle.* files. -#------------------------------------------------------------- - -#----------------------------------------------------------- -# Find the first date with data. Start at today and work -# backwards. If not found, stop after 90 days and exit. -# -PDATE=`${IG_SCRIPTS}/find_cycle.pl --cyc 1 --dir ${TANKverf}` -limit=`$NDATE -2160 $PDATE` # 90 days - -#----------------------------------------------------------- -# Build test_list which will contain all data files for -# one cycle in $PDATE. -# -# Note: with the new nam (rapid refresh) the data file name has changed -# from this: -# angle.[sat_name].[cycle].ieee_d.gz -# to this: -# t[cc]z.angle.[sat_name].[cycle].ieee_d.tm[HH].gz -# -# so to cope we'll have to build a list and step through file names -# until we find "angle" then take the next field to get the sat names. - -data_found=0 -while [[ data_found -eq 0 && $PDATE -ge $limit ]]; do - PDY=`echo $PDATE|cut -c1-8` - - if [[ ${TANK_USE_RUN} -eq 1 ]]; then - test_dir=${TANKverf}/${RUN}.${PDY}/${MONITOR} - else - test_dir=${TANKverf}/${MONITOR}.${PDY} - fi - - if [[ -d ${test_dir} ]]; then - echo " test_dir is GO " - - test00=`ls ${test_dir}/*angle.*${PDY}00*.ieee_d* | wc -l` - test06=`ls ${test_dir}/*angle.*${PDY}06*.ieee_d* | wc -l` - test12=`ls ${test_dir}/*angle.*${PDY}12*.ieee_d* | wc -l` - test18=`ls ${test_dir}/*angle.*${PDY}18*.ieee_d* | wc -l` - if [[ $test00 -gt 0 ]]; then - test_list=`ls ${test_dir}/*angle.*${PDY}00*.ieee_d*` - data_found=1 - elif [[ $test06 -gt 0 ]]; then - test_list=`ls ${test_dir}/*angle.*${PDY}06*.ieee_d*` - data_found=1 - elif [[ $test12 -gt 0 ]]; then - test_list=`ls ${test_dir}/*angle.*${PDY}12*.ieee_d*` - data_found=1 - elif [[ $test18 -gt 0 ]]; then - test_list=`ls ${test_dir}/*angle.*${PDY}18*.ieee_d*` - data_found=1 - fi - else - echo " test_dir is NOGO " - fi - - if [[ data_found -eq 0 ]]; then - PDATE=`$NDATE -24 $PDATE` - fi -done - -if [[ $data_found -eq 0 ]]; then - echo Unable to locate any data files in the past 90 days for $SUFFIX - echo in $TANKverf/angle. - exit -fi - -#----------------------------------------------------------- -# Go through test_list and identify all unique -# sat_instrument combinations. The results are the -# SATYPE list for this source. -# - -for test in ${test_list}; do - this_file=`basename $test` - tmp1=`echo "$this_file" | cut -d. -f1` - tmp2=`echo "$this_file" | cut -d. -f2` - if [[ $tmp1 == "angle" ]]; then - tmp=`echo "$this_file" | cut -d. -f2` - elif [[ $tmp2 == "angle" ]]; then - tmp=`echo "$this_file" | cut -d. -f3` - fi - - #---------------------------------------------------------- - # remove sat/instrument_anl names so we don't end up - # with both "airs_aqua" and "airs_aqua_anl" if analysis - # files are being generated for this source. - #---------------------------------------------------------- - test_anl=`echo $tmp | grep "_anl"` - if [[ $test_anl = "" ]]; then - SATYPE_LIST="$SATYPE_LIST $tmp" - fi -done - - -#---------------------------------------------------------- -# use sort to remove duplicate entries in SATYPE_LIST -#---------------------------------------------------------- -unique_satype=`echo $(printf '%s\n' $SATYPE_LIST | sort -u)` -export SATYPE=$unique_satype - -if [[ ${#SATYPE} -le 0 ]]; then - echo "SATYPE list is zero length, unable to complete html installation" - exit -fi - -#-------------------------------------------------------------- -# Use the SATYPE list to construct the platform table. -# -UNSORTED_LIST=./unsorted.txt ->$UNSORTED_LIST -export SORTED_LIST=./sorted.txt ->$SORTED_LIST - - -for satype in $SATYPE; do - ins=${satype%_*} - tmp="${ins}_" - sat=${satype#$tmp} - - sat_num=`echo $sat | tr -d '[[:alpha:]]'` - - #----------------------------------------------------------------- - # If sat_num has a length > 0 then we have a goes or noaa series - # satellite. Otherwise, convert sat to upper case and stop there. - # - if [[ ${#sat_num} -gt 0 ]]; then - char=`expr substr $sat 1 1` - if [[ $char == "g" ]]; then - sat="GOES-${sat_num}" - elif [[ $char == "n" ]]; then - sat="NOAA-${sat_num}" - else - sat=`echo $sat | tr 'a-z' 'A-Z'` - fi - else - sat=`echo $sat | tr 'a-z' 'A-Z'` - fi - - - #----------------------------------------------------------------- - # - # - amsu_test=`expr match ${ins} "amsu"` - hirs_test=`expr match ${ins} "hirs"` - ins_num=`echo $ins | tr -d '[[:alpha:]]'` - - if [[ $amsu_test -gt 0 ]]; then - char=`expr substr $ins 5 5` - char=`echo $char | tr -s 'a-z' 'A-Z'` - ins="AMSU-${char}" - elif [[ $hirs_test -gt 0 ]]; then - ins="HIRS/${ins_num}" - else - ins=`echo $ins | tr -s 'a-z' 'A-Z'` - fi - - echo $sat $ins $satype >> $UNSORTED_LIST - -done - -#-------------------------------------------------------------- -# Sort the list by Satellite -# -`sort -d $UNSORTED_LIST > $SORTED_LIST` - - -#-------------------------------------------------------------- -# Read the sorted list and create the platform table -# -PLATFORM_TBL=./platform.txt -> ${PLATFORM_TBL} -TIME_PLATFORM_TBL=./time_platform.txt -> ${TIME_PLATFORM_TBL} - -quote='"' -id=' id="' -extra='">' -end_option='' - -while read line; do - sat=`echo $line | gawk '{print $1}'` - ins=`echo $line | gawk '{print $2}'` - satype=`echo $line | gawk '{print $3}'` - - hline='