[develop] Make get_obs tasks day-dependent in workflow; other improvements and bug fixes

doc/UsersGuide/Reference/Glossary.rst

@@ -26,7 +26,7 @@ Glossary
 
    chgres_cube
        The preprocessing software used to create initial and boundary condition files to 
-       “cold start” the forecast model. It is part of :term:`UFS_UTILS`.
+       "cold start" the forecast model. It is part of :term:`UFS_UTILS`.
 
    CIN
       Convective Inhibition.
@@ -87,7 +87,7 @@ Glossary
       Extended Schmidt Gnomonic (ESG) grid. The ESG grid uses the map projection developed by Jim Purser of NOAA :term:`EMC` (:cite:t:`Purser_2020`). 
 
    ESMF
-      `Earth System Modeling Framework <https://earthsystemmodeling.org/docs/release/latest/ESMF_usrdoc/>`__. The ESMF defines itself as “a suite of software tools for developing high-performance, multi-component Earth science modeling applications.” 
+      `Earth System Modeling Framework <https://earthsystemmodeling.org/docs/release/latest/ESMF_usrdoc/>`__. The ESMF defines itself as "a suite of software tools for developing high-performance, multi-component Earth science modeling applications."
 
    ex-scripts
       Scripting layer (contained in ``ufs-srweather-app/scripts/``) that should be called by a :term:`J-job <J-jobs>` for each workflow componentto run a specific task or sub-task in the workflow. The different scripting layers are described in detail in the :nco:`NCO Implementation Standards document <ImplementationStandards.v11.0.0.pdf>`
@@ -215,7 +215,7 @@ Glossary
       The branch of physical geography dealing with mountains.
 
    Parameterizations
-      Simplified functions that approximate the effects of small-scale processes (e.g., microphysics, gravity wave drag) that cannot be explicitly resolved by a model grid’s representation of the earth.
+      Simplified functions that approximate the effects of small-scale processes (e.g., microphysics, gravity wave drag) that cannot be explicitly resolved by a model grid's representation of the earth.
 
    RAP
       `Rapid Refresh <https://rapidrefresh.noaa.gov/>`__. The continental-scale NOAA hourly-updated assimilation/modeling system operational at :term:`NCEP`. RAP covers North America and is comprised primarily of a numerical forecast model and an analysis/assimilation system to initialize that model. RAP is complemented by the higher-resolution 3km High-Resolution Rapid Refresh (:term:`HRRR`) model.

jobs/JREGIONAL_GET_VERIF_OBS

@@ -69,7 +69,6 @@ This is the J-job script for the task that checks, pulls, and stages
 observation data for verification purposes.
 ========================================================================"
 
-#
 #
 #-----------------------------------------------------------------------
 #

modulefiles/tasks/derecho/get_obs.local.lua

@@ -0,0 +1 @@
+load("run_vx.local")

modulefiles/tasks/gaea/get_obs.local.lua

@@ -0,0 +1 @@
+load("run_vx.local")

modulefiles/tasks/hera/get_obs.local.lua

@@ -1,3 +1,2 @@
 load("hpss")
-unload("python")
-load("python_srw")
+load("run_vx.local")

modulefiles/tasks/hercules/get_obs.local.lua

@@ -0,0 +1 @@
+load("run_vx.local")

modulefiles/tasks/jet/get_obs.local.lua

@@ -1,3 +1,2 @@
 load("hpss")
-unload("python")
-load("python_srw")
+load("run_vx.local")

modulefiles/tasks/noaacloud/get_obs.local.lua

@@ -0,0 +1 @@
+load("run_vx.local")

modulefiles/tasks/orion/get_obs.local.lua

@@ -0,0 +1 @@
+load("run_vx.local")

parm/data_locations.yml

@@ -319,18 +319,15 @@ CCPA_obs:
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
     archive_internal_dir:
-      - "./00"
-      - "./06"
-      - "./12"
-      - "./18"
+      - "./{hh}"
     archive_file_names:
       - "com2_ccpa_prod_ccpa.{yyyy}{mm}{dd}.tar"
       - "gpfs_dell1_nco_ops_com_ccpa_prod_ccpa.{yyyy}{mm}{dd}.tar"
       - "com_ccpa_prod_ccpa.{yyyy}{mm}{dd}.tar"
       - "com_ccpa_v4.2_ccpa.{yyyy}{mm}{dd}.tar"
     file_names:
       obs:
-        - "ccpa.t{hh}z.01h.hrap.conus.gb2"
+        - "ccpa.t*z.01h.hrap.conus.gb2"
 
 MRMS_obs:
   hpss:
@@ -365,11 +362,13 @@ NDAS_obs:
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
+      - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}        
     archive_file_names:
       - "com2_nam_prod_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
       - "gpfs_dell1_nco_ops_com_nam_prod_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
       - "com_nam_prod_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
       - "com_obsproc_v1.1_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
+      - "com_obsproc_v1.2_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
     file_names:
       obs:
         - "./nam.t{hh}z.prepbufr.tm*.nr"
@@ -386,6 +385,6 @@ NOHRSC_obs:
       - "dcom_{yyyy}{mm}{dd}.tar"
     file_names:
       obs:
-        - "sfav2_CONUS_*h_{yyyy}{mm}{dd}{hh}_grid184.grb2"
+        - "sfav2_CONUS_6h_{yyyy}{mm}{dd}*_grid184.grb2"
     archive_internal_dir:
       - ./wgrbbul/nohrsc_snowfall/

parm/metplus/EnsembleStat.conf

@@ -31,7 +31,7 @@ INIT_INCREMENT = 3600
 # List of forecast leads to process for each run time (init or valid)
 # In hours if units are not specified
 # If unset, defaults to 0 (don't loop through forecast leads)
-LEAD_SEQ = {{fhr_list}}
+LEAD_SEQ = {{vx_leadhr_list}}
 #
 # Order of loops to process data - Options are times, processes
 # Not relevant if only one item is in the PROCESS_LIST
@@ -87,7 +87,7 @@ OBS_FILE_WINDOW_BEGIN = -300
 OBS_FILE_WINDOW_END = 300
 OBS_{{METPLUS_TOOL_NAME}}_WINDOW_BEGIN = 0
 OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = 0
-{%- elif input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- elif input_field_group in ['SFC', 'UPA'] %}
 OBS_WINDOW_BEGIN = -1799
 OBS_WINDOW_END = 1800
 OBS_{{METPLUS_TOOL_NAME}}_WINDOW_BEGIN = {OBS_WINDOW_BEGIN}
@@ -105,7 +105,7 @@ OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = {OBS_WINDOW_END}
 # ens.vld_thresh value in the MET config file
 {{METPLUS_TOOL_NAME}}_ENS_VLD_THRESH = 1.0
 
-{%- if input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- if input_field_group in ['SFC', 'UPA'] %}
 
 {{METPLUS_TOOL_NAME}}_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA
 #{{METPLUS_TOOL_NAME}}_OBS_QUALITY_EXC =
@@ -118,7 +118,7 @@ OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = {OBS_WINDOW_END}
 {{METPLUS_TOOL_NAME}}_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt
 {%- elif input_field_group in ['REFC', 'RETOP'] %}
 {{METPLUS_TOOL_NAME}}_MET_OBS_ERR_TABLE =
-{%- elif input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- elif input_field_group in ['SFC', 'UPA'] %}
 {{METPLUS_TOOL_NAME}}_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt
 {%- endif %}
 
@@ -135,7 +135,7 @@ OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = {OBS_WINDOW_END}
   {%- set comment_or_null = '' %}
   {%- set regrid_to_grid = 'FCST' %}
   {%- set regrid_method = 'BUDGET' %}
-{%- elif input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- elif input_field_group in ['SFC', 'UPA'] %}
   {%- set comment_or_null = '#' %}
   {%- set regrid_to_grid = 'NONE' %}
   {%- set regrid_method = 'BILIN' %}
@@ -160,8 +160,8 @@ OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = {OBS_WINDOW_END}
 {{METPLUS_TOOL_NAME}}_DUPLICATE_FLAG = NONE
 {{METPLUS_TOOL_NAME}}_SKIP_CONST = TRUE
 {{METPLUS_TOOL_NAME}}_OBS_ERROR_FLAG = FALSE
-{%- elif input_field_group in ['ADPSFC', 'ADPUPA'] %}
-{{METPLUS_TOOL_NAME}}_MESSAGE_TYPE = {{input_field_group}}
+{%- elif input_field_group in ['SFC', 'UPA'] %}
+{{METPLUS_TOOL_NAME}}_MESSAGE_TYPE = {{fieldname_in_met_filedir_names}}
 {{METPLUS_TOOL_NAME}}_DUPLICATE_FLAG = NONE
 {{METPLUS_TOOL_NAME}}_SKIP_CONST = FALSE
 {{METPLUS_TOOL_NAME}}_OBS_ERROR_FLAG = FALSE
@@ -197,7 +197,7 @@ OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = {OBS_WINDOW_END}
   {%- set comment_or_null = '' %}
 {%- elif input_field_group in ['REFC', 'RETOP'] %}
   {%- set comment_or_null = '' %}
-{%- elif input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- elif input_field_group in ['SFC', 'UPA'] %}
   {%- set comment_or_null = '#' %}
 {%- endif %}
 {{comment_or_null}}{{METPLUS_TOOL_NAME}}_CLIMO_CDF_BINS = 1
@@ -207,7 +207,7 @@ OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = {OBS_WINDOW_END}
 {{METPLUS_TOOL_NAME}}_MASK_GRID =
 {%- elif input_field_group in ['REFC', 'RETOP'] %}
 {{METPLUS_TOOL_NAME}}_MASK_GRID = FULL
-{%- elif input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- elif input_field_group in ['SFC', 'UPA'] %}
 {{METPLUS_TOOL_NAME}}_MASK_GRID =
 {%- endif %}
 
@@ -483,7 +483,7 @@ FCST_VAR{{ns.var_count}}_OPTIONS = convert(x) = x * 3.28084 * 0.001; ;; Convert
 {{opts_indent}}ens_phist_bin_size = 0.05;
         {%- endif %}
 
-      {%- elif input_field_group == 'ADPSFC' %}
+      {%- elif input_field_group == 'SFC' %}
 
         {%- if field_fcst == 'HGT' %}
 FCST_VAR{{ns.var_count}}_OPTIONS = GRIB_lvl_typ = 215;
@@ -499,7 +499,7 @@ FCST_VAR{{ns.var_count}}_OPTIONS = interp = { type = [ { method = NEAREST; width
 FCST_VAR{{ns.var_count}}_OPTIONS = GRIB2_pdt = 0; ;; Derive instantaneous 10-m wind from U/V components, overriding max 10-m wind.
         {%- endif %}
 
-      {%- elif input_field_group == 'ADPUPA' %}
+      {%- elif input_field_group == 'UPA' %}
 
         {%- if field_fcst == 'CAPE' %}
 FCST_VAR{{ns.var_count}}_OPTIONS = cnt_thresh = [ >0 ];
@@ -516,15 +516,8 @@ PcpCombine tool.  In that file, the field name consists of the observation
 field name here (field_obs) with the accumulation period appended to it
 (separated by an underscore), so we must do the same here to get an exact
 match.
-
-Note:
-Turns out for ASNOW, PcpCombine is not run for obs, so we exclude that
-from the "if" clause here (so it goes into the "else").  For workflow
-behavior uniformity between APCP and ASNOW, consider running PcpCombine
-for ASNOW observations as well (just as it's run for APCP observations).
-      {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 #}
-      {%- if (input_field_group in ['APCP']) %}
+      {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}_{{accum_hh}}
       {%- else %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}
@@ -615,15 +608,15 @@ OBS_VAR{{ns.var_count}}_OPTIONS = censor_thresh = lt-20;
 {{opts_indent}}ens_phist_bin_size = 0.05;
         {%- endif %}
 
-      {%- elif input_field_group == 'ADPSFC' %}
+      {%- elif input_field_group == 'SFC' %}
 
         {%- if field_obs in ['DPT', 'TMP', 'WIND'] %}
 OBS_VAR{{ns.var_count}}_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; }
         {%- elif field_obs == 'CEILING' %}
 OBS_VAR{{ns.var_count}}_OPTIONS = GRIB_lvl_typ = 215
         {%- endif %}
 
-      {%- elif input_field_group == 'ADPUPA' %}
+      {%- elif input_field_group == 'UPA' %}
 
         {%- if field_obs in ['DPT', 'HGT', 'TMP', 'WIND'] %}
 OBS_VAR{{ns.var_count}}_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; }
@@ -656,15 +649,15 @@ OUTPUT_BASE = {{output_base}}
 #
 # Point observation input directory for {{MetplusToolName}}.
 #
-{%- if input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- if input_field_group in ['SFC', 'UPA'] %}
 OBS_{{METPLUS_TOOL_NAME}}_POINT_INPUT_DIR = {{obs_input_dir}}
 {%- else %}
 OBS_{{METPLUS_TOOL_NAME}}_POINT_INPUT_DIR =
 {%- endif %}
 #
 # Grid observation input directory for {{MetplusToolName}}.
 #
-{%- if input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- if input_field_group in ['SFC', 'UPA'] %}
 OBS_{{METPLUS_TOOL_NAME}}_GRID_INPUT_DIR =
 {%- else %}
 OBS_{{METPLUS_TOOL_NAME}}_GRID_INPUT_DIR = {{obs_input_dir}}
@@ -698,7 +691,7 @@ STAGING_DIR = {{staging_dir}}
 # Template for point observation input to {{MetplusToolName}} relative to
 # OBS_{{METPLUS_TOOL_NAME}}_POINT_INPUT_DIR.
 #
-{%- if input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- if input_field_group in ['SFC', 'UPA'] %}
 OBS_{{METPLUS_TOOL_NAME}}_POINT_INPUT_TEMPLATE = {{obs_input_fn_template}}
 {%- else %}
 OBS_{{METPLUS_TOOL_NAME}}_POINT_INPUT_TEMPLATE =
@@ -707,7 +700,7 @@ OBS_{{METPLUS_TOOL_NAME}}_POINT_INPUT_TEMPLATE =
 # Template for gridded observation input to {{MetplusToolName}} relative to
 # OBS_{{METPLUS_TOOL_NAME}}_GRID_INPUT_DIR.
 #
-{%- if input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- if input_field_group in ['SFC', 'UPA'] %}
 OBS_{{METPLUS_TOOL_NAME}}_GRID_INPUT_TEMPLATE =
 {%- else %}
 OBS_{{METPLUS_TOOL_NAME}}_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}}

parm/metplus/GenEnsProd.conf

@@ -31,7 +31,7 @@ INIT_INCREMENT = 3600
 # List of forecast leads to process for each run time (init or valid)
 # In hours if units are not specified
 # If unset, defaults to 0 (don't loop through forecast leads)
-LEAD_SEQ = {{fhr_list}}
+LEAD_SEQ = {{vx_leadhr_list}}
 #
 # Order of loops to process data - Options are times, processes
 # Not relevant if only one item is in the PROCESS_LIST
@@ -328,7 +328,7 @@ Set forecast field options.
 ENS_VAR{{ns.var_count}}_OPTIONS = convert(x) = x * 3.28084 * 0.001; ;; Convert from meters to kilofeet.
         {%- endif %}
 
-      {%- elif input_field_group == 'ADPSFC' %}
+      {%- elif input_field_group == 'SFC' %}
 
         {%- if field_fcst == 'HGT' %}
 ENS_VAR{{ns.var_count}}_OPTIONS = GRIB_lvl_typ = 215;
@@ -344,7 +344,7 @@ ENS_VAR{{ns.var_count}}_OPTIONS = interp = { type = [ { method = NEAREST; width
 ENS_VAR{{ns.var_count}}_OPTIONS = GRIB2_pdt = 0; ;; Derive instantaneous 10-m wind from U/V components, overriding max 10-m wind.
         {%- endif %}
 
-      {%- elif input_field_group == 'ADPUPA' %}
+      {%- elif input_field_group == 'UPA' %}
 
         {%- if field_fcst == 'CAPE' %}
 ENS_VAR{{ns.var_count}}_OPTIONS = cnt_thresh = [ >0 ];

parm/metplus/GridStat_ensmean.conf

@@ -31,7 +31,7 @@ INIT_INCREMENT = 3600
 # List of forecast leads to process for each run time (init or valid)
 # In hours if units are not specified
 # If unset, defaults to 0 (don't loop through forecast leads)
-LEAD_SEQ = {{fhr_list}}
+LEAD_SEQ = {{vx_leadhr_list}}
 #
 # Order of loops to process data - Options are times, processes
 # Not relevant if only one item is in the PROCESS_LIST
@@ -174,8 +174,8 @@ following dictionary.
        'ASNOW': [],
        'REFC': [],
        'RETOP': [],
-       'ADPSFC': ['TCDC', 'VIS', 'HGT'],
-       'ADPUPA': []}  %}
+       'SFC': ['TCDC', 'VIS', 'HGT'],
+       'UPA': []}  %}
 {%- set fields_fcst_to_exclude = fields_fcst_to_exclude_by_field_group[input_field_group] %}
 
 {#-
@@ -383,7 +383,7 @@ Set forecast field options.
       {%- endif %}
       {%- set opts_indent = ' '*opts_indent_len %}
 
-      {%- if input_field_group == 'ADPUPA' %}
+      {%- if input_field_group == 'UPA' %}
 
         {%- if field_fcst == 'CAPE' %}
 FCST_VAR{{ns.var_count}}_OPTIONS = cnt_thresh = [ >0 ];
@@ -400,15 +400,8 @@ PcpCombine tool.  In that file, the field name consists of the observation
 field name here (field_obs) with the accumulation period appended to it
 (separated by an underscore), so we must do the same here to get an exact
 match.
-
-Note:
-Turns out for ASNOW, PcpCombine is not run for obs, so we exclude that
-from the "if" clause here (so it goes into the "else").  For workflow
-behavior uniformity between APCP and ASNOW, consider running PcpCombine
-for ASNOW observations as well (just as it's run for APCP observations).
-      {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 #}
-      {%- if (input_field_group in ['APCP']) %}
+      {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}_{{accum_hh}}
       {%- else %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}
@@ -469,7 +462,7 @@ Set observation field options.
 OBS_VAR{{ns.var_count}}_OPTIONS = convert(x) = 100.0*x;
         {%- endif %}
 
-      {%- elif input_field_group == 'ADPUPA' %}
+      {%- elif input_field_group == 'UPA' %}
 
         {%- if field_obs == 'CAPE' %}
 OBS_VAR{{ns.var_count}}_OPTIONS = cnt_thresh = [ >0 ];

parm/metplus/GridStat_ensprob.conf

@@ -31,7 +31,7 @@ INIT_INCREMENT = 3600
 # List of forecast leads to process for each run time (init or valid)
 # In hours if units are not specified
 # If unset, defaults to 0 (don't loop through forecast leads)
-LEAD_SEQ = {{fhr_list}}
+LEAD_SEQ = {{vx_leadhr_list}}
 #
 # Order of loops to process data - Options are times, processes
 # Not relevant if only one item is in the PROCESS_LIST
@@ -354,15 +354,8 @@ PcpCombine tool.  In that file, the field name consists of the observation
 field name here (field_obs) with the accumulation period appended to it
 (separated by an underscore), so we must do the same here to get an exact
 match.
-
-Note:
-Turns out for ASNOW, PcpCombine is not run for obs, so we exclude that
-from the "if" clause here (so it goes into the "else").  For workflow
-behavior uniformity between APCP and ASNOW, consider running PcpCombine
-for ASNOW observations as well (just as it's run for APCP observations).
-            {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 #}
-            {%- if (input_field_group in ['APCP']) %}
+            {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}_{{accum_hh}}
             {%- else %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}