diff --git a/build.sh b/build.sh index 42ec34284..6445f80cf 100755 --- a/build.sh +++ b/build.sh @@ -71,7 +71,7 @@ while getopts "p:t:c:hvdfa" opt; do done case ${BUILD_TARGET} in - hera | orion | hercules | wcoss2 | noaacloud | gaea | gaeac6 ) + hera | orion | hercules | wcoss2 | noaacloud | gaeac5 | gaeac6 ) echo "Building GDASApp on $BUILD_TARGET" source $dir_root/ush/module-setup.sh module use $dir_root/modulefiles @@ -103,14 +103,12 @@ WORKFLOW_BUILD=${WORKFLOW_BUILD:-"OFF"} CMAKE_OPTS+=" -DWORKFLOW_TESTS=${WORKFLOW_BUILD}" # JCSDA changed test data things, need to make a dummy CRTM directory -if [[ $BUILD_TARGET == 'hera' ]]; then - if [ -d "$dir_root/bundle/fix/test-data-release/" ]; then rm -rf $dir_root/bundle/fix/test-data-release/; fi - if [ -d "$dir_root/bundle/test-data-release/" ]; then rm -rf $dir_root/bundle/test-data-release/; fi - mkdir -p $dir_root/bundle/fix/test-data-release/ - mkdir -p $dir_root/bundle/test-data-release/ - ln -sf $GDASAPP_TESTDATA/crtm $dir_root/bundle/fix/test-data-release/crtm - ln -sf $GDASAPP_TESTDATA/crtm $dir_root/bundle/test-data-release/crtm -fi +if [ -d "$dir_root/bundle/fix/test-data-release/" ]; then rm -rf $dir_root/bundle/fix/test-data-release/; fi +if [ -d "$dir_root/bundle/test-data-release/" ]; then rm -rf $dir_root/bundle/test-data-release/; fi +mkdir -p $dir_root/bundle/fix/test-data-release/ +mkdir -p $dir_root/bundle/test-data-release/ +ln -sf $GDASAPP_TESTDATA/crtm $dir_root/bundle/fix/test-data-release/crtm +ln -sf $GDASAPP_TESTDATA/crtm $dir_root/bundle/test-data-release/crtm # Configure echo "Configuring ..." diff --git a/mains/gdas.cc b/mains/gdas.cc index f47b82802..10d826bd9 100755 --- a/mains/gdas.cc +++ b/mains/gdas.cc @@ -56,7 +56,7 @@ int runApp(int argc, char** argv, const std::string traits, const std::string ap apps["converttostructuredgrid"] = []() { return std::make_unique>(); - }; + }; apps["convertstate"] = []() { return std::make_unique>(); }; @@ -67,7 +67,7 @@ int runApp(int argc, char** argv, const std::string traits, const std::string ap return std::make_unique>(); }; apps["localensembleda"] = []() { - return std::make_unique>(); + return std::make_unique>(); }; apps["variational"] = []() { return std::make_unique>(); @@ -106,6 +106,7 @@ int main(int argc, char ** argv) { const std::set validApps = { "converttostructuredgrid", "convertstate", + "ensmean", "hofx4d", "localensembleda", "variational" diff --git a/modulefiles/GDAS/gaea.intel.lua b/modulefiles/GDAS/gaeac5.intel.lua similarity index 86% rename from modulefiles/GDAS/gaea.intel.lua rename to modulefiles/GDAS/gaeac5.intel.lua index 85d779669..d1aa1df6c 100644 --- a/modulefiles/GDAS/gaea.intel.lua +++ b/modulefiles/GDAS/gaeac5.intel.lua @@ -10,8 +10,8 @@ prepend_path("MODULEPATH", '/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/u prepend_path("MODULEPATH", '/ncrc/proj/epic/rocoto/modulefiles') -- below two lines get us access to the spack-stack modules -load("stack-intel/2023.1.0") -load("stack-cray-mpich/8.1.25") +load("stack-intel/2023.2.0") +load("stack-cray-mpich/8.1.28") -- JCSDA has 'jedi-fv3-env/unified-dev', but we should load these manually as needed load("cmake/3.23.1") load("gettext/0.20.2") @@ -44,11 +44,12 @@ load("fckit/0.11.0") load("fiat/1.2.0") load("ectrans/1.2.0") load("fms/2023.04") +load("esmf/8.5.0") load("atlas/0.35.1") load("sp/2.5.0") load("gsl-lite/0.37.0") load("libjpeg/2.1.0") -load("krb5/1.16.3") +load("krb5/1.20.1") load("libtirpc/1.3.3") load("hdf/4.2.15") load("jedi-cmake/1.4.0") @@ -84,9 +85,9 @@ local mpinproc = '-n' setenv('MPIEXEC_EXEC', mpiexec) setenv('MPIEXEC_NPROC', mpinproc) -setenv("CRTM_FIX","/gpfs/f5/ufs-ard/world-shared/GDASApp/fix/crtm/2.4.0") -setenv("GDASAPP_TESTDATA","/gpfs/f5/ufs-ard/world-shared/GDASApp/CI/data") -setenv("GDASAPP_UNIT_TEST_DATA_PATH", "/gpfs/f5/ufs-ard/world-shared/GDASApp/CI/data/test") +setenv("CRTM_FIX","/gpfs/f5/nggps_emc/world-shared/GDASApp/fix/crtm/2.4.0") +setenv("GDASAPP_TESTDATA","/gpfs/f5/nggps_emc/world-shared/GDASApp/testdata") +setenv("GDASAPP_UNIT_TEST_DATA_PATH", "/gpfs/f5/nggps_emc/world-shared/GDASApp/unittestdata") whatis("Name: ".. "pkgName") whatis("Version: ".. "pkgVersion") diff --git a/parm/aero/jcb-base.yaml.j2 b/parm/aero/jcb-base.yaml.j2 new file mode 100644 index 000000000..86988206d --- /dev/null +++ b/parm/aero/jcb-base.yaml.j2 @@ -0,0 +1,136 @@ +# Search path for model and obs for JCB +# ------------------------------------- +algorithm_path: "{{PARMgfs}}/gdas/jcb-algorithms" +app_path_algorithm: "{{PARMgfs}}/gdas/jcb-gdas/algorithm/aero" +app_path_model: "{{PARMgfs}}/gdas/jcb-gdas/model/aero" +app_path_observations: "{{PARMgfs}}/gdas/jcb-gdas/observations/aero" +app_path_observation_chronicle: "{{PARMgfs}}/gdas/jcb-gdas/observation_chronicle/aero" + + +# Places where we deviate from the generic file name of a yaml +# ------------------------------------------------------------ +final_increment_to_latlon_file: aero_final_increment_gaussian +final_increment_file: aero_final_increment_cubed_sphere +model_file: aero_model_pseudo +initial_condition_file: aero_background # Initial conditions for 4D apps is background +background_error_file: "{{BERROR_YAML}}" + +# Assimilation standard things (not prepended with model choice) +# ---------------------------- +window_begin: "{{ AERO_WINDOW_BEGIN | to_isotime }}" +window_length: "{{ AERO_WINDOW_LENGTH }}" +bound_to_include: begin +minimizer: DRPCG +final_diagnostics_departures: anlmob +final_prints_frequency: PT3H +number_of_outer_loops: 2 +analysis_variables: [mass_fraction_of_sulfate_in_air, + mass_fraction_of_hydrophobic_black_carbon_in_air, + mass_fraction_of_hydrophilic_black_carbon_in_air, + mass_fraction_of_hydrophobic_organic_carbon_in_air, + mass_fraction_of_hydrophilic_organic_carbon_in_air, + mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air, + mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air, + mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air, + mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air, + mass_fraction_of_sea_salt004_in_air] + +# Model things +# ------------ +# Geometry +aero_layout_x: {{ layout_x | default(1, true) }} +aero_layout_y: {{ layout_y | default(1, true) }} +aero_npx_ges: {{ npx_ges | default(49, true) }} +aero_npy_ges: {{ npy_ges | default(49, true) }} +aero_npz_ges: {{ npz_ges | default(127, true) }} +aero_npx_anl: {{ npx_anl | default(49, true) }} +aero_npy_anl: {{ npy_anl | default(49, true) }} +aero_npz_anl: {{ npz_anl | default(127, true) }} +aero_npx_clim_b: {{ npx_clim_b | default(49, true) }} +aero_npy_clim_b: {{ npy_clim_b | default(49, true) }} +aero_npz_clim_b: {{ npz_anl | default(127, true) }} + +aero_fv3jedi_files_path: ./fv3jedi # Ideally this would be {{DATA}}/fv3jedi but FMS + +# Background +aero_background_path: ./bkg +aero_background_ensemble_path: ./ens/mem%mem% + +# Default background time is for 3D applications +{% if DOIAU == True %} +aero_background_time_iso: "{{ AERO_WINDOW_BEGIN | to_isotime }}" +{% else %} +aero_background_time_iso: "{{ current_cycle | to_isotime }}" +{% endif %} +aero_cycle_time_iso: "{{ current_cycle | to_isotime }}" +aero_cycle_time_fv3: "{{ current_cycle | to_fv3time }}" + +# time for background error calculation for next cycle +{% set offset_td = "+6H" | to_timedelta %} +{% set background_time = current_cycle | add_to_datetime(offset_td) %} +aero_background_error_time_iso: "{{ background_time | to_isotime }}" +aero_background_error_time_fv3: "{{ background_time | to_fv3time }}" + +# Background error +aero_berror_data_directory: "{{ DATA }}/berror" +aero_berror_diffusion_directory: "{{ DATA }}/diffusion" +aero_standard_devation_path: ./stddev +aero_climatological_b_path: ./clm_stddev +aero_diagb_weight: {{ aero_diagb_weight | default(1.0, true) }} +aero_diagb_static_rescale_factor: {{aero_staticb_rescaling_factor | default(1.0, true) }} +aero_diagb_rescale_factor: {{aero_diagb_rescale | default(1.0, true) }} +aero_diagb_n_halo: {{ aero_diagb_n_halo | default(1, true) }} +aero_diagb_n_neighbors: {{ aero_diagb_n_neighbors | default(1, true) }} +aero_diagb_smooth_horiz_iter: {{ aero_diagb_smooth_horiz_iter | default(1, true) }} +aero_diagb_smooth_vert_iter: {{ aero_diagb_smooth_vert_iter | default(1, true) }} +aero_diffusion_iter: {{ aero_diffusion_iter | default(1, true) }} +aero_diffusion_horiz_len: {{ aero_diffusion_horiz_len | default(1.0, true)}} +aero_diffusion_fixed_val: {{ aero_diffusion_fixed_val | default(1.0, true)}} + +# Forecasting +aero_forecast_timestep: "{{ BKG_TSTEP }}" + +# Observation things +# ------------------ +observations: all_observations + +crtm_coefficient_path: "{{ DATA }}/crtm/" + +# Naming conventions for observational files +aero_obsdataroot_path: "{{COM_OBS}}" + +aero_obsdatain_path: "{{aero_obsdatain_path}}" +aero_obsdatain_prefix: "{{OPREFIX}}" +aero_obsdatain_suffix: ".tm00.nc" + +aero_obsdataout_path: "{{aero_obsdataout_path}}" +aero_obsdataout_prefix: diag_ +aero_obsdataout_suffix: "_{{ current_cycle | to_YMDH }}.nc" + +# Naming conventions for bias correction files +aero_obsbiasroot_path: "{{COM_CHEM_ANALYSIS_PREV}}" + +aero_obsbiasin_path: "{{DATA}}/obs/" +aero_obsbiasin_prefix: "{{GPREFIX}}" +aero_obsbiasin_suffix: ".satbias.nc" +aero_obstlapsein_prefix: "{{GPREFIX}}" +aero_obstlapsein_suffix: ".tlapse.txt" +aero_obsbiascovin_prefix: "{{GPREFIX}}" +aero_obsbiascovin_suffix: ".satbias_cov.nc" + +aero_obsbiasout_path: "{{DATA}}/bc/" +aero_obsbiasout_prefix: "{{APREFIX}}" +aero_obsbiasout_suffix: ".satbias.nc" +aero_obsbiascovout_prefix: "{{APREFIX}}" +aero_obsbiascovout_suffix: ".satbias_cov.nc" + +bias_files: + atms_n20: rad_varbc_params.tar + atms_npp: rad_varbc_params.tar + mtiasi_metop-a: rad_varbc_params.tar + mtiasi_metop-b: rad_varbc_params.tar + amsua_n19: rad_varbc_params.tar + ssmis_f17: rad_varbc_params.tar + ssmis_f18: rad_varbc_params.tar + cris-fsr_n20: rad_varbc_params.tar + cris-fsr_npp: rad_varbc_params.tar diff --git a/parm/aero/jcb-prototype_3dvar.yaml.j2 b/parm/aero/jcb-prototype_3dvar.yaml.j2 new file mode 100644 index 000000000..da4739bca --- /dev/null +++ b/parm/aero/jcb-prototype_3dvar.yaml.j2 @@ -0,0 +1,10 @@ +# Algorithm +# --------- +algorithm: 3dfgat + +# Observation things +# ------------------ +observations: +- viirs_n20_aod +- viirs_npp_aod +# - viirs_n21_aod diff --git a/parm/atm/jcb-prototype_3dvar.yaml.j2 b/parm/atm/jcb-prototype_3dvar.yaml.j2 index 7b6c80011..4330a87bd 100644 --- a/parm/atm/jcb-prototype_3dvar.yaml.j2 +++ b/parm/atm/jcb-prototype_3dvar.yaml.j2 @@ -16,8 +16,8 @@ observations: - conventional_ps - gnssro # - gpsro -# - mtiasi_metop-a -# - mtiasi_metop-b +# - iasi_metop-a +# - iasi_metop-b # - ompsnp_n20 - ompsnp_npp # - ompstc_n20 @@ -29,3 +29,4 @@ observations: # - satwnd.viirs_npp # - scatwind_ascat_metop-a # - snowcvr +# - abi_g16 diff --git a/parm/atm/jcb-prototype_lgetkf.yaml.j2 b/parm/atm/jcb-prototype_lgetkf.yaml.j2 index a1b099bb6..2ed04df3b 100644 --- a/parm/atm/jcb-prototype_lgetkf.yaml.j2 +++ b/parm/atm/jcb-prototype_lgetkf.yaml.j2 @@ -25,8 +25,8 @@ observations: - conventional_ps - gnssro # - gpsro -# - mtiasi_metop-a -# - mtiasi_metop-b +# - iasi_metop-a +# - iasi_metop-b # - ompsnp_n20 - ompsnp_npp # - ompstc_n20 diff --git a/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 b/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 index 26654b175..4b800ac8f 100644 --- a/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 +++ b/parm/atm/jcb-prototype_lgetkf_observer.yaml.j2 @@ -25,8 +25,8 @@ observations: - conventional_ps - gnssro # - gpsro -# - mtiasi_metop-a -# - mtiasi_metop-b +# - iasi_metop-a +# - iasi_metop-b # - ompsnp_n20 - ompsnp_npp # - ompstc_n20 diff --git a/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 b/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 index 677934158..b5123dde9 100644 --- a/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 +++ b/parm/atm/jcb-prototype_lgetkf_solver.yaml.j2 @@ -25,8 +25,8 @@ observations: - conventional_ps - gnssro # - gpsro -# - mtiasi_metop-a -# - mtiasi_metop-b +# - iasi_metop-a +# - iasi_metop-b # - ompsnp_n20 - ompsnp_npp # - ompstc_n20 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_gsrcsr.json b/parm/ioda/bufr2ioda/bufr2ioda_gsrcsr.json new file mode 100644 index 000000000..036f7bd4b --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_gsrcsr.json @@ -0,0 +1,16 @@ +{ + "data_format" : "bufr_d", + "data_type" : "gsrcsr", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : ['NC021046'], + "data_description" : "NC021046 ABI, GOES-16; NC021046 ABI, GOES-17, NC021046 ABI, GOES-18", + "data_provider" : "U.S. NOAA/NESDIS", + "sensor_info" : { "sensor_name": "ABI", "sensor_full_name": "Advanced Baseline Imager", "sensor_id": 617 }, + "satellite_info" : [ + { "satellite_name": "g16", "satellite_full_name": "GOES-16", "satellite_id": 270, "launch time": "20171119" }, + { "satellite_name": "g17", "satellite_full_name": "GOES-17", "satellite_id": 271, "launch time": "20180301" }, + { "satellite_name": "g18", "satellite_full_name": "GOES-18", "satellite_id": 272, "launch time": "20220301" } ] +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml b/parm/ioda/bufr2ioda/bufr2ioda_iasi.yaml similarity index 98% rename from parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml rename to parm/ioda/bufr2ioda/bufr2ioda_iasi.yaml index c15df2b62..c11e147c6 100755 --- a/parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml +++ b/parm/ioda/bufr2ioda/bufr2ioda_iasi.yaml @@ -94,7 +94,7 @@ observations: ioda: backend: netcdf - obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.mtiasi_$(splitvar).tm00.nc" + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.iasi_$(splitvar).tm00.nc" dimensions: - name: Channel diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_argo.yaml b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_argo.yaml new file mode 100644 index 000000000..7dd28f21c --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_argo.yaml @@ -0,0 +1,11 @@ +cycle_datetime: '{{ current_cycle | to_YMDH }}' +cycle_type: '{{ RUN }}' +data_description: 6-hrly in situ ARGO profiles +data_format: subpfl +data_provider: U.S. NOAA +data_type: argo +dump_directory: '{{ DMPDIR }}' +ioda_directory: '{{ COM_OBS }}' +source: NCEP data tank +subsets: SUBPFL +ocean_basin: '{{ OCEAN_BASIN_FILE }}' diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.yaml b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.yaml new file mode 100644 index 000000000..f26d341ea --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.yaml @@ -0,0 +1,11 @@ +cycle_datetime: '{{ current_cycle | to_YMDH }}' +cycle_type: '{{ RUN }}' +data_description: 6-hrly in situ Bathythermal profiles +data_format: bathy +data_provider: U.S. NOAA +data_type: bathy +dump_directory: '{{ DMPDIR }}' +ioda_directory: '{{ COM_OBS }}' +source: NCEP data tank +subsets: BATHY +ocean_basin: '{{ OCEAN_BASIN_FILE }}' diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_glider.yaml b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_glider.yaml new file mode 100644 index 000000000..cc732df94 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_glider.yaml @@ -0,0 +1,11 @@ +cycle_datetime: '{{ current_cycle | to_YMDH }}' +cycle_type: '{{ RUN }}' +data_description: 6-hrly in situ GLIDER profiles +data_format: subpfl +data_provider: U.S. NOAA +data_type: glider +dump_directory: '{{ DMPDIR }}' +ioda_directory: '{{ COM_OBS }}' +source: NCEP data tank +subsets: SUBPFL +ocean_basin: '{{ OCEAN_BASIN_FILE }}' diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.yaml b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.yaml new file mode 100644 index 000000000..01dc441ca --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.yaml @@ -0,0 +1,11 @@ +cycle_datetime: '{{ current_cycle | to_YMDH }}' +cycle_type: '{{ RUN }}' +data_description: 6-hrly in situ TESAC profiles +data_format: tesac +data_provider: U.S. NOAA +data_type: tesac +dump_directory: '{{ DMPDIR }}' +ioda_directory: '{{ COM_OBS }}' +source: NCEP data tank +subsets: TESAC +ocean_basin: '{{ OCEAN_BASIN_FILE }}' diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_xbtctd.yaml b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_xbtctd.yaml new file mode 100644 index 000000000..49d7f13d2 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_xbtctd.yaml @@ -0,0 +1,11 @@ +cycle_datetime: '{{ current_cycle | to_YMDH }}' +cycle_type: '{{ RUN }}' +data_description: 6-hrly in situ XBT/XCTD profiles +data_format: xbtctd +data_provider: U.S. NOAA +data_type: xbtctd +dump_directory: '{{ DMPDIR }}' +ioda_directory: '{{ COM_OBS }}' +source: NCEP data tank +subsets: XBTCTD +ocean_basin: '{{ OCEAN_BASIN_FILE }}' diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.yaml b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.yaml new file mode 100644 index 000000000..1385920c1 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.yaml @@ -0,0 +1,11 @@ +cycle_datetime: '{{ current_cycle | to_YMDH }}' +cycle_type: '{{ RUN }}' +data_description: 6-hrly in situ TRACKOB surface +data_format: trkob +data_provider: U.S. NOAA +data_type: trackob +dump_directory: '{{ DMPDIR }}' +ioda_directory: '{{ COM_OBS }}' +source: NCEP data tank +subsets: TRACKOB +ocean_basin: '{{ OCEAN_BASIN_FILE }}' diff --git a/parm/ioda/bufr2ioda/j2y.py b/parm/ioda/bufr2ioda/j2y.py new file mode 100644 index 000000000..c8b158517 --- /dev/null +++ b/parm/ioda/bufr2ioda/j2y.py @@ -0,0 +1,24 @@ +import json +import yaml +import argparse + +def convert_json_to_yaml(input_file, output_file): + # Load the JSON data from the input file + with open(input_file, 'r') as json_file: + json_data = json.load(json_file) + + # Convert and save as YAML in the output file + with open(output_file, 'w') as yaml_file: + yaml.dump(json_data, yaml_file, default_flow_style=False) + +if __name__ == '__main__': + # Set up argument parser + parser = argparse.ArgumentParser(description='Convert JSON to YAML.') + parser.add_argument('input_file', help='Path to the input JSON file') + parser.add_argument('output_file', help='Path to the output YAML file') + + args = parser.parse_args() + + # Perform the conversion + convert_json_to_yaml(args.input_file, args.output_file) + diff --git a/parm/soca/berror/soca_diagb.yaml.j2 b/parm/soca/berror/soca_diagb.yaml.j2 deleted file mode 100644 index c0597d358..000000000 --- a/parm/soca/berror/soca_diagb.yaml.j2 +++ /dev/null @@ -1,37 +0,0 @@ -geometry: - mom6_input_nml: mom_input.nml - fields metadata: ./fields_metadata.yaml - -date: '{{ MARINE_WINDOW_END | to_isotime }}' - -background: - date: '{{ MARINE_WINDOW_END | to_isotime }}' - basename: ./bkg/ - ocn_filename: 'ocean.bkg.f009.nc' - ice_filename: 'ice.bkg.f009.nc' - read_from_file: 1 - -background error: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_MIDDLE | to_isotime }}' - exp: bkgerr_stddev - type: incr - -variables: - name: [tocn, socn, uocn, vocn, hocn, ssh, cicen, hicen, hsnon, mom6_mld] - -rescale: 2.0 # rescales the filtered std. dev. by "rescale" -min sst: 0.0 # Added to sst bkg. err. -max ssh: 0.0 # Limits the amplitude of the unbalanced bkg err -min depth: 500.0 # zero out the bkg. error. at less than min depth -number of halo points: 4 -number of neighbors: 16 - -simple smoothing: - horizontal iterations: 10 - vertical iterations: 1 - -# TODO(G): Too slow for the below scale -#diffusion: -# horizontal: 500.0e3 -# vertical: 3.0 diff --git a/parm/soca/berror/soca_ensb.yaml.j2 b/parm/soca/berror/soca_ensb.yaml.j2 deleted file mode 100644 index 4033f41ba..000000000 --- a/parm/soca/berror/soca_ensb.yaml.j2 +++ /dev/null @@ -1,98 +0,0 @@ -# Configuration for the recentering and re-balancing of the ensemble members -geometry: - mom6_input_nml: mom_input.nml - fields metadata: ./fields_metadata.yaml - -date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - -layers variable: [hocn] - -increment variables: [tocn, socn, uocn, vocn, ssh, hocn, cicen, hicen, hsnon] - -set increment variables to zero: [ssh] - -vertical geometry: - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - basename: ./INPUT/ - ocn_filename: MOM.res.nc - read_from_file: 3 - -add recentering increment: false - -soca increments: # Could also be states, but they are read as increments - number of increments: {{ NMEM_ENS }} - pattern: '%mem%' - template: - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - basename: '{{ ENSPERT_RELPATH }}/ens/' - ocn_filename: 'ocean.%mem%.nc' - ice_filename: 'ice.%mem%.nc' - read_from_file: 3 - -steric height: - linear variable changes: - - linear variable change name: BalanceSOCA # Only the steric balance is applied - -ensemble mean output: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: ens_mean - type: incr - -ssh output: - unbalanced: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: ssh_unbal_stddev - type: incr - - steric: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: ssh_steric_stddev - type: incr - - total: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: ssh_total_stddev - type: incr - - explained variance: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: steric_explained_variance - type: incr - - recentering error: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: ssh_recentering_error - type: incr - -background error output: - datadir: ./staticb/ - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: bkgerr_stddev - type: incr - -linear variable change: - linear variable changes: - - linear variable change name: BalanceSOCA - -trajectory: - state variables: [tocn, socn, uocn, vocn, ssh, hocn, layer_depth, mld, cicen, hicen, hsnon] - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - basename: ./INPUT/ - ocn_filename: MOM.res.nc - ice_filename: cice.res.nc - read_from_file: 1 - -output increment: - # TODO: Revert this when fms can take more than 128 charactres file names - datadir: '{{ ENSPERT_RELPATH }}/enspert/' - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - exp: trash - type: incr - output file: 'ocn.pert.steric.%mem%.nc' - pattern: '%mem%' diff --git a/parm/soca/berror/soca_ensweights.yaml.j2 b/parm/soca/berror/soca_ensweights.yaml.j2 deleted file mode 100644 index f677d2a8d..000000000 --- a/parm/soca/berror/soca_ensweights.yaml.j2 +++ /dev/null @@ -1,37 +0,0 @@ -geometry: - mom6_input_nml: mom_input.nml - fields metadata: ./fields_metadata.yaml - -date: '{{ MARINE_WINDOW_MIDDLE | to_isotime }}' - -variables: - ice: [cicen, hicen, hsnon] - ocean: [tocn, socn, uocn, vocn, ssh] - -background: - date: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' - basename: ./INPUT/ - ocn_filename: MOM.res.nc - ice_filename: cice.res.nc - read_from_file: 1 - -weights: - # Need to provide weights^2 when reading from file - ice: 0.0025 # 5% of original variance - ocean: 0.0625 # 25% " " - # Apply localized weights to the ocean ens. B - ocean local weights: - - lon: -172.0 - lat: 11.0 - amplitude: -1.0 - length scale: 700.0 - - lon: -160.0 - lat: 12.0 - amplitude: -1.0 - length scale: 700.0 - -output: - datadir: ./ - date: '{{ MARINE_WINDOW_MIDDLE | to_isotime }}' - exp: ens_weights - type: incr diff --git a/parm/soca/berror/soca_parameters_diffusion_hz.yaml.j2 b/parm/soca/berror/soca_parameters_diffusion_hz.yaml.j2 deleted file mode 100644 index 7d3a78cfb..000000000 --- a/parm/soca/berror/soca_parameters_diffusion_hz.yaml.j2 +++ /dev/null @@ -1,37 +0,0 @@ -geometry: &geom - mom6_input_nml: mom_input.nml - fields metadata: ./fields_metadata.yaml - -background: - read_from_file: 1 - basename: ./INPUT/ - ocn_filename: MOM.res.nc - ice_filename: cice.res.nc - date: '{{ MARINE_WINDOW_END | to_isotime }}' - state variables: [ssh] - -background error: - covariance model: SABER - saber central block: - saber block name: diffusion - geometry: *geom - calibration: - normalization: - method: randomization - iterations: 10000 - - groups: - - horizontal: - model file: - date: '{{ MARINE_WINDOW_END | to_isotime }}' - basename: ./ - ocn_filename: ocn.cor_rh.incr.0001-01-01T00:00:00Z.nc - model variable: ssh - write: - filepath: ./staticb/hz_ocean - - - horizontal: - as gaussian: true - fixed value: 50000.0 - write: - filepath: ./staticb/hz_ice diff --git a/parm/soca/berror/soca_parameters_diffusion_vt.yaml.j2 b/parm/soca/berror/soca_parameters_diffusion_vt.yaml.j2 deleted file mode 100644 index 76ab67e94..000000000 --- a/parm/soca/berror/soca_parameters_diffusion_vt.yaml.j2 +++ /dev/null @@ -1,33 +0,0 @@ -geometry: &geom - mom6_input_nml: mom_input.nml - fields metadata: ./fields_metadata.yaml - -background: - read_from_file: 1 - basename: ./INPUT/ - ocn_filename: MOM.res.nc - ice_filename: cice.res.nc - date: '{{ MARINE_WINDOW_MIDDLE | to_isotime }}' - state variables: [tocn] - -background error: - covariance model: SABER - saber central block: - saber block name: diffusion - geometry: *geom - calibration: - normalization: - # NOTE, not actually used here, since the normalization spec is only used for hz - method: randomization #< other option is "brute force" - iterations: 1000 #< in the real world you'll want to use 1e4 or so - - groups: - - vertical: - as gaussian: true - model file: - date: '{{ MARINE_WINDOW_MIDDLE | to_isotime }}' - basename: ./ - ocn_filename: vt_scales.nc - model variable: tocn - write: - filepath: ./staticb/vt_ocean diff --git a/parm/soca/berror/soca_setcorscales.yaml b/parm/soca/berror/soca_setcorscales.yaml deleted file mode 100644 index 0a91777a1..000000000 --- a/parm/soca/berror/soca_setcorscales.yaml +++ /dev/null @@ -1,23 +0,0 @@ -resolution: - mom6_input_nml: mom_input.nml - fields metadata: ./fields_metadata.yaml - -date: 0001-01-01T00:00:00Z - -corr variables: [ssh] - -scales: - vert layers: 5 # in units of layer - ssh: - rossby mult: 1.00 - min grid mult: 2.0 - -rh output: - datadir: ./ - exp: cor_rh - type: incr - -rv output: - datadir: ./ - exp: cor_rv - type: incr diff --git a/parm/soca/berror/soca_vtscales.yaml.j2 b/parm/soca/berror/soca_vtscales.yaml.j2 deleted file mode 100644 index 8f68b1517..000000000 --- a/parm/soca/berror/soca_vtscales.yaml.j2 +++ /dev/null @@ -1,13 +0,0 @@ -gridspec_filename: soca_gridspec.nc -restart_filename: ./INPUT/MOM.res.nc -mld_filename: './staticb/ocn.bkgerr_stddev.incr.{{ MARINE_WINDOW_END | to_isotime }}.nc' -output_filename: ./vt_scales.nc -output_variable_vt: Temp -output_variable_hz: ave_ssh - -VT_MIN: 5 -VT_MAX: 15 - -HZ_ROSSBY_MULT: 1.0 -HZ_MAX: 200e3 -HZ_MIN_GRID_MULT: 2.0 diff --git a/parm/soca/ensda/stage_ens_mem.yaml.j2 b/parm/soca/ensda/stage_ens_mem.yaml.j2 index e30a337e7..d0dca6e18 100644 --- a/parm/soca/ensda/stage_ens_mem.yaml.j2 +++ b/parm/soca/ensda/stage_ens_mem.yaml.j2 @@ -9,8 +9,7 @@ # create working directories ###################################### mkdir: -- "{{ DATAenspert }}/ens" - + - "{{ ENSPERT_RELPATH }}/ens" ###################################### # copy ensemble background files ###################################### @@ -22,6 +21,6 @@ copy: '${YMD}':gPDY, '${HH}':gcyc, '${MEMDIR}':"mem" + '%03d' % mem} %} - - ["{{ COM_OCEAN_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/{{ GDUMP_ENS }}.ocean.t{{ gcyc }}z.inst.f006.nc", "{{ DATAenspert }}/ens/ocean.{{ mem }}.nc"] - - ["{{ COM_ICE_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/{{ GDUMP_ENS }}.ice.t{{ gcyc }}z.inst.f006.nc", "{{ DATAenspert }}/ens/ice.{{ mem }}.nc"] + - ["{{ COM_OCEAN_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/{{ GDUMP_ENS }}.ocean.t{{ gcyc }}z.inst.f006.nc", "{{ ENSPERT_RELPATH }}/ens/ocean.{{ mem }}.nc"] + - ["{{ COM_ICE_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/{{ GDUMP_ENS }}.ice.t{{ gcyc }}z.inst.f006.nc", "{{ ENSPERT_RELPATH }}/ens/ice.{{ mem }}.nc"] {% endfor %} diff --git a/parm/soca/gridgen/gridgen.yaml b/parm/soca/gridgen/gridgen.yaml deleted file mode 100644 index 34fbdeca6..000000000 --- a/parm/soca/gridgen/gridgen.yaml +++ /dev/null @@ -1,5 +0,0 @@ -geometry: - geom_grid_file: soca_gridspec.nc - mom6_input_nml: mom_input.nml - fields metadata: fields_metadata.yaml - rossby file: rossrad.nc diff --git a/parm/soca/letkf/letkf.yaml.j2 b/parm/soca/letkf/letkf.yaml.j2 index f4c4d4875..1d5e93d7f 100644 --- a/parm/soca/letkf/letkf.yaml.j2 +++ b/parm/soca/letkf/letkf.yaml.j2 @@ -1,3 +1,5 @@ +{% set gcyc = previous_cycle | strftime("%H") %} + geometry: geom_grid_file: soca_gridspec.nc mom6_input_nml: mom_input.nml @@ -11,11 +13,11 @@ background: members from template: template: date: '{{ WINDOW_MIDDLE | to_isotime }}' - ocn_filename: "{{ RUN }}.ocean.t{{ gcyc }}z.inst.f006.nc" - ice_filename: "{{ RUN }}.ice.t{{ gcyc }}z.inst.f006.nc" + ocn_filename: "ocean.%mem%.nc" + ice_filename: "ice.%mem%.nc" read_from_file: 1 - basename: ./ens/mem%mem% - state variables: [socn, tocn, ssh, uocn, vocn, cicen] + basename: {{ ENSPERT_RELPATH }}/ens/ + state variables: [socn, tocn, ssh, hocn, uocn, vocn, cicen] pattern: '%mem%' nmembers: {{ NMEM_ENS }} @@ -38,32 +40,31 @@ local ensemble DA: mult: 1.1 output: - datadir: data_output/ + datadir: letkf_output/ date: *date exp: letkf type: ens output mean prior: - datadir: data_output/ + datadir: letkf_output/ date: *date - exp: letkf + exp: letkf.mean_prior type: fc output variance prior: - datadir: data_output/ + datadir: letkf_output/ date: *date - exp: letkf + exp: letkf.var_prior type: fc output variance posterior: - datadir: data_output/ + datadir: letkf_output/ date: *date - exp: letkf + exp: letkf.var_post type: an output increment: - datadir: data_output/ + datadir: letkf_output/ date: *date exp: letkf.inc - type: an - + type: ens diff --git a/parm/soca/letkf/letkf_save.yaml.j2 b/parm/soca/letkf/letkf_save.yaml.j2 new file mode 100644 index 000000000..194cf222e --- /dev/null +++ b/parm/soca/letkf/letkf_save.yaml.j2 @@ -0,0 +1,20 @@ +{% set PDY = current_cycle | to_YMD %} +{% set cyc = current_cycle | strftime("%H") %} +{% set timestr = WINDOW_BEGIN | to_isotime %} +###################################### +# save letkf analysis to comout +###################################### + +copy: +{% for mem in range(1, NMEM_ENS + 1) %} + {% set tmpl_dict = {'${ROTDIR}':ROTDIR, + '${RUN}': GDUMP_ENS, + '${YMD}':PDY, + '${HH}':cyc, + '${MEMDIR}':"mem" + '%03d' % mem} %} + {% set COMOUT_OCEAN_LETKF_MEM = COM_OCEAN_LETKF_TMPL | replace_tmpl(tmpl_dict) %} + {% set COMOUT_ICE_LETKF_MEM = COM_ICE_LETKF_TMPL | replace_tmpl(tmpl_dict) %} + + - ["{{ DATA }}/letkf_output/ocn.letkf.ens.{{ mem }}.{{ timestr }}.PT3H.nc", "{{ COMOUT_OCEAN_LETKF_MEM }}/{{ GDUMP_ENS }}.ocean.t{{ cyc }}z.analysis.nc"] + - ["{{ DATA }}/letkf_output/ice.letkf.ens.{{ mem }}.{{ timestr }}.PT3H.nc", "{{ COMOUT_ICE_LETKF_MEM }}/{{ GDUMP_ENS }}.ice.t{{ cyc }}z.analysis.nc"] +{% endfor %} diff --git a/parm/soca/letkf/letkf_stage.yaml.j2 b/parm/soca/letkf/letkf_stage.yaml.j2 index 019e1ba37..233e45eb9 100644 --- a/parm/soca/letkf/letkf_stage.yaml.j2 +++ b/parm/soca/letkf/letkf_stage.yaml.j2 @@ -1,11 +1,30 @@ -###################################### -# create working directories +{% set PDY = current_cycle | to_YMD %} +{% set cyc = current_cycle | strftime("%H") %} +{% set gcyc = previous_cycle | strftime("%H") %} ###################################### mkdir: -- "{{ DATA }}/Data" +- "{{ DATA }}/letkf_output" - "{{ DATA }}/obs" -copy: +- "{{ DATA }}/diags" +- "{{ COMOUT_OCEAN_LETKF }}" +- "{{ COMOUT_ICE_LETKF }}" +###################################### +# make comout directories ###################################### -# copy mom input template +{% for mem in range(1, NMEM_ENS + 1) %} + {% set tmpl_dict = {'${ROTDIR}':ROTDIR, + '${RUN}':GDUMP_ENS, + '${YMD}':PDY, + '${HH}':cyc, + '${MEMDIR}':"mem" + '%03d' % mem} %} +- "{{ COM_OCEAN_LETKF_TMPL | replace_tmpl(tmpl_dict) }}" +- "{{ COM_ICE_LETKF_TMPL | replace_tmpl(tmpl_dict) }}" +{% endfor %} ###################################### +# copy mom input template and det bkg +###################################### +copy: - ["{{ PARMgfs }}/gdas/soca/fms/input.nml", "{{ DATA }}/mom_input.nml.tmpl"] +- ["{{ PARMgfs }}/gdas/soca/fields_metadata.yaml", "{{ DATA }}/fields_metadata.yaml"] +- ["{{ COMIN_OCEAN_HISTORY_PREV }}/gdas.ocean.t{{ gcyc }}z.inst.f009.nc", "{{ DATA }}/INPUT/MOM.res.nc"] +- ["{{ COMIN_ICE_HISTORY_PREV }}/gdas.ice.t{{ gcyc }}z.inst.f009.nc", "{{ DATA }}/INPUT/cice.res.nc"] diff --git a/parm/soca/marine-jcb-base.yaml b/parm/soca/marine-jcb-base.yaml index cb5230f1c..d07edcd8c 100644 --- a/parm/soca/marine-jcb-base.yaml +++ b/parm/soca/marine-jcb-base.yaml @@ -17,7 +17,7 @@ final_increment_file: marine_final_increment # Assimilation standard things (not prepended with model choice) # ---------------------------- -window_begin: '{{MARINE_WINDOW_BEGIN}}' +window_begin: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' window_length: '{{MARINE_WINDOW_LENGTH}}' bound_to_include: begin minimizer: RPCG @@ -31,6 +31,10 @@ analysis_variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh] # ------------ marine_window_begin: '{{MARINE_WINDOW_BEGIN}}' marine_window_middle: '{{MARINE_WINDOW_MIDDLE}}' +marine_window_begin_iso: '{{ MARINE_WINDOW_BEGIN | to_isotime }}' +marine_window_middle_iso: '{{ MARINE_WINDOW_MIDDLE | to_isotime }}' +marine_window_end_iso: '{{ MARINE_WINDOW_END | to_isotime }}' +enspert_relpath: '{{ ENSPERT_RELPATH }}' # Geometry marine_soca_files_path: . @@ -42,11 +46,10 @@ marine_background_time: '{{MARINE_WINDOW_BEGIN_ISO}}' # Pseudo model marine_forecast_timestep: PT3H -marine_pseudo_model_states: !INC 'bkg_list.yaml' # Background error model background_error_file: '{{berror_model}}' -marine_number_ensemble_members: '{{nmem_ens}}' +marine_number_ensemble_members: '{{NMEM_ENS}}' marine_stddev_time: '{{MARINE_WINDOW_MIDDLE}}' # Observations diff --git a/parm/soca/obs/config/adt_rads_all.yaml b/parm/soca/obs/config/adt_rads_all.yaml index 177e58a8f..e5b6dee26 100644 --- a/parm/soca/obs/config/adt_rads_all.yaml +++ b/parm/soca/obs/config/adt_rads_all.yaml @@ -17,6 +17,13 @@ obs operator: name: ADT obs error: covariance model: diagonal +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 obs filters: - filter: Domain Check where: diff --git a/parm/soca/obs/config/icec_amsr2_north.yaml b/parm/soca/obs/config/icec_amsr2_north.yaml index 1b1509671..bd454ae88 100644 --- a/parm/soca/obs/config/icec_amsr2_north.yaml +++ b/parm/soca/obs/config/icec_amsr2_north.yaml @@ -43,3 +43,10 @@ obs filters: where: - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/icec_amsr2_south.yaml b/parm/soca/obs/config/icec_amsr2_south.yaml index 5373b1bee..eedf6800d 100644 --- a/parm/soca/obs/config/icec_amsr2_south.yaml +++ b/parm/soca/obs/config/icec_amsr2_south.yaml @@ -43,3 +43,10 @@ obs filters: where: - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_profile_argo.yaml b/parm/soca/obs/config/insitu_profile_argo.yaml index b2abaab2c..e533966b8 100644 --- a/parm/soca/obs/config/insitu_profile_argo.yaml +++ b/parm/soca/obs/config/insitu_profile_argo.yaml @@ -625,3 +625,10 @@ obs filters: - ObsError/salinity coefs: - 1000.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_abi_g16_l3c.yaml b/parm/soca/obs/config/sst_abi_g16_l3c.yaml index d96135409..e4bef888d 100644 --- a/parm/soca/obs/config/sst_abi_g16_l3c.yaml +++ b/parm/soca/obs/config/sst_abi_g16_l3c.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_abi_g17_l3c.yaml b/parm/soca/obs/config/sst_abi_g17_l3c.yaml index 8843da412..c34f5f777 100644 --- a/parm/soca/obs/config/sst_abi_g17_l3c.yaml +++ b/parm/soca/obs/config/sst_abi_g17_l3c.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_ahi_h08_l3c.yaml b/parm/soca/obs/config/sst_ahi_h08_l3c.yaml index d1842320c..0bf07bab1 100644 --- a/parm/soca/obs/config/sst_ahi_h08_l3c.yaml +++ b/parm/soca/obs/config/sst_ahi_h08_l3c.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_avhrr_ma_l3u.yaml b/parm/soca/obs/config/sst_avhrr_ma_l3u.yaml index 71f5947a7..1223c1f7f 100644 --- a/parm/soca/obs/config/sst_avhrr_ma_l3u.yaml +++ b/parm/soca/obs/config/sst_avhrr_ma_l3u.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_avhrr_mb_l3u.yaml b/parm/soca/obs/config/sst_avhrr_mb_l3u.yaml index 090b47ae6..92cde48fe 100644 --- a/parm/soca/obs/config/sst_avhrr_mb_l3u.yaml +++ b/parm/soca/obs/config/sst_avhrr_mb_l3u.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_avhrr_mc_l3u.yaml b/parm/soca/obs/config/sst_avhrr_mc_l3u.yaml index ab03d548f..fa706616c 100644 --- a/parm/soca/obs/config/sst_avhrr_mc_l3u.yaml +++ b/parm/soca/obs/config/sst_avhrr_mc_l3u.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_viirs_n20_l3u.yaml b/parm/soca/obs/config/sst_viirs_n20_l3u.yaml index 5941d746c..e78f0f77a 100644 --- a/parm/soca/obs/config/sst_viirs_n20_l3u.yaml +++ b/parm/soca/obs/config/sst_viirs_n20_l3u.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/sst_viirs_npp_l3u.yaml b/parm/soca/obs/config/sst_viirs_npp_l3u.yaml index 1d0e447ed..6fd0e47e3 100644 --- a/parm/soca/obs/config/sst_viirs_npp_l3u.yaml +++ b/parm/soca/obs/config/sst_viirs_npp_l3u.yaml @@ -54,3 +54,10 @@ obs filters: - ObsError/seaSurfaceTemperature coefs: - 1.0 +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml index c11dc1ace..0ac8ab5af 100644 --- a/parm/soca/obs/obs_list.yaml +++ b/parm/soca/obs/obs_list.yaml @@ -25,15 +25,15 @@ observers: #- !INC ${MARINE_OBS_YAML_DIR}/icec_ssmis_f17_l2.yaml # in situ: monthly -#- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_bathy.yaml +- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_bathy.yaml - !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_argo.yaml -#- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_glider.yaml -#- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_tesac.yaml +- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_glider.yaml +- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_tesac.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_tesac_salinity.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_marinemammal.yaml -#- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_xbtctd.yaml +- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_xbtctd.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_surface_altkob.yaml -#- !INC ${MARINE_OBS_YAML_DIR}/insitu_surface_trkob.yaml +- !INC ${MARINE_OBS_YAML_DIR}/insitu_surface_trkob.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_surface_trkob_salinity.yaml # in situ: daily diff --git a/parm/soca/soca_ens_bkg_stage.yaml.j2 b/parm/soca/soca_ens_bkg_stage.yaml.j2 index a74853b3b..b6f1416fb 100644 --- a/parm/soca/soca_ens_bkg_stage.yaml.j2 +++ b/parm/soca/soca_ens_bkg_stage.yaml.j2 @@ -18,7 +18,7 @@ copy: # define variables # Declare a dict of search and replace terms to GDUMP on each template {% set tmpl_dict = {'ROTDIR':ROTDIR, - 'RUN': RUN, + 'RUN': GDUMP_ENS, 'YMD':gPDY, 'HH':gcyc, 'MEMDIR':"mem" + '%03d' % mem} %} @@ -35,6 +35,6 @@ copy: {% set com_prev_ocn.COMIN_OCEAN_HISTORY_MEM = com_prev_ocn.COMIN_OCEAN_HISTORY_MEM.replace(search_term, replace_term) %} {% set com_prev_ice.COMIN_ICE_HISTORY_MEM = com_prev_ice.COMIN_ICE_HISTORY_MEM.replace(search_term, replace_term) %} {% endfor %} - - ["{{ com_prev_ocn.COMIN_OCEAN_HISTORY_MEM }}/{{ RUN }}.ocean.t{{ gcyc }}z.inst.f006.nc", "{{ DATAens }}/ens/mem{{ '%03d' % mem }}/{{ RUN }}.ocean.t{{ gcyc }}z.inst.f006.nc"] - - ["{{ com_prev_ice.COMIN_ICE_HISTORY_MEM }}/{{ RUN }}.ice.t{{ gcyc }}z.inst.f006.nc", "{{ DATAens }}/ens/mem{{ '%03d' % mem }}/{{ RUN }}.ice.t{{ gcyc }}z.inst.f006.nc"] + - ["{{ com_prev_ocn.COMIN_OCEAN_HISTORY_MEM }}/{{ GDUMP_ENS }}.ocean.t{{ gcyc }}z.inst.f006.nc", "{{ DATAens }}/ens/mem{{ '%03d' % mem }}/{{ GDUMP_ENS }}.ocean.t{{ gcyc }}z.inst.f006.nc"] + - ["{{ com_prev_ice.COMIN_ICE_HISTORY_MEM }}/{{ GDUMP_ENS }}.ice.t{{ gcyc }}z.inst.f006.nc", "{{ DATAens }}/ens/mem{{ '%03d' % mem }}/{{ GDUMP_ENS }}.ice.t{{ gcyc }}z.inst.f006.nc"] {% endfor %} diff --git a/parm/soca/soca_fix_stage_025.yaml.j2 b/parm/soca/soca_fix_stage_025.yaml.j2 index 2c41dcad4..f7b334e7d 100644 --- a/parm/soca/soca_fix_stage_025.yaml.j2 +++ b/parm/soca/soca_fix_stage_025.yaml.j2 @@ -1,4 +1,3 @@ -# TODO(AFE): make resolution dependent mkdir: - "{{ DATA }}/INPUT" ###################################### diff --git a/parm/soca/soca_fix_stage_100.yaml.j2 b/parm/soca/soca_fix_stage_100.yaml.j2 index 0869f7d34..e2f4137a2 100644 --- a/parm/soca/soca_fix_stage_100.yaml.j2 +++ b/parm/soca/soca_fix_stage_100.yaml.j2 @@ -1,4 +1,3 @@ -# TODO(AFE): make resolution dependent mkdir: - "{{ DATA }}/INPUT" ###################################### diff --git a/parm/soca/soca_fix_stage_500.yaml.j2 b/parm/soca/soca_fix_stage_500.yaml.j2 index 0b25073ba..6d6930e0b 100644 --- a/parm/soca/soca_fix_stage_500.yaml.j2 +++ b/parm/soca/soca_fix_stage_500.yaml.j2 @@ -1,4 +1,3 @@ -# TODO(AFE): make resolution dependent mkdir: - "{{ DATA }}/INPUT" ###################################### @@ -9,7 +8,6 @@ copy: - ["{{ SOCA_INPUT_FIX_DIR }}/field_table", "{{ DATA }}/field_table"] - ["{{ SOCA_INPUT_FIX_DIR }}/diag_table", "{{ DATA }}/diag_table"] - ["{{ SOCA_INPUT_FIX_DIR }}/MOM_input", "{{ DATA }}/MOM_input"] -- ["{{ SOCA_INPUT_FIX_DIR }}/fields_metadata.yaml", "{{ DATA }}/fields_metadata.yaml"] - ["{{ SOCA_INPUT_FIX_DIR }}/obsop_name_map.yaml", "{{ DATA }}/obsop_name_map.yaml"] - ["{{ SOCA_INPUT_FIX_DIR }}/INPUT/grid_spec.nc", "{{ DATA }}/INPUT/grid_spec.nc"] - ["{{ SOCA_INPUT_FIX_DIR }}/INPUT/hycom1_25.nc", "{{ DATA }}/INPUT/hycom1_25.nc"] diff --git a/parm/soca/soca_utils_stage.yaml.j2 b/parm/soca/soca_utils_stage.yaml.j2 index 26570abe3..462cfccd0 100644 --- a/parm/soca/soca_utils_stage.yaml.j2 +++ b/parm/soca/soca_utils_stage.yaml.j2 @@ -4,4 +4,3 @@ copy: - ["{{ HOMEgfs }}/parm/gdas/soca/fields_metadata.yaml", "{{ DATA }}/fields_metadata.yaml"] - ["{{ HOMEgfs }}/parm/gdas/soca/obsop_name_map.yaml", "{{ DATA }}/obsop_name_map.yaml"] -- ["{{ HOMEgfs }}/parm/gdas/soca/gridgen/gridgen.yaml", "{{ DATA }}/gridgen.yaml"] diff --git a/test/atm/global-workflow/setup_workflow_exp.sh b/test/atm/global-workflow/setup_workflow_exp.sh index d793733ae..47bb0d82e 100755 --- a/test/atm/global-workflow/setup_workflow_exp.sh +++ b/test/atm/global-workflow/setup_workflow_exp.sh @@ -10,7 +10,7 @@ idate=2021032312 edate=2021032318 app=ATM starttype='warm' -gfscyc='4' +interval='6' resdetatmos='48' resensatmos='48' nens=3 @@ -37,7 +37,7 @@ $srcdir/../../workflow/setup_expt.py gfs cycled --idate $idate \ --edate $edate \ --app $app \ --start $starttype \ - --gfs_cyc $gfscyc \ + --interval $interval \ --resdetatmos $resdetatmos \ --resensatmos $resensatmos \ --nens $nens \ diff --git a/test/gw-ci/CMakeLists.txt b/test/gw-ci/CMakeLists.txt index 6c9faedf1..4f1a969e3 100644 --- a/test/gw-ci/CMakeLists.txt +++ b/test/gw-ci/CMakeLists.txt @@ -19,18 +19,39 @@ function(add_cycling_tests pslot YAML_PATH HOMEgfs RUNTESTS PROJECT_SOURCE_DIR T list(GET DATES_LIST 1 FULL_CYCLE) # stage IC's - message(STATUS "staging the 1/2 cycle IC's for ${test_name} ctest") + message(STATUS "staging the 1/2 cycle IC's for ${pslot} ctest") add_test(NAME ${test_name}_gdas_stage_ic_${HALF_CYCLE} COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} gdas_stage_ic ${HALF_CYCLE}" WORKING_DIRECTORY ${RUNTESTS}) set_tests_properties(${test_name}_gdas_stage_ic_${HALF_CYCLE} PROPERTIES LABELS "manual") + # stage ensemble ics + if (letkf) + message(STATUS "preparing enkfgdas_stage_ic for ${pslot} ctest") + add_test(NAME ${test_name}_enkfgdas_stage_ic_${HALF_CYCLE} + COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} enkfgdas_stage_ic ${HALF_CYCLE}" + WORKING_DIRECTORY ${RUNTESTS}) + set_tests_properties(${test_name}_enkfgdas_stage_ic_${HALF_CYCLE} PROPERTIES LABELS "manual") + endif() + # 1/2 cycle gdas_fcst message(STATUS "preparing 1/2 cycle gdas_fcst for ${pslot} ctest") - add_test(NAME ${test_name}_gdas_fcst_${HALF_CYCLE} + add_test(NAME ${test_name}_gdas_fcst_seg0_${HALF_CYCLE} COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} gdas_fcst_seg0 ${HALF_CYCLE}" WORKING_DIRECTORY ${RUNTESTS}) - set_tests_properties(${test_name}_gdas_fcst_${HALF_CYCLE} PROPERTIES LABELS "manual") + set_tests_properties(${test_name}_gdas_fcst_seg0_${HALF_CYCLE} PROPERTIES LABELS "manual") + + # 1/2 cycle enkfgdas_fcst + if (letkf) + set(ENS_MEMS "mem001" "mem002" "mem003") + foreach(ENS_MEM ${ENS_MEMS}) + message(STATUS "preparing 1/2 cycle enkfgdas_fcst_${ENS_MEM} for ${pslot} ctest") + add_test(NAME ${test_name}_enkfgdas_fcst_${ENS_MEM}_${HALF_CYCLE} + COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} enkfgdas_fcst_${ENS_MEM} ${HALF_CYCLE}" + WORKING_DIRECTORY ${RUNTESTS}) + set_tests_properties(${test_name}_enkfgdas_fcst_${ENS_MEM}_${HALF_CYCLE} PROPERTIES LABELS "manual") + endforeach() + endif() # Select the list of tasks to run for the full cycle message(STATUS "Tasks ${TASK_LIST}") @@ -62,6 +83,24 @@ if (WORKFLOW_TESTS) "gdas_marineanlfinal" ) add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}") + + # WCDA, low-res, ensemble da + # ------------- + set(pslot "WCDA-hyb-C48mx500") + set(letkf TRUE) + set(YAML_PATH ${HOMEgfs}/ci/cases/pr/C48mx500_hybAOWCDA.yaml) + set(TASK_LIST + "gdas_prepoceanobs" + "gdas_marineanlletkf" + # TODO(AFE) waiting until these are working for hybrid + # "gdas_marinebmat" + # "gdas_marineanlinit" + # "gdas_marineanlvar" + # "gdas_marineanlchkpt" + # "gdas_marineanlfinal" + ) + add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}") + set(letkf FALSE) endif() option(RUN_GW_CI "Enable the global-workflow CI tests" OFF) diff --git a/test/marine/CMakeLists.txt b/test/marine/CMakeLists.txt index 92c880a58..c75a5664b 100644 --- a/test/marine/CMakeLists.txt +++ b/test/marine/CMakeLists.txt @@ -28,6 +28,14 @@ install(FILES ${test_input} # bufr to ioda tests: ########################################################################### +set(TEST_WORKING_DIR ${PROJECT_BINARY_DIR}/test/marine) +set(MARINE_BUFR2IODA_DIR ${PROJECT_SOURCE_DIR}/ush/ioda/bufr2ioda/marine) +set(MARINE_BUFR2IODA_DIR ${MARINE_BUFR2IODA_DIR}/b2i) +set(CONFIG_DIR ${PROJECT_SOURCE_DIR}/test/marine/testinput) +set(TESTREF_DIR ${PROJECT_SOURCE_DIR}/test/marine/testref) +set(PYIODACONV_DIR "${PROJECT_SOURCE_DIR}/build/lib/python3.10/") + + # prepare a test.yaml file from test.yaml.in by replacing # placeholder patterns __BUFRINPUTDIR__ and __IODAOUTPUTDIR__ and __OCEANBASIN__ # with actual directory paths @@ -46,14 +54,6 @@ function(CREATE_CONFIG_FILE endfunction() -set(TEST_WORKING_DIR ${PROJECT_BINARY_DIR}/test/marine) - -set(MARINE_BUFR2IODA_DIR ${PROJECT_SOURCE_DIR}/ush/ioda/bufr2ioda/marine) -set(MARINE_BUFR2IODA_DIR ${MARINE_BUFR2IODA_DIR}/b2i) -set(CONFIG_DIR ${PROJECT_SOURCE_DIR}/test/marine/testinput) -set(TESTREF_DIR ${PROJECT_SOURCE_DIR}/test/marine/testref) - - function(CHECK_AND_SET_PATH PATH1 PATH2 RESULT_VAR) # Check if PATH1 exists if(EXISTS ${PATH1}) @@ -157,9 +157,15 @@ function(ADD_INSITU_TEST testname testbufr) COMMAND ${MARINE_BUFR2IODA_DIR}/${TEST}.py -c ${CONFIG_FILE} -t ${TESTREF_DIR}/${TESTREF_FILE} WORKING_DIRECTORY ${TEST_WORKING_DIR} ) + set_property( + TEST test_gdasapp_${TEST} + APPEND PROPERTY + ENVIRONMENT "PYTHONPATH=${PYIODACONV_DIR}:$ENV{PYTHONPATH}" + ) endfunction() + if (GENERATE_BUFR2IODA_TESTS) ADD_INSITU_TEST("profile_argo" "subpfl") ADD_INSITU_TEST("profile_bathy" "bathy") diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 997c394fa..ab039aebf 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -21,8 +21,8 @@ case $(hostname -f) in dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9 dlogin10.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood10 - gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58 - gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58 + gaea5[1-8]) MACHINE_ID=gaeac5 ;; ### gaea51-58 + gaea5[1-8].ncrc.gov) MACHINE_ID=gaeac5 ;; ### gaea51-58 gaea6[1-8]) MACHINE_ID=gaeac6 ;; ### gaea61-68 gaea6[1-8].ncrc.gov) MACHINE_ID=gaeac6 ;; ### gaea61-68 @@ -84,9 +84,12 @@ elif [[ -d /work ]]; then else MACHINE_ID=orion fi -elif [[ -d /gpfs && -d /ncrc ]]; then - # We are on GAEA. - MACHINE_ID=gaea +elif [[ -d /gpfs/f5 ]]; then + # We are on GAEAC5. + MACHINE_ID=gaeac5 +elif [[ -d /gpfs/f6 ]]; then + # We are on GAEAC6. + MACHINE_ID=gaeac6 elif [[ -d /data/prod ]]; then # We are on SSEC's S4 MACHINE_ID=s4 diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gsrcsr.py b/ush/ioda/bufr2ioda/bufr2ioda_gsrcsr.py new file mode 100755 index 000000000..dc4fc20b0 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_gsrcsr.py @@ -0,0 +1,555 @@ +#!/usr/bin/env python3 +import argparse +import calendar +import datetime +import json +import math +import os +import time +from datetime import datetime + +import numpy as np +import numpy.ma as ma +from wxflow import Logger + +from pyioda import ioda_obs_space as ioda_ospace +from pyiodaconv import bufr + +# Define and initialize global variables +global float32_fill_value +global int32_fill_value +global int64_fill_value + +float32_fill_value = np.float32(0) +int32_fill_value = np.int32(0) +int64_fill_value = np.int64(0) + + +def bufr_to_ioda(config, logger): + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + subsets = config["subsets"] + data_format = config["data_format"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + satellite_info_array = config["satellite_info"] + sensor_name = config["sensor_info"]["sensor_name"] + sensor_full_name = config["sensor_info"]["sensor_full_name"] + sensor_id = config["sensor_info"]["sensor_id"] + + # Get derived parameters + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + reference_time = datetime.strptime(cycle, "%Y%m%d%H") + reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") + + # General informaton + converter = "BUFR to IODA Converter" + process_level = "Level-2" + platform_description = "NOAA Series of Geostationary Operational Environmental Satellites - 3rd generation since 2016" + sensor_description = "Spinning Enhanced Visible and InfraRed Imager;12 channels, 1 narrow-bandwidth, 1 high-resolution broad-bandwidth VIS" + + logger.info(f"sensor_name = {sensor_name}") + logger.info(f"sensor_full_name = {sensor_full_name}") + logger.info(f"sensor_id = {sensor_id}") + logger.info(f"reference_time = {reference_time}") + + bufrfile = f"{cycle_type}.t{hh}z.{data_type}.tm00.{data_format}" + DATA_PATH = os.path.join( + dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), "atmos", bufrfile + ) + if not os.path.isfile(DATA_PATH): + logger.info(f"The DATA_PATH is: {DATA_PATH}") + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + logger.info("Making QuerySet") + q = bufr.QuerySet(subsets) + + # MetaData + q.add("latitude", "*/CLATH") + q.add("longitude", "*/CLONH") + q.add("satelliteId", "*/SAID") + q.add("year", "*/YEAR") + q.add("month", "*/MNTH") + q.add("day", "*/DAYS") + q.add("hour", "*/HOUR") + q.add("minute", "*/MINU") + q.add("second", "*/SECO") + q.add("sensorId", "*/SIID[1]") + q.add("sensorZenithAngle", "*/SAZA") + q.add("sensorCentralFrequency", "*/CSRADSEQ/SCCF") + q.add("solarZenithAngle", "*/SOZA") + q.add("cloudFree", "*/CLFRASEQ{2}/NCLDMNT") + q.add("brightnessTemperature", "*/CSRADSEQ/TMBRST") + q.add("ClearSkyStdDev", "*/SDRADSQ/SDTB") + q.add("solarAzimuthAngle", "*/SOLAZI") + q.add("sensorAzimuthAngle", "*/BEARAZ") + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Processing time for making QuerySet : {running_time} seconds") + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + logger.info("Executing QuerySet to get ResultSet") + with bufr.File(DATA_PATH) as f: + try: + r = f.execute(q) + except Exception as err: + logger.info(f'Return with {err}') + return + # MetaData + satid = r.get("satelliteId") + instid = r.get("sensorId") + year = r.get("year") + month = r.get("month") + day = r.get("day") + hour = r.get("hour") + minute = r.get("minute") + second = r.get("second") + lat = r.get("latitude") + lon = r.get("longitude") + satzenang = r.get("sensorZenithAngle") + chanfreq = r.get("sensorCentralFrequency", type="float") + BT = r.get("brightnessTemperature") + clrStdDev = r.get("ClearSkyStdDev") + cldFree = r.get("cloudFree", type="float") + solzenang = r.get("solarZenithAngle") + solaziang = r.get("solarAzimuthAngle") + sataziang = r.get("sensorAzimuthAngle") + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + timestamp = r.get_datetime( + "year", "month", "day", "hour", "minute", "second" + ).astype(np.int64) + + # Global variables declaration + # Set global fill values + float32_fill_value = satzenang.fill_value + int32_fill_value = satid.fill_value + int64_fill_value = timestamp.fill_value.astype(np.int64) + + end_time = time.time() + running_time = end_time - start_time + logger.info( + f"Processing time for executing QuerySet to get ResultSet : {running_time} seconds" + ) + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + rounded_values = np.where(satzenang % 1 > 0.5, np.ceil(satzenang), np.floor(satzenang)) + # Convert to integer and add 1 + scanpos = rounded_values.astype(np.int32) + 1 + cloudAmount = 100. - cldFree + # Define the conversion factor from degrees to radians + deg2rad = math.pi/180.0 + sataziang = sataziang*deg2rad + viewang = np.full_like(solzenang, float32_fill_value, dtype=np.float32) + # Define Channel dimension for channels 4 to 11 since the other channel values are missing + channel_start = 7 + channel_end = 16 + channum = np.arange(channel_start, channel_end + 1) + # Define wavenumbers for each satellite ID + wavenum_values_dict = { + 270: np.array( + [ + 257037.4, + 162052.9, + 144355.4, + 136322.8, + 118422, + 104089.1, + 96800.1, + 89400.06, + 81529.43, + 75378.98, + ], + dtype=np.float32, + ), + 271: np.array( + [ + 256550.4, + 159490.2, + 136128.6, + 114870.3, + 103420.4, + 92938.72, + 83886.68, + 75122.19, + 83886.68, + 75122.19, + ], + dtype=np.float32, + ), + } + wavenum_fill_value = float32_fill_value + + logger.info("Creating derived variables") + + end_time = time.time() + running_time = end_time - start_time + logger.info( + f"Processing time for creating derived variables : {running_time} seconds" + ) + + # ===================================== + # Split output based on satellite id + # Create IODA ObsSpace + # Write IODA output + # ===================================== + logger.info("Create IODA ObsSpace and Write IODA output based on satellite ID") + + # Find nique satellite identifiers in data to process + unique_satids = np.unique(satid) + logger.info(f"Number of Unique satellite identifiers: {len(unique_satids)}") + logger.info(f"Unique satellite identifiers: {unique_satids}") + logger.debug(f"Loop through unique satellite identifier {unique_satids}") + total_ob_processed = 0 + for sat in unique_satids.tolist(): + start_time = time.time() + + matched = False + for satellite_info in satellite_info_array: + if satellite_info["satellite_id"] == sat: + matched = True + satellite_id = satellite_info["satellite_id"] + satellite_name = satellite_info["satellite_name"] + satinst = sensor_name.lower() + "_" + satellite_name.lower() + logger.debug(f"Split data for {satinst} satid = {sat}") + + if matched: + if satellite_id in wavenum_values_dict: + # Extract the wavenum values for the current satellite ID + Wavenum = wavenum_values_dict[satellite_id] + else: + # If the satellite ID is not in the dictionary + logger.debug(f"satellite ID is not in the dictionary {satellite_id}") + + # Define a boolean mask to subset data from the original data object + satelite_mask = satid == sat + # Define a boolean mask based on the condition 0 < satzenang2 < 80 + satzenang_mask = np.logical_and(0 < satzenang, satzenang < 80) + combined_mask = satzenang_mask & satelite_mask + # MetaData + lon2 = lon[combined_mask] + lat2 = lat[combined_mask] + timestamp2 = timestamp[combined_mask] + satid2 = satid[combined_mask] + instid2 = instid[combined_mask] + satzenang2 = satzenang[combined_mask] + chanfreq2 = chanfreq[6:16] + scanpos2 = scanpos[combined_mask] + solzenang2 = solzenang[combined_mask] + cldFree2 = cldFree[combined_mask] + cloudAmount2 = cloudAmount[combined_mask] + BT2 = BT[combined_mask] + clrStdDev2 = clrStdDev[combined_mask] + viewang2 = viewang.flatten()[combined_mask] + sataziang2 = sataziang.flatten()[combined_mask] + solaziang2 = solaziang.flatten()[combined_mask] + + # Timestamp Range + timestamp2_min = datetime.fromtimestamp(timestamp2.min()) + timestamp2_max = datetime.fromtimestamp(timestamp2.max()) + + # Check unique observation time + unique_timestamp2 = np.unique(timestamp2) + logger.debug(f"Processing output for satid {sat}") + logger.info(f"number of unique_timestamp2 {len(unique_timestamp2)}") + logger.info(f"unique_timestamp2 {unique_timestamp2}") + + # Create the dimensions + dims = { + "Location": np.arange(0, BT2.shape[0]), + "Channel": np.arange(channel_start, channel_end + 1), + } + + # Create IODA ObsSpace + iodafile = f"{cycle_type}.t{hh}z.{satinst}.tm00.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.info(f"Create output file : {OUTPUT_PATH}") + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode="w", dim_dict=dims) + + # Create Global attributes + logger.debug("Write global attributes") + obsspace.write_attr("Converter", converter) + obsspace.write_attr("sourceFiles", bufrfile) + obsspace.write_attr("description", data_description) + obsspace.write_attr("datetimeReference", reference_time) + obsspace.write_attr( + "datetimeRange", [str(timestamp2_min), str(timestamp2_max)] + ) + obsspace.write_attr("sensor", sensor_id) + obsspace.write_attr("platform", satellite_id) + obsspace.write_attr("platformCommonName", satellite_name) + obsspace.write_attr("sensorCommonName", sensor_name) + obsspace.write_attr("processingLevel", process_level) + obsspace.write_attr("platformLongDescription", platform_description) + obsspace.write_attr("sensorLongDescription", sensor_description) + + # Create IODA variables + logger.debug("Write variables: name, type, units, and attributes") + + # Sensor Channel Number + obsspace.create_var( + "MetaData/sensorChannelNumber", + dim_list=["Channel"], + dtype=np.int32, + fillval=int32_fill_value, + ).write_attr("long_name", "Sensor Channel Number").write_data(channum) + + # Sensor Central Frequency + obsspace.create_var( + "MetaData/sensorCentralFrequency", + dim_list=["Channel"], + dtype=chanfreq2.dtype, + fillval=chanfreq2.fill_value, + ).write_attr("units", "Hz").write_attr( + "long_name", "Satellite Channel Center Frequency" + ).write_data( + chanfreq2 + ) + + # Sensor Central Wavenumber + obsspace.create_var( + "MetaData/sensorCentralWavenumber", + dim_list=["Channel"], + dtype=Wavenum.dtype, + fillval=wavenum_fill_value, + ).write_attr("units", "m-1").write_attr( + "long_name", "Sensor Central Wavenumber" + ).write_data( + Wavenum + ) + + if np.any(combined_mask): + # Longitude + obsspace.create_var( + "MetaData/longitude", dtype=lon2.dtype, fillval=lon2.fill_value + ).write_attr("units", "degrees_east").write_attr( + "valid_range", np.array([-180, 180], dtype=np.float32) + ).write_attr( + "long_name", "Longitude" + ).write_data( + lon2 + ) + + # Latitude + obsspace.create_var( + "MetaData/latitude", dtype=lat2.dtype, fillval=lat2.fill_value + ).write_attr("units", "degrees_north").write_attr( + "valid_range", np.array([-90, 90], dtype=np.float32) + ).write_attr( + "long_name", "Latitude" + ).write_data( + lat2 + ) + + # Datetime + obsspace.create_var( + "MetaData/dateTime", dtype=np.int64, fillval=int64_fill_value + ).write_attr("units", "seconds since 1970-01-01T00:00:00Z").write_attr( + "long_name", "Datetime" + ).write_data( + timestamp2 + ) + + # Satellite Identifier + obsspace.create_var( + "MetaData/satelliteIdentifier", + dtype=satid2.dtype, + fillval=satid2.fill_value, + ).write_attr("long_name", "Satellite Identifier").write_data(satid2) + + # Instrument Identifier + obsspace.create_var( + "MetaData/instrumentIdentifier", + dtype=instid2.dtype, + fillval=instid2.fill_value, + ).write_attr("long_name", "Satellite Instrument Identifier").write_data( + instid2 + ) + + # Scan Position (derived variable, need to specified fill value explicitly) + obsspace.create_var( + "MetaData/sensorScanPosition", + dtype=scanpos2.astype(np.int32).dtype, + fillval=int32_fill_value, + ).write_attr("long_name", "Sensor Scan Position").write_data(scanpos2) + + # Sensor Zenith Angle + obsspace.create_var( + "MetaData/sensorZenithAngle", + dtype=satzenang2.dtype, + fillval=satzenang2.fill_value, + ).write_attr("units", "degree").write_attr( + "valid_range", np.array([0, 90], dtype=np.float32) + ).write_attr( + "long_name", "Sensor Zenith Angle" + ).write_data( + satzenang2 + ) + + # Sensor Azimuth Angle + obsspace.create_var( + "MetaData/sensorAzimuthAngle", + dtype=np.float32, + fillval=sataziang2.fill_value, + ).write_attr("units", "degree").write_attr( + "valid_range", np.array([0, 360], dtype=np.float32) + ).write_attr( + "long_name", "Sensor Azimuth Angle" + ).write_data( + sataziang2 + ) + + # Solar Azimuth Angle + obsspace.create_var( + "MetaData/solarAzimuthAngle", + dtype=np.float32, + fillval=solaziang2.fill_value, + ).write_attr("units", "degree").write_attr( + "valid_range", np.array([0, 360], dtype=np.float32) + ).write_attr( + "long_name", "Solar Azimuth Angle" + ).write_data( + solaziang2 + ) + + # Sensor View Angle + obsspace.create_var( + "MetaData/sensorViewAngle", + dtype=np.float32, + fillval=viewang2.fill_value, + ).write_attr("units", "degree").write_attr( + "long_name", "Sensor View Angle" + ).write_data( + viewang2 + ) + + # Solar Zenith Angle + obsspace.create_var( + "MetaData/solarZenithAngle", + dtype=solzenang2.dtype, + fillval=solzenang2.fill_value, + ).write_attr("units", "degree").write_attr( + "valid_range", np.array([0, 180], dtype=np.float32) + ).write_attr( + "long_name", "Solar Zenith Angle" + ).write_data( + solzenang2 + ) + + # Cloud free + obsspace.create_var( + "MetaData/cloudFree", + dtype=cldFree2.dtype, fillval=int32_fill_value + ).write_attr("units", "1").write_attr( + "valid_range", np.array([0, 100], dtype=np.int32) + ).write_attr( + "long_name", "Amount Segment Cloud Free" + ).write_data( + cldFree2 + ) + + # Cloud amount based on computation + obsspace.create_var( + "MetaData/cloudAmount", + dtype=cloudAmount2.dtype, + fillval=cloudAmount2.fill_value, + ).write_attr("units", "1").write_attr( + "valid_range", np.array([0, 100], dtype=np.float32) + ).write_attr( + "long_name", "Amount of cloud coverage in layer" + ).write_data( + cloudAmount2 + ) + + # ObsType based on computation method/spectral band + obsspace.create_var( + "ObsValue/brightnessTemperature", + dim_list=["Location", "Channel"], + dtype=np.float32, + fillval=BT2.fill_value, + ).write_attr("units", "k").write_attr( + "long_name", "Brightness Temperature" + ).write_data( + BT2 + ) + + obsspace.create_var( + "ClearSkyStdDev/brightnessTemperature", + dim_list=["Location", "Channel"], + dtype=np.float32, + fillval=clrStdDev2.fill_value, + ).write_attr( + "long_name", "Standard Deviation Brightness Temperature" + ).write_data( + clrStdDev2 + ) + + else: + logger.debug( + "No valid values (0 None: 'ATM_WINDOW_MIDDLE': window_middle_iso, 'DATA': DATA, 'dump': self.task_config.RUN, - 'fv3jedi_stage_files': self.task_config.FV3JEDI_STAGE_YAML, - 'fv3jedi_stage': self.task_config.FV3JEDI_STAGE_YAML, 'stage_dir': DATA, 'soca_input_fix_dir': self.task_config.SOCA_INPUT_FIX_DIR, 'NMEM_ENS': self.task_config.NMEM_ENS, diff --git a/ush/soca/prep_ocean_obs.py b/ush/soca/prep_ocean_obs.py index 1d6b55e8e..da7b2da6a 100644 --- a/ush/soca/prep_ocean_obs.py +++ b/ush/soca/prep_ocean_obs.py @@ -10,6 +10,7 @@ from wxflow import (chdir, FileHandler, logit, + parse_j2yaml, save_as_yaml, Task, YAMLFile) @@ -74,6 +75,7 @@ def initialize(self): SOCA_INPUT_FIX_DIR = self.task_config['SOCA_INPUT_FIX_DIR'] ocean_mask_src = os.path.join(SOCA_INPUT_FIX_DIR, 'RECCAP2_region_masks_all_v20221025.nc') ocean_mask_dest = os.path.join(self.task_config.DATA, 'RECCAP2_region_masks_all_v20221025.nc') + self.task_config['OCEAN_BASIN_FILE'] = ocean_mask_dest try: FileHandler({'copy': [[ocean_mask_src, ocean_mask_dest]]}).sync() @@ -90,11 +92,15 @@ def initialize(self): logger.critical(f"OBSPREP_YAML file {OBSPREP_YAML} does not exist") raise FileNotFoundError - JSON_TMPL_DIR = self.task_config.JSON_TMPL_DIR - BUFR2IODA_PY_DIR = self.task_config.BUFR2IODA_PY_DIR + # TODO (AFE): this should be in the task config file in g-w + BUFR2IODA_TMPL_DIR = os.path.join(self.task_config.HOMEgfs, 'parm/gdas/ioda/bufr2ioda') + # TODO (AFE): this should be in the task config file in g-w, and reaches into GDASApp + # in order to avoid touching the g-w until we know this will remain a task + BUFR2IODA_PY_DIR = os.path.join(self.task_config.HOMEgfs, 'sorc/gdas.cd/ush/ioda/bufr2ioda/marine/b2i') COMIN_OBS = self.task_config.COMIN_OBS COMOUT_OBS = self.task_config['COMOUT_OBS'] + OCEAN_BASIN_FILE = self.task_config['OCEAN_BASIN_FILE'] if not os.path.exists(COMOUT_OBS): os.makedirs(COMOUT_OBS) @@ -146,32 +152,34 @@ def initialize(self): obsprep_space['window end'] = self.window_end ioda_filename = f"{RUN}.t{cyc:02d}z.{obs_space_name}.{cdatestr}.nc4" obsprep_space['output file'] = ioda_filename + ioda_config_file = obtype + '2ioda.yaml' # set up the config file for conversion to IODA for bufr and # netcdf files respectively if obsprep_space['type'] == 'bufr': - gen_bufr_json_config = {'RUN': RUN, - 'current_cycle': cdate, - 'DMPDIR': COMIN_OBS, - 'COM_OBS': COMIN_OBS} - json_config_file = os.path.join(COMIN_OBS, - f"{obtype}_{cdatestr}.json") - obsprep_space['conversion config file'] = json_config_file + bufrconv_config = { + 'RUN': RUN, + 'current_cycle': cdate, + 'DMPDIR': COMIN_OBS, + 'COM_OBS': COMIN_OBS, + 'OCEAN_BASIN_FILE': OCEAN_BASIN_FILE} + obsprep_space['conversion config file'] = ioda_config_file bufr2iodapy = BUFR2IODA_PY_DIR + '/bufr2ioda_' + obtype + '.py' obsprep_space['bufr2ioda converter'] = bufr2iodapy - tmpl_filename = 'bufr2ioda_' + obtype + '.json' - template = os.path.join(JSON_TMPL_DIR, tmpl_filename) + tmpl_filename = 'bufr2ioda_' + obtype + '.yaml' + bufrconv_template = os.path.join(BUFR2IODA_TMPL_DIR, tmpl_filename) + try: - gen_bufr_json(gen_bufr_json_config, template, json_config_file) + bufrconv = parse_j2yaml(bufrconv_template, bufrconv_config) + bufrconv.save(ioda_config_file) except Exception as e: - logger.warning(f"An exeception {e} occured while trying to run gen_bufr_json") + logger.warning(f"An exeception {e} occured while trying to create BUFR2IODA config") logger.warning(f"obtype {obtype} will be skipped") break # go to next observer in OBS_YAML obsspaces_to_convert.append({"obs space": obsprep_space}) elif obsprep_space['type'] == 'nc': - ioda_config_file = obtype + '2ioda.yaml' obsprep_space['conversion config file'] = ioda_config_file save_as_yaml(obsprep_space, ioda_config_file) @@ -252,14 +260,16 @@ def finalize(self): for obsspace_to_save in obsspaces_to_save['observations']: - output_file = obsspace_to_save['output file'] - conv_config_file = obsspace_to_save['conversion config file'] + output_file = os.path.basename(obsspace_to_save['output file']) + conv_config_file = os.path.basename(obsspace_to_save['conversion config file']) output_file_dest = os.path.join(COMOUT_OBS, output_file) conv_config_file_dest = os.path.join(COMOUT_OBS, conv_config_file) try: FileHandler({'copy': [[output_file, output_file_dest]]}).sync() FileHandler({'copy': [[conv_config_file, conv_config_file_dest]]}).sync() + except Exception as e: + logger.warning(f"An exeception {e} occured while trying to run gen_bufr_json") except OSError: logger.warning(f"Obs file not found, possible IODA converter failure)") continue diff --git a/ush/soca/prep_ocean_obs_utils.py b/ush/soca/prep_ocean_obs_utils.py index 11b18fd37..9ecb06464 100755 --- a/ush/soca/prep_ocean_obs_utils.py +++ b/ush/soca/prep_ocean_obs_utils.py @@ -68,13 +68,12 @@ def run_netcdf_to_ioda(obsspace_to_convert, OCNOBS2IODAEXEC): def run_bufr_to_ioda(obsspace_to_convert): logger.info(f"running run_bufr_to_ioda on {obsspace_to_convert['name']}") - json_output_file = obsspace_to_convert['conversion config file'] + bufrconv_yaml = obsspace_to_convert['conversion config file'] bufr2iodapy = obsspace_to_convert['bufr2ioda converter'] try: - subprocess.run(['python', bufr2iodapy, '-c', json_output_file, '-v'], check=True) - logger.info(f"ran ioda converter on obs space {obsspace_to_convert['name']} successfully") + subprocess.run(['python', bufr2iodapy, '-c', bufrconv_yaml], check=True) return 0 except subprocess.CalledProcessError as e: - logger.warning(f"bufr2ioda converter failed with error {e}, \ + logger.warning(f"bufr2ioda converter failed with error >{e}<, \ return code {e.returncode}") return e.returncode