From 9597117e5ed8d394c76aba4563d20eb73c785411 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 10 Jun 2024 19:24:28 -0500 Subject: [PATCH] Update dependency checking --- .../generated/test_complete_run_chrysalis.cfg | 2 +- ...st_min_case_add_dependencies_chrysalis.cfg | 113 ++++++++++++++++++ tests/integration/template_complete_run.cfg | 2 +- .../template_min_case_add_dependencies.cfg | 113 ++++++++++++++++++ tests/integration/utils.py | 1 + zppy/e3sm_diags.py | 46 ++++--- zppy/global_time_series.py | 32 +++-- zppy/ilamb.py | 42 +++---- zppy/utils.py | 23 ++++ 9 files changed, 317 insertions(+), 57 deletions(-) create mode 100644 tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg create mode 100644 tests/integration/template_min_case_add_dependencies.cfg diff --git a/tests/integration/generated/test_complete_run_chrysalis.cfg b/tests/integration/generated/test_complete_run_chrysalis.cfg index 9737545c..6ab27bfe 100644 --- a/tests/integration/generated/test_complete_run_chrysalis.cfg +++ b/tests/integration/generated/test_complete_run_chrysalis.cfg @@ -194,6 +194,6 @@ nodes = 8 partition = "compute" short_name = 'v2.LR.historical_0201' ts_num_years = 2 -years = "1850:1854:2", +years = "1850:1854:4", [[ land_monthly ]] diff --git a/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg b/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg new file mode 100644 index 00000000..76b12996 --- /dev/null +++ b/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg @@ -0,0 +1,113 @@ +# Test the add_dependencies function +# That touches the following cases: +# e3sm_diags: enso_diags, qbo (depend_on_ts) +# e3sm_diags: streamflow +# e3sm_diags: tropical_subseasonal +# global_time_series: use_atm, use_lnd (custom, original_8_no_ocn, original_8) +# ilamb: with or without land_only (base case, land_only) + +[default] +case = "v3.LR.historical_0051" +constraint = "" +dry_run = "False" +environment_commands = "" +input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0051 +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_add_dependencies_output/unique_id/v3.LR.historical_0051" +partition = "debug" +qos = "regular" +# ts is in 5 year increments +ts_num_years = 5 +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_add_dependencies_www/unique_id" +# We want to produce diagnostics for 10 years. +years = "1985:1995:10", + +[ts] +active = True +e3sm_to_cmip_environment_commands = "" +walltime = "00:50:00" +# ts is in 5 year increments +years = "1985:1995:5" + + [[ atm_monthly_180x360_aave ]] + # for e3sm_diags (depend_on_ts), ilamb + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + ts_fmt = "cmip" + vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST" + + [[ land_monthly ]] + # for ilamb + extra_vars = "landfrac" + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" + ts_fmt = "cmip" + vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" + + [[ rof_monthly ]] + # for e3sm_diags (streamflow) + extra_vars = 'areatotal2' + frequency = "monthly" + input_files = "mosart.h0" + input_subdir = "archive/rof/hist" + mapping_file = "" + vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" + + [[ atm_daily_180x360_aave ]] + # for e3sm_diags (tropical_subseasonal) + frequency = "daily" + input_files = "eam.h1" + vars = "PRECT" + + [[ atm_monthly_glb ]] + # for global_time_series (custom) + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + mapping_file = "glb" + vars = "TREFHT,AODDUST" + + [[ lnd_monthly_glb ]] + # for global_time_series (custom) + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "glb" + vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" + +[e3sm_diags] +active = True +grid = '180x360_aave' +multiprocessing = True +num_workers = 8 +partition = "compute" +qos = "regular" +short_name = "v3.LR.historical_0051" +walltime = "5:00:00" + + [[ atm_monthly_180x360_aave ]] + sets = "enso_diags","qbo","streamflow","tropical_subseasonal", + # streamflow_obs_ts is determined automatically + +[global_time_series] +# ...skipping because of dependency status file missing +# /lcrc/group/e3sm/ac.forsyth2/zppy_min_case_ilamb_output/test_599-min-case-v1/v3.LR.historical_0051/post/scripts/ts_atm_monthly_glb_1985-1994-0010.status +active = True +experiment_name = "v3.LR.historical_0051" +figstr = "v3.LR.historical_0051" +plots_original="" +plots_atm = "TREFHT,AODDUST" +plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +walltime = "00:30:00" + +[ilamb] +active = True +nodes = 8 +partition = "compute" +short_name = "v3.LR.historical_0051" +ts_num_years = 30 +walltime = "2:00:00" diff --git a/tests/integration/template_complete_run.cfg b/tests/integration/template_complete_run.cfg index 8b440bdb..ee8bf927 100644 --- a/tests/integration/template_complete_run.cfg +++ b/tests/integration/template_complete_run.cfg @@ -194,6 +194,6 @@ nodes = 8 partition = "#expand partition_long#" short_name = 'v2.LR.historical_0201' ts_num_years = 2 -years = "1850:1854:2", +years = "1850:1854:4", [[ land_monthly ]] diff --git a/tests/integration/template_min_case_add_dependencies.cfg b/tests/integration/template_min_case_add_dependencies.cfg new file mode 100644 index 00000000..d4012735 --- /dev/null +++ b/tests/integration/template_min_case_add_dependencies.cfg @@ -0,0 +1,113 @@ +# Test the add_dependencies function +# That touches the following cases: +# e3sm_diags: enso_diags, qbo (depend_on_ts) +# e3sm_diags: streamflow +# e3sm_diags: tropical_subseasonal +# global_time_series: use_atm, use_lnd (custom, original_8_no_ocn, original_8) +# ilamb: with or without land_only (base case, land_only) + +[default] +case = "#expand case_name#" +constraint = "#expand constraint#" +dry_run = "#expand dry_run#" +environment_commands = "#expand environment_commands#" +input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/#expand case_name# +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +output = "#expand user_output#zppy_min_case_add_dependencies_output/#expand unique_id#/#expand case_name#" +partition = "#expand partition_short#" +qos = "#expand qos_short#" +# ts is in 5 year increments +ts_num_years = 5 +www = "#expand user_www#zppy_min_case_add_dependencies_www/#expand unique_id#" +# We want to produce diagnostics for 10 years. +years = "1985:1995:10", + +[ts] +active = True +e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" +walltime = "00:50:00" +# ts is in 5 year increments +years = "1985:1995:5" + + [[ atm_monthly_180x360_aave ]] + # for e3sm_diags (depend_on_ts), ilamb + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + ts_fmt = "cmip" + vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST" + + [[ land_monthly ]] + # for ilamb + extra_vars = "landfrac" + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" + ts_fmt = "cmip" + vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" + + [[ rof_monthly ]] + # for e3sm_diags (streamflow) + extra_vars = 'areatotal2' + frequency = "monthly" + input_files = "mosart.h0" + input_subdir = "archive/rof/hist" + mapping_file = "" + vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" + + [[ atm_daily_180x360_aave ]] + # for e3sm_diags (tropical_subseasonal) + frequency = "daily" + input_files = "eam.h1" + vars = "PRECT" + + [[ atm_monthly_glb ]] + # for global_time_series (custom) + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + mapping_file = "glb" + vars = "TREFHT,AODDUST" + + [[ lnd_monthly_glb ]] + # for global_time_series (custom) + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "glb" + vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" + +[e3sm_diags] +active = True +grid = '180x360_aave' +multiprocessing = True +num_workers = 8 +partition = "#expand partition_long#" +qos = "#expand qos_long#" +short_name = "#expand case_name#" +walltime = "#expand diags_walltime#" + + [[ atm_monthly_180x360_aave ]] + sets = "enso_diags","qbo","streamflow","tropical_subseasonal", + # streamflow_obs_ts is determined automatically + +[global_time_series] +# ...skipping because of dependency status file missing +# /lcrc/group/e3sm/ac.forsyth2/zppy_min_case_ilamb_output/test_599-min-case-v1/v3.LR.historical_0051/post/scripts/ts_atm_monthly_glb_1985-1994-0010.status +active = True +experiment_name = "#expand case_name#" +figstr = "#expand case_name#" +plots_original="" +plots_atm = "TREFHT,AODDUST" +plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +walltime = "00:30:00" + +[ilamb] +active = True +nodes = 8 +partition = "#expand partition_long#" +short_name = "#expand case_name#" +ts_num_years = 30 +walltime = "2:00:00" diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 8f24d4fe..9a65b059 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -267,6 +267,7 @@ def generate_cfgs(unified_testing=False, dry_run=False): "bundles", "complete_run", "debug", + "min_case_add_dependencies", "min_case_e3sm_diags_depend_on_climo_mvm_1", "min_case_e3sm_diags_depend_on_climo_mvm_2", "min_case_e3sm_diags_depend_on_climo", diff --git a/zppy/e3sm_diags.py b/zppy/e3sm_diags.py index bac2c2da..76262f98 100644 --- a/zppy/e3sm_diags.py +++ b/zppy/e3sm_diags.py @@ -1,10 +1,12 @@ import os import pprint +from typing import List import jinja2 from zppy.bundle import handle_bundles from zppy.utils import ( + add_dependencies, checkStatus, getTasks, getYears, @@ -31,7 +33,7 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file): # noqa: C901 return existing_bundles # --- Generate and submit e3sm_diags scripts --- - dependencies = [] + dependencies: List[str] = [] for c in tasks: @@ -216,28 +218,34 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file): # noqa: C901 or ("qbo" in c["sets"]) or ("area_mean_time_series" in c["sets"]) ): - dependencies.append( - os.path.join( - scriptDir, - "ts_%s_%04d-%04d-%04d.status" - % (ts_sub, start_yr, end_yr, c["ts_num_years"]), - ) + add_dependencies( + dependencies, + scriptDir, + "ts", + ts_sub, + start_yr, + end_yr, + c["ts_num_years"], ) if "streamflow" in c["sets"]: - dependencies.append( - os.path.join( - scriptDir, - "ts_rof_monthly_%04d-%04d-%04d.status" - % (start_yr, end_yr, c["ts_num_years"]), - ) + add_dependencies( + dependencies, + scriptDir, + "ts", + "rof_monthly", + start_yr, + end_yr, + c["ts_num_years"], ) if "tropical_subseasonal" in c["sets"]: - dependencies.append( - os.path.join( - scriptDir, - "ts_%s_%04d-%04d-%04d.status" - % (ts_daily_sub, start_yr, end_yr, c["ts_num_years"]), - ) + add_dependencies( + dependencies, + scriptDir, + "ts", + ts_daily_sub, + start_yr, + end_yr, + c["ts_num_years"], ) with open(settingsFile, "w") as sf: p = pprint.PrettyPrinter(indent=2, stream=sf) diff --git a/zppy/global_time_series.py b/zppy/global_time_series.py index 9af839fc..0ca682fd 100644 --- a/zppy/global_time_series.py +++ b/zppy/global_time_series.py @@ -1,10 +1,12 @@ import os import pprint +from typing import List import jinja2 from zppy.bundle import handle_bundles from zppy.utils import ( + add_dependencies, checkStatus, getTasks, getYears, @@ -126,30 +128,34 @@ def global_time_series(config, scriptDir, existing_bundles, job_ids_file): # no makeExecutable(scriptFile) # List of dependencies - dependencies = [] + dependencies: List[str] = [] # Add Time Series dependencies if c["use_atm"]: # Iterate from year1 to year2 incrementing by the number of years per time series file. for yr in range(c["year1"], c["year2"], c["ts_num_years"]): start_yr = yr end_yr = yr + c["ts_num_years"] - 1 - dependencies.append( - os.path.join( - scriptDir, - "ts_%s_%04d-%04d-%04d.status" - % ("atm_monthly_glb", start_yr, end_yr, c["ts_num_years"]), - ) + add_dependencies( + dependencies, + scriptDir, + "ts", + "atm_monthly_glb", + start_yr, + end_yr, + c["ts_num_years"], ) if c["use_lnd"]: for yr in range(c["year1"], c["year2"], c["ts_num_years"]): start_yr = yr end_yr = yr + c["ts_num_years"] - 1 - dependencies.append( - os.path.join( - scriptDir, - "ts_%s_%04d-%04d-%04d.status" - % ("lnd_monthly_glb", start_yr, end_yr, c["ts_num_years"]), - ) + add_dependencies( + dependencies, + scriptDir, + "ts", + "lnd_monthly_glb", + start_yr, + end_yr, + c["ts_num_years"], ) if c["use_ocn"]: # Add MPAS Analysis dependencies diff --git a/zppy/ilamb.py b/zppy/ilamb.py index 206582a3..f1928f8b 100644 --- a/zppy/ilamb.py +++ b/zppy/ilamb.py @@ -1,10 +1,12 @@ import os import pprint +from typing import List import jinja2 from zppy.bundle import handle_bundles from zppy.utils import ( + add_dependencies, checkStatus, getTasks, getYears, @@ -30,7 +32,7 @@ def ilamb(config, scriptDir, existing_bundles, job_ids_file): return existing_bundles # --- Generate and submit ilamb scripts --- - dependencies = [] + dependencies: List[str] = [] for c in tasks: @@ -54,30 +56,24 @@ def ilamb(config, scriptDir, existing_bundles, job_ids_file): c["ilamb_obs"] = os.path.join(ilamb_obs_prefix, ilamb_obs_suffix) # List of dependencies - dependencies.append( - os.path.join( - scriptDir, - "ts_%s_%04d-%04d-%04d.status" - % ( - c["ts_land_subsection"], - c["year1"], - c["year2"], - c["ts_num_years"], - ), - ), + add_dependencies( + dependencies, + scriptDir, + "ts", + c["ts_land_subsection"], + c["year1"], + c["year2"], + c["ts_num_years"], ) if not c["land_only"]: - dependencies.append( - os.path.join( - scriptDir, - "ts_%s_%04d-%04d-%04d.status" - % ( - c["ts_atm_subsection"], - c["year1"], - c["year2"], - c["ts_num_years"], - ), - ), + add_dependencies( + dependencies, + scriptDir, + "ts", + c["ts_atm_subsection"], + c["year1"], + c["year2"], + c["ts_num_years"], ) prefix = "ilamb_%04d-%04d" % ( diff --git a/zppy/utils.py b/zppy/utils.py index 77697016..96c3a180 100644 --- a/zppy/utils.py +++ b/zppy/utils.py @@ -297,3 +297,26 @@ def print_url(c, task): print(f"URL: {base_url}{www_suffix}/{case}/{task}") else: print(f"Could not determine URL from www={www}") + + +# ----------------------------------------------------------------------------- +def add_dependencies( + dependencies: List[str], + scriptDir: str, + prefix: str, + sub: str, + start_yr: int, + end_yr: int, + num_years: int, +): + y1: int = start_yr + y2: int = start_yr + num_years - 1 + while y2 <= end_yr: + dependencies.append( + os.path.join( + scriptDir, + "%s_%s_%04d-%04d-%04d.status" % (prefix, sub, y1, y2, num_years), + ) + ) + y1 += num_years + y2 += num_years