Update dependency checking

tests/integration/generated/test_complete_run_chrysalis.cfg

@@ -194,6 +194,6 @@ nodes = 8
 partition = "compute"
 short_name = 'v2.LR.historical_0201'
 ts_num_years = 2
-years = "1850:1854:2",
+years = "1850:1854:4",
 
   [[ land_monthly ]]

tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg

@@ -0,0 +1,113 @@
+# Test the add_dependencies function
+# That touches the following cases:
+# e3sm_diags: enso_diags, qbo (depend_on_ts)
+# e3sm_diags: streamflow
+# e3sm_diags: tropical_subseasonal
+# global_time_series: use_atm, use_lnd (custom, original_8_no_ocn, original_8)
+# ilamb: with or without land_only (base case, land_only)
+
+[default]
+case = "v3.LR.historical_0051"
+constraint = ""
+dry_run = "False"
+environment_commands = ""
+input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0051
+input_subdir = archive/atm/hist
+mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_add_dependencies_output/unique_id/v3.LR.historical_0051"
+partition = "debug"
+qos = "regular"
+# ts is in 5 year increments
+ts_num_years = 5
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_add_dependencies_www/unique_id"
+# We want to produce diagnostics for 10 years.
+years = "1985:1995:10",
+
+[ts]
+active = True
+e3sm_to_cmip_environment_commands = ""
+walltime = "00:50:00"
+# ts is in 5 year increments
+years = "1985:1995:5"
+
+  [[ atm_monthly_180x360_aave ]]
+  # for e3sm_diags (depend_on_ts), ilamb
+  frequency = "monthly"
+  input_files = "eam.h0"
+  input_subdir = "archive/atm/hist"
+  ts_fmt = "cmip"
+  vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"
+
+  [[ land_monthly ]]
+  # for ilamb
+  extra_vars = "landfrac"
+  frequency = "monthly"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
+  ts_fmt = "cmip"
+  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+
+  [[ rof_monthly ]]
+  # for e3sm_diags (streamflow)
+  extra_vars = 'areatotal2'
+  frequency = "monthly"
+  input_files = "mosart.h0"
+  input_subdir = "archive/rof/hist"
+  mapping_file = ""
+  vars = "RIVER_DISCHARGE_OVER_LAND_LIQ"
+
+  [[ atm_daily_180x360_aave ]]
+  # for e3sm_diags (tropical_subseasonal)
+  frequency = "daily"
+  input_files = "eam.h1"
+  vars = "PRECT"
+
+  [[ atm_monthly_glb ]]
+  # for global_time_series (custom)
+  frequency = "monthly"
+  input_files = "eam.h0"
+  input_subdir = "archive/atm/hist"
+  mapping_file = "glb"
+  vars = "TREFHT,AODDUST"
+
+  [[ lnd_monthly_glb ]]
+  # for global_time_series (custom)
+  frequency = "monthly"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = "glb"
+  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+
+[e3sm_diags]
+active = True
+grid = '180x360_aave'
+multiprocessing = True
+num_workers = 8
+partition = "compute"
+qos = "regular"
+short_name = "v3.LR.historical_0051"
+walltime = "5:00:00"
+
+  [[ atm_monthly_180x360_aave ]]
+  sets = "enso_diags","qbo","streamflow","tropical_subseasonal",
+  # streamflow_obs_ts is determined automatically
+
+[global_time_series]
+# ...skipping because of dependency status file missing
+#   /lcrc/group/e3sm/ac.forsyth2/zppy_min_case_ilamb_output/test_599-min-case-v1/v3.LR.historical_0051/post/scripts/ts_atm_monthly_glb_1985-1994-0010.status
+active = True
+experiment_name = "v3.LR.historical_0051"
+figstr = "v3.LR.historical_0051"
+plots_original=""
+plots_atm = "TREFHT,AODDUST"
+plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+walltime = "00:30:00"
+
+[ilamb]
+active = True
+nodes = 8
+partition = "compute"
+short_name = "v3.LR.historical_0051"
+ts_num_years = 30
+walltime = "2:00:00"

tests/integration/template_complete_run.cfg

@@ -194,6 +194,6 @@ nodes = 8
 partition = "#expand partition_long#"
 short_name = 'v2.LR.historical_0201'
 ts_num_years = 2
-years = "1850:1854:2",
+years = "1850:1854:4",
 
   [[ land_monthly ]]

tests/integration/template_min_case_add_dependencies.cfg

@@ -0,0 +1,113 @@
+# Test the add_dependencies function
+# That touches the following cases:
+# e3sm_diags: enso_diags, qbo (depend_on_ts)
+# e3sm_diags: streamflow
+# e3sm_diags: tropical_subseasonal
+# global_time_series: use_atm, use_lnd (custom, original_8_no_ocn, original_8)
+# ilamb: with or without land_only (base case, land_only)
+
+[default]
+case = "#expand case_name#"
+constraint = "#expand constraint#"
+dry_run = "#expand dry_run#"
+environment_commands = "#expand environment_commands#"
+input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/#expand case_name#
+input_subdir = archive/atm/hist
+mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
+output = "#expand user_output#zppy_min_case_add_dependencies_output/#expand unique_id#/#expand case_name#"
+partition = "#expand partition_short#"
+qos = "#expand qos_short#"
+# ts is in 5 year increments
+ts_num_years = 5
+www = "#expand user_www#zppy_min_case_add_dependencies_www/#expand unique_id#"
+# We want to produce diagnostics for 10 years.
+years = "1985:1995:10",
+
+[ts]
+active = True
+e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#"
+walltime = "00:50:00"
+# ts is in 5 year increments
+years = "1985:1995:5"
+
+  [[ atm_monthly_180x360_aave ]]
+  # for e3sm_diags (depend_on_ts), ilamb
+  frequency = "monthly"
+  input_files = "eam.h0"
+  input_subdir = "archive/atm/hist"
+  ts_fmt = "cmip"
+  vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"
+
+  [[ land_monthly ]]
+  # for ilamb
+  extra_vars = "landfrac"
+  frequency = "monthly"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
+  ts_fmt = "cmip"
+  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+
+  [[ rof_monthly ]]
+  # for e3sm_diags (streamflow)
+  extra_vars = 'areatotal2'
+  frequency = "monthly"
+  input_files = "mosart.h0"
+  input_subdir = "archive/rof/hist"
+  mapping_file = ""
+  vars = "RIVER_DISCHARGE_OVER_LAND_LIQ"
+
+  [[ atm_daily_180x360_aave ]]
+  # for e3sm_diags (tropical_subseasonal)
+  frequency = "daily"
+  input_files = "eam.h1"
+  vars = "PRECT"
+
+  [[ atm_monthly_glb ]]
+  # for global_time_series (custom)
+  frequency = "monthly"
+  input_files = "eam.h0"
+  input_subdir = "archive/atm/hist"
+  mapping_file = "glb"
+  vars = "TREFHT,AODDUST"
+
+  [[ lnd_monthly_glb ]]
+  # for global_time_series (custom)
+  frequency = "monthly"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = "glb"
+  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+
+[e3sm_diags]
+active = True
+grid = '180x360_aave'
+multiprocessing = True
+num_workers = 8
+partition = "#expand partition_long#"
+qos = "#expand qos_long#"
+short_name = "#expand case_name#"
+walltime = "#expand diags_walltime#"
+
+  [[ atm_monthly_180x360_aave ]]
+  sets = "enso_diags","qbo","streamflow","tropical_subseasonal",
+  # streamflow_obs_ts is determined automatically
+
+[global_time_series]
+# ...skipping because of dependency status file missing
+#   /lcrc/group/e3sm/ac.forsyth2/zppy_min_case_ilamb_output/test_599-min-case-v1/v3.LR.historical_0051/post/scripts/ts_atm_monthly_glb_1985-1994-0010.status
+active = True
+experiment_name = "#expand case_name#"
+figstr = "#expand case_name#"
+plots_original=""
+plots_atm = "TREFHT,AODDUST"
+plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+walltime = "00:30:00"
+
+[ilamb]
+active = True
+nodes = 8
+partition = "#expand partition_long#"
+short_name = "#expand case_name#"
+ts_num_years = 30
+walltime = "2:00:00"

tests/integration/utils.py

@@ -267,6 +267,7 @@ def generate_cfgs(unified_testing=False, dry_run=False):
         "bundles",
         "complete_run",
         "debug",
+        "min_case_add_dependencies",
         "min_case_e3sm_diags_depend_on_climo_mvm_1",
         "min_case_e3sm_diags_depend_on_climo_mvm_2",
         "min_case_e3sm_diags_depend_on_climo",

zppy/e3sm_diags.py

@@ -1,10 +1,12 @@
 import os
 import pprint
+from typing import List
 
 import jinja2
 
 from zppy.bundle import handle_bundles
 from zppy.utils import (
+    add_dependencies,
     checkStatus,
     getTasks,
     getYears,
@@ -31,7 +33,7 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file):  # noqa: C901
         return existing_bundles
 
     # --- Generate and submit e3sm_diags scripts ---
-    dependencies = []
+    dependencies: List[str] = []
 
     for c in tasks:
 
@@ -216,28 +218,34 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file):  # noqa: C901
                         or ("qbo" in c["sets"])
                         or ("area_mean_time_series" in c["sets"])
                     ):
-                        dependencies.append(
-                            os.path.join(
-                                scriptDir,
-                                "ts_%s_%04d-%04d-%04d.status"
-                                % (ts_sub, start_yr, end_yr, c["ts_num_years"]),
-                            )
+                        add_dependencies(
+                            dependencies,
+                            scriptDir,
+                            "ts",
+                            ts_sub,
+                            start_yr,
+                            end_yr,
+                            c["ts_num_years"],
                         )
                     if "streamflow" in c["sets"]:
-                        dependencies.append(
-                            os.path.join(
-                                scriptDir,
-                                "ts_rof_monthly_%04d-%04d-%04d.status"
-                                % (start_yr, end_yr, c["ts_num_years"]),
-                            )
+                        add_dependencies(
+                            dependencies,
+                            scriptDir,
+                            "ts",
+                            "rof_monthly",
+                            start_yr,
+                            end_yr,
+                            c["ts_num_years"],
                         )
                     if "tropical_subseasonal" in c["sets"]:
-                        dependencies.append(
-                            os.path.join(
-                                scriptDir,
-                                "ts_%s_%04d-%04d-%04d.status"
-                                % (ts_daily_sub, start_yr, end_yr, c["ts_num_years"]),
-                            )
+                        add_dependencies(
+                            dependencies,
+                            scriptDir,
+                            "ts",
+                            ts_daily_sub,
+                            start_yr,
+                            end_yr,
+                            c["ts_num_years"],
                         )
             with open(settingsFile, "w") as sf:
                 p = pprint.PrettyPrinter(indent=2, stream=sf)

zppy/global_time_series.py

@@ -1,10 +1,12 @@
 import os
 import pprint
+from typing import List
 
 import jinja2
 
 from zppy.bundle import handle_bundles
 from zppy.utils import (
+    add_dependencies,
     checkStatus,
     getTasks,
     getYears,
@@ -126,30 +128,34 @@ def global_time_series(config, scriptDir, existing_bundles, job_ids_file):  # no
             makeExecutable(scriptFile)
 
             # List of dependencies
-            dependencies = []
+            dependencies: List[str] = []
             # Add Time Series dependencies
             if c["use_atm"]:
                 # Iterate from year1 to year2 incrementing by the number of years per time series file.
                 for yr in range(c["year1"], c["year2"], c["ts_num_years"]):
                     start_yr = yr
                     end_yr = yr + c["ts_num_years"] - 1
-                    dependencies.append(
-                        os.path.join(
-                            scriptDir,
-                            "ts_%s_%04d-%04d-%04d.status"
-                            % ("atm_monthly_glb", start_yr, end_yr, c["ts_num_years"]),
-                        )
+                    add_dependencies(
+                        dependencies,
+                        scriptDir,
+                        "ts",
+                        "atm_monthly_glb",
+                        start_yr,
+                        end_yr,
+                        c["ts_num_years"],
                     )
             if c["use_lnd"]:
                 for yr in range(c["year1"], c["year2"], c["ts_num_years"]):
                     start_yr = yr
                     end_yr = yr + c["ts_num_years"] - 1
-                    dependencies.append(
-                        os.path.join(
-                            scriptDir,
-                            "ts_%s_%04d-%04d-%04d.status"
-                            % ("lnd_monthly_glb", start_yr, end_yr, c["ts_num_years"]),
-                        )
+                    add_dependencies(
+                        dependencies,
+                        scriptDir,
+                        "ts",
+                        "lnd_monthly_glb",
+                        start_yr,
+                        end_yr,
+                        c["ts_num_years"],
                     )
             if c["use_ocn"]:
                 # Add MPAS Analysis dependencies