ESCOMP tag: cam6_4_023

Merge pull request ESCOMP#958 from jtruesdal/scam_dev_exp

cam6_4_023: SCAM-SE feature addition plus bugfixes and some refactoring.
This update includes some refactoring of SCAM, a few bugfixes, and adding the capability to use spectral elements dycore to do vertical transport in the column. The SE feature addition follows the E3SM implementation where a complete coarse resolution (ne3np4) of the SE dycore is initialized but only a single element is run through vertical transport with all element subcolumns being copies of the single column chosen by scmlat, scmlon.

Like the Eulerian version, SCAM-SE also has a bit for bit test to validate an exact run through the same physics as the full 3d model. Because SCAM updates the solution using a slightly different order of operations, the bfb capability is tested by making a special diagnostic run of CAM where the 3d model derives the phys/dyn tendency each time step and then recalculates the prognostic solution using the derived tendencies and SCAM's prognostic equation. This new solution (which is less precise (roundoff) due to the change in order of operations) is substituted for the full 3d solution at each time step of the model run. The substitution of the roundoff state in the 3d run allows SCAM to reproduce (BFB) each time step using the captured tendencies in the cam iop history file.

The SCAM-SE vertical advection skips the horizontal step and derives the floating level tendency based on the IOP prescribed vertical velocity. The floating levels are subsequently remapped at the end of the vertically Lagrangian dynamics step.

This PR also includes an update to .gitmodules for cice and CDEPS.
- update cice to fix scam regression failure
- update cdeps to fix CDEPS regression test build failures

Closes ESCOMP#957
Closes ESCOMP#853
Closes ESCOMP#742

ESCOMP commit: 6e32d03

cime_config/SystemTests/sct.py

@@ -30,15 +30,17 @@ def __init__(self, case):
 
     def _case_one_setup(self):
         append_to_user_nl_files(caseroot = self._get_caseroot(), component = "cam", contents = "inithist = 'CAMIOP'")
+        if self._case.get_value("CAM_DYCORE") == "se":
+            append_to_user_nl_files(caseroot = self._get_caseroot(), component = "cam", contents = "scale_dry_air_mass     = 0.0D0")
 
         CAM_CONFIG_OPTS = self._case1.get_value("CAM_CONFIG_OPTS")
+        self._case.set_value("BFBFLAG","TRUE")
 
 
 
     def _case_two_setup(self):
         case_name = self._case.get_value("CASE")
         RUN_STARTDATE = self._case1.get_value("RUN_STARTDATE")
-        append_to_user_nl_files(caseroot = self._get_caseroot(), component = "cam", contents = "ncdata = '../"+case_name+".cam.i."+RUN_STARTDATE+"-00000.nc'")
         append_to_user_nl_files(caseroot = self._get_caseroot(), component = "cam", contents = "NDENS    = 1,1,1,1,1,1")
         append_to_user_nl_files(caseroot = self._get_caseroot(), component = "cam", contents = "MFILT    = 1,7,1,1,1,1")
         append_to_user_nl_files(caseroot = self._get_caseroot(), component = "cam", contents = "nhtfrq   = 1,1,1,1,1,1")
@@ -47,25 +49,37 @@ def _case_two_setup(self):
         append_to_user_nl_files(caseroot = self._get_caseroot(), component = "cam", contents = "inithist = 'YEARLY'")
         append_to_user_nl_files(caseroot = self._get_caseroot(), component = "cam", contents = "scm_cambfb_mode                = .true.")
         append_to_user_nl_files(caseroot = self._get_caseroot(), component = "cam", contents = "scm_use_obs_uv         = .true.")
+        append_to_user_nl_files(caseroot = self._get_caseroot(), component = "cam", contents = "scm_relaxation         = .false.")
+        append_to_user_nl_files(caseroot = self._get_caseroot(), component = "cam", contents = "scm_use_3dfrc          = .true.")
         for comp in self._case.get_values("COMP_CLASSES"):
             self._case.set_value("NTASKS_{}".format(comp), 1)
             self._case.set_value("NTHRDS_{}".format(comp), 1)
             self._case.set_value("ROOTPE_{}".format(comp), 0)
 
         if self._case.get_value("COMP_INTERFACE") == "mct":
             self._case.set_value("PTS_MODE","TRUE")
-        self._case.set_value("PTS_LAT",-20.0)
-        self._case.set_value("PTS_LON",140.0)
 
-        CAM_CONFIG_OPTS = self._case1.get_value("CAM_CONFIG_OPTS")
-        self._case.set_value("CAM_CONFIG_OPTS","{} -scam ".format(CAM_CONFIG_OPTS))
+        self._case.set_value("BFBFLAG","TRUE")
+
+        CAM_CONFIG_OPTS = self._case1.get_value("CAM_CONFIG_OPTS").replace('-camiop','')
+        self._case.set_value("CAM_CONFIG_OPTS","{} -scam camfrc ".format(CAM_CONFIG_OPTS))
+        if self._case.get_value("CAM_DYCORE") == "se":
+            self._case.set_value("PTS_LAT",44.80320177421346)
+            self._case.set_value("PTS_LON",276.7082039324993)
+            append_to_user_nl_files(caseroot = self._get_caseroot(), component = "cam", contents = "scale_dry_air_mass     = 0.0D0")
+        else:
+            append_to_user_nl_files(caseroot = self._get_caseroot(), component = "cam", contents = "ncdata = '../"+case_name+".cam.i."+RUN_STARTDATE+"-00000.nc'")
+            self._case.set_value("PTS_LAT",-20.0)
+            self._case.set_value("PTS_LON",140.0)
+
+        self._case.set_value("STOP_N",5)
         self._case.case_setup(test_mode=True, reset=True)
 
     def _component_compare_test(self, suffix1, suffix2,
                                 success_change=False,
                                 ignore_fieldlist_diffs=False):
         with self._test_status:
-            stat,netcdf_filename,err=run_cmd('ls ./run/case2run/*h1i*8400.nc ')
+            stat,netcdf_filename,err=run_cmd('ls ./run/case2run/*h1*0000.nc ')
             stat,DIFFs,err=run_cmd('ncdump -ff -p 9,17 -v QDIFF,TDIFF '+netcdf_filename+' | egrep //\.\*DIFF | sed s/^\ \*// | sed s/^0,/0.0,/ | sed s/^0\;/0.0\;/ | sed s/\[,\;\].\*// |  uniq')
             array_of_DIFFs=DIFFs.split("\n")
             answer=max([abs(float(x)) for x in array_of_DIFFs])

cime_config/buildcpp

@@ -81,10 +81,6 @@ def buildcpp(case):
     if nlev:
         config_opts += ["-nlev", nlev]
 
-    # Some settings for single column mode.
-    if pts_mode:
-        config_opts.append("-scam")
-
     if mpilib == 'mpi-serial':
         config_opts.append("-nospmd")
     else:

cime_config/config_pes.xml

@@ -1,42 +1,8 @@
 <?xml version="1.0"?>
 
 <config_pes version="2.0">
-
   <grid name="any">
     <mach name="any">
-      <pes pesize="any" compset="CAM[45]0%SCAM">
-	<comment>none</comment>
-	<ntasks>
-	  <ntasks_atm>1</ntasks_atm>
-	  <ntasks_lnd>1</ntasks_lnd>
-	  <ntasks_rof>1</ntasks_rof>
-	  <ntasks_ice>1</ntasks_ice>
-	  <ntasks_ocn>1</ntasks_ocn>
-	  <ntasks_glc>1</ntasks_glc>
-	  <ntasks_wav>1</ntasks_wav>
-	  <ntasks_cpl>1</ntasks_cpl>
-	</ntasks>
-	<nthrds>
-	  <nthrds_atm>1</nthrds_atm>
-	  <nthrds_lnd>1</nthrds_lnd>
-	  <nthrds_rof>1</nthrds_rof>
-	  <nthrds_ice>1</nthrds_ice>
-	  <nthrds_ocn>1</nthrds_ocn>
-	  <nthrds_glc>1</nthrds_glc>
-	  <nthrds_wav>1</nthrds_wav>
-	  <nthrds_cpl>1</nthrds_cpl>
-	</nthrds>
-	<rootpe>
-	  <rootpe_atm>0</rootpe_atm>
-	  <rootpe_lnd>0</rootpe_lnd>
-	  <rootpe_rof>0</rootpe_rof>
-	  <rootpe_ice>0</rootpe_ice>
-	  <rootpe_ocn>0</rootpe_ocn>
-	  <rootpe_glc>0</rootpe_glc>
-	  <rootpe_wav>0</rootpe_wav>
-	  <rootpe_cpl>0</rootpe_cpl>
-	</rootpe>
-      </pes>
       <pes pesize="any" compset="any">
 	<comment>none</comment>
 	<ntasks>
@@ -109,6 +75,43 @@
       </pes>
     </mach>
   </grid>
+  <grid name="a%ne3np4">
+    <mach name="any">
+      <pes pesize="any" compset="any">
+        <comment>none</comment>
+        <ntasks>
+          <ntasks_atm>24</ntasks_atm>
+          <ntasks_lnd>24</ntasks_lnd>
+          <ntasks_rof>24</ntasks_rof>
+          <ntasks_ice>24</ntasks_ice>
+          <ntasks_ocn>24</ntasks_ocn>
+          <ntasks_glc>24</ntasks_glc>
+          <ntasks_wav>24</ntasks_wav>
+          <ntasks_cpl>24</ntasks_cpl>
+        </ntasks>
+        <nthrds>
+          <nthrds_atm>1</nthrds_atm>
+          <nthrds_lnd>1</nthrds_lnd>
+          <nthrds_rof>1</nthrds_rof>
+          <nthrds_ice>1</nthrds_ice>
+          <nthrds_ocn>1</nthrds_ocn>
+          <nthrds_glc>1</nthrds_glc>
+          <nthrds_wav>1</nthrds_wav>
+          <nthrds_cpl>1</nthrds_cpl>
+        </nthrds>
+        <rootpe>
+          <rootpe_atm>0</rootpe_atm>
+          <rootpe_lnd>0</rootpe_lnd>
+          <rootpe_rof>0</rootpe_rof>
+          <rootpe_ice>0</rootpe_ice>
+          <rootpe_ocn>0</rootpe_ocn>
+          <rootpe_glc>0</rootpe_glc>
+          <rootpe_wav>0</rootpe_wav>
+          <rootpe_cpl>0</rootpe_cpl>
+        </rootpe>
+      </pes>
+    </mach>
+  </grid>
   <grid name="a%ne30" >
     <mach name="any">
       <pes pesize="any" compset="any">
@@ -2066,6 +2069,39 @@
             <nthrds_cpl>1</nthrds_cpl>
           </nthrds>
         </pes>
+	<pes pesize="any" compset="SCAM.*">
+	  <comment>none</comment>
+	  <ntasks>
+	    <ntasks_atm>1</ntasks_atm>
+	    <ntasks_lnd>1</ntasks_lnd>
+	    <ntasks_rof>1</ntasks_rof>
+	    <ntasks_ice>1</ntasks_ice>
+	    <ntasks_ocn>1</ntasks_ocn>
+	    <ntasks_glc>1</ntasks_glc>
+	    <ntasks_wav>1</ntasks_wav>
+	    <ntasks_cpl>1</ntasks_cpl>
+	  </ntasks>
+	  <nthrds>
+	    <nthrds_atm>1</nthrds_atm>
+	    <nthrds_lnd>1</nthrds_lnd>
+	    <nthrds_rof>1</nthrds_rof>
+	    <nthrds_ice>1</nthrds_ice>
+	    <nthrds_ocn>1</nthrds_ocn>
+	    <nthrds_glc>1</nthrds_glc>
+	    <nthrds_wav>1</nthrds_wav>
+	    <nthrds_cpl>1</nthrds_cpl>
+	  </nthrds>
+	  <rootpe>
+	    <rootpe_atm>0</rootpe_atm>
+	    <rootpe_lnd>0</rootpe_lnd>
+	    <rootpe_rof>0</rootpe_rof>
+	    <rootpe_ice>0</rootpe_ice>
+	    <rootpe_ocn>0</rootpe_ocn>
+	    <rootpe_glc>0</rootpe_glc>
+	    <rootpe_wav>0</rootpe_wav>
+	    <rootpe_cpl>0</rootpe_cpl>
+	  </rootpe>
+	</pes>
       </mach>
     </grid>
   </overrides>