Implement finalization and various enhancements for MPAS dynamical core

cime_config/buildlib

@@ -189,9 +189,9 @@ def _setup_mpas(case: Case) -> None:
 
     shutil.copytree(mpas_dycore_src_root, mpas_dycore_bld_root, copy_function=_copy2_as_needed, dirs_exist_ok=True)
     shutil.move(os.path.join(mpas_dycore_bld_root, "Makefile"), os.path.join(mpas_dycore_bld_root, "Makefile.CESM"))
-    _copy2_as_needed(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "driver", "dyn_mpas_subdriver.F90")), os.path.join(mpas_dycore_bld_root, "driver"))
-    _copy2_as_needed(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "Makefile")), mpas_dycore_bld_root)
-    _copy2_as_needed(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "Makefile.in.CESM")), mpas_dycore_bld_root)
+
+    shutil.copytree(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "assets")), mpas_dycore_bld_root, copy_function=_copy2_as_needed, dirs_exist_ok=True)
+    shutil.copytree(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "driver")), os.path.join(mpas_dycore_bld_root, "driver"), copy_function=_copy2_as_needed, dirs_exist_ok=True)
 
 def _copy2_as_needed(src: str, dst: str) -> None:
     """

cime_config/namelist_definition_cam.xml

@@ -149,8 +149,16 @@
       <value dyn="se" hgrid="ne30np4" nlev="70" sim_year="1850">${DIN_LOC_ROOT}/atm/waccm/ic/waccm5_1850_ne30np4_L70_0001-01-11-00000_c151217.nc</value>
       <value dyn="se" hgrid="ne30np4" nlev="70">${DIN_LOC_ROOT}/atm/waccm/ic/fw2000_ne30np4_L70_c181221.nc</value>
       <value hgrid="64x128" nlev="30" scam="1">${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-09-01_64x128_L30_c031210.nc</value>
-      <value dyn="mpas" hgrid="mpasa120" nlev="32" analytic_ic="1">${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa120_L32_notopo_coords_c201216.nc</value>
-      <value dyn="mpas" hgrid="mpasa480" nlev="32" analytic_ic="1">${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa480_L32_notopo_coords_c201125.nc</value>
+      <value dyn="mpas" hgrid="mpasa480" nlev="32" analytic_ic="1">${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa480_L32_notopo_coords_c240507.nc</value>
+      <value dyn="mpas" hgrid="mpasa120" nlev="32" analytic_ic="1">${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa120_L32_notopo_coords_c240507.nc</value>
+      <value dyn="mpas" hgrid="mpasa60"  nlev="32" analytic_ic="1">${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa60_L32_notopo_coords_c240507.nc</value>
+      <value dyn="mpas" hgrid="mpasa30"  nlev="32" analytic_ic="1">${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa30_L32_notopo_coords_c240507.nc</value>
+      <value dyn="mpas" hgrid="mpasa480" nlev="58" analytic_ic="1">${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa480_L58_notopo_coords_c240814.nc</value>
+      <value dyn="mpas" hgrid="mpasa120" nlev="58" analytic_ic="1">${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa120_L58_notopo_coords_c240814.nc</value>
+      <value dyn="mpas" hgrid="mpasa60"  nlev="58" analytic_ic="1">${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa60_L58_notopo_coords_c240814.nc</value>
+      <value dyn="mpas" hgrid="mpasa480" nlev="93" analytic_ic="1">${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa480_L93_notopo_coords_c240814.nc</value>
+      <value dyn="mpas" hgrid="mpasa120" nlev="93" analytic_ic="1">${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa120_L93_notopo_coords_c240814.nc</value>
+      <value dyn="mpas" hgrid="mpasa60"  nlev="93" analytic_ic="1">${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa60_L93_notopo_coords_c240814.nc</value>
     </values>
   </entry>
   <entry id="pertlim">

src/dynamics/mpas/assets/0001-Prefix-all-MPAS-namelist-group-and-option-names.patch

@@ -0,0 +1,190 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Kuan-Chih Wang <[email protected]>
+Date: Mon, 1 Apr 2024 16:46:50 -0600
+Subject: [PATCH] Prefix all MPAS namelist group and option names
+
+The following transformations are performed for each MPAS namelist group and
+option name:
+
+1. Leading `config_` is removed recursively from the name. Case-insensitive.
+2. Leading `mpas_` is removed recursively from the name. Case-insensitive.
+3. Prepend `mpas_` to the name.
+
+As a result, it is easier to distinguish MPAS namelist groups and options from
+CAM-SIMA ones. Note that only namelist I/O is affected. Internally, MPAS still
+refers to its namelist options by their original names due to compatibility reasons.
+
+This downstream patch is maintained by CAM-SIMA for its particular use case of MPAS.
+---
+ src/tools/registry/gen_inc.c | 80 +++++++++++++++++++++++++++++++++++-
+ src/tools/registry/gen_inc.h |  4 ++
+ 2 files changed, 82 insertions(+), 2 deletions(-)
+
+diff --git a/src/tools/registry/gen_inc.c b/src/tools/registry/gen_inc.c
+index 94f5f714..95b2502c 100644
+--- a/src/tools/registry/gen_inc.c
++++ b/src/tools/registry/gen_inc.c
+@@ -15,6 +15,10 @@
+ #include "fortprintf.h"
+ #include "utility.h"
+
++#ifdef MPAS_CAM_DYCORE
++#include <ctype.h>
++#endif
++
+ void process_core_macro(const char *macro, const char *val, va_list ap);
+ void process_domain_macro(const char *macro, const char *val, va_list ap);
+
+@@ -696,8 +700,18 @@ int parse_namelist_records_from_registry(ezxml_t registry)/*{{{*/
+ 	ezxml_t nmlrecs_xml, nmlopt_xml;
+
+ 	const char *const_core;
+-	const char *nmlrecname, *nmlrecindef, *nmlrecinsub;
+-	const char *nmloptname, *nmlopttype, *nmloptval, *nmloptunits, *nmloptdesc, *nmloptposvals, *nmloptindef;
++#ifdef MPAS_CAM_DYCORE
++	const char *orinmlrecname;
++	const char *orinmloptname;
++	// Fortran variable names have a length limit of 63 characters. +1 for the terminating null character.
++	char nmlrecname[64];
++	char nmloptname[64];
++#else
++	const char *nmlrecname;
++	const char *nmloptname;
++#endif
++	const char *nmlrecindef, *nmlrecinsub;
++	const char *nmlopttype, *nmloptval, *nmloptunits, *nmloptdesc, *nmloptposvals, *nmloptindef;
+
+ 	char pool_name[1024];
+ 	char core_string[1024];
+@@ -743,7 +757,12 @@ int parse_namelist_records_from_registry(ezxml_t registry)/*{{{*/
+
+ 	// Parse Namelist Records
+ 	for (nmlrecs_xml = ezxml_child(registry, "nml_record"); nmlrecs_xml; nmlrecs_xml = nmlrecs_xml->next){
++#ifdef MPAS_CAM_DYCORE
++		orinmlrecname = ezxml_attr(nmlrecs_xml, "name");
++		transform_name(nmlrecname, sizeof(nmlrecname), orinmlrecname);
++#else
+ 		nmlrecname = ezxml_attr(nmlrecs_xml, "name");
++#endif
+ 		nmlrecindef = ezxml_attr(nmlrecs_xml, "in_defaults");
+ 		nmlrecinsub = ezxml_attr(nmlrecs_xml, "in_subpool");
+
+@@ -777,7 +796,12 @@ int parse_namelist_records_from_registry(ezxml_t registry)/*{{{*/
+
+ 		// Define variable definitions prior to reading the namelist in.
+ 		for (nmlopt_xml = ezxml_child(nmlrecs_xml, "nml_option"); nmlopt_xml; nmlopt_xml = nmlopt_xml->next){
++#ifdef MPAS_CAM_DYCORE
++			orinmloptname = ezxml_attr(nmlopt_xml, "name");
++			transform_name(nmloptname, sizeof(nmloptname), orinmloptname);
++#else
+ 			nmloptname = ezxml_attr(nmlopt_xml, "name");
++#endif
+ 			nmlopttype = ezxml_attr(nmlopt_xml, "type");
+ 			nmloptval = ezxml_attr(nmlopt_xml, "default_value");
+ 			nmloptunits = ezxml_attr(nmlopt_xml, "units");
+@@ -809,7 +833,12 @@ int parse_namelist_records_from_registry(ezxml_t registry)/*{{{*/
+ 		// Define the namelist block, to read the namelist record in.
+ 		fortprintf(fd, "      namelist /%s/ &\n", nmlrecname);
+ 		for (nmlopt_xml = ezxml_child(nmlrecs_xml, "nml_option"); nmlopt_xml; nmlopt_xml = nmlopt_xml->next){
++#ifdef MPAS_CAM_DYCORE
++			orinmloptname = ezxml_attr(nmlopt_xml, "name");
++			transform_name(nmloptname, sizeof(nmloptname), orinmloptname);
++#else
+ 			nmloptname = ezxml_attr(nmlopt_xml, "name");
++#endif
+ 			if(nmlopt_xml->next){
+ 				fortprintf(fd, "         %s, &\n", nmloptname);
+ 			} else {
+@@ -840,7 +869,12 @@ int parse_namelist_records_from_registry(ezxml_t registry)/*{{{*/
+ 		// Define broadcast calls for namelist values.
+ 		fortprintf(fd, "      if (ierr <= 0) then\n");
+ 		for (nmlopt_xml = ezxml_child(nmlrecs_xml, "nml_option"); nmlopt_xml; nmlopt_xml = nmlopt_xml->next){
++#ifdef MPAS_CAM_DYCORE
++			orinmloptname = ezxml_attr(nmlopt_xml, "name");
++			transform_name(nmloptname, sizeof(nmloptname), orinmloptname);
++#else
+ 			nmloptname = ezxml_attr(nmlopt_xml, "name");
++#endif
+ 			nmlopttype = ezxml_attr(nmlopt_xml, "type");
+
+ 			if(strncmp(nmlopttype, "real", 1024) == 0){
+@@ -858,7 +892,12 @@ int parse_namelist_records_from_registry(ezxml_t registry)/*{{{*/
+ 		fortprintf(fd, "            call mpas_log_write('    The following values will be used for variables in this record:')\n");
+ 		fortprintf(fd, "            call mpas_log_write(' ')\n");
+ 		for (nmlopt_xml = ezxml_child(nmlrecs_xml, "nml_option"); nmlopt_xml; nmlopt_xml = nmlopt_xml->next){
++#ifdef MPAS_CAM_DYCORE
++			orinmloptname = ezxml_attr(nmlopt_xml, "name");
++			transform_name(nmloptname, sizeof(nmloptname), orinmloptname);
++#else
+ 			nmloptname = ezxml_attr(nmlopt_xml, "name");
++#endif
+ 			nmlopttype = ezxml_attr(nmlopt_xml, "type");
+
+ 			if (strncmp(nmlopttype, "character", 1024) == 0) {
+@@ -885,10 +924,18 @@ int parse_namelist_records_from_registry(ezxml_t registry)/*{{{*/
+ 		fortprintf(fd, "\n");
+
+ 		for (nmlopt_xml = ezxml_child(nmlrecs_xml, "nml_option"); nmlopt_xml; nmlopt_xml = nmlopt_xml->next){
++#ifdef MPAS_CAM_DYCORE
++			orinmloptname = ezxml_attr(nmlopt_xml, "name");
++			transform_name(nmloptname, sizeof(nmloptname), orinmloptname);
++
++			fortprintf(fd, "      call mpas_pool_add_config(%s, '%s', %s)\n", pool_name, orinmloptname, nmloptname);
++			fortprintf(fcg, "      call mpas_pool_get_config(configPool, '%s', %s)\n", orinmloptname, nmloptname);
++#else
+ 			nmloptname = ezxml_attr(nmlopt_xml, "name");
+
+ 			fortprintf(fd, "      call mpas_pool_add_config(%s, '%s', %s)\n", pool_name, nmloptname, nmloptname);
+ 			fortprintf(fcg, "      call mpas_pool_get_config(configPool, '%s', %s)\n", nmloptname, nmloptname);
++#endif
+ 		}
+ 		fortprintf(fd, "\n");
+ 		fortprintf(fcg, "\n");
+@@ -2532,3 +2579,32 @@ int parse_structs_from_registry(ezxml_t registry)/*{{{*/
+
+ 	return 0;
+ }/*}}}*/
++
++
++#ifdef MPAS_CAM_DYCORE
++// Perform transformations for namelist group and option names.
++void transform_name(char *new_name, const size_t new_name_size, const char *old_name) {
++    const char *const new_prefix = "mpas_";
++    const char *const old_prefix = "config_";
++    size_t size = 0;
++
++    if (!new_name || !old_name || new_name_size == 0) return;
++
++    // Remove all leading whitespaces by moving pointer forward.
++    while (*old_name != '\0' && isspace((unsigned char) *old_name)) old_name++;
++
++    // Remove all leading `config_` by moving pointer forward.
++    while (strncasecmp(old_name, old_prefix, strlen(old_prefix)) == 0) old_name += strlen(old_prefix);
++
++    // Remove all leading `mpas_` by moving pointer forward.
++    while (strncasecmp(old_name, new_prefix, strlen(new_prefix)) == 0) old_name += strlen(new_prefix);
++
++    *new_name = '\0';
++    size = snprintf(NULL, 0, "%s%s", new_prefix, old_name) + 1;
++    snprintf(new_name, size > new_name_size ? new_name_size : size, "%s%s", new_prefix, old_name);
++
++    // Remove all trailing whitespaces by zeroing (nulling) out.
++    new_name += strlen(new_name) - 1;
++    while (*new_name != '\0' && isspace((unsigned char) *new_name)) *new_name-- = '\0';
++}
++#endif
+diff --git a/src/tools/registry/gen_inc.h b/src/tools/registry/gen_inc.h
+index fc94e78b..69a76ace 100644
+--- a/src/tools/registry/gen_inc.h
++++ b/src/tools/registry/gen_inc.h
+@@ -38,3 +38,7 @@ int push_attributes(ezxml_t currentPosition);
+ int merge_structs_and_var_arrays(ezxml_t currentPosition);
+ int merge_streams(ezxml_t registry);
+ int parse_structs_from_registry(ezxml_t registry);
++
++#ifdef MPAS_CAM_DYCORE
++void transform_name(char *new_name, const size_t new_name_size, const char *old_name);
++#endif
+-- 
+2.43.0
+

src/dynamics/mpas/assets/0002-Disable-physics-for-MPAS-dycore-only-build.patch

@@ -0,0 +1,42 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Kuan-Chih Wang <[email protected]>
+Date: Thu, 1 Aug 2024 17:09:58 -0600
+Subject: [PATCH] Disable physics for MPAS dycore-only build
+
+When building MPAS as a dycore, all physics-related components are disabled.
+This build configuration is usually used by CAM/CAM-SIMA, and can be achieved by
+defining the `MPAS_CAM_DYCORE` macro in `CPPFLAGS`.
+
+The `PHYSICS` variable controls whether physics are enabled in MPAS, but its logic
+is decoupled from the `MPAS_CAM_DYCORE` macro. Disabling physics in MPAS currently
+requires manual interventions.
+
+Therefore, automatically disable physics when the `MPAS_CAM_DYCORE` macro is found
+in `CPPFLAGS`.
+
+This downstream patch is maintained by CAM-SIMA for its particular use case of MPAS.
+---
+ src/core_atmosphere/Makefile | 7 +++++--
+ 1 file changed, 5 insertions(+), 2 deletions(-)
+
+diff --git a/src/core_atmosphere/Makefile b/src/core_atmosphere/Makefile
+index 8d9f4f1a..cac8255e 100644
+--- a/src/core_atmosphere/Makefile
++++ b/src/core_atmosphere/Makefile
+@@ -4,8 +4,11 @@
+ # To build a dycore-only MPAS-Atmosphere model, comment-out or delete
+ # the definition of PHYSICS, below
+ #
+-PHYSICS=-DDO_PHYSICS
+-
++# If MPAS_CAM_DYCORE is found in CPPFLAGS, PHYSICS will become undefined automatically
++#
++ifeq ($(findstring MPAS_CAM_DYCORE,$(CPPFLAGS)),)
++    PHYSICS = -DDO_PHYSICS
++endif
+
+ ifdef PHYSICS
+     PHYSCORE = physcore
+-- 
+2.43.0
+

src/dynamics/mpas/assets/0003-Declare-constants-at-the-native-precision-of-MPAS.patch

@@ -0,0 +1,94 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Kuan-Chih Wang <[email protected]>
+Date: Wed, 13 Nov 2024 17:19:10 -0700
+Subject: [PATCH] Declare constants at the native precision of MPAS
+
+When building MPAS as a dycore, certain constants in the `mpas_constants` module are
+imported from the `physconst` module, which is a part of CAM/CAM-SIMA. However,
+multiple issues arise if the precision of those constants differs from MPAS.
+
+For example, building MPAS in single precision mode with CAM-SIMA fails due to
+multiple occurrences of type mismatch between actual and dummy arguments.
+
+mpas_geometry_utils.F:885:157:
+
+  885 |          call mpas_log_write('$r', MPAS_LOG_ERR, realArgs=(/mpas_triangle_signed_area_sphere(a,b,c,sphereRadius) - pii/2.0_RKIND*sphereRadius*sphereRadius/))
+      |                                                                                                                                                             1
+Error: Type mismatch in argument 'realargs' at (1); passed REAL(8) to REAL(4)
+
+Here, `pii` is declared by CAM-SIMA to be double precision, and it causes unintended
+floating-point promotion in the expression.
+
+The solution is to ensure that constants in the `mpas_constants` module are declared
+at the native precision of MPAS.
+
+This downstream patch is maintained by CAM-SIMA for its particular use case of MPAS.
+---
+ src/framework/mpas_constants.F | 44 ++++++++++++++++++++++++++--------
+ 1 file changed, 34 insertions(+), 10 deletions(-)
+
+diff --git a/src/framework/mpas_constants.F b/src/framework/mpas_constants.F
+index c98cb810..68164bab 100644
+--- a/src/framework/mpas_constants.F
++++ b/src/framework/mpas_constants.F
+@@ -23,16 +23,30 @@ module mpas_constants
+    use mpas_kind_types
+
+ #ifdef MPAS_CAM_DYCORE
+-   use physconst, only : pii => pi
+-   use physconst, only : gravity => gravit
+-   use physconst, only : omega
+-   use physconst, only : a => rearth
+-   use physconst, only : cp => cpair
+-   use physconst, only : rgas => rair
+-   use physconst, only : rv => rh2o
+-   real (kind=RKIND) :: rvord = huge(1.0_RKIND)                       ! Derived in mpas_constants_compute_derived
+-   real (kind=RKIND) :: cv = huge(1.0_RKIND)                          ! Derived in mpas_constants_compute_derived
+-   real (kind=RKIND) :: cvpm = huge(1.0_RKIND)                        ! Derived in mpas_constants_compute_derived
++   use physconst, only: external_pii => pi
++   use physconst, only: external_a => rearth
++   use physconst, only: external_omega => omega
++   use physconst, only: external_gravity => gravit
++   use physconst, only: external_rgas => rair
++   use physconst, only: external_rv => rh2o
++   use physconst, only: external_cp => cpair
++   private :: external_pii
++   private :: external_a
++   private :: external_omega
++   private :: external_gravity
++   private :: external_rgas
++   private :: external_rv
++   private :: external_cp
++   real (kind=RKIND), protected :: pii = huge(1.0_RKIND)              ! Derived in mpas_constants_compute_derived
++   real (kind=RKIND), protected :: a = huge(1.0_RKIND)                ! Derived in mpas_constants_compute_derived
++   real (kind=RKIND), protected :: omega = huge(1.0_RKIND)            ! Derived in mpas_constants_compute_derived
++   real (kind=RKIND), protected :: gravity = huge(1.0_RKIND)          ! Derived in mpas_constants_compute_derived
++   real (kind=RKIND), protected :: rgas = huge(1.0_RKIND)             ! Derived in mpas_constants_compute_derived
++   real (kind=RKIND), protected :: rv = huge(1.0_RKIND)               ! Derived in mpas_constants_compute_derived
++   real (kind=RKIND), protected :: cp = huge(1.0_RKIND)               ! Derived in mpas_constants_compute_derived
++   real (kind=RKIND), protected :: rvord = huge(1.0_RKIND)            ! Derived in mpas_constants_compute_derived
++   real (kind=RKIND), protected :: cv = huge(1.0_RKIND)               ! Derived in mpas_constants_compute_derived
++   real (kind=RKIND), protected :: cvpm = huge(1.0_RKIND)             ! Derived in mpas_constants_compute_derived
+ #else
+    real (kind=RKIND), parameter :: pii     = 3.141592653589793_RKIND  !< Constant: Pi
+    real (kind=RKIND), parameter :: a       = 6371229.0_RKIND          !< Constant: Spherical Earth radius [m]
+@@ -77,6 +91,16 @@ module mpas_constants
+        implicit none
+
+ #ifdef MPAS_CAM_DYCORE
++       ! Convert external constants to the native precision of MPAS (i.e., `RKIND`).
++
++       pii = real(external_pii, RKIND)
++       a = real(external_a, RKIND)
++       omega = real(external_omega, RKIND)
++       gravity = real(external_gravity, RKIND)
++       rgas = real(external_rgas, RKIND)
++       rv = real(external_rv, RKIND)
++       cp = real(external_cp, RKIND)
++
+        !
+        ! In the case of CAM-MPAS, rgas may depend on a CAM namelist option,
+        ! so physical constants that depend on rgas must be computed here after
+-- 
+2.43.0
+