merge up to head of feature/capgen

CMakeLists.txt

@@ -7,7 +7,7 @@ project(ccpp_framework
 #------------------------------------------------------------------------------
 # Set package definitions
 set(PACKAGE "ccpp-framework")
-set(AUTHORS "Dom Heinzeller" "Grant Firl" "Laurie Carson")
+set(AUTHORS "Dom Heinzeller" "Grant Firl" "Mike Kavulich" "Steve Goldhaber")
 string(TIMESTAMP YEAR "%Y")
 
 #------------------------------------------------------------------------------
@@ -20,12 +20,10 @@ endif (OPENMP)
 #------------------------------------------------------------------------------
 # Set a default build type if none was specified
 if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
-    # DH* TODO - TRY TO CHANGE THIS TO RELEASE AND SEE WHAT HAPPENS TO THE RESULTS
-    message(STATUS "Setting build type to 'Debug' as none was specified.")
+    message(STATUS "Setting build type to 'Release' as none was specified.")
     set(CMAKE_BUILD_TYPE Debug CACHE STRING "Choose the type of build." FORCE)
-    # *DH
     # Set the possible values of build type for cmake-gui
-    set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Bitforbit" "Release" "Coverage")
+    set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "Coverage")
 endif()
 
 #------------------------------------------------------------------------------

CODEOWNERS

@@ -3,7 +3,7 @@
 
 # These owners will be the default owners for everything in the repo.
 #*       @defunkt
-*       @climbfuji @gold2718
+*       @climbfuji @grantfirl @gold2718 @mkavulich
 
 # Order is important. The last matching pattern has the most precedence.
 # So if a pull request only touches javascript files, only these owners

README.md

@@ -5,7 +5,7 @@ This repository contains the Common Community Physics Package (CCPP) Framework:
 ## Documentation
 The CCPP Framework is designed to be used with the [CCPP Physics repository](https://github.com/NCAR/ccpp-physics) and a host model. Links to further information about this repository and its place in the context of the CCPP as a whole can be found on the [GitHub Wiki](https://github.com/NCAR/ccpp-framework/wiki).
 
-The CCPP Single Column Model (SCM) is designed as a simple host model for testing and development of the CCPP; information about building and using the CCPP in this context can be found in [the CCPP SCM Users Guide](https://dtcenter.org/sites/default/files/paragraph/scm-ccpp-guide-v5.0.0.pdf).
+The CCPP Single Column Model (SCM) is designed as a simple host model for testing and development of the CCPP; information about building and using the CCPP in this context can be found in [the CCPP SCM Users Guide](https://dtcenter.org/sites/default/files/paragraph/scm-ccpp-guide-v6.0.0.pdf).
 
-More information about the design and use of CCPP can be found in the [CCPP Technical Documentation](https://ccpp-techdoc.readthedocs.io/en/v5.0.0/).
+More information about the design and use of CCPP can be found in the [CCPP Technical Documentation](https://ccpp-techdoc.readthedocs.io/en/v6.0.0/).
 

scripts/ccpp_prebuild.py

@@ -3,6 +3,7 @@
 # Standard modules
 import argparse
 import collections
+import copy
 import filecmp
 import importlib
 import itertools
@@ -23,6 +24,7 @@
                   TypedefsMakefile, TypedefsCMakefile, TypedefsSourcefile
 from mkdoc import metadata_to_html, metadata_to_latex
 from mkstatic import API, Suite, Group
+from mkstatic import CCPP_SUITE_VARIABLES
 
 ###############################################################################
 # Set up the command line argument parser and other global variables          #
@@ -35,6 +37,7 @@
 parser.add_argument('--debug',      action='store_true', help='enable debugging features in auto-generated code', default=False)
 parser.add_argument('--suites',     action='store', help='suite definition files to use (comma-separated, without path)', default='')
 parser.add_argument('--builddir',   action='store', help='relative path to CCPP build directory', required=False, default=None)
+parser.add_argument('--namespace',  action='store', help='namespace suffix to be added to the name of static api module', required=False, default='')
 
 # BASEDIR is the current directory where this script is executed
 BASEDIR = os.getcwd()
@@ -56,7 +59,8 @@ def parse_arguments():
     else:
         sdfs = None
     builddir = args.builddir
-    return (success, configfile, clean, verbose, debug, sdfs, builddir)
+    namespace = args.namespace
+    return (success, configfile, clean, verbose, debug, sdfs, builddir, namespace)
 
 def import_config(configfile, builddir):
     """Import the configuration from a given configuration file"""
@@ -95,13 +99,14 @@ def import_config(configfile, builddir):
     config['caps_cmakefile']            = ccpp_prebuild_config.CAPS_CMAKEFILE.format(build_dir=builddir)
     config['caps_sourcefile']           = ccpp_prebuild_config.CAPS_SOURCEFILE.format(build_dir=builddir)
     config['caps_dir']                  = ccpp_prebuild_config.CAPS_DIR.format(build_dir=builddir)
-    config['suites_dir']                = ccpp_prebuild_config.SUITES_DIR
+    config['suites_dir']                = ccpp_prebuild_config.SUITES_DIR.format(build_dir=builddir)
     config['host_model']                = ccpp_prebuild_config.HOST_MODEL_IDENTIFIER
     config['html_vartable_file']        = ccpp_prebuild_config.HTML_VARTABLE_FILE.format(build_dir=builddir)
     config['latex_vartable_file']       = ccpp_prebuild_config.LATEX_VARTABLE_FILE.format(build_dir=builddir)
-    # Location of static API file, and shell script to source
+    # Location of static API file, shell script to source, cmake include file
     config['static_api_dir']            = ccpp_prebuild_config.STATIC_API_DIR.format(build_dir=builddir)
-    config['static_api_srcfile']        = ccpp_prebuild_config.STATIC_API_SRCFILE.format(build_dir=builddir)
+    config['static_api_sourcefile']     = ccpp_prebuild_config.STATIC_API_SOURCEFILE.format(build_dir=builddir)
+    config['static_api_cmakefile']      = ccpp_prebuild_config.STATIC_API_CMAKEFILE.format(build_dir=builddir)
 
     # To handle new metadata: import DDT references (if exist)
     try:
@@ -127,10 +132,14 @@ def setup_logging(verbose):
         logging.info('Logging level set to INFO')
     return success
 
-def clean_files(config):
+def clean_files(config, namespace):
     """Clean files created by ccpp_prebuild.py"""
     success = True
     logging.info('Performing clean ....')
+    if namespace:
+        static_api_file = '{api}.F90'.format(api=CCPP_STATIC_API_MODULE+'_'+namespace)
+    else:
+        static_api_file = '{api}.F90'.format(api=CCPP_STATIC_API_MODULE)
     # Create list of files to remove, use wildcards where necessary
     files_to_remove = [
         config['typedefs_makefile'],
@@ -145,15 +154,16 @@ def clean_files(config):
         config['html_vartable_file'],
         config['latex_vartable_file'],
         os.path.join(config['caps_dir'], 'ccpp_*_cap.F90'),
-        os.path.join(config['static_api_dir'], '{api}.F90'.format(api=CCPP_STATIC_API_MODULE)),
-        config['static_api_srcfile'],
+        os.path.join(config['static_api_dir'], static_api_file),
+        config['static_api_sourcefile'],
         ]
     # Not very pythonic, but the easiest way w/o importing another Python module
     cmd = 'rm -vf {0}'.format(' '.join(files_to_remove))
     execute(cmd)
     return success
 
 def get_all_suites(suites_dir):
+    """Assemble a list of all suite definition files in suites_dir"""
     success = False
     logging.info("No suites were given, compiling a list of all suites")
     sdfs = []
@@ -209,7 +219,7 @@ def convert_local_name_from_new_metadata(metadata, standard_name, typedefs_new_m
 
     # The local name (incl. the array reference) is in new metadata format
     local_name = var.local_name
-    logging.info("Converting local name {0} of variable {1} from new to old metadata".format(local_name, standard_name))
+    logging.debug("Converting local name {0} of variable {1} from new to old metadata".format(local_name, standard_name))
     if "(" in local_name:
         (actual_var_name, array_reference) = split_var_name_and_array_reference(local_name)
         indices = array_reference.lstrip('(').rstrip(')').split(',')
@@ -328,6 +338,19 @@ def collect_physics_subroutines(scheme_files):
     os.chdir(BASEDIR)
     return (success, metadata_request, arguments_request, dependencies_request, schemes_in_files)
 
+def check_schemes_in_suites(arguments, suites):
+    """Check that all schemes that are requested in the suites exist"""
+    success = True
+    logging.info("Checking for existence of schemes in suites ...")
+    for suite in suites:
+        for group in suite.groups:
+            for subcycle in group.subcycles:
+                for scheme_name in subcycle.schemes:
+                    if not scheme_name in arguments.keys():
+                        success = False
+                        logging.critical("Scheme {} in suite {} cannot be found".format(scheme_name, suite.name))
+    return success
+
 def filter_metadata(metadata, arguments, dependencies, schemes_in_files, suites):
     """Remove all variables from metadata that are not used in the given suite;
     also remove information on argument lists, dependencies and schemes in files"""
@@ -375,6 +398,16 @@ def filter_metadata(metadata, arguments, dependencies, schemes_in_files, suites)
                 schemes_in_files_filtered[scheme] = schemes_in_files[scheme]
     return (success, metadata_filtered, arguments_filtered, dependencies_filtered, schemes_in_files_filtered)
 
+def add_ccpp_suite_variables(metadata):
+    """ Add variables that are required to construct CCPP suites to the list of requested variables"""
+    success = True
+    logging.info("Adding CCPP suite variables to list of requested variables")
+    for var_name in CCPP_SUITE_VARIABLES.keys():
+        if not var_name in metadata.keys():
+            metadata[var_name] = [copy.deepcopy(CCPP_SUITE_VARIABLES[var_name])]
+            logging.debug("Adding CCPP suite variable {0} to list of requested variables".format(var_name))
+    return (success, metadata)
+
 def generate_list_of_schemes_and_dependencies_to_compile(schemes_in_files, dependencies1, dependencies2):
     """Generate a flat list of schemes and dependencies in two dependency dictionaries to compile"""
     success = True
@@ -488,14 +521,20 @@ def generate_suite_and_group_caps(suites, metadata_request, metadata_define, arg
         success = False
     return (success, suite_and_group_caps)
 
-def generate_static_api(suites, static_api_dir):
+def generate_static_api(suites, static_api_dir, namespace):
     """Generate static API for given suite(s)"""
     success = True
     # Change to caps directory, create if necessary
     if not os.path.isdir(static_api_dir):
         os.makedirs(static_api_dir)
     os.chdir(static_api_dir)
     api = API(suites=suites, directory=static_api_dir)
+    if namespace:
+        base = os.path.splitext(os.path.basename(api.filename))[0]
+        logging.info('Static API file name is ''{}'''.format(api.filename))
+        api.filename = base+'_'+namespace+'.F90'
+        api.module = base+'_'+namespace
+        logging.info('Static API file name is changed to ''{}'''.format(api.filename))
     logging.info('Generating static API {0} in {1} ...'.format(api.filename, static_api_dir))
     api.write()
     os.chdir(BASEDIR)
@@ -654,7 +693,7 @@ def generate_caps_makefile(caps, caps_makefile, caps_cmakefile, caps_sourcefile,
 def main():
     """Main routine that handles the CCPP prebuild for different host models."""
     # Parse command line arguments
-    (success, configfile, clean, verbose, debug, sdfs, builddir) = parse_arguments()
+    (success, configfile, clean, verbose, debug, sdfs, builddir, namespace) = parse_arguments()
     if not success:
         raise Exception('Call to parse_arguments failed.')
 
@@ -668,7 +707,7 @@ def main():
 
     # Perform clean if requested, then exit
     if clean:
-        success = clean_files(config)
+        success = clean_files(config, namespace)
         logging.info('CCPP prebuild clean completed successfully, exiting.')
         sys.exit(0)
 
@@ -698,12 +737,22 @@ def main():
     if not success:
         raise Exception('Call to collect_physics_subroutines failed.')
 
+    # Check that the schemes requested in the suites exist
+    success = check_schemes_in_suites(arguments_request, suites)
+    if not success:
+        raise Exception('Call to check_schemes_in_suites failed.')
+
     # Filter metadata/arguments - remove whatever is not included in suite definition files
     (success, metadata_request, arguments_request, dependencies_request, schemes_in_files) = filter_metadata(
                          metadata_request, arguments_request, dependencies_request, schemes_in_files, suites)
     if not success:
         raise Exception('Call to filter_metadata failed.')
 
+    # Add variables that are required to construct CCPP suites to the list of requested variables
+    (success, metadata_request) = add_ccpp_suite_variables(metadata_request)
+    if not success:
+        raise Exception('Call to add_ccpp_suite_variables failed.')
+
     (success, schemes_and_dependencies_to_compile) = generate_list_of_schemes_and_dependencies_to_compile(
                                               schemes_in_files, dependencies_request, dependencies_define)
     if not success:
@@ -738,13 +787,17 @@ def main():
     if not success:
         raise Exception('Call to generate_suite_and_group_caps failed.')
 
-    (success, api) = generate_static_api(suites, config['static_api_dir'])
-    if not success: 
+    (success, api) = generate_static_api(suites, config['static_api_dir'], namespace)
+    if not success:
         raise Exception('Call to generate_static_api failed.')
 
-    success = api.write_sourcefile(config['static_api_srcfile'])
-    if not success: 
-        raise Exception("Writing API sourcefile {sourcefile} failed".format(sourcefile=config['static_api_srcfile']))
+    success = api.write_includefile(config['static_api_sourcefile'], type='shell')
+    if not success:
+        raise Exception("Writing API sourcefile {sourcefile} failed".format(sourcefile=config['static_api_sourcefile']))
+
+    success = api.write_includefile(config['static_api_cmakefile'], type='cmake')
+    if not success:
+        raise Exception("Writing API cmakefile {cmakefile} failed".format(cmakefile=config['static_api_cmakefile']))
 
     # Add filenames of caps to makefile/cmakefile/shell script
     all_caps = suite_and_group_caps