diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index c02e20b678..581503ccf7 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -54,3 +54,6 @@ aa04d1f7d86cc2503b98b7e2b2d84dbfff6c316b 753fda3ff0147837231a73c9c728dd9ce47b5997 f112ba0bbf96a61d5a4d354dc0dcbd8b0c68145c bd535c710db78420b8e8b9d71d88d8339e899c59 +4b20bbd7003e6f77dab4e3268cc4a43f9b5a3b5d +cf433215b58ba8776ec5edfb0b0d80c0836ed3a0 +16d57ff37859b34dab005693e3085d64e2bcd95a diff --git a/cime_config/SystemTests/ssp.py b/cime_config/SystemTests/ssp.py index bd554aeae9..92cab0a961 100644 --- a/cime_config/SystemTests/ssp.py +++ b/cime_config/SystemTests/ssp.py @@ -81,7 +81,7 @@ def run_phase(self): ) refsec = "00000" - # obtain rpointer files and necessary restart files from short term archiving directory + # obtain necessary restart files from short term archiving directory rundir = self._case.get_value("RUNDIR") rest_path = os.path.join(dout_sr, "rest", "{}-{}".format(refdate, refsec)) @@ -96,9 +96,6 @@ def run_phase(self): else: os.symlink(item, link_name) - for item in glob.glob("{}/*rpointer*".format(rest_path)): - shutil.copy(item, rundir) - self._case.set_value("CLM_ACCELERATED_SPINUP", "off") self._case.set_value("RUN_TYPE", "hybrid") self._case.set_value("GET_REFCASE", False) diff --git a/cime_config/SystemTests/sspmatrixcn.py b/cime_config/SystemTests/sspmatrixcn.py index 29b6dce8e6..98a580a8db 100644 --- a/cime_config/SystemTests/sspmatrixcn.py +++ b/cime_config/SystemTests/sspmatrixcn.py @@ -13,6 +13,7 @@ Step 4: matrix Spinup off """ import shutil, glob, os, sys +from datetime import datetime if __name__ == "__main__": CIMEROOT = os.environ.get("CIMEROOT") @@ -24,6 +25,7 @@ else: from CIME.status import append_testlog +from CIME.case import Case from CIME.XML.standard_module_setup import * from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.SystemTests.test_utils import user_nl_utils @@ -186,6 +188,26 @@ def append_user_nl(self, caseroot, n=0): caseroot=caseroot, component=self.comp, contents=contents_to_append ) + def run_indv(self, nstep, st_archive=True): + """ + Individual run of a given step + """ + suffix = "step{}".format(self.steps[nstep]) + if isinstance(self._case, Case): + super().run_indv(suffix, st_archive=True) + else: + caseroot = self._case.get_value("CASEROOT") + dout_sr = self._case.get_value("DOUT_S_ROOT") + rest_r = os.path.join(dout_sr, "rest") + nyear = 1851 + nstep + rundate = "%s-01-01-00000" % nyear + restdir = os.path.join(rest_r, rundate) + os.mkdir(restdir) + rpoint = os.path.join(restdir, "rpointer.clm." + rundate) + os.mknod(rpoint) + rpoint = os.path.join(restdir, "rpointer.cpl." + rundate) + os.mknod(rpoint) + def run_phase(self): "Run phase" @@ -225,6 +247,7 @@ def run_phase(self): self.append_user_nl(clone_path, n) dout_sr = clone.get_value("DOUT_S_ROOT") + ninst = self._case.get_value("NINST") self._skip_pnl = False # @@ -247,14 +270,24 @@ def run_phase(self): os.makedirs(rundir) os.symlink(item, linkfile) - for item in glob.glob("{}/*rpointer*".format(rest_path)): - shutil.copy(item, rundir) + # For a branch the cpl rpointer file needs to be handled + if self.runtyp[n] == "branch": + + drvrest = "rpointer.cpl" + if ninst > 1: + drvrest += "_0001" + drvrest += rest_time + self._set_drv_restart_pointer(drvrest) + try: + shutil.copy(drvrest, rundir) + except shutil.SameFileError: + pass # # Run the case (Archiving on) # self._case.flush() - self.run_indv(suffix="step{}".format(self.steps[n]), st_archive=True) + self.run_indv(nstep=n, st_archive=True) # # Get the reference case from this step for the next step @@ -267,6 +300,7 @@ def run_phase(self): ) refsec = "00000" rest_path = os.path.join(dout_sr, "rest", "{}-{}".format(refdate, refsec)) + rest_time = "." + refdate + "-" + refsec # # Last step in original case @@ -292,10 +326,22 @@ def run_phase(self): linkfile = os.path.join(rundir, os.path.basename(item)) if os.path.exists(linkfile): os.remove(linkfile) + expect(True, os.path.exists(item), "expected file does NOT exist = " + item) os.symlink(item, linkfile) - for item in glob.glob("{}/*rpointer*".format(rest_path)): - shutil.copy(item, rundir) + # For a branch the cpl rpointer file needs to be handled + if self.runtyp[n] == "branch": + + drvrest = "rpointer.cpl" + if ninst > 1: + drvrest += "_0001" + drvrest += rest_time + + self._set_drv_restart_pointer(drvrest) + try: + shutil.copy(os.path.join(rest_path, drvrest), rundir) + except shutil.SameFileError: + pass self.append_user_nl(clone_path, n) # @@ -306,66 +352,4 @@ def run_phase(self): # Run the case (short term archiving is off) # self._case.flush() - self.run_indv(suffix="step{}".format(self.steps[n]), st_archive=False) - - -# -# Unit testing for above -# -import unittest -from CIME.case import Case -from CIME.utils import _LessThanFilter -from argparse import RawTextHelpFormatter - - -class test_ssp_matrixcn(unittest.TestCase): - def setUp(self): - self.ssp = SSPMATRIXCN() - - def test_logger(self): - # Test the logger - stream_handler = logging.StreamHandler(sys.stdout) - logger.addHandler(stream_handler) - logger.level = logging.DEBUG - logger.info("nyr_forcing = {}".format(self.ssp.nyr_forcing)) - for n in range(self.ssp.n_steps()): - self.ssp.__logger__(n) - if self.ssp.spin[n] == "sasu": - logger.info(" SASU spinup is .true.") - if self.ssp.sasu[n] != -999: - logger.info(" nyr_sasu = {}".format(self.ssp.sasu[n])) - if self.ssp.iloop[n] != -999: - logger.info(" iloop_avg = {}".format(self.ssp.iloop[n])) - - logger.info("Total number of years {}".format(self.ssp.total_years())) - logger.removeHandler(stream_handler) - - def test_n_steps(self): - self.assertTrue(self.ssp.n_steps() == 3) - - def test_valid_n(self): - for n in range(self.ssp.n_steps()): - self.ssp.check_n(n) - - def test_negative_n(self): - self.assertRaises(SystemExit, self.ssp.check_n, -1) - - def test_n_too_big(self): - self.assertRaises(SystemExit, self.ssp.check_n, self.ssp.n_steps()) - - def test_append_user_nl_step2(self): - ufile = "user_nl_clm" - if not os.path.exists(ufile): - os.mknod(ufile) - else: - expect(0, ufile + " file already exists, not overwritting it") - - self.ssp.append_user_nl(caseroot=".", n=2) - print(ufile + " for step 2") - log = open(ufile, "r").read() - print(log) - os.remove(ufile) - - -if __name__ == "__main__": - unittest.main() + self.run_indv(nstep=n, st_archive=False) diff --git a/parse_cime.cs.status b/parse_cime.cs.status index 264ba0708f..daaaef2293 100755 --- a/parse_cime.cs.status +++ b/parse_cime.cs.status @@ -332,7 +332,9 @@ sub print_categories { if ( ! -f $expectedfailfile ) { $expectedfailfile = "$scrdir/cime_config/testdefs/ExpectedTestFails.xml"; } - my @failfiles = ( $expectedfailfile, "$scrdir/components/mizuRoute/cime_config/testdefs/ExpectedTestFails.xml" ); + my @failfiles = ( $expectedfailfile, "$scrdir/components/mizuRoute/cime_config/testdefs/ExpectedTestFails.xml", + #"$scrdir/components/mosart/cime_config/testdefs/ExpectedTestFails.xml", + "$scrdir/components/cmeps/cime_config/ExpectedTestFails.xml" ); my @passes; my @fails; my @pendings; diff --git a/python/ctsm/path_utils.py b/python/ctsm/path_utils.py index 0015822c03..e8cf2989ce 100644 --- a/python/ctsm/path_utils.py +++ b/python/ctsm/path_utils.py @@ -96,6 +96,16 @@ def add_cime_lib_to_path(standalone_only=False): return cime_path +def add_ctsm_systests_to_path(standalone_only=False): + """Adds the CTSM python SystemTests to the python path, to allow importing + modules from that library + """ + cime_path = path_to_cime(standalone_only=standalone_only) + ctsm_systest_dir = os.path.join(cime_path, os.pardir, "cime_config") + prepend_to_python_path(ctsm_systest_dir) + sys.path.insert(1, ctsm_systest_dir) + + # ======================================================================== # Private functions # ======================================================================== diff --git a/python/ctsm/test/test_unit_sspmatrix.py b/python/ctsm/test/test_unit_sspmatrix.py new file mode 100755 index 0000000000..1b1bc60185 --- /dev/null +++ b/python/ctsm/test/test_unit_sspmatrix.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +""" +Unit tests for the SystemTest sspmatrix.py +""" + +import unittest +import os +import sys +import tempfile +import shutil +import logging +from pathlib import Path + +from ctsm.path_utils import add_cime_lib_to_path, add_ctsm_systests_to_path +from ctsm import unit_testing + +add_ctsm_systests_to_path() +add_cime_lib_to_path() + +# pylint: disable=import-error +# pylint: disable=wrong-import-position +# pylint: disable=wrong-import-order +from SystemTests.sspmatrixcn import SSPMATRIXCN + +# pylint: disable=import-error +# pylint: disable=wrong-import-position +# pylint: disable=wrong-import-order +from CIME.tests.case_fake import CaseFake + +# pylint: disable=invalid-name + +logger = logging.getLogger(__name__) + + +class SSPCaseFake(CaseFake): + """ + Extend the CaseFake class with a couple things needed here + """ + + def __init__(self, case_root, tempdir, create_case_root=True): + """ + Initialization handling the tempdir + """ + super().__init__(case_root, create_case_root) + self._tempdir = tempdir + + def create_clone( + self, + newcase, + keepexe=False, + ): + """ + Extend to handle creation of user_nl_clm file + """ + clone = super().create_clone(newcase, keepexe=keepexe) + os.mknod(os.path.join(newcase, "user_nl_clm")) + # Also make the needed case directories + clone.make_case_dirs(self._tempdir) + return clone + + def make_case_dirs(self, tempdir): + """ + Create the directories needed for the CASE + """ + casename = self.get_value("CASE") + dout_s_root = Path(tempdir, "archive", casename) + dout_s_root.mkdir(parents=True) + self.set_value("DOUT_S_ROOT", str(dout_s_root)) + rest_root = Path(dout_s_root, "rest") + rest_root.mkdir() + rundir = Path(tempdir, casename, "run") + rundir.mkdir() + self.set_value("RUNDIR", rundir) + + def __str__(self): + """ + String method + """ + return "caseroot=%s" % (self.get_value("CASEROOT")) + + +class TestSSPMatrix(unittest.TestCase): + """ + Basic class for testing the sspmatrix.py SystemTest + """ + + def setUp(self): + """ + Setup test directory + """ + self._previous_dir = os.getcwd() + self._tempdir = tempfile.mkdtemp() + + # Set up the testing SSPCaseFake + caseroot = os.path.join(self._tempdir, "ssptestcase") + self._case = SSPCaseFake(caseroot, tempdir=self._tempdir, create_case_root=True) + os.chdir(caseroot) + + # Set XML variables that will be needed in the case + self._case.set_value("DATM_YR_START", 2000) + self._case.set_value("DATM_YR_END", 2001) + self._case.set_value("COMP_LND", "clm") + self._case.set_value("NINST", 1) + + self.ssp = SSPMATRIXCN(self._case) + self._case.make_case_dirs(self._tempdir) + + def tearDown(self): + """ + Remove temporary directory + """ + os.chdir(self._previous_dir) + shutil.rmtree(self._tempdir, ignore_errors=True) + + def test_logger(self): + """ + Test the logger + """ + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + logger.level = logging.DEBUG + logger.info("nyr_forcing = %s", self.ssp.nyr_forcing) + for n in range(self.ssp.n_steps()): + self.ssp.__logger__(n) + if self.ssp.spin[n] == "sasu": + logger.info(" SASU spinup is .true.") + if self.ssp.sasu[n] != -999: + logger.info(" nyr_sasu = %s", self.ssp.sasu[n]) + if self.ssp.iloop[n] != -999: + logger.info(" iloop_avg = %s", self.ssp.iloop[n]) + + logger.info("Total number of years %s", self.ssp.total_years()) + logger.removeHandler(stream_handler) + + def test_n_steps(self): + """ + Test that n_steps is as expected + """ + self.assertTrue(self.ssp.n_steps() == 3) + + def test_valid_n(self): + """ + Test that check of n-step is good for the range it runs in + """ + for n in range(self.ssp.n_steps()): + self.ssp.check_n(n) + + def test_negative_n(self): + """ + Test that fails when n-step is negative + """ + self.assertRaises(SystemExit, self.ssp.check_n, -1) + + def test_n_too_big(self): + """ + Test that fails when n-step is too big + """ + self.assertRaises(SystemExit, self.ssp.check_n, self.ssp.n_steps()) + + def test_append_user_nl_step2(self): + """ + Test appending to user_nl_clm file for step 2 + """ + ufile = "user_nl_clm" + if os.path.exists(ufile): + os.remove(ufile) + + os.mknod(ufile) + + expect = "\nhist_nhtfrq = -8760, hist_mfilt = 2\n" + self.ssp.append_user_nl(caseroot=".", n=2) + log = open(ufile, "r").read() + self.assertEqual(expect, log, "Append user_nl_clm file NOT as expected for step 2") + os.remove(ufile) + + def test_run_phase(self): + """ + Test doing the standard run_phase, that does each step + """ + self.ssp.run_phase() + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main()