Skip to content

Commit

Permalink
Merge pull request #2921 from jedwards4b/handle_same_file_error
Browse files Browse the repository at this point in the history 
Handle same file error
  • Loading branch information
@ekluzek
ekluzek authored Feb 1, 2025
2 parents dff976f + e1f21f2 commit 28501f7
Show file tree
Hide file tree
Showing 6 changed files with 254 additions and 73 deletions.
3 changes: 3 additions & 0 deletions .git-blame-ignore-revs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,6 @@ aa04d1f7d86cc2503b98b7e2b2d84dbfff6c316b
753fda3ff0147837231a73c9c728dd9ce47b5997
f112ba0bbf96a61d5a4d354dc0dcbd8b0c68145c
bd535c710db78420b8e8b9d71d88d8339e899c59
4b20bbd7003e6f77dab4e3268cc4a43f9b5a3b5d
cf433215b58ba8776ec5edfb0b0d80c0836ed3a0
16d57ff37859b34dab005693e3085d64e2bcd95a
5 changes: 1 addition & 4 deletions cime_config/SystemTests/ssp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def run_phase(self):
)
refsec = "00000"

# obtain rpointer files and necessary restart files from short term archiving directory
# obtain necessary restart files from short term archiving directory
rundir = self._case.get_value("RUNDIR")

rest_path = os.path.join(dout_sr, "rest", "{}-{}".format(refdate, refsec))
Expand All @@ -96,9 +96,6 @@ def run_phase(self):
else:
os.symlink(item, link_name)

for item in glob.glob("{}/*rpointer*".format(rest_path)):
shutil.copy(item, rundir)

self._case.set_value("CLM_ACCELERATED_SPINUP", "off")
self._case.set_value("RUN_TYPE", "hybrid")
self._case.set_value("GET_REFCASE", False)
Expand Down
120 changes: 52 additions & 68 deletions cime_config/SystemTests/sspmatrixcn.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
Step 4: matrix Spinup off
"""
import shutil, glob, os, sys
from datetime import datetime

if __name__ == "__main__":
CIMEROOT = os.environ.get("CIMEROOT")
Expand All @@ -24,6 +25,7 @@
else:
from CIME.status import append_testlog

from CIME.case import Case
from CIME.XML.standard_module_setup import *
from CIME.SystemTests.system_tests_common import SystemTestsCommon
from CIME.SystemTests.test_utils import user_nl_utils
Expand Down Expand Up @@ -186,6 +188,26 @@ def append_user_nl(self, caseroot, n=0):
caseroot=caseroot, component=self.comp, contents=contents_to_append
)

def run_indv(self, nstep, st_archive=True):
"""
Individual run of a given step
"""
suffix = "step{}".format(self.steps[nstep])
if isinstance(self._case, Case):
super().run_indv(suffix, st_archive=True)
else:
caseroot = self._case.get_value("CASEROOT")
dout_sr = self._case.get_value("DOUT_S_ROOT")
rest_r = os.path.join(dout_sr, "rest")
nyear = 1851 + nstep
rundate = "%s-01-01-00000" % nyear
restdir = os.path.join(rest_r, rundate)
os.mkdir(restdir)
rpoint = os.path.join(restdir, "rpointer.clm." + rundate)
os.mknod(rpoint)
rpoint = os.path.join(restdir, "rpointer.cpl." + rundate)
os.mknod(rpoint)

def run_phase(self):
"Run phase"

Expand Down Expand Up @@ -225,6 +247,7 @@ def run_phase(self):
self.append_user_nl(clone_path, n)

dout_sr = clone.get_value("DOUT_S_ROOT")
ninst = self._case.get_value("NINST")

self._skip_pnl = False
#
Expand All @@ -247,14 +270,24 @@ def run_phase(self):
os.makedirs(rundir)
os.symlink(item, linkfile)

for item in glob.glob("{}/*rpointer*".format(rest_path)):
shutil.copy(item, rundir)
# For a branch the cpl rpointer file needs to be handled
if self.runtyp[n] == "branch":

drvrest = "rpointer.cpl"
if ninst > 1:
drvrest += "_0001"
drvrest += rest_time

self._set_drv_restart_pointer(drvrest)
try:
shutil.copy(drvrest, rundir)
except shutil.SameFileError:
pass
#
# Run the case (Archiving on)
#
self._case.flush()
self.run_indv(suffix="step{}".format(self.steps[n]), st_archive=True)
self.run_indv(nstep=n, st_archive=True)

#
# Get the reference case from this step for the next step
Expand All @@ -267,6 +300,7 @@ def run_phase(self):
)
refsec = "00000"
rest_path = os.path.join(dout_sr, "rest", "{}-{}".format(refdate, refsec))
rest_time = "." + refdate + "-" + refsec

#
# Last step in original case
Expand All @@ -292,10 +326,22 @@ def run_phase(self):
linkfile = os.path.join(rundir, os.path.basename(item))
if os.path.exists(linkfile):
os.remove(linkfile)
expect(True, os.path.exists(item), "expected file does NOT exist = " + item)
os.symlink(item, linkfile)

for item in glob.glob("{}/*rpointer*".format(rest_path)):
shutil.copy(item, rundir)
# For a branch the cpl rpointer file needs to be handled
if self.runtyp[n] == "branch":

drvrest = "rpointer.cpl"
if ninst > 1:
drvrest += "_0001"
drvrest += rest_time

self._set_drv_restart_pointer(drvrest)
try:
shutil.copy(os.path.join(rest_path, drvrest), rundir)
except shutil.SameFileError:
pass

self.append_user_nl(clone_path, n)
#
Expand All @@ -306,66 +352,4 @@ def run_phase(self):
# Run the case (short term archiving is off)
#
self._case.flush()
self.run_indv(suffix="step{}".format(self.steps[n]), st_archive=False)


#
# Unit testing for above
#
import unittest
from CIME.case import Case
from CIME.utils import _LessThanFilter
from argparse import RawTextHelpFormatter


class test_ssp_matrixcn(unittest.TestCase):
def setUp(self):
self.ssp = SSPMATRIXCN()

def test_logger(self):
# Test the logger
stream_handler = logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)
logger.level = logging.DEBUG
logger.info("nyr_forcing = {}".format(self.ssp.nyr_forcing))
for n in range(self.ssp.n_steps()):
self.ssp.__logger__(n)
if self.ssp.spin[n] == "sasu":
logger.info(" SASU spinup is .true.")
if self.ssp.sasu[n] != -999:
logger.info(" nyr_sasu = {}".format(self.ssp.sasu[n]))
if self.ssp.iloop[n] != -999:
logger.info(" iloop_avg = {}".format(self.ssp.iloop[n]))

logger.info("Total number of years {}".format(self.ssp.total_years()))
logger.removeHandler(stream_handler)

def test_n_steps(self):
self.assertTrue(self.ssp.n_steps() == 3)

def test_valid_n(self):
for n in range(self.ssp.n_steps()):
self.ssp.check_n(n)

def test_negative_n(self):
self.assertRaises(SystemExit, self.ssp.check_n, -1)

def test_n_too_big(self):
self.assertRaises(SystemExit, self.ssp.check_n, self.ssp.n_steps())

def test_append_user_nl_step2(self):
ufile = "user_nl_clm"
if not os.path.exists(ufile):
os.mknod(ufile)
else:
expect(0, ufile + " file already exists, not overwritting it")

self.ssp.append_user_nl(caseroot=".", n=2)
print(ufile + " for step 2")
log = open(ufile, "r").read()
print(log)
os.remove(ufile)


if __name__ == "__main__":
unittest.main()
self.run_indv(nstep=n, st_archive=False)
4 changes: 3 additions & 1 deletion parse_cime.cs.status
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,9 @@ sub print_categories {
if ( ! -f $expectedfailfile ) {
$expectedfailfile = "$scrdir/cime_config/testdefs/ExpectedTestFails.xml";
}
my @failfiles = ( $expectedfailfile, "$scrdir/components/mizuRoute/cime_config/testdefs/ExpectedTestFails.xml" );
my @failfiles = ( $expectedfailfile, "$scrdir/components/mizuRoute/cime_config/testdefs/ExpectedTestFails.xml",
#"$scrdir/components/mosart/cime_config/testdefs/ExpectedTestFails.xml",
"$scrdir/components/cmeps/cime_config/ExpectedTestFails.xml" );
my @passes;
my @fails;
my @pendings;
Expand Down
10 changes: 10 additions & 0 deletions python/ctsm/path_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,16 @@ def add_cime_lib_to_path(standalone_only=False):
return cime_path


def add_ctsm_systests_to_path(standalone_only=False):
"""Adds the CTSM python SystemTests to the python path, to allow importing
modules from that library
"""
cime_path = path_to_cime(standalone_only=standalone_only)
ctsm_systest_dir = os.path.join(cime_path, os.pardir, "cime_config")
prepend_to_python_path(ctsm_systest_dir)
sys.path.insert(1, ctsm_systest_dir)


# ========================================================================
# Private functions
# ========================================================================
Expand Down
Loading

0 comments on commit 28501f7

Please sign in to comment.