From 3e9e9a68dacfe36806123b5a6741d238f92f4da6 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Mon, 2 Dec 2024 13:42:00 +1100 Subject: [PATCH 01/13] Reload global platforms --- cylc/flow/commands.py | 24 ++++++++++++- cylc/flow/task_proxy.py | 1 - tests/integration/test_reload.py | 50 ++++++++++++++++++++++++++++ tests/unit/cfgspec/test_globalcfg.py | 26 +++++++++++++++ 4 files changed, 99 insertions(+), 2 deletions(-) diff --git a/cylc/flow/commands.py b/cylc/flow/commands.py index b10474e7a66..c10569bc7a1 100644 --- a/cylc/flow/commands.py +++ b/cylc/flow/commands.py @@ -88,6 +88,7 @@ RunMode, StopMode, ) +from cylc.flow.cfgspec.glbl_cfg import glbl_cfg if TYPE_CHECKING: @@ -329,7 +330,7 @@ async def remove_tasks( @_command('reload_workflow') -async def reload_workflow(schd: 'Scheduler'): +async def reload_workflow(schd: 'Scheduler', reload_global: bool): """Reload workflow configuration.""" yield # pause the workflow if not already @@ -363,6 +364,27 @@ async def reload_workflow(schd: 'Scheduler'): # give commands time to complete sleep(1) # give any remove-init's time to complete + + if reload_global: + # Reload global config if requested + LOG.info("Reloading the global configuration.") + try: + glbl_cfg(reload=True) + except (ParsecError, CylcConfigError) as exc: + if cylc.flow.flags.verbosity > 1: + # log full traceback in debug mode + LOG.exception(exc) + LOG.critical( + f'Reload failed - {exc.__class__.__name__}: {exc}' + '\nThis is probably due to an issue with the new' + ' configuration.' + '\nTo continue with the pre-reload config, un-pause the' + ' workflow.' + '\nOtherwise, fix the configuration and attempt to reload' + ' again.' + ) + + # reload the workflow definition schd.reload_pending = 'loading the workflow definition' schd.update_data_store() # update workflow status msg diff --git a/cylc/flow/task_proxy.py b/cylc/flow/task_proxy.py index e641b2598e1..aa48db41c3d 100644 --- a/cylc/flow/task_proxy.py +++ b/cylc/flow/task_proxy.py @@ -344,7 +344,6 @@ def copy_to_reload_successor( reload_successor.summary = self.summary reload_successor.local_job_file_path = self.local_job_file_path reload_successor.try_timers = self.try_timers - reload_successor.platform = self.platform reload_successor.job_vacated = self.job_vacated reload_successor.poll_timer = self.poll_timer reload_successor.timeout = self.timeout diff --git a/tests/integration/test_reload.py b/tests/integration/test_reload.py index ad96b187722..5d7ea805fbf 100644 --- a/tests/integration/test_reload.py +++ b/tests/integration/test_reload.py @@ -24,6 +24,7 @@ TASK_STATUS_PREPARING, TASK_STATUS_SUBMITTED, ) +from cylc.flow.cfgspec.glbl_cfg import glbl_cfg async def test_reload_waits_for_pending_tasks( @@ -146,3 +147,52 @@ async def test_reload_failure( # the config should be unchanged assert schd.config.cfg['scheduling']['graph']['R1'] == 'one' + + +async def test_reload_global( + flow, + one_conf, + scheduler, + start, + log_filter, + tmp_path, + monkeypatch, + ): + + global_config_path = tmp_path / 'global.cylc' + monkeypatch.setenv("CYLC_CONF_PATH", str(global_config_path.parent)) + + # Original global config file + global_config_path.write_text(""" + [platforms] + [[localhost]] + [[[meta]]] + x = 1 + """) + assert glbl_cfg().get(['platforms','localhost','meta','x']) == '1' + + # Modify the global config file + global_config_path.write_text(""" + [platforms] + [[localhost]] + [[[meta]]] + x = 2 + """) + + id_ = flow(one_conf) + schd = scheduler(id_) + async with start(schd) as log: + + # reload the workflow and global config + await commands.run_cmd(commands.reload_workflow, schd, reload_global=True) + + # Global config should have been reloaded + assert log_filter( + log, + contains=( + 'Reloading the global configuration.' + ) + ) + + # Task platforms reflect the new config + assert schd.pool.get_tasks()[0].platform['meta']['x'] == '2' diff --git a/tests/unit/cfgspec/test_globalcfg.py b/tests/unit/cfgspec/test_globalcfg.py index 996d0fa4d95..83f6ec90246 100644 --- a/tests/unit/cfgspec/test_globalcfg.py +++ b/tests/unit/cfgspec/test_globalcfg.py @@ -174,3 +174,29 @@ def test_platform_ssh_forward_variables(mock_global_config): ''') assert glblcfg.get(['platforms','foo','ssh forward environment variables']) == ["FOO", "BAR"] + +def test_reload(mock_global_config, tmp_path): + # Load a config + glblcfg: GlobalConfig = mock_global_config(f''' + [platforms] + [[foo]] + [[[meta]]] + x = 1 + ''') + + # Update the global config file and reload + conf_path = tmp_path / GlobalConfig.CONF_BASENAME + conf_path.write_text(''' + [platforms] + [[foo]] + [[[meta]]] + x = 2 + ''') + glblcfg.load() + + assert glblcfg.get(['platforms','foo','meta','x']) == '2' + + from cylc.flow.platforms import get_platform + platform = get_platform("foo") + + assert platform['meta']['x'] == "2" From 656c275dc674bbb27a7c1e653554377690b029c9 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Mon, 2 Dec 2024 16:14:29 +1100 Subject: [PATCH 02/13] Add flag to command --- cylc/flow/commands.py | 2 +- cylc/flow/network/schema.py | 4 ++++ cylc/flow/scripts/reload.py | 11 ++++++++++- tests/integration/test_reload.py | 2 +- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/cylc/flow/commands.py b/cylc/flow/commands.py index c10569bc7a1..3481aaaa615 100644 --- a/cylc/flow/commands.py +++ b/cylc/flow/commands.py @@ -330,7 +330,7 @@ async def remove_tasks( @_command('reload_workflow') -async def reload_workflow(schd: 'Scheduler', reload_global: bool): +async def reload_workflow(schd: 'Scheduler', reload_global: bool = False): """Reload workflow configuration.""" yield # pause the workflow if not already diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py index a1b9fc1c50c..84e4afe7a3f 100644 --- a/cylc/flow/network/schema.py +++ b/cylc/flow/network/schema.py @@ -1886,6 +1886,10 @@ class Meta: class Arguments: workflows = graphene.List(WorkflowID, required=True) + reload_global = Boolean( + default_value=False, + description="Also reload global config") + result = GenericScalar() diff --git a/cylc/flow/scripts/reload.py b/cylc/flow/scripts/reload.py index ca11062e9b4..a0250a21c06 100755 --- a/cylc/flow/scripts/reload.py +++ b/cylc/flow/scripts/reload.py @@ -69,10 +69,12 @@ MUTATION = ''' mutation ( - $wFlows: [WorkflowID]! + $wFlows: [WorkflowID]!, + $reloadGlobal: Boolean, ) { reload ( workflows: $wFlows + reloadGlobal: $reloadGlobal ) { result } @@ -87,6 +89,12 @@ def get_option_parser(): multiworkflow=True, argdoc=[WORKFLOW_ID_MULTI_ARG_DOC], ) + + parser.add_option( + "-g", "--global", + help="also reload global configuration.", + action="store_true", default=False, dest="reload_global") + return parser @@ -97,6 +105,7 @@ async def run(options: 'Values', workflow_id: str): 'request_string': MUTATION, 'variables': { 'wFlows': [workflow_id], + 'reloadGlobal': options.reload_global, } } diff --git a/tests/integration/test_reload.py b/tests/integration/test_reload.py index 5d7ea805fbf..0a91c416d05 100644 --- a/tests/integration/test_reload.py +++ b/tests/integration/test_reload.py @@ -169,7 +169,7 @@ async def test_reload_global( [[[meta]]] x = 1 """) - assert glbl_cfg().get(['platforms','localhost','meta','x']) == '1' + assert glbl_cfg(reload=True).get(['platforms','localhost','meta','x']) == '1' # Modify the global config file global_config_path.write_text(""" From 2ae8bb3c4f6b9b03b0bad05bc025c923fbbce7ba Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Mon, 2 Dec 2024 16:29:16 +1100 Subject: [PATCH 03/13] Test errors are handled --- cylc/flow/commands.py | 2 -- tests/integration/test_reload.py | 38 +++++++++++++++++++++++++------- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/cylc/flow/commands.py b/cylc/flow/commands.py index 3481aaaa615..198cd6f71c9 100644 --- a/cylc/flow/commands.py +++ b/cylc/flow/commands.py @@ -364,7 +364,6 @@ async def reload_workflow(schd: 'Scheduler', reload_global: bool = False): # give commands time to complete sleep(1) # give any remove-init's time to complete - if reload_global: # Reload global config if requested LOG.info("Reloading the global configuration.") @@ -384,7 +383,6 @@ async def reload_workflow(schd: 'Scheduler', reload_global: bool = False): ' again.' ) - # reload the workflow definition schd.reload_pending = 'loading the workflow definition' schd.update_data_store() # update workflow status msg diff --git a/tests/integration/test_reload.py b/tests/integration/test_reload.py index 0a91c416d05..2871feb83f1 100644 --- a/tests/integration/test_reload.py +++ b/tests/integration/test_reload.py @@ -171,18 +171,18 @@ async def test_reload_global( """) assert glbl_cfg(reload=True).get(['platforms','localhost','meta','x']) == '1' - # Modify the global config file - global_config_path.write_text(""" - [platforms] - [[localhost]] - [[[meta]]] - x = 2 - """) - id_ = flow(one_conf) schd = scheduler(id_) async with start(schd) as log: + # Modify the global config file + global_config_path.write_text(""" + [platforms] + [[localhost]] + [[[meta]]] + x = 2 + """) + # reload the workflow and global config await commands.run_cmd(commands.reload_workflow, schd, reload_global=True) @@ -196,3 +196,25 @@ async def test_reload_global( # Task platforms reflect the new config assert schd.pool.get_tasks()[0].platform['meta']['x'] == '2' + + # Modify the global config file with an error + global_config_path.write_text(""" + [ERROR] + [[localhost]] + [[[meta]]] + x = 3 + """) + + # reload the workflow and global config + await commands.run_cmd(commands.reload_workflow, schd, reload_global=True) + + # Error is noted in the log + assert log_filter( + log, + contains=( + 'This is probably due to an issue with the new configuration.' + ) + ) + + # Task platforms should be the last valid value + assert schd.pool.get_tasks()[0].platform['meta']['x'] == '2' From 9b7c1e80d386f73dd158b5e49f61a030377913f0 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Tue, 3 Dec 2024 14:31:18 +1100 Subject: [PATCH 04/13] Fix lint and tests --- cylc/flow/network/schema.py | 4 ++-- cylc/flow/scripts/reload.py | 6 +++--- tests/unit/cfgspec/test_globalcfg.py | 26 -------------------------- 3 files changed, 5 insertions(+), 31 deletions(-) diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py index 84e4afe7a3f..f5aaddae6ea 100644 --- a/cylc/flow/network/schema.py +++ b/cylc/flow/network/schema.py @@ -1887,8 +1887,8 @@ class Arguments: workflows = graphene.List(WorkflowID, required=True) reload_global = Boolean( - default_value=False, - description="Also reload global config") + default_value=False, + description="Also reload global config") result = GenericScalar() diff --git a/cylc/flow/scripts/reload.py b/cylc/flow/scripts/reload.py index a0250a21c06..1d1b89ed991 100755 --- a/cylc/flow/scripts/reload.py +++ b/cylc/flow/scripts/reload.py @@ -91,9 +91,9 @@ def get_option_parser(): ) parser.add_option( - "-g", "--global", - help="also reload global configuration.", - action="store_true", default=False, dest="reload_global") + "-g", "--global", + help="also reload global configuration.", + action="store_true", default=False, dest="reload_global") return parser diff --git a/tests/unit/cfgspec/test_globalcfg.py b/tests/unit/cfgspec/test_globalcfg.py index 83f6ec90246..996d0fa4d95 100644 --- a/tests/unit/cfgspec/test_globalcfg.py +++ b/tests/unit/cfgspec/test_globalcfg.py @@ -174,29 +174,3 @@ def test_platform_ssh_forward_variables(mock_global_config): ''') assert glblcfg.get(['platforms','foo','ssh forward environment variables']) == ["FOO", "BAR"] - -def test_reload(mock_global_config, tmp_path): - # Load a config - glblcfg: GlobalConfig = mock_global_config(f''' - [platforms] - [[foo]] - [[[meta]]] - x = 1 - ''') - - # Update the global config file and reload - conf_path = tmp_path / GlobalConfig.CONF_BASENAME - conf_path.write_text(''' - [platforms] - [[foo]] - [[[meta]]] - x = 2 - ''') - glblcfg.load() - - assert glblcfg.get(['platforms','foo','meta','x']) == '2' - - from cylc.flow.platforms import get_platform - platform = get_platform("foo") - - assert platform['meta']['x'] == "2" From 7f3bd41928f3363465c3982bb3e30b527a4eb28d Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Tue, 3 Dec 2024 15:59:40 +1100 Subject: [PATCH 05/13] Update test structure --- tests/integration/test_reload.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_reload.py b/tests/integration/test_reload.py index 2871feb83f1..36eb3e3e47d 100644 --- a/tests/integration/test_reload.py +++ b/tests/integration/test_reload.py @@ -173,7 +173,7 @@ async def test_reload_global( id_ = flow(one_conf) schd = scheduler(id_) - async with start(schd) as log: + async with start(schd): # Modify the global config file global_config_path.write_text(""" @@ -184,11 +184,10 @@ async def test_reload_global( """) # reload the workflow and global config - await commands.run_cmd(commands.reload_workflow, schd, reload_global=True) + await commands.run_cmd(commands.reload_workflow(schd, reload_global=True)) # Global config should have been reloaded assert log_filter( - log, contains=( 'Reloading the global configuration.' ) @@ -206,11 +205,10 @@ async def test_reload_global( """) # reload the workflow and global config - await commands.run_cmd(commands.reload_workflow, schd, reload_global=True) + await commands.run_cmd(commands.reload_workflow(schd, reload_global=True)) # Error is noted in the log assert log_filter( - log, contains=( 'This is probably due to an issue with the new configuration.' ) From 7be6be5fa3184bff4b2fe3e6ebeab82ce1f6a10c Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 5 Dec 2024 15:07:52 +1100 Subject: [PATCH 06/13] Revert proxy change, fix test --- cylc/flow/task_proxy.py | 1 + tests/integration/test_reload.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cylc/flow/task_proxy.py b/cylc/flow/task_proxy.py index aa48db41c3d..e641b2598e1 100644 --- a/cylc/flow/task_proxy.py +++ b/cylc/flow/task_proxy.py @@ -344,6 +344,7 @@ def copy_to_reload_successor( reload_successor.summary = self.summary reload_successor.local_job_file_path = self.local_job_file_path reload_successor.try_timers = self.try_timers + reload_successor.platform = self.platform reload_successor.job_vacated = self.job_vacated reload_successor.poll_timer = self.poll_timer reload_successor.timeout = self.timeout diff --git a/tests/integration/test_reload.py b/tests/integration/test_reload.py index 36eb3e3e47d..c6321e5a842 100644 --- a/tests/integration/test_reload.py +++ b/tests/integration/test_reload.py @@ -25,6 +25,7 @@ TASK_STATUS_SUBMITTED, ) from cylc.flow.cfgspec.glbl_cfg import glbl_cfg +from cylc.flow.platforms import get_platform async def test_reload_waits_for_pending_tasks( @@ -174,6 +175,10 @@ async def test_reload_global( id_ = flow(one_conf) schd = scheduler(id_) async with start(schd): + # Task platforms reflect the original config + rtconf = schd.broadcast_mgr.get_updated_rtconfig(schd.pool.get_tasks()[0]) + platform = get_platform(rtconf) + assert platform['meta']['x'] == '1' # Modify the global config file global_config_path.write_text(""" @@ -194,11 +199,14 @@ async def test_reload_global( ) # Task platforms reflect the new config - assert schd.pool.get_tasks()[0].platform['meta']['x'] == '2' + rtconf = schd.broadcast_mgr.get_updated_rtconfig(schd.pool.get_tasks()[0]) + platform = get_platform(rtconf) + assert platform['meta']['x'] == '2' # Modify the global config file with an error global_config_path.write_text(""" [ERROR] + [platforms] [[localhost]] [[[meta]]] x = 3 @@ -215,4 +223,6 @@ async def test_reload_global( ) # Task platforms should be the last valid value - assert schd.pool.get_tasks()[0].platform['meta']['x'] == '2' + rtconf = schd.broadcast_mgr.get_updated_rtconfig(schd.pool.get_tasks()[0]) + platform = get_platform(rtconf) + assert platform['meta']['x'] == '2' From d6fd8224d3e118cb68f6600f1f45bba1c0cf156c Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 5 Dec 2024 15:21:07 +1100 Subject: [PATCH 07/13] Add notes for settings not reloadable --- cylc/flow/cfgspec/globalcfg.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py index 18570a65c3e..77aeefd31f1 100644 --- a/cylc/flow/cfgspec/globalcfg.py +++ b/cylc/flow/cfgspec/globalcfg.py @@ -108,6 +108,11 @@ Not to be confused with :cylc:conf:`flow.cylc[scheduling]`. +.. note:: + + Scheduler settings cannot be reloaded, the server must be stopped and + restarted for changes to take effect + .. versionchanged:: 8.0.0 {REPLACES}``[cylc]`` @@ -1171,6 +1176,11 @@ def default_for( Symlinks from the the standard ``$HOME/cylc-run`` locations will be created. + .. note:: + + Once a platform has been installed and symlinks created they + cannot be modified for that run. + .. versionadded:: 8.0.0 """): with Conf('', desc=dedent(""" From 172537d98c283a885957b97f25abfbb9613671d9 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 5 Dec 2024 15:23:03 +1100 Subject: [PATCH 08/13] Add changelog entry --- changes.d/6509.feat.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes.d/6509.feat.md diff --git a/changes.d/6509.feat.md b/changes.d/6509.feat.md new file mode 100644 index 00000000000..84063795626 --- /dev/null +++ b/changes.d/6509.feat.md @@ -0,0 +1 @@ +Added --global flag to 'cylc reload' From 49bf3de9258585644a3afc3745eac3b21fd49155 Mon Sep 17 00:00:00 2001 From: Tim Pillinger <26465611+wxtim@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:11:36 +0000 Subject: [PATCH 09/13] Add a store for CLI variables for reload, which can then be accessed by both Cylc VR and Reload. Mirrors practice for VALIDATE, INSTALL, PLAY and RESTART --- cylc/flow/scripts/reload.py | 20 +++++++++++++++----- cylc/flow/scripts/validate_reinstall.py | 3 +++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/cylc/flow/scripts/reload.py b/cylc/flow/scripts/reload.py index 1d1b89ed991..c8e39989d1c 100755 --- a/cylc/flow/scripts/reload.py +++ b/cylc/flow/scripts/reload.py @@ -60,6 +60,7 @@ from cylc.flow.option_parsers import ( WORKFLOW_ID_MULTI_ARG_DOC, CylcOptionParser as COP, + OptionSettings, ) from cylc.flow.terminal import cli_function @@ -67,6 +68,18 @@ from optparse import Values +RELOAD_OPTIONS = [ + OptionSettings( + ['-g', '--global'], + help='also reload global configuration.', + action="store_true", + default=False, + dest="reload_global", + sources={'reload'} + ), +] + + MUTATION = ''' mutation ( $wFlows: [WorkflowID]!, @@ -89,11 +102,8 @@ def get_option_parser(): multiworkflow=True, argdoc=[WORKFLOW_ID_MULTI_ARG_DOC], ) - - parser.add_option( - "-g", "--global", - help="also reload global configuration.", - action="store_true", default=False, dest="reload_global") + for option in RELOAD_OPTIONS: + parser.add_option(*option.args, **option.kwargs) return parser diff --git a/cylc/flow/scripts/validate_reinstall.py b/cylc/flow/scripts/validate_reinstall.py index 4e78d673d7e..afb36ddeab4 100644 --- a/cylc/flow/scripts/validate_reinstall.py +++ b/cylc/flow/scripts/validate_reinstall.py @@ -73,6 +73,7 @@ reinstall_cli as cylc_reinstall, ) from cylc.flow.scripts.reload import ( + RELOAD_OPTIONS, run as cylc_reload ) from cylc.flow.terminal import cli_function @@ -86,6 +87,7 @@ VALIDATE_OPTIONS, REINSTALL_OPTIONS, REINSTALL_CYLC_ROSE_OPTIONS, + RELOAD_OPTIONS, PLAY_OPTIONS, CYLC_ROSE_OPTIONS, modify={'cylc-rose': 'validate, install'} @@ -102,6 +104,7 @@ def get_option_parser() -> COP: for option in VR_OPTIONS: parser.add_option(*option.args, **option.kwargs) parser.set_defaults(is_validate=True) + return parser From 557f74d736c54546ed7d3980194bf6a4a9237de9 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Wed, 11 Dec 2024 15:50:53 +1100 Subject: [PATCH 10/13] Add coverage over verbose --- tests/integration/test_reload.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/integration/test_reload.py b/tests/integration/test_reload.py index c6321e5a842..097f8360c8b 100644 --- a/tests/integration/test_reload.py +++ b/tests/integration/test_reload.py @@ -26,6 +26,7 @@ ) from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.platforms import get_platform +from cylc.flow import flags async def test_reload_waits_for_pending_tasks( @@ -226,3 +227,12 @@ async def test_reload_global( rtconf = schd.broadcast_mgr.get_updated_rtconfig(schd.pool.get_tasks()[0]) platform = get_platform(rtconf) assert platform['meta']['x'] == '2' + + # reload the workflow in verbose mode + flags.verbosity = 2 + await commands.run_cmd(commands.reload_workflow(schd, reload_global=True)) + + # Traceback is shown in the log (match for ERROR, trace is not captured) + assert log_filter( + exact_match = 'ERROR' + ) From b4542e16a99cea6a2dae7e29f94ac2f9cf1ebf52 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 12 Dec 2024 15:23:24 +1100 Subject: [PATCH 11/13] Add global reload functional test --- tests/functional/reload/29-global.t | 47 +++++++++++++++++++++ tests/functional/reload/29-global/flow.cylc | 30 +++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 tests/functional/reload/29-global.t create mode 100644 tests/functional/reload/29-global/flow.cylc diff --git a/tests/functional/reload/29-global.t b/tests/functional/reload/29-global.t new file mode 100644 index 00000000000..2ffa716b0bc --- /dev/null +++ b/tests/functional/reload/29-global.t @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- +# Test reloading global configuration +. "$(dirname "$0")/test_header" +set_test_number 9 + +create_test_global_config "" "" + +TEST_NAME="${TEST_NAME_BASE}" +install_workflow "${TEST_NAME}" "${TEST_NAME_BASE}" + +# Validate the config +run_ok "${TEST_NAME}-validate" cylc validate "${WORKFLOW_NAME}" + +# Run the workflow +workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" --no-detach -v + +# Reload happened +grep_ok "Reloading the global configuration" "${WORKFLOW_RUN_DIR}/log/scheduler/01-start-01.log" + +# Reload hasn't happened in job a, has happened in b and c +grep_fail "global init-script reloaded!" "${WORKFLOW_RUN_DIR}/log/job/1/a/01/job.out" +grep_ok "global init-script reloaded!" "${WORKFLOW_RUN_DIR}/log/job/1/b/01/job.out" +grep_ok "global init-script reloaded!" "${WORKFLOW_RUN_DIR}/log/job/1/c/01/job.out" + +# Events are original in job a, updated in b and c +grep_fail "!!EVENT!! succeeded 1/a" "${WORKFLOW_RUN_DIR}/log/scheduler/01-start-01.log" +grep_ok "!!EVENT!! succeeded 1/b" "${WORKFLOW_RUN_DIR}/log/scheduler/01-start-01.log" +grep_ok "!!EVENT!! succeeded 1/c" "${WORKFLOW_RUN_DIR}/log/scheduler/01-start-01.log" + +purge +exit diff --git a/tests/functional/reload/29-global/flow.cylc b/tests/functional/reload/29-global/flow.cylc new file mode 100644 index 00000000000..e1584c977ae --- /dev/null +++ b/tests/functional/reload/29-global/flow.cylc @@ -0,0 +1,30 @@ +[scheduling] + cycling mode = integer + [[graph]] + R1 = a => reload-global => b & c +[runtime] + [[root]] + script = true + [[a,b]] + platform = localhost + [[c]] + platform = $CYLC_TEST_PLATFORM + [[reload-global]] + platform = localhost + script = """ + # Append to global config + cat >> "$CYLC_CONF_PATH/global.cylc" < Date: Thu, 12 Dec 2024 16:13:00 +1100 Subject: [PATCH 12/13] Test reloading platform groups --- tests/integration/test_reload.py | 81 +++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_reload.py b/tests/integration/test_reload.py index 097f8360c8b..3d17a9f35ea 100644 --- a/tests/integration/test_reload.py +++ b/tests/integration/test_reload.py @@ -151,7 +151,7 @@ async def test_reload_failure( assert schd.config.cfg['scheduling']['graph']['R1'] == 'one' -async def test_reload_global( +async def test_reload_global_platform( flow, one_conf, scheduler, @@ -171,7 +171,8 @@ async def test_reload_global( [[[meta]]] x = 1 """) - assert glbl_cfg(reload=True).get(['platforms','localhost','meta','x']) == '1' + glbl_cfg(reload=True) + assert glbl_cfg().get(['platforms','localhost','meta','x']) == '1' id_ = flow(one_conf) schd = scheduler(id_) @@ -236,3 +237,79 @@ async def test_reload_global( assert log_filter( exact_match = 'ERROR' ) + + +async def test_reload_global_platform_group( + flow, + scheduler, + start, + log_filter, + tmp_path, + monkeypatch, + ): + + global_config_path = tmp_path / 'global.cylc' + monkeypatch.setenv("CYLC_CONF_PATH", str(global_config_path.parent)) + + # Original global config file + global_config_path.write_text(""" + [platforms] + [[foo]] + [[[meta]]] + x = 1 + [platform groups] + [[pg]] + platforms = foo + """) + glbl_cfg(reload=True) + + # Task using the platform group + conf = { + 'scheduler': { + 'allow implicit tasks': True + }, + 'scheduling': { + 'graph': { + 'R1': 'one' + } + }, + 'runtime': { + 'one': { + 'platform': 'pg', + } + }, + } + + id_ = flow(conf) + schd = scheduler(id_) + async with start(schd): + # Task platforms reflect the original config + rtconf = schd.broadcast_mgr.get_updated_rtconfig(schd.pool.get_tasks()[0]) + platform = get_platform(rtconf) + assert platform['meta']['x'] == '1' + + # Modify the global config file + global_config_path.write_text(""" + [platforms] + [[bar]] + [[[meta]]] + x = 2 + [platform groups] + [[pg]] + platforms = bar + """) + + # reload the workflow and global config + await commands.run_cmd(commands.reload_workflow(schd, reload_global=True)) + + # Global config should have been reloaded + assert log_filter( + contains=( + 'Reloading the global configuration.' + ) + ) + + # Task platforms reflect the new config + rtconf = schd.broadcast_mgr.get_updated_rtconfig(schd.pool.get_tasks()[0]) + platform = get_platform(rtconf) + assert platform['meta']['x'] == '2' From 8c933850112789cacf2e2e5ea24794f630940239 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 12 Dec 2024 16:38:14 +1100 Subject: [PATCH 13/13] Update scheduler reload docs --- cylc/flow/cfgspec/globalcfg.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py index 77aeefd31f1..06b7265ec7d 100644 --- a/cylc/flow/cfgspec/globalcfg.py +++ b/cylc/flow/cfgspec/globalcfg.py @@ -110,8 +110,10 @@ .. note:: - Scheduler settings cannot be reloaded, the server must be stopped and - restarted for changes to take effect + The majority of scheduler settings affect the server and cannot be reloaded + with ``cylc reload --global``, the server must be stopped and restarted for + changes to take effect except for the sections [mail] and [events] which + provide workflow defaults. .. versionchanged:: 8.0.0