From c1582128ecc089b70fee52f30d331b6ec1129e51 Mon Sep 17 00:00:00 2001 From: Lyubov Yamshchikova <43475193+YamLyubov@users.noreply.github.com> Date: Mon, 27 Nov 2023 14:43:03 +0300 Subject: [PATCH] Tuner interface update (#234) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add kwargs * Add warning multi-obj * Fix timeout update * Add timeout * Minor * Refactor * зуз8 --- golem/core/tuning/hyperopt_tuner.py | 9 +- golem/core/tuning/iopt_tuner.py | 19 ++-- golem/core/tuning/optuna_tuner.py | 31 ++++--- golem/core/tuning/sequential.py | 129 +++++++++++++-------------- golem/core/tuning/simultaneous.py | 102 +++++++++------------ golem/core/tuning/tuner_interface.py | 51 +++++++++-- test/unit/tuning/test_tuning.py | 2 +- 7 files changed, 176 insertions(+), 167 deletions(-) diff --git a/golem/core/tuning/hyperopt_tuner.py b/golem/core/tuning/hyperopt_tuner.py index 153216ba9..e954aee47 100644 --- a/golem/core/tuning/hyperopt_tuner.py +++ b/golem/core/tuning/hyperopt_tuner.py @@ -10,7 +10,6 @@ from golem.core.adapter import BaseOptimizationAdapter from golem.core.log import default_log from golem.core.optimisers.objective import ObjectiveFunction -from golem.core.optimisers.timer import Timer from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label from golem.core.tuning.tuner_interface import BaseTuner @@ -41,7 +40,7 @@ def __init__(self, objective_evaluate: ObjectiveFunction, timeout: timedelta = timedelta(minutes=5), n_jobs: int = -1, deviation: float = 0.05, - algo: Callable = tpe.suggest): + algo: Callable = tpe.suggest, **kwargs): early_stopping_rounds = early_stopping_rounds or max(100, int(np.sqrt(iterations) * 10)) super().__init__(objective_evaluate, search_space, @@ -50,16 +49,12 @@ def __init__(self, objective_evaluate: ObjectiveFunction, early_stopping_rounds, timeout, n_jobs, - deviation) + deviation, **kwargs) self.early_stop_fn = no_progress_loss(iteration_stop_count=self.early_stopping_rounds) - self.max_seconds = int(timeout.seconds) if timeout is not None else None self.algo = algo self.log = default_log(self) - def _update_remaining_time(self, tuner_timer: Timer): - self.max_seconds = self.max_seconds - tuner_timer.minutes_from_start * 60 - def get_parameter_hyperopt_space(search_space: SearchSpace, operation_name: str, diff --git a/golem/core/tuning/iopt_tuner.py b/golem/core/tuning/iopt_tuner.py index 021baf5f3..a0b1cbd6f 100644 --- a/golem/core/tuning/iopt_tuner.py +++ b/golem/core/tuning/iopt_tuner.py @@ -142,18 +142,12 @@ def __init__(self, objective_evaluate: ObjectiveEvaluate, epsR=np.double(eps_r), refineSolution=refine_solution) - def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainGraphForTune: - graph = self.adapter.adapt(graph) + def _tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainGraphForTune: problem_parameters, initial_parameters = self._get_parameters_for_tune(graph) - no_parameters_to_optimize = (not problem_parameters.discrete_parameters_names and - not problem_parameters.float_parameters_names) - self.init_check(graph) - - if no_parameters_to_optimize: - self._stop_tuning_with_message(f'Graph "{graph.graph_description}" has no parameters to optimize') - final_graph = graph - else: + has_parameters_to_optimize = (len(problem_parameters.discrete_parameters_names) > 0 or + len(problem_parameters.float_parameters_names) > 0) + if self._check_if_tuning_possible(graph, has_parameters_to_optimize): if initial_parameters: initial_point = Point(**initial_parameters) self.solver_parameters.startPoint = initial_point @@ -171,10 +165,9 @@ def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainG final_graph = self.set_arg_graph(graph, best_parameters) self.was_tuned = True + else: + final_graph = graph - # Validate if optimisation did well - graph = self.final_check(final_graph) - final_graph = self.adapter.restore(graph) return final_graph def _get_parameters_for_tune(self, graph: OptGraph) -> Tuple[IOptProblemParameters, dict]: diff --git a/golem/core/tuning/optuna_tuner.py b/golem/core/tuning/optuna_tuner.py index aabcb95d0..6a91c5ea0 100644 --- a/golem/core/tuning/optuna_tuner.py +++ b/golem/core/tuning/optuna_tuner.py @@ -12,6 +12,7 @@ from golem.core.optimisers.objective import ObjectiveFunction from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label from golem.core.tuning.tuner_interface import BaseTuner, DomainGraphForTune +from golem.utilities.data_structures import ensure_wrapped_in_sequence class OptunaTuner(BaseTuner): @@ -22,8 +23,7 @@ def __init__(self, objective_evaluate: ObjectiveFunction, early_stopping_rounds: Optional[int] = None, timeout: timedelta = timedelta(minutes=5), n_jobs: int = -1, - deviation: float = 0.05, - objectives_number: int = 1): + deviation: float = 0.05, **kwargs): super().__init__(objective_evaluate, search_space, adapter, @@ -31,25 +31,24 @@ def __init__(self, objective_evaluate: ObjectiveFunction, early_stopping_rounds, timeout, n_jobs, - deviation) - self.objectives_number = objectives_number + deviation, **kwargs) self.study = None - def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> \ + def _tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> \ Union[DomainGraphForTune, Sequence[DomainGraphForTune]]: - graph = self.adapter.adapt(graph) predefined_objective = partial(self.objective, graph=graph) - is_multi_objective = self.objectives_number > 1 - self.init_check(graph) + self.objectives_number = len(ensure_wrapped_in_sequence(self.init_metric)) + is_multi_objective = self.objectives_number > 1 self.study = optuna.create_study(directions=['minimize'] * self.objectives_number) init_parameters, has_parameters_to_optimize = self._get_initial_point(graph) - if not has_parameters_to_optimize: - self._stop_tuning_with_message(f'Graph {graph.graph_description} has no parameters to optimize') - tuned_graphs = self.init_graph - else: + remaining_time = self._get_remaining_time() + if self._check_if_tuning_possible(graph, + has_parameters_to_optimize, + remaining_time, + supports_multi_objective=True): # Enqueue initial point to try if init_parameters: self.study.enqueue_trial(init_parameters) @@ -60,7 +59,7 @@ def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> \ self.study.optimize(predefined_objective, n_trials=self.iterations, n_jobs=self.n_jobs, - timeout=self.timeout.seconds, + timeout=remaining_time, callbacks=[self.early_stopping_callback], show_progress_bar=show_progress) @@ -75,9 +74,9 @@ def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> \ tuned_graph = self.set_arg_graph(deepcopy(graph), best_parameters) tuned_graphs.append(tuned_graph) self.was_tuned = True - final_graphs = self.final_check(tuned_graphs, is_multi_objective) - final_graphs = self.adapter.restore(final_graphs) - return final_graphs + else: + tuned_graphs = graph + return tuned_graphs def objective(self, trial: Trial, graph: OptGraph) -> Union[float, Sequence[float, ]]: new_parameters = self._get_parameters_from_trial(graph, trial) diff --git a/golem/core/tuning/sequential.py b/golem/core/tuning/sequential.py index fd5924549..003bb1700 100644 --- a/golem/core/tuning/sequential.py +++ b/golem/core/tuning/sequential.py @@ -26,7 +26,7 @@ def __init__(self, objective_evaluate: ObjectiveFunction, n_jobs: int = -1, deviation: float = 0.05, algo: Callable = tpe.suggest, - inverse_node_order: bool = False): + inverse_node_order: bool = False, **kwargs): super().__init__(objective_evaluate, search_space, adapter, @@ -34,61 +34,53 @@ def __init__(self, objective_evaluate: ObjectiveFunction, early_stopping_rounds, timeout, n_jobs, deviation, - algo) + algo, **kwargs) self.inverse_node_order = inverse_node_order - def tune(self, graph: DomainGraphForTune) -> DomainGraphForTune: + def _tune(self, graph: DomainGraphForTune, **kwargs) -> DomainGraphForTune: """ Method for hyperparameters tuning on the entire graph Args: graph: graph which hyperparameters will be tuned """ - graph = self.adapter.adapt(graph) - - # Check source metrics for data - self.init_check(graph) - - # Calculate amount of iterations we can apply per node - nodes_amount = graph.length - iterations_per_node = round(self.iterations / nodes_amount) - iterations_per_node = int(iterations_per_node) - if iterations_per_node == 0: - iterations_per_node = 1 - - # Calculate amount of seconds we can apply per node - if self.max_seconds is not None: - seconds_per_node = round(self.max_seconds / nodes_amount) - seconds_per_node = int(seconds_per_node) - else: - seconds_per_node = None - - # Tuning performed sequentially for every node - so get ids of nodes - nodes_ids = self.get_nodes_order(nodes_number=nodes_amount) - for node_id in nodes_ids: - node = graph.nodes[node_id] - operation_name = node.name - - # Get node's parameters to optimize - node_params = get_node_parameters_for_hyperopt(self.search_space, node_id, operation_name) - - if not node_params: - self.log.info(f'"{operation_name}" operation has no parameters to optimize') + remaining_time = self._get_remaining_time() + if self._check_if_tuning_possible(graph, parameters_to_optimize=True, remaining_time=remaining_time): + # Calculate amount of iterations we can apply per node + nodes_amount = graph.length + iterations_per_node = round(self.iterations / nodes_amount) + iterations_per_node = int(iterations_per_node) + if iterations_per_node == 0: + iterations_per_node = 1 + + # Calculate amount of seconds we can apply per node + if remaining_time is not None: + seconds_per_node = round(remaining_time / nodes_amount) + seconds_per_node = int(seconds_per_node) else: - # Apply tuning for current node - self._optimize_node(node_id=node_id, - graph=graph, - node_params=node_params, - iterations_per_node=iterations_per_node, - seconds_per_node=seconds_per_node) - - # Validate if optimisation did well - final_graph = self.final_check(graph) + seconds_per_node = None + + # Tuning performed sequentially for every node - so get ids of nodes + nodes_ids = self.get_nodes_order(nodes_number=nodes_amount) + for node_id in nodes_ids: + node = graph.nodes[node_id] + operation_name = node.name + + # Get node's parameters to optimize + node_params = get_node_parameters_for_hyperopt(self.search_space, node_id, operation_name) + + if not node_params: + self.log.info(f'"{operation_name}" operation has no parameters to optimize') + else: + # Apply tuning for current node + self._optimize_node(node_id=node_id, + graph=graph, + node_params=node_params, + iterations_per_node=iterations_per_node, + seconds_per_node=seconds_per_node) - self.was_tuned = True - final_graph = self.adapter.restore(final_graph) - - return final_graph + self.was_tuned = True + return graph def get_nodes_order(self, nodes_number: int) -> range: """ Method returns list with indices of nodes in the graph @@ -118,30 +110,33 @@ def tune_node(self, graph: DomainGraphForTune, node_index: int) -> DomainGraphFo """ graph = self.adapter.adapt(graph) - self.init_check(graph) + with self.timer: + self.init_check(graph) - node = graph.nodes[node_index] - operation_name = node.name + node = graph.nodes[node_index] + operation_name = node.name - # Get node's parameters to optimize - node_params = get_node_parameters_for_hyperopt(self.search_space, - node_id=node_index, - operation_name=operation_name) + # Get node's parameters to optimize + node_params = get_node_parameters_for_hyperopt(self.search_space, + node_id=node_index, + operation_name=operation_name) - if not node_params: - self._stop_tuning_with_message(f'"{operation_name}" operation has no parameters to optimize') - else: - # Apply tuning for current node - self._optimize_node(graph=graph, - node_id=node_index, - node_params=node_params, - iterations_per_node=self.iterations, - seconds_per_node=self.max_seconds, - ) - self.was_tuned = True + remaining_time = self._get_remaining_time() + if self._check_if_tuning_possible(graph, len(node_params) > 1, remaining_time): + # Apply tuning for current node + self._optimize_node(graph=graph, + node_id=node_index, + node_params=node_params, + iterations_per_node=self.iterations, + seconds_per_node=remaining_time + ) + self.was_tuned = True - # Validation is the optimization do well - final_graph = self.final_check(graph) + # Validation is the optimization do well + final_graph = self.final_check(graph) + else: + final_graph = graph + self.obtained_metric = self.init_metric final_graph = self.adapter.restore(final_graph) return final_graph @@ -149,7 +144,7 @@ def _optimize_node(self, graph: OptGraph, node_id: int, node_params: dict, iterations_per_node: int, - seconds_per_node: int) -> OptGraph: + seconds_per_node: float) -> OptGraph: """ Method for node optimization diff --git a/golem/core/tuning/simultaneous.py b/golem/core/tuning/simultaneous.py index 465dc81a8..f540114ea 100644 --- a/golem/core/tuning/simultaneous.py +++ b/golem/core/tuning/simultaneous.py @@ -5,7 +5,6 @@ from golem.core.constants import MIN_TIME_FOR_TUNING_IN_SEC from golem.core.optimisers.graph import OptGraph -from golem.core.optimisers.timer import Timer from golem.core.tuning.hyperopt_tuner import HyperoptTuner, get_node_parameters_for_hyperopt from golem.core.tuning.search_space import get_node_operation_parameter_label from golem.core.tuning.tuner_interface import DomainGraphForTune @@ -16,7 +15,7 @@ class SimultaneousTuner(HyperoptTuner): Class for hyperparameters optimization for all nodes simultaneously """ - def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainGraphForTune: + def _tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainGraphForTune: """ Function for hyperparameters tuning on the entire graph Args: @@ -26,69 +25,56 @@ def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainG Returns: Graph with tuned hyperparameters """ - - graph = self.adapter.adapt(graph) parameters_dict, init_parameters = self._get_parameters_for_tune(graph) - - with Timer() as global_tuner_timer: - self.init_check(graph) - self._update_remaining_time(global_tuner_timer) - - if not parameters_dict: - self._stop_tuning_with_message(f'Graph "{graph.graph_description}" has no parameters to optimize') + remaining_time = self._get_remaining_time() + + if self._check_if_tuning_possible(graph, parameters_dict, remaining_time): + trials = Trials() + + try: + # try searching using initial parameters + # (uses original search space with fixed initial parameters) + trials, init_trials_num = self._search_near_initial_parameters(graph, + parameters_dict, + init_parameters, + trials, + remaining_time, + show_progress) + remaining_time = self._get_remaining_time() + if remaining_time > MIN_TIME_FOR_TUNING_IN_SEC: + fmin(partial(self._objective, graph=graph), + parameters_dict, + trials=trials, + algo=self.algo, + max_evals=self.iterations, + show_progressbar=show_progress, + early_stop_fn=self.early_stop_fn, + timeout=remaining_time) + else: + self.log.message('Tunner stopped after initial search due to the lack of time') + + best = space_eval(space=parameters_dict, hp_assignment=trials.argmin) + # check if best point was obtained using search space with fixed initial parameters + is_best_trial_with_init_params = trials.best_trial.get('tid') in range(init_trials_num) + if is_best_trial_with_init_params: + best = {**best, **init_parameters} + + final_graph = self.set_arg_graph(graph=graph, parameters=best) + + self.was_tuned = True + + except Exception as ex: + self.log.warning(f'Exception {ex} occurred during tuning') final_graph = graph - - elif self.max_seconds <= MIN_TIME_FOR_TUNING_IN_SEC: - self._stop_tuning_with_message('Tunner stopped after initial assumption due to the lack of time') - final_graph = graph - - else: - trials = Trials() - - try: - # try searching using initial parameters - # (uses original search space with fixed initial parameters) - trials, init_trials_num = self._search_near_initial_parameters(graph, - parameters_dict, - init_parameters, - trials, - show_progress) - self._update_remaining_time(global_tuner_timer) - if self.max_seconds > MIN_TIME_FOR_TUNING_IN_SEC: - fmin(partial(self._objective, graph=graph), - parameters_dict, - trials=trials, - algo=self.algo, - max_evals=self.iterations, - show_progressbar=show_progress, - early_stop_fn=self.early_stop_fn, - timeout=self.max_seconds) - else: - self.log.message('Tunner stopped after initial search due to the lack of time') - - best = space_eval(space=parameters_dict, hp_assignment=trials.argmin) - # check if best point was obtained using search space with fixed initial parameters - is_best_trial_with_init_params = trials.best_trial.get('tid') in range(init_trials_num) - if is_best_trial_with_init_params: - best = {**best, **init_parameters} - - final_graph = self.set_arg_graph(graph=graph, parameters=best) - - self.was_tuned = True - - except Exception as ex: - self.log.warning(f'Exception {ex} occurred during tuning') - final_graph = graph - - # Validate if optimisation did well - graph = self.final_check(final_graph) - final_graph = self.adapter.restore(graph) + else: + final_graph = graph return final_graph def _search_near_initial_parameters(self, graph: OptGraph, search_space: dict, initial_parameters: dict, trials: Trials, + remaining_time: float, show_progress: bool = True) -> Tuple[Trials, int]: """ Method to search using the search space where parameters initially set for the graph are fixed. This allows not to lose results obtained while composition process @@ -123,7 +109,7 @@ def _search_near_initial_parameters(self, graph: OptGraph, max_evals=init_trials_num, show_progressbar=show_progress, early_stop_fn=self.early_stop_fn, - timeout=self.max_seconds) + timeout=remaining_time) return trials, init_trials_num def _get_parameters_for_tune(self, graph: OptGraph) -> Tuple[dict, dict]: diff --git a/golem/core/tuning/tuner_interface.py b/golem/core/tuning/tuner_interface.py index c4d6ff6e6..098c37347 100644 --- a/golem/core/tuning/tuner_interface.py +++ b/golem/core/tuning/tuner_interface.py @@ -7,12 +7,13 @@ from golem.core.adapter import BaseOptimizationAdapter from golem.core.adapter.adapter import IdentityAdapter -from golem.core.constants import MAX_TUNING_METRIC_VALUE +from golem.core.constants import MAX_TUNING_METRIC_VALUE, MIN_TIME_FOR_TUNING_IN_SEC from golem.core.dag.graph_utils import graph_structure from golem.core.log import default_log from golem.core.optimisers.fitness import SingleObjFitness, MultiObjFitness from golem.core.optimisers.graph import OptGraph from golem.core.optimisers.objective import ObjectiveEvaluate, ObjectiveFunction +from golem.core.optimisers.timer import Timer from golem.core.tuning.search_space import SearchSpace, convert_parameters from golem.utilities.data_structures import ensure_wrapped_in_sequence @@ -43,7 +44,7 @@ def __init__(self, objective_evaluate: ObjectiveFunction, early_stopping_rounds: Optional[int] = None, timeout: timedelta = timedelta(minutes=5), n_jobs: int = -1, - deviation: float = 0.05): + deviation: float = 0.05, **kwargs): self.iterations = iterations self.adapter = adapter or IdentityAdapter() self.search_space = search_space @@ -54,6 +55,7 @@ def __init__(self, objective_evaluate: ObjectiveFunction, self.deviation = deviation self.timeout = timeout + self.timer = Timer() self.early_stopping_rounds = early_stopping_rounds self._default_metric_value = MAX_TUNING_METRIC_VALUE @@ -62,9 +64,9 @@ def __init__(self, objective_evaluate: ObjectiveFunction, self.init_metric = None self.obtained_metric = None self.log = default_log(self) + self.objectives_number = 1 - @abstractmethod - def tune(self, graph: DomainGraphForTune) -> Union[DomainGraphForTune, Sequence[DomainGraphForTune]]: + def tune(self, graph: DomainGraphForTune, **kwargs) -> Union[DomainGraphForTune, Sequence[DomainGraphForTune]]: """ Function for hyperparameters tuning on the graph @@ -75,7 +77,22 @@ def tune(self, graph: DomainGraphForTune) -> Union[DomainGraphForTune, Sequence[ Graph with optimized hyperparameters or pareto front of optimized graphs in case of multi-objective optimization """ - raise NotImplementedError() + graph = self.adapter.adapt(graph) + self.was_tuned = False + with self.timer: + + # Check source metrics for data + self.init_check(graph) + final_graph = self._tune(graph, **kwargs) + # Validate if optimisation did well + final_graph = self.final_check(final_graph, self.objectives_number > 1) + + final_graph = self.adapter.restore(final_graph) + return final_graph + + @abstractmethod + def _tune(self, graph: DomainGraphForTune, **kwargs): + raise NotImplementedError def init_check(self, graph: OptGraph) -> None: """ @@ -163,6 +180,7 @@ def _multi_obj_final_check(self, tuned_graphs: Sequence[OptGraph]) -> Sequence[O else: self.log.message('Initial metric dominates all found solutions. Return initial graph.') final_graphs = self.init_graph + self.obtained_metric = self.init_metric return final_graphs def get_metric_value(self, graph: OptGraph) -> Union[float, Sequence[float]]: @@ -232,6 +250,29 @@ def set_arg_node(graph: OptGraph, node_id: int, node_params: dict) -> OptGraph: return graph + def _check_if_tuning_possible(self, graph: OptGraph, + parameters_to_optimize: bool, + remaining_time: Optional[float] = None, + supports_multi_objective: bool = False) -> bool: + if len(ensure_wrapped_in_sequence(self.init_metric)) > 1 and not supports_multi_objective: + self._stop_tuning_with_message(f'{self.__class__.__name__} does not support multi-objective optimization.') + return False + elif not parameters_to_optimize: + self._stop_tuning_with_message(f'Graph "{graph.graph_description}" has no parameters to optimize') + return False + elif remaining_time is not None: + if remaining_time <= MIN_TIME_FOR_TUNING_IN_SEC: + self._stop_tuning_with_message('Tunner stopped after initial assumption due to the lack of time') + return False + return True + def _stop_tuning_with_message(self, message: str): self.log.message(message) self.obtained_metric = self.init_metric + + def _get_remaining_time(self) -> Optional[float]: + if self.timeout is not None: + remaining_time = self.timeout.seconds - self.timer.seconds_from_start + return remaining_time + else: + return None diff --git a/test/unit/tuning/test_tuning.py b/test/unit/tuning/test_tuning.py index cec2a15a3..e2d7967d4 100644 --- a/test/unit/tuning/test_tuning.py +++ b/test/unit/tuning/test_tuning.py @@ -121,7 +121,7 @@ def test_node_tuning(search_space, graph): is_multi_objective=True)))]) def test_multi_objective_tuning(search_space, tuner_cls, init_graph, adapter, obj_eval): init_metric = obj_eval.evaluate(init_graph) - tuner = tuner_cls(obj_eval, search_space, adapter, iterations=20, objectives_number=2) + tuner = tuner_cls(obj_eval, search_space, adapter, iterations=20) tuned_graphs = tuner.tune(deepcopy(init_graph), show_progress=False) for graph in tuned_graphs: assert type(graph) == type(init_graph)