From 085ca3db85fbfdb8d4dead2c02ea87727d938b48 Mon Sep 17 00:00:00 2001 From: Lyubov Yamshchikova <43475193+YamLyubov@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:24:56 +0300 Subject: [PATCH] Update iopt version (#150) * Add starting point * Add categorical parameters, treat discrete and categorical as discrete * Fix discrete setting * Add n_jobs * Fix for 0.2.22 * Fix eps * Fix initial point * Add multi objective * Rebase * Minor --- docs/source/api/tuning.rst | 5 - examples/tuning_example.py | 72 +++++++++++++++ golem/core/tuning/iopt_tuner.py | 158 ++++++++++++++++++-------------- requirements.txt | 2 +- test/unit/tuning/test_tuning.py | 7 +- 5 files changed, 167 insertions(+), 77 deletions(-) create mode 100644 examples/tuning_example.py diff --git a/docs/source/api/tuning.rst b/docs/source/api/tuning.rst index 9858dd8e9..97d6e8a79 100644 --- a/docs/source/api/tuning.rst +++ b/docs/source/api/tuning.rst @@ -52,11 +52,6 @@ You can tune all parameters of graph nodes simultaneously using ``SimultaneousTu .. note:: ``IOptTuner`` implements deterministic algorithm. - For now ``IOptTuner`` can not be constrained by time, so constrain execution by number of iterations. - - Also ``IOptTuner`` can optimise only `continuous` and `discrete` parameters but not `categorical` ones. - `Categorical` parameters will be ignored while tuning. - ``IOptTuner`` is implemented using `IOpt library`_. See the `documentation`_ (in Russian) to learn more about the optimisation algorithm. diff --git a/examples/tuning_example.py b/examples/tuning_example.py new file mode 100644 index 000000000..6aab55480 --- /dev/null +++ b/examples/tuning_example.py @@ -0,0 +1,72 @@ +from datetime import timedelta + +from golem.core.optimisers.graph import OptNode, OptGraph +from golem.core.optimisers.objective import ObjectiveEvaluate, Objective +from golem.core.tuning.iopt_tuner import IOptTuner +from golem.core.tuning.search_space import SearchSpace +from test.unit.utils import ParamsSumMetric + + +def opt_graph_with_params(): + node_a = OptNode('a') + node_b = OptNode({'name': 'b', 'params': {'b2': 0.7, 'b3': 2}}) + node_c = OptNode('c', nodes_from=[node_a]) + node_d = OptNode('d', nodes_from=[node_b]) + node_final = OptNode('e', nodes_from=[node_c, node_d]) + graph = OptGraph(node_final) + return graph + + +def get_search_space(): + params_per_operation = { + 'a': { + 'a1': { + 'sampling-scope': [2, 7], + 'type': 'discrete' + }, + 'a2': { + 'sampling-scope': [1e-3, 1], + 'type': 'continuous' + }, + 'a3': { + 'sampling-scope': [['A', 'B', 'C']], + 'type': 'categorical' + } + }, + 'b': { + 'b1': { + 'sampling-scope': [["first", "second", "third"]], + 'type': 'categorical' + }, + 'b2': { + 'sampling-scope': [0.04, 1.0], + 'type': 'continuous' + }, + }, + 'e': { + 'e1': { + 'sampling-scope': [0.05, 1.0], + 'type': 'continuous' + }, + 'e2': { + 'sampling-scope': [0.05, 1.0], + 'type': 'continuous' + } + }, + 'k': { + 'k': { + 'sampling-scope': [1e-2, 10.0], + 'type': 'continuous' + } + }} + return SearchSpace(params_per_operation) + + +if __name__ == '__main__': + search_space = get_search_space() + graph = opt_graph_with_params() + # search for parameters that will maximize their sum + obj_eval = ObjectiveEvaluate(Objective({'sum_metric': ParamsSumMetric.get_value})) + + tuner = IOptTuner(obj_eval, search_space, iterations=10, n_jobs=-1) + tuned_graph = tuner.tune(graph) diff --git a/golem/core/tuning/iopt_tuner.py b/golem/core/tuning/iopt_tuner.py index a0b1cbd6f..ade0fc677 100644 --- a/golem/core/tuning/iopt_tuner.py +++ b/golem/core/tuning/iopt_tuner.py @@ -1,18 +1,22 @@ +from copy import deepcopy from dataclasses import dataclass, field +from datetime import timedelta from typing import List, Dict, Generic, Tuple, Any, Optional import numpy as np -from iOpt.method.listener import ConsoleFullOutputListener +from iOpt.output_system.listeners.console_outputers import ConsoleOutputListener from iOpt.problem import Problem from iOpt.solver import Solver from iOpt.solver_parametrs import SolverParameters from iOpt.trial import Point, FunctionValue from golem.core.adapter import BaseOptimizationAdapter +from golem.core.optimisers.genetic.evaluation import determine_n_jobs from golem.core.optimisers.graph import OptGraph from golem.core.optimisers.objective import ObjectiveEvaluate -from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label +from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label, convert_parameters from golem.core.tuning.tuner_interface import BaseTuner, DomainGraphForTune +from golem.utilities.data_structures import ensure_wrapped_in_sequence @dataclass @@ -36,60 +40,50 @@ def from_parameters_dicts(float_parameters_dict: Optional[Dict[str, List]] = Non upper_bounds_of_float_parameters = [bounds[1] for bounds in float_parameters_dict.values()] discrete_parameters_vals = [values_set for values_set in discrete_parameters_dict.values()] - # TODO: Remove - for now IOpt handles only float variables, so we treat discrete parameters as float ones - float_parameters_names.extend(discrete_parameters_names) - lower_bounds_of_discrete_parameters = [bounds[0] for bounds in discrete_parameters_dict.values()] - upper_bounds_of_discrete_parameters = [bounds[1] for bounds in discrete_parameters_dict.values()] - lower_bounds_of_float_parameters.extend(lower_bounds_of_discrete_parameters) - upper_bounds_of_float_parameters.extend(upper_bounds_of_discrete_parameters) - - return IOptProblemParameters(float_parameters_names, discrete_parameters_names, + return IOptProblemParameters(float_parameters_names, + discrete_parameters_names, lower_bounds_of_float_parameters, - upper_bounds_of_float_parameters, discrete_parameters_vals) + upper_bounds_of_float_parameters, + discrete_parameters_vals) class GolemProblem(Problem, Generic[DomainGraphForTune]): def __init__(self, graph: DomainGraphForTune, objective_evaluate: ObjectiveEvaluate, - problem_parameters: IOptProblemParameters): + problem_parameters: IOptProblemParameters, + objectives_number: int = 1): super().__init__() self.objective_evaluate = objective_evaluate self.graph = graph - self.numberOfObjectives = 1 - self.numberOfConstraints = 0 + self.number_of_objectives = objectives_number + self.number_of_constraints = 0 - self.discreteVariableNames = problem_parameters.discrete_parameters_names - self.discreteVariableValues = problem_parameters.discrete_parameters_vals - self.numberOfDiscreteVariables = len(self.discreteVariableNames) + self.discrete_variable_names = problem_parameters.discrete_parameters_names + self.discrete_variable_values = problem_parameters.discrete_parameters_vals + self.number_of_discrete_variables = len(self.discrete_variable_names) - self.floatVariableNames = problem_parameters.float_parameters_names - self.lowerBoundOfFloatVariables = problem_parameters.lower_bounds_of_float_parameters - self.upperBoundOfFloatVariables = problem_parameters.upper_bounds_of_float_parameters - self.numberOfFloatVariables = len(self.floatVariableNames) + self.float_variable_names = problem_parameters.float_parameters_names + self.lower_bound_of_float_variables = problem_parameters.lower_bounds_of_float_parameters + self.upper_bound_of_float_variables = problem_parameters.upper_bounds_of_float_parameters + self.number_of_float_variables = len(self.float_variable_names) self._default_metric_value = np.inf - def Calculate(self, point: Point, functionValue: FunctionValue) -> FunctionValue: + def calculate(self, point: Point, function_value: FunctionValue) -> FunctionValue: new_parameters = self.get_parameters_dict_from_iopt_point(point) BaseTuner.set_arg_graph(self.graph, new_parameters) graph_fitness = self.objective_evaluate(self.graph) metric_value = graph_fitness.value if graph_fitness.valid else self._default_metric_value - functionValue.value = metric_value - return functionValue + function_value.value = metric_value + return function_value def get_parameters_dict_from_iopt_point(self, point: Point) -> Dict[str, Any]: """Constructs a dict with all hyperparameters """ - float_parameters = dict(zip(self.floatVariableNames, point.floatVariables)) \ - if point.floatVariables is not None else {} - discrete_parameters = dict(zip(self.discreteVariableNames, point.discreteVariables)) \ - if point.discreteVariables is not None else {} - - # TODO: Remove workaround - for now IOpt handles only float variables, so discrete parameters - # are optimized as continuous and we need to round them - for parameter_name in float_parameters: - if parameter_name in self.discreteVariableNames: - float_parameters[parameter_name] = round(float_parameters[parameter_name]) + float_parameters = dict(zip(self.float_variable_names, point.float_variables)) \ + if point.float_variables is not None else {} + discrete_parameters = dict(zip(self.discrete_variable_names, point.discrete_variables)) \ + if point.discrete_variables is not None else {} parameters_dict = {**float_parameters, **discrete_parameters} return parameters_dict @@ -122,8 +116,9 @@ def __init__(self, objective_evaluate: ObjectiveEvaluate, search_space: SearchSpace, adapter: Optional[BaseOptimizationAdapter] = None, iterations: int = 100, + timeout: timedelta = timedelta(minutes=5), n_jobs: int = -1, - eps: float = 0.01, + eps: float = 0.001, r: float = 2.0, evolvent_density: int = 10, eps_r: float = 0.001, @@ -133,42 +128,57 @@ def __init__(self, objective_evaluate: ObjectiveEvaluate, search_space, adapter, iterations=iterations, + timeout=timeout, n_jobs=n_jobs, deviation=deviation, **kwargs) + self.n_jobs = determine_n_jobs(self.n_jobs) self.solver_parameters = SolverParameters(r=np.double(r), eps=np.double(eps), - itersLimit=iterations, - evolventDensity=evolvent_density, - epsR=np.double(eps_r), - refineSolution=refine_solution) + iters_limit=iterations, + evolvent_density=evolvent_density, + eps_r=np.double(eps_r), + refine_solution=refine_solution, + number_of_parallel_points=self.n_jobs, + timeout=round(timeout.total_seconds()/60) if self.timeout else -1) def _tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainGraphForTune: problem_parameters, initial_parameters = self._get_parameters_for_tune(graph) has_parameters_to_optimize = (len(problem_parameters.discrete_parameters_names) > 0 or len(problem_parameters.float_parameters_names) > 0) - if self._check_if_tuning_possible(graph, has_parameters_to_optimize): + self.objectives_number = len(ensure_wrapped_in_sequence(self.init_metric)) + is_multi_objective = self.objectives_number > 1 + + if self._check_if_tuning_possible(graph, has_parameters_to_optimize, supports_multi_objective=True): if initial_parameters: initial_point = Point(**initial_parameters) - self.solver_parameters.startPoint = initial_point + self.solver_parameters.start_point = initial_point - problem = GolemProblem(graph, self.objective_evaluate, problem_parameters) + problem = GolemProblem(graph, self.objective_evaluate, problem_parameters, self.objectives_number) solver = Solver(problem, parameters=self.solver_parameters) if show_progress: - console_output = ConsoleFullOutputListener(mode='full') - solver.AddListener(console_output) - - solution = solver.Solve() - best_point = solution.bestTrials[0].point - best_parameters = problem.get_parameters_dict_from_iopt_point(best_point) - final_graph = self.set_arg_graph(graph, best_parameters) - - self.was_tuned = True + console_output = ConsoleOutputListener(mode='full') + solver.add_listener(console_output) + + solver.solve() + solution = solver.get_results() + if not is_multi_objective: + best_point = solution.best_trials[0].point + best_parameters = problem.get_parameters_dict_from_iopt_point(best_point) + tuned_graphs = self.set_arg_graph(graph, best_parameters) + self.was_tuned = True + else: + tuned_graphs = [] + for best_trial in solution.best_trials: + best_parameters = problem.get_parameters_dict_from_iopt_point(best_trial.point) + tuned_graph = self.set_arg_graph(deepcopy(graph), best_parameters) + tuned_graphs.append(tuned_graph) + self.was_tuned = True else: - final_graph = graph + tuned_graphs = graph - return final_graph + return tuned_graphs def _get_parameters_for_tune(self, graph: OptGraph) -> Tuple[IOptProblemParameters, dict]: """ Method for defining the search space @@ -182,26 +192,28 @@ def _get_parameters_for_tune(self, graph: OptGraph) -> Tuple[IOptProblemParamete """ float_parameters_dict = {} discrete_parameters_dict = {} - initial_parameters = {'floatVariables': [], 'discreteVariables': []} + has_init_parameters = any(len(node.parameters) > 0 for node in graph.nodes) + initial_parameters = {'float_variables': [], 'discrete_variables': []} if has_init_parameters else None for node_id, node in enumerate(graph.nodes): operation_name = node.name # Assign unique prefix for each model hyperparameter # label - number of node in the graph - float_node_parameters, discrete_node_parameters = get_node_parameters_for_iopt(self.search_space, - node_id, - operation_name) - - # Set initial parameters for search - for parameter, bounds in float_node_parameters.items(): - # If parameter is not set use parameter minimum possible value - initaial_value = node.parameters.get(parameter) or bounds[0] - initial_parameters['floatVariables'].append(initaial_value) - - for parameter, bounds in discrete_node_parameters.items(): - # If parameter is not set use parameter minimum possible value - initaial_value = node.parameters.get(parameter) or bounds[0] - initial_parameters['discreteVariables'].append(initaial_value) + float_node_parameters, discrete_node_parameters = get_node_parameters_for_iopt( + self.search_space, + node_id, + operation_name) + if has_init_parameters: + # Set initial parameters for search + for parameter, bounds in convert_parameters(float_node_parameters).items(): + # If parameter is not set use parameter minimum possible value + initial_value = node.parameters.get(parameter) or bounds[0] + initial_parameters['float_variables'].append(initial_value) + + for parameter, values in convert_parameters(discrete_node_parameters).items(): + # If parameter is not set use the last value + initial_value = node.parameters.get(parameter) or values[-1] + initial_parameters['discrete_variables'].append(initial_value) float_parameters_dict.update(float_node_parameters) discrete_parameters_dict.update(discrete_node_parameters) @@ -230,16 +242,22 @@ def get_node_parameters_for_iopt(search_space: SearchSpace, node_id: int, operat discrete_parameters_dict = {} float_parameters_dict = {} + categorical_parameters_dict = {} for parameter_name, parameter_properties in parameters_dict.items(): node_op_parameter_name = get_node_operation_parameter_label(node_id, operation_name, parameter_name) parameter_type = parameter_properties.get('type') if parameter_type == 'discrete': - discrete_parameters_dict.update({node_op_parameter_name: parameter_properties - .get('sampling-scope')}) + discrete_parameters_dict.update({node_op_parameter_name: list(range(*parameter_properties + .get('sampling-scope')))}) elif parameter_type == 'continuous': float_parameters_dict.update({node_op_parameter_name: parameter_properties .get('sampling-scope')}) + elif parameter_type == 'categorical': + categorical_parameters_dict.update({node_op_parameter_name: parameter_properties + .get('sampling-scope')[0]}) + # IOpt does not distinguish between discrete and categorical parameters + discrete_parameters_dict = {**discrete_parameters_dict, **categorical_parameters_dict} return float_parameters_dict, discrete_parameters_dict diff --git a/requirements.txt b/requirements.txt index f1a3b42cb..8166f68de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ psutil>=5.9.2 # Optimisation hyperopt>=0.2.7 -iOpt==0.1.6 +iOpt==0.2.22 optuna>=3.2.0 # Tests diff --git a/test/unit/tuning/test_tuning.py b/test/unit/tuning/test_tuning.py index e2d7967d4..08addcd43 100644 --- a/test/unit/tuning/test_tuning.py +++ b/test/unit/tuning/test_tuning.py @@ -35,6 +35,11 @@ def search_space(): 'hyperopt-dist': hp.loguniform, 'sampling-scope': [1e-3, 1], 'type': 'continuous' + }, + 'a3': { + 'hyperopt-dist': hp.choice, + 'sampling-scope': [['A', 'B', 'C']], + 'type': 'categorical' } }, 'b': { @@ -109,7 +114,7 @@ def test_node_tuning(search_space, graph): assert tuner.init_metric >= tuner.obtained_metric -@pytest.mark.parametrize('tuner_cls', [OptunaTuner]) +@pytest.mark.parametrize('tuner_cls', [OptunaTuner, IOptTuner]) @pytest.mark.parametrize('init_graph, adapter, obj_eval', [(mock_graph_with_params(), MockAdapter(), MockObjectiveEvaluate(Objective({'sum_metric': ParamsSumMetric.get_value,