Tuner interface update (#234)

* Add kwargs

* Add warning multi-obj

* Fix timeout update

* Add timeout

* Minor

* Refactor

* зуз8

golem/core/tuning/hyperopt_tuner.py

@@ -10,7 +10,6 @@
 from golem.core.adapter import BaseOptimizationAdapter
 from golem.core.log import default_log
 from golem.core.optimisers.objective import ObjectiveFunction
-from golem.core.optimisers.timer import Timer
 from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label
 from golem.core.tuning.tuner_interface import BaseTuner
 
@@ -41,7 +40,7 @@ def __init__(self, objective_evaluate: ObjectiveFunction,
                  timeout: timedelta = timedelta(minutes=5),
                  n_jobs: int = -1,
                  deviation: float = 0.05,
-                 algo: Callable = tpe.suggest):
+                 algo: Callable = tpe.suggest, **kwargs):
         early_stopping_rounds = early_stopping_rounds or max(100, int(np.sqrt(iterations) * 10))
         super().__init__(objective_evaluate,
                          search_space,
@@ -50,16 +49,12 @@ def __init__(self, objective_evaluate: ObjectiveFunction,
                          early_stopping_rounds,
                          timeout,
                          n_jobs,
-                         deviation)
+                         deviation, **kwargs)
 
         self.early_stop_fn = no_progress_loss(iteration_stop_count=self.early_stopping_rounds)
-        self.max_seconds = int(timeout.seconds) if timeout is not None else None
         self.algo = algo
         self.log = default_log(self)
 
-    def _update_remaining_time(self, tuner_timer: Timer):
-        self.max_seconds = self.max_seconds - tuner_timer.minutes_from_start * 60
-
 
 def get_parameter_hyperopt_space(search_space: SearchSpace,
                                  operation_name: str,

golem/core/tuning/iopt_tuner.py

@@ -142,18 +142,12 @@ def __init__(self, objective_evaluate: ObjectiveEvaluate,
                                                   epsR=np.double(eps_r),
                                                   refineSolution=refine_solution)
 
-    def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainGraphForTune:
-        graph = self.adapter.adapt(graph)
+    def _tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainGraphForTune:
         problem_parameters, initial_parameters = self._get_parameters_for_tune(graph)
 
-        no_parameters_to_optimize = (not problem_parameters.discrete_parameters_names and
-                                     not problem_parameters.float_parameters_names)
-        self.init_check(graph)
-
-        if no_parameters_to_optimize:
-            self._stop_tuning_with_message(f'Graph "{graph.graph_description}" has no parameters to optimize')
-            final_graph = graph
-        else:
+        has_parameters_to_optimize = (len(problem_parameters.discrete_parameters_names) > 0 or
+                                      len(problem_parameters.float_parameters_names) > 0)
+        if self._check_if_tuning_possible(graph, has_parameters_to_optimize):
             if initial_parameters:
                 initial_point = Point(**initial_parameters)
                 self.solver_parameters.startPoint = initial_point
@@ -171,10 +165,9 @@ def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainG
             final_graph = self.set_arg_graph(graph, best_parameters)
 
             self.was_tuned = True
+        else:
+            final_graph = graph
 
-        # Validate if optimisation did well
-        graph = self.final_check(final_graph)
-        final_graph = self.adapter.restore(graph)
         return final_graph
 
     def _get_parameters_for_tune(self, graph: OptGraph) -> Tuple[IOptProblemParameters, dict]:

golem/core/tuning/optuna_tuner.py

@@ -12,6 +12,7 @@
 from golem.core.optimisers.objective import ObjectiveFunction
 from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label
 from golem.core.tuning.tuner_interface import BaseTuner, DomainGraphForTune
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 
 
 class OptunaTuner(BaseTuner):
@@ -22,34 +23,32 @@ def __init__(self, objective_evaluate: ObjectiveFunction,
                  early_stopping_rounds: Optional[int] = None,
                  timeout: timedelta = timedelta(minutes=5),
                  n_jobs: int = -1,
-                 deviation: float = 0.05,
-                 objectives_number: int = 1):
+                 deviation: float = 0.05, **kwargs):
         super().__init__(objective_evaluate,
                          search_space,
                          adapter,
                          iterations,
                          early_stopping_rounds,
                          timeout,
                          n_jobs,
-                         deviation)
-        self.objectives_number = objectives_number
+                         deviation, **kwargs)
         self.study = None
 
-    def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> \
+    def _tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> \
             Union[DomainGraphForTune, Sequence[DomainGraphForTune]]:
-        graph = self.adapter.adapt(graph)
         predefined_objective = partial(self.objective, graph=graph)
-        is_multi_objective = self.objectives_number > 1
 
-        self.init_check(graph)
+        self.objectives_number = len(ensure_wrapped_in_sequence(self.init_metric))
+        is_multi_objective = self.objectives_number > 1
 
         self.study = optuna.create_study(directions=['minimize'] * self.objectives_number)
 
         init_parameters, has_parameters_to_optimize = self._get_initial_point(graph)
-        if not has_parameters_to_optimize:
-            self._stop_tuning_with_message(f'Graph {graph.graph_description} has no parameters to optimize')
-            tuned_graphs = self.init_graph
-        else:
+        remaining_time = self._get_remaining_time()
+        if self._check_if_tuning_possible(graph,
+                                          has_parameters_to_optimize,
+                                          remaining_time,
+                                          supports_multi_objective=True):
             # Enqueue initial point to try
             if init_parameters:
                 self.study.enqueue_trial(init_parameters)
@@ -60,7 +59,7 @@ def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> \
             self.study.optimize(predefined_objective,
                                 n_trials=self.iterations,
                                 n_jobs=self.n_jobs,
-                                timeout=self.timeout.seconds,
+                                timeout=remaining_time,
                                 callbacks=[self.early_stopping_callback],
                                 show_progress_bar=show_progress)
 
@@ -75,9 +74,9 @@ def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> \
                     tuned_graph = self.set_arg_graph(deepcopy(graph), best_parameters)
                     tuned_graphs.append(tuned_graph)
                     self.was_tuned = True
-        final_graphs = self.final_check(tuned_graphs, is_multi_objective)
-        final_graphs = self.adapter.restore(final_graphs)
-        return final_graphs
+        else:
+            tuned_graphs = graph
+        return tuned_graphs
 
     def objective(self, trial: Trial, graph: OptGraph) -> Union[float, Sequence[float, ]]:
         new_parameters = self._get_parameters_from_trial(graph, trial)

golem/core/tuning/sequential.py

@@ -26,69 +26,61 @@ def __init__(self, objective_evaluate: ObjectiveFunction,
                  n_jobs: int = -1,
                  deviation: float = 0.05,
                  algo: Callable = tpe.suggest,
-                 inverse_node_order: bool = False):
+                 inverse_node_order: bool = False, **kwargs):
         super().__init__(objective_evaluate,
                          search_space,
                          adapter,
                          iterations,
                          early_stopping_rounds, timeout,
                          n_jobs,
                          deviation,
-                         algo)
+                         algo, **kwargs)
 
         self.inverse_node_order = inverse_node_order
 
-    def tune(self, graph: DomainGraphForTune) -> DomainGraphForTune:
+    def _tune(self, graph: DomainGraphForTune, **kwargs) -> DomainGraphForTune:
         """ Method for hyperparameters tuning on the entire graph
 
         Args:
             graph: graph which hyperparameters will be tuned
         """
-        graph = self.adapter.adapt(graph)
-
-        # Check source metrics for data
-        self.init_check(graph)
-
-        # Calculate amount of iterations we can apply per node
-        nodes_amount = graph.length
-        iterations_per_node = round(self.iterations / nodes_amount)
-        iterations_per_node = int(iterations_per_node)
-        if iterations_per_node == 0:
-            iterations_per_node = 1
-
-        # Calculate amount of seconds we can apply per node
-        if self.max_seconds is not None:
-            seconds_per_node = round(self.max_seconds / nodes_amount)
-            seconds_per_node = int(seconds_per_node)
-        else:
-            seconds_per_node = None
-
-        # Tuning performed sequentially for every node - so get ids of nodes
-        nodes_ids = self.get_nodes_order(nodes_number=nodes_amount)
-        for node_id in nodes_ids:
-            node = graph.nodes[node_id]
-            operation_name = node.name
-
-            # Get node's parameters to optimize
-            node_params = get_node_parameters_for_hyperopt(self.search_space, node_id, operation_name)
-
-            if not node_params:
-                self.log.info(f'"{operation_name}" operation has no parameters to optimize')
+        remaining_time = self._get_remaining_time()
+        if self._check_if_tuning_possible(graph, parameters_to_optimize=True, remaining_time=remaining_time):
+            # Calculate amount of iterations we can apply per node
+            nodes_amount = graph.length
+            iterations_per_node = round(self.iterations / nodes_amount)
+            iterations_per_node = int(iterations_per_node)
+            if iterations_per_node == 0:
+                iterations_per_node = 1
+
+            # Calculate amount of seconds we can apply per node
+            if remaining_time is not None:
+                seconds_per_node = round(remaining_time / nodes_amount)
+                seconds_per_node = int(seconds_per_node)
             else:
-                # Apply tuning for current node
-                self._optimize_node(node_id=node_id,
-                                    graph=graph,
-                                    node_params=node_params,
-                                    iterations_per_node=iterations_per_node,
-                                    seconds_per_node=seconds_per_node)
-
-        # Validate if optimisation did well
-        final_graph = self.final_check(graph)
+                seconds_per_node = None
+
+            # Tuning performed sequentially for every node - so get ids of nodes
+            nodes_ids = self.get_nodes_order(nodes_number=nodes_amount)
+            for node_id in nodes_ids:
+                node = graph.nodes[node_id]
+                operation_name = node.name
+
+                # Get node's parameters to optimize
+                node_params = get_node_parameters_for_hyperopt(self.search_space, node_id, operation_name)
+
+                if not node_params:
+                    self.log.info(f'"{operation_name}" operation has no parameters to optimize')
+                else:
+                    # Apply tuning for current node
+                    self._optimize_node(node_id=node_id,
+                                        graph=graph,
+                                        node_params=node_params,
+                                        iterations_per_node=iterations_per_node,
+                                        seconds_per_node=seconds_per_node)
 
-        self.was_tuned = True
-        final_graph = self.adapter.restore(final_graph)
-
-        return final_graph
+            self.was_tuned = True
+        return graph
 
     def get_nodes_order(self, nodes_number: int) -> range:
         """ Method returns list with indices of nodes in the graph
@@ -118,38 +110,41 @@ def tune_node(self, graph: DomainGraphForTune, node_index: int) -> DomainGraphFo
         """
         graph = self.adapter.adapt(graph)
 
-        self.init_check(graph)
+        with self.timer:
+            self.init_check(graph)
 
-        node = graph.nodes[node_index]
-        operation_name = node.name
+            node = graph.nodes[node_index]
+            operation_name = node.name
 
-        # Get node's parameters to optimize
-        node_params = get_node_parameters_for_hyperopt(self.search_space,
-                                                       node_id=node_index,
-                                                       operation_name=operation_name)
+            # Get node's parameters to optimize
+            node_params = get_node_parameters_for_hyperopt(self.search_space,
+                                                           node_id=node_index,
+                                                           operation_name=operation_name)
 
-        if not node_params:
-            self._stop_tuning_with_message(f'"{operation_name}" operation has no parameters to optimize')
-        else:
-            # Apply tuning for current node
-            self._optimize_node(graph=graph,
-                                node_id=node_index,
-                                node_params=node_params,
-                                iterations_per_node=self.iterations,
-                                seconds_per_node=self.max_seconds,
-                                )
-            self.was_tuned = True
+            remaining_time = self._get_remaining_time()
+            if self._check_if_tuning_possible(graph, len(node_params) > 1, remaining_time):
+                # Apply tuning for current node
+                self._optimize_node(graph=graph,
+                                    node_id=node_index,
+                                    node_params=node_params,
+                                    iterations_per_node=self.iterations,
+                                    seconds_per_node=remaining_time
+                                    )
+                self.was_tuned = True
 
-        # Validation is the optimization do well
-        final_graph = self.final_check(graph)
+                # Validation is the optimization do well
+                final_graph = self.final_check(graph)
+            else:
+                final_graph = graph
+                self.obtained_metric = self.init_metric
         final_graph = self.adapter.restore(final_graph)
         return final_graph
 
     def _optimize_node(self, graph: OptGraph,
                        node_id: int,
                        node_params: dict,
                        iterations_per_node: int,
-                       seconds_per_node: int) -> OptGraph:
+                       seconds_per_node: float) -> OptGraph:
         """
         Method for node optimization