Add tuning history to OptHistory

examples/molecule_search/experiment.py

@@ -109,7 +109,7 @@ def visualize_results(molecules: Iterable[MolGraph],
 
     # Plot pareto front (if multi-objective)
     if objective.is_multi_objective:
-        visualise_pareto(history.archive_history[-1],
+        visualise_pareto(history.evolution_best_archive[-1],
                          objectives_names=objective.metric_names[:2],
                          folder=str(save_path))
 
@@ -193,7 +193,7 @@ def run_experiment(optimizer_setup: Callable,
             result_dir = Path('results') / exp_name
             result_dir.mkdir(parents=True, exist_ok=True)
             history.save(result_dir / f'history_trial_{trial}.json')
-        trial_results.extend(history.final_choices)
+        trial_results.extend(history.evolution_results)
         trial_histories.append(history)
 
     # Compute mean & std for metrics of trials

examples/synthetic_graph_evolution/experiment_setup.py

@@ -66,7 +66,7 @@ def run_experiments(optimizer_setup: Callable,
             found_graph = found_graphs[0] if isinstance(found_graphs, Sequence) else found_graphs
             history = optimizer.history
 
-            trial_results.extend(history.final_choices)
+            trial_results.extend(history.evolution_results)
             found_nx_graph = BaseNetworkxAdapter().restore(found_graph)
 
             duration = datetime.now() - start_time

experiments/experiment_analyzer.py

@@ -104,7 +104,7 @@ def _analyze_convergence(history: OptHistory, metrics_num: int) -> List[float]:
         # find best final metric for each objective
         best_fitness_per_objective = [np.inf] * metrics_num
         for i in range(metrics_num):
-            for ind in history.final_choices.data:
+            for ind in history.evolution_results.data:
                 if ind.fitness.values[i] < best_fitness_per_objective[i]:
                     best_fitness_per_objective[i] = ind.fitness.values[i]
 
@@ -231,7 +231,8 @@ def plot_convergence(self, path_to_save: str, with_confidence: bool = True,
 
     def analyze_statistical_significance(self, data_to_analyze: Dict[str, Dict[str, List[float]]],
                                          stat_tests: List[Callable], path_to_save: Optional[str] = None,
-                                         test_format: Optional[List[str]] = None) -> Dict[str, Dict[str, Dict[str, float]]]:
+                                         test_format: Optional[List[str]] = None
+                                         ) -> Dict[str, Dict[str, Dict[str, float]]]:
         """ Method to perform statistical analysis of data. Metric data obtained with 'analyze_metrics' and
         convergence data obtained with 'analyze_convergence' can be simply analyzed, for example.
         :param data_to_analyze: data to analyze.
@@ -291,7 +292,8 @@ def analyze_structural_complexity(self, path_to_save: str, dir_name: str, class_
             os.makedirs(path_to_save, exist_ok=True)
 
         for setup, dataset, path_to_launch in self._get_path_to_launch():
-            if not self._check_if_file_or_folder_present(path=os.path.join(path_to_launch, dir_name), is_raise=is_raise):
+            if not self._check_if_file_or_folder_present(path=os.path.join(path_to_launch, dir_name),
+                                                         is_raise=is_raise):
                 continue
 
             path_to_json = None
@@ -324,7 +326,7 @@ def analyze_structural_complexity(self, path_to_save: str, dir_name: str, class_
             saved_results = [int(cur_name.split("_")[0]) for cur_name in os.listdir(cur_path_to_save)
                              if cur_name not in self._folders_to_ignore]
             max_saved_num = max(saved_results) if saved_results else 0
-            cur_path_to_save = os.path.join(cur_path_to_save, f'{max_saved_num+1}_result.png')
+            cur_path_to_save = os.path.join(cur_path_to_save, f'{max_saved_num + 1}_result.png')
             result.show(cur_path_to_save, title=title)
             self._log.info(f"Resulting graph was saved to {cur_path_to_save}")
 

experiments/experiment_launcher.py

@@ -133,7 +133,7 @@ def launch(self, optimizer_setup: Callable, **kwargs):
                 found_graph = found_graphs[0] if isinstance(found_graphs, Sequence) else found_graphs
                 history = optimizer.history
 
-                trial_results.extend(history.final_choices)
+                trial_results.extend(history.evolution_results)
                 found_nx_graph = BaseNetworkxAdapter().restore(found_graph)
 
                 duration = datetime.now() - start_time

golem/core/optimisers/adaptive/experience_buffer.py

@@ -57,7 +57,7 @@ def unroll_trajectories(history: OptHistory) -> List[GraphTrajectory]:
         """Iterates through history and find continuous sequences of applied operator actions."""
         trajectories = []
         seen_uids = set()
-        for terminal_individual in history.final_choices:
+        for terminal_individual in history.evolution_results:
             trajectory = []
             next_ind = terminal_individual
             while True:

golem/core/optimisers/meta/surrogate_optimizer.py

@@ -54,5 +54,5 @@ def optimise(self, objective: ObjectiveFunction) -> Sequence[OptGraph]:
                     break
                 # Adding of new population to history
                 self._update_population(new_population)
-        self._update_population(self.best_individuals, 'final_choices')
+        self._update_population(self.best_individuals, 'evolution_results')
         return [ind.graph for ind in self.best_individuals]

golem/core/optimisers/opt_history_objects/opt_history.py

@@ -6,18 +6,16 @@
 import os
 import shutil
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Sequence, Union, TYPE_CHECKING
+from typing import Any, Dict, List, Optional, Sequence, TYPE_CHECKING, Union
 
 from golem.core.log import default_log
 from golem.core.optimisers.objective.objective import ObjectiveInfo
 from golem.core.optimisers.opt_history_objects.generation import Generation
-
 from golem.core.paths import default_data_dir
 from golem.serializers.serializer import default_load, default_save
 from golem.visualisation.opt_viz import OptHistoryVisualizer
 
 if TYPE_CHECKING:
-    from golem.core.dag.graph import Graph
     from golem.core.optimisers.opt_history_objects.individual import Individual
 
 
@@ -36,8 +34,7 @@ def __init__(self,
                  default_save_dir: Optional[os.PathLike] = None):
         self._objective = objective or ObjectiveInfo()
         self._generations: List[Generation] = []
-        self.archive_history: List[List[Individual]] = []
-        self._tuning_result: Optional[Graph] = None
+        self.evolution_best_archive: List[List[Individual]] = []
 
         # init default save directory
         if default_save_dir:
@@ -61,8 +58,8 @@ def add_to_history(self, individuals: Sequence[Individual], generation_label: Op
         generation = Generation(individuals, self.generations_count, generation_label, generation_metadata)
         self.generations.append(generation)
 
-    def add_to_archive_history(self, individuals: Sequence[Individual]):
-        self.archive_history.append(list(individuals))
+    def add_to_evolution_best_archive(self, individuals: Sequence[Individual]):
+        self.evolution_best_archive.append(list(individuals))
 
     def to_csv(self, save_dir: Optional[os.PathLike] = None, file: os.PathLike = 'history.csv'):
         save_dir = save_dir or self._default_save_dir
@@ -219,27 +216,24 @@ def initial_assumptions(self) -> Optional[Generation]:
                 return gen
 
     @property
-    def final_choices(self) -> Optional[Generation]:
+    def generations_count(self) -> int:
+        return len(self.generations)
+
+    @property
+    def evolution_results(self) -> Optional[Generation]:
         if not self.generations:
             return None
         for gen in reversed(self.generations):
-            if gen.label == 'final_choices':
+            if gen.label == 'evolution_results':
                 return gen
 
     @property
-    def generations_count(self) -> int:
-        return len(self.generations)
-
-    @property
-    def tuning_result(self):
-        if hasattr(self, '_tuning_result'):
-            return self._tuning_result
-        else:
+    def tuning_result(self) -> Optional[Generation]:
+        if not self.generations:
             return None
-
-    @tuning_result.setter
-    def tuning_result(self, val):
-        self._tuning_result = val
+        for gen in reversed(self.generations):
+            if gen.label == 'tuning_result':
+                return gen
 
     @property
     def generations(self):
@@ -270,8 +264,7 @@ def lighten_history(history: OptHistory) -> OptHistory:
     without excessive memory usage. """
     light_history = OptHistory()
     light_history._generations = \
-        [Generation(iterable=gen, generation_num=i) for i, gen in enumerate(history.archive_history)]
-    light_history.archive_history = history.archive_history
+        [Generation(iterable=gen, generation_num=i) for i, gen in enumerate(history.evolution_best_archive)]
+    light_history.evolution_best_archive = history.evolution_best_archive
     light_history._objective = history.objective
-    light_history._tuning_result = history.tuning_result
     return light_history

golem/core/optimisers/populational_optimizer.py

@@ -105,7 +105,7 @@ def optimise(self, objective: ObjectiveFunction) -> Sequence[Graph]:
                 # Adding of new population to history
                 self._update_population(new_population)
         pbar.close()
-        self._update_population(self.best_individuals, 'final_choices')
+        self._update_population(self.best_individuals, 'evolution_results')
         return [ind.graph for ind in self.best_individuals]
 
     @property
@@ -146,7 +146,7 @@ def _update_population(self, next_population: PopulationT, label: Optional[str]
     def _log_to_history(self, population: PopulationT, label: Optional[str] = None,
                         metadata: Optional[Dict[str, Any]] = None):
         self.history.add_to_history(population, label, metadata)
-        self.history.add_to_archive_history(self.generations.best_individuals)
+        self.history.add_to_evolution_best_archive(self.generations.best_individuals)
         if self.requirements.history_dir:
             self.history.save_current_results(self.requirements.history_dir)
 

golem/core/optimisers/random/random_search.py

@@ -53,7 +53,7 @@ def optimise(self, objective: ObjectiveFunction) -> Sequence[OptGraph]:
                 self.current_iteration_num += 1
                 self._update_best_individual(new_individual)
                 pbar.update()
-        self._update_best_individual(self.best_individual, 'final_choices')
+        self._update_best_individual(self.best_individual, 'evolution_results')
         pbar.close()
         return [self.best_individual.graph]
 
@@ -68,7 +68,7 @@ def _update_best_individual(self, new_individual: Individual, label: Optional[st
                       f'Best individuals fitness {str(self.generations)}')
 
         self.history.add_to_history([new_individual], label)
-        self.history.add_to_archive_history(self.generations.best_individuals)
+        self.history.add_to_evolution_best_archive(self.generations.best_individuals)
 
     def _eval_initial_individual(self, evaluator: EvaluationOperator) -> Individual:
         init_ind = Individual(choice(self.initial_graphs)) if self.initial_graphs else self._generate_new_individual()

golem/core/tuning/hyperopt_tuner.py

@@ -10,6 +10,7 @@
 from golem.core.adapter import BaseOptimizationAdapter
 from golem.core.log import default_log
 from golem.core.optimisers.objective import ObjectiveFunction
+from golem.core.optimisers.opt_history_objects.opt_history import OptHistory
 from golem.core.optimisers.timer import Timer
 from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label
 from golem.core.tuning.tuner_interface import BaseTuner
@@ -31,6 +32,7 @@ class HyperoptTuner(BaseTuner, ABC):
         By default, ``deviation=0.05``, which means that tuned graph will be returned
         if it's metric will be at least 0.05% better than the initial.
       algo: algorithm for hyperparameters optimization with signature similar to :obj:`hyperopt.tse.suggest`
+      history: object to store tuning history if needed.
     """
 
     def __init__(self, objective_evaluate: ObjectiveFunction,
@@ -41,7 +43,8 @@ def __init__(self, objective_evaluate: ObjectiveFunction,
                  timeout: timedelta = timedelta(minutes=5),
                  n_jobs: int = -1,
                  deviation: float = 0.05,
-                 algo: Callable = tpe.suggest):
+                 algo: Callable = tpe.suggest,
+                 history: Optional[OptHistory] = None):
         early_stopping_rounds = early_stopping_rounds or max(100, int(np.sqrt(iterations) * 10))
         super().__init__(objective_evaluate,
                          search_space,
@@ -50,7 +53,8 @@ def __init__(self, objective_evaluate: ObjectiveFunction,
                          early_stopping_rounds,
                          timeout,
                          n_jobs,
-                         deviation)
+                         deviation,
+                         history)
 
         self.early_stop_fn = no_progress_loss(iteration_stop_count=self.early_stopping_rounds)
         self.max_seconds = int(timeout.seconds) if timeout is not None else None

golem/core/tuning/iopt_tuner.py

@@ -1,12 +1,12 @@
 from dataclasses import dataclass, field
-from typing import List, Dict, Generic, Tuple, Any, Optional
+from typing import Any, Dict, Generic, List, Optional, Tuple
 
 import numpy as np
 from iOpt.method.listener import ConsoleFullOutputListener
 from iOpt.problem import Problem
 from iOpt.solver import Solver
 from iOpt.solver_parametrs import SolverParameters
-from iOpt.trial import Point, FunctionValue
+from iOpt.trial import FunctionValue, Point
 
 from golem.core.adapter import BaseOptimizationAdapter
 from golem.core.optimisers.graph import OptGraph

golem/core/tuning/optuna_tuner.py

@@ -10,6 +10,7 @@
 from golem.core.adapter import BaseOptimizationAdapter
 from golem.core.optimisers.graph import OptGraph
 from golem.core.optimisers.objective import ObjectiveFunction
+from golem.core.optimisers.opt_history_objects.opt_history import OptHistory
 from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label
 from golem.core.tuning.tuner_interface import BaseTuner, DomainGraphForTune
 
@@ -23,15 +24,17 @@ def __init__(self, objective_evaluate: ObjectiveFunction,
                  timeout: timedelta = timedelta(minutes=5),
                  n_jobs: int = -1,
                  deviation: float = 0.05,
-                 objectives_number: int = 1):
+                 objectives_number: int = 1,
+                 history: Optional[OptHistory] = None):
         super().__init__(objective_evaluate,
                          search_space,
                          adapter,
                          iterations,
                          early_stopping_rounds,
                          timeout,
                          n_jobs,
-                         deviation)
+                         deviation,
+                         history)
         self.objectives_number = objectives_number
         self.study = None
 
@@ -82,7 +85,7 @@ def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> \
     def objective(self, trial: Trial, graph: OptGraph) -> Union[float, Sequence[float, ]]:
         new_parameters = self._get_parameters_from_trial(graph, trial)
         new_graph = BaseTuner.set_arg_graph(graph, new_parameters)
-        metric_value = self.get_metric_value(new_graph)
+        metric_value = self.evaluate_graph(new_graph)
         return metric_value
 
     def _get_parameters_from_trial(self, graph: OptGraph, trial: Trial) -> dict:

golem/core/tuning/sequential.py

@@ -7,6 +7,7 @@
 from golem.core.adapter import BaseOptimizationAdapter
 from golem.core.optimisers.graph import OptGraph
 from golem.core.optimisers.objective import ObjectiveFunction
+from golem.core.optimisers.opt_history_objects.opt_history import OptHistory
 from golem.core.tuning.hyperopt_tuner import HyperoptTuner, get_node_parameters_for_hyperopt
 from golem.core.tuning.search_space import SearchSpace
 from golem.core.tuning.tuner_interface import DomainGraphForTune
@@ -26,15 +27,17 @@ def __init__(self, objective_evaluate: ObjectiveFunction,
                  n_jobs: int = -1,
                  deviation: float = 0.05,
                  algo: Callable = tpe.suggest,
-                 inverse_node_order: bool = False):
+                 inverse_node_order: bool = False,
+                 history: Optional[OptHistory] = None):
         super().__init__(objective_evaluate,
                          search_space,
                          adapter,
                          iterations,
                          early_stopping_rounds, timeout,
                          n_jobs,
                          deviation,
-                         algo)
+                         algo,
+                         history)
 
         self.inverse_node_order = inverse_node_order
 
@@ -191,5 +194,5 @@ def _objective(self, node_params: dict, graph: OptGraph, node_id: int) -> float:
         # Set hyperparameters for node
         graph = self.set_arg_node(graph=graph, node_id=node_id, node_params=node_params)
 
-        metric_value = self.get_metric_value(graph=graph)
+        metric_value = self.evaluate_graph(graph=graph)
         return metric_value

golem/core/tuning/simultaneous.py

@@ -178,6 +178,6 @@ def _objective(self, parameters_dict: dict, graph: OptGraph, unchangeable_parame
         # Set hyperparameters for every node
         graph = self.set_arg_graph(graph, parameters_dict)
 
-        metric_value = self.get_metric_value(graph=graph)
+        metric_value = self.evaluate_graph(graph=graph)
 
         return metric_value