diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 4c2e5ca29..466162d0e 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -29,4 +29,4 @@ jobs: with: user: __token__ password: ${{ secrets.GOLEM_PYPI_PUBLISH }} - repository_url: https://upload.pypi.org/legacy/ + repository-url: https://upload.pypi.org/legacy/ diff --git a/README.rst b/README.rst index e279a9bde..5ccf56415 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,7 @@ |sai| |itmo| - |python| |pypi| |build| |integration| |docs| |license| |tg| |eng| |mirror| + |python| |pypi| |build| |integration| |coverage| |docs| |license| |tg| |eng| |mirror| Оптимизация и обучение графовых моделей эволюционными методами @@ -80,7 +80,7 @@ GOLEM можно установить с помощью ``pip``: Быстрый старт ============= -Следующий пример показывает поиск графа по графу-эталону с помощью метрики расстояния редактирования (Edit Distance). Оптимизатор настраивается с минимальным набором параметров и простыми одноточечными мутациями. Более подробные примеры можно найти в файлах `simple_run.py `_, `graph_search.py `_ и `tree_search.py `_ в директории `examples/synthetic_graph_evolution `_. +Следующий пример показывает поиск графа по графу-эталону с помощью метрики редакционного расстояния (Edit Distance). Оптимизатор настраивается с минимальным набором параметров и простыми одноточечными мутациями. Более подробные примеры можно найти в файлах `simple_run.py `_, `graph_search.py `_ и `tree_search.py `_ в директории `examples/synthetic_graph_evolution `_. .. code-block:: python @@ -106,6 +106,13 @@ GOLEM можно установить с помощью ``pip``: optimiser.history.show.fitness_line() return found_graph +Если проследить предков найденного графа, будет видно, как к нему один за другим применяются генетические операторы (мутации, скрещивания и т.д.), приводящие, в конечном итоге, к целевому графу: + +.. image:: /docs/source/img/evolution_process.gif + :alt: Процесс эволюции + :align: center + +Можно также заметить, что, несмотря на общее улучшение фитнеса вдоль генеалогического пути, оптимизатор иногда жертвует локальным уменьшением редакционного расстояния некоторых графов ради поддержания разнообразия и получения таким образом наилучшего решения в конце. Структура проекта ================= @@ -226,14 +233,14 @@ GOLEM можно установить с помощью ``pip``: .. |eng| image:: https://img.shields.io/badge/lang-en-red.svg :target: /README_en.rst -.. |ITMO| image:: https://github.com/aimclub/open-source-ops/blob/add_badge/badges/ITMO_badge_rus.svg +.. |ITMO| image:: https://raw.githubusercontent.com/aimclub/open-source-ops/43bb283758b43d75ec1df0a6bb4ae3eb20066323/badges/ITMO_badge_rus.svg :alt: Acknowledgement to ITMO :target: https://itmo.ru -.. |SAI| image:: https://github.com/aimclub/open-source-ops/blob/add_badge/badges/SAI_badge.svg +.. |SAI| image:: https://raw.githubusercontent.com/aimclub/open-source-ops/43bb283758b43d75ec1df0a6bb4ae3eb20066323/badges/SAI_badge.svg :alt: Acknowledgement to SAI :target: https://sai.itmo.ru/ -.. |mirror| image:: https://camo.githubusercontent.com/9bd7b8c5b418f1364e72110a83629772729b29e8f3393b6c86bff237a6b784f6/68747470733a2f2f62616467656e2e6e65742f62616467652f6769746c61622f6d6972726f722f6f72616e67653f69636f6e3d6769746c6162 +.. |mirror| image:: https://img.shields.io/badge/mirror-GitLab-orange :alt: GitLab mirror for this repository - :target: https://gitlab.actcognitive.org/itmo-nss-team/GOLEM \ No newline at end of file + :target: https://gitlab.actcognitive.org/itmo-nss-team/GOLEM diff --git a/README_en.rst b/README_en.rst index 0ac42b74f..01dba7cec 100644 --- a/README_en.rst +++ b/README_en.rst @@ -7,7 +7,7 @@ |sai| |itmo| - |python| |pypi| |build| |integration| |docs| |license| |tg| |rus| |mirror| + |python| |pypi| |build| |integration| |coverage| |docs| |license| |tg| |rus| |mirror| Graph Optimization and Learning by Evolutionary Methods @@ -105,6 +105,14 @@ Following example demonstrates graph search using reference graph & edit distanc return found_graph +Tracing the lineage of the found_graph reveals how genetic operators (mutations, crossovers, etc.) are applied to a random graph one after another, eventually leading to the target graph: + +.. image:: /docs/source/img/evolution_process.gif + :alt: Evolution process + :align: center + +One can also notice that despite the fact that the edit distance generally decreases along the genealogical path, the optimizer sometimes sacrifices local fitness gain of some graphs in order to achieve diversity and thus obtain the best possible solution at the end. + Project Structure ================= @@ -223,16 +231,16 @@ There are various cases solved with GOLEM's algorithms: :alt: Powered by GOLEM .. |rus| image:: https://img.shields.io/badge/lang-ru-yellow.svg - :target: /README.rst + :target: /README.rst -.. |ITMO| image:: https://github.com/aimclub/open-source-ops/blob/add_badge/badges/ITMO_badge.svg +.. |ITMO| image:: https://raw.githubusercontent.com/aimclub/open-source-ops/43bb283758b43d75ec1df0a6bb4ae3eb20066323/badges/ITMO_badge.svg :alt: Acknowledgement to ITMO :target: https://en.itmo.ru/en/ -.. |SAI| image:: https://github.com/aimclub/open-source-ops/blob/add_badge/badges/SAI_badge.svg +.. |SAI| image:: https://raw.githubusercontent.com/aimclub/open-source-ops/43bb283758b43d75ec1df0a6bb4ae3eb20066323/badges/SAI_badge.svg :alt: Acknowledgement to SAI :target: https://sai.itmo.ru/ -.. |mirror| image:: https://camo.githubusercontent.com/9bd7b8c5b418f1364e72110a83629772729b29e8f3393b6c86bff237a6b784f6/68747470733a2f2f62616467656e2e6e65742f62616467652f6769746c61622f6d6972726f722f6f72616e67653f69636f6e3d6769746c6162 +.. |mirror| image:: https://img.shields.io/badge/mirror-GitLab-orange :alt: GitLab mirror for this repository - :target: https://gitlab.actcognitive.org/itmo-nss-team/GOLEM \ No newline at end of file + :target: https://gitlab.actcognitive.org/itmo-nss-team/GOLEM diff --git a/docs/source/api/tuning.rst b/docs/source/api/tuning.rst index 9858dd8e9..97d6e8a79 100644 --- a/docs/source/api/tuning.rst +++ b/docs/source/api/tuning.rst @@ -52,11 +52,6 @@ You can tune all parameters of graph nodes simultaneously using ``SimultaneousTu .. note:: ``IOptTuner`` implements deterministic algorithm. - For now ``IOptTuner`` can not be constrained by time, so constrain execution by number of iterations. - - Also ``IOptTuner`` can optimise only `continuous` and `discrete` parameters but not `categorical` ones. - `Categorical` parameters will be ignored while tuning. - ``IOptTuner`` is implemented using `IOpt library`_. See the `documentation`_ (in Russian) to learn more about the optimisation algorithm. diff --git a/docs/source/conf.py b/docs/source/conf.py index ad3e3e349..fa13e6ead 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -24,7 +24,7 @@ author = 'NSS Lab' # The full version, including alpha/beta/rc tags -release = '0.3.3' +release = '0.4.0' # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be diff --git a/docs/source/img/evolution_process.gif b/docs/source/img/evolution_process.gif new file mode 100644 index 000000000..d0d1201ba Binary files /dev/null and b/docs/source/img/evolution_process.gif differ diff --git a/examples/molecule_search/experiment.py b/examples/molecule_search/experiment.py index d8e407d14..79cea2bbe 100644 --- a/examples/molecule_search/experiment.py +++ b/examples/molecule_search/experiment.py @@ -16,6 +16,8 @@ normalized_logp, CLScorer from golem.core.dag.verification_rules import has_no_self_cycled_nodes, has_no_isolated_components, \ has_no_isolated_nodes +from golem.core.optimisers.adaptive.agent_trainer import AgentTrainer +from golem.core.optimisers.adaptive.history_collector import HistoryReader from golem.core.optimisers.adaptive.operator_agent import MutationAgentTypeEnum from golem.core.optimisers.genetic.gp_optimizer import EvoGraphOptimizer from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters @@ -25,6 +27,7 @@ from golem.core.optimisers.objective import Objective from golem.core.optimisers.opt_history_objects.opt_history import OptHistory from golem.core.optimisers.optimizer import GraphGenerationParams, GraphOptimizer +from golem.core.paths import project_root from golem.visualisation.opt_history.multiple_fitness_line import MultipleFitnessLines from golem.visualisation.opt_viz_extra import visualise_pareto @@ -129,6 +132,16 @@ def visualize_results(molecules: Iterable[MolGraph], image.show() +def pretrain_agent(optimizer: EvoGraphOptimizer, objective: Objective, results_dir: str) -> AgentTrainer: + agent = optimizer.mutation.agent + trainer = AgentTrainer(objective, optimizer.mutation, agent) + # load histories + history_reader = HistoryReader(Path(results_dir)) + # train agent + trainer.fit(histories=history_reader.load_histories(), validate_each=1) + return trainer + + def run_experiment(optimizer_setup: Callable, optimizer_cls: Type[GraphOptimizer] = EvoGraphOptimizer, adaptive_kind: MutationAgentTypeEnum = MutationAgentTypeEnum.random, @@ -143,13 +156,14 @@ def run_experiment(optimizer_setup: Callable, trial_iterations: Optional[int] = None, visualize: bool = False, save_history: bool = True, + pretrain_dir: Optional[str] = None, ): + metrics = metrics or ['qed_score'] optimizer_id = optimizer_cls.__name__.lower()[:3] experiment_id = f'Experiment [optimizer={optimizer_id} metrics={", ".join(metrics)} pop_size={pop_size}]' exp_name = f'{optimizer_id}_{adaptive_kind.value}_popsize{pop_size}_min{trial_timeout}_{"_".join(metrics)}' atom_types = atom_types or ['C', 'N', 'O', 'F', 'P', 'S', 'Cl', 'Br'] - metrics = metrics or ['qed_score'] trial_results = [] trial_histories = [] trial_timedelta = timedelta(minutes=trial_timeout) if trial_timeout else None @@ -165,6 +179,9 @@ def run_experiment(optimizer_setup: Callable, pop_size, metrics, initial_molecules) + if pretrain_dir: + pretrain_agent(optimizer, objective, pretrain_dir) + found_graphs = optimizer.optimise(objective) history = optimizer.history @@ -208,10 +225,11 @@ def plot_experiment_comparison(experiment_ids: Sequence[str], metric_id: int = 0 if __name__ == '__main__': run_experiment(molecule_search_setup, - adaptive_kind=MutationAgentTypeEnum.random, + adaptive_kind=MutationAgentTypeEnum.bandit, max_heavy_atoms=38, - trial_timeout=15, + trial_timeout=6, pop_size=50, - metrics=['qed_score', 'cl_score'], visualize=True, - num_trials=5) + num_trials=5, + pretrain_dir=os.path.join(project_root(), 'examples', 'molecule_search', 'histories') + ) diff --git a/examples/molecule_search/experiment_with_api.py b/examples/molecule_search/experiment_with_api.py new file mode 100644 index 000000000..bcb829e3d --- /dev/null +++ b/examples/molecule_search/experiment_with_api.py @@ -0,0 +1,141 @@ +import os.path +from datetime import timedelta +from pathlib import Path +from typing import Type, Optional, Sequence, List + +import numpy as np +from rdkit.Chem.rdchem import BondType + +from examples.molecule_search.experiment import visualize_results, get_methane, get_all_mol_metrics +from examples.molecule_search.mol_adapter import MolAdapter +from examples.molecule_search.mol_advisor import MolChangeAdvisor +from examples.molecule_search.mol_graph import MolGraph +from examples.molecule_search.mol_mutations import CHEMICAL_MUTATIONS +from golem.api.main import GOLEM +from golem.core.dag.verification_rules import has_no_self_cycled_nodes, has_no_isolated_components, \ + has_no_isolated_nodes +from golem.core.optimisers.adaptive.operator_agent import MutationAgentTypeEnum +from golem.core.optimisers.genetic.gp_optimizer import EvoGraphOptimizer +from golem.core.optimisers.genetic.operators.crossover import CrossoverTypesEnum +from golem.core.optimisers.genetic.operators.elitism import ElitismTypesEnum +from golem.core.optimisers.genetic.operators.inheritance import GeneticSchemeTypesEnum +from golem.core.optimisers.objective import Objective +from golem.core.optimisers.opt_history_objects.opt_history import OptHistory +from golem.core.optimisers.optimizer import GraphOptimizer +from golem.core.paths import project_root +from golem.visualisation.opt_history.multiple_fitness_line import MultipleFitnessLines + + +def run_experiment(optimizer_cls: Type[GraphOptimizer] = EvoGraphOptimizer, + adaptive_kind: MutationAgentTypeEnum = MutationAgentTypeEnum.random, + max_heavy_atoms: int = 50, + atom_types: Optional[List[str]] = None, + bond_types: Sequence[BondType] = (BondType.SINGLE, BondType.DOUBLE, BondType.TRIPLE), + initial_molecules: Optional[Sequence[MolGraph]] = None, + pop_size: int = 20, + metrics: Optional[List[str]] = None, + num_trials: int = 1, + trial_timeout: Optional[int] = None, + trial_iterations: Optional[int] = None, + visualize: bool = False, + save_history: bool = True, + pretrain_dir: Optional[str] = None, + ): + metrics = metrics or ['qed_score'] + optimizer_id = optimizer_cls.__name__.lower()[:3] + experiment_id = f'Experiment [optimizer={optimizer_id} metrics={", ".join(metrics)} pop_size={pop_size}]' + exp_name = f'{optimizer_id}_{adaptive_kind.value}_popsize{pop_size}_min{trial_timeout}_{"_".join(metrics)}' + + atom_types = atom_types or ['C', 'N', 'O', 'F', 'P', 'S', 'Cl', 'Br'] + trial_results = [] + trial_histories = [] + trial_timedelta = timedelta(minutes=trial_timeout) if trial_timeout else None + all_metrics = get_all_mol_metrics() + objective = Objective( + quality_metrics={metric_name: all_metrics[metric_name] for metric_name in metrics}, + is_multi_objective=len(metrics) > 1 + ) + + for trial in range(num_trials): + + metrics = metrics or ['qed_score'] + + initial_graphs = initial_molecules or [get_methane()] + initial_graphs = MolAdapter().adapt(initial_graphs) + golem = GOLEM( + n_jobs=1, + timeout=trial_timedelta, + objective=objective, + optimizer=optimizer_cls, + initial_graphs=initial_graphs, + pop_size=pop_size, + max_pop_size=pop_size, + multi_objective=True, + genetic_scheme_type=GeneticSchemeTypesEnum.steady_state, + elitism_type=ElitismTypesEnum.replace_worst, + mutation_types=CHEMICAL_MUTATIONS, + crossover_types=[CrossoverTypesEnum.none], + adaptive_mutation_type=adaptive_kind, + adapter=MolAdapter(), + rules_for_constraint=[has_no_self_cycled_nodes, has_no_isolated_components, has_no_isolated_nodes], + advisor=MolChangeAdvisor(), + max_heavy_atoms=max_heavy_atoms, + available_atom_types=atom_types or ['C', 'N', 'O', 'F', 'P', 'S', 'Cl', 'Br'], + bond_types=bond_types, + early_stopping_timeout=np.inf, + early_stopping_iterations=np.inf, + keep_n_best=4, + num_of_generations=trial_iterations, + keep_history=True, + history_dir=None, + ) + found_graphs = golem.optimise() + history = golem.optimiser.history + + if visualize: + molecules = [MolAdapter().restore(graph) for graph in found_graphs] + save_dir = Path('visualisations') / exp_name / f'trial_{trial}' + visualize_results(set(molecules), objective, history, save_dir) + if save_history: + result_dir = Path('results') / exp_name + result_dir.mkdir(parents=True, exist_ok=True) + history.save(result_dir / f'history_trial_{trial}.json') + trial_results.extend(history.final_choices) + trial_histories.append(history) + + # Compute mean & std for metrics of trials + ff = objective.format_fitness + trial_metrics = np.array([ind.fitness.values for ind in trial_results]) + trial_metrics_mean = trial_metrics.mean(axis=0) + trial_metrics_std = trial_metrics.std(axis=0) + print(f'Experiment {experiment_id}\n' + f'finished with metrics:\n' + f'mean={ff(trial_metrics_mean)}\n' + f' std={ff(trial_metrics_std)}') + + +def plot_experiment_comparison(experiment_ids: Sequence[str], metric_id: int = 0, results_dir='./results'): + root = Path(results_dir) + histories = {} + for exp_name in experiment_ids: + trials = [] + for history_filename in os.listdir(root / exp_name): + if history_filename.startswith('history'): + history = OptHistory.load(root / exp_name / history_filename) + trials.append(history) + histories[exp_name] = trials + print(f'Loaded {len(trials)} trial histories for experiment: {exp_name}') + # Visualize + MultipleFitnessLines.from_histories(histories).visualize(metric_id=metric_id) + return histories + + +if __name__ == '__main__': + run_experiment(adaptive_kind=MutationAgentTypeEnum.bandit, + max_heavy_atoms=38, + trial_timeout=6, + pop_size=50, + visualize=True, + num_trials=5, + pretrain_dir=os.path.join(project_root(), 'examples', 'molecule_search', 'histories') + ) diff --git a/examples/molecule_search/mol_adapter.py b/examples/molecule_search/mol_adapter.py index dfd0a2367..226d6ca31 100644 --- a/examples/molecule_search/mol_adapter.py +++ b/examples/molecule_search/mol_adapter.py @@ -17,8 +17,10 @@ def __init__(self): def _restore(self, opt_graph: OptGraph, metadata: Optional[Dict[str, Any]] = None) -> MolGraph: digraph = self.nx_adapter.restore(opt_graph) - # return to previous node indexing - digraph = nx.relabel_nodes(digraph, dict(digraph.nodes(data='nxid'))) + # to ensure backward compatibility with old individuals without 'nxid' field in nodes + if not any(x is None for x in list(dict(digraph.nodes(data='nxid')).values())): + # return to previous node indexing + digraph = nx.relabel_nodes(digraph, dict(digraph.nodes(data='nxid'))) digraph = restore_edges_params_from_nodes(digraph) nx_graph = digraph.to_undirected() mol_graph = MolGraph.from_nx_graph(nx_graph) @@ -50,7 +52,11 @@ def restore_edges_params_from_nodes(graph: nx.DiGraph) -> nx.DiGraph: all_edges_params = {} for node in graph.nodes(): for predecessor in graph.predecessors(node): - edge_params = edge_params_by_node[node][predecessor] - all_edges_params.update({(predecessor, node): edge_params}) + node_params = edge_params_by_node[node] + # sometimes by unknown reason some nodes are encoded as int, some as str. + # maybe that's deserialization messing up somewhere. + edge_params = node_params.get(predecessor) or node_params.get(str(predecessor)) + if edge_params: + all_edges_params[(predecessor, node)] = edge_params nx.set_edge_attributes(graph, all_edges_params) return graph diff --git a/examples/molecule_search/mol_graph.py b/examples/molecule_search/mol_graph.py index 7db32b687..799c6c8f2 100644 --- a/examples/molecule_search/mol_graph.py +++ b/examples/molecule_search/mol_graph.py @@ -5,7 +5,7 @@ from rdkit import Chem from rdkit.Chem import MolFromSmiles, MolToSmiles, SanitizeMol, Kekulize, MolToInchi from rdkit.Chem.Draw import rdMolDraw2D -from rdkit.Chem.rdchem import Atom, BondType, RWMol, GetPeriodicTable +from rdkit.Chem.rdchem import Atom, BondType, RWMol, GetPeriodicTable, ChiralType, HybridizationType class MolGraph: @@ -32,10 +32,10 @@ def from_nx_graph(graph: nx.Graph): node_to_idx = {} for node in graph.nodes(): a = Chem.Atom(atomic_nums[node]) - a.SetChiralTag(chiral_tags[node]) + a.SetChiralTag(ChiralType(chiral_tags[node])) a.SetFormalCharge(formal_charges[node]) a.SetIsAromatic(node_is_aromatics[node]) - a.SetHybridization(node_hybridizations[node]) + a.SetHybridization(HybridizationType(node_hybridizations[node])) a.SetNumExplicitHs(num_explicit_hss[node]) idx = mol.AddAtom(a) node_to_idx[node] = idx @@ -45,7 +45,7 @@ def from_nx_graph(graph: nx.Graph): first, second = edge ifirst = node_to_idx[first] isecond = node_to_idx[second] - bond_type = bond_types[first, second] + bond_type = BondType(bond_types[first, second]) mol.AddBond(ifirst, isecond, bond_type) SanitizeMol(mol) diff --git a/examples/structural_analysis/opt_graph_optimization.py b/examples/structural_analysis/opt_graph_optimization.py index 39632e99f..6b05446e6 100644 --- a/examples/structural_analysis/opt_graph_optimization.py +++ b/examples/structural_analysis/opt_graph_optimization.py @@ -68,7 +68,8 @@ def complexity_metric(graph: OptGraph, adapter: BaseNetworkxAdapter, metric: Cal seed=1, replacement_number_of_random_operations_nodes=2, replacement_number_of_random_operations_edges=2) - path_to_save = os.path.join(project_root(), 'sa') + path_to_save = os.path.join(project_root(), 'examples', 'sa') + os.makedirs(path_to_save, exist_ok=True) # structural analysis will optimize given graph if at least one of the metrics was increased. sa = GraphStructuralAnalysis(objective=objective, node_factory=node_factory, requirements=requirements, @@ -81,6 +82,7 @@ def complexity_metric(graph: OptGraph, adapter: BaseNetworkxAdapter, metric: Cal optimized_graph = GraphStructuralAnalysis.visualize_on_graph(graph=get_opt_graph(), analysis_result=results, metric_idx_to_optimize_by=0, mode="by_iteration", - font_size_scale=0.6) + font_size_scale=0.6, + save_path=path_to_save) graph.show() diff --git a/examples/synthetic_graph_evolution/simple_run.py b/examples/synthetic_graph_evolution/simple_run.py index fdce35f65..85e338c86 100644 --- a/examples/synthetic_graph_evolution/simple_run.py +++ b/examples/synthetic_graph_evolution/simple_run.py @@ -40,8 +40,9 @@ def run_graph_search(size=16, timeout=8, visualize=True): MutationTypesEnum.single_change], crossover_types=[CrossoverTypesEnum.subtree] ) + adapter = BaseNetworkxAdapter() # Example works with NetworkX graphs graph_gen_params = GraphGenerationParams( - adapter=BaseNetworkxAdapter(), # Example works with NetworkX graphs + adapter=adapter, rules_for_constraint=DEFAULT_DAG_RULES, # We don't want cycles in the graph available_node_types=node_types # Node types that can appear in graphs ) @@ -50,11 +51,18 @@ def run_graph_search(size=16, timeout=8, visualize=True): # Build and run the optimizer optimiser = EvoGraphOptimizer(objective, initial_graphs, *all_parameters) found_graphs = optimiser.optimise(objective) + if visualize: # Restore the NetworkX graph back from internal Graph representation - found_graph = graph_gen_params.adapter.restore(found_graphs[0]) + found_graph = adapter.restore(found_graphs[0]) draw_graphs_subplots(target_graph, found_graph, titles=['Target Graph', 'Found Graph']) optimiser.history.show.fitness_line() + + # Animation of genealogical path of the best individual: + optimiser.history.show.genealogical_path(graph_dist=adapter.adapt_func(tree_edit_dist), + target_graph=adapter.adapt(target_graph), + show=True) + return found_graphs diff --git a/examples/synthetic_graph_evolution/simple_run_with_api.py b/examples/synthetic_graph_evolution/simple_run_with_api.py new file mode 100644 index 000000000..68fe3471e --- /dev/null +++ b/examples/synthetic_graph_evolution/simple_run_with_api.py @@ -0,0 +1,48 @@ +import logging +from functools import partial + +from examples.synthetic_graph_evolution.generators import generate_labeled_graph +from golem.api.main import GOLEM +from golem.core.optimisers.genetic.operators.base_mutations import MutationTypesEnum +from golem.core.optimisers.genetic.operators.crossover import CrossoverTypesEnum +from golem.core.optimisers.genetic.operators.inheritance import GeneticSchemeTypesEnum +from golem.core.optimisers.objective import Objective +from golem.metrics.edit_distance import tree_edit_dist + + +def run_graph_search(size=16, timeout=8, visualize=True): + # Generate target graph that will be sought by optimizer + node_types = ('a', 'b') + target_graph = generate_labeled_graph('tree', size, node_labels=node_types) + + # Generate initial population with small tree graphs + initial_graphs = [generate_labeled_graph('tree', 5, node_types) for _ in range(10)] + # Setup objective: edit distance to target graph + objective = Objective(partial(tree_edit_dist, target_graph)) + + golem = GOLEM(timeout=timeout, + logging_level=logging.INFO, + early_stopping_iterations=100, + initial_graphs=initial_graphs, + objective=objective, + genetic_scheme_type=GeneticSchemeTypesEnum.parameter_free, + max_pop_size=50, + mutation_types=[MutationTypesEnum.single_add, + MutationTypesEnum.single_drop, + MutationTypesEnum.single_change], + crossover_types=[CrossoverTypesEnum.subtree], + available_node_types=node_types # Node types that can appear in graphs + ) + found_graphs = golem.optimise() + + return found_graphs + + +if __name__ == '__main__': + """ + Same as `simple_run.py` but with GOLEM API usage example. + In this example Optimizer is expected to find the target graph + using Tree Edit Distance metric and a random tree (nx.random_tree) as target. + The convergence can be seen from achieved metrics and visually from graph plots. + """ + run_graph_search(visualize=True) diff --git a/examples/synthetic_graph_evolution/utils.py b/examples/synthetic_graph_evolution/utils.py index 84aac2e88..7055a5056 100644 --- a/examples/synthetic_graph_evolution/utils.py +++ b/examples/synthetic_graph_evolution/utils.py @@ -10,7 +10,7 @@ from golem.core.adapter.nx_adapter import BaseNetworkxAdapter from golem.core.optimisers.opt_history_objects.opt_history import OptHistory -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence from golem.visualisation.graph_viz import GraphVisualizer diff --git a/examples/tuning_example.py b/examples/tuning_example.py new file mode 100644 index 000000000..6aab55480 --- /dev/null +++ b/examples/tuning_example.py @@ -0,0 +1,72 @@ +from datetime import timedelta + +from golem.core.optimisers.graph import OptNode, OptGraph +from golem.core.optimisers.objective import ObjectiveEvaluate, Objective +from golem.core.tuning.iopt_tuner import IOptTuner +from golem.core.tuning.search_space import SearchSpace +from test.unit.utils import ParamsSumMetric + + +def opt_graph_with_params(): + node_a = OptNode('a') + node_b = OptNode({'name': 'b', 'params': {'b2': 0.7, 'b3': 2}}) + node_c = OptNode('c', nodes_from=[node_a]) + node_d = OptNode('d', nodes_from=[node_b]) + node_final = OptNode('e', nodes_from=[node_c, node_d]) + graph = OptGraph(node_final) + return graph + + +def get_search_space(): + params_per_operation = { + 'a': { + 'a1': { + 'sampling-scope': [2, 7], + 'type': 'discrete' + }, + 'a2': { + 'sampling-scope': [1e-3, 1], + 'type': 'continuous' + }, + 'a3': { + 'sampling-scope': [['A', 'B', 'C']], + 'type': 'categorical' + } + }, + 'b': { + 'b1': { + 'sampling-scope': [["first", "second", "third"]], + 'type': 'categorical' + }, + 'b2': { + 'sampling-scope': [0.04, 1.0], + 'type': 'continuous' + }, + }, + 'e': { + 'e1': { + 'sampling-scope': [0.05, 1.0], + 'type': 'continuous' + }, + 'e2': { + 'sampling-scope': [0.05, 1.0], + 'type': 'continuous' + } + }, + 'k': { + 'k': { + 'sampling-scope': [1e-2, 10.0], + 'type': 'continuous' + } + }} + return SearchSpace(params_per_operation) + + +if __name__ == '__main__': + search_space = get_search_space() + graph = opt_graph_with_params() + # search for parameters that will maximize their sum + obj_eval = ObjectiveEvaluate(Objective({'sum_metric': ParamsSumMetric.get_value})) + + tuner = IOptTuner(obj_eval, search_space, iterations=10, n_jobs=-1) + tuned_graph = tuner.tune(graph) diff --git a/golem/core/utilities/__init__.py b/golem/api/__init__.py similarity index 100% rename from golem/core/utilities/__init__.py rename to golem/api/__init__.py diff --git a/golem/api/api_utils/__init__.py b/golem/api/api_utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/golem/api/api_utils/api_params.py b/golem/api/api_utils/api_params.py new file mode 100644 index 000000000..2530d7254 --- /dev/null +++ b/golem/api/api_utils/api_params.py @@ -0,0 +1,102 @@ +import datetime +from collections import UserDict + +from typing import Dict, Any + +from golem.core.adapter.nx_adapter import BaseNetworkxAdapter +from golem.core.dag.verification_rules import DEFAULT_DAG_RULES +from golem.core.log import LoggerAdapter, default_log +from golem.core.optimisers.dynamic_graph_requirements import DynamicGraphRequirements +from golem.core.optimisers.genetic.gp_optimizer import EvoGraphOptimizer +from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters +from golem.core.optimisers.optimization_parameters import GraphRequirements +from golem.core.optimisers.optimizer import GraphGenerationParams +from golem.utilities.utilities import determine_n_jobs + + +class ApiParams(UserDict): + """ + Class to further distribute params specified params in API between the following classes: + `GraphRequirements`, `GraphGenerationParams`, `GPAlgorithmParameters`. + """ + def __init__(self, input_params: Dict[str, Any], n_jobs: int = -1, timeout: float = 5): + self.log: LoggerAdapter = default_log(self) + self.n_jobs: int = determine_n_jobs(n_jobs) + self.timeout = timeout + + self._input_params = input_params + self._input_params['timeout'] = timeout if isinstance(timeout, datetime.timedelta) else datetime.timedelta(minutes=timeout) + self._default_common_params = self.get_default_common_params() + super().__init__(self._input_params) + + def get_default_common_params(self): + """ Common params that do not belong to any category + (from `GPAlgorithmParameters`, `GraphGenerationParams`, `GraphRequirements`). """ + default_common_params = { + 'optimizer': EvoGraphOptimizer, + 'initial_graphs': list(), + 'objective': None + } + self.log.info("EvoGraphOptimizer was used as default optimizer, " + "will be overwritten by specified one if there is any.") + return default_common_params + + def get_default_graph_generation_params(self): + """ Default graph generations params to minimize the number of arguments that must be specified in API. + Need to be hardcoded like that since the list of input arguments is not the same as the class fields list. """ + default_graph_generation_params = { + 'adapter': BaseNetworkxAdapter(), + 'rules_for_constraint': DEFAULT_DAG_RULES, + 'advisor': None, + 'node_factory': None, + 'random_graph_factory': None, + 'available_node_types': None, + 'remote_evaluator': None + } + self.log.info("BaseNetworkxAdapter was used as default adapter, " + "will be overwritten by specified one if there is any.") + return default_graph_generation_params + + def get_gp_algorithm_parameters(self) -> GPAlgorithmParameters: + default_gp_algorithm_params_dict = dict(list(vars(GPAlgorithmParameters()).items())) + k_pop = [] + for k, v in self._input_params.items(): + if k in default_gp_algorithm_params_dict: + default_gp_algorithm_params_dict[k] = self._input_params[k] + k_pop.append(k) + for k in k_pop: + self._input_params.pop(k) + return GPAlgorithmParameters(**default_gp_algorithm_params_dict) + + def get_graph_generation_parameters(self) -> GraphGenerationParams: + default_graph_generation_params_dict = self.get_default_graph_generation_params() + k_pop = [] + for k, v in self._input_params.items(): + if k in default_graph_generation_params_dict: + default_graph_generation_params_dict[k] = self._input_params[k] + k_pop.append(k) + for k in k_pop: + self._input_params.pop(k) + ggp = GraphGenerationParams(**default_graph_generation_params_dict) + return ggp + + def get_graph_requirements(self) -> GraphRequirements: + default_graph_requirements_params_dict = dict(list(vars(GraphRequirements()).items())) + # if there are any custom domain specific graph requirements params + is_custom_graph_requirements_params = \ + any([k not in default_graph_requirements_params_dict for k in self._input_params]) + for k, v in self._input_params.items(): + # add all parameters except common left unused after GPAlgorithmParameters and GraphGenerationParams + # initialization, since it can be custom domain specific params + if k not in self._default_common_params: + default_graph_requirements_params_dict[k] = self._input_params[k] + if is_custom_graph_requirements_params: + return DynamicGraphRequirements(default_graph_requirements_params_dict) + else: + return GraphRequirements(**default_graph_requirements_params_dict) + + def get_actual_common_params(self) -> Dict[str, Any]: + for k, v in self._input_params.items(): + if k in self._default_common_params: + self._default_common_params[k] = v + return self._default_common_params diff --git a/golem/api/main.py b/golem/api/main.py new file mode 100644 index 000000000..326bb914b --- /dev/null +++ b/golem/api/main.py @@ -0,0 +1,139 @@ +import logging +from typing import Optional + +from golem.api.api_utils.api_params import ApiParams +from golem.core.constants import DEFAULT_API_TIMEOUT_MINUTES +from golem.core.log import Log, default_log +from golem.utilities.utilities import set_random_seed + + +class GOLEM: + """ + Main class for GOLEM API. + + Args: + :param timeout: timeout for optimization. + :param seed: value for a fixed random seed. + :param logging_level: logging levels are the same as in `logging `_. + + .. details:: Possible options: + + - ``50`` -> critical + - ``40`` -> error + - ``30`` -> warning + - ``20`` -> info + - ``10`` -> debug + - ``0`` -> nonset + :param n_jobs: num of ``n_jobs`` for parallelization (set to ``-1`` to use all cpu's). Defaults to ``-1``. + :param graph_requirements_class: class to specify custom graph requirements. + Must be inherited from GraphRequirements class. + + :param crossover_prob: crossover probability (chance that two individuals will be mated). + + ``GPAlgorithmParameters`` parameters + :param mutation_prob: mutation probability (chance that an individual will be mutated). + :param variable_mutation_num: flag to apply mutation one or few times for individual in each iteration. + :param max_num_of_operator_attempts: max number of unsuccessful evo operator attempts before continuing. + :param mutation_strength: strength of mutation in tree (using in certain mutation types) + :param min_pop_size_with_elitism: minimal population size with which elitism is applicable + :param required_valid_ratio: ratio of valid individuals on next population to continue optimization. + + Used in `ReproductionController` to compensate for invalid individuals. See the class for details. + + :param adaptive_mutation_type: enables adaptive Mutation agent. + :param context_agent_type: enables graph encoding for Mutation agent. + + Adaptive mutation agent uses specified algorithm. 'random' type is the default non-adaptive version. + Requires crossover_types to be CrossoverTypesEnum.none for correct adaptive learning, + so that fitness changes depend only on agent's actions (chosen mutations). + ``MutationAgentTypeEnum.bandit`` uses Multi-Armed Bandit (MAB) learning algorithm. + ``MutationAgentTypeEnum.contextual_bandit`` uses contextual MAB learning algorithm. + ``MutationAgentTypeEnum.neural_bandit`` uses contextual MAB learning algorithm with Deep Neural encoding. + + Parameter `context_agent_type` specifies implementation of graph/node encoder for adaptive + mutation agent. It is relevant for contextual and neural bandits. + + :param decaying_factor: decaying factor for Multi-Armed Bandits for managing the profit from operators + The smaller the value of decaying_factor, the larger the influence for the best operator. + :param window_size: the size of sliding window for Multi-Armed Bandits to decrease variance. + The window size is measured by the number of individuals to consider. + + + :param selection_types: Sequence of selection operators types + :param crossover_types: Sequence of crossover operators types + :param mutation_types: Sequence of mutation operators types + :param elitism_type: type of elitism operator evolution + + :param regularization_type: type of regularization operator + + Regularization attempts to cut off the subtrees of the graph. If the truncated graph + is not worse than the original, then it enters the new generation as a simpler solution. + Regularization is not used by default, it must be explicitly enabled. + + :param genetic_scheme_type: type of genetic evolutionary scheme + + The `generational` scheme is a standard scheme of the evolutionary algorithm. + It specifies that at each iteration the entire generation is updated. + + In the `steady_state` individuals from previous populations are mixed with the ones from new population. + UUIDs of individuals do not repeat within one population. + + The `parameter_free` scheme is same as `steady_state` for now. + + ``GraphGenerationParams`` parameters + :param adapter: instance of domain graph adapter for adaptation + between domain and optimization graphs + :param rules_for_constraint: collection of constraints for graph verification + :param advisor: instance providing task and context-specific advices for graph changes + :param node_factory: instance for generating new nodes in the process of graph search + :param remote_evaluator: instance of delegate evaluator for evaluation of graphs + + ``GraphRequirements`` parameters + :param start_depth: start value of adaptive tree depth + :param max_depth: max depth of the resulting graph + :param min_arity: min number of parents for node + :param max_arity: max number of parents for node + + Also, custom domain specific parameters can be specified here. These parameters can be then used in + ``DynamicGraphRequirements`` as fields. + """ + def __init__(self, + timeout: Optional[float] = DEFAULT_API_TIMEOUT_MINUTES, + seed: Optional[int] = None, + logging_level: int = logging.INFO, + n_jobs: int = -1, + **all_parameters): + set_random_seed(seed) + self.log = self._init_logger(logging_level) + + self.api_params = ApiParams(input_params=all_parameters, + n_jobs=n_jobs, + timeout=timeout) + self.gp_algorithm_parameters = self.api_params.get_gp_algorithm_parameters() + self.graph_generation_parameters = self.api_params.get_graph_generation_parameters() + self.graph_requirements = self.api_params.get_graph_requirements() + + def optimise(self, **custom_optimiser_parameters): + """ Method to start optimisation process. + `custom_optimiser_parameters` parameters can be specified additionally to use it directly in optimiser. + """ + common_params = self.api_params.get_actual_common_params() + optimizer_cls = common_params['optimizer'] + objective = common_params['objective'] + initial_graphs = common_params['initial_graphs'] + + self.optimiser = optimizer_cls(objective, + initial_graphs, + self.graph_requirements, + self.graph_generation_parameters, + self.gp_algorithm_parameters, + **custom_optimiser_parameters) + + found_graphs = self.optimiser.optimise(objective) + return found_graphs + + @staticmethod + def _init_logger(logging_level: int): + # reset logging level for Singleton + Log().reset_logging_level(logging_level) + return default_log(prefix='GOLEM logger') diff --git a/golem/core/adapter/adapt_registry.py b/golem/core/adapter/adapt_registry.py index 39c9554a2..850c24187 100644 --- a/golem/core/adapter/adapt_registry.py +++ b/golem/core/adapter/adapt_registry.py @@ -2,7 +2,7 @@ from functools import partial from typing import Callable -from golem.core.utilities.singleton_meta import SingletonMeta +from golem.utilities.singleton_meta import SingletonMeta class AdaptRegistry(metaclass=SingletonMeta): diff --git a/golem/core/adapter/nx_adapter.py b/golem/core/adapter/nx_adapter.py index 91645b71d..183e4456d 100644 --- a/golem/core/adapter/nx_adapter.py +++ b/golem/core/adapter/nx_adapter.py @@ -22,13 +22,14 @@ def __init__(self): def _node_restore(self, node: GraphNode) -> Dict: """Transforms GraphNode to dict of NetworkX node attributes. Override for custom behavior.""" + parameters = {} if hasattr(node, 'parameters'): - parameters = node.parameters - if node.name: - parameters['name'] = node.name - return deepcopy(parameters) - else: - return {} + parameters = deepcopy(node.parameters) + + if node.name: + parameters['name'] = node.name + + return parameters def _node_adapt(self, data: Dict) -> OptNode: """Transforms a dict of NetworkX node attributes to GraphNode. diff --git a/golem/core/constants.py b/golem/core/constants.py index 4104ead3a..c9fad3b72 100644 --- a/golem/core/constants.py +++ b/golem/core/constants.py @@ -7,3 +7,4 @@ EVALUATION_ATTEMPTS_NUMBER = 5 # Min pop size to avoid getting stuck in local maximum during optimization. MIN_POP_SIZE = 5 +DEFAULT_API_TIMEOUT_MINUTES = 5.0 diff --git a/golem/core/dag/graph.py b/golem/core/dag/graph.py index 8c64280be..239bb23da 100644 --- a/golem/core/dag/graph.py +++ b/golem/core/dag/graph.py @@ -1,7 +1,9 @@ from abc import ABC, abstractmethod from enum import Enum from os import PathLike -from typing import Dict, List, Optional, Sequence, Union, Tuple, TypeVar +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, TypeVar, Union + +import networkx as nx from golem.core.dag.graph_node import GraphNode from golem.visualisation.graph_viz import GraphVisualizer, NodeColorType @@ -33,8 +35,9 @@ def add_node(self, node: GraphNode): def update_node(self, old_node: GraphNode, new_node: GraphNode): """Replaces ``old_node`` node with ``new_node`` - :param old_node: node to be replaced - :param new_node: node to be placed instead + Args: + old_node: node to be replaced + new_node: node to be placed instead """ raise NotImplementedError() @@ -207,7 +210,9 @@ def show(self, save_path: Optional[Union[PathLike, str]] = None, engine: Optiona node_size_scale: Optional[float] = None, font_size_scale: Optional[float] = None, edge_curvature_scale: Optional[float] = None, title: Optional[str] = None, - nodes_labels: Dict[int, str] = None, edges_labels: Dict[int, str] = None): + node_names_placement: Optional[Literal['auto', 'nodes', 'legend', 'none']] = None, + nodes_labels: Dict[int, str] = None, edges_labels: Dict[int, str] = None, + nodes_layout_function: Optional[Callable[[nx.DiGraph], Dict[Any, Tuple[float, float]]]] = None): """Visualizes graph or saves its picture to the specified ``path`` Args: @@ -219,14 +224,28 @@ def show(self, save_path: Optional[Union[PathLike, str]] = None, engine: Optiona edge_curvature_scale: use to make edges more or less curved. Supported only for the engine 'matplotlib'. dpi: DPI of the output image. Not supported for the engine 'pyvis'. title: title for plot + node_names_placement: variant of node names displaying. Defaults to ``auto``. + + Possible options: + + - ``auto`` -> empirical rule by node size + + - ``nodes`` -> place node names on top of the nodes + + - ``legend`` -> place node names at the legend + + - ``none`` -> do not show node names + nodes_labels: labels to display near nodes edges_labels: labels to display near edges + nodes_layout_function: any of `Networkx layout functions \ + `_ . """ - GraphVisualizer(graph=self)\ + GraphVisualizer(graph=self) \ .visualise(save_path=save_path, engine=engine, node_color=node_color, dpi=dpi, node_size_scale=node_size_scale, font_size_scale=font_size_scale, - edge_curvature_scale=edge_curvature_scale, - title=title, + edge_curvature_scale=edge_curvature_scale, node_names_placement=node_names_placement, + title=title, nodes_layout_function=nodes_layout_function, nodes_labels=nodes_labels, edges_labels=edges_labels) @property diff --git a/golem/core/dag/graph_utils.py b/golem/core/dag/graph_utils.py index 99a7d7cc8..4516c736f 100644 --- a/golem/core/dag/graph_utils.py +++ b/golem/core/dag/graph_utils.py @@ -1,6 +1,6 @@ from typing import Sequence, List, TYPE_CHECKING, Callable, Union -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence if TYPE_CHECKING: from golem.core.dag.graph import Graph diff --git a/golem/core/dag/linked_graph.py b/golem/core/dag/linked_graph.py index 8eceb77a8..789c3f6ab 100644 --- a/golem/core/dag/linked_graph.py +++ b/golem/core/dag/linked_graph.py @@ -8,7 +8,7 @@ from golem.core.dag.graph_node import GraphNode from golem.core.dag.graph_utils import ordered_subnodes_hierarchy, node_depth, graph_has_cycle from golem.core.paths import copy_doc -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence, Copyable, remove_items +from golem.utilities.data_structures import ensure_wrapped_in_sequence, Copyable, remove_items NodePostprocessCallable = Callable[[Graph, Sequence[GraphNode]], Any] diff --git a/golem/core/dag/linked_graph_node.py b/golem/core/dag/linked_graph_node.py index 964b5824f..20a19ac32 100644 --- a/golem/core/dag/linked_graph_node.py +++ b/golem/core/dag/linked_graph_node.py @@ -1,6 +1,6 @@ from typing import Union, Optional, Iterable, List from golem.core.dag.graph_node import GraphNode -from golem.core.utilities.data_structures import UniqueList +from golem.utilities.data_structures import UniqueList class LinkedGraphNode(GraphNode): diff --git a/golem/core/log.py b/golem/core/log.py index a6b9ea9a5..bcf447162 100644 --- a/golem/core/log.py +++ b/golem/core/log.py @@ -8,8 +8,10 @@ from logging.handlers import RotatingFileHandler from typing import Optional, Tuple, Union -from golem.core.utilities.singleton_meta import SingletonMeta +from typing_extensions import Literal + from golem.core.paths import default_data_dir +from golem.utilities.singleton_meta import SingletonMeta DEFAULT_LOG_PATH = pathlib.Path(default_data_dir(), 'log.log') @@ -160,46 +162,57 @@ def process(self, msg, kwargs): self.logger.setLevel(self.logging_level) return '%s - %s' % (self.extra['prefix'], msg), kwargs - def debug(self, msg, *args, **kwargs): - raise_if_test(msg, **kwargs) - super().debug(msg, *args, **kwargs) - - def info(self, msg, *args, **kwargs): - raise_if_test(msg, **kwargs) - super().info(msg, *args, **kwargs) - - def warning(self, msg, *args, **kwargs): - raise_if_test(msg, **kwargs) - super().warning(msg, *args, **kwargs) + def message(self, msg: str, **kwargs): + """ Record the message to user. + Message is an intermediate logging level between info and warning + to display main info about optimization process """ + level = 45 + self.log(level, msg, **kwargs) - def error(self, msg, *args, **kwargs): - raise_if_test(msg, **kwargs) - super().error(msg, *args, **kwargs) + def log_or_raise( + self, level: Union[int, Literal['debug', 'info', 'warning', 'error', 'critical', 'message']], + exc: Union[BaseException, object], + **log_kwargs): + """ Logs the given exception with the given logging level or raises it if the current + session is a test one. - def exception(self, msg, *args, exc_info=True, **kwargs): - raise_if_test(msg, **kwargs) - super().exception(msg, *args, **kwargs) + The given exception is logged with its traceback. If this method is called inside an ``except`` block, + the exception caught earlier is used as a cause for the given exception. - def critical(self, msg, *args, **kwargs): - raise_if_test(msg, **kwargs) - super().critical(msg, *args, **kwargs) + Args: - def log(self, level, msg, *args, **kwargs): - """ - Delegate a log call to the underlying logger, after adding - contextual information from this adapter instance. + level: the same as in :py:func:`logging.log`, but may be specified as a lower-case string literal + for convenience. For example, the value ``warning`` is equivalent for ``logging.WARNING``. + This includes a custom "message" logging level that equals to 45. + exc: the exception/message to log/raise. Given a message, an ``Exception`` instance is initialized + based on the message. + log_kwargs: keyword arguments for :py:func:`logging.log`. """ - raise_if_test(msg, **kwargs) - super().log(level, msg, *args, **kwargs) - - def message(self, msg: str, **kwargs): - """ Record the message to user. - Message is an intermediate logging level between info and warning - to display main info about optimization process """ - raise_if_test(msg, **kwargs) - message_logging_level = 45 - if message_logging_level >= self.logging_level: - self.critical(msg=msg) + _, recent_exc, _ = sys.exc_info() # Catch the most recent exception + if not isinstance(exc, BaseException): + exc = Exception(exc) + try: + # Raise anyway to combine tracebacks + raise exc from recent_exc + except type(exc) as exc_info: + # Raise further if test session + if is_test_session(): + raise + # Log otherwise + level_map = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warning': logging.WARNING, + 'error': logging.ERROR, + 'critical': logging.CRITICAL, + 'message': 45, + } + if isinstance(level, str): + level = level_map[level] + self.log(level, exc, + exc_info=log_kwargs.pop('exc_info', exc_info), + stacklevel=log_kwargs.pop('stacklevel', 2), + **log_kwargs) def __str__(self): return f'LoggerAdapter object for {self.extra["prefix"]} module' @@ -208,11 +221,6 @@ def __repr__(self): return self.__str__() -def raise_if_test(msg, **kwargs): - if kwargs.get('raise_if_test', False) is True and is_test_session: - raise Exception(msg) - - def is_test_session(): return 'PYTEST_CURRENT_TEST' in os.environ diff --git a/golem/core/optimisers/adaptive/agent_trainer.py b/golem/core/optimisers/adaptive/agent_trainer.py new file mode 100644 index 000000000..80e9b6c99 --- /dev/null +++ b/golem/core/optimisers/adaptive/agent_trainer.py @@ -0,0 +1,198 @@ +import operator +from copy import deepcopy +from functools import reduce +from typing import Sequence, Optional, Any, Tuple, List, Iterable + +import numpy as np + +from golem.core.dag.graph import Graph +from golem.core.log import default_log +from golem.core.optimisers.adaptive.common_types import TrajectoryStep, GraphTrajectory +from golem.core.optimisers.adaptive.experience_buffer import ExperienceBuffer +from golem.core.optimisers.adaptive.operator_agent import OperatorAgent +from golem.core.optimisers.fitness import Fitness +from golem.core.optimisers.genetic.operators.mutation import Mutation +from golem.core.optimisers.objective import Objective +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.optimisers.opt_history_objects.opt_history import OptHistory +from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator +from golem.utilities.data_structures import unzip + + +class AgentTrainer: + """Utility class providing fit/validate logic for adaptive Mutation agents. + Works in tandem with `HistoryReader`. + + How to use offline training: + + 1. Collect histories to some directory using `ExperimentLauncher` + 2. Create optimizer & Pretrain mutation agent on these histories using `HistoryReader` and `AgentTrainer` + 3. Optionally, validate the Agent on validation set of histories + 4. Run optimization with pretrained agent + """ + + def __init__(self, + objective: Objective, + mutation_operator: Mutation, + agent: Optional[OperatorAgent] = None, + ): + self._log = default_log(self) + self.agent = agent if agent is not None else mutation_operator.agent + self.mutation = mutation_operator + self.objective = objective + self._adapter = self.mutation.graph_generation_params.adapter + + def fit(self, histories: Iterable[OptHistory], validate_each: int = -1) -> OperatorAgent: + """ + Method to fit trainer on collected histories. + param histories: histories to use in training. + param validate_each: validate agent once in validate_each generation. + """ + # Set mutation probabilities to 1.0 + initial_req = deepcopy(self.mutation.requirements) + self.mutation.requirements.mutation_prob = 1.0 + + for i, history in enumerate(histories): + # Preliminary validity check + # This allows to filter out histories with different objectives automatically + if history.objective.metric_names != self.objective.metric_names: + self._log.warning(f'History #{i+1} has different objective! ' + f'Expected {self.objective}, got {history.objective}.') + continue + + # Build datasets + experience = ExperienceBuffer.from_history(history) + val_experience = None + if validate_each > 0 and i % validate_each == 0: + experience, val_experience = experience.split(ratio=0.8, shuffle=True) + + # Train + self._log.info(f'Training on history #{i+1} with {len(history.generations)} generations') + self.agent.partial_fit(experience) + + # Validate + if val_experience: + reward_loss, reward_target = self.validate_agent(experience=val_experience) + self._log.info(f'Agent validation for history #{i+1} & {experience}: ' + f'Reward target={reward_target:.3f}, loss={reward_loss:.3f}') + + # Reset mutation probabilities to default + self.mutation.update_requirements(requirements=initial_req) + return self.agent + + def validate_on_rollouts(self, histories: Sequence[OptHistory]) -> float: + """Validates rollouts of agent vs. historic trajectories, comparing + their mean total rewards (i.e. total fitness gain over the trajectory).""" + + # Collect all trajectories from all histories; and their rewards + trajectories = concat_lists(map(ExperienceBuffer.unroll_trajectories, histories)) + + mean_traj_len = int(np.mean([len(tr) for tr in trajectories])) + traj_rewards = [sum(reward for _, reward, _ in traj) for traj in trajectories] + mean_baseline_reward = np.mean(traj_rewards) + + # Collect same number of trajectories of the same length; and their rewards + agent_trajectories = [self._sample_trajectory(initial=tr[0][0], length=mean_traj_len) + for tr in trajectories] + agent_traj_rewards = [sum(reward for _, reward, _ in traj) for traj in agent_trajectories] + mean_agent_reward = np.mean(agent_traj_rewards) + + # Compute improvement score of agent over baseline histories + improvement = mean_agent_reward - mean_baseline_reward + return improvement + + def validate_history(self, history: OptHistory) -> Tuple[float, float]: + """Validates history of mutated individuals against optimal policy.""" + history_trajectories = ExperienceBuffer.unroll_trajectories(history) + return self._validate_against_optimal(history_trajectories) + + def validate_agent(self, + graphs: Optional[Sequence[Graph]] = None, + experience: Optional[ExperienceBuffer] = None) -> Tuple[float, float]: + """Validates agent policy against optimal policy on given graphs.""" + if experience: + agent_steps = experience.retrieve_trajectories() + elif graphs: + agent_steps = [self._make_action_step(Individual(g)) for g in graphs] + else: + self._log.warning('Either graphs or history must not be None for validation!') + return 0., 0. + return self._validate_against_optimal(trajectories=[agent_steps]) + + def _validate_against_optimal(self, trajectories: Sequence[GraphTrajectory]) -> Tuple[float, float]: + """Validates a policy trajectories against optimal policy + that at each step always chooses the best action with max reward.""" + reward_losses = [] + reward_targets = [] + for trajectory in trajectories: + inds, actions, rewards = unzip(trajectory) + _, best_actions, best_rewards = self._apply_best_action(inds) + reward_loss = self._compute_reward_loss(rewards, best_rewards) + reward_losses.append(reward_loss) + reward_targets.append(np.mean(best_rewards)) + reward_loss = float(np.mean(reward_losses)) + reward_target = float(np.mean(reward_targets)) + return reward_loss, reward_target + + @staticmethod + def _compute_reward_loss(rewards, optimal_rewards, normalized=False) -> float: + """Returns difference (or deviation) from optimal reward. + When normalized, 0. means actual rewards match optimal rewards completely, + 0.5 means they on average deviate by 50% from optimal rewards, + and 2.2 means they on average deviate by more than 2 times from optimal reward.""" + reward_losses = np.subtract(optimal_rewards, rewards) # always positive + if normalized: + reward_losses = reward_losses / np.abs(optimal_rewards) \ + if np.count_nonzero(optimal_rewards) == optimal_rewards.size else reward_losses + means = np.mean(reward_losses) + return float(means) + + def _apply_best_action(self, inds: Sequence[Individual]) -> TrajectoryStep: + """Returns greedily optimal mutation for given graph and associated reward.""" + candidates = [] + for ind in inds: + for mutation_id in self.agent.available_actions: + try: + values = self._apply_action(mutation_id, ind) + candidates.append(values) + except Exception as e: + self._log.warning(f'Eval error for mutation <{mutation_id}> ' + f'on graph: {ind.graph.descriptive_id}:\n{e}') + continue + best_step = max(candidates, key=lambda step: step[-1]) + return best_step + + def _apply_action(self, action: Any, ind: Individual) -> TrajectoryStep: + new_graph, applied = self.mutation._adapt_and_apply_mutation(ind.graph, action) + fitness = self._eval_objective(new_graph) if applied else None + parent_op = ParentOperator(type_='mutation', operators=applied, parent_individuals=ind) + new_ind = Individual(new_graph, fitness=fitness, parent_operator=parent_op) + + prev_fitness = ind.fitness or self._eval_objective(ind.graph) + if prev_fitness and fitness: + reward = prev_fitness.value - fitness.value + elif prev_fitness and not fitness: + reward = -1. + else: + reward = 0. + return new_ind, action, reward + + def _eval_objective(self, graph: Graph) -> Fitness: + return self._adapter.adapt_func(self.objective)(graph) + + def _make_action_step(self, ind: Individual) -> TrajectoryStep: + action = self.agent.choose_action(ind.graph) + return self._apply_action(action, ind) + + def _sample_trajectory(self, initial: Individual, length: int) -> GraphTrajectory: + trajectory = [] + past_ind = initial + for i in range(length): + next_ind, action, reward = self._make_action_step(past_ind) + trajectory.append((next_ind, action, reward)) + past_ind = next_ind + return trajectory + + +def concat_lists(lists: Iterable[List]) -> List: + return reduce(operator.add, lists, []) diff --git a/golem/core/optimisers/adaptive/common_types.py b/golem/core/optimisers/adaptive/common_types.py new file mode 100644 index 000000000..528ebcc84 --- /dev/null +++ b/golem/core/optimisers/adaptive/common_types.py @@ -0,0 +1,11 @@ +from typing import Union, Hashable, Tuple, Sequence + +from golem.core.dag.graph import Graph +from golem.core.optimisers.opt_history_objects.individual import Individual + +ObsType = Union[Individual, Graph] +ActType = Hashable +# Trajectory step includes (past observation, action, reward) +TrajectoryStep = Tuple[Individual, ActType, float] +# Trajectory is a sequence of applied mutations and received rewards +GraphTrajectory = Sequence[TrajectoryStep] diff --git a/golem/core/optimisers/adaptive/experience_buffer.py b/golem/core/optimisers/adaptive/experience_buffer.py new file mode 100644 index 000000000..6b4852dd1 --- /dev/null +++ b/golem/core/optimisers/adaptive/experience_buffer.py @@ -0,0 +1,142 @@ +from collections import deque +from typing import List, Iterable, Tuple, Optional + +import numpy as np + +from golem.core.optimisers.adaptive.common_types import ObsType, ActType, TrajectoryStep, GraphTrajectory +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.optimisers.opt_history_objects.opt_history import OptHistory + + +class ExperienceBuffer: + """Buffer for learning experience of ``OperatorAgent``. + Keeps (State, Action, Reward) lists until retrieval.""" + + def __init__(self, window_size: Optional[int] = None, inds=None, actions=None, rewards=None): + self.window_size = window_size + self._prev_pop = set() + self._next_pop = set() + + if inds and not (len(inds) == len(actions) == len(rewards)): + raise ValueError('lengths of buffers do not match') + self._individuals = deque(inds) if inds else deque(maxlen=self.window_size) + self._actions = deque(actions) if actions else deque(maxlen=self.window_size) + self._rewards = deque(rewards) if rewards else deque(maxlen=self.window_size) + + @staticmethod + def from_history(history: OptHistory) -> 'ExperienceBuffer': + exp = ExperienceBuffer() + exp.collect_history(history) + return exp + + def _reset(self): + self._prev_pop = set() + self._next_pop = set() + + # if window size was not specified than there is no need to store these values for reuse. + # Otherwise, if the window_size is specified, then storages will be updated automatically in queue + if self.window_size is None: + self._individuals = deque(maxlen=self.window_size) + self._actions = deque(maxlen=self.window_size) + self._rewards = deque(maxlen=self.window_size) + + @staticmethod + def unroll_action_step(result: Individual) -> TrajectoryStep: + """Unrolls individual's history to get its source individual, action and resulting reward.""" + if not result.parent_operator or result.parent_operator.type_ != 'mutation': + return None, None, np.nan + source_ind = result.parent_operator.parent_individuals[0] + action = result.parent_operator.operators[0] + # we're minimising the fitness, that's why less is better + reward = (source_ind.fitness.value - result.fitness.value) / abs(source_ind.fitness.value)\ + if source_ind.fitness and source_ind.fitness.value != 0. else 0. + return source_ind, action, reward + + @staticmethod + def unroll_trajectories(history: OptHistory) -> List[GraphTrajectory]: + """Iterates through history and find continuous sequences of applied operator actions.""" + trajectories = [] + seen_uids = set() + for terminal_individual in history.final_choices: + trajectory = [] + next_ind = terminal_individual + while True: + seen_uids.add(next_ind.uid) + source_ind, action, reward = ExperienceBuffer.unroll_action_step(next_ind) + if source_ind is None or source_ind.uid in seen_uids: + break + # prepend step to keep historical direction + trajectory.insert(0, (source_ind, action, reward)) + next_ind = source_ind + trajectories.append(trajectory) + return trajectories + + def collect_history(self, history: OptHistory): + seen = set() + # We don't need the initial assumptions, as they have no parent operators, hence [1:] + for generation in history.generations[1:]: + for ind in generation: + if ind.uid not in seen: + seen.add(ind.uid) + self.collect_result(ind) + + def collect_results(self, results: Iterable[Individual]): + for ind in results: + self.collect_result(ind) + + def collect_result(self, result: Individual): + if result.uid in self._prev_pop: + # avoid collecting results from individuals that didn't change + return + self._next_pop.add(result.uid) + + source_ind, action, reward = self.unroll_action_step(result) + if action is None: + return + self.collect_experience(source_ind, action, reward) + + def collect_experience(self, obs: Individual, action: ActType, reward: float): + self._individuals.append(obs) + self._actions.append(action) + self._rewards.append(reward) + + def retrieve_experience(self, as_graphs: bool = True) -> Tuple[List[ObsType], List[ActType], List[float]]: + """Get all collected experience and clear the experience buffer. + Args: + as_graphs: if True (by default) returns observations as graphs, otherwise as individuals. + Return: + Unzipped trajectories (tuple of lists of observations, actions, rewards). + """ + individuals, actions, rewards = self._individuals, self._actions, self._rewards + observations = [ind.graph for ind in individuals] if as_graphs else individuals + next_pop = self._next_pop + self._reset() + self._prev_pop = next_pop + return list(observations), list(actions), list(rewards) + + def retrieve_trajectories(self) -> GraphTrajectory: + """Same as `retrieve_experience` but in the form of zipped trajectories that consist from steps.""" + trajectories = list(zip(*self.retrieve_experience(as_graphs=False))) + return trajectories + + def split(self, ratio: float = 0.8, shuffle: bool = False + ) -> Tuple['ExperienceBuffer', 'ExperienceBuffer']: + """Splits buffer in 2 parts, useful for train/validation split.""" + mask_train = np.full_like(self._individuals, False, dtype=bool) + num_train = int(len(self._individuals) * ratio) + mask_train[-num_train:] = True + if shuffle: + np.random.default_rng().shuffle(mask_train) + buffer_train = ExperienceBuffer(inds=np.array(self._individuals)[mask_train].tolist(), + actions=np.array(self._actions)[mask_train].tolist(), + rewards=np.array(self._rewards)[mask_train].tolist()) + buffer_val = ExperienceBuffer(inds=np.array(self._individuals)[~mask_train].tolist(), + actions=np.array(self._actions)[~mask_train].tolist(), + rewards=np.array(self._rewards)[~mask_train].tolist()) + return buffer_train, buffer_val + + def __len__(self): + return len(self._individuals) + + def __str__(self): + return f'{self.__class__.__name__}({len(self)})' diff --git a/golem/core/optimisers/adaptive/history_collector.py b/golem/core/optimisers/adaptive/history_collector.py new file mode 100644 index 000000000..b33c2197c --- /dev/null +++ b/golem/core/optimisers/adaptive/history_collector.py @@ -0,0 +1,42 @@ +import os +from pathlib import Path +from typing import Optional, Iterable + +from golem.core.log import default_log +from golem.core.optimisers.opt_history_objects.opt_history import OptHistory + + +class HistoryReader: + """Simplifies reading a bunch of histories from single directory.""" + + def __init__(self, save_path: Optional[Path] = None): + self.log = default_log(self) + self.save_path = save_path or Path("results") + self.save_path.mkdir(parents=True, exist_ok=True) + + def load_histories(self) -> Iterable[OptHistory]: + """Iteratively loads saved histories one-by-ony.""" + num_histories = 0 + total_individuals = 0 + for history_path in HistoryReader.traverse_histories(self.save_path): + history = OptHistory.load(history_path) + num_histories += 1 + total_individuals += sum(map(len, history.generations)) + yield history + + if num_histories == 0 or total_individuals == 0: + raise ValueError(f'Could not load any individuals.' + f'Possibly, path {self.save_path} does not exist or is empty.') + else: + self.log.info(f'Loaded {num_histories} histories ' + f'with {total_individuals} individuals in total.') + + @staticmethod + def traverse_histories(path) -> Iterable[Path]: + if path.exists(): + # recursive traversal of the save directory + for root, dirs, files in os.walk(path): + for history_filename in files: + if history_filename.startswith('history'): + full_path = Path(root) / history_filename + yield full_path diff --git a/golem/core/optimisers/adaptive/mab_agents/contextual_mab_agent.py b/golem/core/optimisers/adaptive/mab_agents/contextual_mab_agent.py index 01abda14c..ccd66480d 100644 --- a/golem/core/optimisers/adaptive/mab_agents/contextual_mab_agent.py +++ b/golem/core/optimisers/adaptive/mab_agents/contextual_mab_agent.py @@ -82,6 +82,14 @@ def partial_fit(self, experience: ExperienceBuffer): contexts = self.get_context(obs=obs) self._agent.partial_fit(decisions=arms, rewards=processed_rewards, contexts=contexts) + def _get_experience(self, experience: ExperienceBuffer): + """ Get experience from ExperienceBuffer, process rewards and log. """ + obs, actions, rewards = experience.retrieve_experience() + arms = [self._arm_by_action[action] for action in actions] + # there is no need to process rewards as in MAB, since this processing unifies rewards for all contexts + self._dbg_log(obs, actions, rewards) + return obs, arms, rewards + def get_context(self, obs: Union[List[ObsType], ObsType]) -> np.array: """ Returns contexts based on specified context agent. """ if not isinstance(obs, list): diff --git a/golem/core/optimisers/adaptive/mab_agents/mab_agent.py b/golem/core/optimisers/adaptive/mab_agents/mab_agent.py index a07c75e05..538c2ed2f 100644 --- a/golem/core/optimisers/adaptive/mab_agents/mab_agent.py +++ b/golem/core/optimisers/adaptive/mab_agents/mab_agent.py @@ -22,12 +22,12 @@ def __init__(self, decaying_factor: float = 1.0, path_to_save: Optional[str] = None, is_initial_fit: bool = True): - super().__init__(enable_logging) + super().__init__(actions=actions, enable_logging=enable_logging) self.actions = list(actions) self._indices = list(range(len(actions))) self._arm_by_action = dict(zip(actions, self._indices)) self._agent = MAB(arms=self._indices, - learning_policy=LearningPolicy.UCB1(alpha=1.25), + learning_policy=LearningPolicy.EpsilonGreedy(epsilon=0.4), n_jobs=n_jobs) self._reward_agent = FitnessRateRankRewardTransformer(decaying_factor=decaying_factor) if is_initial_fit: diff --git a/golem/core/optimisers/adaptive/mab_agents/neural_contextual_mab_agent.py b/golem/core/optimisers/adaptive/mab_agents/neural_contextual_mab_agent.py index ad7510319..1265ca6ac 100644 --- a/golem/core/optimisers/adaptive/mab_agents/neural_contextual_mab_agent.py +++ b/golem/core/optimisers/adaptive/mab_agents/neural_contextual_mab_agent.py @@ -3,7 +3,7 @@ from golem.core.optimisers.adaptive.mab_agents.contextual_mab_agent import ContextualMultiArmedBanditAgent from golem.core.optimisers.adaptive.neural_mab import NeuralMAB from golem.core.optimisers.adaptive.context_agents import ContextAgentTypeEnum -from golem.core.optimisers.adaptive.operator_agent import ActType +from golem.core.optimisers.adaptive.common_types import ActType class NeuralContextualMultiArmedBanditAgent(ContextualMultiArmedBanditAgent): diff --git a/golem/core/optimisers/adaptive/operator_agent.py b/golem/core/optimisers/adaptive/operator_agent.py index ac3c906e5..7b1034504 100644 --- a/golem/core/optimisers/adaptive/operator_agent.py +++ b/golem/core/optimisers/adaptive/operator_agent.py @@ -1,19 +1,15 @@ import random from abc import ABC, abstractmethod -from collections import deque from enum import Enum -from typing import Union, Sequence, Hashable, Tuple, Optional, List +from typing import Union, Sequence, Optional import numpy as np from golem.core.dag.graph import Graph from golem.core.dag.graph_node import GraphNode from golem.core.log import default_log -from golem.core.optimisers.genetic.operators.base_mutations import MutationTypesEnum -from golem.core.optimisers.opt_history_objects.individual import Individual - -ObsType = Graph -ActType = Hashable +from golem.core.optimisers.adaptive.common_types import ObsType, ActType +from golem.core.optimisers.adaptive.experience_buffer import ExperienceBuffer class MutationAgentTypeEnum(Enum): @@ -24,77 +20,16 @@ class MutationAgentTypeEnum(Enum): neural_bandit = 'neural_bandit' -class ExperienceBuffer: - """ - Buffer for learning experience of ``OperatorAgent``. - Keeps (State, Action, Reward) lists until retrieval. - Can be used with window_size for actualizing experience. - """ - - def __init__(self, window_size: Optional[int] = None): - self.window_size = window_size - self._reset_main_storages() - self.reset() - - def reset(self): - self._current_observations = [] - self._current_actions = [] - self._current_rewards = [] - self._prev_pop = set() - self._next_pop = set() - - # if window size was not specified than there is no need to store these values for reuse - if self.window_size is None: - self._reset_main_storages() - - def _reset_main_storages(self): - self._observations = deque(maxlen=self.window_size) - self._actions = deque(maxlen=self.window_size) - self._rewards = deque(maxlen=self.window_size) - - def collect_results(self, results: Sequence[Individual]): - for ind in results: - self.collect_result(ind) - self._observations += self._current_observations - self._actions += self._current_actions - self._rewards += self._current_rewards - - def collect_result(self, result: Individual): - if result.uid in self._prev_pop: - return - if not result.parent_operator or result.parent_operator.type_ != 'mutation': - return - self._next_pop.add(result.uid) - obs = result.graph - action = result.parent_operator.operators[0] - prev_fitness = result.parent_operator.parent_individuals[0].fitness.value - # we're minimising the fitness, that's why less is better - # reward is defined as fitness improvement rate (FIR) to stabilize the algorithm - reward = (prev_fitness - result.fitness.value) / abs(prev_fitness) \ - if prev_fitness is not None and prev_fitness != 0 else 0. - self.collect_experience(obs, action, reward) - - def collect_experience(self, obs: ObsType, action: ActType, reward: float): - self._current_observations.append(obs) - self._current_actions.append(action) - self._current_rewards.append(reward) - - def retrieve_experience(self) -> Tuple[List[ObsType], List[ActType], List[float]]: - """Get all collected experience and clear the experience buffer.""" - observations, actions, rewards = self._observations, self._actions, self._rewards - next_pop = self._next_pop - self.reset() - self._prev_pop = next_pop - return list(observations), \ - list(actions), \ - list(rewards) - - class OperatorAgent(ABC): - def __init__(self, enable_logging: bool = True): + def __init__(self, actions: Sequence[ActType], enable_logging: bool = True): + self.actions = list(actions) self._enable_logging = enable_logging self._log = default_log(self) + @property + def available_actions(self) -> Sequence[ActType]: + return self.actions + @abstractmethod def partial_fit(self, experience: ExperienceBuffer): raise NotImplementedError() @@ -104,7 +39,7 @@ def choose_action(self, obs: Optional[ObsType]) -> ActType: raise NotImplementedError() @abstractmethod - def choose_nodes(self, graph: Graph, num_nodes: int = 1) -> Union[GraphNode, Sequence[GraphNode]]: + def choose_nodes(self, graph: ObsType, num_nodes: int = 1) -> Union[GraphNode, Sequence[GraphNode]]: raise NotImplementedError() @abstractmethod @@ -122,14 +57,11 @@ def _dbg_log(self, obs, actions, rewards): nonzero = rr[rr.nonzero()] msg = f'len={len(rr)} nonzero={len(nonzero)} ' if len(nonzero) > 0: - msg += (f'avg={nonzero.mean()} std={nonzero.std()} ' - f'min={nonzero.min()} max={nonzero.max()} ') + msg += (f'avg={nonzero.mean():.3f} std={nonzero.std():.3f} ' + f'min={nonzero.min():.3f} max={nonzero.max():.3f} ') self._log.info(msg) - - actions_names = [action.name if isinstance(action, MutationTypesEnum) else action.__name__ - for action in actions] - self._log.info(f'actions/rewards: {list(zip(actions_names, rr))}') + self._log.info(f'actions/rewards: {list(zip(actions, rr))}') action_values = list(map(self.get_action_values, obs)) action_probs = list(map(self.get_action_probs, obs)) @@ -145,11 +77,10 @@ def __init__(self, actions: Sequence[ActType], probs: Optional[Sequence[float]] = None, enable_logging: bool = True): - self.actions = list(actions) + super().__init__(actions, enable_logging) self._probs = probs or [1. / len(actions)] * len(actions) - super().__init__(enable_logging) - def choose_action(self, obs: ObsType) -> ActType: + def choose_action(self, obs: Graph) -> ActType: action = np.random.choice(self.actions, p=self.get_action_probs(obs)) return action @@ -161,8 +92,8 @@ def partial_fit(self, experience: ExperienceBuffer): obs, actions, rewards = experience.retrieve_experience() self._dbg_log(obs, actions, rewards) - def get_action_probs(self, obs: Optional[ObsType] = None) -> Sequence[float]: + def get_action_probs(self, obs: Optional[Graph] = None) -> Sequence[float]: return self._probs - def get_action_values(self, obs: Optional[ObsType] = None) -> Sequence[float]: + def get_action_values(self, obs: Optional[Graph] = None) -> Sequence[float]: return self._probs diff --git a/golem/core/optimisers/adaptive/reward_agent.py b/golem/core/optimisers/adaptive/reward_agent.py index f5677c4b3..08f17dce7 100644 --- a/golem/core/optimisers/adaptive/reward_agent.py +++ b/golem/core/optimisers/adaptive/reward_agent.py @@ -1,9 +1,5 @@ from typing import List, Tuple -from numpy import sign - -from golem.core.optimisers.adaptive.operator_agent import ObsType - class FitnessRateRankRewardTransformer: """ @@ -32,4 +28,4 @@ def get_decay_values_for_arms(self, rewards: List[float], arms: List[int]) -> Tu def get_fitness_rank_rate(decay_values: List[float]) -> List[float]: # abs() is used to save the initial sign of each decay value total_decay_sum = abs(sum(decay_values)) - return [decay / total_decay_sum for decay in decay_values] if total_decay_sum != 0 else [0.] + return [decay / total_decay_sum for decay in decay_values] if total_decay_sum != 0 else [0.]*len(decay_values) diff --git a/golem/core/optimisers/advisor.py b/golem/core/optimisers/advisor.py index a7f50624a..f53d1602d 100644 --- a/golem/core/optimisers/advisor.py +++ b/golem/core/optimisers/advisor.py @@ -1,6 +1,6 @@ from typing import List, Any, TypeVar, Generic -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum NodeType = TypeVar('NodeType') diff --git a/golem/core/optimisers/dynamic_graph_requirements.py b/golem/core/optimisers/dynamic_graph_requirements.py new file mode 100644 index 000000000..67145dadf --- /dev/null +++ b/golem/core/optimisers/dynamic_graph_requirements.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass + +from golem.core.optimisers.optimization_parameters import GraphRequirements + + +@dataclass +class DynamicGraphRequirements(GraphRequirements): + """ Class for using custom domain specific graph requirements. """ + def __init__(self, attributes: dict): + for attribute, value in attributes.items(): + setattr(self, attribute, value) diff --git a/golem/core/optimisers/fitness/fitness.py b/golem/core/optimisers/fitness/fitness.py index 7b2310004..eeb2b3f0f 100644 --- a/golem/core/optimisers/fitness/fitness.py +++ b/golem/core/optimisers/fitness/fitness.py @@ -3,7 +3,7 @@ import numpy as np -from golem.core.utilities.data_structures import Comparable +from golem.utilities.data_structures import Comparable class Fitness(Comparable): @@ -102,6 +102,9 @@ def __eq__(self, other: 'Fitness') -> bool: self.valid and other.valid and self.allclose(self.values, other.values)) + def __bool__(self) -> bool: + return self.valid + @staticmethod def allclose(values1, values2) -> bool: return np.allclose(values1, values2, rtol=1e-8, atol=1e-10) diff --git a/golem/core/optimisers/genetic/evaluation.py b/golem/core/optimisers/genetic/evaluation.py index f3459473e..14ee73f82 100644 --- a/golem/core/optimisers/genetic/evaluation.py +++ b/golem/core/optimisers/genetic/evaluation.py @@ -7,7 +7,7 @@ from functools import partial from typing import List, Optional, Sequence, Tuple, TypeVar, Dict -from joblib import Parallel, cpu_count, delayed +from joblib import Parallel, delayed from golem.core.adapter import BaseOptimizationAdapter from golem.core.dag.graph import Graph @@ -18,15 +18,15 @@ from golem.core.optimisers.objective import GraphFunction, ObjectiveFunction from golem.core.optimisers.opt_history_objects.individual import GraphEvalResult from golem.core.optimisers.timer import Timer, get_forever_timer -from golem.core.utilities.serializable import Serializable +from golem.utilities.serializable import Serializable from golem.utilities.memory import MemoryAnalytics +from golem.utilities.utilities import determine_n_jobs # the percentage of successful evaluations, # at which evolution is not threatened with stagnation at the moment STAGNATION_EVALUATION_PERCENTAGE = 0.5 -OptionalEvalResult = Optional[GraphEvalResult] -EvalResultsList = List[OptionalEvalResult] +EvalResultsList = List[GraphEvalResult] G = TypeVar('G', bound=Serializable) @@ -153,12 +153,12 @@ def population_evaluation_info(self, pop_size: int, evaluated_pop_size: int): f"were evaluated successfully.") @abstractmethod - def evaluate_population(self, individuals: PopulationT) -> Optional[PopulationT]: + def evaluate_population(self, individuals: PopulationT) -> PopulationT: raise NotImplementedError() def evaluate_single(self, graph: OptGraph, uid_of_individual: str, with_time_limit: bool = True, cache_key: Optional[str] = None, - logs_initializer: Optional[Tuple[int, pathlib.Path]] = None) -> OptionalEvalResult: + logs_initializer: Optional[Tuple[int, pathlib.Path]] = None) -> GraphEvalResult: graph = self.evaluation_cache.get(cache_key, graph) @@ -193,7 +193,7 @@ def _evaluate_graph(self, domain_graph: Graph) -> Tuple[Fitness, Graph]: return fitness, domain_graph - def evaluate_with_cache(self, population: PopulationT) -> Optional[PopulationT]: + def evaluate_with_cache(self, population: PopulationT) -> PopulationT: reversed_population = list(reversed(population)) self._remote_compute_cache(reversed_population) evaluated_population = self.evaluate_population(reversed_population) @@ -239,7 +239,7 @@ def dispatch(self, objective: ObjectiveFunction, timer: Optional[Timer] = None) super().dispatch(objective, timer) return self.evaluate_with_cache - def evaluate_population(self, individuals: PopulationT) -> Optional[PopulationT]: + def evaluate_population(self, individuals: PopulationT) -> PopulationT: individuals_to_evaluate, individuals_to_skip = self.split_individuals_to_evaluate(individuals) # Evaluate individuals without valid fitness in parallel. n_jobs = determine_n_jobs(self._n_jobs, self.logger) @@ -256,7 +256,7 @@ def evaluate_population(self, individuals: PopulationT) -> Optional[PopulationT] if not successful_evals: for single_ind in individuals: evaluation_result = eval_func(single_ind.graph, single_ind.uid, with_time_limit=False) - successful_evals = self.apply_evaluation_results([single_ind], [evaluation_result]) or None + successful_evals = self.apply_evaluation_results([single_ind], [evaluation_result]) if successful_evals: break MemoryAnalytics.log(self.logger, @@ -271,23 +271,9 @@ class SequentialDispatcher(BaseGraphEvaluationDispatcher): Usage: call `dispatch(objective_function)` to get evaluation function. """ - def evaluate_population(self, individuals: PopulationT) -> Optional[PopulationT]: + def evaluate_population(self, individuals: PopulationT) -> PopulationT: individuals_to_evaluate, individuals_to_skip = self.split_individuals_to_evaluate(individuals) evaluation_results = [self.evaluate_single(ind.graph, ind.uid) for ind in individuals_to_evaluate] individuals_evaluated = self.apply_evaluation_results(individuals_to_evaluate, evaluation_results) - evaluated_population = individuals_evaluated + individuals_to_skip or None + evaluated_population = individuals_evaluated + individuals_to_skip return evaluated_population - - -def determine_n_jobs(n_jobs=-1, logger=None): - cpu_num = cpu_count() - if n_jobs > cpu_num: - n_jobs = cpu_num - elif n_jobs <= 0: - if n_jobs <= -cpu_num - 1 or n_jobs == 0: - raise ValueError(f"Unproper `n_jobs` = {n_jobs}. " - f"`n_jobs` should be between ({-cpu_num}, {cpu_num}) except 0") - n_jobs = cpu_num + 1 + n_jobs - if logger: - logger.info(f"Number of used CPU's: {n_jobs}") - return n_jobs diff --git a/golem/core/optimisers/genetic/gp_optimizer.py b/golem/core/optimisers/genetic/gp_optimizer.py index eec6e4ae1..36bc1db00 100644 --- a/golem/core/optimisers/genetic/gp_optimizer.py +++ b/golem/core/optimisers/genetic/gp_optimizer.py @@ -33,8 +33,11 @@ def __init__(self, initial_graphs: Sequence[Union[Graph, Any]], requirements: GraphRequirements, graph_generation_params: GraphGenerationParams, - graph_optimizer_params: GPAlgorithmParameters): - super().__init__(objective, initial_graphs, requirements, graph_generation_params, graph_optimizer_params) + graph_optimizer_params: GPAlgorithmParameters, + **custom_optimizer_params + ): + super().__init__(objective, initial_graphs, requirements, + graph_generation_params, graph_optimizer_params, **custom_optimizer_params) # Define genetic operators self.regularization = Regularization(graph_optimizer_params, graph_generation_params) self.selection = Selection(graph_optimizer_params) diff --git a/golem/core/optimisers/genetic/gp_params.py b/golem/core/optimisers/genetic/gp_params.py index 7579dc965..b977fec89 100644 --- a/golem/core/optimisers/genetic/gp_params.py +++ b/golem/core/optimisers/genetic/gp_params.py @@ -83,8 +83,7 @@ class GPAlgorithmParameters(AlgorithmParameters): adaptive_mutation_type: MutationAgentTypeEnum = MutationAgentTypeEnum.default context_agent_type: Union[ContextAgentTypeEnum, Callable] = ContextAgentTypeEnum.nodes_num - selection_types: Sequence[SelectionTypesEnum] = \ - (SelectionTypesEnum.tournament,) + selection_types: Optional[Sequence[Union[SelectionTypesEnum, Any]]] = None crossover_types: Sequence[Union[CrossoverTypesEnum, Any]] = \ (CrossoverTypesEnum.one_point,) mutation_types: Sequence[Union[MutationTypesEnum, Any]] = simple_mutation_set @@ -96,7 +95,9 @@ class GPAlgorithmParameters(AlgorithmParameters): window_size: Optional[int] = None def __post_init__(self): + if not self.selection_types: + self.selection_types = (SelectionTypesEnum.spea2,) if self.multi_objective \ + else (SelectionTypesEnum.tournament,) if self.multi_objective: - self.selection_types = (SelectionTypesEnum.spea2,) # TODO add possibility of using regularization in MO alg self.regularization_type = RegularizationTypesEnum.none diff --git a/golem/core/optimisers/genetic/operators/base_mutations.py b/golem/core/optimisers/genetic/operators/base_mutations.py index 6982dd725..1677359fe 100644 --- a/golem/core/optimisers/genetic/operators/base_mutations.py +++ b/golem/core/optimisers/genetic/operators/base_mutations.py @@ -1,7 +1,10 @@ +from copy import deepcopy from functools import partial -from random import choice, randint, random, sample +from random import choice, randint, random, sample, shuffle from typing import TYPE_CHECKING, Optional +import numpy as np + from golem.core.adapter import register_native from golem.core.dag.graph import ReconnectType from golem.core.dag.graph_node import GraphNode @@ -11,7 +14,7 @@ from golem.core.optimisers.opt_node_factory import OptNodeFactory from golem.core.optimisers.optimization_parameters import GraphRequirements from golem.core.optimisers.optimizer import GraphGenerationParams, AlgorithmParameters -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum if TYPE_CHECKING: from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters @@ -69,7 +72,6 @@ def simple_mutation(graph: OptGraph, :param graph: graph to mutate """ - exchange_node = graph_gen_params.node_factory.exchange_node visited_nodes = set() @@ -138,56 +140,67 @@ def nodes_not_cycling(source_node: OptNode, target_node: OptNode): @register_native def add_intermediate_node(graph: OptGraph, - node_to_mutate: OptNode, node_factory: OptNodeFactory) -> OptGraph: - # add between node and parent - new_node = node_factory.get_parent_node(node_to_mutate, is_primary=False) - if not new_node: - return graph - - # rewire old children to new parent - new_node.nodes_from = node_to_mutate.nodes_from - node_to_mutate.nodes_from = [new_node] - - # add new node to graph - graph.add_node(new_node) + nodes_with_parents = [node for node in graph.nodes if node.nodes_from] + if len(nodes_with_parents) > 0: + shuffle(nodes_with_parents) + for node_to_mutate in nodes_with_parents: + # add between node and parent + new_node = node_factory.get_parent_node(node_to_mutate, is_primary=False) + if not new_node: + continue + + # rewire old children to new parent + new_node.nodes_from = node_to_mutate.nodes_from + node_to_mutate.nodes_from = [new_node] + + # add new node to graph + graph.add_node(new_node) + break return graph @register_native def add_separate_parent_node(graph: OptGraph, - node_to_mutate: OptNode, node_factory: OptNodeFactory) -> OptGraph: - # add as separate parent - new_node = node_factory.get_parent_node(node_to_mutate, is_primary=True) - if not new_node: - # there is no possible operators - return graph - if node_to_mutate.nodes_from: - node_to_mutate.nodes_from.append(new_node) - else: - node_to_mutate.nodes_from = [new_node] - graph.nodes.append(new_node) + node_idx = np.arange(len(graph.nodes)) + shuffle(node_idx) + for idx in node_idx: + node_to_mutate = graph.nodes[idx] + # add as separate parent + new_node = node_factory.get_parent_node(node_to_mutate, is_primary=True) + if not new_node: + # there is no possible operators + continue + if node_to_mutate.nodes_from: + node_to_mutate.nodes_from.append(new_node) + else: + node_to_mutate.nodes_from = [new_node] + graph.nodes.append(new_node) + break return graph @register_native def add_as_child(graph: OptGraph, - node_to_mutate: OptNode, node_factory: OptNodeFactory) -> OptGraph: - # add as child - old_node_children = graph.node_children(node_to_mutate) - new_node_child = choice(old_node_children) if old_node_children else None - new_node = node_factory.get_node(is_primary=False) - if not new_node: - return graph - graph.add_node(new_node) - graph.connect_nodes(node_parent=node_to_mutate, node_child=new_node) - if new_node_child: - graph.connect_nodes(node_parent=new_node, node_child=new_node_child) - graph.disconnect_nodes(node_parent=node_to_mutate, node_child=new_node_child, - clean_up_leftovers=True) - + node_idx = np.arange(len(graph.nodes)) + shuffle(node_idx) + for idx in node_idx: + node_to_mutate = graph.nodes[idx] + # add as child + old_node_children = graph.node_children(node_to_mutate) + new_node_child = choice(old_node_children) if old_node_children else None + new_node = node_factory.get_node(is_primary=False) + if not new_node: + continue + graph.add_node(new_node) + graph.connect_nodes(node_parent=node_to_mutate, node_child=new_node) + if new_node_child: + graph.connect_nodes(node_parent=new_node, node_child=new_node_child) + graph.disconnect_nodes(node_parent=node_to_mutate, node_child=new_node_child, + clean_up_leftovers=True) + break return graph @@ -202,20 +215,20 @@ def single_add_mutation(graph: OptGraph, :param graph: graph to mutate """ - if graph.depth >= requirements.max_depth: # add mutation is not possible return graph - node_to_mutate = choice(graph.nodes) - - single_add_strategies = [add_as_child, add_separate_parent_node] - if node_to_mutate.nodes_from: - single_add_strategies.append(add_intermediate_node) - strategy = choice(single_add_strategies) - - result = strategy(graph, node_to_mutate, graph_gen_params.node_factory) - return result + new_graph = deepcopy(graph) + single_add_strategies = [add_as_child, add_separate_parent_node, add_intermediate_node] + shuffle(single_add_strategies) + for strategy in single_add_strategies: + new_graph = strategy(new_graph, graph_gen_params.node_factory) + # maximum three equality check + if new_graph == graph: + continue + break + return new_graph @register_native @@ -229,11 +242,15 @@ def single_change_mutation(graph: OptGraph, :param graph: graph to mutate """ - node = choice(graph.nodes) - new_node = graph_gen_params.node_factory.exchange_node(node) - if not new_node: - return graph - graph.update_node(node, new_node) + node_idx = np.arange(len(graph.nodes)) + shuffle(node_idx) + for idx in node_idx: + node = graph.nodes[idx] + new_node = graph_gen_params.node_factory.exchange_node(node) + if not new_node: + continue + graph.update_node(node, new_node) + break return graph @@ -289,22 +306,27 @@ def tree_growth(graph: OptGraph, selected random node, if false then previous depth of selected node doesn't affect to new subtree depth, maximal depth of new subtree just should satisfy depth constraint in parent tree """ - node_from_graph = choice(graph.nodes) - if local_growth: - max_depth = distance_to_primary_level(node_from_graph) - is_primary_node_selected = (not node_from_graph.nodes_from) or (node_from_graph != graph.root_node and - randint(0, 1)) - else: - max_depth = requirements.max_depth - distance_to_root_level(graph, node_from_graph) - is_primary_node_selected = \ - distance_to_root_level(graph, node_from_graph) >= requirements.max_depth and randint(0, 1) - if is_primary_node_selected: - new_subtree = graph_gen_params.node_factory.get_node(is_primary=True) - if not new_subtree: - return graph - else: - new_subtree = graph_gen_params.random_graph_factory(requirements, max_depth).root_node - graph.update_subtree(node_from_graph, new_subtree) + node_idx = np.arange(len(graph.nodes)) + shuffle(node_idx) + for idx in node_idx: + node_from_graph = graph.nodes[idx] + if local_growth: + max_depth = distance_to_primary_level(node_from_graph) + is_primary_node_selected = (not node_from_graph.nodes_from) or (node_from_graph != graph.root_node and + randint(0, 1)) + else: + max_depth = requirements.max_depth - distance_to_root_level(graph, node_from_graph) + is_primary_node_selected = \ + distance_to_root_level(graph, node_from_graph) >= requirements.max_depth and randint(0, 1) + if is_primary_node_selected: + new_subtree = graph_gen_params.node_factory.get_node(is_primary=True) + if not new_subtree: + continue + else: + new_subtree = graph_gen_params.random_graph_factory(requirements, max_depth).root_node + + graph.update_subtree(node_from_graph, new_subtree) + break return graph @@ -349,16 +371,18 @@ def reduce_mutation(graph: OptGraph, return graph nodes = [node for node in graph.nodes if node is not graph.root_node] - node_to_del = choice(nodes) - children = graph.node_children(node_to_del) - is_possible_to_delete = all([len(child.nodes_from) - 1 >= requirements.min_arity for child in children]) - if is_possible_to_delete: - graph.delete_subtree(node_to_del) - else: - primary_node = graph_gen_params.node_factory.get_node(is_primary=True) - if not primary_node: - return graph - graph.update_subtree(node_to_del, primary_node) + shuffle(nodes) + for node_to_del in nodes: + children = graph.node_children(node_to_del) + is_possible_to_delete = all([len(child.nodes_from) - 1 >= requirements.min_arity for child in children]) + if is_possible_to_delete: + graph.delete_subtree(node_to_del) + else: + primary_node = graph_gen_params.node_factory.get_node(is_primary=True) + if not primary_node: + continue + graph.update_subtree(node_to_del, primary_node) + break return graph diff --git a/golem/core/optimisers/genetic/operators/crossover.py b/golem/core/optimisers/genetic/operators/crossover.py index 508623bfd..866b0d279 100644 --- a/golem/core/optimisers/genetic/operators/crossover.py +++ b/golem/core/optimisers/genetic/operators/crossover.py @@ -13,7 +13,7 @@ from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator from golem.core.optimisers.optimization_parameters import GraphRequirements from golem.core.optimisers.optimizer import GraphGenerationParams -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum if TYPE_CHECKING: from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters diff --git a/golem/core/optimisers/genetic/operators/elitism.py b/golem/core/optimisers/genetic/operators/elitism.py index 6e4e4a052..e462659ae 100644 --- a/golem/core/optimisers/genetic/operators/elitism.py +++ b/golem/core/optimisers/genetic/operators/elitism.py @@ -1,7 +1,7 @@ from random import shuffle from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum class ElitismTypesEnum(Enum): diff --git a/golem/core/optimisers/genetic/operators/inheritance.py b/golem/core/optimisers/genetic/operators/inheritance.py index 7b64432bb..b0e8807f1 100644 --- a/golem/core/optimisers/genetic/operators/inheritance.py +++ b/golem/core/optimisers/genetic/operators/inheritance.py @@ -2,7 +2,7 @@ from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator from golem.core.optimisers.genetic.operators.selection import Selection -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum if TYPE_CHECKING: from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters diff --git a/golem/core/optimisers/genetic/operators/mutation.py b/golem/core/optimisers/genetic/operators/mutation.py index 2b1127299..8c52e7518 100644 --- a/golem/core/optimisers/genetic/operators/mutation.py +++ b/golem/core/optimisers/genetic/operators/mutation.py @@ -6,14 +6,14 @@ from golem.core.dag.graph import Graph from golem.core.optimisers.adaptive.mab_agents.contextual_mab_agent import ContextualMultiArmedBanditAgent -from golem.core.optimisers.adaptive.mab_agents.neural_contextual_mab_agent import NeuralContextualMultiArmedBanditAgent from golem.core.optimisers.adaptive.mab_agents.mab_agent import MultiArmedBanditAgent +from golem.core.optimisers.adaptive.mab_agents.neural_contextual_mab_agent import NeuralContextualMultiArmedBanditAgent from golem.core.optimisers.adaptive.operator_agent import \ - OperatorAgent, RandomAgent, ExperienceBuffer, MutationAgentTypeEnum + OperatorAgent, RandomAgent, MutationAgentTypeEnum +from golem.core.optimisers.adaptive.experience_buffer import ExperienceBuffer from golem.core.optimisers.genetic.operators.base_mutations import \ base_mutations_repo, MutationTypesEnum from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator -from golem.core.optimisers.graph import OptGraph from golem.core.optimisers.opt_history_objects.individual import Individual from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator from golem.core.optimisers.optimization_parameters import GraphRequirements, OptimizationParameters @@ -81,74 +81,59 @@ def __call__(self, population: Union[Individual, PopulationT]) -> Union[Individu if isinstance(population, Individual): population = [population] - final_population, mutations_applied, application_attempts = tuple(zip(*map(self._mutation, population))) + final_population, application_attempts = tuple(zip(*map(self._mutation, population))) # drop individuals to which mutations could not be applied final_population = [ind for ind, init_ind, attempt in zip(final_population, population, application_attempts) - if not attempt or ind.graph != init_ind.graph] + if not(attempt and ind.graph == init_ind.graph)] if len(population) == 1: return final_population[0] if final_population else final_population return final_population - def _mutation(self, individual: Individual) -> Tuple[Individual, Optional[MutationIdType], bool]: + def _mutation(self, individual: Individual) -> Tuple[Individual, bool]: """ Function applies mutation operator to graph """ - application_attempt = False - mutation_applied = None - for _ in range(self.parameters.max_num_of_operator_attempts): - new_graph = deepcopy(individual.graph) - - new_graph, mutation_applied = self._apply_mutations(new_graph) - if mutation_applied is None: - continue - application_attempt = True - is_correct_graph = self.graph_generation_params.verifier(new_graph) - if is_correct_graph: - parent_operator = ParentOperator(type_='mutation', - operators=mutation_applied, - parent_individuals=individual) - individual = Individual(new_graph, parent_operator, - metadata=self.requirements.static_individual_metadata) - break + mutation_type = self._operator_agent.choose_action(individual.graph) + is_applied = self._will_mutation_be_applied(mutation_type) + if is_applied: + for _ in range(self.parameters.max_num_of_operator_attempts): + new_graph = deepcopy(individual.graph) + + new_graph = self._apply_mutations(new_graph, mutation_type) + is_correct_graph = self.graph_generation_params.verifier(new_graph) + if is_correct_graph: + parent_operator = ParentOperator(type_='mutation', + operators=mutation_type, + parent_individuals=individual) + individual = Individual(new_graph, parent_operator, + metadata=self.requirements.static_individual_metadata) + break else: # Collect invalid actions - self.agent_experience.collect_experience(individual.graph, mutation_applied, reward=-1.0) - else: - self.log.debug('Number of mutation attempts exceeded. ' - 'Please check optimization parameters for correctness.') - return individual, mutation_applied, application_attempt + self.agent_experience.collect_experience(individual, mutation_type, reward=-1.0) - def _sample_num_of_mutations(self) -> int: + self.log.debug(f'Number of attempts for {mutation_type} mutation application exceeded. ' + 'Please check optimization parameters for correctness.') + return individual, is_applied + + def _sample_num_of_mutations(self, mutation_type: Union[MutationTypesEnum, Callable]) -> int: # most of the time returns 1 or rarely several mutations - if self.parameters.variable_mutation_num: + is_custom_mutation = isinstance(mutation_type, Callable) + if self.parameters.variable_mutation_num and not is_custom_mutation: num_mut = max(int(round(np.random.lognormal(0, sigma=0.5))), 1) else: num_mut = 1 return num_mut - def _apply_mutations(self, new_graph: OptGraph) -> Tuple[OptGraph, Optional[MutationIdType]]: + def _apply_mutations(self, new_graph: Graph, mutation_type: Union[MutationTypesEnum, Callable]) -> Graph: """Apply mutation 1 or few times iteratively""" - mutation_type = self._operator_agent.choose_action(new_graph) - mutation_applied = None - for _ in range(self._sample_num_of_mutations()): - new_graph, applied = self._adapt_and_apply_mutation(new_graph, mutation_type) - if applied: - mutation_applied = mutation_type - is_custom_mutation = isinstance(mutation_type, Callable) - if is_custom_mutation: # custom mutation occurs once - break - return new_graph, mutation_applied - - def _adapt_and_apply_mutation(self, new_graph: OptGraph, mutation_type) -> Tuple[OptGraph, bool]: - applied = self._will_mutation_be_applied(mutation_type) - if applied: - # get the mutation function and adapt it + for _ in range(self._sample_num_of_mutations(mutation_type)): mutation_func = self._get_mutation_func(mutation_type) new_graph = mutation_func(new_graph, requirements=self.requirements, graph_gen_params=self.graph_generation_params, parameters=self.parameters) - return new_graph, applied + return new_graph def _will_mutation_be_applied(self, mutation_type: Union[MutationTypesEnum, Callable]) -> bool: return random() <= self.parameters.mutation_prob and mutation_type is not MutationTypesEnum.none diff --git a/golem/core/optimisers/genetic/operators/regularization.py b/golem/core/optimisers/genetic/operators/regularization.py index 9b3d024bc..7d8e7dbe2 100644 --- a/golem/core/optimisers/genetic/operators/regularization.py +++ b/golem/core/optimisers/genetic/operators/regularization.py @@ -7,7 +7,7 @@ from golem.core.optimisers.opt_history_objects.individual import Individual from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator from golem.core.optimisers.optimizer import GraphGenerationParams -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum if TYPE_CHECKING: from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters diff --git a/golem/core/optimisers/genetic/operators/reproduction.py b/golem/core/optimisers/genetic/operators/reproduction.py index 60de62f37..bbaacde4c 100644 --- a/golem/core/optimisers/genetic/operators/reproduction.py +++ b/golem/core/optimisers/genetic/operators/reproduction.py @@ -10,7 +10,7 @@ from golem.core.optimisers.genetic.operators.operator import PopulationT, EvaluationOperator from golem.core.optimisers.genetic.operators.selection import Selection from golem.core.optimisers.populational_optimizer import EvaluationAttemptsError -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence class ReproductionController: diff --git a/golem/core/optimisers/genetic/operators/selection.py b/golem/core/optimisers/genetic/operators/selection.py index a06b234b0..c2f1ab729 100644 --- a/golem/core/optimisers/genetic/operators/selection.py +++ b/golem/core/optimisers/genetic/operators/selection.py @@ -5,7 +5,7 @@ from typing import Callable, List, Optional from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum class SelectionTypesEnum(Enum): @@ -33,6 +33,8 @@ def _selection_by_type(selection_type: SelectionTypesEnum) -> Callable[[Populati } if selection_type in selections: return selections[selection_type] + elif isinstance(selection_type, Callable): + return selection_type else: raise ValueError(f'Required selection not found: {selection_type}') diff --git a/golem/core/optimisers/genetic/parameters/mutation_prob.py b/golem/core/optimisers/genetic/parameters/mutation_prob.py index 2ea40c5ca..fc8a20961 100644 --- a/golem/core/optimisers/genetic/parameters/mutation_prob.py +++ b/golem/core/optimisers/genetic/parameters/mutation_prob.py @@ -9,7 +9,7 @@ class AdaptiveMutationProb(AdaptiveParameter[float]): def __init__(self, default_prob: float = 0.5): self._current_std = 0. self._max_std = 0. - self._min_proba = 0.05 + self._min_proba = 0.1 self._default_prob = default_prob @property diff --git a/golem/core/optimisers/genetic/parameters/population_size.py b/golem/core/optimisers/genetic/parameters/population_size.py index fbdebfd63..275f05812 100644 --- a/golem/core/optimisers/genetic/parameters/population_size.py +++ b/golem/core/optimisers/genetic/parameters/population_size.py @@ -7,8 +7,8 @@ from golem.core.optimisers.genetic.operators.inheritance import GeneticSchemeTypesEnum from golem.core.optimisers.genetic.operators.operator import PopulationT from golem.core.optimisers.genetic.parameters.parameter import AdaptiveParameter -from golem.core.utilities.data_structures import BidirectionalIterator -from golem.core.utilities.sequence_iterator import fibonacci_sequence, SequenceIterator +from golem.utilities.data_structures import BidirectionalIterator +from golem.utilities.sequence_iterator import fibonacci_sequence, SequenceIterator PopulationSize = AdaptiveParameter[int] diff --git a/golem/core/optimisers/meta/surrogate_evaluator.py b/golem/core/optimisers/meta/surrogate_evaluator.py index 3eedd39b0..cf7a70c4f 100644 --- a/golem/core/optimisers/meta/surrogate_evaluator.py +++ b/golem/core/optimisers/meta/surrogate_evaluator.py @@ -5,7 +5,7 @@ from golem.core.adapter import BaseOptimizationAdapter from golem.core.log import Log -from golem.core.optimisers.genetic.evaluation import OptionalEvalResult, DelegateEvaluator, SequentialDispatcher +from golem.core.optimisers.genetic.evaluation import DelegateEvaluator, SequentialDispatcher from golem.core.optimisers.graph import OptGraph from golem.core.optimisers.meta.surrogate_model import SurrogateModel, RandomValuesSurrogateModel from golem.core.optimisers.objective.objective import to_fitness, GraphFunction @@ -30,7 +30,7 @@ def __init__(self, def evaluate_single(self, graph: OptGraph, uid_of_individual: str, with_time_limit: bool = True, cache_key: Optional[str] = None, - logs_initializer: Optional[Tuple[int, pathlib.Path]] = None) -> OptionalEvalResult: + logs_initializer: Optional[Tuple[int, pathlib.Path]] = None) -> GraphEvalResult: graph = self.evaluation_cache.get(cache_key, graph) if logs_initializer is not None: # in case of multiprocessing run diff --git a/golem/core/optimisers/meta/surrogate_optimizer.py b/golem/core/optimisers/meta/surrogate_optimizer.py index d7d4c9cb7..0b1b40dc2 100644 --- a/golem/core/optimisers/meta/surrogate_optimizer.py +++ b/golem/core/optimisers/meta/surrogate_optimizer.py @@ -51,7 +51,7 @@ def optimise(self, objective: ObjectiveFunction) -> Sequence[OptGraph]: new_population = self._evolve_population(evaluator) except EvaluationAttemptsError as ex: self.log.warning(f'Composition process was stopped due to: {ex}') - return [ind.graph for ind in self.best_individuals] + break # Adding of new population to history self._update_population(new_population) self._update_population(self.best_individuals, 'final_choices') diff --git a/golem/core/optimisers/objective/objective.py b/golem/core/optimisers/objective/objective.py index db10b5ed1..e9649b3e3 100644 --- a/golem/core/optimisers/objective/objective.py +++ b/golem/core/optimisers/objective/objective.py @@ -19,6 +19,9 @@ class ObjectiveInfo: is_multi_objective: bool = False metric_names: Sequence[str] = () + def __str__(self): + return f'{self.__class__.__name__}(multi={self.is_multi_objective}, metrics={self.metric_names})' + def format_fitness(self, fitness: Union[Fitness, Sequence[float]]) -> str: """Returns formatted fitness string. Example for 3 metrics: ``""" diff --git a/golem/core/optimisers/opt_history_objects/generation.py b/golem/core/optimisers/opt_history_objects/generation.py index 4106bf853..3f385e2e6 100644 --- a/golem/core/optimisers/opt_history_objects/generation.py +++ b/golem/core/optimisers/opt_history_objects/generation.py @@ -4,7 +4,7 @@ from copy import deepcopy, copy from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Union -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence if TYPE_CHECKING: from golem.core.optimisers.opt_history_objects.individual import Individual diff --git a/golem/core/optimisers/opt_history_objects/individual.py b/golem/core/optimisers/opt_history_objects/individual.py index 4868ed6f9..f43ae66c1 100644 --- a/golem/core/optimisers/opt_history_objects/individual.py +++ b/golem/core/optimisers/opt_history_objects/individual.py @@ -115,14 +115,14 @@ def __eq__(self, other: Individual): return self.uid == other.uid def __copy__(self): - default_log(self).warning(INDIVIDUAL_COPY_RESTRICTION_MESSAGE, raise_if_test=True) + default_log(self).log_or_raise('warning', INDIVIDUAL_COPY_RESTRICTION_MESSAGE) cls = self.__class__ result = cls.__new__(cls) result.__dict__.update(self.__dict__) return result def __deepcopy__(self, memo): - default_log(self).warning(INDIVIDUAL_COPY_RESTRICTION_MESSAGE, raise_if_test=True) + default_log(self).log_or_raise('warning', INDIVIDUAL_COPY_RESTRICTION_MESSAGE) cls = self.__class__ result = cls.__new__(cls) memo[id(self)] = result diff --git a/golem/core/optimisers/opt_history_objects/opt_history.py b/golem/core/optimisers/opt_history_objects/opt_history.py index b15dd5bc1..dafc84ca4 100644 --- a/golem/core/optimisers/opt_history_objects/opt_history.py +++ b/golem/core/optimisers/opt_history_objects/opt_history.py @@ -23,7 +23,7 @@ class OptHistory: """ - Contains optimization history, save history to csv. + Contains optimization history, saves history to csv. Can be used for any type of graph that is serializable with Serializer. Args: @@ -99,14 +99,21 @@ def save_current_results(self, save_dir: Optional[os.PathLike] = None): last_gen = self.generations[last_gen_id] for individual in last_gen: ind_path = Path(save_dir, str(last_gen_id), str(individual.uid)) - if not os.path.isdir(ind_path): - os.makedirs(ind_path) - individual.save(json_file_path=Path(ind_path, f'{str(individual.uid)}.json')) + ind_path.mkdir(exist_ok=True, parents=True) + individual.save(json_file_path=ind_path / f'{str(individual.uid)}.json') except Exception as ex: self._log.exception(ex) - def save(self, json_file_path: Union[str, os.PathLike] = None) -> Optional[str]: - return default_save(obj=self, json_file_path=json_file_path) + def save(self, json_file_path: Union[str, os.PathLike] = None, is_save_light: bool = False) -> Optional[str]: + """ Saves history to specified path. + Args: + json_file_path: path to json file where to save history. + is_save_light: bool parameter to specify whether there is a need to save full history or a light version. + NB! For experiments and etc. full histories must be saved. However, to make the analysis of results faster + (show fitness plots, for example) the light version of histories can be saved too. + """ + history_to_save = lighten_history(self) if is_save_light else self + return default_save(obj=history_to_save, json_file_path=json_file_path) @staticmethod def load(json_str_or_file_path: Union[str, os.PathLike] = None) -> OptHistory: @@ -256,3 +263,15 @@ def individuals(self, value): @property def _log(self): return default_log(self) + + +def lighten_history(history: OptHistory) -> OptHistory: + """ Keeps the most informative field in OptHistory object to show most of the visualizations + without excessive memory usage. """ + light_history = OptHistory() + light_history._generations = \ + [Generation(iterable=gen, generation_num=i) for i, gen in enumerate(history.archive_history)] + light_history.archive_history = history.archive_history + light_history._objective = history.objective + light_history._tuning_result = history.tuning_result + return light_history diff --git a/golem/core/optimisers/opt_history_objects/parent_operator.py b/golem/core/optimisers/opt_history_objects/parent_operator.py index d80c51c10..55e944b0a 100644 --- a/golem/core/optimisers/opt_history_objects/parent_operator.py +++ b/golem/core/optimisers/opt_history_objects/parent_operator.py @@ -5,7 +5,7 @@ from uuid import uuid4 from golem.core.optimisers.opt_history_objects.individual import Individual -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence @dataclass(frozen=True) diff --git a/golem/core/optimisers/optimization_parameters.py b/golem/core/optimisers/optimization_parameters.py index 430635c08..cd6d3601b 100644 --- a/golem/core/optimisers/optimization_parameters.py +++ b/golem/core/optimisers/optimization_parameters.py @@ -5,6 +5,7 @@ from typing import Optional from golem.core.paths import default_data_dir +from golem.utilities.utilities import determine_n_jobs @dataclass @@ -85,9 +86,9 @@ class GraphRequirements(OptimizationParameters): max_arity: int = 4 def __post_init__(self): - excluded_fields = ['n_jobs'] + # check and convert n_jobs to non-negative + self.n_jobs = determine_n_jobs(self.n_jobs) + for field_name, field_value in dataclasses.asdict(self).items(): - if field_name in excluded_fields: - continue if isinstance(field_value, Number) and field_value < 0: raise ValueError(f'Value of {field_name} must be non-negative') diff --git a/golem/core/optimisers/optimizer.py b/golem/core/optimisers/optimizer.py index 2459b9a3e..86adf1a66 100644 --- a/golem/core/optimisers/optimizer.py +++ b/golem/core/optimisers/optimizer.py @@ -18,7 +18,7 @@ from golem.core.optimisers.opt_history_objects.opt_history import OptHistory from golem.core.optimisers.opt_node_factory import DefaultOptNodeFactory, OptNodeFactory from golem.core.optimisers.random_graph_factory import RandomGraphFactory, RandomGrowthGraphFactory -from golem.core.utilities.random import RandomStateHandler +from golem.utilities.random import RandomStateHandler STRUCTURAL_DIVERSITY_FREQUENCY_CHECK = 5 @@ -102,6 +102,8 @@ class GraphOptimizer: :param requirements: implementation-independent requirements for graph optimizer :param graph_generation_params: parameters for new graph generation :param graph_optimizer_params: parameters for specific implementation of graph optimizer + + Additional custom params can be specified with `custom_optimizer_params`. """ def __init__(self, @@ -110,7 +112,8 @@ def __init__(self, # TODO: rename params to avoid confusion requirements: Optional[OptimizationParameters] = None, graph_generation_params: Optional[GraphGenerationParams] = None, - graph_optimizer_params: Optional[AlgorithmParameters] = None): + graph_optimizer_params: Optional[AlgorithmParameters] = None, + **custom_optimizer_params): self.log = default_log(self) self._objective = objective initial_graphs = graph_generation_params.adapter.adapt(initial_graphs) if initial_graphs else None diff --git a/golem/core/optimisers/populational_optimizer.py b/golem/core/optimisers/populational_optimizer.py index 14f92d513..8a8704d9d 100644 --- a/golem/core/optimisers/populational_optimizer.py +++ b/golem/core/optimisers/populational_optimizer.py @@ -13,7 +13,7 @@ from golem.core.optimisers.optimization_parameters import GraphRequirements from golem.core.optimisers.optimizer import GraphGenerationParams, GraphOptimizer, AlgorithmParameters from golem.core.optimisers.timer import OptimisationTimer -from golem.core.utilities.grouped_condition import GroupedCondition +from golem.utilities.grouped_condition import GroupedCondition class PopulationalOptimizer(GraphOptimizer): @@ -30,6 +30,8 @@ class PopulationalOptimizer(GraphOptimizer): requirements: implementation-independent requirements for graph optimizer graph_generation_params: parameters for new graph generation graph_optimizer_params: parameters for specific implementation of graph optimizer + + Additional custom params can be specified with `custom_optimizer_params`. """ def __init__(self, @@ -38,8 +40,10 @@ def __init__(self, requirements: GraphRequirements, graph_generation_params: GraphGenerationParams, graph_optimizer_params: Optional['AlgorithmParameters'] = None, + **custom_optimizer_params ): - super().__init__(objective, initial_graphs, requirements, graph_generation_params, graph_optimizer_params) + super().__init__(objective, initial_graphs, requirements, + graph_generation_params, graph_optimizer_params, **custom_optimizer_params) self.population = None self.generations = GenerationKeeper(self.objective, keep_n_best=requirements.keep_n_best) self.timer = OptimisationTimer(timeout=self.requirements.timeout) @@ -57,7 +61,7 @@ def __init__(self, max_stagnation_time = requirements.early_stopping_timeout or self.timer.timeout self.stop_optimization = \ GroupedCondition(results_as_message=True).add_condition( - lambda: self.timer.is_time_limit_reached(self.current_generation_num), + lambda: self.timer.is_time_limit_reached(self.current_generation_num - 1), 'Optimisation stopped: Time limit is reached' ).add_condition( lambda: (requirements.num_of_generations is not None and @@ -101,7 +105,7 @@ def optimise(self, objective: ObjectiveFunction) -> Sequence[Graph]: pbar.update() except EvaluationAttemptsError as ex: self.log.warning(f'Composition process was stopped due to: {ex}') - return [ind.graph for ind in self.best_individuals] + break # Adding of new population to history self._update_population(new_population) pbar.close() diff --git a/golem/core/optimisers/random/random_mutation_optimizer.py b/golem/core/optimisers/random/random_mutation_optimizer.py index 6110dfe51..b6adc7061 100644 --- a/golem/core/optimisers/random/random_mutation_optimizer.py +++ b/golem/core/optimisers/random/random_mutation_optimizer.py @@ -10,7 +10,7 @@ from golem.core.optimisers.optimizer import GraphGenerationParams from golem.core.optimisers.populational_optimizer import PopulationalOptimizer from golem.core.optimisers.random.random_search import RandomSearchOptimizer -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence class PopulationalRandomMutationOptimizer(PopulationalOptimizer): diff --git a/golem/core/optimisers/random/random_search.py b/golem/core/optimisers/random/random_search.py index 0a7e7e501..0ca330108 100644 --- a/golem/core/optimisers/random/random_search.py +++ b/golem/core/optimisers/random/random_search.py @@ -12,7 +12,7 @@ from golem.core.optimisers.optimization_parameters import GraphRequirements from golem.core.optimisers.optimizer import GraphOptimizer, GraphGenerationParams from golem.core.optimisers.timer import OptimisationTimer -from golem.core.utilities.grouped_condition import GroupedCondition +from golem.utilities.grouped_condition import GroupedCondition class RandomSearchOptimizer(GraphOptimizer): diff --git a/golem/core/optimisers/timer.py b/golem/core/optimisers/timer.py index 11aaff830..34cae0318 100644 --- a/golem/core/optimisers/timer.py +++ b/golem/core/optimisers/timer.py @@ -48,9 +48,9 @@ def __init__(self, timeout: datetime.timedelta = None): def _is_next_iteration_possible(self, time_constraint: float, iteration_num: int = None) -> bool: minutes = self.minutes_from_start - if iteration_num is not None: + if iteration_num is not None and iteration_num != 0: evo_proc_minutes = minutes - self.init_time - possible = time_constraint > (minutes + (evo_proc_minutes / (iteration_num + 1))) + possible = time_constraint > (minutes + (evo_proc_minutes / iteration_num)) else: possible = time_constraint > minutes if not possible: diff --git a/golem/core/tuning/hyperopt_tuner.py b/golem/core/tuning/hyperopt_tuner.py index 153216ba9..cf6471a66 100644 --- a/golem/core/tuning/hyperopt_tuner.py +++ b/golem/core/tuning/hyperopt_tuner.py @@ -1,20 +1,28 @@ from abc import ABC from datetime import timedelta -from typing import Optional, Callable, Dict +from typing import Callable, Dict, Optional import numpy as np -from hyperopt import tpe, hp +from hyperopt import hp, tpe from hyperopt.early_stop import no_progress_loss -from hyperopt.pyll import Apply +from hyperopt.pyll import Apply, scope +from hyperopt.pyll_utils import validate_label from golem.core.adapter import BaseOptimizationAdapter from golem.core.log import default_log from golem.core.optimisers.objective import ObjectiveFunction -from golem.core.optimisers.timer import Timer from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label from golem.core.tuning.tuner_interface import BaseTuner +@validate_label +def hp_randint(label, *args, **kwargs): + return scope.int(scope.hyperopt_param(label, scope.randint(*args, **kwargs))) + + +hp.randint = hp_randint + + class HyperoptTuner(BaseTuner, ABC): """Base class for hyperparameters optimization based on hyperopt library @@ -41,7 +49,7 @@ def __init__(self, objective_evaluate: ObjectiveFunction, timeout: timedelta = timedelta(minutes=5), n_jobs: int = -1, deviation: float = 0.05, - algo: Callable = tpe.suggest): + algo: Callable = tpe.suggest, **kwargs): early_stopping_rounds = early_stopping_rounds or max(100, int(np.sqrt(iterations) * 10)) super().__init__(objective_evaluate, search_space, @@ -50,16 +58,12 @@ def __init__(self, objective_evaluate: ObjectiveFunction, early_stopping_rounds, timeout, n_jobs, - deviation) + deviation, **kwargs) self.early_stop_fn = no_progress_loss(iteration_stop_count=self.early_stopping_rounds) - self.max_seconds = int(timeout.seconds) if timeout is not None else None self.algo = algo self.log = default_log(self) - def _update_remaining_time(self, tuner_timer: Timer): - self.max_seconds = self.max_seconds - tuner_timer.minutes_from_start * 60 - def get_parameter_hyperopt_space(search_space: SearchSpace, operation_name: str, diff --git a/golem/core/tuning/iopt_tuner.py b/golem/core/tuning/iopt_tuner.py index 021baf5f3..ade0fc677 100644 --- a/golem/core/tuning/iopt_tuner.py +++ b/golem/core/tuning/iopt_tuner.py @@ -1,18 +1,22 @@ +from copy import deepcopy from dataclasses import dataclass, field +from datetime import timedelta from typing import List, Dict, Generic, Tuple, Any, Optional import numpy as np -from iOpt.method.listener import ConsoleFullOutputListener +from iOpt.output_system.listeners.console_outputers import ConsoleOutputListener from iOpt.problem import Problem from iOpt.solver import Solver from iOpt.solver_parametrs import SolverParameters from iOpt.trial import Point, FunctionValue from golem.core.adapter import BaseOptimizationAdapter +from golem.core.optimisers.genetic.evaluation import determine_n_jobs from golem.core.optimisers.graph import OptGraph from golem.core.optimisers.objective import ObjectiveEvaluate -from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label +from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label, convert_parameters from golem.core.tuning.tuner_interface import BaseTuner, DomainGraphForTune +from golem.utilities.data_structures import ensure_wrapped_in_sequence @dataclass @@ -36,60 +40,50 @@ def from_parameters_dicts(float_parameters_dict: Optional[Dict[str, List]] = Non upper_bounds_of_float_parameters = [bounds[1] for bounds in float_parameters_dict.values()] discrete_parameters_vals = [values_set for values_set in discrete_parameters_dict.values()] - # TODO: Remove - for now IOpt handles only float variables, so we treat discrete parameters as float ones - float_parameters_names.extend(discrete_parameters_names) - lower_bounds_of_discrete_parameters = [bounds[0] for bounds in discrete_parameters_dict.values()] - upper_bounds_of_discrete_parameters = [bounds[1] for bounds in discrete_parameters_dict.values()] - lower_bounds_of_float_parameters.extend(lower_bounds_of_discrete_parameters) - upper_bounds_of_float_parameters.extend(upper_bounds_of_discrete_parameters) - - return IOptProblemParameters(float_parameters_names, discrete_parameters_names, + return IOptProblemParameters(float_parameters_names, + discrete_parameters_names, lower_bounds_of_float_parameters, - upper_bounds_of_float_parameters, discrete_parameters_vals) + upper_bounds_of_float_parameters, + discrete_parameters_vals) class GolemProblem(Problem, Generic[DomainGraphForTune]): def __init__(self, graph: DomainGraphForTune, objective_evaluate: ObjectiveEvaluate, - problem_parameters: IOptProblemParameters): + problem_parameters: IOptProblemParameters, + objectives_number: int = 1): super().__init__() self.objective_evaluate = objective_evaluate self.graph = graph - self.numberOfObjectives = 1 - self.numberOfConstraints = 0 + self.number_of_objectives = objectives_number + self.number_of_constraints = 0 - self.discreteVariableNames = problem_parameters.discrete_parameters_names - self.discreteVariableValues = problem_parameters.discrete_parameters_vals - self.numberOfDiscreteVariables = len(self.discreteVariableNames) + self.discrete_variable_names = problem_parameters.discrete_parameters_names + self.discrete_variable_values = problem_parameters.discrete_parameters_vals + self.number_of_discrete_variables = len(self.discrete_variable_names) - self.floatVariableNames = problem_parameters.float_parameters_names - self.lowerBoundOfFloatVariables = problem_parameters.lower_bounds_of_float_parameters - self.upperBoundOfFloatVariables = problem_parameters.upper_bounds_of_float_parameters - self.numberOfFloatVariables = len(self.floatVariableNames) + self.float_variable_names = problem_parameters.float_parameters_names + self.lower_bound_of_float_variables = problem_parameters.lower_bounds_of_float_parameters + self.upper_bound_of_float_variables = problem_parameters.upper_bounds_of_float_parameters + self.number_of_float_variables = len(self.float_variable_names) self._default_metric_value = np.inf - def Calculate(self, point: Point, functionValue: FunctionValue) -> FunctionValue: + def calculate(self, point: Point, function_value: FunctionValue) -> FunctionValue: new_parameters = self.get_parameters_dict_from_iopt_point(point) BaseTuner.set_arg_graph(self.graph, new_parameters) graph_fitness = self.objective_evaluate(self.graph) metric_value = graph_fitness.value if graph_fitness.valid else self._default_metric_value - functionValue.value = metric_value - return functionValue + function_value.value = metric_value + return function_value def get_parameters_dict_from_iopt_point(self, point: Point) -> Dict[str, Any]: """Constructs a dict with all hyperparameters """ - float_parameters = dict(zip(self.floatVariableNames, point.floatVariables)) \ - if point.floatVariables is not None else {} - discrete_parameters = dict(zip(self.discreteVariableNames, point.discreteVariables)) \ - if point.discreteVariables is not None else {} - - # TODO: Remove workaround - for now IOpt handles only float variables, so discrete parameters - # are optimized as continuous and we need to round them - for parameter_name in float_parameters: - if parameter_name in self.discreteVariableNames: - float_parameters[parameter_name] = round(float_parameters[parameter_name]) + float_parameters = dict(zip(self.float_variable_names, point.float_variables)) \ + if point.float_variables is not None else {} + discrete_parameters = dict(zip(self.discrete_variable_names, point.discrete_variables)) \ + if point.discrete_variables is not None else {} parameters_dict = {**float_parameters, **discrete_parameters} return parameters_dict @@ -122,8 +116,9 @@ def __init__(self, objective_evaluate: ObjectiveEvaluate, search_space: SearchSpace, adapter: Optional[BaseOptimizationAdapter] = None, iterations: int = 100, + timeout: timedelta = timedelta(minutes=5), n_jobs: int = -1, - eps: float = 0.01, + eps: float = 0.001, r: float = 2.0, evolvent_density: int = 10, eps_r: float = 0.001, @@ -133,49 +128,57 @@ def __init__(self, objective_evaluate: ObjectiveEvaluate, search_space, adapter, iterations=iterations, + timeout=timeout, n_jobs=n_jobs, deviation=deviation, **kwargs) + self.n_jobs = determine_n_jobs(self.n_jobs) self.solver_parameters = SolverParameters(r=np.double(r), eps=np.double(eps), - itersLimit=iterations, - evolventDensity=evolvent_density, - epsR=np.double(eps_r), - refineSolution=refine_solution) - - def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainGraphForTune: - graph = self.adapter.adapt(graph) + iters_limit=iterations, + evolvent_density=evolvent_density, + eps_r=np.double(eps_r), + refine_solution=refine_solution, + number_of_parallel_points=self.n_jobs, + timeout=round(timeout.total_seconds()/60) if self.timeout else -1) + + def _tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainGraphForTune: problem_parameters, initial_parameters = self._get_parameters_for_tune(graph) - no_parameters_to_optimize = (not problem_parameters.discrete_parameters_names and - not problem_parameters.float_parameters_names) - self.init_check(graph) + has_parameters_to_optimize = (len(problem_parameters.discrete_parameters_names) > 0 or + len(problem_parameters.float_parameters_names) > 0) + self.objectives_number = len(ensure_wrapped_in_sequence(self.init_metric)) + is_multi_objective = self.objectives_number > 1 - if no_parameters_to_optimize: - self._stop_tuning_with_message(f'Graph "{graph.graph_description}" has no parameters to optimize') - final_graph = graph - else: + if self._check_if_tuning_possible(graph, has_parameters_to_optimize, supports_multi_objective=True): if initial_parameters: initial_point = Point(**initial_parameters) - self.solver_parameters.startPoint = initial_point + self.solver_parameters.start_point = initial_point - problem = GolemProblem(graph, self.objective_evaluate, problem_parameters) + problem = GolemProblem(graph, self.objective_evaluate, problem_parameters, self.objectives_number) solver = Solver(problem, parameters=self.solver_parameters) if show_progress: - console_output = ConsoleFullOutputListener(mode='full') - solver.AddListener(console_output) - - solution = solver.Solve() - best_point = solution.bestTrials[0].point - best_parameters = problem.get_parameters_dict_from_iopt_point(best_point) - final_graph = self.set_arg_graph(graph, best_parameters) - - self.was_tuned = True + console_output = ConsoleOutputListener(mode='full') + solver.add_listener(console_output) + + solver.solve() + solution = solver.get_results() + if not is_multi_objective: + best_point = solution.best_trials[0].point + best_parameters = problem.get_parameters_dict_from_iopt_point(best_point) + tuned_graphs = self.set_arg_graph(graph, best_parameters) + self.was_tuned = True + else: + tuned_graphs = [] + for best_trial in solution.best_trials: + best_parameters = problem.get_parameters_dict_from_iopt_point(best_trial.point) + tuned_graph = self.set_arg_graph(deepcopy(graph), best_parameters) + tuned_graphs.append(tuned_graph) + self.was_tuned = True + else: + tuned_graphs = graph - # Validate if optimisation did well - graph = self.final_check(final_graph) - final_graph = self.adapter.restore(graph) - return final_graph + return tuned_graphs def _get_parameters_for_tune(self, graph: OptGraph) -> Tuple[IOptProblemParameters, dict]: """ Method for defining the search space @@ -189,26 +192,28 @@ def _get_parameters_for_tune(self, graph: OptGraph) -> Tuple[IOptProblemParamete """ float_parameters_dict = {} discrete_parameters_dict = {} - initial_parameters = {'floatVariables': [], 'discreteVariables': []} + has_init_parameters = any(len(node.parameters) > 0 for node in graph.nodes) + initial_parameters = {'float_variables': [], 'discrete_variables': []} if has_init_parameters else None for node_id, node in enumerate(graph.nodes): operation_name = node.name # Assign unique prefix for each model hyperparameter # label - number of node in the graph - float_node_parameters, discrete_node_parameters = get_node_parameters_for_iopt(self.search_space, - node_id, - operation_name) - - # Set initial parameters for search - for parameter, bounds in float_node_parameters.items(): - # If parameter is not set use parameter minimum possible value - initaial_value = node.parameters.get(parameter) or bounds[0] - initial_parameters['floatVariables'].append(initaial_value) - - for parameter, bounds in discrete_node_parameters.items(): - # If parameter is not set use parameter minimum possible value - initaial_value = node.parameters.get(parameter) or bounds[0] - initial_parameters['discreteVariables'].append(initaial_value) + float_node_parameters, discrete_node_parameters = get_node_parameters_for_iopt( + self.search_space, + node_id, + operation_name) + if has_init_parameters: + # Set initial parameters for search + for parameter, bounds in convert_parameters(float_node_parameters).items(): + # If parameter is not set use parameter minimum possible value + initial_value = node.parameters.get(parameter) or bounds[0] + initial_parameters['float_variables'].append(initial_value) + + for parameter, values in convert_parameters(discrete_node_parameters).items(): + # If parameter is not set use the last value + initial_value = node.parameters.get(parameter) or values[-1] + initial_parameters['discrete_variables'].append(initial_value) float_parameters_dict.update(float_node_parameters) discrete_parameters_dict.update(discrete_node_parameters) @@ -237,16 +242,22 @@ def get_node_parameters_for_iopt(search_space: SearchSpace, node_id: int, operat discrete_parameters_dict = {} float_parameters_dict = {} + categorical_parameters_dict = {} for parameter_name, parameter_properties in parameters_dict.items(): node_op_parameter_name = get_node_operation_parameter_label(node_id, operation_name, parameter_name) parameter_type = parameter_properties.get('type') if parameter_type == 'discrete': - discrete_parameters_dict.update({node_op_parameter_name: parameter_properties - .get('sampling-scope')}) + discrete_parameters_dict.update({node_op_parameter_name: list(range(*parameter_properties + .get('sampling-scope')))}) elif parameter_type == 'continuous': float_parameters_dict.update({node_op_parameter_name: parameter_properties .get('sampling-scope')}) + elif parameter_type == 'categorical': + categorical_parameters_dict.update({node_op_parameter_name: parameter_properties + .get('sampling-scope')[0]}) + # IOpt does not distinguish between discrete and categorical parameters + discrete_parameters_dict = {**discrete_parameters_dict, **categorical_parameters_dict} return float_parameters_dict, discrete_parameters_dict diff --git a/golem/core/tuning/optuna_tuner.py b/golem/core/tuning/optuna_tuner.py index aabcb95d0..aac3990f3 100644 --- a/golem/core/tuning/optuna_tuner.py +++ b/golem/core/tuning/optuna_tuner.py @@ -12,6 +12,7 @@ from golem.core.optimisers.objective import ObjectiveFunction from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label from golem.core.tuning.tuner_interface import BaseTuner, DomainGraphForTune +from golem.utilities.data_structures import ensure_wrapped_in_sequence class OptunaTuner(BaseTuner): @@ -22,8 +23,7 @@ def __init__(self, objective_evaluate: ObjectiveFunction, early_stopping_rounds: Optional[int] = None, timeout: timedelta = timedelta(minutes=5), n_jobs: int = -1, - deviation: float = 0.05, - objectives_number: int = 1): + deviation: float = 0.05, **kwargs): super().__init__(objective_evaluate, search_space, adapter, @@ -31,25 +31,24 @@ def __init__(self, objective_evaluate: ObjectiveFunction, early_stopping_rounds, timeout, n_jobs, - deviation) - self.objectives_number = objectives_number + deviation, **kwargs) self.study = None - def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> \ + def _tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> \ Union[DomainGraphForTune, Sequence[DomainGraphForTune]]: - graph = self.adapter.adapt(graph) predefined_objective = partial(self.objective, graph=graph) - is_multi_objective = self.objectives_number > 1 - self.init_check(graph) + self.objectives_number = len(ensure_wrapped_in_sequence(self.init_metric)) + is_multi_objective = self.objectives_number > 1 self.study = optuna.create_study(directions=['minimize'] * self.objectives_number) init_parameters, has_parameters_to_optimize = self._get_initial_point(graph) - if not has_parameters_to_optimize: - self._stop_tuning_with_message(f'Graph {graph.graph_description} has no parameters to optimize') - tuned_graphs = self.init_graph - else: + remaining_time = self._get_remaining_time() + if self._check_if_tuning_possible(graph, + has_parameters_to_optimize, + remaining_time, + supports_multi_objective=True): # Enqueue initial point to try if init_parameters: self.study.enqueue_trial(init_parameters) @@ -60,8 +59,8 @@ def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> \ self.study.optimize(predefined_objective, n_trials=self.iterations, n_jobs=self.n_jobs, - timeout=self.timeout.seconds, - callbacks=[self.early_stopping_callback], + timeout=remaining_time, + callbacks=[self.early_stopping_callback] if not is_multi_objective else None, show_progress_bar=show_progress) if not is_multi_objective: @@ -75,9 +74,9 @@ def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> \ tuned_graph = self.set_arg_graph(deepcopy(graph), best_parameters) tuned_graphs.append(tuned_graph) self.was_tuned = True - final_graphs = self.final_check(tuned_graphs, is_multi_objective) - final_graphs = self.adapter.restore(final_graphs) - return final_graphs + else: + tuned_graphs = graph + return tuned_graphs def objective(self, trial: Trial, graph: OptGraph) -> Union[float, Sequence[float, ]]: new_parameters = self._get_parameters_from_trial(graph, trial) diff --git a/golem/core/tuning/sequential.py b/golem/core/tuning/sequential.py index fd5924549..003bb1700 100644 --- a/golem/core/tuning/sequential.py +++ b/golem/core/tuning/sequential.py @@ -26,7 +26,7 @@ def __init__(self, objective_evaluate: ObjectiveFunction, n_jobs: int = -1, deviation: float = 0.05, algo: Callable = tpe.suggest, - inverse_node_order: bool = False): + inverse_node_order: bool = False, **kwargs): super().__init__(objective_evaluate, search_space, adapter, @@ -34,61 +34,53 @@ def __init__(self, objective_evaluate: ObjectiveFunction, early_stopping_rounds, timeout, n_jobs, deviation, - algo) + algo, **kwargs) self.inverse_node_order = inverse_node_order - def tune(self, graph: DomainGraphForTune) -> DomainGraphForTune: + def _tune(self, graph: DomainGraphForTune, **kwargs) -> DomainGraphForTune: """ Method for hyperparameters tuning on the entire graph Args: graph: graph which hyperparameters will be tuned """ - graph = self.adapter.adapt(graph) - - # Check source metrics for data - self.init_check(graph) - - # Calculate amount of iterations we can apply per node - nodes_amount = graph.length - iterations_per_node = round(self.iterations / nodes_amount) - iterations_per_node = int(iterations_per_node) - if iterations_per_node == 0: - iterations_per_node = 1 - - # Calculate amount of seconds we can apply per node - if self.max_seconds is not None: - seconds_per_node = round(self.max_seconds / nodes_amount) - seconds_per_node = int(seconds_per_node) - else: - seconds_per_node = None - - # Tuning performed sequentially for every node - so get ids of nodes - nodes_ids = self.get_nodes_order(nodes_number=nodes_amount) - for node_id in nodes_ids: - node = graph.nodes[node_id] - operation_name = node.name - - # Get node's parameters to optimize - node_params = get_node_parameters_for_hyperopt(self.search_space, node_id, operation_name) - - if not node_params: - self.log.info(f'"{operation_name}" operation has no parameters to optimize') + remaining_time = self._get_remaining_time() + if self._check_if_tuning_possible(graph, parameters_to_optimize=True, remaining_time=remaining_time): + # Calculate amount of iterations we can apply per node + nodes_amount = graph.length + iterations_per_node = round(self.iterations / nodes_amount) + iterations_per_node = int(iterations_per_node) + if iterations_per_node == 0: + iterations_per_node = 1 + + # Calculate amount of seconds we can apply per node + if remaining_time is not None: + seconds_per_node = round(remaining_time / nodes_amount) + seconds_per_node = int(seconds_per_node) else: - # Apply tuning for current node - self._optimize_node(node_id=node_id, - graph=graph, - node_params=node_params, - iterations_per_node=iterations_per_node, - seconds_per_node=seconds_per_node) - - # Validate if optimisation did well - final_graph = self.final_check(graph) + seconds_per_node = None + + # Tuning performed sequentially for every node - so get ids of nodes + nodes_ids = self.get_nodes_order(nodes_number=nodes_amount) + for node_id in nodes_ids: + node = graph.nodes[node_id] + operation_name = node.name + + # Get node's parameters to optimize + node_params = get_node_parameters_for_hyperopt(self.search_space, node_id, operation_name) + + if not node_params: + self.log.info(f'"{operation_name}" operation has no parameters to optimize') + else: + # Apply tuning for current node + self._optimize_node(node_id=node_id, + graph=graph, + node_params=node_params, + iterations_per_node=iterations_per_node, + seconds_per_node=seconds_per_node) - self.was_tuned = True - final_graph = self.adapter.restore(final_graph) - - return final_graph + self.was_tuned = True + return graph def get_nodes_order(self, nodes_number: int) -> range: """ Method returns list with indices of nodes in the graph @@ -118,30 +110,33 @@ def tune_node(self, graph: DomainGraphForTune, node_index: int) -> DomainGraphFo """ graph = self.adapter.adapt(graph) - self.init_check(graph) + with self.timer: + self.init_check(graph) - node = graph.nodes[node_index] - operation_name = node.name + node = graph.nodes[node_index] + operation_name = node.name - # Get node's parameters to optimize - node_params = get_node_parameters_for_hyperopt(self.search_space, - node_id=node_index, - operation_name=operation_name) + # Get node's parameters to optimize + node_params = get_node_parameters_for_hyperopt(self.search_space, + node_id=node_index, + operation_name=operation_name) - if not node_params: - self._stop_tuning_with_message(f'"{operation_name}" operation has no parameters to optimize') - else: - # Apply tuning for current node - self._optimize_node(graph=graph, - node_id=node_index, - node_params=node_params, - iterations_per_node=self.iterations, - seconds_per_node=self.max_seconds, - ) - self.was_tuned = True + remaining_time = self._get_remaining_time() + if self._check_if_tuning_possible(graph, len(node_params) > 1, remaining_time): + # Apply tuning for current node + self._optimize_node(graph=graph, + node_id=node_index, + node_params=node_params, + iterations_per_node=self.iterations, + seconds_per_node=remaining_time + ) + self.was_tuned = True - # Validation is the optimization do well - final_graph = self.final_check(graph) + # Validation is the optimization do well + final_graph = self.final_check(graph) + else: + final_graph = graph + self.obtained_metric = self.init_metric final_graph = self.adapter.restore(final_graph) return final_graph @@ -149,7 +144,7 @@ def _optimize_node(self, graph: OptGraph, node_id: int, node_params: dict, iterations_per_node: int, - seconds_per_node: int) -> OptGraph: + seconds_per_node: float) -> OptGraph: """ Method for node optimization diff --git a/golem/core/tuning/simultaneous.py b/golem/core/tuning/simultaneous.py index 465dc81a8..f540114ea 100644 --- a/golem/core/tuning/simultaneous.py +++ b/golem/core/tuning/simultaneous.py @@ -5,7 +5,6 @@ from golem.core.constants import MIN_TIME_FOR_TUNING_IN_SEC from golem.core.optimisers.graph import OptGraph -from golem.core.optimisers.timer import Timer from golem.core.tuning.hyperopt_tuner import HyperoptTuner, get_node_parameters_for_hyperopt from golem.core.tuning.search_space import get_node_operation_parameter_label from golem.core.tuning.tuner_interface import DomainGraphForTune @@ -16,7 +15,7 @@ class SimultaneousTuner(HyperoptTuner): Class for hyperparameters optimization for all nodes simultaneously """ - def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainGraphForTune: + def _tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainGraphForTune: """ Function for hyperparameters tuning on the entire graph Args: @@ -26,69 +25,56 @@ def tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainG Returns: Graph with tuned hyperparameters """ - - graph = self.adapter.adapt(graph) parameters_dict, init_parameters = self._get_parameters_for_tune(graph) - - with Timer() as global_tuner_timer: - self.init_check(graph) - self._update_remaining_time(global_tuner_timer) - - if not parameters_dict: - self._stop_tuning_with_message(f'Graph "{graph.graph_description}" has no parameters to optimize') + remaining_time = self._get_remaining_time() + + if self._check_if_tuning_possible(graph, parameters_dict, remaining_time): + trials = Trials() + + try: + # try searching using initial parameters + # (uses original search space with fixed initial parameters) + trials, init_trials_num = self._search_near_initial_parameters(graph, + parameters_dict, + init_parameters, + trials, + remaining_time, + show_progress) + remaining_time = self._get_remaining_time() + if remaining_time > MIN_TIME_FOR_TUNING_IN_SEC: + fmin(partial(self._objective, graph=graph), + parameters_dict, + trials=trials, + algo=self.algo, + max_evals=self.iterations, + show_progressbar=show_progress, + early_stop_fn=self.early_stop_fn, + timeout=remaining_time) + else: + self.log.message('Tunner stopped after initial search due to the lack of time') + + best = space_eval(space=parameters_dict, hp_assignment=trials.argmin) + # check if best point was obtained using search space with fixed initial parameters + is_best_trial_with_init_params = trials.best_trial.get('tid') in range(init_trials_num) + if is_best_trial_with_init_params: + best = {**best, **init_parameters} + + final_graph = self.set_arg_graph(graph=graph, parameters=best) + + self.was_tuned = True + + except Exception as ex: + self.log.warning(f'Exception {ex} occurred during tuning') final_graph = graph - - elif self.max_seconds <= MIN_TIME_FOR_TUNING_IN_SEC: - self._stop_tuning_with_message('Tunner stopped after initial assumption due to the lack of time') - final_graph = graph - - else: - trials = Trials() - - try: - # try searching using initial parameters - # (uses original search space with fixed initial parameters) - trials, init_trials_num = self._search_near_initial_parameters(graph, - parameters_dict, - init_parameters, - trials, - show_progress) - self._update_remaining_time(global_tuner_timer) - if self.max_seconds > MIN_TIME_FOR_TUNING_IN_SEC: - fmin(partial(self._objective, graph=graph), - parameters_dict, - trials=trials, - algo=self.algo, - max_evals=self.iterations, - show_progressbar=show_progress, - early_stop_fn=self.early_stop_fn, - timeout=self.max_seconds) - else: - self.log.message('Tunner stopped after initial search due to the lack of time') - - best = space_eval(space=parameters_dict, hp_assignment=trials.argmin) - # check if best point was obtained using search space with fixed initial parameters - is_best_trial_with_init_params = trials.best_trial.get('tid') in range(init_trials_num) - if is_best_trial_with_init_params: - best = {**best, **init_parameters} - - final_graph = self.set_arg_graph(graph=graph, parameters=best) - - self.was_tuned = True - - except Exception as ex: - self.log.warning(f'Exception {ex} occurred during tuning') - final_graph = graph - - # Validate if optimisation did well - graph = self.final_check(final_graph) - final_graph = self.adapter.restore(graph) + else: + final_graph = graph return final_graph def _search_near_initial_parameters(self, graph: OptGraph, search_space: dict, initial_parameters: dict, trials: Trials, + remaining_time: float, show_progress: bool = True) -> Tuple[Trials, int]: """ Method to search using the search space where parameters initially set for the graph are fixed. This allows not to lose results obtained while composition process @@ -123,7 +109,7 @@ def _search_near_initial_parameters(self, graph: OptGraph, max_evals=init_trials_num, show_progressbar=show_progress, early_stop_fn=self.early_stop_fn, - timeout=self.max_seconds) + timeout=remaining_time) return trials, init_trials_num def _get_parameters_for_tune(self, graph: OptGraph) -> Tuple[dict, dict]: diff --git a/golem/core/tuning/tuner_interface.py b/golem/core/tuning/tuner_interface.py index 0d9886578..098c37347 100644 --- a/golem/core/tuning/tuner_interface.py +++ b/golem/core/tuning/tuner_interface.py @@ -7,14 +7,15 @@ from golem.core.adapter import BaseOptimizationAdapter from golem.core.adapter.adapter import IdentityAdapter -from golem.core.constants import MAX_TUNING_METRIC_VALUE +from golem.core.constants import MAX_TUNING_METRIC_VALUE, MIN_TIME_FOR_TUNING_IN_SEC from golem.core.dag.graph_utils import graph_structure from golem.core.log import default_log from golem.core.optimisers.fitness import SingleObjFitness, MultiObjFitness from golem.core.optimisers.graph import OptGraph from golem.core.optimisers.objective import ObjectiveEvaluate, ObjectiveFunction +from golem.core.optimisers.timer import Timer from golem.core.tuning.search_space import SearchSpace, convert_parameters -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence DomainGraphForTune = TypeVar('DomainGraphForTune') @@ -43,7 +44,7 @@ def __init__(self, objective_evaluate: ObjectiveFunction, early_stopping_rounds: Optional[int] = None, timeout: timedelta = timedelta(minutes=5), n_jobs: int = -1, - deviation: float = 0.05): + deviation: float = 0.05, **kwargs): self.iterations = iterations self.adapter = adapter or IdentityAdapter() self.search_space = search_space @@ -54,6 +55,7 @@ def __init__(self, objective_evaluate: ObjectiveFunction, self.deviation = deviation self.timeout = timeout + self.timer = Timer() self.early_stopping_rounds = early_stopping_rounds self._default_metric_value = MAX_TUNING_METRIC_VALUE @@ -62,9 +64,9 @@ def __init__(self, objective_evaluate: ObjectiveFunction, self.init_metric = None self.obtained_metric = None self.log = default_log(self) + self.objectives_number = 1 - @abstractmethod - def tune(self, graph: DomainGraphForTune) -> Union[DomainGraphForTune, Sequence[DomainGraphForTune]]: + def tune(self, graph: DomainGraphForTune, **kwargs) -> Union[DomainGraphForTune, Sequence[DomainGraphForTune]]: """ Function for hyperparameters tuning on the graph @@ -75,7 +77,22 @@ def tune(self, graph: DomainGraphForTune) -> Union[DomainGraphForTune, Sequence[ Graph with optimized hyperparameters or pareto front of optimized graphs in case of multi-objective optimization """ - raise NotImplementedError() + graph = self.adapter.adapt(graph) + self.was_tuned = False + with self.timer: + + # Check source metrics for data + self.init_check(graph) + final_graph = self._tune(graph, **kwargs) + # Validate if optimisation did well + final_graph = self.final_check(final_graph, self.objectives_number > 1) + + final_graph = self.adapter.restore(final_graph) + return final_graph + + @abstractmethod + def _tune(self, graph: DomainGraphForTune, **kwargs): + raise NotImplementedError def init_check(self, graph: OptGraph) -> None: """ @@ -163,6 +180,7 @@ def _multi_obj_final_check(self, tuned_graphs: Sequence[OptGraph]) -> Sequence[O else: self.log.message('Initial metric dominates all found solutions. Return initial graph.') final_graphs = self.init_graph + self.obtained_metric = self.init_metric return final_graphs def get_metric_value(self, graph: OptGraph) -> Union[float, Sequence[float]]: @@ -232,6 +250,29 @@ def set_arg_node(graph: OptGraph, node_id: int, node_params: dict) -> OptGraph: return graph + def _check_if_tuning_possible(self, graph: OptGraph, + parameters_to_optimize: bool, + remaining_time: Optional[float] = None, + supports_multi_objective: bool = False) -> bool: + if len(ensure_wrapped_in_sequence(self.init_metric)) > 1 and not supports_multi_objective: + self._stop_tuning_with_message(f'{self.__class__.__name__} does not support multi-objective optimization.') + return False + elif not parameters_to_optimize: + self._stop_tuning_with_message(f'Graph "{graph.graph_description}" has no parameters to optimize') + return False + elif remaining_time is not None: + if remaining_time <= MIN_TIME_FOR_TUNING_IN_SEC: + self._stop_tuning_with_message('Tunner stopped after initial assumption due to the lack of time') + return False + return True + def _stop_tuning_with_message(self, message: str): self.log.message(message) self.obtained_metric = self.init_metric + + def _get_remaining_time(self) -> Optional[float]: + if self.timeout is not None: + remaining_time = self.timeout.seconds - self.timer.seconds_from_start + return remaining_time + else: + return None diff --git a/golem/serializers/serializer.py b/golem/serializers/serializer.py index 4504f5287..6308a4834 100644 --- a/golem/serializers/serializer.py +++ b/golem/serializers/serializer.py @@ -88,7 +88,7 @@ def _register_default_coders(): from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator from golem.core.optimisers.fitness.fitness import Fitness from golem.core.optimisers.objective.objective import ObjectiveInfo - from golem.core.utilities.data_structures import ComparableEnum + from golem.utilities.data_structures import ComparableEnum from .any_serialization import any_from_json, any_to_json diff --git a/golem/structural_analysis/graph_sa/results/sa_analysis_results.py b/golem/structural_analysis/graph_sa/results/sa_analysis_results.py index e2c805c7d..75a22a2ce 100644 --- a/golem/structural_analysis/graph_sa/results/sa_analysis_results.py +++ b/golem/structural_analysis/graph_sa/results/sa_analysis_results.py @@ -6,7 +6,7 @@ from golem.core.dag.graph import Graph from golem.core.log import default_log from golem.core.paths import project_root -from golem.core.utilities.serializable import Serializable +from golem.utilities.serializable import Serializable from golem.serializers import Serializer from golem.structural_analysis.graph_sa.results.deletion_sa_approach_result import DeletionSAApproachResult from golem.structural_analysis.graph_sa.results.object_sa_result import ObjectSAResult, \ diff --git a/golem/core/utilities/data_structures.py b/golem/utilities/data_structures.py similarity index 96% rename from golem/core/utilities/data_structures.py rename to golem/utilities/data_structures.py index 2e36e3361..c2f1c1610 100644 --- a/golem/core/utilities/data_structures.py +++ b/golem/utilities/data_structures.py @@ -1,10 +1,11 @@ import collections.abc +import dataclasses from abc import ABC, abstractmethod from copy import deepcopy from enum import Enum from typing import Callable, Container, Generic, Iterable, Iterator, List, Optional, Sequence, Sized, TypeVar, Union, \ - Tuple + Tuple, Any, Dict T = TypeVar('T') @@ -306,5 +307,11 @@ def __iter__(self): return self -def unzip(tuples: Iterable[Tuple]) -> Tuple[Sequence, Sequence]: +def unzip(tuples: Iterable[Tuple]) -> Tuple[Sequence, ...]: return tuple(zip(*tuples)) + + +def update_dataclass(base_dc: Any, update_dc: Union[Any, Dict]) -> Any: + update_dict = dataclasses.asdict(update_dc) if dataclasses.is_dataclass(update_dc) else update_dc + new_base = dataclasses.replace(base_dc, **update_dict) + return new_base diff --git a/golem/core/utilities/grouped_condition.py b/golem/utilities/grouped_condition.py similarity index 100% rename from golem/core/utilities/grouped_condition.py rename to golem/utilities/grouped_condition.py diff --git a/golem/core/utilities/random.py b/golem/utilities/random.py similarity index 100% rename from golem/core/utilities/random.py rename to golem/utilities/random.py diff --git a/golem/core/utilities/sequence_iterator.py b/golem/utilities/sequence_iterator.py similarity index 97% rename from golem/core/utilities/sequence_iterator.py rename to golem/utilities/sequence_iterator.py index 97f8b7c0b..188b89b80 100644 --- a/golem/core/utilities/sequence_iterator.py +++ b/golem/utilities/sequence_iterator.py @@ -1,6 +1,6 @@ from typing import Callable, Optional -from golem.core.utilities.data_structures import BidirectionalIterator +from golem.utilities.data_structures import BidirectionalIterator class SequenceIterator(BidirectionalIterator[int]): diff --git a/golem/core/utilities/serializable.py b/golem/utilities/serializable.py similarity index 100% rename from golem/core/utilities/serializable.py rename to golem/utilities/serializable.py diff --git a/golem/core/utilities/singleton_meta.py b/golem/utilities/singleton_meta.py similarity index 100% rename from golem/core/utilities/singleton_meta.py rename to golem/utilities/singleton_meta.py diff --git a/golem/utilities/utilities.py b/golem/utilities/utilities.py new file mode 100644 index 000000000..7cff0b201 --- /dev/null +++ b/golem/utilities/utilities.py @@ -0,0 +1,30 @@ +import logging +from typing import Optional + +import numpy as np +from joblib import cpu_count + +from golem.utilities import random +from golem.utilities.random import RandomStateHandler + + +def determine_n_jobs(n_jobs=-1, logger=None): + cpu_num = cpu_count() + if n_jobs > cpu_num: + n_jobs = cpu_num + elif n_jobs <= 0: + if n_jobs <= -cpu_num - 1 or n_jobs == 0: + raise ValueError(f"Unproper `n_jobs` = {n_jobs}. " + f"`n_jobs` should be between ({-cpu_num}, {cpu_num}) except 0") + n_jobs = cpu_num + 1 + n_jobs + if logger: + logger.info(f"Number of used CPU's: {n_jobs}") + return n_jobs + + +def set_random_seed(seed: Optional[int]): + """ Sets random seed for evaluation of models. """ + if seed is not None: + np.random.seed(seed) + random.seed(seed) + RandomStateHandler.MODEL_FITTING_SEED = seed diff --git a/golem/visualisation/graph_viz.py b/golem/visualisation/graph_viz.py index fac8bd78b..09517a86c 100644 --- a/golem/visualisation/graph_viz.py +++ b/golem/visualisation/graph_viz.py @@ -5,7 +5,7 @@ from copy import deepcopy from pathlib import Path from textwrap import wrap -from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, Optional, Sequence, Tuple, Union, List +from typing import Any, Callable, Dict, Iterable, Literal, Optional, Sequence, TYPE_CHECKING, Tuple, Union from uuid import uuid4 import networkx as nx @@ -16,16 +16,18 @@ from pyvis.network import Network from seaborn import color_palette -from golem.core.dag.graph_utils import distance_to_primary_level from golem.core.dag.convert import graph_structure_as_nx_graph +from golem.core.dag.graph_utils import distance_to_primary_level from golem.core.log import default_log from golem.core.paths import default_data_dir if TYPE_CHECKING: from golem.core.dag.graph import Graph from golem.core.optimisers.graph import OptGraph + from golem.core.dag.graph_node import GraphNode GraphType = Union[Graph, OptGraph] + GraphConvertType = Callable[[GraphType], Tuple[nx.DiGraph, Dict[uuid4, GraphNode]]] PathType = Union[os.PathLike, str] @@ -36,106 +38,141 @@ class GraphVisualizer: - def __init__(self, graph: GraphType, visuals_params: Optional[Dict[str, Any]] = None, ): + def __init__(self, graph: GraphType, visuals_params: Optional[Dict[str, Any]] = None, + to_nx_convert_func: GraphConvertType = graph_structure_as_nx_graph): visuals_params = visuals_params or {} default_visuals_params = dict( engine='matplotlib', dpi=100, - node_color=self.__get_colors_by_labels, + node_color=self._get_colors_by_labels, node_size_scale=1.0, font_size_scale=1.0, edge_curvature_scale=1.0, - graph_to_nx_convert_func=graph_structure_as_nx_graph + node_names_placement='auto', + nodes_layout_function=GraphVisualizer._get_hierarchy_pos_by_distance_to_primary_level, + figure_size=(7, 7), + save_path=None, ) default_visuals_params.update(visuals_params) self.visuals_params = default_visuals_params - self.graph = graph - + self.to_nx_convert_func = to_nx_convert_func + self._update_graph(graph) self.log = default_log(self) + def _update_graph(self, graph: GraphType): + self.graph = graph + self.nx_graph, self.nodes_dict = self.to_nx_convert_func(self.graph) + def visualise(self, save_path: Optional[PathType] = None, engine: Optional[str] = None, node_color: Optional[NodeColorType] = None, dpi: Optional[int] = None, - node_size_scale: Optional[float] = None, - font_size_scale: Optional[float] = None, edge_curvature_scale: Optional[float] = None, - title: Optional[str] = None, - nodes_labels: Dict[int, str] = None, edges_labels: Dict[int, str] = None): - engine = engine or self.get_predefined_value('engine') + node_size_scale: Optional[float] = None, font_size_scale: Optional[float] = None, + edge_curvature_scale: Optional[float] = None, figure_size: Optional[Tuple[int, int]] = None, + nodes_labels: Dict[int, str] = None, edges_labels: Dict[int, str] = None, + node_names_placement: Optional[Literal['auto', 'nodes', 'legend', 'none']] = None, + nodes_layout_function: Optional[Callable[[nx.DiGraph], Dict[Any, Tuple[float, float]]]] = None, + title: Optional[str] = None): + engine = engine or self._get_predefined_value('engine') if not self.graph.nodes: raise ValueError('Empty graph can not be visualized.') if engine == 'matplotlib': - self.__draw_with_networkx(save_path=save_path, node_color=node_color, dpi=dpi, - node_size_scale=node_size_scale, font_size_scale=font_size_scale, - edge_curvature_scale=edge_curvature_scale, - title=title, nodes_labels=nodes_labels, edges_labels=edges_labels) + self._draw_with_networkx(save_path=save_path, node_color=node_color, dpi=dpi, + node_size_scale=node_size_scale, font_size_scale=font_size_scale, + edge_curvature_scale=edge_curvature_scale, figure_size=figure_size, + title=title, nodes_labels=nodes_labels, edges_labels=edges_labels, + nodes_layout_function=nodes_layout_function, + node_names_placement=node_names_placement) elif engine == 'pyvis': - self.__draw_with_pyvis(save_path, node_color) + self._draw_with_pyvis(save_path, node_color) elif engine == 'graphviz': - self.__draw_with_graphviz(save_path, node_color, dpi) + self._draw_with_graphviz(save_path, node_color, dpi) else: raise NotImplementedError(f'Unexpected visualization engine: {engine}. ' 'Possible values: matplotlib, pyvis, graphviz.') - @staticmethod - def __get_colors_by_labels(labels: Iterable[str]) -> LabelsColorMapType: - unique_labels = list(set(labels)) - palette = color_palette('tab10', len(unique_labels)) - return {label: palette[unique_labels.index(label)] for label in labels} + def draw_nx_dag( + self, ax: Optional[plt.Axes] = None, node_color: Optional[NodeColorType] = None, + node_size_scale: Optional[float] = None, font_size_scale: Optional[float] = None, + edge_curvature_scale: Optional[float] = None, nodes_labels: Dict[int, str] = None, + edges_labels: Dict[int, str] = None, + nodes_layout_function: Optional[Callable[[nx.DiGraph], Dict[Any, Tuple[float, float]]]] = None, + node_names_placement: Optional[Literal['auto', 'nodes', 'legend', 'none']] = None): + node_color = node_color or self._get_predefined_value('node_color') + node_size_scale = node_size_scale or self._get_predefined_value('node_size_scale') + font_size_scale = font_size_scale or self._get_predefined_value('font_size_scale') + edge_curvature_scale = (edge_curvature_scale if edge_curvature_scale is not None + else self._get_predefined_value('edge_curvature_scale')) + nodes_layout_function = nodes_layout_function or self._get_predefined_value('nodes_layout_function') + node_names_placement = node_names_placement or self._get_predefined_value('node_names_placement') - def __draw_with_graphviz(self, save_path: Optional[PathType] = None, node_color: Optional[NodeColorType] = None, - dpi: Optional[int] = None, graph_to_nx_convert_func: Optional[Callable] = None): - save_path = save_path or self.get_predefined_value('save_path') - node_color = node_color or self.get_predefined_value('node_color') - dpi = dpi or self.get_predefined_value('dpi') - graph_to_nx_convert_func = graph_to_nx_convert_func or self.get_predefined_value('graph_to_nx_convert_func') + nx_graph, nodes = self.nx_graph, self.nodes_dict + + if ax is None: + ax = plt.gca() - nx_graph, nodes = graph_to_nx_convert_func(self.graph) # Define colors if callable(node_color): - colors = node_color([str(node) for node in nodes.values()]) - elif isinstance(node_color, dict): - colors = node_color + node_color = node_color([str(node) for node in nodes.values()]) + if isinstance(node_color, dict): + node_color = [node_color.get(str(node), node_color.get(None)) for node in nodes.values()] else: - colors = {str(node): node_color for node in nodes.values()} - for n, data in nx_graph.nodes(data=True): - label = str(nodes[n]) - data['label'] = label.replace('_', ' ') - data['color'] = to_hex(colors.get(label, colors.get(None))) + node_color = [node_color for _ in nodes] + # Get node positions + if nodes_layout_function == GraphVisualizer._get_hierarchy_pos_by_distance_to_primary_level: + pos = nodes_layout_function(nx_graph, nodes) + else: + pos = nodes_layout_function(nx_graph) - gv_graph = nx.nx_agraph.to_agraph(nx_graph) - kwargs = {'prog': 'dot', 'args': f'-Gnodesep=0.5 -Gdpi={dpi} -Grankdir="LR"'} + node_size = self._get_scaled_node_size(len(nodes), node_size_scale) - if save_path: - gv_graph.draw(save_path, **kwargs) - else: - save_path = Path(default_data_dir(), 'graph_plots', str(uuid4()) + '.png') - save_path.parent.mkdir(exist_ok=True) - gv_graph.draw(save_path, **kwargs) + with_node_names = node_names_placement != 'none' - img = plt.imread(str(save_path)) - plt.imshow(img) - plt.gca().axis('off') - plt.gcf().set_dpi(dpi) - plt.tight_layout() + if node_names_placement in ('auto', 'none'): + node_names_placement = GraphVisualizer._define_node_names_placement(node_size) + + if node_names_placement == 'nodes': + self._draw_nx_big_nodes(ax, pos, nodes, node_color, node_size, font_size_scale, with_node_names) + elif node_names_placement == 'legend': + self._draw_nx_small_nodes(ax, pos, nodes, node_color, node_size, font_size_scale, with_node_names) + self._draw_nx_curved_edges(ax, pos, node_size, edge_curvature_scale) + self._draw_nx_labels(ax, pos, font_size_scale, nodes_labels, edges_labels) + + def _get_predefined_value(self, param: str): + if param not in self.visuals_params: + self.log.warning(f'No default param found: {param}.') + return self.visuals_params.get(param) + + def _draw_with_networkx( + self, save_path: Optional[PathType] = None, node_color: Optional[NodeColorType] = None, + dpi: Optional[int] = None, node_size_scale: Optional[float] = None, + font_size_scale: Optional[float] = None, edge_curvature_scale: Optional[float] = None, + figure_size: Optional[Tuple[int, int]] = None, title: Optional[str] = None, + nodes_labels: Dict[int, str] = None, edges_labels: Dict[int, str] = None, + nodes_layout_function: Optional[Callable[[nx.DiGraph], Dict[Any, Tuple[float, float]]]] = None, + node_names_placement: Optional[Literal['auto', 'nodes', 'legend', 'none']] = None): + save_path = save_path or self._get_predefined_value('save_path') + node_color = node_color or self._get_predefined_value('node_color') + dpi = dpi or self._get_predefined_value('dpi') + figure_size = figure_size or self._get_predefined_value('figure_size') + + ax = GraphVisualizer._setup_matplotlib_figure(figure_size, dpi, title) + self.draw_nx_dag(ax, node_color, node_size_scale, font_size_scale, edge_curvature_scale, + nodes_labels, edges_labels, nodes_layout_function, node_names_placement) + GraphVisualizer._rescale_matplotlib_figure(ax) + if not save_path: plt.show() - remove_old_files_from_dir(save_path.parent) + else: + plt.savefig(save_path, dpi=dpi) + plt.close() - def __draw_with_pyvis(self, save_path: Optional[PathType] = None, node_color: Optional[NodeColorType] = None, - graph_to_nx_convert_func: Optional[Callable] = None): - save_path = save_path or self.get_predefined_value('save_path') - node_color = node_color or self.get_predefined_value('node_color') - graph_to_nx_convert_func = graph_to_nx_convert_func or self.get_predefined_value('graph_to_nx_convert_func') + def _draw_with_pyvis(self, save_path: Optional[PathType] = None, node_color: Optional[NodeColorType] = None): + save_path = save_path or self._get_predefined_value('save_path') + node_color = node_color or self._get_predefined_value('node_color') net = Network('500px', '1000px', directed=True) - nx_graph, nodes = graph_to_nx_convert_func(self.graph) - # Define colors - if callable(node_color): - colors = node_color([str(node) for node in nodes.values()]) - elif isinstance(node_color, dict): - colors = node_color - else: - colors = {str(node): node_color for node in nodes.values()} + nx_graph, nodes = self.nx_graph, self.nodes_dict + node_color = self._define_colors(node_color, nodes) for n, data in nx_graph.nodes(data=True): operation = nodes[n] label = str(operation) @@ -146,7 +183,7 @@ def __draw_with_pyvis(self, save_path: Optional[PathType] = None, node_color: Op params = str(params)[1:-1] data['title'] = params data['level'] = distance_to_primary_level(operation) - data['color'] = to_hex(colors.get(label, colors.get(None))) + data['color'] = to_hex(node_color.get(label, node_color.get(None))) data['font'] = '20px' data['labelHighlightBold'] = True @@ -163,82 +200,140 @@ def __draw_with_pyvis(self, save_path: Optional[PathType] = None, node_color: Op net.show(str(save_path)) remove_old_files_from_dir(save_path.parent) - def __draw_with_networkx(self, save_path: Optional[PathType] = None, - node_color: Optional[NodeColorType] = None, - dpi: Optional[int] = None, node_size_scale: Optional[float] = None, - font_size_scale: Optional[float] = None, edge_curvature_scale: Optional[float] = None, - graph_to_nx_convert_func: Optional[Callable] = None, title: Optional[str] = None, - nodes_labels: Dict[int, str] = None, edges_labels: Dict[int, str] = None): - save_path = save_path or self.get_predefined_value('save_path') - node_color = node_color or self.get_predefined_value('node_color') - dpi = dpi or self.get_predefined_value('dpi') - node_size_scale = node_size_scale or self.get_predefined_value('node_size_scale') - font_size_scale = font_size_scale or self.get_predefined_value('font_size_scale') - edge_curvature_scale = (edge_curvature_scale if edge_curvature_scale is not None - else self.get_predefined_value('edge_curvature_scale')) - graph_to_nx_convert_func = graph_to_nx_convert_func or self.get_predefined_value('graph_to_nx_convert_func') + def _draw_with_graphviz(self, save_path: Optional[PathType] = None, node_color: Optional[NodeColorType] = None, + dpi: Optional[int] = None): + save_path = save_path or self._get_predefined_value('save_path') + node_color = node_color or self._get_predefined_value('node_color') + dpi = dpi or self._get_predefined_value('dpi') - fig, ax = plt.subplots(figsize=(7, 7)) - fig.set_dpi(dpi) + nx_graph, nodes = self.nx_graph, self.nodes_dict + node_color = self._define_colors(node_color, nodes) + for n, data in nx_graph.nodes(data=True): + label = str(nodes[n]) + data['label'] = label.replace('_', ' ') + data['color'] = to_hex(node_color.get(label, node_color.get(None))) - plt.title(title) - self.draw_nx_dag(ax, node_color, node_size_scale, font_size_scale, edge_curvature_scale, - graph_to_nx_convert_func, nodes_labels, edges_labels) - if not save_path: - plt.show() + gv_graph = nx.nx_agraph.to_agraph(nx_graph) + kwargs = {'prog': 'dot', 'args': f'-Gnodesep=0.5 -Gdpi={dpi} -Grankdir="LR"'} + + if save_path: + gv_graph.draw(save_path, **kwargs) else: - plt.savefig(save_path, dpi=dpi) - plt.close() + save_path = Path(default_data_dir(), 'graph_plots', str(uuid4()) + '.png') + save_path.parent.mkdir(exist_ok=True) + gv_graph.draw(save_path, **kwargs) - def draw_nx_dag(self, ax: Optional[plt.Axes] = None, - node_color: Optional[NodeColorType] = None, - node_size_scale: float = 1, font_size_scale: float = 1, edge_curvature_scale: float = 1, - graph_to_nx_convert_func: Callable = graph_structure_as_nx_graph, - nodes_labels: Dict[int, str] = None, edges_labels: Dict[int, str] = None): - - def draw_nx_labels(pos, node_labels, ax, max_sequence_length, font_size_scale=1.0): - def get_scaled_font_size(nodes_amount): - min_size = 2 - max_size = 20 - - size = min_size + int((max_size - min_size) / np.log2(max(nodes_amount, 2))) - return size - - if ax is None: - ax = plt.gca() - for node, (x, y) in pos.items(): - text = '\n'.join(wrap(node_labels[node].replace('_', ' ').replace('-', ' '), 10)) - ax.text(x, y, text, ha='center', va='center', - fontsize=get_scaled_font_size(max_sequence_length) * font_size_scale, - bbox=dict(alpha=0.9, color='w', boxstyle='round')) - - def get_scaled_node_size(nodes_amount): - min_size = 500 - max_size = 5000 - size = min_size + int((max_size - min_size) / np.log2(max(nodes_amount, 2))) - return size + img = plt.imread(str(save_path)) + plt.imshow(img) + plt.gca().axis('off') + plt.gcf().set_dpi(dpi) + plt.tight_layout() + plt.show() + remove_old_files_from_dir(save_path.parent) - if ax is None: - ax = plt.gca() + @staticmethod + def _get_scaled_node_size(nodes_amount: int, size_scale: float) -> float: + min_size = 150 + max_size = 12000 + size = max(max_size * (1 - np.log10(nodes_amount)), min_size) + return size * size_scale - nx_graph, nodes = graph_to_nx_convert_func(self.graph) - # Define colors + @staticmethod + def _get_scaled_font_size(nodes_amount: int, size_scale: float) -> float: + min_size = 14 + max_size = 30 + size = max(max_size * (1 - np.log10(nodes_amount)), min_size) + return size * size_scale + + @staticmethod + def _get_colors_by_labels(labels: Iterable[str]) -> LabelsColorMapType: + unique_labels = list(set(labels)) + palette = color_palette('tab10', len(unique_labels)) + return {label: palette[unique_labels.index(label)] for label in labels} + + @staticmethod + def _define_colors(node_color, nodes): if callable(node_color): - node_color = node_color([str(node) for node in nodes.values()]) - if isinstance(node_color, dict): - node_color = [node_color.get(str(node), node_color.get(None)) for node in nodes.values()] - # Define hierarchy_level - for node_id, node_data in nx_graph.nodes(data=True): - node_data['hierarchy_level'] = distance_to_primary_level(nodes[node_id]) - # Get nodes positions - pos, longest_sequence = get_hierarchy_pos(nx_graph) - node_size = get_scaled_node_size(longest_sequence) * node_size_scale + colors = node_color([str(node) for node in nodes.values()]) + elif isinstance(node_color, dict): + colors = node_color + else: + colors = {str(node): node_color for node in nodes.values()} + return colors + + @staticmethod + def _setup_matplotlib_figure(figure_size: Tuple[float, float], dpi: int, title: Optional[str] = None) -> plt.Axes: + fig, ax = plt.subplots(figsize=figure_size) + fig.set_dpi(dpi) + plt.title(title) + return ax + + @staticmethod + def _rescale_matplotlib_figure(ax): + """Rescale the figure for all nodes to fit in.""" + + x_1, x_2 = ax.get_xlim() + y_1, y_2 = ax.get_ylim() + offset = 0.2 + x_offset = x_2 * offset + y_offset = y_2 * offset + ax.set_xlim(x_1 - x_offset, x_2 + x_offset) + ax.set_ylim(y_1 - y_offset, y_2 + y_offset) + ax.axis('off') + plt.tight_layout() + + def _draw_nx_big_nodes(self, ax, pos, nodes, node_color, node_size, font_size_scale, with_node_names): # Draw the graph's nodes. - nx.draw_networkx_nodes(nx_graph, pos, node_size=node_size, ax=ax, node_color='w', linewidths=3, + nx.draw_networkx_nodes(self.nx_graph, pos, node_size=node_size, ax=ax, node_color='w', linewidths=3, edgecolors=node_color) + if not with_node_names: + return # Draw the graph's node labels. - draw_nx_labels(pos, {node_id: str(node) for node_id, node in nodes.items()}, ax, longest_sequence, - font_size_scale) + node_labels = {node_id: str(node) for node_id, node in nodes.items()} + font_size = GraphVisualizer._get_scaled_font_size(len(nodes), font_size_scale) + for node, (x, y) in pos.items(): + text = '\n'.join(wrap(node_labels[node].replace('_', ' ').replace('-', ' '), 10)) + ax.text(x, y, text, + ha='center', va='center', + fontsize=font_size, + bbox=dict(alpha=0.9, color='w', boxstyle='round')) + + def _draw_nx_small_nodes(self, ax, pos, nodes, node_color, node_size, font_size_scale, with_node_names): + nx_graph = self.nx_graph + markers = 'os^>v len(markers) - 1: + self.log.warning(f'Too much node labels derive the same color: {color}. The markers may repeat.\n' + '\tSpecify the parameter "node_color" to set distinct colors.') + color_count = color_count % len(markers) + marker = markers[color_count] + label_markers[label] = marker + color_counts[color] = color_count + 1 + nx.draw_networkx_nodes(nx_graph, pos, [node_id], ax=ax, node_color=[color], node_size=node_size, + node_shape=marker) + if label in labels_added: + continue + ax.plot([], [], marker=marker, linestyle='None', color=color, label=label) + labels_added.add(label) + if not with_node_names: + return + # @morrisnein took the following code from https://stackoverflow.com/a/27512450 + handles, labels = ax.get_legend_handles_labels() + # Sort both labels and handles by labels + labels, handles = zip(*sorted(zip(labels, handles), key=lambda t: t[0])) + ax.legend(handles, labels, prop={'size': round(20 * font_size_scale)}) + + def _draw_nx_curved_edges(self, ax, pos, node_size, edge_curvature_scale): + nx_graph = self.nx_graph # The ongoing section defines curvature for all edges. # This is 'connection style' for an edge that does not intersect any nodes. connection_style = 'arc3' @@ -261,9 +356,9 @@ def get_scaled_node_size(nodes_amount): continue # The node is adjacent to the edge. p_3 = np.array(pos[node_id]) distance_to_node = abs(np.cross(p_1_2, p_3 - p_1)) / p_1_2_length - if (distance_to_node > min(node_distance_gap, min_distance_found) or # The node is too far. - ((p_3 - p_1) @ p_1_2) < 0 or # There's no perpendicular from the node to the edge. - ((p_3 - p_2) @ -p_1_2) < 0): + if (distance_to_node > min(node_distance_gap, min_distance_found) # The node is too far. + or ((p_3 - p_1) @ p_1_2) < 0 # There's no perpendicular from the node to the edge. + or ((p_3 - p_2) @ -p_1_2) < 0): continue min_distance_found = distance_to_node closest_node_id = node_id @@ -281,148 +376,138 @@ def get_scaled_node_size(nodes_amount): # Then, its ordinate shows on which side of the edge it is, "on the left" or "on the right". rotation_matrix = np.array([[cos_alpha, sin_alpha], [-sin_alpha, cos_alpha]]) p_1_3_rotated = rotation_matrix @ p_1_3 - curvature_direction = (-1) ** (p_1_3_rotated[1] < 0) # +1 is a "boat" \/, -1 is a "cat" /\. + curvature_direction = (-1) ** (p_1_3_rotated[1] < 0) # +1 is a "cup" \/, -1 is a "cat" /\. edge_curvature = curvature_direction * curvature_strength e['connectionstyle'] = connection_style_curved_template.format(edge_curvature) + # Define edge center position for labels. + edge_center_position = np.mean([p_1, p_2], axis=0) + edge_curvature_shift = np.linalg.inv(rotation_matrix) @ [0, -1 * edge_curvature / 4] + edge_center_position += edge_curvature_shift + e['edge_center_position'] = edge_center_position # Draw the graph's edges. arrow_style = ArrowStyle('Simple', head_length=1.5, head_width=0.8) for u, v, e in nx_graph.edges(data=True): nx.draw_networkx_edges(nx_graph, pos, edgelist=[(u, v)], node_size=node_size, ax=ax, arrowsize=10, arrowstyle=arrow_style, connectionstyle=e['connectionstyle']) - if nodes_labels or edges_labels: - self._set_labels(ax, pos, nx_graph, - longest_sequence, longest_sequence, font_size_scale, - nodes_labels, edges_labels) - # Rescale the figure for all nodes to fit in. - x_1, x_2 = ax.get_xlim() - y_1, y_2 = ax.get_ylim() - offset = 0.2 - x_offset = x_2 * offset - y_offset = y_2 * offset - ax.set_xlim(x_1 - x_offset, x_2 + x_offset) - ax.set_ylim(y_1 - y_offset, y_2 + y_offset) - ax.axis('off') - plt.tight_layout() + self._rescale_matplotlib_figure(ax) - def get_predefined_value(self, param: str): - return self.visuals_params.get(param) - - def _set_labels(self, ax: plt.Axes, pos: Any, nx_graph: nx.DiGraph, - longest_sequence: int, longest_y_sequence: int, font_size_scale: float, - nodes_labels: Dict[int, str], edges_labels: Dict[int, str]): + def _draw_nx_labels(self, ax: plt.Axes, pos: Any, font_size_scale: float, + nodes_labels: Dict[int, str], edges_labels: Dict[int, str]): """ Set labels with scores to nodes and edges. """ - def calculate_labels_bias(ax: plt.Axes, longest_y_sequence: int): + def calculate_labels_bias(ax: plt.Axes, y_span: int): y_1, y_2 = ax.get_ylim() y_size = y_2 - y_1 - if longest_y_sequence == 1: + if y_span == 1: bias_scale = 0.25 # Fits between the central line and the upper bound. else: - bias_scale = 1 / longest_y_sequence / 3 * 0.9 # Fits between the narrowest horizontal rows. + bias_scale = 1 / y_span / 3 * 0.5 # Fits between the narrowest horizontal rows. bias = y_size * bias_scale return bias - def _get_scaled_font_size(nodes_amount: int, size_scale: float) -> float: - min_size = 11 - max_size = 25 - size = max(max_size * (1 - np.log10(nodes_amount)), min_size) - return size * size_scale - def match_labels_with_nx_nodes(nx_graph: nx.DiGraph, labels: Dict[int, str]) -> Dict[str, str]: """ Matches index of node in GOLEM graph with networkx node name. """ nx_nodes = list(nx_graph.nodes.keys()) nx_labels = {} - for index in labels: - nx_labels[nx_nodes[index]] = labels[index] + for index, label in labels.items(): + nx_labels[nx_nodes[index]] = label return nx_labels def match_labels_with_nx_edges(nx_graph: nx.DiGraph, labels: Dict[int, str]) \ - -> Dict[Tuple[str, str], List[str]]: + -> Dict[Tuple[str, str], str]: """ Matches index of edge in GOLEM graph with tuple of networkx nodes names. """ nx_nodes = list(nx_graph.nodes.keys()) edges = self.graph.get_edges() nx_labels = {} - for index in labels: + for index, label in labels.items(): edge = edges[index] parent_node_nx = nx_nodes[self.graph.nodes.index(edge[0])] child_node_nx = nx_nodes[self.graph.nodes.index(edge[1])] - nx_labels[(parent_node_nx, child_node_nx)] = labels[index] + nx_labels[(parent_node_nx, child_node_nx)] = label return nx_labels - if not edges_labels and not nodes_labels: - return - - bias = calculate_labels_bias(ax, longest_y_sequence) - if nodes_labels: - # Set labels for nodes + def draw_node_labels(node_labels, ax, bias, font_size, nx_graph, pos): labels_pos = deepcopy(pos) - font_size = _get_scaled_font_size(longest_sequence, font_size_scale * 0.7) - bbox = dict(alpha=0.9, color='w') for value in labels_pos.values(): value[1] += bias + bbox = dict(alpha=0.9, color='w') - nodes_nx_labels = match_labels_with_nx_nodes(nx_graph=nx_graph, labels=nodes_labels) + nodes_nx_labels = match_labels_with_nx_nodes(nx_graph=nx_graph, labels=node_labels) nx.draw_networkx_labels( nx_graph, labels_pos, + ax=ax, labels=nodes_nx_labels, font_color='black', font_size=font_size, bbox=bbox ) - if not edges_labels: + def draw_edge_labels(edge_labels, ax, bias, font_size, nx_graph, pos): + labels_pos_edges = deepcopy(pos) + label_bias_y = 2 / 3 * bias + if len(set([coord[1] for coord in pos.values()])) == 1 and len(list(pos.values())) > 2: + for value in labels_pos_edges.values(): + value[1] += label_bias_y + edges_nx_labels = match_labels_with_nx_edges(nx_graph=nx_graph, labels=edge_labels) + bbox = dict(alpha=0.9, color='w') + # Set labels for edges + for u, v, e in nx_graph.edges(data=True): + if (u, v) not in edges_nx_labels: + continue + current_pos = labels_pos_edges + if 'edge_center_position' in e: + x, y = e['edge_center_position'] + plt.text(x, y, edges_nx_labels[(u, v)], bbox=bbox, fontsize=font_size) + else: + nx.draw_networkx_edge_labels( + nx_graph, current_pos, {(u, v): edges_nx_labels[(u, v)]}, + label_pos=0.5, ax=ax, + font_color='black', + font_size=font_size, + rotate=False, + bbox=bbox + ) + + if not (edges_labels or nodes_labels): return - labels_pos_edges = deepcopy(pos) - label_bias_y = 2 / 3 * bias - if len(set([coord[1] for coord in pos.values()])) == 1 and len(list(pos.values())) > 2: - for value in labels_pos_edges.values(): - value[1] += label_bias_y - - edges_nx_labels = match_labels_with_nx_edges(nx_graph=nx_graph, labels=edges_labels) - # Set labels for edges - for u, v, e in nx_graph.edges(data=True): - if (u, v) not in edges_nx_labels: - continue - current_pos = labels_pos_edges - if 'edge_center_position' in e: - x, y = e['edge_center_position'] - plt.text(x, y, edges_nx_labels[(u, v)], bbox=bbox, fontsize=font_size) - else: - nx.draw_networkx_edge_labels( - nx_graph, current_pos, {(u, v): edges_nx_labels[(u, v)]}, - label_pos=0.5, ax=ax, - font_color='black', - font_size=font_size, - rotate=False, - bbox=bbox - ) - - -def get_hierarchy_pos(graph: nx.DiGraph, max_line_length: int = 6) -> Tuple[Dict[Any, Tuple[float, float]], int]: - """By default, returns 'networkx.multipartite_layout' positions based on 'hierarchy_level` from node data - the - property must be set beforehand. - If line of nodes reaches 'max_line_length', the result is the combination of 'networkx.multipartite_layout' and - 'networkx.spring_layout'. - :param graph: the graph. - :param max_line_length: the limit for common nodes horizontal or vertical line. - """ - longest_path = nx.dag_longest_path(graph, weight=None) - longest_sequence = len(longest_path) - - pos = nx.multipartite_layout(graph, subset_key='hierarchy_level') - - y_level_nodes_count = {} - for x, _ in pos.values(): - y_level_nodes_count[x] = y_level_nodes_count.get(x, 0) + 1 - nodes_on_level = y_level_nodes_count[x] - if nodes_on_level > longest_sequence: - longest_sequence = nodes_on_level - - if longest_sequence > max_line_length: - pos = {n: np.array(x_y) + (np.random.random(2) - 0.5) * 0.001 for n, x_y in pos.items()} - pos = nx.spring_layout(graph, k=2, iterations=5, pos=pos, seed=42) - - return pos, longest_sequence + + nodes_amount = len(pos) + font_size = GraphVisualizer._get_scaled_font_size(nodes_amount, font_size_scale * 0.75) + _, y_span = GraphVisualizer._get_x_y_span(pos) + bias = calculate_labels_bias(ax, y_span) + + if nodes_labels: + draw_node_labels(nodes_labels, ax, bias, font_size, self.nx_graph, pos) + + if edges_labels: + draw_edge_labels(edges_labels, ax, bias, font_size, self.nx_graph, pos) + + @staticmethod + def _get_hierarchy_pos_by_distance_to_primary_level(nx_graph: nx.DiGraph, nodes: Dict + ) -> Dict[Any, Tuple[float, float]]: + """By default, returns 'networkx.multipartite_layout' positions based on 'hierarchy_level` + from node data - the property must be set beforehand. + :param graph: the graph. + """ + for node_id, node_data in nx_graph.nodes(data=True): + node_data['hierarchy_level'] = distance_to_primary_level(nodes[node_id]) + + return nx.multipartite_layout(nx_graph, subset_key='hierarchy_level') + + @staticmethod + def _get_x_y_span(pos: Dict[Any, Tuple[float, float]]) -> Tuple[int, int]: + pos_x, pos_y = np.split(np.array(tuple(pos.values())), 2, axis=1) + x_span = max(pos_x) - min(pos_x) + y_span = max(pos_y) - min(pos_y) + return x_span, y_span + + @staticmethod + def _define_node_names_placement(node_size): + if node_size >= 1000: + node_names_placement = 'nodes' + else: + node_names_placement = 'legend' + return node_names_placement def remove_old_files_from_dir(dir_: Path, time_interval=datetime.timedelta(minutes=10)): diff --git a/golem/visualisation/opt_history/genealogical_path.py b/golem/visualisation/opt_history/genealogical_path.py new file mode 100644 index 000000000..1e9a42251 --- /dev/null +++ b/golem/visualisation/opt_history/genealogical_path.py @@ -0,0 +1,131 @@ +import math +import os +from functools import partial +from typing import Callable, List, Union, Optional + +from matplotlib import pyplot as plt, animation + +from golem.core.dag.graph import Graph +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.visualisation.graph_viz import GraphVisualizer +from golem.visualisation.opt_history.history_visualization import HistoryVisualization + + +class GenealogicalPath(HistoryVisualization): + def visualize(self, graph_dist: Callable[[Graph, Graph], float] = None, target_graph: Graph = None, + evolution_time_s: float = 8., hold_result_time_s: float = 2., + save_path: Optional[Union[os.PathLike, str]] = None, show: bool = False): + """ + Takes the best individual from the resultant generation and traces its genealogical path + taking the most similar parent each time (or the first parent if no similarity measure is provided). + That makes the picture more stable (and hence comprehensible) and the evolution process more apparent. + + Saves the result as a GIF with the following layout: + - target graph (if provided) is displayed on the left, + - evolving graphs go as the next subplot, they evolve from the first generation to the last, + - and the fitness plot on the right shows fitness dynamics as the graphs evolve. + + :param graph_dist: a function to measure the distance between two graphs. If not provided, all graphs are + treated as equally distant. + Works on optimization graphs, not domain graphs. If your distance metric works on domain graphs, + adapt it with `adapter.adapt_func(your_metric)`. + :param target_graph: the graph to compare the genealogical path with. Again, optimization graph is expected. + If provided, it will be displayed on the left throughout the animation. + :param save_path: path to save the visualization (won't be saved if it's None). + GIF of video extension is expected. + :param show: whether to show the visualization. + :param evolution_time_s: time in seconds for the part of the animation where the evolution process is shown. + :param hold_result_time_s: time in seconds for the part of the animation where the final result is shown. + """ + # Treating all graphs as equally distant if there's no reasonable way to compare them: + graph_dist = graph_dist or (lambda g1, g2: 1) + + def draw_graph(graph: Graph, ax, title, highlight_title=False): + ax.clear() + ax.set_title(title, fontsize=22, color='green' if highlight_title else 'black') + GraphVisualizer(graph).draw_nx_dag(ax, node_names_placement='legend') + + try: + last_internal_graph = self.history.archive_history[-1][0] + genealogical_path = trace_genealogical_path(last_internal_graph, graph_dist) + except Exception as e: + # At least `Individual.parents_from_prev_generation` my fail + self.log.error(f"Failed to trace genealogical path: {e}") + return + + figure_width = 5 + width_ratios = [1.3, 0.7] + if target_graph is not None: + width_ratios = [1.3] + width_ratios + + fig, axes = plt.subplots( + 1, len(width_ratios), + figsize=(figure_width * sum(width_ratios), figure_width), + gridspec_kw={'width_ratios': width_ratios} + ) + evo_ax, fitness_ax = axes[-2:] + if target_graph is not None: + draw_graph(target_graph, axes[0], "Target graph") # Persists throughout the animation + + fitnesses_along_path = list(map(lambda ind: ind.fitness.value, genealogical_path)) + generations_along_path = list(map(lambda ind: ind.native_generation, genealogical_path)) + + def render_frame(frame_index): + path_index = min(frame_index, len(genealogical_path) - 1) + is_hold_stage = frame_index >= len(genealogical_path) + + draw_graph( + genealogical_path[path_index].graph, evo_ax, + f"Evolution process,\ngeneration {generations_along_path[path_index]}/{generations_along_path[-1]}", + highlight_title=is_hold_stage + ) + # Select only the genealogical path + fitness_ax.clear() + plot_fitness_with_axvline( + generations=generations_along_path, + fitnesses=fitnesses_along_path, + ax=fitness_ax, + axvline_x=generations_along_path[path_index], + current_fitness=fitnesses_along_path[path_index] + ) + return evo_ax, fitness_ax + + frames = len(genealogical_path) + int( + math.ceil(len(genealogical_path) * hold_result_time_s / (hold_result_time_s + evolution_time_s)) + ) + seconds_per_frame = (evolution_time_s + hold_result_time_s) / frames + fps = math.ceil(1 / seconds_per_frame) + + anim = animation.FuncAnimation(fig, render_frame, repeat=False, frames=frames, + interval=1000 * seconds_per_frame) + + try: + if save_path is not None: + anim.save(save_path, fps=fps) + if show: + plt.show() + except Exception as e: + self.log.error(f"Failed to render the genealogical path: {e}") + + +def trace_genealogical_path(individual: Individual, graph_dist: Callable[[Graph, Graph], float]) -> List[Individual]: + # Choose nearest parent each time: + genealogical_path: List[Individual] = [individual] + while genealogical_path[-1].parents_from_prev_generation: + genealogical_path.append(max( + genealogical_path[-1].parents_from_prev_generation, + key=partial(graph_dist, genealogical_path[-1]) + )) + + return list(reversed(genealogical_path)) + + +def plot_fitness_with_axvline(generations: List[int], fitnesses: List[float], ax: plt.Axes, current_fitness: float, + axvline_x: int = None): + ax.plot(generations, fitnesses) + ax.set_title(f'Fitness dynamic,\ncurrent: {current_fitness}', fontsize=22) + ax.set_xlabel('Generation') + ax.set_ylabel('Fitness') + if axvline_x is not None: + ax.axvline(x=axvline_x, color='black') + return ax diff --git a/golem/visualisation/opt_history/multiple_fitness_line.py b/golem/visualisation/opt_history/multiple_fitness_line.py index 231556553..38c7267d6 100644 --- a/golem/visualisation/opt_history/multiple_fitness_line.py +++ b/golem/visualisation/opt_history/multiple_fitness_line.py @@ -8,7 +8,7 @@ from golem.core.log import default_log from golem.core.optimisers.opt_history_objects.opt_history import OptHistory -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence from golem.visualisation.opt_history.arg_constraint_wrapper import ArgConstraintWrapper from golem.visualisation.opt_history.fitness_line import setup_fitness_plot from golem.visualisation.opt_history.utils import show_or_save_figure diff --git a/golem/visualisation/opt_viz.py b/golem/visualisation/opt_viz.py index 0c8252023..bd588ea2c 100644 --- a/golem/visualisation/opt_viz.py +++ b/golem/visualisation/opt_viz.py @@ -7,6 +7,7 @@ from golem.visualisation.opt_history.diversity import DiversityLine, DiversityPopulation from golem.visualisation.opt_history.fitness_box import FitnessBox from golem.visualisation.opt_history.fitness_line import FitnessLine, FitnessLineInteractive +from golem.visualisation.opt_history.genealogical_path import GenealogicalPath from golem.visualisation.opt_history.operations_animated_bar import OperationsAnimatedBar from golem.visualisation.opt_history.operations_kde import OperationsKDE @@ -22,6 +23,7 @@ class PlotTypesEnum(Enum): operations_animated_bar = OperationsAnimatedBar diversity_line = DiversityLine diversity_population = DiversityPopulation + genealogical_path = GenealogicalPath @classmethod def member_names(cls): @@ -60,6 +62,7 @@ def __init__(self, history: OptHistory, visuals_params: Optional[Dict[str, Any]] self.operations_animated_bar = OperationsAnimatedBar(history, self.visuals_params).visualize self.diversity_line = DiversityLine(history, self.visuals_params).visualize self.diversity_population = DiversityPopulation(history, self.visuals_params).visualize + self.genealogical_path = GenealogicalPath(history, self.visuals_params).visualize self.log = default_log(self) diff --git a/golem/visualisation/opt_viz_extra.py b/golem/visualisation/opt_viz_extra.py index 2b71606ca..13eb83a24 100644 --- a/golem/visualisation/opt_viz_extra.py +++ b/golem/visualisation/opt_viz_extra.py @@ -9,6 +9,8 @@ import numpy as np import pandas as pd import seaborn as sns +from PIL import Image +from imageio import get_writer, v2 from matplotlib import pyplot as plt from golem.core.dag.graph import Graph @@ -17,8 +19,6 @@ from golem.core.optimisers.opt_history_objects.opt_history import OptHistory from golem.core.paths import default_data_dir from golem.visualisation.graph_viz import GraphVisualizer -from PIL import Image -from imageio import get_writer, v2 class OptHistoryExtraVisualizer: diff --git a/other_requirements/molecules.txt b/other_requirements/molecules.txt index e8021e9b3..a33309c49 100644 --- a/other_requirements/molecules.txt +++ b/other_requirements/molecules.txt @@ -2,6 +2,6 @@ rdkit>=2018.09.1.0 guacamol>=0.5.4 joblib>=0.12.5 requests>=2.30.0 -mol2vec @ git+https://github.com/samoturk/mol2vec +mol2vec==0.2.2 gensim>=4.3.2 torch>=2.0.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f1a3b42cb..7d9797570 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,8 @@ zss>=1.2.0 # Plotting matplotlib>=3.3.1; python_version >= '3.8' pyvis==0.2.1 +# needs to be installed explicitly since no clear dependency is specified in pyvis->Jinja2->MarkupSafe +MarkupSafe==2.1.1 seaborn>=0.9.0 imageio>=2.28.1 Pillow>=9.5.0 @@ -24,7 +26,7 @@ psutil>=5.9.2 # Optimisation hyperopt>=0.2.7 -iOpt==0.1.6 +iOpt==0.2.22 optuna>=3.2.0 # Tests diff --git a/setup.py b/setup.py index 9c248bb9d..3f98be05f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ # The text of the README file NAME = 'thegolem' -VERSION = '0.3.3' +VERSION = '0.4.0' AUTHOR = 'NSS Lab' SHORT_DESCRIPTION = 'Framework for Graph Optimization and Learning by Evolutionary Methods' diff --git a/test/integration/test_external_history_visualize.py b/test/integration/test_external_history_visualize.py index e4550ec7d..a74c2d947 100644 --- a/test/integration/test_external_history_visualize.py +++ b/test/integration/test_external_history_visualize.py @@ -2,18 +2,19 @@ import pytest -from golem.core.optimisers.opt_history_objects.opt_history import OptHistory +from golem.core.optimisers.opt_history_objects.opt_history import OptHistory, lighten_history from golem.core.paths import project_root from golem.visualisation.opt_viz import PlotTypesEnum, OptHistoryVisualizer -@pytest.mark.parametrize('history_path', [ - 'test/data/history_composite_bn_healthcare.json', -]) +@pytest.mark.parametrize('history_path', ['test/data/history_composite_bn_healthcare.json']) @pytest.mark.parametrize('plot_type', PlotTypesEnum) -def test_visualizations_for_external_history(tmp_path, history_path, plot_type): +@pytest.mark.parametrize('is_light_history', [True, False]) +def test_visualizations_for_external_history(tmp_path, history_path, plot_type, is_light_history): history_path = project_root() / history_path history = OptHistory.load(history_path) + if is_light_history: + history = lighten_history(history) save_path = Path(tmp_path, plot_type.name) save_path = save_path.with_suffix('.gif') if plot_type is PlotTypesEnum.operations_animated_bar \ else save_path.with_suffix('.png') diff --git a/test/integration/test_genetic_schemes.py b/test/integration/test_genetic_schemes.py index 02a81a2e1..f720ae5ce 100644 --- a/test/integration/test_genetic_schemes.py +++ b/test/integration/test_genetic_schemes.py @@ -1,16 +1,14 @@ -from functools import partial +from typing import Sequence import numpy as np import pytest -from examples.synthetic_graph_evolution.experiment_setup import run_trial from examples.synthetic_graph_evolution.generators import generate_labeled_graph from examples.synthetic_graph_evolution.tree_search import tree_search_setup from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters from golem.core.optimisers.genetic.operators.base_mutations import MutationTypesEnum from golem.core.optimisers.genetic.operators.crossover import CrossoverTypesEnum from golem.core.optimisers.genetic.operators.inheritance import GeneticSchemeTypesEnum -from golem.utilities.utils import set_random_seed def set_up_params(genetic_scheme: GeneticSchemeTypesEnum): @@ -18,7 +16,7 @@ def set_up_params(genetic_scheme: GeneticSchemeTypesEnum): multi_objective=False, mutation_types=[ MutationTypesEnum.single_add, - MutationTypesEnum.single_drop, + MutationTypesEnum.single_drop ], crossover_types=[CrossoverTypesEnum.none], genetic_scheme_type=genetic_scheme @@ -28,17 +26,20 @@ def set_up_params(genetic_scheme: GeneticSchemeTypesEnum): @pytest.mark.parametrize('genetic_type', GeneticSchemeTypesEnum) def test_genetic_scheme_types(genetic_type): - target_graph = generate_labeled_graph('tree', 4, node_labels=['x']) + target_graph = generate_labeled_graph('tree', 30, node_labels=['x']) num_iterations = 30 gp_params = set_up_params(genetic_type) - set_random_seed(42) - found_graph, history = run_trial(target_graph=target_graph, - optimizer_setup=partial(tree_search_setup, algorithm_parameters=gp_params), - num_iterations=num_iterations) + optimizer, objective = tree_search_setup(target_graph, + num_iterations=num_iterations, + algorithm_parameters=gp_params) + found_graphs = optimizer.optimise(objective) + found_graph = found_graphs[0] if isinstance(found_graphs, Sequence) else found_graphs + history = optimizer.history assert found_graph is not None # at least 20% more generation than early_stopping_iterations were evaluated - assert history.generations_count >= num_iterations // 3 * 1.2 + # (+2 gen for initial assumption and final choice) + assert history.generations_count >= num_iterations // 3 * 1.2 + 2 # metric improved assert np.mean([ind.fitness.value for ind in history.generations[0].data]) > \ np.mean([ind.fitness.value for ind in history.generations[-1].data]) diff --git a/test/unit/api/__init__.py b/test/unit/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/unit/api/test_api.py b/test/unit/api/test_api.py new file mode 100644 index 000000000..7712f24ef --- /dev/null +++ b/test/unit/api/test_api.py @@ -0,0 +1,73 @@ +import datetime +import logging +from functools import partial + +import pickle + +from examples.synthetic_graph_evolution.generators import generate_labeled_graph +from golem.api.main import GOLEM +from golem.core.adapter.nx_adapter import BaseNetworkxAdapter +from golem.core.dag.verification_rules import DEFAULT_DAG_RULES +from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters +from golem.core.optimisers.genetic.operators.base_mutations import MutationTypesEnum +from golem.core.optimisers.genetic.operators.crossover import CrossoverTypesEnum +from golem.core.optimisers.genetic.operators.inheritance import GeneticSchemeTypesEnum +from golem.core.optimisers.objective import Objective +from golem.core.optimisers.optimization_parameters import GraphRequirements +from golem.core.optimisers.optimizer import GraphGenerationParams +from golem.metrics.edit_distance import tree_edit_dist + + +def test_specifying_parameters_through_api(): + """ Tests that parameters for optimizer are specified correctly. """ + + timeout = 1 + size = 16 + node_types = ('a', 'b') + target_graph = generate_labeled_graph('tree', size, node_labels=node_types) + + # Generate initial population with small tree graphs + initial_graphs = [generate_labeled_graph('tree', 5, node_types) for _ in range(10)] + # Setup objective: edit distance to target graph + objective = Objective(partial(tree_edit_dist, target_graph)) + + golem = GOLEM(timeout=timeout, + logging_level=logging.INFO, + early_stopping_iterations=100, + initial_graphs=initial_graphs, + objective=objective, + genetic_scheme_type=GeneticSchemeTypesEnum.parameter_free, + max_pop_size=50, + mutation_types=[MutationTypesEnum.single_add, + MutationTypesEnum.single_drop, + MutationTypesEnum.single_change], + crossover_types=[CrossoverTypesEnum.subtree], + available_node_types=node_types # Node types that can appear in graphs + ) + + # setup with externally specifying params + requirements = GraphRequirements( + early_stopping_iterations=100, + timeout=datetime.timedelta(minutes=timeout), + n_jobs=1, + ) + gp_params = GPAlgorithmParameters( + genetic_scheme_type=GeneticSchemeTypesEnum.parameter_free, + max_pop_size=50, + mutation_types=[MutationTypesEnum.single_add, + MutationTypesEnum.single_drop, + MutationTypesEnum.single_change], + crossover_types=[CrossoverTypesEnum.subtree] + ) + graph_gen_params = GraphGenerationParams( + adapter=BaseNetworkxAdapter(), # Example works with NetworkX graphs + rules_for_constraint=DEFAULT_DAG_RULES, # We don't want cycles in the graph + available_node_types=node_types # Node types that can appear in graphs + ) + + assert golem.gp_algorithm_parameters == gp_params + # compared by pickle dump since there are lots of inner classes with not implemented __eq__ magic methods + # probably needs to be fixed + assert pickle.dumps(golem.graph_generation_parameters) == pickle.dumps(graph_gen_params) + # need to be compared by dicts since the classes itself are different + assert golem.graph_requirements.__dict__ == requirements.__dict__ diff --git a/test/unit/optimizers/gp_operators/test_mutation.py b/test/unit/optimizers/gp_operators/test_mutation.py index fe392c3ab..f89bd7571 100644 --- a/test/unit/optimizers/gp_operators/test_mutation.py +++ b/test/unit/optimizers/gp_operators/test_mutation.py @@ -79,13 +79,17 @@ def test_add_as_parent_node(graph): Test correctness of adding as a parent """ new_graph = deepcopy(graph) - node_to_mutate = new_graph.nodes[1] params = get_mutation_params() node_factory = params['graph_gen_params'].node_factory - add_separate_parent_node(new_graph, node_to_mutate, node_factory) + prev_nodes = new_graph.nodes[:] + add_separate_parent_node(new_graph, node_factory) + new_nodes = [node for node in new_graph.nodes if node not in prev_nodes] - assert len(node_to_mutate.nodes_from) > len(graph.nodes[1].nodes_from) + assert len(new_nodes) == 1 + assert not new_nodes[0].nodes_from + assert new_graph.node_children(new_nodes[0]) + assert new_graph.length > graph.length @pytest.mark.parametrize('graph', [simple_linear_graph(), tree_graph(), simple_cycled_graph()]) @@ -94,14 +98,16 @@ def test_add_as_child_node(graph): Test correctness of adding as a child """ new_graph = deepcopy(graph) - node_to_mutate = new_graph.nodes[1] params = get_mutation_params() node_factory = params['graph_gen_params'].node_factory - add_as_child(new_graph, node_to_mutate, node_factory) + prev_nodes = new_graph.nodes[:] + add_as_child(new_graph, node_factory) + new_nodes = [node for node in new_graph.nodes if node not in prev_nodes] + assert len(new_nodes) == 1 + assert new_nodes[0].nodes_from assert new_graph.length > graph.length - assert new_graph.node_children(node_to_mutate) != graph.node_children(node_to_mutate) @pytest.mark.parametrize('graph', [simple_linear_graph(), tree_graph(), simple_cycled_graph()]) @@ -110,14 +116,16 @@ def test_add_as_intermediate_node(graph): Test correctness of adding as an intermediate node """ new_graph = deepcopy(graph) - node_to_mutate = new_graph.nodes[1] params = get_mutation_params() node_factory = params['graph_gen_params'].node_factory + prev_nodes = new_graph.nodes[:] + add_intermediate_node(new_graph, node_factory) + new_nodes = [node for node in new_graph.nodes if node not in prev_nodes] - add_intermediate_node(new_graph, node_to_mutate, node_factory) - + assert len(new_nodes) == 1 + assert new_nodes[0].nodes_from + assert new_graph.node_children(new_nodes[0]) assert new_graph.length > graph.length - assert node_to_mutate.nodes_from[0] != graph.nodes[1].nodes_from[0] @pytest.mark.parametrize('graph', [simple_linear_graph(), tree_graph(), simple_cycled_graph()]) diff --git a/test/unit/optimizers/gp_operators/test_population_size.py b/test/unit/optimizers/gp_operators/test_population_size.py index 56ee91c80..fb3de1d43 100644 --- a/test/unit/optimizers/gp_operators/test_population_size.py +++ b/test/unit/optimizers/gp_operators/test_population_size.py @@ -1,13 +1,11 @@ -import pytest - from golem.core.optimisers.archive import GenerationKeeper from golem.core.optimisers.fitness import SingleObjFitness -from golem.core.optimisers.genetic.parameters.population_size import PopulationSize, AdaptivePopulationSize, \ +from golem.core.optimisers.genetic.parameters.population_size import AdaptivePopulationSize, \ ConstRatePopulationSize from golem.core.optimisers.graph import OptGraph, OptNode from golem.core.optimisers.objective import Objective from golem.core.optimisers.opt_history_objects.individual import Individual -from golem.core.utilities.sequence_iterator import SequenceIterator, fibonacci_sequence +from golem.utilities.sequence_iterator import SequenceIterator def custom_objective(): diff --git a/test/unit/optimizers/gp_operators/test_selection.py b/test/unit/optimizers/gp_operators/test_selection.py index b0a8e67eb..922fe03fe 100644 --- a/test/unit/optimizers/gp_operators/test_selection.py +++ b/test/unit/optimizers/gp_operators/test_selection.py @@ -1,9 +1,15 @@ from golem.core.adapter import DirectAdapter from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters +from golem.core.optimisers.genetic.operators.operator import PopulationT from golem.core.optimisers.genetic.operators.selection import Selection, SelectionTypesEnum, random_selection from golem.core.optimisers.opt_history_objects.individual import Individual from test.unit.optimizers.test_evaluation import get_objective from test.unit.utils import graph_first, graph_second, graph_third, graph_fourth, graph_fifth +from random import sample + + +def custom_selection(population: PopulationT, pop_size: int): + return sample(population, pop_size) def get_population(): @@ -56,3 +62,13 @@ def test_individuals_selection_equality_individuals(): selected_individuals_ref = [str(ind) for ind in selected_individuals] assert (len(selected_individuals) == num_of_inds and len(set(selected_individuals_ref)) == 1) + + +def test_custom_selection(): + num_of_inds = 3 + population = get_population() + requirements = GPAlgorithmParameters(selection_types=[custom_selection], pop_size=num_of_inds) + selection = Selection(requirements) + selected_individuals = selection(population) + assert (all([ind in population for ind in selected_individuals]) and + len(selected_individuals) == num_of_inds) diff --git a/test/unit/optimizers/test_composing_history.py b/test/unit/optimizers/test_composing_history.py index 76977ad1c..e001cc7d4 100644 --- a/test/unit/optimizers/test_composing_history.py +++ b/test/unit/optimizers/test_composing_history.py @@ -60,7 +60,8 @@ def generate_history(request) -> OptHistory: ind.set_native_generation(gen_num) new_pop.append(ind) history.add_to_history(new_pop) - history.add_to_archive_history(new_pop) + # since only n best individuals need to be added to archive history + history.add_to_archive_history([sorted(new_pop, key=lambda ind: ind.fitness.values[0], reverse=False)[0]]) return history @@ -266,7 +267,8 @@ def test_newly_generated_history(n_jobs: int): def test_history_show_saving_plots(tmp_path, plot_type: PlotTypesEnum, generate_history): save_path = Path(tmp_path, plot_type.name) gif_plots = [PlotTypesEnum.operations_animated_bar, - PlotTypesEnum.diversity_population] + PlotTypesEnum.diversity_population, + PlotTypesEnum.genealogical_path] save_path = save_path.with_suffix('.gif') if plot_type in gif_plots \ else save_path.with_suffix('.png') history: OptHistory = generate_history @@ -329,6 +331,38 @@ def test_load_zero_generations_history(): assert history.objective is not None +@pytest.mark.parametrize('generate_history', [[100, 100, create_individual]], indirect=True) +def test_save_load_light_history(generate_history): + history = generate_history + file_name = 'light_history.json' + path_to_dir = os.path.join(project_root(), 'test', 'data') + path_to_history = os.path.join(path_to_dir, file_name) + history.save(json_file_path=path_to_history, is_save_light=True) + assert file_name in os.listdir(path_to_dir) + loaded_history = OptHistory().load(path_to_history) + assert isinstance(loaded_history, OptHistory) + assert len(loaded_history.archive_history) == len(loaded_history.generations) == 100 + for i, _ in enumerate(loaded_history.generations): + assert len(loaded_history.generations[i]) == len(loaded_history.archive_history[i]) == 1 + os.remove(path=os.path.join(path_to_dir, file_name)) + + +@pytest.mark.parametrize('generate_history', [[50, 30, create_individual]], indirect=True) +def test_light_history_is_significantly_lighter(generate_history): + """ Checks if light version of history weights signif """ + history = generate_history + file_name_light = 'light_history.json' + file_name_heavy = 'heavy_history.json' + path_to_dir = os.path.join(project_root(), 'test', 'data') + history.save(json_file_path=os.path.join(path_to_dir, file_name_light), is_save_light=True) + history.save(json_file_path=os.path.join(path_to_dir, file_name_heavy), is_save_light=False) + light_history_size = os.stat(os.path.join(path_to_dir, file_name_light)).st_size + heavy_history_size = os.stat(os.path.join(path_to_dir, file_name_heavy)).st_size + assert light_history_size * 25 <= heavy_history_size + os.remove(path=os.path.join(path_to_dir, file_name_light)) + os.remove(path=os.path.join(path_to_dir, file_name_heavy)) + + def assert_intermediate_metrics(graph: MockDomainStructure): seen_metrics = [] for node in graph.nodes: diff --git a/test/unit/optimizers/test_evaluation.py b/test/unit/optimizers/test_evaluation.py index 60cf42a64..45623ab54 100644 --- a/test/unit/optimizers/test_evaluation.py +++ b/test/unit/optimizers/test_evaluation.py @@ -8,11 +8,12 @@ from golem.core.dag.graph import Graph from golem.core.optimisers.fitness import Fitness, null_fitness from golem.core.optimisers.genetic.evaluation import MultiprocessingDispatcher, SequentialDispatcher, \ - ObjectiveEvaluationDispatcher, determine_n_jobs + ObjectiveEvaluationDispatcher from golem.core.optimisers.meta.surrogate_evaluator import SurrogateDispatcher from golem.core.optimisers.objective import Objective from golem.core.optimisers.opt_history_objects.individual import Individual from golem.core.optimisers.timer import OptimisationTimer +from golem.utilities.utilities import determine_n_jobs from test.unit.utils import graph_first, graph_second, graph_third, graph_fourth, RandomMetric @@ -61,7 +62,7 @@ def test_dispatchers_with_faulty_objectives(objective, dispatcher): adapter, population = set_up_tests() evaluator = dispatcher.dispatch(objective) - assert evaluator(population) is None + assert evaluator(population) == [] @pytest.mark.parametrize('dispatcher', [ diff --git a/test/unit/optimizers/test_random_graph_factory.py b/test/unit/optimizers/test_random_graph_factory.py index 1acea2379..44669c916 100644 --- a/test/unit/optimizers/test_random_graph_factory.py +++ b/test/unit/optimizers/test_random_graph_factory.py @@ -31,4 +31,4 @@ def test_gp_composer_random_graph_generation_looping(max_depth): assert verifier(graph) is True assert graph.depth <= requirements.max_depth # at least one graph has depth greater than a max_depth quarter - assert np.any([graph.depth >= math.ceil(max_depth / 4) for graph in graphs]) + assert np.any([graph.depth >= math.ceil(max_depth * 0.25) for graph in graphs]) diff --git a/test/unit/test_logger.py b/test/unit/test_logger.py index e28999937..3097cdf73 100644 --- a/test/unit/test_logger.py +++ b/test/unit/test_logger.py @@ -1,13 +1,16 @@ import logging import os +import traceback +from copy import copy from importlib import reload from pathlib import Path +from unittest import mock import pytest from golem.core.log import DEFAULT_LOG_PATH, Log, default_log -from golem.core.utilities.grouped_condition import GroupedCondition -from golem.core.utilities.singleton_meta import SingletonMeta +from golem.utilities.grouped_condition import GroupedCondition +from golem.utilities.singleton_meta import SingletonMeta @pytest.fixture() @@ -122,11 +125,52 @@ def test_reset_logging_level(): log.reset_logging_level(20) b.message('test_message_4') # should be shown since logging level is info now - c.message('test_message_5') # should be shown since logging level is info now + c.message('test_message_5') # should be shown since logging level is info now content = '' if Path(DEFAULT_LOG_PATH).exists(): content = Path(DEFAULT_LOG_PATH).read_text() - assert (lambda message: message in content, ['test_message_1', 'test_message_4', 'test_message_5']) - assert (lambda message: message not in content, ['test_message_2', 'test_message_3']) + assert all(map(lambda message: message in content, ['test_message_1', 'test_message_4', 'test_message_5'])) + assert all(map(lambda message: message not in content, ['test_message_2', 'test_message_3'])) + + +def get_log_or_raise_output(exc_message, exc_type, imitate_non_test_launch): + if imitate_non_test_launch: + environ_mock = copy(os.environ) + del environ_mock['PYTEST_CURRENT_TEST'] + with mock.patch.dict(os.environ, environ_mock, clear=True): + default_log().log_or_raise('message', exc_message) + output = Path(DEFAULT_LOG_PATH).read_text() + else: + with pytest.raises(exc_type, match=str(exc_message)) as exc_info: + default_log().log_or_raise('message', exc_message) + output = ''.join(traceback.format_exception(exc_type, exc_info.value, exc_info.tb)) + return output + + +@pytest.mark.parametrize('exc_message, exc_type', + [('And therefore could not continue.', Exception), + (ValueError('And therefore could not continue.'), ValueError)]) +@pytest.mark.parametrize('imitate_non_test_launch', [False, True]) +def test_log_or_raise(exc_message, exc_type, imitate_non_test_launch): + output = get_log_or_raise_output(exc_message, exc_type, imitate_non_test_launch) + assert str(exc_message) in output + + +@pytest.mark.parametrize('cause', + [ArithmeticError('Something went wrong.'), ValueError('Unbelievable!')]) +@pytest.mark.parametrize('exc_message, exc_type', + [('And therefore could not continue.', Exception), + (ValueError('And therefore could not continue.'), ValueError)]) +@pytest.mark.parametrize('imitate_non_test_launch', [False, True]) +def test_log_or_raise_with_cause_exception(cause, exc_message, exc_type, imitate_non_test_launch): + try: + raise cause + except type(cause): + cause_formatted = traceback.format_exc() + output = get_log_or_raise_output(exc_message, exc_type, imitate_non_test_launch) + assert all(map(lambda text: text in output, + [cause_formatted, + 'The above exception was the direct cause of the following exception:', + str(exc_message)])) diff --git a/test/unit/tuning/test_tuning.py b/test/unit/tuning/test_tuning.py index cec2a15a3..943d3dcac 100644 --- a/test/unit/tuning/test_tuning.py +++ b/test/unit/tuning/test_tuning.py @@ -9,9 +9,9 @@ from golem.core.tuning.search_space import SearchSpace from golem.core.tuning.sequential import SequentialTuner from golem.core.tuning.simultaneous import SimultaneousTuner -from test.unit.mocks.common_mocks import MockAdapter, MockObjectiveEvaluate, mock_graph_with_params, \ - opt_graph_with_params, MockNode, MockDomainStructure -from test.unit.utils import ParamsSumMetric, ParamsProductMetric +from test.unit.mocks.common_mocks import (MockAdapter, MockDomainStructure, MockNode, MockObjectiveEvaluate, + mock_graph_with_params, opt_graph_with_params) +from test.unit.utils import ParamsProductMetric, ParamsSumMetric def not_tunable_mock_graph(): @@ -35,8 +35,12 @@ def search_space(): 'hyperopt-dist': hp.loguniform, 'sampling-scope': [1e-3, 1], 'type': 'continuous' - } - }, + }, + 'a3': { + 'hyperopt-dist': hp.choice, + 'sampling-scope': [['A', 'B', 'C']], + 'type': 'categorical' + }}, 'b': { 'b1': { 'hyperopt-dist': hp.choice, @@ -48,6 +52,11 @@ def search_space(): 'sampling-scope': [0.05, 1.0], 'type': 'continuous' }, + 'b3': { + 'hyperopt-dist': hp.randint, + 'sampling-scope': [1, 1000], + 'type': 'discrete' + } }, 'e': { 'e1': { @@ -109,7 +118,7 @@ def test_node_tuning(search_space, graph): assert tuner.init_metric >= tuner.obtained_metric -@pytest.mark.parametrize('tuner_cls', [OptunaTuner]) +@pytest.mark.parametrize('tuner_cls', [OptunaTuner, IOptTuner]) @pytest.mark.parametrize('init_graph, adapter, obj_eval', [(mock_graph_with_params(), MockAdapter(), MockObjectiveEvaluate(Objective({'sum_metric': ParamsSumMetric.get_value, @@ -121,10 +130,23 @@ def test_node_tuning(search_space, graph): is_multi_objective=True)))]) def test_multi_objective_tuning(search_space, tuner_cls, init_graph, adapter, obj_eval): init_metric = obj_eval.evaluate(init_graph) - tuner = tuner_cls(obj_eval, search_space, adapter, iterations=20, objectives_number=2) + tuner = tuner_cls(obj_eval, search_space, adapter, iterations=20, early_stopping_rounds=3) tuned_graphs = tuner.tune(deepcopy(init_graph), show_progress=False) for graph in tuned_graphs: assert type(graph) == type(init_graph) final_metric = obj_eval.evaluate(graph) assert final_metric is not None assert not init_metric.dominates(final_metric) + + +@pytest.mark.parametrize('tuner_cls', [SequentialTuner, SimultaneousTuner]) +def test_hyperopt_returns_native_types(search_space, tuner_cls): + obj_eval = MockObjectiveEvaluate(Objective({'sum_metric': ParamsSumMetric.get_value})) + adapter = MockAdapter() + graph = opt_graph_with_params() + tuner = tuner_cls(obj_eval, search_space, adapter, iterations=20) + tuned_graph = tuner.tune(deepcopy(graph)) + for node in tuned_graph.nodes: + for param, val in node.parameters.items(): + assert val.__class__.__module__ != 'numpy', (f'The parameter "{param}" should not be a numpy type. ' + f'Got "{type(val)}".') diff --git a/test/unit/utilities/test_data_structures.py b/test/unit/utilities/test_data_structures.py index 8661e6675..91f33ef45 100644 --- a/test/unit/utilities/test_data_structures.py +++ b/test/unit/utilities/test_data_structures.py @@ -1,4 +1,4 @@ -from golem.core.utilities.data_structures import UniqueList, ensure_wrapped_in_sequence +from golem.utilities.data_structures import UniqueList, ensure_wrapped_in_sequence def test_init(): diff --git a/test/unit/utilities/test_iterator.py b/test/unit/utilities/test_iterator.py index 8bcf4ebc9..b39ad34e5 100644 --- a/test/unit/utilities/test_iterator.py +++ b/test/unit/utilities/test_iterator.py @@ -1,4 +1,4 @@ -from golem.core.utilities.sequence_iterator import fibonacci_sequence, SequenceIterator +from golem.utilities.sequence_iterator import fibonacci_sequence, SequenceIterator def test_iterator_without_constraints(): diff --git a/test/unit/visualisation/test_visualisation_utils.py b/test/unit/visualisation/test_visualisation_utils.py index cd470c696..cbd875a29 100644 --- a/test/unit/visualisation/test_visualisation_utils.py +++ b/test/unit/visualisation/test_visualisation_utils.py @@ -1,9 +1,8 @@ from golem.core.adapter import DirectAdapter from golem.core.dag.convert import graph_structure_as_nx_graph -from golem.core.dag.graph_utils import distance_to_primary_level from golem.core.optimisers.fitness.multi_objective_fitness import MultiObjFitness from golem.core.optimisers.opt_history_objects.individual import Individual -from golem.visualisation.graph_viz import get_hierarchy_pos +from golem.visualisation.graph_viz import GraphVisualizer from golem.visualisation.opt_viz_extra import extract_objectives from test.unit.utils import graph_first @@ -41,12 +40,10 @@ def test_hierarchy_pos(): 1: ['a', 'b'], 2: ['a']} - graph, node_labels = graph_structure_as_nx_graph(graph) - for n, data in graph.nodes(data=True): - data['hierarchy_level'] = distance_to_primary_level(node_labels[n]) - node_labels[n] = str(node_labels[n]) + nx_graph, nodes_dict = graph_structure_as_nx_graph(graph) + node_labels = {uid: str(node) for uid, node in nodes_dict.items()} - pos, _ = get_hierarchy_pos(graph) + pos = GraphVisualizer._get_hierarchy_pos_by_distance_to_primary_level(nx_graph, nodes_dict) comparable_lists_y = make_comparable_lists(pos, real_hierarchy_levels_y, node_labels, 1, reverse=True) comparable_lists_x = make_comparable_lists(pos, real_hierarchy_levels_x,