From 52ad42087cacaf2bced42f18743b64a69c7ec8da Mon Sep 17 00:00:00 2001 From: Sergey Date: Fri, 27 Oct 2023 13:36:58 +0300 Subject: [PATCH] Fix some issues (#230) Fix Problem with OptHistory #229 Fix Refactor utilities #173 Fix potential problem with n_jobs. It may appear if some multiprocessing modules cannot work with n_jobs < 0. Now n_jobs is checked and converted to positive value in GraphRequirements Fix problem with retaining composing time calculation. Initial generation is taked as full generation, but it does not consume any time for preparing. Also fix problem with generation num increment. Change version info from `0.3.3` to `0.4.0`. --- docs/source/conf.py | 2 +- examples/synthetic_graph_evolution/utils.py | 2 +- golem/core/adapter/adapt_registry.py | 2 +- golem/core/dag/graph_utils.py | 2 +- golem/core/dag/linked_graph.py | 2 +- golem/core/dag/linked_graph_node.py | 2 +- golem/core/log.py | 2 +- .../core/optimisers/adaptive/agent_trainer.py | 2 +- golem/core/optimisers/advisor.py | 2 +- golem/core/optimisers/fitness/fitness.py | 2 +- golem/core/optimisers/genetic/evaluation.py | 19 +++---------------- .../genetic/operators/base_mutations.py | 2 +- .../optimisers/genetic/operators/crossover.py | 2 +- .../optimisers/genetic/operators/elitism.py | 2 +- .../genetic/operators/inheritance.py | 2 +- .../genetic/operators/regularization.py | 2 +- .../genetic/operators/reproduction.py | 2 +- .../optimisers/genetic/operators/selection.py | 2 +- .../genetic/parameters/population_size.py | 4 ++-- .../opt_history_objects/generation.py | 2 +- .../opt_history_objects/opt_history.py | 2 +- .../opt_history_objects/parent_operator.py | 2 +- .../optimisers/optimization_parameters.py | 7 ++++--- golem/core/optimisers/optimizer.py | 2 +- .../core/optimisers/populational_optimizer.py | 4 ++-- .../random/random_mutation_optimizer.py | 2 +- golem/core/optimisers/random/random_search.py | 2 +- golem/core/optimisers/timer.py | 4 ++-- golem/core/tuning/tuner_interface.py | 2 +- golem/core/utilities/__init__.py | 0 golem/serializers/serializer.py | 2 +- .../graph_sa/results/sa_analysis_results.py | 2 +- golem/{core => }/utilities/data_structures.py | 0 .../{core => }/utilities/grouped_condition.py | 0 golem/{core => }/utilities/random.py | 0 .../{core => }/utilities/sequence_iterator.py | 2 +- golem/{core => }/utilities/serializable.py | 0 golem/{core => }/utilities/singleton_meta.py | 0 golem/utilities/utilities.py | 15 +++++++++++++++ .../opt_history/multiple_fitness_line.py | 2 +- setup.py | 2 +- .../gp_operators/test_population_size.py | 6 ++---- test/unit/optimizers/test_evaluation.py | 3 ++- test/unit/test_logger.py | 4 ++-- test/unit/utilities/test_data_structures.py | 2 +- test/unit/utilities/test_iterator.py | 2 +- 46 files changed, 65 insertions(+), 63 deletions(-) delete mode 100644 golem/core/utilities/__init__.py rename golem/{core => }/utilities/data_structures.py (100%) rename golem/{core => }/utilities/grouped_condition.py (100%) rename golem/{core => }/utilities/random.py (100%) rename golem/{core => }/utilities/sequence_iterator.py (97%) rename golem/{core => }/utilities/serializable.py (100%) rename golem/{core => }/utilities/singleton_meta.py (100%) create mode 100644 golem/utilities/utilities.py diff --git a/docs/source/conf.py b/docs/source/conf.py index ad3e3e349..fa13e6ead 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -24,7 +24,7 @@ author = 'NSS Lab' # The full version, including alpha/beta/rc tags -release = '0.3.3' +release = '0.4.0' # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be diff --git a/examples/synthetic_graph_evolution/utils.py b/examples/synthetic_graph_evolution/utils.py index 84aac2e88..7055a5056 100644 --- a/examples/synthetic_graph_evolution/utils.py +++ b/examples/synthetic_graph_evolution/utils.py @@ -10,7 +10,7 @@ from golem.core.adapter.nx_adapter import BaseNetworkxAdapter from golem.core.optimisers.opt_history_objects.opt_history import OptHistory -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence from golem.visualisation.graph_viz import GraphVisualizer diff --git a/golem/core/adapter/adapt_registry.py b/golem/core/adapter/adapt_registry.py index 39c9554a2..850c24187 100644 --- a/golem/core/adapter/adapt_registry.py +++ b/golem/core/adapter/adapt_registry.py @@ -2,7 +2,7 @@ from functools import partial from typing import Callable -from golem.core.utilities.singleton_meta import SingletonMeta +from golem.utilities.singleton_meta import SingletonMeta class AdaptRegistry(metaclass=SingletonMeta): diff --git a/golem/core/dag/graph_utils.py b/golem/core/dag/graph_utils.py index 99a7d7cc8..4516c736f 100644 --- a/golem/core/dag/graph_utils.py +++ b/golem/core/dag/graph_utils.py @@ -1,6 +1,6 @@ from typing import Sequence, List, TYPE_CHECKING, Callable, Union -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence if TYPE_CHECKING: from golem.core.dag.graph import Graph diff --git a/golem/core/dag/linked_graph.py b/golem/core/dag/linked_graph.py index 8eceb77a8..789c3f6ab 100644 --- a/golem/core/dag/linked_graph.py +++ b/golem/core/dag/linked_graph.py @@ -8,7 +8,7 @@ from golem.core.dag.graph_node import GraphNode from golem.core.dag.graph_utils import ordered_subnodes_hierarchy, node_depth, graph_has_cycle from golem.core.paths import copy_doc -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence, Copyable, remove_items +from golem.utilities.data_structures import ensure_wrapped_in_sequence, Copyable, remove_items NodePostprocessCallable = Callable[[Graph, Sequence[GraphNode]], Any] diff --git a/golem/core/dag/linked_graph_node.py b/golem/core/dag/linked_graph_node.py index 964b5824f..20a19ac32 100644 --- a/golem/core/dag/linked_graph_node.py +++ b/golem/core/dag/linked_graph_node.py @@ -1,6 +1,6 @@ from typing import Union, Optional, Iterable, List from golem.core.dag.graph_node import GraphNode -from golem.core.utilities.data_structures import UniqueList +from golem.utilities.data_structures import UniqueList class LinkedGraphNode(GraphNode): diff --git a/golem/core/log.py b/golem/core/log.py index a6b9ea9a5..83a93e538 100644 --- a/golem/core/log.py +++ b/golem/core/log.py @@ -8,7 +8,7 @@ from logging.handlers import RotatingFileHandler from typing import Optional, Tuple, Union -from golem.core.utilities.singleton_meta import SingletonMeta +from golem.utilities.singleton_meta import SingletonMeta from golem.core.paths import default_data_dir DEFAULT_LOG_PATH = pathlib.Path(default_data_dir(), 'log.log') diff --git a/golem/core/optimisers/adaptive/agent_trainer.py b/golem/core/optimisers/adaptive/agent_trainer.py index d01f5a4f6..80e9b6c99 100644 --- a/golem/core/optimisers/adaptive/agent_trainer.py +++ b/golem/core/optimisers/adaptive/agent_trainer.py @@ -16,7 +16,7 @@ from golem.core.optimisers.opt_history_objects.individual import Individual from golem.core.optimisers.opt_history_objects.opt_history import OptHistory from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator -from golem.core.utilities.data_structures import unzip +from golem.utilities.data_structures import unzip class AgentTrainer: diff --git a/golem/core/optimisers/advisor.py b/golem/core/optimisers/advisor.py index a7f50624a..f53d1602d 100644 --- a/golem/core/optimisers/advisor.py +++ b/golem/core/optimisers/advisor.py @@ -1,6 +1,6 @@ from typing import List, Any, TypeVar, Generic -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum NodeType = TypeVar('NodeType') diff --git a/golem/core/optimisers/fitness/fitness.py b/golem/core/optimisers/fitness/fitness.py index 33cdbd446..eeb2b3f0f 100644 --- a/golem/core/optimisers/fitness/fitness.py +++ b/golem/core/optimisers/fitness/fitness.py @@ -3,7 +3,7 @@ import numpy as np -from golem.core.utilities.data_structures import Comparable +from golem.utilities.data_structures import Comparable class Fitness(Comparable): diff --git a/golem/core/optimisers/genetic/evaluation.py b/golem/core/optimisers/genetic/evaluation.py index 8c59fb479..14ee73f82 100644 --- a/golem/core/optimisers/genetic/evaluation.py +++ b/golem/core/optimisers/genetic/evaluation.py @@ -7,7 +7,7 @@ from functools import partial from typing import List, Optional, Sequence, Tuple, TypeVar, Dict -from joblib import Parallel, cpu_count, delayed +from joblib import Parallel, delayed from golem.core.adapter import BaseOptimizationAdapter from golem.core.dag.graph import Graph @@ -18,8 +18,9 @@ from golem.core.optimisers.objective import GraphFunction, ObjectiveFunction from golem.core.optimisers.opt_history_objects.individual import GraphEvalResult from golem.core.optimisers.timer import Timer, get_forever_timer -from golem.core.utilities.serializable import Serializable +from golem.utilities.serializable import Serializable from golem.utilities.memory import MemoryAnalytics +from golem.utilities.utilities import determine_n_jobs # the percentage of successful evaluations, # at which evolution is not threatened with stagnation at the moment @@ -276,17 +277,3 @@ def evaluate_population(self, individuals: PopulationT) -> PopulationT: individuals_evaluated = self.apply_evaluation_results(individuals_to_evaluate, evaluation_results) evaluated_population = individuals_evaluated + individuals_to_skip return evaluated_population - - -def determine_n_jobs(n_jobs=-1, logger=None): - cpu_num = cpu_count() - if n_jobs > cpu_num: - n_jobs = cpu_num - elif n_jobs <= 0: - if n_jobs <= -cpu_num - 1 or n_jobs == 0: - raise ValueError(f"Unproper `n_jobs` = {n_jobs}. " - f"`n_jobs` should be between ({-cpu_num}, {cpu_num}) except 0") - n_jobs = cpu_num + 1 + n_jobs - if logger: - logger.info(f"Number of used CPU's: {n_jobs}") - return n_jobs diff --git a/golem/core/optimisers/genetic/operators/base_mutations.py b/golem/core/optimisers/genetic/operators/base_mutations.py index 6982dd725..9dc24d7f1 100644 --- a/golem/core/optimisers/genetic/operators/base_mutations.py +++ b/golem/core/optimisers/genetic/operators/base_mutations.py @@ -11,7 +11,7 @@ from golem.core.optimisers.opt_node_factory import OptNodeFactory from golem.core.optimisers.optimization_parameters import GraphRequirements from golem.core.optimisers.optimizer import GraphGenerationParams, AlgorithmParameters -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum if TYPE_CHECKING: from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters diff --git a/golem/core/optimisers/genetic/operators/crossover.py b/golem/core/optimisers/genetic/operators/crossover.py index 508623bfd..866b0d279 100644 --- a/golem/core/optimisers/genetic/operators/crossover.py +++ b/golem/core/optimisers/genetic/operators/crossover.py @@ -13,7 +13,7 @@ from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator from golem.core.optimisers.optimization_parameters import GraphRequirements from golem.core.optimisers.optimizer import GraphGenerationParams -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum if TYPE_CHECKING: from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters diff --git a/golem/core/optimisers/genetic/operators/elitism.py b/golem/core/optimisers/genetic/operators/elitism.py index 6e4e4a052..e462659ae 100644 --- a/golem/core/optimisers/genetic/operators/elitism.py +++ b/golem/core/optimisers/genetic/operators/elitism.py @@ -1,7 +1,7 @@ from random import shuffle from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum class ElitismTypesEnum(Enum): diff --git a/golem/core/optimisers/genetic/operators/inheritance.py b/golem/core/optimisers/genetic/operators/inheritance.py index 7b64432bb..b0e8807f1 100644 --- a/golem/core/optimisers/genetic/operators/inheritance.py +++ b/golem/core/optimisers/genetic/operators/inheritance.py @@ -2,7 +2,7 @@ from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator from golem.core.optimisers.genetic.operators.selection import Selection -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum if TYPE_CHECKING: from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters diff --git a/golem/core/optimisers/genetic/operators/regularization.py b/golem/core/optimisers/genetic/operators/regularization.py index 9b3d024bc..7d8e7dbe2 100644 --- a/golem/core/optimisers/genetic/operators/regularization.py +++ b/golem/core/optimisers/genetic/operators/regularization.py @@ -7,7 +7,7 @@ from golem.core.optimisers.opt_history_objects.individual import Individual from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator from golem.core.optimisers.optimizer import GraphGenerationParams -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum if TYPE_CHECKING: from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters diff --git a/golem/core/optimisers/genetic/operators/reproduction.py b/golem/core/optimisers/genetic/operators/reproduction.py index 60de62f37..bbaacde4c 100644 --- a/golem/core/optimisers/genetic/operators/reproduction.py +++ b/golem/core/optimisers/genetic/operators/reproduction.py @@ -10,7 +10,7 @@ from golem.core.optimisers.genetic.operators.operator import PopulationT, EvaluationOperator from golem.core.optimisers.genetic.operators.selection import Selection from golem.core.optimisers.populational_optimizer import EvaluationAttemptsError -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence class ReproductionController: diff --git a/golem/core/optimisers/genetic/operators/selection.py b/golem/core/optimisers/genetic/operators/selection.py index a06b234b0..14850a83f 100644 --- a/golem/core/optimisers/genetic/operators/selection.py +++ b/golem/core/optimisers/genetic/operators/selection.py @@ -5,7 +5,7 @@ from typing import Callable, List, Optional from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum class SelectionTypesEnum(Enum): diff --git a/golem/core/optimisers/genetic/parameters/population_size.py b/golem/core/optimisers/genetic/parameters/population_size.py index fbdebfd63..275f05812 100644 --- a/golem/core/optimisers/genetic/parameters/population_size.py +++ b/golem/core/optimisers/genetic/parameters/population_size.py @@ -7,8 +7,8 @@ from golem.core.optimisers.genetic.operators.inheritance import GeneticSchemeTypesEnum from golem.core.optimisers.genetic.operators.operator import PopulationT from golem.core.optimisers.genetic.parameters.parameter import AdaptiveParameter -from golem.core.utilities.data_structures import BidirectionalIterator -from golem.core.utilities.sequence_iterator import fibonacci_sequence, SequenceIterator +from golem.utilities.data_structures import BidirectionalIterator +from golem.utilities.sequence_iterator import fibonacci_sequence, SequenceIterator PopulationSize = AdaptiveParameter[int] diff --git a/golem/core/optimisers/opt_history_objects/generation.py b/golem/core/optimisers/opt_history_objects/generation.py index 4106bf853..3f385e2e6 100644 --- a/golem/core/optimisers/opt_history_objects/generation.py +++ b/golem/core/optimisers/opt_history_objects/generation.py @@ -4,7 +4,7 @@ from copy import deepcopy, copy from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Union -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence if TYPE_CHECKING: from golem.core.optimisers.opt_history_objects.individual import Individual diff --git a/golem/core/optimisers/opt_history_objects/opt_history.py b/golem/core/optimisers/opt_history_objects/opt_history.py index b14cd551e..dafc84ca4 100644 --- a/golem/core/optimisers/opt_history_objects/opt_history.py +++ b/golem/core/optimisers/opt_history_objects/opt_history.py @@ -99,7 +99,7 @@ def save_current_results(self, save_dir: Optional[os.PathLike] = None): last_gen = self.generations[last_gen_id] for individual in last_gen: ind_path = Path(save_dir, str(last_gen_id), str(individual.uid)) - ind_path.mkdir(exist_ok=True) + ind_path.mkdir(exist_ok=True, parents=True) individual.save(json_file_path=ind_path / f'{str(individual.uid)}.json') except Exception as ex: self._log.exception(ex) diff --git a/golem/core/optimisers/opt_history_objects/parent_operator.py b/golem/core/optimisers/opt_history_objects/parent_operator.py index d80c51c10..55e944b0a 100644 --- a/golem/core/optimisers/opt_history_objects/parent_operator.py +++ b/golem/core/optimisers/opt_history_objects/parent_operator.py @@ -5,7 +5,7 @@ from uuid import uuid4 from golem.core.optimisers.opt_history_objects.individual import Individual -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence @dataclass(frozen=True) diff --git a/golem/core/optimisers/optimization_parameters.py b/golem/core/optimisers/optimization_parameters.py index 430635c08..cd6d3601b 100644 --- a/golem/core/optimisers/optimization_parameters.py +++ b/golem/core/optimisers/optimization_parameters.py @@ -5,6 +5,7 @@ from typing import Optional from golem.core.paths import default_data_dir +from golem.utilities.utilities import determine_n_jobs @dataclass @@ -85,9 +86,9 @@ class GraphRequirements(OptimizationParameters): max_arity: int = 4 def __post_init__(self): - excluded_fields = ['n_jobs'] + # check and convert n_jobs to non-negative + self.n_jobs = determine_n_jobs(self.n_jobs) + for field_name, field_value in dataclasses.asdict(self).items(): - if field_name in excluded_fields: - continue if isinstance(field_value, Number) and field_value < 0: raise ValueError(f'Value of {field_name} must be non-negative') diff --git a/golem/core/optimisers/optimizer.py b/golem/core/optimisers/optimizer.py index 2459b9a3e..5a54bb501 100644 --- a/golem/core/optimisers/optimizer.py +++ b/golem/core/optimisers/optimizer.py @@ -18,7 +18,7 @@ from golem.core.optimisers.opt_history_objects.opt_history import OptHistory from golem.core.optimisers.opt_node_factory import DefaultOptNodeFactory, OptNodeFactory from golem.core.optimisers.random_graph_factory import RandomGraphFactory, RandomGrowthGraphFactory -from golem.core.utilities.random import RandomStateHandler +from golem.utilities.random import RandomStateHandler STRUCTURAL_DIVERSITY_FREQUENCY_CHECK = 5 diff --git a/golem/core/optimisers/populational_optimizer.py b/golem/core/optimisers/populational_optimizer.py index 6c73f1ed7..c95be9ce2 100644 --- a/golem/core/optimisers/populational_optimizer.py +++ b/golem/core/optimisers/populational_optimizer.py @@ -13,7 +13,7 @@ from golem.core.optimisers.optimization_parameters import GraphRequirements from golem.core.optimisers.optimizer import GraphGenerationParams, GraphOptimizer, AlgorithmParameters from golem.core.optimisers.timer import OptimisationTimer -from golem.core.utilities.grouped_condition import GroupedCondition +from golem.utilities.grouped_condition import GroupedCondition class PopulationalOptimizer(GraphOptimizer): @@ -57,7 +57,7 @@ def __init__(self, max_stagnation_time = requirements.early_stopping_timeout or self.timer.timeout self.stop_optimization = \ GroupedCondition(results_as_message=True).add_condition( - lambda: self.timer.is_time_limit_reached(self.current_generation_num), + lambda: self.timer.is_time_limit_reached(self.current_generation_num - 1), 'Optimisation stopped: Time limit is reached' ).add_condition( lambda: (requirements.num_of_generations is not None and diff --git a/golem/core/optimisers/random/random_mutation_optimizer.py b/golem/core/optimisers/random/random_mutation_optimizer.py index 6110dfe51..b6adc7061 100644 --- a/golem/core/optimisers/random/random_mutation_optimizer.py +++ b/golem/core/optimisers/random/random_mutation_optimizer.py @@ -10,7 +10,7 @@ from golem.core.optimisers.optimizer import GraphGenerationParams from golem.core.optimisers.populational_optimizer import PopulationalOptimizer from golem.core.optimisers.random.random_search import RandomSearchOptimizer -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence class PopulationalRandomMutationOptimizer(PopulationalOptimizer): diff --git a/golem/core/optimisers/random/random_search.py b/golem/core/optimisers/random/random_search.py index 0a7e7e501..0ca330108 100644 --- a/golem/core/optimisers/random/random_search.py +++ b/golem/core/optimisers/random/random_search.py @@ -12,7 +12,7 @@ from golem.core.optimisers.optimization_parameters import GraphRequirements from golem.core.optimisers.optimizer import GraphOptimizer, GraphGenerationParams from golem.core.optimisers.timer import OptimisationTimer -from golem.core.utilities.grouped_condition import GroupedCondition +from golem.utilities.grouped_condition import GroupedCondition class RandomSearchOptimizer(GraphOptimizer): diff --git a/golem/core/optimisers/timer.py b/golem/core/optimisers/timer.py index 11aaff830..34cae0318 100644 --- a/golem/core/optimisers/timer.py +++ b/golem/core/optimisers/timer.py @@ -48,9 +48,9 @@ def __init__(self, timeout: datetime.timedelta = None): def _is_next_iteration_possible(self, time_constraint: float, iteration_num: int = None) -> bool: minutes = self.minutes_from_start - if iteration_num is not None: + if iteration_num is not None and iteration_num != 0: evo_proc_minutes = minutes - self.init_time - possible = time_constraint > (minutes + (evo_proc_minutes / (iteration_num + 1))) + possible = time_constraint > (minutes + (evo_proc_minutes / iteration_num)) else: possible = time_constraint > minutes if not possible: diff --git a/golem/core/tuning/tuner_interface.py b/golem/core/tuning/tuner_interface.py index 0d9886578..c4d6ff6e6 100644 --- a/golem/core/tuning/tuner_interface.py +++ b/golem/core/tuning/tuner_interface.py @@ -14,7 +14,7 @@ from golem.core.optimisers.graph import OptGraph from golem.core.optimisers.objective import ObjectiveEvaluate, ObjectiveFunction from golem.core.tuning.search_space import SearchSpace, convert_parameters -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence DomainGraphForTune = TypeVar('DomainGraphForTune') diff --git a/golem/core/utilities/__init__.py b/golem/core/utilities/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/golem/serializers/serializer.py b/golem/serializers/serializer.py index 4504f5287..6308a4834 100644 --- a/golem/serializers/serializer.py +++ b/golem/serializers/serializer.py @@ -88,7 +88,7 @@ def _register_default_coders(): from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator from golem.core.optimisers.fitness.fitness import Fitness from golem.core.optimisers.objective.objective import ObjectiveInfo - from golem.core.utilities.data_structures import ComparableEnum + from golem.utilities.data_structures import ComparableEnum from .any_serialization import any_from_json, any_to_json diff --git a/golem/structural_analysis/graph_sa/results/sa_analysis_results.py b/golem/structural_analysis/graph_sa/results/sa_analysis_results.py index e2c805c7d..75a22a2ce 100644 --- a/golem/structural_analysis/graph_sa/results/sa_analysis_results.py +++ b/golem/structural_analysis/graph_sa/results/sa_analysis_results.py @@ -6,7 +6,7 @@ from golem.core.dag.graph import Graph from golem.core.log import default_log from golem.core.paths import project_root -from golem.core.utilities.serializable import Serializable +from golem.utilities.serializable import Serializable from golem.serializers import Serializer from golem.structural_analysis.graph_sa.results.deletion_sa_approach_result import DeletionSAApproachResult from golem.structural_analysis.graph_sa.results.object_sa_result import ObjectSAResult, \ diff --git a/golem/core/utilities/data_structures.py b/golem/utilities/data_structures.py similarity index 100% rename from golem/core/utilities/data_structures.py rename to golem/utilities/data_structures.py diff --git a/golem/core/utilities/grouped_condition.py b/golem/utilities/grouped_condition.py similarity index 100% rename from golem/core/utilities/grouped_condition.py rename to golem/utilities/grouped_condition.py diff --git a/golem/core/utilities/random.py b/golem/utilities/random.py similarity index 100% rename from golem/core/utilities/random.py rename to golem/utilities/random.py diff --git a/golem/core/utilities/sequence_iterator.py b/golem/utilities/sequence_iterator.py similarity index 97% rename from golem/core/utilities/sequence_iterator.py rename to golem/utilities/sequence_iterator.py index 97f8b7c0b..188b89b80 100644 --- a/golem/core/utilities/sequence_iterator.py +++ b/golem/utilities/sequence_iterator.py @@ -1,6 +1,6 @@ from typing import Callable, Optional -from golem.core.utilities.data_structures import BidirectionalIterator +from golem.utilities.data_structures import BidirectionalIterator class SequenceIterator(BidirectionalIterator[int]): diff --git a/golem/core/utilities/serializable.py b/golem/utilities/serializable.py similarity index 100% rename from golem/core/utilities/serializable.py rename to golem/utilities/serializable.py diff --git a/golem/core/utilities/singleton_meta.py b/golem/utilities/singleton_meta.py similarity index 100% rename from golem/core/utilities/singleton_meta.py rename to golem/utilities/singleton_meta.py diff --git a/golem/utilities/utilities.py b/golem/utilities/utilities.py new file mode 100644 index 000000000..7d27cbb60 --- /dev/null +++ b/golem/utilities/utilities.py @@ -0,0 +1,15 @@ +from joblib import cpu_count + + +def determine_n_jobs(n_jobs=-1, logger=None): + cpu_num = cpu_count() + if n_jobs > cpu_num: + n_jobs = cpu_num + elif n_jobs <= 0: + if n_jobs <= -cpu_num - 1 or n_jobs == 0: + raise ValueError(f"Unproper `n_jobs` = {n_jobs}. " + f"`n_jobs` should be between ({-cpu_num}, {cpu_num}) except 0") + n_jobs = cpu_num + 1 + n_jobs + if logger: + logger.info(f"Number of used CPU's: {n_jobs}") + return n_jobs diff --git a/golem/visualisation/opt_history/multiple_fitness_line.py b/golem/visualisation/opt_history/multiple_fitness_line.py index 231556553..38c7267d6 100644 --- a/golem/visualisation/opt_history/multiple_fitness_line.py +++ b/golem/visualisation/opt_history/multiple_fitness_line.py @@ -8,7 +8,7 @@ from golem.core.log import default_log from golem.core.optimisers.opt_history_objects.opt_history import OptHistory -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence from golem.visualisation.opt_history.arg_constraint_wrapper import ArgConstraintWrapper from golem.visualisation.opt_history.fitness_line import setup_fitness_plot from golem.visualisation.opt_history.utils import show_or_save_figure diff --git a/setup.py b/setup.py index 9c248bb9d..3f98be05f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ # The text of the README file NAME = 'thegolem' -VERSION = '0.3.3' +VERSION = '0.4.0' AUTHOR = 'NSS Lab' SHORT_DESCRIPTION = 'Framework for Graph Optimization and Learning by Evolutionary Methods' diff --git a/test/unit/optimizers/gp_operators/test_population_size.py b/test/unit/optimizers/gp_operators/test_population_size.py index 56ee91c80..fb3de1d43 100644 --- a/test/unit/optimizers/gp_operators/test_population_size.py +++ b/test/unit/optimizers/gp_operators/test_population_size.py @@ -1,13 +1,11 @@ -import pytest - from golem.core.optimisers.archive import GenerationKeeper from golem.core.optimisers.fitness import SingleObjFitness -from golem.core.optimisers.genetic.parameters.population_size import PopulationSize, AdaptivePopulationSize, \ +from golem.core.optimisers.genetic.parameters.population_size import AdaptivePopulationSize, \ ConstRatePopulationSize from golem.core.optimisers.graph import OptGraph, OptNode from golem.core.optimisers.objective import Objective from golem.core.optimisers.opt_history_objects.individual import Individual -from golem.core.utilities.sequence_iterator import SequenceIterator, fibonacci_sequence +from golem.utilities.sequence_iterator import SequenceIterator def custom_objective(): diff --git a/test/unit/optimizers/test_evaluation.py b/test/unit/optimizers/test_evaluation.py index 47bc39515..45623ab54 100644 --- a/test/unit/optimizers/test_evaluation.py +++ b/test/unit/optimizers/test_evaluation.py @@ -8,11 +8,12 @@ from golem.core.dag.graph import Graph from golem.core.optimisers.fitness import Fitness, null_fitness from golem.core.optimisers.genetic.evaluation import MultiprocessingDispatcher, SequentialDispatcher, \ - ObjectiveEvaluationDispatcher, determine_n_jobs + ObjectiveEvaluationDispatcher from golem.core.optimisers.meta.surrogate_evaluator import SurrogateDispatcher from golem.core.optimisers.objective import Objective from golem.core.optimisers.opt_history_objects.individual import Individual from golem.core.optimisers.timer import OptimisationTimer +from golem.utilities.utilities import determine_n_jobs from test.unit.utils import graph_first, graph_second, graph_third, graph_fourth, RandomMetric diff --git a/test/unit/test_logger.py b/test/unit/test_logger.py index e28999937..495f1d87e 100644 --- a/test/unit/test_logger.py +++ b/test/unit/test_logger.py @@ -6,8 +6,8 @@ import pytest from golem.core.log import DEFAULT_LOG_PATH, Log, default_log -from golem.core.utilities.grouped_condition import GroupedCondition -from golem.core.utilities.singleton_meta import SingletonMeta +from golem.utilities.grouped_condition import GroupedCondition +from golem.utilities.singleton_meta import SingletonMeta @pytest.fixture() diff --git a/test/unit/utilities/test_data_structures.py b/test/unit/utilities/test_data_structures.py index 8661e6675..91f33ef45 100644 --- a/test/unit/utilities/test_data_structures.py +++ b/test/unit/utilities/test_data_structures.py @@ -1,4 +1,4 @@ -from golem.core.utilities.data_structures import UniqueList, ensure_wrapped_in_sequence +from golem.utilities.data_structures import UniqueList, ensure_wrapped_in_sequence def test_init(): diff --git a/test/unit/utilities/test_iterator.py b/test/unit/utilities/test_iterator.py index 8bcf4ebc9..b39ad34e5 100644 --- a/test/unit/utilities/test_iterator.py +++ b/test/unit/utilities/test_iterator.py @@ -1,4 +1,4 @@ -from golem.core.utilities.sequence_iterator import fibonacci_sequence, SequenceIterator +from golem.utilities.sequence_iterator import fibonacci_sequence, SequenceIterator def test_iterator_without_constraints():