From a35ecd232b2f4f2bdcd6abad7515e652863b2476 Mon Sep 17 00:00:00 2001 From: anpolol Date: Thu, 23 Nov 2023 18:26:25 +0300 Subject: [PATCH] k2 method added --- .../GraphGenerator.py | 0 .../one_graph_search_k2.py | 391 ++++++++++++++++++ .../one_graph_search_kronecker.py | 27 +- .../parallel_graph_adapter.py | 0 .../synthetic_graph_evolution/simple_run.py | 56 ++- golem/core/optimisers/genetic/gp_optimizer.py | 2 +- 6 files changed, 451 insertions(+), 25 deletions(-) create mode 100644 examples/synthetic_graph_evolution/GraphGenerator.py create mode 100644 examples/synthetic_graph_evolution/one_graph_search_k2.py create mode 100644 examples/synthetic_graph_evolution/parallel_graph_adapter.py diff --git a/examples/synthetic_graph_evolution/GraphGenerator.py b/examples/synthetic_graph_evolution/GraphGenerator.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/synthetic_graph_evolution/one_graph_search_k2.py b/examples/synthetic_graph_evolution/one_graph_search_k2.py new file mode 100644 index 000000000..60cb25960 --- /dev/null +++ b/examples/synthetic_graph_evolution/one_graph_search_k2.py @@ -0,0 +1,391 @@ +import collections +import math +from datetime import timedelta +from functools import partial +from wrapt_timeout_decorator import timeout +from datetime import datetime +from typing import Optional, Union, List + +import pandas as pd +import time +# import timeout_decorator + +from golem.core.dag.graph_delegate import GraphDelegate +from golem.core.dag.linked_graph_node import LinkedGraphNode + +import igraph as ig +import matplotlib.pyplot as plt +import networkx as nx +from random import choice, random, randint, sample, choices + +from examples.synthetic_graph_evolution.generators import generate_labeled_graph +from golem.core.adapter.nx_adapter import BaseNetworkxAdapter +from golem.core.optimisers.genetic.gp_optimizer import EvoGraphOptimizer +from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters + +from golem.core.optimisers.adaptive.operator_agent import MutationAgentTypeEnum +from golem.core.optimisers.adaptive.context_agents import ContextAgentTypeEnum + +from golem.core.optimisers.genetic.operators.base_mutations import MutationTypesEnum +from golem.core.optimisers.genetic.operators.crossover import CrossoverTypesEnum +from golem.core.optimisers.genetic.operators.inheritance import GeneticSchemeTypesEnum +from golem.core.optimisers.objective import Objective +from golem.core.optimisers.optimization_parameters import GraphRequirements +from golem.core.optimisers.optimizer import GraphGenerationParams +import numpy as np +from golem.core.dag.verification_rules import has_no_self_cycled_nodes +import random + +import pickle +from functools import partial + +class GeneratorModel(GraphDelegate): + def __init__(self, nodes: Optional[Union[LinkedGraphNode, List[LinkedGraphNode]]] = None,codes_in=None,nlevels=3): + super().__init__(nodes) + self.nodes = nodes + self.unique_pipeline_id = 1 + self.nlevels = nlevels + self.Nverts = pow(2, nlevels) + c = 1 + self.sizes = [] + for _ in range(nlevels): + self.sizes.append(c) + c *= 2 + self.patterns = [[], [(0, 0)], [(0, 0), (1, 1)], [(0, 1), (1, 0)], [(0, 0), (1, 0), (1, 1)], + [(0, 0), (0, 1), (1, 0), (1, 1)]] + self.weights = [pow(k, 0.3) for k in list(range(1, 7))] + + if codes_in is None: + codes = [] + for i in range(nlevels): + for _ in range(100): + cd = self.gen_pattern(self.sizes[i] * self.sizes[i]) + if any(cd): + break + codes.append(cd) + self.codes = codes + else: + self.codes = codes_in + + def number_of_nodes(self): + return len(self.nodes) + + def degree(self): + edges = self.get_edges() + dic = {} + for num, i in enumerate(self.nodes): + dic[int(i.name)] = 0 + + if len(edges) > 0: + edges = np.array(edges) + edges = np.transpose(edges) + edge_1, edge_2 = edges.tolist() + dic_1 = collections.Counter(edge_1) + dic_2 = collections.Counter(edge_2) + for node in dic_1: + dic[int(node.name)] += dic_1[node] + for node in dic_2: + dic[int(node.name)] += dic_2[node] + return dic + + def gen_pattern(self, k): + code_symbols = list(range(len(self.patterns))) + return random.choices(code_symbols, self.weights, k=k) + def code_to_graph(self, p_prev, ilevel, coords): + for e in self.patterns[p_prev]: + if not e: + continue + i, j = e + coords_new = (coords[0] + i * self.sizes[-ilevel], coords[1] + j * self.sizes[-ilevel]) + if ilevel == self.nlevels: + self.edges.append(coords_new) + else: + ic = i * self.sizes[ilevel] + j + p_new = self.codes[ilevel][ic] + self.code_to_graph(p_new, ilevel + 1, coords_new) + + def gen_edges(self): + self.edges = [] + self.code_to_graph(self.codes[0][0], 1, (0, 0)) + if not self.edges: + return np.array([]) + edge_list = np.array(self.edges).T + c, r = edge_list + edge_list = edge_list[:, c > r] # take only low triangular matrix + edge_list = np.concatenate((edge_list, edge_list[::-1, :]), axis=1) + + for edge in edge_list.transpose(): + node_1, node_2 = edge + for node in self.nodes: + + if node.name == str(node_1): + V1=node + elif node.name==str(node_2): + V2=node + self.connect_nodes(V1,V2) + + return edge_list + + def mutate(self): + # choose code level + lev = random.randint(0, self.nlevels - 1) + len_lev = len(self.codes[lev]) + + # choose element + n_to_change = (lev - 1) * 2 + # n_to_change = 1 # predefined!!!!!!!! + if lev < 2: + n_to_change = 1 + + new_codes = [c.copy() for c in self.codes] + for _ in range(100): + new_els = self.gen_pattern(n_to_change) + new_code = new_codes[lev].copy() + for i, v in enumerate(random.sample(range(len_lev), k=n_to_change)): + new_code[v] = new_els[i] + if any(new_code): + new_codes[lev] = new_code + break + + return GeneratorModel(nodes = self.nodes ,codes_in=new_codes,nlevels=self.nlevels) + +def gen_with_deg(nodes, deg_exp, delta = 0.3,nlvl=3): + deg_curr = 0 + for _ in range(1000): + gen = GeneratorModel(nodes = nodes, nlevels=nlvl) + ee = gen.gen_edges() + if ee.size == 0: + continue + + deg_curr = ee.shape[1]/gen.Nverts + if abs(deg_curr - deg_exp) 0: + edge_1, edge_2 = edges.tolist() + for i in G.nodes: + s_l = 0 + t = 0 + ind_1 = [] + ind_2 = [] + for l, node in enumerate(edge_1): + if node == i: + ind_1.append(l) + if edge_2[l] == i: + ind_2.append(l) + + result_1 = np.array(edge_2)[ind_1] + result_2 = np.array(edge_1)[ind_2] + + for neigbour in result_1: + t += 1 + if (neigbour.content['label'] == i.content["label"]): + s_l += 1 + for neigbour in result_2: + t += 1 + if (neigbour.content['label'] == i.content["label"]): + s_l += 1 + if t > 0: + label_assort += s_l / t + + label_assort = label_assort / len(G.nodes) + return label_assort + + def lab_assort_count(des_label_assort, G): + fact_label = label_assortativity(G) + return (des_label_assort - fact_label) * (des_label_assort - fact_label) + + def asp_count(des_shortest_paths, G): + + G_new = nx.Graph() + G_new.add_nodes_from(G.nodes) + G_new.add_edges_from(G.get_edges()) + + fact_asp = shortest_paths(G_new) + return (des_shortest_paths - fact_asp) * (des_shortest_paths - fact_asp) + + def avg_deg_count(des_d, G): + d = np.mean(list(G.degree().values())) + return (d - des_d) * (d - des_d) + + def avg_cluster_count(des_cl, G): + G_new = nx.Graph() + + G_new.add_nodes_from(G.nodes) + G_new.add_edges_from(G.get_edges()) + + d = nx.average_clustering(G_new.to_undirected()) + return (d - des_cl) * (d - des_cl) + + # Generate initial population with random graphs + initial_graphs = [] + + + for i in range(20): + print(i) + Init2=gen_with_deg(nodes=[GeneratorNode(nodes_from=[], + content={'name': vertex, + 'label': random.choices([0, 1], weights=[ + 0.5 + 0.5 * des_label_assort, + 0.5 - 0.5 * des_label_assort], k=1)}) + for vertex in range(des_num_nodes)], deg_exp=des_degree, nlvl=int(math.log(des_num_nodes,2))) + + initial_graphs.append(Init2) + + + print('avg degree of random graph: {} vs des:{}'.format(np.mean(list(dict(Init2.degree()).values())), des_degree)) + + nodes = [] + for num, i in enumerate(Init2.nodes): + nodes.append(num) + + + G_new = nx.Graph() + G_new.add_nodes_from(nodes) + for edge in Init2.get_edges(): + G_new.add_edge(int(edge[0].name), int(edge[1].name)) + + print('connected_components',nx.number_connected_components(G_new)) + print('clustering coefficient of random graph: {} vs des:{}'.format(nx.average_clustering(G_new.to_undirected()), + des_cluster)) + print('shortest paths real: {} vs des: {}'.format(shortest_paths(G_new.to_undirected()), des_asp)) + objective = Objective({'avg degree': partial(avg_deg_count, des_degree), + 'cluster coef': partial(avg_cluster_count, des_cluster), + 'shortest paths': partial(asp_count, des_asp)}, is_multi_objective=True) + + + + # Setup optimization parameters + max_graph_size = des_num_nodes + requirements = GraphRequirements( + max_arity=max_graph_size, + max_depth=max_graph_size * 10000, + num_of_generations=600, + early_stopping_iterations=100, + timeout=timedelta(minutes=timeout), + n_jobs=-1, + num_edges=num_edges + ) + + mutation_types = [MutationTypesEnum.single_edge, + + #MutationTypesEnum.batch_edge_3, + ] + + + + gp_params = GPAlgorithmParameters( + max_pop_size=10, + crossover_prob=0.8, + mutation_prob=1, + genetic_scheme_type=GeneticSchemeTypesEnum.parameter_free, + multi_objective=objective.is_multi_objective, + mutation_types=mutation_types, + adaptive_mutation_type=MutationAgentTypeEnum.default, + context_agent_type=ContextAgentTypeEnum.adjacency_matrix, + crossover_types=[CrossoverTypesEnum.none] + ) + + graph_gen_params = GraphGenerationParams(adapter=BaseNetworkxAdapter()) + all_parameters = (requirements, graph_gen_params, gp_params) + + # Build and run the optimizer + time_before = datetime.now() + optimiser = EvoGraphOptimizer(objective, initial_graphs, *all_parameters) + found_graphs = optimiser.optimise(objective) + time_after = datetime.now() + + if visualize: + # Restore the NetworkX graph back from internal Graph representation + found_graph = graph_gen_params.adapter.restore(found_graphs[0]) + act_ad = np.mean(list(dict(found_graph.degree()).values())) + print('avg degree of found graph real: {} vs des:{}'.format(act_ad, + des_degree)) + + nodes = [] + colors = [] + for num, i in enumerate(found_graph.nodes): + nodes.append(i) + if i.content['label'] == 0: + colors.append('red') + else: + colors.append('blue') + G_new = nx.Graph() + G_new.add_nodes_from(nodes) + for edge in found_graph.get_edges(): + G_new.add_edge(edge[0], edge[1]) + + G_new.add_edges_from(found_graph.get_edges()) + + act_cl = nx.average_clustering(G_new.to_undirected()) + act_asp = shortest_paths(G_new.to_undirected()) + + + + + print('clustering coefficient real: {} vs des:{}'.format(act_cl, des_cluster)) + # print('label assortativity real: {} vs des: {} '.format(label_assortativity(found_graphs[0]), des_label_assort)) + print('shortest paths real: {} vs des: {}'.format(act_asp, des_asp)) + print('number of nodes', G_new.number_of_nodes()) + return G_new, time_after, time_before, act_cl, act_asp,act_ad diff --git a/examples/synthetic_graph_evolution/one_graph_search_kronecker.py b/examples/synthetic_graph_evolution/one_graph_search_kronecker.py index 57c46f22e..5d8d22a6a 100644 --- a/examples/synthetic_graph_evolution/one_graph_search_kronecker.py +++ b/examples/synthetic_graph_evolution/one_graph_search_kronecker.py @@ -27,7 +27,7 @@ from golem.core.optimisers.optimization_parameters import GraphRequirements from golem.core.optimisers.optimizer import GraphGenerationParams import numpy as np -from golem.core.dag.verification_rules import has_no_self_cycled_nodes + import random import pickle @@ -35,8 +35,6 @@ import math from itertools import combinations - - class GeneratorModel(GraphDelegate): def __init__(self, nodes: Optional[Union[LinkedGraphNode, List[LinkedGraphNode]]] = None): super().__init__(nodes) @@ -200,7 +198,7 @@ def label_assortativity(G): def lab_assort_count(des_label_assort, G): fact_label = label_assortativity(G) - return (des_label_assort - fact_label) * (des_label_assort - fact_label) + return (des_label_assort/fact_label - 1) * (des_label_assort/fact_label - 1) def asp_count(des_shortest_paths, G): @@ -211,13 +209,12 @@ def asp_count(des_shortest_paths, G): return (des_shortest_paths - fact_asp) * (des_shortest_paths - fact_asp) def avg_deg_count(des_d, G): - G_new = nx.Graph() - G_new.add_nodes_from(G.nodes) - G_new.add_edges_from(G.get_edges()) + + G_new = G.to_networkx() G_new.add_edges_from(list(map(lambda x: (x, x), range(len(G_new.nodes()))))) G = nx.tensor_product(G_new, G_new) d = np.mean(list(zip(*nx.degree(G)))[1]) - return (d - des_d) * (d - des_d) + return (des_d - d) * (des_d - d) def avg_cluster_count(des_cl, G): G_new = nx.Graph() @@ -227,12 +224,12 @@ def avg_cluster_count(des_cl, G): G = nx.tensor_product(G_new, G_new) d = nx.average_clustering(G.to_undirected()) - return (d - des_cl) * (d - des_cl) + return (des_cl - d) * (des_cl - d) # Generate initial population with random graphs initial_graphs = [] - for i in range(20): + for i in range(100): Init2 = GeneratorModel(nodes=[GeneratorNode(nodes_from=[], content={'name': vertex, 'label': random.choices([0, 1], weights=[ @@ -287,16 +284,16 @@ def avg_cluster_count(des_cl, G): requirements = GraphRequirements( max_arity=max_graph_size, max_depth=max_graph_size * 10000, - num_of_generations=600, - early_stopping_iterations=100, + num_of_generations=200, + early_stopping_iterations=30, timeout=timedelta(minutes=timeout), n_jobs=-1, num_edges=num_edges ) mutation_types = [MutationTypesEnum.single_edge, - MutationTypesEnum.single_edge_add, - MutationTypesEnum.single_edge_del, + #MutationTypesEnum.single_edge_add, + #MutationTypesEnum.single_edge_del, MutationTypesEnum.batch_edge_5, MutationTypesEnum.star_edge_5, MutationTypesEnum.path_edge_5, @@ -323,7 +320,7 @@ def avg_cluster_count(des_cl, G): adaptive_mutation_type=MutationAgentTypeEnum.default, context_agent_type=ContextAgentTypeEnum.adjacency_matrix, - crossover_types=[CrossoverTypesEnum.none] + crossover_types=[CrossoverTypesEnum.exchange_edges,CrossoverTypesEnum.exchange_parents_one, CrossoverTypesEnum.exchange_parents_both] ) graph_gen_params = GraphGenerationParams(adapter=BaseNetworkxAdapter()) diff --git a/examples/synthetic_graph_evolution/parallel_graph_adapter.py b/examples/synthetic_graph_evolution/parallel_graph_adapter.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/synthetic_graph_evolution/simple_run.py b/examples/synthetic_graph_evolution/simple_run.py index aee92f041..2e40b687f 100644 --- a/examples/synthetic_graph_evolution/simple_run.py +++ b/examples/synthetic_graph_evolution/simple_run.py @@ -1,14 +1,15 @@ -from one_graph_search import run_graph_search - +from one_graph_search_k2 import run_graph_search +import pandas as pd +import random import pickle if __name__ == '__main__': # задаю параметры - des_num_nodes = max_graph_size = 30 - des_degree = 3 + des_num_nodes = max_graph_size = 40# int(math.sqrt(700)) + des_degree = 5 num_edges = 5 - des_cluster = 0.3 + des_cluster = .01 des_asp = 2 des_label_assort = 1 @@ -19,7 +20,44 @@ star = False - G_new = run_graph_search(dense=dense,cycle=cycle,path=path,star=star, size=16, num_edges=num_edges, des_degree=des_degree, - des_cluster=des_cluster, des_num_nodes=des_num_nodes, - des_label_assort=des_label_assort,des_asp=des_asp, visualize=True) - # pickle.dump(G_new.to_undirected(), open('G_40_8.pickle', 'wb')) + def funad(nn): + if nn == 16: + return [2, 5, 10] + if nn == 32: + return [2, 5, 10, 15,20] + if nn == 64: + return [2, 5, 10, 15, 20,25,30] + + all_combinations = [] + for nn in [64]: + for dd in [35]:#funad(nn): + for cl in [0.1]: + for asp in [2.5]: + all_combinations.append((nn,dd,cl,asp)) + + #df = pd.read_csv('analysis_data_for_AAAI_workshop_1_3_edges.csv') + #while 1>0: + (nn,dd,cl,asp) = random.choice(all_combinations) + des_cluster =cl + des_num_nodes = nn + des_degree = dd + des_asp =asp + print(des_num_nodes, des_cluster, des_asp, des_degree) + #if len(df[(df['num nodes']==des_num_nodes) & (df['des cl']==des_cluster) & (df['des asp']==des_asp) & (df['des ad']==des_degree)])==0: + G_new, time_after, time_before, act_cl, act_asp,act_ad = run_graph_search(dense=dense,cycle=cycle,path=path,star=star, size=16, num_edges=num_edges, des_degree=des_degree, + des_cluster=des_cluster, des_num_nodes=des_num_nodes, + des_label_assort=des_label_assort, des_asp=des_asp, visualize=True) + + #new_row = pd.Series([des_num_nodes, des_cluster, des_asp, des_degree, act_cl, act_asp, act_ad, + # time_after - time_before, False], index=df.columns) + + # df = pd.concat([df, new_row], ignore_index=True) + #df = df.append(new_row, ignore_index=True) + + #df.to_csv('analysis_data_for_AAAI_workshop_1_3_edges.csv') + + # with open('G_' + str(des_num_nodes) + '_' + str(des_cluster) + '_' + str(des_asp) + '_' + str( + # des_degree) + '_1_3_edges.txt', 'a') as f: + # for edge in (G_new.edges()): + # f.write(str(edge[0]) + ',' + str(edge[1]) + '\n') + diff --git a/golem/core/optimisers/genetic/gp_optimizer.py b/golem/core/optimisers/genetic/gp_optimizer.py index 64eb7c628..f2cb598b9 100644 --- a/golem/core/optimisers/genetic/gp_optimizer.py +++ b/golem/core/optimisers/genetic/gp_optimizer.py @@ -116,7 +116,7 @@ def _evolve_population(self, evaluator: EvaluationOperator) -> PopulationT: # Adaptive agent experience collection & learning # Must be called after reproduction (that collects the new experience) experience = self.mutation.agent_experience - print('experience',experience) + experience.collect_results(new_population) self.mutation.agent.partial_fit(experience) # Use some part of previous pop in the next pop