diff --git a/Tests/test.py b/Tests/test.py index 6421aae..56bd5f6 100644 --- a/Tests/test.py +++ b/Tests/test.py @@ -1,8 +1,8 @@ from src import * +import pickle -coo = Dense.Layer(shape=Dense.Shape(784, 398), +coo = Dense.Layer(shape=Dense.Shape(100, 200, 300), initializer=Initializers.Xavier(5), optimizer=Optimizers.Adam(epsilon=2), activationFunction=Activators.PRelu()) -print(coo, id(coo)) -print(coo2 := load(*coo.__save__()), id(coo2)) +print(coo.__save__()[2]["shape"].save) diff --git a/main.py b/main.py index cfd68de..df7f6cf 100644 --- a/main.py +++ b/main.py @@ -6,6 +6,8 @@ # todo: string hyperparams # todo: look into "NamedTuple" # fixme: in DunderSaveLoad for __dict__ saves save only req stuff, maybe by subtracting the vars of base __dict__ +import pickle + from src import * from DataSets import dataSet from Models import model @@ -15,27 +17,23 @@ db2 = DataBase.load(dataSet.TestSets.EmnistBalanced, normalizeInp=1, reshapeInp=(-1, 1), name='TestSets.EmnistBalanced') # db2 = False -dense_nn = Dense.NN(shape=Dense.Shape(db.inpShape[0], *(392, 196), db.tarShape[0]), - initializers=None, - activators=None, - lossFunction=None) +# dense_nn = Dense.NN(shape=Dense.Shape(db.inpShape[0], *(392, 196), db.tarShape[0]), +# initializers=None, +# activators=None, +# lossFunction=None) # dense_nn.train(epochs=1, # batchSize=256, # trainDataBase=db, # optimizers=None, # profile=False, # test=db2) -# print(dense_nn) - - -def save(): - import pickle as dill - dbb, dbb2 = dense_nn.trainDataBase, dense_nn.testDataBase - dense_nn.trainDataBase, dense_nn.testDataBase = None, None - dill.dump(dense_nn, open('t1.nntp', 'wb')) - dense_nn.trainDataBase, dense_nn.testDataBase = dbb, dbb2 - - -def load(): - import pickle as dill - return dill.load(open('t1.nntp', 'rb')) +# # print(dense_nn) +# +# coo = dense_nn +# print(coo, id(coo), sep='\n') +# save = coo.__save__() +# with open('temp.save', 'wb') as f: +# pickle.dump(save, f) +with open('temp.save', 'rb') as f: + save2 = pickle.load(f) +print(coo2 := load(*save2), id(coo2), sep='\n') diff --git a/src/NeuralNetworks/base.py b/src/NeuralNetworks/base.py index 5c036a2..7da374a 100644 --- a/src/NeuralNetworks/base.py +++ b/src/NeuralNetworks/base.py @@ -1,3 +1,8 @@ +from typing import TYPE_CHECKING, Union +if TYPE_CHECKING: + from ..tools import * + from ..Topologies import * + import time import warnings import cProfile @@ -5,19 +10,14 @@ import pstats import os from abc import ABCMeta, abstractmethod -from typing import TYPE_CHECKING, Union import numpy as np -from ..tools import MagicProperty, makeMetaMagicProperty, PrintCols, iterable, secToHMS, statPrinter, DunderSaveLoad +from ..tools import PrintCols, iterable, secToHMS, statPrinter, DunderSaveLoad from ..Topologies import Activators, Initializers, Optimizers, LossFunction, DataBase -if TYPE_CHECKING: - from ..tools import * - from ..Topologies import * - -class BaseShape(DunderSaveLoad, metaclass=makeMetaMagicProperty(ABCMeta)): +class BaseShape(DunderSaveLoad, metaclass=ABCMeta): """ """ @@ -27,7 +27,7 @@ def __repr__(self): def __getitem__(self, item): shapes = self.RAW_SHAPES[item] - return self.__class__(*shapes) if isinstance(item, slice) and shapes else self.SHAPES[item] + return type(self)(*shapes) if isinstance(item, slice) and shapes else self.SHAPES[item] def __hash__(self): return hash(self.SHAPES) @@ -65,7 +65,7 @@ def _formatShapes(shapes) -> tuple: return shapes -class BaseLayer(DunderSaveLoad, metaclass=makeMetaMagicProperty(ABCMeta)): +class BaseLayer(DunderSaveLoad, metaclass=ABCMeta): """ """ @@ -79,6 +79,14 @@ def __str__(self): DEPS = ': '.join(f"{dName}:shape{getattr(self, dName).shape}" for dName in self.DEPS) return f"{self.__repr__()[:-1]}:\n{DEPS=}>" + # todo: add ignore to DunderSaveLoad + def __save__(self): + _input, _output, inputDelta, outputDelta = self.input, self.output, self.inputDelta, self.outputDelta + self.input = self.output = self.inputDelta = self.outputDelta = None + _return = super(BaseLayer, self).__save__() + self.input, self.output, self.inputDelta, self.outputDelta = _input, _output, inputDelta, outputDelta + return _return + def __init__(self, shape: "BaseShape", initializer: "Initializers.Base", optimizer: "Optimizers.Base", @@ -92,10 +100,10 @@ def __init__(self, shape: "BaseShape", self.optimizer = optimizer self.ACTIVATION_FUNCTION = activationFunction - self.input = np.zeros(self.SHAPE[0], dtype=np.float32) - self.output = np.zeros(self.SHAPE[-1], dtype=np.float32) - self.inputDelta = np.zeros(self.SHAPE[-1], dtype=np.float32) - self.outputDelta = np.zeros(self.SHAPE[0], dtype=np.float32) + self.input = None + self.output = None + self.inputDelta = None + self.outputDelta = None self.DEPS = self._defineDeps(*depArgs, **depKwargs) self._initializeDepOptimizer() @@ -148,13 +156,13 @@ def _wire(self) -> "np.ndarray": """ -class BasePlot(metaclass=makeMetaMagicProperty(ABCMeta)): +class BasePlot: """ """ -class Network: +class Network(DunderSaveLoad): """ """ @@ -167,9 +175,6 @@ def __str__(self): layers = "\n\t\t".join(repr(layer) for layer in self.LAYERS) return f"{super(Network, self).__str__()}:\n\t\t{layers}" - def __save__(self): - pass - def __init__(self, inputLayer: "BaseLayer", *layers: "BaseLayer", lossFunction: "LossFunction.Base"): assert len(layers) > 0 self.LAYERS = inputLayer, *layers @@ -214,21 +219,25 @@ def backPropagation(self, _delta) -> "np.ndarray": return self.INPUT_LAYER.backProp(_delta) -class BaseNN(metaclass=makeMetaMagicProperty(ABCMeta)): +class BaseNN(DunderSaveLoad, metaclass=ABCMeta): """ """ STAT_PRINT_INTERVAL = 1 __optimizers = Optimizers(Optimizers.Adam(), ..., Optimizers.AdaGrad()) + _dict = True - @MagicProperty + @property def optimizers(self): return self.__optimizers - @optimizers.setter - def optimizers(self, _optimizers: "Optimizers"): - self.__optimizers = _optimizers - self.NETWORK.changeOptimizer(self.__optimizers) + # todo: add ignore to DunderSaveLoad + def __save__(self): + trainDataBase, testDataBase = self.trainDataBase, self.testDataBase + self.trainDataBase = self.testDataBase = None + _return = super(BaseNN, self).__save__() + self.trainDataBase, self.testDataBase = trainDataBase, testDataBase + return _return def __repr__(self): Shape = self.SHAPE @@ -237,13 +246,10 @@ def __repr__(self): return f"<{self.__class__.__name__}:Acc={acc[0]}%,{acc[1]}%: {Cost=:07.4f}: {Time=}: {Epochs=}: {Shape=}>" def __str__(self): - Optimizers = self.optimizers # noqa + Optimizers = self.__optimizers # noqa TrainDataBase, TestDataBase = self.trainDataBase, self.testDataBase return f"{self.__repr__()[1:-1]}:\n\t{Optimizers=}\n\t{TrainDataBase=}\n\t{TestDataBase=}\n\t{self.NETWORK}" - def __save__(self): - pass - def __init__(self, shape: "BaseShape", initializers: "Initializers" = None, activators: "Activators" = None, @@ -341,6 +347,8 @@ def train(self, epochs: int = None, profile: bool = False, test: Union[bool, "DataBase"] = None): # todo: implement "runs" + assert isinstance(epochs, int) or epochs is None + assert isinstance(batchSize, int) or batchSize is None if epochs is not None: self.numEpochs = epochs if batchSize is not None: self.batchSize = batchSize if trainDataBase is not None: self.trainDataBase = trainDataBase diff --git a/src/NeuralNetworks/dense.py b/src/NeuralNetworks/dense.py index f04b3d1..bcbaf7d 100644 --- a/src/NeuralNetworks/dense.py +++ b/src/NeuralNetworks/dense.py @@ -59,7 +59,8 @@ def _fire(self) -> "np.ndarray": # BottleNeck def _wire(self) -> "np.ndarray": self.weight -= self.weightOptimizer(self.__gradWeight, self.weight) self.biases -= self.biasesOptimizer(self.__gradBiases, self.biases) - return self.delta + delta, self.delta, self.activeDerivedDelta = self.delta, None, None + return delta class DensePlot(BasePlot): diff --git a/src/Topologies/optimizer.py b/src/Topologies/optimizer.py index 9cf0128..6920cb2 100644 --- a/src/Topologies/optimizer.py +++ b/src/Topologies/optimizer.py @@ -36,7 +36,7 @@ def __init__(self, learningRate: float = None): def _optimize(self, grad: Callable[["np.ndarray"], "np.ndarray"], theta: "np.ndarray") -> "np.ndarray": delta = grad(theta) - (local_dict := vars(self)).update(locals()) + (local_dict := vars(self).copy()).update(locals()) return ne.evaluate("delta * LEARNING_RATE", local_dict=local_dict) @@ -56,7 +56,7 @@ def _optimize(self, grad: Callable[["np.ndarray"], "np.ndarray"], theta: "np.nda delta = grad(theta) self.decayCounter += self.ONE locals()['ONE'] = self.ONE - (local_dict := vars(self)).update(locals()) + (local_dict := vars(self).copy()).update(locals()) return ne.evaluate("delta * LEARNING_RATE / (ONE + decayCounter * DECAY)", local_dict=local_dict) @@ -74,7 +74,7 @@ def __init__(self, learningRate: float = None, moment: float = None): def _optimize(self, grad: Callable[["np.ndarray"], "np.ndarray"], theta: "np.ndarray") -> "np.ndarray": delta = grad(theta) - (local_dict := vars(self)).update(locals()) + (local_dict := vars(self).copy()).update(locals()) self.prevDelta = momentDelta = ne.evaluate("LEARNING_RATE * delta + MOMENT * prevDelta", local_dict=local_dict) return momentDelta @@ -93,7 +93,7 @@ def __init__(self, learningRate: float = None, moment: float = None): def _optimize(self, grad: Callable[["np.ndarray"], "np.ndarray"], theta: "np.ndarray") -> "np.ndarray": delta = grad(theta - self.MOMENT * self.prevDelta) - (local_dict := vars(self)).update(locals()) + (local_dict := vars(self).copy()).update(locals()) self.prevDelta = momentDelta = ne.evaluate("LEARNING_RATE * delta + MOMENT * prevDelta", local_dict=local_dict) return momentDelta @@ -112,9 +112,9 @@ def __init__(self, learningRate: float = None, epsilon: float = None): def _optimize(self, grad: Callable[["np.ndarray"], "np.ndarray"], theta: "np.ndarray") -> "np.ndarray": delta = grad(theta) - (local_dict := vars(self)).update(locals()) + (local_dict := vars(self).copy()).update(locals()) self.summationSquareDelta = ne.evaluate('summationSquareDelta + delta * delta', local_dict=local_dict) - (local_dict := vars(self)).update(locals()) + (local_dict := vars(self).copy()).update(locals()) return ne.evaluate('delta * LEARNING_RATE / sqrt(summationSquareDelta + EPSILON)', global_dict=local_dict) @@ -154,20 +154,20 @@ def __init__(self, learningRate: float = None, beta1: float = None, beta2: float def _optimize(self, grad: Callable[["np.ndarray"], "np.ndarray"], theta: "np.ndarray") -> "np.ndarray": delta = grad(theta) - (local_dict := vars(self)).update(locals()) + (local_dict := vars(self).copy()).update(locals()) self.weightedSummationDelta = ne.evaluate( "BETA1 * weightedSummationDelta + BETA1_BAR * delta", local_dict=local_dict) self.weightedSummationSquareDelta = ne.evaluate( "BETA2 * weightedSummationSquareDelta + BETA2_BAR * delta * delta", local_dict=local_dict) - (local_dict := vars(self)).update(locals()) + (local_dict := vars(self).copy()).update(locals()) weightedSummationDeltaHat = ne.evaluate( "weightedSummationDelta / (1 - BETA1 ** decayCounter)", local_dict=local_dict) weightedSummationSquareDeltaHat = ne.evaluate( "weightedSummationSquareDelta / (1 - BETA2 ** decayCounter)", local_dict=local_dict) self.decayCounter += self.ONE - (local_dict := vars(self)).update(locals()) + (local_dict := vars(self).copy()).update(locals()) return ne.evaluate( "LEARNING_RATE * weightedSummationDeltaHat / sqrt(weightedSummationSquareDeltaHat + EPSILON)", local_dict=local_dict) diff --git a/src/tools/helperClass.py b/src/tools/helperClass.py index c8d09ad..c18e90a 100644 --- a/src/tools/helperClass.py +++ b/src/tools/helperClass.py @@ -62,24 +62,39 @@ class DunderSaveLoad: _dict = False def __new__(cls, *args, **kwargs): - cls.__RAW_ARGS = [arg if not isinstance(arg, DunderSaveLoad) else Dunder(arg.__save__()) - for arg in args] - cls.__RAW_KWARGS = {key: arg if not isinstance(arg, DunderSaveLoad) else Dunder(arg.__save__()) - for key, arg in kwargs.items()} - return super().__new__(cls) + self = super().__new__(cls) + self.__RAW_ARGS = args + self.__RAW_KWARGS = kwargs + return self def __save__(self): cls_name = f"{self.__module__}.{type(self).__name__}" - return cls_name, self.__RAW_ARGS, self.__RAW_KWARGS, *([] if not self._dict else [{'_dict': self.__dict__}]) + return cls_name, self.checkForDunderObjects(self.__RAW_ARGS, "encode"), \ + self.checkForDunderObjects(self.__RAW_KWARGS, "encode"), \ + *([] if not self._dict else [{'_dict': self.checkForDunderObjects(self.__dict__, "encode")}]) @classmethod def __load__(cls, raw_args, raw_kwargs, **kwargs): - raw_args = [load(*arg.save) if isinstance(arg, Dunder) else arg for arg in raw_args] - raw_kwargs = {key: load(*arg.save) if isinstance(arg, Dunder) else arg for key, arg in raw_kwargs.items()} + raw_args = cls.checkForDunderObjects(raw_args, "decode") + raw_kwargs = cls.checkForDunderObjects(raw_kwargs, "decode") self = cls(*raw_args, **raw_kwargs) - if self._dict: self.__dict__.update(kwargs['_dict']) + if self._dict and '_dict' in kwargs: self.__dict__.update(self.checkForDunderObjects(kwargs['_dict'], "decode")) return self - # todo: - def checkForDunderObjects(self, _obj): - raise NotImplementedError + @classmethod + def checkForDunderObjects(cls, _obj, _type): + assert _type in (types := ("encode", "decode")) + if isinstance(_obj, dict): + keys, vals = _obj.keys(), _obj.values() + return {key: item for item, key in zip(cls.checkForDunderObjects(list(vals), _type), keys)} + elif isinstance(_obj, list): + return [cls.checkForDunderObjects(ob, _type) for ob in _obj] + elif isinstance(_obj, tuple): + return tuple([cls.checkForDunderObjects(ob, _type) for ob in _obj]) + else: + if _type == types[0] and isinstance(_obj, DunderSaveLoad): + return Dunder(_obj.__save__()) + elif _type == types[1] and isinstance(_obj, Dunder): + return load(*_obj.save) + else: + return _obj diff --git a/src/tools/helperFunction.py b/src/tools/helperFunction.py index efd531f..618364f 100644 --- a/src/tools/helperFunction.py +++ b/src/tools/helperFunction.py @@ -34,7 +34,7 @@ def statPrinter(key, value, *, prefix='', suffix=PrintCols.CEND, end=' '): # fixme: improve -def getSize(obj, seen=None, ref=''): +def getSize(obj, seen=None, depth=0): """Recursively finds size of objects""" size = sys.getsizeof(obj) if seen is None: seen = set() @@ -42,18 +42,15 @@ def getSize(obj, seen=None, ref=''): # Important mark as seen *before* entering recursion to gracefully handle # self-referential objects seen.add(obj_id) - ref += str(obj.__class__) if isinstance(obj, dict): - size += sum([getSize(obj[k], seen, ref + str(k)) for k in obj.keys()]) - size += sum([getSize(k, seen, ref) for k in obj.keys()]) + for k in obj.keys(): + siz = getSize(obj[k], seen, depth + 1) + getSize(k, seen, depth + 1) + print('\t' * depth, 'dict', k, siz, sep=': ') + size += siz elif hasattr(obj, '__dict__'): - size += getSize(obj.__dict__, seen, ref) + size += getSize(obj.__dict__, seen, depth + 1) elif hasattr(obj, '__iter__') and not isinstance(obj, (str, bytes, bytearray)): - size += sum([getSize(i, seen, ref) for i in obj]) - - if size > 1024 * 10: # show files > 10Mb - print(obj.__class__, size) - print(ref, '\n') + size += sum([getSize(i, seen) for i in obj], depth + 1) return size diff --git a/temp.save b/temp.save new file mode 100644 index 0000000..27811f8 Binary files /dev/null and b/temp.save differ