-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathGD.py
105 lines (85 loc) · 3.52 KB
/
GD.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
import numpy as np
import scipy as sp
import cvxpy as cvx
#%%
class GradientDescentnorm:
def __init__(self, domain, eta, delta = 1e-8, outcome = "last"):
self.domain = domain
self.eta = eta
if eta == "auto":
sys.exit("Automatic setting of eta not available")
self.hessian = np.repeat(delta, domain.dimension)
self.weights = domain.center(1).reshape(domain.dimension)
self.domain.diag = True
self.outcome = outcome
self.name = "GDnorm"
def getweights(self):
return(self.weights)
def update(self, gradient):
self.hessian = self.hessian + np.dot(gradient, gradient)
wtilde = self.weights - self.eta / np.sqrt(self.hessian) * gradient
if not self.domain.testdomain(wtilde):
if self.domain.name == 'linfball':
self.weights = self.domain.project(wtilde, self.hessian).reshape(self.domain.dimension)
else:
self.weights = self.domain.project(wtilde, np.diag(np.sqrt(self.hessian))).reshape(self.domain.dimension)
else:
self.weights = wtilde
def getname(self):
return(self.name)
def LACeLupdate(self, datarow):
if self.outcome == "last":
xt = datarow[0:-1]
if self.outcome == "first":
xt = datarow[1:]
if self.outcome == "none":
xt = datarow
self.weights = self.domain.futureproject(self.weights, np.diag(1/np.sqrt(self.hessian)), xt)
def effectiveDim(self):
return(1/self.domain.dimension)
class GradientDescentt:
def __init__(self, domain, G, eta = "auto", delta = 1e-8, Ghat = False, Gupdate = False, outcome = "last"):
self.domain = domain
self.eta = eta
if eta == "auto":
sys.exit("Automatic setting of eta not available")
self.hessian = np.repeat(delta, domain.dimension)
self.weights = domain.center(1).reshape(domain.dimension)
self.domain.diag = True
self.G = G
self.Ghat = Ghat
self.Gupdate = Gupdate
self.name = "GDt"
self.outcome = outcome
self.delta = delta
def getweights(self):
return(self.weights)
def update(self, gradient):
if self.Ghat:
self.Ghat = False
self.G = np.max([np.sqrt(np.dot(gradient, gradient)), self.delta])
if self.Gupdate:
tempG = np.sqrt(np.dot(gradient, gradient))
if self.G < tempG:
self.G = tempG
self.hessian = self.hessian + 1
wtilde = self.weights - self.eta / (self.G * np.sqrt((self.hessian))) * gradient
if not self.domain.testdomain(wtilde):
if self.domain.name == 'linfball':
self.weights = self.domain.project(wtilde, self.hessian).reshape(self.domain.dimension)
else:
self.weights = self.domain.project(wtilde, np.diag(self.G*np.sqrt(self.hessian))).reshape(self.domain.dimension)
else:
self.weights = wtilde
def getname(self):
return(self.name)
def LACeLupdate(self, datarow):
if self.outcome == "last":
xt = datarow[0:-1]
if self.outcome == "first":
xt = datarow[1:]
if self.outcome == "none":
xt = datarow
self.weights = self.domain.futureproject(self.weights, np.diag(1 / (self.G * np.sqrt((self.hessian)))), xt)
def effectiveDim(self):
return(1/self.domain.dimension)