-
Notifications
You must be signed in to change notification settings - Fork 28
/
Copy pathCOptimisable.h
258 lines (241 loc) · 5.61 KB
/
COptimisable.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
/*This file contains a base class for optimisation. In particular it contains an implementation of the Scaled Conjugate Gradient optimisation routine.
05/11/2005 Changed use of abs (which was calling f2c code) to fabs to improve compatability with MSVC.
21/10/2005 More minor changes from Bill.
20/10/2005 Minor changes by William V. Baxter for MSVC compatibility. This principally removing an unused variable and changing the file in which the constants were declared. */
#ifndef COPTIMISABLE_H
#define COPTIMISABLE_H
#include "CMatrix.h"
#include "ndlutil.h"
#include "ndlfortran.h"
// abstract base class for making a class optimisable.
class COptimisable {
public:
enum
{
CG,
SCG,
GD,
BFGS,
LBFGS
};
COptimisable()
{
// set default optimisation parameters
setVerbosity(2);
setDefaultOptimiser(SCG);
setFuncEvalTerminate(false);
setIterTerminate(true);
setMaxFuncEvals(1000);
setMaxIters(1000);
setObjectiveTol(1e-6);
setParamTol(1e-6);
setLearnRate(0.01);
setMomentum(0.9);
iter = 0;
funcEval = 0;
}
virtual ~COptimisable() {}
virtual inline void setVerbosity(int val) const
{
verbosity = val;
}
virtual inline int getVerbosity() const
{
return verbosity;
}
virtual unsigned int getOptNumParams() const=0;
virtual void getOptParams(CMatrix& param) const=0;
virtual void setOptParams(const CMatrix& param)=0;
virtual double computeObjectiveGradParams(CMatrix& g) const=0;
virtual double computeObjectiveVal() const=0;
inline void setDirection(const CMatrix& vals)
{
DIMENSIONMATCH(vals.getCols()==getOptNumParams());
DIMENSIONMATCH(vals.getRows()==1);
direction.deepCopy(vals);
}
inline void getDirection(CMatrix& vals) const
{
DIMENSIONMATCH(vals.getCols()==getOptNumParams());
DIMENSIONMATCH(vals.getRows()==1);
DIMENSIONMATCH(direction.dimensionsMatch(vals));
vals.deepCopy(direction);
}
void checkGradients();
void gdOptimise();
void gdPullbackOptimise();
//void netlabScgOptimise();
void lbfgsOptimise();
void scgOptimise();
void cgOptimise();
//void lineMinimisation(const CMatrix& direction);
double oneDObjectiveVal(double val);
//void lineMinimisation();
//void bracketMinimum(double& a, double& b, double& c, double& fa, unsigned int maxStep);
void setLearnRate(double val)
{
learnRate = val;
}
double getLearnRate() const
{
return learnRate;
}
void setMomentum(double val)
{
momentum = val;
}
double getMomentum() const
{
return momentum;
}
void setFuncEvalTerminate(bool val)
{
funcEvalTerminate = val;
}
bool isFuncEvalTerminate() const
{
return funcEvalTerminate;
}
void setIterTerminate(bool val)
{
iterTerminate = val;
}
bool isIterTerminate() const
{
return iterTerminate;
}
void setMaxFuncEvals(unsigned int val)
{
maxFuncVal = val;
}
unsigned int getMaxFuncEvals() const
{
return maxFuncVal;
}
void setMaxIters(unsigned int val)
{
maxIters = val;
}
unsigned int getMaxIters() const
{
return maxIters;
}
void setObjectiveTol(double val)
{
objectiveTol = val;
}
double getObjectiveTol() const
{
return objectiveTol;
}
void setParamTol(double val)
{
parameterTol = val;
}
double getParamTol() const
{
return parameterTol;
}
void setDefaultOptimiser(int val) const
{
defaultOptimiser = val;
}
int getDefaultOptimiser() const
{
return defaultOptimiser;
}
void setDefaultOptimiserStr(string val)
{
if(val == "conjgrad")
defaultOptimiser = CG;
else if(val == "scg")
defaultOptimiser = SCG;
else if(val == "quasinew")
defaultOptimiser = BFGS;
else if(val == "graddesc")
defaultOptimiser = GD;
else
throw ndlexceptions::NotImplementedError("Unknown optimisation");
}
string getDefaultOptimiserStr() const
{
switch(defaultOptimiser)
{
case CG:
return "conjgrad";
case SCG:
return "scg";
case BFGS:
return "quasinew";
case GD:
return "graddesc";
default:
throw ndlexceptions::NotImplementedError("Unknown optimisation.");
}
}
void runDefaultOptimiser()
{
switch(defaultOptimiser)
{
case CG:
cgOptimise();
break;
case SCG:
scgOptimise();
break;
case BFGS:
lbfgsOptimise();
break;
case GD:
gdOptimise();
break;
default:
throw ndlexceptions::NotImplementedError("Unknown optimisation.");
}
}
private:
double objectiveTol;
double parameterTol;
#ifndef SWIG
const static bool evalFunc=true;
const static double phi;
const static double cphi;
const static double smallNum;
#else
static bool evalFunc=true;
static double phi;
static double cphi;
static double smallNum;
#endif
double learnRate;
double momentum;
unsigned int iter;
unsigned int funcEval;
unsigned int maxIters;
unsigned int maxFuncVal;
mutable int verbosity;
mutable int defaultOptimiser;
bool funcEvalTerminate;
bool iterTerminate;
CMatrix direction; // direction for 1-D optimisation.
CMatrix paramStoreOne;
CMatrix paramStoreTwo;
};
class CProbabilisticOptimisable : public COptimisable
{
public:
CProbabilisticOptimisable() : COptimisable() {}
virtual double logLikelihood() const=0;
virtual double logLikelihoodGradient(CMatrix& g) const=0;
virtual double computeObjectiveGradParams(CMatrix& g) const
{
double L = logLikelihoodGradient(g);
g.negate();
return -L;
}
virtual double computeObjectiveVal() const
{
return -logLikelihood();
}
};
#endif