00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016 #ifndef __QUASINEWTONMETHOD_H__
00017 #define __QUASINEWTONMETHOD_H__
00018
00019
00020
00021 #include "../performance_functional/performance_functional.h"
00022
00023 #include "training_algorithm.h"
00024 #include "training_rate_algorithm.h"
00025
00026
00027
00028 #include "../../parsers/tinyxml/tinyxml.h"
00029
00030 namespace OpenNN
00031 {
00032
00036
00037 class QuasiNewtonMethod : public TrainingAlgorithm
00038 {
00039
00040 public:
00041
00042
00043
00045
00046 enum InverseHessianApproximationMethod{DFP, BFGS};
00047
00048
00049
00050
00051 explicit QuasiNewtonMethod(void);
00052
00053
00054
00055 explicit QuasiNewtonMethod(PerformanceFunctional*);
00056
00057
00058
00059 explicit QuasiNewtonMethod(TiXmlElement*);
00060
00061
00062
00063
00064 virtual ~QuasiNewtonMethod(void);
00065
00066
00067
00071
00072 struct QuasiNetwonMethodResults : public TrainingAlgorithm::Results
00073 {
00074
00075
00077
00078 Vector< Vector<double> > parameters_history;
00079
00081
00082 Vector<double> parameters_norm_history;
00083
00085
00086 Vector<double> evaluation_history;
00087
00089
00090 Vector<double> generalization_evaluation_history;
00091
00093
00094 Vector< Vector<double> > gradient_history;
00095
00097
00098 Vector<double> gradient_norm_history;
00099
00101
00102 Vector< Matrix<double> > inverse_Hessian_history;
00103
00105
00106 Vector< Vector<double> > training_direction_history;
00107
00109
00110 Vector<double> training_rate_history;
00111
00113
00114 Vector<double> elapsed_time_history;
00115
00116 void resize_training_history(const unsigned int&);
00117
00118
00119
00121
00122 Vector<double> final_parameters;
00123
00125
00126 double final_parameters_norm;
00127
00129
00130 double final_evaluation;
00131
00133
00134 double final_generalization_evaluation;
00135
00137
00138 Vector<double> final_gradient;
00139
00141
00142 double final_gradient_norm;
00143
00145
00146 Matrix<double> final_inverse_Hessian_approximation;
00147
00149
00150 Vector<double> final_training_direction;
00151
00153
00154 double final_training_rate;
00155
00157
00158 double elapsed_time;
00159
00160 std::string to_string(void) const;
00161 };
00162
00163
00164
00165
00166
00167 const TrainingRateAlgorithm& get_training_rate_algorithm(void) const;
00168 TrainingRateAlgorithm* get_training_rate_algorithm_pointer(void);
00169
00170 const InverseHessianApproximationMethod& get_inverse_Hessian_approximation_method(void) const;
00171 std::string write_inverse_Hessian_approximation_method(void) const;
00172
00173
00174
00175 const double& get_warning_parameters_norm(void) const;
00176 const double& get_warning_gradient_norm(void) const;
00177 const double& get_warning_training_rate(void) const;
00178
00179 const double& get_error_parameters_norm(void) const;
00180 const double& get_error_gradient_norm(void) const;
00181 const double& get_error_training_rate(void) const;
00182
00183
00184
00185 const double& get_minimum_parameters_increment_norm(void) const;
00186
00187 const double& get_minimum_performance_increase(void) const;
00188 const double& get_performance_goal(void) const;
00189 const double& get_gradient_norm_goal(void) const;
00190 const unsigned int& get_maximum_generalization_evaluation_decreases(void) const;
00191
00192 const unsigned int& get_maximum_epochs_number(void) const;
00193 const double& get_maximum_time(void) const;
00194
00195
00196
00197 const bool& get_reserve_parameters_history(void) const;
00198 const bool& get_reserve_parameters_norm_history(void) const;
00199
00200 const bool& get_reserve_evaluation_history(void) const;
00201 const bool& get_reserve_gradient_history(void) const;
00202 const bool& get_reserve_gradient_norm_history(void) const;
00203 const bool& get_reserve_inverse_Hessian_history(void) const;
00204 const bool& get_reserve_generalization_evaluation_history(void) const;
00205
00206 const bool& get_reserve_training_direction_history(void) const;
00207 const bool& get_reserve_training_rate_history(void) const;
00208 const bool& get_reserve_elapsed_time_history(void) const;
00209
00210
00211
00212 const unsigned int& get_display_period(void) const;
00213
00214
00215
00216 void set_inverse_Hessian_approximation_method(const InverseHessianApproximationMethod&);
00217 void set_inverse_Hessian_approximation_method(const std::string&);
00218
00219 void set_default(void);
00220
00221
00222
00223 void set_warning_parameters_norm(const double&);
00224 void set_warning_gradient_norm(const double&);
00225 void set_warning_training_rate(const double&);
00226
00227 void set_error_parameters_norm(const double&);
00228 void set_error_gradient_norm(const double&);
00229 void set_error_training_rate(const double&);
00230
00231
00232
00233 void set_minimum_parameters_increment_norm(const double&);
00234
00235 void set_minimum_performance_increase(const double&);
00236 void set_performance_goal(const double&);
00237 void set_gradient_norm_goal(const double&);
00238 void set_maximum_generalization_evaluation_decreases(const unsigned int&);
00239
00240 void set_maximum_epochs_number(const unsigned int&);
00241 void set_maximum_time(const double&);
00242
00243
00244
00245 void set_reserve_parameters_history(const bool&);
00246 void set_reserve_parameters_norm_history(const bool&);
00247
00248 void set_reserve_evaluation_history(const bool&);
00249 void set_reserve_gradient_history(const bool&);
00250 void set_reserve_gradient_norm_history(const bool&);
00251 void set_reserve_inverse_Hessian_history(const bool&);
00252 void set_reserve_generalization_evaluation_history(const bool&);
00253
00254 void set_reserve_training_direction_history(const bool&);
00255 void set_reserve_training_rate_history(const bool&);
00256 void set_reserve_elapsed_time_history(const bool&);
00257
00258
00259
00260 void set_display_period(const unsigned int&);
00261
00262
00263
00264 Vector<double> calculate_training_direction(const Vector<double>&, const Matrix<double>&) const;
00265
00266 Matrix<double> calculate_inverse_Hessian_approximation(const Vector<double>&, const Vector<double>&, const Vector<double>&, const Vector<double>&, const Matrix<double>&) const;
00267
00268 Vector<double> calculate_gradient_descent_training_direction(const Vector<double>&) const;
00269
00270 Matrix<double> calculate_DFP_inverse_Hessian
00271 (const Vector<double>&, const Vector<double>&, const Vector<double>&, const Vector<double>&, const Matrix<double>&) const;
00272
00273 Matrix<double> calculate_BFGS_inverse_Hessian
00274 (const Vector<double>&, const Vector<double>&, const Vector<double>&, const Vector<double>&, const Matrix<double>&) const;
00275
00276 QuasiNetwonMethodResults* perform_training(void);
00277
00278
00279
00280 void set_reserve_all_training_history(const bool&);
00281
00282 std::string write_training_algorithm_type(void) const;
00283
00284
00285
00286 TiXmlElement* to_XML(void) const;
00287 void from_XML(TiXmlElement*);
00288
00289
00290 private:
00291
00293
00294 TrainingRateAlgorithm training_rate_algorithm;
00295
00297
00298 InverseHessianApproximationMethod inverse_Hessian_approximation_method;
00299
00300
00302
00303 double warning_parameters_norm;
00304
00306
00307 double warning_gradient_norm;
00308
00310
00311 double warning_training_rate;
00312
00314
00315 double error_parameters_norm;
00316
00318
00319 double error_gradient_norm;
00320
00322
00323 double error_training_rate;
00324
00325
00326
00327
00329
00330 double minimum_parameters_increment_norm;
00331
00333
00334 double minimum_performance_increase;
00335
00337
00338 double performance_goal;
00339
00341
00342 double gradient_norm_goal;
00343
00345
00346 unsigned int maximum_generalization_evaluation_decreases;
00347
00349
00350 unsigned int maximum_epochs_number;
00351
00353
00354 double maximum_time;
00355
00356
00357
00359
00360 bool reserve_parameters_history;
00361
00363
00364 bool reserve_parameters_norm_history;
00365
00367
00368 bool reserve_evaluation_history;
00369
00371
00372 bool reserve_gradient_history;
00373
00375
00376 bool reserve_gradient_norm_history;
00377
00379
00380 bool reserve_inverse_Hessian_history;
00381
00383
00384 bool reserve_training_direction_history;
00385
00387
00388 bool reserve_training_rate_history;
00389
00391
00392 bool reserve_elapsed_time_history;
00393
00395
00396 bool reserve_generalization_evaluation_history;
00397
00399
00400 unsigned int display_period;
00401
00402 };
00403
00404 }
00405
00406 #endif
00407
00408
00409
00410
00411
00412
00413
00414
00415
00416
00417
00418
00419
00420
00421
00422
00423
00424