00001 /****************************************************************************************************************/ 00002 /* */ 00003 /* OpenNN: Open Neural Networks Library */ 00004 /* www.opennn.cimne.com */ 00005 /* */ 00006 /* N E W T O N M E T H O D C L A S S H E A D E R */ 00007 /* */ 00008 /* Roberto Lopez */ 00009 /* International Center for Numerical Methods in Engineering (CIMNE) */ 00010 /* Technical University of Catalonia (UPC) */ 00011 /* Barcelona, Spain */ 00012 /* E-mail: rlopez@cimne.upc.edu */ 00013 /* */ 00014 /****************************************************************************************************************/ 00015 00016 #ifndef __NEWTONMETHOD_H__ 00017 #define __NEWTONMETHOD_H__ 00018 00019 // OpenNN includes 00020 00021 #include "../performance_functional/performance_functional.h" 00022 00023 #include "training_algorithm.h" 00024 #include "training_rate_algorithm.h" 00025 00026 // TinyXml includes 00027 00028 #include "../../parsers/tinyxml/tinyxml.h" 00029 00030 namespace OpenNN 00031 { 00032 00036 00037 class NewtonMethod : public TrainingAlgorithm 00038 { 00039 00040 public: 00041 00042 // DEFAULT CONSTRUCTOR 00043 00044 explicit NewtonMethod(void); 00045 00046 // PERFORMANCE FUNCTIONAL CONSTRUCTOR 00047 00048 explicit NewtonMethod(PerformanceFunctional*); 00049 00050 // XML CONSTRUCTOR 00051 00052 explicit NewtonMethod(TiXmlElement*); 00053 00054 00055 // DESTRUCTOR 00056 00057 virtual ~NewtonMethod(void); 00058 00059 00060 // STRUCTURES 00061 00065 00066 struct NewtonMethodResults : public TrainingAlgorithm::Results 00067 { 00068 // Training history 00069 00071 00072 Vector< Vector<double> > parameters_history; 00073 00075 00076 Vector<double> parameters_norm_history; 00077 00079 00080 Vector<double> evaluation_history; 00081 00083 00084 Vector<double> generalization_evaluation_history; 00085 00087 00088 Vector< Vector<double> > gradient_history; 00089 00091 00092 Vector<double> gradient_norm_history; 00093 00095 00096 Vector< Matrix<double> > inverse_Hessian_history; 00097 00099 00100 Vector< Vector<double> > training_direction_history; 00101 00103 00104 Vector<double> training_rate_history; 00105 00107 00108 Vector<double> elapsed_time_history; 00109 00110 // Final values 00111 00113 00114 Vector<double> final_parameters; 00115 00117 00118 double final_parameters_norm; 00119 00121 00122 double final_evaluation; 00123 00125 00126 double final_generalization_evaluation; 00127 00129 00130 Vector<double> final_gradient; 00131 00133 00134 double final_gradient_norm; 00135 00137 00138 Vector<double> final_training_direction; 00139 00141 00142 double final_training_rate; 00143 00145 00146 double elapsed_time; 00147 00148 void resize_training_history(const unsigned int&); 00149 std::string to_string(void) const; 00150 }; 00151 00152 00153 // METHODS 00154 00155 const TrainingRateAlgorithm& get_training_rate_algorithm(void) const; 00156 TrainingRateAlgorithm* get_training_rate_algorithm_pointer(void); 00157 00158 // Training parameters 00159 00160 const double& get_warning_parameters_norm(void) const; 00161 const double& get_warning_gradient_norm(void) const; 00162 const double& get_warning_training_rate(void) const; 00163 00164 const double& get_error_parameters_norm(void) const; 00165 const double& get_error_gradient_norm(void) const; 00166 const double& get_error_training_rate(void) const; 00167 00168 // Stopping criteria 00169 00170 const double& get_minimum_parameters_increment_norm(void) const; 00171 00172 const double& get_minimum_performance_increase(void) const; 00173 const double& get_performance_goal(void) const; 00174 const double& get_gradient_norm_goal(void) const; 00175 const unsigned int& get_maximum_generalization_evaluation_decreases(void) const; 00176 00177 const unsigned int& get_maximum_epochs_number(void) const; 00178 const double& get_maximum_time(void) const; 00179 00180 // Reserve training history 00181 00182 const bool& get_reserve_parameters_history(void) const; 00183 const bool& get_reserve_parameters_norm_history(void) const; 00184 00185 const bool& get_reserve_evaluation_history(void) const; 00186 const bool& get_reserve_gradient_history(void) const; 00187 const bool& get_reserve_gradient_norm_history(void) const; 00188 const bool& get_reserve_inverse_Hessian_history(void) const; 00189 const bool& get_reserve_generalization_evaluation_history(void) const; 00190 00191 const bool& get_reserve_training_direction_history(void) const; 00192 const bool& get_reserve_training_rate_history(void) const; 00193 const bool& get_reserve_elapsed_time_history(void) const; 00194 00195 // Utilities 00196 00197 const unsigned int& get_display_period(void) const; 00198 00199 void set_default(void); 00200 00201 // Training parameters 00202 00203 void set_warning_parameters_norm(const double&); 00204 void set_warning_gradient_norm(const double&); 00205 void set_warning_training_rate(const double&); 00206 00207 void set_error_parameters_norm(const double&); 00208 void set_error_gradient_norm(const double&); 00209 void set_error_training_rate(const double&); 00210 00211 // Stopping criteria 00212 00213 void set_minimum_parameters_increment_norm(const double&); 00214 00215 void set_minimum_performance_increase(const double&); 00216 void set_performance_goal(const double&); 00217 void set_gradient_norm_goal(const double&); 00218 void set_maximum_generalization_evaluation_decreases(const unsigned int&); 00219 00220 void set_maximum_epochs_number(const unsigned int&); 00221 void set_maximum_time(const double&); 00222 00223 // Reserve training history 00224 00225 void set_reserve_parameters_history(const bool&); 00226 void set_reserve_parameters_norm_history(const bool&); 00227 00228 void set_reserve_evaluation_history(const bool&); 00229 void set_reserve_gradient_history(const bool&); 00230 void set_reserve_gradient_norm_history(const bool&); 00231 void set_reserve_inverse_Hessian_history(const bool&); 00232 void set_reserve_generalization_evaluation_history(const bool&); 00233 00234 void set_reserve_training_direction_history(const bool&); 00235 void set_reserve_training_rate_history(const bool&); 00236 void set_reserve_elapsed_time_history(const bool&); 00237 00239 00240 void set_reserve_all_training_history(const bool&); 00241 00242 // Utilities 00243 00244 void set_display_period(const unsigned int&); 00245 00246 // Training methods 00247 00248 Vector<double> calculate_gradient_descent_training_direction(const Vector<double>&) const; 00249 Vector<double> calculate_training_direction(const Vector<double>&, const Matrix<double>&) const; 00250 00251 NewtonMethodResults* perform_training(void); 00252 00253 std::string write_training_algorithm_type(void) const; 00254 00255 // Serialization methods 00256 00257 TiXmlElement* to_XML(void) const; 00258 void from_XML(TiXmlElement*); 00259 00260 private: 00261 00262 TrainingRateAlgorithm training_rate_algorithm; 00263 00265 00266 double warning_parameters_norm; 00267 00269 00270 double warning_gradient_norm; 00271 00273 00274 double warning_training_rate; 00275 00277 00278 double error_parameters_norm; 00279 00281 00282 double error_gradient_norm; 00283 00285 00286 double error_training_rate; 00287 00288 00289 // STOPPING CRITERIA 00290 00292 00293 double minimum_parameters_increment_norm; 00294 00296 00297 double minimum_performance_increase; 00298 00300 00301 double performance_goal; 00302 00304 00305 double gradient_norm_goal; 00306 00307 unsigned int maximum_generalization_evaluation_decreases; 00308 00310 00311 unsigned int maximum_epochs_number; 00312 00314 00315 double maximum_time; 00316 00317 // TRAINING HISTORY 00318 00320 00321 bool reserve_parameters_history; 00322 00324 00325 bool reserve_parameters_norm_history; 00326 00328 00329 bool reserve_evaluation_history; 00330 00332 00333 bool reserve_gradient_history; 00334 00336 00337 bool reserve_gradient_norm_history; 00338 00340 00341 bool reserve_inverse_Hessian_history; 00342 00344 00345 bool reserve_training_direction_history; 00346 00348 00349 bool reserve_training_rate_history; 00350 00352 00353 bool reserve_elapsed_time_history; 00354 00356 00357 bool reserve_generalization_evaluation_history; 00358 00360 00361 unsigned int display_period; 00362 00363 }; 00364 00365 } 00366 00367 #endif 00368 00369 00370 // OpenNN: Open Neural Networks Library. 00371 // Copyright (C) 2005-2012 Roberto Lopez 00372 // 00373 // This library is free software; you can redistribute it and/or 00374 // modify it under the terms of the GNU Lesser General Public 00375 // License as published by the Free Software Foundation; either 00376 // version 2.1 of the License, or any later version. 00377 // 00378 // This library is distributed in the hope that it will be useful, 00379 // but WITHOUT ANY WARRANTY; without even the implied warranty of 00380 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 00381 // Lesser General Public License for more details. 00382 00383 // You should have received a copy of the GNU Lesser General Public 00384 // License along with this library; if not, write to the Free Software 00385 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 00386