00001 /****************************************************************************************************************/ 00002 /* */ 00003 /* OpenNN: Open Neural Networks Library */ 00004 /* www.opennn.cimne.com */ 00005 /* */ 00006 /* L E V E N B E R G - M A R Q U A R D T A L G O R I T H M C L A S S H E A D E R */ 00007 /* */ 00008 /* Roberto Lopez */ 00009 /* International Center for Numerical Methods in Engineering (CIMNE) */ 00010 /* Technical University of Catalonia (UPC) */ 00011 /* Barcelona, Spain */ 00012 /* E-mail: rlopez@cimne.upc.edu */ 00013 /* */ 00014 /****************************************************************************************************************/ 00015 00016 #ifndef __LEVENBERGMARQUARDTALGORITHM_H__ 00017 #define __LEVENBERGMARQUARDTALGORITHM_H__ 00018 00019 // OpenNN includes 00020 00021 #include "../performance_functional/sum_squared_error.h" 00022 #include "../performance_functional/mean_squared_error.h" 00023 #include "../performance_functional/normalized_squared_error.h" 00024 00025 #include "training_algorithm.h" 00026 00027 #include "../utilities/linear_algebraic_equations.h" 00028 00029 // TinyXml includes 00030 00031 #include "../../parsers/tinyxml/tinyxml.h" 00032 00033 namespace OpenNN 00034 { 00035 00038 00039 class LevenbergMarquardtAlgorithm : public TrainingAlgorithm 00040 { 00041 00042 public: 00043 00044 // DEFAULT CONSTRUCTOR 00045 00046 explicit LevenbergMarquardtAlgorithm(void); 00047 00048 // PERFORMANCE FUNCTIONAL CONSTRUCTOR 00049 00050 explicit LevenbergMarquardtAlgorithm(PerformanceFunctional*); 00051 00052 // XML CONSTRUCTOR 00053 00054 explicit LevenbergMarquardtAlgorithm(TiXmlElement*); 00055 00056 // DESTRUCTOR 00057 00058 virtual ~LevenbergMarquardtAlgorithm(void); 00059 00060 // STRUCTURES 00061 00065 00066 struct LevenbergMarquardtAlgorithmResults : public TrainingAlgorithm::Results 00067 { 00068 // Training history 00069 00071 00072 Vector< Vector<double> > parameters_history; 00073 00075 00076 Vector<double> parameters_norm_history; 00077 00079 00080 Vector<double> evaluation_history; 00081 00083 00084 Vector<double> generalization_evaluation_history; 00085 00087 00088 Vector< Vector<double> > gradient_history; 00089 00091 00092 Vector<double> gradient_norm_history; 00093 00095 00096 Vector< Matrix<double> > Hessian_approximation_history; 00097 00099 00100 Vector<double> damping_parameter_history; 00101 00103 00104 Vector<double> elapsed_time_history; 00105 00106 // Final values 00107 00109 00110 Vector<double> final_parameters; 00111 00113 00114 double final_parameters_norm; 00115 00117 00118 double final_evaluation; 00119 00121 00122 double final_generalization_evaluation; 00123 00125 00126 Vector<double> final_gradient; 00127 00129 00130 double final_gradient_norm; 00131 00133 00134 double elapsed_time; 00135 00136 void resize_training_history(const unsigned int&); 00137 std::string to_string(void) const; 00138 }; 00139 00140 // METHODS 00141 00142 // Get methods 00143 00144 // Training parameters 00145 00146 const double& get_warning_parameters_norm(void) const; 00147 const double& get_warning_gradient_norm(void) const; 00148 00149 const double& get_error_parameters_norm(void) const; 00150 const double& get_error_gradient_norm(void) const; 00151 00152 // Stopping criteria 00153 00154 const double& get_minimum_parameters_increment_norm(void) const; 00155 00156 const double& get_minimum_performance_increase(void) const; 00157 const double& get_performance_goal(void) const; 00158 const double& get_gradient_norm_goal(void) const; 00159 const unsigned int& get_maximum_generalization_evaluation_decreases(void) const; 00160 00161 const unsigned int& get_maximum_epochs_number(void) const; 00162 const double& get_maximum_time(void) const; 00163 00164 // Reserve training history 00165 00166 const bool& get_reserve_parameters_history(void) const; 00167 const bool& get_reserve_parameters_norm_history(void) const; 00168 00169 const bool& get_reserve_evaluation_history(void) const; 00170 const bool& get_reserve_gradient_history(void) const; 00171 const bool& get_reserve_gradient_norm_history(void) const; 00172 const bool& get_reserve_inverse_Hessian_history(void) const; 00173 const bool& get_reserve_generalization_evaluation_history(void) const; 00174 00175 const bool& get_reserve_elapsed_time_history(void) const; 00176 00177 // Utilities 00178 00179 const unsigned int& get_display_period(void) const; 00180 00181 const double& get_damping_parameter(void) const; 00182 00183 const double& get_damping_parameter_factor(void) const; 00184 00185 const double& get_minimum_damping_parameter(void) const; 00186 const double& get_maximum_damping_parameter(void) const; 00187 00188 const bool& get_reserve_damping_parameter_history(void) const; 00189 00190 const Vector<double>& get_damping_parameter_history(void) const; 00191 00192 // Set methods 00193 00194 void set_default(void); 00195 00196 void set_damping_parameter(const double&); 00197 00198 void set_damping_parameter_factor(const double&); 00199 00200 void set_minimum_damping_parameter(const double&); 00201 void set_maximum_damping_parameter(const double&); 00202 00203 void set_reserve_damping_parameter_history(const bool&); 00204 00205 // Training parameters 00206 00207 void set_warning_parameters_norm(const double&); 00208 void set_warning_gradient_norm(const double&); 00209 00210 void set_error_parameters_norm(const double&); 00211 void set_error_gradient_norm(const double&); 00212 00213 // Stopping criteria 00214 00215 void set_minimum_parameters_increment_norm(const double&); 00216 00217 void set_minimum_performance_increase(const double&); 00218 void set_performance_goal(const double&); 00219 void set_gradient_norm_goal(const double&); 00220 void set_maximum_generalization_evaluation_decreases(const unsigned int&); 00221 00222 void set_maximum_epochs_number(const unsigned int&); 00223 void set_maximum_time(const double&); 00224 00225 // Reserve training history 00226 00227 void set_reserve_parameters_history(const bool&); 00228 void set_reserve_parameters_norm_history(const bool&); 00229 00230 void set_reserve_evaluation_history(const bool&); 00231 void set_reserve_gradient_history(const bool&); 00232 void set_reserve_gradient_norm_history(const bool&); 00233 void set_reserve_inverse_Hessian_history(const bool&); 00234 void set_reserve_generalization_evaluation_history(const bool&); 00235 00236 void set_reserve_elapsed_time_history(const bool&); 00237 00239 00240 virtual void set_reserve_all_training_history(const bool&); 00241 00242 // Utilities 00243 00244 void set_display_period(const unsigned int&); 00245 00246 // Training methods 00247 00248 void check(void) const; 00249 00250 Vector<double> calculate_gradient(const Vector<double>&, const Matrix<double>&) const; 00251 Matrix<double> calculate_Hessian_approximation(const Matrix<double>&) const; 00252 00253 LevenbergMarquardtAlgorithmResults* perform_training(void); 00254 00255 std::string write_training_algorithm_type(void) const; 00256 00257 // Serialization methods 00258 00259 TiXmlElement* to_XML(void) const; 00260 00261 void from_XML(TiXmlElement*); 00262 00263 private: 00264 00265 // MEMBERS 00266 00268 00269 LinearAlgebraicEquations linear_algebraic_equations; 00270 00272 00273 double damping_parameter; 00274 00276 00277 double minimum_damping_parameter; 00278 00280 00281 double maximum_damping_parameter; 00282 00284 00285 double damping_parameter_factor; 00286 00288 00289 bool reserve_damping_parameter_history; 00290 00292 00293 Vector<double> damping_parameter_history; 00294 00295 00297 00298 double warning_parameters_norm; 00299 00301 00302 double warning_gradient_norm; 00303 00305 00306 double error_parameters_norm; 00307 00309 00310 double error_gradient_norm; 00311 00312 00313 // STOPPING CRITERIA 00314 00316 00317 double minimum_parameters_increment_norm; 00318 00320 00321 double minimum_performance_increase; 00322 00324 00325 double performance_goal; 00326 00328 00329 double gradient_norm_goal; 00330 00331 unsigned int maximum_generalization_evaluation_decreases; 00332 00334 00335 unsigned int maximum_epochs_number; 00336 00338 00339 double maximum_time; 00340 00341 // TRAINING HISTORY 00342 00344 00345 bool reserve_parameters_history; 00346 00348 00349 bool reserve_parameters_norm_history; 00350 00352 00353 bool reserve_evaluation_history; 00354 00356 00357 bool reserve_gradient_history; 00358 00360 00361 bool reserve_gradient_norm_history; 00362 00364 00365 bool reserve_inverse_Hessian_history; 00366 00368 00369 bool reserve_elapsed_time_history; 00370 00372 00373 bool reserve_generalization_evaluation_history; 00374 00376 00377 unsigned int display_period; 00378 }; 00379 00380 } 00381 00382 #endif 00383 00384 00385 // OpenNN: Open Neural Networks Library. 00386 // Copyright (C) 2005-2012 Roberto Lopez 00387 // 00388 // This library is free software; you can redistribute it and/or 00389 // modify it under the terms of the GNU Lesser General Public 00390 // License as published by the Free Software Foundation; either 00391 // version 2.1 of the License, or any later version. 00392 // 00393 // This library is distributed in the hope that it will be useful, 00394 // but WITHOUT ANY WARRANTY; without even the implied warranty of 00395 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 00396 // Lesser General Public License for more details. 00397 00398 // You should have received a copy of the GNU Lesser General Public 00399 // License along with this library; if not, write to the Free Software 00400 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA