00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018 #include <string>
00019 #include <sstream>
00020 #include <iostream>
00021 #include <fstream>
00022 #include <algorithm>
00023 #include <functional>
00024 #include <limits>
00025 #include <cmath>
00026 #include <ctime>
00027
00028
00029
00030 #include "gradient_descent.h"
00031
00032 namespace OpenNN
00033 {
00034
00035
00036
00040
00041 GradientDescent::GradientDescent(void)
00042 : TrainingAlgorithm()
00043 {
00044 set_default();
00045 }
00046
00047
00048
00049
00054
00055 GradientDescent::GradientDescent(PerformanceFunctional* new_performance_functional_pointer)
00056 : TrainingAlgorithm(new_performance_functional_pointer)
00057 {
00058 training_rate_algorithm.set_performance_functional_pointer(new_performance_functional_pointer);
00059
00060 set_default();
00061 }
00062
00063
00064
00065
00070
00071 GradientDescent::GradientDescent(TiXmlElement* gradient_descent_element) : TrainingAlgorithm(gradient_descent_element)
00072 {
00073 set_default();
00074
00075 from_XML(gradient_descent_element);
00076 }
00077
00078
00079
00080
00082
00083 GradientDescent::~GradientDescent(void)
00084 {
00085 }
00086
00087
00088
00089
00090
00091
00093
00094 const TrainingRateAlgorithm& GradientDescent::get_training_rate_algorithm(void) const
00095 {
00096 return(training_rate_algorithm);
00097 }
00098
00099
00100
00101
00103
00104 TrainingRateAlgorithm* GradientDescent::get_training_rate_algorithm_pointer(void)
00105 {
00106 return(&training_rate_algorithm);
00107 }
00108
00109
00110
00111
00114
00115 const double& GradientDescent::get_warning_parameters_norm(void) const
00116 {
00117 return(warning_parameters_norm);
00118 }
00119
00120
00121
00122
00125
00126 const double& GradientDescent::get_warning_gradient_norm(void) const
00127 {
00128 return(warning_gradient_norm);
00129 }
00130
00131
00132
00133
00136
00137 const double& GradientDescent::get_warning_training_rate(void) const
00138 {
00139 return(warning_training_rate);
00140 }
00141
00142
00143
00144
00147
00148 const double& GradientDescent::get_error_parameters_norm(void) const
00149 {
00150 return(error_parameters_norm);
00151 }
00152
00153
00154
00155
00158
00159 const double& GradientDescent::get_error_gradient_norm(void) const
00160 {
00161 return(error_gradient_norm);
00162 }
00163
00164
00165
00166
00169
00170 const double& GradientDescent::get_error_training_rate(void) const
00171 {
00172 return(error_training_rate);
00173 }
00174
00175
00176
00177
00179
00180 const double& GradientDescent::get_minimum_parameters_increment_norm(void) const
00181 {
00182 return(minimum_parameters_increment_norm);
00183 }
00184
00185
00186
00187
00189
00190 const double& GradientDescent::get_minimum_performance_increase(void) const
00191 {
00192 return(minimum_performance_increase);
00193 }
00194
00195
00196
00197
00200
00201 const double& GradientDescent::get_performance_goal(void) const
00202 {
00203 return(performance_goal);
00204 }
00205
00206
00207
00208
00211
00212 const double& GradientDescent::get_gradient_norm_goal(void) const
00213 {
00214 return(gradient_norm_goal);
00215 }
00216
00217
00218
00219
00221
00222 const unsigned int& GradientDescent::get_maximum_generalization_evaluation_decreases(void) const
00223 {
00224 return(maximum_generalization_evaluation_decreases);
00225 }
00226
00227
00228
00229
00231
00232 const unsigned int& GradientDescent::get_maximum_epochs_number(void) const
00233 {
00234 return(maximum_epochs_number);
00235 }
00236
00237
00238
00239
00241
00242 const double& GradientDescent::get_maximum_time(void) const
00243 {
00244 return(maximum_time);
00245 }
00246
00247
00248
00249
00251
00252 const bool& GradientDescent::get_reserve_parameters_history(void) const
00253 {
00254 return(reserve_parameters_history);
00255 }
00256
00257
00258
00259
00261
00262 const bool& GradientDescent::get_reserve_parameters_norm_history(void) const
00263 {
00264 return(reserve_parameters_norm_history);
00265 }
00266
00267
00268
00269
00271
00272 const bool& GradientDescent::get_reserve_evaluation_history(void) const
00273 {
00274 return(reserve_evaluation_history);
00275 }
00276
00277
00278
00279
00281
00282 const bool& GradientDescent::get_reserve_gradient_history(void) const
00283 {
00284 return(reserve_gradient_history);
00285 }
00286
00287
00288
00289
00291
00292 const bool& GradientDescent::get_reserve_gradient_norm_history(void) const
00293 {
00294 return(reserve_gradient_norm_history);
00295 }
00296
00297
00298
00299
00301
00302 const bool& GradientDescent::get_reserve_training_direction_history(void) const
00303 {
00304 return(reserve_training_direction_history);
00305 }
00306
00307
00308
00309
00311
00312 const bool& GradientDescent::get_reserve_training_rate_history(void) const
00313 {
00314 return(reserve_training_rate_history);
00315 }
00316
00317
00318
00319
00321
00322 const bool& GradientDescent::get_reserve_elapsed_time_history(void) const
00323 {
00324 return(reserve_elapsed_time_history);
00325 }
00326
00327
00328
00329
00331
00332 const bool& GradientDescent::get_reserve_generalization_evaluation_history(void) const
00333 {
00334 return(reserve_generalization_evaluation_history);
00335 }
00336
00337
00338
00339
00341
00342 const unsigned int& GradientDescent::get_display_period(void) const
00343 {
00344 return(display_period);
00345 }
00346
00347
00348
00349
00352
00353 void GradientDescent::set_training_rate_algorithm(const TrainingRateAlgorithm& new_training_rate_algorithm)
00354 {
00355 training_rate_algorithm = new_training_rate_algorithm;
00356 }
00357
00358
00359
00360
00361 void GradientDescent::set_default(void)
00362 {
00363
00364
00365 warning_parameters_norm = 1.0e6;
00366 warning_gradient_norm = 1.0e6;
00367 warning_training_rate = 1.0e6;
00368
00369 error_parameters_norm = 1.0e9;
00370 error_gradient_norm = 1.0e9;
00371 error_training_rate = 1.0e9;
00372
00373
00374
00375 minimum_parameters_increment_norm = 0.0;
00376
00377 minimum_performance_increase = 0.0;
00378 performance_goal = -1.0e99;
00379 gradient_norm_goal = 0.0;
00380 maximum_generalization_evaluation_decreases = 1000000;
00381
00382 maximum_epochs_number = 1000;
00383 maximum_time = 1000.0;
00384
00385
00386
00387 reserve_parameters_history = false;
00388 reserve_parameters_norm_history = false;
00389
00390 reserve_evaluation_history = true;
00391 reserve_gradient_history = false;
00392 reserve_gradient_norm_history = false;
00393 reserve_generalization_evaluation_history = false;
00394
00395 reserve_training_direction_history = false;
00396 reserve_training_rate_history = false;
00397 reserve_elapsed_time_history = false;
00398
00399
00400
00401 display = true;
00402 display_period = 100;
00403 }
00404
00405
00406
00407
00421
00422 void GradientDescent::set_reserve_all_training_history(const bool& new_reserve_all_training_history)
00423 {
00424
00425
00426 reserve_parameters_history = new_reserve_all_training_history;
00427 reserve_parameters_norm_history = new_reserve_all_training_history;
00428
00429
00430
00431 reserve_evaluation_history = new_reserve_all_training_history;
00432 reserve_gradient_history = new_reserve_all_training_history;
00433 reserve_gradient_norm_history = new_reserve_all_training_history;
00434
00435 reserve_generalization_evaluation_history = new_reserve_all_training_history;
00436
00437
00438
00439 reserve_training_direction_history = new_reserve_all_training_history;
00440 reserve_training_rate_history = new_reserve_all_training_history;
00441
00442 reserve_elapsed_time_history = new_reserve_all_training_history;
00443 }
00444
00445
00446
00447
00451
00452 void GradientDescent::set_warning_parameters_norm(const double& new_warning_parameters_norm)
00453 {
00454
00455
00456 #ifdef _DEBUG
00457
00458 if(new_warning_parameters_norm < 0.0)
00459 {
00460 std::ostringstream buffer;
00461
00462 buffer << "OpenNN Exception: TrainingAlgorithm class.\n"
00463 << "void set_warning_parameters_norm(const double&) method.\n"
00464 << "Warning parameters norm must be equal or greater than 0.\n";
00465
00466 throw std::logic_error(buffer.str().c_str());
00467 }
00468
00469 #endif
00470
00471
00472
00473 warning_parameters_norm = new_warning_parameters_norm;
00474 }
00475
00476
00477
00478
00482
00483 void GradientDescent::set_warning_gradient_norm(const double& new_warning_gradient_norm)
00484 {
00485
00486
00487 #ifdef _DEBUG
00488
00489 if(new_warning_gradient_norm < 0.0)
00490 {
00491 std::ostringstream buffer;
00492
00493 buffer << "OpenNN Exception: TrainingAlgorithm class.\n"
00494 << "void set_warning_gradient_norm(const double&) method.\n"
00495 << "Warning gradient norm must be equal or greater than 0.\n";
00496
00497 throw std::logic_error(buffer.str().c_str());
00498 }
00499
00500 #endif
00501
00502
00503
00504 warning_gradient_norm = new_warning_gradient_norm;
00505 }
00506
00507
00508
00509
00513
00514 void GradientDescent::set_warning_training_rate(const double& new_warning_training_rate)
00515 {
00516
00517
00518 #ifdef _DEBUG
00519
00520 if(new_warning_training_rate < 0.0)
00521 {
00522 std::ostringstream buffer;
00523
00524 buffer << "OpenNN Exception: TrainingAlgorithm class.\n"
00525 << "void set_warning_training_rate(const double&) method.\n"
00526 << "Warning training rate must be equal or greater than 0.\n";
00527
00528 throw std::logic_error(buffer.str().c_str());
00529 }
00530
00531 #endif
00532
00533 warning_training_rate = new_warning_training_rate;
00534 }
00535
00536
00537
00538
00542
00543 void GradientDescent::set_error_parameters_norm(const double& new_error_parameters_norm)
00544 {
00545
00546
00547 #ifdef _DEBUG
00548
00549 if(new_error_parameters_norm < 0.0)
00550 {
00551 std::ostringstream buffer;
00552
00553 buffer << "OpenNN Exception: TrainingAlgorithm class.\n"
00554 << "void set_error_parameters_norm(const double&) method.\n"
00555 << "Error parameters norm must be equal or greater than 0.\n";
00556
00557 throw std::logic_error(buffer.str().c_str());
00558 }
00559
00560 #endif
00561
00562
00563
00564 error_parameters_norm = new_error_parameters_norm;
00565 }
00566
00567
00568
00569
00573
00574 void GradientDescent::set_error_gradient_norm(const double& new_error_gradient_norm)
00575 {
00576
00577
00578 #ifdef _DEBUG
00579
00580 if(new_error_gradient_norm < 0.0)
00581 {
00582 std::ostringstream buffer;
00583
00584 buffer << "OpenNN Exception: TrainingAlgorithm class.\n"
00585 << "void set_error_gradient_norm(const double&) method.\n"
00586 << "Error gradient norm must be equal or greater than 0.\n";
00587
00588 throw std::logic_error(buffer.str().c_str());
00589 }
00590
00591 #endif
00592
00593
00594
00595 error_gradient_norm = new_error_gradient_norm;
00596 }
00597
00598
00599
00600
00604
00605 void GradientDescent::set_error_training_rate(const double& new_error_training_rate)
00606 {
00607
00608
00609 #ifdef _DEBUG
00610
00611 if(new_error_training_rate < 0.0)
00612 {
00613 std::ostringstream buffer;
00614
00615 buffer << "OpenNN Exception: TrainingAlgorithm class.\n"
00616 << "void set_error_training_rate(const double&) method.\n"
00617 << "Error training rate must be equal or greater than 0.\n";
00618
00619 throw std::logic_error(buffer.str().c_str());
00620 }
00621
00622 #endif
00623
00624
00625
00626 error_training_rate = new_error_training_rate;
00627 }
00628
00629
00630
00631
00634
00635 void GradientDescent::set_minimum_parameters_increment_norm(const double& new_minimum_parameters_increment_norm)
00636 {
00637
00638
00639 #ifdef _DEBUG
00640
00641 if(new_minimum_parameters_increment_norm < 0.0)
00642 {
00643 std::ostringstream buffer;
00644
00645 buffer << "OpenNN Exception: TrainingAlgorithm class.\n"
00646 << "void new_minimum_parameters_increment_norm(const double&) method.\n"
00647 << "Minimum parameters increment norm must be equal or greater than 0.\n";
00648
00649 throw std::logic_error(buffer.str().c_str());
00650 }
00651
00652 #endif
00653
00654
00655
00656 minimum_parameters_increment_norm = new_minimum_parameters_increment_norm;
00657 }
00658
00659
00660
00661
00664
00665 void GradientDescent::set_minimum_performance_increase(const double& new_minimum_performance_increase)
00666 {
00667
00668
00669 #ifdef _DEBUG
00670
00671 if(new_minimum_performance_increase < 0.0)
00672 {
00673 std::ostringstream buffer;
00674
00675 buffer << "OpenNN Exception: TrainingAlgorithm class.\n"
00676 << "void set_minimum_performance_increase(const double&) method.\n"
00677 << "Minimum performance improvement must be equal or greater than 0.\n";
00678
00679 throw std::logic_error(buffer.str().c_str());
00680 }
00681
00682 #endif
00683
00684
00685
00686 minimum_performance_increase = new_minimum_performance_increase;
00687 }
00688
00689
00690
00691
00695
00696 void GradientDescent::set_performance_goal(const double& new_performance_goal)
00697 {
00698 performance_goal = new_performance_goal;
00699 }
00700
00701
00702
00703
00707
00708 void GradientDescent::set_gradient_norm_goal(const double& new_gradient_norm_goal)
00709 {
00710
00711
00712 #ifdef _DEBUG
00713
00714 if(new_gradient_norm_goal < 0.0)
00715 {
00716 std::ostringstream buffer;
00717
00718 buffer << "OpenNN Exception: GradientDescent class.\n"
00719 << "void set_gradient_norm_goal(const double&) method.\n"
00720 << "Gradient norm goal must be equal or greater than 0.\n";
00721
00722 throw std::logic_error(buffer.str().c_str());
00723 }
00724
00725 #endif
00726
00727
00728
00729 gradient_norm_goal = new_gradient_norm_goal;
00730 }
00731
00732
00733
00734
00737
00738 void GradientDescent::set_maximum_generalization_evaluation_decreases(const unsigned int& new_maximum_generalization_evaluation_decreases)
00739 {
00740
00741
00742 #ifdef _DEBUG
00743
00744 if(new_maximum_generalization_evaluation_decreases < 0)
00745 {
00746 std::ostringstream buffer;
00747
00748 buffer << "OpenNN Exception: TrainingAlgorithm class.\n"
00749 << "void set_maximum_generalization_evaluation_decreases(const unsigned int&) method.\n"
00750 << "Number of generalization performance decreases must be equal or greater than 0.\n";
00751
00752 throw std::logic_error(buffer.str().c_str());
00753 }
00754
00755 #endif
00756
00757
00758
00759 maximum_generalization_evaluation_decreases = new_maximum_generalization_evaluation_decreases;
00760 }
00761
00762
00763
00764
00767
00768 void GradientDescent::set_maximum_epochs_number(const unsigned int& new_maximum_epochs_number)
00769 {
00770
00771
00772 #ifdef _DEBUG
00773
00774 if(new_maximum_epochs_number < 0)
00775 {
00776 std::ostringstream buffer;
00777
00778 buffer << "OpenNN Exception: TrainingAlgorithm class.\n"
00779 << "void set_maximum_epochs_number(unsigned int) method.\n"
00780 << "Number of epochs must be equal or greater than 0.\n";
00781
00782 throw std::logic_error(buffer.str().c_str());
00783 }
00784
00785 #endif
00786
00787
00788
00789 maximum_epochs_number = new_maximum_epochs_number;
00790 }
00791
00792
00793
00794
00797
00798 void GradientDescent::set_maximum_time(const double& new_maximum_time)
00799 {
00800
00801
00802 #ifdef _DEBUG
00803
00804 if(new_maximum_time < 0.0)
00805 {
00806 std::ostringstream buffer;
00807
00808 buffer << "OpenNN Exception: TrainingAlgorithm class.\n"
00809 << "void set_maximum_time(const double&) method.\n"
00810 << "Maximum time must be equal or greater than 0.\n";
00811
00812 throw std::logic_error(buffer.str().c_str());
00813 }
00814
00815 #endif
00816
00817
00818
00819 maximum_time = new_maximum_time;
00820 }
00821
00822
00823
00824
00827
00828 void GradientDescent::set_reserve_parameters_history(const bool& new_reserve_parameters_history)
00829 {
00830 reserve_parameters_history = new_reserve_parameters_history;
00831 }
00832
00833
00834
00835
00838
00839 void GradientDescent::set_reserve_parameters_norm_history(const bool& new_reserve_parameters_norm_history)
00840 {
00841 reserve_parameters_norm_history = new_reserve_parameters_norm_history;
00842 }
00843
00844
00845
00846
00849
00850 void GradientDescent::set_reserve_evaluation_history(const bool& new_reserve_evaluation_history)
00851 {
00852 reserve_evaluation_history = new_reserve_evaluation_history;
00853 }
00854
00855
00856
00857
00860
00861 void GradientDescent::set_reserve_gradient_history(const bool& new_reserve_gradient_history)
00862 {
00863 reserve_gradient_history = new_reserve_gradient_history;
00864 }
00865
00866
00867
00868
00872
00873 void GradientDescent::set_reserve_gradient_norm_history(const bool& new_reserve_gradient_norm_history)
00874 {
00875 reserve_gradient_norm_history = new_reserve_gradient_norm_history;
00876 }
00877
00878
00879
00880
00884
00885 void GradientDescent::set_reserve_training_direction_history(const bool& new_reserve_training_direction_history)
00886 {
00887 reserve_training_direction_history = new_reserve_training_direction_history;
00888 }
00889
00890
00891
00892
00896
00897 void GradientDescent::set_reserve_training_rate_history(const bool& new_reserve_training_rate_history)
00898 {
00899 reserve_training_rate_history = new_reserve_training_rate_history;
00900 }
00901
00902
00903
00904
00908
00909 void GradientDescent::set_reserve_elapsed_time_history(const bool& new_reserve_elapsed_time_history)
00910 {
00911 reserve_elapsed_time_history = new_reserve_elapsed_time_history;
00912 }
00913
00914
00915
00916
00920
00921 void GradientDescent::set_reserve_generalization_evaluation_history(const bool& new_reserve_generalization_evaluation_history)
00922 {
00923 reserve_generalization_evaluation_history = new_reserve_generalization_evaluation_history;
00924 }
00925
00926
00927
00928
00932
00933 void GradientDescent::set_display_period(const unsigned int& new_display_period)
00934 {
00935
00936
00937 #ifdef _DEBUG
00938
00939 if(new_display_period <= 0)
00940 {
00941 std::ostringstream buffer;
00942
00943 buffer << "OpenNN Exception: TrainingAlgorithm class.\n"
00944 << "void set_display_period(const double&) method.\n"
00945 << "First training rate must be greater than 0.\n";
00946
00947 throw std::logic_error(buffer.str().c_str());
00948 }
00949
00950 #endif
00951
00952 display_period = new_display_period;
00953 }
00954
00955
00956
00957
00958
00960
00961
00962 Vector<double> GradientDescent::calculate_training_direction(const Vector<double>& gradient) const
00963 {
00964 const double gradient_norm = gradient.calculate_norm();
00965
00966 return(gradient*(-1.0)/gradient_norm);
00967 }
00968
00969
00970
00971
00972 std::string GradientDescent::GradientDescentResults::to_string(void) const
00973 {
00974 std::ostringstream buffer;
00975
00976
00977
00978 if(!parameters_history.empty())
00979 {
00980 if(!parameters_history[0].empty())
00981 {
00982 buffer << "% Parameters history:\n"
00983 << parameters_history << "\n";
00984 }
00985 }
00986
00987
00988
00989 if(!parameters_norm_history.empty())
00990 {
00991 buffer << "% Parameters norm history:\n"
00992 << parameters_norm_history << "\n";
00993 }
00994
00995
00996
00997 if(!evaluation_history.empty())
00998 {
00999 buffer << "% Performance history:\n"
01000 << evaluation_history << "\n";
01001 }
01002
01003
01004
01005 if(!generalization_evaluation_history.empty())
01006 {
01007 buffer << "% Generalization evaluation history:\n"
01008 << generalization_evaluation_history << "\n";
01009 }
01010
01011
01012
01013 if(!gradient_history.empty())
01014 {
01015 if(!gradient_history[0].empty())
01016 {
01017 buffer << "% Gradient history:\n"
01018 << gradient_history << "\n";
01019 }
01020 }
01021
01022
01023
01024 if(!gradient_norm_history.empty())
01025 {
01026 buffer << "% Gradient norm history:\n"
01027 << gradient_norm_history << "\n";
01028 }
01029
01030
01031
01032 if(!training_direction_history.empty())
01033 {
01034 if(!training_direction_history[0].empty())
01035 {
01036 buffer << "% Training direction history:\n"
01037 << training_direction_history << "\n";
01038 }
01039 }
01040
01041
01042
01043 if(!training_rate_history.empty())
01044 {
01045 buffer << "% Training rate history:\n"
01046 << training_rate_history << "\n";
01047 }
01048
01049
01050
01051 if(!elapsed_time_history.empty())
01052 {
01053 buffer << "% Elapsed time history:\n"
01054 << elapsed_time_history << "\n";
01055 }
01056
01057 return(buffer.str());
01058 }
01059
01060
01061
01062
01065
01066 void GradientDescent::GradientDescentResults::resize_training_history(const unsigned int& new_size)
01067 {
01068 parameters_history.resize(new_size);
01069 parameters_norm_history.resize(new_size);
01070
01071 evaluation_history.resize(new_size);
01072 generalization_evaluation_history.resize(new_size);
01073 gradient_history.resize(new_size);
01074 gradient_norm_history.resize(new_size);
01075
01076 training_direction_history.resize(new_size);
01077 training_rate_history.resize(new_size);
01078 elapsed_time_history.resize(new_size);
01079 }
01080
01081
01082
01083
01085
01086 GradientDescent::GradientDescentResults* GradientDescent::perform_training(void)
01087 {
01088 GradientDescentResults* training_results_pointer = new GradientDescentResults;
01089
01090
01091
01092 #ifdef _DEBUG
01093
01094 check();
01095
01096 #endif
01097
01098
01099
01100 if(display)
01101 {
01102 std::cout << "Training with gradient descent...\n";
01103 }
01104
01105
01106
01107 NeuralNetwork* neural_network_pointer = performance_functional_pointer->get_neural_network_pointer();
01108
01109 unsigned int parameters_number = neural_network_pointer->count_parameters_number();
01110
01111 Vector<double> parameters(parameters_number);
01112 double parameters_norm;
01113
01114 Vector<double> parameters_increment(parameters_number);
01115 double parameters_increment_norm;
01116
01117
01118
01119 double generalization_evaluation = 0.0;
01120 double old_generalization_evaluation = 0.0;
01121 double generalization_evaluation_increment = 0.0;
01122
01123 double performance = 0.0;
01124 double old_performance = 0.0;
01125 double performance_increase = 0.0;
01126
01127 Vector<double> gradient(parameters_number);
01128 double gradient_norm;
01129
01130 PerformanceFunctional::FirstOrderEvaluation first_order_evaluation;
01131
01132
01133
01134 unsigned int generalization_evaluation_decreases_count = 0;
01135
01136 Vector<double> training_direction(parameters_number);
01137
01138 double initial_training_rate = 0.0;
01139 double training_rate = 0.0;
01140 double old_training_rate = 0.0;
01141
01142 Vector<double> directional_point(2);
01143 directional_point[0] = 0.0;
01144 directional_point[1] = 0.0;
01145
01146 bool stop_training = false;
01147
01148 time_t beginning_time, current_time;
01149 time(&beginning_time);
01150 double elapsed_time;
01151
01152 training_results_pointer->resize_training_history(maximum_epochs_number+1);
01153
01154
01155
01156 for(unsigned int epoch = 0; epoch <= maximum_epochs_number; epoch++)
01157 {
01158
01159
01160 parameters = neural_network_pointer->arrange_parameters();
01161
01162 parameters_norm = parameters.calculate_norm();
01163
01164 if(display && parameters_norm >= warning_parameters_norm)
01165 {
01166 std::cout << "OpenNN Warning: Parameters norm is " << parameters_norm << ".\n";
01167 }
01168
01169
01170
01171 if(epoch == 0)
01172 {
01173 performance = performance_functional_pointer->calculate_evaluation();
01174 performance_increase = 0.0;
01175 }
01176 else
01177 {
01178 performance = directional_point[1];
01179 performance_increase = old_performance - performance;
01180 }
01181
01182 gradient = performance_functional_pointer->calculate_gradient();
01183
01184 gradient_norm = gradient.calculate_norm();
01185
01186 if(display && gradient_norm >= warning_gradient_norm)
01187 {
01188 std::cout << "OpenNN Warning: Gradient norm is " << gradient_norm << ".\n";
01189 }
01190
01191 generalization_evaluation = performance_functional_pointer->calculate_generalization_evaluation();
01192
01193 if(epoch != 0 && generalization_evaluation > old_generalization_evaluation)
01194 {
01195 generalization_evaluation_decreases_count++;
01196 }
01197
01198
01199
01200 training_direction = calculate_training_direction(gradient);
01201
01202 if(epoch == 0)
01203 {
01204 initial_training_rate = training_rate_algorithm.get_first_training_rate();
01205 }
01206 else
01207 {
01208 initial_training_rate = old_training_rate;
01209 }
01210
01211 directional_point = training_rate_algorithm.calculate_directional_point(performance, training_direction, initial_training_rate);
01212
01213 training_rate = directional_point[0];
01214
01215 parameters_increment = training_direction*training_rate;
01216 parameters_increment_norm = parameters_increment.calculate_norm();
01217
01218
01219
01220 time(¤t_time);
01221 elapsed_time = difftime(current_time, beginning_time);
01222
01223
01224
01225 if(reserve_parameters_history)
01226 {
01227 training_results_pointer->parameters_history[epoch] = parameters;
01228 }
01229
01230 if(reserve_parameters_norm_history)
01231 {
01232 training_results_pointer->parameters_norm_history[epoch] = parameters_norm;
01233 }
01234
01235
01236
01237 if(reserve_evaluation_history)
01238 {
01239 training_results_pointer->evaluation_history[epoch] = performance;
01240 }
01241
01242 if(reserve_gradient_history)
01243 {
01244 training_results_pointer->gradient_history[epoch] = gradient;
01245 }
01246
01247 if(reserve_gradient_norm_history)
01248 {
01249 training_results_pointer->gradient_norm_history[epoch] = gradient_norm;
01250 }
01251
01252 if(reserve_generalization_evaluation_history)
01253 {
01254 training_results_pointer->generalization_evaluation_history[epoch] = generalization_evaluation;
01255 }
01256
01257
01258
01259 if(reserve_training_direction_history)
01260 {
01261 training_results_pointer->training_direction_history[epoch] = training_direction;
01262 }
01263
01264 if(reserve_training_rate_history)
01265 {
01266 training_results_pointer->training_rate_history[epoch] = training_rate;
01267 }
01268
01269 if(reserve_elapsed_time_history)
01270 {
01271 training_results_pointer->elapsed_time_history[epoch] = elapsed_time;
01272 }
01273
01274
01275
01276 if(parameters_increment_norm <= minimum_parameters_increment_norm)
01277 {
01278 if(display)
01279 {
01280 std::cout << "Epoch " << epoch << ": Minimum parameters increment norm reached.\n";
01281 std::cout << "Parameters increment norm: " << parameters_increment_norm << std::endl;
01282 }
01283
01284 stop_training = true;
01285 }
01286
01287 else if(epoch != 0 && performance_increase <= minimum_performance_increase)
01288 {
01289 if(display)
01290 {
01291 std::cout << "Epoch " << epoch << ": Minimum performance increase reached.\n"
01292 << "Performance increase: " << performance_increase << std::endl;
01293 }
01294
01295 stop_training = true;
01296 }
01297
01298 else if(performance <= performance_goal)
01299 {
01300 if(display)
01301 {
01302 std::cout << "Epoch " << epoch << ": Performance goal reached.\n";
01303 }
01304
01305 stop_training = true;
01306 }
01307
01308 else if(epoch != 0 && generalization_evaluation_increment > 0.0)
01309 {
01310 if(display)
01311 {
01312 std::cout << "Epoch " << epoch << ": Validation performance stopped improving.\n";
01313 std::cout << "Validation performance increment: " << generalization_evaluation_increment << std::endl;
01314 }
01315
01316 stop_training = true;
01317 }
01318
01319 else if(gradient_norm <= gradient_norm_goal)
01320 {
01321 if(display)
01322 {
01323 std::cout << "Epoch " << epoch << ": Gradient norm goal reached.\n";
01324 }
01325
01326 stop_training = true;
01327 }
01328
01329 else if(epoch == maximum_epochs_number)
01330 {
01331 if(display)
01332 {
01333 std::cout << "Epoch " << epoch << ": Maximum number of epochs reached.\n";
01334 }
01335
01336 stop_training = true;
01337 }
01338
01339 else if(elapsed_time >= maximum_time)
01340 {
01341 if(display)
01342 {
01343 std::cout << "Epoch " << epoch << ": Maximum training time reached.\n";
01344 }
01345
01346 stop_training = true;
01347 }
01348
01349 if(stop_training)
01350 {
01351 if(display)
01352 {
01353 std::cout << "Parameters norm: " << parameters_norm << "\n"
01354 << "Performance: " << performance << "\n"
01355 << "Gradient norm: " << gradient_norm << "\n"
01356 << performance_functional_pointer->write_information()
01357 << "Training rate: " << training_rate << "\n"
01358 << "Elapsed time: " << elapsed_time << std::endl;
01359
01360 if(generalization_evaluation != 0)
01361 {
01362 std::cout << "Validation performance: " << generalization_evaluation << std::endl;
01363 }
01364
01365 }
01366
01367 training_results_pointer->resize_training_history(1+epoch);
01368
01369 break;
01370 }
01371 else if(display && epoch % display_period == 0)
01372 {
01373 std::cout << "Epoch " << epoch << ";\n"
01374 << "Parameters norm: " << parameters_norm << "\n"
01375 << "Performance: " << performance << "\n"
01376 << "Gradient norm: " << gradient_norm << "\n"
01377 << performance_functional_pointer->write_information()
01378 << "Training rate: " << training_rate << "\n"
01379 << "Elapsed time: " << elapsed_time << std::endl;
01380
01381 if(generalization_evaluation != 0)
01382 {
01383 std::cout << "Validation performance: " << generalization_evaluation << std::endl;
01384 }
01385
01386 }
01387
01388
01389
01390 parameters += parameters_increment;
01391
01392 neural_network_pointer->set_parameters(parameters);
01393
01394
01395
01396 old_performance = performance;
01397 old_generalization_evaluation = generalization_evaluation;
01398
01399 old_training_rate = training_rate;
01400 }
01401
01402 return(training_results_pointer);
01403 }
01404
01405
01406
01407
01408 std::string GradientDescent::write_training_algorithm_type(void) const
01409 {
01410 return("GRADIENT_DESCENT");
01411 }
01412
01413
01414
01415
01418
01419 TiXmlElement* GradientDescent::to_XML(void) const
01420 {
01421 std::ostringstream buffer;
01422
01423
01424
01425 TiXmlElement* gradient_descent_element = new TiXmlElement("GradientDescent");
01426 gradient_descent_element->SetAttribute("Version", 4);
01427
01428
01429
01430
01431 TiXmlElement* warning_parameters_norm_element = new TiXmlElement("WarningParametersNorm");
01432 gradient_descent_element->LinkEndChild(warning_parameters_norm_element);
01433
01434 buffer.str("");
01435 buffer << warning_parameters_norm;
01436
01437 TiXmlText* warning_parameters_norm_text = new TiXmlText(buffer.str().c_str());
01438 warning_parameters_norm_element->LinkEndChild(warning_parameters_norm_text);
01439
01440
01441
01442 TiXmlElement* warning_gradient_norm_element = new TiXmlElement("WarningGradientNorm");
01443 gradient_descent_element->LinkEndChild(warning_gradient_norm_element);
01444
01445 buffer.str("");
01446 buffer << warning_gradient_norm;
01447
01448 TiXmlText* warning_gradient_norm_text = new TiXmlText(buffer.str().c_str());
01449 warning_gradient_norm_element->LinkEndChild(warning_gradient_norm_text);
01450
01451
01452
01453 TiXmlElement* warning_training_rate_element = new TiXmlElement("WarningTrainingRate");
01454 gradient_descent_element->LinkEndChild(warning_training_rate_element);
01455
01456 buffer.str("");
01457 buffer << warning_training_rate;
01458
01459 TiXmlText* warning_training_rate_text = new TiXmlText(buffer.str().c_str());
01460 warning_training_rate_element->LinkEndChild(warning_training_rate_text);
01461
01462
01463
01464 TiXmlElement* error_parameters_norm_element = new TiXmlElement("ErrorParametersNorm");
01465 gradient_descent_element->LinkEndChild(error_parameters_norm_element);
01466
01467 buffer.str("");
01468 buffer << error_parameters_norm;
01469
01470 TiXmlText* error_parameters_norm_text = new TiXmlText(buffer.str().c_str());
01471 error_parameters_norm_element->LinkEndChild(error_parameters_norm_text);
01472
01473
01474
01475 TiXmlElement* error_gradient_norm_element = new TiXmlElement("ErrorGradientNorm");
01476 gradient_descent_element->LinkEndChild(error_gradient_norm_element);
01477
01478 buffer.str("");
01479 buffer << error_gradient_norm;
01480
01481 TiXmlText* error_gradient_norm_text = new TiXmlText(buffer.str().c_str());
01482 error_gradient_norm_element->LinkEndChild(error_gradient_norm_text);
01483
01484
01485
01486 TiXmlElement* error_training_rate_element = new TiXmlElement("ErrorTrainingRate");
01487 gradient_descent_element->LinkEndChild(error_training_rate_element);
01488
01489 buffer.str("");
01490 buffer << error_training_rate;
01491
01492 TiXmlText* error_training_rate_text = new TiXmlText(buffer.str().c_str());
01493 error_training_rate_element->LinkEndChild(error_training_rate_text);
01494
01495
01496
01497 TiXmlElement* minimum_parameters_increment_norm_element = new TiXmlElement("MinimumParametersIncrement");
01498 gradient_descent_element->LinkEndChild(minimum_parameters_increment_norm_element);
01499
01500 buffer.str("");
01501 buffer << minimum_parameters_increment_norm;
01502
01503 TiXmlText* minimum_parameters_increment_norm_text = new TiXmlText(buffer.str().c_str());
01504 minimum_parameters_increment_norm_element->LinkEndChild(minimum_parameters_increment_norm_text);
01505
01506
01507
01508 TiXmlElement* minimum_performance_increase_element = new TiXmlElement("MinimumPerformanceIncrease");
01509 gradient_descent_element->LinkEndChild(minimum_performance_increase_element);
01510
01511 buffer.str("");
01512 buffer << minimum_performance_increase;
01513
01514 TiXmlText* minimum_performance_increase_text = new TiXmlText(buffer.str().c_str());
01515 minimum_performance_increase_element->LinkEndChild(minimum_performance_increase_text);
01516
01517
01518
01519 TiXmlElement* performance_goal_element = new TiXmlElement("PerformanceGoal");
01520 gradient_descent_element->LinkEndChild(performance_goal_element);
01521
01522 buffer.str("");
01523 buffer << performance_goal;
01524
01525 TiXmlText* performance_goal_text = new TiXmlText(buffer.str().c_str());
01526 performance_goal_element->LinkEndChild(performance_goal_text);
01527
01528
01529
01530 TiXmlElement* gradient_norm_goal_element = new TiXmlElement("GradientNormGoal");
01531 gradient_descent_element->LinkEndChild(gradient_norm_goal_element);
01532
01533 buffer.str("");
01534 buffer << gradient_norm_goal;
01535
01536 TiXmlText* gradient_norm_goal_text = new TiXmlText(buffer.str().c_str());
01537 gradient_norm_goal_element->LinkEndChild(gradient_norm_goal_text);
01538
01539
01540
01541 TiXmlElement* maximum_generalization_evaluation_decreases_element = new TiXmlElement("MaximumGeneralizationEvaluationDecreases");
01542 gradient_descent_element->LinkEndChild(maximum_generalization_evaluation_decreases_element);
01543
01544 buffer.str("");
01545 buffer << maximum_generalization_evaluation_decreases;
01546
01547 TiXmlText* maximum_generalization_evaluation_decreases_text = new TiXmlText(buffer.str().c_str());
01548 maximum_generalization_evaluation_decreases_element->LinkEndChild(maximum_generalization_evaluation_decreases_text);
01549
01550
01551
01552 TiXmlElement* maximum_epochs_number_element = new TiXmlElement("MaximumEpochsNumber");
01553 gradient_descent_element->LinkEndChild(maximum_epochs_number_element);
01554
01555 buffer.str("");
01556 buffer << maximum_epochs_number;
01557
01558 TiXmlText* maximum_epochs_number_text = new TiXmlText(buffer.str().c_str());
01559 maximum_epochs_number_element->LinkEndChild(maximum_epochs_number_text);
01560
01561
01562
01563 TiXmlElement* maximum_time_element = new TiXmlElement("MaximumTime");
01564 gradient_descent_element->LinkEndChild(maximum_time_element);
01565
01566 buffer.str("");
01567 buffer << maximum_time;
01568
01569 TiXmlText* maximum_time_text = new TiXmlText(buffer.str().c_str());
01570 maximum_time_element->LinkEndChild(maximum_time_text);
01571
01572
01573
01574 TiXmlElement* reserve_parameters_history_element = new TiXmlElement("ReserveParametersHistory");
01575 gradient_descent_element->LinkEndChild(reserve_parameters_history_element);
01576
01577 buffer.str("");
01578 buffer << reserve_parameters_history;
01579
01580 TiXmlText* reserve_parameters_history_text = new TiXmlText(buffer.str().c_str());
01581 reserve_parameters_history_element->LinkEndChild(reserve_parameters_history_text);
01582
01583
01584
01585 TiXmlElement* reserve_parameters_norm_history_element = new TiXmlElement("ReserveParametersNormHistory");
01586 gradient_descent_element->LinkEndChild(reserve_parameters_norm_history_element);
01587
01588 buffer.str("");
01589 buffer << reserve_parameters_norm_history;
01590
01591 TiXmlText* reserve_parameters_norm_history_text = new TiXmlText(buffer.str().c_str());
01592 reserve_parameters_norm_history_element->LinkEndChild(reserve_parameters_norm_history_text);
01593
01594
01595
01596 TiXmlElement* reserve_evaluation_history_element = new TiXmlElement("ReservePerformanceHistory");
01597 gradient_descent_element->LinkEndChild(reserve_evaluation_history_element);
01598
01599 buffer.str("");
01600 buffer << reserve_evaluation_history;
01601
01602 TiXmlText* reserve_evaluation_history_text = new TiXmlText(buffer.str().c_str());
01603 reserve_evaluation_history_element->LinkEndChild(reserve_evaluation_history_text);
01604
01605
01606
01607 TiXmlElement* reserve_gradient_history_element = new TiXmlElement("ReserveGradientHistory");
01608 gradient_descent_element->LinkEndChild(reserve_gradient_history_element);
01609
01610 buffer.str("");
01611 buffer << reserve_gradient_history;
01612
01613 TiXmlText* reserve_gradient_history_text = new TiXmlText(buffer.str().c_str());
01614 reserve_gradient_history_element->LinkEndChild(reserve_gradient_history_text);
01615
01616
01617
01618 TiXmlElement* reserve_gradient_norm_history_element = new TiXmlElement("ReserveGradientNormHistory");
01619 gradient_descent_element->LinkEndChild(reserve_gradient_norm_history_element);
01620
01621 buffer.str("");
01622 buffer << reserve_gradient_norm_history;
01623
01624 TiXmlText* reserve_gradient_norm_history_text = new TiXmlText(buffer.str().c_str());
01625 reserve_gradient_norm_history_element->LinkEndChild(reserve_gradient_norm_history_text);
01626
01627
01628
01629 TiXmlElement* reserve_training_direction_history_element = new TiXmlElement("ReserveTrainingDirectionHistory");
01630 gradient_descent_element->LinkEndChild(reserve_training_direction_history_element);
01631
01632 buffer.str("");
01633 buffer << reserve_training_direction_history;
01634
01635 TiXmlText* reserve_training_direction_history_text = new TiXmlText(buffer.str().c_str());
01636 reserve_training_direction_history_element->LinkEndChild(reserve_training_direction_history_text);
01637
01638
01639
01640 TiXmlElement* reserve_training_rate_history_element = new TiXmlElement("ReserveTrainingRateHistory");
01641 gradient_descent_element->LinkEndChild(reserve_training_rate_history_element);
01642
01643 buffer.str("");
01644 buffer << reserve_training_rate_history;
01645
01646 TiXmlText* reserve_training_rate_history_text = new TiXmlText(buffer.str().c_str());
01647 reserve_training_rate_history_element->LinkEndChild(reserve_training_rate_history_text);
01648
01649
01650
01651 TiXmlElement* reserve_elapsed_time_history_element = new TiXmlElement("ReserveElapsedTimeHistory");
01652 gradient_descent_element->LinkEndChild(reserve_elapsed_time_history_element);
01653
01654 buffer.str("");
01655 buffer << reserve_elapsed_time_history;
01656
01657 TiXmlText* reserve_elapsed_time_history_text = new TiXmlText(buffer.str().c_str());
01658 reserve_elapsed_time_history_element->LinkEndChild(reserve_elapsed_time_history_text);
01659
01660
01661
01662 TiXmlElement* reserve_generalization_evaluation_history_element = new TiXmlElement("ReserveGeneralizationPerformanceHistory");
01663 gradient_descent_element->LinkEndChild(reserve_generalization_evaluation_history_element);
01664
01665 buffer.str("");
01666 buffer << reserve_generalization_evaluation_history;
01667
01668 TiXmlText* reserve_generalization_evaluation_history_text = new TiXmlText(buffer.str().c_str());
01669 reserve_generalization_evaluation_history_element->LinkEndChild(reserve_generalization_evaluation_history_text);
01670
01671
01672
01673 TiXmlElement* display_period_element = new TiXmlElement("DisplayPeriod");
01674 gradient_descent_element->LinkEndChild(display_period_element);
01675
01676 buffer.str("");
01677 buffer << display_period;
01678
01679 TiXmlText* display_period_text = new TiXmlText(buffer.str().c_str());
01680 display_period_element->LinkEndChild(display_period_text);
01681
01682
01683
01684 TiXmlElement* display_element = new TiXmlElement("Display");
01685 gradient_descent_element->LinkEndChild(display_element);
01686
01687 buffer.str("");
01688 buffer << display;
01689
01690 TiXmlText* display_text = new TiXmlText(buffer.str().c_str());
01691 display_element->LinkEndChild(display_text);
01692
01693 return(gradient_descent_element);
01694 }
01695
01696
01697
01698
01699 void GradientDescent::from_XML(TiXmlElement* gradient_descent_element)
01700 {
01701
01702
01703 TiXmlElement* training_rate_algorithm_element = gradient_descent_element->FirstChildElement("TrainingRateAlgorithm");
01704
01705 if(training_rate_algorithm_element)
01706 {
01707 try
01708 {
01709 training_rate_algorithm.from_XML(training_rate_algorithm_element);
01710 }
01711 catch(std::exception& e)
01712 {
01713 std::cout << e.what() << std::endl;
01714 }
01715 }
01716
01717
01718
01719 TiXmlElement* warning_parameters_norm_element = gradient_descent_element->FirstChildElement("WarningParametersNorm");
01720
01721 if(warning_parameters_norm_element)
01722 {
01723 double new_warning_parameters_norm = atof(warning_parameters_norm_element->GetText());
01724
01725 try
01726 {
01727 set_warning_parameters_norm(new_warning_parameters_norm);
01728 }
01729 catch(std::exception& e)
01730 {
01731 std::cout << e.what() << std::endl;
01732 }
01733 }
01734
01735
01736
01737 TiXmlElement* warning_gradient_norm_element = gradient_descent_element->FirstChildElement("WarningGradientNorm");
01738
01739 if(warning_gradient_norm_element)
01740 {
01741 double new_warning_gradient_norm = atof(warning_gradient_norm_element->GetText());
01742
01743 try
01744 {
01745 set_warning_gradient_norm(new_warning_gradient_norm);
01746 }
01747 catch(std::exception& e)
01748 {
01749 std::cout << e.what() << std::endl;
01750 }
01751 }
01752
01753
01754
01755 TiXmlElement* warning_training_rate_element = gradient_descent_element->FirstChildElement("WarningTrainingRate");
01756
01757 if(warning_training_rate_element)
01758 {
01759 double new_warning_training_rate = atof(warning_training_rate_element->GetText());
01760
01761 try
01762 {
01763 set_warning_training_rate(new_warning_training_rate);
01764 }
01765 catch(std::exception& e)
01766 {
01767 std::cout << e.what() << std::endl;
01768 }
01769 }
01770
01771
01772
01773 TiXmlElement* error_parameters_norm_element = gradient_descent_element->FirstChildElement("ErrorParametersNorm");
01774
01775 if(error_parameters_norm_element)
01776 {
01777 double new_error_parameters_norm = atof(error_parameters_norm_element->GetText());
01778
01779 try
01780 {
01781 set_error_parameters_norm(new_error_parameters_norm);
01782 }
01783 catch(std::exception& e)
01784 {
01785 std::cout << e.what() << std::endl;
01786 }
01787 }
01788
01789
01790
01791 TiXmlElement* error_gradient_norm_element = gradient_descent_element->FirstChildElement("ErrorGradientNorm");
01792
01793 if(error_gradient_norm_element)
01794 {
01795 double new_error_gradient_norm = atof(error_gradient_norm_element->GetText());
01796
01797 try
01798 {
01799 set_error_gradient_norm(new_error_gradient_norm);
01800 }
01801 catch(std::exception& e)
01802 {
01803 std::cout << e.what() << std::endl;
01804 }
01805 }
01806
01807
01808
01809 TiXmlElement* error_training_rate_element = gradient_descent_element->FirstChildElement("ErrorTrainingRate");
01810
01811 if(error_training_rate_element)
01812 {
01813 double new_error_training_rate = atof(error_training_rate_element->GetText());
01814
01815 try
01816 {
01817 set_error_training_rate(new_error_training_rate);
01818 }
01819 catch(std::exception& e)
01820 {
01821 std::cout << e.what() << std::endl;
01822 }
01823 }
01824
01825
01826
01827 TiXmlElement* minimum_parameters_increment_norm_element = gradient_descent_element->FirstChildElement("MinimumParametersIncrementNorm");
01828
01829 if(minimum_parameters_increment_norm_element)
01830 {
01831 double new_minimum_parameters_increment_norm = atof(minimum_parameters_increment_norm_element->GetText());
01832
01833 try
01834 {
01835 set_minimum_parameters_increment_norm(new_minimum_parameters_increment_norm);
01836 }
01837 catch(std::exception& e)
01838 {
01839 std::cout << e.what() << std::endl;
01840 }
01841 }
01842
01843
01844
01845 TiXmlElement* minimum_performance_increase_element = gradient_descent_element->FirstChildElement("MinimumPerformanceIncrease");
01846
01847 if(minimum_performance_increase_element)
01848 {
01849 double new_minimum_performance_increase = atof(minimum_performance_increase_element->GetText());
01850
01851 try
01852 {
01853 set_minimum_performance_increase(new_minimum_performance_increase);
01854 }
01855 catch(std::exception& e)
01856 {
01857 std::cout << e.what() << std::endl;
01858 }
01859 }
01860
01861
01862
01863 TiXmlElement* performance_goal_element = gradient_descent_element->FirstChildElement("PerformanceGoal");
01864
01865 if(performance_goal_element)
01866 {
01867 double new_performance_goal = atof(performance_goal_element->GetText());
01868
01869 try
01870 {
01871 set_performance_goal(new_performance_goal);
01872 }
01873 catch(std::exception& e)
01874 {
01875 std::cout << e.what() << std::endl;
01876 }
01877 }
01878
01879
01880
01881 TiXmlElement* gradient_norm_goal_element = gradient_descent_element->FirstChildElement("GradientNormGoal");
01882
01883 if(gradient_norm_goal_element)
01884 {
01885 double new_gradient_norm_goal = atof(gradient_norm_goal_element->GetText());
01886
01887 try
01888 {
01889 set_gradient_norm_goal(new_gradient_norm_goal);
01890 }
01891 catch(std::exception& e)
01892 {
01893 std::cout << e.what() << std::endl;
01894 }
01895 }
01896
01897
01898
01899 TiXmlElement* maximum_generalization_evaluation_decreases_element = gradient_descent_element->FirstChildElement("MaximumGeneralizationEvaluationDecreases");
01900
01901 if(maximum_generalization_evaluation_decreases_element)
01902 {
01903 unsigned int new_maximum_generalization_evaluation_decreases = atoi(maximum_generalization_evaluation_decreases_element->GetText());
01904
01905 try
01906 {
01907 set_maximum_generalization_evaluation_decreases(new_maximum_generalization_evaluation_decreases);
01908 }
01909 catch(std::exception& e)
01910 {
01911 std::cout << e.what() << std::endl;
01912 }
01913 }
01914
01915
01916
01917 TiXmlElement* maximum_epochs_number_element = gradient_descent_element->FirstChildElement("MaximumEpochsNumber");
01918
01919 if(maximum_epochs_number_element)
01920 {
01921 unsigned int new_maximum_epochs_number = atoi(maximum_epochs_number_element->GetText());
01922
01923 try
01924 {
01925 set_maximum_epochs_number(new_maximum_epochs_number);
01926 }
01927 catch(std::exception& e)
01928 {
01929 std::cout << e.what() << std::endl;
01930 }
01931 }
01932
01933
01934
01935 TiXmlElement* maximum_time_element = gradient_descent_element->FirstChildElement("MaximumTime");
01936
01937 if(maximum_time_element)
01938 {
01939 double new_maximum_time = atof(maximum_time_element->GetText());
01940
01941 try
01942 {
01943 set_maximum_time(new_maximum_time);
01944 }
01945 catch(std::exception& e)
01946 {
01947 std::cout << e.what() << std::endl;
01948 }
01949 }
01950
01951
01952
01953 TiXmlElement* reserve_parameters_history_element = gradient_descent_element->FirstChildElement("ReserveParametersHistory");
01954
01955 if(reserve_parameters_history_element)
01956 {
01957 std::string new_reserve_parameters_history = reserve_parameters_history_element->GetText();
01958
01959 try
01960 {
01961 set_reserve_parameters_history(new_reserve_parameters_history != "0");
01962 }
01963 catch(std::exception& e)
01964 {
01965 std::cout << e.what() << std::endl;
01966 }
01967 }
01968
01969
01970
01971 TiXmlElement* reserve_parameters_norm_history_element = gradient_descent_element->FirstChildElement("ReserveParametersNormHistory");
01972
01973 if(reserve_parameters_norm_history_element)
01974 {
01975 std::string new_reserve_parameters_norm_history = reserve_parameters_norm_history_element->GetText();
01976
01977 try
01978 {
01979 set_reserve_parameters_norm_history(new_reserve_parameters_norm_history != "0");
01980 }
01981 catch(std::exception& e)
01982 {
01983 std::cout << e.what() << std::endl;
01984 }
01985 }
01986
01987
01988
01989 TiXmlElement* reserve_evaluation_history_element = gradient_descent_element->FirstChildElement("ReservePerformanceHistory");
01990
01991 if(reserve_evaluation_history_element)
01992 {
01993 std::string new_reserve_evaluation_history = reserve_evaluation_history_element->GetText();
01994
01995 try
01996 {
01997 set_reserve_evaluation_history(new_reserve_evaluation_history != "0");
01998 }
01999 catch(std::exception& e)
02000 {
02001 std::cout << e.what() << std::endl;
02002 }
02003 }
02004
02005
02006
02007 TiXmlElement* reserve_gradient_history_element = gradient_descent_element->FirstChildElement("ReserveGradientHistory");
02008
02009 if(reserve_gradient_history_element)
02010 {
02011 std::string new_reserve_gradient_history = reserve_gradient_history_element->GetText();
02012
02013 try
02014 {
02015 set_reserve_gradient_history(new_reserve_gradient_history != "0");
02016 }
02017 catch(std::exception& e)
02018 {
02019 std::cout << e.what() << std::endl;
02020 }
02021 }
02022
02023
02024
02025 TiXmlElement* reserve_gradient_norm_history_element = gradient_descent_element->FirstChildElement("ReserveGradientNormHistory");
02026
02027 if(reserve_gradient_norm_history_element)
02028 {
02029 std::string new_reserve_gradient_norm_history = reserve_gradient_norm_history_element->GetText();
02030
02031 try
02032 {
02033 set_reserve_gradient_norm_history(new_reserve_gradient_norm_history != "0");
02034 }
02035 catch(std::exception& e)
02036 {
02037 std::cout << e.what() << std::endl;
02038 }
02039 }
02040
02041
02042
02043 TiXmlElement* reserve_training_direction_history_element = gradient_descent_element->FirstChildElement("ReserveTrainingDirectionHistory");
02044
02045 if(reserve_training_direction_history_element)
02046 {
02047 std::string new_reserve_training_direction_history = reserve_training_direction_history_element->GetText();
02048
02049 try
02050 {
02051 set_reserve_training_direction_history(new_reserve_training_direction_history != "0");
02052 }
02053 catch(std::exception& e)
02054 {
02055 std::cout << e.what() << std::endl;
02056 }
02057 }
02058
02059
02060
02061 TiXmlElement* reserve_training_rate_history_element = gradient_descent_element->FirstChildElement("ReserveTrainingRateHistory");
02062
02063 if(reserve_training_rate_history_element)
02064 {
02065 std::string new_reserve_training_rate_history = reserve_training_rate_history_element->GetText();
02066
02067 try
02068 {
02069 set_reserve_training_rate_history(new_reserve_training_rate_history != "0");
02070 }
02071 catch(std::exception& e)
02072 {
02073 std::cout << e.what() << std::endl;
02074 }
02075 }
02076
02077
02078
02079 TiXmlElement* reserve_elapsed_time_history_element = gradient_descent_element->FirstChildElement("ReserveElapsedTimeHistory");
02080
02081 if(reserve_elapsed_time_history_element)
02082 {
02083 std::string new_reserve_elapsed_time_history = reserve_elapsed_time_history_element->GetText();
02084
02085 try
02086 {
02087 set_reserve_elapsed_time_history(new_reserve_elapsed_time_history != "0");
02088 }
02089 catch(std::exception& e)
02090 {
02091 std::cout << e.what() << std::endl;
02092 }
02093 }
02094
02095
02096
02097 TiXmlElement* reserve_generalization_evaluation_history_element = gradient_descent_element->FirstChildElement("ReserveGeneralizationPerformanceHistory");
02098
02099 if(reserve_generalization_evaluation_history_element)
02100 {
02101 std::string new_reserve_generalization_evaluation_history = reserve_generalization_evaluation_history_element->GetText();
02102
02103 try
02104 {
02105 set_reserve_generalization_evaluation_history(new_reserve_generalization_evaluation_history != "0");
02106 }
02107 catch(std::exception& e)
02108 {
02109 std::cout << e.what() << std::endl;
02110 }
02111 }
02112
02113
02114
02115 TiXmlElement* display_period_element = gradient_descent_element->FirstChildElement("DisplayPeriod");
02116
02117 if(display_period_element)
02118 {
02119 unsigned int new_display_period = atoi(display_period_element->GetText());
02120
02121 try
02122 {
02123 set_display_period(new_display_period);
02124 }
02125 catch(std::exception& e)
02126 {
02127 std::cout << e.what() << std::endl;
02128 }
02129 }
02130
02131
02132
02133 TiXmlElement* display_element = gradient_descent_element->FirstChildElement("Display");
02134
02135 if(display_element)
02136 {
02137 std::string new_display = display_element->GetText();
02138
02139 try
02140 {
02141 set_display(new_display != "0");
02142 }
02143 catch(std::exception& e)
02144 {
02145 std::cout << e.what() << std::endl;
02146 }
02147 }
02148 }
02149
02150 }
02151
02152
02153
02154
02155
02156
02157
02158
02159
02160
02161
02162
02163
02164
02165
02166
02167
02168
02169