00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018 #include <cmath>
00019 #include <cstdlib>
00020 #include <fstream>
00021 #include <iostream>
00022 #include <string>
00023 #include <sstream>
00024
00025
00026
00027 #include "multilayer_perceptron.h"
00028
00029
00030 namespace OpenNN
00031 {
00032
00033
00034
00038
00039 MultilayerPerceptron::MultilayerPerceptron(void)
00040 {
00041 set();
00042 }
00043
00044
00045
00046
00050
00051 MultilayerPerceptron::MultilayerPerceptron(const Vector<PerceptronLayer>& new_layers)
00052 {
00053 set(new_layers);
00054 }
00055
00056
00057
00058
00066
00067 MultilayerPerceptron::MultilayerPerceptron(const Vector<unsigned int>& new_architecture)
00068 {
00069 set(new_architecture);
00070 }
00071
00072
00073
00074
00084
00085 MultilayerPerceptron::MultilayerPerceptron(const unsigned int& new_inputs_number, const unsigned int& new_neurons_number)
00086 {
00087 set(new_inputs_number, new_neurons_number);
00088 }
00089
00090
00091
00092
00101
00102 MultilayerPerceptron::MultilayerPerceptron(const unsigned int& new_inputs_number, const unsigned int& new_hidden_neurons_number, const unsigned int& new_outputs_number)
00103 {
00104 set(new_inputs_number, new_hidden_neurons_number, new_outputs_number);
00105
00106 set_default();
00107 }
00108
00109
00110
00111
00115
00116 MultilayerPerceptron::MultilayerPerceptron(const MultilayerPerceptron& other_multilayer_perceptron)
00117 {
00118 set(other_multilayer_perceptron);
00119 }
00120
00121
00122
00123
00126
00127 MultilayerPerceptron::~MultilayerPerceptron(void)
00128 {
00129 }
00130
00131
00132
00133
00137
00138 MultilayerPerceptron& MultilayerPerceptron::operator = (const MultilayerPerceptron& other_multilayer_perceptron)
00139 {
00140 if(this != &other_multilayer_perceptron)
00141 {
00142 layers = other_multilayer_perceptron.layers;
00143
00144 display = other_multilayer_perceptron.display;
00145 }
00146
00147 return(*this);
00148 }
00149
00150
00151
00152
00153
00154
00159
00160 bool MultilayerPerceptron::operator == (const MultilayerPerceptron& other_multilayer_perceptron) const
00161 {
00162 if(layers == other_multilayer_perceptron.layers
00163 && display == other_multilayer_perceptron.display)
00164 {
00165 return(true);
00166 }
00167 else
00168 {
00169 return(false);
00170 }
00171 }
00172
00173
00174
00175
00176
00177
00181
00182 const Vector<PerceptronLayer>& MultilayerPerceptron::get_layers(void) const
00183 {
00184 return(layers);
00185 }
00186
00187
00188
00189
00192
00193 const PerceptronLayer& MultilayerPerceptron::get_layer(const unsigned int& i) const
00194 {
00195
00196
00197 #ifdef _DEBUG
00198
00199 const unsigned int layers_number = count_layers_number();
00200
00201 if(i >= layers_number)
00202 {
00203 std::ostringstream buffer;
00204
00205 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00206 << "const PerceptronLayer get_layer(const unsigned int&) const method.\n"
00207 << "Index of layer must be less than number of layers.\n";
00208
00209 throw std::logic_error(buffer.str());
00210 }
00211
00212 #endif
00213
00214 return(layers[i]);
00215 }
00216
00217
00218
00219
00222
00223 PerceptronLayer* MultilayerPerceptron::get_layer_pointer(const unsigned int& i)
00224 {
00225
00226
00227 #ifdef _DEBUG
00228
00229 const unsigned int layers_number = count_layers_number();
00230
00231 if(i >= layers_number)
00232 {
00233 std::ostringstream buffer;
00234
00235 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00236 << "PerceptronLayer* get_layer_pointer(const unsigned int&) const method.\n"
00237 << "Index of layer must be less than number of layers.\n";
00238
00239 throw std::logic_error(buffer.str());
00240 }
00241
00242 #endif
00243
00244 return(&layers[i]);
00245 }
00246
00247
00248
00249
00252
00253 unsigned int MultilayerPerceptron::count_perceptrons_number(void) const
00254 {
00255 const Vector<unsigned int> layers_perceptrons_number = arrange_layers_perceptrons_numbers();
00256
00257 return(layers_perceptrons_number.calculate_sum());
00258 }
00259
00260
00261
00262
00264
00265 Vector<unsigned int> MultilayerPerceptron::count_cumulative_perceptrons_number(void) const
00266 {
00267 const unsigned int layers_number = count_layers_number();
00268
00269 Vector<unsigned int> cumulative_neurons_number(layers_number);
00270
00271 if(layers_number != 0)
00272 {
00273 Vector<unsigned int> layers_size = arrange_layers_perceptrons_numbers();
00274
00275 cumulative_neurons_number[0] = layers_size[0];
00276
00277 for(unsigned int i = 1; i < layers_number; i++)
00278 {
00279 cumulative_neurons_number[i] = cumulative_neurons_number[i-1] + layers_size[i];
00280 }
00281 }
00282
00283 return(cumulative_neurons_number);
00284 }
00285
00286
00287
00288
00291
00292 Vector<unsigned int> MultilayerPerceptron::arrange_layers_parameters_number(void) const
00293 {
00294 const unsigned int layers_number = count_layers_number();
00295
00296 Vector<unsigned int> layers_parameters_number(layers_number);
00297
00298 for(unsigned int i = 0; i < layers_number; i++)
00299 {
00300 layers_parameters_number[i] = layers[i].count_parameters_number();
00301 }
00302
00303 return(layers_parameters_number);
00304 }
00305
00306
00307
00308
00311
00312 Vector<unsigned int> MultilayerPerceptron::arrange_layers_cumulative_parameters_number(void) const
00313 {
00314 const unsigned int layers_number = count_layers_number();
00315
00316 Vector<unsigned int> layers_cumulative_parameters_number(layers_number);
00317
00318 layers_cumulative_parameters_number[0] = layers[0].count_parameters_number();
00319
00320 for(unsigned int i = 1; i < layers_number; i++)
00321 {
00322 layers_cumulative_parameters_number[i] = layers_cumulative_parameters_number[i-1] + layers[i].count_parameters_number();
00323 }
00324
00325 return(layers_cumulative_parameters_number);
00326 }
00327
00328
00329
00330
00335
00336 Vector< Vector<double> > MultilayerPerceptron::arrange_layers_biases(void) const
00337 {
00338 const unsigned int layers_number = count_layers_number();
00339
00340 Vector< Vector<double> > layers_biases(layers_number);
00341
00342 for(unsigned int i = 0; i < layers_number; i++)
00343 {
00344 layers_biases[i] = layers[i].arrange_biases();
00345 }
00346
00347 return(layers_biases);
00348 }
00349
00350
00351
00352
00358
00359 Vector< Matrix<double> > MultilayerPerceptron::arrange_layers_synaptic_weights(void) const
00360 {
00361 const unsigned int layers_number = count_layers_number();
00362
00363 Vector< Matrix<double> > layers_synaptic_weights(layers_number);
00364
00365 for(unsigned int i = 0; i < layers_number; i++)
00366 {
00367 layers_synaptic_weights[i] = layers[i].arrange_synaptic_weights();
00368 }
00369
00370 return(layers_synaptic_weights);
00371 }
00372
00373
00374
00375
00382
00383 Vector< Vector<double> > MultilayerPerceptron::get_layers_parameters(void) const
00384 {
00385 const unsigned int layers_number = count_layers_number();
00386
00387 Vector< Vector<double> > layers_parameters(layers_number);
00388
00389 for(unsigned int i = 0; i < layers_number; i++)
00390 {
00391 layers_parameters[i] = layers[i].arrange_parameters();
00392 }
00393
00394 return(layers_parameters);
00395 }
00396
00397
00398
00399
00401
00402 unsigned int MultilayerPerceptron::count_parameters_number(void) const
00403 {
00404 const unsigned int layers_number = count_layers_number();
00405
00406 unsigned int parameters_number = 0;
00407
00408 for(unsigned int i = 0; i < layers_number; i++)
00409 {
00410 parameters_number += layers[i].count_parameters_number();
00411 }
00412
00413 return(parameters_number);
00414 }
00415
00416
00417
00418
00420
00421 Vector<double> MultilayerPerceptron::arrange_parameters(void) const
00422 {
00423 const unsigned int layers_number = count_layers_number();
00424
00425 const unsigned int parameters_number = count_parameters_number();
00426
00427 Vector<double> parameters(parameters_number);
00428
00429 unsigned int position = 0;
00430
00431 for(unsigned int i = 0; i < layers_number; i++)
00432 {
00433 const Vector<double> layer_parameters = layers[i].arrange_parameters();
00434 const unsigned int layer_parameters_number = layer_parameters.size();
00435
00436 parameters.tuck_in(position, layer_parameters);
00437 position += layer_parameters_number;
00438 }
00439
00440 return(parameters);
00441 }
00442
00443
00444
00445
00448
00449 unsigned int MultilayerPerceptron::get_layer_index(const unsigned int& neuron_index) const
00450 {
00451
00452
00453 #ifdef _DEBUG
00454
00455 const unsigned int neurons_number = count_perceptrons_number();
00456
00457 if(neuron_index >= neurons_number)
00458 {
00459 std::ostringstream buffer;
00460
00461 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00462 << "int get_layer_index(const unsigned int&) const method.\n"
00463 << "Index of neuron must be less than number of neurons.\n";
00464
00465 throw std::logic_error(buffer.str());
00466 }
00467
00468 #endif
00469
00470 const Vector<unsigned int> cumulative_neurons_number = count_cumulative_perceptrons_number();
00471
00472 const unsigned int layer_index = cumulative_neurons_number.calculate_cumulative_index(neuron_index);
00473
00474 return(layer_index);
00475 }
00476
00477
00478
00479
00483
00484 unsigned int MultilayerPerceptron::get_perceptron_index(const unsigned int& layer_index, const unsigned int& perceptron_position) const
00485 {
00486 if(layer_index == 0)
00487 {
00488 return(perceptron_position);
00489 }
00490 else
00491 {
00492 const Vector<unsigned int> cumulative_neurons_number = count_cumulative_perceptrons_number();
00493
00494 return(cumulative_neurons_number[layer_index-1] + perceptron_position);
00495 }
00496 }
00497
00498
00499
00500
00504
00505 unsigned int MultilayerPerceptron::get_layer_bias_index(const unsigned int& layer_index, const unsigned int& perceptron_index) const
00506 {
00507
00508
00509 #ifdef _DEBUG
00510
00511 const unsigned int layers_number = count_layers_number();
00512
00513 if(layer_index >= layers_number)
00514 {
00515 std::ostringstream buffer;
00516
00517 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00518 << "unsigned int get_layer_bias_index(const unsigned int&, const unsigned int&) const method.\n"
00519 << "Index of layer must be less than number of layers.\n";
00520
00521 throw std::logic_error(buffer.str());
00522 }
00523
00524 const unsigned int layer_perceptrons_number = layers[layer_index].count_perceptrons_number();
00525
00526 if(perceptron_index >= layer_perceptrons_number)
00527 {
00528 std::ostringstream buffer;
00529
00530 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00531 << "unsigned int get_layer_bias_index(const unsigned int&, const unsigned int&) const method.\n"
00532 << "Index of perceptron must be less than number of perceptrons in that layer.\n";
00533
00534 throw std::logic_error(buffer.str());
00535 }
00536
00537 #endif
00538
00539 unsigned int layer_bias_index = 0;
00540
00541
00542
00543 for(unsigned int i = 0; i < layer_index; i++)
00544 {
00545 layer_bias_index += layers[i].count_parameters_number();
00546 }
00547
00548
00549
00550 for(unsigned int j = 0; j < perceptron_index; j++)
00551 {
00552 layer_bias_index += layers[layer_index].get_perceptron(j).count_parameters_number();
00553 }
00554
00555 return(layer_bias_index);
00556 }
00557
00558
00559
00560
00565
00566 unsigned int MultilayerPerceptron::get_layer_synaptic_weight_index
00567 (const unsigned int& layer_index, const unsigned int& perceptron_index, const unsigned int& input_index) const
00568 {
00569
00570
00571 #ifdef _DEBUG
00572
00573 const unsigned int layers_number = count_layers_number();
00574
00575 if(layer_index >= layers_number)
00576 {
00577 std::ostringstream buffer;
00578
00579 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00580 << "unsigned int get_layer_synaptic_weight_index(const unsigned int&, const unsigned int&, const unsigned int&) method.\n"
00581 << "Index of layer must be less than number of layers.\n";
00582
00583 throw std::logic_error(buffer.str());
00584 }
00585
00586 const unsigned int layer_perceptrons_number = layers[layer_index].count_perceptrons_number();
00587
00588 if(perceptron_index >= layer_perceptrons_number)
00589 {
00590 std::ostringstream buffer;
00591
00592 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00593 << "unsigned int get_layer_synaptic_weight_index(const unsigned int&, const unsigned int&, const unsigned int&) method.\n"
00594 << "Index of perceptron must be less than number of perceptrons in layer.\n";
00595
00596 throw std::logic_error(buffer.str());
00597 }
00598
00599 const unsigned int layer_inputs_number = layers[layer_index].count_inputs_number();
00600
00601 if(input_index >= layer_inputs_number)
00602 {
00603 std::ostringstream buffer;
00604
00605 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00606 << "unsigned int get_layer_synaptic_weight_index(const unsigned int&, const unsigned int&, const unsigned int&) method.\n"
00607 << "Index of inputs must be less than number of inputs in perceptron.\n";
00608
00609 throw std::logic_error(buffer.str());
00610 }
00611
00612 #endif
00613
00614 unsigned int layer_synaptic_weight_index = 0;
00615
00616
00617
00618 if(layer_index > 0)
00619 {
00620 for(unsigned int i = 0; i < layer_index-1; i++)
00621 {
00622 layer_synaptic_weight_index += layers[layer_index].count_parameters_number();
00623 }
00624 }
00625
00626
00627
00628 if(perceptron_index > 0)
00629 {
00630 for(unsigned int i = 0; i < perceptron_index-1; i++)
00631 {
00632 layer_synaptic_weight_index += layers[layer_index].get_perceptron(i).count_parameters_number();
00633 }
00634 }
00635
00636
00637
00638 layer_synaptic_weight_index += 1;
00639
00640
00641
00642 layer_synaptic_weight_index += input_index;
00643
00644 return(layer_synaptic_weight_index);
00645 }
00646
00647
00648
00649
00652
00653 Vector<unsigned int> MultilayerPerceptron::arrange_parameter_indices(const unsigned int& parameter_index) const
00654 {
00655
00656
00657 #ifdef _DEBUG
00658
00659 const unsigned int parameters_number = count_parameters_number();
00660
00661 if(parameter_index >= parameters_number)
00662 {
00663 std::ostringstream buffer;
00664
00665 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00666 << "Vector<int> arrange_parameter_indices(const unsigned int&) const method.\n"
00667 << "Index of neural parameter must be less than number of multilayer_perceptron_pointer parameters.\n";
00668
00669 throw std::logic_error(buffer.str());
00670 }
00671
00672 #endif
00673
00674 return(arrange_parameters_indices().arrange_row(parameter_index));
00675 }
00676
00677
00678
00679
00684
00685 Matrix<unsigned int> MultilayerPerceptron::arrange_parameters_indices(void) const
00686 {
00687 unsigned int perceptron_parameters_number;
00688
00689 const unsigned int layers_number = count_layers_number();
00690 const Vector<unsigned int> layers_size = arrange_layers_perceptrons_numbers();
00691
00692 unsigned int parameters_number = count_parameters_number();
00693
00694 Matrix<unsigned int> parameters_indices(parameters_number, 3);
00695
00696 unsigned int parameter_index = 0;
00697
00698 for(unsigned int i = 0; i < layers_number; i++)
00699 {
00700 for(unsigned int j = 0; j < layers_size[i]; j++)
00701 {
00702 perceptron_parameters_number = layers[i].get_perceptron(j).count_parameters_number();
00703
00704 for(unsigned int k = 0; k < perceptron_parameters_number; k++)
00705 {
00706 parameters_indices[parameter_index][0] = i;
00707 parameters_indices[parameter_index][1] = j;
00708 parameters_indices[parameter_index][2] = k;
00709 parameter_index++;
00710 }
00711 }
00712 }
00713
00714 return(parameters_indices);
00715 }
00716
00717
00718
00719
00721
00722 Vector<Perceptron::ActivationFunction> MultilayerPerceptron::get_layers_activation_function(void) const
00723 {
00724 const unsigned int layers_number = count_layers_number();
00725
00726 Vector<Perceptron::ActivationFunction> layers_activation_function(layers_number);
00727
00728 for(unsigned int i = 0; i < layers_number; i++)
00729 {
00730 layers_activation_function[i] = layers[i].get_activation_function();
00731 }
00732
00733 return(layers_activation_function);
00734 }
00735
00736
00737
00738
00741
00742 Vector<std::string> MultilayerPerceptron::write_layers_activation_function(void) const
00743 {
00744 const unsigned int layers_number = count_layers_number();
00745
00746 Vector<std::string> layers_activation_function_name(layers_number);
00747
00748 for(unsigned int i = 0; i < layers_number; i++)
00749 {
00750 layers_activation_function_name[i] = layers[i].write_activation_function_name();
00751 }
00752
00753 return(layers_activation_function_name);
00754 }
00755
00756
00757
00758
00761
00762 const bool& MultilayerPerceptron::get_display(void) const
00763 {
00764 return(display);
00765 }
00766
00767
00768
00769
00776
00777 void MultilayerPerceptron::set_default(void)
00778 {
00779
00780
00781 const unsigned int layers_number = count_layers_number();
00782
00783 if(layers_number > 0)
00784 {
00785 for(unsigned int i = 0; i < layers_number-1; i++)
00786 {
00787 layers[i].set_activation_function(Perceptron::HyperbolicTangent);
00788 }
00789
00790 layers[layers_number-1].set_activation_function(Perceptron::Linear);
00791 }
00792
00793
00794
00795 set_display(true);
00796 }
00797
00798
00799
00800
00802
00803 void MultilayerPerceptron::set(void)
00804 {
00805 layers.set();
00806 }
00807
00808
00809
00810
00813
00814 void MultilayerPerceptron::set(const Vector<PerceptronLayer>& new_layers)
00815 {
00816 layers = new_layers;
00817 }
00818
00819
00820
00821
00829
00830 void MultilayerPerceptron::set(const Vector<unsigned int>& new_architecture)
00831 {
00832 std::ostringstream buffer;
00833
00834 const unsigned int new_architecture_size = new_architecture.size();
00835
00836 if(new_architecture_size == 0)
00837 {
00838 set();
00839 }
00840 else if(new_architecture_size == 1)
00841 {
00842 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00843 << "void set_architecture(const Vector<unsigned int>&) method.\n"
00844 << "Size of architecture cannot be one.\n";
00845
00846 throw std::logic_error(buffer.str());
00847 }
00848 else
00849 {
00850 for(unsigned int i = 0; i < new_architecture_size; i++)
00851 {
00852 if(new_architecture[i] == 0)
00853 {
00854 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00855 << "void set_architecture(const Vector<unsigned int>&) method.\n"
00856 << "Size " << i << " must be greater than zero.\n";
00857
00858 throw std::logic_error(buffer.str());
00859 }
00860 }
00861
00862 const unsigned int new_layers_number = new_architecture_size-1;
00863 layers.set(new_layers_number);
00864
00865
00866
00867 for(unsigned int i = 0; i < new_layers_number; i++)
00868 {
00869 layers[i].set_perceptrons_number(new_architecture[i+1]);
00870 layers[i].set_inputs_number(new_architecture[i]);
00871 }
00872
00873
00874
00875 for(unsigned int i = 0; i < new_layers_number-1; i++)
00876 {
00877 layers[i].set_activation_function(Perceptron::HyperbolicTangent);
00878 }
00879
00880 layers[new_layers_number-1].set_activation_function(Perceptron::Linear);
00881 }
00882 }
00883
00884
00885
00886
00890
00891 void MultilayerPerceptron::set(const unsigned int& new_inputs_number, const unsigned int& new_perceptrons_number)
00892 {
00893
00894
00895 #ifdef _DEBUG
00896
00897 std::ostringstream buffer;
00898
00899 if(new_inputs_number == 0)
00900 {
00901 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00902 << "void set(const unsigned int&, const unsigned int&) method.\n"
00903 << "Number of inputs cannot be zero.\n";
00904
00905 throw std::logic_error(buffer.str());
00906 }
00907 else if(new_perceptrons_number == 0)
00908 {
00909 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00910 << "void set_architecture(const unsigned int&, const unsigned int&) method.\n"
00911 << "Number of perceptrons cannot be zero.\n";
00912
00913 throw std::logic_error(buffer.str());
00914 }
00915
00916 #endif
00917
00918 layers.set(1);
00919
00920 layers[0].set(new_inputs_number, new_perceptrons_number);
00921 }
00922
00923
00924
00925
00930
00931 void MultilayerPerceptron::set(const unsigned int& new_inputs_number, const unsigned int& new_hidden_neurons_number, const unsigned int& new_outputs_number)
00932 {
00933
00934
00935 #ifdef _DEBUG
00936
00937 std::ostringstream buffer;
00938
00939 if(new_inputs_number == 0)
00940 {
00941 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00942 << "void set(const unsigned int&, const unsigned int&, const unsigned int&) method.\n"
00943 << "Number of inputs must be greater than zero.\n";
00944
00945 throw std::logic_error(buffer.str());
00946 }
00947 else if(new_hidden_neurons_number == 0)
00948 {
00949 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00950 << "void set(const unsigned int&, const unsigned int&, const unsigned int&) method.\n"
00951 << "Number of hidden neurons must be greater than zero.\n";
00952
00953 throw std::logic_error(buffer.str());
00954 }
00955 else if(new_outputs_number == 0)
00956 {
00957 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
00958 << "void set(const unsigned int&, const unsigned int&, const unsigned int&) method.\n"
00959 << "Number of outputs must be greater than zero.\n";
00960
00961 throw std::logic_error(buffer.str());
00962 }
00963
00964 #endif
00965
00966 layers.set(2);
00967
00968 layers[0].set(new_inputs_number, new_hidden_neurons_number);
00969 layers[0].set_activation_function(Perceptron::HyperbolicTangent);
00970
00971 layers[1].set(new_hidden_neurons_number, new_outputs_number);
00972 layers[1].set_activation_function(Perceptron::Linear);
00973 }
00974
00975
00976
00977
00980
00981 void MultilayerPerceptron::set(const MultilayerPerceptron& other_multilayer_perceptron)
00982 {
00983 layers = other_multilayer_perceptron.layers;
00984
00985 display = other_multilayer_perceptron.display;
00986 }
00987
00988
00989
00990
00993
00994 void MultilayerPerceptron::set_inputs_number(const unsigned int new_inputs_number)
00995 {
00996 if(is_empty())
00997 {
00998 std::ostringstream buffer;
00999
01000 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01001 << "void set_inputs_number(const unsigned int&) method.\n"
01002 << "Multilayer perceptron is empty.\n";
01003
01004 throw std::logic_error(buffer.str());
01005 }
01006
01007 layers[0].set_inputs_number(new_inputs_number);
01008 }
01009
01010
01011
01012
01017
01018 void MultilayerPerceptron::set_layers_perceptrons_number(const Vector<unsigned int>& new_layers_size)
01019 {
01020 const Vector<unsigned int> inputs_number(1, count_inputs_number());
01021
01022 set(inputs_number.get_assembly(new_layers_size));
01023 }
01024
01025
01026
01027
01032
01033 void MultilayerPerceptron::set_layer_perceptrons_number(const unsigned int& layer_index, const unsigned int& new_layer_perceptrons_number)
01034 {
01035 const unsigned int layer_inputs_number = layers[layer_index].count_inputs_number();
01036
01037 layers[layer_index].set_perceptrons_number(new_layer_perceptrons_number);
01038
01039 layers[layer_index].set_inputs_number(layer_inputs_number);
01040
01041 const unsigned int layers_number = count_layers_number();
01042
01043 if(layer_index < layers_number-1)
01044 {
01045 layers[layer_index+1].set_inputs_number(new_layer_perceptrons_number);
01046 }
01047 }
01048
01049
01050
01051
01054
01055 void MultilayerPerceptron::set_layers(const Vector<PerceptronLayer>& new_layers)
01056 {
01057 layers = new_layers;
01058 }
01059
01060
01061
01062
01068
01069 void MultilayerPerceptron::set_layers_biases(const Vector< Vector<double> >& new_layers_biases)
01070 {
01071 const unsigned int layers_number = count_layers_number();
01072
01073
01074
01075 #ifdef _DEBUG
01076
01077 const unsigned int size = new_layers_biases.size();
01078
01079 if(size != layers_number)
01080 {
01081 std::ostringstream buffer;
01082
01083 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01084 << "void set_layers_biases(const Vector< Vector<double> >&) method.\n"
01085 << "Size (" << size << ") must be equal to number of layers (" << layers_number << ").\n";
01086
01087 throw std::logic_error(buffer.str());
01088 }
01089
01090 #endif
01091
01092 const Vector<unsigned int> layers_size = arrange_layers_perceptrons_numbers();
01093
01094 for(unsigned int i = 0; i < layers_number; i++)
01095 {
01096 layers[i].set_biases(new_layers_biases[i]);
01097 }
01098 }
01099
01100
01101
01102
01109
01110 void MultilayerPerceptron::set_layers_synaptic_weights(const Vector< Matrix<double> >& new_layers_synaptic_weights)
01111 {
01112 const unsigned int layers_number = count_layers_number();
01113
01114
01115
01116 #ifdef _DEBUG
01117
01118 const unsigned int size = new_layers_synaptic_weights.size();
01119
01120 if(size != layers_number)
01121 {
01122 std::ostringstream buffer;
01123
01124 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01125 << "void set_layers_synaptic_weights(const Vector< Matrix<double> >&) method.\n"
01126 << "Size must be equal to number of layers.\n";
01127
01128 throw std::logic_error(buffer.str());
01129 }
01130
01131 #endif
01132
01133 for(unsigned int i = 0; i < layers_number; i++)
01134 {
01135 layers[i].set_synaptic_weights(new_layers_synaptic_weights[i]);
01136 }
01137
01138 }
01139
01140
01141
01142
01148
01149 void MultilayerPerceptron::set_layers_parameters(const Vector< Vector<double> >& new_layers_parameters)
01150 {
01151 const unsigned int layers_number = count_layers_number();
01152
01153
01154
01155 #ifdef _DEBUG
01156
01157 const unsigned int new_layers_parameters_size = new_layers_parameters.size();
01158
01159 if(new_layers_parameters_size != layers_number)
01160 {
01161 std::ostringstream buffer;
01162
01163 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01164 << "void set_layers_parameters(const Vector< Vector<double> >&) method.\n"
01165 << "Size of layer parameters must be equal to number of layers.\n";
01166
01167 throw std::logic_error(buffer.str());
01168 }
01169
01170 #endif
01171
01172 for(unsigned int i = 0; i < layers_number; i++)
01173 {
01174 layers[i].set_parameters(new_layers_parameters[i]);
01175 }
01176 }
01177
01178
01179
01180
01183
01184 void MultilayerPerceptron::set_parameters(const Vector<double>& new_parameters)
01185 {
01186
01187
01188 #ifdef _DEBUG
01189
01190 const unsigned int size = new_parameters.size();
01191
01192 const unsigned int parameters_number = count_parameters_number();
01193
01194 if(size != parameters_number)
01195 {
01196 std::ostringstream buffer;
01197
01198 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01199 << "void set_parameters(const Vector<double>&) method.\n"
01200 << "Size must be equal to number of biases and synaptic weights.\n";
01201
01202 throw std::logic_error(buffer.str());
01203 }
01204
01205 #endif
01206
01207 const unsigned int layers_number = count_layers_number();
01208
01209 unsigned int layer_parameters_number;
01210 Vector<double> layer_parameters;
01211
01212 unsigned int position = 0;
01213
01214 for(unsigned int i = 0; i < layers_number; i++)
01215 {
01216 layer_parameters_number = layers[i].count_parameters_number();
01217 layer_parameters = layers[i].arrange_parameters();
01218
01219 layer_parameters = new_parameters.take_out(position, layer_parameters_number);
01220 layers[i].set_parameters(layer_parameters);
01221 position += layer_parameters_number;
01222 }
01223 }
01224
01225
01226
01227
01232
01233 void MultilayerPerceptron::set_layers_activation_function
01234 (const Vector<Perceptron::ActivationFunction>& new_layers_activation_function)
01235 {
01236 const unsigned int layers_number = count_layers_number();
01237
01238
01239
01240 #ifdef _DEBUG
01241
01242 const unsigned int new_layers_activation_function_size = new_layers_activation_function.size();
01243
01244 if(new_layers_activation_function_size != layers_number)
01245 {
01246 std::ostringstream buffer;
01247
01248 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01249 << "void set_layers_activation_function(const Vector<Perceptron::ActivationFunction>&) method.\n"
01250 << "Size of activation function of layers must be equal to number of layers.\n";
01251
01252 throw std::logic_error(buffer.str());
01253 }
01254
01255 #endif
01256
01257 for(unsigned int i = 0; i < layers_number; i++)
01258 {
01259 layers[i].set_activation_function(new_layers_activation_function[i]);
01260 }
01261 }
01262
01263
01264
01265
01269
01270 void MultilayerPerceptron::set_layers_activation_function(const Vector<std::string>& new_layers_activation_function)
01271 {
01272 const unsigned int layers_number = count_layers_number();
01273
01274 for(unsigned int i = 0; i < layers_number; i++)
01275 {
01276 layers[i].set_activation_function(new_layers_activation_function[i]);
01277 }
01278 }
01279
01280
01281
01282
01287
01288 void MultilayerPerceptron::set_display(const bool& new_display)
01289 {
01290 display = new_display;
01291 }
01292
01293
01294
01295
01297
01298 bool MultilayerPerceptron::is_empty(void) const
01299 {
01300 if(layers.empty())
01301 {
01302 return(true);
01303 }
01304 else
01305 {
01306 return(false);
01307 }
01308 }
01309
01310
01311
01312
01314
01315 void MultilayerPerceptron::grow_input(const unsigned int&)
01316 {
01317 }
01318
01319
01320
01321
01323
01324 void MultilayerPerceptron::grow_layer(const unsigned int&, const unsigned int&)
01325 {
01326 }
01327
01328
01329
01330
01332
01333 void MultilayerPerceptron::prune_input(const unsigned int&)
01334 {
01335 }
01336
01337
01338
01339
01341
01342 void MultilayerPerceptron::prune_layer(const unsigned int&, const unsigned int&)
01343 {
01344 }
01345
01346
01347
01348
01352
01353 void MultilayerPerceptron::initialize_random(void)
01354 {
01355 const unsigned int architecture_size = rand()%10 + 2;
01356
01357 Vector<unsigned int> architecture(architecture_size);
01358
01359 for(unsigned int i = 0; i < architecture_size; i++)
01360 {
01361 architecture[i] = rand()%10 + 1;
01362 }
01363
01364 set(architecture);
01365
01366 const unsigned int layers_number = count_layers_number();
01367
01368
01369
01370 for(unsigned int i = 0; i < layers_number; i++)
01371 {
01372 switch(rand()%5)
01373 {
01374 case 0:
01375 {
01376 layers[i].set_activation_function(Perceptron::Logistic);
01377 }
01378 break;
01379
01380 case 1:
01381 {
01382 layers[i].set_activation_function(Perceptron::HyperbolicTangent);
01383 }
01384 break;
01385
01386 case 2:
01387 {
01388 layers[i].set_activation_function(Perceptron::Threshold);
01389 }
01390 break;
01391
01392 case 3:
01393 {
01394 layers[i].set_activation_function(Perceptron::SymmetricThreshold);
01395 }
01396 break;
01397
01398 case 4:
01399 {
01400 layers[i].set_activation_function(Perceptron::Linear);
01401 }
01402 break;
01403
01404 default:
01405 {
01406 std::ostringstream buffer;
01407
01408 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01409 << "void initialize_random(void) method.\n"
01410 << "Unknown layer activation function.\n";
01411
01412 throw std::logic_error(buffer.str());
01413 }
01414 break;
01415 }
01416 }
01417
01418
01419
01420 set_display(true);
01421 }
01422
01423
01424
01425
01428
01429 void MultilayerPerceptron::initialize_biases(const double& value)
01430 {
01431 const unsigned int layers_number = count_layers_number();
01432
01433 for(unsigned int i = 0; i < layers_number; i++)
01434 {
01435 layers[i].initialize_biases(value);
01436 }
01437 }
01438
01439
01440
01441
01444
01445 void MultilayerPerceptron::initialize_synaptic_weights(const double& value)
01446 {
01447 const unsigned int layers_number = count_layers_number();
01448 const Vector<unsigned int> layers_size = arrange_layers_perceptrons_numbers();
01449
01450 for(unsigned int i = 0; i < layers_number; i++)
01451 {
01452 layers[i].initialize_synaptic_weights(value);
01453 }
01454 }
01455
01456
01457
01458
01461
01462 void MultilayerPerceptron::initialize_parameters(const double& value)
01463 {
01464 const unsigned int parameters_number = count_parameters_number();
01465
01466 const Vector<double> parameters(parameters_number, value);
01467
01468 set_parameters(parameters);
01469 }
01470
01471
01472
01473
01476
01477 void MultilayerPerceptron::initialize_parameters_uniform(void)
01478 {
01479 const unsigned int parameters_number = count_parameters_number();
01480
01481 Vector<double> parameters(parameters_number);
01482
01483 parameters.initialize_uniform();
01484
01485 set_parameters(parameters);
01486 }
01487
01488
01489
01490
01495
01496 void MultilayerPerceptron::initialize_parameters_uniform(const double& minimum, const double& maximum)
01497 {
01498 const unsigned int parameters_number = count_parameters_number();
01499
01500 Vector<double> parameters(parameters_number);
01501
01502 parameters.initialize_uniform(minimum, maximum);
01503
01504 set_parameters(parameters);
01505 }
01506
01507
01508
01509
01514
01515 void MultilayerPerceptron::initialize_parameters_uniform(const Vector<double>& minimum, const Vector<double>& maximum)
01516 {
01517 const unsigned int parameters_number = count_parameters_number();
01518
01519 Vector<double> parameters(parameters_number);
01520
01521 parameters.initialize_uniform(minimum, maximum);
01522
01523 set_parameters(parameters);
01524 }
01525
01526
01527
01528
01535
01536 void MultilayerPerceptron::initialize_parameters_uniform(const Vector< Vector<double> >& minimum_maximum)
01537 {
01538 const unsigned int parameters_number = count_parameters_number();
01539
01540 Vector<double> parameters(parameters_number);
01541
01542 parameters.initialize_uniform(minimum_maximum[0], minimum_maximum[1]);
01543
01544 set_parameters(parameters);
01545 }
01546
01547
01548
01549
01552
01553 void MultilayerPerceptron::initialize_parameters_normal(void)
01554 {
01555 const unsigned int parameters_number = count_parameters_number();
01556
01557 Vector<double> parameters(parameters_number);
01558
01559 parameters.initialize_normal();
01560
01561 set_parameters(parameters);
01562 }
01563
01564
01565
01566
01571
01572 void MultilayerPerceptron::initialize_parameters_normal(const double& mean, const double& standard_deviation)
01573 {
01574 const unsigned int parameters_number = count_parameters_number();
01575
01576 Vector<double> parameters(parameters_number);
01577
01578 parameters.initialize_normal(mean, standard_deviation);
01579
01580 set_parameters(parameters);
01581 }
01582
01583
01584
01585
01590
01591 void MultilayerPerceptron::initialize_parameters_normal(const Vector<double>& mean, const Vector<double>& standard_deviation)
01592 {
01593 const unsigned int parameters_number = count_parameters_number();
01594
01595 Vector<double> parameters(parameters_number);
01596
01597 parameters.initialize_normal(mean, standard_deviation);
01598
01599 set_parameters(parameters);
01600 }
01601
01602
01603
01604
01611
01612 void MultilayerPerceptron::initialize_parameters_normal(const Vector< Vector<double> >& mean_standard_deviation)
01613 {
01614 const unsigned int parameters_number = count_parameters_number();
01615
01616 Vector<double> parameters(parameters_number);
01617
01618 parameters.initialize_normal(mean_standard_deviation[0], mean_standard_deviation[1]);
01619
01620 set_parameters(parameters);
01621 }
01622
01623
01624
01625
01627
01628 void MultilayerPerceptron::initialize_parameters(void)
01629 {
01630 initialize_parameters_normal();
01631 }
01632
01633
01634
01635
01637
01638 double MultilayerPerceptron::calculate_parameters_norm(void) const
01639 {
01640 const Vector<double> parameters = arrange_parameters();
01641
01642 const double parameters_norm = parameters.calculate_norm();
01643
01644 return(parameters_norm);
01645 }
01646
01647
01648
01649
01653
01654 Vector<double> MultilayerPerceptron::calculate_outputs(const Vector<double>& inputs) const
01655 {
01656
01657
01658 #ifdef _DEBUG
01659
01660 const unsigned int size = inputs.size();
01661
01662 const unsigned int inputs_number = count_inputs_number();
01663
01664 if(size != inputs_number)
01665 {
01666 std::ostringstream buffer;
01667
01668 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01669 << "Vector<double> calculate_outputs(const Vector<double>&) const method.\n"
01670 << "Size of inputs (" << size <<") must be equal to number of inputs (" << inputs_number << ").\n";
01671
01672 throw std::logic_error(buffer.str());
01673 }
01674
01675 #endif
01676
01677 const unsigned int layers_number = count_layers_number();
01678
01679 Vector<double> outputs;
01680
01681 if(layers_number == 0)
01682 {
01683 return(outputs);
01684 }
01685 else
01686 {
01687 outputs = layers[0].calculate_outputs(inputs);
01688
01689 for(unsigned int i = 1; i < layers_number; i++)
01690 {
01691 outputs = layers[i].calculate_outputs(outputs);
01692 }
01693 }
01694
01695 return(outputs);
01696 }
01697
01698
01699
01700
01704
01705 Matrix<double> MultilayerPerceptron::calculate_Jacobian(const Vector<double>& inputs) const
01706 {
01707 #ifdef _DEBUG
01708
01709 const unsigned int size = inputs.size();
01710
01711 const unsigned int inputs_number = count_inputs_number();
01712
01713 if(size != inputs_number)
01714 {
01715 std::ostringstream buffer;
01716
01717 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01718 << "Matrix<double> calculate_Jacobian(const Vector<double>&) const method.\n"
01719 << "Size must be equal to number of inputs.\n";
01720
01721 throw std::logic_error(buffer.str());
01722 }
01723
01724 #endif
01725
01726 const unsigned int layers_number = count_layers_number();
01727
01728 if(layers_number == 0)
01729 {
01730 Matrix<double> Jacobian;
01731
01732 return(Jacobian);
01733 }
01734 else
01735 {
01736 const Vector< Matrix<double> > layers_Jacobian = calculate_layers_Jacobian(inputs);
01737
01738 Matrix<double> Jacobian = layers_Jacobian[layers_number-1];
01739
01740 for(int i = layers_number-2; i > -1; i--)
01741 {
01742 Jacobian = Jacobian.dot(layers_Jacobian[i]);
01743 }
01744
01745 return(Jacobian);
01746 }
01747 }
01748
01749
01750
01751
01755
01756 Vector< Matrix<double> > MultilayerPerceptron::calculate_Hessian_form(const Vector<double>& inputs) const
01757 {
01758 #ifdef _DEBUG
01759
01760 const unsigned int size = inputs.size();
01761
01762 const unsigned int inputs_number = count_inputs_number();
01763
01764 if(size != inputs_number)
01765 {
01766 std::ostringstream buffer;
01767
01768 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01769 << "Vector< Matrix<double> > calculate_Hessian_form(const Vector<double>&) const method.\n"
01770 << "Size must be equal to number of inputs.\n";
01771
01772 throw std::logic_error(buffer.str());
01773 }
01774
01775 #endif
01776
01777 const unsigned int layers_number = count_layers_number();
01778 const Vector<unsigned int> layers_size = arrange_layers_perceptrons_numbers();
01779 const unsigned int outputs_number = count_outputs_number();
01780
01781
01782
01783 const Vector< Matrix<double> > layers_Jacobian = calculate_layers_Jacobian(inputs);
01784
01785 const Vector< Vector< Matrix<double> > > layers_Hessian_form = calculate_layers_Hessian_form(inputs);
01786
01787
01788
01789 Vector< Matrix<double> > Hessian_form = layers_Hessian_form[layers_number-1];
01790 Matrix<double> Jacobian = layers_Jacobian[layers_number-1];
01791
01792 for(unsigned int layer_index = layers_number-2; layer_index != 0; layer_index--)
01793 {
01794 for(unsigned int output_index = 0; output_index < outputs_number; output_index++)
01795 {
01796 Hessian_form[output_index] = (layers_Jacobian[layer_index].calculate_transpose()).dot(Hessian_form[output_index]).dot(layers_Jacobian[layer_index]);
01797
01798 for(unsigned int neuron_index = 0; neuron_index < layers_size[layer_index]; neuron_index++)
01799 {
01800 Hessian_form[output_index] += layers_Hessian_form[layer_index][neuron_index]*Jacobian[output_index][neuron_index];
01801 }
01802 }
01803
01804 Jacobian = Jacobian.dot(layers_Jacobian[layer_index]);
01805 }
01806
01807
01808
01809
01810
01811
01812
01813
01814
01815
01816
01817
01818
01819
01820
01821
01822
01823
01824
01825 return(Hessian_form);
01826 }
01827
01828
01829
01830
01834
01835 Vector<double> MultilayerPerceptron::calculate_layer_combination_combination(const unsigned int& layer_index, const Vector<double>& previous_layer_combination) const
01836 {
01837
01838
01839 #ifdef _DEBUG
01840
01841 const unsigned int layers_number = count_layers_number();
01842
01843 if(layer_index == 0)
01844 {
01845 std::ostringstream buffer;
01846
01847 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01848 << "Matrix<double> calculate_layer_combination_combination(const unsigned int&, const Vector<double>&) const.\n"
01849 << "Index of layer must be greater than zero.\n";
01850
01851 throw std::logic_error(buffer.str());
01852 }
01853 else if(layer_index >= layers_number)
01854 {
01855 std::ostringstream buffer;
01856
01857 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01858 << "Matrix<double> calculate_layer_combination_combination(const unsigned int&, const Vector<double>&) const.\n"
01859 << "Index of layer must be less than numnber of layers.\n";
01860
01861 throw std::logic_error(buffer.str());
01862 }
01863
01864 const unsigned int size = previous_layer_combination.size();
01865 const unsigned int previous_layer_perceptrons_number = layers[layer_index-1].count_perceptrons_number();
01866
01867 if(size != previous_layer_perceptrons_number)
01868 {
01869 std::ostringstream buffer;
01870
01871 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01872 << "Matrix<double> calculate_layer_combination_combination(const unsigned int&, const Vector<double>&) const.\n"
01873 << "Size must be equal to size of previous layer.\n";
01874
01875 throw std::logic_error(buffer.str());
01876 }
01877
01878 #endif
01879
01880
01881
01882 const Vector<double> previous_layer_activation = layers[layer_index-1].calculate_activation(previous_layer_combination);
01883
01884 const Vector<double> layer_combination_combination = layers[layer_index].calculate_combination(previous_layer_activation);
01885
01886 return(layer_combination_combination);
01887 }
01888
01889
01890
01891
01897
01898 Matrix<double> MultilayerPerceptron::calculate_layer_combination_combination_Jacobian(const unsigned int& layer_index, const Vector<double>& previous_layer_activation_derivative) const
01899 {
01900 const unsigned int previous_layer_perceptrons_number = layers[layer_index-1].count_perceptrons_number();
01901
01902
01903
01904 #ifdef _DEBUG
01905
01906 const unsigned int layers_number = count_layers_number();
01907
01908 if(layer_index == 0)
01909 {
01910 std::ostringstream buffer;
01911
01912 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01913 << "Matrix<double> calculate_layer_combination_combination_Jacobian(const unsigned int&, const Vector<double>&) const.\n"
01914 << "Index of layer must be greater than zero.\n";
01915
01916 throw std::logic_error(buffer.str());
01917 }
01918 else if(layer_index >= layers_number)
01919 {
01920 std::ostringstream buffer;
01921
01922 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01923 << "Matrix<double> calculate_layer_combination_combination_Jacobian(const unsigned int&, const Vector<double>&) const.\n"
01924 << "Index of layer must be less than numnber of layers.\n";
01925
01926 throw std::logic_error(buffer.str());
01927 }
01928
01929 const unsigned int size = previous_layer_activation_derivative.size();
01930
01931 if(size != previous_layer_perceptrons_number)
01932 {
01933 std::ostringstream buffer;
01934
01935 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01936 << "Matrix<double> calculate_layer_combination_combination_Jacobian(const unsigned int&, const Vector<double>&).\n"
01937 << "Size of activation derivative must be equal to size of previous layer.\n";
01938
01939 throw std::logic_error(buffer.str());
01940 }
01941
01942 #endif
01943
01944 const Matrix<double> layer_synaptic_weights = layers[layer_index].arrange_synaptic_weights();
01945
01946 Matrix<double> previous_layer_activation_Jacobian(previous_layer_perceptrons_number, previous_layer_perceptrons_number, 0.0);
01947 previous_layer_activation_Jacobian.set_diagonal(previous_layer_activation_derivative);
01948
01949 return(layer_synaptic_weights.dot(previous_layer_activation_Jacobian));
01950 }
01951
01952
01953
01954
01959
01960 Vector<double> MultilayerPerceptron::calculate_interlayer_combination_combination(const unsigned int& domain_layer_index, const unsigned int& image_layer_index, const Vector<double>& domain_layer_combination) const
01961 {
01962 if(domain_layer_index > image_layer_index)
01963 {
01964 std::ostringstream buffer;
01965
01966 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
01967 << "Vector<double> calculate_interlayer_combination_combination(const unsigned int&, const unsigned int&, const Vector<double>&) const method.\n"
01968 << "Index of domain layer must be less or equal than index of image layer.\n";
01969
01970 throw std::logic_error(buffer.str());
01971 }
01972
01973 if(domain_layer_index == image_layer_index)
01974 {
01975 return(domain_layer_combination);
01976 }
01977 else
01978 {
01979 Vector<double> interlayer_combination_combination(domain_layer_combination);
01980
01981 for(unsigned int i = domain_layer_index+1; i <= image_layer_index; i++)
01982 {
01983 interlayer_combination_combination = calculate_layer_combination_combination(i, interlayer_combination_combination);
01984 }
01985
01986 return(interlayer_combination_combination);
01987 }
01988 }
01989
01990
01991
01992
01998
01999 Matrix<double> MultilayerPerceptron::calculate_interlayer_combination_combination_Jacobian(const unsigned int& domain_layer_index, const unsigned int& image_layer_index, const Vector<double>& domain_layer_combination) const
02000 {
02001 Matrix<double> interlayer_combination_combination_Jacobian;
02002
02003 if(domain_layer_index < image_layer_index)
02004 {
02005 const unsigned int size = image_layer_index-domain_layer_index;
02006
02007 Vector< Vector<double> > layers_combination_combination(size);
02008 Vector< Matrix<double> > layers_combination_combination_Jacobian(size);
02009
02010 layers_combination_combination[0] = calculate_layer_combination_combination(domain_layer_index+1, domain_layer_combination);
02011 layers_combination_combination_Jacobian[0] = calculate_layer_combination_combination_Jacobian(domain_layer_index+1, layers[domain_layer_index].calculate_activation_derivative(domain_layer_combination));
02012
02013 for(unsigned int i = 1; i < size; i++)
02014 {
02015 layers_combination_combination[i] = calculate_layer_combination_combination(domain_layer_index+i+1, layers_combination_combination[i-1]);
02016
02017 layers_combination_combination_Jacobian[i] = calculate_layer_combination_combination_Jacobian(domain_layer_index+i+1, layers_combination_combination[i-1]);
02018 }
02019
02020 interlayer_combination_combination_Jacobian = layers_combination_combination_Jacobian[size-1];
02021
02022 for(int i = size-2; i > -1; i--)
02023 {
02024 interlayer_combination_combination_Jacobian = interlayer_combination_combination_Jacobian.dot(layers_combination_combination_Jacobian[i]);
02025 }
02026 }
02027 else if(domain_layer_index == image_layer_index)
02028 {
02029 const unsigned int image_layer_perceptrons_number = layers[image_layer_index].count_perceptrons_number();
02030
02031 interlayer_combination_combination_Jacobian.set_identity(image_layer_perceptrons_number);
02032 }
02033 else
02034 {
02035 const unsigned int image_layer_perceptrons_number = layers[image_layer_index].count_perceptrons_number();
02036 const unsigned int domain_layer_perceptrons_number = layers[domain_layer_index].count_perceptrons_number();
02037
02038 interlayer_combination_combination_Jacobian.set(image_layer_perceptrons_number, domain_layer_perceptrons_number, 0.0);
02039 }
02040
02041 return(interlayer_combination_combination_Jacobian);
02042 }
02043
02044
02045
02046
02050
02051 Vector<double> MultilayerPerceptron::calculate_output_layer_combination(const unsigned int& layer_index, const Vector<double>& layer_combinations) const
02052 {
02053
02054
02055 #ifdef _DEBUG
02056
02057 const unsigned int size = layer_combinations.size();
02058
02059 const unsigned int layer_perceptrons_number = layers[layer_index].count_perceptrons_number();
02060
02061 if(size != layer_perceptrons_number)
02062 {
02063 std::ostringstream buffer;
02064
02065 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02066 << "Vector<double> calculate_output_layer_combination(const unsigned int&, const Vector<double>&) const method.\n"
02067 << "Size must be equal to layer size.\n";
02068
02069 throw std::logic_error(buffer.str());
02070 }
02071
02072 #endif
02073
02074 const unsigned int layers_number = count_layers_number();
02075
02076 Vector<double> outputs = layers[layer_index].calculate_activation(layer_combinations);
02077
02078 if(layer_index < layers_number)
02079 {
02080 for(unsigned int i = layer_index+1; i < layers_number; i++)
02081 {
02082 outputs = layers[i].calculate_outputs(outputs);
02083 }
02084 }
02085
02086 return(outputs);
02087 }
02088
02089
02090
02091
02095
02096 Vector< Matrix<double> > MultilayerPerceptron::calculate_output_layers_delta(const Vector< Vector<double> >& layers_activation_derivative) const
02097 {
02098
02099
02100 const unsigned int layers_number = count_layers_number();
02101 const Vector<unsigned int> layers_size = arrange_layers_perceptrons_numbers();
02102
02103
02104
02105 #ifdef _DEBUG
02106
02107
02108
02109 const unsigned int layers_activation_derivative_size = layers_activation_derivative.size();
02110
02111 if(layers_activation_derivative_size != layers_number)
02112 {
02113 std::ostringstream buffer;
02114
02115 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02116 << "Vector< Vector<double> > calculate_output_layers_delta(const Vector< Vector<double> >&) method.\n"
02117 << "Size of forward propagation activation derivative vector must be equal to number of layers.\n";
02118
02119 throw std::logic_error(buffer.str().c_str());
02120 }
02121
02122 #endif
02123
02124 Vector< Matrix<double> > output_layers_delta(layers_number);
02125
02126 if(layers_number > 0)
02127 {
02128
02129
02130 output_layers_delta[layers_number-1] = layers[layers_number-1].arrange_activation_Jacobian(layers_activation_derivative[layers_number-1]);
02131
02132
02133
02134 for(int i = layers_number-2; i >= 0; i--)
02135 {
02136 output_layers_delta[i] = output_layers_delta[i+1].dot(layers[i+1].arrange_synaptic_weights()).dot(layers[i].arrange_activation_Jacobian(layers_activation_derivative[i]));
02137 }
02138 }
02139
02140 return(output_layers_delta);
02141 }
02142
02143
02144
02145
02150
02151 Matrix< Vector< Matrix<double> > > MultilayerPerceptron::calculate_output_interlayers_Delta
02152 (const Vector< Vector< Vector<double> > >& second_order_forward_propagation,
02153 const Matrix< Matrix<double> >& interlayers_combination_combination_Jacobian,
02154 const Vector< Matrix<double> >& output_layers_delta) const
02155 {
02156 const unsigned int layers_number = count_layers_number();
02157 const Vector<unsigned int> layers_size = arrange_layers_perceptrons_numbers();
02158 const unsigned int outputs_number = count_outputs_number();
02159
02160
02161
02162 #ifdef _DEBUG
02163
02164 std::ostringstream buffer;
02165
02166
02167
02168 const unsigned int second_order_forward_propagation_size = second_order_forward_propagation.size();
02169
02170 if(second_order_forward_propagation_size != 3)
02171 {
02172 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02173 << "Matrix< Vector< Matrix<double> > > calculate_output_interlayers_delta(@todo) method.\n"
02174 << "Size of second order forward propagation must be three.\n";
02175
02176 throw std::logic_error(buffer.str().c_str());
02177 }
02178
02179
02180
02181 const unsigned int interlayers_combination_combination_Jacobian_rows_number = interlayers_combination_combination_Jacobian.get_rows_number();
02182
02183 if(interlayers_combination_combination_Jacobian_rows_number != layers_number)
02184 {
02185 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02186 << "Matrix< Vector< Matrix<double> > > calculate_output_interlayers_delta(@todo) method.\n"
02187 << "Number of rows of interlayers combination-combination Jacobian must be equal to number of layers.\n";
02188
02189 throw std::logic_error(buffer.str().c_str());
02190 }
02191
02192
02193
02194 const unsigned int output_layers_delta_size = output_layers_delta.size();
02195
02196 if(output_layers_delta_size != layers_number)
02197 {
02198 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02199 << "Matrix< Vector< Matrix<double> > > calculate_output_interlayers_delta(@todo) method.\n"
02200 << "Size of multilayer_perceptron_pointer outputs layers delta must be equal to number of layers.\n";
02201
02202 throw std::logic_error(buffer.str().c_str());
02203 }
02204
02205 #endif
02206
02207 const Vector< Matrix<double> > layers_synaptic_weights = arrange_layers_synaptic_weights();
02208
02209 const Vector< Vector<double> >& layers_activation_derivative = second_order_forward_propagation[1];
02210 const Vector< Vector<double> >& layers_activation_second_derivative = second_order_forward_propagation[2];
02211
02212
02213
02214 Matrix< Vector< Matrix<double> > > output_interlayers_Delta(layers_number, layers_number);
02215
02216
02217
02218 for(unsigned int i = 0; i < layers_number; i++)
02219 {
02220 for(unsigned int j = 0; j < layers_number; j++)
02221 {
02222 output_interlayers_Delta[i][j].set(outputs_number);
02223 }
02224 }
02225
02226
02227
02228 for(unsigned int i = 0; i < layers_number; i++)
02229 {
02230 for(unsigned int j = 0; j < layers_number; j++)
02231 {
02232 for(unsigned int k = 0; k < outputs_number; k++)
02233 {
02234 output_interlayers_Delta[i][j][k].set(layers_size[i], layers_size[j]);
02235 }
02236 }
02237 }
02238
02239 if(layers_number > 0)
02240 {
02241
02242
02243 output_interlayers_Delta[layers_number-1][layers_number-1] = layers[layers_number-1].arrange_activation_Hessian_form(layers_activation_second_derivative[layers_number-1]);
02244
02245
02246
02247 double sum_1;
02248 double sum_2;
02249
02250 for(unsigned int L = layers_number-1; L != 0; L--)
02251 {
02252 for(unsigned int M = layers_number-1; M >= L; M--)
02253 {
02254 if(!(L == layers_number-1 && M == layers_number-1))
02255 {
02256 for(unsigned int i = 0; i < outputs_number; i++)
02257 {
02258 for(unsigned int j = 0; j < layers_size[L]; j++)
02259 {
02260 sum_1 = 0.0;
02261
02262 for(unsigned int l = 0; l < layers_size[L+1]; l++)
02263 {
02264 sum_1 += output_layers_delta[L+1][i][l]*layers_synaptic_weights[L+1][l][j];
02265 }
02266
02267 for(unsigned int k = 0; k < layers_size[M]; k++)
02268 {
02269 sum_2 = 0.0;
02270
02271 for(unsigned int l = 0; l < layers_size[L+1]; l++)
02272 {
02273 sum_2 += output_interlayers_Delta[L+1][M][i][l][k]*layers_synaptic_weights[L+1][l][j];
02274 }
02275
02276 output_interlayers_Delta[L][M][i][j][k]
02277 = layers_activation_second_derivative[L][j]*interlayers_combination_combination_Jacobian[L][M][j][k]*sum_1
02278 + layers_activation_derivative[L][j]*sum_2;
02279
02280 output_interlayers_Delta[M][L][i][k][j] = output_interlayers_Delta[L][M][i][j][k];
02281 }
02282 }
02283 }
02284 }
02285 }
02286 }
02287 }
02288
02289 return(output_interlayers_Delta);
02290 }
02291
02292
02293
02294
02297
02298 Matrix<double> MultilayerPerceptron::calculate_parameters_Jacobian(const Vector<double>& inputs, const Vector<double>&) const
02299 {
02300 #ifdef _DEBUG
02301
02302 const unsigned int inputs_number = count_inputs_number();
02303 const unsigned int size = inputs.size();
02304
02305 if(size != inputs_number)
02306 {
02307 std::ostringstream buffer;
02308
02309 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02310 << "void calculate_parameters_Jacobian(Vector<double>&) const method.\n"
02311 << "Size must be equal to number of inputs.\n";
02312
02313 throw std::logic_error(buffer.str());
02314 }
02315
02316 #endif
02317
02318
02319
02320 const unsigned int layers_number = count_layers_number();
02321 const Vector<unsigned int> layers_size = arrange_layers_perceptrons_numbers();
02322
02323
02324
02325 const Vector< Vector< Vector<double> > > first_order_forward_propagation = calculate_first_order_forward_propagation(inputs);
02326 const Vector< Vector<double> >& layers_activation = first_order_forward_propagation[0];
02327 const Vector< Vector<double> >& layers_activation_derivative = first_order_forward_propagation[1];
02328
02329 const Vector< Vector<double> > layers_inputs = arrange_layers_input(inputs, layers_activation);
02330
02331 const Vector< Matrix<double> > layers_combination_parameters_Jacobian = calculate_layers_combination_parameters_Jacobian(layers_inputs);
02332
02333 const Vector< Matrix<double> > output_layers_delta = calculate_output_layers_delta(layers_activation_derivative);
02334
02335 Matrix<double> parameters_Jacobian = output_layers_delta[0].dot(layers_combination_parameters_Jacobian[0]);
02336
02337 for(unsigned int i = 1; i < layers_number; i++)
02338 {
02339 parameters_Jacobian = parameters_Jacobian.get_assembly_columns(output_layers_delta[i].dot(layers_combination_parameters_Jacobian[i]));
02340 }
02341
02342 return(parameters_Jacobian);
02343 }
02344
02345
02346
02347
02350
02351 Vector< Matrix<double> > MultilayerPerceptron::calculate_parameters_Hessian_form(const Vector<double>& inputs, const Vector<double>&) const
02352 {
02353
02354
02355 const unsigned int layers_number = count_layers_number();
02356 const Vector<unsigned int> layers_size = arrange_layers_perceptrons_numbers();
02357 const unsigned int outputs_number = count_outputs_number();
02358
02359 const unsigned int parameters_number = count_parameters_number();
02360
02361 const Vector< Matrix<double> > layers_synaptic_weights = arrange_layers_synaptic_weights();
02362
02363
02364
02365 const Vector< Vector< Vector<double> > > second_order_forward_propagation = calculate_second_order_forward_propagation(inputs);
02366
02367 const Vector< Vector<double> >& layers_activation = second_order_forward_propagation[0];
02368 const Vector< Vector<double> >& layers_activation_derivative = second_order_forward_propagation[1];
02369
02370
02371 const Vector< Vector<double> > layers_inputs = arrange_layers_input(inputs, layers_activation);
02372
02373 const Vector< Vector< Vector<double> > > perceptrons_combination_parameters_gradient = calculate_perceptrons_combination_parameters_gradient(layers_inputs);
02374
02375 const Matrix< Matrix<double> > interlayers_combination_combination_Jacobian = calculate_interlayers_combination_combination_Jacobian(inputs);
02376
02377 const Vector< Matrix<double> > output_layers_delta = calculate_output_layers_delta(layers_activation_derivative);
02378
02379 const Matrix< Vector< Matrix<double> > > output_interlayers_Delta
02380 = calculate_output_interlayers_Delta(second_order_forward_propagation, interlayers_combination_combination_Jacobian, output_layers_delta);
02381
02382
02383
02384 Vector< Matrix<double> > parameters_Hessian_form(outputs_number);
02385
02386 for(unsigned int i = 0; i < outputs_number; i++)
02387 {
02388 parameters_Hessian_form[i].set(parameters_number, parameters_number);
02389 }
02390
02391
02392
02393 if(layers_number > 0)
02394 {
02395 Matrix<unsigned int> parameters_indices = arrange_parameters_indices();
02396
02397 unsigned int layer_j;
02398 unsigned int neuron_j;
02399 unsigned int parameter_j;
02400
02401 unsigned int layer_k;
02402 unsigned int neuron_k;
02403 unsigned int parameter_k;
02404
02405 Vector<double> neuron_j_combination_parameters_gradient;
02406 Vector<double> neuron_k_combination_parameters_gradient;
02407
02408 for(unsigned int i = 0; i < outputs_number; i++)
02409 {
02410 for(unsigned int j = 0; j < parameters_number; j++)
02411 {
02412 layer_j = parameters_indices[j][0];
02413 neuron_j = parameters_indices[j][1];
02414 parameter_j = parameters_indices[j][2];
02415
02416 for(unsigned int k = j; k < parameters_number; k++)
02417 {
02418 layer_k = parameters_indices[k][0];
02419 neuron_k = parameters_indices[k][1];
02420 parameter_k = parameters_indices[k][2];
02421
02422 parameters_Hessian_form[i][j][k] =
02423 output_interlayers_Delta[layer_j][layer_k][i][neuron_j][neuron_k]
02424 *perceptrons_combination_parameters_gradient[layer_j][neuron_j][parameter_j]
02425 *perceptrons_combination_parameters_gradient[layer_k][neuron_k][parameter_k]
02426 ;
02427
02428 if(layer_j != 0 && parameter_j != 0)
02429 {
02430
02431
02432
02433
02434
02435 ;
02436 }
02437
02438 if(layer_k != 0 && parameter_k != 0)
02439 {
02440 parameters_Hessian_form[i][j][k] +=
02441 output_layers_delta[layer_k][i][neuron_k]
02442 *layers_activation_derivative[layer_k-1][parameter_k-1]
02443 *interlayers_combination_combination_Jacobian[layer_k-1][layer_j][parameter_k-1][neuron_j]
02444 *perceptrons_combination_parameters_gradient[layer_j][neuron_j][parameter_j]
02445 ;
02446 }
02447
02448 parameters_Hessian_form[i][k][j] = parameters_Hessian_form[i][j][k];
02449 }
02450 }
02451 }
02452 }
02453
02454 return(parameters_Hessian_form);
02455 }
02456
02457
02458
02459
02466
02467 Vector< Vector<double> > MultilayerPerceptron::arrange_layers_input(const Vector<double>& inputs, const Vector< Vector<double> >& layers_activation) const
02468 {
02469
02470
02471 const unsigned int layers_number = count_layers_number();
02472
02473 #ifdef _DEBUG
02474
02475 const unsigned int inputs_number = count_inputs_number();
02476 const unsigned int inputs_size = inputs.size();
02477
02478 if(inputs_size != inputs_number)
02479 {
02480 std::ostringstream buffer;
02481
02482 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02483 << "Vector< Vector<double> > arrange_layers_input(const Vector<double>&, const Vector< Vector<double> >&) const method.\n"
02484 << "Size must be equal to number of inputs.\n";
02485
02486 throw std::logic_error(buffer.str());
02487 }
02488
02489 const unsigned int layers_activation_size = layers_activation.size();
02490
02491 if(layers_activation_size != layers_number)
02492 {
02493 std::ostringstream buffer;
02494
02495 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02496 << "Vector< Vector<double> > arrange_layers_input(const Vector<double>&, const Vector< Vector<double> >&) const method.\n"
02497 << "Size must be equal to number of inputs.\n";
02498
02499 throw std::logic_error(buffer.str());
02500 }
02501
02502 #endif
02503
02504
02505
02506 Vector< Vector<double> > layers_inputs(layers_number);
02507
02508 if(layers_number != 0)
02509 {
02510 layers_inputs[0] = inputs;
02511
02512 for(unsigned int j = 1; j < layers_number; j++)
02513 {
02514 layers_inputs[j] = layers_activation[j-1];
02515 }
02516 }
02517
02518 return(layers_inputs);
02519 }
02520
02521
02522
02526
02527 Vector< Vector<double> > MultilayerPerceptron::calculate_layers_input(const Vector<double>& inputs) const
02528 {
02529 const unsigned int layers_number = count_layers_number();
02530
02531 Vector< Vector<double> > layers_inputs(layers_number);
02532
02533 layers_inputs[0] = inputs;
02534
02535 for(unsigned int i = 1; i < layers_number; i++)
02536 {
02537 layers_inputs[i] = layers[i-1].calculate_outputs(layers_inputs[i-1]);
02538 }
02539
02540 return(layers_inputs);
02541 }
02542
02543
02544
02545
02549
02550 Vector< Vector<double> > MultilayerPerceptron::calculate_layers_combination(const Vector<double>& inputs) const
02551 {
02552
02553
02554 #ifdef _DEBUG
02555
02556 const unsigned int inputs_size = inputs.size();
02557
02558 const unsigned int inputs_number = count_inputs_number();
02559
02560 if(inputs_size != inputs_number)
02561 {
02562 std::ostringstream buffer;
02563
02564 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02565 << "Vector< Vector<double> > calculate_layers_combination(const Vector<double>&) const method.\n"
02566 << "Size must be equal to number of inputs.\n";
02567
02568 throw std::logic_error(buffer.str());
02569 }
02570
02571 #endif
02572
02573 const unsigned int layers_number = count_layers_number();
02574
02575 Vector< Vector<double> > layers_combination(layers_number);
02576
02577 if(layers_number > 0)
02578 {
02579 Vector< Vector<double> > layers_activation(layers_number);
02580
02581 layers_combination[0] = layers[0].calculate_combination(inputs);
02582
02583 layers_activation[0] = layers[0].calculate_activation(layers_combination[0]);
02584
02585 for(unsigned int i = 1; i < layers_number; i++)
02586 {
02587 layers_combination[i] = layers[i].calculate_combination(layers_activation[i-1]);
02588
02589 layers_activation[i] = layers[i].calculate_activation(layers_combination[i]);
02590 }
02591 }
02592
02593 return(layers_combination);
02594 }
02595
02596
02597
02598
02601
02602 Vector< Matrix<double> > MultilayerPerceptron::calculate_layers_combination_Jacobian(const Vector<double>& inputs) const
02603 {
02604 const unsigned int layers_number = count_layers_number();
02605
02606 Vector< Matrix<double> > layers_combination_Jacobian(layers_number-2);
02607
02608 Vector< Vector<double> > layers_output(layers_number);
02609
02610 Vector< Matrix<double> > layers_Jacobian(layers_number);
02611
02612
02613
02614 layers_output[0] = layers[0].calculate_outputs(inputs);
02615
02616 layers_combination_Jacobian[0] = layers[0].calculate_combination_Jacobian(inputs);
02617
02618 for(unsigned int i = 1; i < layers_number; i++)
02619 {
02620 layers_output[i] = layers[i].calculate_outputs(layers_output[i-1]);
02621
02622 layers_Jacobian[i] = layers[i].calculate_Jacobian(layers_output[i-1]);
02623 }
02624
02625 Matrix<double> output_layer_Jacobian = layers[layers_number-1].calculate_Jacobian(layers_output[layers_number-1]);
02626
02627
02628
02629 layers_Jacobian[layers_number] = output_layer_Jacobian;
02630
02631 for(int i = layers_number-1; i > -1; i--)
02632 {
02633 layers_Jacobian[layers_number] = layers_Jacobian[layers_number].dot(layers_Jacobian[i]);
02634 }
02635
02636 for(int i = layers_number-1; i > -1; i--)
02637 {
02638 layers_Jacobian[i] = layers_Jacobian[i];
02639
02640 for(int j = i-1; j > -1; j--)
02641 {
02642 layers_Jacobian[i] = layers_Jacobian[i].dot(layers_Jacobian[j]);
02643 }
02644 }
02645
02646 return(layers_combination_Jacobian);
02647 }
02648
02649
02650
02651
02656
02657 Vector< Matrix<double> > MultilayerPerceptron::calculate_layers_combination_parameters_Jacobian(const Vector< Vector<double> >& layers_inputs) const
02658 {
02659 const unsigned int layers_number = count_layers_number();
02660
02661 Vector< Matrix<double> > layers_combination_parameters_Jacobian(layers_number);
02662
02663 const Vector<double> dummy;
02664
02665 for(unsigned int i = 0; i < layers_number; i++)
02666 {
02667 layers_combination_parameters_Jacobian[i] = layers[i].calculate_combination_parameters_Jacobian(layers_inputs[i], dummy);
02668 }
02669
02670 return(layers_combination_parameters_Jacobian);
02671 }
02672
02673
02674
02675
02683
02684 Vector< Vector< Vector<double> > > MultilayerPerceptron::calculate_perceptrons_combination_parameters_gradient(const Vector< Vector<double> >& layers_inputs) const
02685 {
02686 const unsigned int layers_number = count_layers_number();
02687
02688
02689
02690 #ifdef _DEBUG
02691
02692 const unsigned int layers_input_size = layers_inputs.size();
02693
02694 if(layers_input_size != layers_number)
02695 {
02696 std::ostringstream buffer;
02697
02698 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02699 << "Vector< Vector< Vector<double> > > calculate_perceptrons_combination_parameters_gradient(const Vector< Vector<double> >&) const method.\n"
02700 << "Size must be equal to number of layers.\n";
02701
02702 throw std::logic_error(buffer.str());
02703 }
02704
02705 #endif
02706
02707 const Vector<unsigned int> layers_inputs_number = get_layers_inputs_number();
02708
02709 #ifdef _DEBUG
02710
02711 for(unsigned int i = 0; i < layers_number; i++)
02712 {
02713 if(layers_inputs[i].size() != layers_inputs_number[i])
02714 {
02715 std::ostringstream buffer;
02716
02717 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02718 << "Vector< Vector< Vector<double> > > calculate_perceptrons_combination_parameters_gradient(const Vector< Vector<double> >&) const method.\n"
02719 << "Size of inputs to layer " << i << " must be equal to size of that layer.\n";
02720
02721 throw std::logic_error(buffer.str());
02722 }
02723 }
02724
02725 #endif
02726
02727 const Vector<unsigned int> layers_size = arrange_layers_perceptrons_numbers();
02728
02729 Vector< Vector< Vector<double> > > perceptrons_combination_parameters_gradient(layers_number);
02730
02731 for(unsigned int i = 0; i < layers_number; i++)
02732 {
02733 perceptrons_combination_parameters_gradient[i].set(layers_size[i]);
02734
02735 for(unsigned int j = 0; j < layers_size[i]; j++)
02736 {
02737 const Perceptron& perceptron = layers[i].get_perceptron(j);
02738
02739 perceptrons_combination_parameters_gradient[i][j] = perceptron.calculate_combination_parameters_gradient(layers_inputs[i]);
02740 }
02741 }
02742
02743 return(perceptrons_combination_parameters_gradient);
02744 }
02745
02746
02747
02748
02752
02753 Vector< Vector<double> > MultilayerPerceptron::calculate_layers_activation(const Vector<double>& inputs) const
02754 {
02755
02756
02757 #ifdef _DEBUG
02758
02759 const unsigned int inputs_size = inputs.size();
02760
02761 const unsigned int inputs_number = count_inputs_number();
02762
02763 if(inputs_size != inputs_number)
02764 {
02765 std::ostringstream buffer;
02766
02767 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02768 << "Vector< Vector<double> > calculate_layers_activation(const Vector<double>&) const method.\n"
02769 << "Size must be equal to number of inputs.\n";
02770
02771 throw std::logic_error(buffer.str());
02772 }
02773
02774 #endif
02775
02776 const unsigned int layers_number = count_layers_number();
02777
02778 Vector< Vector<double> > layers_combination(layers_number);
02779 Vector< Vector<double> > layers_activation(layers_number);
02780
02781 layers_combination[0] = layers[0].calculate_combination(inputs);
02782 layers_activation[0] = layers[0].calculate_activation(layers_combination[0]);
02783
02784 for(unsigned int i = 1; i < layers_number; i++)
02785 {
02786 layers_combination[i] = layers[i].calculate_combination(layers_activation[i-1]);
02787 layers_activation[i] = layers[i].calculate_activation(layers_combination[i]);
02788 }
02789
02790 return(layers_activation);
02791 }
02792
02793
02794
02795
02799
02800 Vector< Vector<double> > MultilayerPerceptron::calculate_layers_activation_derivative(const Vector<double>& inputs) const
02801 {
02802
02803
02804 #ifdef _DEBUG
02805
02806 const unsigned int inputs_size = inputs.size();
02807 const unsigned int inputs_number = count_inputs_number();
02808
02809 if(inputs_size != inputs_number)
02810 {
02811 std::ostringstream buffer;
02812
02813 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02814 << "Vector< Vector<double> > calculate_layers_activation_derivative(const Vector<double>&) const method.\n"
02815 << "Size must be equal to number of inputs.\n";
02816
02817 throw std::logic_error(buffer.str());
02818 }
02819
02820 #endif
02821
02822 const unsigned int layers_number = count_layers_number();
02823
02824 Vector< Vector<double> > layers_combination(layers_number);
02825 Vector< Vector<double> > layers_activation(layers_number);
02826 Vector< Vector<double> > layers_activation_derivative(layers_number);
02827
02828 if(layers_number != 0)
02829 {
02830 layers_combination[0] = layers[0].calculate_combination(inputs);
02831 layers_activation[0] = layers[0].calculate_activation(layers_combination[0]);
02832 layers_activation_derivative[0] = layers[0].calculate_activation_derivative(layers_combination[0]);
02833
02834 for(unsigned int i = 1; i < layers_number; i++)
02835 {
02836 layers_combination[i] = layers[i].calculate_combination(layers_activation[i-1]);
02837 layers_activation[i] = layers[i].calculate_activation(layers_combination[i]);
02838 layers_activation_derivative[i] = layers[i].calculate_activation_derivative(layers_combination[i]);
02839 }
02840 }
02841
02842 return(layers_activation_derivative);
02843 }
02844
02845
02846
02847
02856
02857 Vector< Vector<double> > MultilayerPerceptron::calculate_layers_activation_second_derivative(const Vector<double>& inputs) const
02858 {
02859
02860
02861 #ifdef _DEBUG
02862
02863 const unsigned int inputs_size = inputs.size();
02864
02865 const unsigned int inputs_number = count_inputs_number();
02866
02867 if(inputs_size != inputs_number)
02868 {
02869 std::ostringstream buffer;
02870
02871 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02872 << "Vector< Vector<double> > calculate_layers_activation_second_derivative(const Vector<double>&) const method.\n"
02873 << "Size must be equal to number of inputs.\n";
02874
02875 throw std::logic_error(buffer.str());
02876 }
02877
02878 #endif
02879
02880 const unsigned int layers_number = count_layers_number();
02881
02882 Vector< Vector<double> > layers_combination(layers_number);
02883 Vector< Vector<double> > layers_activation(layers_number);
02884 Vector< Vector<double> > layers_activation_second_derivative(layers_number);
02885
02886 layers_combination[0] = layers[0].calculate_combination(inputs);
02887 layers_activation[0] = layers[0].calculate_activation(layers_combination[0]);
02888 layers_activation_second_derivative[0] = layers[0].calculate_activation_second_derivative(layers_combination[0]);
02889
02890 for(unsigned int i = 1; i < layers_number; i++)
02891 {
02892 layers_combination[i] = layers[i].calculate_combination(layers_activation[i-1]);
02893
02894 layers_activation[i] = layers[i].calculate_activation(layers_combination[i]);
02895
02896 layers_activation_second_derivative[i] = layers[i].calculate_activation_second_derivative(layers_combination[i]);
02897 }
02898
02899 return(layers_activation_second_derivative);
02900 }
02901
02902
02903
02904
02909
02910 Vector< Matrix<double> > MultilayerPerceptron::calculate_layers_Jacobian(const Vector<double>& inputs) const
02911 {
02912 const unsigned int layers_number = count_layers_number();
02913
02914 Vector<Vector<double> > layers_output(layers_number);
02915 Vector< Matrix<double> > layers_Jacobian(layers_number);
02916
02917 layers_output[0] = layers[0].calculate_outputs(inputs);
02918 layers_Jacobian[0] = layers[0].calculate_Jacobian(inputs);
02919
02920 for(unsigned int i = 1; i < layers_number; i++)
02921 {
02922 layers_output[i] = layers[i].calculate_outputs(layers_output[i-1]);
02923
02924 layers_Jacobian[i] = layers[i].calculate_Jacobian(layers_output[i-1]);
02925 }
02926
02927 return(layers_Jacobian);
02928 }
02929
02930
02931
02932
02937
02938 Vector< Vector< Matrix<double> > > MultilayerPerceptron::calculate_layers_Hessian_form(const Vector<double>& inputs) const
02939 {
02940 const unsigned int layers_number = count_layers_number();
02941
02942 Vector<Vector<double> > layers_output(layers_number);
02943 Vector< Vector< Matrix<double> > > layers_Hessian_form(layers_number);
02944
02945 layers_output[0] = layers[0].calculate_outputs(inputs);
02946 layers_Hessian_form[0] = layers[0].calculate_Hessian_form(inputs);
02947
02948 for(unsigned int i = 1; i < layers_number; i++)
02949 {
02950 layers_output[i] = layers[i].calculate_outputs(layers_output[i-1]);
02951
02952 layers_Hessian_form[i] = layers[i].calculate_Hessian_form(layers_output[i-1]);
02953 }
02954
02955 return(layers_Hessian_form);
02956 }
02957
02958
02959
02960
02964
02965 Matrix< Matrix<double> > MultilayerPerceptron::calculate_interlayers_combination_combination_Jacobian(const Vector<double>& inputs) const
02966 {
02967 #ifdef _DEBUG
02968
02969 const unsigned int size = inputs.size();
02970
02971 const unsigned int inputs_number = count_inputs_number();
02972
02973 if(size != inputs_number)
02974 {
02975 std::ostringstream buffer;
02976
02977 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
02978 << "Matrix< Matrix<double> > calculate_interlayers_combination_combination_Jacobian(const Vector<double>&) const method.\n"
02979 << "Size of inpouts must be equal to number of inputs.\n";
02980
02981 throw std::logic_error(buffer.str());
02982 }
02983
02984 #endif
02985
02986 const Vector< Vector<double> > layers_combination = calculate_layers_combination(inputs);
02987
02988 return(calculate_interlayers_combination_combination_Jacobian(layers_combination));
02989 }
02990
02991
02992
02993
02997
02998 Matrix < Matrix<double> > MultilayerPerceptron::calculate_interlayers_combination_combination_Jacobian(const Vector< Vector<double> >& layers_combination) const
02999 {
03000 const unsigned int layers_number = count_layers_number();
03001
03002 #ifdef _DEBUG
03003
03004 unsigned int size;
03005
03006 const Vector<unsigned int> layers_size = arrange_layers_perceptrons_numbers();
03007
03008 for(unsigned int i = 0; i < layers_number; i++)
03009 {
03010 size = layers_combination[i].size();
03011
03012 if(size != layers_size[i])
03013 {
03014 std::ostringstream buffer;
03015
03016 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
03017 << "Matrix< Matrix<double> > calculate_interlayers_combination_combination_Jacobian(const Vector< Vector<double> >&) const method.\n"
03018 << "Size must be equal to size of layer.\n";
03019
03020 throw std::logic_error(buffer.str());
03021 }
03022 }
03023
03024 #endif
03025
03026
03027
03028 Matrix< Matrix<double> > interlayers_combination_combination_Jacobian(layers_number, layers_number);
03029
03030 for(unsigned int image_index = 0; image_index < layers_number; image_index++)
03031 {
03032 for(unsigned int domain_index = 0; domain_index < layers_number; domain_index++)
03033 {
03034 interlayers_combination_combination_Jacobian[image_index][domain_index] = calculate_interlayer_combination_combination_Jacobian(domain_index, image_index, layers_combination[domain_index]);
03035 }
03036 }
03037
03038 return(interlayers_combination_combination_Jacobian);
03039 }
03040
03041
03042
03043
03051
03052 Vector< Vector< Vector<double> > > MultilayerPerceptron::calculate_first_order_forward_propagation(const Vector<double>& inputs) const
03053 {
03054
03055
03056 #ifdef _DEBUG
03057
03058 const unsigned int inputs_size = inputs.size();
03059
03060 const unsigned int inputs_number = count_inputs_number();
03061
03062 if(inputs_size != inputs_number)
03063 {
03064 std::ostringstream buffer;
03065
03066 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
03067 << "Vector< Vector< Vector<double> > > calculate_first_order_forward_propagation(const Vector<double>&) const method.\n"
03068 << "Size must be equal to number of inputs.\n";
03069
03070 throw std::logic_error(buffer.str());
03071 }
03072
03073 #endif
03074
03075 const unsigned int layers_number = count_layers_number();
03076
03077 Vector< Vector<double> > layers_combination(layers_number);
03078
03079 Vector< Vector< Vector<double> > > first_order_forward_propagation(2);
03080
03081 first_order_forward_propagation[0].set(layers_number);
03082 first_order_forward_propagation[1].set(layers_number);
03083
03084 layers_combination[0] = layers[0].calculate_combination(inputs);
03085
03086 first_order_forward_propagation[0][0] = layers[0].calculate_activation(layers_combination[0]);
03087
03088 first_order_forward_propagation[1][0] = layers[0].calculate_activation_derivative(layers_combination[0]);
03089
03090 for(unsigned int i = 1; i < layers_number; i++)
03091 {
03092 layers_combination[i] = layers[i].calculate_combination(first_order_forward_propagation[0][i-1]);
03093
03094 first_order_forward_propagation[0][i] = layers[i].calculate_activation(layers_combination[i]);
03095
03096 first_order_forward_propagation[1][i] = layers[i].calculate_activation_derivative(layers_combination[i]);
03097 }
03098
03099 return(first_order_forward_propagation);
03100 }
03101
03102
03103
03104
03112
03113 Vector< Vector< Vector<double> > > MultilayerPerceptron::calculate_second_order_forward_propagation(const Vector<double>& inputs) const
03114 {
03115
03116
03117 #ifdef _DEBUG
03118
03119 const unsigned int inputs_size = inputs.size();
03120
03121 const unsigned int inputs_number = count_inputs_number();
03122
03123 if(inputs_size != inputs_number)
03124 {
03125 std::ostringstream buffer;
03126
03127 buffer << "OpenNN Exception: MultilayerPerceptron class.\n"
03128 << "Vector< Vector< Vector<double> > > calculate_second_order_forward_propagation(const Vector<double>&) const method.\n"
03129 << "Size of multilayer_perceptron_pointer inputs must be equal to number of inputs.\n";
03130
03131 throw std::logic_error(buffer.str());
03132 }
03133
03134 #endif
03135
03136 const unsigned int layers_number = count_layers_number();
03137
03138 Vector< Vector<double> > layers_combination(layers_number);
03139
03140 Vector< Vector< Vector<double> > > second_order_forward_propagation(3);
03141
03142 second_order_forward_propagation[0].set(layers_number);
03143 second_order_forward_propagation[1].set(layers_number);
03144 second_order_forward_propagation[2].set(layers_number);
03145
03146 layers_combination[0] = layers[0].calculate_combination(inputs);
03147
03148 second_order_forward_propagation[0][0] = layers[0].calculate_activation(layers_combination[0]);
03149
03150 second_order_forward_propagation[1][0] = layers[0].calculate_activation_derivative(layers_combination[0]);
03151
03152 second_order_forward_propagation[2][0] = layers[0].calculate_activation_second_derivative(layers_combination[0]);
03153
03154 for(unsigned int i = 1; i < layers_number; i++)
03155 {
03156 layers_combination[i] = layers[i].calculate_combination(second_order_forward_propagation[0][i-1]);
03157
03158 second_order_forward_propagation[0][i] = layers[i].calculate_activation(layers_combination[i]);
03159
03160 second_order_forward_propagation[1][i] = layers[i].calculate_activation_derivative(layers_combination[i]);
03161
03162 second_order_forward_propagation[2][i] = layers[i].calculate_activation_second_derivative(layers_combination[i]);
03163 }
03164
03165 return(second_order_forward_propagation);
03166 }
03167
03168
03169
03170
03172
03173 std::string MultilayerPerceptron::to_string(void) const
03174 {
03175 std::ostringstream buffer;
03176
03177 buffer << "MultilayerPerceptron\n"
03178 << "Architecture: " << arrange_architecture() << "\n"
03179 << "Layers activation function: " << write_layers_activation_function() << "\n"
03180 << "Parameters: " << arrange_parameters() << "\n"
03181 << "Display: " << display << "\n";
03182
03183 return(buffer.str());
03184 }
03185
03186
03187
03188
03191
03192 TiXmlElement* MultilayerPerceptron::to_XML(void) const
03193 {
03194 std::ostringstream buffer;
03195
03196 TiXmlElement* multilayer_perceptron_element = new TiXmlElement("MultilayerPerceptron");
03197 multilayer_perceptron_element->SetAttribute("Version", 4);
03198
03199
03200 {
03201 TiXmlElement* architecture_element = new TiXmlElement("Architecture");
03202 multilayer_perceptron_element->LinkEndChild(architecture_element);
03203
03204 std::string architecture_string = arrange_architecture().to_string();
03205
03206 TiXmlText* architecture_text = new TiXmlText(architecture_string.c_str());
03207 architecture_element->LinkEndChild(architecture_text);
03208 }
03209
03210
03211 {
03212 TiXmlElement* layers_activation_function_element = new TiXmlElement("LayersActivationFunction");
03213 multilayer_perceptron_element->LinkEndChild(layers_activation_function_element);
03214
03215 std::string layers_activation_function_string = write_layers_activation_function().to_string();
03216
03217 TiXmlText* layers_activation_function_text = new TiXmlText(layers_activation_function_string.c_str());
03218 layers_activation_function_element->LinkEndChild(layers_activation_function_text);
03219 }
03220
03221
03222 {
03223 TiXmlElement* parameters_element = new TiXmlElement("Parameters");
03224 multilayer_perceptron_element->LinkEndChild(parameters_element);
03225
03226 std::string parameters_string = arrange_parameters().to_string();
03227
03228 TiXmlText* parameters_text = new TiXmlText(parameters_string.c_str());
03229 parameters_element->LinkEndChild(parameters_text);
03230 }
03231
03232
03233 {
03234 TiXmlElement* display_element = new TiXmlElement("Display");
03235 multilayer_perceptron_element->LinkEndChild(display_element);
03236
03237 buffer.str("");
03238 buffer << display;
03239
03240 TiXmlText* display_text = new TiXmlText(buffer.str().c_str());
03241 display_element->LinkEndChild(display_text);
03242 }
03243
03244 return(multilayer_perceptron_element);
03245 }
03246
03247
03248
03249
03252
03253 void MultilayerPerceptron::from_XML(TiXmlElement* multilayer_perceptron_element)
03254 {
03255 if(multilayer_perceptron_element)
03256 {
03257
03258 {
03259 TiXmlElement* architecture_element = multilayer_perceptron_element->FirstChildElement("Architecture");
03260
03261 if(architecture_element)
03262 {
03263 const char* architecture_text = architecture_element->GetText();
03264
03265 if(architecture_text)
03266 {
03267 Vector<unsigned int> new_architecture;
03268 new_architecture.parse(architecture_text);
03269
03270 try
03271 {
03272 set(new_architecture);
03273 }
03274 catch(std::exception& e)
03275 {
03276 std::cout << e.what() << std::endl;
03277 }
03278 }
03279 }
03280 }
03281
03282
03283 {
03284 TiXmlElement* layers_activation_function_element = multilayer_perceptron_element->FirstChildElement("LayersActivationFunction");
03285
03286 if(layers_activation_function_element)
03287 {
03288 const char* layers_activation_function_text = layers_activation_function_element->GetText();
03289
03290 if(layers_activation_function_text)
03291 {
03292 Vector<std::string> new_layers_activation_function;
03293 new_layers_activation_function.parse(layers_activation_function_text);
03294
03295 try
03296 {
03297 set_layers_activation_function(new_layers_activation_function);
03298 }
03299 catch(std::exception& e)
03300 {
03301 std::cout << e.what() << std::endl;
03302 }
03303 }
03304 }
03305 }
03306
03307
03308 {
03309 TiXmlElement* parameters_element = multilayer_perceptron_element->FirstChildElement("Parameters");
03310
03311 if(parameters_element)
03312 {
03313 const char* parameters_text = parameters_element->GetText();
03314
03315 if(parameters_text)
03316 {
03317 Vector<double> new_parameters;
03318 new_parameters.parse(parameters_text);
03319
03320 try
03321 {
03322 set_parameters(new_parameters);
03323 }
03324 catch(std::exception& e)
03325 {
03326 std::cout << e.what() << std::endl;
03327 }
03328 }
03329 }
03330 }
03331
03332
03333 {
03334 TiXmlElement* display_element = multilayer_perceptron_element->FirstChildElement("Display");
03335
03336 if(display)
03337 {
03338 std::string new_display_string = display_element->GetText();
03339
03340 try
03341 {
03342 set_display(new_display_string != "0");
03343 }
03344 catch(std::exception& e)
03345 {
03346 std::cout << e.what() << std::endl;
03347 }
03348 }
03349 }
03350 }
03351 }
03352
03353
03354
03355
03358
03359 std::string MultilayerPerceptron::write_expression(const Vector<std::string>& inputs_name, const Vector<std::string>& outputs_name) const
03360 {
03361 const unsigned int layers_number = count_layers_number();
03362 const Vector<unsigned int> layers_perceptrons_number = arrange_layers_perceptrons_numbers();
03363
03364 std::ostringstream buffer;
03365
03366 if(layers_number == 0)
03367 {
03368 }
03369 else if(layers_number == 1)
03370 {
03371 buffer << layers[0].write_expression(inputs_name, outputs_name) << "\n";
03372 }
03373 else
03374 {
03375 Vector< Vector<std::string> > layers_outputs_name(layers_number);
03376
03377 for(unsigned int i = 0; i < layers_number; i++)
03378 {
03379 layers_outputs_name[i].set(layers_perceptrons_number[i]);
03380
03381 for(unsigned int j = 0; j < layers_perceptrons_number[i]; j++)
03382 {
03383 std::ostringstream new_buffer;
03384 new_buffer << "y_" << i+1 << "_" << j+1;
03385 layers_outputs_name[i][j] = new_buffer.str();
03386 }
03387 }
03388
03389 buffer << layers[0].write_expression(inputs_name, layers_outputs_name[0]);
03390
03391 for(unsigned int i = 1; i < layers_number-1; i++)
03392 {
03393 buffer << layers[i].write_expression(layers_outputs_name[i-1], layers_outputs_name[i]);
03394 }
03395
03396 buffer << layers[layers_number-1].write_expression(layers_outputs_name[layers_number-2], outputs_name);
03397 }
03398
03399 return(buffer.str());
03400 }
03401
03402 }
03403
03404
03405
03406
03407
03408
03409
03410
03411
03412
03413
03414
03415
03416
03417
03418
03419