Adding Initializers and Loss functions

CMakeLists.txt

@@ -15,8 +15,9 @@ target_sources(afml
   src/nn/Modules/Activations.cpp
   src/nn/Modules/Container.cpp
   src/nn/Modules/Linear.cpp
+  src/nn/Modules/Loss.cpp
   src/nn/Modules/Module.cpp
-  src/nn/Types.cpp
+  src/nn/Init.cpp
   )
 
 target_include_directories(afml

examples/autograd.cpp

@@ -8,6 +8,7 @@
  ********************************************************/
 
 #include <af/autograd.h>
+#include <af/nn.h>
 
 #define VERIFY(VAL) do {                                    \
         auto res = af::allTrue<bool>(af::abs(VAL) < 1E-5);  \
@@ -128,11 +129,11 @@ void test_tanh()
     VERIFY(dx.array() - (1 + af::tanh(x.array())) * (1 - af::tanh(x.array())));
 }
 
-void test_expand()
+void test_tile()
 {
     auto x = Variable(af::randu(5), true);
     auto y = Variable(af::randu(5, 2), true);
-    auto z = y * expandAs(x, y);
+    auto z = y * tileAs(x, y);
     auto dz = Variable(af::constant(1.0, 5, 2), false);
     z.backward(dz);
     auto dy = y.grad();
@@ -141,11 +142,11 @@ void test_expand()
     VERIFY(dx.array() - af::sum(y.array(), 1));
 }
 
-void test_reduce()
+void test_sum()
 {
     auto x = Variable(af::randu(5), true);
     auto y = Variable(af::randu(5, 2), true);
-    auto z = x * reduceAs(y, x);
+    auto z = x * sumAs(y, x);
     auto dz = Variable(af::constant(1.0, 5), false);
     z.backward(dz);
     auto dy = y.grad();
@@ -154,6 +155,19 @@ void test_reduce()
     VERIFY(dx.array() - af::sum(y.array(), 1));
 }
 
+void test_mean()
+{
+    auto x = Variable(af::randu(5), true);
+    auto y = Variable(af::randu(5, 3, 2), true);
+    auto z = x * mean(y, {1,2});
+    auto dz = Variable(af::constant(1.0, 5), false);
+    z.backward(dz);
+    auto dy = y.grad();
+    auto dx = x.grad();
+    VERIFY(dy.array() - 6 * af::tile(x.array(), 1, 3, 2));
+    VERIFY(dx.array() - af::mean(af::mean(y.array(), 1), 2));
+}
+
 int main()
 {
     af::info();
@@ -166,7 +180,8 @@ int main()
     test_exp();
     test_sigmoid();
     test_tanh();
-    test_expand();
-    test_reduce();
+    test_tile();
+    test_sum();
+    test_mean();
     return 0;
 }

examples/perceptron.cpp

@@ -39,7 +39,9 @@ int main()
     perceptron.add(nn::Linear(inputSize, outputSize));
     perceptron.add(nn::Sigmoid());
 
-    Variable result;
+    auto loss = nn::MeanSquaredError();
+
+    Variable result, l;
     for (int i = 0; i < 1000; i++) {
         for (int j = 0; j < numSamples; j++) {
             perceptron.train();
@@ -52,17 +54,15 @@ int main()
             result = perceptron.forward(nn::input(in_j));
 
             // Calculate loss
-            // TODO: Use loss function
-            af::array diff = out_j - result.array();
+            l = loss.forward(result, nn::noGrad(out_j));
 
             // Backward propagation
-            auto d_result = Variable(diff, false);
-            result.backward(d_result);
+            l.backward();
 
             // Update parameters
             // TODO: Should use optimizer
             for (auto &param : perceptron.parameters()) {
-                param.array() += lr * param.grad().array();
+                param.array() -= lr * param.grad().array();
                 param.array().eval();
             }
         }

include/af/autograd/Functions.hpp

@@ -8,6 +8,9 @@
  ********************************************************/
 #pragma once
 
+#include <arrayfire.h>
+#include <vector>
+
 namespace af {
     namespace autograd {
 
@@ -46,27 +49,35 @@ namespace af {
         Variable reciprocal(const Variable &input);
 
         Variable exp(const Variable &input);
+        Variable log(const Variable &input);
         Variable sin(const Variable &input);
         Variable cos(const Variable &input);
         Variable tanh(const Variable &input);
         Variable sigmoid(const Variable &input);
-      
+
         Variable max(const Variable &lhs, const Variable &rhs);
         Variable max(const Variable &lhs, const double &rhs);
         Variable max(const double &lhs, const Variable &rhs);
-      
+
         Variable min(const Variable &lhs, const Variable &rhs);
         Variable min(const Variable &lhs, const double &rhs);
         Variable min(const double &lhs, const Variable &rhs);
-      
+
         Variable transpose(const Variable &input);
-        Variable expandAs(const Variable &input, const Variable &reference);
-        Variable reduceAs(const Variable &input, const Variable &reference);
+        Variable tileAs(const Variable &input, const Variable &reference);
+        Variable sumAs(const Variable &input, const Variable &reference);
+
+        Variable tile(const Variable &input, const std::vector<int> &repeats);
+        Variable sum(const Variable &input, const std::vector<int> &axes);
+        Variable mean(const Variable &input, const std::vector<int> &axes);
 
         Variable matmul(const Variable &lhs, const Variable &rhs);
         Variable matmulTN(const Variable &lhs, const Variable &rhs);
         Variable matmulNT(const Variable &lhs, const Variable &rhs);
 
+        Variable abs(const Variable &input);
 
+        Variable flat(const Variable &input);
+        Variable moddims(const Variable &input, const dim4 &dims);
     }
 }

include/af/autograd/Variable.hpp

@@ -60,6 +60,8 @@ namespace af {
 
             bool isGradAvailable() const;
 
+            af::dim4 dims() const;
+
             void zeroGrad();
 
             void setCalcGrad(bool calc_grad);
@@ -70,6 +72,7 @@ namespace af {
 
             void backward(const Variable &grad, bool retain_grad_graph = false);
 
+            void backward(bool retain_grad_graph = false);
 
         private:
             void evalGrad(bool retain_grad_graph = false);

include/af/nn.h

@@ -10,4 +10,4 @@
 #pragma once
 
 #include <af/nn/Modules.hpp>
-#include <af/nn/Types.hpp>
+#include <af/nn/Init.hpp>

include/af/nn/Init.hpp

@@ -0,0 +1,76 @@
+/*******************************************************
+ * Copyright (c) 2017, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+#include <af/autograd/Variable.hpp>
+
+namespace af {
+    namespace nn {
+
+        autograd::Variable input(const af::array &arr);
+
+        autograd::Variable noGrad(const af::array &arr);
+
+        autograd::Variable parameter(const af::array &arr);
+
+        autograd::Variable uniform(int input_size, int output_size,
+                                   double min = 0, double max = 1,
+                                   af::dtype type = f32, bool calc_grad=true);
+
+        autograd::Variable uniform(af::dim4 dims,
+                                   double min = 0, double max = 1,
+                                   af::dtype type = f32, bool calc_grad=true);
+
+        autograd::Variable normal(int input_size, int output_size,
+                                  double stdv = 1, double mean = 0,
+                                  af::dtype type = f32, bool calc_grad=true);
+
+        autograd::Variable normal(af::dim4 dims,
+                                  double stdv = 1, double mean = 0,
+                                  af::dtype type = f32, bool calc_grad=true);
+
+        autograd::Variable lecunUniform(int input_size, int output_size,
+                                        af::dtype type = f32, bool calc_grad=true);
+
+        autograd::Variable lecunUniform(af::dim4 dims,
+                                        af::dtype type = f32, bool calc_grad=true);
+
+        autograd::Variable lecunNormal(int input_size, int output_size,
+                                       af::dtype type = f32, bool calc_grad=true);
+
+        autograd::Variable lecunNormal(af::dim4 dims,
+                                       af::dtype type = f32, bool calc_grad=true);
+
+        autograd::Variable glorotUniform(int input_size, int output_size,
+                                         af::dtype type = f32, bool calc_grad=true);
+
+        autograd::Variable glorotUniform(af::dim4 dims,
+                                         af::dtype type = f32, bool calc_grad=true);
+
+        autograd::Variable glorotNormal(int input_size, int output_size,
+                                        af::dtype type = f32, bool calc_grad=true);
+
+        autograd::Variable glorotNormal(af::dim4 dims,
+                                        af::dtype type = f32, bool calc_grad=true);
+
+
+        autograd::Variable constant(double val, int input_size, int output_size,
+                                    af::dtype type = f32, bool calc_grad=true);
+
+        autograd::Variable constant(double val, af::dim4 dims,
+                                    af::dtype type = f32, bool calc_grad=true);
+
+        autograd::Variable identity(int input_size, int output_size,
+                                    af::dtype type = f32, bool calc_grad=true);
+
+        autograd::Variable identity(af::dim4 dims,
+                                    af::dtype type = f32, bool calc_grad=true);
+
+    }
+}

include/af/nn/Modules.hpp

@@ -12,3 +12,4 @@
 #include <af/nn/Modules/Linear.hpp>
 #include <af/nn/Modules/Container.hpp>
 #include <af/nn/Modules/Activations.hpp>
+#include <af/nn/Modules/Loss.hpp>

include/af/nn/Modules/Activations.hpp

@@ -35,26 +35,26 @@ namespace af
         {
         public:
             ReLU();
-         
+
             autograd::Variable forward(const autograd::Variable &input);
         };
-      
+
         class LeakyReLU : public Module
         {
         private:
             double m_slope;
         public:
             LeakyReLU(double slope = 0.0);
-         
+
             autograd::Variable forward(const autograd::Variable &input);
         };
 
         class PReLU : public Module
         {
         public:
-            PReLU(int size, double spread = 1.0);
+            PReLU(int size, double value = 1.0);
             PReLU(const autograd::Variable &w);
-         
+
             autograd::Variable forward(const autograd::Variable &input);
         };
 
@@ -74,11 +74,11 @@ namespace af
             double m_threshold;
         public:
             ThresholdReLU(double threshold = 1.0);
-         
+
             autograd::Variable forward(const autograd::Variable &input);
         };
 
-      
+
 
     }
 }

include/af/nn/Modules/Container.hpp

@@ -41,8 +41,6 @@ namespace af
             ModulePtr get(int id);
 
             std::vector<ModulePtr> modules();
-
-            virtual autograd::Variable forward(const autograd::Variable &input) = 0;
         };
 
         class Sequential : public Container
@@ -51,7 +49,7 @@ namespace af
 
             Sequential();
 
-            virtual autograd::Variable forward(const autograd::Variable &input);
+            autograd::Variable forward(const autograd::Variable &input);
         };
     }
 }

include/af/nn/Modules/Loss.hpp

@@ -0,0 +1,64 @@
+/*******************************************************
+ * Copyright (c) 2017, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+#include <af/nn/Modules/Module.hpp>
+
+namespace af
+{
+    namespace nn
+    {
+        class Loss : public Module
+        {
+        public:
+            Loss() {}
+
+            virtual autograd::Variable forward(const autograd::Variable &inputs,
+                                               const autograd::Variable &targets) = 0;
+
+            autograd::Variable forward(const autograd::Variable &inputs);
+        };
+
+        class MeanSquaredError : public Loss
+        {
+        public:
+            MeanSquaredError() {}
+
+            autograd::Variable forward(const autograd::Variable &inputs,
+                                       const autograd::Variable &targets);
+        };
+
+        class MeanAbsoluteError : public Loss
+        {
+        public:
+            MeanAbsoluteError() {}
+
+            autograd::Variable forward(const autograd::Variable &inputs,
+                                       const autograd::Variable &targets);
+        };
+
+        class BinaryCrossEntropyLoss : public Loss
+        {
+        public:
+            BinaryCrossEntropyLoss() {}
+
+            autograd::Variable forward(const autograd::Variable &inputs,
+                                       const autograd::Variable &targets);
+
+            autograd::Variable forward(const autograd::Variable &inputs,
+                                       const autograd::Variable &targets,
+                                       const autograd::Variable &weights);
+        };
+
+        typedef MeanSquaredError MSE;
+        typedef MeanAbsoluteError MAE;
+        typedef MeanAbsoluteError L1Loss;
+        typedef BinaryCrossEntropyLoss BCELoss;
+    }
+}