Adding MeanSquaredError, MeanAbsoluteError loss functions.

CMakeLists.txt

@@ -15,6 +15,7 @@ target_sources(afml
   src/nn/Modules/Activations.cpp
   src/nn/Modules/Container.cpp
   src/nn/Modules/Linear.cpp
+  src/nn/Modules/Loss.cpp
   src/nn/Modules/Module.cpp
   src/nn/Init.cpp
   )

examples/autograd.cpp

@@ -8,6 +8,7 @@
  ********************************************************/
 
 #include <af/autograd.h>
+#include <af/nn.h>
 
 #define VERIFY(VAL) do {                                    \
         auto res = af::allTrue<bool>(af::abs(VAL) < 1E-5);  \

examples/perceptron.cpp

@@ -39,7 +39,9 @@ int main()
     perceptron.add(nn::Linear(inputSize, outputSize));
     perceptron.add(nn::Sigmoid());
 
-    Variable result;
+    auto loss = nn::MeanSquaredError();
+
+    Variable result, l;
     for (int i = 0; i < 1000; i++) {
         for (int j = 0; j < numSamples; j++) {
             perceptron.train();
@@ -52,17 +54,15 @@ int main()
             result = perceptron.forward(nn::input(in_j));
 
             // Calculate loss
-            // TODO: Use loss function
-            af::array diff = out_j - result.array();
+            l = loss.forward(result, nn::target(out_j));
 
             // Backward propagation
-            auto d_result = Variable(diff, false);
-            result.backward(d_result);
+            l.backward();
 
             // Update parameters
             // TODO: Should use optimizer
             for (auto &param : perceptron.parameters()) {
-                param.array() += lr * param.grad().array();
+                param.array() -= lr * param.grad().array();
                 param.array().eval();
             }
         }

include/af/autograd/Variable.hpp

@@ -72,6 +72,7 @@ namespace af {
 
             void backward(const Variable &grad, bool retain_grad_graph = false);
 
+            void backward(bool retain_grad_graph = false);
 
         private:
             void evalGrad(bool retain_grad_graph = false);

include/af/nn/Init.hpp

@@ -15,6 +15,8 @@ namespace af {
 
         autograd::Variable input(const af::array &arr);
 
+        autograd::Variable target(const af::array &arr);
+
         autograd::Variable parameter(const af::array &arr);
 
         autograd::Variable uniform(int input_size, int output_size,

include/af/nn/Modules.hpp

@@ -12,3 +12,4 @@
 #include <af/nn/Modules/Linear.hpp>
 #include <af/nn/Modules/Container.hpp>
 #include <af/nn/Modules/Activations.hpp>
+#include <af/nn/Modules/Loss.hpp>

include/af/nn/Modules/Container.hpp

@@ -41,8 +41,6 @@ namespace af
             ModulePtr get(int id);
 
             std::vector<ModulePtr> modules();
-
-            virtual autograd::Variable forward(const autograd::Variable &input) = 0;
         };
 
         class Sequential : public Container
@@ -51,7 +49,7 @@ namespace af
 
             Sequential();
 
-            virtual autograd::Variable forward(const autograd::Variable &input);
+            autograd::Variable forward(const autograd::Variable &input);
         };
     }
 }

include/af/nn/Modules/Loss.hpp

@@ -0,0 +1,46 @@
+/*******************************************************
+ * Copyright (c) 2017, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+#include <af/nn/Modules/Module.hpp>
+
+namespace af
+{
+    namespace nn
+    {
+        class Loss : public Module
+        {
+        public:
+            Loss() {}
+
+            virtual autograd::Variable forward(const autograd::Variable &input,
+                                               const autograd::Variable &target) = 0;
+
+            autograd::Variable forward(const autograd::Variable &input);
+        };
+
+        class MeanSquaredError : public Loss
+        {
+        public:
+            MeanSquaredError() {}
+
+            autograd::Variable forward(const autograd::Variable &input,
+                                       const autograd::Variable &target);
+        };
+
+        class MeanAbsoluteError : public Loss
+        {
+        public:
+            MeanAbsoluteError() {}
+
+            autograd::Variable forward(const autograd::Variable &input,
+                                       const autograd::Variable &target);
+        };
+    }
+}

src/autograd/Variable.cpp

@@ -172,6 +172,12 @@ namespace af {
             }
         }
 
+        void Variable::backward(bool retain_grad_graph)
+        {
+            auto ones = Variable(af::constant(1, this->dims()), false);
+            this->backward(ones, retain_grad_graph);
+        }
+
         Variable::DAG_t Variable::build(const Variable &var)
         {
             Cache_t cache;

src/nn/Init.cpp

@@ -21,6 +21,11 @@ namespace af {
             return Variable(arr, false);
         }
 
+        Variable target(const af::array &arr)
+        {
+            return Variable(arr, false);
+        }
+
         Variable parameter(const af::array &arr)
         {
             return Variable(arr, true);

src/nn/Modules/Loss.cpp

@@ -0,0 +1,39 @@
+/*******************************************************
+ * Copyright (c) 2017, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#include <af/autograd/Functions.hpp>
+
+#include <af/nn/Modules/Loss.hpp>
+
+
+namespace af
+{
+    namespace nn
+    {
+        using namespace autograd;
+
+        autograd::Variable Loss::forward(const autograd::Variable &input)
+        {
+            throw af::exception("Loss module requires both input and targets");
+        }
+
+        autograd::Variable MeanSquaredError::forward(const autograd::Variable &input,
+                                                     const autograd::Variable &target)
+        {
+            auto df = input - target;
+            return mean(df * df, {0});
+        }
+
+        autograd::Variable MeanAbsoluteError::forward(const autograd::Variable &input,
+                                                      const autograd::Variable &target)
+        {
+            auto df = input - target;
+            return mean(abs(df), {0});
+        }
+    }
+}