Merge pull request #8 from denosaurs/models

feat: models

.gitignore

@@ -2,6 +2,9 @@
 .idea/
 build/
 xor_model.bin
+esr.bin
 node_modules/
+output.png
 digit_model.bin
-bench/tfjs/node_modules
+package-lock.json
+test.ts

README.md

@@ -27,7 +27,12 @@
 ### Usage
 
 ```typescript
-import { DenseLayer, NeuralNetwork } from "https://deno.land/x/netsaur/mod.ts";
+import {
+  DenseLayer,
+  NeuralNetwork,
+  tensor1D,
+  tensor2D,
+} from "https://deno.land/x/netsaur/mod.ts";
 
 const net = new NeuralNetwork({
   silent: true,
@@ -40,11 +45,17 @@ const net = new NeuralNetwork({
 
 await net.train(
   [
-    { inputs: [0, 0, 1, 0, 0, 1, 1, 1], outputs: [0, 1, 1, 0] },
+    {
+      inputs: await tensor2D([
+        [0, 0],
+        [1, 0],
+        [0, 1],
+        [1, 1],
+      ]),
+      outputs: await tensor1D([0, 1, 1, 0]),
+    },
   ],
   5000,
-  4,
-  0.1,
 );
 
 console.log(await net.predict(new Float32Array([0, 0])));
@@ -57,7 +68,10 @@ console.log(await net.predict(new Float32Array([1, 1])));
 
 ```typescript
 import { DenseLayer, NeuralNetwork } from "https://deno.land/x/netsaur/mod.ts";
-import { Matrix, Native } from "https://deno.land/x/netsaur/backends/native.ts";
+import {
+  Matrix,
+  Native,
+} from "https://deno.land/x/netsaur/backends/native/mod.ts";
 
 const network = await new NeuralNetwork({
   input: 2,
@@ -95,3 +109,54 @@ console.log(
   ),
 );
 ```
+
+### Saving Models
+
+```typescript
+import {
+  DenseLayer,
+  NeuralNetwork,
+  tensor1D,
+  tensor2D,
+} from "https://deno.land/x/netsaur/mod.ts";
+import { Model } from "https://deno.land/x/netsaur/model/mod.ts";
+
+const net = new NeuralNetwork({
+  silent: true,
+  layers: [
+    new DenseLayer({ size: 3, activation: "sigmoid" }),
+    new DenseLayer({ size: 1, activation: "sigmoid" }),
+  ],
+  cost: "crossentropy",
+});
+
+await net.train(
+  [
+    {
+      inputs: await tensor2D([
+        [0, 0],
+        [1, 0],
+        [0, 1],
+        [1, 1],
+      ]),
+      outputs: await tensor1D([0, 1, 1, 0]),
+    },
+  ],
+  5000,
+);
+
+await Model.save("./network.json", net);
+```
+
+### Loading & Running Models
+
+```typescript
+import { Model } from "https://deno.land/x/netsaur/model/mod.ts";
+
+const net = await Model.load("./network.json");
+
+console.log(await net.predict(new Float32Array([0, 0])));
+console.log(await net.predict(new Float32Array([1, 0])));
+console.log(await net.predict(new Float32Array([0, 1])));
+console.log(await net.predict(new Float32Array([1, 1])));
+```

src/cpu/activation.ts → backends/cpu/activation.ts

@@ -1,4 +1,5 @@
 export interface CPUActivationFn {
+  name: string;
   activate(val: number): number;
   prime(val: number, error?: number): number;
 }
@@ -7,6 +8,7 @@ export interface CPUActivationFn {
  * Linear activation function f(x) = x
  */
 export class Linear implements CPUActivationFn {
+  name = "linear";
   activate(val: number): number {
     return val;
   }
@@ -20,6 +22,7 @@ export class Linear implements CPUActivationFn {
  * Sigmoid activation function f(x) = 1 / (1 + e^(-x))
  */
 export class Sigmoid implements CPUActivationFn {
+  name = "sigmoid";
   activate(val: number): number {
     return 1 / (1 + Math.exp(-val));
   }
@@ -34,6 +37,7 @@ export class Sigmoid implements CPUActivationFn {
  * This is the same as the sigmoid function, but is more robust to outliers
  */
 export class Tanh implements CPUActivationFn {
+  name = "tanh";
   activate(val: number): number {
     return Math.tanh(val);
   }
@@ -48,6 +52,7 @@ export class Tanh implements CPUActivationFn {
  * This is a rectified linear unit, which is a smooth approximation to the sigmoid function.
  */
 export class Relu implements CPUActivationFn {
+  name = "relu";
   activate(val: number): number {
     return Math.max(0, val);
   }
@@ -62,6 +67,7 @@ export class Relu implements CPUActivationFn {
  * This is a rectified linear unit with a 6-value output range.
  */
 export class Relu6 implements CPUActivationFn {
+  name = "relu6";
   activate(val: number): number {
     return Math.min(Math.max(0, val), 6);
   }
@@ -75,6 +81,7 @@ export class Relu6 implements CPUActivationFn {
  * Leaky ReLU activation function f(x) = x if x > 0, 0.01 * x otherwise
  */
 export class LeakyRelu implements CPUActivationFn {
+  name = "leakyrelu";
   activate(val: number): number {
     return val > 0 ? val : 0.01 * val;
   }
@@ -89,6 +96,7 @@ export class LeakyRelu implements CPUActivationFn {
  * This is a rectified linear unit with an exponential output range.
  */
 export class Elu implements CPUActivationFn {
+  name = "elu";
   activate(val: number): number {
     return val >= 0 ? val : Math.exp(val) - 1;
   }
@@ -103,6 +111,7 @@ export class Elu implements CPUActivationFn {
  * This is a scaled version of the Elu function, which is a smoother approximation to the ReLU function.
  */
 export class Selu implements CPUActivationFn {
+  name = "selu";
   activate(val: number): number {
     return val >= 0 ? val : 1.0507 * (Math.exp(val) - 1);
   }

src/cpu/backend.ts → backends/cpu/backend.ts

@@ -1,4 +1,5 @@
 import type { DataTypeArray } from "../../deps.ts";
+import { ConvLayer, DenseLayer, PoolLayer } from "../../mod.ts";
 import type {
   Backend,
   ConvLayerConfig,
@@ -11,8 +12,8 @@ import type {
   NetworkJSON,
   PoolLayerConfig,
   Size,
-} from "../types.ts";
-import { iterate1D, to1D } from "../util.ts";
+} from "../../core/types.ts";
+import { iterate1D } from "../../core/util.ts";
 import { CPUCostFunction, CrossEntropy, Hinge } from "./cost.ts";
 import { ConvCPULayer } from "./layers/conv.ts";
 import { DenseCPULayer } from "./layers/dense.ts";
@@ -33,7 +34,9 @@ export class CPUBackend implements Backend {
     this.silent = config.silent ?? false;
     config.layers.slice(0, -1).map(this.addLayer.bind(this));
     const output = config.layers[config.layers.length - 1];
-    this.output = new DenseCPULayer(output.config as DenseLayerConfig);
+    this.output = output.load
+      ? DenseCPULayer.fromJSON(output.data!)
+      : new DenseCPULayer(output.config as DenseLayerConfig);
     this.setCost(config.cost);
   }
 
@@ -54,13 +57,25 @@ export class CPUBackend implements Backend {
   addLayer(layer: Layer): void {
     switch (layer.type) {
       case "dense":
-        this.layers.push(new DenseCPULayer(layer.config as DenseLayerConfig));
+        this.layers.push(
+          layer.load
+            ? DenseCPULayer.fromJSON(layer.data!)
+            : new DenseCPULayer(layer.config as DenseLayerConfig),
+        );
         break;
       case "conv":
-        this.layers.push(new ConvCPULayer(layer.config as ConvLayerConfig));
+        this.layers.push(
+          layer.load
+            ? ConvCPULayer.fromJSON(layer.data!)
+            : new ConvCPULayer(layer.config as ConvLayerConfig),
+        );
         break;
       case "pool":
-        this.layers.push(new PoolCPULayer(layer.config as PoolLayerConfig));
+        this.layers.push(
+          layer.load
+            ? PoolCPULayer.fromJSON(layer.data!)
+            : new PoolCPULayer(layer.config as PoolLayerConfig),
+        );
         break;
       default:
         throw new Error(
@@ -114,29 +129,20 @@ export class CPUBackend implements Backend {
 
   train(
     datasets: DataSet[],
-    epochs: number,
-    batches: number,
-    rate: number,
+    epochs = 5000,
+    batches = 1,
+    rate = 0.1,
   ): void {
-    const inputSize = this.input || datasets[0].inputs.length / batches;
+
+    batches = datasets[0].inputs.y || batches;
+    const inputSize = datasets[0].inputs.x || this.input;
 
     this.initialize(inputSize, batches);
 
-    if (!(datasets[0].inputs as DataTypeArray).BYTES_PER_ELEMENT) {
-      for (const dataset of datasets) {
-        dataset.inputs = new Float32Array(dataset.inputs);
-        dataset.outputs = new Float32Array(dataset.outputs);
-      }
-    }
     iterate1D(epochs, (e: number) => {
       if (!this.silent) console.log(`Epoch ${e + 1}/${epochs}`);
       for (const dataset of datasets) {
-        const input = new CPUMatrix(
-          dataset.inputs as DataTypeArray,
-          to1D(inputSize),
-          batches,
-        );
-        this.feedForward(input);
+        this.feedForward(dataset.inputs);
         this.backpropagate(dataset.outputs as DataTypeArray, rate);
       }
     });
@@ -168,6 +174,7 @@ export class CPUBackend implements Backend {
 
   toJSON(): NetworkJSON {
     return {
+      costFn: this.costFn.name,
       type: "NeuralNetwork",
       sizes: this.layers.map((layer) => layer.outputSize),
       input: this.input,
@@ -176,6 +183,32 @@ export class CPUBackend implements Backend {
     };
   }
 
+  static fromJSON(data: NetworkJSON): CPUBackend {
+    const layers = data.layers.map((layer) => {
+      switch (layer.type) {
+        case "dense":
+          return DenseLayer.fromJSON(layer);
+        case "conv":
+          return ConvLayer.fromJSON(layer);
+        case "pool":
+          return PoolLayer.fromJSON(layer);
+        default:
+          throw new Error(
+            `${
+              layer.type.charAt(0).toUpperCase() + layer.type.slice(1)
+            }Layer not implemented for the CPU backend`,
+          );
+      }
+    });
+    layers.push(DenseLayer.fromJSON(data.output));
+    const backend = new CPUBackend({
+      input: data.input,
+      layers,
+      cost: data.costFn! as Cost,
+    });
+    return backend;
+  }
+
   save(_str: string): void {
     throw new Error("Not implemented");
   }

src/cpu/cost.ts → backends/cpu/cost.ts

@@ -1,7 +1,8 @@
 import { DataType, DataTypeArray } from "../../deps.ts";
-import { iterate1D } from "../util.ts";
+import { iterate1D } from "../../core/util.ts";
 
 export interface CPUCostFunction<T extends DataType = DataType> {
+  name: string;
   /** Return the cost associated with an output `a` and desired output `y`. */
   cost(yHat: DataTypeArray<T>, y: DataTypeArray<T>): number;
 
@@ -14,6 +15,7 @@ export interface CPUCostFunction<T extends DataType = DataType> {
  */
 export class CrossEntropy<T extends DataType = DataType>
   implements CPUCostFunction {
+  name = "crossentropy";
   cost(yHat: DataTypeArray<T>, y: DataTypeArray<T>) {
     let sum = 0;
     iterate1D(yHat.length, (i: number) => {
@@ -31,6 +33,7 @@ export class CrossEntropy<T extends DataType = DataType>
  * Hinge cost function is the standard cost function for multiclass classification.
  */
 export class Hinge<T extends DataType = DataType> implements CPUCostFunction {
+  name = "hinge";
   cost(yHat: DataTypeArray<T>, y: DataTypeArray<T>) {
     let max = -Infinity;
     iterate1D(yHat.length, (i: number) => {