diff --git a/.gitignore b/.gitignore index f0139c5..6658ca8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ .idea/ build/ xor_model.bin +esr.bin node_modules/ +output.png digit_model.bin -bench/tfjs/node_modules \ No newline at end of file +package-lock.json +test.ts \ No newline at end of file diff --git a/README.md b/README.md index e50ee16..9799e38 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,12 @@ ### Usage ```typescript -import { DenseLayer, NeuralNetwork } from "https://deno.land/x/netsaur/mod.ts"; +import { + DenseLayer, + NeuralNetwork, + tensor1D, + tensor2D, +} from "https://deno.land/x/netsaur/mod.ts"; const net = new NeuralNetwork({ silent: true, @@ -40,11 +45,17 @@ const net = new NeuralNetwork({ await net.train( [ - { inputs: [0, 0, 1, 0, 0, 1, 1, 1], outputs: [0, 1, 1, 0] }, + { + inputs: await tensor2D([ + [0, 0], + [1, 0], + [0, 1], + [1, 1], + ]), + outputs: await tensor1D([0, 1, 1, 0]), + }, ], 5000, - 4, - 0.1, ); console.log(await net.predict(new Float32Array([0, 0]))); @@ -57,7 +68,10 @@ console.log(await net.predict(new Float32Array([1, 1]))); ```typescript import { DenseLayer, NeuralNetwork } from "https://deno.land/x/netsaur/mod.ts"; -import { Matrix, Native } from "https://deno.land/x/netsaur/backends/native.ts"; +import { + Matrix, + Native, +} from "https://deno.land/x/netsaur/backends/native/mod.ts"; const network = await new NeuralNetwork({ input: 2, @@ -95,3 +109,54 @@ console.log( ), ); ``` + +### Saving Models + +```typescript +import { + DenseLayer, + NeuralNetwork, + tensor1D, + tensor2D, +} from "https://deno.land/x/netsaur/mod.ts"; +import { Model } from "https://deno.land/x/netsaur/model/mod.ts"; + +const net = new NeuralNetwork({ + silent: true, + layers: [ + new DenseLayer({ size: 3, activation: "sigmoid" }), + new DenseLayer({ size: 1, activation: "sigmoid" }), + ], + cost: "crossentropy", +}); + +await net.train( + [ + { + inputs: await tensor2D([ + [0, 0], + [1, 0], + [0, 1], + [1, 1], + ]), + outputs: await tensor1D([0, 1, 1, 0]), + }, + ], + 5000, +); + +await Model.save("./network.json", net); +``` + +### Loading & Running Models + +```typescript +import { Model } from "https://deno.land/x/netsaur/model/mod.ts"; + +const net = await Model.load("./network.json"); + +console.log(await net.predict(new Float32Array([0, 0]))); +console.log(await net.predict(new Float32Array([1, 0]))); +console.log(await net.predict(new Float32Array([0, 1]))); +console.log(await net.predict(new Float32Array([1, 1]))); +``` diff --git a/backends/cpu.ts b/backends/cpu.ts deleted file mode 100644 index 500bad9..0000000 --- a/backends/cpu.ts +++ /dev/null @@ -1 +0,0 @@ -export { CPU } from "../src/cpu/mod.ts"; diff --git a/src/cpu/activation.ts b/backends/cpu/activation.ts similarity index 94% rename from src/cpu/activation.ts rename to backends/cpu/activation.ts index 12a6226..d000ed3 100644 --- a/src/cpu/activation.ts +++ b/backends/cpu/activation.ts @@ -1,4 +1,5 @@ export interface CPUActivationFn { + name: string; activate(val: number): number; prime(val: number, error?: number): number; } @@ -7,6 +8,7 @@ export interface CPUActivationFn { * Linear activation function f(x) = x */ export class Linear implements CPUActivationFn { + name = "linear"; activate(val: number): number { return val; } @@ -20,6 +22,7 @@ export class Linear implements CPUActivationFn { * Sigmoid activation function f(x) = 1 / (1 + e^(-x)) */ export class Sigmoid implements CPUActivationFn { + name = "sigmoid"; activate(val: number): number { return 1 / (1 + Math.exp(-val)); } @@ -34,6 +37,7 @@ export class Sigmoid implements CPUActivationFn { * This is the same as the sigmoid function, but is more robust to outliers */ export class Tanh implements CPUActivationFn { + name = "tanh"; activate(val: number): number { return Math.tanh(val); } @@ -48,6 +52,7 @@ export class Tanh implements CPUActivationFn { * This is a rectified linear unit, which is a smooth approximation to the sigmoid function. */ export class Relu implements CPUActivationFn { + name = "relu"; activate(val: number): number { return Math.max(0, val); } @@ -62,6 +67,7 @@ export class Relu implements CPUActivationFn { * This is a rectified linear unit with a 6-value output range. */ export class Relu6 implements CPUActivationFn { + name = "relu6"; activate(val: number): number { return Math.min(Math.max(0, val), 6); } @@ -75,6 +81,7 @@ export class Relu6 implements CPUActivationFn { * Leaky ReLU activation function f(x) = x if x > 0, 0.01 * x otherwise */ export class LeakyRelu implements CPUActivationFn { + name = "leakyrelu"; activate(val: number): number { return val > 0 ? val : 0.01 * val; } @@ -89,6 +96,7 @@ export class LeakyRelu implements CPUActivationFn { * This is a rectified linear unit with an exponential output range. */ export class Elu implements CPUActivationFn { + name = "elu"; activate(val: number): number { return val >= 0 ? val : Math.exp(val) - 1; } @@ -103,6 +111,7 @@ export class Elu implements CPUActivationFn { * This is a scaled version of the Elu function, which is a smoother approximation to the ReLU function. */ export class Selu implements CPUActivationFn { + name = "selu"; activate(val: number): number { return val >= 0 ? val : 1.0507 * (Math.exp(val) - 1); } diff --git a/src/cpu/backend.ts b/backends/cpu/backend.ts similarity index 71% rename from src/cpu/backend.ts rename to backends/cpu/backend.ts index ba13b2c..3cf797f 100644 --- a/src/cpu/backend.ts +++ b/backends/cpu/backend.ts @@ -1,4 +1,5 @@ import type { DataTypeArray } from "../../deps.ts"; +import { ConvLayer, DenseLayer, PoolLayer } from "../../mod.ts"; import type { Backend, ConvLayerConfig, @@ -11,8 +12,8 @@ import type { NetworkJSON, PoolLayerConfig, Size, -} from "../types.ts"; -import { iterate1D, to1D } from "../util.ts"; +} from "../../core/types.ts"; +import { iterate1D } from "../../core/util.ts"; import { CPUCostFunction, CrossEntropy, Hinge } from "./cost.ts"; import { ConvCPULayer } from "./layers/conv.ts"; import { DenseCPULayer } from "./layers/dense.ts"; @@ -33,7 +34,9 @@ export class CPUBackend implements Backend { this.silent = config.silent ?? false; config.layers.slice(0, -1).map(this.addLayer.bind(this)); const output = config.layers[config.layers.length - 1]; - this.output = new DenseCPULayer(output.config as DenseLayerConfig); + this.output = output.load + ? DenseCPULayer.fromJSON(output.data!) + : new DenseCPULayer(output.config as DenseLayerConfig); this.setCost(config.cost); } @@ -54,13 +57,25 @@ export class CPUBackend implements Backend { addLayer(layer: Layer): void { switch (layer.type) { case "dense": - this.layers.push(new DenseCPULayer(layer.config as DenseLayerConfig)); + this.layers.push( + layer.load + ? DenseCPULayer.fromJSON(layer.data!) + : new DenseCPULayer(layer.config as DenseLayerConfig), + ); break; case "conv": - this.layers.push(new ConvCPULayer(layer.config as ConvLayerConfig)); + this.layers.push( + layer.load + ? ConvCPULayer.fromJSON(layer.data!) + : new ConvCPULayer(layer.config as ConvLayerConfig), + ); break; case "pool": - this.layers.push(new PoolCPULayer(layer.config as PoolLayerConfig)); + this.layers.push( + layer.load + ? PoolCPULayer.fromJSON(layer.data!) + : new PoolCPULayer(layer.config as PoolLayerConfig), + ); break; default: throw new Error( @@ -114,29 +129,20 @@ export class CPUBackend implements Backend { train( datasets: DataSet[], - epochs: number, - batches: number, - rate: number, + epochs = 5000, + batches = 1, + rate = 0.1, ): void { - const inputSize = this.input || datasets[0].inputs.length / batches; + + batches = datasets[0].inputs.y || batches; + const inputSize = datasets[0].inputs.x || this.input; this.initialize(inputSize, batches); - if (!(datasets[0].inputs as DataTypeArray).BYTES_PER_ELEMENT) { - for (const dataset of datasets) { - dataset.inputs = new Float32Array(dataset.inputs); - dataset.outputs = new Float32Array(dataset.outputs); - } - } iterate1D(epochs, (e: number) => { if (!this.silent) console.log(`Epoch ${e + 1}/${epochs}`); for (const dataset of datasets) { - const input = new CPUMatrix( - dataset.inputs as DataTypeArray, - to1D(inputSize), - batches, - ); - this.feedForward(input); + this.feedForward(dataset.inputs); this.backpropagate(dataset.outputs as DataTypeArray, rate); } }); @@ -168,6 +174,7 @@ export class CPUBackend implements Backend { toJSON(): NetworkJSON { return { + costFn: this.costFn.name, type: "NeuralNetwork", sizes: this.layers.map((layer) => layer.outputSize), input: this.input, @@ -176,6 +183,32 @@ export class CPUBackend implements Backend { }; } + static fromJSON(data: NetworkJSON): CPUBackend { + const layers = data.layers.map((layer) => { + switch (layer.type) { + case "dense": + return DenseLayer.fromJSON(layer); + case "conv": + return ConvLayer.fromJSON(layer); + case "pool": + return PoolLayer.fromJSON(layer); + default: + throw new Error( + `${ + layer.type.charAt(0).toUpperCase() + layer.type.slice(1) + }Layer not implemented for the CPU backend`, + ); + } + }); + layers.push(DenseLayer.fromJSON(data.output)); + const backend = new CPUBackend({ + input: data.input, + layers, + cost: data.costFn! as Cost, + }); + return backend; + } + save(_str: string): void { throw new Error("Not implemented"); } diff --git a/src/cpu/cost.ts b/backends/cpu/cost.ts similarity index 92% rename from src/cpu/cost.ts rename to backends/cpu/cost.ts index 00359a3..33163b0 100644 --- a/src/cpu/cost.ts +++ b/backends/cpu/cost.ts @@ -1,7 +1,8 @@ import { DataType, DataTypeArray } from "../../deps.ts"; -import { iterate1D } from "../util.ts"; +import { iterate1D } from "../../core/util.ts"; export interface CPUCostFunction { + name: string; /** Return the cost associated with an output `a` and desired output `y`. */ cost(yHat: DataTypeArray, y: DataTypeArray): number; @@ -14,6 +15,7 @@ export interface CPUCostFunction { */ export class CrossEntropy implements CPUCostFunction { + name = "crossentropy"; cost(yHat: DataTypeArray, y: DataTypeArray) { let sum = 0; iterate1D(yHat.length, (i: number) => { @@ -31,6 +33,7 @@ export class CrossEntropy * Hinge cost function is the standard cost function for multiclass classification. */ export class Hinge implements CPUCostFunction { + name = "hinge"; cost(yHat: DataTypeArray, y: DataTypeArray) { let max = -Infinity; iterate1D(yHat.length, (i: number) => { diff --git a/backends/cpu/layers/conv.ts b/backends/cpu/layers/conv.ts new file mode 100644 index 0000000..45a2b73 --- /dev/null +++ b/backends/cpu/layers/conv.ts @@ -0,0 +1,155 @@ +import { + Activation, + ConvLayerConfig, + LayerJSON, + MatrixJSON, + Size, + Size2D, +} from "../../../core/types.ts"; +import { ActivationError, iterate2D, to2D } from "../../../core/util.ts"; +import { + CPUActivationFn, + Elu, + LeakyRelu, + Linear, + Relu, + Relu6, + Selu, + Sigmoid, + Tanh, +} from "../activation.ts"; +import { CPUMatrix } from "../matrix.ts"; + +// https://github.com/mnielsen/neural-networks-and-deep-learning +// https://ml-cheatsheet.readthedocs.io/en/latest/backpropagation.html#applying-the-chain-rule + +/** + * Convolutional 2D layer. + */ +export class ConvCPULayer { + outputSize!: Size2D; + padding: number; + strides: Size2D; + activationFn: CPUActivationFn = new Sigmoid(); + + input!: CPUMatrix; + kernel!: CPUMatrix; + padded!: CPUMatrix; + output!: CPUMatrix; + + constructor(config: ConvLayerConfig) { + this.kernel = new CPUMatrix( + config.kernel, + config.kernelSize.x, + config.kernelSize.y, + ); + this.padding = config.padding || 0; + this.strides = to2D(config.strides); + this.setActivation(config.activation ?? "linear"); + } + + reset(_batches: number) { + } + + initialize(inputSize: Size, _batches: number) { + const wp = (inputSize as Size2D).x + 2 * this.padding; + const hp = (inputSize as Size2D).y + 2 * this.padding; + if (this.padding > 0) { + this.padded = CPUMatrix.with(wp, hp); + this.padded.fill(255); + } + const wo = 1 + Math.floor((wp - this.kernel.x) / this.strides.x); + const ho = 1 + Math.floor((hp - this.kernel.y) / this.strides.y); + this.output = CPUMatrix.with(wo, ho); + this.outputSize = { x: wo, y: ho }; + } + + setActivation(activation: Activation) { + switch (activation) { + case "sigmoid": + this.activationFn = new Sigmoid(); + break; + case "leakyrelu": + this.activationFn = new LeakyRelu(); + break; + case "tanh": + this.activationFn = new Tanh(); + break; + case "relu": + this.activationFn = new Relu(); + break; + case "relu6": + this.activationFn = new Relu6(); + break; + case "elu": + this.activationFn = new Elu(); + break; + case "selu": + this.activationFn = new Selu(); + break; + case "linear": + this.activationFn = new Linear(); + break; + default: + throw new ActivationError(activation); + } + } + + feedForward(input: CPUMatrix): CPUMatrix { + if (this.padding > 0) { + iterate2D(input, (i: number, j: number) => { + const idx = this.padded.x * (this.padding + j) + this.padding + i; + this.padded.data[idx] = input.data[j * input.x + i]; + }); + } else { + this.padded = input; + } + iterate2D(this.output, (i: number, j: number) => { + let sum = 0; + iterate2D(this.kernel, (x: number, y: number) => { + const k = this.padded.x * (j * this.strides.y + y) + + (i * this.strides.x + x); + const l = y * this.kernel.x + x; + sum += this.padded.data[k] * this.kernel.data[l]; + }); + this.output.data[j * this.output.x + i] = this.activationFn.activate(sum); + }); + return this.output; + } + + backPropagate(_error: CPUMatrix, _rate: number) { + } + + toJSON(): LayerJSON { + return { + outputSize: this.outputSize, + type: "conv", + input: this.input.toJSON(), + kernel: this.kernel.toJSON(), + padded: this.padded.toJSON(), + output: this.output.toJSON(), + strides: this.strides, + padding: this.padding, + }; + } + + static fromJSON( + { outputSize, input, kernel, padded, output, strides, padding }: LayerJSON, + ): ConvCPULayer { + const layer = new ConvCPULayer({ + kernel: (kernel as MatrixJSON).data, + kernelSize: { x: (kernel as MatrixJSON).x, y: (kernel as MatrixJSON).y }, + padding, + strides, + }); + layer.input = new CPUMatrix(input.data, input.x, input.y); + layer.outputSize = outputSize as Size2D; + layer.padded = new CPUMatrix( + (padded as MatrixJSON).data, + (padded as MatrixJSON).x, + (padded as MatrixJSON).y, + ); + layer.output = new CPUMatrix(output.data, output.x, output.y); + return layer; + } +} diff --git a/src/cpu/layers/dense.ts b/backends/cpu/layers/dense.ts similarity index 69% rename from src/cpu/layers/dense.ts rename to backends/cpu/layers/dense.ts index b225b0d..878166f 100644 --- a/src/cpu/layers/dense.ts +++ b/backends/cpu/layers/dense.ts @@ -1,5 +1,11 @@ -import { Activation, DenseLayerConfig, LayerJSON, Size } from "../../types.ts"; -import { ActivationError, to1D } from "../../util.ts"; +import { + Activation, + DenseLayerConfig, + LayerJSON, + MatrixJSON, + Size, +} from "../../../core/types.ts"; +import { ActivationError, to1D } from "../../../core/util.ts"; import { CPUActivationFn, Elu, @@ -11,6 +17,7 @@ import { Sigmoid, Tanh, } from "../activation.ts"; +import { CPUCostFunction, CrossEntropy } from "../cost.ts"; import { CPUMatrix } from "../matrix.ts"; // https://github.com/mnielsen/neural-networks-and-deep-learning @@ -22,6 +29,7 @@ import { CPUMatrix } from "../matrix.ts"; export class DenseCPULayer { outputSize: number; activationFn: CPUActivationFn = new Sigmoid(); + costFunction: CPUCostFunction = new CrossEntropy(); input!: CPUMatrix; weights!: CPUMatrix; @@ -87,7 +95,7 @@ export class DenseCPULayer { return this.output; } - backPropagate(error: CPUMatrix, rate: number) { + backPropagate(error: CPUMatrix, rate: number, _costFn: CPUCostFunction = this.costFunction,) { const cost = CPUMatrix.with(error.x, error.y); for (const i in cost.data) { const activation = this.activationFn.prime(this.output.data[i]); @@ -106,8 +114,35 @@ export class DenseCPULayer { toJSON(): LayerJSON { return { outputSize: this.outputSize, - activation: this.activationFn, + activationFn: this.activationFn.name, + costFn: this.costFunction.name, type: "dense", + input: this.input.toJSON(), + weights: this.weights.toJSON(), + biases: this.biases.toJSON(), + output: this.output.toJSON(), }; } + + static fromJSON( + { outputSize, activationFn, input, weights, biases, output }: LayerJSON, + ): DenseCPULayer { + const layer = new DenseCPULayer({ + size: outputSize, + activation: (activationFn as Activation) || "sigmoid", + }); + layer.input = new CPUMatrix(input.data, input.x, input.y); + layer.weights = new CPUMatrix( + (weights as MatrixJSON).data, + (weights as MatrixJSON).x, + (weights as MatrixJSON).y, + ); + layer.biases = new CPUMatrix( + (biases as MatrixJSON).data, + (biases as MatrixJSON).x, + (biases as MatrixJSON).y, + ); + layer.output = new CPUMatrix(output.data, output.x, output.y); + return layer; + } } diff --git a/backends/cpu/layers/pool.ts b/backends/cpu/layers/pool.ts new file mode 100644 index 0000000..5774635 --- /dev/null +++ b/backends/cpu/layers/pool.ts @@ -0,0 +1,78 @@ +import { + LayerJSON, + PoolLayerConfig, + Size, + Size2D, +} from "../../../core/types.ts"; +import { average, to2D } from "../../../core/util.ts"; +import { CPUMatrix } from "../matrix.ts"; + +// https://github.com/mnielsen/neural-networks-and-deep-learning +// https://ml-cheatsheet.readthedocs.io/en/latest/backpropagation.html#applying-the-chain-rule + +/** + * Pooling layer. + */ +export class PoolCPULayer { + outputSize!: Size2D; + strides: Size2D; + mode: "max" | "avg"; + input!: CPUMatrix; + output!: CPUMatrix; + + constructor(config: PoolLayerConfig) { + this.strides = to2D(config.strides); + this.mode = config.mode ?? "max"; + } + + reset(_batches: number) { + } + + initialize(inputSize: Size, _batches: number) { + const w = (inputSize as Size2D).x / this.strides.x; + const h = (inputSize as Size2D).y / this.strides.y; + this.output = CPUMatrix.with(w, h); + this.outputSize = { x: w, y: h }; + } + + feedForward(input: CPUMatrix) { + for (let i = 0; i < this.output.x; i++) { + for (let j = 0; j < this.output.y; j++) { + const pool = []; + for (let x = 0; x < this.strides.x; x++) { + for (let y = 0; y < this.strides.y; y++) { + const idx = (j * this.strides.y + y) * input.x + + i * this.strides.x + x; + pool.push(input.data[idx]); + } + } + this.output.data[j * this.output.x + i] = (this.mode === "max" ? Math.max : average)(...pool); + } + } + return this.output; + } + + backPropagate(_error: CPUMatrix, _rate: number) { + } + + toJSON(): LayerJSON { + return { + outputSize: this.outputSize, + type: "pool", + input: this.input.toJSON(), + output: this.output.toJSON(), + strides: this.strides, + mode: this.mode, + }; + } + + static fromJSON( + { outputSize, input, output, strides, mode }: LayerJSON, + ): PoolCPULayer { + const layer = new PoolCPULayer({ strides, mode }); + layer.input = new CPUMatrix(input.data, input.x, input.y); + layer.outputSize = outputSize as Size2D; + layer.output = new CPUMatrix(output.data, output.x, output.y); + return layer; + } +} diff --git a/src/cpu/matrix.ts b/backends/cpu/matrix.ts similarity index 98% rename from src/cpu/matrix.ts rename to backends/cpu/matrix.ts index 6f7b49c..1103b84 100644 --- a/src/cpu/matrix.ts +++ b/backends/cpu/matrix.ts @@ -1,5 +1,5 @@ import { DataType, DataTypeArray } from "../../deps.ts"; -import { iterate1D, iterate2D } from "../util.ts"; +import { iterate1D, iterate2D } from "../../core/util.ts"; export class CPUMatrix { deltas: DataTypeArray; constructor( diff --git a/backends/cpu/mod.ts b/backends/cpu/mod.ts new file mode 100644 index 0000000..44b0066 --- /dev/null +++ b/backends/cpu/mod.ts @@ -0,0 +1,16 @@ +import { Backend, NetworkConfig, NetworkJSON } from "../../core/types.ts"; + +import { CPUBackend } from "./backend.ts"; +import { TensorCPUBackend } from "./tensor.ts"; + + +export const CPU = { + // deno-lint-ignore require-await + backend: async (config: NetworkConfig): Promise => new CPUBackend(config), + // deno-lint-ignore require-await + model: async (data: NetworkJSON, _silent=false): Promise => CPUBackend.fromJSON(data), + tensor: () => new TensorCPUBackend() +} +export { CPUBackend }; +export * from "./matrix.ts"; +export type { DataSet } from "../../core/types.ts"; \ No newline at end of file diff --git a/backends/cpu/tensor.ts b/backends/cpu/tensor.ts new file mode 100644 index 0000000..2d84cf4 --- /dev/null +++ b/backends/cpu/tensor.ts @@ -0,0 +1,13 @@ +import { TensorBackend, TensorLike, Tensor2DCPU, TypedArray } from "../../core/types.ts"; +import { flatten } from "../../core/util.ts"; +import { CPUMatrix } from "./matrix.ts" + + +export class TensorCPUBackend implements TensorBackend{ + tensor2D(values: TensorLike, width: number, height: number): Tensor2DCPU { + return new CPUMatrix(new Float32Array(flatten(values as TypedArray) as ArrayBufferLike), width, height) + } + tensor1D(values: TensorLike): Float32Array { + return new Float32Array(values as TypedArray); + } +} \ No newline at end of file diff --git a/backends/gpu.ts b/backends/gpu.ts deleted file mode 100644 index 0aaffc1..0000000 --- a/backends/gpu.ts +++ /dev/null @@ -1 +0,0 @@ -export { GPU } from "../src/gpu/mod.ts"; diff --git a/src/gpu/activation.ts b/backends/gpu/activation.ts similarity index 95% rename from src/gpu/activation.ts rename to backends/gpu/activation.ts index 603ddc0..340de81 100644 --- a/src/gpu/activation.ts +++ b/backends/gpu/activation.ts @@ -1,4 +1,5 @@ export interface GPUActivationFn { + name: string; activate(type: string): string; prime(type: string): string; } @@ -7,6 +8,7 @@ export interface GPUActivationFn { * Linear activation function f(x) = x */ export class Linear implements GPUActivationFn { + name = "linear"; activate(_: string): string { return `return weighted_sum`; } @@ -20,6 +22,7 @@ export class Linear implements GPUActivationFn { * Sigmoid activation function f(x) = 1 / (1 + e^(-x)) */ export class Sigmoid implements GPUActivationFn { + name = "sigmoid"; activate(type: string): string { return `return ${type}(1) / (${type}(1) + exp(-weighted_sum))`; } @@ -34,6 +37,7 @@ export class Sigmoid implements GPUActivationFn { * This is the same as the sigmoid function, but is more robust to outliers */ export class Tanh implements GPUActivationFn { + name = "tanh"; activate(_: string): string { return `return tanh(weighted_sum)`; } @@ -48,6 +52,7 @@ export class Tanh implements GPUActivationFn { * This is a rectified linear unit, which is a smooth approximation to the sigmoid function. */ export class Relu implements GPUActivationFn { + name = "relu"; activate(type: string): string { return `return max(${type}(0), weighted_sum)`; } @@ -65,6 +70,7 @@ export class Relu implements GPUActivationFn { * This is a rectified linear unit with a 6-value output range. */ export class Relu6 implements GPUActivationFn { + name = "relu6"; activate(type: string): string { return `return min(max(${type}(0), weighted_sum), ${type}(6))`; } @@ -84,6 +90,7 @@ export class Relu6 implements GPUActivationFn { * Leaky ReLU activation function f(x) = x if x > 0, 0.01 * x otherwise */ export class LeakyRelu implements GPUActivationFn { + name = "leakyrelu"; activate(type: string): string { return `if (weighted_sum > ${type}(0)) { return weighted_sum; @@ -104,6 +111,7 @@ export class LeakyRelu implements GPUActivationFn { * This is a rectified linear unit with an exponential output range. */ export class Elu implements GPUActivationFn { + name = "elu"; activate(type: string): string { return `if (weighted_sum > ${type}(0)) { return weighted_sum; @@ -124,6 +132,7 @@ export class Elu implements GPUActivationFn { * This is a scaled version of the Elu function, which is a smoother approximation to the ReLU function. */ export class Selu implements GPUActivationFn { + name = "selu"; activate(type: string): string { return `return ${type}(weighted_sum) + ${type}(weighted_sum) * (${type}(1) - ${type}(weighted_sum)) * ${type}(1.67326)`; } diff --git a/src/gpu/backend.ts b/backends/gpu/backend.ts similarity index 60% rename from src/gpu/backend.ts rename to backends/gpu/backend.ts index da00136..d7fb53f 100644 --- a/src/gpu/backend.ts +++ b/backends/gpu/backend.ts @@ -8,7 +8,7 @@ import type { NetworkConfig, NetworkJSON, Size, -} from "../types.ts"; +} from "../../core/types.ts"; import { CrossEntropy, GPUCostFunction, Hinge } from "./cost.ts"; import { DataType, @@ -16,9 +16,9 @@ import { WebGPUBackend, WebGPUData, } from "../../deps.ts"; -import { fromType, getType, to1D } from "../util.ts"; +import { getType } from "../../core/util.ts"; import { GPUMatrix } from "./matrix.ts"; -import { DenseLayer } from "../mod.ts"; +import { DenseLayer } from "../../mod.ts"; export class GPUBackend implements Backend { input?: Size; @@ -27,6 +27,7 @@ export class GPUBackend implements Backend { silent: boolean; costFn: GPUCostFunction = new CrossEntropy(); backend: WebGPUBackend; + imported = false; constructor(config: NetworkConfig, backend: WebGPUBackend) { this.backend = backend; @@ -69,17 +70,24 @@ export class GPUBackend implements Backend { } } - async initialize(type: DataType, inputSize: Size, batches: number) { - await this.layers[0].initialize(type, inputSize, batches); - + async initialize(inputSize: Size, batches: number, type: DataType) { + await this.layers[0].initialize( inputSize, batches, type); for (let i = 1; i < this.layers.length; i++) { const current = this.layers[i]; const previous = this.layers[i - 1]; - await current.initialize(type, previous.outputSize, batches); + await current.initialize( + previous.outputSize, + batches, + type, + ); } const lastLayer = this.layers[this.layers.length - 1]; - await this.output.initialize(type, lastLayer.outputSize, batches); + await this.output.initialize( + lastLayer.outputSize, + batches, + type, + ); } async feedForward(input: GPUMatrix) { @@ -91,9 +99,15 @@ export class GPUBackend implements Backend { } async backpropagate(output: GPUMatrix, rate: number) { - await this.output.backPropagate(output, output, rate, 0, this.costFn); + await this.output.backPropagate( + output, + output, + rate, + 0, + this.costFn, + ); // todo: update for convolutional layer - let weights = (this.output as DenseGPULayer).weights; + let weights = this.output.weights; let last = 1; for (let i = this.layers.length - 1; i >= 0; i--) { await this.layers[i].backPropagate( @@ -103,42 +117,35 @@ export class GPUBackend implements Backend { last, ); // todo: update for convolutional layer - weights = (this.layers[i] as DenseGPULayer).weights; + weights = this.layers[i].weights; last++; } } async train( datasets: DataSet[], - epochs: number, - batches: number, - rate: number, + epochs = 5000, + batches = 1, + rate = 0.1, ) { - const type = getType(datasets[0].inputs as DataTypeArray); - const inputSize = this.input || datasets[0].inputs.length / batches; + const type = (datasets[0].inputs as GPUMatrix).type; + batches = (datasets[0].inputs as GPUMatrix).y || batches; + const inputSize = (datasets[0].inputs as GPUMatrix).x || this.input; const outputSize = datasets[0].outputs.length / batches; - await this.initialize(type, inputSize, batches); + + await this.initialize(inputSize!, batches, type); const databuffers = []; for (const dataset of datasets) { - const inputArray = new (fromType(type))(dataset.inputs); - const outputArray = new (fromType(type))(dataset.outputs); - - const input = await GPUMatrix.from( - this.backend, - inputArray, - to1D(inputSize), - batches, - ); + // const outputArray = new (fromType(type))(); const output = await GPUMatrix.from( this.backend, - outputArray, + dataset.outputs, outputSize, batches, ); - - databuffers.push({ input, output }); + databuffers.push({ input: dataset.inputs, output }); } for (let e = 0; e < epochs; e++) { @@ -159,23 +166,70 @@ export class GPUBackend implements Backend { const type = getType(data); const gpuData = await WebGPUData.from(this.backend, data); const input = new GPUMatrix(gpuData, gpuData.length, 1, type); - for (const layer of this.layers) { + this.layers.forEach(async (layer) => { await layer.reset(type, 1); - } + }); + await this.output.reset(type, 1); return await (await this.feedForward(input)).data.get(); } - toJSON(): NetworkJSON { + async toJSON(): Promise { + const layers = await Promise.all( + this.layers.map(async (layer) => await layer.toJSON()), + ); return { + costFn: this.costFn.name, type: "NeuralNetwork", sizes: this.layers.map((layer) => layer.outputSize), input: this.input, - layers: this.layers.map((layer) => layer.toJSON()), - output: this.output.toJSON(), + layers, + output: await this.output.toJSON(), }; } + static async fromJSON( + data: NetworkJSON, + backend: WebGPUBackend, + ): Promise { + const layers = data.layers.map((layer) => { + switch (layer.type) { + case "dense": + return DenseLayer.fromJSON(layer); + default: + throw new Error( + `${ + layer.type.charAt(0).toUpperCase() + layer.type.slice(1) + }Layer not implemented for the CPU backend`, + ); + } + }); + layers.push(DenseLayer.fromJSON(data.output)); + const gpubackend = new GPUBackend({ + input: data.input, + layers, + cost: data.costFn! as Cost, + }, backend); + gpubackend.output = await DenseGPULayer.fromJSON( + (layers.at(-1) as DenseLayer)!.data!, + backend, + ); + layers.slice(0, -1).forEach(async (layer) => { + if (layer.type === "dense") { + gpubackend.layers.push( + await DenseGPULayer.fromJSON(layer.data!, backend), + ); + } else { + throw new Error( + `${ + layer.type.charAt(0).toUpperCase() + layer.type.slice(1) + }Layer not implemented for the GPU backend`, + ); + } + }); + return gpubackend; + } + save(_str: string): void { throw new Error("Not implemented"); } diff --git a/src/gpu/cost.ts b/backends/gpu/cost.ts similarity index 95% rename from src/gpu/cost.ts rename to backends/gpu/cost.ts index 1412203..24f0b83 100644 --- a/src/gpu/cost.ts +++ b/backends/gpu/cost.ts @@ -2,6 +2,7 @@ import { WebGPUData } from "../../deps.ts"; export interface GPUCostFunction { + name: string; /** Return the cost associated with an output `a` and desired output `y`. */ cost(type: string): string; @@ -13,6 +14,7 @@ export interface GPUCostFunction { * Cross entropy cost function is the standard cost function for binary classification. */ export class CrossEntropy implements GPUCostFunction { + name = "crossentropy" cost(type: string) { return `var sum: ${type} = ${type}(0); for (var i = ${type}(0); i < yHat.length; i++) { @@ -30,6 +32,7 @@ export class CrossEntropy implements GPUCostFunction { * Hinge cost function is the standard cost function for multiclass classification. */ export class Hinge implements GPUCostFunction { + name = "hinge" cost(type: string) { return `var max: ${type} = ${type}(0); for (var i = ${type}(0); i < yHat.length; i++) { diff --git a/src/gpu/kernels/backpropagate.ts b/backends/gpu/kernels/backpropagate.ts similarity index 100% rename from src/gpu/kernels/backpropagate.ts rename to backends/gpu/kernels/backpropagate.ts diff --git a/src/gpu/kernels/feedforward.ts b/backends/gpu/kernels/feedforward.ts similarity index 100% rename from src/gpu/kernels/feedforward.ts rename to backends/gpu/kernels/feedforward.ts diff --git a/src/gpu/kernels/reduce.ts b/backends/gpu/kernels/reduce.ts similarity index 100% rename from src/gpu/kernels/reduce.ts rename to backends/gpu/kernels/reduce.ts diff --git a/src/gpu/layers/dense.ts b/backends/gpu/layers/dense.ts similarity index 67% rename from src/gpu/layers/dense.ts rename to backends/gpu/layers/dense.ts index e4f5463..50a465c 100644 --- a/src/gpu/layers/dense.ts +++ b/backends/gpu/layers/dense.ts @@ -1,6 +1,12 @@ -import { DataType, WebGPUBackend } from "../../../deps.ts"; -import { Activation, DenseLayerConfig, LayerJSON, Size } from "../../types.ts"; -import { ActivationError, fromType, to1D } from "../../util.ts"; +import { DataType, WebGPUBackend, WebGPUData } from "../../../deps.ts"; +import { + Activation, + DenseLayerConfig, + LayerJSON, + MatrixJSON, + Size, +} from "../../../core/types.ts"; +import { ActivationError, fromType, to1D } from "../../../core/util.ts"; import { Elu, GPUActivationFn, @@ -49,7 +55,7 @@ export class DenseGPULayer { } } - async initialize(type: DataType, inputSize: Size, batches: number) { + async initialize(inputSize: Size, batches: number, type: DataType) { const b = this.#backend; const weights = new (fromType(type))(this.outputSize * to1D(inputSize)) .map(() => Math.random() * 2 - 1); @@ -140,11 +146,57 @@ export class DenseGPULayer { return this.output; } - toJSON(): LayerJSON { + async toJSON(): Promise { + const input = await this.input.toJSON(); + const weights = await this.weights.toJSON(); + const biases = await this.biases.toJSON(); + const output = await this.output.toJSON(); + const error = await this.error.toJSON(); + const cost = await this.cost.toJSON(); + return { outputSize: this.outputSize, - activation: this.activationFn, + activationFn: this.activationFn.name, + costFn: this.costFunction.name, type: "dense", + input, + weights, + biases, + output, + error, + cost, }; } + + static async fromJSON( + { outputSize, activationFn, input, weights, biases, output }: + LayerJSON, + backend: WebGPUBackend, + ): Promise { + const layer = new DenseGPULayer({ + size: outputSize, + activation: (activationFn as Activation) || "sigmoid", + }, backend); + layer.input = new GPUMatrix( + await WebGPUData.from(backend, input.data, "f32"), + input.x, + input.y, + ); + layer.weights = new GPUMatrix( + await WebGPUData.from(backend, (weights as MatrixJSON).data, "f32"), + (weights as MatrixJSON).x, + (weights as MatrixJSON).y, + ); + layer.biases = new GPUMatrix( + await WebGPUData.from(backend, (biases as MatrixJSON).data, "f32"), + (biases as MatrixJSON).x, + (biases as MatrixJSON).y, + ); + layer.output = new GPUMatrix( + await WebGPUData.from(backend, output.data, "f32"), + output.x, + output.y, + ); + return layer; + } } diff --git a/src/gpu/matrix.ts b/backends/gpu/matrix.ts similarity index 82% rename from src/gpu/matrix.ts rename to backends/gpu/matrix.ts index ea72ae5..dd3e84d 100644 --- a/src/gpu/matrix.ts +++ b/backends/gpu/matrix.ts @@ -4,7 +4,8 @@ import { WebGPUBackend, WebGPUData, } from "../../deps.ts"; -import { fromType } from "../util.ts"; +import { MatrixJSON } from "../../core/types.ts"; +import { fromType } from "../../core/util.ts"; export class GPUMatrix { constructor( @@ -35,9 +36,10 @@ export class GPUMatrix { const buf = await WebGPUData.from(backend, data); return new this(buf, x, y, type); } - toJSON() { + async toJSON(): Promise { + const data = await this.data.get(); return { - data: this.data, + data, x: this.x, y: this.y, type: this.type, diff --git a/backends/gpu/mod.ts b/backends/gpu/mod.ts new file mode 100644 index 0000000..30a4f61 --- /dev/null +++ b/backends/gpu/mod.ts @@ -0,0 +1,42 @@ +import { Backend, NetworkConfig, NetworkJSON } from "../../core/types.ts"; + +import { Core, WebGPUBackend } from "../../deps.ts"; +import { GPUBackend } from "./backend.ts"; +import { TensorGPUBackend } from "./tensor.ts"; + + +class GPUInstance { + static core = new Core(); + static backend?: WebGPUBackend; + static initialized = false; + + static async init(silent = false): Promise { + if (GPUInstance.initialized) return; + await GPUInstance.core.initialize(); + GPUInstance.backend = GPUInstance.core.backends.get("webgpu")! as WebGPUBackend; + GPUInstance.initialized = true; + if (!GPUInstance.backend.adapter) throw new Error("No backend adapter found!"); + if (!silent) console.log(`Using adapter: ${GPUInstance.backend.adapter}`); + const features = [...GPUInstance.backend.adapter.features.values()]; + if (!silent) console.log(`Supported features: ${features.join(", ")}`); + } +} +export const GPU = { + backend: async (config: NetworkConfig): Promise => { + await GPUInstance.init(config.silent); + return new GPUBackend(config, GPUInstance.backend!); + }, + model: async (data: NetworkJSON, silent = false): Promise => { + await GPUInstance.init(silent); + const gpubackend = await GPUBackend.fromJSON(data, GPUInstance.backend!); + return gpubackend; + }, + tensor: async () => { + await GPUInstance.init(true); + return new TensorGPUBackend(GPUInstance.backend!); + }, +}; + +export { GPUBackend }; +export * from "./matrix.ts"; +export type { DataSet } from "../../core/types.ts"; diff --git a/backends/gpu/tensor.ts b/backends/gpu/tensor.ts new file mode 100644 index 0000000..dd80728 --- /dev/null +++ b/backends/gpu/tensor.ts @@ -0,0 +1,30 @@ +import { + Tensor2DGPU, + TensorBackend, + TensorLike, + TypedArray, +} from "../../core/types.ts"; +import { flatten } from "../../core/util.ts"; +import { WebGPUBackend } from "../../deps.ts"; +import { GPUMatrix } from "./matrix.ts"; + +export class TensorGPUBackend implements TensorBackend { + constructor(public backend: WebGPUBackend) { + } + async tensor2D( + values: TensorLike, + width: number, + height: number, + ): Promise { + return await GPUMatrix.from( + this.backend, + // deno-lint-ignore no-explicit-any + new Float32Array(flatten(values as TypedArray) as any), + width, + height, + ); + } + tensor1D(values: TensorLike): Float32Array { + return new Float32Array(values as TypedArray); + } +} diff --git a/backends/native.ts b/backends/native.ts deleted file mode 100644 index e4ac90e..0000000 --- a/backends/native.ts +++ /dev/null @@ -1,4 +0,0 @@ -export { Native } from "../src/native/mod.ts"; -export { Matrix } from "../src/native/matrix.ts"; -export type { DataType } from "../src/native/matrix.ts"; -export type { Dataset } from "../src/native/backend.ts"; diff --git a/src/native/backend.ts b/backends/native/backend.ts similarity index 86% rename from src/native/backend.ts rename to backends/native/backend.ts index 0748ba4..a9d5e9e 100644 --- a/src/native/backend.ts +++ b/backends/native/backend.ts @@ -3,13 +3,13 @@ import { DenseLayerConfig, Layer, NetworkConfig, -} from "../types.ts"; -import { to1D } from "../util.ts"; + Size, +} from "../../core/types.ts"; +import { to1D } from "../../core/util.ts"; import ffi, { cstr } from "./ffi.ts"; import { DenseNativeLayer } from "./layers/dense.ts"; import { Matrix } from "./matrix.ts"; - const { network_free, network_create, @@ -32,8 +32,6 @@ const NetworkFinalizer = new FinalizationRegistry( export type NativeLayer = DenseNativeLayer; - - export interface Dataset { inputs: Matrix<"f32">; outputs: Matrix<"f32">; @@ -46,7 +44,10 @@ export class NativeBackend implements Backend { get unsafePointer() { return this.#ptr; } - + get layers() { + // TODO: get layers from backend + return []; + } constructor(config: NetworkConfig | Deno.PointerValue) { this.#ptr = typeof config === "object" ? network_create( @@ -63,7 +64,6 @@ export class NativeBackend implements Backend { } addLayer(_layer: Layer): void { - } encodeLayer(layer: Layer): NativeLayer { @@ -79,11 +79,14 @@ export class NativeBackend implements Backend { } } - predict(input: Matrix<"f32">): Matrix<"f32"> { + initialize(_inputSize: Size, _batches: number): void { + } + + feedForward(input: Matrix<"f32">): Matrix<"f32"> { return new Matrix(network_feed_forward(this.#ptr, input.unsafePointer)); } - train(datasets: Dataset[], epochs: number, learningRate: number) { + train(datasets: Dataset[], epochs = 5000, _batches = 1, rate = 0.1) { const datasetBuffers = datasets.map((e) => new BigUint64Array( [e.inputs.unsafePointer, e.outputs.unsafePointer].map(BigInt), @@ -97,18 +100,22 @@ export class NativeBackend implements Backend { datasets.length, datasetBufferPointers, epochs, - learningRate, + rate, ); } - static load(path: string): NativeBackend { - return new NativeBackend(network_load(cstr(path))); + predict(input: Matrix<"f32">): Matrix<"f32"> { + return new Matrix(network_feed_forward(this.#ptr, input.unsafePointer)); } save(path: string) { network_save(this.#ptr, cstr(path)); } + static load(path: string): NativeBackend { + return new NativeBackend(network_load(cstr(path))); + } + free(): void { if (this.#ptr) { network_free(this.#ptr); diff --git a/src/native/ffi.ts b/backends/native/ffi.ts similarity index 96% rename from src/native/ffi.ts rename to backends/native/ffi.ts index 4a3d800..2aa367a 100644 --- a/src/native/ffi.ts +++ b/backends/native/ffi.ts @@ -1,4 +1,7 @@ -import { dlopen, FetchOptions } from "https://deno.land/x/plug@1.0.0-rc.3/mod.ts" +import { + dlopen, + FetchOptions, +} from "https://deno.land/x/plug@1.0.0-rc.3/mod.ts"; const symbols = { matrix_new: { @@ -169,12 +172,12 @@ const symbols = { const opts: FetchOptions = { name: "netsaur", - url: "https://github.com/denosaurs/netsaur/releases/download/0.1.4/", + url: "https://github.com/denosaurs/netsaur/releases/download/0.1.5/", prefixes: { darwin: "lib", windows: "lib", linux: "lib", - } + }, }; const mod = await dlopen(opts, symbols); diff --git a/backends/native/layers/dense.ts b/backends/native/layers/dense.ts new file mode 100644 index 0000000..b59beb3 --- /dev/null +++ b/backends/native/layers/dense.ts @@ -0,0 +1,32 @@ +import ffi from "../ffi.ts"; + +import { DenseLayerConfig } from "../../../core/types.ts"; +import { to1D } from "../../../core/util.ts"; + +const { + layer_dense, +} = ffi; + +const C_ACTIVATION: { [key: string]: number } = { + "sigmoid": 0, + "tanh": 1, + "relu": 2, + "relu6": 5, + "leakyrelu": 4, + "elu": 6, + "linear": 3, + "selu": 7, +}; +// export type Activation = keyof typeof C_ACTIVATION; + +export class DenseNativeLayer { + #ptr: Deno.PointerValue; + + get unsafePointer() { + return this.#ptr; + } + + constructor(config: DenseLayerConfig) { + this.#ptr = layer_dense(to1D(config.size), C_ACTIVATION[config.activation]); + } +} diff --git a/src/native/matrix.ts b/backends/native/matrix.ts similarity index 100% rename from src/native/matrix.ts rename to backends/native/matrix.ts diff --git a/backends/native/mod.ts b/backends/native/mod.ts new file mode 100644 index 0000000..9107b15 --- /dev/null +++ b/backends/native/mod.ts @@ -0,0 +1,15 @@ +import { Backend, NetworkConfig } from "../../core/types.ts"; + +import { NativeBackend } from "./backend.ts"; +import { TensorNativeBackend } from "./tensor.ts"; + +export const Native = { + // deno-lint-ignore require-await + backend: async (config: NetworkConfig): Promise => + // deno-lint-ignore no-explicit-any + new NativeBackend(config as any), + tensor: () => new TensorNativeBackend(), +}; +export * from "./backend.ts"; +export * from "./matrix.ts"; +export type { DataSet } from "../../core/types.ts"; diff --git a/backends/native/tensor.ts b/backends/native/tensor.ts new file mode 100644 index 0000000..0877ee7 --- /dev/null +++ b/backends/native/tensor.ts @@ -0,0 +1,29 @@ +import { + Tensor2DNative, + TensorBackend, + TensorLike, +} from "../../core/types.ts"; +import { Matrix } from "./matrix.ts"; + +export class TensorNativeBackend implements TensorBackend { + constructor() { + } + tensor2D( + values: TensorLike, + width: number, + height: number, + ): Tensor2DNative { + return new Matrix<"f32">( + height, + width, + new Float32Array((values as number[][]).flat()), + ); + } + tensor1D(values: TensorLike) { + return new Matrix( + (values as number[]).length, + 1, + new Float32Array(values as number[]), + ); + } +} diff --git a/bench/netsaur/xor.ts b/bench/netsaur/xor.ts deleted file mode 100644 index 9870641..0000000 --- a/bench/netsaur/xor.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { NeuralNetwork, DenseLayer } from "../../mod.ts"; -import { Native, Matrix } from "../../backends/native.ts"; - -const start = Date.now(); - -const network = await new NeuralNetwork({ - input: 2, - layers: [ - new DenseLayer({ size: 3, activation: "sigmoid" }), - new DenseLayer({ size: 1, activation: "sigmoid" }), - ], - cost: "crossentropy", -}).setupBackend(Native); - -network.train( - [ - { - inputs: Matrix.of([ - [0, 0], - [0, 1], - [1, 0], - [1, 1], - ]), - outputs: Matrix.column([0, 1, 1, 0]), - }, - ], - 5000, - 0.1, -); - -console.log("training time", Date.now() - start); - -console.log( - network.predict( - Matrix.of([ - [0, 0], - [0, 1], - [1, 0], - [1, 1], - ]), - ), -); diff --git a/bench/tensorflow/xor.py b/bench/tensorflow/xor.py deleted file mode 100644 index 9acb85c..0000000 --- a/bench/tensorflow/xor.py +++ /dev/null @@ -1,36 +0,0 @@ -import tensorflow as tf -import numpy as np -import time - - -start = time.time() - -#tf.get_logger().setLevel('INFO') -#tf.autograph.set_verbosity(3) - -x_train, y_train = (tf.constant([[0,0],[0,1],[1,0],[1,1]], "float32"), tf.constant([[0],[1],[1],[0]], "float32")) -XOR_True = [(1, 0), (0, 1)] -XOR_False = [(0, 0), (1, 1)] - -model = tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(2,)), - tf.keras.layers.Dense(3, activation=tf.nn.sigmoid), # hidden layer - tf.keras.layers.Dense(1, activation=tf.nn.sigmoid) # output layer -]) - -model.compile( - # optimizer='adam', - loss='binary_crossentropy', - #loss='mean_squared_error', # try this too; treat as regression problem - metrics=['accuracy']) - - -model.fit(x_train, y_train, epochs=5000, verbose=0) -print("Training took", time.time() - start, "ms") - -print("XOR True") -for x in XOR_True: - print(model.predict(np.array([x]))) -print("XOR False") -for x in XOR_False: - print(model.predict(np.array([x]))) diff --git a/bench/tfjs/package.json b/bench/tfjs/package.json deleted file mode 100644 index a36cb43..0000000 --- a/bench/tfjs/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "bench", - "version": "1.0.0", - "description": "", - "main": "xor.mjs", - "scripts": { - "start": "node xor.mjs" - }, - "keywords": [], - "author": "", - "license": "ISC", - "dependencies": { - "@tensorflow/tfjs-node": "^3.20.0" - } -} diff --git a/bench/tfjs/xor.mjs b/bench/tfjs/xor.mjs deleted file mode 100644 index b25a66e..0000000 --- a/bench/tfjs/xor.mjs +++ /dev/null @@ -1,27 +0,0 @@ -// deno-lint-ignore-file -import * as tf from '@tensorflow/tfjs-node'; - -const model = tf.sequential(); -model.add(tf.layers.dense({inputShape:[2], units: 3, activation: 'sigmoid'})); -model.add(tf.layers.dense({units: 1, activation: 'sigmoid'})); -model.compile({ loss: "meanSquaredError", optimizer: "sgd" }); - -const xs = tf.tensor2d([ - [0, 0], - [0, 1], - [1, 0], - [1, 1], -]); - -const ys = tf.tensor2d([ - [0], - [1], - [1], - [0], -]); - -const start = Date.now(); -model.fit(xs, ys, {epochs: 5000, verbose:0}).then(() => { - console.log("Training took", Date.now() - start, "ms"); - model.predict(xs).print(); -}); diff --git a/src/mod.ts b/core/mod.ts similarity index 58% rename from src/mod.ts rename to core/mod.ts index 0beca92..b952e16 100644 --- a/src/mod.ts +++ b/core/mod.ts @@ -1,14 +1,12 @@ import { DataTypeArray } from "../deps.ts"; -import { CPUBackend } from "./cpu/backend.ts"; +import { CPUBackend } from "../backends/cpu/backend.ts"; import { - ConvLayerConfig, + Backend, DataSet, - DenseLayerConfig, Layer, - Backend, NetworkConfig, NetworkJSON, - PoolLayerConfig, +Size, } from "./types.ts"; /** @@ -26,12 +24,21 @@ export class NeuralNetwork { /** * setup backend and initialize network */ - async setupBackend(backendLoader: (config: NetworkConfig) => Promise) { - const backend = await backendLoader(this.config); + async setupBackend( + backendLoader: {backend: (config: NetworkConfig) => Promise}, + ) { + const backend = await backendLoader.backend(this.config); this.backend = backend ?? new CPUBackend(this.config); return this; } + /** + * initialize the backend + */ + initialize(inputSize: Size, batches = 1) { + this.backend.initialize(inputSize, batches); + } + /** * add layer to network */ @@ -39,6 +46,13 @@ export class NeuralNetwork { this.backend.addLayer(layer); } + /** + * feed an input through the layers + */ + // deno-lint-ignore no-explicit-any + async feedForward(input: any): Promise { + return await this.backend.feedForward(input); + } /** * train network */ @@ -61,21 +75,28 @@ export class NeuralNetwork { /** * Export the network in a JSON format */ - toJSON(): NetworkJSON | undefined{ - return this.backend.toJSON(); + async toJSON() { + return await this.backend.toJSON(); + } + /** + * Import the network in a JSON format + */ + static async fromJSON( + data: NetworkJSON, + helper?: (data: NetworkJSON, silent: boolean) => Promise, + silent = false, + ) { + return helper ? await helper(data, silent) : CPUBackend.fromJSON(data); } - /** * Load model from binary file */ static load(_str: string) { - } - /** * save model to binary file */ - save(str: string) { + save(str: string) { this.backend.save(str); } /** @@ -91,19 +112,13 @@ export class NeuralNetwork { getBiases() { return this.backend.getBiases(); } + /** + * get layers from the backend + */ + getLayer(index: number) { + return this.backend.layers[index]; + } } -export class DenseLayer { - public type = "dense"; - constructor(public config: DenseLayerConfig) {} -} -export class ConvLayer { - public type = "conv"; - constructor(public config: ConvLayerConfig) {} -} -export class PoolLayer { - public type = "pool"; - constructor(public config: PoolLayerConfig) {} -} diff --git a/core/tensor.ts b/core/tensor.ts new file mode 100644 index 0000000..b7f4984 --- /dev/null +++ b/core/tensor.ts @@ -0,0 +1,30 @@ +import { TensorCPUBackend } from "../backends/cpu/tensor.ts"; +import { TensorBackend, TensorLike } from "./types.ts"; +import { inferShape } from "./util.ts"; + +export class Tensor { + static backend: TensorBackend = new TensorCPUBackend(); + + static async setupBackend( + backend: { tensor: () => TensorBackend | Promise }, + ) { + Tensor.backend = await backend.tensor(); + } +} + +export async function tensor2D( + values: TensorLike, +) { + const shape = inferShape(values).slice(); + if (shape.length > 2) throw new Error("Invalid 2D Tensor"); + // values + return await Tensor.backend.tensor2D(values, shape[1], shape[0]); +} + +export async function tensor1D( + values: TensorLike, +) { + // deno-lint-ignore no-explicit-any + if (Array.isArray((values as any)[0])) throw new Error("Invalid 1D Tensor"); + return await Tensor.backend.tensor1D(values); +} diff --git a/core/types.ts b/core/types.ts new file mode 100644 index 0000000..5804b9b --- /dev/null +++ b/core/types.ts @@ -0,0 +1,199 @@ +import { DataType, DataTypeArray } from "../deps.ts"; +import { ConvLayer, DenseLayer, PoolLayer } from "../mod.ts"; +import { ConvCPULayer } from "../backends/cpu/layers/conv.ts"; +import { DenseCPULayer } from "../backends/cpu/layers/dense.ts"; +import { DenseGPULayer } from "../backends/gpu/layers/dense.ts"; +import { GPUMatrix } from "../backends/gpu/matrix.ts"; +import { CPUMatrix } from "../backends/cpu/matrix.ts"; +import { PoolCPULayer } from "../backends/cpu/layers/pool.ts"; + +export interface LayerJSON { + outputSize: number | Size2D; + activationFn?: string; + costFn?: string; + type: string; + input: MatrixJSON; + weights?: MatrixJSON; + biases?: MatrixJSON; + output: MatrixJSON; + error?: MatrixJSON; + cost?: MatrixJSON; + kernel?: MatrixJSON; + padded?: MatrixJSON; + strides?: Size; + padding?: number; + mode?: "max" | "avg"; +} + +export interface NetworkJSON { + costFn?: string; + type: "NeuralNetwork"; + sizes: (number | Size2D)[]; + input: Size | undefined; + layers: LayerJSON[]; + output: LayerJSON; +} + +export interface MatrixJSON { + // deno-lint-ignore no-explicit-any + data: any; + x: number; + y: number; + type?: DataType; +} + +export interface Backend { + // deno-lint-ignore no-explicit-any + layers: Array; + initialize( + inputSize: Size, + batches: number, + type?: DataType, + ): void | Promise; + // deno-lint-ignore no-explicit-any + addLayer(layer: Layer | any, index?: number): void; + train( + // deno-lint-ignore no-explicit-any + datasets: DataSet[] | any, + epochs: number, + batches: number, + learningRate: number, + ): void; + // deno-lint-ignore no-explicit-any + predict(input: DataTypeArray | any): DataTypeArray | any; + // deno-lint-ignore no-explicit-any + feedForward(input: any): Promise | any; + save(input: string): void; + toJSON(): NetworkJSON | Promise | undefined; + // deno-lint-ignore no-explicit-any + getWeights(): (GPUMatrix | CPUMatrix | any)[]; + // deno-lint-ignore no-explicit-any + getBiases(): (GPUMatrix | CPUMatrix | any)[]; +} + +export interface TensorBackend { + tensor2D(values: TensorLike, width: number, height: number): Tensor2D; + tensor1D(values: TensorLike): Tensor1D; +} + +export type Tensor2DCPU = CPUMatrix; +export type Tensor2DGPU = GPUMatrix; +// deno-lint-ignore no-explicit-any +export type Tensor2DNative = any; +export type Tensor2D = Tensor2DCPU | Tensor2DGPU | Tensor2DNative; +// deno-lint-ignore no-explicit-any +export type Tensor1D = Float32Array | any; + +/** + * NetworkConfig represents the configuration of a neural network. + */ +export interface NetworkConfig { + input?: Size; + layers: Layer[]; + cost: Cost; + silent?: boolean; +} + +export type Layer = DenseLayer | ConvLayer | PoolLayer; + +export type CPULayer = ConvCPULayer | DenseCPULayer | PoolCPULayer; + +export type GPULayer = DenseGPULayer; + +export interface DenseLayerConfig { + size: Size; + activation: Activation; +} + +export interface ConvLayerConfig { + activation?: Activation; + kernel: DataTypeArray; + kernelSize: Size2D; + padding?: number; + strides?: Size; +} + +export interface PoolLayerConfig { + strides?: Size; + mode?: "max" | "avg"; +} + +export type Size = number | Size2D; + +export type Size1D = number | { x: number }; + +export type Size2D = { x: number; y: number }; + +export type Size3D = { x: number; y: number; z: number }; + +/** + * Activation functions are used to transform the output of a layer into a new output. + */ +export type Activation = + | "sigmoid" + | "tanh" + | "relu" + | "relu6" + | "leakyrelu" + | "elu" + | "linear" + | "selu"; + +export type Cost = "crossentropy" | "hinge" | "mse"; + +export type ArrayMap = + | number + | number[] + | number[][] + | number[][][] + | number[][][][] + | number[][][][][] + | number[][][][][][]; + +export type TypedArray = Float32Array | Int32Array | Uint8Array; + +export type NumberArray = + | DataTypeArray + | Array + // deno-lint-ignore no-explicit-any + | any; + +/** + * DataSet is a container for training data. + */ +export type DataSet = { + inputs: Tensor2D; + outputs: Tensor1D; +}; + +/** @docalias TypedArray|Array */ +export type TensorLike = + | TypedArray + | number[][] + | number[] + | ArrayMap + | TypedArray[]; + +export type ScalarLike = number | Uint8Array; + +/** @docalias TypedArray|Array */ +export type TensorLike1D = + | TypedArray + | number[] + | Uint8Array[]; + +/** @docalias TypedArray|Array */ +export type TensorLike2D = + | TypedArray + | number[] + | number[][] + | Uint8Array[] + | Uint8Array[][]; + +/** @docalias TypedArray|Array */ +export type TensorLike3D = + | TypedArray + | number[] + | number[][][] + | Uint8Array[] + | Uint8Array[][][]; diff --git a/core/util.ts b/core/util.ts new file mode 100644 index 0000000..88b2577 --- /dev/null +++ b/core/util.ts @@ -0,0 +1,310 @@ +import { DataType, DataTypeArray, DataTypeArrayConstructor } from "../deps.ts"; +import { CPUMatrix } from "../backends/cpu/matrix.ts"; +import type { Size, Size2D, TensorLike, TypedArray } from "./types.ts"; + +export const isTypedArray = ( + // deno-lint-ignore ban-types + a: {}, +): a is Float32Array | Int32Array | Uint8Array | Uint8ClampedArray => + a instanceof Float32Array || a instanceof Int32Array || + a instanceof Uint8Array || a instanceof Uint8ClampedArray; + +export function getType(type: DataTypeArray) { + return ( + type instanceof Uint32Array + ? "u32" + : type instanceof Int32Array + ? "i32" + : "f32" + ); +} +export function fromType(type: string) { + return ( + type === "u32" + ? Uint32Array + : type === "i32" + ? Int32Array + : type === "f32" + ? Float32Array + : Uint32Array + ) as DataTypeArrayConstructor; +} +export function toType(type: string) { + return ( + type === "u32" + ? Uint32Array + : type === "i32" + ? Int32Array + : type === "f32" + ? Float32Array + : Uint32Array + ) as DataTypeArrayConstructor; +} + +export class ActivationError extends Error { + constructor(activation: string) { + super( + `Unknown activation function: ${activation}. Available: "sigmoid", "tanh", "relu", "relu6" , "leakyrelu", "elu", "linear", "selu"`, + ); + } +} + +export const randomWeight = (): number => Math.random() * 0.4 - 0.2; + +export const randomFloat = (min: number, max: number): number => + Math.random() * (max - min) + min; + +export const gaussRandom = (): number => { + if (gaussRandom.returnV) { + gaussRandom.returnV = false; + return gaussRandom.vVal; + } + const u = 2 * Math.random() - 1; + const v = 2 * Math.random() - 1; + const r = u * u + v * v; + if (r === 0 || r > 1) { + return gaussRandom(); + } + const c = Math.sqrt((-2 * Math.log(r)) / r); + gaussRandom.vVal = v * c; + gaussRandom.returnV = true; + return u * c; +}; +gaussRandom.returnV = false; +gaussRandom.vVal = 0; + +export const randomInteger = (min: number, max: number): number => + Math.floor(Math.random() * (max - min) + min); + +export const randomN = (mu: number, std: number): number => + mu + gaussRandom() * std; +export const max = ( + values: + | Float32Array + | { + [key: string]: number; + }, +): number => + (Array.isArray(values) || values instanceof Float32Array) + ? Math.max(...values) + : Math.max(...Object.values(values)); + +export const mse = (errors: Float32Array): number => { + let sum = 0; + for (let i = 0; i < errors.length; i++) { + sum += errors[i] ** 2; + } + return sum / errors.length; +}; + +export function to1D(size: Size): number { + const size2d = (size as Size2D); + if (size2d.y) { + return size2d.x * size2d.y; + } else { + return size as number; + } +} + +export function to2D(size: Size = 1): Size2D { + return Number(size) + ? { x: size as number, y: size as number } as Size2D + : size as Size2D; +} + +export function iterate2D( + mat: { x: number; y: number } | CPUMatrix, + callback: (i: number, j: number) => void, +): void { + for (let i = 0; i < mat.x; i++) { + for (let j = 0; j < mat.y; j++) { + callback(i, j); + } + } +} + +export function iterate1D(length: number, callback: (i: number) => void): void { + for (let i = 0; i < length; i++) { + callback(i); + } +} + +export function swap( + object: { [index: number]: T }, + left: number, + right: number, +) { + const temp = object[left]; + object[left] = object[right]; + object[right] = temp; +} + +export function shuffle( + // deno-lint-ignore no-explicit-any + array: any[] | Uint32Array | Int32Array | Float32Array, +): void { + let counter = array.length; + let index = 0; + while (counter > 0) { + index = (Math.random() * counter) | 0; + counter--; + swap(array, counter, index); + } +} + +export function shuffleCombo( + // deno-lint-ignore no-explicit-any + array: any[] | Uint32Array | Int32Array | Float32Array, + // deno-lint-ignore no-explicit-any + array2: any[] | Uint32Array | Int32Array | Float32Array, +): void { + if (array.length !== array2.length) { + throw new Error( + `Array sizes must match to be shuffled together ` + + `First array length was ${array.length}` + + `Second array length was ${array2.length}`, + ); + } + let counter = array.length; + let index = 0; + while (counter > 0) { + index = (Math.random() * counter) | 0; + counter--; + swap(array, counter, index); + swap(array2, counter, index); + } +} +export function createShuffledIndices(n: number): Uint32Array { + const shuffledIndices = new Uint32Array(n); + for (let i = 0; i < n; ++i) { + shuffledIndices[i] = i; + } + shuffle(shuffledIndices); + return shuffledIndices; +} + +export function randUniform(a: number, b: number) { + const r = Math.random(); + return (b * r) + (1 - r) * a; +} + +export function flatten< + T extends number | Promise | TypedArray, +>( + arr: T, + result: T[] = [], + skipTypedArray = false, +): T[] | ArrayBufferLike { + if (result == null) { + result = []; + } + if (Array.isArray(arr) || isTypedArray(arr) && !skipTypedArray) { + for (let i = 0; i < arr.length; ++i) { + flatten(arr[i], result, skipTypedArray); + } + } else { + result.push(arr as T); + } + return result; +} + +export function sizeFromShape(shape: number[]): number { + if (shape.length === 0) { + return 1; + } + let size = shape[0]; + for (let i = 1; i < shape.length; i++) { + size *= shape[i]; + } + return size; +} + +export function computeStrides(shape: number[]): number[] { + const rank = shape.length; + if (rank < 2) { + return []; + } + const strides = new Array(rank - 1); + strides[rank - 2] = shape[rank - 1]; + for (let i = rank - 3; i >= 0; --i) { + strides[i] = strides[i + 1] * shape[i + 1]; + } + return strides; +} + +function createNestedArray( + offset: number, + shape: number[], + a: TypedArray, + isComplex = false, +) { + // deno-lint-ignore no-array-constructor + const ret = new Array(); + if (shape.length === 1) { + const d = shape[0] * (isComplex ? 2 : 1); + for (let i = 0; i < d; i++) { + ret[i] = a[offset + i]; + } + } else { + const d = shape[0]; + const rest = shape.slice(1); + const len = rest.reduce((acc, c) => acc * c) * (isComplex ? 2 : 1); + for (let i = 0; i < d; i++) { + ret[i] = createNestedArray(offset + i * len, rest, a, isComplex); + } + } + return ret; +} + +export function toNestedArray( + shape: number[], + a: TypedArray, + isComplex = false, +) { + if (shape.length === 0) { + return a[0]; + } + const size = shape.reduce((acc, c) => acc * c) * (isComplex ? 2 : 1); + if (size === 0) { + return []; + } + if (size !== a.length) { + throw new Error( + `[${shape}] does not match the input size ${a.length}${ + isComplex ? " for a complex tensor" : "" + }.`, + ); + } + + return createNestedArray(0, shape, a, isComplex); +} + +export const average = (...args: number[]) => + args.reduce((a, b) => a + b) / args.length; + +export function inferShape(val: TensorLike): number[] { + let firstElem: typeof val = val; + + if (isTypedArray(val)) { + return [val.length]; + } + if (!Array.isArray(val)) { + return []; + } + const shape: number[] = []; + + while ( + Array.isArray(firstElem) || + isTypedArray(firstElem) + ) { + shape.push(firstElem.length); + firstElem = firstElem[0]; + } + if ( + Array.isArray(val) + ) { + // TODO: assert shape + } + + return shape; +} diff --git a/deno.json b/deno.json index 8176618..a9932c9 100644 --- a/deno.json +++ b/deno.json @@ -2,9 +2,13 @@ "tasks": { "train_xor": "deno run -A --unstable ./examples/train_xor_cpu.ts", "train_xor_gpu": "deno run -A --unstable ./examples/train_xor_gpu.ts", + "train_xor_native": "deno run -A --unstable ./examples/train_xor_native.ts", "train_letter": "deno run -A --unstable ./examples/train_letter.ts", "train_emoticon": "deno run -A --unstable ./examples/train_emoticon.ts", "train_conv": "deno run -A --unstable ./examples/train_conv.ts", + "test_train": "deno run -A --unstable ./examples/train_and_run/train.ts", + "test_run": "deno run -A --unstable ./examples/train_and_run/run.ts", + "train_filters": "deno run -A --unstable ./examples/filters/conv.ts", "perf_test": "deno run -A --unstable ./examples/perf_test.ts", "build": "cd native/build && cmake .. && make" } diff --git a/examples/filters/conv.ts b/examples/filters/conv.ts new file mode 100644 index 0000000..cd3d289 --- /dev/null +++ b/examples/filters/conv.ts @@ -0,0 +1,93 @@ +import { ConvLayer, DenseLayer, NeuralNetwork } from "../../mod.ts"; +import { CPU, CPUMatrix } from "../../backends/cpu/mod.ts"; +import { ConvCPULayer } from "../../backends/cpu/layers/conv.ts"; +import { PoolCPULayer } from "../../backends/cpu/layers/pool.ts"; +import { PoolLayer } from "../../layers/mod.ts"; + +import { decode } from "https://deno.land/x/pngs@0.1.1/mod.ts"; +import { DataTypeArray } from "../../deps.ts"; + +// import { Canvas } from "https://deno.land/x/neko@1.1.3/canvas/mod.ts"; +import { createCanvas } from "https://deno.land/x/canvas@v1.4.1/mod.ts"; + +const canvas = createCanvas(600, 600); +// const canvas = new Canvas({ +// title: "Netsaur Convolutions", +// width: 600, +// height: 600, +// fps: 60, +// }); + +const ctx = canvas.getContext("2d"); +ctx.fillStyle = "white"; +ctx.fillRect(0, 0, 600, 600); + +const dim = 28; + +//Credit: Hashrock (https://github.com/hashrock) +const img = decode(Deno.readFileSync("./examples/filters/deno.png")).image; +const buf = new Float32Array(dim * dim) as DataTypeArray<"f32">; + +for (let i = 0; i < dim * dim; i++) { + buf[i] = img[i * 4]; +} + +const kernel = [ + [-1, 1, 0], + [-1, 1, 0], + [-1, 1, 0], +].flat(); + +const net = await new NeuralNetwork({ + silent: true, + layers: [ + new ConvLayer({ + activation: "linear", + kernel: new Float32Array(kernel), + kernelSize: { x: 3, y: 3 }, + padding: 1, + strides: 1, + }), + new PoolLayer({ strides: 2, mode: "max" }), + new DenseLayer({ size: 1, activation: "sigmoid" }), + ], + cost: "crossentropy", + input: 2, +}).setupBackend(CPU); + +const input = new CPUMatrix(buf, dim, dim); + +const conv = net.getLayer(0) as ConvCPULayer; +const pool = net.getLayer(1) as PoolCPULayer; + +net.initialize({ x: dim, y: dim }, 1); +net.feedForward(input); + +for (let i = 0; i < dim; i++) { + for (let j = 0; j < dim; j++) { + const pixel = buf[j * dim + i]; + ctx.fillStyle = `rgb(${pixel}, ${pixel}, ${pixel})`; + ctx.fillRect(i * 10, j * 10, 10, 10); + } +} + +for (let i = 0; i < conv.output.x; i++) { + for (let j = 0; j < conv.output.y; j++) { + const pixel = Math.round( + Math.max(Math.min(conv.output.data[j * conv.output.x + i], 255), 0), + ); + ctx.fillStyle = `rgb(${pixel}, ${pixel}, ${pixel})`; + ctx.fillRect(i * 10 + dim * 10, j * 10, 10, 10); + } +} + +for (let i = 0; i < pool.output.x; i++) { + for (let j = 0; j < pool.output.y; j++) { + const pixel = Math.round( + Math.max(Math.min(pool.output.data[j * pool.output.x + i], 255), 0), + ); + ctx.fillStyle = `rgb(${pixel}, ${pixel}, ${pixel})`; + ctx.fillRect(i * 20 + dim * 10, j * 20 + dim * 10, 20, 20); + } +} +await Deno.writeFile("./examples/filters/output.png", canvas.toBuffer()); diff --git a/tests/deno.png b/examples/filters/deno.png similarity index 100% rename from tests/deno.png rename to examples/filters/deno.png diff --git a/examples/mnist_digits/common.ts b/examples/mnist_digits/common.ts index 4b347e1..2cf3cff 100644 --- a/examples/mnist_digits/common.ts +++ b/examples/mnist_digits/common.ts @@ -1,6 +1,5 @@ -import type { Dataset } from "../../backends/native.ts"; -import { Matrix } from "../../backends/native.ts"; - +import type { Dataset } from "../../backends/native/mod.ts"; +import { Matrix } from "../../backends/native/mod.ts"; export function assert(condition: boolean, message?: string) { if (!condition) { diff --git a/examples/mnist_digits/predict.ts b/examples/mnist_digits/predict.ts index 1871ffb..de98eb1 100644 --- a/examples/mnist_digits/predict.ts +++ b/examples/mnist_digits/predict.ts @@ -1,5 +1,4 @@ -import { NativeBackend } from "../../src/native/backend.ts"; -import { DataType, Matrix } from "../../backends/native.ts"; +import { DataType, Matrix, NativeBackend } from "../../backends/native/mod.ts"; import { loadDataset } from "./common.ts"; const network = NativeBackend.load("digit_model.bin"); @@ -25,4 +24,6 @@ const correct = testSet.filter((e) => { }); console.log(`${correct.length} / ${testSet.length} correct`); -console.log(`accuracy: ${((correct.length / testSet.length) * 100).toFixed(2)}%`); \ No newline at end of file +console.log( + `accuracy: ${((correct.length / testSet.length) * 100).toFixed(2)}%`, +); diff --git a/examples/mnist_digits/train.ts b/examples/mnist_digits/train.ts index e09e7bb..e9324ea 100644 --- a/examples/mnist_digits/train.ts +++ b/examples/mnist_digits/train.ts @@ -1,5 +1,5 @@ import { DenseLayer, NeuralNetwork } from "../../mod.ts"; -import { Native } from "../../backends/native.ts"; +import { Native } from "../../backends/native/mod.ts"; import { loadDataset } from "./common.ts"; const network = await new NeuralNetwork({ diff --git a/examples/train_and_run/.gitignore b/examples/train_and_run/.gitignore new file mode 100644 index 0000000..1ef32e2 --- /dev/null +++ b/examples/train_and_run/.gitignore @@ -0,0 +1,3 @@ +layer.json +output-layer.json +network.json \ No newline at end of file diff --git a/examples/train_and_run/run.ts b/examples/train_and_run/run.ts new file mode 100644 index 0000000..272ec6f --- /dev/null +++ b/examples/train_and_run/run.ts @@ -0,0 +1,9 @@ +import { CPU } from "../../backends/cpu/mod.ts"; +import { Model } from "../../model/mod.ts"; + +const net = await Model.load("./examples/train_and_run/network.json", CPU); + +console.log(await net.predict(new Float32Array([0, 0]))); +console.log(await net.predict(new Float32Array([1, 0]))); +console.log(await net.predict(new Float32Array([0, 1]))); +console.log(await net.predict(new Float32Array([1, 1]))); diff --git a/examples/train_and_run/train.ts b/examples/train_and_run/train.ts new file mode 100644 index 0000000..e36afdf --- /dev/null +++ b/examples/train_and_run/train.ts @@ -0,0 +1,35 @@ +import { DenseLayer, NeuralNetwork, tensor1D, tensor2D } from "../../mod.ts"; +import { CPU } from "../../backends/cpu/mod.ts"; +import { Model } from "../../model/mod.ts"; + +const net = await new NeuralNetwork({ + silent: true, + layers: [ + new DenseLayer({ size: 3, activation: "sigmoid" }), + new DenseLayer({ size: 1, activation: "sigmoid" }), + ], + cost: "crossentropy", +}).setupBackend(CPU); + +const time = performance.now(); + +await net.train( + [ + { + inputs: await tensor2D([ + [0, 0], + [1, 0], + [0, 1], + [1, 1], + ]), + outputs: await tensor1D([0, 1, 1, 0]), + }, + ], + 5000, + 4, + 0.1, +); + +console.log(`training time: ${performance.now() - time}ms`); + +await Model.save("./examples/train_and_run/network.json", net); diff --git a/examples/train_conv.ts b/examples/train_conv.ts index 94653a8..71c3f92 100644 --- a/examples/train_conv.ts +++ b/examples/train_conv.ts @@ -1,73 +1,44 @@ -import { ConvLayer, DenseLayer, NeuralNetwork, PoolLayer } from "../mod.ts"; -import { ConvCPULayer } from "../src/cpu/layers/conv.ts"; -import { PoolCPULayer } from "../src/cpu/layers/pool.ts"; -import { CPUMatrix } from "../src/cpu/matrix.ts"; -import { CPUBackend } from "../src/cpu/backend.ts"; -import { CPU } from "../backends/cpu.ts"; - -const kernel = new Float32Array([ - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, -]); +import { + ConvLayer, + DenseLayer, + NeuralNetwork, + PoolLayer, + tensor2D, +} from "../mod.ts"; +import { ConvCPULayer } from "../backends/cpu/layers/conv.ts"; +import { PoolCPULayer } from "../backends/cpu/layers/pool.ts"; +import { CPU } from "../backends/cpu/mod.ts"; const net = await new NeuralNetwork({ silent: true, layers: [ new ConvLayer({ - activation: "sigmoid", - kernel: kernel, + activation: "linear", + kernel: new Float32Array([1, 1, 1, 1, 1, 1, 1, 1, 1]), kernelSize: { x: 3, y: 3 }, padding: 2, - stride: 2, + strides: 2, }), - new PoolLayer({ stride: 2 }), + new PoolLayer({ strides: 2 }), new DenseLayer({ size: 1, activation: "sigmoid" }), ], cost: "crossentropy", input: 2, }).setupBackend(CPU); -const buf = new Float32Array([ - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, +const input = await tensor2D([ + [1, 1, 1, 1, 1], + [1, 1, 1, 1, 1], + [1, 1, 1, 1, 1], + [1, 1, 1, 1, 1], + [1, 1, 1, 1, 1], ]); -const input = new CPUMatrix(buf, 5, 5); -const network = net.backend as CPUBackend; -const conv = network.layers[0] as ConvCPULayer; -const pool = network.layers[1] as PoolCPULayer; -network.initialize({ x: 5, y: 5 }, 1); -network.layers[0].feedForward(input); -network.layers[1].feedForward(conv.output); + +const conv = net.getLayer(0) as ConvCPULayer; +const pool = net.getLayer(1) as PoolCPULayer; +net.initialize(input, 1); +net.feedForward(input); + console.log(conv.padded.fmt()); console.log(conv.output.fmt()); console.log(pool.output.fmt()); diff --git a/examples/train_emotion.ts b/examples/train_emoticon.ts similarity index 89% rename from examples/train_emotion.ts rename to examples/train_emoticon.ts index 6d008df..2099693 100644 --- a/examples/train_emotion.ts +++ b/examples/train_emoticon.ts @@ -1,9 +1,9 @@ import { DataType, DataTypeArray } from "../deps.ts"; -import { DenseLayer, NeuralNetwork } from "../mod.ts"; -import { CPU } from "../backends/cpu.ts"; +import { DenseLayer, NeuralNetwork, tensor1D, tensor2D } from "../mod.ts"; +import { CPU } from "../backends/cpu/mod.ts"; -const character = (string: string): Float32Array => - Float32Array.from(string.trim().split("").map(integer)); +const character = (string: string) => + Array.from(string.trim().split("").map(integer)); const integer = (character: string): number => character === "#" ? 1 : 0; @@ -69,8 +69,14 @@ const net = await new NeuralNetwork({ net.train( [ - { inputs: happy, outputs: ["a".charCodeAt(0) / 255] }, - { inputs: sad, outputs: ["b".charCodeAt(0) / 255] }, + { + inputs: await tensor2D([happy]), + outputs: await tensor1D(["a".charCodeAt(0) / 255]), + }, + { + inputs: await tensor2D([sad]), + outputs: await tensor1D(["b".charCodeAt(0) / 255]), + }, ], 5000, 1, diff --git a/examples/train_letter.ts b/examples/train_letter.ts index 4ad92e1..0e778fb 100644 --- a/examples/train_letter.ts +++ b/examples/train_letter.ts @@ -1,6 +1,6 @@ import { DataType, DataTypeArray } from "../deps.ts"; -import { DenseLayer, NeuralNetwork } from "../mod.ts"; -import { CPU } from "../backends/cpu.ts"; +import { DenseLayer, NeuralNetwork, tensor1D, tensor2D } from "../mod.ts"; +import { CPU } from "../backends/cpu/mod.ts"; // https://github.com/BrainJS/brain.js/blob/master/examples/typescript/which-letter-simple.ts const character = (string: string): Float32Array => @@ -46,9 +46,14 @@ const net = await new NeuralNetwork({ net.train( [ - { inputs: a, outputs: ["a".charCodeAt(0) / 255] }, - { inputs: b, outputs: ["b".charCodeAt(0) / 255] }, - { inputs: c, outputs: ["c".charCodeAt(0) / 255] }, + { + inputs: await tensor2D([a, b, c]), + outputs: await tensor1D([ + "a".charCodeAt(0) / 255, + "b".charCodeAt(0) / 255, + "c".charCodeAt(0) / 255, + ]), + }, ], 5000, 1, diff --git a/examples/train_xor_cpu.ts b/examples/train_xor_cpu.ts index b83942e..1c87312 100644 --- a/examples/train_xor_cpu.ts +++ b/examples/train_xor_cpu.ts @@ -1,5 +1,5 @@ -import { DenseLayer, NeuralNetwork } from "../mod.ts"; -import { CPU } from "../backends/cpu.ts"; +import { DenseLayer, NeuralNetwork, tensor1D, tensor2D } from "../mod.ts"; +import { CPU } from "../backends/cpu/mod.ts"; const net = await new NeuralNetwork({ silent: true, @@ -10,18 +10,24 @@ const net = await new NeuralNetwork({ cost: "crossentropy", }).setupBackend(CPU); -const time = Date.now(); +const time = performance.now(); await net.train( [ - { inputs: [0, 0, 1, 0, 0, 1, 1, 1], outputs: [0, 1, 1, 0] }, + { + inputs: await tensor2D([ + [0, 0], + [1, 0], + [0, 1], + [1, 1], + ]), + outputs: await tensor1D([0, 1, 1, 0]), + }, ], 5000, - 4, - 0.1, ); -console.log(`training time: ${Date.now() - time}ms`); +console.log(`training time: ${performance.now() - time}ms`); console.log(await net.predict(new Float32Array([0, 0]))); console.log(await net.predict(new Float32Array([1, 0]))); console.log(await net.predict(new Float32Array([0, 1]))); diff --git a/examples/train_xor_gpu.ts b/examples/train_xor_gpu.ts index 45ff27e..9ec90a2 100644 --- a/examples/train_xor_gpu.ts +++ b/examples/train_xor_gpu.ts @@ -1,5 +1,7 @@ -import { DenseLayer, NeuralNetwork } from "../mod.ts"; -import { GPU } from "../backends/gpu.ts"; +import { DenseLayer, NeuralNetwork, Tensor, tensor2D, tensor1D } from "../mod.ts"; +import { GPU } from "../backends/gpu/mod.ts"; + +await Tensor.setupBackend(GPU); const net = await new NeuralNetwork({ silent: true, @@ -10,18 +12,24 @@ const net = await new NeuralNetwork({ cost: "crossentropy", }).setupBackend(GPU); -const time = Date.now(); +const time = performance.now(); await net.train( [ - { inputs: [0, 0, 1, 0, 0, 1, 1, 1], outputs: [0, 1, 1, 0] }, + { + inputs: await tensor2D([ + [0, 0], + [1, 0], + [0, 1], + [1, 1], + ]), + outputs: await tensor1D([0, 1, 1, 0]), + }, ], 5000, - 4, - 0.1, ); -console.log(`training time: ${Date.now() - time}ms`); +console.log(`training time: ${performance.now() - time}ms`); console.log(await net.predict(new Float32Array([0, 0]))); console.log(await net.predict(new Float32Array([1, 0]))); console.log(await net.predict(new Float32Array([0, 1]))); diff --git a/examples/train_xor_native.ts b/examples/train_xor_native.ts index aba7ea9..8260913 100644 --- a/examples/train_xor_native.ts +++ b/examples/train_xor_native.ts @@ -1,7 +1,8 @@ -import { NeuralNetwork, DenseLayer } from "../mod.ts"; -import { Native, Matrix } from "../backends/native.ts"; +import { NeuralNetwork, DenseLayer, Tensor, tensor2D, tensor1D } from "../mod.ts"; +import { Native } from "../backends/native/mod.ts"; -const start = Date.now(); +Tensor.setupBackend(Native); +const start = performance.now(); const network = await new NeuralNetwork({ input: 2, @@ -15,24 +16,24 @@ const network = await new NeuralNetwork({ network.train( [ { - inputs: Matrix.of([ + inputs: await tensor2D([ [0, 0], [0, 1], [1, 0], [1, 1], ]), - outputs: Matrix.column([0, 1, 1, 0]), + outputs: await tensor1D([0, 1, 1, 0]), }, ], 5000, 0.1, ); -console.log("training time", Date.now() - start, "milliseconds"); +console.log("training time", performance.now() - start, "milliseconds"); console.log( await network.predict( - Matrix.of([ + await tensor2D([ [0, 0], [0, 1], [1, 0], diff --git a/layers/conv.ts b/layers/conv.ts new file mode 100644 index 0000000..0e1a51b --- /dev/null +++ b/layers/conv.ts @@ -0,0 +1,32 @@ +import { ConvLayerConfig, LayerJSON } from "../core/types.ts"; + +/** + * Convolutional layer. + */ +export class ConvLayer { + type = "conv"; + load = false; + data?: LayerJSON; + constructor(public config: ConvLayerConfig) {} + static fromJSON(layerJSON: LayerJSON): ConvLayer { + if (layerJSON.type !== "conv") { + throw new Error( + "Cannot cannot create a Convolutional layer from a" + + layerJSON.type.charAt(0).toUpperCase() + layerJSON.type.slice(1) + + "Layer", + ); + } + if (layerJSON.padded === undefined || layerJSON.kernel === undefined) { + throw new Error("Layer imported must be initialized"); + } + const layer = new ConvLayer({ + kernel: layerJSON.kernel!.data, + kernelSize: { x: layerJSON.kernel!.x, y: layerJSON.kernel!.y }, + padding: layerJSON.padding, + strides: layerJSON.strides, + }); + layer.load = true; + layer.data = layerJSON; + return layer; + } +} diff --git a/layers/dense.ts b/layers/dense.ts new file mode 100644 index 0000000..9dcf2fc --- /dev/null +++ b/layers/dense.ts @@ -0,0 +1,30 @@ +import { Activation, DenseLayerConfig, LayerJSON } from "../core/types.ts"; + +/** + * Base class for all layers. + */ +export class DenseLayer { + type = "dense"; + load = false; + data?: LayerJSON; + constructor(public config: DenseLayerConfig) {} + static fromJSON(layerJSON: LayerJSON): DenseLayer { + if (layerJSON.type !== "dense") { + throw new Error( + "Cannot cannot create a Dense layer from a" + + layerJSON.type.charAt(0).toUpperCase() + layerJSON.type.slice(1) + + "Layer", + ); + } + if (layerJSON.weights === undefined || layerJSON.biases === undefined) { + throw new Error("Layer imported must be initialized"); + } + const layer = new DenseLayer({ + size: layerJSON.outputSize, + activation: (layerJSON.activationFn as Activation) || "sigmoid", + }); + layer.load = true; + layer.data = layerJSON; + return layer; + } +} diff --git a/layers/mod.ts b/layers/mod.ts new file mode 100644 index 0000000..9ba3afa --- /dev/null +++ b/layers/mod.ts @@ -0,0 +1,3 @@ +export { ConvLayer } from "./conv.ts"; +export { DenseLayer } from "./dense.ts"; +export { PoolLayer } from "./pool.ts"; diff --git a/layers/pool.ts b/layers/pool.ts new file mode 100644 index 0000000..603004f --- /dev/null +++ b/layers/pool.ts @@ -0,0 +1,27 @@ +import { LayerJSON, PoolLayerConfig } from "../core/types.ts"; + +/** + * MaxPool layer. + */ +export class PoolLayer { + type = "pool"; + load = false; + data?: LayerJSON; + constructor(public config: PoolLayerConfig) {} + static fromJSON(layerJSON: LayerJSON): PoolLayer { + if (layerJSON.type !== "pool") { + throw new Error( + "Cannot cannot create a MaxPooling layer from a" + + layerJSON.type.charAt(0).toUpperCase() + layerJSON.type.slice(1) + + "Layer", + ); + } + if (layerJSON.strides === undefined) { + throw new Error("Layer imported must be initialized"); + } + const layer = new PoolLayer({ strides: layerJSON.strides! }); + layer.load = true; + layer.data = layerJSON; + return layer; + } +} diff --git a/mod.ts b/mod.ts index 29a0b68..3c4ebde 100644 --- a/mod.ts +++ b/mod.ts @@ -1 +1,3 @@ -export { ConvLayer, DenseLayer, NeuralNetwork, PoolLayer } from "./src/mod.ts"; +export { NeuralNetwork } from "./core/mod.ts"; +export { ConvLayer, DenseLayer, PoolLayer } from "./layers/mod.ts"; +export * from "./core/tensor.ts"; diff --git a/model/json.ts b/model/json.ts new file mode 100644 index 0000000..11b3dcb --- /dev/null +++ b/model/json.ts @@ -0,0 +1,25 @@ +import { NeuralNetwork } from "../core/mod.ts"; +import { NetworkJSON } from "../core/types.ts"; +import type { ModelFormat } from "./types.ts"; +import { staticImplements } from "./util.ts"; + +@staticImplements() +export class JSONModel { + static async load( + path: string, + ): Promise { + if (!path.endsWith(".json")) path = path + ".json"; + return (path.startsWith("http://") || path.startsWith("https://")) + ? await (await fetch("https://api.github.com/users/denoland")) + .json() as NetworkJSON + : JSON.parse(await Deno.readTextFile(path)) as NetworkJSON; + } + + static async save(path: string, net: NeuralNetwork) { + if (!path.endsWith(".json")) path = path + ".json"; + await Deno.writeTextFile( + path, + JSON.stringify(await net.backend.toJSON() as NetworkJSON), + ); + } +} diff --git a/model/keras.ts b/model/keras.ts new file mode 100644 index 0000000..8e85c9e --- /dev/null +++ b/model/keras.ts @@ -0,0 +1,33 @@ +import { NeuralNetwork } from "../core/mod.ts"; +import { NetworkJSON } from "../core/types.ts"; +import type { ModelFormat } from "./types.ts"; +import { staticImplements } from "./util.ts"; +import proto from "./proto/keras_proto.js"; + +// deno-lint-ignore no-unused-vars +function fromKeraslayerType(type: string): string { + switch (type) { + case "Conv2D": + return "conv"; + default: + throw new Error("Unknown Layer type: " + type); + } +} + +@staticImplements() +export class KerasModel { + static async load( + path: string, + ): Promise { + const _config: Partial = {}; + if (!path.endsWith(".bin")) path = path + ".bin"; + const model = proto.Model.decode(await Deno.readFile(path)); + const _model_config = JSON.parse(model.modelConfig); + throw new Error("Keras not yet implemented"); + } + + // deno-lint-ignore require-await + static async save(_path: string, _net: NeuralNetwork) { + throw new Error("Keras not yet implemented"); + } +} diff --git a/model/mod.ts b/model/mod.ts new file mode 100644 index 0000000..2bdac3b --- /dev/null +++ b/model/mod.ts @@ -0,0 +1,31 @@ +import { Backend, NetworkJSON } from "../core/types.ts"; +import { NeuralNetwork } from "../core/mod.ts"; +import { ModelFormat } from "./types.ts"; +import { JSONModel } from "./json.ts"; + +/** + * Model Loader Class + */ +export class Model { + static async load( + path: string, + helper?: { + model: (data: NetworkJSON, silent: boolean) => Promise; + }, + format: ModelFormat = JSONModel, + // deno-lint-ignore no-explicit-any + ): Promise { + return await NeuralNetwork.fromJSON( + await format.load(path), + helper ? helper.model : undefined, + ); + } + + static async save( + path: string, + net: NeuralNetwork, + format: ModelFormat = JSONModel, + ): Promise { + await format.save(path, net); + } +} diff --git a/model/proto/keras_proto.js b/model/proto/keras_proto.js new file mode 100644 index 0000000..03257e7 --- /dev/null +++ b/model/proto/keras_proto.js @@ -0,0 +1,677 @@ +// deno-lint-ignore-file +import $protobuf from "npm:protobufjs"; + + +// Common aliases +const $Reader = $protobuf.Reader, $Writer = $protobuf.Writer, $util = $protobuf.util; + +// Exported root namespace +const $root = $protobuf.roots["default"] || ($protobuf.roots["default"] = {}); + +export const Weights = $root.Weights = (() => { + + /** + * Properties of a Weights. + * @exports IWeights + * @interface IWeights + * @property {string} [layerName] Weights layerName + * @property {string} [weightName] Weights weightName + * @property {Array.} [shape] Weights shape + * @property {string} [type] Weights type + * @property {Uint8Array} [data] Weights data + * @property {number} [quantizeMin] Weights quantizeMin + * @property {number} [quantizeMax] Weights quantizeMax + */ + + /** + * Constructs a new Weights. + * @exports Weights + * @classdesc Represents a Weights. + * @constructor + * @param {IWeights=} [properties] Properties to set + */ + function Weights(properties) { + this.shape = []; + if (properties) + for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i) + if (properties[keys[i]] != null) + this[keys[i]] = properties[keys[i]]; + } + + /** + * Weights layerName. + * @member {string}layerName + * @memberof Weights + * @instance + */ + Weights.prototype.layerName = ""; + + /** + * Weights weightName. + * @member {string}weightName + * @memberof Weights + * @instance + */ + Weights.prototype.weightName = ""; + + /** + * Weights shape. + * @member {Array.}shape + * @memberof Weights + * @instance + */ + Weights.prototype.shape = $util.emptyArray; + + /** + * Weights type. + * @member {string}type + * @memberof Weights + * @instance + */ + Weights.prototype.type = ""; + + /** + * Weights data. + * @member {Uint8Array}data + * @memberof Weights + * @instance + */ + Weights.prototype.data = $util.newBuffer([]); + + /** + * Weights quantizeMin. + * @member {number}quantizeMin + * @memberof Weights + * @instance + */ + Weights.prototype.quantizeMin = 0; + + /** + * Weights quantizeMax. + * @member {number}quantizeMax + * @memberof Weights + * @instance + */ + Weights.prototype.quantizeMax = 0; + + /** + * Creates a new Weights instance using the specified properties. + * @function create + * @memberof Weights + * @static + * @param {IWeights=} [properties] Properties to set + * @returns {Weights} Weights instance + */ + Weights.create = function create(properties) { + return new Weights(properties); + }; + + /** + * Encodes the specified Weights message. Does not implicitly {@link Weights.verify|verify} messages. + * @function encode + * @memberof Weights + * @static + * @param {IWeights} message Weights message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + Weights.encode = function encode(message, writer) { + if (!writer) + writer = $Writer.create(); + if (message.layerName != null && message.hasOwnProperty("layerName")) + writer.uint32(/* id 1, wireType 2 =*/10).string(message.layerName); + if (message.weightName != null && message.hasOwnProperty("weightName")) + writer.uint32(/* id 2, wireType 2 =*/18).string(message.weightName); + if (message.shape != null && message.shape.length) { + writer.uint32(/* id 3, wireType 2 =*/26).fork(); + for (let i = 0; i < message.shape.length; ++i) + writer.uint32(message.shape[i]); + writer.ldelim(); + } + if (message.type != null && message.hasOwnProperty("type")) + writer.uint32(/* id 4, wireType 2 =*/34).string(message.type); + if (message.data != null && message.hasOwnProperty("data")) + writer.uint32(/* id 5, wireType 2 =*/42).bytes(message.data); + if (message.quantizeMin != null && message.hasOwnProperty("quantizeMin")) + writer.uint32(/* id 6, wireType 5 =*/53).float(message.quantizeMin); + if (message.quantizeMax != null && message.hasOwnProperty("quantizeMax")) + writer.uint32(/* id 7, wireType 5 =*/61).float(message.quantizeMax); + return writer; + }; + + /** + * Encodes the specified Weights message, length delimited. Does not implicitly {@link Weights.verify|verify} messages. + * @function encodeDelimited + * @memberof Weights + * @static + * @param {IWeights} message Weights message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + Weights.encodeDelimited = function encodeDelimited(message, writer) { + return this.encode(message, writer).ldelim(); + }; + + /** + * Decodes a Weights message from the specified reader or buffer. + * @function decode + * @memberof Weights + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @param {number} [length] Message length if known beforehand + * @returns {Weights} Weights + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + Weights.decode = function decode(reader, length) { + if (!(reader instanceof $Reader)) + reader = $Reader.create(reader); + let end = length === undefined ? reader.len : reader.pos + length, message = new $root.Weights(); + while (reader.pos < end) { + let tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + message.layerName = reader.string(); + break; + case 2: + message.weightName = reader.string(); + break; + case 3: + if (!(message.shape && message.shape.length)) + message.shape = []; + if ((tag & 7) === 2) { + let end2 = reader.uint32() + reader.pos; + while (reader.pos < end2) + message.shape.push(reader.uint32()); + } else + message.shape.push(reader.uint32()); + break; + case 4: + message.type = reader.string(); + break; + case 5: + message.data = reader.bytes(); + break; + case 6: + message.quantizeMin = reader.float(); + break; + case 7: + message.quantizeMax = reader.float(); + break; + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }; + + /** + * Decodes a Weights message from the specified reader or buffer, length delimited. + * @function decodeDelimited + * @memberof Weights + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @returns {Weights} Weights + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + Weights.decodeDelimited = function decodeDelimited(reader) { + if (!(reader instanceof $Reader)) + reader = new $Reader(reader); + return this.decode(reader, reader.uint32()); + }; + + /** + * Verifies a Weights message. + * @function verify + * @memberof Weights + * @static + * @param {Object.} message Plain object to verify + * @returns {string|null} `null` if valid, otherwise the reason why it is not + */ + Weights.verify = function verify(message) { + if (typeof message !== "object" || message === null) + return "object expected"; + if (message.layerName != null && message.hasOwnProperty("layerName")) + if (!$util.isString(message.layerName)) + return "layerName: string expected"; + if (message.weightName != null && message.hasOwnProperty("weightName")) + if (!$util.isString(message.weightName)) + return "weightName: string expected"; + if (message.shape != null && message.hasOwnProperty("shape")) { + if (!Array.isArray(message.shape)) + return "shape: array expected"; + for (let i = 0; i < message.shape.length; ++i) + if (!$util.isInteger(message.shape[i])) + return "shape: integer[] expected"; + } + if (message.type != null && message.hasOwnProperty("type")) + if (!$util.isString(message.type)) + return "type: string expected"; + if (message.data != null && message.hasOwnProperty("data")) + if (!(message.data && typeof message.data.length === "number" || $util.isString(message.data))) + return "data: buffer expected"; + if (message.quantizeMin != null && message.hasOwnProperty("quantizeMin")) + if (typeof message.quantizeMin !== "number") + return "quantizeMin: number expected"; + if (message.quantizeMax != null && message.hasOwnProperty("quantizeMax")) + if (typeof message.quantizeMax !== "number") + return "quantizeMax: number expected"; + return null; + }; + + /** + * Creates a Weights message from a plain object. Also converts values to their respective internal types. + * @function fromObject + * @memberof Weights + * @static + * @param {Object.} object Plain object + * @returns {Weights} Weights + */ + Weights.fromObject = function fromObject(object) { + if (object instanceof $root.Weights) + return object; + let message = new $root.Weights(); + if (object.layerName != null) + message.layerName = String(object.layerName); + if (object.weightName != null) + message.weightName = String(object.weightName); + if (object.shape) { + if (!Array.isArray(object.shape)) + throw TypeError(".Weights.shape: array expected"); + message.shape = []; + for (let i = 0; i < object.shape.length; ++i) + message.shape[i] = object.shape[i] >>> 0; + } + if (object.type != null) + message.type = String(object.type); + if (object.data != null) + if (typeof object.data === "string") + $util.base64.decode(object.data, message.data = $util.newBuffer($util.base64.length(object.data)), 0); + else if (object.data.length) + message.data = object.data; + if (object.quantizeMin != null) + message.quantizeMin = Number(object.quantizeMin); + if (object.quantizeMax != null) + message.quantizeMax = Number(object.quantizeMax); + return message; + }; + + /** + * Creates a plain object from a Weights message. Also converts values to other types if specified. + * @function toObject + * @memberof Weights + * @static + * @param {Weights} message Weights + * @param {$protobuf.IConversionOptions} [options] Conversion options + * @returns {Object.} Plain object + */ + Weights.toObject = function toObject(message, options) { + if (!options) + options = {}; + let object = {}; + if (options.arrays || options.defaults) + object.shape = []; + if (options.defaults) { + object.layerName = ""; + object.weightName = ""; + object.type = ""; + object.data = options.bytes === String ? "" : []; + object.quantizeMin = 0; + object.quantizeMax = 0; + } + if (message.layerName != null && message.hasOwnProperty("layerName")) + object.layerName = message.layerName; + if (message.weightName != null && message.hasOwnProperty("weightName")) + object.weightName = message.weightName; + if (message.shape && message.shape.length) { + object.shape = []; + for (let j = 0; j < message.shape.length; ++j) + object.shape[j] = message.shape[j]; + } + if (message.type != null && message.hasOwnProperty("type")) + object.type = message.type; + if (message.data != null && message.hasOwnProperty("data")) + object.data = options.bytes === String ? $util.base64.encode(message.data, 0, message.data.length) : options.bytes === Array ? Array.prototype.slice.call(message.data) : message.data; + if (message.quantizeMin != null && message.hasOwnProperty("quantizeMin")) + object.quantizeMin = options.json && !isFinite(message.quantizeMin) ? String(message.quantizeMin) : message.quantizeMin; + if (message.quantizeMax != null && message.hasOwnProperty("quantizeMax")) + object.quantizeMax = options.json && !isFinite(message.quantizeMax) ? String(message.quantizeMax) : message.quantizeMax; + return object; + }; + + /** + * Converts this Weights to JSON. + * @function toJSON + * @memberof Weights + * @instance + * @returns {Object.} JSON object + */ + Weights.prototype.toJSON = function toJSON() { + return this.constructor.toObject(this, $protobuf.util.toJSONOptions); + }; + + return Weights; +})(); + +export const Model = $root.Model = (() => { + + /** + * Properties of a Model. + * @exports IModel + * @interface IModel + * @property {string} [id] Model id + * @property {string} [name] Model name + * @property {string} [kerasVersion] Model kerasVersion + * @property {string} [backend] Model backend + * @property {string} [modelConfig] Model modelConfig + * @property {Array.} [modelWeights] Model modelWeights + */ + + /** + * Constructs a new Model. + * @exports Model + * @classdesc Represents a Model. + * @constructor + * @param {IModel=} [properties] Properties to set + */ + function Model(properties) { + this.modelWeights = []; + if (properties) + for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i) + if (properties[keys[i]] != null) + this[keys[i]] = properties[keys[i]]; + } + + /** + * Model id. + * @member {string}id + * @memberof Model + * @instance + */ + Model.prototype.id = ""; + + /** + * Model name. + * @member {string}name + * @memberof Model + * @instance + */ + Model.prototype.name = ""; + + /** + * Model kerasVersion. + * @member {string}kerasVersion + * @memberof Model + * @instance + */ + Model.prototype.kerasVersion = ""; + + /** + * Model backend. + * @member {string}backend + * @memberof Model + * @instance + */ + Model.prototype.backend = ""; + + /** + * Model modelConfig. + * @member {string}modelConfig + * @memberof Model + * @instance + */ + Model.prototype.modelConfig = ""; + + /** + * Model modelWeights. + * @member {Array.}modelWeights + * @memberof Model + * @instance + */ + Model.prototype.modelWeights = $util.emptyArray; + + /** + * Creates a new Model instance using the specified properties. + * @function create + * @memberof Model + * @static + * @param {IModel=} [properties] Properties to set + * @returns {Model} Model instance + */ + Model.create = function create(properties) { + return new Model(properties); + }; + + /** + * Encodes the specified Model message. Does not implicitly {@link Model.verify|verify} messages. + * @function encode + * @memberof Model + * @static + * @param {IModel} message Model message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + Model.encode = function encode(message, writer) { + if (!writer) + writer = $Writer.create(); + if (message.id != null && message.hasOwnProperty("id")) + writer.uint32(/* id 1, wireType 2 =*/10).string(message.id); + if (message.name != null && message.hasOwnProperty("name")) + writer.uint32(/* id 2, wireType 2 =*/18).string(message.name); + if (message.kerasVersion != null && message.hasOwnProperty("kerasVersion")) + writer.uint32(/* id 3, wireType 2 =*/26).string(message.kerasVersion); + if (message.backend != null && message.hasOwnProperty("backend")) + writer.uint32(/* id 4, wireType 2 =*/34).string(message.backend); + if (message.modelConfig != null && message.hasOwnProperty("modelConfig")) + writer.uint32(/* id 5, wireType 2 =*/42).string(message.modelConfig); + if (message.modelWeights != null && message.modelWeights.length) + for (let i = 0; i < message.modelWeights.length; ++i) + $root.Weights.encode(message.modelWeights[i], writer.uint32(/* id 6, wireType 2 =*/50).fork()).ldelim(); + return writer; + }; + + /** + * Encodes the specified Model message, length delimited. Does not implicitly {@link Model.verify|verify} messages. + * @function encodeDelimited + * @memberof Model + * @static + * @param {IModel} message Model message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + Model.encodeDelimited = function encodeDelimited(message, writer) { + return this.encode(message, writer).ldelim(); + }; + + /** + * Decodes a Model message from the specified reader or buffer. + * @function decode + * @memberof Model + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @param {number} [length] Message length if known beforehand + * @returns {Model} Model + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + Model.decode = function decode(reader, length) { + if (!(reader instanceof $Reader)) + reader = $Reader.create(reader); + let end = length === undefined ? reader.len : reader.pos + length, message = new $root.Model(); + while (reader.pos < end) { + let tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + message.id = reader.string(); + break; + case 2: + message.name = reader.string(); + break; + case 3: + message.kerasVersion = reader.string(); + break; + case 4: + message.backend = reader.string(); + break; + case 5: + message.modelConfig = reader.string(); + break; + case 6: + if (!(message.modelWeights && message.modelWeights.length)) + message.modelWeights = []; + message.modelWeights.push($root.Weights.decode(reader, reader.uint32())); + break; + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }; + + /** + * Decodes a Model message from the specified reader or buffer, length delimited. + * @function decodeDelimited + * @memberof Model + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @returns {Model} Model + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + Model.decodeDelimited = function decodeDelimited(reader) { + if (!(reader instanceof $Reader)) + reader = new $Reader(reader); + return this.decode(reader, reader.uint32()); + }; + + /** + * Verifies a Model message. + * @function verify + * @memberof Model + * @static + * @param {Object.} message Plain object to verify + * @returns {string|null} `null` if valid, otherwise the reason why it is not + */ + Model.verify = function verify(message) { + if (typeof message !== "object" || message === null) + return "object expected"; + if (message.id != null && message.hasOwnProperty("id")) + if (!$util.isString(message.id)) + return "id: string expected"; + if (message.name != null && message.hasOwnProperty("name")) + if (!$util.isString(message.name)) + return "name: string expected"; + if (message.kerasVersion != null && message.hasOwnProperty("kerasVersion")) + if (!$util.isString(message.kerasVersion)) + return "kerasVersion: string expected"; + if (message.backend != null && message.hasOwnProperty("backend")) + if (!$util.isString(message.backend)) + return "backend: string expected"; + if (message.modelConfig != null && message.hasOwnProperty("modelConfig")) + if (!$util.isString(message.modelConfig)) + return "modelConfig: string expected"; + if (message.modelWeights != null && message.hasOwnProperty("modelWeights")) { + if (!Array.isArray(message.modelWeights)) + return "modelWeights: array expected"; + for (let i = 0; i < message.modelWeights.length; ++i) { + let error = $root.Weights.verify(message.modelWeights[i]); + if (error) + return "modelWeights." + error; + } + } + return null; + }; + + /** + * Creates a Model message from a plain object. Also converts values to their respective internal types. + * @function fromObject + * @memberof Model + * @static + * @param {Object.} object Plain object + * @returns {Model} Model + */ + Model.fromObject = function fromObject(object) { + if (object instanceof $root.Model) + return object; + let message = new $root.Model(); + if (object.id != null) + message.id = String(object.id); + if (object.name != null) + message.name = String(object.name); + if (object.kerasVersion != null) + message.kerasVersion = String(object.kerasVersion); + if (object.backend != null) + message.backend = String(object.backend); + if (object.modelConfig != null) + message.modelConfig = String(object.modelConfig); + if (object.modelWeights) { + if (!Array.isArray(object.modelWeights)) + throw TypeError(".Model.modelWeights: array expected"); + message.modelWeights = []; + for (let i = 0; i < object.modelWeights.length; ++i) { + if (typeof object.modelWeights[i] !== "object") + throw TypeError(".Model.modelWeights: object expected"); + message.modelWeights[i] = $root.Weights.fromObject(object.modelWeights[i]); + } + } + return message; + }; + + /** + * Creates a plain object from a Model message. Also converts values to other types if specified. + * @function toObject + * @memberof Model + * @static + * @param {Model} message Model + * @param {$protobuf.IConversionOptions} [options] Conversion options + * @returns {Object.} Plain object + */ + Model.toObject = function toObject(message, options) { + if (!options) + options = {}; + let object = {}; + if (options.arrays || options.defaults) + object.modelWeights = []; + if (options.defaults) { + object.id = ""; + object.name = ""; + object.kerasVersion = ""; + object.backend = ""; + object.modelConfig = ""; + } + if (message.id != null && message.hasOwnProperty("id")) + object.id = message.id; + if (message.name != null && message.hasOwnProperty("name")) + object.name = message.name; + if (message.kerasVersion != null && message.hasOwnProperty("kerasVersion")) + object.kerasVersion = message.kerasVersion; + if (message.backend != null && message.hasOwnProperty("backend")) + object.backend = message.backend; + if (message.modelConfig != null && message.hasOwnProperty("modelConfig")) + object.modelConfig = message.modelConfig; + if (message.modelWeights && message.modelWeights.length) { + object.modelWeights = []; + for (let j = 0; j < message.modelWeights.length; ++j) + object.modelWeights[j] = $root.Weights.toObject(message.modelWeights[j], options); + } + return object; + }; + + /** + * Converts this Model to JSON. + * @function toJSON + * @memberof Model + * @instance + * @returns {Object.} JSON object + */ + Model.prototype.toJSON = function toJSON() { + return this.constructor.toObject(this, $protobuf.util.toJSONOptions); + }; + + return Model; +})(); + +export { $root as default }; diff --git a/model/types.ts b/model/types.ts new file mode 100644 index 0000000..40e8614 --- /dev/null +++ b/model/types.ts @@ -0,0 +1,50 @@ +import { NetworkJSON } from "../core/types.ts"; +import { NeuralNetwork } from "../mod.ts"; + + +export interface ModelFormat { + load: (path: string) => Promise; + save: (path: string, net: NeuralNetwork) => Promise; +} + +export type KerasLayerType = "InputLayer" | "Conv2D"; + +export interface KerasLayerConfig { + name?: string; + trainable?: boolean; + filters?: number; + // deno-lint-ignore no-explicit-any + kernel_size?: any[]; + strides?: number[]; + padding?: string; + data_format?: string; + // deno-lint-ignore no-explicit-any + dilation_rate?: any[]; + activation?: string; + use_bias?: boolean; + // deno-lint-ignore no-explicit-any + kernel_initializer?: any; + // deno-lint-ignore no-explicit-any + bias_initializer?: any; + // deno-lint-ignore no-explicit-any + kernel_regularizer?: any; + // deno-lint-ignore no-explicit-any + bias_regularizer?: any; + // deno-lint-ignore no-explicit-any + activity_regularizer?: any; + // deno-lint-ignore no-explicit-any + kernel_constraints?: any; + // deno-lint-ignore no-explicit-any + bias_constraints?: any; + // deno-lint-ignore no-explicit-any + batch_input_shape?: any[]; + dtype?: string; + sparse?: boolean; +} +export interface KerasLayer { + name: string; + class_name: KerasLayerType; + config: KerasLayerConfig; + // deno-lint-ignore no-explicit-any + inbound_nodes: any[] +} \ No newline at end of file diff --git a/model/util.ts b/model/util.ts new file mode 100644 index 0000000..f962c41 --- /dev/null +++ b/model/util.ts @@ -0,0 +1,5 @@ +export function staticImplements() { + return (constructor: U) => { + constructor; + }; +} diff --git a/src/cpu/layers/conv.ts b/src/cpu/layers/conv.ts deleted file mode 100644 index b403648..0000000 --- a/src/cpu/layers/conv.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { ConvLayerConfig, LayerJSON, Size, Size2D } from "../../types.ts"; -import { iterate2D } from "../../util.ts"; -import { CPUMatrix } from "../matrix.ts"; - -// https://github.com/mnielsen/neural-networks-and-deep-learning -// https://ml-cheatsheet.readthedocs.io/en/latest/backpropagation.html#applying-the-chain-rule - -/** - * Convolutional layer. - */ -export class ConvCPULayer { - outputSize!: Size2D; - padding: number; - stride: number; - - input!: CPUMatrix; - kernel!: CPUMatrix; - padded!: CPUMatrix; - output!: CPUMatrix; - - constructor(config: ConvLayerConfig) { - this.kernel = new CPUMatrix( - config.kernel, - config.kernelSize.x, - config.kernelSize.y, - ); - this.padding = config.padding || 0; - this.stride = config.stride || 1; - } - - reset(_batches: number) { - } - - initialize(inputSize: Size, _batches: number) { - const wp = (inputSize as Size2D).x + 2 * this.padding; - const hp = (inputSize as Size2D).y + 2 * this.padding; - if (this.padding > 0) { - this.padded = CPUMatrix.with(wp, hp); - this.padded.fill(255); - } - const wo = 1 + Math.floor((wp - this.kernel.x) / this.stride); - const ho = 1 + Math.floor((hp - this.kernel.y) / this.stride); - this.output = CPUMatrix.with(wo, ho); - this.outputSize = { x: wo, y: ho }; - } - - feedForward(input: CPUMatrix): CPUMatrix { - if (this.padding > 0) { - iterate2D(input, (i: number, j: number) => { - const idx = this.padded.x * (this.padding + j) + this.padding + i; - this.padded.data[idx] = input.data[j * input.x + i]; - }); - } else { - this.padded = input; - } - iterate2D(this.output, (i: number, j: number) => { - let sum = 0; - iterate2D(this.kernel, (x: number, y: number) => { - const k = this.padded.x * (j * this.stride + y) + (i * this.stride + x); - const l = y * this.kernel.x + x; - sum += this.padded.data[k] * this.kernel.data[l]; - }); - this.output.data[j * this.output.x + i] = sum; - }); - return this.output; - } - - backPropagate(_error: CPUMatrix, _rate: number) { - } - - toJSON(): LayerJSON { - return { - outputSize: this.outputSize, - type: "conv", - }; - } -} diff --git a/src/cpu/layers/pool.ts b/src/cpu/layers/pool.ts deleted file mode 100644 index 0beab15..0000000 --- a/src/cpu/layers/pool.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { LayerJSON, PoolLayerConfig, Size, Size2D } from "../../types.ts"; -import { CPUMatrix } from "../matrix.ts"; - -// https://github.com/mnielsen/neural-networks-and-deep-learning -// https://ml-cheatsheet.readthedocs.io/en/latest/backpropagation.html#applying-the-chain-rule - -/** - * MaxPool layer. - */ -export class PoolCPULayer { - outputSize!: Size2D; - stride: number; - - input!: CPUMatrix; - output!: CPUMatrix; - - constructor(config: PoolLayerConfig) { - this.stride = config.stride; - } - - reset(_batches: number) { - } - - initialize(inputSize: Size, _batches: number) { - const w = (inputSize as Size2D).x / this.stride; - const h = (inputSize as Size2D).y / this.stride; - this.output = CPUMatrix.with(w, h); - this.outputSize = { x: w, y: h }; - } - - feedForward(input: CPUMatrix) { - for (let i = 0; i < this.output.x; i++) { - for (let j = 0; j < this.output.y; j++) { - const pool = []; - for (let x = 0; x < this.stride; x++) { - for (let y = 0; y < this.stride; y++) { - const idx = (j * this.stride + y) * input.x + - i * this.stride + x; - pool.push(input.data[idx]); - } - } - this.output.data[j * this.output.x + i] = Math.max(...pool); - } - } - return this.output; - } - - backPropagate(_error: CPUMatrix, _rate: number) { - } - - toJSON(): LayerJSON { - return { - outputSize: this.outputSize, - type: "pool", - }; - } -} diff --git a/src/cpu/mod.ts b/src/cpu/mod.ts deleted file mode 100644 index bec0734..0000000 --- a/src/cpu/mod.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Backend, NetworkConfig } from "../types.ts"; - -import { CPUBackend } from "./backend.ts"; -// import { Backend, NetworkConfig } from "../types.ts"; - -// deno-lint-ignore require-await -export async function CPU(config: NetworkConfig): Promise { - return new CPUBackend(config); -} diff --git a/src/gpu/mod.ts b/src/gpu/mod.ts deleted file mode 100644 index 89dcfd9..0000000 --- a/src/gpu/mod.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { Backend, NetworkConfig } from "../types.ts"; - -import { Core, WebGPUBackend } from "../../deps.ts"; -import { GPUBackend } from "./backend.ts"; - -export async function GPU(config: NetworkConfig): Promise { - const silent = config.silent; - const core = new Core(); - await core.initialize(); - const backend = core.backends.get("webgpu")! as WebGPUBackend; - if (!backend.adapter) throw new Error("No backend adapter found!"); - if (!silent) console.log(`Using adapter: ${backend.adapter}`); - const features = [...backend.adapter.features.values()]; - if (!silent) console.log(`Supported features: ${features.join(", ")}`); - return new GPUBackend(config, backend); -} diff --git a/src/native/layers/dense.ts b/src/native/layers/dense.ts deleted file mode 100644 index 23bd0ac..0000000 --- a/src/native/layers/dense.ts +++ /dev/null @@ -1,32 +0,0 @@ -import ffi from "../ffi.ts"; - -import { DenseLayerConfig } from "../../types.ts"; -import { to1D } from "../../util.ts"; - -const { - layer_dense, -} = ffi; - -const C_ACTIVATION: { [key : string] : number} = { - "sigmoid": 0, - "tanh": 1, - "relu": 2, - "relu6": 5, - "leakyrelu": 4, - "elu": 6, - "linear": 3, - "selu": 7, - } -// export type Activation = keyof typeof C_ACTIVATION; - -export class DenseNativeLayer { - #ptr: Deno.PointerValue; - - get unsafePointer() { - return this.#ptr; - } - - constructor(config: DenseLayerConfig) { - this.#ptr = layer_dense(to1D(config.size), C_ACTIVATION[config.activation]); - } -} \ No newline at end of file diff --git a/src/native/mod.ts b/src/native/mod.ts deleted file mode 100644 index 527330a..0000000 --- a/src/native/mod.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Backend, NetworkConfig } from "../types.ts"; - -import { NativeBackend } from "./backend.ts"; -// import { Backend, NetworkConfig } from "../types.ts"; - -// deno-lint-ignore require-await -export async function Native(config: NetworkConfig): Promise { - // deno-lint-ignore no-explicit-any - return new NativeBackend(config as any); -} diff --git a/src/types.ts b/src/types.ts deleted file mode 100644 index 4ffe36f..0000000 --- a/src/types.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { DataType, DataTypeArray } from "../deps.ts"; -import { ConvLayer, DenseLayer, PoolLayer } from "./mod.ts"; -import { ConvCPULayer } from "./cpu/layers/conv.ts"; -import { DenseCPULayer } from "./cpu/layers/dense.ts"; -import { DenseGPULayer } from "./gpu/layers/dense.ts"; -import { CPUActivationFn } from "./cpu/activation.ts"; -import { GPUActivationFn } from "./gpu/activation.ts"; -import { GPUMatrix } from "./gpu/matrix.ts"; -import { CPUMatrix } from "./cpu/matrix.ts"; -import { PoolCPULayer } from "./cpu/layers/pool.ts"; - -export interface LayerJSON { - outputSize: number | Size2D; - activation?: CPUActivationFn | GPUActivationFn; - type: string; -} - -export interface NetworkJSON { - type: "NeuralNetwork"; - sizes: (number | Size2D)[]; - input: Size | undefined; - layers: LayerJSON[]; - output: LayerJSON; -} - -export interface Backend { - // deno-lint-ignore no-explicit-any - addLayer(layer: Layer | any): void; - // getOutput(): DataTypeArray | any; - train( - // deno-lint-ignore no-explicit-any - datasets: DataSet[] | any, - epochs: number, - batches: number, - learningRate: number, - ): void; - // deno-lint-ignore no-explicit-any - predict(input: DataTypeArray | any): DataTypeArray | any; - save(input: string): void; - toJSON(): NetworkJSON | undefined; - // deno-lint-ignore no-explicit-any - getWeights(): (GPUMatrix | CPUMatrix | any)[]; - // deno-lint-ignore no-explicit-any - getBiases(): (GPUMatrix | CPUMatrix | any)[]; -} - -/** - * NetworkConfig represents the configuration of a neural network. - */ -export interface NetworkConfig { - input?: Size; - layers: Layer[]; - cost: Cost; - silent?: boolean; -} - -export type Layer = DenseLayer | ConvLayer | PoolLayer; - -export type CPULayer = ConvCPULayer | DenseCPULayer | PoolCPULayer; - -export type GPULayer = DenseGPULayer; - -export interface DenseLayerConfig { - size: Size; - activation: Activation; -} - -export interface ConvLayerConfig { - activation: Activation; - kernel: DataTypeArray; - kernelSize: Size2D; - padding?: number; - stride?: number; -} - -export interface PoolLayerConfig { - stride: number; -} - -export type Size = number | Size2D; - -export type Size2D = { x: number; y: number }; - -/** - * Activation functions are used to transform the output of a layer into a new output. - */ -export type Activation = - | "sigmoid" - | "tanh" - | "relu" - | "relu6" - | "leakyrelu" - | "elu" - | "linear" - | "selu"; - -export type Cost = "crossentropy" | "hinge" | "mse"; - -export type Shape = number; -/** - * NumberArray is a typed array of numbers. - */ -export type NumberArray = - | DataTypeArray - | Array - // TODO: fix - // deno-lint-ignore no-explicit-any - | any; -/** - * DataSet is a container for training data. - */ -export type DataSet = { - inputs: NumberArray; - outputs: NumberArray; -}; diff --git a/src/util.ts b/src/util.ts deleted file mode 100644 index dd9e0ed..0000000 --- a/src/util.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { DataType, DataTypeArray, DataTypeArrayConstructor } from "../deps.ts"; -import { CPUMatrix } from "./cpu/matrix.ts"; -import type { Size, Size2D } from "./types.ts"; - -export function getType(type: DataTypeArray) { - return ( - type instanceof Uint32Array - ? "u32" - : type instanceof Int32Array - ? "i32" - : "f32" - ); -} -export function fromType(type: string) { - return ( - type === "u32" - ? Uint32Array - : type === "i32" - ? Int32Array - : type === "f32" - ? Float32Array - : Uint32Array - ) as DataTypeArrayConstructor; -} -export function toType(type: string) { - return ( - type === "u32" - ? Uint32Array - : type === "i32" - ? Int32Array - : type === "f32" - ? Float32Array - : Uint32Array - ) as DataTypeArrayConstructor; -} - -export class ActivationError extends Error { - constructor(activation: string) { - super( - `Unknown activation function: ${activation}. Available: "sigmoid", "tanh", "relu", "relu6" , "leakyrelu", "elu", "linear", "selu"`, - ); - } -} - -export const zeros = (size: number): Float32Array => new Float32Array(size); -export const zeros2D = (width: number, height: number): Float32Array[] => { - const result: Float32Array[] = new Array(height); - for (let y = 0; y < height; y++) { - result[y] = zeros(width); - } - return result; -}; -export const zeros3D = ( - width: number, - height: number, - depth: number, -): Float32Array[][] => { - const result: Float32Array[][] = new Array(depth); - for (let z = 0; z < depth; z++) { - result[z] = zeros2D(width, height); - } - return result; -}; -export const ones = (size: number): Float32Array => - new Float32Array(size).fill(1); -export const ones2D = (width: number, height: number): Float32Array[] => { - const result = new Array(height); - for (let y = 0; y < height; y++) { - result[y] = ones(width); - } - return result; -}; -export const values = (size: number, value: number): Float32Array => - new Float32Array(size).fill(value); -export const values2D = ( - width: number, - height: number, - value: number, -): Float32Array[] => { - const result: Float32Array[] = new Array(height); - for (let y = 0; y < height; y++) { - result[y] = values(width, value); - } - return result; -}; -export const values3D = ( - width: number, - height: number, - depth: number, - value: number, -): Float32Array[][] => { - const result: Float32Array[][] = new Array(depth); - for (let z = 0; z < depth; z++) { - result[z] = values2D(width, height, value); - } - return result; -}; -export const randomWeight = (): number => Math.random() * 0.4 - 0.2; -export const randomFloat = (min: number, max: number): number => - Math.random() * (max - min) + min; -export const gaussRandom = (): number => { - if (gaussRandom.returnV) { - gaussRandom.returnV = false; - return gaussRandom.vVal; - } - const u = 2 * Math.random() - 1; - const v = 2 * Math.random() - 1; - const r = u * u + v * v; - if (r === 0 || r > 1) { - return gaussRandom(); - } - const c = Math.sqrt((-2 * Math.log(r)) / r); - gaussRandom.vVal = v * c; - gaussRandom.returnV = true; - return u * c; -}; -gaussRandom.returnV = false; -gaussRandom.vVal = 0; - -export const randomInteger = (min: number, max: number): number => - Math.floor(Math.random() * (max - min) + min); - -export const randomN = (mu: number, std: number): number => - mu + gaussRandom() * std; -export const max = ( - values: - | Float32Array - | { - [key: string]: number; - }, -): number => - (Array.isArray(values) || values instanceof Float32Array) - ? Math.max(...values) - : Math.max(...Object.values(values)); -export const mse = (errors: Float32Array): number => { - let sum = 0; - for (let i = 0; i < errors.length; i++) { - sum += errors[i] ** 2; - } - return sum / errors.length; -}; - -export function to1D(size: Size): number { - const size2d = (size as Size2D); - if (size2d.y) { - return size2d.x * size2d.y; - } else { - return size as number; - } -} - -export function iterate2D( - mat: { x: number; y: number } | CPUMatrix, - callback: (i: number, j: number) => void, -): void { - for (let i = 0; i < mat.x; i++) { - for (let j = 0; j < mat.y; j++) { - callback(i, j); - } - } -} - -export function iterate1D(length: number, callback: (i: number) => void): void { - for (let i = 0; i < length; i++) { - callback(i); - } -} diff --git a/tests/conv.ts b/tests/conv.ts deleted file mode 100644 index 9383aba..0000000 --- a/tests/conv.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { ConvLayer, DenseLayer, NeuralNetwork } from "../mod.ts"; -import { ConvCPULayer } from "../src/cpu/layers/conv.ts"; -import { CPUMatrix } from "../src/cpu/matrix.ts"; -import { CPUBackend } from "../src/cpu/backend.ts"; -import { CPU } from "../backends/cpu.ts"; -import { PoolCPULayer } from "../src/cpu/layers/pool.ts"; -import { PoolLayer } from "../src/mod.ts"; - -import { decode } from "https://deno.land/x/pngs@0.1.1/mod.ts"; -import { DataTypeArray } from "../deps.ts"; - - -import { Canvas } from "https://deno.land/x/neko@1.1.3/canvas/mod.ts"; - -const canvas = new Canvas({ - title: "Netsaur Convolutions", - width: 600, - height: 600, - fps: 60, -}); - -const ctx = canvas.getContext("2d"); -ctx.fillStyle = "white"; -ctx.fillRect(0, 0, 600, 600); - -const dim = 28; - -//Credit: Hashrock (https://github.com/hashrock) -const img = decode(Deno.readFileSync("./tests/deno.png")).image; -const buf = new Float32Array(dim * dim) as DataTypeArray<"f32">; - -for (let i = 0; i < dim * dim; i++) { - buf[i] = img[i * 4]; -} - -const kernel = new Float32Array([ - -1, - 1, - 0, - -1, - 1, - 0, - -1, - 1, - 0, -]) as DataTypeArray<"f32">; - -const net = await new NeuralNetwork({ - silent: true, - layers: [ - new ConvLayer({ - activation: "sigmoid", - kernel: kernel, - kernelSize: { x: 3, y: 3 }, - padding: 1, - stride: 1, - }), - new PoolLayer({ stride: 2 }), - new DenseLayer({ size: 1, activation: "sigmoid" }), - ], - cost: "crossentropy", - input: 2, -}).setupBackend(CPU); - -const input = new CPUMatrix(buf, dim, dim); -const network = net.backend as CPUBackend; -const conv = network.layers[0] as ConvCPULayer; -const pool = network.layers[1] as PoolCPULayer; -network.initialize({ x: dim, y: dim }, 1); -network.layers[0].feedForward(input); -network.layers[1].feedForward(conv.output); -const cv = conv.output; -const out = pool.output; - -for (let i = 0; i < dim; i++) { - for (let j = 0; j < dim; j++) { - const pixel = buf[j * dim + i]; - ctx.fillStyle = `rgb(${pixel}, ${pixel}, ${pixel})`; - ctx.fillRect(i * 10, j * 10, 10, 10); - } -} - -for (let i = 0; i < cv.x; i++) { - for (let j = 0; j < cv.y; j++) { - const pixel = Math.round(Math.max(Math.min(cv.data[j * cv.x + i], 255), 0)); - ctx.fillStyle = `rgb(${pixel}, ${pixel}, ${pixel})`; - ctx.fillRect(i * 10 + dim * 10, j * 10, 10, 10); - } -} - -for (let i = 0; i < out.x; i++) { - for (let j = 0; j < out.y; j++) { - const pixel = Math.round( - Math.max(Math.min(out.data[j * out.x + i], 255), 0), - ); - ctx.fillStyle = `rgb(${pixel}, ${pixel}, ${pixel})`; - ctx.fillRect(i * 20 + dim * 10, j * 20 + dim * 10, 20, 20); - } -} diff --git a/tests/linear.ts b/tests/linear.ts index c8f98a2..ab545c8 100644 --- a/tests/linear.ts +++ b/tests/linear.ts @@ -1,5 +1,5 @@ import { NeuralNetwork, DenseLayer } from "../mod.ts"; -import { Native, Matrix } from "../backends/native.ts"; +import { Native, Matrix } from "../backends/native/mod.ts"; const start = performance.now(); diff --git a/types.ts b/types.ts new file mode 100644 index 0000000..b7800a2 --- /dev/null +++ b/types.ts @@ -0,0 +1 @@ +export * from "./core/types.ts"; \ No newline at end of file