From ee5fc8d544ffa8e76860c251f9640c401f4710cd Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Sat, 15 Jun 2024 09:54:57 -0400 Subject: [PATCH 01/29] Implement a loss function for GPU --- src/neural-network-gpu.ts | 8 +++++--- src/neural-network.ts | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/neural-network-gpu.ts b/src/neural-network-gpu.ts index 01bd6ea9..ab32cecf 100644 --- a/src/neural-network-gpu.ts +++ b/src/neural-network-gpu.ts @@ -96,7 +96,7 @@ function weightedSumTanh( return Math.tanh(sum); } -function calcErrorOutput(output: number, target: number): number { +function loss(output: number, target: number): number { return target - output; } @@ -400,6 +400,8 @@ export class NeuralNetworkGPU< ); } + let _loss = typeof this.loss === "function" ? this.loss : loss; + calcDeltas = alias( utils.getMinifySafeName(() => calcDeltas), calcDeltas @@ -411,7 +413,7 @@ export class NeuralNetworkGPU< // @ts-expect-error this.backwardPropagate[this.outputLayer] = this.gpu.createKernelMap( { - error: calcErrorOutput, + error: _loss, }, function ( this: IKernelFunctionThis, @@ -422,7 +424,7 @@ export class NeuralNetworkGPU< const target = targets[this.thread.x]; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error - return calcDeltas(calcErrorOutput(output, target), output); + return calcDeltas(loss(output, target), output); }, { output: [this.sizes[this.outputLayer]], diff --git a/src/neural-network.ts b/src/neural-network.ts index 5e851d40..6c421764 100644 --- a/src/neural-network.ts +++ b/src/neural-network.ts @@ -13,6 +13,8 @@ import { mse } from './utilities/mse'; import { randos } from './utilities/randos'; import { zeros } from './utilities/zeros'; +export type LossFunction = (actual: number, expected: number) => number; + type NeuralNetworkFormatter = | ((v: INumberHash) => Float32Array) | ((v: number[]) => Float32Array); @@ -107,6 +109,7 @@ export interface INeuralNetworkTrainOptions { errorThresh: number; log: boolean | ((status: INeuralNetworkState) => void); logPeriod: number; + loss: boolean | LossFunction; leakyReluAlpha: number; learningRate: number; momentum: number; @@ -126,6 +129,7 @@ export function trainDefaults(): INeuralNetworkTrainOptions { errorThresh: 0.005, // the acceptable error percentage from training data log: false, // true to use console.log, when a function is supplied it is used logPeriod: 10, // iterations between logging out + loss: false, leakyReluAlpha: 0.01, learningRate: 0.3, // multiply's against the input and the delta then adds to momentum momentum: 0.1, // multiply's against the specified "change" then adds to learning rate for change @@ -188,6 +192,8 @@ export class NeuralNetwork< return this.calculateDeltas(output); }; + loss: boolean | LossFunction = false; + // adam biasChangesLow: Float32Array[] = []; biasChangesHigh: Float32Array[] = []; @@ -470,6 +476,10 @@ export class NeuralNetwork< const val = options.logPeriod; return typeof val === 'number' && val > 0; }, + loss: () => { + const val = options.loss; + return typeof val === 'function' || typeof val === 'boolean'; + }, leakyReluAlpha: () => { const val = options.leakyReluAlpha; return typeof val === 'number' && val > 0 && val < 1; @@ -666,6 +676,8 @@ export class NeuralNetwork< data: Array, Partial>>, options: Partial = {} ): INeuralNetworkState { + this.loss = options.loss ?? false; + const { preparedData, status, endTime } = this.prepTraining( data as Array>, options @@ -826,6 +838,33 @@ export class NeuralNetwork< } } + _calculateDeltasLoss(target: Float32Array): void { + for (let layer = this.outputLayer; layer >= 0; layer--) { + const activeSize = this.sizes[layer]; + const activeOutput = this.outputs[layer]; + const activeError = this.errors[layer]; + const activeDeltas = this.deltas[layer]; + const nextLayer = this.weights[layer + 1]; + + for (let node = 0; node < activeSize; node++) { + const output = activeOutput[node]; + + let error = 0; + if (layer === this.outputLayer) { + if (typeof this.loss === "function") error = this.loss(output, target[node]); + else error = target[node] - output; + } else { + const deltas = this.deltas[layer + 1]; + for (let k = 0; k < deltas.length; k++) { + error += deltas[k] * nextLayer[k][node]; + } + } + activeError[node] = error; + activeDeltas[node] = error * output * (1 - output); + } + } + } + /** * * Changes weights of networks From 6baf8c621686df95c076823c4f21ad8ab7ec35ed Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Sat, 15 Jun 2024 21:04:58 -0400 Subject: [PATCH 02/29] Add partial support for loss functions Add support for loss functions in CPU-based neural networks --- src/neural-network.ts | 93 +++++++++++++++++++++++------------------ src/utilities/loss.ts | 10 +++++ src/utilities/to-svg.ts | 1 + 3 files changed, 63 insertions(+), 41 deletions(-) create mode 100644 src/utilities/loss.ts diff --git a/src/neural-network.ts b/src/neural-network.ts index 6c421764..e4309812 100644 --- a/src/neural-network.ts +++ b/src/neural-network.ts @@ -12,8 +12,7 @@ import { max } from './utilities/max'; import { mse } from './utilities/mse'; import { randos } from './utilities/randos'; import { zeros } from './utilities/zeros'; - -export type LossFunction = (actual: number, expected: number) => number; +import { LossFunction, LossFunctionInputs, LossFunctionState } from './utilities/loss'; type NeuralNetworkFormatter = | ((v: INumberHash) => Float32Array) @@ -70,6 +69,7 @@ export interface INeuralNetworkOptions { outputSize: number; binaryThresh: number; hiddenLayers?: number[]; + lossStateSize: number; } export function defaults(): INeuralNetworkOptions { @@ -77,6 +77,7 @@ export function defaults(): INeuralNetworkOptions { inputSize: 0, outputSize: 0, binaryThresh: 0.5, + lossStateSize: 1 }; } @@ -110,6 +111,8 @@ export interface INeuralNetworkTrainOptions { log: boolean | ((status: INeuralNetworkState) => void); logPeriod: number; loss: boolean | LossFunction; + lossState?: LossFunctionState; + lossStateSize: number; leakyReluAlpha: number; learningRate: number; momentum: number; @@ -130,6 +133,7 @@ export function trainDefaults(): INeuralNetworkTrainOptions { log: false, // true to use console.log, when a function is supplied it is used logPeriod: 10, // iterations between logging out loss: false, + lossStateSize: 1, leakyReluAlpha: 0.01, learningRate: 0.3, // multiply's against the input and the delta then adds to momentum momentum: 0.1, // multiply's against the specified "change" then adds to learning rate for change @@ -180,16 +184,19 @@ export class NeuralNetwork< _formatInput: NeuralNetworkFormatter | null = null; _formatOutput: NeuralNetworkFormatter | null = null; + _lossState: LossFunctionState; + runInput: (input: Float32Array) => Float32Array = (input: Float32Array) => { this.setActivation(); return this.runInput(input); }; - calculateDeltas: (output: Float32Array) => void = ( - output: Float32Array + calculateDeltas: (output: Float32Array, input: LossFunctionInputs) => void = ( + output: Float32Array, + input: LossFunctionInputs ): void => { this.setActivation(); - return this.calculateDeltas(output); + return this.calculateDeltas(output, input); }; loss: boolean | LossFunction = false; @@ -211,6 +218,9 @@ export class NeuralNetwork< if (inputSize && outputSize) { this.sizes = [inputSize].concat(hiddenLayers ?? []).concat([outputSize]); } + + const { lossStateSize } = this.options ?? 0; + this._lossState = this.trainOpts.lossState ?? this.replaceLossState(lossStateSize); } /** @@ -287,6 +297,10 @@ export class NeuralNetwork< return this.sizes.length > 0; } + public get lossState(): LossFunctionState { + return this._lossState; + } + run(input: Partial): OutputType { if (!this.isRunnable) { throw new Error('network not runnable'); @@ -724,7 +738,7 @@ export class NeuralNetwork< this.runInput(value.input); // back propagate - this.calculateDeltas(value.output); + this.calculateDeltas(value.output, value.input); this.adjustWeights(); if (logErrorRate) { @@ -733,7 +747,7 @@ export class NeuralNetwork< return null; } - _calculateDeltasSigmoid(target: Float32Array): void { + _calculateDeltasSigmoid(target: Float32Array, input: LossFunctionInputs): void { for (let layer = this.outputLayer; layer >= 0; layer--) { const activeSize = this.sizes[layer]; const activeOutput = this.outputs[layer]; @@ -746,7 +760,8 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - error = target[node] - output; + if (typeof this.loss === "function") error = this.loss.call({thread: {x: node}}, output, target[node], input, this.lossState); + else error = target[node] - output; } else { const deltas = this.deltas[layer + 1]; for (let k = 0; k < deltas.length; k++) { @@ -759,7 +774,7 @@ export class NeuralNetwork< } } - _calculateDeltasRelu(target: Float32Array): void { + _calculateDeltasRelu(target: Float32Array, input: LossFunctionInputs): void { for (let layer = this.outputLayer; layer >= 0; layer--) { const currentSize = this.sizes[layer]; const currentOutputs = this.outputs[layer]; @@ -773,7 +788,8 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - error = target[node] - output; + if (typeof this.loss === "function") error = this.loss.call({thread: {x: node}}, output, target[node], input, this.lossState); + else error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { error += nextDeltas[k] * nextWeights[k][node]; @@ -785,7 +801,7 @@ export class NeuralNetwork< } } - _calculateDeltasLeakyRelu(target: Float32Array): void { + _calculateDeltasLeakyRelu(target: Float32Array, input: LossFunctionInputs): void { const alpha = this.trainOpts.leakyReluAlpha; for (let layer = this.outputLayer; layer >= 0; layer--) { const currentSize = this.sizes[layer]; @@ -800,7 +816,8 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - error = target[node] - output; + if (typeof this.loss === "function") error = this.loss.call({thread: {x: node}}, output, target[node], input, this.lossState); + else error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { error += nextDeltas[k] * nextWeights[k][node]; @@ -812,7 +829,7 @@ export class NeuralNetwork< } } - _calculateDeltasTanh(target: Float32Array): void { + _calculateDeltasTanh(target: Float32Array, input: LossFunctionInputs): void { for (let layer = this.outputLayer; layer >= 0; layer--) { const currentSize = this.sizes[layer]; const currentOutputs = this.outputs[layer]; @@ -826,7 +843,8 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - error = target[node] - output; + if (typeof this.loss === "function") error = this.loss.call({thread: {x: node}}, output, target[node], input, this.lossState); + else error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { error += nextDeltas[k] * nextWeights[k][node]; @@ -838,33 +856,6 @@ export class NeuralNetwork< } } - _calculateDeltasLoss(target: Float32Array): void { - for (let layer = this.outputLayer; layer >= 0; layer--) { - const activeSize = this.sizes[layer]; - const activeOutput = this.outputs[layer]; - const activeError = this.errors[layer]; - const activeDeltas = this.deltas[layer]; - const nextLayer = this.weights[layer + 1]; - - for (let node = 0; node < activeSize; node++) { - const output = activeOutput[node]; - - let error = 0; - if (layer === this.outputLayer) { - if (typeof this.loss === "function") error = this.loss(output, target[node]); - else error = target[node] - output; - } else { - const deltas = this.deltas[layer + 1]; - for (let k = 0; k < deltas.length; k++) { - error += deltas[k] * nextLayer[k][node]; - } - } - activeError[node] = error; - activeDeltas[node] = error * output * (1 - output); - } - } - } - /** * * Changes weights of networks @@ -1367,4 +1358,24 @@ export class NeuralNetwork< input: Partial ) => OutputType; } + + private createLossState( + lossStateSize: number + ): LossFunctionState { + const lossState: LossFunctionState = []; + for (let layer = 0; layer < this.sizes.length; layer++) { + lossState[layer] = []; + for (let neuron = 0; neuron < this.sizes.length; neuron++) { + lossState[layer][neuron] = new Float32Array(lossStateSize); + } + } + return lossState; + } + + private replaceLossState( + lossState: number | LossFunctionState + ): LossFunctionState { + if (typeof lossState === "number") return this._lossState = this.createLossState(lossState); + return this._lossState = lossState; + } } diff --git a/src/utilities/loss.ts b/src/utilities/loss.ts new file mode 100644 index 00000000..3e07072f --- /dev/null +++ b/src/utilities/loss.ts @@ -0,0 +1,10 @@ +export type LossFunctionInputs = number[] | number[][] | number[][][] | Float32Array | Float32Array[] | Float32Array[][]; + +export type LossFunctionState = number[][][] | Float32Array[][]; + +export type LossFunction = ( + actual: number, + expected: number, + inputs: LossFunctionInputs, + state: LossFunctionState +) => number; diff --git a/src/utilities/to-svg.ts b/src/utilities/to-svg.ts index 6e4bb835..87546c59 100644 --- a/src/utilities/to-svg.ts +++ b/src/utilities/to-svg.ts @@ -467,6 +467,7 @@ export function toSVG< // Get network size array for NeuralNetwork or NeuralNetworkGPU let sizes: number[] = []; if (net instanceof NeuralNetwork || net instanceof NeuralNetworkGPU) { + // @ts-ignore sizes = getNeuralNetworkSizes(net); } // get network size for Recurrent From eb909c9549f60342151f509b7a89df39bfdc999a Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Sat, 15 Jun 2024 22:15:09 -0400 Subject: [PATCH 03/29] Update loss.ts --- src/utilities/loss.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utilities/loss.ts b/src/utilities/loss.ts index 3e07072f..ed614ced 100644 --- a/src/utilities/loss.ts +++ b/src/utilities/loss.ts @@ -1,4 +1,4 @@ -export type LossFunctionInputs = number[] | number[][] | number[][][] | Float32Array | Float32Array[] | Float32Array[][]; +export type LossFunctionInputs = number[] | Float32Array; export type LossFunctionState = number[][][] | Float32Array[][]; From 1f2c6812b9b4f8b0f6ef0363c4da1c9bf275eed4 Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Sat, 15 Jun 2024 23:50:30 -0400 Subject: [PATCH 04/29] OMG OMG OMG!!!!!! ZOOOOOMIESSS <3333 * IT WORKS FOR GPU NOW, DON'T TOUCH ANYTHING YET LMAO * Bugs noted: It throws if you don't provide a loss function; should be an extremely trivial fix, but I had this working, broke it, panicked, and had to fix it over the course of half an hour. So please, understand that I HAD TO COMMIT!!! --- src/neural-network-gpu.ts | 39 +++++++++++++++++++++++++++++++-------- src/neural-network.ts | 38 +++++++++++++++++++++++--------------- src/utilities/loss.ts | 2 +- src/utilities/to-svg.ts | 1 + 4 files changed, 56 insertions(+), 24 deletions(-) diff --git a/src/neural-network-gpu.ts b/src/neural-network-gpu.ts index ab32cecf..21097ef6 100644 --- a/src/neural-network-gpu.ts +++ b/src/neural-network-gpu.ts @@ -22,6 +22,16 @@ import { NeuralNetwork, } from './neural-network'; import { release } from './utilities/kernel'; +import { LossFunction, LossFunctionInputs, LossFunctionState } from './utilities/loss'; + +function loss( + actual: number, + expected: number, + inputs: LossFunctionInputs, + state: LossFunctionState +) { + return expected - actual; +} export interface INeuralNetworkGPUDatumFormatted { input: KernelOutput; @@ -96,8 +106,8 @@ function weightedSumTanh( return Math.tanh(sum); } -function loss(output: number, target: number): number { - return target - output; +function calcErrorOutput(value: number): number { + return value; } function calcDeltasSigmoid(error: number, output: number): number { @@ -266,6 +276,16 @@ export class NeuralNetworkGPU< this.gpu = new GPU({ mode: options.mode }); } + get lossFunction(): LossFunction { + return typeof this._lossFunction === "function" ? this._lossFunction : loss + } + + set lossFunction( + value: LossFunction + ) { + this._lossFunction = value; + } + initialize(): void { super.initialize(); this.buildRunInput(); @@ -380,7 +400,7 @@ export class NeuralNetworkGPU< }; buildCalculateDeltas(): void { - let calcDeltas: GPUFunction<[number, number]>; + let calcDeltas: GPUFunction<[number, number, LossFunctionInputs, LossFunctionState]>; switch (this.trainOpts.activation) { case 'sigmoid': calcDeltas = calcDeltasSigmoid; @@ -400,31 +420,34 @@ export class NeuralNetworkGPU< ); } - let _loss = typeof this.loss === "function" ? this.loss : loss; + const loss: LossFunction = this._lossFunction ?? function loss(actual, expected) { return expected - actual; }; calcDeltas = alias( utils.getMinifySafeName(() => calcDeltas), calcDeltas ); this.gpu.addFunction(calcDeltas); + this.gpu.addFunction(loss); for (let layer = this.outputLayer; layer > 0; layer--) { if (layer === this.outputLayer) { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error this.backwardPropagate[this.outputLayer] = this.gpu.createKernelMap( { - error: _loss, + error: calcErrorOutput, }, function ( this: IKernelFunctionThis, outputs: number[], - targets: number[] + targets: number[], + inputs: LossFunctionInputs, + state: LossFunctionState ): number { const output = outputs[this.thread.x]; const target = targets[this.thread.x]; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error - return calcDeltas(loss(output, target), output); + return calcDeltas(calcErrorOutput(loss(output, target, inputs, state)), output); }, { output: [this.sizes[this.outputLayer]], @@ -480,7 +503,7 @@ export class NeuralNetworkGPU< if (layer === this.outputLayer) { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error - output = this.backwardPropagate[layer](this.outputs[layer], target); + output = this.backwardPropagate[layer](this.outputs[layer], target, this.outputs[0], this.lossState); } else { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error diff --git a/src/neural-network.ts b/src/neural-network.ts index e4309812..8d334a0f 100644 --- a/src/neural-network.ts +++ b/src/neural-network.ts @@ -13,7 +13,6 @@ import { mse } from './utilities/mse'; import { randos } from './utilities/randos'; import { zeros } from './utilities/zeros'; import { LossFunction, LossFunctionInputs, LossFunctionState } from './utilities/loss'; - type NeuralNetworkFormatter = | ((v: INumberHash) => Float32Array) | ((v: number[]) => Float32Array); @@ -41,6 +40,15 @@ export function getTypedArrayFn( }; } +function loss( + actual: number, + expected: number, + inputs: LossFunctionInputs, + state: LossFunctionState +) { + return expected - actual; +} + export type NeuralNetworkActivation = | 'sigmoid' | 'relu' @@ -110,7 +118,7 @@ export interface INeuralNetworkTrainOptions { errorThresh: number; log: boolean | ((status: INeuralNetworkState) => void); logPeriod: number; - loss: boolean | LossFunction; + loss?: LossFunction; lossState?: LossFunctionState; lossStateSize: number; leakyReluAlpha: number; @@ -132,7 +140,7 @@ export function trainDefaults(): INeuralNetworkTrainOptions { errorThresh: 0.005, // the acceptable error percentage from training data log: false, // true to use console.log, when a function is supplied it is used logPeriod: 10, // iterations between logging out - loss: false, + loss, lossStateSize: 1, leakyReluAlpha: 0.01, learningRate: 0.3, // multiply's against the input and the delta then adds to momentum @@ -191,15 +199,15 @@ export class NeuralNetwork< return this.runInput(input); }; - calculateDeltas: (output: Float32Array, input: LossFunctionInputs) => void = ( + calculateDeltas: (output: Float32Array, input: Float32Array) => void = ( output: Float32Array, - input: LossFunctionInputs + input: Float32Array ): void => { this.setActivation(); return this.calculateDeltas(output, input); }; - loss: boolean | LossFunction = false; + _lossFunction?: LossFunction; // adam biasChangesLow: Float32Array[] = []; @@ -690,7 +698,7 @@ export class NeuralNetwork< data: Array, Partial>>, options: Partial = {} ): INeuralNetworkState { - this.loss = options.loss ?? false; + this._lossFunction ??= options.loss; const { preparedData, status, endTime } = this.prepTraining( data as Array>, @@ -747,7 +755,7 @@ export class NeuralNetwork< return null; } - _calculateDeltasSigmoid(target: Float32Array, input: LossFunctionInputs): void { + _calculateDeltasSigmoid(target: Float32Array, input: Float32Array): void { for (let layer = this.outputLayer; layer >= 0; layer--) { const activeSize = this.sizes[layer]; const activeOutput = this.outputs[layer]; @@ -760,7 +768,7 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - if (typeof this.loss === "function") error = this.loss.call({thread: {x: node}}, output, target[node], input, this.lossState); + if (typeof this._lossFunction === "function") error = this._lossFunction.call({thread: {x: node}}, output, target[node], input, this.lossState); else error = target[node] - output; } else { const deltas = this.deltas[layer + 1]; @@ -774,7 +782,7 @@ export class NeuralNetwork< } } - _calculateDeltasRelu(target: Float32Array, input: LossFunctionInputs): void { + _calculateDeltasRelu(target: Float32Array, input: Float32Array): void { for (let layer = this.outputLayer; layer >= 0; layer--) { const currentSize = this.sizes[layer]; const currentOutputs = this.outputs[layer]; @@ -788,7 +796,7 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - if (typeof this.loss === "function") error = this.loss.call({thread: {x: node}}, output, target[node], input, this.lossState); + if (typeof this._lossFunction === "function") error = this._lossFunction.call({thread: {x: node}}, output, target[node], input, this.lossState); else error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { @@ -801,7 +809,7 @@ export class NeuralNetwork< } } - _calculateDeltasLeakyRelu(target: Float32Array, input: LossFunctionInputs): void { + _calculateDeltasLeakyRelu(target: Float32Array, input: Float32Array): void { const alpha = this.trainOpts.leakyReluAlpha; for (let layer = this.outputLayer; layer >= 0; layer--) { const currentSize = this.sizes[layer]; @@ -816,7 +824,7 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - if (typeof this.loss === "function") error = this.loss.call({thread: {x: node}}, output, target[node], input, this.lossState); + if (typeof this._lossFunction === "function") error = this._lossFunction.call({thread: {x: node}}, output, target[node], input, this.lossState); else error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { @@ -829,7 +837,7 @@ export class NeuralNetwork< } } - _calculateDeltasTanh(target: Float32Array, input: LossFunctionInputs): void { + _calculateDeltasTanh(target: Float32Array, input: Float32Array): void { for (let layer = this.outputLayer; layer >= 0; layer--) { const currentSize = this.sizes[layer]; const currentOutputs = this.outputs[layer]; @@ -843,7 +851,7 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - if (typeof this.loss === "function") error = this.loss.call({thread: {x: node}}, output, target[node], input, this.lossState); + if (typeof this._lossFunction === "function") error = this._lossFunction.call({thread: {x: node}}, output, target[node], input, this.lossState); else error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { diff --git a/src/utilities/loss.ts b/src/utilities/loss.ts index ed614ced..3e07072f 100644 --- a/src/utilities/loss.ts +++ b/src/utilities/loss.ts @@ -1,4 +1,4 @@ -export type LossFunctionInputs = number[] | Float32Array; +export type LossFunctionInputs = number[] | number[][] | number[][][] | Float32Array | Float32Array[] | Float32Array[][]; export type LossFunctionState = number[][][] | Float32Array[][]; diff --git a/src/utilities/to-svg.ts b/src/utilities/to-svg.ts index 87546c59..6c6e0f83 100644 --- a/src/utilities/to-svg.ts +++ b/src/utilities/to-svg.ts @@ -467,6 +467,7 @@ export function toSVG< // Get network size array for NeuralNetwork or NeuralNetworkGPU let sizes: number[] = []; if (net instanceof NeuralNetwork || net instanceof NeuralNetworkGPU) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore sizes = getNeuralNetworkSizes(net); } From ba12f82f80d20d59941cf120707005912291d119 Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Sat, 15 Jun 2024 23:55:02 -0400 Subject: [PATCH 05/29] Fixed the bug~! <3 `NeuralNetworkGPU` no longer throws an error when a loss function is not provided --- src/neural-network-gpu.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/neural-network-gpu.ts b/src/neural-network-gpu.ts index 21097ef6..56117f98 100644 --- a/src/neural-network-gpu.ts +++ b/src/neural-network-gpu.ts @@ -33,6 +33,8 @@ function loss( return expected - actual; } +const DEFAULT_LOSS_FUNCTION = loss; + export interface INeuralNetworkGPUDatumFormatted { input: KernelOutput; output: KernelOutput; @@ -420,7 +422,7 @@ export class NeuralNetworkGPU< ); } - const loss: LossFunction = this._lossFunction ?? function loss(actual, expected) { return expected - actual; }; + const loss: LossFunction = this._lossFunction ?? DEFAULT_LOSS_FUNCTION; calcDeltas = alias( utils.getMinifySafeName(() => calcDeltas), From 7371a23c242e62411594a765b0c6aad09091de30 Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Sun, 16 Jun 2024 00:20:25 -0400 Subject: [PATCH 06/29] Generalize loss function for both CPU and GPU Allow for using the same loss function between both CPU and GPU neural networks --- src/neural-network.ts | 28 ++++++++++++++++++++++++---- src/utilities/loss.ts | 3 +++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/neural-network.ts b/src/neural-network.ts index 8d334a0f..b341cb4e 100644 --- a/src/neural-network.ts +++ b/src/neural-network.ts @@ -768,7 +768,12 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - if (typeof this._lossFunction === "function") error = this._lossFunction.call({thread: {x: node}}, output, target[node], input, this.lossState); + if (typeof this._lossFunction === "function") { + const kernelFunctionThis = { thread: { x: node, y: layer, z: 0 } }; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.lossState); + } else error = target[node] - output; } else { const deltas = this.deltas[layer + 1]; @@ -796,7 +801,12 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - if (typeof this._lossFunction === "function") error = this._lossFunction.call({thread: {x: node}}, output, target[node], input, this.lossState); + if (typeof this._lossFunction === "function") { + const kernelFunctionThis = { thread: { x: node, y: layer, z: 0 } }; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.lossState); + } else error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { @@ -824,7 +834,12 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - if (typeof this._lossFunction === "function") error = this._lossFunction.call({thread: {x: node}}, output, target[node], input, this.lossState); + if (typeof this._lossFunction === "function") { + const kernelFunctionThis = { thread: { x: node, y: layer, z: 0 } }; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.lossState); + } else error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { @@ -851,7 +866,12 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - if (typeof this._lossFunction === "function") error = this._lossFunction.call({thread: {x: node}}, output, target[node], input, this.lossState); + if (typeof this._lossFunction === "function") { + const kernelFunctionThis = { thread: { x: node, y: layer, z: 0 } }; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.lossState); + } else error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { diff --git a/src/utilities/loss.ts b/src/utilities/loss.ts index 3e07072f..56a4da66 100644 --- a/src/utilities/loss.ts +++ b/src/utilities/loss.ts @@ -1,8 +1,11 @@ +import { IKernelFunctionThis } from "gpu.js"; + export type LossFunctionInputs = number[] | number[][] | number[][][] | Float32Array | Float32Array[] | Float32Array[][]; export type LossFunctionState = number[][][] | Float32Array[][]; export type LossFunction = ( + this: IKernelFunctionThis, actual: number, expected: number, inputs: LossFunctionInputs, From 2a7840a47757c4529cf1bcffb939cf8923d1b14c Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Sun, 16 Jun 2024 04:42:17 -0400 Subject: [PATCH 07/29] Add memory function --- src/neural-network-gpu.ts | 41 +++++++++++++++++----- src/neural-network.ts | 71 ++++++++++++++++++++++++--------------- src/utilities/loss.ts | 14 ++++++-- 3 files changed, 88 insertions(+), 38 deletions(-) diff --git a/src/neural-network-gpu.ts b/src/neural-network-gpu.ts index 56117f98..57adbf9b 100644 --- a/src/neural-network-gpu.ts +++ b/src/neural-network-gpu.ts @@ -22,18 +22,37 @@ import { NeuralNetwork, } from './neural-network'; import { release } from './utilities/kernel'; -import { LossFunction, LossFunctionInputs, LossFunctionState } from './utilities/loss'; +import { LossFunction, LossFunctionInputs, MemoryFunction, NeuralNetworkMemory } from './utilities/loss'; function loss( + this: IKernelFunctionThis, actual: number, expected: number, inputs: LossFunctionInputs, - state: LossFunctionState + memory: NeuralNetworkMemory ) { return expected - actual; } +function updateMemory( + this: IKernelFunctionThis, + actual: number, + expected: number, + inputs: LossFunctionInputs, + memory: NeuralNetworkMemory, + memorySize: number, + loss: number +) { + const layer = this.thread.z; + const neuron = this.thread.y; + const signal = this.thread.x; + + // Maintain the same signal magnitude. + return memory[layer][neuron][signal]; +} + const DEFAULT_LOSS_FUNCTION = loss; +const DEFAULT_MEMORY_FUNCTION = updateMemory; export interface INeuralNetworkGPUDatumFormatted { input: KernelOutput; @@ -402,7 +421,7 @@ export class NeuralNetworkGPU< }; buildCalculateDeltas(): void { - let calcDeltas: GPUFunction<[number, number, LossFunctionInputs, LossFunctionState]>; + let calcDeltas: GPUFunction<[number, number, LossFunctionInputs, NeuralNetworkMemory]>; switch (this.trainOpts.activation) { case 'sigmoid': calcDeltas = calcDeltasSigmoid; @@ -423,6 +442,7 @@ export class NeuralNetworkGPU< } const loss: LossFunction = this._lossFunction ?? DEFAULT_LOSS_FUNCTION; + const updateMemory: MemoryFunction = this._memoryFunction ?? DEFAULT_MEMORY_FUNCTION; calcDeltas = alias( utils.getMinifySafeName(() => calcDeltas), @@ -436,14 +456,14 @@ export class NeuralNetworkGPU< // @ts-expect-error this.backwardPropagate[this.outputLayer] = this.gpu.createKernelMap( { - error: calcErrorOutput, + error: calcErrorOutput }, function ( this: IKernelFunctionThis, outputs: number[], targets: number[], inputs: LossFunctionInputs, - state: LossFunctionState + state: NeuralNetworkMemory ): number { const output = outputs[this.thread.x]; const target = targets[this.thread.x]; @@ -503,9 +523,8 @@ export class NeuralNetworkGPU< let output; if (layer === this.outputLayer) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - output = this.backwardPropagate[layer](this.outputs[layer], target, this.outputs[0], this.lossState); + // @ts-ignore + output = this.backwardPropagate[layer](this.outputs[layer], target, this.outputs[0], this.memory); } else { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error @@ -731,11 +750,17 @@ export class NeuralNetworkGPU< : (layerBiases as Float32Array) ) ); + const jsonLayerMemory = this.memory.map((layerMemory, layerIndex) => + layerMemory.map(nodeMemory => + Array.from(nodeMemory) + ) + ); const jsonLayers: IJSONLayer[] = []; for (let i = 0; i <= this.outputLayer; i++) { jsonLayers.push({ weights: jsonLayerWeights[i] ?? [], biases: jsonLayerBiases[i] ?? [], + memory: jsonLayerMemory[i] ?? [] }); } return { diff --git a/src/neural-network.ts b/src/neural-network.ts index b341cb4e..9e3e939c 100644 --- a/src/neural-network.ts +++ b/src/neural-network.ts @@ -12,7 +12,7 @@ import { max } from './utilities/max'; import { mse } from './utilities/mse'; import { randos } from './utilities/randos'; import { zeros } from './utilities/zeros'; -import { LossFunction, LossFunctionInputs, LossFunctionState } from './utilities/loss'; +import { LossFunction, LossFunctionInputs, MemoryFunction, NeuralNetworkMemory } from './utilities/loss'; type NeuralNetworkFormatter = | ((v: INumberHash) => Float32Array) | ((v: number[]) => Float32Array); @@ -44,7 +44,7 @@ function loss( actual: number, expected: number, inputs: LossFunctionInputs, - state: LossFunctionState + state: NeuralNetworkMemory ) { return expected - actual; } @@ -58,6 +58,7 @@ export type NeuralNetworkActivation = export interface IJSONLayer { biases: number[]; weights: number[][]; + memory: number[][]; } export interface INeuralNetworkJSON { @@ -77,7 +78,7 @@ export interface INeuralNetworkOptions { outputSize: number; binaryThresh: number; hiddenLayers?: number[]; - lossStateSize: number; + memorySize: number; } export function defaults(): INeuralNetworkOptions { @@ -85,7 +86,7 @@ export function defaults(): INeuralNetworkOptions { inputSize: 0, outputSize: 0, binaryThresh: 0.5, - lossStateSize: 1 + memorySize: 1 }; } @@ -119,8 +120,8 @@ export interface INeuralNetworkTrainOptions { log: boolean | ((status: INeuralNetworkState) => void); logPeriod: number; loss?: LossFunction; - lossState?: LossFunctionState; - lossStateSize: number; + memory?: MemoryFunction; + memorySize: number; leakyReluAlpha: number; learningRate: number; momentum: number; @@ -141,7 +142,7 @@ export function trainDefaults(): INeuralNetworkTrainOptions { log: false, // true to use console.log, when a function is supplied it is used logPeriod: 10, // iterations between logging out loss, - lossStateSize: 1, + memorySize: 1, leakyReluAlpha: 0.01, learningRate: 0.3, // multiply's against the input and the delta then adds to momentum momentum: 0.1, // multiply's against the specified "change" then adds to learning rate for change @@ -192,7 +193,7 @@ export class NeuralNetwork< _formatInput: NeuralNetworkFormatter | null = null; _formatOutput: NeuralNetworkFormatter | null = null; - _lossState: LossFunctionState; + _memory: NeuralNetworkMemory; runInput: (input: Float32Array) => Float32Array = (input: Float32Array) => { this.setActivation(); @@ -208,6 +209,7 @@ export class NeuralNetwork< }; _lossFunction?: LossFunction; + _memoryFunction?: MemoryFunction; // adam biasChangesLow: Float32Array[] = []; @@ -227,8 +229,9 @@ export class NeuralNetwork< this.sizes = [inputSize].concat(hiddenLayers ?? []).concat([outputSize]); } - const { lossStateSize } = this.options ?? 0; - this._lossState = this.trainOpts.lossState ?? this.replaceLossState(lossStateSize); + // Initialize memory matrix + const { memorySize } = this.options ?? 0; + this._memory = this.replaceMemory(memorySize); } /** @@ -305,8 +308,8 @@ export class NeuralNetwork< return this.sizes.length > 0; } - public get lossState(): LossFunctionState { - return this._lossState; + public get memory(): NeuralNetworkMemory { + return this._memory; } run(input: Partial): OutputType { @@ -772,7 +775,7 @@ export class NeuralNetwork< const kernelFunctionThis = { thread: { x: node, y: layer, z: 0 } }; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.lossState); + error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.memory); } else error = target[node] - output; } else { @@ -805,7 +808,7 @@ export class NeuralNetwork< const kernelFunctionThis = { thread: { x: node, y: layer, z: 0 } }; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.lossState); + error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.memory); } else error = target[node] - output; } else { @@ -838,7 +841,7 @@ export class NeuralNetwork< const kernelFunctionThis = { thread: { x: node, y: layer, z: 0 } }; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.lossState); + error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.memory); } else error = target[node] - output; } else { @@ -870,7 +873,7 @@ export class NeuralNetwork< const kernelFunctionThis = { thread: { x: node, y: layer, z: 0 } }; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.lossState); + error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.memory); } else error = target[node] - output; } else { @@ -1231,12 +1234,18 @@ export class NeuralNetwork< const jsonLayerBiases = this.biases.map((layerBiases) => Array.from(layerBiases) ); + const jsonLayerMemory = this.memory.map(layerMemory => + layerMemory.map( + nodeMemory => Array.from(nodeMemory) + ) + ); const jsonLayers: IJSONLayer[] = []; const outputLength = this.sizes.length - 1; for (let i = 0; i <= outputLength; i++) { jsonLayers.push({ weights: jsonLayerWeights[i] ?? [], biases: jsonLayerBiases[i] ?? [], + memory: jsonLayerMemory[i] ?? [] }); } return { @@ -1281,9 +1290,15 @@ export class NeuralNetwork< const layerBiases = this.biases.map((layerBiases, layerIndex) => Float32Array.from(jsonLayers[layerIndex].biases) ); + const layerMemory = this.memory.map((memory, layerIndex) => + Array.from(jsonLayers[layerIndex].memory).map(nodeMemory => + Float32Array.from(nodeMemory) + ) + ); for (let i = 0; i <= this.outputLayer; i++) { this.weights[i] = layerWeights[i] || []; this.biases[i] = layerBiases[i] || []; + this.memory[i] = layerMemory[i] || []; } return this; } @@ -1387,23 +1402,23 @@ export class NeuralNetwork< ) => OutputType; } - private createLossState( - lossStateSize: number - ): LossFunctionState { - const lossState: LossFunctionState = []; + private createMemory( + memorySize: number + ): NeuralNetworkMemory { + const memory: NeuralNetworkMemory = []; for (let layer = 0; layer < this.sizes.length; layer++) { - lossState[layer] = []; + memory[layer] = []; for (let neuron = 0; neuron < this.sizes.length; neuron++) { - lossState[layer][neuron] = new Float32Array(lossStateSize); + memory[layer][neuron] = new Float32Array(memorySize); } } - return lossState; + return memory; } - private replaceLossState( - lossState: number | LossFunctionState - ): LossFunctionState { - if (typeof lossState === "number") return this._lossState = this.createLossState(lossState); - return this._lossState = lossState; + private replaceMemory( + memory: number | NeuralNetworkMemory + ): NeuralNetworkMemory { + if (typeof memory === "number") return this._memory = this.createMemory(memory); + return this._memory = memory; } } diff --git a/src/utilities/loss.ts b/src/utilities/loss.ts index 56a4da66..7467745c 100644 --- a/src/utilities/loss.ts +++ b/src/utilities/loss.ts @@ -2,12 +2,22 @@ import { IKernelFunctionThis } from "gpu.js"; export type LossFunctionInputs = number[] | number[][] | number[][][] | Float32Array | Float32Array[] | Float32Array[][]; -export type LossFunctionState = number[][][] | Float32Array[][]; +export type NeuralNetworkMemory = Float32Array[][]; export type LossFunction = ( this: IKernelFunctionThis, actual: number, expected: number, inputs: LossFunctionInputs, - state: LossFunctionState + memory: NeuralNetworkMemory +) => number; + +export type MemoryFunction = ( + this: IKernelFunctionThis, + actual: number, + expected: number, + inputs: LossFunctionInputs, + memory: NeuralNetworkMemory, + memorySize: number, + loss: number ) => number; From a762a48c199e6d0fd353a81f50abcd2cadb2d2cb Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Sun, 16 Jun 2024 15:21:03 -0400 Subject: [PATCH 08/29] Backup: Another thunderstorm, power outage risk --- src/neural-network-gpu.ts | 26 ++++++++++++---- src/neural-network.ts | 62 +++++++++++++++++++++++++-------------- src/utilities/loss.ts | 7 ++--- 3 files changed, 64 insertions(+), 31 deletions(-) diff --git a/src/neural-network-gpu.ts b/src/neural-network-gpu.ts index 57adbf9b..d7bb2b4f 100644 --- a/src/neural-network-gpu.ts +++ b/src/neural-network-gpu.ts @@ -36,12 +36,11 @@ function loss( function updateMemory( this: IKernelFunctionThis, - actual: number, - expected: number, - inputs: LossFunctionInputs, memory: NeuralNetworkMemory, memorySize: number, - loss: number + loss: number, + outputs: number[][], + sizes: number[] ) { const layer = this.thread.z; const neuron = this.thread.y; @@ -291,10 +290,19 @@ export class NeuralNetworkGPU< // @ts-expect-error biases: KernelOutput[] = []; + + losses: KernelOutput[] = []; + ram: KernelOutput = ; + constructor(options: Partial = {}) { super(options); this.errorCheckInterval = 100; this.gpu = new GPU({ mode: options.mode }); + this._memoryFunction = options.memory ?? DEFAULT_MEMORY_FUNCTION; + } + + get loss(): number { + return this._loss; } get lossFunction(): LossFunction { @@ -391,6 +399,14 @@ export class NeuralNetworkGPU< }); } + // Determine whether `options.updateMemory` was passed to the constructor. + if (this._memoryFunction) { + // Get the defined memory function. + const updateMemory = this._memoryKernel; + // Update the neural network's memory matrix after each feed-forward pass. + updateMemory(this.outputs, this.memory, this.sizes, this.memorySize, this.loss); + } + this.texturizeInputData = this.gpu.createKernel( function (value: number[]): number { return value[this.thread.x]; @@ -442,7 +458,6 @@ export class NeuralNetworkGPU< } const loss: LossFunction = this._lossFunction ?? DEFAULT_LOSS_FUNCTION; - const updateMemory: MemoryFunction = this._memoryFunction ?? DEFAULT_MEMORY_FUNCTION; calcDeltas = alias( utils.getMinifySafeName(() => calcDeltas), @@ -523,6 +538,7 @@ export class NeuralNetworkGPU< let output; if (layer === this.outputLayer) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore output = this.backwardPropagate[layer](this.outputs[layer], target, this.outputs[0], this.memory); } else { diff --git a/src/neural-network.ts b/src/neural-network.ts index 9e3e939c..943c06da 100644 --- a/src/neural-network.ts +++ b/src/neural-network.ts @@ -58,7 +58,7 @@ export type NeuralNetworkActivation = export interface IJSONLayer { biases: number[]; weights: number[][]; - memory: number[][]; + memory?: number[][]; } export interface INeuralNetworkJSON { @@ -78,6 +78,7 @@ export interface INeuralNetworkOptions { outputSize: number; binaryThresh: number; hiddenLayers?: number[]; + memory?: MemoryFunction; memorySize: number; } @@ -193,7 +194,7 @@ export class NeuralNetwork< _formatInput: NeuralNetworkFormatter | null = null; _formatOutput: NeuralNetworkFormatter | null = null; - _memory: NeuralNetworkMemory; + _memory?: NeuralNetworkMemory; runInput: (input: Float32Array) => Float32Array = (input: Float32Array) => { this.setActivation(); @@ -229,9 +230,23 @@ export class NeuralNetwork< this.sizes = [inputSize].concat(hiddenLayers ?? []).concat([outputSize]); } + this._memoryFunction = options.memory; // Initialize memory matrix const { memorySize } = this.options ?? 0; - this._memory = this.replaceMemory(memorySize); + if (memorySize) this._memory = this.replaceMemory(memorySize); + } + + public get memory(): NeuralNetworkMemory | undefined { + return this._memory; + } + + public get memoryFunction(): MemoryFunction | undefined { + return this._memoryFunction; + } + + public get memorySize(): number { + if (!this._memory || !this._memory[0] || !this._memory[0][0]) return 0; + return this._memory[0][0].length; } /** @@ -308,10 +323,6 @@ export class NeuralNetwork< return this.sizes.length > 0; } - public get memory(): NeuralNetworkMemory { - return this._memory; - } - run(input: Partial): OutputType { if (!this.isRunnable) { throw new Error('network not runnable'); @@ -1234,19 +1245,23 @@ export class NeuralNetwork< const jsonLayerBiases = this.biases.map((layerBiases) => Array.from(layerBiases) ); - const jsonLayerMemory = this.memory.map(layerMemory => - layerMemory.map( - nodeMemory => Array.from(nodeMemory) - ) - ); + let jsonLayerMemory; + if (this.memory) { + jsonLayerMemory = this.memory.map(layerMemory => + layerMemory.map( + nodeMemory => Array.from(nodeMemory) + ) + ); + } const jsonLayers: IJSONLayer[] = []; const outputLength = this.sizes.length - 1; for (let i = 0; i <= outputLength; i++) { - jsonLayers.push({ + const jsonLayer: IJSONLayer = { weights: jsonLayerWeights[i] ?? [], - biases: jsonLayerBiases[i] ?? [], - memory: jsonLayerMemory[i] ?? [] - }); + biases: jsonLayerBiases[i] ?? [] + }; + if (jsonLayerMemory) jsonLayer.memory = jsonLayerMemory[i] ?? []; + jsonLayers.push(jsonLayer); } return { type: 'NeuralNetwork', @@ -1290,15 +1305,18 @@ export class NeuralNetwork< const layerBiases = this.biases.map((layerBiases, layerIndex) => Float32Array.from(jsonLayers[layerIndex].biases) ); - const layerMemory = this.memory.map((memory, layerIndex) => - Array.from(jsonLayers[layerIndex].memory).map(nodeMemory => - Float32Array.from(nodeMemory) - ) - ); + let layerMemory; + if (this.memory) { + layerMemory = this.memory.map((memory, layerIndex) => + Array.from(jsonLayers[layerIndex].memory ?? []).map(nodeMemory => + Float32Array.from(nodeMemory) + ) + ); + } for (let i = 0; i <= this.outputLayer; i++) { this.weights[i] = layerWeights[i] || []; this.biases[i] = layerBiases[i] || []; - this.memory[i] = layerMemory[i] || []; + if (layerMemory && this.memory) this.memory[i] = (layerMemory && layerMemory[i]) ?? []; } return this; } diff --git a/src/utilities/loss.ts b/src/utilities/loss.ts index 7467745c..f861243f 100644 --- a/src/utilities/loss.ts +++ b/src/utilities/loss.ts @@ -14,10 +14,9 @@ export type LossFunction = ( export type MemoryFunction = ( this: IKernelFunctionThis, - actual: number, - expected: number, - inputs: LossFunctionInputs, memory: NeuralNetworkMemory, memorySize: number, - loss: number + loss: number, + outputs: number[][], + sizes: number[] ) => number; From 94fc99a0bfa0ef6482f792fa05ef869dd1af2476 Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Sun, 16 Jun 2024 19:15:18 -0400 Subject: [PATCH 09/29] Revert "Backup: Another thunderstorm, power outage risk" This reverts commit a762a48c199e6d0fd353a81f50abcd2cadb2d2cb. --- src/neural-network-gpu.ts | 26 ++++------------ src/neural-network.ts | 62 ++++++++++++++------------------------- src/utilities/loss.ts | 7 +++-- 3 files changed, 31 insertions(+), 64 deletions(-) diff --git a/src/neural-network-gpu.ts b/src/neural-network-gpu.ts index d7bb2b4f..57adbf9b 100644 --- a/src/neural-network-gpu.ts +++ b/src/neural-network-gpu.ts @@ -36,11 +36,12 @@ function loss( function updateMemory( this: IKernelFunctionThis, + actual: number, + expected: number, + inputs: LossFunctionInputs, memory: NeuralNetworkMemory, memorySize: number, - loss: number, - outputs: number[][], - sizes: number[] + loss: number ) { const layer = this.thread.z; const neuron = this.thread.y; @@ -290,19 +291,10 @@ export class NeuralNetworkGPU< // @ts-expect-error biases: KernelOutput[] = []; - - losses: KernelOutput[] = []; - ram: KernelOutput = ; - constructor(options: Partial = {}) { super(options); this.errorCheckInterval = 100; this.gpu = new GPU({ mode: options.mode }); - this._memoryFunction = options.memory ?? DEFAULT_MEMORY_FUNCTION; - } - - get loss(): number { - return this._loss; } get lossFunction(): LossFunction { @@ -399,14 +391,6 @@ export class NeuralNetworkGPU< }); } - // Determine whether `options.updateMemory` was passed to the constructor. - if (this._memoryFunction) { - // Get the defined memory function. - const updateMemory = this._memoryKernel; - // Update the neural network's memory matrix after each feed-forward pass. - updateMemory(this.outputs, this.memory, this.sizes, this.memorySize, this.loss); - } - this.texturizeInputData = this.gpu.createKernel( function (value: number[]): number { return value[this.thread.x]; @@ -458,6 +442,7 @@ export class NeuralNetworkGPU< } const loss: LossFunction = this._lossFunction ?? DEFAULT_LOSS_FUNCTION; + const updateMemory: MemoryFunction = this._memoryFunction ?? DEFAULT_MEMORY_FUNCTION; calcDeltas = alias( utils.getMinifySafeName(() => calcDeltas), @@ -538,7 +523,6 @@ export class NeuralNetworkGPU< let output; if (layer === this.outputLayer) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore output = this.backwardPropagate[layer](this.outputs[layer], target, this.outputs[0], this.memory); } else { diff --git a/src/neural-network.ts b/src/neural-network.ts index 943c06da..9e3e939c 100644 --- a/src/neural-network.ts +++ b/src/neural-network.ts @@ -58,7 +58,7 @@ export type NeuralNetworkActivation = export interface IJSONLayer { biases: number[]; weights: number[][]; - memory?: number[][]; + memory: number[][]; } export interface INeuralNetworkJSON { @@ -78,7 +78,6 @@ export interface INeuralNetworkOptions { outputSize: number; binaryThresh: number; hiddenLayers?: number[]; - memory?: MemoryFunction; memorySize: number; } @@ -194,7 +193,7 @@ export class NeuralNetwork< _formatInput: NeuralNetworkFormatter | null = null; _formatOutput: NeuralNetworkFormatter | null = null; - _memory?: NeuralNetworkMemory; + _memory: NeuralNetworkMemory; runInput: (input: Float32Array) => Float32Array = (input: Float32Array) => { this.setActivation(); @@ -230,23 +229,9 @@ export class NeuralNetwork< this.sizes = [inputSize].concat(hiddenLayers ?? []).concat([outputSize]); } - this._memoryFunction = options.memory; // Initialize memory matrix const { memorySize } = this.options ?? 0; - if (memorySize) this._memory = this.replaceMemory(memorySize); - } - - public get memory(): NeuralNetworkMemory | undefined { - return this._memory; - } - - public get memoryFunction(): MemoryFunction | undefined { - return this._memoryFunction; - } - - public get memorySize(): number { - if (!this._memory || !this._memory[0] || !this._memory[0][0]) return 0; - return this._memory[0][0].length; + this._memory = this.replaceMemory(memorySize); } /** @@ -323,6 +308,10 @@ export class NeuralNetwork< return this.sizes.length > 0; } + public get memory(): NeuralNetworkMemory { + return this._memory; + } + run(input: Partial): OutputType { if (!this.isRunnable) { throw new Error('network not runnable'); @@ -1245,23 +1234,19 @@ export class NeuralNetwork< const jsonLayerBiases = this.biases.map((layerBiases) => Array.from(layerBiases) ); - let jsonLayerMemory; - if (this.memory) { - jsonLayerMemory = this.memory.map(layerMemory => - layerMemory.map( - nodeMemory => Array.from(nodeMemory) - ) - ); - } + const jsonLayerMemory = this.memory.map(layerMemory => + layerMemory.map( + nodeMemory => Array.from(nodeMemory) + ) + ); const jsonLayers: IJSONLayer[] = []; const outputLength = this.sizes.length - 1; for (let i = 0; i <= outputLength; i++) { - const jsonLayer: IJSONLayer = { + jsonLayers.push({ weights: jsonLayerWeights[i] ?? [], - biases: jsonLayerBiases[i] ?? [] - }; - if (jsonLayerMemory) jsonLayer.memory = jsonLayerMemory[i] ?? []; - jsonLayers.push(jsonLayer); + biases: jsonLayerBiases[i] ?? [], + memory: jsonLayerMemory[i] ?? [] + }); } return { type: 'NeuralNetwork', @@ -1305,18 +1290,15 @@ export class NeuralNetwork< const layerBiases = this.biases.map((layerBiases, layerIndex) => Float32Array.from(jsonLayers[layerIndex].biases) ); - let layerMemory; - if (this.memory) { - layerMemory = this.memory.map((memory, layerIndex) => - Array.from(jsonLayers[layerIndex].memory ?? []).map(nodeMemory => - Float32Array.from(nodeMemory) - ) - ); - } + const layerMemory = this.memory.map((memory, layerIndex) => + Array.from(jsonLayers[layerIndex].memory).map(nodeMemory => + Float32Array.from(nodeMemory) + ) + ); for (let i = 0; i <= this.outputLayer; i++) { this.weights[i] = layerWeights[i] || []; this.biases[i] = layerBiases[i] || []; - if (layerMemory && this.memory) this.memory[i] = (layerMemory && layerMemory[i]) ?? []; + this.memory[i] = layerMemory[i] || []; } return this; } diff --git a/src/utilities/loss.ts b/src/utilities/loss.ts index f861243f..7467745c 100644 --- a/src/utilities/loss.ts +++ b/src/utilities/loss.ts @@ -14,9 +14,10 @@ export type LossFunction = ( export type MemoryFunction = ( this: IKernelFunctionThis, + actual: number, + expected: number, + inputs: LossFunctionInputs, memory: NeuralNetworkMemory, memorySize: number, - loss: number, - outputs: number[][], - sizes: number[] + loss: number ) => number; From ba03eb375b5d92b2c74e34f6db709342b2c84e37 Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Sun, 16 Jun 2024 19:22:28 -0400 Subject: [PATCH 10/29] Add parameter `lossDelta` --- src/utilities/loss.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utilities/loss.ts b/src/utilities/loss.ts index 7467745c..2986e5f2 100644 --- a/src/utilities/loss.ts +++ b/src/utilities/loss.ts @@ -19,5 +19,6 @@ export type MemoryFunction = ( inputs: LossFunctionInputs, memory: NeuralNetworkMemory, memorySize: number, - loss: number + loss: number, + lossDelta: number ) => number; From aa337f3406fc6a3401127418b8ce594e9e5f88ac Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Sun, 16 Jun 2024 19:49:11 -0400 Subject: [PATCH 11/29] Rename memory to RAM --- src/neural-network-gpu.ts | 18 +++++++++--------- src/neural-network.ts | 24 ++++++++++++------------ src/utilities/loss.ts | 10 +++++----- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/neural-network-gpu.ts b/src/neural-network-gpu.ts index 57adbf9b..1f8e5646 100644 --- a/src/neural-network-gpu.ts +++ b/src/neural-network-gpu.ts @@ -22,24 +22,24 @@ import { NeuralNetwork, } from './neural-network'; import { release } from './utilities/kernel'; -import { LossFunction, LossFunctionInputs, MemoryFunction, NeuralNetworkMemory } from './utilities/loss'; +import { LossFunction, LossFunctionInputs, RAMFunction, NeuralNetworkRAM } from './utilities/loss'; function loss( this: IKernelFunctionThis, actual: number, expected: number, inputs: LossFunctionInputs, - memory: NeuralNetworkMemory + memory: NeuralNetworkRAM ) { return expected - actual; } -function updateMemory( +function updateRAM( this: IKernelFunctionThis, actual: number, expected: number, inputs: LossFunctionInputs, - memory: NeuralNetworkMemory, + memory: NeuralNetworkRAM, memorySize: number, loss: number ) { @@ -52,7 +52,7 @@ function updateMemory( } const DEFAULT_LOSS_FUNCTION = loss; -const DEFAULT_MEMORY_FUNCTION = updateMemory; +const DEFAULT_MEMORY_FUNCTION = updateRAM; export interface INeuralNetworkGPUDatumFormatted { input: KernelOutput; @@ -421,7 +421,7 @@ export class NeuralNetworkGPU< }; buildCalculateDeltas(): void { - let calcDeltas: GPUFunction<[number, number, LossFunctionInputs, NeuralNetworkMemory]>; + let calcDeltas: GPUFunction<[number, number, LossFunctionInputs, NeuralNetworkRAM]>; switch (this.trainOpts.activation) { case 'sigmoid': calcDeltas = calcDeltasSigmoid; @@ -442,7 +442,7 @@ export class NeuralNetworkGPU< } const loss: LossFunction = this._lossFunction ?? DEFAULT_LOSS_FUNCTION; - const updateMemory: MemoryFunction = this._memoryFunction ?? DEFAULT_MEMORY_FUNCTION; + const updateRAM: RAMFunction = this._ramFunction ?? DEFAULT_MEMORY_FUNCTION; calcDeltas = alias( utils.getMinifySafeName(() => calcDeltas), @@ -463,13 +463,13 @@ export class NeuralNetworkGPU< outputs: number[], targets: number[], inputs: LossFunctionInputs, - state: NeuralNetworkMemory + ram: NeuralNetworkRAM ): number { const output = outputs[this.thread.x]; const target = targets[this.thread.x]; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error - return calcDeltas(calcErrorOutput(loss(output, target, inputs, state)), output); + return calcDeltas(calcErrorOutput(loss(output, target, inputs, ram)), output); }, { output: [this.sizes[this.outputLayer]], diff --git a/src/neural-network.ts b/src/neural-network.ts index 9e3e939c..14c33473 100644 --- a/src/neural-network.ts +++ b/src/neural-network.ts @@ -12,7 +12,7 @@ import { max } from './utilities/max'; import { mse } from './utilities/mse'; import { randos } from './utilities/randos'; import { zeros } from './utilities/zeros'; -import { LossFunction, LossFunctionInputs, MemoryFunction, NeuralNetworkMemory } from './utilities/loss'; +import { LossFunction, LossFunctionInputs, RAMFunction, NeuralNetworkRAM } from './utilities/loss'; type NeuralNetworkFormatter = | ((v: INumberHash) => Float32Array) | ((v: number[]) => Float32Array); @@ -44,7 +44,7 @@ function loss( actual: number, expected: number, inputs: LossFunctionInputs, - state: NeuralNetworkMemory + state: NeuralNetworkRAM ) { return expected - actual; } @@ -120,8 +120,8 @@ export interface INeuralNetworkTrainOptions { log: boolean | ((status: INeuralNetworkState) => void); logPeriod: number; loss?: LossFunction; - memory?: MemoryFunction; - memorySize: number; + updateRAM?: RAMFunction; + ramSize: number; leakyReluAlpha: number; learningRate: number; momentum: number; @@ -142,7 +142,7 @@ export function trainDefaults(): INeuralNetworkTrainOptions { log: false, // true to use console.log, when a function is supplied it is used logPeriod: 10, // iterations between logging out loss, - memorySize: 1, + ramSize: 1, leakyReluAlpha: 0.01, learningRate: 0.3, // multiply's against the input and the delta then adds to momentum momentum: 0.1, // multiply's against the specified "change" then adds to learning rate for change @@ -193,7 +193,7 @@ export class NeuralNetwork< _formatInput: NeuralNetworkFormatter | null = null; _formatOutput: NeuralNetworkFormatter | null = null; - _memory: NeuralNetworkMemory; + _memory: NeuralNetworkRAM; runInput: (input: Float32Array) => Float32Array = (input: Float32Array) => { this.setActivation(); @@ -209,7 +209,7 @@ export class NeuralNetwork< }; _lossFunction?: LossFunction; - _memoryFunction?: MemoryFunction; + _ramFunction?: RAMFunction; // adam biasChangesLow: Float32Array[] = []; @@ -308,7 +308,7 @@ export class NeuralNetwork< return this.sizes.length > 0; } - public get memory(): NeuralNetworkMemory { + public get memory(): NeuralNetworkRAM { return this._memory; } @@ -1404,8 +1404,8 @@ export class NeuralNetwork< private createMemory( memorySize: number - ): NeuralNetworkMemory { - const memory: NeuralNetworkMemory = []; + ): NeuralNetworkRAM { + const memory: NeuralNetworkRAM = []; for (let layer = 0; layer < this.sizes.length; layer++) { memory[layer] = []; for (let neuron = 0; neuron < this.sizes.length; neuron++) { @@ -1416,8 +1416,8 @@ export class NeuralNetwork< } private replaceMemory( - memory: number | NeuralNetworkMemory - ): NeuralNetworkMemory { + memory: number | NeuralNetworkRAM + ): NeuralNetworkRAM { if (typeof memory === "number") return this._memory = this.createMemory(memory); return this._memory = memory; } diff --git a/src/utilities/loss.ts b/src/utilities/loss.ts index 2986e5f2..3bd9b8ae 100644 --- a/src/utilities/loss.ts +++ b/src/utilities/loss.ts @@ -2,23 +2,23 @@ import { IKernelFunctionThis } from "gpu.js"; export type LossFunctionInputs = number[] | number[][] | number[][][] | Float32Array | Float32Array[] | Float32Array[][]; -export type NeuralNetworkMemory = Float32Array[][]; +export type NeuralNetworkRAM = Float32Array[][]; export type LossFunction = ( this: IKernelFunctionThis, actual: number, expected: number, inputs: LossFunctionInputs, - memory: NeuralNetworkMemory + memory: NeuralNetworkRAM ) => number; -export type MemoryFunction = ( +export type RAMFunction = ( this: IKernelFunctionThis, actual: number, expected: number, inputs: LossFunctionInputs, - memory: NeuralNetworkMemory, - memorySize: number, + ram: NeuralNetworkRAM, + ramSize: number, loss: number, lossDelta: number ) => number; From c655c52c8c7bc26f436b6a87bdef889f97e50fb4 Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Mon, 17 Jun 2024 03:02:41 -0400 Subject: [PATCH 12/29] Add `updateRAM` Add `updateRAM` as: * property `NeuralNetwork.updateRAM: RAMFunction` * option `NeuralNetworkOptions.updateRAM` * option `NeuralNetworkTrainOptions.updateRAM` *(temporary sets `this.updateRAM` for the duration of the training session)* --- src/neural-network-gpu.ts | 92 +++++++-------- src/neural-network.ts | 240 ++++++++++++++++++++++++++++++++------ src/utilities/loss.ts | 24 ---- 3 files changed, 249 insertions(+), 107 deletions(-) delete mode 100644 src/utilities/loss.ts diff --git a/src/neural-network-gpu.ts b/src/neural-network-gpu.ts index 1f8e5646..9ea80365 100644 --- a/src/neural-network-gpu.ts +++ b/src/neural-network-gpu.ts @@ -4,6 +4,7 @@ import { GPUFunction, IKernelFunctionThis, IKernelMapRunShortcut, + IKernelRunShortcut, IMappedKernelResult, KernelOutput, Texture, @@ -22,37 +23,7 @@ import { NeuralNetwork, } from './neural-network'; import { release } from './utilities/kernel'; -import { LossFunction, LossFunctionInputs, RAMFunction, NeuralNetworkRAM } from './utilities/loss'; - -function loss( - this: IKernelFunctionThis, - actual: number, - expected: number, - inputs: LossFunctionInputs, - memory: NeuralNetworkRAM -) { - return expected - actual; -} - -function updateRAM( - this: IKernelFunctionThis, - actual: number, - expected: number, - inputs: LossFunctionInputs, - memory: NeuralNetworkRAM, - memorySize: number, - loss: number -) { - const layer = this.thread.z; - const neuron = this.thread.y; - const signal = this.thread.x; - - // Maintain the same signal magnitude. - return memory[layer][neuron][signal]; -} - -const DEFAULT_LOSS_FUNCTION = loss; -const DEFAULT_MEMORY_FUNCTION = updateRAM; +import { LossFunction, NeuralNetworkIO, RAMFunction, NeuralNetworkRAM } from './neural-network'; export interface INeuralNetworkGPUDatumFormatted { input: KernelOutput; @@ -211,7 +182,9 @@ export interface INeuralNetworkGPUOptions extends INeuralNetworkOptions { export type BackPropagateOutput = ( this: IKernelFunctionThis, outputs: KernelOutput, - targets: KernelOutput + targets: KernelOutput, + inputs: NeuralNetworkIO, + ram: NeuralNetworkRAM ) => { result: KernelOutput; error: KernelOutput }; export type BackPropagateLayer = ( @@ -291,20 +264,36 @@ export class NeuralNetworkGPU< // @ts-expect-error biases: KernelOutput[] = []; + _ramKernel?: IKernelRunShortcut; + constructor(options: Partial = {}) { super(options); this.errorCheckInterval = 100; this.gpu = new GPU({ mode: options.mode }); + // Compile the accelerated learning functions. + this.lossFunction = this._lossFunction; + this.ramFunction = this._ramFunction; } - get lossFunction(): LossFunction { - return typeof this._lossFunction === "function" ? this._lossFunction : loss + public get lossFunction(): LossFunction { + return super.lossFunction; } - set lossFunction( + public set lossFunction( value: LossFunction ) { - this._lossFunction = value; + this.gpu.addFunction(value); + super.lossFunction = value; + } + + public set ramFunction( + value: RAMFunction | undefined + ) { + if (!value) { + if (this._ramKernel) delete this._ramKernel; + } + else this._ramKernel = this.gpu.createKernel(value); + super.ramFunction = value; } initialize(): void { @@ -391,6 +380,16 @@ export class NeuralNetworkGPU< }); } + const updateRAM: IKernelRunShortcut | undefined = this._ramKernel; + + if (updateRAM) { + const input = this.outputs[0]; + const output = this.outputs[this.outputLayer]; + const loss = this.loss.current.mean; + const deltaLoss = loss - this.loss.previous.mean; + updateRAM(this.ram, this.ramSize, input, output, this.sizes, loss, deltaLoss); + } + this.texturizeInputData = this.gpu.createKernel( function (value: number[]): number { return value[this.thread.x]; @@ -421,7 +420,7 @@ export class NeuralNetworkGPU< }; buildCalculateDeltas(): void { - let calcDeltas: GPUFunction<[number, number, LossFunctionInputs, NeuralNetworkRAM]>; + let calcDeltas: GPUFunction<[number, number, NeuralNetworkIO, NeuralNetworkRAM]>; switch (this.trainOpts.activation) { case 'sigmoid': calcDeltas = calcDeltasSigmoid; @@ -441,8 +440,7 @@ export class NeuralNetworkGPU< ); } - const loss: LossFunction = this._lossFunction ?? DEFAULT_LOSS_FUNCTION; - const updateRAM: RAMFunction = this._ramFunction ?? DEFAULT_MEMORY_FUNCTION; + const loss: LossFunction = this.lossFunction; calcDeltas = alias( utils.getMinifySafeName(() => calcDeltas), @@ -462,7 +460,7 @@ export class NeuralNetworkGPU< this: IKernelFunctionThis, outputs: number[], targets: number[], - inputs: LossFunctionInputs, + inputs: NeuralNetworkIO, ram: NeuralNetworkRAM ): number { const output = outputs[this.thread.x]; @@ -523,8 +521,9 @@ export class NeuralNetworkGPU< let output; if (layer === this.outputLayer) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - output = this.backwardPropagate[layer](this.outputs[layer], target, this.outputs[0], this.memory); + output = this.backwardPropagate[layer](this.outputs[layer], target, this.outputs[0], this.ram); } else { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error @@ -750,18 +749,19 @@ export class NeuralNetworkGPU< : (layerBiases as Float32Array) ) ); - const jsonLayerMemory = this.memory.map((layerMemory, layerIndex) => + const jsonLayerMemory = this.ram?.map((layerMemory, layerIndex) => layerMemory.map(nodeMemory => Array.from(nodeMemory) ) ); const jsonLayers: IJSONLayer[] = []; for (let i = 0; i <= this.outputLayer; i++) { - jsonLayers.push({ + const jsonLayer: IJSONLayer = { weights: jsonLayerWeights[i] ?? [], - biases: jsonLayerBiases[i] ?? [], - memory: jsonLayerMemory[i] ?? [] - }); + biases: jsonLayerBiases[i] ?? [] + }; + if (jsonLayerMemory) jsonLayer.ram = jsonLayerMemory[i] ?? []; + jsonLayers.push(jsonLayer); } return { type: 'NeuralNetworkGPU', diff --git a/src/neural-network.ts b/src/neural-network.ts index 14c33473..3ce0e408 100644 --- a/src/neural-network.ts +++ b/src/neural-network.ts @@ -12,7 +12,71 @@ import { max } from './utilities/max'; import { mse } from './utilities/mse'; import { randos } from './utilities/randos'; import { zeros } from './utilities/zeros'; -import { LossFunction, LossFunctionInputs, RAMFunction, NeuralNetworkRAM } from './utilities/loss'; +import { IKernelFunctionThis, KernelOutput } from 'gpu.js'; + +export type NeuralNetworkIO = number[] | number[][] | number[][][]; + +export type NeuralNetworkRAM = number[][][]; + +export type LossFunction = ( + this: IKernelFunctionThis, + actual: number, + expected: number, + inputs: NeuralNetworkIO, + memory: NeuralNetworkRAM +) => number; + +export type RAMFunction = ( + this: IKernelFunctionThis, + ram: NeuralNetworkRAM, + ramSize: number, + inputs: NeuralNetworkIO, + outputs: NeuralNetworkIO, + sizes: number[], + loss: number, + lossDelta: number +) => number; + +export interface ILossAnalyticsSnapshot { + mean: number; + median: number; + total: number; +} + +const EMPTY_LOSS_SNAPSHOT: ILossAnalyticsSnapshot = { + mean: NaN, + median: NaN, + total: NaN +}; + +Object.freeze(EMPTY_LOSS_SNAPSHOT); + +function createLossAnalyticsSnapshot() { + return JSON.parse(JSON.stringify(EMPTY_LOSS_SNAPSHOT)); +} + +export interface ILossAnalytics { + current: ILossAnalyticsSnapshot; + max: ILossAnalyticsSnapshot; + min: ILossAnalyticsSnapshot; + previous: ILossAnalyticsSnapshot; + projected: ILossAnalyticsSnapshot; +} + +const EMPTY_LOSS: ILossAnalytics = { + current: createLossAnalyticsSnapshot(), + max: createLossAnalyticsSnapshot(), + min: createLossAnalyticsSnapshot(), + previous: createLossAnalyticsSnapshot(), + projected: createLossAnalyticsSnapshot() +}; + +Object.freeze(EMPTY_LOSS); + +export function createLossAnalytics() { + return JSON.parse(JSON.stringify(EMPTY_LOSS)); +} + type NeuralNetworkFormatter = | ((v: INumberHash) => Float32Array) | ((v: number[]) => Float32Array); @@ -41,14 +105,32 @@ export function getTypedArrayFn( } function loss( + this: IKernelFunctionThis, actual: number, expected: number, - inputs: LossFunctionInputs, - state: NeuralNetworkRAM + inputs: NeuralNetworkIO, + memory: NeuralNetworkRAM ) { return expected - actual; } +// function updateRAM( +// this: IKernelFunctionThis, +// inputs: NeuralNetworkIO, +// ram: NeuralNetworkRAM, +// ramSize: number, +// loss: number +// ) { +// const layer = this.thread.z; +// const neuron = this.thread.y; +// const signal = this.thread.x; + +// // Maintain the same signal magnitude. +// return ram[layer][neuron][signal]; +// } + +const DEFAULT_LOSS_FUNCTION = loss; + export type NeuralNetworkActivation = | 'sigmoid' | 'relu' @@ -58,7 +140,7 @@ export type NeuralNetworkActivation = export interface IJSONLayer { biases: number[]; weights: number[][]; - memory: number[][]; + ram?: number[][]; } export interface INeuralNetworkJSON { @@ -78,7 +160,8 @@ export interface INeuralNetworkOptions { outputSize: number; binaryThresh: number; hiddenLayers?: number[]; - memorySize: number; + loss: LossFunction; + ramSize: number; } export function defaults(): INeuralNetworkOptions { @@ -86,7 +169,8 @@ export function defaults(): INeuralNetworkOptions { inputSize: 0, outputSize: 0, binaryThresh: 0.5, - memorySize: 1 + loss, + ramSize: NaN }; } @@ -121,7 +205,6 @@ export interface INeuralNetworkTrainOptions { logPeriod: number; loss?: LossFunction; updateRAM?: RAMFunction; - ramSize: number; leakyReluAlpha: number; learningRate: number; momentum: number; @@ -142,7 +225,6 @@ export function trainDefaults(): INeuralNetworkTrainOptions { log: false, // true to use console.log, when a function is supplied it is used logPeriod: 10, // iterations between logging out loss, - ramSize: 1, leakyReluAlpha: 0.01, learningRate: 0.3, // multiply's against the input and the delta then adds to momentum momentum: 0.1, // multiply's against the specified "change" then adds to learning rate for change @@ -193,7 +275,8 @@ export class NeuralNetwork< _formatInput: NeuralNetworkFormatter | null = null; _formatOutput: NeuralNetworkFormatter | null = null; - _memory: NeuralNetworkRAM; + _lossAnalytics: ILossAnalytics; + _ram: NeuralNetworkRAM; runInput: (input: Float32Array) => Float32Array = (input: Float32Array) => { this.setActivation(); @@ -208,7 +291,7 @@ export class NeuralNetwork< return this.calculateDeltas(output, input); }; - _lossFunction?: LossFunction; + _lossFunction: LossFunction = loss; _ramFunction?: RAMFunction; // adam @@ -229,9 +312,12 @@ export class NeuralNetwork< this.sizes = [inputSize].concat(hiddenLayers ?? []).concat([outputSize]); } - // Initialize memory matrix - const { memorySize } = this.options ?? 0; - this._memory = this.replaceMemory(memorySize); + // Initialize the memory matrix. + const { ramSize } = this.options ?? NaN; + this._ram = this.replaceRAM(ramSize ?? 0); + // Initialize the loss function. + this._lossAnalytics = createLossAnalytics(); + if (options.loss) this._lossFunction = options.loss; } /** @@ -308,8 +394,37 @@ export class NeuralNetwork< return this.sizes.length > 0; } - public get memory(): NeuralNetworkRAM { - return this._memory; + public get loss(): ILossAnalytics { + return this._lossAnalytics; + } + + public get lossFunction(): LossFunction { + return typeof this._lossFunction === "function" ? this._lossFunction : loss + } + + public set lossFunction( + value: LossFunction + ) { + this._lossFunction = value; + } + + public get ram(): NeuralNetworkRAM { + return this._ram; + } + + public get ramFunction(): RAMFunction | undefined { + return this._ramFunction; + } + + public set ramFunction( + value: RAMFunction | undefined + ) { + this._ramFunction = value; + } + + public get ramSize(): number { + if (!this.ram || !this.ram[0] || !this.ram[0][0]) return NaN; + return this.ram[0][0].length; } run(input: Partial): OutputType { @@ -701,7 +816,11 @@ export class NeuralNetwork< data: Array, Partial>>, options: Partial = {} ): INeuralNetworkState { - this._lossFunction ??= options.loss; + let lossFunctionBackup; + if (options.loss) { + lossFunctionBackup = this.lossFunction; + this.lossFunction = options.loss; + } const { preparedData, status, endTime } = this.prepTraining( data as Array>, @@ -713,6 +832,9 @@ export class NeuralNetwork< break; } } + + if (lossFunctionBackup) this.lossFunction = lossFunctionBackup; + return status; } @@ -752,6 +874,42 @@ export class NeuralNetwork< this.calculateDeltas(value.output, value.input); this.adjustWeights(); + if (this.ram) { + const updateRAM: RAMFunction | undefined = this.ramFunction; + + if (updateRAM) { + const input = this.outputs[0]; + const output = this.outputs[this.outputLayer]; + const loss = this.loss.current.mean; + const deltaLoss = loss - this.loss.previous.mean; + this.ram.map( + (layerRAM, layer) => layerRAM.map( + (neuronRAM, neuron) => neuronRAM.map( + (value, index) => { + const kernelFunctionThis: IKernelFunctionThis = { + color: function color(r: number, g: number = 0, b: number = 0, a: number = 0) {}, + constants: {}, + output: { + x: NaN, + y: NaN, + z: NaN + }, + thread: { + x: index, + y: neuron, + z: layer + } + }; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + return updateRAM.call(kernelFunctionThis, this.ram, this.ramSize, input, output, this.sizes, loss, deltaLoss); + } + ) + ) + ) + } + } + if (logErrorRate) { return mse(this.errors[this.outputLayer]); } @@ -775,7 +933,7 @@ export class NeuralNetwork< const kernelFunctionThis = { thread: { x: node, y: layer, z: 0 } }; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.memory); + error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.ram); } else error = target[node] - output; } else { @@ -808,7 +966,7 @@ export class NeuralNetwork< const kernelFunctionThis = { thread: { x: node, y: layer, z: 0 } }; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.memory); + error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.ram); } else error = target[node] - output; } else { @@ -841,7 +999,7 @@ export class NeuralNetwork< const kernelFunctionThis = { thread: { x: node, y: layer, z: 0 } }; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.memory); + error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.ram); } else error = target[node] - output; } else { @@ -873,7 +1031,7 @@ export class NeuralNetwork< const kernelFunctionThis = { thread: { x: node, y: layer, z: 0 } }; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.memory); + error = this._lossFunction.call(kernelFunctionThis, output, target[node], input, this.ram); } else error = target[node] - output; } else { @@ -1234,7 +1392,7 @@ export class NeuralNetwork< const jsonLayerBiases = this.biases.map((layerBiases) => Array.from(layerBiases) ); - const jsonLayerMemory = this.memory.map(layerMemory => + const jsonLayerMemory = this.ram?.map(layerMemory => layerMemory.map( nodeMemory => Array.from(nodeMemory) ) @@ -1242,11 +1400,12 @@ export class NeuralNetwork< const jsonLayers: IJSONLayer[] = []; const outputLength = this.sizes.length - 1; for (let i = 0; i <= outputLength; i++) { - jsonLayers.push({ + const jsonLayer: IJSONLayer = { weights: jsonLayerWeights[i] ?? [], - biases: jsonLayerBiases[i] ?? [], - memory: jsonLayerMemory[i] ?? [] - }); + biases: jsonLayerBiases[i] ?? [] + }; + if (jsonLayerMemory) jsonLayer.ram = jsonLayerMemory[i] ?? []; + jsonLayers.push(jsonLayer); } return { type: 'NeuralNetwork', @@ -1290,15 +1449,21 @@ export class NeuralNetwork< const layerBiases = this.biases.map((layerBiases, layerIndex) => Float32Array.from(jsonLayers[layerIndex].biases) ); - const layerMemory = this.memory.map((memory, layerIndex) => - Array.from(jsonLayers[layerIndex].memory).map(nodeMemory => - Float32Array.from(nodeMemory) + const ramSize = this.ramSize; + const layerRAM + = isFinite(ramSize) + ? this.ram?.map((ram, layerIndex) => + Array.from(jsonLayers[layerIndex].ram ?? []).map(nodeRAM => + Float32Array.from(nodeRAM) + ) ) - ); + : undefined; for (let i = 0; i <= this.outputLayer; i++) { this.weights[i] = layerWeights[i] || []; this.biases[i] = layerBiases[i] || []; - this.memory[i] = layerMemory[i] || []; + } + if (layerRAM) { + if (!this.ram) this._ram = this.replaceRAM(ramSize ?? 0); } return this; } @@ -1402,23 +1567,24 @@ export class NeuralNetwork< ) => OutputType; } - private createMemory( - memorySize: number + private createRAM( + ramSize: number ): NeuralNetworkRAM { + if (!isFinite(ramSize) || ramSize < 0) ramSize = 0; const memory: NeuralNetworkRAM = []; for (let layer = 0; layer < this.sizes.length; layer++) { memory[layer] = []; for (let neuron = 0; neuron < this.sizes.length; neuron++) { - memory[layer][neuron] = new Float32Array(memorySize); + memory[layer][neuron] = new Array(ramSize).fill(0); } } return memory; } - private replaceMemory( - memory: number | NeuralNetworkRAM + private replaceRAM( + ram: number | NeuralNetworkRAM ): NeuralNetworkRAM { - if (typeof memory === "number") return this._memory = this.createMemory(memory); - return this._memory = memory; + if (typeof ram === "number") return this._ram = this.createRAM(ram); + return this._ram = ram; } } diff --git a/src/utilities/loss.ts b/src/utilities/loss.ts deleted file mode 100644 index 3bd9b8ae..00000000 --- a/src/utilities/loss.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { IKernelFunctionThis } from "gpu.js"; - -export type LossFunctionInputs = number[] | number[][] | number[][][] | Float32Array | Float32Array[] | Float32Array[][]; - -export type NeuralNetworkRAM = Float32Array[][]; - -export type LossFunction = ( - this: IKernelFunctionThis, - actual: number, - expected: number, - inputs: LossFunctionInputs, - memory: NeuralNetworkRAM -) => number; - -export type RAMFunction = ( - this: IKernelFunctionThis, - actual: number, - expected: number, - inputs: LossFunctionInputs, - ram: NeuralNetworkRAM, - ramSize: number, - loss: number, - lossDelta: number -) => number; From b45d581058d5456b49dd28a4963a4e8e1aff92a2 Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Mon, 17 Jun 2024 03:08:09 -0400 Subject: [PATCH 13/29] Fix bug that required `ramSize` to be defined --- src/neural-network.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/neural-network.ts b/src/neural-network.ts index 3ce0e408..acac0550 100644 --- a/src/neural-network.ts +++ b/src/neural-network.ts @@ -170,7 +170,7 @@ export function defaults(): INeuralNetworkOptions { outputSize: 0, binaryThresh: 0.5, loss, - ramSize: NaN + ramSize: 1 }; } @@ -314,7 +314,7 @@ export class NeuralNetwork< // Initialize the memory matrix. const { ramSize } = this.options ?? NaN; - this._ram = this.replaceRAM(ramSize ?? 0); + this._ram = this.replaceRAM(ramSize ?? 1); // Initialize the loss function. this._lossAnalytics = createLossAnalytics(); if (options.loss) this._lossFunction = options.loss; @@ -1463,7 +1463,7 @@ export class NeuralNetwork< this.biases[i] = layerBiases[i] || []; } if (layerRAM) { - if (!this.ram) this._ram = this.replaceRAM(ramSize ?? 0); + if (!this.ram) this._ram = this.replaceRAM(ramSize ?? 1); } return this; } @@ -1570,7 +1570,7 @@ export class NeuralNetwork< private createRAM( ramSize: number ): NeuralNetworkRAM { - if (!isFinite(ramSize) || ramSize < 0) ramSize = 0; + if (!isFinite(ramSize) || ramSize < 0) ramSize = 1; const memory: NeuralNetworkRAM = []; for (let layer = 0; layer < this.sizes.length; layer++) { memory[layer] = []; From b703e4aa99449ee5a4c5ce256be32b5af46c636b Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Mon, 17 Jun 2024 03:09:02 -0400 Subject: [PATCH 14/29] Prune unused code --- src/neural-network.ts | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/neural-network.ts b/src/neural-network.ts index acac0550..b81de42a 100644 --- a/src/neural-network.ts +++ b/src/neural-network.ts @@ -114,23 +114,6 @@ function loss( return expected - actual; } -// function updateRAM( -// this: IKernelFunctionThis, -// inputs: NeuralNetworkIO, -// ram: NeuralNetworkRAM, -// ramSize: number, -// loss: number -// ) { -// const layer = this.thread.z; -// const neuron = this.thread.y; -// const signal = this.thread.x; - -// // Maintain the same signal magnitude. -// return ram[layer][neuron][signal]; -// } - -const DEFAULT_LOSS_FUNCTION = loss; - export type NeuralNetworkActivation = | 'sigmoid' | 'relu' From fda0349cf2bf403acdc214e7637d98b7479891ab Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Mon, 17 Jun 2024 04:22:35 -0400 Subject: [PATCH 15/29] Run `updateRAM` on both CPU and GPU nets --- src/neural-network-gpu.ts | 37 +++++++++++----- src/neural-network.ts | 93 ++++++++++++++++++++++----------------- 2 files changed, 78 insertions(+), 52 deletions(-) diff --git a/src/neural-network-gpu.ts b/src/neural-network-gpu.ts index 9ea80365..e83d4e89 100644 --- a/src/neural-network-gpu.ts +++ b/src/neural-network-gpu.ts @@ -292,7 +292,22 @@ export class NeuralNetworkGPU< if (!value) { if (this._ramKernel) delete this._ramKernel; } - else this._ramKernel = this.gpu.createKernel(value); + else { + const layerCount = this.sizes.length; + const maxNeuronsPerLayer = this.sizes.reduce( + (eax, edx) => edx > eax ? edx : eax + ); + const ramSize = this.ramSize; + this._ramKernel = this.gpu.createKernel( + value, + { + constants: { + ramSize + }, + output: [ layerCount, maxNeuronsPerLayer, ramSize ] + } + ); + } super.ramFunction = value; } @@ -380,16 +395,6 @@ export class NeuralNetworkGPU< }); } - const updateRAM: IKernelRunShortcut | undefined = this._ramKernel; - - if (updateRAM) { - const input = this.outputs[0]; - const output = this.outputs[this.outputLayer]; - const loss = this.loss.current.mean; - const deltaLoss = loss - this.loss.previous.mean; - updateRAM(this.ram, this.ramSize, input, output, this.sizes, loss, deltaLoss); - } - this.texturizeInputData = this.gpu.createKernel( function (value: number[]): number { return value[this.thread.x]; @@ -416,6 +421,16 @@ export class NeuralNetworkGPU< ); output = input = this.outputs[layer]; } + const updateRAM: IKernelRunShortcut | undefined = this._ramKernel; + if (updateRAM) { + const input = this.outputs[0]; + const output = this.outputs[this.outputLayer]; + const loss = this.loss.current.mean; + const deltaLoss = loss - this.loss.previous.mean; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + this._ram = updateRAM(this.ram, input, output, this.sizes, loss, deltaLoss); + } return output; }; diff --git a/src/neural-network.ts b/src/neural-network.ts index b81de42a..0c0f4f34 100644 --- a/src/neural-network.ts +++ b/src/neural-network.ts @@ -29,7 +29,6 @@ export type LossFunction = ( export type RAMFunction = ( this: IKernelFunctionThis, ram: NeuralNetworkRAM, - ramSize: number, inputs: NeuralNetworkIO, outputs: NeuralNetworkIO, sizes: number[], @@ -44,9 +43,9 @@ export interface ILossAnalyticsSnapshot { } const EMPTY_LOSS_SNAPSHOT: ILossAnalyticsSnapshot = { - mean: NaN, - median: NaN, - total: NaN + mean: Number.MAX_SAFE_INTEGER, + median: Number.MAX_SAFE_INTEGER, + total: Number.MAX_SAFE_INTEGER }; Object.freeze(EMPTY_LOSS_SNAPSHOT); @@ -263,7 +262,9 @@ export class NeuralNetwork< runInput: (input: Float32Array) => Float32Array = (input: Float32Array) => { this.setActivation(); - return this.runInput(input); + const output = this.runInput(input); + this._updateRAM(); + return output; }; calculateDeltas: (output: Float32Array, input: Float32Array) => void = ( @@ -301,6 +302,7 @@ export class NeuralNetwork< // Initialize the loss function. this._lossAnalytics = createLossAnalytics(); if (options.loss) this._lossFunction = options.loss; + if (options.updateRAM) this._ramFunction = options.updateRAM; } /** @@ -426,6 +428,7 @@ export class NeuralNetwork< } this.validateInput(formattedInput); const output = this.runInput(formattedInput).slice(0); + this._updateRAM(); if (this.outputLookup) { return (lookup.toObject( this.outputLookup, @@ -435,6 +438,47 @@ export class NeuralNetwork< return (output as unknown) as OutputType; } + protected _updateRAM() { + if (this.ram) { + const updateRAM: RAMFunction | undefined = this.ramFunction; + + if (updateRAM) { + const ramSize = this.ramSize; + const input = this.outputs[0]; + const output = this.outputs[this.outputLayer]; + const loss = this.loss.current.mean; + const deltaLoss = loss - this.loss.previous.mean; + this._ram = this.ram.map( + (layerRAM, layer) => layerRAM.map( + (neuronRAM, neuron) => neuronRAM.map( + (value, index) => { + const kernelFunctionThis: IKernelFunctionThis = { + color: function color(r: number, g: number = 0, b: number = 0, a: number = 0) {}, + constants: { + ramSize + }, + output: { + x: NaN, + y: NaN, + z: NaN + }, + thread: { + x: index, + y: neuron, + z: layer + } + }; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + return updateRAM.call(kernelFunctionThis, this.ram, this.ramSize, input, output, this.sizes, loss, deltaLoss); + } + ) + ) + ); + } + } + } + _runInputSigmoid(input: Float32Array): Float32Array { this.outputs[0] = input; // set output state of input layer @@ -852,47 +896,12 @@ export class NeuralNetwork< ): number | null { // forward propagate this.runInput(value.input); + this._updateRAM(); // back propagate this.calculateDeltas(value.output, value.input); this.adjustWeights(); - if (this.ram) { - const updateRAM: RAMFunction | undefined = this.ramFunction; - - if (updateRAM) { - const input = this.outputs[0]; - const output = this.outputs[this.outputLayer]; - const loss = this.loss.current.mean; - const deltaLoss = loss - this.loss.previous.mean; - this.ram.map( - (layerRAM, layer) => layerRAM.map( - (neuronRAM, neuron) => neuronRAM.map( - (value, index) => { - const kernelFunctionThis: IKernelFunctionThis = { - color: function color(r: number, g: number = 0, b: number = 0, a: number = 0) {}, - constants: {}, - output: { - x: NaN, - y: NaN, - z: NaN - }, - thread: { - x: index, - y: neuron, - z: layer - } - }; - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - return updateRAM.call(kernelFunctionThis, this.ram, this.ramSize, input, output, this.sizes, loss, deltaLoss); - } - ) - ) - ) - } - } - if (logErrorRate) { return mse(this.errors[this.outputLayer]); } @@ -1290,6 +1299,7 @@ export class NeuralNetwork< for (let i = 0; i < preparedData.length; i++) { const output = this.runInput(preparedData[i].input); + this._updateRAM(); const target = preparedData[i].output; const actual = output[0] > this.options.binaryThresh ? 1 : 0; const expected = target[0]; @@ -1337,6 +1347,7 @@ export class NeuralNetwork< for (let i = 0; i < preparedData.length; i++) { const output = this.runInput(preparedData[i].input); + this._updateRAM(); const target = preparedData[i].output; const actual = output.indexOf(max(output)); const expected = target.indexOf(max(target)); From 3d392f1778c3bb781ec1416ac497d8f8ba9c71f2 Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Mon, 17 Jun 2024 08:36:33 -0400 Subject: [PATCH 16/29] Design custom loss function for autoencoders --- README.md | 8 +-- src/autoencoder.test.ts | 6 +- src/autoencoder.ts | 70 ++++++++++++-------- src/errors/untrained-neural-network-error.ts | 5 +- src/index.ts | 4 +- src/neural-network.ts | 4 +- 6 files changed, 57 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 2d07e04c..c45ece39 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ GPU accelerated Neural networks in JavaScript for Browsers and Node.js - [For training with NeuralNetwork](#for-training-with-neuralnetwork) - [For training with `RNNTimeStep`, `LSTMTimeStep` and `GRUTimeStep`](#for-training-with-rnntimestep-lstmtimestep-and-grutimestep) - [For training with `RNN`, `LSTM` and `GRU`](#for-training-with-rnn-lstm-and-gru) - - [For training with `AE`](#for-training-with-ae) + - [For training with `AutoencoderGPU`](#for-training-with-ae) - [Training Options](#training-options) - [Async Training](#async-training) - [Cross Validation](#cross-validation) @@ -318,7 +318,7 @@ net.train([ const output = net.run('I feel great about the world!'); // 'happy' ``` -#### For training with `AE` +#### For training with `AutoencoderGPU` Each training pattern can either: @@ -328,7 +328,7 @@ Each training pattern can either: Training an autoencoder to compress the values of a XOR calculation: ```javascript -const net = new brain.AE( +const net = new brain.AutoencoderGPU( { hiddenLayers: [ 5, 2, 5 ] } @@ -644,7 +644,7 @@ The user interface used: - [`brain.NeuralNetwork`](src/neural-network.ts) - [Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation - [`brain.NeuralNetworkGPU`](src/neural-network-gpu.ts) - [Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation, GPU version -- [`brain.AE`](src/autoencoder.ts) - [Autoencoder or "AE"](https://en.wikipedia.org/wiki/Autoencoder) with backpropogation and GPU support +- [`brain.AutoencoderGPU`](src/autoencoder.ts) - [Autoencoder or "AutoencoderGPU"](https://en.wikipedia.org/wiki/Autoencoder) with backpropogation and GPU support - [`brain.recurrent.RNNTimeStep`](src/recurrent/rnn-time-step.ts) - [Time Step Recurrent Neural Network or "RNN"](https://en.wikipedia.org/wiki/Recurrent_neural_network) - [`brain.recurrent.LSTMTimeStep`](src/recurrent/lstm-time-step.ts) - [Time Step Long Short Term Memory Neural Network or "LSTM"](https://en.wikipedia.org/wiki/Long_short-term_memory) - [`brain.recurrent.GRUTimeStep`](src/recurrent/gru-time-step.ts) - [Time Step Gated Recurrent Unit or "GRU"](https://en.wikipedia.org/wiki/Gated_recurrent_unit) diff --git a/src/autoencoder.test.ts b/src/autoencoder.test.ts index 7838fe1e..a841ea01 100644 --- a/src/autoencoder.test.ts +++ b/src/autoencoder.test.ts @@ -1,4 +1,4 @@ -import AE from "./autoencoder"; +import AutoencoderGPU from "./autoencoder"; const trainingData = [ [0, 0, 0], @@ -7,9 +7,9 @@ const trainingData = [ [1, 1, 0] ]; -const xornet = new AE( +const xornet = new AutoencoderGPU( { - decodedSize: 3, + inputSize: 3, hiddenLayers: [ 5, 2, 5 ] } ); diff --git a/src/autoencoder.ts b/src/autoencoder.ts index e799b042..bebed4dc 100644 --- a/src/autoencoder.ts +++ b/src/autoencoder.ts @@ -1,41 +1,54 @@ -import { KernelOutput, Texture, TextureArrayOutput } from "gpu.js"; -import { IJSONLayer, INeuralNetworkData, INeuralNetworkDatum, INeuralNetworkTrainOptions } from "./neural-network"; +import { IKernelFunctionThis, KernelOutput, Texture, TextureArrayOutput } from "gpu.js"; +import { IJSONLayer, INeuralNetworkData, INeuralNetworkDatum, INeuralNetworkTrainOptions, NeuralNetworkIO, NeuralNetworkRAM } from "./neural-network"; import { INeuralNetworkGPUOptions, NeuralNetworkGPU } from "./neural-network-gpu"; import { INeuralNetworkState } from "./neural-network-types"; import { UntrainedNeuralNetworkError } from "./errors/untrained-neural-network-error"; -export interface IAEOptions { - binaryThresh: number; - decodedSize: number; - hiddenLayers: number[]; +function loss( + this: IKernelFunctionThis, + actual: number, + expected: number, + inputs: NeuralNetworkIO, + ram: NeuralNetworkRAM +) { + let error = expected - actual; + + // if ( o ≈ i0 ) then return 10% of the loss value. + // Otherwise, return 1000% of the full loss value. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + if (Math.round(actual) !== Math.round(inputs[this.thread.x])) error *= 32; + else error *= 0.03125; + + return error; } /** * An autoencoder learns to compress input data down to relevant features and reconstruct input data from its compressed representation. */ -export class AE { +export class AutoencoderGPU extends NeuralNetworkGPU { private decoder?: NeuralNetworkGPU; - private denoiser: NeuralNetworkGPU; constructor ( - options?: Partial + options?: Partial ) { // Create default options for the autoencoder. options ??= {}; - // Create default options for the autoencoder's denoiser subnet. - const denoiserOptions: Partial = {}; - // Inherit the binary threshold of the parent autoencoder. - denoiserOptions.binaryThresh = options.binaryThresh; + options.binaryThresh = options.binaryThresh; // Inherit the hidden layers of the parent autoencoder. - denoiserOptions.hiddenLayers = options.hiddenLayers; + options.hiddenLayers = options.hiddenLayers; + + const decodedSize = options.inputSize ?? options.outputSize ?? 1; // Define the denoiser subnet's input and output sizes. - if (options.decodedSize) denoiserOptions.inputSize = denoiserOptions.outputSize = options.decodedSize; + if (decodedSize) options.inputSize = options.outputSize = decodedSize; + + options.loss ??= loss; - // Create the denoiser subnet of the autoencoder. - this.denoiser = new NeuralNetworkGPU(options); + // Create the autoencoder. + super(options); } /** @@ -51,7 +64,7 @@ export class AE} options * @returns {INeuralNetworkState} */ - train(data: DecodedData[], options?: Partial): INeuralNetworkState { + train(data: Partial[] | INeuralNetworkDatum, Partial>[], options?: Partial): INeuralNetworkState { const preprocessedData: INeuralNetworkDatum, Partial>[] = []; + if (data.length && data.length > 0) for (let datum of data) { - preprocessedData.push( { input: datum, output: datum } ); + preprocessedData.push( { input: datum as Partial, output: datum as Partial } ); } - const results = this.denoiser.train(preprocessedData, options); + const results = super.train(preprocessedData, options); this.decoder = this.createDecoder(); @@ -151,12 +165,12 @@ export class AE} */ private createDecoder() { - const json = this.denoiser.toJSON(); + const json = this.toJSON(); const layers: IJSONLayer[] = []; const sizes: number[] = []; - for (let i = this.encodedLayerIndex; i < this.denoiser.sizes.length; i++) { + for (let i = this.encodedLayerIndex; i < this.sizes.length; i++) { layers.push(json.layers[i]); sizes.push(json.sizes[i]); } @@ -175,15 +189,15 @@ export class AE number; export type RAMFunction = ( @@ -108,7 +108,7 @@ function loss( actual: number, expected: number, inputs: NeuralNetworkIO, - memory: NeuralNetworkRAM + ram: NeuralNetworkRAM ) { return expected - actual; } From ce98bf1727eacb1994e1dae21052c0ad63ab80b7 Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:38:24 -0400 Subject: [PATCH 17/29] Fix CI task errors --- src/errors/untrained-neural-network-error.ts | 5 +-- src/neural-network-gpu.ts | 4 ++ src/neural-network.ts | 45 +++++++++++++++++--- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/errors/untrained-neural-network-error.ts b/src/errors/untrained-neural-network-error.ts index 3884fcc6..d916b21f 100644 --- a/src/errors/untrained-neural-network-error.ts +++ b/src/errors/untrained-neural-network-error.ts @@ -1,7 +1,6 @@ -import { INeuralNetworkData, NeuralNetwork } from "../neural-network"; -import { NeuralNetworkGPU } from "../neural-network-gpu"; +import { NeuralNetwork, NeuralNetworkIO } from "../neural-network"; -export class UntrainedNeuralNetworkError extends Error { +export class UntrainedNeuralNetworkError> extends Error { constructor ( neuralNetwork: object ) { diff --git a/src/neural-network-gpu.ts b/src/neural-network-gpu.ts index e83d4e89..5e2dd329 100644 --- a/src/neural-network-gpu.ts +++ b/src/neural-network-gpu.ts @@ -286,6 +286,10 @@ export class NeuralNetworkGPU< super.lossFunction = value; } + public get ramFunction(): RAMFunction | undefined { + return super.ramFunction; + } + public set ramFunction( value: RAMFunction | undefined ) { diff --git a/src/neural-network.ts b/src/neural-network.ts index ca8d5f26..89b6d14e 100644 --- a/src/neural-network.ts +++ b/src/neural-network.ts @@ -12,12 +12,33 @@ import { max } from './utilities/max'; import { mse } from './utilities/mse'; import { randos } from './utilities/randos'; import { zeros } from './utilities/zeros'; -import { IKernelFunctionThis, KernelOutput } from 'gpu.js'; +import { IKernelFunctionThis } from 'gpu.js'; +/** + * An input or output layer of a neural network. + * This data type exists to allow kernel functions to operate on individual layers. + * This functionality is essential to custom loss functions. + */ export type NeuralNetworkIO = number[] | number[][] | number[][][]; +/** + * A read-write state matrix designed to hold metadata for use by `loss` functions. + * This data is read-only to the `loss` function. + * To the `updateRAM` kernel function, this data is read-write. + * To the neural network consumer, + * the `ram` property is made public to allow for modifications to be made in addition to reading. + */ export type NeuralNetworkRAM = number[][][]; +/** + * A loss function determines how fit a neural network currently is. + * The higher the value returned by this function, + * the less accurate the network is. + * The lower the value returned by this function is, + * the more accurate the network is. + * + * Here, `ram` is read-only. + */ export type LossFunction = ( this: IKernelFunctionThis, actual: number, @@ -26,6 +47,13 @@ export type LossFunction = ( ram: NeuralNetworkRAM ) => number; +/** + * A RAM function updates the RAM matrix of the neural network. + * + * Here, `ram` is read-write. + * The actual matrix passed to the function is read-only. + * However, the return value of the function directly corresponds to a value within the RAM matrix. + */ export type RAMFunction = ( this: IKernelFunctionThis, ram: NeuralNetworkRAM, @@ -36,6 +64,10 @@ export type RAMFunction = ( lossDelta: number ) => number; +/** + * Each time the loss is calculated, + * a snapshot is taken of various loss analytics. + */ export interface ILossAnalyticsSnapshot { mean: number; median: number; @@ -50,10 +82,13 @@ const EMPTY_LOSS_SNAPSHOT: ILossAnalyticsSnapshot = { Object.freeze(EMPTY_LOSS_SNAPSHOT); -function createLossAnalyticsSnapshot() { +function createLossAnalyticsSnapshot(): ILossAnalyticsSnapshot { return JSON.parse(JSON.stringify(EMPTY_LOSS_SNAPSHOT)); } +/** + * A collection of analytics pertaining to the results of the loss function. + */ export interface ILossAnalytics { current: ILossAnalyticsSnapshot; max: ILossAnalyticsSnapshot; @@ -72,7 +107,7 @@ const EMPTY_LOSS: ILossAnalytics = { Object.freeze(EMPTY_LOSS); -export function createLossAnalytics() { +export function createLossAnalytics(): ILossAnalytics { return JSON.parse(JSON.stringify(EMPTY_LOSS)); } @@ -109,7 +144,7 @@ function loss( expected: number, inputs: NeuralNetworkIO, ram: NeuralNetworkRAM -) { +): number { return expected - actual; } @@ -438,7 +473,7 @@ export class NeuralNetwork< return (output as unknown) as OutputType; } - protected _updateRAM() { + protected _updateRAM(): void { if (this.ram) { const updateRAM: RAMFunction | undefined = this.ramFunction; From 51b9aa9ddc4f97faa0a7377dc9e9542c7b47095f Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:47:44 -0400 Subject: [PATCH 18/29] Fix a CI task related to type coersion --- src/errors/untrained-neural-network-error.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/errors/untrained-neural-network-error.ts b/src/errors/untrained-neural-network-error.ts index d916b21f..bd7722f3 100644 --- a/src/errors/untrained-neural-network-error.ts +++ b/src/errors/untrained-neural-network-error.ts @@ -1,6 +1,4 @@ -import { NeuralNetwork, NeuralNetworkIO } from "../neural-network"; - -export class UntrainedNeuralNetworkError> extends Error { +export class UntrainedNeuralNetworkError extends Error { constructor ( neuralNetwork: object ) { From c5c84383ba5fac5e4630e6a940e5284be1f90326 Mon Sep 17 00:00:00 2001 From: voidvoxel <160202125+voidvoxel@users.noreply.github.com> Date: Mon, 17 Jun 2024 15:13:15 -0400 Subject: [PATCH 19/29] TypeScript hates me today --- src/autoencoder.ts | 9 +++------ src/errors/untrained-neural-network-error.ts | 8 +++++++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/autoencoder.ts b/src/autoencoder.ts index bebed4dc..71d6b386 100644 --- a/src/autoencoder.ts +++ b/src/autoencoder.ts @@ -35,15 +35,12 @@ export class AutoencoderGPU