Update activation and shuffle methods (#313)

Co-authored-by: Nigel (MacBook) <nigel@laptop>
stSoftwareAU · Mar 13, 2024 · 6723603 · 6723603
1 parent d2cb4fc
commit 6723603
Show file tree

Hide file tree

Showing 10 changed files with 50 additions and 97 deletions.
diff --git a/bench/Activate.ts b/bench/Activate.ts
@@ -18,7 +18,7 @@ const inputs = makeInputs(creature);
 
 export function perform() {
   const config = new BackPropagationConfig();
-  for (let i = 0; i < 10000; i++) {
+  for (let i = 0; i < 50; i++) {
     const input = inputs[i % inputs.length];
     creature.activateAndTrace(input);
     creature.propagate([i % 2], config);

diff --git a/src/Creature.ts b/src/Creature.ts
@@ -983,7 +983,8 @@ export class Creature implements CreatureInternal {
       );
     }
 
-    const indices = Array.from({ length: this.output }, (_, i) => i); // Create an array of indices
+    this.state.cacheAdjustedActivation.clear();
+    const indices = Int32Array.from({ length: this.output }, (_, i) => i); // Create an array of indices
 
     if (!config.disableRandomSamples) {
       CreatureUtil.shuffle(indices);

diff --git a/src/architecture/BackPropagation.ts b/src/architecture/BackPropagation.ts
@@ -114,7 +114,7 @@ export function adjustedBias(
   config: BackPropagationConfig,
 ): number {
   if (node.type == "constant") {
-    return node.bias ? node.bias : 0;
+    return node.bias;
   } else {
     const ns = node.creature.state.node(node.index);
 
@@ -151,54 +151,6 @@ export function adjustedBias(
   }
 }
 
-// export function adjustedBiasV2(
-//   node: Neuron,
-//   config: BackPropagationConfig,
-// ): number {
-//   if (node.type == "constant") {
-//     return node.bias ? node.bias : 0;
-//   } else {
-//     const ns = node.creature.state.node(node.index);
-
-//     if (ns.count) {
-//       const totalDifferenceBias = ns.totalValue - ns.totalWeightedSum;
-//       const totalBias = totalDifferenceBias + (node.bias * config.generations);
-//       const samples = ns.count + config.generations;
-
-//       const averageDifferenceBias = totalBias /
-//         samples;
-
-//       // if (Math.abs(averageDifferenceBias - node.bias) > 0.0001) {
-//       //   console.info(
-//       //     `averageDifferenceBias: ${averageDifferenceBias} node.bias: ${node.bias}`,
-//       //   );
-//       // }
-//       const unaccountedRatioBias = 1 - (ns.totalValue / ns.totalWeightedSum);
-
-//       if (
-//         config.useAverageDifferenceBias == "Yes" ||
-//         Number.isFinite(unaccountedRatioBias) == false
-//       ) {
-//         if (Number.isFinite(averageDifferenceBias)) {
-//           return limitBias(averageDifferenceBias, node.bias, config);
-//         }
-//       } else if (
-//         config.useAverageDifferenceBias == "No" ||
-//         (
-//           Math.abs(averageDifferenceBias - node.bias) <
-//             Math.abs(unaccountedRatioBias - node.bias)
-//         )
-//       ) {
-//         return limitBias(unaccountedRatioBias, node.bias, config);
-//       } else {
-//         return limitBias(averageDifferenceBias, node.bias, config);
-//       }
-//     }
-
-//     return node.bias;
-//   }
-// }
-
 export function limitActivationToRange(node: Neuron, activation: number) {
   if (node.type == "input" || node.type == "constant") {
     return activation;
@@ -232,15 +184,10 @@ export function toValue(neuron: Neuron, activation: number, hint?: number) {
     return activation;
   }
   const squash = neuron.findSquash();
-  if (((squash as unknown) as UnSquashInterface).unSquash != undefined) {
-    const unSquasher = (squash as unknown) as UnSquashInterface;
-    const value = unSquasher.unSquash(activation, hint);
+  const unSquash = (squash as UnSquashInterface).unSquash;
+  if (unSquash !== undefined) {
+    const value = unSquash.call(squash, activation, hint);
 
-    if (!Number.isFinite(value)) {
-      throw new Error(
-        `${neuron.index}: ${neuron.squash}.unSquash(${activation}) invalid -> ${value}`,
-      );
-    }
     return limitValue(value);
   } else {
     return activation;

diff --git a/src/architecture/CreatureState.ts b/src/architecture/CreatureState.ts
@@ -46,31 +46,7 @@ export class NeuronState implements NeuronStateInterface {
     const activationDifference = Math.abs(targetActivation - activation);
     if (activationDifference < config.plankConstant) {
       difference = 0;
-    } //else {
-    // const activationPercent = activationDifference /
-    //   Math.max(Math.abs(targetActivation), Math.abs(activation));
-    // const valuePercent = Math.abs(difference) /
-    //   Math.max(Math.abs(targetValue), Math.abs(value));
-
-    // if (activationPercent > valuePercent) {
-    //   const originalDifference = difference;
-    //   const adjustPercent = valuePercent / activationPercent;
-    //   difference = originalDifference * adjustPercent;
-    // console.info(
-    //   `restrict ${originalDifference} to ${difference} as activation percentage ${
-    //     activationPercent.toFixed(3)
-    //   } versus ${valuePercent.toFixed(3)} adjustPercent ${
-    //     adjustPercent.toFixed(3)
-    //   }`,
-    // );
-    // }
-    // }
-
-    // if (Math.abs(difference) > PLANK_CONSTANT) {
-    //   console.info(
-    //     `difference ${difference} targetValue ${targetValue} value ${value} targetActivation ${targetActivation} activation ${activation}`,
-    //   );
-    // }
+    }
 
     if (!config.disableExponentialScaling) {
       // Squash the difference using the hyperbolic tangent function and scale it
@@ -100,11 +76,13 @@ export class CreatureState {
   private network;
   public activations: Float32Array = new Float32Array(0);
   public propagated = false;
+  readonly cacheAdjustedActivation;
 
   constructor(network: Creature) {
     this.network = network;
     this.nodeMap = new Map<number, NeuronState>();
     this.connectionMap = new Map<number, Map<number, SynapseState>>();
+    this.cacheAdjustedActivation = new Map<number, number>();
   }
 
   connection(from: number, to: number): SynapseState {

diff --git a/src/architecture/CreatureUtils.ts b/src/architecture/CreatureUtils.ts
@@ -6,7 +6,7 @@ export class CreatureUtil {
   private static NAMESPACE = "843dc7df-f60b-47f6-823d-2992e0a4295c";
 
   /* Shuffle array in place using the Fisher-Yates shuffle algorithm */
-  static shuffle<T>(array: T[]): void {
+  static shuffle(array: Int32Array): void {
     if (array.length > 1) {
       for (let i = array.length; i--;) {
         const j = Math.round(Math.random() * i);

diff --git a/src/architecture/Neuron.ts b/src/architecture/Neuron.ts
@@ -383,7 +383,7 @@ export class Neuron implements TagsInterface, NeuronInternal {
       const listLength = toList.length;
 
       if (listLength) {
-        const indices = Array.from({ length: listLength }, (_, i) => i); // Create an array of indices
+        const indices = Int32Array.from({ length: listLength }, (_, i) => i); // Create an array of indices
 
         if (!config.disableRandomSamples) {
           CreatureUtil.shuffle(indices);
@@ -467,16 +467,33 @@ export class Neuron implements TagsInterface, NeuronInternal {
       }
     }
     ns.traceActivation(limitedActivation);
+    if (Math.abs(limitedActivation - activation) > config.plankConstant) {
+      this.creature.state.cacheAdjustedActivation.delete(this.index);
+    }
 
     return limitedActivation;
   }
 
+  /**
+   * Adjusts the activation based on the current state
+   */
   adjustedActivation(config: BackPropagationConfig) {
-    if (this.type == "input") {
-      return this.creature.state.activations[this.index];
+    const adjustedActivationValue = this.creature.state.cacheAdjustedActivation
+      .get(this.index);
+
+    if (adjustedActivationValue !== undefined) {
+      return adjustedActivationValue;
     }
+    const value = this.rawAdjustedActivation(config);
 
-    if (this.type == "constant") {
+    this.creature.state.cacheAdjustedActivation.set(this.index, value);
+    return value;
+  }
+
+  rawAdjustedActivation(config: BackPropagationConfig) {
+    if (this.type == "input") {
+      return this.creature.state.activations[this.index];
+    } else if (this.type == "constant") {
       return this.bias;
     } else {
       const aBias = adjustedBias(this, config);

diff --git a/src/architecture/Training.ts b/src/architecture/Training.ts
@@ -57,7 +57,7 @@ export async function trainDir(
     Math.max(0, options.trainingSampleRate ?? Math.max(Math.random(), 0.3)),
   );
 
-  const indxMap = new Map<string, number[]>();
+  const indxMap = new Map<string, Int32Array>();
   const files: string[] = dataFiles(dataDir).map((fn) => dataDir + "/" + fn);
   const cached = files.length == 1;
   if (!cached) {
@@ -67,7 +67,10 @@ export async function trainDir(
 
   // Randomize the list of files
   if (!options.disableRandomSamples) {
-    CreatureUtil.shuffle(files);
+    for (let i = files.length; i--;) {
+      const j = Math.round(Math.random() * i);
+      [files[i], files[j]] = [files[j], files[i]];
+    }
   }
 
   // Loops the training process
@@ -123,12 +126,16 @@ export async function trainDir(
       let indices = indxMap.get(fn);
 
       if (!indices) {
-        indices = Array.from({ length: json.length }, (_, i) => i); // Create an array of indices
+        const tmpIndexes = Int32Array.from(
+          { length: json.length },
+          (_, i) => i,
+        ); // Create an array of indices
 
         if (!options.disableRandomSamples) {
-          CreatureUtil.shuffle(indices);
+          CreatureUtil.shuffle(tmpIndexes);
         }
-        indices.length = len;
+        indices = tmpIndexes.slice(0, len);
+
         if (len != json.length) {
           /* No need to cache what we wont use */
           indxMap.set(fn, indices);

diff --git a/src/compact/CompactUnused.ts b/src/compact/CompactUnused.ts
@@ -11,7 +11,10 @@ export async function compactUnused(
   const clean = Creature.fromJSON(start.exportJSON());
   const compacted = Creature.fromJSON(clean.exportJSON());
 
-  const indices = Array.from({ length: traced.neurons.length }, (_, i) => i); // Create an array of indices
+  const indices = Int32Array.from(
+    { length: traced.neurons.length },
+    (_, i) => i,
+  ); // Create an array of indices
 
   CreatureUtil.shuffle(indices);
 

diff --git a/src/methods/activations/aggregate/IF.ts b/src/methods/activations/aggregate/IF.ts
@@ -321,7 +321,7 @@ export class IF implements NeuronActivationInterface, ApplyLearningsInterface {
     let targetWeightedSum = 0;
 
     const listLength = toList.length;
-    const indices = Array.from({ length: listLength }, (_, i) => i); // Create an array of indices
+    const indices = Int32Array.from({ length: listLength }, (_, i) => i); // Create an array of indices
 
     if (!config.disableRandomSamples) {
       CreatureUtil.shuffle(indices);

diff --git a/test/Propagate/SingleNeuron.ts b/test/Propagate/SingleNeuron.ts
@@ -258,7 +258,7 @@ Deno.test("ManySame", () => {
 });
 
 Deno.test("propagateSingleNeuronKnown", () => {
-  const traceDir = ".trace/propagateSingleNeuronKnown";
+  const traceDir = ".test/propagateSingleNeuronKnown";
   ensureDirSync(traceDir);
 
   for (let attempts = 0; true; attempts++) {