From 1d5a750260b7be076228eccb88bb601ad8f9f410 Mon Sep 17 00:00:00 2001 From: "Pranev (NeTT)" Date: Fri, 20 Sep 2024 22:46:47 +0530 Subject: [PATCH] feat: Time per epoch and ETA logging when `silent=false` (#64) * add time logger * return vec on fit * use Set * use logistic reg --- crates/core/src/cpu/backend.rs | 43 +++++++++++++++++--- deno.lock | 4 ++ examples/classification/spam.ts | 29 ++++++------- packages/utilities/src/text/vectorizer.ts | 3 +- packages/utilities/src/utils/array/unique.ts | 3 +- 5 files changed, 58 insertions(+), 24 deletions(-) diff --git a/crates/core/src/cpu/backend.rs b/crates/core/src/cpu/backend.rs index 1273ff8..91051fe 100644 --- a/crates/core/src/cpu/backend.rs +++ b/crates/core/src/cpu/backend.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::time::Instant; use ndarray::{ArrayD, ArrayViewD, IxDyn}; use safetensors::{serialize, SafeTensors}; @@ -110,7 +111,10 @@ impl Backend { match layers { Some(layer_indices) => { for layer_index in layer_indices { - let layer = self.layers.get_mut(layer_index).expect(&format!("Layer #{} does not exist.", layer_index)); + let layer = self + .layers + .get_mut(layer_index) + .expect(&format!("Layer #{} does not exist.", layer_index)); inputs = layer.forward_propagate(inputs, training); } } @@ -141,6 +145,10 @@ impl Backend { let mut disappointments = 0; let mut best_net = self.save(); let mut cost = 0f32; + let mut time: u128; + let mut total_time = 0u128; + let start = Instant::now(); + let total_iter = epochs * datasets.len(); while epoch < epochs { let mut total = 0.0; for (i, dataset) in datasets.iter().enumerate() { @@ -152,7 +160,19 @@ impl Backend { let minibatch = outputs.dim()[0]; if !self.silent && ((i + 1) * minibatch) % batches == 0 { cost = total / (batches) as f32; - let msg = format!("Epoch={}, Dataset={}, Cost={}", epoch, i * minibatch, cost); + time = start.elapsed().as_millis() - total_time; + total_time += time; + let current_iter = epoch * datasets.len() + i; + let msg = format!( + "Epoch={}, Dataset={}, Cost={}, Time={}s, ETA={}s", + epoch, + i * minibatch, + cost, + (time as f32) / 1000.0, + (((total_time as f32) / current_iter as f32) + * (total_iter - current_iter) as f32) + / 1000.0 + ); (self.logger.log)(msg); total = 0.0; } @@ -165,17 +185,28 @@ impl Backend { disappointments = 0; best_cost = cost; best_net = self.save(); - } else { + } else { disappointments += 1; if !self.silent { - println!("Patience counter: {} disappointing epochs out of {}.", disappointments, self.patience); + println!( + "Patience counter: {} disappointing epochs out of {}.", + disappointments, self.patience + ); } } if disappointments >= self.patience { if !self.silent { - println!("No improvement for {} epochs. Stopping early at cost={}", disappointments, best_cost); + println!( + "No improvement for {} epochs. Stopping early at cost={}", + disappointments, best_cost + ); } - let net = Self::load(&best_net, Logger { log: |x| println!("{}", x) }); + let net = Self::load( + &best_net, + Logger { + log: |x| println!("{}", x), + }, + ); self.layers = net.layers; break; } diff --git a/deno.lock b/deno.lock index 74d33ec..a11e2b8 100644 --- a/deno.lock +++ b/deno.lock @@ -4,6 +4,7 @@ "jsr:@denosaurs/netsaur@0.4.0": "0.4.0", "jsr:@denosaurs/plug@1.0.3": "1.0.3", "jsr:@std/assert@~0.213.1": "0.213.1", + "jsr:@std/csv@1.0.3": "1.0.3", "jsr:@std/encoding@0.213.1": "0.213.1", "jsr:@std/fmt@0.213.1": "0.213.1", "jsr:@std/fs@0.213.1": "0.213.1", @@ -29,6 +30,9 @@ "@std/assert@0.213.1": { "integrity": "24c28178b30c8e0782c18e8e94ea72b16282207569cdd10ffb9d1d26f2edebfe" }, + "@std/csv@1.0.3": { + "integrity": "623acf0dcb88d62ba727c3611ad005df7f109ede8cac833e3986f540744562e5" + }, "@std/encoding@0.213.1": { "integrity": "fcbb6928713dde941a18ca5db88ca1544d0755ec8fb20fe61e2dc8144b390c62" }, diff --git a/examples/classification/spam.ts b/examples/classification/spam.ts index 38939b0..c4cabb1 100644 --- a/examples/classification/spam.ts +++ b/examples/classification/spam.ts @@ -14,12 +14,12 @@ import { // Import helpers for metrics import { ClassificationReport, - CountVectorizer, - SplitTokenizer, - TfIdfTransformer, + TextCleaner, + TextVectorizer, // Split the dataset useSplit, } from "../../packages/utilities/mod.ts"; +import { SigmoidLayer } from "../../mod.ts"; // Define classes const ymap = ["spam", "ham"]; @@ -32,25 +32,21 @@ const data = parse(_data); const x = data.map((msg) => msg[1]); // Get the classes -const y = data.map((msg) => (ymap.indexOf(msg[0]) === 0 ? -1 : 1)); +const y = data.map((msg) => (ymap.indexOf(msg[0]) === 0 ? 0 : 1)); // Split the dataset for training and testing const [train, test] = useSplit({ ratio: [7, 3], shuffle: true }, x, y); // Vectorize the text messages -const tokenizer = new SplitTokenizer({ - skipWords: "english", - standardize: { lowercase: true }, -}).fit(train[0]); +const textCleaner = new TextCleaner({ lowercase: true }); -const vec = new CountVectorizer(tokenizer.vocabulary.size); +train[0] = textCleaner.clean(train[0]) -const x_vec = vec.transform(tokenizer.transform(train[0]), "f32") +const vec = new TextVectorizer("tfidf").fit(train[0]); -const tfidf = new TfIdfTransformer(); +const x_vec = vec.transform(train[0], "f32"); -const x_tfidf = tfidf.fit(x_vec).transform(x_vec) // Setup the CPU backend for Netsaur await setupBackend(CPU); @@ -73,14 +69,15 @@ const net = new Sequential({ // A dense layer with 1 neuron DenseLayer({ size: [1] }), // A sigmoid activation layer + SigmoidLayer() ], // We are using Log Loss for finding cost - cost: Cost.Hinge, + cost: Cost.BinCrossEntropy, optimizer: NadamOptimizer(), }); -const inputs = tensor(x_tfidf); +const inputs = tensor(x_vec); const time = performance.now(); // Train the network @@ -99,10 +96,10 @@ net.train( console.log(`training time: ${performance.now() - time}ms`); -const x_vec_test = tfidf.transform(vec.transform(tokenizer.transform(test[0]), "f32")); +const x_vec_test = vec.transform(test[0], "f32"); // Calculate metrics const res = await net.predict(tensor(x_vec_test)); -const y1 = res.data.map((i) => (i < 0 ? -1 : 1)); +const y1 = res.data.map((i) => (i < 0.5 ? 0 : 1)); const cMatrix = new ClassificationReport(test[1], y1); console.log("Confusion Matrix: ", cMatrix); diff --git a/packages/utilities/src/text/vectorizer.ts b/packages/utilities/src/text/vectorizer.ts index 0f7e3ba..0293319 100644 --- a/packages/utilities/src/text/vectorizer.ts +++ b/packages/utilities/src/text/vectorizer.ts @@ -13,7 +13,7 @@ export class TextVectorizer { this.mode = mode; this.mapper = new DiscreteMapper(); } - fit(document: string | string[]) { + fit(document: string | string[]): TextVectorizer { this.mapper.fit( (Array.isArray(document) ? document.join(" ") : document).split(" ") ); @@ -27,6 +27,7 @@ export class TextVectorizer { this.transformer.fit(this.encoder.transform(tokens, "f32")); } } + return this; } transform
( document: string | string[], diff --git a/packages/utilities/src/utils/array/unique.ts b/packages/utilities/src/utils/array/unique.ts index 6a4dcea..4124d16 100644 --- a/packages/utilities/src/utils/array/unique.ts +++ b/packages/utilities/src/utils/array/unique.ts @@ -6,5 +6,6 @@ */ export function useUnique(arr: ArrayLike): T[] { const array = Array.from(arr); - return array.filter((x, i) => array.indexOf(x) === i); + return [...new Set(array)] +// return array.filter((x, i) => array.indexOf(x) === i); }