From 9253c4713dd84d2877e63a9756a24ad65a10a209 Mon Sep 17 00:00:00 2001 From: tharvik Date: Wed, 28 Feb 2024 19:13:10 +0100 Subject: [PATCH] discojs-{node,web}: don't reexport core Closes: #638 --- cli/src/args.ts | 3 ++- cli/src/cli.ts | 3 ++- cli/src/data.ts | 10 +++++----- cli/src/utils.ts | 2 +- .../discojs-core/src/aggregator/secure.spec.ts | 2 +- .../src/validation/validator.spec.ts | 11 +++++------ .../src/weights/aggregation.spec.ts | 2 +- .../data_loader => data}/image_loader.spec.ts | 17 +++++++++-------- .../data_loader => data}/image_loader.ts | 2 +- discojs/discojs-node/src/data/index.ts | 2 ++ .../data_loader => data}/tabular_loader.spec.ts | 8 +++++--- .../data_loader => data}/tabular_loader.ts | 2 +- .../data_loader => data}/text_loader.ts | 0 .../src/dataset/data_loader/index.ts | 2 -- discojs/discojs-node/src/imports.ts | 1 - discojs/discojs-node/src/index.ts | 4 +--- .../data_loader => data}/image_loader.ts | 4 ++-- discojs/discojs-web/src/data/index.ts | 3 +++ .../data_loader => data}/tabular_loader.ts | 4 ++-- .../data_loader => data}/text_loader.ts | 4 ++-- .../src/dataset/data_loader/index.ts | 3 --- discojs/discojs-web/src/imports.ts | 2 -- discojs/discojs-web/src/index.ts | 5 ++--- discojs/discojs-web/src/memory/memory.ts | 3 ++- server/src/config.ts | 2 +- server/src/get_server.ts | 2 +- server/src/router/decentralized/server.ts | 4 ++-- server/src/router/federated/server.ts | 16 +++++++++------- server/src/router/server.ts | 2 +- server/src/router/tasks.ts | 4 ++-- server/src/tasks.ts | 4 ++-- server/tests/client/decentralized.spec.ts | 3 ++- server/tests/client/federated.spec.ts | 3 ++- server/tests/e2e/decentralized.spec.ts | 5 +++-- server/tests/e2e/federated.spec.ts | 9 +++++---- server/tests/utils.ts | 2 +- web-client/cypress/e2e/tasks.cy.ts | 2 +- web-client/src/clients.ts | 2 +- web-client/src/components/data/Data.vue | 2 +- .../src/components/data/DataHarmonization.vue | 2 +- .../data/dataset_input/DatasetInput.vue | 2 +- web-client/src/components/pages/TaskList.vue | 2 +- .../src/components/sidebar/ModelLibrary.vue | 5 +++-- .../components/task_creation_form/TaskForm.vue | 2 +- web-client/src/components/testing/Tester.vue | 5 +++-- web-client/src/components/testing/Testing.vue | 9 +++++---- .../src/components/training/Description.vue | 2 +- web-client/src/components/training/Finished.vue | 6 ++++-- .../src/components/training/ModelCaching.vue | 5 +++-- web-client/src/components/training/Trainer.vue | 5 +++-- web-client/src/components/training/Training.vue | 9 +++++---- .../components/training/TrainingInformation.vue | 2 +- web-client/src/store/memory.ts | 2 +- web-client/src/store/tasks.ts | 2 +- web-client/src/store/training.ts | 2 +- web-client/src/store/validation.ts | 2 +- 56 files changed, 119 insertions(+), 106 deletions(-) rename discojs/discojs-node/src/{dataset/data_loader => data}/image_loader.spec.ts (92%) rename discojs/discojs-node/src/{dataset/data_loader => data}/image_loader.ts (81%) create mode 100644 discojs/discojs-node/src/data/index.ts rename discojs/discojs-node/src/{dataset/data_loader => data}/tabular_loader.spec.ts (89%) rename discojs/discojs-node/src/{dataset/data_loader => data}/tabular_loader.ts (83%) rename discojs/discojs-node/src/{dataset/data_loader => data}/text_loader.ts (100%) delete mode 100644 discojs/discojs-node/src/dataset/data_loader/index.ts delete mode 100644 discojs/discojs-node/src/imports.ts rename discojs/discojs-web/src/{dataset/data_loader => data}/image_loader.ts (62%) create mode 100644 discojs/discojs-web/src/data/index.ts rename discojs/discojs-web/src/{dataset/data_loader => data}/tabular_loader.ts (67%) rename discojs/discojs-web/src/{dataset/data_loader => data}/text_loader.ts (77%) delete mode 100644 discojs/discojs-web/src/dataset/data_loader/index.ts delete mode 100644 discojs/discojs-web/src/imports.ts diff --git a/cli/src/args.ts b/cli/src/args.ts index dd8890089..e8c5b6937 100644 --- a/cli/src/args.ts +++ b/cli/src/args.ts @@ -1,6 +1,7 @@ import { parse } from 'ts-command-line-args' import { Map } from 'immutable' -import { defaultTasks, type Task } from '@epfml/discojs-node' +import type { Task } from '@epfml/discojs-core' +import { defaultTasks } from '@epfml/discojs-core' interface BenchmarkArguments { task: Task diff --git a/cli/src/cli.ts b/cli/src/cli.ts index d4fc3b248..f483933bf 100644 --- a/cli/src/cli.ts +++ b/cli/src/cli.ts @@ -1,6 +1,7 @@ import { Range } from 'immutable' -import { Disco, TrainingSchemes, type TrainerLog, type data, type Task } from '@epfml/discojs-node' +import type { TrainerLog, data, Task } from '@epfml/discojs-core' +import { Disco, TrainingSchemes } from '@epfml/discojs-core' import { startServer, saveLog } from './utils' import { getTaskData } from './data' diff --git a/cli/src/data.ts b/cli/src/data.ts index c099ba9bf..c850a7019 100644 --- a/cli/src/data.ts +++ b/cli/src/data.ts @@ -3,8 +3,8 @@ import fs from 'node:fs' import fs_promises from 'fs/promises' import path from 'node:path' -import type { Task, data } from '@epfml/discojs-node' -import { node } from '@epfml/discojs-node' +import type { Task, data } from '@epfml/discojs-core' +import { NodeImageLoader, NodeTabularLoader } from '@epfml/discojs-node' function filesFromFolder (dir: string, folder: string, fractionToKeep: number): string[] { const f = fs.readdirSync(dir + folder) @@ -35,7 +35,7 @@ async function simplefaceData (task: Task): Promise { const labels = filesPerFolder.flatMap((files, index) => Array(files.length).fill(index)) const files = filesPerFolder.flat() - return await new node.data.NodeImageLoader(task).loadAll(files, { labels }) + return await new NodeImageLoader(task).loadAll(files, { labels }) } async function cifar10Data (cifar10: Task): Promise { @@ -43,13 +43,13 @@ async function cifar10Data (cifar10: Task): Promise { const files = (await fs_promises.readdir(dir)).map((file) => path.join(dir, file)) const labels = Range(0, 24).map((label) => (label % 10).toString()).toArray() - return await new node.data.NodeImageLoader(cifar10).loadAll(files, { labels }) + return await new NodeImageLoader(cifar10).loadAll(files, { labels }) } async function titanicData (titanic: Task): Promise { const dir = '../example_training_data/titanic_train.csv' - const data = await (new node.data.NodeTabularLoader(titanic, ',').loadAll( + const data = await (new NodeTabularLoader(titanic, ',').loadAll( ['file://'.concat(dir)], { features: titanic.trainingInformation?.inputColumns, diff --git a/cli/src/utils.ts b/cli/src/utils.ts index 42c27b879..3c1f0dc9c 100644 --- a/cli/src/utils.ts +++ b/cli/src/utils.ts @@ -1,7 +1,7 @@ import type http from 'node:http' import fs from 'node:fs' -import { type TrainerLog } from '@epfml/discojs-node' +import type { TrainerLog } from '@epfml/discojs-core' import { Disco } from '@epfml/disco-server' export async function startServer (): Promise<[http.Server, URL]> { diff --git a/discojs/discojs-core/src/aggregator/secure.spec.ts b/discojs/discojs-core/src/aggregator/secure.spec.ts index 088a70fb6..0f5f33797 100644 --- a/discojs/discojs-core/src/aggregator/secure.spec.ts +++ b/discojs/discojs-core/src/aggregator/secure.spec.ts @@ -1,7 +1,7 @@ import { List, Set, Range } from 'immutable' import { assert } from 'chai' -import { aggregator as aggregators, aggregation, WeightsContainer, defaultTasks } from '@epfml/discojs-node' +import { aggregator as aggregators, aggregation, WeightsContainer, defaultTasks } from '@epfml/discojs-core' describe('secret shares test', function () { const epsilon = 1e-4 diff --git a/discojs/discojs-core/src/validation/validator.spec.ts b/discojs/discojs-core/src/validation/validator.spec.ts index 63534cab4..24d3a7db5 100644 --- a/discojs/discojs-core/src/validation/validator.spec.ts +++ b/discojs/discojs-core/src/validation/validator.spec.ts @@ -1,10 +1,9 @@ import { assert } from 'chai' import fs from 'fs' -import { - type Task, node, Validator, ConsoleLogger, EmptyMemory, - client as clients, type data, aggregator, defaultTasks -} from '@epfml/discojs-node' +import type { Task, data } from '../..' +import { Validator, ConsoleLogger, EmptyMemory, client as clients, aggregator, defaultTasks } from '../..' +import { NodeImageLoader, NodeTabularLoader } from '@epfml/discojs-node' const simplefaceMock = { taskID: 'simple_face', @@ -32,7 +31,7 @@ describe('validator', () => { .map((file: string) => dir + subdir + file)) const labels = files.flatMap((files, index) => Array(files.length).fill(index)) - const data = (await new node.data.NodeImageLoader(simplefaceMock) + const data = (await new NodeImageLoader(simplefaceMock) .loadAll(files.flat(), { labels })).train const meanAggregator = new aggregator.MeanAggregator(simplefaceMock) const client = new clients.Local(new URL('http://localhost:8080'), simplefaceMock, meanAggregator) @@ -63,7 +62,7 @@ describe('validator', () => { it('titanic validator', async () => { const titanicTask = defaultTasks.titanic.getTask() const files = ['../../example_training_data/titanic_train.csv'] - const data: data.Data = (await new node.data.NodeTabularLoader(titanicTask, ',').loadAll(files, { + const data: data.Data = (await new NodeTabularLoader(titanicTask, ',').loadAll(files, { features: titanicTask.trainingInformation.inputColumns, labels: titanicTask.trainingInformation.outputColumns, shuffle: false diff --git a/discojs/discojs-core/src/weights/aggregation.spec.ts b/discojs/discojs-core/src/weights/aggregation.spec.ts index 97774bdc0..fff526cbc 100644 --- a/discojs/discojs-core/src/weights/aggregation.spec.ts +++ b/discojs/discojs-core/src/weights/aggregation.spec.ts @@ -1,6 +1,6 @@ import { assert } from 'chai' -import { WeightsContainer, aggregation } from '@epfml/discojs-node' +import { WeightsContainer, aggregation } from '.' describe('weights aggregation', () => { it('avg of weights with two operands', async () => { diff --git a/discojs/discojs-node/src/dataset/data_loader/image_loader.spec.ts b/discojs/discojs-node/src/data/image_loader.spec.ts similarity index 92% rename from discojs/discojs-node/src/dataset/data_loader/image_loader.spec.ts rename to discojs/discojs-node/src/data/image_loader.spec.ts index c38417b56..dda02de96 100644 --- a/discojs/discojs-node/src/dataset/data_loader/image_loader.spec.ts +++ b/discojs/discojs-node/src/data/image_loader.spec.ts @@ -1,11 +1,12 @@ import { assert, expect } from 'chai' import { List, Map, Range } from 'immutable' import fs from 'fs' - import tf from '@tensorflow/tfjs' import { node as tfNode } from '@tensorflow/tfjs-node' -import { node, type Task } from '../..' +import type { Task } from '@epfml/discojs-core' + +import { ImageLoader } from './image_loader' const readFilesFromDir = (dir: string): string[] => fs.readdirSync(dir).map((file: string) => dir + file) @@ -35,8 +36,8 @@ const mnistMock: Task = { } as unknown as Task const LOADERS = { - CIFAR10: new node.data.NodeImageLoader(cifar10Mock), - MNIST: new node.data.NodeImageLoader(mnistMock) + CIFAR10: new ImageLoader(cifar10Mock), + MNIST: new ImageLoader(mnistMock) } const FILES = Map(DIRS).map((readFilesFromDir)).toObject() @@ -93,7 +94,7 @@ describe('image loader', () => { }) it('loads samples in order', async () => { - const loader = new node.data.NodeImageLoader(cifar10Mock) + const loader = new ImageLoader(cifar10Mock) const dataset = await ((await loader.loadAll(FILES.CIFAR10, { shuffle: false })).train.dataset).toArray() List(dataset).zip(List(FILES.CIFAR10)) @@ -105,7 +106,7 @@ describe('image loader', () => { }) it('shuffles list', async () => { - const loader = new node.data.NodeImageLoader(cifar10Mock) + const loader = new ImageLoader(cifar10Mock) const list = Range(0, 100_000).toArray() const shuffled = [...list] @@ -117,7 +118,7 @@ describe('image loader', () => { }) it('shuffles samples', async () => { - const loader = new node.data.NodeImageLoader(cifar10Mock) + const loader = new ImageLoader(cifar10Mock) const dataset = await (await loader.loadAll(FILES.CIFAR10, { shuffle: false })).train.dataset.toArray() const shuffled = await (await loader.loadAll(FILES.CIFAR10, { shuffle: true })).train.dataset.toArray() @@ -129,7 +130,7 @@ describe('image loader', () => { it('validation split', async () => { const validationSplit = 0.2 const imagesContent = FILES.CIFAR10.map((file) => tfNode.decodeImage(fs.readFileSync(file))) - const datasetContent = await new node.data.NodeImageLoader(cifar10Mock) + const datasetContent = await new ImageLoader(cifar10Mock) .loadAll(FILES.CIFAR10, { shuffle: false, validationSplit }) const trainSize = Math.floor(imagesContent.length * (1 - validationSplit)) diff --git a/discojs/discojs-node/src/dataset/data_loader/image_loader.ts b/discojs/discojs-node/src/data/image_loader.ts similarity index 81% rename from discojs/discojs-node/src/dataset/data_loader/image_loader.ts rename to discojs/discojs-node/src/data/image_loader.ts index 58b1d7954..2c7fee5db 100644 --- a/discojs/discojs-node/src/dataset/data_loader/image_loader.ts +++ b/discojs/discojs-node/src/data/image_loader.ts @@ -4,7 +4,7 @@ import { node } from '@tensorflow/tfjs-node' import { data } from '@epfml/discojs-core' -export class NodeImageLoader extends data.ImageLoader { +export class ImageLoader extends data.ImageLoader { async readImageFrom (source: string): Promise { return node.decodeImage(fs.readFileSync(source)) as tf.Tensor3D } diff --git a/discojs/discojs-node/src/data/index.ts b/discojs/discojs-node/src/data/index.ts new file mode 100644 index 000000000..612a1c891 --- /dev/null +++ b/discojs/discojs-node/src/data/index.ts @@ -0,0 +1,2 @@ +export { ImageLoader as NodeImageLoader } from './image_loader' +export { TabularLoader as NodeTabularLoader } from './tabular_loader' diff --git a/discojs/discojs-node/src/dataset/data_loader/tabular_loader.spec.ts b/discojs/discojs-node/src/data/tabular_loader.spec.ts similarity index 89% rename from discojs/discojs-node/src/dataset/data_loader/tabular_loader.spec.ts rename to discojs/discojs-node/src/data/tabular_loader.spec.ts index d1745556e..22c969d9c 100644 --- a/discojs/discojs-node/src/dataset/data_loader/tabular_loader.spec.ts +++ b/discojs/discojs-node/src/data/tabular_loader.spec.ts @@ -2,7 +2,9 @@ import { assert, expect } from 'chai' import { List } from 'immutable' import tf from '@tensorflow/tfjs' -import { node, type Task } from '../..' +import type { Task } from '@epfml/discojs-core' + +import { TabularLoader } from './tabular_loader' const inputFiles = ['../../example_training_data/titanic_train.csv'] @@ -26,7 +28,7 @@ const titanicMock: Task = { describe('tabular loader', () => { it('loads a single sample', async () => { - const loaded = new node.data.NodeTabularLoader(titanicMock, ',').loadAll( + const loaded = new TabularLoader(titanicMock, ',').loadAll( inputFiles, { features: titanicMock.trainingInformation?.inputColumns, @@ -51,7 +53,7 @@ describe('tabular loader', () => { it('shuffles samples', async () => { const titanic = titanicMock - const loader = new node.data.NodeTabularLoader(titanic, ',') + const loader = new TabularLoader(titanic, ',') const config = { features: titanic.trainingInformation?.inputColumns, labels: titanic.trainingInformation?.outputColumns, diff --git a/discojs/discojs-node/src/dataset/data_loader/tabular_loader.ts b/discojs/discojs-node/src/data/tabular_loader.ts similarity index 83% rename from discojs/discojs-node/src/dataset/data_loader/tabular_loader.ts rename to discojs/discojs-node/src/data/tabular_loader.ts index 79aec87a2..b2dca6e86 100644 --- a/discojs/discojs-node/src/dataset/data_loader/tabular_loader.ts +++ b/discojs/discojs-node/src/data/tabular_loader.ts @@ -2,7 +2,7 @@ import { data as tfData } from '@tensorflow/tfjs-node' import { data } from '@epfml/discojs-core' -export class NodeTabularLoader extends data.TabularLoader { +export class TabularLoader extends data.TabularLoader { async loadDatasetFrom (source: string, csvConfig: Record): Promise { const prefix = 'file://' if (source.slice(0, 7) !== prefix) { diff --git a/discojs/discojs-node/src/dataset/data_loader/text_loader.ts b/discojs/discojs-node/src/data/text_loader.ts similarity index 100% rename from discojs/discojs-node/src/dataset/data_loader/text_loader.ts rename to discojs/discojs-node/src/data/text_loader.ts diff --git a/discojs/discojs-node/src/dataset/data_loader/index.ts b/discojs/discojs-node/src/dataset/data_loader/index.ts deleted file mode 100644 index 0ea714444..000000000 --- a/discojs/discojs-node/src/dataset/data_loader/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { NodeImageLoader } from './image_loader' -export { NodeTabularLoader } from './tabular_loader' diff --git a/discojs/discojs-node/src/imports.ts b/discojs/discojs-node/src/imports.ts deleted file mode 100644 index f4628360c..000000000 --- a/discojs/discojs-node/src/imports.ts +++ /dev/null @@ -1 +0,0 @@ -export * as data from './dataset/data_loader' diff --git a/discojs/discojs-node/src/index.ts b/discojs/discojs-node/src/index.ts index 8b4108de4..7f165348f 100644 --- a/discojs/discojs-node/src/index.ts +++ b/discojs/discojs-node/src/index.ts @@ -1,3 +1 @@ -export * from '@epfml/discojs-core' - -export * as node from './imports' +export * from './data' diff --git a/discojs/discojs-web/src/dataset/data_loader/image_loader.ts b/discojs/discojs-web/src/data/image_loader.ts similarity index 62% rename from discojs/discojs-web/src/dataset/data_loader/image_loader.ts rename to discojs/discojs-web/src/data/image_loader.ts index 6cabcc0b7..537dfe26c 100644 --- a/discojs/discojs-web/src/dataset/data_loader/image_loader.ts +++ b/discojs/discojs-web/src/data/image_loader.ts @@ -1,8 +1,8 @@ import tf from '@tensorflow/tfjs' -import { data } from '../..' +import { data } from '@epfml/discojs-core' -export class WebImageLoader extends data.ImageLoader { +export class ImageLoader extends data.ImageLoader { async readImageFrom (source: File): Promise { return tf.browser.fromPixels(await createImageBitmap(source)) } diff --git a/discojs/discojs-web/src/data/index.ts b/discojs/discojs-web/src/data/index.ts new file mode 100644 index 000000000..705a83eb9 --- /dev/null +++ b/discojs/discojs-web/src/data/index.ts @@ -0,0 +1,3 @@ +export { ImageLoader as WebImageLoader } from './image_loader' +export { TabularLoader as WebTabularLoader } from './tabular_loader' +export { TextLoader as WebTextLoader } from './text_loader' diff --git a/discojs/discojs-web/src/dataset/data_loader/tabular_loader.ts b/discojs/discojs-web/src/data/tabular_loader.ts similarity index 67% rename from discojs/discojs-web/src/dataset/data_loader/tabular_loader.ts rename to discojs/discojs-web/src/data/tabular_loader.ts index f4ab7425c..edb9e7d6c 100644 --- a/discojs/discojs-web/src/dataset/data_loader/tabular_loader.ts +++ b/discojs/discojs-web/src/data/tabular_loader.ts @@ -1,8 +1,8 @@ import tf from '@tensorflow/tfjs' -import { data } from '../..' +import { data } from '@epfml/discojs-core' -export class WebTabularLoader extends data.TabularLoader { +export class TabularLoader extends data.TabularLoader { async loadDatasetFrom (source: File, csvConfig: Record): Promise { return new tf.data.CSVDataset(new tf.data.FileDataSource(source), csvConfig) } diff --git a/discojs/discojs-web/src/dataset/data_loader/text_loader.ts b/discojs/discojs-web/src/data/text_loader.ts similarity index 77% rename from discojs/discojs-web/src/dataset/data_loader/text_loader.ts rename to discojs/discojs-web/src/data/text_loader.ts index eab2c98b5..702368ca3 100644 --- a/discojs/discojs-web/src/dataset/data_loader/text_loader.ts +++ b/discojs/discojs-web/src/data/text_loader.ts @@ -1,8 +1,8 @@ import tf from '@tensorflow/tfjs' -import { data } from '../..' +import { data } from '@epfml/discojs-core' -export class WebTextLoader extends data.TextLoader { +export class TextLoader extends data.TextLoader { async loadDatasetFrom (source: File, config?: Record): Promise { const file = new tf.data.FileDataSource(source) if (config !== undefined) { diff --git a/discojs/discojs-web/src/dataset/data_loader/index.ts b/discojs/discojs-web/src/dataset/data_loader/index.ts deleted file mode 100644 index 2dd904507..000000000 --- a/discojs/discojs-web/src/dataset/data_loader/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export { WebImageLoader } from './image_loader' -export { WebTabularLoader } from './tabular_loader' -export { WebTextLoader } from './text_loader' diff --git a/discojs/discojs-web/src/imports.ts b/discojs/discojs-web/src/imports.ts deleted file mode 100644 index 8b334abde..000000000 --- a/discojs/discojs-web/src/imports.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * as data from './dataset/data_loader' -export { IndexedDB } from './memory' diff --git a/discojs/discojs-web/src/index.ts b/discojs/discojs-web/src/index.ts index aef4db385..f55da3c7a 100644 --- a/discojs/discojs-web/src/index.ts +++ b/discojs/discojs-web/src/index.ts @@ -1,3 +1,2 @@ -export * from '@epfml/discojs-core' - -export * as browser from './imports' +export * from './data' +export * from './memory' diff --git a/discojs/discojs-web/src/memory/memory.ts b/discojs/discojs-web/src/memory/memory.ts index d821812c7..ab7296d20 100644 --- a/discojs/discojs-web/src/memory/memory.ts +++ b/discojs/discojs-web/src/memory/memory.ts @@ -10,7 +10,8 @@ import { Map } from 'immutable' import path from 'path' import * as tf from '@tensorflow/tfjs' -import { Memory, ModelType, type Path, type ModelInfo, type ModelSource } from '..' +import type { Path, ModelInfo, ModelSource } from '@epfml/discojs-core' +import { Memory, ModelType } from '@epfml/discojs-core' export class IndexedDB extends Memory { pathFor (source: ModelSource): Path { diff --git a/server/src/config.ts b/server/src/config.ts index 9c295acc0..b1e27fcd2 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -1,6 +1,6 @@ import path from 'path' -import { type Path, type TaskID } from '@epfml/discojs-node' +import type { Path, TaskID } from '@epfml/discojs-core' export class Config { public readonly serverUrl: URL diff --git a/server/src/get_server.ts b/server/src/get_server.ts index a124f77f8..68bf491ef 100644 --- a/server/src/get_server.ts +++ b/server/src/get_server.ts @@ -3,7 +3,7 @@ import express from 'express' import expressWS from 'express-ws' import type tf from '@tensorflow/tfjs' -import type { Task, TaskProvider } from '@epfml/discojs-node' +import type { Task, TaskProvider } from '@epfml/discojs-core' import { CONFIG } from './config' import { Router } from './router' diff --git a/server/src/router/decentralized/server.ts b/server/src/router/decentralized/server.ts index d779720a9..99f780a5f 100644 --- a/server/src/router/decentralized/server.ts +++ b/server/src/router/decentralized/server.ts @@ -7,8 +7,8 @@ import type { ParsedQs } from 'qs' import { Map, Set } from 'immutable' import type tf from '@tensorflow/tfjs' -import type { Task, TaskID } from '@epfml/discojs-node' -import { client } from '@epfml/discojs-node' +import type { Task, TaskID } from '@epfml/discojs-core' +import { client } from '@epfml/discojs-core' import { Server } from '../server' diff --git a/server/src/router/federated/server.ts b/server/src/router/federated/server.ts index 5444a15b1..fc3c5cf96 100644 --- a/server/src/router/federated/server.ts +++ b/server/src/router/federated/server.ts @@ -5,17 +5,19 @@ import { List, Map } from 'immutable' import msgpack from 'msgpack-lite' import type tf from '@tensorflow/tfjs' +import type { + Task, + TaskID, + WeightsContainer, + MetadataKey, + MetadataValue +} from '@epfml/discojs-core' import { client, serialization, AsyncInformant, - type Task, - type TaskID, - aggregator as aggregators, - type WeightsContainer, - type MetadataKey, - type MetadataValue -} from '@epfml/discojs-node' + aggregator as aggregators +} from '@epfml/discojs-core' import { Server } from '../server' diff --git a/server/src/router/server.ts b/server/src/router/server.ts index 7ee7ba9fb..d2a6fe023 100644 --- a/server/src/router/server.ts +++ b/server/src/router/server.ts @@ -3,7 +3,7 @@ import type expressWS from 'express-ws' import type WebSocket from 'ws' import type tf from '@tensorflow/tfjs' -import type { Task } from '@epfml/discojs-node' +import type { Task } from '@epfml/discojs-core' import type { TasksAndModels } from '../tasks' diff --git a/server/src/router/tasks.ts b/server/src/router/tasks.ts index 51bc58fe3..2eef48bd8 100644 --- a/server/src/router/tasks.ts +++ b/server/src/router/tasks.ts @@ -3,8 +3,8 @@ import express from 'express' import { Set } from 'immutable' import type tf from '@tensorflow/tfjs' -import type { Task, TaskID } from '@epfml/discojs-node' -import { serialization, isTask } from '@epfml/discojs-node' +import type { Task, TaskID } from '@epfml/discojs-core' +import { serialization, isTask } from '@epfml/discojs-core' import type { Config } from '../config' import type { TasksAndModels } from '../tasks' diff --git a/server/src/tasks.ts b/server/src/tasks.ts index 25f23b7f0..e141fb333 100644 --- a/server/src/tasks.ts +++ b/server/src/tasks.ts @@ -4,8 +4,8 @@ import fs from 'node:fs' import tf from '@tensorflow/tfjs' import '@tensorflow/tfjs-node' -import type { Task, Path, Digest, TaskProvider } from '@epfml/discojs-node' -import { isTaskProvider, defaultTasks } from '@epfml/discojs-node' +import type { Task, Path, Digest, TaskProvider } from '@epfml/discojs-core' +import { isTaskProvider, defaultTasks } from '@epfml/discojs-core' // default tasks and added ones // register 'taskAndModel' event to get tasks diff --git a/server/tests/client/decentralized.spec.ts b/server/tests/client/decentralized.spec.ts index fc74aafb2..48bf560d0 100644 --- a/server/tests/client/decentralized.spec.ts +++ b/server/tests/client/decentralized.spec.ts @@ -1,6 +1,7 @@ import type * as http from 'http' -import { aggregator as aggregators, client as clients, type Task, defaultTasks } from '@epfml/discojs-node' +import type { Task } from '@epfml/discojs-core' +import { aggregator as aggregators, client as clients, defaultTasks } from '@epfml/discojs-core' import { getClient, startServer } from '../utils' diff --git a/server/tests/client/federated.spec.ts b/server/tests/client/federated.spec.ts index d6b4f3387..c57772c72 100644 --- a/server/tests/client/federated.spec.ts +++ b/server/tests/client/federated.spec.ts @@ -1,6 +1,7 @@ import type * as http from 'http' -import { aggregator as aggregators, client as clients, informant, defaultTasks, type Task } from '@epfml/discojs-node' +import type { Task } from '@epfml/discojs-core' +import { aggregator as aggregators, client as clients, informant, defaultTasks } from '@epfml/discojs-core' import { getClient, startServer } from '../utils' diff --git a/server/tests/e2e/decentralized.spec.ts b/server/tests/e2e/decentralized.spec.ts index 769415807..67ad64a40 100644 --- a/server/tests/e2e/decentralized.spec.ts +++ b/server/tests/e2e/decentralized.spec.ts @@ -2,9 +2,10 @@ import type { Server } from 'node:http' import { List } from 'immutable' import { assert } from 'chai' +import type { Task } from '@epfml/discojs-core' import { - aggregator as aggregators, informant as informants, client as clients, type Task, WeightsContainer, defaultTasks, aggregation -} from '@epfml/discojs-node' + aggregator as aggregators, informant as informants, client as clients, WeightsContainer, defaultTasks, aggregation +} from '@epfml/discojs-core' import { getClient, startServer } from '../utils' diff --git a/server/tests/e2e/federated.spec.ts b/server/tests/e2e/federated.spec.ts index 53f05359f..095baf2e7 100644 --- a/server/tests/e2e/federated.spec.ts +++ b/server/tests/e2e/federated.spec.ts @@ -5,9 +5,10 @@ import { Range } from 'immutable' import { assert } from 'chai' import { - WeightsContainer, node, Disco, TrainingSchemes, client as clients, + WeightsContainer, Disco, TrainingSchemes, client as clients, aggregator as aggregators, informant, defaultTasks -} from '@epfml/discojs-node' +} from '@epfml/discojs-core' +import { NodeImageLoader, NodeTabularLoader } from '@epfml/discojs-node' import { getClient, startServer } from '../utils' @@ -31,7 +32,7 @@ describe('end-to-end federated', function () { const cifar10Task = defaultTasks.cifar10.getTask() - const data = await new node.data.NodeImageLoader(cifar10Task).loadAll(files, { labels }) + const data = await new NodeImageLoader(cifar10Task).loadAll(files, { labels }) const aggregator = new aggregators.MeanAggregator(cifar10Task) const client = await getClient(clients.federated.FederatedClient, server, cifar10Task, aggregator) @@ -51,7 +52,7 @@ describe('end-to-end federated', function () { const titanicTask = defaultTasks.titanic.getTask() titanicTask.trainingInformation.epochs = 5 - const data = await (new node.data.NodeTabularLoader(titanicTask, ',').loadAll( + const data = await (new NodeTabularLoader(titanicTask, ',').loadAll( files, { features: titanicTask.trainingInformation.inputColumns, diff --git a/server/tests/utils.ts b/server/tests/utils.ts index aceca4432..de71f9a91 100644 --- a/server/tests/utils.ts +++ b/server/tests/utils.ts @@ -1,6 +1,6 @@ import type { Server } from 'node:http' -import type { aggregator, client, Task } from '@epfml/discojs-node' +import type { aggregator, client, Task } from '@epfml/discojs-core' import { runDefaultServer } from '../src/get_server' diff --git a/web-client/cypress/e2e/tasks.cy.ts b/web-client/cypress/e2e/tasks.cy.ts index 37d961342..b1b8030a3 100644 --- a/web-client/cypress/e2e/tasks.cy.ts +++ b/web-client/cypress/e2e/tasks.cy.ts @@ -1,5 +1,5 @@ /* eslint-disable no-undef */ -import { defaultTasks } from '@epfml/discojs' +import { defaultTasks } from '@epfml/discojs-core' // most basic disco tasks export const TASK_LIST = [ diff --git a/web-client/src/clients.ts b/web-client/src/clients.ts index 2eae7078d..6e890cc78 100644 --- a/web-client/src/clients.ts +++ b/web-client/src/clients.ts @@ -1,4 +1,4 @@ -import { client as clients, aggregator as aggregators, Task, TrainingSchemes } from '@epfml/discojs' +import { client as clients, aggregator as aggregators, Task, TrainingSchemes } from '@epfml/discojs-core' import { CONFIG } from './config' diff --git a/web-client/src/components/data/Data.vue b/web-client/src/components/data/Data.vue index a910d53ca..ba4e0eff2 100644 --- a/web-client/src/components/data/Data.vue +++ b/web-client/src/components/data/Data.vue @@ -12,7 +12,7 @@