discojs-{node,web}: don't reexport core

Closes: #638

cli/src/args.ts

@@ -1,6 +1,7 @@
 import { parse } from 'ts-command-line-args'
 import { Map } from 'immutable'
-import { defaultTasks, type Task } from '@epfml/discojs-node'
+import type { Task } from '@epfml/discojs-core'
+import { defaultTasks } from '@epfml/discojs-core'
 
 interface BenchmarkArguments {
   task: Task

cli/src/cli.ts

@@ -1,6 +1,7 @@
 import { Range } from 'immutable'
 
-import { Disco, TrainingSchemes, type TrainerLog, type data, type Task } from '@epfml/discojs-node'
+import type { TrainerLog, data, Task } from '@epfml/discojs-core'
+import { Disco, TrainingSchemes } from '@epfml/discojs-core'
 
 import { startServer, saveLog } from './utils'
 import { getTaskData } from './data'

cli/src/data.ts

@@ -3,8 +3,8 @@ import fs from 'node:fs'
 import fs_promises from 'fs/promises'
 import path from 'node:path'
 
-import type { Task, data } from '@epfml/discojs-node'
-import { node } from '@epfml/discojs-node'
+import type { Task, data } from '@epfml/discojs-core'
+import { NodeImageLoader, NodeTabularLoader } from '@epfml/discojs-node'
 
 function filesFromFolder (dir: string, folder: string, fractionToKeep: number): string[] {
   const f = fs.readdirSync(dir + folder)
@@ -35,21 +35,21 @@ async function simplefaceData (task: Task): Promise<data.DataSplit> {
   const labels = filesPerFolder.flatMap((files, index) => Array(files.length).fill(index))
   const files = filesPerFolder.flat()
 
-  return await new node.data.NodeImageLoader(task).loadAll(files, { labels })
+  return await new NodeImageLoader(task).loadAll(files, { labels })
 }
 
 async function cifar10Data (cifar10: Task): Promise<data.DataSplit> {
   const dir = '../example_training_data/CIFAR10/'
   const files = (await fs_promises.readdir(dir)).map((file) => path.join(dir, file))
   const labels = Range(0, 24).map((label) => (label % 10).toString()).toArray()
 
-  return await new node.data.NodeImageLoader(cifar10).loadAll(files, { labels })
+  return await new NodeImageLoader(cifar10).loadAll(files, { labels })
 }
 
 async function titanicData (titanic: Task): Promise<data.DataSplit> {
   const dir = '../example_training_data/titanic_train.csv'
 
-  const data = await (new node.data.NodeTabularLoader(titanic, ',').loadAll(
+  const data = await (new NodeTabularLoader(titanic, ',').loadAll(
     ['file://'.concat(dir)],
     {
       features: titanic.trainingInformation?.inputColumns,

cli/src/utils.ts

@@ -1,7 +1,7 @@
 import type http from 'node:http'
 import fs from 'node:fs'
 
-import { type TrainerLog } from '@epfml/discojs-node'
+import type { TrainerLog } from '@epfml/discojs-core'
 import { Disco } from '@epfml/disco-server'
 
 export async function startServer (): Promise<[http.Server, URL]> {

discojs/discojs-core/src/aggregator/secure.spec.ts

@@ -1,7 +1,7 @@
 import { List, Set, Range } from 'immutable'
 import { assert } from 'chai'
 
-import { aggregator as aggregators, aggregation, WeightsContainer, defaultTasks } from '@epfml/discojs-node'
+import { aggregator as aggregators, aggregation, WeightsContainer, defaultTasks } from '@epfml/discojs-core'
 
 describe('secret shares test', function () {
   const epsilon = 1e-4

discojs/discojs-core/src/validation/validator.spec.ts

@@ -1,10 +1,9 @@
 import { assert } from 'chai'
 import fs from 'fs'
 
-import {
-  type Task, node, Validator, ConsoleLogger, EmptyMemory,
-  client as clients, type data, aggregator, defaultTasks
-} from '@epfml/discojs-node'
+import type { Task, data } from '../..'
+import { Validator, ConsoleLogger, EmptyMemory, client as clients, aggregator, defaultTasks } from '../..'
+import { NodeImageLoader, NodeTabularLoader } from '@epfml/discojs-node'
 
 const simplefaceMock = {
   taskID: 'simple_face',
@@ -32,7 +31,7 @@ describe('validator', () => {
         .map((file: string) => dir + subdir + file))
     const labels = files.flatMap((files, index) => Array(files.length).fill(index))
 
-    const data = (await new node.data.NodeImageLoader(simplefaceMock)
+    const data = (await new NodeImageLoader(simplefaceMock)
       .loadAll(files.flat(), { labels })).train
     const meanAggregator = new aggregator.MeanAggregator(simplefaceMock)
     const client = new clients.Local(new URL('http://localhost:8080'), simplefaceMock, meanAggregator)
@@ -63,7 +62,7 @@ describe('validator', () => {
   it('titanic validator', async () => {
     const titanicTask = defaultTasks.titanic.getTask()
     const files = ['../../example_training_data/titanic_train.csv']
-    const data: data.Data = (await new node.data.NodeTabularLoader(titanicTask, ',').loadAll(files, {
+    const data: data.Data = (await new NodeTabularLoader(titanicTask, ',').loadAll(files, {
       features: titanicTask.trainingInformation.inputColumns,
       labels: titanicTask.trainingInformation.outputColumns,
       shuffle: false

discojs/discojs-core/src/weights/aggregation.spec.ts

@@ -1,6 +1,6 @@
 import { assert } from 'chai'
 
-import { WeightsContainer, aggregation } from '@epfml/discojs-node'
+import { WeightsContainer, aggregation } from '.'
 
 describe('weights aggregation', () => {
   it('avg of weights with two operands', async () => {

discojs/discojs-node/src/dataset/data_loader/image_loader.spec.ts → discojs/discojs-node/src/data/image_loader.spec.ts

@@ -1,11 +1,12 @@
 import { assert, expect } from 'chai'
 import { List, Map, Range } from 'immutable'
 import fs from 'fs'
-
 import tf from '@tensorflow/tfjs'
 import { node as tfNode } from '@tensorflow/tfjs-node'
 
-import { node, type Task } from '../..'
+import type { Task } from '@epfml/discojs-core'
+
+import { ImageLoader } from './image_loader'
 
 const readFilesFromDir = (dir: string): string[] =>
   fs.readdirSync(dir).map((file: string) => dir + file)
@@ -35,8 +36,8 @@ const mnistMock: Task = {
 } as unknown as Task
 
 const LOADERS = {
-  CIFAR10: new node.data.NodeImageLoader(cifar10Mock),
-  MNIST: new node.data.NodeImageLoader(mnistMock)
+  CIFAR10: new ImageLoader(cifar10Mock),
+  MNIST: new ImageLoader(mnistMock)
 }
 const FILES = Map(DIRS).map((readFilesFromDir)).toObject()
 
@@ -93,7 +94,7 @@ describe('image loader', () => {
   })
 
   it('loads samples in order', async () => {
-    const loader = new node.data.NodeImageLoader(cifar10Mock)
+    const loader = new ImageLoader(cifar10Mock)
     const dataset = await ((await loader.loadAll(FILES.CIFAR10, { shuffle: false })).train.dataset).toArray()
 
     List(dataset).zip(List(FILES.CIFAR10))
@@ -105,7 +106,7 @@ describe('image loader', () => {
   })
 
   it('shuffles list', async () => {
-    const loader = new node.data.NodeImageLoader(cifar10Mock)
+    const loader = new ImageLoader(cifar10Mock)
     const list = Range(0, 100_000).toArray()
     const shuffled = [...list]
 
@@ -117,7 +118,7 @@ describe('image loader', () => {
   })
 
   it('shuffles samples', async () => {
-    const loader = new node.data.NodeImageLoader(cifar10Mock)
+    const loader = new ImageLoader(cifar10Mock)
     const dataset = await (await loader.loadAll(FILES.CIFAR10, { shuffle: false })).train.dataset.toArray()
     const shuffled = await (await loader.loadAll(FILES.CIFAR10, { shuffle: true })).train.dataset.toArray()
 
@@ -129,7 +130,7 @@ describe('image loader', () => {
   it('validation split', async () => {
     const validationSplit = 0.2
     const imagesContent = FILES.CIFAR10.map((file) => tfNode.decodeImage(fs.readFileSync(file)))
-    const datasetContent = await new node.data.NodeImageLoader(cifar10Mock)
+    const datasetContent = await new ImageLoader(cifar10Mock)
       .loadAll(FILES.CIFAR10, { shuffle: false, validationSplit })
 
     const trainSize = Math.floor(imagesContent.length * (1 - validationSplit))

discojs/discojs-node/src/dataset/data_loader/image_loader.ts → discojs/discojs-node/src/data/image_loader.ts

@@ -4,7 +4,7 @@ import { node } from '@tensorflow/tfjs-node'
 
 import { data } from '@epfml/discojs-core'
 
-export class NodeImageLoader extends data.ImageLoader<string> {
+export class ImageLoader extends data.ImageLoader<string> {
   async readImageFrom (source: string): Promise<tf.Tensor3D> {
     return node.decodeImage(fs.readFileSync(source)) as tf.Tensor3D
   }

discojs/discojs-node/src/data/index.ts

@@ -0,0 +1,2 @@
+export { ImageLoader as NodeImageLoader } from './image_loader'
+export { TabularLoader as NodeTabularLoader } from './tabular_loader'

discojs/discojs-node/src/dataset/data_loader/tabular_loader.spec.ts → discojs/discojs-node/src/data/tabular_loader.spec.ts

@@ -2,7 +2,9 @@ import { assert, expect } from 'chai'
 import { List } from 'immutable'
 import tf from '@tensorflow/tfjs'
 
-import { node, type Task } from '../..'
+import type { Task } from '@epfml/discojs-core'
+
+import { TabularLoader } from './tabular_loader'
 
 const inputFiles = ['../../example_training_data/titanic_train.csv']
 
@@ -26,7 +28,7 @@ const titanicMock: Task = {
 
 describe('tabular loader', () => {
   it('loads a single sample', async () => {
-    const loaded = new node.data.NodeTabularLoader(titanicMock, ',').loadAll(
+    const loaded = new TabularLoader(titanicMock, ',').loadAll(
       inputFiles,
       {
         features: titanicMock.trainingInformation?.inputColumns,
@@ -51,7 +53,7 @@ describe('tabular loader', () => {
 
   it('shuffles samples', async () => {
     const titanic = titanicMock
-    const loader = new node.data.NodeTabularLoader(titanic, ',')
+    const loader = new TabularLoader(titanic, ',')
     const config = {
       features: titanic.trainingInformation?.inputColumns,
       labels: titanic.trainingInformation?.outputColumns,

discojs/discojs-node/src/dataset/data_loader/tabular_loader.ts → discojs/discojs-node/src/data/tabular_loader.ts

@@ -2,7 +2,7 @@ import { data as tfData } from '@tensorflow/tfjs-node'
 
 import { data } from '@epfml/discojs-core'
 
-export class NodeTabularLoader extends data.TabularLoader<string> {
+export class TabularLoader extends data.TabularLoader<string> {
   async loadDatasetFrom (source: string, csvConfig: Record<string, unknown>): Promise<data.Dataset> {
     const prefix = 'file://'
     if (source.slice(0, 7) !== prefix) {

discojs/discojs-node/src/index.ts

@@ -1,3 +1 @@
-export * from '@epfml/discojs-core'
-
-export * as node from './imports'
+export * from './data'

discojs/discojs-web/src/dataset/data_loader/image_loader.ts → discojs/discojs-web/src/data/image_loader.ts

@@ -1,8 +1,8 @@
 import tf from '@tensorflow/tfjs'
 
-import { data } from '../..'
+import { data } from '@epfml/discojs-core'
 
-export class WebImageLoader extends data.ImageLoader<File> {
+export class ImageLoader extends data.ImageLoader<File> {
   async readImageFrom (source: File): Promise<tf.Tensor3D> {
     return tf.browser.fromPixels(await createImageBitmap(source))
   }

discojs/discojs-web/src/data/index.ts

@@ -0,0 +1,3 @@
+export { ImageLoader as WebImageLoader } from './image_loader'
+export { TabularLoader as WebTabularLoader } from './tabular_loader'
+export { TextLoader as WebTextLoader } from './text_loader'

discojs/discojs-web/src/dataset/data_loader/tabular_loader.ts → discojs/discojs-web/src/data/tabular_loader.ts

@@ -1,8 +1,8 @@
 import tf from '@tensorflow/tfjs'
 
-import { data } from '../..'
+import { data } from '@epfml/discojs-core'
 
-export class WebTabularLoader extends data.TabularLoader<File> {
+export class TabularLoader extends data.TabularLoader<File> {
   async loadDatasetFrom (source: File, csvConfig: Record<string, unknown>): Promise<data.Dataset> {
     return new tf.data.CSVDataset(new tf.data.FileDataSource(source), csvConfig)
   }

discojs/discojs-web/src/dataset/data_loader/text_loader.ts → discojs/discojs-web/src/data/text_loader.ts

@@ -1,8 +1,8 @@
 import tf from '@tensorflow/tfjs'
 
-import { data } from '../..'
+import { data } from '@epfml/discojs-core'
 
-export class WebTextLoader extends data.TextLoader<File> {
+export class TextLoader extends data.TextLoader<File> {
   async loadDatasetFrom (source: File, config?: Record<string, unknown>): Promise<data.Dataset> {
     const file = new tf.data.FileDataSource(source)
     if (config !== undefined) {

discojs/discojs-web/src/index.ts

@@ -1,3 +1,2 @@
-export * from '@epfml/discojs-core'
-
-export * as browser from './imports'
+export * from './data'
+export * from './memory'

discojs/discojs-web/src/memory/memory.ts

@@ -10,7 +10,8 @@ import { Map } from 'immutable'
 import path from 'path'
 import * as tf from '@tensorflow/tfjs'
 
-import { Memory, ModelType, type Path, type ModelInfo, type ModelSource } from '..'
+import type { Path, ModelInfo, ModelSource } from '@epfml/discojs-core'
+import { Memory, ModelType } from '@epfml/discojs-core'
 
 export class IndexedDB extends Memory {
   pathFor (source: ModelSource): Path {

server/src/config.ts

@@ -1,6 +1,6 @@
 import path from 'path'
 
-import { type Path, type TaskID } from '@epfml/discojs-node'
+import type { Path, TaskID } from '@epfml/discojs-core'
 
 export class Config {
   public readonly serverUrl: URL

server/src/get_server.ts

@@ -3,7 +3,7 @@ import express from 'express'
 import expressWS from 'express-ws'
 import type tf from '@tensorflow/tfjs'
 
-import type { Task, TaskProvider } from '@epfml/discojs-node'
+import type { Task, TaskProvider } from '@epfml/discojs-core'
 
 import { CONFIG } from './config'
 import { Router } from './router'

server/src/router/decentralized/server.ts

@@ -7,8 +7,8 @@ import type { ParsedQs } from 'qs'
 import { Map, Set } from 'immutable'
 import type tf from '@tensorflow/tfjs'
 
-import type { Task, TaskID } from '@epfml/discojs-node'
-import { client } from '@epfml/discojs-node'
+import type { Task, TaskID } from '@epfml/discojs-core'
+import { client } from '@epfml/discojs-core'
 
 import { Server } from '../server'
 

server/src/router/federated/server.ts

@@ -5,17 +5,19 @@ import { List, Map } from 'immutable'
 import msgpack from 'msgpack-lite'
 import type tf from '@tensorflow/tfjs'
 
+import type {
+  Task,
+  TaskID,
+  WeightsContainer,
+  MetadataKey,
+  MetadataValue
+} from '@epfml/discojs-core'
 import {
   client,
   serialization,
   AsyncInformant,
-  type Task,
-  type TaskID,
-  aggregator as aggregators,
-  type WeightsContainer,
-  type MetadataKey,
-  type MetadataValue
-} from '@epfml/discojs-node'
+  aggregator as aggregators
+} from '@epfml/discojs-core'
 
 import { Server } from '../server'
 

server/src/router/server.ts

@@ -3,7 +3,7 @@ import type expressWS from 'express-ws'
 import type WebSocket from 'ws'
 import type tf from '@tensorflow/tfjs'
 
-import type { Task } from '@epfml/discojs-node'
+import type { Task } from '@epfml/discojs-core'
 
 import type { TasksAndModels } from '../tasks'
 

server/src/router/tasks.ts

@@ -3,8 +3,8 @@ import express from 'express'
 import { Set } from 'immutable'
 import type tf from '@tensorflow/tfjs'
 
-import type { Task, TaskID } from '@epfml/discojs-node'
-import { serialization, isTask } from '@epfml/discojs-node'
+import type { Task, TaskID } from '@epfml/discojs-core'
+import { serialization, isTask } from '@epfml/discojs-core'
 
 import type { Config } from '../config'
 import type { TasksAndModels } from '../tasks'

server/src/tasks.ts

@@ -4,8 +4,8 @@ import fs from 'node:fs'
 import tf from '@tensorflow/tfjs'
 import '@tensorflow/tfjs-node'
 
-import type { Task, Path, Digest, TaskProvider } from '@epfml/discojs-node'
-import { isTaskProvider, defaultTasks } from '@epfml/discojs-node'
+import type { Task, Path, Digest, TaskProvider } from '@epfml/discojs-core'
+import { isTaskProvider, defaultTasks } from '@epfml/discojs-core'
 
 // default tasks and added ones
 // register 'taskAndModel' event to get tasks

server/tests/client/decentralized.spec.ts

@@ -1,6 +1,7 @@
 import type * as http from 'http'
 
-import { aggregator as aggregators, client as clients, type Task, defaultTasks } from '@epfml/discojs-node'
+import type { Task } from '@epfml/discojs-core'
+import { aggregator as aggregators, client as clients, defaultTasks } from '@epfml/discojs-core'
 
 import { getClient, startServer } from '../utils'
 

server/tests/client/federated.spec.ts

@@ -1,6 +1,7 @@
 import type * as http from 'http'
 
-import { aggregator as aggregators, client as clients, informant, defaultTasks, type Task } from '@epfml/discojs-node'
+import type { Task } from '@epfml/discojs-core'
+import { aggregator as aggregators, client as clients, informant, defaultTasks } from '@epfml/discojs-core'
 
 import { getClient, startServer } from '../utils'