feat: Typed loaders (#2448)

modules/arrow/src/arrow-loader.ts

@@ -1,4 +1,6 @@
+// loaders.gl, MIT license
 import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {ArrowTable} from '@loaders.gl/schema';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -10,14 +12,8 @@ export type ArrowLoaderOptions = LoaderOptions & {
   };
 };
 
-const DEFAULT_ARROW_LOADER_OPTIONS = {
-  arrow: {
-    shape: 'columnar-table'
-  }
-};
-
 /** ArrowJS table loader */
-export const ArrowLoader = {
+export const ArrowLoader: Loader<ArrowTable, never, ArrowLoaderOptions> = {
   name: 'Apache Arrow',
   id: 'arrow',
   module: 'arrow',
@@ -32,7 +28,11 @@ export const ArrowLoader = {
   ],
   binary: true,
   tests: ['ARROW'],
-  options: DEFAULT_ARROW_LOADER_OPTIONS
+  options: {
+    arrow: {
+      shape: 'columnar-table'
+    }
+  }
 };
 
 export const _typecheckArrowLoader: Loader = ArrowLoader;

modules/arrow/src/arrow-writer.ts

@@ -1,15 +1,18 @@
-// import type {Writer} from '@loaders.gl/loader-utils';
-import type {WriterOptions} from '@loaders.gl/loader-utils';
+// import type {} from '@loaders.gl/loader-utils';
+import type {Writer, WriterOptions} from '@loaders.gl/loader-utils';
+import {ColumnarTable} from './lib/encode-arrow';
 import {encodeArrowSync} from './lib/encode-arrow';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-type ArrowWriterOptions = WriterOptions;
+type ArrowWriterOptions = WriterOptions & {
+  arrow?: {};
+};
 
 /** Apache Arrow writer */
-export const ArrowWriter = {
+export const ArrowWriter: Writer<ColumnarTable, never, ArrowWriterOptions> = {
   name: 'Apache Arrow',
   id: 'arrow',
   module: 'arrow',
@@ -20,11 +23,9 @@ export const ArrowWriter = {
     'application/vnd.apache.arrow.stream',
     'application/octet-stream'
   ],
-  encodeSync,
+  encodeSync(data, options?) {
+    return encodeArrowSync(data);
+  },
   binary: true,
   options: {}
 };
-
-function encodeSync(data, options?: ArrowWriterOptions) {
-  return encodeArrowSync(data);
-}

modules/arrow/src/lib/encode-arrow.ts

@@ -1,7 +1,7 @@
 import {Table, Vector, tableToIPC, vectorFromArray} from 'apache-arrow';
 import {AnyArrayType, VECTOR_TYPES} from '../types';
 
-type ColumnarTable = {
+export type ColumnarTable = {
   name: string;
   array: AnyArrayType;
   type: number;

modules/bson/src/bson-loader.ts

@@ -19,7 +19,7 @@ const DEFAULT_BSON_LOADER_OPTIONS = {
   bson: {}
 };
 
-export const BSONLoader: LoaderWithParser = {
+export const BSONLoader: LoaderWithParser<Record<string, unknown>, never, BSONLoaderOptions> = {
   name: 'BSON',
   id: 'bson',
   module: 'bson',

modules/bson/src/bson-writer.ts

@@ -1,23 +1,25 @@
 // loaders.gl, MIT license
 
 import type {Writer, WriterOptions} from '@loaders.gl/loader-utils';
+import type {EncodeBSONOptions} from './lib/encoders/encode-bson';
 import {encodeBSONSync} from './lib/encoders/encode-bson';
-// import type {Writer} from '@loaders.gl/loader-utils';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-export const BSONWriter: Writer = {
+export type BSONWriterOptions = WriterOptions & {
+  bson?: EncodeBSONOptions
+}
+
+export const BSONWriter: Writer<Record<string, unknown>, never, BSONWriterOptions> = {
   name: 'BSON',
   id: 'bson',
   module: 'bson',
   version: VERSION,
   extensions: ['bson'],
   options: {
-    image: {
-      mimeType: 'application/bson'
-    }
+    bson: {}
   },
   async encode(data: Record<string, unknown>, options?: WriterOptions): Promise<ArrayBuffer> {
     return encodeBSONSync(data, {}); // options

modules/gltf/src/glb-loader.ts

@@ -13,7 +13,7 @@ export type GLBLoaderOptions = LoaderOptions & {
  * GLB Loader -
  * GLB is the binary container format for GLTF
  */
-export const GLBLoader: LoaderWithParser = {
+export const GLBLoader: LoaderWithParser<GLB, never, GLBLoaderOptions> = {
   name: 'GLB',
   id: 'glb',
   module: 'gltf',

modules/gltf/src/glb-writer.ts

@@ -1,12 +1,18 @@
-import type {Writer} from '@loaders.gl/loader-utils';
-import {VERSION} from './lib/utils/version';
+import type {Writer, WriterOptions} from '@loaders.gl/loader-utils';
+import type {GLB} from './lib/types/glb-types';
+import type {GLBEncodeOptions} from './lib/encoders/encode-glb';
 import encodeGLBSync from './lib/encoders/encode-glb';
+import {VERSION} from './lib/utils/version';
+
+export type GLBWriterOptions = WriterOptions & {
+  glb?: GLBEncodeOptions;
+};
 
 /**
  * GLB exporter
  * GLB is the binary container format for GLTF
  */
-export const GLBWriter = {
+export const GLBWriter: Writer<GLB, never, GLBWriterOptions> = {
   name: 'GLB',
   id: 'glb',
   module: 'gltf',

modules/gltf/src/lib/encoders/encode-glb.ts

@@ -10,10 +10,17 @@ const MAGIC_BIN = 0x004e4942; // BIN\0 in ASCII
 
 const LE = true; // Binary GLTF is little endian.
 
+export type GLBEncodeOptions = {};
+
 // Encode the full GLB buffer with header etc
 // https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#
 // glb-file-format-specification
-export default function encodeGLBSync(glb, dataView, byteOffset = 0, options = {}) {
+export default function encodeGLBSync(
+  glb,
+  dataView,
+  byteOffset = 0,
+  options: GLBEncodeOptions = {}
+) {
   const {magic = MAGIC_glTF, version = 2, json = {}, binary} = glb;
 
   const byteOffsetStart = byteOffset;

modules/gltf/src/lib/encoders/encode-gltf.ts

@@ -1,15 +1,24 @@
 import encodeGLBSync from './encode-glb';
 
-// Encode the full glTF file as a binary GLB file
-// Returns an ArrayBuffer that represents the complete GLB image that can be saved to file
-//
-// TODO - Does not support encoding to non-GLB versions of glTF format
-// - Encode as a textual JSON file with binary data in base64 data URLs.
-// - Encode as a JSON with all images (and buffers?) in separate binary files
-//
-// glb-file-format-specification
-// https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#
+export type GLTFEncodeOptions = Record<string, any>;
 
+/**
+ * Encode the full glTF file as a binary GLB file
+ * Returns an ArrayBuffer that represents the complete GLB image that can be saved to file
+ *
+ * @todo - Does not support encoding to non-GLB versions of glTF format. Other formats
+ * - Encode as a textual JSON file with binary data in base64 data URLs.
+ * - Encode as a JSON with all images (and buffers?) in separate binary files
+ *
+ * glb-file-format-specification
+ * @see https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#
+ *
+ * @param gltf
+ * @param arrayBuffer
+ * @param byteOffset
+ * @param options
+ * @returns
+ */
 export function encodeGLTFSync(gltf, arrayBuffer, byteOffset, options) {
   convertBuffersToBase64(gltf);
 

modules/gltf/src/lib/types/glb-types.ts

@@ -16,6 +16,6 @@ export type GLB = {
   };
 
   // Per spec we must iterate over chunks, ignoring all except JSON and BIN
-  json: {};
+  json: Record<string, any>;
   binChunks: GLBBinChunk[];
 };

modules/parquet/src/lib/arrow/convert-schema-from-parquet.ts

@@ -4,7 +4,7 @@ import {Schema, Field, DataType} from '@loaders.gl/schema';
 
 import type {ParquetSchema} from '../../parquetjs/schema/schema';
 import type {FieldDefinition, ParquetField, ParquetType} from '../../parquetjs/schema/declare';
-import {FileMetaData} from '@loaders.gl/parquet/parquetjs/parquet-thrift';
+import {FileMetaData} from '../../parquetjs/parquet-thrift';
 
 export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = {
   BOOLEAN: 'bool',

modules/ply/src/index.ts

@@ -25,5 +25,3 @@ export const PLYLoader: LoaderWithParser<PLYMesh, any, PLYLoaderOptions> = {
   parseSync: (arrayBuffer, options) => parsePLY(arrayBuffer, options?.ply),
   parseInBatches: (arrayBuffer, options) => parsePLYInBatches(arrayBuffer, options?.ply)
 };
-
-export const _typecheckPLYLoader: LoaderWithParser = PLYLoader;

modules/schema/src/lib/table/simple-table/table-accessors.ts

@@ -14,15 +14,16 @@ export function getTableLength(table: Table): number {
     case 'geojson-row-table':
       return table.data.length;
 
+    case 'arrow-table':
+      return table.data.numRows;
+
     case 'columnar-table':
       for (const column of Object.values(table.data)) {
         return column.length || 0;
       }
       return 0;
-
-    case 'arrow-table':
     default:
-      return table.data.numRows;
+      throw new Error('table');
   }
 }
 
@@ -48,8 +49,9 @@ export function getTableNumCols(table: Table): number {
       return Object.keys(table.data).length;
 
     case 'arrow-table':
-    default:
       return table.data.numCols;
+    default:
+      throw new Error('table');
   }
 }
 
@@ -69,7 +71,10 @@ export function getTableCell(table: Table, rowIndex: number, columnName: string)
       return column[rowIndex];
 
     case 'arrow-table':
-      return table.data.getChild(columnName)?.get(rowIndex);
+      const arrowColumnIndex = table.data.schema.fields.findIndex(
+        (field) => field.name === columnName
+      );
+      return table.data.getChildAt(arrowColumnIndex)?.get(rowIndex);
 
     default:
       throw new Error('todo');