From 6ddb4b6459066d13e33aa21cf25f3661fc27068b Mon Sep 17 00:00:00 2001 From: etowahadams Date: Tue, 26 Sep 2023 09:03:35 -0400 Subject: [PATCH] feat(data-fetcher,core): Add support for getting data from URLs that require certain request initializations (#965) * feat(core): pass fetch options into compiler * feat(data-fetch): make datafetchers use the fetch options * test(core): test that compiler can create higlass spec with the fetch options --------- Co-authored-by: Sehi L'Yi --- src/compiler/compile.test.ts | 101 +++++++++++++++++ src/compiler/compile.ts | 7 +- src/compiler/create-higlass-models.ts | 6 +- src/compiler/gosling-to-higlass.ts | 15 ++- src/core/gosling-component.tsx | 9 +- src/core/gosling-embed.ts | 5 +- src/data-fetchers/bam/bam-worker.ts | 41 +++---- src/data-fetchers/bed/bed-worker.ts | 22 +++- .../bigwig/bigwig-data-fetcher.ts | 4 +- .../csv/csv-data-fetcher.test.ts | 102 +++++++++++------- src/data-fetchers/csv/csv-data-fetcher.ts | 16 +-- src/data-fetchers/gff/gff-worker.ts | 18 +++- src/data-fetchers/utils.ts | 1 + src/data-fetchers/vcf/vcf-worker.ts | 20 +++- src/higlass-schema/higlass.schema.json | 18 ++++ src/higlass-schema/higlass.schema.ts | 14 ++- 16 files changed, 304 insertions(+), 95 deletions(-) diff --git a/src/compiler/compile.test.ts b/src/compiler/compile.test.ts index 47e20965..8a22b568 100644 --- a/src/compiler/compile.test.ts +++ b/src/compiler/compile.test.ts @@ -246,6 +246,107 @@ describe('Dummy track', () => { }); }); +describe('Compiler with UrlToFetchOptions', () => { + it('passes UrlToFetchOptions to the data fetcher', () => { + const urlToFetchOptions = { 'https://my-csv-url.com': { headers: { Authentication: 'Bearer 1234' } } }; + const spec: GoslingSpec = { + tracks: [ + { + id: 'track-id', + data: { + type: 'csv', + url: 'https://my-csv-url.com' + }, + mark: 'rect', + width: 100, + height: 100 + } + ] + }; + compile( + spec, + hgSpec => { + // @ts-ignore + expect(hgSpec.views[0].tracks.center[0].contents[0].data).toMatchInlineSnapshot(` + { + "assembly": "hg38", + "indexUrlFetchOptions": {}, + "type": "csv", + "url": "https://my-csv-url.com", + "urlFetchOptions": { + "headers": { + "Authentication": "Bearer 1234", + }, + }, + "x": undefined, + "x1": undefined, + "x1e": undefined, + "xe": undefined, + } + `); + }, + [], + getTheme(), + {}, + urlToFetchOptions + ); + }); + + it('passes UrlToFetchOptions and IndexUrlFetchOptions to the data fetcher', () => { + const urlToFetchOptions = { + 'https://file.gff': { headers: { Authentication: 'Bearer 1234' } }, + 'https://file.gff.tbi': { headers: { Authentication: 'Bearer 4321' } } + }; + const spec: GoslingSpec = { + tracks: [ + { + id: 'track-id', + data: { + type: 'gff', + url: 'https://file.gff', + indexUrl: 'https://file.gff.tbi' + }, + mark: 'rect', + width: 100, + height: 100 + } + ] + }; + compile( + spec, + hgSpec => { + // @ts-ignore + expect(hgSpec.views[0].tracks.center[0].contents[0].data).toMatchInlineSnapshot(` + { + "assembly": "hg38", + "indexUrl": "https://file.gff.tbi", + "indexUrlFetchOptions": { + "headers": { + "Authentication": "Bearer 4321", + }, + }, + "type": "gff", + "url": "https://file.gff", + "urlFetchOptions": { + "headers": { + "Authentication": "Bearer 1234", + }, + }, + "x": undefined, + "x1": undefined, + "x1e": undefined, + "xe": undefined, + } + `); + }, + [], + getTheme(), + {}, + urlToFetchOptions + ); + }); +}); + describe('Maintain IDs', () => { it('Overlaid tracks', () => { const twoTracksWithDiffData: SingleView = { diff --git a/src/compiler/compile.ts b/src/compiler/compile.ts index 004ad8c6..1ae08d63 100644 --- a/src/compiler/compile.ts +++ b/src/compiler/compile.ts @@ -4,6 +4,7 @@ import { traverseToFixSpecDownstream, overrideDataTemplates } from './spec-prepr import { replaceTrackTemplates } from '../core/utils/template'; import { getRelativeTrackInfo, type Size } from './bounding-box'; import type { CompleteThemeDeep } from '../core/utils/theme'; +import type { UrlToFetchOptions } from 'src/core/gosling-component'; import { renderHiGlass as createHiGlassModels } from './create-higlass-models'; import { manageResponsiveSpecs } from './responsive'; import type { IdTable } from '../api/track-and-view-ids'; @@ -17,6 +18,7 @@ export type CompileCallback = ( idTable: IdTable ) => void; + export function compile( spec: GoslingSpec, callback: CompileCallback, @@ -25,7 +27,8 @@ export function compile( containerStatus: { containerSize?: { width: number; height: number }; containerParentSize?: { width: number; height: number }; - } + }, + urlToFetchOptions?: UrlToFetchOptions ) { // Make sure to keep the original spec as-is const specCopy = JSON.parse(JSON.stringify(spec)); @@ -68,5 +71,5 @@ export function compile( } // Make HiGlass models for individual tracks - createHiGlassModels(specCopy, trackInfos, callback, theme); + createHiGlassModels(specCopy, trackInfos, callback, theme, urlToFetchOptions); } diff --git a/src/compiler/create-higlass-models.ts b/src/compiler/create-higlass-models.ts index 0e761eb1..dc1811a7 100644 --- a/src/compiler/create-higlass-models.ts +++ b/src/compiler/create-higlass-models.ts @@ -12,6 +12,7 @@ import type { } from '@gosling-lang/gosling-schema'; import type { CompleteThemeDeep } from '../core/utils/theme'; import type { CompileCallback } from './compile'; +import type { UrlToFetchOptions } from 'src/core/gosling-component'; import { getViewApiData } from '../api/api-data'; import { GoslingToHiGlassIdMapper } from '../api/track-and-view-ids'; import { IsDummyTrack } from '@gosling-lang/gosling-schema'; @@ -20,7 +21,8 @@ export function renderHiGlass( spec: GoslingSpec, trackInfos: TrackInfo[], callback: CompileCallback, - theme: CompleteThemeDeep + theme: CompleteThemeDeep, + urlToFetchOptions?: UrlToFetchOptions ) { if (trackInfos.length === 0) { // no tracks to render @@ -36,7 +38,7 @@ export function renderHiGlass( /* Update the HiGlass model by iterating tracks */ trackInfos.forEach(tb => { const { track, boundingBox: bb, layout } = tb; - goslingToHiGlass(hgModel, track, bb, layout, theme, idMapper); + goslingToHiGlass(hgModel, track, bb, layout, theme, idMapper, urlToFetchOptions); }); /* Add linking information to the HiGlass model */ diff --git a/src/compiler/gosling-to-higlass.ts b/src/compiler/gosling-to-higlass.ts index a0dd6ab0..33424d78 100644 --- a/src/compiler/gosling-to-higlass.ts +++ b/src/compiler/gosling-to-higlass.ts @@ -20,6 +20,7 @@ import { DEWFAULT_TITLE_PADDING_ON_TOP_AND_BOTTOM } from './defaults'; import type { CompleteThemeDeep } from '../core/utils/theme'; import { DEFAULT_TEXT_STYLE } from '../core/utils/text-style'; import type { GoslingToHiGlassIdMapper } from '../api/track-and-view-ids'; +import type { UrlToFetchOptions } from 'src/core/gosling-component'; /** * Convert a gosling track into a HiGlass view and add it into a higlass model. @@ -30,7 +31,8 @@ export function goslingToHiGlass( bb: BoundingBox, layout: RelativePosition, theme: Required, - idMapper: GoslingToHiGlassIdMapper + idMapper: GoslingToHiGlassIdMapper, + urlToFetchOptions?: UrlToFetchOptions ): HiGlassModel { // TODO: check whether there are multiple track.data across superposed tracks // ... @@ -152,10 +154,19 @@ export function goslingToHiGlass( x1: getFieldName('x1'), x1e: getFieldName('x1e') } as const; - // use gosling's custom data fetchers + + // Check whether there are any URL specific fetch options + const urlFetchOptions = + ('url' in firstResolvedSpec.data && urlToFetchOptions?.[firstResolvedSpec.data.url]) || {}; + const indexUrlFetchOptions = + ('indexUrl' in firstResolvedSpec.data && urlToFetchOptions?.[firstResolvedSpec.data.indexUrl]) || {}; + + // This object will be passed to the data fetchers hgTrack.data = { ...firstResolvedSpec.data, ...xFields, + urlFetchOptions, + indexUrlFetchOptions, // Additionally, add assembly, otherwise, a default genome build is used assembly // TODO: should look all sub tracks' `dataTransform` and apply OR operation. diff --git a/src/core/gosling-component.tsx b/src/core/gosling-component.tsx index d5fe1e13..23abed90 100644 --- a/src/core/gosling-component.tsx +++ b/src/core/gosling-component.tsx @@ -1,6 +1,7 @@ /* eslint-disable react/prop-types */ import { type HiGlassApi, HiGlassComponentWrapper } from './higlass-component-wrapper'; import type { TemplateTrackDef, VisUnitApiData } from '@gosling-lang/gosling-schema'; +import type { RequestInit } from '@gosling-lang/higlass-schema'; import React, { useState, useEffect, useMemo, useRef, forwardRef, useCallback, useImperativeHandle } from 'react'; import ResizeSensor from 'css-element-queries/src/ResizeSensor'; import * as gosling from '..'; @@ -17,6 +18,10 @@ import type { IdTable } from '../api/track-and-view-ids'; // If HiGlass is rendered and then the container resizes, the viewport position changes, unmatching `xDomain` specified by users. const DELAY_FOR_CONTAINER_RESIZE_BEFORE_RERENDER = 300; +/** Matches URLs to specific fetch options so that datafetchers have access URL specific fetch options */ +export interface UrlToFetchOptions { + [url: string]: RequestInit; +} interface GoslingCompProps { spec?: gosling.GoslingSpec; compiled?: (goslingSpec: gosling.GoslingSpec, higlassSpec: gosling.HiGlassSpec) => void; @@ -27,6 +32,7 @@ interface GoslingCompProps { className?: string; theme?: Theme; templates?: TemplateTrackDef[]; + urlToFetchOptions?: UrlToFetchOptions; experimental?: { reactive?: boolean; }; @@ -157,7 +163,8 @@ export const GoslingComponent = forwardRef((props, { containerSize: wrapperSize.current, containerParentSize: wrapperParentSize.current - } + }, + props.urlToFetchOptions ); } }, [props.spec, theme]); diff --git a/src/core/gosling-embed.ts b/src/core/gosling-embed.ts index 883e191d..0082bfd8 100644 --- a/src/core/gosling-embed.ts +++ b/src/core/gosling-embed.ts @@ -6,6 +6,7 @@ import type { HiGlassSpec } from '@gosling-lang/higlass-schema'; import { validateGoslingSpec } from '@gosling-lang/gosling-schema'; import { compile } from '../compiler/compile'; +import type { UrlToFetchOptions } from './gosling-component'; import { getTheme, type Theme } from './utils/theme'; import { GoslingTemplates } from './utils/template'; import { type GoslingApi, createApi } from '../api/api'; @@ -20,6 +21,7 @@ export type GoslingEmbedOptions = Omit> = new Map(); // indexed by bam url @@ -270,7 +261,7 @@ const init = async ( options: Partial = {} ) => { if (!bamFileCache.has(bam.url)) { - const bamFile = BamFile.fromUrl(bam.url, bam.indexUrl); + const bamFile = BamFile.fromUrl(bam.url, bam.indexUrl, options.urlFetchOptions, options.indexUrlFetchOptions); await bamFile.getHeader(); // reads bam/bai headers // Infer the correct chromosome names between 'chr1' and '1' diff --git a/src/data-fetchers/bed/bed-worker.ts b/src/data-fetchers/bed/bed-worker.ts index 0c84e6e6..32bb0eee 100644 --- a/src/data-fetchers/bed/bed-worker.ts +++ b/src/data-fetchers/bed/bed-worker.ts @@ -4,16 +4,20 @@ */ import { TabixIndexedFile } from '@gmod/tabix'; -import { expose, Transfer } from 'threads/worker'; import { sampleSize } from 'lodash-es'; +import { expose, Transfer } from 'threads/worker'; import type { TilesetInfo } from '@higlass/types'; + import type { ChromSizes } from '@gosling-lang/gosling-schema'; + import { DataSource, RemoteFile } from '../utils'; import BedParser from './bed-parser'; export type BedFileOptions = { sampleLength: number; customFields?: string[]; + urlFetchOptions?: RequestInit; + indexUrlFetchOptions?: RequestInit; }; /** @@ -55,12 +59,20 @@ export class BedFile { * @param url A string which is the URL of the bed file * @param indexUrl A string which is the URL of the bed index file * @param uid A unique identifier for the worker + * @param urlFetchOptions When the url is fetched, these options will be used + * @param indexUrlFetchOptions When the index URL is fetched, these options will be used * @returns an instance of BedFile */ - static fromUrl(url: string, indexUrl: string, uid: string) { + static fromUrl( + url: string, + indexUrl: string, + uid: string, + urlFetchOptions?: RequestInit, + indexUrlFetchOptions?: RequestInit + ) { const tbi = new TabixIndexedFile({ - filehandle: new RemoteFile(url), - tbiFilehandle: new RemoteFile(indexUrl) + filehandle: new RemoteFile(url, { overrides: urlFetchOptions }), + tbiFilehandle: new RemoteFile(indexUrl, { overrides: indexUrlFetchOptions }) }); return new BedFile(tbi, uid); } @@ -184,7 +196,7 @@ function init( ) { let bedFile = bedFiles.get(bed.url); if (!bedFile) { - bedFile = BedFile.fromUrl(bed.url, bed.indexUrl, uid); + bedFile = BedFile.fromUrl(bed.url, bed.indexUrl, uid, options.urlFetchOptions, options.indexUrlFetchOptions); if (options.customFields) bedFile.customFields = options.customFields; } const dataSource = new DataSource(bedFile, chromSizes, { diff --git a/src/data-fetchers/bigwig/bigwig-data-fetcher.ts b/src/data-fetchers/bigwig/bigwig-data-fetcher.ts index 9038f10c..f89e7e6a 100644 --- a/src/data-fetchers/bigwig/bigwig-data-fetcher.ts +++ b/src/data-fetchers/bigwig/bigwig-data-fetcher.ts @@ -86,10 +86,10 @@ function BigWigDataFetcher(HGC: import('@higlass/types').HGC, dataConfig: BigWig this.dataPromises.push(this.loadBBI(dataConfig)); } - async loadBBI(dataConfig: { url: string }) { + async loadBBI(dataConfig: BigWigDataConfig) { if (dataConfig.url) { this.bwFile = new BigWig({ - filehandle: new RemoteFile(dataConfig.url) + filehandle: new RemoteFile(dataConfig.url, { overrides: dataConfig.urlFetchOptions }) }); return this.bwFile.getHeader().then((h: BigWigHeader) => { this.bwFileHeader = h; diff --git a/src/data-fetchers/csv/csv-data-fetcher.test.ts b/src/data-fetchers/csv/csv-data-fetcher.test.ts index e5b4b80f..00b441d1 100644 --- a/src/data-fetchers/csv/csv-data-fetcher.test.ts +++ b/src/data-fetchers/csv/csv-data-fetcher.test.ts @@ -1,20 +1,29 @@ +import { vi, describe, expect } from 'vitest'; import type { TilesetInfo } from '@higlass/types'; import CsvDataFetcher, { type LoadedTiles, CsvDataFetcherClass } from './csv-data-fetcher'; -import fetch from 'cross-fetch'; +import { RemoteFile } from 'generic-filehandle'; -if (!globalThis.fetch) globalThis.fetch = fetch; +/** + * This mocks RemoteFile. It returns the contents of a csv file when the readFile method is called + */ +vi.mock('generic-filehandle', () => { + const str = 'c1,s1,e1,c2,s2,e2\n1,486,76975,15,100263879,100338121\n1,342608,393885,15,100218755,100268630'; + const RemoteFile = vi.fn(); + RemoteFile.prototype.readFile = vi.fn().mockResolvedValue(str); + RemoteFile.prototype.fetch = vi.fn(); + return { + RemoteFile + }; +}); describe('CSV data fetcher', () => { - const fetcher = new (CsvDataFetcher as any)( - {}, - { - url: 'https://raw.githubusercontent.com/sehilyi/gemini-datasets/master/data/cytogenetic_band.csv', - type: 'csv', - chromosomeField: 'Chr.', - genomicFields: ['ISCN_start', 'ISCN_stop', 'Basepair_start', 'Basepair_stop'] - }, - {} - ) as CsvDataFetcherClass; + const fetcher = new CsvDataFetcherClass({ + url: '', + type: 'csv', + chromosomeField: 'c1', + genomicFields: ['s1', 'e1'], + assembly: 'hg16' + }); it('creates tileset metadata', () => new Promise(resolve => { @@ -22,10 +31,10 @@ describe('CSV data fetcher', () => { expect(tile).toMatchInlineSnapshot(` { "max_pos": [ - 3088269832, - 3088269832, + 3070144630, + 3070144630, ], - "max_width": 3088269832, + "max_width": 3070144630, "max_zoom": 22, "min_pos": [ 0, @@ -42,17 +51,24 @@ describe('CSV data fetcher', () => { new Promise(resolve => { fetcher.fetchTilesDebounced( (loadedTile: LoadedTiles) => { - expect(Object.keys(loadedTile['0.0'].tabularData[0])).toMatchInlineSnapshot(` + expect(loadedTile['0.0'].tabularData).toMatchInlineSnapshot(` [ - "Chr.", - "Arm", - "Band", - "ISCN_start", - "ISCN_stop", - "Basepair_start", - "Basepair_stop", - "Stain", - "Density", + { + "c1": "1", + "c2": "15", + "e1": "76975", + "e2": "100338121", + "s1": "486", + "s2": "100263879", + }, + { + "c1": "1", + "c2": "15", + "e1": "393885", + "e2": "100268630", + "s1": "342608", + "s2": "100218755", + }, ] `); resolve(); @@ -60,18 +76,28 @@ describe('CSV data fetcher', () => { ['0.0'] ); })); +}); - it('puts data into multiple tiles', () => - new Promise(resolve => { - fetcher.fetchTilesDebounced( - (loadedTile: LoadedTiles) => { - expect(loadedTile['1.0']).not.toBeUndefined(); - expect(loadedTile['1.1']).not.toBeUndefined(); - expect(loadedTile['1.0'].tabularData.length).toBeGreaterThan(0); - expect(loadedTile['1.1'].tabularData.length).toBeGreaterThan(0); - resolve(); - }, - ['1.0', '1.1'] - ); - })); +test('CSV data fetcher can take fetch options', () => { + const overrides = { + headers: { + Authorization: 'Bearer 1234' + } + }; + new (CsvDataFetcher as any)( + {}, + { + url: 'https://raw.githubusercontent.com/sehilyi/gemini-datasets/master/data/cytogenetic_band.csv', + type: 'csv', + chromosomeField: 'Chr.', + genomicFields: ['ISCN_start', 'ISCN_stop', 'Basepair_start', 'Basepair_stop'], + urlFetchOptions: overrides + }, + {} + ) as CsvDataFetcherClass; + + expect(RemoteFile).toHaveBeenCalledWith( + 'https://raw.githubusercontent.com/sehilyi/gemini-datasets/master/data/cytogenetic_band.csv', + { overrides: overrides } + ); }); diff --git a/src/data-fetchers/csv/csv-data-fetcher.ts b/src/data-fetchers/csv/csv-data-fetcher.ts index e610928a..ac54d9dc 100644 --- a/src/data-fetchers/csv/csv-data-fetcher.ts +++ b/src/data-fetchers/csv/csv-data-fetcher.ts @@ -3,9 +3,9 @@ import { computeChromSizes } from '../../core/utils/assembly'; import { sampleSize } from 'lodash-es'; import type { Assembly, CsvData, FilterTransform, Datum } from '@gosling-lang/gosling-schema'; import { filterData } from '../../core/utils/data-transform'; -import { type CommonDataConfig, filterUsingGenoPos, sanitizeChrName } from '../utils'; +import { type CommonDataConfig, filterUsingGenoPos, sanitizeChrName, RemoteFile } from '../utils'; -type CsvDataConfig = CsvData & CommonDataConfig & { filter: FilterTransform[] }; +type CsvDataConfig = CsvData & CommonDataConfig & { filter?: FilterTransform[] }; interface ChromSizes { chrToAbs: (chrom: string, chromPos: number) => number; @@ -53,6 +53,7 @@ export class CsvDataFetcherClass { #parsedData!: DSVRowString[]; // Either set in the constructor or in #fetchCsv() #assembly: Assembly; #filter: FilterTransform[] | undefined; + #file: RemoteFile; constructor(dataConfig: CsvDataConfig) { this.dataConfig = dataConfig; @@ -64,6 +65,10 @@ export class CsvDataFetcherClass { console.error('Please provide the `url` of the data'); } + // Use any headers for this particular URL + const { urlFetchOptions, url } = dataConfig; + this.#file = new RemoteFile(url, { overrides: urlFetchOptions }); + this.#chromSizes = this.#generateChomSizeInfo(); this.#dataPromise = this.#fetchCsv(); } @@ -72,14 +77,13 @@ export class CsvDataFetcherClass { * Fetches CSV file from url, parses it, and sets this.#parsedData */ async #fetchCsv(): Promise { - const { url, chromosomeField, genomicFields, headerNames, longToWideId, genomicFieldsToConvert } = - this.dataConfig; + const { chromosomeField, genomicFields, headerNames, longToWideId, genomicFieldsToConvert } = this.dataConfig; const separator = this.dataConfig.separator ?? ','; try { - const response = await fetch(url); - const text = await (response.ok ? response.text() : Promise.reject(response.status)); + const buffer = await this.#file.readFile(); + const text = buffer.toString(); const textWithHeader = headerNames ? `${headerNames.join(separator)}\n${text}` : text; const parsedCSV = d3dsvFormat(separator).parse(textWithHeader, (row: DSVRowString) => diff --git a/src/data-fetchers/gff/gff-worker.ts b/src/data-fetchers/gff/gff-worker.ts index a02ac867..2b742743 100644 --- a/src/data-fetchers/gff/gff-worker.ts +++ b/src/data-fetchers/gff/gff-worker.ts @@ -11,6 +11,8 @@ import { isGFF3Feature, makeRandomSortedArray } from './utils'; export type GffFileOptions = { sampleLength: number; attributesToFields?: { attribute: string; defaultValue: string }[]; + urlFetchOptions?: RequestInit; + indexUrlFetchOptions?: RequestInit; }; export interface GffTile extends GFF3FeatureLineWithRefs { @@ -38,12 +40,20 @@ export class GffFile { * @param url A string which is the URL of the bed file * @param indexUrl A string which is the URL of the bed index file * @param uid A unique identifier for the worker + * @param urlFetchOptions When the url is fetched, these options will be used + * @param indexUrlFetchOptions When the index URL is fetched, these options will be used * @returns an instance of BedFile */ - static fromUrl(url: string, indexUrl: string, uid: string) { + static fromUrl( + url: string, + indexUrl: string, + uid: string, + urlFetchOptions?: RequestInit, + indexUrlFetchOptions?: RequestInit + ): GffFile { const tbi = new TabixIndexedFile({ - filehandle: new RemoteFile(url), - tbiFilehandle: new RemoteFile(indexUrl) + filehandle: new RemoteFile(url, { overrides: urlFetchOptions }), + tbiFilehandle: new RemoteFile(indexUrl, { overrides: indexUrlFetchOptions }) }); return new GffFile(tbi, uid); } @@ -252,7 +262,7 @@ function init( ) { let gffFile = gffFiles.get(bed.url); if (!gffFile) { - gffFile = GffFile.fromUrl(bed.url, bed.indexUrl, uid); + gffFile = GffFile.fromUrl(bed.url, bed.indexUrl, uid, options.urlFetchOptions, options.indexUrlFetchOptions); } const dataSource = new DataSource(gffFile, chromSizes, { sampleLength: 1000, // default sampleLength diff --git a/src/data-fetchers/utils.ts b/src/data-fetchers/utils.ts index b63c72e7..f2212fe2 100644 --- a/src/data-fetchers/utils.ts +++ b/src/data-fetchers/utils.ts @@ -10,6 +10,7 @@ export type CommonDataConfig = { xe?: string; x1?: string; x1e?: string; + urlFetchOptions?: RequestInit; }; export class DataSource { diff --git a/src/data-fetchers/vcf/vcf-worker.ts b/src/data-fetchers/vcf/vcf-worker.ts index 6bb8ad96..5289a505 100644 --- a/src/data-fetchers/vcf/vcf-worker.ts +++ b/src/data-fetchers/vcf/vcf-worker.ts @@ -19,6 +19,8 @@ const vcfFiles: Map = new Map(); type VcfFileOptions = { sampleLength: number; + urlFetchOptions?: RequestInit; + indexUrlFetchOptions?: RequestInit; }; /** @@ -33,14 +35,22 @@ class VcfFile { /** * Function to create an instance of VcfFile from URLs * @param url A string which is the URL of the bed file - * @param indexUrl A string which is the URL of the bed index file + * @param indexUrl A string which is the URL of the bed index file * @param uid A unique identifier for the worker + * @param urlFetchOptions + * @param indexUrlFetchOptions * @returns an instance of VcfFile */ - static fromUrl(url: string, indexUrl: string, uid: string) { + static fromUrl( + url: string, + indexUrl: string, + uid: string, + urlFetchOptions?: RequestInit, + indexUrlFetchOptions?: RequestInit + ) { const tbi = new TabixIndexedFile({ - filehandle: new RemoteFile(url), - tbiFilehandle: new RemoteFile(indexUrl) + filehandle: new RemoteFile(url, { overrides: urlFetchOptions }), + tbiFilehandle: new RemoteFile(indexUrl, { overrides: indexUrlFetchOptions }) }); return new VcfFile(tbi, uid); } @@ -132,7 +142,7 @@ function init( ) { let vcfFile = vcfFiles.get(vcf.url); if (!vcfFile) { - vcfFile = VcfFile.fromUrl(vcf.url, vcf.indexUrl, uid); + vcfFile = VcfFile.fromUrl(vcf.url, vcf.indexUrl, uid, options.urlFetchOptions, options.indexUrlFetchOptions); } const dataSource = new DataSource(vcfFile, chromSizes, { sampleLength: 1000, diff --git a/src/higlass-schema/higlass.schema.json b/src/higlass-schema/higlass.schema.json index 346fa160..adef2f22 100644 --- a/src/higlass-schema/higlass.schema.json +++ b/src/higlass-schema/higlass.schema.json @@ -107,6 +107,12 @@ }, "type": "array" }, + "indexUrl": { + "type": "string" + }, + "indexUrlFetchOptions": { + "$ref": "#/definitions/RequestInit" + }, "tiles": {}, "tilesetInfo": {}, "type": { @@ -114,6 +120,9 @@ }, "url": { "type": "string" + }, + "urlFetchOptions": { + "$ref": "#/definitions/RequestInit" } }, "type": "object" @@ -622,6 +631,15 @@ ], "type": "object" }, + "RequestInit": { + "additionalProperties": false, + "properties": { + "headers": { + "type": "object" + } + }, + "type": "object" + }, "Track": { "anyOf": [ { diff --git a/src/higlass-schema/higlass.schema.ts b/src/higlass-schema/higlass.schema.ts index 8f0cc9de..2ac62e95 100644 --- a/src/higlass-schema/higlass.schema.ts +++ b/src/higlass-schema/higlass.schema.ts @@ -100,21 +100,31 @@ export interface EnumTrack { y?: number; options?: any; } + export interface Data { - type?: string; // TODO: What kinds of types exist? + type?: string; children?: any[]; tiles?: any; tilesetInfo?: any; url?: string; // Options Gosling internally use - assembly?: Assembly; + assembly?: Assembly; + urlFetchOptions?: RequestInit; // Options used when URL is fetched + indexUrl?: string; // For datatypes that have indexes + indexUrlFetchOptions?: RequestInit; // Options used when index URL is fetched // Option to filter datasets // This has been added to filter data properly during fetching tiles filter?: FilterTransform[]; } +// When using the built-in RequestInit type, `yarn schema` doesn't work. So we are using our own RequestInit. +// Currently, we only forsee using the headers option. +export interface RequestInit { + headers?: Record; +} + export interface Overlay { uid?: string; type?: string; // TODO: What kinds of types exist?