Merge pull request #1471 from nextstrain/fix/large-auspice-json-local…

…-storage

packages/nextclade-web/.eslintrc.js

@@ -130,6 +130,7 @@ module.exports = {
     'no-shadow': 'off',
     'no-unused-vars': 'off',
     'only-ascii/only-ascii': 'warn',
+    'prefer-const': ['warn', { destructuring: 'all' }],
     'prefer-for-of': 'off',
     'prettier/prettier': 'warn',
     'react-hooks/exhaustive-deps': [
@@ -148,7 +149,7 @@ module.exports = {
     'react/state-in-constructor': 'off',
     'security/detect-non-literal-fs-filename': 'off',
     'security/detect-object-injection': 'off',
-    'sonarjs/cognitive-complexity': ['warn', 20],
+    'sonarjs/cognitive-complexity': 'off',
     'unicorn/consistent-function-scoping': 'off',
     'unicorn/escape-case': 'off',
     'unicorn/filename-case': 'off',

packages/nextclade-web/src/components/Main/SuggestionAlertMainPage.tsx

@@ -13,7 +13,6 @@ export function SuggestionAlertMainPage({ ...restProps }) {
   const { numSuggestions, datasetsActive: datasetSuggestions } = useDatasetSuggestionResults()
   const autodetectRunState = useRecoilValue(autodetectRunStateAtom)
 
-  // eslint-disable-next-line sonarjs/cognitive-complexity
   const alert = useMemo(() => {
     const hasMatch = datasetCurrent && datasetSuggestions.some((suggestion) => suggestion.path === datasetCurrent.path)
 

packages/nextclade-web/src/io/fetchDatasets.ts

@@ -9,7 +9,7 @@ import {
   parseGithubRepoUrl,
 } from 'src/io/fetchSingleDatasetFromGithub'
 
-import { type AuspiceTree, Dataset } from 'src/types'
+import { Dataset } from 'src/types'
 import {
   fetchDatasetsIndex,
   filterDatasets,
@@ -128,11 +128,7 @@ export async function initializeDatasets(datasetServerUrl: string, urlQuery: Par
   const minimizerIndexVersion = await getCompatibleMinimizerIndexVersion(datasetServerUrl, datasetsIndexJson)
 
   // Check if URL params specify dataset params and try to find the corresponding dataset
-  const currentDataset:
-    | (Dataset & {
-        auspiceJson?: AuspiceTree
-      })
-    | undefined = await getDatasetFromUrlParams(urlQuery, datasets)
+  const currentDataset = await getDatasetFromUrlParams(urlQuery, datasets)
 
   return { datasets, currentDataset, minimizerIndexVersion }
 }

packages/nextclade-web/src/io/fetchSingleDatasetAuspice.ts

@@ -39,7 +39,7 @@ export async function fetchSingleDatasetAuspice(datasetJsonUrl_: string) {
     }
   }
 
-  const currentDataset: Dataset & { auspiceJson?: AuspiceTree } = {
+  const currentDataset: Dataset = {
     path: datasetJsonUrl,
     capabilities: {
       primers: false,
@@ -50,8 +50,8 @@ export async function fetchSingleDatasetAuspice(datasetJsonUrl_: string) {
       name,
       ...pathogen?.attributes,
     },
+    type: 'auspiceJson',
     version,
-    auspiceJson,
   }
 
   const datasets = [currentDataset]
@@ -60,5 +60,5 @@ export async function fetchSingleDatasetAuspice(datasetJsonUrl_: string) {
   const defaultDatasetName = currentDatasetName
   const defaultDatasetNameFriendly = attrStrMaybe(currentDataset.attributes, 'name') ?? currentDatasetName
 
-  return { datasets, defaultDataset, defaultDatasetName, defaultDatasetNameFriendly, currentDataset }
+  return { datasets, defaultDataset, defaultDatasetName, defaultDatasetNameFriendly, currentDataset, auspiceJson }
 }

packages/nextclade-web/src/io/fetchSingleDatasetDirectory.ts

@@ -2,7 +2,7 @@ import axios from 'axios'
 import urljoin from 'url-join'
 import { mapValues } from 'lodash'
 import { concurrent } from 'fasy'
-import { attrStrMaybe, AuspiceTree, Dataset, VirusProperties } from 'src/types'
+import { attrStrMaybe, Dataset, VirusProperties } from 'src/types'
 import { removeTrailingSlash } from 'src/io/url'
 import { axiosFetch, axiosHead, axiosHeadOrUndefined } from 'src/io/axiosFetch'
 import { sanitizeError } from 'src/helpers/sanitizeError'
@@ -17,14 +17,15 @@ export async function fetchSingleDatasetDirectory(
 
   const files = mapValues(pathogen.files, (file) => (file ? urljoin(datasetRootUrl, file) : file))
 
-  const currentDataset: Dataset & { auspiceJson?: AuspiceTree } = {
+  const currentDataset: Dataset = {
     path: datasetRootUrl,
     capabilities: {
       primers: false,
       qc: [],
     },
     ...pathogen,
     files,
+    type: 'directory',
   }
 
   const datasets = [currentDataset]
@@ -51,7 +52,14 @@ export async function fetchSingleDatasetDirectory(
     Object.entries(files).filter(([_, key]) => !['examples', 'readme'].includes(key)),
   )
 
-  return { datasets, defaultDataset, defaultDatasetName, defaultDatasetNameFriendly, currentDataset }
+  return {
+    datasets,
+    defaultDataset,
+    defaultDatasetName,
+    defaultDatasetNameFriendly,
+    currentDataset,
+    auspiceJson: undefined,
+  }
 }
 
 async function fetchPathogenJson(datasetRootUrl: string) {

packages/nextclade-web/src/io/fetchSingleDatasetFromGithub.ts

@@ -1,4 +1,4 @@
-/* eslint-disable security/detect-unsafe-regex,prefer-const */
+/* eslint-disable security/detect-unsafe-regex */
 import type { Optional } from 'utility-types'
 import { isEmpty, isNil, trim } from 'lodash'
 import pMemoize from 'p-memoize'

packages/nextclade-web/src/pages/_app.tsx

@@ -8,7 +8,6 @@ import { RecoilEnv, RecoilRoot, useRecoilCallback, useRecoilState, useRecoilValu
 import { AppProps } from 'next/app'
 import { useRouter } from 'next/router'
 import dynamic from 'next/dynamic'
-import type { Dataset, AuspiceTree } from 'src/types'
 import { sanitizeError } from 'src/helpers/sanitizeError'
 import { useRunAnalysis } from 'src/hooks/useRunAnalysis'
 import i18nAuspice, { changeAuspiceLocale } from 'src/i18n/i18n.auspice'
@@ -97,24 +96,35 @@ function RecoilStateInitializer() {
 
         const datasetInfo = await fetchSingleDataset(urlQuery)
         if (!isNil(datasetInfo)) {
-          const { datasets, currentDataset } = datasetInfo
-          return { datasets, currentDataset, minimizerIndexVersion: undefined }
+          const { datasets, currentDataset, auspiceJson } = datasetInfo
+          return { datasets, currentDataset, minimizerIndexVersion: undefined, auspiceJson }
         }
-        return { datasets, currentDataset, minimizerIndexVersion }
+        return { datasets, currentDataset, minimizerIndexVersion, auspiceJson: undefined }
       })
       .catch((error) => {
         // Dataset error is fatal and we want error to be handled in the ErrorBoundary
         setInitialized(false)
         set(globalErrorAtom, sanitizeError(error))
         throw error
       })
-      .then(async ({ datasets, currentDataset, minimizerIndexVersion }) => {
+      .then(async ({ datasets, currentDataset, minimizerIndexVersion, auspiceJson }) => {
         set(datasetsAtom, { datasets })
-        const previousDataset = await getPromise(datasetCurrentAtom)
-        const dataset: (Dataset & { auspiceJson?: AuspiceTree }) | undefined = currentDataset ?? previousDataset
+        let previousDataset = await getPromise(datasetCurrentAtom)
+        if (previousDataset?.type === 'auspiceJson') {
+          // Disregard previously saved dataset if it's Auspice dataset, because the data is no longer available.
+          // We might re-fetch instead, but need to persist URL for that somehow.
+          previousDataset = undefined
+        }
+
+        const dataset = currentDataset ?? previousDataset
         set(datasetCurrentAtom, dataset)
         set(minimizerIndexVersionAtom, minimizerIndexVersion)
-        set(datasetJsonAtom, dataset?.auspiceJson)
+
+        if (dataset?.type === 'auspiceJson' && !isNil(auspiceJson)) {
+          set(datasetJsonAtom, auspiceJson)
+        } else {
+          set(datasetJsonAtom, undefined)
+        }
         return dataset
       })
       .then(async (dataset) => {

packages/nextclade/src/io/dataset.rs

@@ -78,6 +78,9 @@ pub struct Dataset {
   #[serde(default, skip_serializing_if = "DatasetVersion::is_empty")]
   pub version: DatasetVersion,
 
+  #[serde(default, skip_serializing_if = "Option::is_none")]
+  pub r#type: Option<DatasetType>,
+
   #[serde(flatten)]
   pub other: serde_json::Value,
 }
@@ -386,3 +389,11 @@ pub struct MinimizerIndexVersion {
   #[serde(flatten)]
   pub other: serde_json::Value,
 }
+
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub enum DatasetType {
+  Directory,
+  AuspiceJson,
+  Other,
+}