Merge pull request #1743 from nextstrain/james/search-all

Enable filtering for all available metadata

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Changelog
 
+* Sidebar filtering now contains all non-continuous metadata defined across the tree (i.e. all data within `node.node_attrs`). The traits listed in `meta.filters` are now only used to determine which filters to list in the footer of the page. ([#1743](https://github.com/nextstrain/auspice/pull/1743))
 * Added a link to this changelog from the Auspice view. ([#1727](https://github.com/nextstrain/auspice/pull/1727))
 
 ## version 2.51.0 - 2023/11/16

src/actions/recomputeReduxState.js

@@ -8,7 +8,7 @@ import { constructVisibleTipLookupBetweenTrees } from "../util/treeTangleHelpers
 import { getDefaultControlsState, shouldDisplayTemporalConfidence } from "../reducers/controls";
 import { getDefaultFrequenciesState } from "../reducers/frequencies";
 import { getDefaultMeasurementsState } from "../reducers/measurements";
-import { countTraitsAcrossTree, calcTotalTipsInTree } from "../util/treeCountingHelpers";
+import { countTraitsAcrossTree, calcTotalTipsInTree, gatherTraitNames } from "../util/treeCountingHelpers";
 import { calcEntropyInView } from "../util/entropy";
 import { treeJsonToState } from "../util/treeJsonProcessing";
 import { castIncorrectTypes } from "../util/castJsonTypes";
@@ -100,16 +100,21 @@ const modifyStateViaURLQuery = (state, query) => {
     state["dateMax"] = query.dmax;
     state["dateMaxNumeric"] = calendarToNumeric(query.dmax);
   }
+
+  /** Queries 's', 'gt', and 'f_<name>' correspond to active filters */
   for (const filterKey of Object.keys(query).filter((c) => c.startsWith('f_'))) {
-    state.filters[filterKey.replace('f_', '')] = query[filterKey].split(',')
-      .map((value) => ({value, active: true})); /* all filters in the URL are "active" */
+    const filterName = filterKey.replace('f_', '');
+    const filterValues = query[filterKey] ? query[filterKey].split(',') : [];
+    state.filters[filterName] = filterValues.map((value) => ({value, active: true}))
   }
-  if (query.s) {   // selected strains are a filter too
-    state.filters[strainSymbol] = query.s.split(',').map((value) => ({value, active: true}));
+  if (query.s) {
+    const filterValues = query.s ? query.s.split(',') : [];
+    state.filters[strainSymbol] = filterValues.map((value) => ({value, active: true}));
   }
   if (query.gt) {
-    state.filters[genotypeSymbol] = decodeGenotypeFilters(query.gt);
+    state.filters[genotypeSymbol] = decodeGenotypeFilters(query.gt||"");
   }
+
   if (query.animate) {
     const params = query.animate.split(',');
     // console.log("start animation!", params);
@@ -225,19 +230,16 @@ const modifyStateViaMetadata = (state, metadata, genomeMap) => {
     state["analysisSlider"] = {key: metadata.analysisSlider, valid: false};
   }
   if (metadata.filters) {
-    /* the `meta -> filters` JSON spec should define which filters are displayed in the footer.
-    Note that this UI may change, and if so then we can change this state name. */
+    /**
+     * this spec previously defined both the footer-filters and the
+     * sidebar-filters, however it now only defines the former as the sidebar
+     * surfaces all available attributes.
+     */
     state.filtersInFooter = [...metadata.filters];
-    /* TODO - these will be searchable => all available traits should be added and this block shifted up */
-    metadata.filters.forEach((v) => {
-      state.filters[v] = [];
-    });
   } else {
     console.warn("JSON did not include any filters");
     state.filtersInFooter = [];
   }
-  state.filters[strainSymbol] = [];
-  state.filters[genotypeSymbol] = []; // this doesn't necessitate that mutations are defined
   if (metadata.displayDefaults) {
     const keysToCheckFor = ["geoResolution", "colorBy", "distanceMeasure", "layout", "mapTriplicate", "selectedBranchLabel", "tipLabelKey", 'sidebar', "showTransmissionLines", "normalizeFrequencies"];
     const expectedTypes =  ["string",        "string",  "string",          "string", "boolean",       "string",              'string',      'string',  "boolean"              , "boolean"];
@@ -579,32 +581,51 @@ const checkAndCorrectErrorsInState = (state, metadata, genomeMap, query, tree, v
     delete query.ci; // rm ci from the query if it doesn't apply
   }
 
-  /* ensure selected filters (via the URL query) are valid. If not, modify state + URL. */
-  const filterNames = Object.keys(state.filters).filter((filterName) => state.filters[filterName].length);
-  const stateCounts = countTraitsAcrossTree(tree.nodes, filterNames, false, true);
-  filterNames.forEach((filterName) => {
-    const validItems = state.filters[filterName]
-      .filter((item) => stateCounts[filterName].has(item.value));
-    state.filters[filterName] = validItems;
-    if (!validItems.length) {
-      delete query[`f_${filterName}`];
+  /**
+   * Any filters currently set are done so via the URL query, which we validate now
+   * (and update the URL query accordingly)
+   */
+  const _queryKey = (traitName) => (traitName === strainSymbol) ? 's' :
+    (traitName === genotypeSymbol) ? 'gt' :
+      `f_${traitName}`;
+
+  for (const traitName of Reflect.ownKeys(state.filters)) {
+    /* delete empty filters, e.g. "?f_country" or "?f_country=" */
+    if (!state.filters[traitName].length) {
+      delete state.filters[traitName];
+      delete query[_queryKey(traitName)];
+      continue
+    }
+    /* delete filter names (e.g. country, region) which aren't observed on the tree */
+    if (!Object.keys(tree.totalStateCounts).includes(traitName) && traitName!==strainSymbol && traitName!==genotypeSymbol) {
+      delete state.filters[traitName];
+      delete query[_queryKey(traitName)];
+      continue
+    }
+    /* delete filter values (e.g. USA, Oceania) which aren't valid, i.e. observed on the tree */
+    const traitValues = state.filters[traitName].map((f) => f.value);
+    let validTraitValues;
+    if (traitName === strainSymbol) {
+      const nodeNames = new Set(tree.nodes.map((n) => n.name));
+      validTraitValues = traitValues.filter((v) => nodeNames.has(v));
+    } else if (traitName === genotypeSymbol) {
+      const observedMutations = collectGenotypeStates(tree.nodes);
+      validTraitValues = traitValues.filter((v) => observedMutations.has(v));
     } else {
-      query[`f_${filterName}`] = validItems.map((x) => x.value).join(",");
+      validTraitValues = traitValues.filter((value) => tree.totalStateCounts[traitName].has(value));
+    }
+    if (validTraitValues.length===0) {
+      delete state.filters[traitName];
+      delete query[_queryKey(traitName)];
+    } else if (traitValues.length !== validTraitValues.length) {
+      state.filters[traitName] = validTraitValues.map((value) => ({value, active: true}));
+      query[_queryKey(traitName)] = traitName === genotypeSymbol ?
+        encodeGenotypeFilters(state.filters[traitName]) :
+        validTraitValues.join(",");
     }
-  });
-  if (state.filters[strainSymbol]) {
-    const validNames = tree.nodes.map((n) => n.name);
-    state.filters[strainSymbol] = state.filters[strainSymbol]
-      .filter((strainFilter) => validNames.includes(strainFilter.value));
-    query.s = state.filters[strainSymbol].map((f) => f.value).join(",");
-    if (!query.s) delete query.s;
-  }
-  if (state.filters[genotypeSymbol]) {
-    const observedMutations = collectGenotypeStates(tree.nodes);
-    state.filters[genotypeSymbol] = state.filters[genotypeSymbol]
-      .filter((f) => observedMutations.has(f.value));
-    query.gt = encodeGenotypeFilters(state.filters[genotypeSymbol]);
   }
+  /* Also remove any traitNames from the footer-displayed filters if they're not present on the tree */
+  state.filtersInFooter = state.filtersInFooter.filter((traitName) => traitName in tree.totalStateCounts);
 
   /* can we display branch length by div or num_date? */
   if (query.m && state.branchLengthsToDisplay !== "divAndDate") {
@@ -667,11 +688,7 @@ const modifyTreeStateVisAndBranchThickness = (oldState, zoomSelected, controlsSt
   );
 
   const newState = Object.assign({}, oldState, visAndThicknessData);
-  newState.stateCountAttrs = Object.keys(controlsState.filters);
   newState.idxOfInViewRootNode = newIdxRoot;
-  newState.visibleStateCounts = countTraitsAcrossTree(newState.nodes, newState.stateCountAttrs, newState.visibility, true);
-  newState.totalStateCounts   = countTraitsAcrossTree(newState.nodes, newState.stateCountAttrs, false,               true);
-
   return newState;
 };
 
@@ -868,6 +885,10 @@ export const createStateFromQueryOrJSONs = ({
   }
 
   const viewingNarrative = (narrativeBlocks || (oldState && oldState.narrative.display));
+
+  const stateCountAttrs = gatherTraitNames(tree.nodes, metadata.colorings);
+  tree.totalStateCounts = countTraitsAcrossTree(tree.nodes, stateCountAttrs, false, true);
+
   controls = checkAndCorrectErrorsInState(controls, metadata, entropy.genomeMap, query, tree, viewingNarrative); /* must run last */
 
 

src/actions/tree.js

@@ -97,7 +97,6 @@ export const updateVisibleTipsAndBranchThicknesses = (
       idxOfFilteredRoot: data.idxOfFilteredRoot,
       cladeName: cladeSelected,
       selectedClade: cladeSelected,
-      stateCountAttrs: Object.keys(controls.filters)
     };
 
     if (controls.showTreeToo) {
@@ -168,7 +167,6 @@ export const changeDateFilter = ({newMin = false, newMax = false, quickdraw = fa
       branchThickness: data.branchThickness,
       branchThicknessVersion: data.branchThicknessVersion,
       idxOfInViewRootNode: tree.idxOfInViewRootNode,
-      stateCountAttrs: Object.keys(controls.filters)
     };
     if (controls.showTreeToo) {
       const dataToo = calculateVisiblityAndBranchThickness(treeToo, controls, dates);
@@ -274,7 +272,7 @@ export const applyFilter = (mode, trait, values) => {
           console.error(`trying to ${mode} values from an un-initialised filter!`);
           return;
         }
-        newValues = controls.filters[trait].slice();
+        newValues = controls.filters[trait].map((f) => ({...f}));
         const currentItemNames = newValues.map((i) => i.value);
         for (const item of values) {
           const idx = currentItemNames.indexOf(item);
@@ -408,7 +406,6 @@ export const explodeTree = (attr) => (dispatch, getState) => {
     {dateMinNumeric: controls.dateMinNumeric, dateMaxNumeric: controls.dateMaxNumeric}
   );
   visData.idxOfInViewRootNode = 0;
-  visData.stateCountAttrs = Object.keys(controls.filters);
   /* Changes in visibility require a recomputation of which legend items we wish to display */
   visData.visibleLegendValues = createVisibleLegendValues({
     colorBy: controls.colorBy,

src/components/controls/filter.js

@@ -2,7 +2,7 @@ import React from "react";
 import { connect } from "react-redux";
 import AsyncSelect from "react-select/async";
 import { debounce } from 'lodash';
-import { controlsWidth, isValueValid, strainSymbol, genotypeSymbol} from "../../util/globals";
+import { controlsWidth, strainSymbol, genotypeSymbol} from "../../util/globals";
 import { collectGenotypeStates } from "../../util/treeMiscHelpers";
 import { applyFilter } from "../../actions/tree";
 import { removeAllFieldFilters, toggleAllFieldFilters, applyMeasurementFilter } from "../../actions/measurements";
@@ -24,6 +24,7 @@ const DEBOUNCE_TIME = 200;
     activeFilters: state.controls.filters,
     colorings: state.metadata.colorings,
     totalStateCounts: state.tree.totalStateCounts,
+    canFilterByGenotype: !!state.entropy.genomeMap,
     nodes: state.tree.nodes,
     measurementsFieldsMap: state.measurements.collectionToDisplay.fields,
     measurementsFiltersMap: state.measurements.collectionToDisplay.filters,
@@ -54,22 +55,40 @@ class FilterData extends React.Component {
      * each time a filter is toggled on / off.
      */
     const options = [];
-    Object.keys(this.props.activeFilters)
-      .forEach((filterName) => {
-        const filterTitle = this.getFilterTitle(filterName);
-        const filterValuesCurrentlyActive = this.props.activeFilters[filterName].filter((x) => x.active).map((x) => x.value);
-        Array.from(this.props.totalStateCounts[filterName].keys())
-          .filter((itemName) => isValueValid(itemName)) // remove invalid values present across the tree
-          .filter((itemName) => !filterValuesCurrentlyActive.includes(itemName)) // remove already enabled filters
-          .sort() // filters are sorted alphabetically - probably not necessary for a select component
-          .forEach((itemName) => {
-            options.push({
-              label: `${filterTitle} → ${itemName}`,
-              value: [filterName, itemName]
-            });
-          });
-      });
-    if (genotypeSymbol in this.props.activeFilters) {
+
+    /**
+     * First set of options is from the totalStateCounts -- i.e. every node attr
+     * which we know about (minus any currently selected filters). Note that we
+     * can't filter the filters to those set on visible nodes, as selecting a
+     * filter from outside this is perfectly fine in many situations.
+     *
+     * Those which are colorings appear first (and in the order defined in
+     * colorings). Within each trait, the values are alphabetical
+     */
+    const coloringKeys = Object.keys(this.props.colorings||{});
+    const unorderedTraitNames = Object.keys(this.props.totalStateCounts);
+    const traitNames = [
+      ...coloringKeys.filter((name) => unorderedTraitNames.includes(name)),
+      ...unorderedTraitNames.filter((name) => !coloringKeys.includes(name))
+    ]
+    for (const traitName of traitNames) {
+      const traitData = this.props.totalStateCounts[traitName];
+      const traitTitle = this.getFilterTitle(traitName);
+      const filterValuesCurrentlyActive = new Set((this.props.activeFilters[traitName] || []).filter((x) => x.active).map((x) => x.value));
+      for (const traitValue of Array.from(traitData.keys()).sort()) {
+        if (filterValuesCurrentlyActive.has(traitValue)) continue;
+        options.push({
+          label: `${traitTitle} → ${traitValue}`,
+          value: [traitName, traitValue]
+        });
+      }
+    }
+
+    /**
+     * Genotype filter options are numerous, they're the set of all observed
+     * mutations
+     */
+    if (this.props.canFilterByGenotype) {
       Array.from(collectGenotypeStates(this.props.nodes))
         .sort()
         .forEach((o) => {
@@ -79,16 +98,16 @@ class FilterData extends React.Component {
           });
         });
     }
-    if (strainSymbol in this.props.activeFilters) {
-      this.props.nodes
-        .filter((n) => !n.hasChildren)
-        .forEach((n) => {
-          options.push({
-            label: `sample → ${n.name}`,
-            value: [strainSymbol, n.name]
-          });
+
+    this.props.nodes
+      .filter((n) => !n.hasChildren)
+      .forEach((n) => {
+        options.push({
+          label: `sample → ${n.name}`,
+          value: [strainSymbol, n.name]
         });
-    }
+      });
+
     if (this.props.measurementsOn && this.props.measurementsFiltersMap && this.props.measurementsFieldsMap) {
       this.props.measurementsFiltersMap.forEach(({values}, filterField) => {
         const { title } = this.props.measurementsFieldsMap.get(filterField);
@@ -148,7 +167,7 @@ class FilterData extends React.Component {
     const measurementsFilters = this.summariseMeasurementsFilters();
     /* When filter categories were dynamically created (via metadata drag&drop) the `options` here updated but `<Async>`
     seemed to use a cached version of all values & wouldn't update. Changing the key forces a rerender, but it's not ideal */
-    const divKey = String(Object.keys(this.props.activeFilters).length);
+    const divKey = String(Object.keys(this.props.totalStateCounts).join(","));
     return (
       <div style={styles.base} key={divKey}>
         <AsyncSelect

src/components/download/downloadButtons.js

@@ -16,7 +16,7 @@ const iconWidth = 25;
  * A React Component displaying buttons which trigger data-downloads. Intended for display within the
  * larger Download modal component
  */
-export const DownloadButtons = ({dispatch, t, tree, entropy, metadata, colorBy, distanceMeasure, panelsToDisplay, panelLayout, filters, visibility, visibleStateCounts, relevantPublications}) => {
+export const DownloadButtons = ({dispatch, t, tree, entropy, metadata, colorBy, distanceMeasure, panelsToDisplay, panelLayout, visibility, relevantPublications}) => {
   const totalTipCount = metadata.mainTreeNumTips;
   const selectedTipsCount = getNumSelectedTips(tree.nodes, tree.visibility);
   const partialData = selectedTipsCount !== totalTipCount;
@@ -100,9 +100,7 @@ export const DownloadButtons = ({dispatch, t, tree, entropy, metadata, colorBy,
           t,
           metadata,
           tree.nodes,
-          filters,
           visibility,
-          visibleStateCounts,
           getFilePrefix(),
           panelsToDisplay,
           panelLayout,

src/components/download/downloadModal.js

@@ -50,8 +50,6 @@ export const publications = {
   entropy: state.entropy,
   tree: state.tree,
   nodes: state.tree.nodes,
-  visibleStateCounts: state.tree.visibleStateCounts,
-  filters: state.controls.filters,
   visibility: state.tree.visibility,
   panelsToDisplay: state.controls.panelsToDisplay,
   panelLayout: state.controls.panelLayout
@@ -164,9 +162,7 @@ class DownloadModal extends React.Component {
             {datasetSummary({
               mainTreeNumTips: this.props.metadata.mainTreeNumTips,
               nodes: this.props.nodes,
-              filters: this.props.filters,
               visibility: this.props.visibility,
-              visibleStateCounts: this.props.visibleStateCounts,
               t: this.props.t
             })}
           </div>

src/components/download/helperFunctions.js

@@ -565,7 +565,7 @@ const writeSVGPossiblyIncludingMap = (dispatch, filePrefix, panelsInDOM, panelLa
   }
 };
 
-export const SVG = (dispatch, t, metadata, nodes, filters, visibility, visibleStateCounts, filePrefix, panelsInDOM, panelLayout, publications) => {
+export const SVG = (dispatch, t, metadata, nodes, visibility, filePrefix, panelsInDOM, panelLayout, publications) => {
   /* make the text strings */
   const textStrings = [];
   textStrings.push(metadata.title);
@@ -575,9 +575,7 @@ export const SVG = (dispatch, t, metadata, nodes, filters, visibility, visibleSt
   textStrings.push(datasetSummary({
     mainTreeNumTips: metadata.mainTreeNumTips,
     nodes,
-    filters,
     visibility,
-    visibleStateCounts,
     t
   }));
   textStrings.push("");