Filter on all available data

The sidebar filtering now surfaces all valid node-attrs defined across
the (terminal) nodes. URL queries (`?f_${attrName}=value1,value2,...`)
also work for all known attrs. Attributes which are known to be
continuous (via a colorings definition) are excluded, as the filtering
UI is not (yet) able to handle these; if a non-coloring continuous
attribute is set on the nodes then this will end up as a multitude of
numerical options in the sidebar.

As part of this implementation we have removed `stateCountAttrs` from
redux state and improved the validation of (filtering) URL queries.

The behaviour of filtering, and the restriction to collecting attributes
from terminal nodes only, is unchanged. See #1275 for more context.

Closes #1251

src/actions/recomputeReduxState.js

@@ -8,7 +8,7 @@ import { constructVisibleTipLookupBetweenTrees } from "../util/treeTangleHelpers
 import { getDefaultControlsState, shouldDisplayTemporalConfidence } from "../reducers/controls";
 import { getDefaultFrequenciesState } from "../reducers/frequencies";
 import { getDefaultMeasurementsState } from "../reducers/measurements";
-import { countTraitsAcrossTree, calcTotalTipsInTree } from "../util/treeCountingHelpers";
+import { countTraitsAcrossTree, calcTotalTipsInTree, gatherTraitNames } from "../util/treeCountingHelpers";
 import { calcEntropyInView } from "../util/entropy";
 import { treeJsonToState } from "../util/treeJsonProcessing";
 import { castIncorrectTypes } from "../util/castJsonTypes";
@@ -100,16 +100,21 @@ const modifyStateViaURLQuery = (state, query) => {
     state["dateMax"] = query.dmax;
     state["dateMaxNumeric"] = calendarToNumeric(query.dmax);
   }
+
+  /** Queries 's', 'gt', and 'f_<name>' correspond to active filters */
   for (const filterKey of Object.keys(query).filter((c) => c.startsWith('f_'))) {
-    state.filters[filterKey.replace('f_', '')] = query[filterKey].split(',')
-      .map((value) => ({value, active: true})); /* all filters in the URL are "active" */
+    const filterName = filterKey.replace('f_', '');
+    const filterValues = query[filterKey] ? query[filterKey].split(',') : [];
+    state.filters[filterName] = filterValues.map((value) => ({value, active: true}))
   }
-  if (query.s) {   // selected strains are a filter too
-    state.filters[strainSymbol] = query.s.split(',').map((value) => ({value, active: true}));
+  if (query.s) {
+    const filterValues = query.s ? query.s.split(',') : [];
+    state.filters[strainSymbol] = filterValues.map((value) => ({value, active: true}));
   }
   if (query.gt) {
-    state.filters[genotypeSymbol] = decodeGenotypeFilters(query.gt);
+    state.filters[genotypeSymbol] = decodeGenotypeFilters(query.gt||"");
   }
+
   if (query.animate) {
     const params = query.animate.split(',');
     // console.log("start animation!", params);
@@ -225,19 +230,16 @@ const modifyStateViaMetadata = (state, metadata, genomeMap) => {
     state["analysisSlider"] = {key: metadata.analysisSlider, valid: false};
   }
   if (metadata.filters) {
-    /* the `meta -> filters` JSON spec should define which filters are displayed in the footer.
-    Note that this UI may change, and if so then we can change this state name. */
+    /**
+     * this spec previously defined both the footer-filters and the
+     * sidebar-filters, however it now only defines the former as the sidebar
+     * surfaces all available attributes.
+     */
     state.filtersInFooter = [...metadata.filters];
-    /* TODO - these will be searchable => all available traits should be added and this block shifted up */
-    metadata.filters.forEach((v) => {
-      state.filters[v] = [];
-    });
   } else {
     console.warn("JSON did not include any filters");
     state.filtersInFooter = [];
   }
-  state.filters[strainSymbol] = [];
-  state.filters[genotypeSymbol] = []; // this doesn't necessitate that mutations are defined
   if (metadata.displayDefaults) {
     const keysToCheckFor = ["geoResolution", "colorBy", "distanceMeasure", "layout", "mapTriplicate", "selectedBranchLabel", "tipLabelKey", 'sidebar', "showTransmissionLines", "normalizeFrequencies"];
     const expectedTypes =  ["string",        "string",  "string",          "string", "boolean",       "string",              'string',      'string',  "boolean"              , "boolean"];
@@ -579,31 +581,48 @@ const checkAndCorrectErrorsInState = (state, metadata, genomeMap, query, tree, v
     delete query.ci; // rm ci from the query if it doesn't apply
   }
 
-  /* ensure selected filters (via the URL query) are valid. If not, modify state + URL. */
-  const filterNames = Object.keys(state.filters).filter((filterName) => state.filters[filterName].length);
-  const stateCounts = countTraitsAcrossTree(tree.nodes, filterNames, false, true);
-  filterNames.forEach((filterName) => {
-    const validItems = state.filters[filterName]
-      .filter((item) => stateCounts[filterName].has(item.value));
-    state.filters[filterName] = validItems;
-    if (!validItems.length) {
-      delete query[`f_${filterName}`];
+  /**
+   * Any filters currently set are done so via the URL query, which we validate now
+   * (and update the URL query accordingly)
+   */
+  const _queryKey = (traitName) => (traitName === strainSymbol) ? 's' :
+    (traitName === genotypeSymbol) ? 'gt' :
+      `f_${traitName}`;
+
+  for (const traitName of Reflect.ownKeys(state.filters)) {
+    /* delete empty filters, e.g. "?f_country" or "?f_country=" */
+    if (!state.filters[traitName].length) {
+      delete state.filters[traitName];
+      delete query[_queryKey(traitName)];
+      continue
+    }
+    /* delete filter names (e.g. country, region) which aren't observed on the tree */
+    if (!Object.keys(tree.totalStateCounts).includes(traitName) && traitName!==strainSymbol && traitName!==genotypeSymbol) {
+      delete state.filters[traitName];
+      delete query[_queryKey(traitName)];
+      continue
+    }
+    /* delete filter values (e.g. USA, Oceania) which aren't valid, i.e. observed on the tree */
+    const traitValues = state.filters[traitName].map((f) => f.value);
+    let validTraitValues;
+    if (traitName === strainSymbol) {
+      const nodeNames = new Set(tree.nodes.map((n) => n.name));
+      validTraitValues = traitValues.filter((v) => nodeNames.has(v));
+    } else if (traitName === genotypeSymbol) {
+      const observedMutations = collectGenotypeStates(tree.nodes);
+      validTraitValues = traitValues.filter((v) => observedMutations.has(v));
     } else {
-      query[`f_${filterName}`] = validItems.map((x) => x.value).join(",");
+      validTraitValues = traitValues.filter((value) => tree.totalStateCounts[traitName].has(value));
+    }
+    if (validTraitValues.length===0) {
+      delete state.filters[traitName];
+      delete query[_queryKey(traitName)];
+    } else if (traitValues.length !== validTraitValues.length) {
+      state.filters[traitName] = validTraitValues.map((value) => ({value, active: true}));
+      query[_queryKey(traitName)] = traitName === genotypeSymbol ?
+        encodeGenotypeFilters(state.filters[traitName]) :
+        validTraitValues.join(",");
     }
-  });
-  if (state.filters[strainSymbol]) {
-    const validNames = tree.nodes.map((n) => n.name);
-    state.filters[strainSymbol] = state.filters[strainSymbol]
-      .filter((strainFilter) => validNames.includes(strainFilter.value));
-    query.s = state.filters[strainSymbol].map((f) => f.value).join(",");
-    if (!query.s) delete query.s;
-  }
-  if (state.filters[genotypeSymbol]) {
-    const observedMutations = collectGenotypeStates(tree.nodes);
-    state.filters[genotypeSymbol] = state.filters[genotypeSymbol]
-      .filter((f) => observedMutations.has(f.value));
-    query.gt = encodeGenotypeFilters(state.filters[genotypeSymbol]);
   }
 
   /* can we display branch length by div or num_date? */
@@ -667,10 +686,7 @@ const modifyTreeStateVisAndBranchThickness = (oldState, zoomSelected, controlsSt
   );
 
   const newState = Object.assign({}, oldState, visAndThicknessData);
-  newState.stateCountAttrs = Object.keys(controlsState.filters);
   newState.idxOfInViewRootNode = newIdxRoot;
-  newState.totalStateCounts = countTraitsAcrossTree(newState.nodes, newState.stateCountAttrs, false, true);
-
   return newState;
 };
 
@@ -867,6 +883,10 @@ export const createStateFromQueryOrJSONs = ({
   }
 
   const viewingNarrative = (narrativeBlocks || (oldState && oldState.narrative.display));
+
+  const stateCountAttrs = gatherTraitNames(tree.nodes, metadata.colorings);
+  tree.totalStateCounts = countTraitsAcrossTree(tree.nodes, stateCountAttrs, false, true);
+
   controls = checkAndCorrectErrorsInState(controls, metadata, entropy.genomeMap, query, tree, viewingNarrative); /* must run last */
 
 

src/actions/tree.js

@@ -272,7 +272,7 @@ export const applyFilter = (mode, trait, values) => {
           console.error(`trying to ${mode} values from an un-initialised filter!`);
           return;
         }
-        newValues = controls.filters[trait].slice();
+        newValues = controls.filters[trait].map((f) => ({...f}));
         const currentItemNames = newValues.map((i) => i.value);
         for (const item of values) {
           const idx = currentItemNames.indexOf(item);

src/components/controls/filter.js

@@ -2,7 +2,7 @@ import React from "react";
 import { connect } from "react-redux";
 import AsyncSelect from "react-select/async";
 import { debounce } from 'lodash';
-import { controlsWidth, isValueValid, strainSymbol, genotypeSymbol} from "../../util/globals";
+import { controlsWidth, strainSymbol, genotypeSymbol} from "../../util/globals";
 import { collectGenotypeStates } from "../../util/treeMiscHelpers";
 import { applyFilter } from "../../actions/tree";
 import { removeAllFieldFilters, toggleAllFieldFilters, applyMeasurementFilter } from "../../actions/measurements";
@@ -24,6 +24,7 @@ const DEBOUNCE_TIME = 200;
     activeFilters: state.controls.filters,
     colorings: state.metadata.colorings,
     totalStateCounts: state.tree.totalStateCounts,
+    canFilterByGenotype: !!state.entropy.genomeMap,
     nodes: state.tree.nodes,
     measurementsFieldsMap: state.measurements.collectionToDisplay.fields,
     measurementsFiltersMap: state.measurements.collectionToDisplay.filters,
@@ -54,22 +55,38 @@ class FilterData extends React.Component {
      * each time a filter is toggled on / off.
      */
     const options = [];
-    Object.keys(this.props.activeFilters)
-      .forEach((filterName) => {
-        const filterTitle = this.getFilterTitle(filterName);
-        const filterValuesCurrentlyActive = this.props.activeFilters[filterName].filter((x) => x.active).map((x) => x.value);
-        Array.from(this.props.totalStateCounts[filterName].keys())
-          .filter((itemName) => isValueValid(itemName)) // remove invalid values present across the tree
-          .filter((itemName) => !filterValuesCurrentlyActive.includes(itemName)) // remove already enabled filters
-          .sort() // filters are sorted alphabetically - probably not necessary for a select component
-          .forEach((itemName) => {
-            options.push({
-              label: `${filterTitle} → ${itemName}`,
-              value: [filterName, itemName]
-            });
-          });
-      });
-    if (genotypeSymbol in this.props.activeFilters) {
+
+    /**
+     * First set of options is from the totalStateCounts -- i.e. every node attr
+     * which we know about (minus any currently selected filters). Note that we
+     * can't filter the filters to those set on visible nodes, as selecting a
+     * filter from outside this is perfectly fine in many situations.
+     *
+     * Those which are colorings appear first (and in the order defined in
+     * colorings). Within each trait, the values are alphabetical
+     */
+    const coloringKeys = Object.values(this.props.colorings||{}).map((c) => c.key);
+    const traitNames = Object.keys(this.props.totalStateCounts).sort((a,b) => {
+      return coloringKeys.indexOf(a) - coloringKeys.indexOf(b)
+    })
+    for (const traitName of traitNames) {
+      const traitData = this.props.totalStateCounts[traitName];
+      const traitTitle = this.getFilterTitle(traitName);
+      const filterValuesCurrentlyActive = new Set((this.props.activeFilters[traitName] || []).filter((x) => x.active).map((x) => x.value));
+      for (const traitValue of Array.from(traitData.keys()).sort()) {
+        if (filterValuesCurrentlyActive.has(traitValue)) continue;
+        options.push({
+          label: `${traitTitle} → ${traitValue}`,
+          value: [traitName, traitValue]
+        });
+      }
+    }
+
+    /**
+     * Genotype filter options are numerous, they're the set of all observed
+     * mutations
+     */
+    if (this.props.canFilterByGenotype) {
       Array.from(collectGenotypeStates(this.props.nodes))
         .sort()
         .forEach((o) => {
@@ -79,16 +96,16 @@ class FilterData extends React.Component {
           });
         });
     }
-    if (strainSymbol in this.props.activeFilters) {
-      this.props.nodes
-        .filter((n) => !n.hasChildren)
-        .forEach((n) => {
-          options.push({
-            label: `sample → ${n.name}`,
-            value: [strainSymbol, n.name]
-          });
+
+    this.props.nodes
+      .filter((n) => !n.hasChildren)
+      .forEach((n) => {
+        options.push({
+          label: `sample → ${n.name}`,
+          value: [strainSymbol, n.name]
         });
-    }
+      });
+
     if (this.props.measurementsOn && this.props.measurementsFiltersMap && this.props.measurementsFieldsMap) {
       this.props.measurementsFiltersMap.forEach(({values}, filterField) => {
         const { title } = this.props.measurementsFieldsMap.get(filterField);
@@ -148,7 +165,7 @@ class FilterData extends React.Component {
     const measurementsFilters = this.summariseMeasurementsFilters();
     /* When filter categories were dynamically created (via metadata drag&drop) the `options` here updated but `<Async>`
     seemed to use a cached version of all values & wouldn't update. Changing the key forces a rerender, but it's not ideal */
-    const divKey = String(Object.keys(this.props.activeFilters).length);
+    const divKey = String(Object.keys(this.props.totalStateCounts).join(","));
     return (
       <div style={styles.base} key={divKey}>
         <AsyncSelect

src/components/framework/footer.js

@@ -180,7 +180,7 @@ export const getAcknowledgments = (metadata, dispatch) => {
 };
 
 const dispatchFilter = (dispatch, activeFilters, key, value) => {
-  const activeValuesOfFilter = activeFilters[key].map((f) => f.value);
+  const activeValuesOfFilter = (activeFilters[key] || []).map((f) => f.value);
   const mode = activeValuesOfFilter.indexOf(value) === -1 ? "add" : "remove";
   dispatch(applyFilter(mode, key, [value]));
 };
@@ -246,10 +246,10 @@ class Footer extends React.Component {
     const { t } = this.props;
     const totalStateCount = this.props.totalStateCounts[filterName];
     const filterTitle = this.props.metadata.colorings[filterName] ? this.props.metadata.colorings[filterName].title : filterName;
-    const activeFilterItems = this.props.activeFilters[filterName].filter((x) => x.active).map((x) => x.value);
+    const activeFilterItems = (this.props.activeFilters[filterName] || []).filter((x) => x.active).map((x) => x.value);
     const title = (<div>
       {t("Filter by {{filterTitle}}", {filterTitle: filterTitle}) + ` (n=${totalStateCount.size})`}
-      {this.props.activeFilters[filterName].length ? removeFiltersButton(this.props.dispatch, [filterName], "inlineRight", t("Clear {{filterName}} filter", { filterName: filterName})) : null}
+      {this.props.activeFilters?.[filterName]?.length ? removeFiltersButton(this.props.dispatch, [filterName], "inlineRight", t("Clear {{filterName}} filter", { filterName: filterName})) : null}
     </div>);
     return (
       <div>
@@ -293,16 +293,12 @@ class Footer extends React.Component {
           <div className='line'/>
           {getAcknowledgments(this.props.metadata, this.props.dispatch)}
           <div className='line'/>
-          {Object.keys(this.props.activeFilters)
-            .filter((name) => this.props.filtersInFooter.includes(name))
-            .map((name) => {
-              return (
-                <div key={name}>
-                  {this.displayFilter(name)}
-                  <div className='line'/>
-                </div>
-              );
-            })}
+          {this.props.filtersInFooter.map((name) => (
+            <div key={name}>
+              {this.displayFilter(name)}
+              <div className='line'/>
+            </div>
+          ))}
         </div>
       </FooterStyles>
     );

src/reducers/controls.ts

@@ -247,7 +247,11 @@ const Controls = (state: ControlsState = getDefaultControlsState(), action): Con
     case types.APPLY_FILTER: {
       // values arrive as array
       const filters = Object.assign({}, state.filters, {});
-      filters[action.trait] = action.values;
+      if (action.values.length) { // set the filters to the new values
+        filters[action.trait] = action.values;
+      } else {                    // remove if no active+inactive filters
+        delete filters[action.trait]
+      }
       return Object.assign({}, state, {
         filters
       });
@@ -284,7 +288,6 @@ const Controls = (state: ControlsState = getDefaultControlsState(), action): Con
       return Object.assign({}, state, { legendOpen: action.value });
     case types.ADD_EXTRA_METADATA: {
       for (const colorBy of Object.keys(action.newColorings)) {
-        state.filters[colorBy] = [];
         state.coloringsPresentOnTree.add(colorBy);
       }
       let newState = Object.assign({}, state, { coloringsPresentOnTree: state.coloringsPresentOnTree, filters: state.filters });

src/util/getGenotype.js

@@ -102,6 +102,7 @@ export const decodeGenotypeFilters = (query) => {
       }
       return `${currentGene} ${x}`;
     })
+    .filter((value) => !!value)
     .map((value) => ({active: true, value})); // all URL filters _start_ active
 };
 

src/util/treeCountingHelpers.js

@@ -33,6 +33,35 @@ export const countTraitsAcrossTree = (nodes, traits, visibility, terminalOnly) =
   return counts;
 };
 
+
+/**
+ * Scan terminal nodes and gather all trait names with at least one valid value.
+ * Includes a hardcoded list of trait names we will ignore, as well as any trait
+ * which we know is continuous (via a colouring definition) because the
+ * filtering is not designed for these kinds of data (yet).
+ * @param {Array} nodes 
+ * @param {Object} colorings
+ * @returns {Array} list of trait names
+ */
+export const gatherTraitNames = (nodes, colorings) => {
+  const ignore = new Set([
+    'num_date',
+    ...Object.entries(colorings).filter(([_, info]) => info.type==='continuous').map(([name, _]) => name),
+  ])
+  const names = new Set();
+  for (const node of nodes) {
+    if (node.hasChildren) continue;
+    for (const traitName in node.node_attrs || {}) {
+      if (ignore.has(traitName)) continue;
+      if (names.has(traitName)) continue;
+      if (getTraitFromNode(node, traitName)) { // ensures validity
+        names.add(traitName);
+      }
+    }
+  }
+  return [...names]
+}
+
 /**
 * for each node, calculate the number of subtending tips which are visible
 * side effects: n.tipCount for each node