Skip to content

Commit

Permalink
ALS-4947: Allow multiple sets of anyRecordOf filters
Browse files Browse the repository at this point in the history
  • Loading branch information
@ramari16
ramari16 committed Aug 25, 2023
1 parent 5912b01 commit 5bdb474
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 18 deletions.
19 changes: 17 additions & 2 deletions client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public Query(Query query) {
private List<String> fields = new ArrayList<>();
private List<String> requiredFields = new ArrayList<>();
private List<String> anyRecordOf = new ArrayList<>();
private List<List<String>> anyRecordOfMulti = new ArrayList<>();
private Map<String, DoubleFilter> numericFilters = new HashMap<>();
private Map<String, String[]> categoryFilters = new HashMap<>();
private List<VariantInfoFilter> variantInfoFilters = new ArrayList<>();
Expand All @@ -62,6 +63,14 @@ public List<String> getRequiredFields() {
public List<String> getAnyRecordOf() {
return anyRecordOf;
}
public List<List<String>> getAnyRecordOfMulti() {
return anyRecordOfMulti;
}
public List<List<String>> getAllAnyRecordOf() {
List<List<String>> anyRecordOfMultiCopy = new ArrayList<>(anyRecordOfMulti);
anyRecordOfMultiCopy.add(anyRecordOf);
return anyRecordOfMultiCopy;
}

public Map<String, DoubleFilter> getNumericFilters() {
return numericFilters;
Expand Down Expand Up @@ -98,6 +107,9 @@ public void setRequiredFields(Collection<String> requiredFields) {
public void setAnyRecordOf(Collection<String> anyRecordOf) {
this.anyRecordOf = anyRecordOf != null ? new ArrayList<>(anyRecordOf) : new ArrayList<>();
}
public void setAnyRecordOfMulti(Collection<List<String>> anyRecordOfMulti) {
this.anyRecordOfMulti = anyRecordOfMulti != null ? new ArrayList<>(anyRecordOfMulti) : new ArrayList<>();
}

public void setNumericFilters(Map<String, DoubleFilter> numericFilters) {
this.numericFilters = numericFilters != null ? new HashMap<>(numericFilters) : new HashMap<>();
Expand Down Expand Up @@ -191,7 +203,10 @@ public String toString() {
writePartFormat("Numeric filters", numericFilters, builder);
writePartFormat("Category filters", categoryFilters, builder);
writePartFormat("Variant Info filters", variantInfoFilters, builder, false);
writePartFormat("Any-Record-Of filters", anyRecordOf, builder, true);

List<List<String>> allAnyRecordOf = new ArrayList<>(anyRecordOfMulti);
allAnyRecordOf.add(anyRecordOf);
writePartFormat("Any-Record-Of filters", allAnyRecordOf, builder, true);

return builder.toString();
}
Expand Down Expand Up @@ -234,7 +249,7 @@ private static void showTopLevelValues(Collection varList, StringBuilder builder

Integer count = countMap.get(firstLevel);
if(count == null) {
count = new Integer(1);
count = 1;
} else {
count = count + 1;
}
Expand Down
37 changes: 21 additions & 16 deletions ...essing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import java.util.Map.Entry;
import java.util.concurrent.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.GZIPInputStream;

import com.google.common.util.concurrent.UncheckedExecutionException;
Expand Down Expand Up @@ -195,24 +196,27 @@ protected Set<Integer> applyBooleanLogic(List<Set<Integer>> filteredIdSets) {
* @return
*/
protected List<Set<Integer>> idSetsForEachFilter(Query query) {
ArrayList<Set<Integer>> filteredIdSets = new ArrayList<Set<Integer>>();
final ArrayList<Set<Integer>> filteredIdSets = new ArrayList<>();

try {
addIdSetsForAnyRecordOf(query, filteredIdSets);
query.getAllAnyRecordOf().forEach(anyRecordOfFilterList -> {
addIdSetsForAnyRecordOf(anyRecordOfFilterList, filteredIdSets);
});
addIdSetsForRequiredFields(query, filteredIdSets);
addIdSetsForNumericFilters(query, filteredIdSets);
addIdSetsForCategoryFilters(query, filteredIdSets);
} catch (InvalidCacheLoadException e) {
log.warn("Invalid query supplied: " + e.getLocalizedMessage());
filteredIdSets.add(new HashSet<Integer>()); // if an invalid path is supplied, no patients should match.
filteredIdSets.add(new HashSet<>()); // if an invalid path is supplied, no patients should match.
}

//AND logic to make sure all patients match each filter
if(filteredIdSets.size()>1) {
filteredIdSets = new ArrayList<Set<Integer>>(List.of(applyBooleanLogic(filteredIdSets)));
List<Set<Integer>> processedFilteredIdSets = new ArrayList<>(List.of(applyBooleanLogic(filteredIdSets)));
return addIdSetsForVariantInfoFilters(query, processedFilteredIdSets);
} else {
return addIdSetsForVariantInfoFilters(query, filteredIdSets);
}

return addIdSetsForVariantInfoFilters(query, filteredIdSets);
}

/**
Expand Down Expand Up @@ -248,7 +252,7 @@ public TreeSet<Integer> getPatientSubsetForQuery(Query query) {
private void addIdSetsForRequiredFields(Query query, ArrayList<Set<Integer>> filteredIdSets) {
if(!query.getRequiredFields().isEmpty()) {
VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<>();
filteredIdSets.addAll(query.getRequiredFields().parallelStream().map(path->{
filteredIdSets.addAll(query.getRequiredFields().stream().map(path->{
if(VariantUtils.pathIsVariantSpec(path)) {
TreeSet<Integer> patientsInScope = new TreeSet<>();
addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1","1/1"}, path, patientsInScope, bucketCache);
Expand All @@ -260,18 +264,19 @@ private void addIdSetsForRequiredFields(Query query, ArrayList<Set<Integer>> fil
}
}

private void addIdSetsForAnyRecordOf(Query query, ArrayList<Set<Integer>> filteredIdSets) {
if(!query.getAnyRecordOf().isEmpty()) {
private void addIdSetsForAnyRecordOf(List<String> anyRecordOfFilters, ArrayList<Set<Integer>> filteredIdSets) {
if(!anyRecordOfFilters.isEmpty()) {
VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<>();
filteredIdSets.addAll(query.getAnyRecordOf().parallelStream().map(path->{
if(VariantUtils.pathIsVariantSpec(path)) {
Set<Integer> anyRecordOfPatientSet = anyRecordOfFilters.stream().flatMap(path -> {
if (VariantUtils.pathIsVariantSpec(path)) {
TreeSet<Integer> patientsInScope = new TreeSet<>();
addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1","1/1"}, path, patientsInScope, bucketCache);
return patientsInScope;
addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1", "1/1"}, path, patientsInScope, bucketCache);
return patientsInScope.stream();
} else {
return new TreeSet<Integer>(getCube(path).keyBasedIndex());
return (Stream<Integer>) getCube(path).keyBasedIndex().stream();
}
}).collect(Collectors.toSet()));
}).collect(Collectors.toSet());
filteredIdSets.add(anyRecordOfPatientSet);
}
}

Expand All @@ -286,7 +291,7 @@ private void addIdSetsForNumericFilters(Query query, ArrayList<Set<Integer>> fil
private void addIdSetsForCategoryFilters(Query query, ArrayList<Set<Integer>> filteredIdSets) {
if(!query.getCategoryFilters().isEmpty()) {
VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<VariantMasks>();
Set<Set<Integer>> idsThatMatchFilters = (Set<Set<Integer>>)query.getCategoryFilters().entrySet().parallelStream().map(entry->{
Set<Set<Integer>> idsThatMatchFilters = (Set<Set<Integer>>)query.getCategoryFilters().entrySet().stream().map(entry->{
Set<Integer> ids = new TreeSet<Integer>();
if(VariantUtils.pathIsVariantSpec(entry.getKey())) {
addIdSetsForVariantSpecCategoryFilters(entry.getValue(), entry.getKey(), ids, bucketCache);
Expand Down

0 comments on commit 5bdb474

Please sign in to comment.