From d90486f35af46852f5c06b2d8828b3be54544687 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Thu, 27 Jun 2024 00:43:55 +0530 Subject: [PATCH] includes Count Aggregator Signed-off-by: Sarthak Aggarwal --- .../lucene/index/BaseStarTreeBuilder.java | 195 +++++++++--------- .../common/inject/util/Modules.java | 1 + .../aggregators/CountValueAggregator.java | 22 +- ...criptor.java => MetricAggregatorInfo.java} | 58 ++---- .../aggregators/SumValueAggregator.java | 22 +- .../startree/aggregators/ValueAggregator.java | 10 +- .../aggregators/ValueAggregatorFactory.java | 4 +- .../numerictype/StarTreeNumericType.java | 11 +- .../StarTreeNumericTypeConverters.java | 14 ++ ...uilder.java => OnHeapStarTreeBuilder.java} | 13 +- ...eTreeBuilder.java => StarTreeBuilder.java} | 4 +- .../StarTreeDocValuesIteratorAdapter.java | 14 +- .../startree/builder/StarTreesBuilder.java | 33 +-- .../startree/utils/StarTreeBuilderUtils.java | 37 +++- .../CountValueAggregatorTests.java | 56 +++++ ...ts.java => MetricAggregatorInfoTests.java} | 43 ++-- .../aggregators/SumValueAggregatorTests.java | 30 +-- .../ValueAggregatorFactoryTests.java | 4 +- .../builder/BaseStarTreeBuilderTests.java | 21 +- ...s.java => OnHeapStarTreeBuilderTests.java} | 177 +++++++++------- .../StarTreeValuesIteratorFactoryTests.java | 21 +- 21 files changed, 448 insertions(+), 342 deletions(-) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/{MetricAggregationDescriptor.java => MetricAggregatorInfo.java} (63%) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/{OnHeapSingleTreeBuilder.java => OnHeapStarTreeBuilder.java} (93%) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/{SingleTreeBuilder.java => StarTreeBuilder.java} (87%) create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java rename server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/{MetricAggregationDescriptorTests.java => MetricAggregatorInfoTests.java} (63%) rename server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/{OnHeapSingleTreeBuilderTests.java => OnHeapStarTreeBuilderTests.java} (79%) diff --git a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index 425101678ba58..1d851603369a3 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -9,21 +9,20 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.search.DocIdSetIterator; -import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregationDescriptor; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregator; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; -import org.opensearch.index.compositeindex.datacube.startree.builder.SingleTreeBuilder; +import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeBuilder; import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorAdapter; +import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeBuilderUtils; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.Mapper; @@ -41,10 +40,12 @@ import java.util.Set; /** - * Base class for star-tree builder + * Builder for star tree. Defines the algorithm to construct star-tree + * See {@link StarTreesBuilder} for information around the construction of star-trees based on star-tree fields + * * @opensearch.experimental */ -public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { +public abstract class BaseStarTreeBuilder implements StarTreeBuilder { private static final Logger logger = LogManager.getLogger(BaseStarTreeBuilder.class); @@ -52,7 +53,7 @@ public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { protected final Set skipStarNodeCreationForDimensions; - protected final List metricAggregationDescriptors; + protected final List metricAggregatorInfos; protected final int numMetrics; protected final int numDimensions; protected int numStarTreeDocs; @@ -65,24 +66,21 @@ public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { protected DocIdSetIterator[] dimensionReaders; protected Map fieldProducerMap; - protected DocValuesConsumer docValuesConsumer; private final StarTreeDocValuesIteratorAdapter starTreeDocValuesIteratorFactory; private final StarTreeField starTreeField; /** - * Constructor for base star-tree builder + * Reads all the configuration related to dimensions and metrics, builds a star-tree based on the different construction parameters. * - * @param starTreeField holds the configuration for the star tree - * @param fieldProducerMap helps return the doc values iterator for each type based on field name - * @param docValuesConsumer to consume the new aggregated metrics during flush - * @param state stores the segment write state - * @param mapperService helps to find the original type of the field + * @param starTreeField holds the configuration for the star tree + * @param fieldProducerMap helps return the doc values iterator for each type based on field name + * @param state stores the segment write state + * @param mapperService helps to find the original type of the field */ protected BaseStarTreeBuilder( StarTreeField starTreeField, Map fieldProducerMap, - DocValuesConsumer docValuesConsumer, SegmentWriteState state, MapperService mapperService ) throws IOException { @@ -91,7 +89,6 @@ protected BaseStarTreeBuilder( this.starTreeField = starTreeField; StarTreeFieldConfiguration starTreeFieldSpec = starTreeField.getStarTreeConfig(); - this.docValuesConsumer = docValuesConsumer; this.fieldProducerMap = fieldProducerMap; this.starTreeDocValuesIteratorFactory = new StarTreeDocValuesIteratorAdapter(); @@ -117,50 +114,53 @@ protected BaseStarTreeBuilder( ); } - this.metricAggregationDescriptors = generateMetricStatFieldPairs(mapperService, state); - this.numMetrics = metricAggregationDescriptors.size(); + this.metricAggregatorInfos = generateMetricAggregatorInfos(mapperService, state); + this.numMetrics = metricAggregatorInfos.size(); this.maxLeafDocuments = starTreeFieldSpec.maxLeafDocs(); } /** - * Generates the MetricStatFieldPairs for all the metrics on a field + * Generates the configuration required to perform aggregation for all the metrics on a field * - * @return list of metric stat mapped with respective fields + * @return list of MetricAggregatorInfo */ - public List generateMetricStatFieldPairs(MapperService mapperService, SegmentWriteState state) + public List generateMetricAggregatorInfos(MapperService mapperService, SegmentWriteState state) throws IOException { - List metricAggregationDescriptors = new ArrayList<>(); - IndexNumericFieldData.NumericType numericType; - DocIdSetIterator metricStatReader = null; + List metricAggregatorInfos = new ArrayList<>(); for (Metric metric : this.starTreeField.getMetrics()) { for (MetricStat metricType : metric.getMetrics()) { + IndexNumericFieldData.NumericType numericType; + DocIdSetIterator metricStatReader = null; Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metric.getField()); if (fieldMapper instanceof NumberFieldMapper) { numericType = ((NumberFieldMapper) fieldMapper).fieldType().numericType(); } else { - logger.error("metric mapper is not of type number field mapper"); - throw new IllegalStateException("metric mapper is not of type number field mapper"); + logger.error("unsupported mapper type"); + throw new IllegalStateException("unsupported mapper type"); } - // Ignore the column for COUNT aggregation function + + FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metric.getField()); + DocValuesType metricDocValuesType = metricFieldInfos.getDocValuesType(); if (metricType != MetricStat.COUNT) { - FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metric.getField()); - DocValuesType metricDocValuesType = metricFieldInfos.getDocValuesType(); + // Need not initialize the metric reader for COUNT metric type metricStatReader = starTreeDocValuesIteratorFactory.getDocValuesIterator( metricDocValuesType, metricFieldInfos, fieldProducerMap.get(metricFieldInfos.name) ); } - MetricAggregationDescriptor metricAggregationDescriptor = new MetricAggregationDescriptor( + + MetricAggregatorInfo metricAggregatorInfo = new MetricAggregatorInfo( metricType, metric.getField(), + starTreeField.getName(), numericType, metricStatReader ); - metricAggregationDescriptors.add(metricAggregationDescriptor); + metricAggregatorInfos.add(metricAggregatorInfo); } } - return metricAggregationDescriptors; + return metricAggregatorInfos; } /** @@ -185,7 +185,7 @@ public List generateMetricStatFieldPairs(MapperServ * * @return Star tree documents */ - public abstract List getStarTreeDocuments() throws IOException; + public abstract List getStarTreeDocuments(); /** * Returns the value of the dimension for the given dimension id and document in the star-tree. @@ -206,7 +206,7 @@ public List generateMetricStatFieldPairs(MapperServ public abstract Iterator sortMergeAndAggregateStarTreeDocument(int numDocs) throws IOException; /** - * Generates aggregated star-tree document for star-node. + * Generates aggregated star-tree documents for star-node. * * @param startDocId start document id (inclusive) in the star-tree * @param endDocId end document id (exclusive) in the star-tree @@ -218,6 +218,7 @@ public abstract Iterator generateStarTreeDocumentsForStarNode( /** * Returns the star-tree document from the segment + * * @throws IOException when we are unable to build a star tree document from the segment */ protected StarTreeDocument getSegmentStarTreeDocument() throws IOException { @@ -235,25 +236,18 @@ protected StarTreeDocument getSegmentStarTreeDocument() throws IOException { long[] getStarTreeDimensionsFromSegment() throws IOException { long[] dimensions = new long[numDimensions]; for (int i = 0; i < numDimensions; i++) { - try { - dimensionReaders[i].nextDoc(); - } catch (IOException e) { - logger.error("unable to iterate to next doc", e); - } catch (NullPointerException e) { - logger.error("dimension does not have an associated reader", e); - throw new IllegalStateException("dimension should have a valid associated reader", e); - } catch (Exception e) { - logger.error("unable to read the dimension values from the segment", e); - throw new IllegalStateException("unable to read the dimension values from the segment", e); - } + if (dimensionReaders[i] != null) { + try { + dimensionReaders[i].nextDoc(); + } catch (IOException e) { + logger.error("unable to iterate to next doc", e); + throw e; + } catch (Exception e) { + logger.error("unable to read the dimension values from the segment", e); + throw new IllegalStateException("unable to read the dimension values from the segment", e); + } - if (starTreeField.getDimensionsOrder().get(i) instanceof DateDimension) { - dimensions[i] = handleDateDimension( - starTreeField.getDimensionsOrder().get(i).getField(), - starTreeDocValuesIteratorFactory.getNextOrd(dimensionReaders[i]) - ); - } else { - dimensions[i] = starTreeDocValuesIteratorFactory.getNextOrd(dimensionReaders[i]); + dimensions[i] = starTreeDocValuesIteratorFactory.getNextValue(dimensionReaders[i]); } } return dimensions; @@ -268,21 +262,18 @@ long[] getStarTreeDimensionsFromSegment() throws IOException { private Object[] getStarTreeMetricsFromSegment() throws IOException { Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { - // Ignore the column for COUNT aggregation function - DocIdSetIterator metricStatReader = metricAggregationDescriptors.get(i).getMetricStatReader(); + DocIdSetIterator metricStatReader = metricAggregatorInfos.get(i).getMetricStatReader(); if (metricStatReader != null) { try { metricStatReader.nextDoc(); } catch (IOException e) { logger.error("unable to iterate to next doc", e); - } catch (NullPointerException e) { - logger.error("metric does not have an associated reader", e); - throw new IllegalStateException("metric should have a valid associated reader", e); + throw e; } catch (Exception e) { logger.error("unable to read the metric values from the segment", e); throw new IllegalStateException("unable to read the metric values from the segment", e); } - metrics[i] = starTreeDocValuesIteratorFactory.getNextOrd(metricStatReader); + metrics[i] = starTreeDocValuesIteratorFactory.getNextValue(metricStatReader); } } return metrics; @@ -305,40 +296,62 @@ protected StarTreeDocument reduceSegmentStarTreeDocuments( Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { - ValueAggregator metricValueAggregator = metricAggregationDescriptors.get(i).getValueAggregators(); - StarTreeNumericType starTreeNumericType = metricAggregationDescriptors.get(i).getStarTreeNumericType(); - metrics[i] = metricValueAggregator.getInitialAggregatedValue((Long) segmentDocument.metrics[i], starTreeNumericType); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - logger.error("Cannot parse initial segment doc value", e); - throw new IllegalStateException("Cannot parse initial segment doc value [" + segmentDocument.metrics[i] + "]"); + ValueAggregator metricValueAggregator = metricAggregatorInfos.get(i).getValueAggregators(); + StarTreeNumericType starTreeNumericType = metricAggregatorInfos.get(i).getAggregatedValueType(); + metrics[i] = metricValueAggregator.getInitialAggregatedValueForSegmentDocValue( + getLong(segmentDocument.metrics[i]), + starTreeNumericType + ); } catch (Exception e) { logger.error("Cannot parse initial segment doc value", e); - throw new RuntimeException("Cannot parse initial segment doc value [" + segmentDocument.metrics[i] + "]"); + throw new IllegalStateException("Cannot parse initial segment doc value [" + segmentDocument.metrics[i] + "]"); } } return new StarTreeDocument(dimensions, metrics); } else { for (int i = 0; i < numMetrics; i++) { try { - ValueAggregator metricValueAggregator = metricAggregationDescriptors.get(i).getValueAggregators(); - StarTreeNumericType starTreeNumericType = metricAggregationDescriptors.get(i).getStarTreeNumericType(); - aggregatedSegmentDocument.metrics[i] = metricValueAggregator.applySegmentRawValue( + ValueAggregator metricValueAggregator = metricAggregatorInfos.get(i).getValueAggregators(); + StarTreeNumericType starTreeNumericType = metricAggregatorInfos.get(i).getAggregatedValueType(); + aggregatedSegmentDocument.metrics[i] = metricValueAggregator.mergeAggregatedValueAndSegmentValue( aggregatedSegmentDocument.metrics[i], - (Long) segmentDocument.metrics[i], + getLong(segmentDocument.metrics[i]), starTreeNumericType ); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - logger.error("Cannot apply segment doc value for aggregation", e); - throw new IllegalStateException("Cannot apply segment doc value for aggregation [" + segmentDocument.metrics[i] + "]"); } catch (Exception e) { logger.error("Cannot apply segment doc value for aggregation", e); - throw new RuntimeException("Cannot apply segment doc value for aggregation [" + segmentDocument.metrics[i] + "]"); + throw new IllegalStateException("Cannot apply segment doc value for aggregation [" + segmentDocument.metrics[i] + "]"); } } return aggregatedSegmentDocument; } } + /** + * Safely converts the metric value of object type to long. + * + * @param metric value of the metric + * @return converted metric value to long + */ + private static long getLong(Object metric) { + + Long metricValue = null; + try { + if (metric instanceof Long) { + metricValue = (long) metric; + } else if (metric != null) { + metricValue = Long.valueOf(String.valueOf(metric)); + } + } catch (Exception e) { + throw new IllegalStateException("unable to cast segment metric", e); + } + + if (metricValue == null) { + throw new IllegalStateException("unable to cast segment metric"); + } + return metricValue; + } + /** * Merges a star-tree document into an aggregated star-tree document. * A new aggregated star-tree document is created if the aggregated document is null. @@ -354,30 +367,22 @@ public StarTreeDocument reduceStarTreeDocuments(StarTreeDocument aggregatedDocum Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { - metrics[i] = metricAggregationDescriptors.get(i).getValueAggregators().getAggregatedValue(starTreeDocument.metrics[i]); - } catch (IllegalArgumentException | NullPointerException e) { - logger.error("Cannot get aggregated value", e); - throw new IllegalArgumentException("Cannot get aggregated value[" + starTreeDocument.metrics[i] + "]"); + metrics[i] = metricAggregatorInfos.get(i).getValueAggregators().getInitialAggregatedValue(starTreeDocument.metrics[i]); } catch (Exception e) { logger.error("Cannot get value for aggregation", e); - throw new RuntimeException("Cannot get value for aggregation[" + starTreeDocument.metrics[i] + "]"); + throw new IllegalStateException("Cannot get value for aggregation[" + starTreeDocument.metrics[i] + "]"); } } return new StarTreeDocument(dimensions, metrics); } else { for (int i = 0; i < numMetrics; i++) { try { - aggregatedDocument.metrics[i] = metricAggregationDescriptors.get(i) + aggregatedDocument.metrics[i] = metricAggregatorInfos.get(i) .getValueAggregators() - .applyAggregatedValue(starTreeDocument.metrics[i], aggregatedDocument.metrics[i]); - } catch (IllegalArgumentException | NullPointerException e) { - logger.error("Cannot apply value to aggregated document for aggregation", e); - throw new IllegalArgumentException( - "Cannot apply value to aggregated document for aggregation[" + starTreeDocument.metrics[i] + "]" - ); + .mergeAggregatedValues(starTreeDocument.metrics[i], aggregatedDocument.metrics[i]); } catch (Exception e) { logger.error("Cannot apply value to aggregated document for aggregation", e); - throw new RuntimeException( + throw new IllegalStateException( "Cannot apply value to aggregated document for aggregation [" + starTreeDocument.metrics[i] + "]" ); } @@ -393,7 +398,7 @@ public StarTreeDocument reduceStarTreeDocuments(StarTreeDocument aggregatedDocum */ public void build() throws IOException { long startTime = System.currentTimeMillis(); - logger.debug("Star-tree build is a go with star tree field{}", starTreeField); + logger.debug("Star-tree build is a go with star tree field {}", starTreeField.getName()); Iterator starTreeDocumentIterator = sortMergeAndAggregateStarTreeDocument(totalSegmentDocs); logger.debug("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); @@ -465,9 +470,9 @@ private StarTreeBuilderUtils.TreeNode getNewNode() { /** * Implements the algorithm to construct a star-tree * - * @param node star-tree node - * @param startDocId start document id - * @param endDocId end document id + * @param node star-tree node + * @param startDocId start document id + * @param endDocId end document id * @throws IOException throws an exception if we are unable to construct the tree */ private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocId, int endDocId) throws IOException { @@ -498,8 +503,8 @@ private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocI /** * Constructs non star tree nodes * - * @param startDocId start document id - * @param endDocId end document id + * @param startDocId start document id (inclusive) + * @param endDocId end document id (exclusive) * @param dimensionId id of the dimension in the star tree * @return root node with non-star nodes constructed * @throws IOException throws an exception if we are unable to construct non-star nodes @@ -535,8 +540,8 @@ private Map constructNonStarNodes(int start /** * Constructs star tree nodes * - * @param startDocId start document id - * @param endDocId end document id + * @param startDocId start document id (inclusive) + * @param endDocId end document id (exclusive) * @param dimensionId id of the dimension in the star tree * @return root node with star nodes constructed * @throws IOException throws an exception if we are unable to construct non-star nodes @@ -576,7 +581,6 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node for (int i = node.startDocId; i < node.endDocId; i++) { aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, getStarTreeDocument(i)); } - assert aggregatedStarTreeDocument != null; if (null == aggregatedStarTreeDocument) { throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); } @@ -603,7 +607,6 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); } - assert aggregatedStarTreeDocument != null; if (null == aggregatedStarTreeDocument) { throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); } diff --git a/server/src/main/java/org/opensearch/common/inject/util/Modules.java b/server/src/main/java/org/opensearch/common/inject/util/Modules.java index 71eafc182c573..ae37cb3d29a68 100644 --- a/server/src/main/java/org/opensearch/common/inject/util/Modules.java +++ b/server/src/main/java/org/opensearch/common/inject/util/Modules.java @@ -208,6 +208,7 @@ public Void visit(Binding binding) { scopeInstancesInUse.put(scope, binding.getSource()); } } + return null; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java index b4837809b9706..193c66cfa033c 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java @@ -17,7 +17,7 @@ * @opensearch.experimental */ public class CountValueAggregator implements ValueAggregator { - public static final StarTreeNumericType STAR_TREE_NUMERIC_TYPE = StarTreeNumericType.DOUBLE; + public static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.DOUBLE; @Override public MetricStat getAggregationType() { @@ -25,27 +25,27 @@ public MetricStat getAggregationType() { } @Override - public StarTreeNumericType getStarTreeNumericType() { - return STAR_TREE_NUMERIC_TYPE; + public StarTreeNumericType getAggregatedValueType() { + return VALUE_AGGREGATOR_TYPE; } @Override - public Double getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + public Double getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { return 1.0; } @Override - public Double applySegmentRawValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + public Double mergeAggregatedValueAndSegmentValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { return value + 1; } @Override - public Double applyAggregatedValue(Double value, Double aggregatedValue) { + public Double mergeAggregatedValues(Double value, Double aggregatedValue) { return value + aggregatedValue; } @Override - public Double getAggregatedValue(Double value) { + public Double getInitialAggregatedValue(Double value) { return value; } @@ -58,8 +58,8 @@ public int getMaxAggregatedValueByteSize() { public Long toLongValue(Double value) { try { return NumericUtils.doubleToSortableLong(value); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - throw new IllegalArgumentException("Cannot convert " + value + " to sortable long", e); + } catch (Exception e) { + throw new IllegalStateException("Cannot convert " + value + " to sortable long", e); } } @@ -67,8 +67,8 @@ public Long toLongValue(Double value) { public Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType type) { try { return type.getDoubleValue(value); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - throw new IllegalArgumentException("Cannot convert " + value + " to sortable aggregation type", e); + } catch (Exception e) { + throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); } } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java similarity index 63% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java index f5e1055b6ef24..03e3de0831ed6 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java @@ -18,18 +18,11 @@ * Builds aggregation function and doc values field pair to support various aggregations * @opensearch.experimental */ -public class MetricAggregationDescriptor implements Comparable { - - public static final String DELIMITER = "__"; - public static final String STAR = "*"; - public static final MetricAggregationDescriptor COUNT_STAR = new MetricAggregationDescriptor( - MetricStat.COUNT, - STAR, - IndexNumericFieldData.NumericType.DOUBLE, - null - ); - - private final String metricStatName; +public class MetricAggregatorInfo implements Comparable { + + public static final String DELIMITER = "_"; + private final String metric; + private final String starFieldName; private final MetricStat metricStat; private final String field; private final ValueAggregator valueAggregators; @@ -37,11 +30,12 @@ public class MetricAggregationDescriptor implements Comparable o.field) - .thenComparing((MetricAggregationDescriptor o) -> o.metricStat) + public int compareTo(MetricAggregatorInfo other) { + return Comparator.comparing((MetricAggregatorInfo o) -> o.field) + .thenComparing((MetricAggregatorInfo o) -> o.metricStat) .compare(this, other); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java index d9e55caae8784..982860d16bfa3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java @@ -19,7 +19,7 @@ */ public class SumValueAggregator implements ValueAggregator { - public static final StarTreeNumericType STAR_TREE_NUMERIC_TYPE = StarTreeNumericType.DOUBLE; + public static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.DOUBLE; @Override public MetricStat getAggregationType() { @@ -27,17 +27,17 @@ public MetricStat getAggregationType() { } @Override - public StarTreeNumericType getStarTreeNumericType() { - return STAR_TREE_NUMERIC_TYPE; + public StarTreeNumericType getAggregatedValueType() { + return VALUE_AGGREGATOR_TYPE; } @Override - public Double getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + public Double getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { return starTreeNumericType.getDoubleValue(segmentDocValue); } @Override - public Double applySegmentRawValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + public Double mergeAggregatedValueAndSegmentValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { CompensatedSum kahanSummation = new CompensatedSum(0, 0); kahanSummation.add(value); kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue)); @@ -45,7 +45,7 @@ public Double applySegmentRawValue(Double value, Long segmentDocValue, StarTreeN } @Override - public Double applyAggregatedValue(Double value, Double aggregatedValue) { + public Double mergeAggregatedValues(Double value, Double aggregatedValue) { CompensatedSum kahanSummation = new CompensatedSum(0, 0); kahanSummation.add(value); kahanSummation.add(aggregatedValue); @@ -53,7 +53,7 @@ public Double applyAggregatedValue(Double value, Double aggregatedValue) { } @Override - public Double getAggregatedValue(Double value) { + public Double getInitialAggregatedValue(Double value) { return value; } @@ -66,8 +66,8 @@ public int getMaxAggregatedValueByteSize() { public Long toLongValue(Double value) { try { return NumericUtils.doubleToSortableLong(value); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - throw new IllegalArgumentException("Cannot convert " + value + " to sortable long", e); + } catch (Exception e) { + throw new IllegalStateException("Cannot convert " + value + " to sortable long", e); } } @@ -75,8 +75,8 @@ public Long toLongValue(Double value) { public Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType type) { try { return type.getDoubleValue(value); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - throw new IllegalArgumentException("Cannot convert " + value + " to sortable aggregation type", e); + } catch (Exception e) { + throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); } } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java index 397f497d0105e..3dd1f85845c17 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java @@ -25,27 +25,27 @@ public interface ValueAggregator { /** * Returns the data type of the aggregated value. */ - StarTreeNumericType getStarTreeNumericType(); + StarTreeNumericType getAggregatedValueType(); /** * Returns the initial aggregated value. */ - A getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType); + A getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType); /** * Applies a segment doc value to the current aggregated value. */ - A applySegmentRawValue(A value, Long segmentDocValue, StarTreeNumericType starTreeNumericType); + A mergeAggregatedValueAndSegmentValue(A value, Long segmentDocValue, StarTreeNumericType starTreeNumericType); /** * Applies an aggregated value to the current aggregated value. */ - A applyAggregatedValue(A value, A aggregatedValue); + A mergeAggregatedValues(A value, A aggregatedValue); /** * Clones an aggregated value. */ - A getAggregatedValue(A value); + A getInitialAggregatedValue(A value); /** * Returns the maximum size in bytes of the aggregated values seen so far. diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java index 2bde2f16f91f8..4ee0b0b5b13f8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java @@ -46,9 +46,9 @@ public static StarTreeNumericType getAggregatedValueType(MetricStat aggregationT switch (aggregationType) { // other metric types (count, min, max, avg) will be supported in the future case SUM: - return SumValueAggregator.STAR_TREE_NUMERIC_TYPE; + return SumValueAggregator.VALUE_AGGREGATOR_TYPE; case COUNT: - return CountValueAggregator.STAR_TREE_NUMERIC_TYPE; + return CountValueAggregator.VALUE_AGGREGATOR_TYPE; default: throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java index 28bfd82c69fdf..f09fb1567cb26 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java @@ -20,7 +20,10 @@ public enum StarTreeNumericType { HALF_FLOAT(IndexNumericFieldData.NumericType.HALF_FLOAT, StarTreeNumericTypeConverters::halfFloatPointToDouble), FLOAT(IndexNumericFieldData.NumericType.FLOAT, StarTreeNumericTypeConverters::floatPointToDouble), LONG(IndexNumericFieldData.NumericType.LONG, StarTreeNumericTypeConverters::longToDouble), - DOUBLE(IndexNumericFieldData.NumericType.DOUBLE, StarTreeNumericTypeConverters::sortableLongtoDouble); + DOUBLE(IndexNumericFieldData.NumericType.DOUBLE, StarTreeNumericTypeConverters::sortableLongtoDouble), + INT(IndexNumericFieldData.NumericType.INT, StarTreeNumericTypeConverters::intToDouble), + SHORT(IndexNumericFieldData.NumericType.SHORT, StarTreeNumericTypeConverters::shortToDouble), + UNSIGNED_LONG(IndexNumericFieldData.NumericType.UNSIGNED_LONG, StarTreeNumericTypeConverters::unsignedlongToDouble); final IndexNumericFieldData.NumericType numericType; final Function converter; @@ -44,6 +47,12 @@ public static StarTreeNumericType fromNumericType(IndexNumericFieldData.NumericT return StarTreeNumericType.LONG; case DOUBLE: return StarTreeNumericType.DOUBLE; + case INT: + return StarTreeNumericType.INT; + case SHORT: + return StarTreeNumericType.SHORT; + case UNSIGNED_LONG: + return StarTreeNumericType.UNSIGNED_LONG; default: throw new UnsupportedOperationException("Unknown numeric type [" + numericType + "]"); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java index f3fdd0d1162c3..b840d9436a26a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java @@ -10,6 +10,7 @@ import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.util.NumericUtils; +import org.opensearch.common.Numbers; import org.opensearch.common.annotation.ExperimentalApi; /** @@ -31,7 +32,20 @@ public static double longToDouble(Long value) { return (double) value; } + public static double intToDouble(Long value) { + return (double) value; + } + + public static double shortToDouble(Long value) { + return (double) value; + } + public static Double sortableLongtoDouble(Long value) { return NumericUtils.sortableLongToDouble(value); } + + public static double unsignedlongToDouble(Long value) { + return Numbers.unsignedLongToDouble(value); + } + } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java similarity index 93% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 0289a56ce5b5f..7e00509a74ac0 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -7,7 +7,6 @@ */ package org.opensearch.index.compositeindex.datacube.startree.builder; -import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BaseStarTreeBuilder; import org.apache.lucene.index.SegmentWriteState; @@ -28,28 +27,26 @@ * @opensearch.experimental */ @ExperimentalApi -public class OnHeapSingleTreeBuilder extends BaseStarTreeBuilder { +public class OnHeapStarTreeBuilder extends BaseStarTreeBuilder { private final List starTreeDocuments = new ArrayList<>(); /** - * Constructor for OnHeapSingleTreeBuilder + * Constructor for OnHeapStarTreeBuilder * * @param starTreeField star-tree field * @param fieldProducerMap helps with document values producer for a particular field - * @param docValuesConsumer document values consumer * @param segmentWriteState segment write state * @param mapperService helps with the numeric type of field * @throws IOException throws an exception we are unable to construct an onheap star-tree */ - public OnHeapSingleTreeBuilder( + public OnHeapStarTreeBuilder( StarTreeField starTreeField, Map fieldProducerMap, - DocValuesConsumer docValuesConsumer, SegmentWriteState segmentWriteState, MapperService mapperService ) throws IOException { - super(starTreeField, fieldProducerMap, docValuesConsumer, segmentWriteState, mapperService); + super(starTreeField, fieldProducerMap, segmentWriteState, mapperService); } @Override @@ -63,7 +60,7 @@ public StarTreeDocument getStarTreeDocument(int docId) throws IOException { } @Override - public List getStarTreeDocuments() throws IOException { + public List getStarTreeDocuments() { return starTreeDocuments; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java similarity index 87% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java index 44866caba938b..ef542a1c848f2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java @@ -18,11 +18,11 @@ * @opensearch.experimental */ @ExperimentalApi -public interface SingleTreeBuilder extends Closeable { +public interface StarTreeBuilder extends Closeable { /** * Builds the star tree based on star-tree field * @throws IOException when we are unable to build star-tree */ - void build() throws Exception; + void build() throws IOException; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java index e4cf2a45e131b..7c035ed6f21be 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.common.annotation.ExperimentalApi; @@ -31,8 +30,6 @@ public class StarTreeDocValuesIteratorAdapter { */ public DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException { switch (type) { - case SORTED_SET: - return producer.getSortedSet(field); case SORTED_NUMERIC: return producer.getSortedNumeric(field); default: @@ -41,12 +38,10 @@ public DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field } /** - * Returns the next ordinal for the given iterator + * Returns the next value for the given iterator */ - public long getNextOrd(DocIdSetIterator iterator) throws IOException { - if (iterator instanceof SortedSetDocValues) { - return ((SortedSetDocValues) iterator).nextOrd(); - } else if (iterator instanceof SortedNumericDocValues) { + public long getNextValue(DocIdSetIterator iterator) throws IOException { + if (iterator instanceof SortedNumericDocValues) { return ((SortedNumericDocValues) iterator).nextValue(); } else { throw new IllegalArgumentException("Unsupported Iterator: " + iterator.toString()); @@ -54,7 +49,8 @@ public long getNextOrd(DocIdSetIterator iterator) throws IOException { } /** - * Returns the doc id for the next document from the given iterator + * Moves to the next doc in the iterator + * Returns the doc id for the next document from the given iterator */ public int nextDoc(DocIdSetIterator iterator) throws IOException { return iterator.nextDoc(); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index 4d110609462a5..7b6d774698447 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.SegmentWriteState; import org.opensearch.common.annotation.ExperimentalApi; @@ -25,7 +24,7 @@ import java.util.Map; /** - * Builder to construct star tree based on multiple star tree configs + * Builder to construct star-trees based on multiple star-tree fields. * * @opensearch.experimental */ @@ -36,7 +35,6 @@ public class StarTreesBuilder implements Closeable { private final List starTreeFields; private final StarTreeFieldConfiguration.StarTreeBuildMode buildMode; - private final DocValuesConsumer docValuesConsumer; private final SegmentWriteState state; private final Map fieldProducerMap; private final MapperService mapperService; @@ -45,17 +43,15 @@ public StarTreesBuilder( List starTreeFields, StarTreeFieldConfiguration.StarTreeBuildMode buildMode, Map fieldProducerMap, - DocValuesConsumer docValuesConsumer, SegmentWriteState segmentWriteState, MapperService mapperService ) { this.starTreeFields = starTreeFields; if (starTreeFields == null || starTreeFields.isEmpty()) { - throw new IllegalArgumentException("Must provide star-tree builder configs"); + throw new IllegalArgumentException("Must provide star-tree field to build star trees"); } this.buildMode = buildMode; this.fieldProducerMap = fieldProducerMap; - this.docValuesConsumer = docValuesConsumer; this.state = segmentWriteState; this.mapperService = mapperService; } @@ -63,32 +59,22 @@ public StarTreesBuilder( /** * Builds the star-trees. */ - public void build() throws Exception { + public void build() throws IOException { long startTime = System.currentTimeMillis(); int numStarTrees = starTreeFields.size(); - logger.debug("Starting building {} star-trees with configs: {} using {} builder", numStarTrees, starTreeFields, buildMode); + logger.debug("Starting building {} star-trees with star-tree fields using {} builder", numStarTrees, buildMode); // Build all star-trees for (int i = 0; i < numStarTrees; i++) { StarTreeField starTreeField = starTreeFields.get(i); - try ( - SingleTreeBuilder singleTreeBuilder = getSingleTreeBuilder( - starTreeField, - buildMode, - fieldProducerMap, - docValuesConsumer, - state, - mapperService - ) - ) { - singleTreeBuilder.build(); + try (StarTreeBuilder starTreeBuilder = getSingleTreeBuilder(starTreeField, buildMode, fieldProducerMap, state, mapperService)) { + starTreeBuilder.build(); } } logger.debug( - "Took {} ms to building {} star-trees with configs: {} using {} builder", + "Took {} ms to building {} star-trees with star-tree fields using {} builder", System.currentTimeMillis() - startTime, numStarTrees, - starTreeFields, buildMode ); } @@ -98,17 +84,16 @@ public void close() throws IOException { } - private static SingleTreeBuilder getSingleTreeBuilder( + private static StarTreeBuilder getSingleTreeBuilder( StarTreeField starTreeField, StarTreeFieldConfiguration.StarTreeBuildMode buildMode, Map fieldProducerMap, - DocValuesConsumer docValuesConsumer, SegmentWriteState state, MapperService mapperService ) throws IOException { switch (buildMode) { case ON_HEAP: - return new OnHeapSingleTreeBuilder(starTreeField, fieldProducerMap, docValuesConsumer, state, mapperService); + return new OnHeapStarTreeBuilder(starTreeField, fieldProducerMap, state, mapperService); default: throw new IllegalArgumentException( String.format(Locale.ROOT, "No star tree implementation is available for [%s] build mode", buildMode) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java index 656196c7ffe0f..96e6681d40a76 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java @@ -22,15 +22,50 @@ private StarTreeBuilderUtils() {} public static final int ALL = -1; - /** Tree node representation */ + /** + * Represents a node in a tree data structure, specifically designed for a star-tree implementation. + * A star-tree node will represent both star and non-star nodes. + */ public static class TreeNode { + + /** + * The dimension id for the dimension (field) associated with this star-tree node. + */ public int dimensionId = ALL; + + /** + * The starting document id (inclusive) associated with this star-tree node. + */ public int startDocId = ALL; + + /** + * The ending document id (exclusive) associated with this star-tree node. + */ public int endDocId = ALL; + + /** + * The aggregated document id associated with this star-tree node. + */ public int aggregatedDocId = ALL; + + /** + * The child dimension identifier associated with this star-tree node. + */ public int childDimensionId = ALL; + + /** + * The value of the dimension associated with this star-tree node. + */ public long dimensionValue = ALL; + + /** + * A flag indicating whether this node is a star node (a node that represents an aggregation of all dimensions). + */ public boolean isStarNode = false; + + /** + * A map containing the child nodes of this star-tree node, keyed by their dimension id. + */ public Map children; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java new file mode 100644 index 0000000000000..e7d659ee177f3 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.apache.lucene.util.NumericUtils; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.test.OpenSearchTestCase; + +public class CountValueAggregatorTests extends OpenSearchTestCase { + private final CountValueAggregator aggregator = new CountValueAggregator(); + + public void testGetAggregationType() { + assertEquals(MetricStat.COUNT.getTypeName(), aggregator.getAggregationType().getTypeName()); + } + + public void testGetAggregatedValueType() { + assertEquals(CountValueAggregator.VALUE_AGGREGATOR_TYPE, aggregator.getAggregatedValueType()); + } + + public void testGetInitialAggregatedValueForSegmentDocValue() { + assertEquals(1.0, aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong(), StarTreeNumericType.LONG), 0.0); + } + + public void testMergeAggregatedValueAndSegmentValue() { + assertEquals(3.0, aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L, StarTreeNumericType.LONG), 0.0); + } + + public void testMergeAggregatedValues() { + assertEquals(5.0, aggregator.mergeAggregatedValues(2.0, 3.0), 0.0); + } + + public void testGetInitialAggregatedValue() { + assertEquals(3.0, aggregator.getInitialAggregatedValue(3.0), 0.0); + } + + public void testGetMaxAggregatedValueByteSize() { + assertEquals(Double.BYTES, aggregator.getMaxAggregatedValueByteSize()); + } + + public void testToLongValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(NumericUtils.doubleToSortableLong(3.0), aggregator.toLongValue(3.0), 0.0); + } + + public void testToStarTreeNumericTypeValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(NumericUtils.sortableLongToDouble(3L), aggregator.toStarTreeNumericTypeValue(3L, StarTreeNumericType.DOUBLE), 0.0); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java similarity index 63% rename from server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java rename to server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java index 98b79141df9eb..d08f637a3f0a9 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java @@ -12,12 +12,13 @@ import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.test.OpenSearchTestCase; -public class MetricAggregationDescriptorTests extends OpenSearchTestCase { +public class MetricAggregatorInfoTests extends OpenSearchTestCase { public void testConstructor() { - MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricAggregatorInfo pair = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); @@ -26,57 +27,66 @@ public void testConstructor() { } public void testCountStarConstructor() { - MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricAggregatorInfo pair = new MetricAggregatorInfo( MetricStat.COUNT, "anything", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); assertEquals(MetricStat.COUNT, pair.getMetricStat()); - assertEquals("*", pair.getField()); + assertEquals("anything", pair.getField()); } public void testToFieldName() { - MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricAggregatorInfo pair = new MetricAggregatorInfo( MetricStat.SUM, "column2", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - assertEquals("sum__column2", pair.toFieldName()); + assertEquals("star_tree_field_column2_sum", pair.toFieldName()); } public void testEquals() { - MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair1 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); assertEquals(pair1, pair2); assertNotEquals( pair1, - new MetricAggregationDescriptor(MetricStat.COUNT, "column1", IndexNumericFieldData.NumericType.DOUBLE, null) + new MetricAggregatorInfo(MetricStat.COUNT, "column1", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null) + ); + assertNotEquals( + pair1, + new MetricAggregatorInfo(MetricStat.SUM, "column2", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null) ); - assertNotEquals(pair1, new MetricAggregationDescriptor(MetricStat.SUM, "column2", IndexNumericFieldData.NumericType.DOUBLE, null)); } public void testHashCode() { - MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair1 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); @@ -84,21 +94,24 @@ public void testHashCode() { } public void testCompareTo() { - MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair1 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column2", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - MetricAggregationDescriptor pair3 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair3 = new MetricAggregatorInfo( MetricStat.COUNT, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java index 287deeead9d9d..99af5a7b153ee 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java @@ -21,26 +21,32 @@ public void testGetAggregationType() { assertEquals(MetricStat.SUM.getTypeName(), aggregator.getAggregationType().getTypeName()); } - public void testGetStarTreeNumericType() { - assertEquals(SumValueAggregator.STAR_TREE_NUMERIC_TYPE, aggregator.getStarTreeNumericType()); + public void testGetAggregatedValueType() { + assertEquals(SumValueAggregator.VALUE_AGGREGATOR_TYPE, aggregator.getAggregatedValueType()); } - public void testGetInitialAggregatedValue() { - assertEquals(1.0, aggregator.getInitialAggregatedValue(1L, StarTreeNumericType.LONG), 0.0); - assertThrows(NullPointerException.class, () -> aggregator.getInitialAggregatedValue(null, StarTreeNumericType.DOUBLE)); + public void testGetInitialAggregatedValueForSegmentDocValue() { + assertEquals(1.0, aggregator.getInitialAggregatedValueForSegmentDocValue(1L, StarTreeNumericType.LONG), 0.0); + assertThrows( + NullPointerException.class, + () -> aggregator.getInitialAggregatedValueForSegmentDocValue(null, StarTreeNumericType.DOUBLE) + ); } - public void testApplySegmentRawValue() { - assertEquals(5.0, aggregator.applySegmentRawValue(2.0, 3L, StarTreeNumericType.LONG), 0.0); - assertThrows(NullPointerException.class, () -> aggregator.applySegmentRawValue(3.14, null, StarTreeNumericType.DOUBLE)); + public void testMergeAggregatedValueAndSegmentValue() { + assertEquals(5.0, aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L, StarTreeNumericType.LONG), 0.0); + assertThrows( + NullPointerException.class, + () -> aggregator.mergeAggregatedValueAndSegmentValue(3.14, null, StarTreeNumericType.DOUBLE) + ); } - public void testApplyAggregatedValue() { - assertEquals(5.0, aggregator.applyAggregatedValue(2.0, 3.0), 0.0); + public void testMergeAggregatedValues() { + assertEquals(5.0, aggregator.mergeAggregatedValues(2.0, 3.0), 0.0); } - public void testGetAggregatedValue() { - assertEquals(3.14, aggregator.getAggregatedValue(3.14), 0.0); + public void testGetInitialAggregatedValue() { + assertEquals(3.14, aggregator.getInitialAggregatedValue(3.14), 0.0); } public void testGetMaxAggregatedValueByteSize() { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java index 21e2d1304f085..ce61ab839cc61 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java @@ -20,8 +20,8 @@ public void testGetValueAggregatorForSumType() { assertEquals(SumValueAggregator.class, aggregator.getClass()); } - public void testGetStarTreeNumericTypeForSumType() { + public void testGetAggregatedValueTypeForSumType() { StarTreeNumericType starTreeNumericType = ValueAggregatorFactory.getAggregatedValueType(MetricStat.SUM); - assertEquals(SumValueAggregator.STAR_TREE_NUMERIC_TYPE, starTreeNumericType); + assertEquals(SumValueAggregator.VALUE_AGGREGATOR_TYPE, starTreeNumericType); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 7a13ab7701912..eba396ffa175a 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -30,7 +30,7 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregationDescriptor; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; @@ -75,6 +75,7 @@ public class BaseStarTreeBuilderTests extends OpenSearchTestCase { private static Directory directory; private static FieldInfo[] fieldsInfo; private static SegmentWriteState state; + private static StarTreeField starTreeField; @BeforeClass public static void setup() throws IOException { @@ -82,7 +83,7 @@ public static void setup() throws IOException { dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); metrics = List.of(new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM))); - StarTreeField compositeField = new StarTreeField( + starTreeField = new StarTreeField( "test", dimensionsOrder, metrics, @@ -150,7 +151,7 @@ public static void setup() throws IOException { ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new BaseStarTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, state, mapperService) { + builder = new BaseStarTreeBuilder(starTreeField, fieldProducerMap, state, mapperService) { @Override public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException {} @@ -160,7 +161,7 @@ public StarTreeDocument getStarTreeDocument(int docId) throws IOException { } @Override - public List getStarTreeDocuments() throws IOException { + public List getStarTreeDocuments() { return List.of(); } @@ -182,13 +183,13 @@ public Iterator generateStarTreeDocumentsForStarNode(int start }; } - public void test_generateMetricStatFieldPairs() throws IOException { - List metricAggregationDescriptors = builder.generateMetricStatFieldPairs(mapperService, state); - List expectedMetricAggregationDescriptors = List.of( - new MetricAggregationDescriptor(MetricStat.SUM, "field2", IndexNumericFieldData.NumericType.DOUBLE, null), - new MetricAggregationDescriptor(MetricStat.SUM, "field4", IndexNumericFieldData.NumericType.DOUBLE, null) + public void test_generateMetricAggregatorInfos() throws IOException { + List metricAggregatorInfos = builder.generateMetricAggregatorInfos(mapperService, state); + List expectedMetricAggregatorInfos = List.of( + new MetricAggregatorInfo(MetricStat.SUM, "field2", starTreeField.getName(), IndexNumericFieldData.NumericType.DOUBLE, null), + new MetricAggregatorInfo(MetricStat.SUM, "field4", starTreeField.getName(), IndexNumericFieldData.NumericType.DOUBLE, null) ); - assertEquals(metricAggregationDescriptors, expectedMetricAggregationDescriptors); + assertEquals(metricAggregatorInfos, expectedMetricAggregatorInfos); } public void test_reduceStarTreeDocuments() { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java similarity index 79% rename from server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java rename to server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java index 73955080c36ae..e6277dfa7ec5a 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java @@ -8,7 +8,6 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; -import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.index.DocValuesType; @@ -53,9 +52,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class OnHeapSingleTreeBuilderTests extends OpenSearchTestCase { +public class OnHeapStarTreeBuilderTests extends OpenSearchTestCase { - private OnHeapSingleTreeBuilder builder; + private OnHeapStarTreeBuilder builder; private MapperService mapperService; private List dimensionsOrder; private List fields = List.of( @@ -75,15 +74,17 @@ public class OnHeapSingleTreeBuilderTests extends OpenSearchTestCase { private FieldInfo[] fieldsInfo; private StarTreeField compositeField; private Map fieldProducerMap; - private DocValuesConsumer docValuesConsumer; private SegmentWriteState writeState; @Before public void setup() throws IOException { dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); - metrics = List.of(new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM))); + metrics = List.of( + new Metric("field2", List.of(MetricStat.SUM)), + new Metric("field4", List.of(MetricStat.SUM)), + new Metric("field6", List.of(MetricStat.COUNT)) + ); - docValuesConsumer = mock(DocValuesConsumer.class); DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); compositeField = new StarTreeField( @@ -143,15 +144,17 @@ public void setup() throws IOException { .build(new Mapper.BuilderContext(settings, new ContentPath())); NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.DOUBLE, false, true) .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2), + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3), Collections.emptyList(), Collections.emptyList(), 0, null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService); } public void test_sortMergeAndAggregateStarTreeDocument() throws IOException { @@ -159,15 +162,15 @@ public void test_sortMergeAndAggregateStarTreeDocument() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0 }) + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0, 3.0 }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -175,7 +178,8 @@ public void test_sortMergeAndAggregateStarTreeDocument() throws IOException { for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -192,6 +196,7 @@ public void test_sortMergeAndAggregateStarTreeDocument() throws IOException { assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); numOfAggregatedDocuments++; } @@ -205,12 +210,12 @@ public void test_sortMergeAndAggregateStarTreeDocument_nullMetric() throws IOExc int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, null }); - StarTreeDocument expectedStarTreeDocument = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, null, randomDouble() }); + StarTreeDocument expectedStarTreeDocument = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { @@ -218,7 +223,8 @@ public void test_sortMergeAndAggregateStarTreeDocument_nullMetric() throws IOExc Long metric2 = starTreeDocuments[i].metrics[1] != null ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) : null; - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Object[] { metric1, metric2 }); + Long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Object[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -246,15 +252,15 @@ public void test_sortMergeAndAggregateStarTreeDocument_longMaxAndLongMinDimensio int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 14.0, 12.0 }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 11.0, 16.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 35.0, 34.0 }) + new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 35.0, 34.0, 3.0 }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -262,7 +268,8 @@ public void test_sortMergeAndAggregateStarTreeDocument_longMaxAndLongMinDimensio for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -279,6 +286,7 @@ public void test_sortMergeAndAggregateStarTreeDocument_longMaxAndLongMinDimensio assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); numOfAggregatedDocuments++; } @@ -292,15 +300,15 @@ public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, Double.MIN_VALUE }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, Double.MIN_VALUE, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE + 9, 14.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, Double.MIN_VALUE + 22 }) + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE + 9, 14.0, 2.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, Double.MIN_VALUE + 22, 3.0 }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -308,7 +316,8 @@ public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -325,6 +334,7 @@ public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); numOfAggregatedDocuments++; } @@ -343,38 +353,40 @@ public void test_build_halfFloatMetrics() throws IOException { .build(new Mapper.BuilderContext(settings, new ContentPath())); NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2), + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3), Collections.emptyList(), Collections.emptyList(), 0, null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService); int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; starTreeDocuments[0] = new StarTreeDocument( new long[] { 2, 4, 3, 4 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf1", 12), new HalfFloatPoint("hf6", 10) } + new HalfFloatPoint[] { new HalfFloatPoint("hf1", 12), new HalfFloatPoint("hf6", 10), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[1] = new StarTreeDocument( new long[] { 3, 4, 2, 1 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf2", 10), new HalfFloatPoint("hf7", 6) } + new HalfFloatPoint[] { new HalfFloatPoint("hf2", 10), new HalfFloatPoint("hf7", 6), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[2] = new StarTreeDocument( new long[] { 3, 4, 2, 1 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf3", 14), new HalfFloatPoint("hf8", 12) } + new HalfFloatPoint[] { new HalfFloatPoint("hf3", 14), new HalfFloatPoint("hf8", 12), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[3] = new StarTreeDocument( new long[] { 2, 4, 3, 4 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf4", 9), new HalfFloatPoint("hf9", 4) } + new HalfFloatPoint[] { new HalfFloatPoint("hf4", 9), new HalfFloatPoint("hf9", 4), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[4] = new StarTreeDocument( new long[] { 3, 4, 2, 1 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf5", 11), new HalfFloatPoint("hf10", 16) } + new HalfFloatPoint[] { new HalfFloatPoint("hf5", 11), new HalfFloatPoint("hf10", 16), new HalfFloatPoint("field6", 10) } ); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; @@ -385,7 +397,10 @@ public void test_build_halfFloatMetrics() throws IOException { long metric2 = HalfFloatPoint.halfFloatToSortableShort( ((HalfFloatPoint) starTreeDocuments[i].metrics[1]).numericValue().floatValue() ); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = HalfFloatPoint.halfFloatToSortableShort( + ((HalfFloatPoint) starTreeDocuments[i].metrics[2]).numericValue().floatValue() + ); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -410,30 +425,33 @@ public void test_build_floatMetrics() throws IOException { .build(new Mapper.BuilderContext(settings, new ContentPath())); NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.FLOAT, false, true) .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2), + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3), Collections.emptyList(), Collections.emptyList(), 0, null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService); int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 12.0F, 10.0F }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 10.0F, 6.0F }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 14.0F, 12.0F }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 9.0F, 4.0F }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 11.0F, 16.0F }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 12.0F, 10.0F, randomFloat() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 10.0F, 6.0F, randomFloat() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 14.0F, 12.0F, randomFloat() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 9.0F, 4.0F, randomFloat() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 11.0F, 16.0F, randomFloat() }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -458,30 +476,33 @@ public void test_build_longMetrics() throws IOException { .build(new Mapper.BuilderContext(settings, new ContentPath())); NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.LONG, false, true) .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.LONG, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2), + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3), Collections.emptyList(), Collections.emptyList(), 0, null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService); int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 12L, 10L }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 10L, 6L }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 14L, 12L }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 9L, 4L }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 11L, 16L }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 12L, 10L, randomLong() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 10L, 6L, randomLong() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 14L, 12L, randomLong() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 9L, 4L, randomLong() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 11L, 16L, randomLong() }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = (Long) starTreeDocuments[i].metrics[0]; long metric2 = (Long) starTreeDocuments[i].metrics[1]; - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = (Long) starTreeDocuments[i].metrics[2]; + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -498,14 +519,14 @@ public void test_build_longMetrics() throws IOException { private static Iterator getExpectedStarTreeDocumentIterator() { List expectedStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0 }), - new StarTreeDocument(new long[] { -1, 4, 2, 1 }, new Double[] { 35.0, 34.0 }), - new StarTreeDocument(new long[] { -1, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { -1, 4, -1, 1 }, new Double[] { 35.0, 34.0 }), - new StarTreeDocument(new long[] { -1, 4, -1, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { -1, 4, -1, -1 }, new Double[] { 56.0, 48.0 }), - new StarTreeDocument(new long[] { -1, -1, -1, -1 }, new Double[] { 56.0, 48.0 }) + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0, 3.0 }), + new StarTreeDocument(new long[] { -1, 4, 2, 1 }, new Double[] { 35.0, 34.0, 3.0 }), + new StarTreeDocument(new long[] { -1, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, 1 }, new Double[] { 35.0, 34.0, 3.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, -1 }, new Double[] { 56.0, 48.0, 5.0 }), + new StarTreeDocument(new long[] { -1, -1, -1, -1 }, new Double[] { 56.0, 48.0, 5.0 }) ); return expectedStarTreeDocuments.iterator(); } @@ -515,17 +536,18 @@ public void test_build() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0, randomDouble() }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -555,6 +577,7 @@ private static void assertStarTreeDocuments( assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java index f5d53afb8f774..59cadfa737786 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java @@ -13,7 +13,6 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.DocIdSetIterator; @@ -56,14 +55,6 @@ public static void setup() { ); } - public void testCreateIterator_SortedSet() throws IOException { - DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); - SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); - when(producer.getSortedSet(mockFieldInfo)).thenReturn(iterator); - DocIdSetIterator result = factory.getDocValuesIterator(DocValuesType.SORTED_SET, mockFieldInfo, producer); - assertEquals(iterator.getClass(), result.getClass()); - } - public void testCreateIterator_SortedNumeric() throws IOException { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); @@ -80,26 +71,18 @@ public void testCreateIterator_UnsupportedType() { assertEquals("Unsupported DocValuesType: BINARY", exception.getMessage()); } - public void testGetNextValue_SortedSet() throws IOException { - SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); - when(iterator.nextOrd()).thenReturn(42L); - - long result = factory.getNextOrd(iterator); - assertEquals(42L, result); - } - public void testGetNextValue_SortedNumeric() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(iterator.nextValue()).thenReturn(123L); - long result = factory.getNextOrd(iterator); + long result = factory.getNextValue(iterator); assertEquals(123L, result); } public void testGetNextValue_UnsupportedIterator() { DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { factory.getNextOrd(iterator); }); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { factory.getNextValue(iterator); }); assertEquals("Unsupported Iterator: " + iterator.toString(), exception.getMessage()); }