Skip to content

Commit

Permalink
debugging
Browse files Browse the repository at this point in the history
Signed-off-by: Bharathwaj G <[email protected]>
  • Loading branch information
bharath-techie committed Feb 9, 2024
1 parent c0c8a28 commit 8ff255d
Show file tree
Hide file tree
Showing 7 changed files with 29 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ public abstract class BaseSingleTreeBuilder {
}

// TODO : Removing hardcoding
_maxLeafRecords = 100; // builderConfig.getMaxLeafRecords();
_maxLeafRecords = 1; // builderConfig.getMaxLeafRecords();
}

private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocId, int endDocId) throws IOException {
Expand Down Expand Up @@ -317,6 +317,7 @@ private void createDocValuesIndices(DocValuesConsumer docValuesConsumer) throws

for (int docId = 0; docId < _numDocs; docId++) {
Record record = getStarTreeRecord(docId);
//logger.info("Record during doc values indices : {} ", record.toString());
for (int i = 0; i < record._dimensions.length; i++) {
long val = record._dimensions[i];
pendingDimArr[i].add(val);
Expand Down Expand Up @@ -381,6 +382,7 @@ private void appendToStarTree(Record record) throws IOException {
// if(star) {
// System.out.println("======Overall sum =====" + (long) record._metrics[0]);
// }
//logger.info("Record : {}", record.toString());
appendRecord(record);
_numDocs++;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ private static void writeNodes(IndexOutput output, TreeNode rootNode) throws IOE
}

private static void writeNode(IndexOutput output, TreeNode node, int firstChildId, int lastChildId) throws IOException {
//logger.info("Builder util : dim id : {} , dim value : {}", node._dimensionId, node._dimensionValue);
output.writeInt(node._dimensionId);
output.writeLong(node._dimensionValue);
output.writeInt(node._startDocId);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
*/
package org.opensearch.index.codec.freshstartree.codec;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.lucene90.IndexedDISI;
Expand Down Expand Up @@ -47,6 +49,8 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.opensearch.index.codec.freshstartree.query.StarTreeFilter;


/**
* Created a copy to initialize producer without field info stored in state which is the case for
Expand All @@ -55,6 +59,9 @@
* We don't create aggregated doc value fields in traditional add/update doc workflow where fieldInfo gets populated
*/
public class Lucene90DocValuesProducerCopy extends DocValuesProducer {

private static final Logger logger = LogManager.getLogger(Lucene90DocValuesProducerCopy.class);

private final Map<String, NumericEntry> numerics;
private final Map<String, SortedNumericEntry> sortedNumerics;
private final IndexInput data;
Expand Down Expand Up @@ -211,6 +218,7 @@ public FieldInfo[] getFieldInfoArr() {

private void readFields(IndexInput meta, FieldInfo[] infos) throws IOException {
for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) {
logger.info("fieldNumber : {}" ,fieldNumber);
FieldInfo info = infos[fieldNumber];
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
*/
package org.opensearch.index.codec.freshstartree.codec;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.BinaryDocValues;
Expand All @@ -34,9 +36,13 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.opensearch.index.codec.freshstartree.query.StarTreeFilter;


/** Custom star tree doc values reader */
public class StarTreeDocValuesReader extends DocValuesProducer {
private static final Logger logger = LogManager.getLogger(StarTreeDocValuesReader.class);

private DocValuesProducer delegate;

private IndexInput data;
Expand Down Expand Up @@ -71,6 +77,7 @@ public NumericDocValues getNumeric(FieldInfo field) throws IOException {
public StarTreeAggregatedValues getAggregatedDocValues() throws IOException {
// starTree.printTree(new HashMap<>());
List<String> dimensionsSplitOrder = starTree.getDimensionNames();
//logger.info("Dimension order {}", dimensionsSplitOrder);
for (int i = 0; i < dimensionsSplitOrder.size(); i++) {
dimensionValues.put(dimensionsSplitOrder.get(i), valuesProducer.getNumeric(dimensionsSplitOrder.get(i) + "_dim"));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
*/
package org.opensearch.index.codec.freshstartree.node;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;

Expand All @@ -26,9 +28,12 @@
import java.util.List;
import java.util.Map;
import java.util.StringJoiner;
import org.opensearch.index.codec.freshstartree.builder.OffHeapBufferedSingleTreeBuilder;


/** Off heap implementation of star tree. */
public class OffHeapStarTree implements StarTree {
private static final Logger logger = LogManager.getLogger(OffHeapStarTree.class);
public static final long MAGIC_MARKER = 0xBADDA55B00DAD00DL;
public static final int VERSION = 1;
private final OffHeapStarTreeNode _root;
Expand Down Expand Up @@ -108,6 +113,7 @@ private void printTreeHelper(Map<String, Map<String, String>> dictionaryMap, Off
.toString();

stringBuilder.append(formattedOutput);
logger.info(stringBuilder.toString());

if (!node.isLeaf()) {
Iterator<OffHeapStarTreeNode> childrenIterator = node.getChildrenIterator();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,9 @@ private StarTreeResult traverseStarTree() throws IOException {
// If all predicate columns and group-by columns are matched, we can use aggregated document
if (remainingPredicateColumns.isEmpty() && remainingGroupByColumns.isEmpty()) {
adder = docsWithField.grow(1);
adder.add(starTreeNode.getAggregatedDocId());
int aggrDocId = starTreeNode.getAggregatedDocId();
logger.info("Adding aggregated doc id {}", aggrDocId);
adder.add(aggrDocId);
continue;
}

Expand All @@ -178,6 +180,7 @@ private StarTreeResult traverseStarTree() throws IOException {
if (starTreeNode.isLeaf()) {
for (long i = starTreeNode.getStartDocId(); i < starTreeNode.getEndDocId(); i++) {
adder = docsWithField.grow(1);
logger.info("Adding normal doc id {}", i);
adder.add((int) i);
}
continue;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,6 @@ public void collect(int doc, long bucket) throws IOException {
for (String field : fieldCols) {
fieldColToDocValuesMap.put(field, aggrVals.dimensionValues.get(field));
}

NumericDocValues dv = aggrVals.metricValues.get("status_sum");
if (dv.advanceExact(doc)) {

Expand Down

0 comments on commit 8ff255d

Please sign in to comment.