fix indexes

NucleoTeam · Dec 8, 2023 · 97a4902 · 97a4902
1 parent 91fa668
commit 97a4902
Show file tree

Hide file tree

Showing 8 changed files with 220 additions and 40 deletions.
diff --git a/.github/workflows/library.yaml b/.github/workflows/library.yaml
@@ -18,7 +18,7 @@ jobs:
       - name: Publish package
         uses: gradle/gradle-build-action@749f47bda3e44aa060e82d7b3ef7e40d953bd629
         with:
-          arguments: "build publish"
+          arguments: "test build publish"
         env:
           SYNLOAD_REPO_USER: ${{ secrets.SYNLOAD_REPO_USER}}
           SYNLOAD_REPO_PASS: ${{ secrets.SYNLOAD_REPO_PASS}}
diff --git a/build.gradle b/build.gradle
@@ -5,7 +5,7 @@ plugins {
 }
 
 group = 'com.nucleodb'
-version = '1.13.8'
+version = '1.13.9'
 
 repositories {
     mavenCentral()
@@ -26,6 +26,11 @@ java {
 compileJava {
     options.release = 19
 }
+
+test {
+    useJUnitPlatform()
+}
+
 dependencies {
     api group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.2.3'
     api group: 'org.apache.kafka', name: 'kafka-clients', version: '2.1.0'
@@ -40,6 +45,10 @@ dependencies {
     api 'org.dizitart:nitrite:3.4.3'
     api group: 'com.fasterxml.jackson.datatype', name:'jackson-datatype-jsr310', version:'2.15.2'
     api 'com.github.jsqlparser:jsqlparser:4.6'
+    dependencies {
+        testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1'
+        testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1'
+    }
 }
 
 task sourceJar(type: Jar) {

diff --git a/src/main/java/com/nucleodb/library/database/index/IndexWrapper.java b/src/main/java/com/nucleodb/library/database/index/IndexWrapper.java
@@ -5,6 +5,7 @@
 import com.google.common.collect.Sets;
 import com.nucleodb.library.database.index.trie.Entry;
 import com.nucleodb.library.database.tables.table.DataEntry;
+import com.nucleodb.library.database.utils.Serializer;
 import com.nucleodb.library.database.utils.exceptions.InvalidIndexTypeException;
 import org.jetbrains.annotations.NotNull;
 
@@ -33,7 +34,7 @@ public List<Object> getIndexValue(T object) throws JsonProcessingException {
     }else{
       obj = object;
     }
-    return getValues(Queues.newConcurrentLinkedQueue(Arrays.asList(this.indexedKeyStr.split("\\."))), object);
+    return getValues(Queues.newConcurrentLinkedQueue(Arrays.asList(this.indexedKeyStr.split("\\."))), obj);
     /*String json = dataEntry.getReference().toString();
     try (JsonReader reader = Json.createReader(new StringReader(json))) {
       System.out.println(json);
@@ -68,15 +69,12 @@ public List<Object> getValues(Queue<String> pointer, Object start) throws JsonPr
     }
     try {
       String name = pointer.poll();
-      //Serializer.log(name);
-      //Serializer.log(current.getClass().getName());
       current = new PropertyDescriptor(name, current.getClass()).getReadMethod().invoke(current);
     } catch (IntrospectionException | InvocationTargetException | IllegalAccessException e) {
       //e.printStackTrace();
       return new LinkedList<>();
     }
     if(current instanceof Collection){
-      //System.out.println(current.getClass().getName());
       return ((Collection<?>) current).stream().map(c-> {
         try {
           return getValues(Queues.newConcurrentLinkedQueue(pointer), c);

diff --git a/src/main/java/com/nucleodb/library/database/index/TreeIndex.java b/src/main/java/com/nucleodb/library/database/index/TreeIndex.java
@@ -1,6 +1,8 @@
 package com.nucleodb.library.database.index;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 import com.nucleodb.library.database.utils.TreeSetExt;
 import org.jetbrains.annotations.NotNull;
 
@@ -21,7 +23,7 @@ public TreeIndex() {
 
 
   private boolean unique;
-  private TreeMap<T, Set<Set<T>>> reverseMap = new TreeMap<>();
+  private TreeMap<T, List<Object>> reverseMap = new TreeMap<>();
   private TreeMap<Object, Set<T>> index = new TreeMap<>();
 
 
@@ -37,24 +39,25 @@ public void add(T dataEntry) throws JsonProcessingException {
     List<Object> values = getIndexValue(dataEntry);
     //System.out.println(Serializer.getObjectMapper().getOm().writeValueAsString(values));
     values.forEach(val->{
-      Set<T> entries;
+      synchronized (reverseMap) {
+        List<Object> rMap = reverseMap.get(dataEntry);
+        if (rMap == null) {
+          rMap = Lists.newLinkedList();
+          reverseMap.put(dataEntry, rMap);
+        }
+        rMap.add(val);
+      }
       synchronized (index) {
+        Set<T> entries;
         entries = index.get(val);
         if (entries == null) {
           entries = new TreeSetExt<>();
           index.put(val, entries);
+
         }
+        entries.add(dataEntry);
       }
-      entries.add(dataEntry);
-      Set<Set<T>> rMap;
-      synchronized (reverseMap) {
-        rMap = reverseMap.get(dataEntry);
-        if (rMap == null) {
-          rMap = new TreeSetExt<>();
-          reverseMap.put(dataEntry, rMap);
-        }
-      }
-      rMap.add(entries);
+
       //System.out.println("Add, "+ this.getIndexedKey() + " = " +val);
 
     });
@@ -63,9 +66,18 @@ public void add(T dataEntry) throws JsonProcessingException {
   @Override
   public void delete(T dataEntry) {
     //System.out.println("Delete "+dataEntry);
-    Set<Set<T>> i = reverseMap.get(dataEntry);
+    List<Object> i = reverseMap.get(dataEntry);
     if(i!=null)
-      i.forEach(c -> c.remove(dataEntry));
+      i.forEach(c -> {
+        Set<T> ts = index.get(c);
+        if(ts!=null) {
+          ts.remove(dataEntry);
+          if(ts.isEmpty()){
+            index.remove(c);
+          }
+        }
+      });
+
     reverseMap.remove(dataEntry);
     //System.out.println(reverseMap.get(dataEntry));
   }
@@ -140,11 +152,11 @@ public Set<T> greaterThanEqual(Object searchObj) {
   }
 
 
-  public TreeMap<T, Set<Set<T>>> getReverseMap() {
+  public TreeMap<T, List<Object>> getReverseMap() {
     return reverseMap;
   }
 
-  public void setReverseMap(TreeMap<T, Set<Set<T>>> reverseMap) {
+  public void setReverseMap(TreeMap<T, List<Object>> reverseMap) {
     this.reverseMap = reverseMap;
   }
 

diff --git a/src/main/java/com/nucleodb/library/database/index/TrieIndex.java b/src/main/java/com/nucleodb/library/database/index/TrieIndex.java
@@ -55,8 +55,6 @@ public void add(T obj) throws JsonProcessingException {
     for (Object o : getIndexValue(obj)) {
       if(o instanceof String){
         insert(entry, (String) o);
-      }else{
-        Serializer.log(o.getClass().getName());
       }
     }
   }
@@ -165,21 +163,7 @@ public Set<T> searchData(String findString) {
     return search(findString).stream().map(e->(T)e.getData()).collect(Collectors.toSet());
   }
   public List<Entry> search(String findString) {
-    PrimitiveIterator.OfInt iterator = findString.chars().iterator();
-    Node tmp = this.root;
-    while (iterator.hasNext()) {
-      int character = iterator.next();
-      Node nodeTmp = getNodeFromIntBuffer(tmp.getNodes(), character);
-      if (tmp.getNodes() != null && nodeTmp == null) {
-        tmp = null;
-        break;
-      }
-      tmp = nodeTmp;
-    }
-    if (tmp == null) {
-      return Lists.newLinkedList();
-    }
-    return tmp.getEntries().stream().collect(Collectors.toList());
+    return partial(findString);
   }
 
   public Set<T> partialData(String findString) {
@@ -197,7 +181,7 @@ public List<Entry> partial(String findString) {
       Node tmp = (Node) r;
       for (int i = 1; i < charArray.length; i++) {
         int character = charArray[i];
-        Node nodeTmp = getNodeFromIntBuffer(tmp.getNodes(), character);
+        Node nodeTmp = tmp.getNodes().get(character);
         if (nodeTmp == null) {
           tmp = null;
           break;

diff --git a/src/test/java/com/nucleodb/library/database/index/TreeIndexTest.java b/src/test/java/com/nucleodb/library/database/index/TreeIndexTest.java
@@ -0,0 +1,56 @@
+package com.nucleodb.library.database.index;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.nucleodb.library.database.index.helpers.TestClass;
+import com.nucleodb.library.database.tables.table.DataEntry;
+import com.nucleodb.library.database.utils.Serializer;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class TreeIndexTest{
+
+  @Test
+  void add() throws JsonProcessingException {
+    TreeIndex index = new TreeIndex<TestClass>("name");
+    index.add(new DataEntry(new TestClass("taco test")));
+    assertEquals(1, index.contains("taco").size());
+  }
+
+  @Test
+  void delete() throws JsonProcessingException {
+    DataEntry tacoTest = new DataEntry(new TestClass("taco test"));
+    TreeIndex index = new TreeIndex<TestClass>("name");
+    index.add(tacoTest);
+    assertEquals(1, index.contains("taco").size());
+    assertEquals(1, index.getIndex().size());
+    assertEquals(1, index.getReverseMap().size());
+    index.delete(tacoTest);
+    assertEquals(0, index.contains("taco").size());
+    assertEquals(0, index.getIndex().size());
+    assertEquals(0, index.getReverseMap().size());
+  }
+
+  @Test
+  void modify() throws JsonProcessingException {
+    DataEntry<TestClass> tacoTest = new DataEntry(new TestClass("taco test"));
+    TreeIndex index = new TreeIndex<TestClass>("name");
+    index.add(tacoTest);
+    assertEquals(1, index.contains("taco").size());
+    assertEquals(1, index.getIndex().size());
+    assertEquals(1, index.getReverseMap().size());
+    tacoTest.getData().setName("just test");
+    index.modify(tacoTest);
+    assertEquals(1, index.contains("just").size());
+    assertEquals(1, index.getIndex().size());
+    assertEquals(1, index.getReverseMap().size());
+  }
+
+  @Test
+  void get() {
+  }
+
+  @Test
+  void contains() {
+  }
+}
diff --git a/src/test/java/com/nucleodb/library/database/index/TrieIndexTest.java b/src/test/java/com/nucleodb/library/database/index/TrieIndexTest.java
@@ -0,0 +1,101 @@
+package com.nucleodb.library.database.index;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.nucleodb.library.database.index.helpers.TestClass;
+import com.nucleodb.library.database.index.trie.Node;
+import com.nucleodb.library.database.tables.table.DataEntry;
+import com.nucleodb.library.database.utils.Serializer;
+import org.junit.jupiter.api.Test;
+
+import java.util.Stack;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.*;
+
+
+class TrieIndexTest{
+
+  @Test
+  void add() throws JsonProcessingException {
+    TrieIndex index = new TrieIndex<TestClass>("name");
+    index.add(new DataEntry(new TestClass("taco test")));
+    assertEquals(1, index.search("taco").size());
+  }
+
+  @Test
+  void modify() throws JsonProcessingException {
+    DataEntry<TestClass> tacoTest = new DataEntry(new TestClass("taco test"));
+    TrieIndex index = new TrieIndex<TestClass>("name");
+    index.add(tacoTest);
+    tacoTest.getData().setName("burrito");
+    index.modify(tacoTest);
+    assertEquals(0, index.search("taco").size());
+    Stack<Node> nodes = new Stack<Node>();
+    nodes.add(index.root);
+    int x = 0;
+    while(!nodes.isEmpty()){
+      Node n = nodes.pop();
+      nodes.addAll(n.getNodes().values());
+      x++;
+    }
+    assertEquals(8, x); // characters + 1(root)
+    assertEquals(1, index.search("burrito").size());
+  }
+
+  @Test
+  void get() throws JsonProcessingException {
+    String name = "taco test";
+    DataEntry<TestClass> tacoTest = new DataEntry(new TestClass(name));
+    TrieIndex index = new TrieIndex<TestClass>("name");
+    index.add(tacoTest);
+    assertEquals(1, index.get(name).size());
+    Stack<Node> nodes = new Stack<>();
+    nodes.add(index.root);
+    int x = 0;
+    while(!nodes.isEmpty()){
+      Node n = nodes.pop();
+      nodes.addAll(n.getNodes().values());
+      x++;
+    }
+    assertEquals(name.length()+1, x); // characters + 1(root)
+  }
+
+  @Test
+  void contains() throws JsonProcessingException {
+    String name = "vestibulum rhoncus est pellentesque elit ullamcorper dignissim cras tincidunt lobortis feugiat vivamus at augue eget arcu dictum varius duis at consectetur lorem donec massa sapien";
+    DataEntry<TestClass> tacoTest = new DataEntry(new TestClass(name));
+    TrieIndex index = new TrieIndex<TestClass>("name");
+    index.add(tacoTest);
+    assertEquals(1, index.search("rho").size());
+    Stack<Node> nodes = new Stack<>();
+    nodes.add(index.root);
+    int x = 0;
+    while(!nodes.isEmpty()){
+      Node n = nodes.pop();
+      nodes.addAll(n.getNodes().values());
+      x++;
+    }
+    assertEquals(name.length()+1, x); // characters + 1(root)
+    assertEquals(1, index.entries.size());
+  }
+
+  @Test
+  void delete() throws JsonProcessingException {
+    String name = "taco test";
+    DataEntry<TestClass> tacoTest = new DataEntry(new TestClass(name));
+    TrieIndex index = new TrieIndex<TestClass>("name");
+    index.add(tacoTest);
+    assertEquals(1, index.get(name).size());
+    index.delete(tacoTest);
+    Stack<Node> nodes = new Stack<>();
+    nodes.add(index.root);
+    int x = 0;
+    while(!nodes.isEmpty()){
+      Node n = nodes.pop();
+      nodes.addAll(n.getNodes().values());
+      x++;
+    }
+    assertEquals(1, x); // characters + 1(root)
+    assertEquals(0, index.entries.size());
+  }
+}
diff --git a/src/test/java/com/nucleodb/library/database/index/helpers/TestClass.java b/src/test/java/com/nucleodb/library/database/index/helpers/TestClass.java
@@ -0,0 +1,20 @@
+package com.nucleodb.library.database.index.helpers;
+
+public class TestClass {
+    String name = "test";
+
+    public TestClass() {
+    }
+
+    public TestClass(String name) {
+      this.name = name;
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    public void setName(String name) {
+      this.name = name;
+    }
+  }