From 97a490285b844e3df54cd8e459db5823f7114f11 Mon Sep 17 00:00:00 2001 From: Nathaniel Date: Fri, 8 Dec 2023 00:31:58 -0700 Subject: [PATCH] fix indexes --- .github/workflows/library.yaml | 2 +- build.gradle | 11 +- .../library/database/index/IndexWrapper.java | 6 +- .../library/database/index/TreeIndex.java | 44 +++++--- .../library/database/index/TrieIndex.java | 20 +--- .../library/database/index/TreeIndexTest.java | 56 ++++++++++ .../library/database/index/TrieIndexTest.java | 101 ++++++++++++++++++ .../database/index/helpers/TestClass.java | 20 ++++ 8 files changed, 220 insertions(+), 40 deletions(-) create mode 100644 src/test/java/com/nucleodb/library/database/index/TreeIndexTest.java create mode 100644 src/test/java/com/nucleodb/library/database/index/TrieIndexTest.java create mode 100644 src/test/java/com/nucleodb/library/database/index/helpers/TestClass.java diff --git a/.github/workflows/library.yaml b/.github/workflows/library.yaml index 3f0da36..5770f6f 100644 --- a/.github/workflows/library.yaml +++ b/.github/workflows/library.yaml @@ -18,7 +18,7 @@ jobs: - name: Publish package uses: gradle/gradle-build-action@749f47bda3e44aa060e82d7b3ef7e40d953bd629 with: - arguments: "build publish" + arguments: "test build publish" env: SYNLOAD_REPO_USER: ${{ secrets.SYNLOAD_REPO_USER}} SYNLOAD_REPO_PASS: ${{ secrets.SYNLOAD_REPO_PASS}} \ No newline at end of file diff --git a/build.gradle b/build.gradle index c805306..98487dc 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ plugins { } group = 'com.nucleodb' -version = '1.13.8' +version = '1.13.9' repositories { mavenCentral() @@ -26,6 +26,11 @@ java { compileJava { options.release = 19 } + +test { + useJUnitPlatform() +} + dependencies { api group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.2.3' api group: 'org.apache.kafka', name: 'kafka-clients', version: '2.1.0' @@ -40,6 +45,10 @@ dependencies { api 'org.dizitart:nitrite:3.4.3' api group: 'com.fasterxml.jackson.datatype', name:'jackson-datatype-jsr310', version:'2.15.2' api 'com.github.jsqlparser:jsqlparser:4.6' + dependencies { + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' + testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1' + } } task sourceJar(type: Jar) { diff --git a/src/main/java/com/nucleodb/library/database/index/IndexWrapper.java b/src/main/java/com/nucleodb/library/database/index/IndexWrapper.java index 7f00564..dcea3a8 100644 --- a/src/main/java/com/nucleodb/library/database/index/IndexWrapper.java +++ b/src/main/java/com/nucleodb/library/database/index/IndexWrapper.java @@ -5,6 +5,7 @@ import com.google.common.collect.Sets; import com.nucleodb.library.database.index.trie.Entry; import com.nucleodb.library.database.tables.table.DataEntry; +import com.nucleodb.library.database.utils.Serializer; import com.nucleodb.library.database.utils.exceptions.InvalidIndexTypeException; import org.jetbrains.annotations.NotNull; @@ -33,7 +34,7 @@ public List getIndexValue(T object) throws JsonProcessingException { }else{ obj = object; } - return getValues(Queues.newConcurrentLinkedQueue(Arrays.asList(this.indexedKeyStr.split("\\."))), object); + return getValues(Queues.newConcurrentLinkedQueue(Arrays.asList(this.indexedKeyStr.split("\\."))), obj); /*String json = dataEntry.getReference().toString(); try (JsonReader reader = Json.createReader(new StringReader(json))) { System.out.println(json); @@ -68,15 +69,12 @@ public List getValues(Queue pointer, Object start) throws JsonPr } try { String name = pointer.poll(); - //Serializer.log(name); - //Serializer.log(current.getClass().getName()); current = new PropertyDescriptor(name, current.getClass()).getReadMethod().invoke(current); } catch (IntrospectionException | InvocationTargetException | IllegalAccessException e) { //e.printStackTrace(); return new LinkedList<>(); } if(current instanceof Collection){ - //System.out.println(current.getClass().getName()); return ((Collection) current).stream().map(c-> { try { return getValues(Queues.newConcurrentLinkedQueue(pointer), c); diff --git a/src/main/java/com/nucleodb/library/database/index/TreeIndex.java b/src/main/java/com/nucleodb/library/database/index/TreeIndex.java index 88e1d4e..82233db 100644 --- a/src/main/java/com/nucleodb/library/database/index/TreeIndex.java +++ b/src/main/java/com/nucleodb/library/database/index/TreeIndex.java @@ -1,6 +1,8 @@ package com.nucleodb.library.database.index; import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.nucleodb.library.database.utils.TreeSetExt; import org.jetbrains.annotations.NotNull; @@ -21,7 +23,7 @@ public TreeIndex() { private boolean unique; - private TreeMap>> reverseMap = new TreeMap<>(); + private TreeMap> reverseMap = new TreeMap<>(); private TreeMap> index = new TreeMap<>(); @@ -37,24 +39,25 @@ public void add(T dataEntry) throws JsonProcessingException { List values = getIndexValue(dataEntry); //System.out.println(Serializer.getObjectMapper().getOm().writeValueAsString(values)); values.forEach(val->{ - Set entries; + synchronized (reverseMap) { + List rMap = reverseMap.get(dataEntry); + if (rMap == null) { + rMap = Lists.newLinkedList(); + reverseMap.put(dataEntry, rMap); + } + rMap.add(val); + } synchronized (index) { + Set entries; entries = index.get(val); if (entries == null) { entries = new TreeSetExt<>(); index.put(val, entries); + } + entries.add(dataEntry); } - entries.add(dataEntry); - Set> rMap; - synchronized (reverseMap) { - rMap = reverseMap.get(dataEntry); - if (rMap == null) { - rMap = new TreeSetExt<>(); - reverseMap.put(dataEntry, rMap); - } - } - rMap.add(entries); + //System.out.println("Add, "+ this.getIndexedKey() + " = " +val); }); @@ -63,9 +66,18 @@ public void add(T dataEntry) throws JsonProcessingException { @Override public void delete(T dataEntry) { //System.out.println("Delete "+dataEntry); - Set> i = reverseMap.get(dataEntry); + List i = reverseMap.get(dataEntry); if(i!=null) - i.forEach(c -> c.remove(dataEntry)); + i.forEach(c -> { + Set ts = index.get(c); + if(ts!=null) { + ts.remove(dataEntry); + if(ts.isEmpty()){ + index.remove(c); + } + } + }); + reverseMap.remove(dataEntry); //System.out.println(reverseMap.get(dataEntry)); } @@ -140,11 +152,11 @@ public Set greaterThanEqual(Object searchObj) { } - public TreeMap>> getReverseMap() { + public TreeMap> getReverseMap() { return reverseMap; } - public void setReverseMap(TreeMap>> reverseMap) { + public void setReverseMap(TreeMap> reverseMap) { this.reverseMap = reverseMap; } diff --git a/src/main/java/com/nucleodb/library/database/index/TrieIndex.java b/src/main/java/com/nucleodb/library/database/index/TrieIndex.java index c5b0d2d..587c076 100644 --- a/src/main/java/com/nucleodb/library/database/index/TrieIndex.java +++ b/src/main/java/com/nucleodb/library/database/index/TrieIndex.java @@ -55,8 +55,6 @@ public void add(T obj) throws JsonProcessingException { for (Object o : getIndexValue(obj)) { if(o instanceof String){ insert(entry, (String) o); - }else{ - Serializer.log(o.getClass().getName()); } } } @@ -165,21 +163,7 @@ public Set searchData(String findString) { return search(findString).stream().map(e->(T)e.getData()).collect(Collectors.toSet()); } public List search(String findString) { - PrimitiveIterator.OfInt iterator = findString.chars().iterator(); - Node tmp = this.root; - while (iterator.hasNext()) { - int character = iterator.next(); - Node nodeTmp = getNodeFromIntBuffer(tmp.getNodes(), character); - if (tmp.getNodes() != null && nodeTmp == null) { - tmp = null; - break; - } - tmp = nodeTmp; - } - if (tmp == null) { - return Lists.newLinkedList(); - } - return tmp.getEntries().stream().collect(Collectors.toList()); + return partial(findString); } public Set partialData(String findString) { @@ -197,7 +181,7 @@ public List partial(String findString) { Node tmp = (Node) r; for (int i = 1; i < charArray.length; i++) { int character = charArray[i]; - Node nodeTmp = getNodeFromIntBuffer(tmp.getNodes(), character); + Node nodeTmp = tmp.getNodes().get(character); if (nodeTmp == null) { tmp = null; break; diff --git a/src/test/java/com/nucleodb/library/database/index/TreeIndexTest.java b/src/test/java/com/nucleodb/library/database/index/TreeIndexTest.java new file mode 100644 index 0000000..21ee1d8 --- /dev/null +++ b/src/test/java/com/nucleodb/library/database/index/TreeIndexTest.java @@ -0,0 +1,56 @@ +package com.nucleodb.library.database.index; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.nucleodb.library.database.index.helpers.TestClass; +import com.nucleodb.library.database.tables.table.DataEntry; +import com.nucleodb.library.database.utils.Serializer; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class TreeIndexTest{ + + @Test + void add() throws JsonProcessingException { + TreeIndex index = new TreeIndex("name"); + index.add(new DataEntry(new TestClass("taco test"))); + assertEquals(1, index.contains("taco").size()); + } + + @Test + void delete() throws JsonProcessingException { + DataEntry tacoTest = new DataEntry(new TestClass("taco test")); + TreeIndex index = new TreeIndex("name"); + index.add(tacoTest); + assertEquals(1, index.contains("taco").size()); + assertEquals(1, index.getIndex().size()); + assertEquals(1, index.getReverseMap().size()); + index.delete(tacoTest); + assertEquals(0, index.contains("taco").size()); + assertEquals(0, index.getIndex().size()); + assertEquals(0, index.getReverseMap().size()); + } + + @Test + void modify() throws JsonProcessingException { + DataEntry tacoTest = new DataEntry(new TestClass("taco test")); + TreeIndex index = new TreeIndex("name"); + index.add(tacoTest); + assertEquals(1, index.contains("taco").size()); + assertEquals(1, index.getIndex().size()); + assertEquals(1, index.getReverseMap().size()); + tacoTest.getData().setName("just test"); + index.modify(tacoTest); + assertEquals(1, index.contains("just").size()); + assertEquals(1, index.getIndex().size()); + assertEquals(1, index.getReverseMap().size()); + } + + @Test + void get() { + } + + @Test + void contains() { + } +} \ No newline at end of file diff --git a/src/test/java/com/nucleodb/library/database/index/TrieIndexTest.java b/src/test/java/com/nucleodb/library/database/index/TrieIndexTest.java new file mode 100644 index 0000000..4406d9f --- /dev/null +++ b/src/test/java/com/nucleodb/library/database/index/TrieIndexTest.java @@ -0,0 +1,101 @@ +package com.nucleodb.library.database.index; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.nucleodb.library.database.index.helpers.TestClass; +import com.nucleodb.library.database.index.trie.Node; +import com.nucleodb.library.database.tables.table.DataEntry; +import com.nucleodb.library.database.utils.Serializer; +import org.junit.jupiter.api.Test; + +import java.util.Stack; + +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + + +class TrieIndexTest{ + + @Test + void add() throws JsonProcessingException { + TrieIndex index = new TrieIndex("name"); + index.add(new DataEntry(new TestClass("taco test"))); + assertEquals(1, index.search("taco").size()); + } + + @Test + void modify() throws JsonProcessingException { + DataEntry tacoTest = new DataEntry(new TestClass("taco test")); + TrieIndex index = new TrieIndex("name"); + index.add(tacoTest); + tacoTest.getData().setName("burrito"); + index.modify(tacoTest); + assertEquals(0, index.search("taco").size()); + Stack nodes = new Stack(); + nodes.add(index.root); + int x = 0; + while(!nodes.isEmpty()){ + Node n = nodes.pop(); + nodes.addAll(n.getNodes().values()); + x++; + } + assertEquals(8, x); // characters + 1(root) + assertEquals(1, index.search("burrito").size()); + } + + @Test + void get() throws JsonProcessingException { + String name = "taco test"; + DataEntry tacoTest = new DataEntry(new TestClass(name)); + TrieIndex index = new TrieIndex("name"); + index.add(tacoTest); + assertEquals(1, index.get(name).size()); + Stack nodes = new Stack<>(); + nodes.add(index.root); + int x = 0; + while(!nodes.isEmpty()){ + Node n = nodes.pop(); + nodes.addAll(n.getNodes().values()); + x++; + } + assertEquals(name.length()+1, x); // characters + 1(root) + } + + @Test + void contains() throws JsonProcessingException { + String name = "vestibulum rhoncus est pellentesque elit ullamcorper dignissim cras tincidunt lobortis feugiat vivamus at augue eget arcu dictum varius duis at consectetur lorem donec massa sapien"; + DataEntry tacoTest = new DataEntry(new TestClass(name)); + TrieIndex index = new TrieIndex("name"); + index.add(tacoTest); + assertEquals(1, index.search("rho").size()); + Stack nodes = new Stack<>(); + nodes.add(index.root); + int x = 0; + while(!nodes.isEmpty()){ + Node n = nodes.pop(); + nodes.addAll(n.getNodes().values()); + x++; + } + assertEquals(name.length()+1, x); // characters + 1(root) + assertEquals(1, index.entries.size()); + } + + @Test + void delete() throws JsonProcessingException { + String name = "taco test"; + DataEntry tacoTest = new DataEntry(new TestClass(name)); + TrieIndex index = new TrieIndex("name"); + index.add(tacoTest); + assertEquals(1, index.get(name).size()); + index.delete(tacoTest); + Stack nodes = new Stack<>(); + nodes.add(index.root); + int x = 0; + while(!nodes.isEmpty()){ + Node n = nodes.pop(); + nodes.addAll(n.getNodes().values()); + x++; + } + assertEquals(1, x); // characters + 1(root) + assertEquals(0, index.entries.size()); + } +} \ No newline at end of file diff --git a/src/test/java/com/nucleodb/library/database/index/helpers/TestClass.java b/src/test/java/com/nucleodb/library/database/index/helpers/TestClass.java new file mode 100644 index 0000000..43ea858 --- /dev/null +++ b/src/test/java/com/nucleodb/library/database/index/helpers/TestClass.java @@ -0,0 +1,20 @@ +package com.nucleodb.library.database.index.helpers; + +public class TestClass { + String name = "test"; + + public TestClass() { + } + + public TestClass(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } \ No newline at end of file