From 3c653fb39048e18f4c321ab11c3170e5c09e5582 Mon Sep 17 00:00:00 2001 From: silvanheller Date: Sat, 7 May 2022 13:32:11 +0200 Subject: [PATCH 1/2] adding test showcasing how export fails for nan --- .../cottontail/cli/ExportImportCommandTest.kt | 34 +++++++++++++++---- .../vitrivr/cottontail/test/GrpcTestUtils.kt | 33 +++++++++++------- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/cottontaildb-cli/src/test/kotlin/org/vitrivr/cottontail/cli/ExportImportCommandTest.kt b/cottontaildb-cli/src/test/kotlin/org/vitrivr/cottontail/cli/ExportImportCommandTest.kt index 693a0d8ef..8b2cc00f1 100644 --- a/cottontaildb-cli/src/test/kotlin/org/vitrivr/cottontail/cli/ExportImportCommandTest.kt +++ b/cottontaildb-cli/src/test/kotlin/org/vitrivr/cottontail/cli/ExportImportCommandTest.kt @@ -9,8 +9,10 @@ import org.vitrivr.cottontail.cli.entity.TruncateEntityCommand import org.vitrivr.cottontail.core.database.Name import org.vitrivr.cottontail.data.Format import org.vitrivr.cottontail.test.AbstractClientTest +import org.vitrivr.cottontail.test.GrpcTestUtils import org.vitrivr.cottontail.test.GrpcTestUtils.countElements import org.vitrivr.cottontail.test.TestConstants +import org.vitrivr.cottontail.test.TestConstants.TEST_ENTITY_NAME import java.nio.file.Path import kotlin.io.path.Path import kotlin.io.path.deleteIfExists @@ -39,13 +41,7 @@ class ExportImportCommandTest : AbstractClientTest() { fun exportCreatesFile() { formats.forEach { format -> TestConstants.ALL_ENTITY_NAMES.forEach { name -> - val path = exportFolder() - path.toFile().mkdirs() - val exported = exportFile(format, name) - exported.deleteIfExists() - DumpEntityCommand.dumpEntity(name, path, format, this.client) - assert(exported.toFile().exists()) - assert(exported.toFile().totalSpace > 1) + exportEntity(format, name) } } } @@ -67,4 +63,28 @@ class ExportImportCommandTest : AbstractClientTest() { } } } + + @Test + fun exportNan() { + GrpcTestUtils.insertIntoTestEntity(client, double = Double.NaN) + GrpcTestUtils.insertIntoTestEntity(client) + formats.forEach { format -> + exportEntity(format, TEST_ENTITY_NAME) + val exportFile = exportFile(format, TEST_ENTITY_NAME) + val count = countElements(this.client, TEST_ENTITY_NAME) + TruncateEntityCommand.truncate(TEST_ENTITY_NAME, this.client, true) + ImportDataCommand.importData(TEST_ENTITY_NAME, exportFile, format, this.client, true) + assert(count == countElements(this.client, TEST_ENTITY_NAME)) + } + } + + private fun exportEntity(format: Format, fqn: Name.EntityName) { + val path = exportFolder() + path.toFile().mkdirs() + val exported = exportFile(format, fqn) + exported.deleteIfExists() + DumpEntityCommand.dumpEntity(fqn, path, format, this.client) + assert(exported.toFile().exists()) + assert(exported.toFile().totalSpace > 1) + } } \ No newline at end of file diff --git a/cottontaildb-dbms/src/testFixtures/kotlin/org/vitrivr/cottontail/test/GrpcTestUtils.kt b/cottontaildb-dbms/src/testFixtures/kotlin/org/vitrivr/cottontail/test/GrpcTestUtils.kt index 039f7eed4..aa752b105 100644 --- a/cottontaildb-dbms/src/testFixtures/kotlin/org/vitrivr/cottontail/test/GrpcTestUtils.kt +++ b/cottontaildb-dbms/src/testFixtures/kotlin/org/vitrivr/cottontail/test/GrpcTestUtils.kt @@ -5,9 +5,13 @@ import org.vitrivr.cottontail.client.SimpleClient import org.vitrivr.cottontail.client.language.basics.Type import org.vitrivr.cottontail.client.language.ddl.* import org.vitrivr.cottontail.client.language.dml.BatchInsert +import org.vitrivr.cottontail.client.language.dml.Insert import org.vitrivr.cottontail.client.language.dql.Query import org.vitrivr.cottontail.core.database.Name import org.vitrivr.cottontail.grpc.CottontailGrpc +import org.vitrivr.cottontail.test.TestConstants.DOUBLE_COLUMN_NAME +import org.vitrivr.cottontail.test.TestConstants.INT_COLUMN_NAME +import org.vitrivr.cottontail.test.TestConstants.STRING_COLUMN_NAME import kotlin.random.Random @@ -43,9 +47,9 @@ object GrpcTestUtils { */ fun createTestEntity(client: SimpleClient) { val create = CreateEntity(TestConstants.TEST_ENTITY_NAME.fqn) - .column(TestConstants.STRING_COLUMN_NAME, Type.STRING) - .column(TestConstants.INT_COLUMN_NAME, Type.INTEGER) - .column(TestConstants.DOUBLE_COLUMN_NAME, Type.DOUBLE) + .column(STRING_COLUMN_NAME, Type.STRING) + .column(INT_COLUMN_NAME, Type.INTEGER) + .column(DOUBLE_COLUMN_NAME, Type.DOUBLE) client.create(create) } @@ -56,9 +60,9 @@ object GrpcTestUtils { */ fun createTestVectorEntity(client: SimpleClient) { val create = CreateEntity(TestConstants.TEST_VECTOR_ENTITY_NAME.fqn) - .column(TestConstants.STRING_COLUMN_NAME, Type.STRING) - .column(TestConstants.INT_COLUMN_NAME, Type.INTEGER) - .column(TestConstants.TWOD_COLUMN_NAME, Type.FLOAT_VECTOR, 2) + .column(STRING_COLUMN_NAME, Type.STRING) + .column(INT_COLUMN_NAME, Type.INTEGER) + .column(TestConstants.TWOD_COLUMN_NAME, Type.FLOAT_VECTOR, 2) client.create(create) } @@ -68,7 +72,7 @@ object GrpcTestUtils { * @param client [SimpleClient] to use. */ fun populateTestEntity(client: SimpleClient) { - val batch = BatchInsert().into(TestConstants.TEST_ENTITY_NAME.fqn).columns(TestConstants.STRING_COLUMN_NAME, TestConstants.INT_COLUMN_NAME, TestConstants.DOUBLE_COLUMN_NAME) + val batch = BatchInsert().into(TestConstants.TEST_ENTITY_NAME.fqn).columns(STRING_COLUMN_NAME, INT_COLUMN_NAME, DOUBLE_COLUMN_NAME) val random = Random.Default repeat(TestConstants.TEST_COLLECTION_SIZE) { batch.append( @@ -80,11 +84,16 @@ object GrpcTestUtils { client.insert(batch) } + fun insertIntoTestEntity(client: SimpleClient, string: String = RandomStringUtils.randomAlphabetic(5), int: Int = Random.nextInt(0, 100), double: Double = Random.nextDouble(1.0)) { + val insert = Insert().into(TestConstants.TEST_ENTITY_NAME.fqn).values(Pair(STRING_COLUMN_NAME, string), Pair(INT_COLUMN_NAME, int), Pair(DOUBLE_COLUMN_NAME, double)) + client.insert(insert) + } + /** * Creates a Lucene index on the [TestConstants.TEST_ENTITY_NAME]. */ fun createLuceneIndexOnTestEntity(client: SimpleClient) { - client.create(CreateIndex(TestConstants.TEST_ENTITY_NAME.fqn, TestConstants.STRING_COLUMN_NAME, CottontailGrpc.IndexType.LUCENE)) + client.create(CreateIndex(TestConstants.TEST_ENTITY_NAME.fqn, STRING_COLUMN_NAME, CottontailGrpc.IndexType.LUCENE)) client.optimize(OptimizeEntity(TestConstants.TEST_ENTITY_NAME.fqn)) } @@ -95,16 +104,16 @@ object GrpcTestUtils { */ fun populateVectorEntity(client: SimpleClient) { val batch = BatchInsert().into(TestConstants.TEST_VECTOR_ENTITY_NAME.fqn) - .columns(TestConstants.STRING_COLUMN_NAME, TestConstants.INT_COLUMN_NAME, TestConstants.TWOD_COLUMN_NAME) + .columns(STRING_COLUMN_NAME, INT_COLUMN_NAME, TestConstants.TWOD_COLUMN_NAME) val random = Random.Default repeat(TestConstants.TEST_COLLECTION_SIZE) { val lat = random.nextFloat() + random.nextInt(0, 50) val lon = random.nextFloat() + random.nextInt(0, 50) val arr = floatArrayOf(lat, lon) batch.append( - RandomStringUtils.randomAlphabetic(5), - random.nextInt(0, 10), - arr + RandomStringUtils.randomAlphabetic(5), + random.nextInt(0, 10), + arr ) } client.insert(batch) From 6eef7916f062a3b05a61ed13feaae7f6a73c7975 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Mon, 9 May 2022 10:25:33 +0200 Subject: [PATCH 2/2] Fixed issue using JsonReader/JsonWritter's isLenient flag. --- .../vitrivr/cottontail/cli/AbstractCottontailCommand.kt | 2 +- .../vitrivr/cottontail/cli/entity/DumpEntityCommand.kt | 2 +- .../vitrivr/cottontail/data/exporter/JsonDataExporter.kt | 9 ++------- .../vitrivr/cottontail/data/importer/JsonDataImporter.kt | 8 +++++++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/AbstractCottontailCommand.kt b/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/AbstractCottontailCommand.kt index 914e2e76d..0af9f216b 100644 --- a/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/AbstractCottontailCommand.kt +++ b/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/AbstractCottontailCommand.kt @@ -187,7 +187,7 @@ sealed class AbstractCottontailCommand(name: String, help: String, val expand: B } } println("Executing and exporting query took $duration.") - } catch (e: StatusException) { + } catch (e: Throwable) { print("A ${e::class.java.simpleName} occurred while executing and exporting query: ${e.message}.") } } diff --git a/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/DumpEntityCommand.kt b/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/DumpEntityCommand.kt index e7ee404da..137210c81 100644 --- a/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/DumpEntityCommand.kt +++ b/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/DumpEntityCommand.kt @@ -45,7 +45,7 @@ class DumpEntityCommand(client: SimpleClient) : AbstractCottontailCommand.Entity } dataExporter.close() } - println("Dumping ${entityName} took $duration.") + println("Dumping $entityName took $duration.") } catch (e: Throwable) { print("A ${e::class.java.simpleName} occurred while executing and exporting query: ${e.message}.") } diff --git a/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/data/exporter/JsonDataExporter.kt b/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/data/exporter/JsonDataExporter.kt index f1111af70..e1d797524 100644 --- a/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/data/exporter/JsonDataExporter.kt +++ b/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/data/exporter/JsonDataExporter.kt @@ -26,15 +26,10 @@ class JsonDataExporter(override val path: Path, val indent: String = "") : DataE private set /** The [JsonWriter] instance used to read the JSON file. */ - private val writer = JsonWriter( - Files.newBufferedWriter( - this.path, - StandardOpenOption.CREATE_NEW, - StandardOpenOption.WRITE - ) - ) + private val writer = JsonWriter(Files.newBufferedWriter(this.path, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) init { + this.writer.isLenient = true this.writer.setIndent(this.indent) this.writer.beginArray() /* Starts writer the JSON array, which is the expected input. */ } diff --git a/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/data/importer/JsonDataImporter.kt b/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/data/importer/JsonDataImporter.kt index 47b05dc8e..871c95455 100644 --- a/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/data/importer/JsonDataImporter.kt +++ b/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/data/importer/JsonDataImporter.kt @@ -1,5 +1,6 @@ package org.vitrivr.cottontail.data.importer +import com.google.gson.GsonBuilder import com.google.gson.stream.JsonReader import com.google.gson.stream.JsonToken import it.unimi.dsi.fastutil.objects.Object2ObjectArrayMap @@ -24,9 +25,14 @@ import java.nio.file.Path class JsonDataImporter(override val path: Path, override val schema: List>) : DataImporter { /** The [JsonReader] instance used to read the JSON file. */ - private val reader = JsonReader(Files.newBufferedReader(this.path)) + private val reader = GsonBuilder() + .serializeNulls() + .serializeSpecialFloatingPointValues() + .create() + .newJsonReader(Files.newBufferedReader(this.path)) init { + this.reader.isLenient = true this.reader.beginArray() /* Starts reading the JSON array, which is the expected input. */ }