Merge remote-tracking branch 'origin/master'

cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/AbstractCottontailCommand.kt

@@ -187,7 +187,7 @@ sealed class AbstractCottontailCommand(name: String, help: String, val expand: B
                     }
                 }
                 println("Executing and exporting query took $duration.")
-            } catch (e: StatusException) {
+            } catch (e: Throwable) {
                 print("A ${e::class.java.simpleName} occurred while executing and exporting query: ${e.message}.")
             }
         }

cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/DumpEntityCommand.kt

@@ -45,7 +45,7 @@ class DumpEntityCommand(client: SimpleClient) : AbstractCottontailCommand.Entity
                     }
                     dataExporter.close()
                 }
-                println("Dumping ${entityName} took $duration.")
+                println("Dumping $entityName took $duration.")
             } catch (e: Throwable) {
                 print("A ${e::class.java.simpleName} occurred while executing and exporting query: ${e.message}.")
             }

cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/data/exporter/JsonDataExporter.kt

@@ -26,15 +26,10 @@ class JsonDataExporter(override val path: Path, val indent: String = "") : DataE
         private set
 
     /** The [JsonWriter] instance used to read the JSON file. */
-    private val writer = JsonWriter(
-        Files.newBufferedWriter(
-            this.path,
-            StandardOpenOption.CREATE_NEW,
-            StandardOpenOption.WRITE
-        )
-    )
+    private val writer = JsonWriter(Files.newBufferedWriter(this.path, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE))
 
     init {
+        this.writer.isLenient = true
         this.writer.setIndent(this.indent)
         this.writer.beginArray() /* Starts writer the JSON array, which is the expected input. */
     }

cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/data/importer/JsonDataImporter.kt

@@ -1,5 +1,6 @@
 package org.vitrivr.cottontail.data.importer
 
+import com.google.gson.GsonBuilder
 import com.google.gson.stream.JsonReader
 import com.google.gson.stream.JsonToken
 import it.unimi.dsi.fastutil.objects.Object2ObjectArrayMap
@@ -24,9 +25,14 @@ import java.nio.file.Path
 class JsonDataImporter(override val path: Path, override val schema: List<ColumnDef<*>>) : DataImporter {
 
     /** The [JsonReader] instance used to read the JSON file. */
-    private val reader = JsonReader(Files.newBufferedReader(this.path))
+    private val reader = GsonBuilder()
+        .serializeNulls()
+        .serializeSpecialFloatingPointValues()
+        .create()
+        .newJsonReader(Files.newBufferedReader(this.path))
 
     init {
+        this.reader.isLenient = true
         this.reader.beginArray() /* Starts reading the JSON array, which is the expected input. */
     }
 

cottontaildb-cli/src/test/kotlin/org/vitrivr/cottontail/cli/ExportImportCommandTest.kt

@@ -9,8 +9,10 @@ import org.vitrivr.cottontail.cli.entity.TruncateEntityCommand
 import org.vitrivr.cottontail.core.database.Name
 import org.vitrivr.cottontail.data.Format
 import org.vitrivr.cottontail.test.AbstractClientTest
+import org.vitrivr.cottontail.test.GrpcTestUtils
 import org.vitrivr.cottontail.test.GrpcTestUtils.countElements
 import org.vitrivr.cottontail.test.TestConstants
+import org.vitrivr.cottontail.test.TestConstants.TEST_ENTITY_NAME
 import java.nio.file.Path
 import kotlin.io.path.Path
 import kotlin.io.path.deleteIfExists
@@ -39,13 +41,7 @@ class ExportImportCommandTest : AbstractClientTest() {
     fun exportCreatesFile() {
         formats.forEach { format ->
             TestConstants.ALL_ENTITY_NAMES.forEach { name ->
-                val path = exportFolder()
-                path.toFile().mkdirs()
-                val exported = exportFile(format, name)
-                exported.deleteIfExists()
-                DumpEntityCommand.dumpEntity(name, path, format, this.client)
-                assert(exported.toFile().exists())
-                assert(exported.toFile().totalSpace > 1)
+                exportEntity(format, name)
             }
         }
     }
@@ -67,4 +63,28 @@ class ExportImportCommandTest : AbstractClientTest() {
             }
         }
     }
+
+    @Test
+    fun exportNan() {
+        GrpcTestUtils.insertIntoTestEntity(client, double = Double.NaN)
+        GrpcTestUtils.insertIntoTestEntity(client)
+        formats.forEach { format ->
+            exportEntity(format, TEST_ENTITY_NAME)
+            val exportFile = exportFile(format, TEST_ENTITY_NAME)
+            val count = countElements(this.client, TEST_ENTITY_NAME)
+            TruncateEntityCommand.truncate(TEST_ENTITY_NAME, this.client, true)
+            ImportDataCommand.importData(TEST_ENTITY_NAME, exportFile, format, this.client, true)
+            assert(count == countElements(this.client, TEST_ENTITY_NAME))
+        }
+    }
+
+    private fun exportEntity(format: Format, fqn: Name.EntityName) {
+        val path = exportFolder()
+        path.toFile().mkdirs()
+        val exported = exportFile(format, fqn)
+        exported.deleteIfExists()
+        DumpEntityCommand.dumpEntity(fqn, path, format, this.client)
+        assert(exported.toFile().exists())
+        assert(exported.toFile().totalSpace > 1)
+    }
 }

cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/entity/DefaultEntity.kt

@@ -389,7 +389,8 @@ class DefaultEntity(override val name: Name.EntityName, override val parent: Def
                 inserts[column.columnDef] = value
 
                 /* Check if null value is allowed. */
-                if (value == null && !column.columnDef.nullable) throw DatabaseException("Cannot insert NULL value into column ${column.columnDef}.")
+                if (value == null && !column.columnDef.nullable)
+                    throw DatabaseException.ValidationException("Cannot INSERT a NULL value into column ${column.columnDef}.")
                 (this.context.getTx(column) as ColumnTx<Value>).add(nextTupleId, value)
             }
 
@@ -418,10 +419,10 @@ class DefaultEntity(override val name: Name.EntityName, override val parent: Def
             /* Execute UPDATE on column level. */
             val updates = Object2ObjectArrayMap<ColumnDef<*>, Pair<Value?, Value?>>(record.columns.size)
             for (def in record.columns) {
-                val column = this.columns[def.name] ?: throw DatabaseException("Record with tuple ID ${record.tupleId} cannot be updated for column $def, because column does not exist on entity.")
+                val column = this.columns[def.name] ?: throw DatabaseException.ColumnDoesNotExistException(def.name)
                 val columnTx = (this.context.getTx(column) as ColumnTx<*>)
                 val value = record[def]
-                if (value == null && !def.nullable) throw DatabaseException("Record with tuple ID ${record.tupleId} cannot be updated with NULL value for column $def, because column is not nullable.")
+                if (value == null && !def.nullable) throw DatabaseException.ValidationException("Record ${record.tupleId} cannot be updated with NULL value for column $def, because column is not nullable.")
                 updates[def] = Pair((columnTx as ColumnTx<Value>).update(record.tupleId, value), value) /* Map: ColumnDef -> Pair[Old, New]. */
             }
 

cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/exceptions/DatabaseException.kt

@@ -3,7 +3,10 @@ package org.vitrivr.cottontail.dbms.exceptions
 import org.vitrivr.cottontail.core.database.Name
 import org.vitrivr.cottontail.dbms.column.Column
 import org.vitrivr.cottontail.dbms.entity.Entity
+import org.vitrivr.cottontail.dbms.general.DBO
 import org.vitrivr.cottontail.dbms.general.DBOVersion
+import org.vitrivr.cottontail.dbms.index.Index
+import org.vitrivr.cottontail.dbms.schema.Schema
 
 open class DatabaseException(message: String, cause: Throwable? = null) : Throwable(message, cause) {
     /**
@@ -15,64 +18,64 @@ open class DatabaseException(message: String, cause: Throwable? = null) : Throwa
     class VersionMismatchException(val expected: DBOVersion, val found: DBOVersion) : DatabaseException("Version mismatch for DBO: Expected $expected but found $found.")
 
     /**
-     * Thrown when trying to create a [Schema][org.vitrivr.cottontail.dbms.schema.DefaultSchema]
+     * Thrown when trying to create a [Schema]
      * that does already exist.
      *
-     * @param schema [Name] of the [Schema][org.vitrivr.cottontail.dbms.schema.DefaultSchema].
+     * @param schema [Name] of the [Schema].
      */
     class SchemaAlreadyExistsException(val schema: Name.SchemaName) : DatabaseException("Schema '$schema' does already exist!")
 
     /**
-     * Thrown when trying to access a [Schema][org.vitrivr.cottontail.dbms.schema.DefaultSchema]
+     * Thrown when trying to access a [Schema]
      * that does not exist.
      *
-     * @param schema [Name] of the [Schema][org.vitrivr.cottontail.dbms.schema.DefaultSchema].
+     * @param schema [Name] of the [Schema].
      */
     class SchemaDoesNotExistException(val schema: Name.SchemaName) : DatabaseException("Schema '$schema' does not exist!")
 
     /**
-     * Thrown when trying to create an [Entity][org.vitrivr.cottontail.dbms.entity.DefaultEntity]
+     * Thrown when trying to create an [Entity]
      * that does already exist.
      *
-     * @param entity [Name] of the [Entity][org.vitrivr.cottontail.dbms.entity.DefaultEntity].
+     * @param entity [Name] of the [Entity].
      */
     class EntityAlreadyExistsException(val entity: Name.EntityName) : DatabaseException("Entity '$entity' does already exist!")
 
     /**
-     * Thrown when trying to access an [Entity][org.vitrivr.cottontail.dbms.entity.DefaultEntity]
+     * Thrown when trying to access an [Entity]
      * that does not exist.
      *
-     * @param entity [Name] of the [Entity][org.vitrivr.cottontail.dbms.entity.DefaultEntity].
+     * @param entity [Name] of the [Entity].
      */
     class EntityDoesNotExistException(val entity: Name.EntityName) : DatabaseException("Entity '$entity' does not exist!")
 
     /**
-     * Thrown whenever trying to create an [Index][org.vitrivr.cottontail.dbms.index.AbstractIndex]
+     * Thrown whenever trying to create an [Index]
      * that does already exist.
      *
-     * @param index The [Name] of the [Index][org.vitrivr.cottontail.dbms.index.AbstractIndex]
+     * @param index The [Name] of the [Index]
      */
     class IndexAlreadyExistsException(val index: Name.IndexName) : DatabaseException("Index '$index' does already exist!")
 
     /**
-     * Thrown whenever trying to access an [Index][org.vitrivr.cottontail.dbms.index.AbstractIndex]
+     * Thrown whenever trying to access an [Index]
      * that does not exist.
      *
-     * @param index The [Name] of the [Index][org.vitrivr.cottontail.dbms.index.AbstractIndex]
+     * @param index The [Name] of the [Index]
      */
     class IndexDoesNotExistException(val index: Name) : DatabaseException("Index '$index' does not exist!")
 
     /**
-     * Thrown whenever trying to create an [Index][[org.vitrivr.cottontail.dbms.index.AbstractIndex] that is not supported (yet). *
+     * Thrown whenever trying to create an [Index]that is not supported (yet). *
      *
-     * @param index The [Name] of the [Index][org.vitrivr.cottontail.dbms.index.AbstractIndex]
+     * @param index The [Name] of the [Index]
      */
-    class IndexNotSupportedException(val index: Name.IndexName, val reason: String) : DatabaseException("Index '$index' could not be created: $reason")
+    class IndexNotSupportedException(val index: Name.IndexName, reason: String) : DatabaseException("Index '$index' could not be created: $reason")
 
     /**
      * Thrown upon creation of an [Entity] if the definition contains no column.
      *
-     * @param entity [Name] of the affected [Entity][org.vitrivr.cottontail.dbms.entity.DefaultEntity]
+     * @param entity [Name] of the affected [Entity]
      */
     class NoColumnException(entity: Name.EntityName) : DatabaseException("Entity '$entity' could not be created because it does not contain a column.")
 
@@ -85,8 +88,7 @@ open class DatabaseException(message: String, cause: Throwable? = null) : Throwa
     class DuplicateColumnException(entity: Name.EntityName, name: Name.ColumnName) : DatabaseException("Entity '$entity' could not be created because it contains duplicate column names '$name'.")
 
     /**
-     * Thrown whenever trying to access a [Column][org.vitrivr.cottontail.dbms.column.Column]
-     * that does not exist.
+     * Thrown whenever trying to access a [Column][org.vitrivr.cottontail.dbms.column.Column]that does not exist.
      *
      * @param column The [Name] of the [Column][org.vitrivr.cottontail.dbms.column.Column].
      */
@@ -112,6 +114,13 @@ open class DatabaseException(message: String, cause: Throwable? = null) : Throwa
      * @param message Description of the issue.
      */
     class ReservedValueException(message: String): DatabaseException(message)
+
+    /**
+     * Write could not be executed because it failed a validation step. This is often caused by a user error, providing erroneous data.
+     *
+     * @param message Description of the validation error.
+     */
+    class ValidationException(message: String) : TransactionException(message)
 }
 
 

cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/exceptions/TransactionException.kt

@@ -20,12 +20,20 @@ sealed class TransactionException(message: String) : DatabaseException(message)
     class DBOClosed(tid: TransactionId, dbo: DBO) : TransactionException("Tx object for transaction $tid could not be created for DBO '${dbo.name}': Enclosing DBO was closed.")
 
     /**
-     * Write could not be executed because it failed a validation step. This is often caused by a user error, providing erroneous data.
+     * COMMIT could not be executed because.
      *
      * @param tid The [TransactionId] of the [Tx] in which this error occurred.
      * @param message Description of the validation error.
      */
-    class Validation(tid: TransactionId, message: String) : TransactionException("Transaction $tid reported validation error: $message")
+    class Commit(tid: TransactionId, override val message: String?) : TransactionException("Transaction $tid could not be committed: $message")
+
+    /**
+     * ROLLBACK could not be executed because.
+     *
+     * @param tid The [TransactionId] of the [Tx] in which this error occurred.
+     * @param message Description of the validation error.
+     */
+    class Rollback(tid: TransactionId, override val message: String?) : TransactionException("Transaction $tid could not be rolled back: $message")
 
     /**
      * Thrown if a [Transaction] could not be committed, because it is in conflict with another [Transaction].

cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/execution/transactions/TransactionManager.kt

@@ -203,9 +203,8 @@ class TransactionManager(val executionManager: ExecutionManager, transactionTabl
          */
         override fun commit() = runBlocking {
             this@TransactionImpl.mutex.withLock { /* Synchronise with ongoing COMMITS, ROLLBACKS or queries that are being scheduled. */
-                check(this@TransactionImpl.state.canCommit) {
-                    "Unable to commit transaction ${this@TransactionImpl.txId} because it is in wrong state (s = ${this@TransactionImpl.state})."
-                }
+                if (!this@TransactionImpl.state.canCommit)
+                    throw TransactionException.Commit(this@TransactionImpl.txId, "Unable to COMMIT because transaction is in wrong state (s = ${this@TransactionImpl.state}).")
                 this@TransactionImpl.state = TransactionStatus.FINALIZING
                 var commit = false
                 try {
@@ -233,9 +232,8 @@ class TransactionManager(val executionManager: ExecutionManager, transactionTabl
          */
         override fun rollback() = runBlocking {
             this@TransactionImpl.mutex.withLock {
-                check(this@TransactionImpl.state.canRollback) {
-                    "Unable to rollback transaction ${this@TransactionImpl.txId} because it is in wrong state (s = ${this@TransactionImpl.state})."
-                }
+                if (!this@TransactionImpl.state.canRollback)
+                    throw TransactionException.Rollback(this@TransactionImpl.txId, "Unable to ROLLBACK because transaction is in wrong state (s = ${this@TransactionImpl.state}).")
                 this@TransactionImpl.performRollback()
             }
         }

cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/index/hash/BTreeIndex.kt

@@ -24,7 +24,6 @@ import org.vitrivr.cottontail.dbms.catalogue.storeName
 import org.vitrivr.cottontail.dbms.entity.DefaultEntity
 import org.vitrivr.cottontail.dbms.entity.EntityTx
 import org.vitrivr.cottontail.dbms.exceptions.DatabaseException
-import org.vitrivr.cottontail.dbms.exceptions.TransactionException
 import org.vitrivr.cottontail.dbms.execution.transactions.TransactionContext
 import org.vitrivr.cottontail.dbms.index.*
 import org.vitrivr.cottontail.dbms.index.lucene.LuceneIndex
@@ -242,11 +241,10 @@ class BTreeIndex(name: Name.IndexName, parent: DefaultEntity) : AbstractIndex(na
             /* Iterate over entity and update index with entries. */
             val cursor = entityTx.cursor(this.columns)
             cursor.forEach { record ->
-                val value = record[this.columns[0]] ?: throw TransactionException.Validation(
-                    this.context.txId,
-                    "A value cannot be null for instances of NonUniqueHashIndex ${this@BTreeIndex.name} but given value is (value = null, tupleId = ${record.tupleId})."
-                )
-                this.addMapping(value, record.tupleId)
+                val value = record[this.columns[0]]
+                if (value != null) {
+                    this.addMapping(value, record.tupleId)
+                }
             }
 
             /* Close cursor. */

cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/index/hash/UQBTreeIndex.kt

@@ -23,7 +23,6 @@ import org.vitrivr.cottontail.dbms.catalogue.storeName
 import org.vitrivr.cottontail.dbms.entity.DefaultEntity
 import org.vitrivr.cottontail.dbms.entity.EntityTx
 import org.vitrivr.cottontail.dbms.exceptions.DatabaseException
-import org.vitrivr.cottontail.dbms.exceptions.TransactionException
 import org.vitrivr.cottontail.dbms.execution.transactions.TransactionContext
 import org.vitrivr.cottontail.dbms.index.*
 import org.vitrivr.cottontail.dbms.index.lucene.LuceneIndex
@@ -143,13 +142,11 @@ class UQBTreeIndex(name: Name.IndexName, parent: DefaultEntity) : AbstractIndex(
          *
          * This is an internal function and can be used safely with values o
          */
-        private fun addMapping(key: Value, tupleId: TupleId): Boolean {
+        private fun addMapping(key: Value, tupleId: TupleId) {
             val keyRaw = this.binding.valueToEntry(key)
             val tupleIdRaw = LongBinding.longToCompressedEntry(tupleId)
-            return if (this.dataStore.get(this.context.xodusTx, keyRaw) == null) {
-                this.dataStore.put(this.context.xodusTx, keyRaw, tupleIdRaw)
-            } else {
-                false
+            if (!this.dataStore.add(this.context.xodusTx, keyRaw, tupleIdRaw)) {
+                throw DatabaseException.ValidationException("Mapping of $key to tuple $tupleId could be added to UniqueHashIndex, because value must be unique.")
             }
         }
 
@@ -228,9 +225,9 @@ class UQBTreeIndex(name: Name.IndexName, parent: DefaultEntity) : AbstractIndex(
             /* Iterate over entity and update index with entries. */
             val cursor = entityTx.cursor(this.columns)
             cursor.forEach { record ->
-                val value = record[this.columns[0]] ?: throw TransactionException.Validation(this.context.txId, "Value cannot be null for UniqueHashIndex ${this@UQBTreeIndex.name} given value is (value = null, tupleId = ${record.tupleId}).")
-                if (!this.addMapping(value, record.tupleId)) {
-                    throw TransactionException.Validation(this.context.txId, "Value must be unique for UniqueHashIndex ${this@UQBTreeIndex.name} but is not (value = $value, tupleId = ${record.tupleId}).")
+                val value = record[this.columns[0]]
+                if (value != null) {
+                    this.addMapping(value, record.tupleId)
                 }
             }
 

cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/server/grpc/services/TransactionalGrpcService.kt

@@ -191,7 +191,8 @@ internal interface TransactionalGrpcService {
             is DatabaseException.EntityAlreadyExistsException,
             is DatabaseException.IndexAlreadyExistsException -> Status.ALREADY_EXISTS.withCause(e)
             is DatabaseException.NoColumnException,
-            is DatabaseException.DuplicateColumnException -> Status.INVALID_ARGUMENT.withCause(e)
+            is DatabaseException.DuplicateColumnException,
+            is DatabaseException.ValidationException -> Status.INVALID_ARGUMENT.withCause(e)
             is DeadlockException,
             is TransactionException.InConflict -> Status.ABORTED.withCause(e)
             is ExecutionException,