From 9057feed914f0f746ead99080ea286cedfa6029c Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Tue, 19 Dec 2023 17:38:35 +0100 Subject: [PATCH] =?UTF-8?q?Added=20(experimental)=20support=20f=C3=BCr=20H?= =?UTF-8?q?alfVectorValues.=20Thanks=20to=20@lucaro.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../vitrivr/cottontail/core/DataExtensions.kt | 195 ++++++++---- .../cottontail/core/DescriptionExtensions.kt | 1 + .../vitrivr/cottontail/core/types/Types.kt | 2 +- .../core/values/BooleanVectorValue.kt | 2 +- .../core/values/Complex32VectorValue.kt | 2 +- .../core/values/Complex64VectorValue.kt | 3 +- .../core/values/DoubleVectorValue.kt | 2 +- .../core/values/FloatVectorValue.kt | 3 +- .../cottontail/core/values/HalfVectorValue.kt | 295 ++++++++++++++++++ .../cottontail/core/values/IntVectorValue.kt | 2 +- .../cottontail/core/values/LongVectorValue.kt | 2 +- .../cottontail/core/values/PublicValue.kt | 1 - .../core/values/ShortVectorValue.kt | 3 +- .../serialization/SerializationExtension.kt | 2 +- .../src/main/protobuf/cottontail.proto | 16 +- .../math/distance/binary/EuclideanDistance.kt | 32 ++ .../generators/FloatVectorValueGenerator.kt | 3 +- .../generators/HalfVectorValueGenerator.kt | 47 +++ .../core/values/tablets/FloatVectorTablet.kt | 2 +- .../core/values/tablets/HalfVectorTablet.kt | 26 ++ .../cottontail/core/values/tablets/Tablet.kt | 1 + .../utilities/hashing/ValueFunnel.kt | 4 + .../vitrivr/cottontail/utilities/math/Half.kt | 27 +- .../dbms/queries/binding/GrpcQueryBinder.kt | 2 +- .../tablets/LZ4TabletSerializer.kt | 4 +- .../tablets/SnappyTabletSerializer.kt | 1 + .../serializers/tuples/TupleSerializer.kt | 2 + .../values/HalfVectorValueValueSerializer.kt | 15 +- .../tablets/TabletSerializationTest.kt | 22 +- .../test/EmbeddedCottontailGrpcServer.kt | 2 - 30 files changed, 602 insertions(+), 119 deletions(-) create mode 100644 cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/HalfVectorValue.kt create mode 100644 cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/generators/HalfVectorValueGenerator.kt create mode 100644 cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/HalfVectorTablet.kt diff --git a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/DataExtensions.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/DataExtensions.kt index 873d55799..983368ead 100644 --- a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/DataExtensions.kt +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/DataExtensions.kt @@ -84,13 +84,13 @@ fun CottontailGrpc.Literal.toValue(type: Types<*>): PublicValue? = when (type) { is Types.IntVector -> this.toIntVectorValue() is Types.LongVector -> this.toLongVectorValue() is Types.FloatVector -> this.toFloatVectorValue() + is Types.HalfVector -> this.toHalfVectorValue() is Types.DoubleVector -> this.toDoubleVectorValue() is Types.BooleanVector -> this.toBooleanVectorValue() is Types.Complex32Vector -> this.toComplex32VectorValue() is Types.Complex64Vector -> this.toComplex64VectorValue() is Types.ShortVector -> this.toShortVectorValue() is Types.ByteString -> this.toByteStringValue() - is Types.HalfVector -> this.toFloatVectorValue() } /** @@ -112,16 +112,18 @@ fun CottontailGrpc.Literal.toValue(): PublicValue? = when(this.dataCase) { CottontailGrpc.Literal.DataCase.COMPLEX64DATA -> Complex64Value(this.complex64Data.real, this.complex64Data.imaginary) CottontailGrpc.Literal.DataCase.BYTESTRINGDATA -> ByteStringValue(this.byteStringData.toByteArray()) CottontailGrpc.Literal.DataCase.VECTORDATA -> when(this.vectorData.vectorDataCase) { - CottontailGrpc.Vector.VectorDataCase.DOUBLEVECTOR -> DoubleVectorValue(this.vectorData.doubleVector.vectorList.toTypedArray()) - CottontailGrpc.Vector.VectorDataCase.FLOATVECTOR -> FloatVectorValue(this.vectorData.floatVector.vectorList.toTypedArray()) - CottontailGrpc.Vector.VectorDataCase.LONGVECTOR -> LongVectorValue(this.vectorData.longVector.vectorList.toTypedArray()) - CottontailGrpc.Vector.VectorDataCase.INTVECTOR -> IntVectorValue(this.vectorData.intVector.vectorList.toTypedArray()) - CottontailGrpc.Vector.VectorDataCase.BOOLVECTOR -> BooleanVectorValue(this.vectorData.boolVector.vectorList.toTypedArray()) - CottontailGrpc.Vector.VectorDataCase.COMPLEX32VECTOR -> Complex32VectorValue(Array(this.vectorData.complex32Vector.vectorList.size) { - Complex32Value(FloatValue(this.vectorData.complex32Vector.vectorList[it].real), FloatValue(this.vectorData.complex32Vector.vectorList[it].imaginary)) + CottontailGrpc.Vector.VectorDataCase.DOUBLE -> DoubleVectorValue(this.vectorData.double.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.FLOAT -> FloatVectorValue(this.vectorData.float.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.HALF -> HalfVectorValue(this.vectorData.half.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.LONG -> LongVectorValue(this.vectorData.long.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.INT -> IntVectorValue(this.vectorData.int.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.SHORT -> ShortVectorValue(this.vectorData.short.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.BOOL -> BooleanVectorValue(this.vectorData.bool.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.COMPLEX32-> Complex32VectorValue(Array(this.vectorData.complex32.vectorList.size) { + Complex32Value(FloatValue(this.vectorData.complex32.vectorList[it].real), FloatValue(this.vectorData.complex32.vectorList[it].imaginary)) }) - CottontailGrpc.Vector.VectorDataCase.COMPLEX64VECTOR -> Complex64VectorValue(Array(this.vectorData.complex64Vector.vectorList.size) { - Complex32Value(FloatValue(this.vectorData.complex64Vector.vectorList[it].real), FloatValue(this.vectorData.complex64Vector.vectorList[it].imaginary)) + CottontailGrpc.Vector.VectorDataCase.COMPLEX64 -> Complex64VectorValue(Array(this.vectorData.complex64.vectorList.size) { + Complex32Value(FloatValue(this.vectorData.complex64.vectorList[it].real), FloatValue(this.vectorData.complex64.vectorList[it].imaginary)) }) else -> throw IllegalArgumentException("Literal malformed: Cannot convert value of type ${this.vectorData.vectorDataCase}.") } @@ -148,13 +150,15 @@ fun CottontailGrpc.Literal.toType(): Types<*> = when(this.dataCase) { CottontailGrpc.Literal.DataCase.COMPLEX64DATA -> Types.Complex64 CottontailGrpc.Literal.DataCase.BYTESTRINGDATA -> Types.ByteString CottontailGrpc.Literal.DataCase.VECTORDATA -> when(this.vectorData.vectorDataCase) { - CottontailGrpc.Vector.VectorDataCase.DOUBLEVECTOR -> Types.DoubleVector(this.vectorData.doubleVector.vectorCount) - CottontailGrpc.Vector.VectorDataCase.FLOATVECTOR -> Types.FloatVector(this.vectorData.floatVector.vectorCount) - CottontailGrpc.Vector.VectorDataCase.LONGVECTOR -> Types.LongVector(this.vectorData.longVector.vectorCount) - CottontailGrpc.Vector.VectorDataCase.INTVECTOR -> Types.IntVector(this.vectorData.intVector.vectorCount) - CottontailGrpc.Vector.VectorDataCase.BOOLVECTOR -> Types.BooleanVector(this.vectorData.boolVector.vectorCount) - CottontailGrpc.Vector.VectorDataCase.COMPLEX32VECTOR -> Types.Complex32Vector(this.vectorData.complex32Vector.vectorCount) - CottontailGrpc.Vector.VectorDataCase.COMPLEX64VECTOR -> Types.Complex64Vector(this.vectorData.complex64Vector.vectorCount) + CottontailGrpc.Vector.VectorDataCase.DOUBLE -> Types.DoubleVector(this.vectorData.double.vectorCount) + CottontailGrpc.Vector.VectorDataCase.FLOAT -> Types.FloatVector(this.vectorData.float.vectorCount) + CottontailGrpc.Vector.VectorDataCase.HALF -> Types.HalfVector(this.vectorData.half.vectorCount) + CottontailGrpc.Vector.VectorDataCase.LONG -> Types.LongVector(this.vectorData.long.vectorCount) + CottontailGrpc.Vector.VectorDataCase.INT -> Types.IntVector(this.vectorData.int.vectorCount) + CottontailGrpc.Vector.VectorDataCase.SHORT -> Types.ShortVector(this.vectorData.short.vectorCount) + CottontailGrpc.Vector.VectorDataCase.BOOL -> Types.BooleanVector(this.vectorData.bool.vectorCount) + CottontailGrpc.Vector.VectorDataCase.COMPLEX32 -> Types.Complex32Vector(this.vectorData.complex32.vectorCount) + CottontailGrpc.Vector.VectorDataCase.COMPLEX64 -> Types.Complex64Vector(this.vectorData.complex64.vectorCount) CottontailGrpc.Vector.VectorDataCase.VECTORDATA_NOT_SET, null -> throw IllegalArgumentException("Type cannot be determined for a value of NULL.") } @@ -484,6 +488,27 @@ fun CottontailGrpc.Literal.toFloatVectorValue(): FloatVectorValue? = when (this. else -> throw IllegalArgumentException("Malformed literal: ${this.dataCase} cannot be cast to VECTOR[DOUBLE].") } +/** + * Returns the value of [CottontailGrpc.Literal] as [FloatVectorValue]. + * + * @return [FloatVectorValue] or null. + * @throws IllegalArgumentException If cast is not possible. + */ +fun CottontailGrpc.Literal.toHalfVectorValue(): HalfVectorValue? = when (this.dataCase) { + CottontailGrpc.Literal.DataCase.BOOLEANDATA -> HalfVectorValue(FloatArray(1) { this.booleanData.toFloat() }) + CottontailGrpc.Literal.DataCase.INTDATA -> HalfVectorValue(FloatArray(1) { this.intData.toFloat() }) + CottontailGrpc.Literal.DataCase.LONGDATA -> HalfVectorValue(FloatArray(1) { this.longData.toFloat() }) + CottontailGrpc.Literal.DataCase.FLOATDATA -> HalfVectorValue(FloatArray(1) { this.floatData }) + CottontailGrpc.Literal.DataCase.DOUBLEDATA -> HalfVectorValue(FloatArray(1) { this.doubleData.toFloat() }) + CottontailGrpc.Literal.DataCase.STRINGDATA -> HalfVectorValue(FloatArray(1) { + this.stringData.toFloatOrNull() ?: throw IllegalArgumentException("A value of type STRING (v='${this.stringData}') cannot be cast to VECTOR[FLOAT].") + }) + CottontailGrpc.Literal.DataCase.DATEDATA -> HalfVectorValue(FloatArray(1) { this.dateData.toFloat() }) + CottontailGrpc.Literal.DataCase.VECTORDATA -> this.vectorData.toHalfVectorValue() + CottontailGrpc.Literal.DataCase.NULLDATA -> null + else -> throw IllegalArgumentException("Malformed literal: ${this.dataCase} cannot be cast to VECTOR[DOUBLE].") +} + /** * Returns the value of [CottontailGrpc.Literal] as [DoubleVectorValue]. @@ -632,11 +657,13 @@ fun CottontailGrpc.Literal.toComplex64VectorValue(): Complex64VectorValue? = whe * @throws IllegalArgumentException If cast is not possible. */ fun CottontailGrpc.Vector.toDoubleVectorValue(): DoubleVectorValue = when (this.vectorDataCase) { - CottontailGrpc.Vector.VectorDataCase.DOUBLEVECTOR -> DoubleVectorValue(this.doubleVector.vectorList.toTypedArray()) - CottontailGrpc.Vector.VectorDataCase.FLOATVECTOR -> DoubleVectorValue(this.floatVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.LONGVECTOR -> DoubleVectorValue(this.longVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.INTVECTOR -> DoubleVectorValue(this.intVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.BOOLVECTOR -> DoubleVectorValue(this.boolVector.vectorList.map { if (it) 1.0 else 0.0 }) + CottontailGrpc.Vector.VectorDataCase.DOUBLE -> DoubleVectorValue(this.double.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.FLOAT -> DoubleVectorValue(this.float.vectorList) + CottontailGrpc.Vector.VectorDataCase.HALF -> DoubleVectorValue(this.half.vectorList) + CottontailGrpc.Vector.VectorDataCase.LONG -> DoubleVectorValue(this.long.vectorList) + CottontailGrpc.Vector.VectorDataCase.INT -> DoubleVectorValue(this.int.vectorList) + CottontailGrpc.Vector.VectorDataCase.SHORT -> DoubleVectorValue(this.short.vectorList) + CottontailGrpc.Vector.VectorDataCase.BOOL -> DoubleVectorValue(this.bool.vectorList.map { if (it) 1.0 else 0.0 }) else -> throw IllegalArgumentException("${this.vectorDataCase} cannot be cast to VECTOR[COMPLEX32].") } @@ -647,11 +674,30 @@ fun CottontailGrpc.Vector.toDoubleVectorValue(): DoubleVectorValue = when (this. * @throws IllegalArgumentException If cast is not possible. */ fun CottontailGrpc.Vector.toFloatVectorValue(): FloatVectorValue = when (this.vectorDataCase) { - CottontailGrpc.Vector.VectorDataCase.DOUBLEVECTOR -> FloatVectorValue(this.doubleVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.FLOATVECTOR -> FloatVectorValue(this.floatVector.vectorList.toTypedArray()) - CottontailGrpc.Vector.VectorDataCase.LONGVECTOR -> FloatVectorValue(this.longVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.INTVECTOR -> FloatVectorValue(this.intVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.BOOLVECTOR -> FloatVectorValue(this.boolVector.vectorList.map { if (it) 1.0f else 0.0f }) + CottontailGrpc.Vector.VectorDataCase.DOUBLE -> FloatVectorValue(this.double.vectorList) + CottontailGrpc.Vector.VectorDataCase.FLOAT -> FloatVectorValue(this.float.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.HALF -> FloatVectorValue(this.half.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.LONG -> FloatVectorValue(this.long.vectorList) + CottontailGrpc.Vector.VectorDataCase.INT -> FloatVectorValue(this.int.vectorList) + CottontailGrpc.Vector.VectorDataCase.SHORT -> FloatVectorValue(this.short.vectorList) + CottontailGrpc.Vector.VectorDataCase.BOOL -> FloatVectorValue(this.bool.vectorList.map { if (it) 1.0f else 0.0f }) + else -> throw IllegalArgumentException("${this.vectorDataCase} cannot be cast to VECTOR[FLOAT].") +} + +/** + * Returns the value of [CottontailGrpc.Vector] as [HalfVectorValue]. + * + * @return [HalfVectorValue] values + * @throws IllegalArgumentException If cast is not possible. + */ +fun CottontailGrpc.Vector.toHalfVectorValue(): HalfVectorValue = when (this.vectorDataCase) { + CottontailGrpc.Vector.VectorDataCase.DOUBLE -> HalfVectorValue(this.double.vectorList) + CottontailGrpc.Vector.VectorDataCase.FLOAT -> HalfVectorValue(this.float.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.HALF -> HalfVectorValue(this.half.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.LONG -> HalfVectorValue(this.long.vectorList) + CottontailGrpc.Vector.VectorDataCase.INT -> HalfVectorValue(this.int.vectorList) + CottontailGrpc.Vector.VectorDataCase.SHORT -> HalfVectorValue(this.short.vectorList) + CottontailGrpc.Vector.VectorDataCase.BOOL -> HalfVectorValue(this.bool.vectorList.map { if (it) 1.0f else 0.0f }) else -> throw IllegalArgumentException("${this.vectorDataCase} cannot be cast to VECTOR[FLOAT].") } @@ -662,11 +708,13 @@ fun CottontailGrpc.Vector.toFloatVectorValue(): FloatVectorValue = when (this.ve * @throws IllegalArgumentException If cast is not possible. */ fun CottontailGrpc.Vector.toLongVectorValue(): LongVectorValue = when (this.vectorDataCase) { - CottontailGrpc.Vector.VectorDataCase.DOUBLEVECTOR -> LongVectorValue(this.doubleVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.FLOATVECTOR -> LongVectorValue(this.floatVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.LONGVECTOR -> LongVectorValue(this.longVector.vectorList.toTypedArray()) - CottontailGrpc.Vector.VectorDataCase.INTVECTOR -> LongVectorValue(this.intVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.BOOLVECTOR -> LongVectorValue(this.boolVector.vectorList.map { if (it) 1L else 0L }) + CottontailGrpc.Vector.VectorDataCase.DOUBLE -> LongVectorValue(this.double.vectorList) + CottontailGrpc.Vector.VectorDataCase.FLOAT -> LongVectorValue(this.float.vectorList) + CottontailGrpc.Vector.VectorDataCase.HALF -> LongVectorValue(this.half.vectorList) + CottontailGrpc.Vector.VectorDataCase.LONG -> LongVectorValue(this.long.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.INT-> LongVectorValue(this.int.vectorList) + CottontailGrpc.Vector.VectorDataCase.SHORT -> LongVectorValue(this.short.vectorList) + CottontailGrpc.Vector.VectorDataCase.BOOL -> LongVectorValue(this.bool.vectorList.map { if (it) 1L else 0L }) else -> throw IllegalArgumentException("${this.vectorDataCase} cannot be cast to VECTOR[LONG].") } @@ -677,20 +725,30 @@ fun CottontailGrpc.Vector.toLongVectorValue(): LongVectorValue = when (this.vect * @throws IllegalArgumentException If cast is not possible. */ fun CottontailGrpc.Vector.toIntVectorValue(): IntVectorValue = when (this.vectorDataCase) { - CottontailGrpc.Vector.VectorDataCase.DOUBLEVECTOR -> IntVectorValue(this.doubleVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.FLOATVECTOR -> IntVectorValue(this.floatVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.LONGVECTOR -> IntVectorValue(this.longVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.INTVECTOR -> IntVectorValue(this.intVector.vectorList.toTypedArray()) - CottontailGrpc.Vector.VectorDataCase.BOOLVECTOR -> IntVectorValue(this.boolVector.vectorList.map { if (it) 1 else 0 }) + CottontailGrpc.Vector.VectorDataCase.DOUBLE -> IntVectorValue(this.double.vectorList) + CottontailGrpc.Vector.VectorDataCase.FLOAT -> IntVectorValue(this.float.vectorList) + CottontailGrpc.Vector.VectorDataCase.HALF -> IntVectorValue(this.float.vectorList) + CottontailGrpc.Vector.VectorDataCase.LONG -> IntVectorValue(this.long.vectorList) + CottontailGrpc.Vector.VectorDataCase.INT -> IntVectorValue(this.int.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.SHORT -> IntVectorValue(this.short.vectorList) + CottontailGrpc.Vector.VectorDataCase.BOOL -> IntVectorValue(this.bool.vectorList.map { if (it) 1 else 0 }) else -> throw IllegalArgumentException("${this.vectorDataCase} cannot be cast to VECTOR[INT].") } +/** + * Returns the value of [CottontailGrpc.Vector] as [IntArray]. + * + * @return [IntArray] values + * @throws IllegalArgumentException If cast is not possible. + */ fun CottontailGrpc.Vector.toShortVectorValue(): ShortVectorValue = when (this.vectorDataCase) { - CottontailGrpc.Vector.VectorDataCase.DOUBLEVECTOR -> ShortVectorValue(this.doubleVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.FLOATVECTOR -> ShortVectorValue(this.floatVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.LONGVECTOR -> ShortVectorValue(this.longVector.vectorList.toTypedArray()) - CottontailGrpc.Vector.VectorDataCase.INTVECTOR -> ShortVectorValue(this.intVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.BOOLVECTOR -> ShortVectorValue(this.boolVector.vectorList.map { if (it) 1 else 0 }) + CottontailGrpc.Vector.VectorDataCase.DOUBLE -> ShortVectorValue(this.double.vectorList) + CottontailGrpc.Vector.VectorDataCase.FLOAT -> ShortVectorValue(this.float.vectorList) + CottontailGrpc.Vector.VectorDataCase.HALF -> ShortVectorValue(this.half.vectorList) + CottontailGrpc.Vector.VectorDataCase.LONG -> ShortVectorValue(this.long.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.INT-> ShortVectorValue(this.int.vectorList) + CottontailGrpc.Vector.VectorDataCase.SHORT-> ShortVectorValue(this.short.vectorList) + CottontailGrpc.Vector.VectorDataCase.BOOL -> ShortVectorValue(this.bool.vectorList.map { if (it) 1 else 0 }) else -> throw IllegalArgumentException("${this.vectorDataCase} cannot be cast to VECTOR[SHORT].") } @@ -701,11 +759,13 @@ fun CottontailGrpc.Vector.toShortVectorValue(): ShortVectorValue = when (this.ve * @throws IllegalArgumentException If cast is not possible. */ fun CottontailGrpc.Vector.toBooleanVectorValue(): BooleanVectorValue = when (this.vectorDataCase) { - CottontailGrpc.Vector.VectorDataCase.DOUBLEVECTOR -> BooleanVectorValue(this.doubleVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.FLOATVECTOR -> BooleanVectorValue(this.floatVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.LONGVECTOR -> BooleanVectorValue(this.longVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.INTVECTOR -> BooleanVectorValue(this.intVector.vectorList) - CottontailGrpc.Vector.VectorDataCase.BOOLVECTOR -> BooleanVectorValue(this.boolVector.vectorList.toTypedArray()) + CottontailGrpc.Vector.VectorDataCase.DOUBLE -> BooleanVectorValue(this.double.vectorList) + CottontailGrpc.Vector.VectorDataCase.FLOAT -> BooleanVectorValue(this.float.vectorList) + CottontailGrpc.Vector.VectorDataCase.HALF -> BooleanVectorValue(this.half.vectorList) + CottontailGrpc.Vector.VectorDataCase.LONG -> BooleanVectorValue(this.long.vectorList) + CottontailGrpc.Vector.VectorDataCase.INT -> BooleanVectorValue(this.int.vectorList) + CottontailGrpc.Vector.VectorDataCase.SHORT -> BooleanVectorValue(this.short.vectorList) + CottontailGrpc.Vector.VectorDataCase.BOOL -> BooleanVectorValue(this.bool.vectorList.toTypedArray()) else -> throw IllegalArgumentException("${this.vectorDataCase} cannot be cast to VECTOR[BOOL].") } @@ -716,20 +776,19 @@ fun CottontailGrpc.Vector.toBooleanVectorValue(): BooleanVectorValue = when (thi * @throws IllegalArgumentException If cast is not possible. */ fun CottontailGrpc.Vector.toComplex32VectorValue(): Complex32VectorValue = when (this.vectorDataCase) { - CottontailGrpc.Vector.VectorDataCase.DOUBLEVECTOR -> Complex32VectorValue(Array(this.doubleVector.vectorList.size) { Complex32Value(this.doubleVector.vectorList[it], 0.0f) }) - CottontailGrpc.Vector.VectorDataCase.FLOATVECTOR -> Complex32VectorValue(Array(this.floatVector.vectorList.size) { Complex32Value(this.floatVector.vectorList[it], 0.0f) }) - CottontailGrpc.Vector.VectorDataCase.LONGVECTOR -> Complex32VectorValue(Array(this.longVector.vectorList.size) { Complex32Value(this.longVector.vectorList[it], 0.0f) }) - CottontailGrpc.Vector.VectorDataCase.INTVECTOR -> Complex32VectorValue(Array(this.intVector.vectorList.size) { Complex32Value(this.intVector.vectorList[it], 0.0f) }) - CottontailGrpc.Vector.VectorDataCase.COMPLEX32VECTOR -> Complex32VectorValue(Array(this.complex32Vector.vectorList.size) { - Complex32Value( - FloatValue(this.complex32Vector.vectorList[it].real), - FloatValue(this.complex32Vector.vectorList[it].imaginary) - ) + CottontailGrpc.Vector.VectorDataCase.DOUBLE -> Complex32VectorValue(Array(this.double.vectorList.size) { Complex32Value(this.double.vectorList[it], 0.0f) }) + CottontailGrpc.Vector.VectorDataCase.FLOAT -> Complex32VectorValue(Array(this.float.vectorList.size) { Complex32Value(this.float.vectorList[it], 0.0f) }) + CottontailGrpc.Vector.VectorDataCase.HALF -> Complex32VectorValue(Array(this.half.vectorList.size) { Complex32Value(this.half.vectorList[it], 0.0f) }) + CottontailGrpc.Vector.VectorDataCase.LONG -> Complex32VectorValue(Array(this.long.vectorList.size) { Complex32Value(this.long.vectorList[it], 0.0f) }) + CottontailGrpc.Vector.VectorDataCase.INT-> Complex32VectorValue(Array(this.int.vectorList.size) { Complex32Value(this.int.vectorList[it], 0.0f) }) + CottontailGrpc.Vector.VectorDataCase.SHORT-> Complex32VectorValue(Array(this.short.vectorList.size) { Complex32Value(this.short.vectorList[it], 0.0f) }) + CottontailGrpc.Vector.VectorDataCase.COMPLEX32 -> Complex32VectorValue(Array(this.complex32.vectorList.size) { + Complex32Value(this.complex32.vectorList[it].real,this.complex32.vectorList[it].imaginary) }) - CottontailGrpc.Vector.VectorDataCase.COMPLEX64VECTOR -> Complex32VectorValue(Array(this.complex64Vector.vectorList.size) { + CottontailGrpc.Vector.VectorDataCase.COMPLEX64 -> Complex32VectorValue(Array(this.complex64.vectorList.size) { Complex32Value( - FloatValue(this.complex64Vector.vectorList[it].real), - FloatValue(this.complex64Vector.vectorList[it].imaginary) + FloatValue(this.complex64.vectorList[it].real), + FloatValue(this.complex64.vectorList[it].imaginary) ) }) else -> throw IllegalArgumentException("${this.vectorDataCase} cannot be cast to VECTOR[COMPLEX32].") @@ -742,11 +801,17 @@ fun CottontailGrpc.Vector.toComplex32VectorValue(): Complex32VectorValue = when * @throws IllegalArgumentException If cast is not possible. */ fun CottontailGrpc.Vector.toComplex64VectorValue(): Complex64VectorValue = when (this.vectorDataCase) { - CottontailGrpc.Vector.VectorDataCase.DOUBLEVECTOR -> Complex64VectorValue(Array(this.doubleVector.vectorList.size) { Complex64Value(this.doubleVector.vectorList[it], 0.0) }) - CottontailGrpc.Vector.VectorDataCase.FLOATVECTOR -> Complex64VectorValue(Array(this.floatVector.vectorList.size) { Complex64Value(this.floatVector.vectorList[it], 0.0) }) - CottontailGrpc.Vector.VectorDataCase.LONGVECTOR -> Complex64VectorValue(Array(this.longVector.vectorList.size) { Complex64Value(this.longVector.vectorList[it], 0.0) }) - CottontailGrpc.Vector.VectorDataCase.INTVECTOR -> Complex64VectorValue(Array(this.intVector.vectorList.size) { Complex64Value(this.intVector.vectorList[it], 0.0) }) - CottontailGrpc.Vector.VectorDataCase.COMPLEX32VECTOR -> Complex64VectorValue(Array(this.complex32Vector.vectorList.size) { Complex64Value(this.complex32Vector.vectorList[it].real, this.complex32Vector.vectorList[it].imaginary) }) - CottontailGrpc.Vector.VectorDataCase.COMPLEX64VECTOR -> Complex64VectorValue(Array(this.complex64Vector.vectorList.size) { Complex64Value(this.complex64Vector.vectorList[it].real, this.complex64Vector.vectorList[it].imaginary) }) + CottontailGrpc.Vector.VectorDataCase.DOUBLE -> Complex64VectorValue(Array(this.double.vectorList.size) { Complex64Value(this.double.vectorList[it], 0.0) }) + CottontailGrpc.Vector.VectorDataCase.FLOAT-> Complex64VectorValue(Array(float.vectorList.size) { Complex64Value(float.vectorList[it], 0.0) }) + CottontailGrpc.Vector.VectorDataCase.HALF -> Complex64VectorValue(Array(this.half.vectorList.size) { Complex64Value(this.half.vectorList[it], 0.0) }) + CottontailGrpc.Vector.VectorDataCase.LONG -> Complex64VectorValue(Array(this.long.vectorList.size) { Complex64Value(this.long.vectorList[it], 0.0) }) + CottontailGrpc.Vector.VectorDataCase.INT -> Complex64VectorValue(Array(this.int.vectorList.size) { Complex64Value(this.int.vectorList[it], 0.0) }) + CottontailGrpc.Vector.VectorDataCase.SHORT-> Complex64VectorValue(Array(this.short.vectorList.size) { Complex32Value(this.short.vectorList[it], 0.0) }) + CottontailGrpc.Vector.VectorDataCase.COMPLEX32 -> Complex64VectorValue(Array(this.complex32.vectorList.size) { + Complex64Value(this.complex32.vectorList[it].real, this.complex32.vectorList[it].imaginary) + }) + CottontailGrpc.Vector.VectorDataCase.COMPLEX64 -> Complex64VectorValue(Array(this.complex64.vectorList.size) { + Complex64Value(this.complex64.vectorList[it].real, this.complex64.vectorList[it].imaginary) + }) else -> throw IllegalArgumentException("${this.vectorDataCase} cannot be cast to VECTOR[COMPLEX64].") } \ No newline at end of file diff --git a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/DescriptionExtensions.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/DescriptionExtensions.kt index 9561a9855..976748c4c 100644 --- a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/DescriptionExtensions.kt +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/DescriptionExtensions.kt @@ -28,6 +28,7 @@ fun PublicValue.toDescription(vectorSeparator: String = ";", max: Int = 4): Stri is BooleanVectorValue -> this.toDescription(vectorSeparator, max) is DoubleVectorValue -> this.toDescription(vectorSeparator, max) is FloatVectorValue -> this.toDescription(vectorSeparator, max) + is HalfVectorValue -> this.toDescription(vectorSeparator, max) is IntVectorValue -> this.toDescription(vectorSeparator, max) is LongVectorValue -> this.toDescription(vectorSeparator, max) is Complex32VectorValue -> this.toDescription(vectorSeparator, max) diff --git a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/types/Types.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/types/Types.kt index 094ec3cfb..ed09266a0 100644 --- a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/types/Types.kt +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/types/Types.kt @@ -350,7 +350,7 @@ sealed class Types { @Serializable @SerialName("HALF_VECTOR") - class HalfVector(override val logicalSize: kotlin.Int): Vector() { + class HalfVector(override val logicalSize: kotlin.Int): Vector() { init { require(this.logicalSize > 0) { "Logical size of a vector must be greater than zero." } } diff --git a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/BooleanVectorValue.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/BooleanVectorValue.kt index 291c07a75..868eed113 100644 --- a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/BooleanVectorValue.kt +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/BooleanVectorValue.kt @@ -57,7 +57,7 @@ value class BooleanVectorValue(val data: BooleanArray) : RealVectorValue, P * @return [CottontailGrpc.Literal] */ override fun toGrpc(): CottontailGrpc.Literal - = CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setBoolVector(CottontailGrpc.BoolVector.newBuilder().addAllVector(this.data.toList()))).build() + = CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setBool(CottontailGrpc.BoolVector.newBuilder().addAllVector(this.data.toList()))).build() /** diff --git a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/Complex32VectorValue.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/Complex32VectorValue.kt index 8b67ee1f9..2c3226b52 100644 --- a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/Complex32VectorValue.kt +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/Complex32VectorValue.kt @@ -129,7 +129,7 @@ value class Complex32VectorValue(val data: FloatArray) : ComplexVectorValue, * @return [CottontailGrpc.Literal] */ override fun toGrpc(): CottontailGrpc.Literal - = CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setDoubleVector(CottontailGrpc.DoubleVector.newBuilder().addAllVector(this.map { it.value }))).build() + = CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setDouble(CottontailGrpc.DoubleVector.newBuilder().addAllVector(this.map { it.value }))).build() /** * Returns the indices of this [DoubleVectorValue]. diff --git a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/FloatVectorValue.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/FloatVectorValue.kt index ee243aa24..6afc0e5f2 100644 --- a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/FloatVectorValue.kt +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/FloatVectorValue.kt @@ -29,7 +29,6 @@ value class FloatVectorValue(val data: FloatArray) : RealVectorValue, Pub constructor(input: FloatBuffer) : this(FloatArray(input.remaining()) { input[it] }) constructor(input: ByteBuffer) : this(input.asFloatBuffer()) - /** The logical size of this [FloatVectorValue]. */ override val logicalSize: Int get() = this.data.size @@ -64,7 +63,7 @@ value class FloatVectorValue(val data: FloatArray) : RealVectorValue, Pub * @return [CottontailGrpc.Literal] */ override fun toGrpc(): CottontailGrpc.Literal - = CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloatVector(CottontailGrpc.FloatVector.newBuilder().addAllVector(this.map { it.value }))).build() + = CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setFloat(CottontailGrpc.FloatVector.newBuilder().addAllVector(this.map { it.value }))).build() /** * Returns the indices of this [FloatVectorValue]. diff --git a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/HalfVectorValue.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/HalfVectorValue.kt new file mode 100644 index 000000000..6e7e3664a --- /dev/null +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/HalfVectorValue.kt @@ -0,0 +1,295 @@ +package org.vitrivr.cottontail.core.values + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import org.vitrivr.cottontail.core.types.* +import org.vitrivr.cottontail.grpc.CottontailGrpc +import java.nio.FloatBuffer +import java.util.* +import kotlin.math.absoluteValue +import kotlin.math.pow + +/** + * + * @author Ralph Gasser + * @version 1.0 + */ +@Serializable +@SerialName("HalfVector") +@JvmInline +value class HalfVectorValue(val data: FloatArray) : RealVectorValue, PublicValue { + + constructor(input: List) : this(FloatArray(input.size) { input[it].toFloat() }) + constructor(input: Array) : this(FloatArray(input.size) { input[it].toFloat() }) + constructor(input: DoubleArray) : this(FloatArray(input.size) { input[it].toFloat() }) + constructor(input: LongArray) : this(FloatArray(input.size) { input[it].toFloat() }) + constructor(input: IntArray) : this(FloatArray(input.size) { input[it].toFloat() }) + constructor(input: FloatBuffer) : this(FloatArray(input.remaining()) { input[it] }) + + /** The logical size of this [HalfVectorValue]. */ + override val logicalSize: Int + get() = this.data.size + + /** The [Types] of this [HalfVectorValue]. */ + override val type: Types<*> + get() = Types.HalfVector(this.logicalSize) + + /** + * Compares this [HalfVectorValue] to another [HalfVectorValue]. The comparison is done lexicographically. + * + * @param other [Value] to compare to. + * @return True if equal, false otherwise. + */ + override fun compareTo(other: Value): Int { + require(other is HalfVectorValue) { "HalfVectorValue can only be compared to another HalfVectorValue. This is a programmer's error!"} + return Arrays.compare(this.data, other.data) + } + + /** + * Checks for equality between this [HalfVectorValue] and the other [Value]. Equality can only be + * established if the other [Value] is also a [HalfVectorValue] and holds the same value. + * + * @param other [Value] to compare to. + * @return True if equal, false otherwise. + */ + override fun isEqual(other: Value): Boolean = (other is HalfVectorValue) && (this.data.contentEquals(other.data)) + + /** + * Converts this [HalfVectorValue] to a [CottontailGrpc.Literal] gRCP representation. + * + * @return [CottontailGrpc.Literal] + */ + override fun toGrpc(): CottontailGrpc.Literal + = CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setHalf(CottontailGrpc.FloatVector.newBuilder().addAllVector(this.map { it.value }))).build() + + /** + * Returns the indices of this [HalfVectorValue]. + * + * @return The indices of this [HalfVectorValue] + */ + override val indices: IntRange + get() = this.data.indices + + /** + * Returns the i-th entry of this [HalfVectorValue]. + * + * @param i Index of the entry. + * @return The value at index i. + */ + override fun get(i: Int): FloatValue = FloatValue(this.data[i]) + + /** + * Returns a sub vector of this [HalfVectorValue] starting at the component [start] and + * containing [length] components. + * + * @param start Index of the first entry of the returned vector. + * @param length how many elements, including start, to return + * + * @return The [HalfVectorValue] representing the sub-vector. + */ + override fun slice(start: Int, length: Int) = HalfVectorValue(this.data.copyOfRange(start, start + length)) + + /** + * Returns the i-th entry of this [HalfVectorValue] as [Boolean]. + * + * @param i Index of the entry. + * @return The value at index i. + */ + override fun getAsBool(i: Int) = this.data[i] != 0.0f + + /** + * Returns true, if this [HalfVectorValue] consists of all zeroes, i.e. [0, 0, ... 0] + * + * @return True, if this [HalfVectorValue] consists of all zeroes + */ + override fun allZeros(): Boolean = this.data.all { it == 0.0f } + + /** + * Returns true, if this [HalfVectorValue] consists of all ones, i.e. [1, 1, ... 1] + * + * @return True, if this [HalfVectorValue] consists of all ones + */ + override fun allOnes(): Boolean = this.data.all { it == 1.0f } + + /** + * Creates and returns a copy of this [HalfVectorValue]. + * + * @return Exact copy of this [HalfVectorValue]. + */ + override fun copy(): HalfVectorValue = HalfVectorValue(this.data.copyOf(this.data.size)) + + /** + * Creates and returns a new instance of [HalfVectorValue] of the same size. + * + * @return New instance of [HalfVectorValue] + */ + override fun new(): HalfVectorValue = HalfVectorValue(FloatArray(this.data.size)) + + override fun plus(other: VectorValue<*>) = when (other) { + is HalfVectorValue -> HalfVectorValue(FloatArray(this.data.size) { + (this.data[it] + other.data[it]) + }) + else -> HalfVectorValue(FloatArray(this.data.size) { + (this.data[it] + other[it].asFloat().value) + }) + } + + override operator fun minus(other: VectorValue<*>) = when (other) { + is HalfVectorValue -> HalfVectorValue(FloatArray(this.data.size) { + (this.data[it] - other.data[it]) + }) + else -> HalfVectorValue(FloatArray(this.data.size) { + (this.data[it] - other[it].asFloat().value) + }) + } + + override fun times(other: VectorValue<*>) = when (other) { + is HalfVectorValue -> HalfVectorValue(FloatArray(this.data.size) { + (this.data[it] * other.data[it]) + }) + else -> HalfVectorValue(FloatArray(this.data.size) { + (this.data[it] * other[it].asFloat().value) + }) + } + + override fun div(other: VectorValue<*>) = when (other) { + is HalfVectorValue -> HalfVectorValue(FloatArray(this.data.size) { + (this.data[it] / other.data[it]) + }) + else -> HalfVectorValue(FloatArray(this.data.size) { + (this.data[it] / other[it].asFloat().value) + }) + } + + override fun plus(other: NumericValue<*>): HalfVectorValue { + val otherAsFloat = other.asFloat().value + return HalfVectorValue(FloatArray(this.logicalSize) { + (this.data[it] + otherAsFloat) + }) + } + + override fun minus(other: NumericValue<*>): HalfVectorValue { + val otherAsFloat = other.asFloat().value + return HalfVectorValue(FloatArray(this.logicalSize) { + (this.data[it] - otherAsFloat) + }) + } + + override fun times(other: NumericValue<*>): HalfVectorValue { + val otherAsFloat = other.asFloat().value + return HalfVectorValue(FloatArray(this.logicalSize) { + (this.data[it] * otherAsFloat) + }) + } + + override fun div(other: NumericValue<*>): HalfVectorValue { + val otherAsFloat = other.asFloat().value + return HalfVectorValue(FloatArray(this.logicalSize) { + (this.data[it] / otherAsFloat) + }) + } + + override fun pow(x: Int) = HalfVectorValue(FloatArray(this.data.size) { + this.data[it].pow(x) + }) + + override fun sqrt() = HalfVectorValue(FloatArray(this.data.size) { + kotlin.math.sqrt(this.data[it]) + }) + + override fun abs() = HalfVectorValue(FloatArray(this.data.size) { + kotlin.math.abs(this.data[it]) + }) + + override fun sum(): FloatValue = FloatValue(this.data.sum()) + + override fun norm2(): FloatValue { + var sum = 0.0f + for (i in this.indices) { + sum += this.data[i].pow(2) + } + return FloatValue(kotlin.math.sqrt(sum)) + } + + override fun dot(other: VectorValue<*>): FloatValue = when (other) { + is HalfVectorValue -> { + var sum = 0.0f + for (i in this.data.indices) { + sum = Math.fma(this.data[i], other.data[i], sum) + } + FloatValue(sum) + } + else -> { + var sum = 0.0f + for (i in this.data.indices) { + sum += Math.fma(this.data[i], other[i].value.toFloat(), sum) + } + FloatValue(sum) + } + } + + override fun l1(other: VectorValue<*>): DoubleValue = when (other) { + is HalfVectorValue -> { + var sum = 0.0 + for (i in this.data.indices) { + sum += kotlin.math.abs(this.data[i] - other.data[i]) + } + DoubleValue(sum) + } + else -> { + var sum = 0.0 + for (i in this.data.indices) { + sum += kotlin.math.abs(this.data[i] - other[i].value.toDouble()) + } + DoubleValue(sum) + } + } + + override fun l2(other: VectorValue<*>): DoubleValue = when (other) { + is HalfVectorValue -> { + var sum = 0.0 + for (i in this.data.indices) { + sum += (this.data[i] - other.data[i]).pow(2) + } + DoubleValue(kotlin.math.sqrt(sum)) + } + else -> { + var sum = 0.0 + for (i in this.data.indices) { + sum += (this.data[i] - other[i].value.toDouble()).pow(2) + } + DoubleValue(kotlin.math.sqrt(sum)) + } + } + + override fun lp(other: VectorValue<*>, p: Int) = when (other) { + is HalfVectorValue -> { + var sum = 0.0 + for (i in this.data.indices) { + sum += (this.data[i] - other.data[i]).absoluteValue.pow(p) + } + DoubleValue(sum.pow(1.0 / p)) + } + else -> { + var sum = 0.0 + for (i in this.data.indices) { + sum += (this.data[i] - other[i].value.toDouble()).absoluteValue.pow(p) + } + DoubleValue(sum.pow(1.0 / p)) + } + } + + override fun hamming(other: VectorValue<*>): FloatValue = when (other) { + is HalfVectorValue -> { + var sum = 0f + val start = Arrays.mismatch(this.data, other.data) + for (i in start until other.data.size) { + if (this.data[i] != other.data[i]) { + sum += 1.0f + } + } + FloatValue(sum) + } + else -> FloatValue(this.data.size) + } +} \ No newline at end of file diff --git a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/IntVectorValue.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/IntVectorValue.kt index 585a3206b..1b389c8b9 100644 --- a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/IntVectorValue.kt +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/IntVectorValue.kt @@ -61,7 +61,7 @@ value class IntVectorValue(val data: IntArray) : RealVectorValue, PublicVal * @return [CottontailGrpc.Literal] */ override fun toGrpc(): CottontailGrpc.Literal - = CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setIntVector(CottontailGrpc.IntVector.newBuilder().addAllVector(this.map { it.value }))).build() + = CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setInt(CottontailGrpc.IntVector.newBuilder().addAllVector(this.map { it.value }))).build() /** diff --git a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/LongVectorValue.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/LongVectorValue.kt index bed24ee55..b540ede3f 100644 --- a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/LongVectorValue.kt +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/LongVectorValue.kt @@ -61,7 +61,7 @@ value class LongVectorValue(val data: LongArray) : RealVectorValue, Public * @return [CottontailGrpc.Literal] */ override fun toGrpc(): CottontailGrpc.Literal - = CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setLongVector(CottontailGrpc.LongVector.newBuilder().addAllVector(this.map { it.value }))).build() + = CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setLong(CottontailGrpc.LongVector.newBuilder().addAllVector(this.map { it.value }))).build() /** * Returns the indices of this [LongVectorValue]. diff --git a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/PublicValue.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/PublicValue.kt index db8797171..adf4089d7 100644 --- a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/PublicValue.kt +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/PublicValue.kt @@ -18,5 +18,4 @@ sealed interface PublicValue: Value { * @return [CottontailGrpc.Literal] */ fun toGrpc(): CottontailGrpc.Literal - } \ No newline at end of file diff --git a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/ShortVectorValue.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/ShortVectorValue.kt index 737605dc0..a6c6a4ae7 100644 --- a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/ShortVectorValue.kt +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/values/ShortVectorValue.kt @@ -5,7 +5,6 @@ import kotlinx.serialization.Serializable import org.vitrivr.cottontail.core.types.* import org.vitrivr.cottontail.grpc.CottontailGrpc import java.nio.ByteBuffer -import java.nio.LongBuffer import java.nio.ShortBuffer import java.util.* import kotlin.math.pow @@ -60,7 +59,7 @@ value class ShortVectorValue(val data: ShortArray) : RealVectorValue, Pub * @return [CottontailGrpc.Literal] */ override fun toGrpc(): CottontailGrpc.Literal - = CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setIntVector(CottontailGrpc.IntVector.newBuilder().addAllVector(this.map { it.value.toInt() }))).build() + = CottontailGrpc.Literal.newBuilder().setVectorData(CottontailGrpc.Vector.newBuilder().setShort(CottontailGrpc.IntVector.newBuilder().addAllVector(this.map { it.value.toInt() }))).build() /** * Returns the indices of this [ShortVectorValue]. diff --git a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/serialization/SerializationExtension.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/serialization/SerializationExtension.kt index c560c4a23..266aee492 100644 --- a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/serialization/SerializationExtension.kt +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/serialization/SerializationExtension.kt @@ -31,8 +31,8 @@ fun Types<*>.serializer(): KSerializer = when (this) { is Types.Complex32Vector -> Complex32VectorValue.serializer() is Types.Complex64Vector -> Complex64VectorValue.serializer() is Types.DoubleVector -> DoubleVectorValue.serializer() - is Types.HalfVector, is Types.FloatVector -> FloatVectorValue.serializer() + is Types.HalfVector -> HalfVectorValue.serializer() is Types.IntVector -> IntVectorValue.serializer() is Types.LongVector -> LongVectorValue.serializer() is Types.ShortVector -> ShortVectorValue.serializer() diff --git a/cottontaildb-client/src/main/protobuf/cottontail.proto b/cottontaildb-client/src/main/protobuf/cottontail.proto index ca3df87ff..4069fdf46 100644 --- a/cottontaildb-client/src/main/protobuf/cottontail.proto +++ b/cottontaildb-client/src/main/protobuf/cottontail.proto @@ -290,13 +290,15 @@ message Expressions { /** Vector data (as opposed to scalar data). */ message Vector { oneof vectorData { - FloatVector floatVector = 1; - DoubleVector doubleVector = 2; - IntVector intVector = 3; - LongVector longVector = 4; - BoolVector boolVector = 5; - Complex32Vector complex32Vector = 6; - Complex64Vector complex64Vector = 7; + FloatVector half = 1; + FloatVector float = 2; + DoubleVector double = 3; + IntVector short = 4; + IntVector int = 5; + LongVector long = 6; + BoolVector bool = 7; + Complex32Vector complex32 = 8; + Complex64Vector complex64 = 9; } } diff --git a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/queries/functions/math/distance/binary/EuclideanDistance.kt b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/queries/functions/math/distance/binary/EuclideanDistance.kt index bc19b91af..b1f621e07 100644 --- a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/queries/functions/math/distance/binary/EuclideanDistance.kt +++ b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/queries/functions/math/distance/binary/EuclideanDistance.kt @@ -36,6 +36,8 @@ sealed class EuclideanDistance>(type: Types.Vector): Min is Types.Complex32Vector -> Complex32Vector(type) is Types.DoubleVector -> DoubleVector(type) is Types.FloatVector -> FloatVector(type) + is Types.HalfVector -> HalfVector(type) + is Types.LongVector -> LongVector(type) is Types.IntVector -> IntVector(type) else -> throw FunctionNotSupportedException("Function generator ${Companion.signature} cannot generate function with signature $signature.") @@ -145,6 +147,36 @@ sealed class EuclideanDistance>(type: Types.Vector): Min override fun vectorized() = FloatVectorVectorized(this.type) } + /** + * [EuclideanDistance] for a [FloatVectorValue]. + */ + class HalfVector(type: Types.Vector): EuclideanDistance(type) { + override val name: Name.FunctionName = FUNCTION_NAME + override fun invoke(vararg arguments: Value?): DoubleValue { + val probing = arguments[0] as HalfVectorValue + val query = arguments[1] as HalfVectorValue + var sum = 0.0 + for (i in 0 until this.vectorSize) { + sum += (query.data[i] - probing.data[i]).pow(2) + } + return DoubleValue(sqrt(sum)) + } + + override fun invokeOrMaximum(left: VectorValue<*>, right: VectorValue<*>, maximum: DoubleValue): DoubleValue { + val max = maximum.value.pow(2) + val probing = left as HalfVectorValue + val query = right as HalfVectorValue + var sum = 0.0 + for (i in 0 until this.vectorSize) { + sum += (query.data[i] - probing.data[i]).pow(2) + if (sum >= max) return maximum + } + return DoubleValue(sqrt(sum)) + } + + override fun copy(d: Int) = HalfVector(Types.HalfVector(d)) + } + /** * SIMD implementation: [EuclideanDistance] for a [FloatVectorValue] */ diff --git a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/generators/FloatVectorValueGenerator.kt b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/generators/FloatVectorValueGenerator.kt index 003718443..c83c7cdc4 100644 --- a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/generators/FloatVectorValueGenerator.kt +++ b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/generators/FloatVectorValueGenerator.kt @@ -1,12 +1,11 @@ package org.vitrivr.cottontail.core.values.generators import org.vitrivr.cottontail.core.types.VectorValue -import org.vitrivr.cottontail.core.values.DoubleVectorValue import org.vitrivr.cottontail.core.values.FloatVectorValue import java.util.random.RandomGenerator /** - * A [VectorValueGenerator] for [DoubleVectorValue]s. + * A [VectorValueGenerator] for [FloatVectorValue]s. * * @author Ralph Gasser * @version 1.0.0 diff --git a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/generators/HalfVectorValueGenerator.kt b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/generators/HalfVectorValueGenerator.kt new file mode 100644 index 000000000..2c2caa5dd --- /dev/null +++ b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/generators/HalfVectorValueGenerator.kt @@ -0,0 +1,47 @@ +package org.vitrivr.cottontail.core.values.generators + +import org.vitrivr.cottontail.core.types.VectorValue +import org.vitrivr.cottontail.core.values.FloatVectorValue +import org.vitrivr.cottontail.core.values.HalfVectorValue +import java.util.random.RandomGenerator + +/** + * A [VectorValueGenerator] for [HalfVectorValue]s. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +object HalfVectorValueGenerator: VectorValueGenerator { + /** + * Generates a [HalfVectorValue] of the given size initialized with random numbers. + * + * @param size Size of the new [HalfVectorValue] + * @param rnd A [RandomGenerator] to generate the random numbers. + * @return The generated [HalfVectorValue] + */ + override fun random(size: Int, rnd: RandomGenerator) = HalfVectorValue(FloatArray(size) { rnd.nextFloat() }) + + /** + * Generates a [HalfVectorValue] of the given size initialized with ones. + * + * @param size Size of the new [HalfVectorValue] + * @return The generated [HalfVectorValue] + */ + override fun one(size: Int) = HalfVectorValue(FloatArray(size) { 1.0f }) + + /** + * Generates a [FloatVectorValue] of the given size initialized with zeros. + * + * @param size Size of the new [HalfVectorValue] + * @return The generated [HalfVectorValue] + */ + override fun zero(size: Int) = HalfVectorValue(FloatArray(size)) + + /** + * Generates a [HalfVectorValue] given [Array] of [Number]s + * + * @param values List of [Number]s to generate the [VectorValue] for. + * @return [HalfVectorValue] + */ + override fun with(values: Array) = HalfVectorValue(values) +} \ No newline at end of file diff --git a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/FloatVectorTablet.kt b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/FloatVectorTablet.kt index 41b10b7bc..7d1d115dc 100644 --- a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/FloatVectorTablet.kt +++ b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/FloatVectorTablet.kt @@ -4,7 +4,7 @@ import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.FloatVectorValue /** - * A [AbstractTablet] implementation for [FloatVectorValue]s. + * A [AbstractTablet] implementation for [FloatVectorValue]s (single-precision). * * @author Ralph Gasser * @version 1.0.0 diff --git a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/HalfVectorTablet.kt b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/HalfVectorTablet.kt new file mode 100644 index 000000000..95b402a80 --- /dev/null +++ b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/HalfVectorTablet.kt @@ -0,0 +1,26 @@ +package org.vitrivr.cottontail.core.values.tablets + +import org.vitrivr.cottontail.core.types.Types +import org.vitrivr.cottontail.core.values.FloatVectorValue +import org.vitrivr.cottontail.core.values.HalfVectorValue +import org.vitrivr.cottontail.utilities.math.Half + +/** + * A [AbstractTablet] implementation for [FloatVectorValue]s (half-precision). + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class HalfVectorTablet(size: Int, logicalSize: Int, direct: Boolean): AbstractTablet(size, Types.HalfVector(logicalSize), direct) { + override fun internalGet(index: Int): HalfVectorValue { + val buffer = this.buffer.slice(indexToPosition(index), this.type.physicalSize) + return HalfVectorValue(FloatArray(this.type.logicalSize) { Half(buffer.getShort().toUShort()).toFloat() }) + } + override fun internalSet(index: Int, value: HalfVectorValue) { + this.buffer.position(indexToPosition(index)) + value.data.forEach { + this.buffer.putShort(Half(it).v.toShort()) + } + this.buffer.position(0) + } +} \ No newline at end of file diff --git a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/Tablet.kt b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/Tablet.kt index a597967bf..c3c4d88e5 100644 --- a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/Tablet.kt +++ b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/Tablet.kt @@ -39,6 +39,7 @@ sealed interface Tablet { is Types.Complex64Vector -> Complex64VectorTablet(size, types.logicalSize, direct) is Types.DoubleVector -> DoubleVectorTablet(size, types.logicalSize, direct) is Types.FloatVector -> FloatVectorTablet(size, types.logicalSize, direct) + is Types.HalfVector -> HalfVectorTablet(size, types.logicalSize, direct) is Types.IntVector -> IntVectorTablet(size, types.logicalSize, direct) is Types.LongVector -> LongVectorTablet(size, types.logicalSize, direct) else -> throw UnsupportedOperationException("The type $types cannot be represented in a tablet.") diff --git a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/hashing/ValueFunnel.kt b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/hashing/ValueFunnel.kt index 31845c54e..a0d0d16d7 100644 --- a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/hashing/ValueFunnel.kt +++ b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/hashing/ValueFunnel.kt @@ -74,6 +74,10 @@ object ValueFunnel: Funnel { into.putInt(value.logicalSize) value.data.forEach { into.putFloat(it) } } + is HalfVectorValue -> { + into.putInt(value.logicalSize) + value.data.forEach { into.putFloat(it) } + } is DoubleVectorValue -> { into.putInt(value.logicalSize) value.data.forEach { into.putDouble(it) } diff --git a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/math/Half.kt b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/math/Half.kt index 69198ee0a..ec04faa84 100644 --- a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/math/Half.kt +++ b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/math/Half.kt @@ -22,7 +22,6 @@ package org.vitrivr.cottontail.utilities.math import org.vitrivr.cottontail.utilities.math.Half.Companion.POSITIVE_INFINITY import org.vitrivr.cottontail.utilities.math.Half.Companion.POSITIVE_ZERO -import kotlin.jvm.JvmInline /** * Converts the specified double-precision float value into a @@ -247,7 +246,7 @@ fun String.toHalf() = Half(floatToHalf(toFloat())) * This table shows that numbers higher than 1024 lose all fractional precision. */ @JvmInline -value class Half(private val v: UShort) : Comparable { +value class Half(public val v: UShort) : Comparable { companion object { /** * The number of bits used to represent a half-precision float value. @@ -401,33 +400,33 @@ value class Half(private val v: UShort) : Comparable { * Returns the value of this [Half] as a `byte` after a narrowing primitive conversion. * * @return The half-precision float value represented by this object converted to type `byte` - * @see halfToShort + * @see halfToFloat */ - fun toByte() = halfToShort(v).toInt().toByte() + fun toByte() = halfToFloat(v).toInt().toByte() /** * Returns the value of this [Half] as a `short` after a narrowing primitive conversion. * * @return The half-precision float value represented by this object converted to type `short` - * @see halfToShort + * @see halfToFloat */ - fun toShort() = halfToShort(v).toInt().toShort() + fun toShort() = halfToFloat(v).toInt().toShort() /** * Returns the value of this [Half] as a `int` after a narrowing primitive conversion. * * @return The half-precision float value represented by this object converted to type `int` - * @see halfToShort + * @see halfToFloat * */ - fun toInt() = halfToShort(v).toInt() + fun toInt() = halfToFloat(v).toInt() /** * Returns the value of this [Half] as a `long` after a narrowing primitive conversion. * * @return The half-precision float value represented by this object converted to type `long` - * @see halfToShort + * @see halfToFloat */ - fun toLong() = halfToShort(v).toLong() + fun toLong() = halfToFloat(v).toLong() /** * Returns the value of this [Half] as a `float` after a widening primitive conversion. @@ -442,7 +441,7 @@ value class Half(private val v: UShort) : Comparable { * * @return The half-precision float value represented by this object converted to type `float` */ - fun toFloat() = halfToShort(v) + fun toFloat() = halfToFloat(v) /** * Returns the value of this [Half] as a `double` after a widening primitive conversion. @@ -457,7 +456,7 @@ value class Half(private val v: UShort) : Comparable { * * @return The half-precision float value represented by this object converted to type `double` */ - fun toDouble() = halfToShort(v).toDouble() + fun toDouble() = halfToFloat(v).toDouble() /** * Returns true if this half-precision float value represents a Not-a-Number, false otherwise. @@ -1077,7 +1076,7 @@ private const val FP32_QNAN_MASK = 0x400000 private const val FP32_DENORMAL_MAGIC = 126 shl 23 private val FP32_DENORMAL_FLOAT = Float.fromBits(FP32_DENORMAL_MAGIC) -private fun floatToHalf(f: Float): UShort { +public fun floatToHalf(f: Float): UShort { val bits: Int = f.toBits() val s = bits ushr FP32_SIGN_SHIFT var e = (bits ushr FP32_EXPONENT_SHIFT) and FP32_EXPONENT_MASK @@ -1115,7 +1114,7 @@ private fun floatToHalf(f: Float): UShort { return (s shl FP16_SIGN_SHIFT or (outE shl FP16_EXPONENT_SHIFT) or outM).toUShort() } -private fun halfToShort(h: UShort): Float { +private fun halfToFloat(h: UShort): Float { val bits = h.toInt() val s = bits and FP16_SIGN_MASK val e = bits ushr FP16_EXPONENT_SHIFT and FP16_EXPONENT_MASK diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/queries/binding/GrpcQueryBinder.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/queries/binding/GrpcQueryBinder.kt index 6fa23c108..00904ead3 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/queries/binding/GrpcQueryBinder.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/queries/binding/GrpcQueryBinder.kt @@ -173,7 +173,7 @@ object GrpcQueryBinder { try { val tuples: MutableList = insert.insertsList.map { ins -> TupleBinding(-1L, columns, Array(ins.valuesCount) { i -> - val literal = ins.valuesList[i].toValue() + val literal = ins.valuesList[i].toValue(columns[i].type) if (literal == null) { this@QueryContext.bindings.bindNull(columns[i].type) } else { diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tablets/LZ4TabletSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tablets/LZ4TabletSerializer.kt index be47fbb89..d67af6038 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tablets/LZ4TabletSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tablets/LZ4TabletSerializer.kt @@ -43,7 +43,9 @@ class LZ4TabletSerializer(override val type: Types, val size: Int): override fun fromEntry(entry: ByteIterable): Tablet { /* Transfer data into buffer. */ this.compressBuffer.clear() - for (b in entry) { this.compressBuffer.put(b) } + for (b in entry) { + this.compressBuffer.put(b) + } /* Decompress payload using LZ4. */ val tablet = Tablet.of(this.size, this.type) diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tablets/SnappyTabletSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tablets/SnappyTabletSerializer.kt index ee836ba66..18425962e 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tablets/SnappyTabletSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tablets/SnappyTabletSerializer.kt @@ -42,6 +42,7 @@ class SnappyTabletSerializer(override val type: Types, val size: In is Types.LongVector -> BitShuffleType.LONG Types.Float, Types.Complex32, + is Types.HalfVector, is Types.FloatVector, is Types.Complex32Vector -> BitShuffleType.FLOAT Types.Int, diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tuples/TupleSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tuples/TupleSerializer.kt index ab06384ef..90cb366c3 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tuples/TupleSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tuples/TupleSerializer.kt @@ -40,6 +40,7 @@ class TupleSerializer(val schema: Array>) { is DoubleVectorValue -> value.logicalSize * Float.SIZE_BYTES is FloatValue -> Float.SIZE_BYTES is FloatVectorValue -> value.logicalSize * Float.SIZE_BYTES + is HalfVectorValue -> value.logicalSize * Short.SIZE_BYTES is IntValue -> Int.SIZE_BYTES is IntVectorValue -> value.logicalSize * Int.SIZE_BYTES is LongValue -> Long.SIZE_BYTES @@ -169,6 +170,7 @@ class TupleSerializer(val schema: Array>) { is BooleanVectorValue -> TODO() is DoubleVectorValue -> value.data.forEach { buffer.putDouble(it) } is FloatVectorValue -> value.data.forEach { buffer.putFloat(it) } + is HalfVectorValue -> value.data.forEach { buffer.putShort(Half(it).toShort()) } is IntVectorValue -> value.data.forEach { buffer.putInt(it) } is LongVectorValue -> value.data.forEach { buffer.putLong(it) } is ShortVectorValue -> value.data.forEach { buffer.putShort(it) } diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/HalfVectorValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/HalfVectorValueValueSerializer.kt index d4d06e87a..32ad8cd6d 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/HalfVectorValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/HalfVectorValueValueSerializer.kt @@ -4,6 +4,7 @@ import jetbrains.exodus.ArrayByteIterable import jetbrains.exodus.ByteIterable import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.FloatVectorValue +import org.vitrivr.cottontail.core.values.HalfVectorValue import org.vitrivr.cottontail.utilities.math.Half import org.xerial.snappy.Snappy import java.nio.ByteBuffer @@ -11,25 +12,21 @@ import java.nio.ByteBuffer /** * A [ValueSerializer] for [FloatVectorValue] serialization and deserialization with 16 bit precision. */ -class HalfVectorValueValueSerializer(size: Int) : ValueSerializer { +class HalfVectorValueValueSerializer(size: Int) : ValueSerializer { init { require(size > 0) { "Cannot initialize vector value binding with size value of $size." } } - override val type: Types = Types.HalfVector(size) - override fun fromEntry(entry: ByteIterable): FloatVectorValue { - + override val type: Types = Types.HalfVector(size) + override fun fromEntry(entry: ByteIterable): HalfVectorValue { val buffer = ByteBuffer.wrap(Snappy.uncompress(entry.bytesUnsafe)).asShortBuffer() val floats = FloatArray(buffer.remaining()) { Half(buffer[it].toUShort()).toFloat() } - - return FloatVectorValue(floats) + return HalfVectorValue(floats) } - override fun toEntry(value: FloatVectorValue): ByteIterable { - + override fun toEntry(value: HalfVectorValue): ByteIterable { val halfs = ShortArray(value.data.size) { Half(value.data[it]).toShort() } - val compressed = Snappy.compress(halfs) return ArrayByteIterable(compressed, compressed.size) } diff --git a/cottontaildb-dbms/src/test/kotlin/org/vitrivr/cottontail/storage/serializer/tablets/TabletSerializationTest.kt b/cottontaildb-dbms/src/test/kotlin/org/vitrivr/cottontail/storage/serializer/tablets/TabletSerializationTest.kt index daa01f28a..29d39b609 100644 --- a/cottontaildb-dbms/src/test/kotlin/org/vitrivr/cottontail/storage/serializer/tablets/TabletSerializationTest.kt +++ b/cottontaildb-dbms/src/test/kotlin/org/vitrivr/cottontail/storage/serializer/tablets/TabletSerializationTest.kt @@ -18,7 +18,7 @@ import java.util.* * A series of test cases that test various flavours of [TabletSerializer] * * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ class TabletSerializationTest { /** The [SplittableRandom] used to generate random numbers. */ @@ -27,8 +27,6 @@ class TabletSerializationTest { /** The [Tablet] size to test. */ private val tabletSize = 128 - - /** * Tests de-/serialization of [AbstractTablet]s of [IntVectorValue]s. */ @@ -288,6 +286,24 @@ class TabletSerializationTest { this.test(tablet, compression) } + /** + * Tests de-/serialization of [AbstractTablet]s of [FloatVectorValue]s. + */ + @ParameterizedTest + @EnumSource(Compression::class) + fun testHalfVectorTabletSerialization(compression: Compression) { + val type = Types.HalfVector(this.random.nextInt(4, 2048)) + val tablet = Tablet.of(this.tabletSize, type, compression.direct) + repeat(this.tabletSize) { + if (this.random.nextBoolean()) { + tablet[it] = HalfVectorValueGenerator.random(type.logicalSize, this.random) + } + } + + /* Create tablet and test de-/serialization. */ + this.test(tablet, compression) + } + /** * Tests de-/serialization of [AbstractTablet]s of [DoubleVectorValue]s. */ diff --git a/cottontaildb-dbms/src/testFixtures/kotlin/org/vitrivr/cottontail/test/EmbeddedCottontailGrpcServer.kt b/cottontaildb-dbms/src/testFixtures/kotlin/org/vitrivr/cottontail/test/EmbeddedCottontailGrpcServer.kt index d341548ac..27d89370c 100644 --- a/cottontaildb-dbms/src/testFixtures/kotlin/org/vitrivr/cottontail/test/EmbeddedCottontailGrpcServer.kt +++ b/cottontaildb-dbms/src/testFixtures/kotlin/org/vitrivr/cottontail/test/EmbeddedCottontailGrpcServer.kt @@ -5,8 +5,6 @@ import org.vitrivr.cottontail.config.Config import org.vitrivr.cottontail.dbms.catalogue.DefaultCatalogue import org.vitrivr.cottontail.dbms.execution.ExecutionManager import org.vitrivr.cottontail.dbms.execution.services.AutoRebuilderService -import org.vitrivr.cottontail.dbms.statistics.StatisticsManager -import org.vitrivr.cottontail.dbms.execution.transactions.TransactionManager import org.vitrivr.cottontail.server.CottontailServer import org.vitrivr.cottontail.server.grpc.services.DDLService import org.vitrivr.cottontail.server.grpc.services.DMLService