From 8515728dc0a8583c4a997fac49855b15e3785218 Mon Sep 17 00:00:00 2001 From: Joseph Ivie Date: Tue, 21 Nov 2023 15:36:16 -0700 Subject: [PATCH] Null attempt --- .../csv/decode/AbstractDecoderAlt.kt | 68 ++++++++++++++ .../csv/decode/ClassCsvDecoder.kt | 13 ++- .../serialization/csv/decode/CsvDecoder.kt | 83 +++++++++++++++-- .../csv/decode/RootCsvDecoder.kt | 20 +++- .../serialization/csv/encode/CsvEncoder.kt | 47 ++++++++-- .../csv/config/CsvHasHeaderRecordTest.kt | 9 +- .../csv/config/CsvIgnoreUnknownColumnsTest.kt | 6 +- .../csv/records/CsvNestedRecordTest.kt | 93 +++++++++++++++++-- .../csv/records/SampleClasses.kt | 14 ++- 9 files changed, 319 insertions(+), 34 deletions(-) create mode 100644 library/src/main/kotlin/kotlinx/serialization/csv/decode/AbstractDecoderAlt.kt diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/AbstractDecoderAlt.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/AbstractDecoderAlt.kt new file mode 100644 index 0000000..622d235 --- /dev/null +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/AbstractDecoderAlt.kt @@ -0,0 +1,68 @@ +package kotlinx.serialization.csv.decode + +import kotlinx.serialization.DeserializationStrategy +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.SerializationException +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.CompositeDecoder +import kotlinx.serialization.encoding.Decoder + +@ExperimentalSerializationApi +public abstract class AbstractDecoderAlt : Decoder, CompositeDecoder { + + /** + * Invoked to decode a value when specialized `encode*` method was not overridden. + */ + public open fun decodeValue(): Any = throw SerializationException("${this::class} can't retrieve untyped values") + + override fun decodeNotNullMark(): Boolean = true + override fun decodeNull(): Nothing? = null + override fun decodeBoolean(): Boolean = decodeValue() as Boolean + override fun decodeByte(): Byte = decodeValue() as Byte + override fun decodeShort(): Short = decodeValue() as Short + override fun decodeInt(): Int = decodeValue() as Int + override fun decodeLong(): Long = decodeValue() as Long + override fun decodeFloat(): Float = decodeValue() as Float + override fun decodeDouble(): Double = decodeValue() as Double + override fun decodeChar(): Char = decodeValue() as Char + override fun decodeString(): String = decodeValue() as String + override fun decodeEnum(enumDescriptor: SerialDescriptor): Int = decodeValue() as Int + + // overwrite by default + public open fun decodeSerializableValue( + deserializer: DeserializationStrategy, + previousValue: T? = null + ): T = decodeSerializableValue(deserializer) + + override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder = this + + override fun endStructure(descriptor: SerialDescriptor) { + } + + final override fun decodeBooleanElement(descriptor: SerialDescriptor, index: Int): Boolean = decodeBoolean() + final override fun decodeByteElement(descriptor: SerialDescriptor, index: Int): Byte = decodeByte() + final override fun decodeShortElement(descriptor: SerialDescriptor, index: Int): Short = decodeShort() + final override fun decodeIntElement(descriptor: SerialDescriptor, index: Int): Int = decodeInt() + final override fun decodeLongElement(descriptor: SerialDescriptor, index: Int): Long = decodeLong() + final override fun decodeFloatElement(descriptor: SerialDescriptor, index: Int): Float = decodeFloat() + final override fun decodeDoubleElement(descriptor: SerialDescriptor, index: Int): Double = decodeDouble() + final override fun decodeCharElement(descriptor: SerialDescriptor, index: Int): Char = decodeChar() + final override fun decodeStringElement(descriptor: SerialDescriptor, index: Int): String = decodeString() + + final override fun decodeSerializableElement( + descriptor: SerialDescriptor, + index: Int, + deserializer: DeserializationStrategy, + previousValue: T? + ): T = decodeSerializableValue(deserializer, previousValue) + + override fun decodeNullableSerializableElement( + descriptor: SerialDescriptor, + index: Int, + deserializer: DeserializationStrategy, + previousValue: T? + ): T? { + val isNullabilitySupported = deserializer.descriptor.isNullable + return if (isNullabilitySupported || decodeNotNullMark()) decodeSerializableValue(deserializer, previousValue) else decodeNull() + } +} \ No newline at end of file diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/ClassCsvDecoder.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/ClassCsvDecoder.kt index b523ed7..cdab6b2 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/decode/ClassCsvDecoder.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/ClassCsvDecoder.kt @@ -29,17 +29,19 @@ internal class ClassCsvDecoder( elementIndex >= descriptor.elementsCount -> DECODE_DONE classHeaders != null && columnIndex >= classHeaders.size -> DECODE_DONE - classHeaders != null -> + classHeaders != null -> { when (val result = classHeaders[columnIndex]) { UNKNOWN_NAME -> { ignoreColumn() decodeElementIndex(descriptor) } + null -> UNKNOWN_NAME - else -> result + else -> result.also { println("${descriptor.serialName} decoded ${it} (${descriptor.getElementName(it)}) due to column ${columnIndex} in ${classHeaders.toString(descriptor)}") } } + } - else -> elementIndex + else -> elementIndex.also { println("${descriptor.serialName} decoded ${it} (${descriptor.getElementName(it)})") } } override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder { @@ -84,4 +86,9 @@ internal class ClassCsvDecoder( reader.readColumn() columnIndex++ } + + override fun virtualColumnAdvance() { + columnIndex++ + elementIndex++ + } } diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvDecoder.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvDecoder.kt index d64c89e..5760c1b 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvDecoder.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvDecoder.kt @@ -2,15 +2,15 @@ package kotlinx.serialization.csv.decode import kotlinx.serialization.DeserializationStrategy import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.KSerializer +import kotlinx.serialization.builtins.nullable import kotlinx.serialization.csv.Csv -import kotlinx.serialization.csv.HeadersNotSupportedForSerialDescriptorException import kotlinx.serialization.csv.UnknownColumnHeaderException import kotlinx.serialization.csv.UnsupportedSerialDescriptorException import kotlinx.serialization.csv.config.CsvConfig import kotlinx.serialization.descriptors.PolymorphicKind import kotlinx.serialization.descriptors.SerialDescriptor import kotlinx.serialization.descriptors.StructureKind -import kotlinx.serialization.encoding.AbstractDecoder import kotlinx.serialization.encoding.CompositeDecoder import kotlinx.serialization.encoding.CompositeDecoder.Companion.UNKNOWN_NAME import kotlinx.serialization.modules.SerializersModule @@ -24,7 +24,7 @@ internal abstract class CsvDecoder( protected val csv: Csv, protected val reader: CsvReader, private val parent: CsvDecoder? -) : AbstractDecoder() { +) : AbstractDecoderAlt() { override val serializersModule: SerializersModule get() = csv.serializersModule @@ -111,11 +111,18 @@ internal abstract class CsvDecoder( return null } + fun skipEmpty(): Nothing? { + val value = reader.readColumn() + println("SKIPPING $value") + require(value == config.nullString) { "Expected '${config.nullString}' but was '$value'." } + return null + } + override fun decodeEnum(enumDescriptor: SerialDescriptor): Int { return enumDescriptor.getElementIndex(decodeColumn()) } - protected open fun decodeColumn() = reader.readColumn() + protected open fun decodeColumn() = reader.readColumn().also { println("READ COLUMN $it") } protected fun readHeaders(descriptor: SerialDescriptor) { if (config.hasHeaderRecord && headers == null) { @@ -144,6 +151,7 @@ internal abstract class CsvDecoder( val headerIndex = descriptor.getElementIndex(header) if (headerIndex != UNKNOWN_NAME) { headers[position] = headerIndex + println("SET $position TO $header") reader.unmark() } else { val name = header.substringBefore(config.headerSeparator) @@ -152,7 +160,6 @@ internal abstract class CsvDecoder( val childDesc = descriptor.getElementDescriptor(nameIndex) if (childDesc.kind is StructureKind.CLASS) { reader.reset() - headers[position] = nameIndex headers[nameIndex] = readHeaders(childDesc, "$prefix$name.") } else { reader.unmark() @@ -168,7 +175,7 @@ internal abstract class CsvDecoder( } position++ } - return headers + return headers.also { println(it.toString(descriptor)) } } protected fun readTrailingDelimiter() { @@ -197,18 +204,74 @@ internal abstract class CsvDecoder( operator fun set(key: Int, value: Headers) { subHeaders[key] = value } + + override fun toString(): String { + return "Headers(map=${map}, subHeaders=${subHeaders})" + } + fun toString(context: SerialDescriptor): String { + return "Headers(map=${map.mapValues { if(it.value in 0 until context.elementsCount) context.getElementName(it.value) else "???" }})" + } } override fun decodeSerializableValue(deserializer: DeserializationStrategy): T { - return if (config.deferToFormatWhenVariableColumns != null) { + if (config.deferToFormatWhenVariableColumns != null) { when (deserializer.descriptor.kind) { is StructureKind.LIST, is StructureKind.MAP, is PolymorphicKind.OPEN -> { - config.deferToFormatWhenVariableColumns!!.decodeFromString(deserializer, decodeColumn()) + return config.deferToFormatWhenVariableColumns!!.decodeFromString(deserializer, decodeColumn()) + } + else -> {} + } + } + if(deserializer.descriptor.isNullable && deserializer.descriptor.kind == StructureKind.CLASS && deserializer.descriptor.elementsCount > 0) { + val isPresent = reader.readColumn().toBoolean() + println("READ PRESENT $isPresent") + if(isPresent) { + return deserializer.deserialize(this) + } else { + virtualColumnAdvance() + decodeNulls(deserializer.descriptor, deserializer.descriptor.serialName) + @Suppress("UNCHECKED_CAST") + return null as T + } + } else { + return deserializer.deserialize(this) + } + } + + protected open fun virtualColumnAdvance() {} + + private fun decodeNulls(serializer: SerialDescriptor, name: String) { + if(serializer.kind == StructureKind.CLASS) { + for(index in (0 until serializer.elementsCount)) { + val sub = serializer.getElementDescriptor(index) + if(sub.isNullable) { + println("Skipping present ${serializer.getElementName(index)}") + skipEmpty() } - else -> super.decodeSerializableValue(deserializer) + decodeNulls(sub, serializer.getElementName(index)) } - } else super.decodeSerializableValue(deserializer) + println("Skipping ${name} end") + } else { + println("Skipping $name") + skipEmpty() + } + } + + override fun decodeNullableSerializableValue(deserializer: DeserializationStrategy): T? { + val isNullabilitySupported = deserializer.descriptor.isNullable + return if (isNullabilitySupported || decodeNotNullMark()) decodeSerializableValue(deserializer) else decodeSerializableValue((deserializer as KSerializer).nullable) + } + + override fun decodeNullableSerializableElement( + descriptor: SerialDescriptor, + index: Int, + deserializer: DeserializationStrategy, + previousValue: T? + ): T? { + val isNullabilitySupported = deserializer.descriptor.isNullable + return if (isNullabilitySupported) decodeSerializableValue(deserializer, previousValue) else decodeSerializableValue((deserializer as KSerializer).nullable, previousValue) } } + diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/RootCsvDecoder.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/RootCsvDecoder.kt index 0260c37..714a737 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/decode/RootCsvDecoder.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/RootCsvDecoder.kt @@ -2,6 +2,8 @@ package kotlinx.serialization.csv.decode import kotlinx.serialization.DeserializationStrategy import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.KSerializer +import kotlinx.serialization.builtins.nullable import kotlinx.serialization.csv.Csv import kotlinx.serialization.descriptors.PolymorphicKind import kotlinx.serialization.descriptors.SerialDescriptor @@ -55,13 +57,25 @@ internal class RootCsvDecoder( } override fun decodeSerializableValue(deserializer: DeserializationStrategy): T { - return if (config.deferToFormatWhenVariableColumns != null) { + if (config.deferToFormatWhenVariableColumns != null) { when (deserializer.descriptor.kind) { is PolymorphicKind.OPEN -> { config.deferToFormatWhenVariableColumns!!.decodeFromString(deserializer, decodeColumn()) } - else -> deserializer.deserialize(this) + else -> {} } - } else deserializer.deserialize(this) + } + if(deserializer.descriptor.isNullable && deserializer.descriptor.kind == StructureKind.CLASS && deserializer.descriptor.elementsCount > 0) { + val isPresent = decodeBoolean() + println("Decoded present boolean ${isPresent}") + if(isPresent) { + return deserializer.deserialize(this) + } else { + @Suppress("UNCHECKED_CAST") + return null as T + } + } else { + return deserializer.deserialize(this) + } } } diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/encode/CsvEncoder.kt b/library/src/main/kotlin/kotlinx/serialization/csv/encode/CsvEncoder.kt index 477926e..a4c64a0 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/encode/CsvEncoder.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/encode/CsvEncoder.kt @@ -1,14 +1,13 @@ package kotlinx.serialization.csv.encode import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.KSerializer import kotlinx.serialization.SerializationStrategy +import kotlinx.serialization.builtins.nullable import kotlinx.serialization.csv.Csv import kotlinx.serialization.csv.HeadersNotSupportedForSerialDescriptorException import kotlinx.serialization.csv.UnsupportedSerialDescriptorException -import kotlinx.serialization.descriptors.PolymorphicKind -import kotlinx.serialization.descriptors.SerialDescriptor -import kotlinx.serialization.descriptors.SerialKind -import kotlinx.serialization.descriptors.StructureKind +import kotlinx.serialization.descriptors.* import kotlinx.serialization.encoding.AbstractEncoder import kotlinx.serialization.encoding.CompositeEncoder import kotlinx.serialization.modules.SerializersModule @@ -146,6 +145,7 @@ internal abstract class CsvEncoder( Unit childDesc.elementsCount > 0 -> { + if(childDesc.isNullable) writer.printColumn(name) val headerSeparator = config.headerSeparator printHeader("$name$headerSeparator", childDesc) } @@ -179,9 +179,44 @@ internal abstract class CsvEncoder( isNumeric = false, isNull = false ) + return } - else -> super.encodeSerializableValue(serializer, value) + else -> {} } - } else super.encodeSerializableValue(serializer, value) + } + if(serializer.descriptor.isNullable && serializer.descriptor.kind == StructureKind.CLASS && serializer.descriptor.elementsCount > 0) { + if(value == null) { + encodeBoolean(false) + encodeNulls(serializer.descriptor) + } else { + encodeBoolean(true) + serializer.serialize(this, value) + } + } else { + serializer.serialize(this, value) + } + } + + private fun encodeNulls(serializer: SerialDescriptor) { + if(serializer.kind == StructureKind.CLASS) { + for(index in (0 until serializer.elementsCount)) { + val sub = serializer.getElementDescriptor(index) + if(sub.isNullable) { + encodeNull() + } + encodeNulls(sub) + } + } else { + encodeNull() + } + } + + override fun encodeNullableSerializableValue(serializer: SerializationStrategy, value: T?) { + val isNullabilitySupported = serializer.descriptor.isNullable + if (isNullabilitySupported) { + // Instead of `serializer.serialize` to be able to intercept this + return encodeSerializableValue(serializer as SerializationStrategy, value) + } + return encodeSerializableValue((serializer as KSerializer).nullable, value) } } diff --git a/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvHasHeaderRecordTest.kt b/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvHasHeaderRecordTest.kt index a0c6fc0..8b65fce 100644 --- a/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvHasHeaderRecordTest.kt +++ b/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvHasHeaderRecordTest.kt @@ -124,7 +124,8 @@ class CsvHasHeaderRecordTest { 0.0, 1.0 ), 100, "info" - ) + ), + "Albert" ), NestedRecord.serializer() ) @@ -143,7 +144,8 @@ class CsvHasHeaderRecordTest { 0.0, 1.0 ), 100, "info" - ) + ), + "Albert" ), NestedRecord( 1, @@ -153,7 +155,8 @@ class CsvHasHeaderRecordTest { 10.0, 20.0 ), 50, "info2" - ) + ), + "Bill" ) ), ListSerializer(NestedRecord.serializer()) diff --git a/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvIgnoreUnknownColumnsTest.kt b/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvIgnoreUnknownColumnsTest.kt index 87cd4d9..7aa2101 100644 --- a/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvIgnoreUnknownColumnsTest.kt +++ b/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvIgnoreUnknownColumnsTest.kt @@ -70,7 +70,8 @@ internal class CsvIgnoreUnknownColumnsTest { ), speed = 100, info = "info" - ) + ), + alternative = "Albert" ), NestedRecord( time = 1, @@ -82,7 +83,8 @@ internal class CsvIgnoreUnknownColumnsTest { ), speed = 50, info = "info2" - ) + ), + alternative = "Bill" ) ), ListSerializer(NestedRecord.serializer()) diff --git a/library/src/test/kotlin/kotlinx/serialization/csv/records/CsvNestedRecordTest.kt b/library/src/test/kotlin/kotlinx/serialization/csv/records/CsvNestedRecordTest.kt index 9718fc1..af87a88 100644 --- a/library/src/test/kotlin/kotlinx/serialization/csv/records/CsvNestedRecordTest.kt +++ b/library/src/test/kotlin/kotlinx/serialization/csv/records/CsvNestedRecordTest.kt @@ -20,7 +20,8 @@ class CsvNestedRecordTest { 0.0, 1.0 ), 100, "info" - ) + ), + "Albert" ), NestedRecord.serializer() ) @@ -37,7 +38,8 @@ class CsvNestedRecordTest { 0.0, 1.0 ), 100, "info" - ) + ), + "Albert" ), NestedRecord( 1, @@ -47,7 +49,8 @@ class CsvNestedRecordTest { 10.0, 20.0 ), 50, "info2" - ) + ), + "Bob" ) ), ListSerializer(NestedRecord.serializer()) @@ -66,7 +69,8 @@ class CsvNestedRecordTest { 0.0, 1.0 ), 100, "info" - ) + ), + "Albert" ), NestedRecord.serializer() ) @@ -85,7 +89,8 @@ class CsvNestedRecordTest { 0.0, 1.0 ), 100, "info" - ) + ), + "Albert" ), NestedRecord( 1, @@ -95,9 +100,85 @@ class CsvNestedRecordTest { 10.0, 20.0 ), 50, "info2" - ) + ), + "Bill" ) ), ListSerializer(NestedRecord.serializer()) ) + + @Test + fun testNestedRecordNullableListWithHeader() = Csv { + hasHeaderRecord = true + }.assertEncodeAndDecode( + "time,name,data,data.location,data.location.lat,data.location.lon,data.speed,data.info,alternative\n0,Alice,true,true,0.0,1.0,100,info,Albert\n1,Bob,false,,,,,,Bill\n3,Charlie,true,false,,,120,info3,Celina", + listOf( + NestedRecordWithNullableField( + 0, + "Alice", + DataWithNullableField( + Location( + 0.0, + 1.0 + ), 100, "info" + ), + "Albert" + ), + NestedRecordWithNullableField( + 1, + "Bob", + null, + "Bill" + ), + NestedRecordWithNullableField( + 3, + "Charlie", + DataWithNullableField( + null, 120, "info3" + ), + "Celina" + ) + ), + ListSerializer(NestedRecordWithNullableField.serializer()), + printResult = true + ) + + @Test + fun testNestedRecordNullableListWithoutHeader() = Csv { + hasHeaderRecord = false + }.assertEncodeAndDecode( + "0,Alice,true,true,0.0,1.0,100,info,Albert\n1,Bob,false,,,,,,Bill\n3,Charlie,true,false,,,120,info3,Celina", + listOf( + NestedRecordWithNullableField( + 0, + "Alice", + DataWithNullableField( + Location( + 0.0, + 1.0 + ), 100, "info" + ), + "Albert" + ), + NestedRecordWithNullableField( + 1, + "Bob", + null, + "Bill" + ), + NestedRecordWithNullableField( + 3, + "Charlie", + DataWithNullableField( + null, 120, "info3" + ), + "Celina" + ) + ), + ListSerializer(NestedRecordWithNullableField.serializer()), + printResult = true + ) } + +//Headers(map={0=0, 1=1, 2=2}, subHeaders={2=Headers(map={0=0, 1=1, 2=2}, subHeaders={0=Headers(map={0=0, 1=1}, subHeaders={})})}) +//Headers(map={0=0, 1=1, 2=2, 3=2}, subHeaders={2=Headers(map={0=0, 1=0, 2=1, 3=2}, subHeaders={0=Headers(map={0=0, 1=1}, subHeaders={})})}) diff --git a/library/src/test/kotlin/kotlinx/serialization/csv/records/SampleClasses.kt b/library/src/test/kotlin/kotlinx/serialization/csv/records/SampleClasses.kt index e9b3ca4..fd39219 100644 --- a/library/src/test/kotlin/kotlinx/serialization/csv/records/SampleClasses.kt +++ b/library/src/test/kotlin/kotlinx/serialization/csv/records/SampleClasses.kt @@ -70,12 +70,24 @@ data class SerialNameRecord( data class NestedRecord( val time: Int, val name: String, - val data: Data + val data: Data, + val alternative: String +) + +@Serializable +data class NestedRecordWithNullableField( + val time: Int, + val name: String, + val data: DataWithNullableField?, + val alternative: String, ) @Serializable data class Data(val location: Location, val speed: Int, val info: String) +@Serializable +data class DataWithNullableField(val location: Location?, val speed: Int, val info: String) + @Serializable data class Location(val lat: Double, val lon: Double)