From 8515728dc0a8583c4a997fac49855b15e3785218 Mon Sep 17 00:00:00 2001 From: Joseph Ivie Date: Tue, 21 Nov 2023 15:36:16 -0700 Subject: [PATCH 1/2] Null attempt --- .../csv/decode/AbstractDecoderAlt.kt | 68 ++++++++++++++ .../csv/decode/ClassCsvDecoder.kt | 13 ++- .../serialization/csv/decode/CsvDecoder.kt | 83 +++++++++++++++-- .../csv/decode/RootCsvDecoder.kt | 20 +++- .../serialization/csv/encode/CsvEncoder.kt | 47 ++++++++-- .../csv/config/CsvHasHeaderRecordTest.kt | 9 +- .../csv/config/CsvIgnoreUnknownColumnsTest.kt | 6 +- .../csv/records/CsvNestedRecordTest.kt | 93 +++++++++++++++++-- .../csv/records/SampleClasses.kt | 14 ++- 9 files changed, 319 insertions(+), 34 deletions(-) create mode 100644 library/src/main/kotlin/kotlinx/serialization/csv/decode/AbstractDecoderAlt.kt diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/AbstractDecoderAlt.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/AbstractDecoderAlt.kt new file mode 100644 index 0000000..622d235 --- /dev/null +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/AbstractDecoderAlt.kt @@ -0,0 +1,68 @@ +package kotlinx.serialization.csv.decode + +import kotlinx.serialization.DeserializationStrategy +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.SerializationException +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.CompositeDecoder +import kotlinx.serialization.encoding.Decoder + +@ExperimentalSerializationApi +public abstract class AbstractDecoderAlt : Decoder, CompositeDecoder { + + /** + * Invoked to decode a value when specialized `encode*` method was not overridden. + */ + public open fun decodeValue(): Any = throw SerializationException("${this::class} can't retrieve untyped values") + + override fun decodeNotNullMark(): Boolean = true + override fun decodeNull(): Nothing? = null + override fun decodeBoolean(): Boolean = decodeValue() as Boolean + override fun decodeByte(): Byte = decodeValue() as Byte + override fun decodeShort(): Short = decodeValue() as Short + override fun decodeInt(): Int = decodeValue() as Int + override fun decodeLong(): Long = decodeValue() as Long + override fun decodeFloat(): Float = decodeValue() as Float + override fun decodeDouble(): Double = decodeValue() as Double + override fun decodeChar(): Char = decodeValue() as Char + override fun decodeString(): String = decodeValue() as String + override fun decodeEnum(enumDescriptor: SerialDescriptor): Int = decodeValue() as Int + + // overwrite by default + public open fun decodeSerializableValue( + deserializer: DeserializationStrategy, + previousValue: T? = null + ): T = decodeSerializableValue(deserializer) + + override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder = this + + override fun endStructure(descriptor: SerialDescriptor) { + } + + final override fun decodeBooleanElement(descriptor: SerialDescriptor, index: Int): Boolean = decodeBoolean() + final override fun decodeByteElement(descriptor: SerialDescriptor, index: Int): Byte = decodeByte() + final override fun decodeShortElement(descriptor: SerialDescriptor, index: Int): Short = decodeShort() + final override fun decodeIntElement(descriptor: SerialDescriptor, index: Int): Int = decodeInt() + final override fun decodeLongElement(descriptor: SerialDescriptor, index: Int): Long = decodeLong() + final override fun decodeFloatElement(descriptor: SerialDescriptor, index: Int): Float = decodeFloat() + final override fun decodeDoubleElement(descriptor: SerialDescriptor, index: Int): Double = decodeDouble() + final override fun decodeCharElement(descriptor: SerialDescriptor, index: Int): Char = decodeChar() + final override fun decodeStringElement(descriptor: SerialDescriptor, index: Int): String = decodeString() + + final override fun decodeSerializableElement( + descriptor: SerialDescriptor, + index: Int, + deserializer: DeserializationStrategy, + previousValue: T? + ): T = decodeSerializableValue(deserializer, previousValue) + + override fun decodeNullableSerializableElement( + descriptor: SerialDescriptor, + index: Int, + deserializer: DeserializationStrategy, + previousValue: T? + ): T? { + val isNullabilitySupported = deserializer.descriptor.isNullable + return if (isNullabilitySupported || decodeNotNullMark()) decodeSerializableValue(deserializer, previousValue) else decodeNull() + } +} \ No newline at end of file diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/ClassCsvDecoder.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/ClassCsvDecoder.kt index b523ed7..cdab6b2 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/decode/ClassCsvDecoder.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/ClassCsvDecoder.kt @@ -29,17 +29,19 @@ internal class ClassCsvDecoder( elementIndex >= descriptor.elementsCount -> DECODE_DONE classHeaders != null && columnIndex >= classHeaders.size -> DECODE_DONE - classHeaders != null -> + classHeaders != null -> { when (val result = classHeaders[columnIndex]) { UNKNOWN_NAME -> { ignoreColumn() decodeElementIndex(descriptor) } + null -> UNKNOWN_NAME - else -> result + else -> result.also { println("${descriptor.serialName} decoded ${it} (${descriptor.getElementName(it)}) due to column ${columnIndex} in ${classHeaders.toString(descriptor)}") } } + } - else -> elementIndex + else -> elementIndex.also { println("${descriptor.serialName} decoded ${it} (${descriptor.getElementName(it)})") } } override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder { @@ -84,4 +86,9 @@ internal class ClassCsvDecoder( reader.readColumn() columnIndex++ } + + override fun virtualColumnAdvance() { + columnIndex++ + elementIndex++ + } } diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvDecoder.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvDecoder.kt index d64c89e..5760c1b 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvDecoder.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvDecoder.kt @@ -2,15 +2,15 @@ package kotlinx.serialization.csv.decode import kotlinx.serialization.DeserializationStrategy import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.KSerializer +import kotlinx.serialization.builtins.nullable import kotlinx.serialization.csv.Csv -import kotlinx.serialization.csv.HeadersNotSupportedForSerialDescriptorException import kotlinx.serialization.csv.UnknownColumnHeaderException import kotlinx.serialization.csv.UnsupportedSerialDescriptorException import kotlinx.serialization.csv.config.CsvConfig import kotlinx.serialization.descriptors.PolymorphicKind import kotlinx.serialization.descriptors.SerialDescriptor import kotlinx.serialization.descriptors.StructureKind -import kotlinx.serialization.encoding.AbstractDecoder import kotlinx.serialization.encoding.CompositeDecoder import kotlinx.serialization.encoding.CompositeDecoder.Companion.UNKNOWN_NAME import kotlinx.serialization.modules.SerializersModule @@ -24,7 +24,7 @@ internal abstract class CsvDecoder( protected val csv: Csv, protected val reader: CsvReader, private val parent: CsvDecoder? -) : AbstractDecoder() { +) : AbstractDecoderAlt() { override val serializersModule: SerializersModule get() = csv.serializersModule @@ -111,11 +111,18 @@ internal abstract class CsvDecoder( return null } + fun skipEmpty(): Nothing? { + val value = reader.readColumn() + println("SKIPPING $value") + require(value == config.nullString) { "Expected '${config.nullString}' but was '$value'." } + return null + } + override fun decodeEnum(enumDescriptor: SerialDescriptor): Int { return enumDescriptor.getElementIndex(decodeColumn()) } - protected open fun decodeColumn() = reader.readColumn() + protected open fun decodeColumn() = reader.readColumn().also { println("READ COLUMN $it") } protected fun readHeaders(descriptor: SerialDescriptor) { if (config.hasHeaderRecord && headers == null) { @@ -144,6 +151,7 @@ internal abstract class CsvDecoder( val headerIndex = descriptor.getElementIndex(header) if (headerIndex != UNKNOWN_NAME) { headers[position] = headerIndex + println("SET $position TO $header") reader.unmark() } else { val name = header.substringBefore(config.headerSeparator) @@ -152,7 +160,6 @@ internal abstract class CsvDecoder( val childDesc = descriptor.getElementDescriptor(nameIndex) if (childDesc.kind is StructureKind.CLASS) { reader.reset() - headers[position] = nameIndex headers[nameIndex] = readHeaders(childDesc, "$prefix$name.") } else { reader.unmark() @@ -168,7 +175,7 @@ internal abstract class CsvDecoder( } position++ } - return headers + return headers.also { println(it.toString(descriptor)) } } protected fun readTrailingDelimiter() { @@ -197,18 +204,74 @@ internal abstract class CsvDecoder( operator fun set(key: Int, value: Headers) { subHeaders[key] = value } + + override fun toString(): String { + return "Headers(map=${map}, subHeaders=${subHeaders})" + } + fun toString(context: SerialDescriptor): String { + return "Headers(map=${map.mapValues { if(it.value in 0 until context.elementsCount) context.getElementName(it.value) else "???" }})" + } } override fun decodeSerializableValue(deserializer: DeserializationStrategy): T { - return if (config.deferToFormatWhenVariableColumns != null) { + if (config.deferToFormatWhenVariableColumns != null) { when (deserializer.descriptor.kind) { is StructureKind.LIST, is StructureKind.MAP, is PolymorphicKind.OPEN -> { - config.deferToFormatWhenVariableColumns!!.decodeFromString(deserializer, decodeColumn()) + return config.deferToFormatWhenVariableColumns!!.decodeFromString(deserializer, decodeColumn()) + } + else -> {} + } + } + if(deserializer.descriptor.isNullable && deserializer.descriptor.kind == StructureKind.CLASS && deserializer.descriptor.elementsCount > 0) { + val isPresent = reader.readColumn().toBoolean() + println("READ PRESENT $isPresent") + if(isPresent) { + return deserializer.deserialize(this) + } else { + virtualColumnAdvance() + decodeNulls(deserializer.descriptor, deserializer.descriptor.serialName) + @Suppress("UNCHECKED_CAST") + return null as T + } + } else { + return deserializer.deserialize(this) + } + } + + protected open fun virtualColumnAdvance() {} + + private fun decodeNulls(serializer: SerialDescriptor, name: String) { + if(serializer.kind == StructureKind.CLASS) { + for(index in (0 until serializer.elementsCount)) { + val sub = serializer.getElementDescriptor(index) + if(sub.isNullable) { + println("Skipping present ${serializer.getElementName(index)}") + skipEmpty() } - else -> super.decodeSerializableValue(deserializer) + decodeNulls(sub, serializer.getElementName(index)) } - } else super.decodeSerializableValue(deserializer) + println("Skipping ${name} end") + } else { + println("Skipping $name") + skipEmpty() + } + } + + override fun decodeNullableSerializableValue(deserializer: DeserializationStrategy): T? { + val isNullabilitySupported = deserializer.descriptor.isNullable + return if (isNullabilitySupported || decodeNotNullMark()) decodeSerializableValue(deserializer) else decodeSerializableValue((deserializer as KSerializer).nullable) + } + + override fun decodeNullableSerializableElement( + descriptor: SerialDescriptor, + index: Int, + deserializer: DeserializationStrategy, + previousValue: T? + ): T? { + val isNullabilitySupported = deserializer.descriptor.isNullable + return if (isNullabilitySupported) decodeSerializableValue(deserializer, previousValue) else decodeSerializableValue((deserializer as KSerializer).nullable, previousValue) } } + diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/RootCsvDecoder.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/RootCsvDecoder.kt index 0260c37..714a737 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/decode/RootCsvDecoder.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/RootCsvDecoder.kt @@ -2,6 +2,8 @@ package kotlinx.serialization.csv.decode import kotlinx.serialization.DeserializationStrategy import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.KSerializer +import kotlinx.serialization.builtins.nullable import kotlinx.serialization.csv.Csv import kotlinx.serialization.descriptors.PolymorphicKind import kotlinx.serialization.descriptors.SerialDescriptor @@ -55,13 +57,25 @@ internal class RootCsvDecoder( } override fun decodeSerializableValue(deserializer: DeserializationStrategy): T { - return if (config.deferToFormatWhenVariableColumns != null) { + if (config.deferToFormatWhenVariableColumns != null) { when (deserializer.descriptor.kind) { is PolymorphicKind.OPEN -> { config.deferToFormatWhenVariableColumns!!.decodeFromString(deserializer, decodeColumn()) } - else -> deserializer.deserialize(this) + else -> {} } - } else deserializer.deserialize(this) + } + if(deserializer.descriptor.isNullable && deserializer.descriptor.kind == StructureKind.CLASS && deserializer.descriptor.elementsCount > 0) { + val isPresent = decodeBoolean() + println("Decoded present boolean ${isPresent}") + if(isPresent) { + return deserializer.deserialize(this) + } else { + @Suppress("UNCHECKED_CAST") + return null as T + } + } else { + return deserializer.deserialize(this) + } } } diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/encode/CsvEncoder.kt b/library/src/main/kotlin/kotlinx/serialization/csv/encode/CsvEncoder.kt index 477926e..a4c64a0 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/encode/CsvEncoder.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/encode/CsvEncoder.kt @@ -1,14 +1,13 @@ package kotlinx.serialization.csv.encode import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.KSerializer import kotlinx.serialization.SerializationStrategy +import kotlinx.serialization.builtins.nullable import kotlinx.serialization.csv.Csv import kotlinx.serialization.csv.HeadersNotSupportedForSerialDescriptorException import kotlinx.serialization.csv.UnsupportedSerialDescriptorException -import kotlinx.serialization.descriptors.PolymorphicKind -import kotlinx.serialization.descriptors.SerialDescriptor -import kotlinx.serialization.descriptors.SerialKind -import kotlinx.serialization.descriptors.StructureKind +import kotlinx.serialization.descriptors.* import kotlinx.serialization.encoding.AbstractEncoder import kotlinx.serialization.encoding.CompositeEncoder import kotlinx.serialization.modules.SerializersModule @@ -146,6 +145,7 @@ internal abstract class CsvEncoder( Unit childDesc.elementsCount > 0 -> { + if(childDesc.isNullable) writer.printColumn(name) val headerSeparator = config.headerSeparator printHeader("$name$headerSeparator", childDesc) } @@ -179,9 +179,44 @@ internal abstract class CsvEncoder( isNumeric = false, isNull = false ) + return } - else -> super.encodeSerializableValue(serializer, value) + else -> {} } - } else super.encodeSerializableValue(serializer, value) + } + if(serializer.descriptor.isNullable && serializer.descriptor.kind == StructureKind.CLASS && serializer.descriptor.elementsCount > 0) { + if(value == null) { + encodeBoolean(false) + encodeNulls(serializer.descriptor) + } else { + encodeBoolean(true) + serializer.serialize(this, value) + } + } else { + serializer.serialize(this, value) + } + } + + private fun encodeNulls(serializer: SerialDescriptor) { + if(serializer.kind == StructureKind.CLASS) { + for(index in (0 until serializer.elementsCount)) { + val sub = serializer.getElementDescriptor(index) + if(sub.isNullable) { + encodeNull() + } + encodeNulls(sub) + } + } else { + encodeNull() + } + } + + override fun encodeNullableSerializableValue(serializer: SerializationStrategy, value: T?) { + val isNullabilitySupported = serializer.descriptor.isNullable + if (isNullabilitySupported) { + // Instead of `serializer.serialize` to be able to intercept this + return encodeSerializableValue(serializer as SerializationStrategy, value) + } + return encodeSerializableValue((serializer as KSerializer).nullable, value) } } diff --git a/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvHasHeaderRecordTest.kt b/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvHasHeaderRecordTest.kt index a0c6fc0..8b65fce 100644 --- a/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvHasHeaderRecordTest.kt +++ b/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvHasHeaderRecordTest.kt @@ -124,7 +124,8 @@ class CsvHasHeaderRecordTest { 0.0, 1.0 ), 100, "info" - ) + ), + "Albert" ), NestedRecord.serializer() ) @@ -143,7 +144,8 @@ class CsvHasHeaderRecordTest { 0.0, 1.0 ), 100, "info" - ) + ), + "Albert" ), NestedRecord( 1, @@ -153,7 +155,8 @@ class CsvHasHeaderRecordTest { 10.0, 20.0 ), 50, "info2" - ) + ), + "Bill" ) ), ListSerializer(NestedRecord.serializer()) diff --git a/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvIgnoreUnknownColumnsTest.kt b/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvIgnoreUnknownColumnsTest.kt index 87cd4d9..7aa2101 100644 --- a/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvIgnoreUnknownColumnsTest.kt +++ b/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvIgnoreUnknownColumnsTest.kt @@ -70,7 +70,8 @@ internal class CsvIgnoreUnknownColumnsTest { ), speed = 100, info = "info" - ) + ), + alternative = "Albert" ), NestedRecord( time = 1, @@ -82,7 +83,8 @@ internal class CsvIgnoreUnknownColumnsTest { ), speed = 50, info = "info2" - ) + ), + alternative = "Bill" ) ), ListSerializer(NestedRecord.serializer()) diff --git a/library/src/test/kotlin/kotlinx/serialization/csv/records/CsvNestedRecordTest.kt b/library/src/test/kotlin/kotlinx/serialization/csv/records/CsvNestedRecordTest.kt index 9718fc1..af87a88 100644 --- a/library/src/test/kotlin/kotlinx/serialization/csv/records/CsvNestedRecordTest.kt +++ b/library/src/test/kotlin/kotlinx/serialization/csv/records/CsvNestedRecordTest.kt @@ -20,7 +20,8 @@ class CsvNestedRecordTest { 0.0, 1.0 ), 100, "info" - ) + ), + "Albert" ), NestedRecord.serializer() ) @@ -37,7 +38,8 @@ class CsvNestedRecordTest { 0.0, 1.0 ), 100, "info" - ) + ), + "Albert" ), NestedRecord( 1, @@ -47,7 +49,8 @@ class CsvNestedRecordTest { 10.0, 20.0 ), 50, "info2" - ) + ), + "Bob" ) ), ListSerializer(NestedRecord.serializer()) @@ -66,7 +69,8 @@ class CsvNestedRecordTest { 0.0, 1.0 ), 100, "info" - ) + ), + "Albert" ), NestedRecord.serializer() ) @@ -85,7 +89,8 @@ class CsvNestedRecordTest { 0.0, 1.0 ), 100, "info" - ) + ), + "Albert" ), NestedRecord( 1, @@ -95,9 +100,85 @@ class CsvNestedRecordTest { 10.0, 20.0 ), 50, "info2" - ) + ), + "Bill" ) ), ListSerializer(NestedRecord.serializer()) ) + + @Test + fun testNestedRecordNullableListWithHeader() = Csv { + hasHeaderRecord = true + }.assertEncodeAndDecode( + "time,name,data,data.location,data.location.lat,data.location.lon,data.speed,data.info,alternative\n0,Alice,true,true,0.0,1.0,100,info,Albert\n1,Bob,false,,,,,,Bill\n3,Charlie,true,false,,,120,info3,Celina", + listOf( + NestedRecordWithNullableField( + 0, + "Alice", + DataWithNullableField( + Location( + 0.0, + 1.0 + ), 100, "info" + ), + "Albert" + ), + NestedRecordWithNullableField( + 1, + "Bob", + null, + "Bill" + ), + NestedRecordWithNullableField( + 3, + "Charlie", + DataWithNullableField( + null, 120, "info3" + ), + "Celina" + ) + ), + ListSerializer(NestedRecordWithNullableField.serializer()), + printResult = true + ) + + @Test + fun testNestedRecordNullableListWithoutHeader() = Csv { + hasHeaderRecord = false + }.assertEncodeAndDecode( + "0,Alice,true,true,0.0,1.0,100,info,Albert\n1,Bob,false,,,,,,Bill\n3,Charlie,true,false,,,120,info3,Celina", + listOf( + NestedRecordWithNullableField( + 0, + "Alice", + DataWithNullableField( + Location( + 0.0, + 1.0 + ), 100, "info" + ), + "Albert" + ), + NestedRecordWithNullableField( + 1, + "Bob", + null, + "Bill" + ), + NestedRecordWithNullableField( + 3, + "Charlie", + DataWithNullableField( + null, 120, "info3" + ), + "Celina" + ) + ), + ListSerializer(NestedRecordWithNullableField.serializer()), + printResult = true + ) } + +//Headers(map={0=0, 1=1, 2=2}, subHeaders={2=Headers(map={0=0, 1=1, 2=2}, subHeaders={0=Headers(map={0=0, 1=1}, subHeaders={})})}) +//Headers(map={0=0, 1=1, 2=2, 3=2}, subHeaders={2=Headers(map={0=0, 1=0, 2=1, 3=2}, subHeaders={0=Headers(map={0=0, 1=1}, subHeaders={})})}) diff --git a/library/src/test/kotlin/kotlinx/serialization/csv/records/SampleClasses.kt b/library/src/test/kotlin/kotlinx/serialization/csv/records/SampleClasses.kt index e9b3ca4..fd39219 100644 --- a/library/src/test/kotlin/kotlinx/serialization/csv/records/SampleClasses.kt +++ b/library/src/test/kotlin/kotlinx/serialization/csv/records/SampleClasses.kt @@ -70,12 +70,24 @@ data class SerialNameRecord( data class NestedRecord( val time: Int, val name: String, - val data: Data + val data: Data, + val alternative: String +) + +@Serializable +data class NestedRecordWithNullableField( + val time: Int, + val name: String, + val data: DataWithNullableField?, + val alternative: String, ) @Serializable data class Data(val location: Location, val speed: Int, val info: String) +@Serializable +data class DataWithNullableField(val location: Location?, val speed: Int, val info: String) + @Serializable data class Location(val lat: Double, val lon: Double) From 643a15792218197d421f9154517f579715272660 Mon Sep 17 00:00:00 2001 From: Joseph Ivie Date: Tue, 21 Nov 2023 16:22:53 -0700 Subject: [PATCH 2/2] All tests passing... --- .../csv/decode/ClassCsvDecoder.kt | 34 +++++++++++-------- .../csv/decode/CollectionRecordCsvDecoder.kt | 3 ++ .../serialization/csv/decode/CsvDecoder.kt | 20 ++++++----- .../csv/decode/RecordListCsvDecoder.kt | 17 ++++++++++ .../csv/decode/RootCsvDecoder.kt | 3 +- .../csv/encode/RecordListCsvEncoder.kt | 22 ++++++++++++ .../csv/config/CsvHasHeaderRecordTest.kt | 9 ++--- .../csv/config/CsvIgnoreUnknownColumnsTest.kt | 6 ++-- .../csv/records/CsvNestedRecordTest.kt | 18 ++++------ .../csv/records/SampleClasses.kt | 3 +- 10 files changed, 86 insertions(+), 49 deletions(-) diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/ClassCsvDecoder.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/ClassCsvDecoder.kt index cdab6b2..df98b01 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/decode/ClassCsvDecoder.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/ClassCsvDecoder.kt @@ -24,24 +24,28 @@ internal class ClassCsvDecoder( private var elementIndex = 0 private var columnIndex = 0 - override fun decodeElementIndex(descriptor: SerialDescriptor): Int = when { - reader.isDone -> DECODE_DONE - elementIndex >= descriptor.elementsCount -> DECODE_DONE - classHeaders != null && columnIndex >= classHeaders.size -> DECODE_DONE + override fun decodeElementIndex(descriptor: SerialDescriptor): Int { +// if(descriptor != classHeaders?.descriptor) throw Exception("ClassCsvDecoder with ${descriptor.serialName} cannot decode an index for ${classHeaders?.descriptor?.serialName}") + return when { + reader.isDone -> DECODE_DONE + elementIndex >= descriptor.elementsCount -> DECODE_DONE + classHeaders != null && columnIndex >= classHeaders.size -> DECODE_DONE - classHeaders != null -> { - when (val result = classHeaders[columnIndex]) { - UNKNOWN_NAME -> { - ignoreColumn() - decodeElementIndex(descriptor) - } + classHeaders != null -> { + println("${descriptor.serialName} decoded column index ${columnIndex} to be ${classHeaders[columnIndex]} (${classHeaders[columnIndex]?.takeIf { it >= 0 }?.let { descriptor.getElementName(it)}}) due to column ${columnIndex} in ${classHeaders}") + when (val result = classHeaders[columnIndex]) { + UNKNOWN_NAME -> { + ignoreColumn() + decodeElementIndex(descriptor) + } - null -> UNKNOWN_NAME - else -> result.also { println("${descriptor.serialName} decoded ${it} (${descriptor.getElementName(it)}) due to column ${columnIndex} in ${classHeaders.toString(descriptor)}") } + null -> UNKNOWN_NAME + else -> result + } } - } - else -> elementIndex.also { println("${descriptor.serialName} decoded ${it} (${descriptor.getElementName(it)})") } + else -> elementIndex.also { println("${descriptor.serialName} decoded ${it} (${descriptor.getElementName(it)}) because there were no class headers") } + } } override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder { @@ -51,7 +55,7 @@ internal class ClassCsvDecoder( csv, reader, this, - classHeaders?.getSubHeaders(decodeElementIndex(descriptor)) + classHeaders?.let { it.getSubHeaders(columnIndex) ?: throw IllegalStateException("Could not find sub headers at index $elementIndex; see $classHeaders}") } ) else -> diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/CollectionRecordCsvDecoder.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/CollectionRecordCsvDecoder.kt index 470be84..f5a2e24 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/decode/CollectionRecordCsvDecoder.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/CollectionRecordCsvDecoder.kt @@ -1,8 +1,11 @@ package kotlinx.serialization.csv.decode +import kotlinx.serialization.DeserializationStrategy import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.csv.Csv +import kotlinx.serialization.descriptors.PolymorphicKind import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.descriptors.StructureKind import kotlinx.serialization.encoding.CompositeDecoder.Companion.DECODE_DONE /** diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvDecoder.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvDecoder.kt index 5760c1b..b735cba 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvDecoder.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvDecoder.kt @@ -10,6 +10,7 @@ import kotlinx.serialization.csv.UnsupportedSerialDescriptorException import kotlinx.serialization.csv.config.CsvConfig import kotlinx.serialization.descriptors.PolymorphicKind import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.descriptors.SerialKind import kotlinx.serialization.descriptors.StructureKind import kotlinx.serialization.encoding.CompositeDecoder import kotlinx.serialization.encoding.CompositeDecoder.Companion.UNKNOWN_NAME @@ -133,7 +134,7 @@ internal abstract class CsvDecoder( } private fun readHeaders(descriptor: SerialDescriptor, prefix: String): Headers { - val headers = Headers() + val headers = Headers(descriptor) var position = 0 while (!reader.isDone && reader.isFirstRecord) { val offset = reader.offset @@ -153,6 +154,8 @@ internal abstract class CsvDecoder( headers[position] = headerIndex println("SET $position TO $header") reader.unmark() + val desc = descriptor.getElementDescriptor(headerIndex) + if(desc.kind == StructureKind.CLASS && desc.isNullable) position-- } else { val name = header.substringBefore(config.headerSeparator) val nameIndex = descriptor.getElementIndex(name) @@ -160,7 +163,9 @@ internal abstract class CsvDecoder( val childDesc = descriptor.getElementDescriptor(nameIndex) if (childDesc.kind is StructureKind.CLASS) { reader.reset() - headers[nameIndex] = readHeaders(childDesc, "$prefix$name.") + if(headers[position] == null) + headers[position] = nameIndex + headers[position] = readHeaders(childDesc, "$prefix$name.") } else { reader.unmark() } @@ -175,7 +180,7 @@ internal abstract class CsvDecoder( } position++ } - return headers.also { println(it.toString(descriptor)) } + return headers.also { println(it.toString()) } } protected fun readTrailingDelimiter() { @@ -184,7 +189,7 @@ internal abstract class CsvDecoder( } } - internal class Headers { + internal class Headers(val descriptor: SerialDescriptor) { private val map = mutableMapOf() private val subHeaders = mutableMapOf() @@ -206,10 +211,7 @@ internal abstract class CsvDecoder( } override fun toString(): String { - return "Headers(map=${map}, subHeaders=${subHeaders})" - } - fun toString(context: SerialDescriptor): String { - return "Headers(map=${map.mapValues { if(it.value in 0 until context.elementsCount) context.getElementName(it.value) else "???" }})" + return "Headers(descriptor=${descriptor.serialName}, map=${map.mapValues { if(it.value in 0 until descriptor.elementsCount) descriptor.getElementName(it.value) else "???" }}, subHeaders=${subHeaders})" } } @@ -224,7 +226,7 @@ internal abstract class CsvDecoder( else -> {} } } - if(deserializer.descriptor.isNullable && deserializer.descriptor.kind == StructureKind.CLASS && deserializer.descriptor.elementsCount > 0) { + if(deserializer.descriptor.isNullable && deserializer.descriptor.kind == StructureKind.CLASS) { val isPresent = reader.readColumn().toBoolean() println("READ PRESENT $isPresent") if(isPresent) { diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/RecordListCsvDecoder.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/RecordListCsvDecoder.kt index 6ea46ce..015f6f9 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/decode/RecordListCsvDecoder.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/RecordListCsvDecoder.kt @@ -1,7 +1,9 @@ package kotlinx.serialization.csv.decode +import kotlinx.serialization.DeserializationStrategy import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.csv.Csv +import kotlinx.serialization.descriptors.PolymorphicKind import kotlinx.serialization.descriptors.SerialDescriptor import kotlinx.serialization.descriptors.StructureKind import kotlinx.serialization.encoding.CompositeDecoder @@ -70,4 +72,19 @@ internal class RecordListCsvDecoder( reader.reset() } } + + override fun decodeSerializableValue(deserializer: DeserializationStrategy): T { + println("DECODING") + if (config.deferToFormatWhenVariableColumns != null) { + when (deserializer.descriptor.kind) { + is StructureKind.LIST, + is StructureKind.MAP, + is PolymorphicKind.OPEN -> { + return config.deferToFormatWhenVariableColumns!!.decodeFromString(deserializer, decodeColumn()) + } + else -> {} + } + } + return deserializer.deserialize(this) + } } diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/RootCsvDecoder.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/RootCsvDecoder.kt index 714a737..9c698fa 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/decode/RootCsvDecoder.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/RootCsvDecoder.kt @@ -31,6 +31,7 @@ internal class RootCsvDecoder( } override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder { + println("BEGIN STRUCT ${descriptor.kind}") return when (descriptor.kind) { StructureKind.LIST -> // Top level list is treated as list of multiple records @@ -65,7 +66,7 @@ internal class RootCsvDecoder( else -> {} } } - if(deserializer.descriptor.isNullable && deserializer.descriptor.kind == StructureKind.CLASS && deserializer.descriptor.elementsCount > 0) { + if(deserializer.descriptor.isNullable && deserializer.descriptor.kind == StructureKind.CLASS) { val isPresent = decodeBoolean() println("Decoded present boolean ${isPresent}") if(isPresent) { diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/encode/RecordListCsvEncoder.kt b/library/src/main/kotlin/kotlinx/serialization/csv/encode/RecordListCsvEncoder.kt index b1247e4..817be5b 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/encode/RecordListCsvEncoder.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/encode/RecordListCsvEncoder.kt @@ -1,8 +1,11 @@ package kotlinx.serialization.csv.encode import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.SerializationStrategy import kotlinx.serialization.csv.Csv +import kotlinx.serialization.descriptors.PolymorphicKind import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.descriptors.StructureKind import kotlinx.serialization.encoding.CompositeEncoder /** @@ -41,4 +44,23 @@ internal class RecordListCsvEncoder( super.encodeColumn(value, isNumeric, isNull) writer.endRecord() } + + override fun encodeSerializableValue(serializer: SerializationStrategy, value: T) { + if (config.deferToFormatWhenVariableColumns != null) { + when (serializer.descriptor.kind) { + is StructureKind.LIST, + is StructureKind.MAP, + is PolymorphicKind.OPEN -> { + encodeColumn( + value = config.deferToFormatWhenVariableColumns!!.encodeToString(serializer, value), + isNumeric = false, + isNull = false + ) + return + } + else -> {} + } + } + serializer.serialize(this, value) + } } diff --git a/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvHasHeaderRecordTest.kt b/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvHasHeaderRecordTest.kt index 8b65fce..a0c6fc0 100644 --- a/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvHasHeaderRecordTest.kt +++ b/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvHasHeaderRecordTest.kt @@ -124,8 +124,7 @@ class CsvHasHeaderRecordTest { 0.0, 1.0 ), 100, "info" - ), - "Albert" + ) ), NestedRecord.serializer() ) @@ -144,8 +143,7 @@ class CsvHasHeaderRecordTest { 0.0, 1.0 ), 100, "info" - ), - "Albert" + ) ), NestedRecord( 1, @@ -155,8 +153,7 @@ class CsvHasHeaderRecordTest { 10.0, 20.0 ), 50, "info2" - ), - "Bill" + ) ) ), ListSerializer(NestedRecord.serializer()) diff --git a/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvIgnoreUnknownColumnsTest.kt b/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvIgnoreUnknownColumnsTest.kt index 7aa2101..87cd4d9 100644 --- a/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvIgnoreUnknownColumnsTest.kt +++ b/library/src/test/kotlin/kotlinx/serialization/csv/config/CsvIgnoreUnknownColumnsTest.kt @@ -70,8 +70,7 @@ internal class CsvIgnoreUnknownColumnsTest { ), speed = 100, info = "info" - ), - alternative = "Albert" + ) ), NestedRecord( time = 1, @@ -83,8 +82,7 @@ internal class CsvIgnoreUnknownColumnsTest { ), speed = 50, info = "info2" - ), - alternative = "Bill" + ) ) ), ListSerializer(NestedRecord.serializer()) diff --git a/library/src/test/kotlin/kotlinx/serialization/csv/records/CsvNestedRecordTest.kt b/library/src/test/kotlin/kotlinx/serialization/csv/records/CsvNestedRecordTest.kt index af87a88..0d475de 100644 --- a/library/src/test/kotlin/kotlinx/serialization/csv/records/CsvNestedRecordTest.kt +++ b/library/src/test/kotlin/kotlinx/serialization/csv/records/CsvNestedRecordTest.kt @@ -20,8 +20,7 @@ class CsvNestedRecordTest { 0.0, 1.0 ), 100, "info" - ), - "Albert" + ) ), NestedRecord.serializer() ) @@ -38,8 +37,7 @@ class CsvNestedRecordTest { 0.0, 1.0 ), 100, "info" - ), - "Albert" + ) ), NestedRecord( 1, @@ -49,8 +47,7 @@ class CsvNestedRecordTest { 10.0, 20.0 ), 50, "info2" - ), - "Bob" + ) ) ), ListSerializer(NestedRecord.serializer()) @@ -69,8 +66,7 @@ class CsvNestedRecordTest { 0.0, 1.0 ), 100, "info" - ), - "Albert" + ) ), NestedRecord.serializer() ) @@ -89,8 +85,7 @@ class CsvNestedRecordTest { 0.0, 1.0 ), 100, "info" - ), - "Albert" + ) ), NestedRecord( 1, @@ -100,8 +95,7 @@ class CsvNestedRecordTest { 10.0, 20.0 ), 50, "info2" - ), - "Bill" + ) ) ), ListSerializer(NestedRecord.serializer()) diff --git a/library/src/test/kotlin/kotlinx/serialization/csv/records/SampleClasses.kt b/library/src/test/kotlin/kotlinx/serialization/csv/records/SampleClasses.kt index fd39219..d0b72e1 100644 --- a/library/src/test/kotlin/kotlinx/serialization/csv/records/SampleClasses.kt +++ b/library/src/test/kotlin/kotlinx/serialization/csv/records/SampleClasses.kt @@ -70,8 +70,7 @@ data class SerialNameRecord( data class NestedRecord( val time: Int, val name: String, - val data: Data, - val alternative: String + val data: Data ) @Serializable