-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Streaming and Excel BOM handling (#19)
* Support for streaming via Reader and Appendable * Handle Microsoft Excel's insistence on using a byte order marker * Cleaning up new unit tests for FetchSourceTest * Removed commented debugging println's for FetchSource * Cleanup formatting --------- Co-authored-by: Sven Obser <[email protected]>
- Loading branch information
1 parent
e92334e
commit bc2e62e
Showing
6 changed files
with
317 additions
and
15 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
86 changes: 86 additions & 0 deletions
86
library/src/main/kotlin/kotlinx/serialization/csv/decode/FetchSource.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| package kotlinx.serialization.csv.decode | ||
|
|
||
| import java.io.Reader | ||
|
|
||
| internal class FetchSource( | ||
| private val getChar: () -> Char?, | ||
| ) : Source { | ||
|
|
||
| constructor( | ||
| reader: Reader, | ||
| ) : this( | ||
| getChar = { | ||
| reader.read().let { | ||
| if (it == -1) null else it.toChar() | ||
| } | ||
| }, | ||
| ) | ||
|
|
||
| private var nextPosition = 0 | ||
| override var offset: Int = 0 | ||
| private set | ||
|
|
||
| private var next: Char? = getChar() | ||
| private fun nextChar(): Char { | ||
| val n = next ?: throw IllegalStateException("Out of characters") | ||
| next = getChar() | ||
| nextPosition++ | ||
| return n | ||
| } | ||
|
|
||
| private var queue = ArrayList<Char>(2048) | ||
| private var marks = ArrayList<Int>(2048) | ||
| private var queueOffset = 0 | ||
|
|
||
| override fun canRead(): Boolean = offset <= nextPosition | ||
|
|
||
| override fun read(): Char? { | ||
| if (offset > nextPosition) { | ||
| return null | ||
| } else if (offset == nextPosition) { | ||
| if (next == null) { | ||
| offset++ | ||
| if (marks.isEmpty()) queue.clear() | ||
| return null | ||
| } | ||
| val c = nextChar() | ||
| if (marks.isNotEmpty()) { | ||
| if (queue.isEmpty()) { | ||
| queueOffset = offset | ||
| } | ||
| queue.add(c) | ||
| } else { | ||
| queue.clear() | ||
| } | ||
| offset++ | ||
| return c | ||
| } else { | ||
| val indexToCheck = offset - queueOffset | ||
| val result = queue[indexToCheck] | ||
| offset++ | ||
| return result | ||
| } | ||
| } | ||
|
|
||
| override fun peek(): Char? = | ||
| if (offset > nextPosition) { | ||
| null | ||
| } else if (offset == nextPosition) { | ||
| next | ||
| } else { | ||
| queue[offset - queueOffset] | ||
| } | ||
|
|
||
| override fun mark() { | ||
| marks.add(offset) | ||
| } | ||
|
|
||
| override fun unmark() { | ||
| marks.removeAt(marks.lastIndex) | ||
| } | ||
|
|
||
| override fun reset() { | ||
| offset = marks[marks.lastIndex] | ||
| marks.removeAt(marks.lastIndex) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
129 changes: 129 additions & 0 deletions
129
library/src/test/kotlin/kotlinx/serialization/csv/decode/FetchSourceTest.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| package kotlinx.serialization.csv.decode | ||
|
|
||
| import java.io.StringReader | ||
| import kotlin.test.Test | ||
| import kotlin.test.assertEquals | ||
| import kotlin.test.assertFalse | ||
| import kotlin.test.assertNull | ||
| import kotlin.test.assertTrue | ||
|
|
||
| class FetchSourceTest { | ||
|
|
||
| @Test | ||
| fun testCanRead() { | ||
| val source = FetchSource("") | ||
| assertTrue(source.canRead()) | ||
| } | ||
|
|
||
| @Test | ||
| fun testNotCanRead() { | ||
| val source = FetchSource("") | ||
| source.read() | ||
| assertFalse(source.canRead()) | ||
| } | ||
|
|
||
| @Test | ||
| fun testRead() { | ||
| val source = FetchSource("abc") | ||
| assertEquals('a', source.read()) | ||
| assertEquals('b', source.read()) | ||
| assertEquals('c', source.read()) | ||
| assertNull(source.read()) | ||
| } | ||
|
|
||
| @Test | ||
| fun testReadEof() { | ||
| val source = FetchSource("") | ||
| assertNull(source.read()) | ||
| } | ||
|
|
||
| @Test | ||
| fun testPeek() { | ||
| val source = FetchSource("abc") | ||
| assertEquals('a', source.peek()) | ||
| } | ||
|
|
||
| @Test | ||
| fun testPeekMultipleTimes() { | ||
| val source = FetchSource("abc") | ||
| assertEquals('a', source.peek()) | ||
| assertEquals('a', source.peek()) | ||
| assertEquals('a', source.peek()) | ||
| } | ||
|
|
||
| @Test | ||
| fun testPeekEof() { | ||
| val source = FetchSource("") | ||
| assertNull(source.peek()) | ||
| } | ||
|
|
||
| @Test | ||
| fun testMarkUnmark() { | ||
| val source = FetchSource("abc") | ||
| assertEquals('a', source.read()) | ||
| source.mark() | ||
| assertEquals('b', source.read()) | ||
| source.unmark() | ||
| assertEquals('c', source.read()) | ||
| } | ||
|
|
||
| @Test | ||
| fun testMarkMarkUnmarkReset() { | ||
| val source = FetchSource("0123456789") | ||
| assertEquals('0', source.read()) | ||
| source.mark() | ||
| assertEquals('1', source.read()) | ||
| source.mark() | ||
| assertEquals('2', source.read()) | ||
| source.unmark() | ||
| assertEquals('3', source.read()) | ||
| source.reset() | ||
| assertEquals('1', source.read()) | ||
| } | ||
|
|
||
| @Test | ||
| fun testMarkReset() { | ||
| val source = FetchSource("abc") | ||
| assertEquals('a', source.read()) | ||
| source.mark() | ||
| assertEquals('b', source.read()) | ||
| source.reset() | ||
| assertEquals('b', source.read()) | ||
| } | ||
|
|
||
| @Test | ||
| fun testMarkResetMultiple() { | ||
| val source = FetchSource("abcdef") | ||
| assertEquals('a', source.read()) | ||
| source.mark() | ||
| assertEquals('b', source.read()) | ||
| source.mark() | ||
| assertEquals('c', source.read()) | ||
| source.reset() | ||
| assertEquals('c', source.read()) | ||
| source.reset() | ||
| assertEquals('b', source.read()) | ||
| } | ||
|
|
||
| @Test | ||
| fun testMarkPeekRead() { | ||
| val source = FetchSource("abc") | ||
| assertEquals('a', source.read()) | ||
| source.mark() | ||
| assertEquals('b', source.peek()) | ||
| assertEquals('b', source.read()) | ||
| source.reset() | ||
| assertEquals('b', source.peek()) | ||
| assertEquals('b', source.read()) | ||
| source.mark() | ||
| assertEquals('c', source.peek()) | ||
| assertEquals('c', source.read()) | ||
| source.reset() | ||
| assertEquals('c', source.peek()) | ||
| assertEquals('c', source.read()) | ||
| } | ||
| } | ||
|
|
||
| @Suppress("TestFunctionName") | ||
| private fun FetchSource(string: String): FetchSource = | ||
| FetchSource(StringReader(string)) |
Oops, something went wrong.