From 2558b418772fa3f689bc793ef15cbef9e979382f Mon Sep 17 00:00:00 2001
From: Arne Blankerts <Arne@Blankerts.de>
Date: Sun, 19 Nov 2023 23:34:40 +0100
Subject: [PATCH] Fix #18

---
 src/Tokenizer.php         |  14 ++++++++++++++
 tests/Issue18Test.php     |  19 +++++++++++++++++++
 tests/_files/Issue-18.php | Bin 0 -> 1139 bytes
 3 files changed, 33 insertions(+)
 create mode 100644 tests/Issue18Test.php
 create mode 100644 tests/_files/Issue-18.php

diff --git a/src/Tokenizer.php b/src/Tokenizer.php
index f582d95..0a455ee 100644
--- a/src/Tokenizer.php
+++ b/src/Tokenizer.php
@@ -1,6 +1,8 @@
 <?php declare(strict_types = 1);
 namespace TheSeer\Tokenizer;
 
+use function var_dump;
+
 class Tokenizer {
 
     /**
@@ -70,6 +72,18 @@ public function parse(string $source): TokenCollection {
             $line   = $tok[2];
             $values = \preg_split('/\R+/Uu', $tok[1]);
 
+            if (!$values) {
+                $result->addToken(
+                    new Token(
+                        $line,
+                        \token_name($tok[0]),
+                        '{binary data}'
+                    )
+                );
+
+                continue;
+            }
+
             foreach ($values as $v) {
                 $token = new Token(
                     $line,
diff --git a/tests/Issue18Test.php b/tests/Issue18Test.php
new file mode 100644
index 0000000..951ae08
--- /dev/null
+++ b/tests/Issue18Test.php
@@ -0,0 +1,19 @@
+<?php declare(strict_types = 1);
+namespace TheSeer\Tokenizer;
+
+use DOMXPath;
+use PHPUnit\Framework\TestCase;
+
+class Issue18Test extends TestCase {
+
+    public function testIssueNoLongerOccurs() {
+        $result = (new Tokenizer)->parse(\file_get_contents(__DIR__ . '/_files/Issue-18.php'));
+
+        $dom = (new XMLSerializer())->toDom($result);
+
+        $node = (new DOMXPath($dom))->query('//*[@no="18"]/*')->item(1);
+
+        $this->assertEquals('{binary data}', $node->textContent);
+    }
+
+}
diff --git a/tests/_files/Issue-18.php b/tests/_files/Issue-18.php
new file mode 100644
index 0000000000000000000000000000000000000000..20bd73bd13fc7f13528c76216710ffc229bd188e
GIT binary patch
literal 1139
zcmah|+iuf95N$4Xl4(;Sztk!`OiM&lsRY#WfKm!-)GJa+h4N@A+Sr?Ph2yo>Yovhs
zgM0=O|G*=&-pI6SDVDsu9-ldLb}sKeB-aT%$1@&t#WW9bJV$-aF;P;>Kt?#uCKIkO
zkh4TYTy+tT2l#0)#FOKT;U>AJGE)KX;f$v#oAPe%2@o?fBrM?A&!$twr%X%b4;4%N
zAy2h`Wdit@T(gjAb^s7WEKTv!vcPvhdXj7$i2zg0G;M8>jzlc9V3GJnQV(&$qLh>G
zee5KQ11)5X#$gZ#yt9Lr(_q_)q^ljGe<tNL;#LgZ)W3)|S22tHW1a>|B&7L@puOId
z>mo<S(%D&rHFRQv9jmd|yC?DxwaWPI=!lBzOvMMeZbNWWoN{F^-;0HgqcAMOo>Klh
z1<<%{!>=kd?<C%ZRA*ybgstBds4Wue0+dDQm9W&Qx=)}+^_;Od^vYyt7(>G5kz}D)
z$qoBN%S1eyE$ZV{vC>)Dd)EES-d!-$Zfoeap)!%us}sZ8-ugq(Qz_jwaHx(3*dSot
zU4^RY01xPE)qM^1BIXKN{uA?j$=wC#nzN8AZ#BnnJQVmmM-ciAG-fPeBeQtT{8F0_
zIb70z^Pa?7%{N{^y}0EHRKD<ep2>#WgypX+%DBzrMjIR($$bp<3YvH59|QNwZ%bS2
zP%)XS(KZ;gmdv4l^tn~)J5V**`dF<mw;WoXR*lwLtKL8PQGR-R@a*K-(X;LTv)q0h
g;1|<5cJiU#-!8gu*EaR78Te=a=SbV?H*f>+2d$JyivR!s

literal 0
HcmV?d00001