Tokenizer: apply tab replacement to heredoc/nowdoc opener

While probably exceedingly rare to be found in actual codebases, the PHP tokenizer apparently allows for whitespace between the `<<<` and the heredoc/nowdoc identifier. See: https://3v4l.org/NUHZd

Both spaces as well as tabs are allowed. New lines are not allowed.

The PHPCS `Tokenizer` did no execute tab replacement on these token leading to unexpected `'content'` and incorrect `'length'` values in the `File::$tokens` array, which in turn could lead to incorrect sniff results and incorrect fixes.

This commit adds the `T_START_HEREDOC`/`T_START_NOWDOC` tokens to the array of tokens for which to do tab replacement to make them more consistent with the rest of PHPCS.

Includes unit tests safeguarding this change.

src/Tokenizers/Tokenizer.php

@@ -193,6 +193,8 @@ private function createPositionMap()
             T_DOC_COMMENT_STRING       => true,
             T_CONSTANT_ENCAPSED_STRING => true,
             T_DOUBLE_QUOTED_STRING     => true,
+            T_START_HEREDOC            => true,
+            T_START_NOWDOC             => true,
             T_HEREDOC                  => true,
             T_NOWDOC                   => true,
             T_END_HEREDOC              => true,

tests/Core/Tokenizer/Tokenizer/HeredocNowdocOpenerTest.inc

@@ -0,0 +1,31 @@
+<?php
+
+/* testHeredocOpenerNoSpace */
+$heredoc = <<<EOD
+some text
+EOD;
+
+/* testNowdocOpenerNoSpace */
+$nowdoc = <<<'EOD'
+some text
+EOD;
+
+/* testHeredocOpenerHasSpace */
+$heredoc = <<< END
+some text
+END;
+
+/* testNowdocOpenerHasSpace */
+$nowdoc = <<<             'END'
+some text
+END;
+
+/* testHeredocOpenerHasTab */
+$heredoc = <<<			"END"
+some text
+END;
+
+/* testNowdocOpenerHasTab */
+$nowdoc = <<<	'END'
+some text
+END;

tests/Core/Tokenizer/Tokenizer/HeredocNowdocOpenerTest.php

@@ -0,0 +1,121 @@
+<?php
+/**
+ * Tests the tokenization of heredoc/nowdoc opener tokens.
+ *
+ * @author    Juliette Reinders Folmer <[email protected]>
+ * @copyright 2024 PHPCSStandards and contributors
+ * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer\Tokenizer;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
+
+/**
+ * Heredoc/nowdoc opener token test.
+ */
+final class HeredocNowdocOpenerTest extends AbstractTokenizerTestCase
+{
+
+
+    /**
+     * Verify that spaces/tabs in a heredoc/nowdoc opener token get the tab replacement treatment.
+     *
+     * @param string                         $testMarker The comment prefacing the target token.
+     * @param array<string, int|string|null> $expected   Expectations for the token array.
+     *
+     * @dataProvider dataHeredocNowdocOpenerTabReplacement
+     * @covers       PHP_CodeSniffer\Tokenizers\Tokenizer::createPositionMap
+     *
+     * @return void
+     */
+    public function testHeredocNowdocOpenerTabReplacement($testMarker, $expected)
+    {
+        $tokens = $this->phpcsFile->getTokens();
+        $opener = $this->getTargetToken($testMarker, [T_START_HEREDOC, T_START_NOWDOC]);
+
+        foreach ($expected as $key => $value) {
+            if ($key === 'orig_content' && $value === null) {
+                $this->assertArrayNotHasKey($key, $tokens[$opener], "Unexpected 'orig_content' key found in the token array.");
+                continue;
+            }
+
+            $this->assertArrayHasKey($key, $tokens[$opener], "Key $key not found in the token array.");
+            $this->assertSame($value, $tokens[$opener][$key], "Value for key $key does not match expectation.");
+        }
+
+    }//end testHeredocNowdocOpenerTabReplacement()
+
+
+    /**
+     * Data provider.
+     *
+     * @see testHeredocNowdocOpenerTabReplacement()
+     *
+     * @return array<string, array<string, string|array<string, int|string|null>>>
+     */
+    public static function dataHeredocNowdocOpenerTabReplacement()
+    {
+        return [
+            'Heredoc opener without space' => [
+                'testMarker' => '/* testHeredocOpenerNoSpace */',
+                'expected'   => [
+                    'length'       => 6,
+                    'content'      => '<<<EOD
+',
+                    'orig_content' => null,
+                ],
+            ],
+            'Nowdoc opener without space'  => [
+                'testMarker' => '/* testNowdocOpenerNoSpace */',
+                'expected'   => [
+                    'length'       => 8,
+                    'content'      => "<<<'EOD'
+",
+                    'orig_content' => null,
+                ],
+            ],
+            'Heredoc opener with space(s)' => [
+                'testMarker' => '/* testHeredocOpenerHasSpace */',
+                'expected'   => [
+                    'length'       => 7,
+                    'content'      => '<<< END
+',
+                    'orig_content' => null,
+                ],
+            ],
+            'Nowdoc opener with space(s)'  => [
+                'testMarker' => '/* testNowdocOpenerHasSpace */',
+                'expected'   => [
+                    'length'       => 21,
+                    'content'      => "<<<             'END'
+",
+                    'orig_content' => null,
+                ],
+            ],
+            'Heredoc opener with tab(s)'   => [
+                'testMarker' => '/* testHeredocOpenerHasTab */',
+                'expected'   => [
+                    'length'       => 18,
+                    'content'      => '<<<          "END"
+',
+                    'orig_content' => '<<<			"END"
+',
+                ],
+            ],
+            'Nowdoc opener with tab(s)'    => [
+                'testMarker' => '/* testNowdocOpenerHasTab */',
+                'expected'   => [
+                    'length'       => 11,
+                    'content'      => "<<<   'END'
+",
+                    'orig_content' => "<<<	'END'
+",
+                ],
+            ],
+        ];
+
+    }//end dataHeredocNowdocOpenerTabReplacement()
+
+
+}//end class