From 2b7bdb344d1527bcdb157e6ee60c6ce9fcca07ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaroslav=20Hansl=C3=ADk?= Date: Sun, 21 Nov 2021 11:51:25 +0100 Subject: [PATCH 1/2] Improved tokenizing of context sensitive keywords --- package.xml | 6 + src/Tokenizers/PHP.php | 265 +++++----- src/Util/Tokens.php | 75 +++ .../ContextSensitiveKeywordsTest.inc | 208 ++++++++ .../ContextSensitiveKeywordsTest.php | 469 ++++++++++++++++++ 5 files changed, 874 insertions(+), 149 deletions(-) create mode 100644 tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc create mode 100644 tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php diff --git a/package.xml b/package.xml index 3836361538..f66e4fda60 100644 --- a/package.xml +++ b/package.xml @@ -138,6 +138,8 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + @@ -2097,6 +2099,8 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + @@ -2191,6 +2195,8 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index 5b7abf1cb9..7ddfa35350 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -589,6 +589,64 @@ protected function tokenize($string) echo PHP_EOL; } + /* + Tokenize context sensitive keyword as string when it should be string. + */ + + if ($tokenIsArray === true + && isset(Util\Tokens::$contextSensitiveKeywords[$token[0]]) === true + && isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true + ) { + $preserveKeyword = false; + + // `new class` should be preserved + if ($token[0] === T_CLASS && $finalTokens[$lastNotEmptyToken]['code'] === T_NEW) { + $preserveKeyword = true; + } + + // `new class extends` `new class implements` should be preserved + if (($token[0] === T_EXTENDS || $token[0] === T_IMPLEMENTS) + && $finalTokens[$lastNotEmptyToken]['code'] === T_CLASS + ) { + $preserveKeyword = true; + } + + // `namespace\` should be preserved + if ($token[0] === T_NAMESPACE) { + for ($i = ($stackPtr + 1); $i < $numTokens; $i++) { + if (is_array($tokens[$i]) === false) { + break; + } + + if (isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true) { + continue; + } + + if ($tokens[$i][0] === T_NS_SEPARATOR) { + $preserveKeyword = true; + } + + break; + } + } + + if ($preserveKeyword === false) { + if (PHP_CODESNIFFER_VERBOSITY > 1) { + $type = Util\Tokens::tokenName($token[0]); + echo "\t\t* token $stackPtr changed from $type to T_STRING".PHP_EOL; + } + + $finalTokens[$newStackPtr] = [ + 'code' => T_STRING, + 'type' => 'T_STRING', + 'content' => $token[1], + ]; + + $newStackPtr++; + continue; + } + }//end if + /* Parse doc blocks into something that can be easily iterated over. */ @@ -1113,6 +1171,7 @@ protected function tokenize($string) && $tokenIsArray === true && $token[0] === T_STRING && strtolower($token[1]) === 'yield' + && isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === false ) { if (isset($tokens[($stackPtr + 1)]) === true && isset($tokens[($stackPtr + 2)]) === true @@ -1446,57 +1505,42 @@ protected function tokenize($string) if ($tokenIsArray === true && $token[0] === T_DEFAULT + && isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === false ) { - if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === false) { - for ($x = ($stackPtr + 1); $x < $numTokens; $x++) { - if ($tokens[$x] === ',') { - // Skip over potential trailing comma (supported in PHP). - continue; - } - - if (is_array($tokens[$x]) === false - || isset(Util\Tokens::$emptyTokens[$tokens[$x][0]]) === false - ) { - // Non-empty, non-comma content. - break; - } + for ($x = ($stackPtr + 1); $x < $numTokens; $x++) { + if ($tokens[$x] === ',') { + // Skip over potential trailing comma (supported in PHP). + continue; } - if (isset($tokens[$x]) === true - && is_array($tokens[$x]) === true - && $tokens[$x][0] === T_DOUBLE_ARROW + if (is_array($tokens[$x]) === false + || isset(Util\Tokens::$emptyTokens[$tokens[$x][0]]) === false ) { - // Modify the original token stack for the double arrow so that - // future checks can disregard the double arrow token more easily. - // For match expression "case" statements, this is handled - // in PHP::processAdditional(). - $tokens[$x][0] = T_MATCH_ARROW; - if (PHP_CODESNIFFER_VERBOSITY > 1) { - echo "\t\t* token $x changed from T_DOUBLE_ARROW to T_MATCH_ARROW".PHP_EOL; - } - - $newToken = []; - $newToken['code'] = T_MATCH_DEFAULT; - $newToken['type'] = 'T_MATCH_DEFAULT'; - $newToken['content'] = $token[1]; + // Non-empty, non-comma content. + break; + } + } - if (PHP_CODESNIFFER_VERBOSITY > 1) { - echo "\t\t* token $stackPtr changed from T_DEFAULT to T_MATCH_DEFAULT".PHP_EOL; - } + if (isset($tokens[$x]) === true + && is_array($tokens[$x]) === true + && $tokens[$x][0] === T_DOUBLE_ARROW + ) { + // Modify the original token stack for the double arrow so that + // future checks can disregard the double arrow token more easily. + // For match expression "case" statements, this is handled + // in PHP::processAdditional(). + $tokens[$x][0] = T_MATCH_ARROW; + if (PHP_CODESNIFFER_VERBOSITY > 1) { + echo "\t\t* token $x changed from T_DOUBLE_ARROW to T_MATCH_ARROW".PHP_EOL; + } - $finalTokens[$newStackPtr] = $newToken; - $newStackPtr++; - continue; - }//end if - } else { - // Definitely not the "default" keyword. $newToken = []; - $newToken['code'] = T_STRING; - $newToken['type'] = 'T_STRING'; + $newToken['code'] = T_MATCH_DEFAULT; + $newToken['type'] = 'T_MATCH_DEFAULT'; $newToken['content'] = $token[1]; if (PHP_CODESNIFFER_VERBOSITY > 1) { - echo "\t\t* token $stackPtr changed from T_DEFAULT to T_STRING".PHP_EOL; + echo "\t\t* token $stackPtr changed from T_DEFAULT to T_MATCH_DEFAULT".PHP_EOL; } $finalTokens[$newStackPtr] = $newToken; @@ -1693,14 +1737,9 @@ protected function tokenize($string) } /* - The string-like token after a function keyword should always be - tokenized as T_STRING even if it appears to be a different token, - such as when writing code like: function default(): foo - so go forward and change the token type before it is processed. - - Note: this should not be done for `function Level\Name` within a - group use statement for the PHP 8 identifier name tokens as it - would interfere with the re-tokenization of those. + This is a special condition for T_ARRAY tokens used for + function return types. We want to keep the parenthesis map clean, + so let's tag these tokens as T_STRING. */ if ($tokenIsArray === true @@ -1708,37 +1747,6 @@ protected function tokenize($string) || $token[0] === T_FN) && $finalTokens[$lastNotEmptyToken]['code'] !== T_USE ) { - if ($token[0] === T_FUNCTION) { - for ($x = ($stackPtr + 1); $x < $numTokens; $x++) { - if (is_array($tokens[$x]) === false - || (isset(Util\Tokens::$emptyTokens[$tokens[$x][0]]) === false - && $tokens[$x][1] !== '&') - ) { - // Non-empty content. - break; - } - } - - if ($x < $numTokens - && is_array($tokens[$x]) === true - && $tokens[$x][0] !== T_STRING - && $tokens[$x][0] !== T_NAME_QUALIFIED - ) { - if (PHP_CODESNIFFER_VERBOSITY > 1) { - $oldType = Util\Tokens::tokenName($tokens[$x][0]); - echo "\t\t* token $x changed from $oldType to T_STRING".PHP_EOL; - } - - $tokens[$x][0] = T_STRING; - } - }//end if - - /* - This is a special condition for T_ARRAY tokens used for - function return types. We want to keep the parenthesis map clean, - so let's tag these tokens as T_STRING. - */ - // Go looking for the colon to start the return type hint. // Start by finding the closing parenthesis of the function. $parenthesisStack = []; @@ -1926,31 +1934,31 @@ function return types. We want to keep the parenthesis map clean, $newStackPtr++; } } else { - if ($tokenIsArray === true && $token[0] === T_STRING) { - // Some T_STRING tokens should remain that way - // due to their context. - if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) { - // Special case for syntax like: return new self - // where self should not be a string. - if ($finalTokens[$lastNotEmptyToken]['code'] === T_NEW - && strtolower($token[1]) === 'self' - ) { - $finalTokens[$newStackPtr] = [ - 'content' => $token[1], - 'code' => T_SELF, - 'type' => 'T_SELF', - ]; - } else { - $finalTokens[$newStackPtr] = [ - 'content' => $token[1], - 'code' => T_STRING, - 'type' => 'T_STRING', - ]; - } + // Some T_STRING tokens should remain that way due to their context. + if ($tokenIsArray === true + && $token[0] === T_STRING + && isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true + ) { + // Special case for syntax like: return new self + // where self should not be a string. + if ($finalTokens[$lastNotEmptyToken]['code'] === T_NEW + && strtolower($token[1]) === 'self' + ) { + $finalTokens[$newStackPtr] = [ + 'content' => $token[1], + 'code' => T_SELF, + 'type' => 'T_SELF', + ]; + } else { + $finalTokens[$newStackPtr] = [ + 'content' => $token[1], + 'code' => T_STRING, + 'type' => 'T_STRING', + ]; + } - $newStackPtr++; - continue; - }//end if + $newStackPtr++; + continue; }//end if $newToken = null; @@ -2114,16 +2122,6 @@ function return types. We want to keep the parenthesis map clean, $newToken['type'] = 'T_FINALLY'; } - // This is a special case for the PHP 5.5 classname::class syntax - // where "class" should be T_STRING instead of T_CLASS. - if (($newToken['code'] === T_CLASS - || $newToken['code'] === T_FUNCTION) - && $finalTokens[$lastNotEmptyToken]['code'] === T_DOUBLE_COLON - ) { - $newToken['code'] = T_STRING; - $newToken['type'] = 'T_STRING'; - } - // This is a special case for PHP 5.6 use function and use const // where "function" and "const" should be T_STRING instead of T_FUNCTION // and T_CONST. @@ -2819,34 +2817,11 @@ protected function processAdditional() $this->tokens[$i]['code'] = T_STRING; $this->tokens[$i]['type'] = 'T_STRING'; } - } else if ($this->tokens[$i]['code'] === T_CONST) { - // Context sensitive keywords support. - for ($x = ($i + 1); $i < $numTokens; $x++) { - if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) { - // Non-whitespace content. - break; - } - } - - if ($this->tokens[$x]['code'] !== T_STRING) { - if (PHP_CODESNIFFER_VERBOSITY > 1) { - $line = $this->tokens[$x]['line']; - $type = $this->tokens[$x]['type']; - echo "\t* token $x on line $line changed from $type to T_STRING".PHP_EOL; - } - - $this->tokens[$x]['code'] = T_STRING; - $this->tokens[$x]['type'] = 'T_STRING'; - } - } else if ($this->tokens[$i]['code'] === T_READONLY - || ($this->tokens[$i]['code'] === T_STRING - && strtolower($this->tokens[$i]['content']) === 'readonly') + } else if ($this->tokens[$i]['code'] === T_STRING + && strtolower($this->tokens[$i]['content']) === 'readonly' ) { /* - Adds "readonly" keyword support: - PHP < 8.1: Converts T_STRING to T_READONLY - PHP >= 8.1: Converts some T_READONLY to T_STRING because token_get_all() - without the TOKEN_PARSE flag cannot distinguish between them in some situations. + Adds "readonly" keyword support for PHP < 8.1. */ $allowedAfter = [ @@ -2890,7 +2865,7 @@ protected function processAdditional() } } - if ($this->tokens[$i]['code'] === T_STRING && $shouldBeReadonly === true) { + if ($shouldBeReadonly === true) { if (PHP_CODESNIFFER_VERBOSITY > 1) { $line = $this->tokens[$i]['line']; echo "\t* token $i on line $line changed from T_STRING to T_READONLY".PHP_EOL; @@ -2898,14 +2873,6 @@ protected function processAdditional() $this->tokens[$i]['code'] = T_READONLY; $this->tokens[$i]['type'] = 'T_READONLY'; - } else if ($this->tokens[$i]['code'] === T_READONLY && $shouldBeReadonly === false) { - if (PHP_CODESNIFFER_VERBOSITY > 1) { - $line = $this->tokens[$i]['line']; - echo "\t* token $i on line $line changed from T_READONLY to T_STRING".PHP_EOL; - } - - $this->tokens[$i]['code'] = T_STRING; - $this->tokens[$i]['type'] = 'T_STRING'; } continue; diff --git a/src/Util/Tokens.php b/src/Util/Tokens.php index 0bc1747275..b05eb6161a 100644 --- a/src/Util/Tokens.php +++ b/src/Util/Tokens.php @@ -653,6 +653,81 @@ final class Tokens T_TRAIT_C => T_TRAIT_C, ]; + /** + * Tokens representing context sensitive keywords in PHP. + * + * @var array + * + * https://wiki.php.net/rfc/context_sensitive_lexer + */ + public static $contextSensitiveKeywords = [ + T_ABSTRACT => T_ABSTRACT, + T_ARRAY => T_ARRAY, + T_AS => T_AS, + T_BREAK => T_BREAK, + T_CALLABLE => T_CALLABLE, + T_CASE => T_CASE, + T_CATCH => T_CATCH, + T_CLASS => T_CLASS, + T_CLONE => T_CLONE, + T_CONST => T_CONST, + T_CONTINUE => T_CONTINUE, + T_DECLARE => T_DECLARE, + T_DEFAULT => T_DEFAULT, + T_DO => T_DO, + T_ECHO => T_ECHO, + T_ELSE => T_ELSE, + T_ELSEIF => T_ELSEIF, + T_ENDDECLARE => T_ENDDECLARE, + T_ENDFOR => T_ENDFOR, + T_ENDFOREACH => T_ENDFOREACH, + T_ENDIF => T_ENDIF, + T_ENDSWITCH => T_ENDSWITCH, + T_ENDWHILE => T_ENDWHILE, + T_EXIT => T_EXIT, + T_EXTENDS => T_EXTENDS, + T_FINAL => T_FINAL, + T_FINALLY => T_FINALLY, + T_FN => T_FN, + T_FOR => T_FOR, + T_FOREACH => T_FOREACH, + T_FUNCTION => T_FUNCTION, + T_GLOBAL => T_GLOBAL, + T_GOTO => T_GOTO, + T_IF => T_IF, + T_IMPLEMENTS => T_IMPLEMENTS, + T_INCLUDE => T_INCLUDE, + T_INCLUDE_ONCE => T_INCLUDE_ONCE, + T_INSTANCEOF => T_INSTANCEOF, + T_INSTEADOF => T_INSTEADOF, + T_INTERFACE => T_INTERFACE, + T_LIST => T_LIST, + T_LOGICAL_AND => T_LOGICAL_AND, + T_LOGICAL_OR => T_LOGICAL_OR, + T_LOGICAL_XOR => T_LOGICAL_XOR, + T_MATCH => T_MATCH, + T_NAMESPACE => T_NAMESPACE, + T_NEW => T_NEW, + T_PRINT => T_PRINT, + T_PRIVATE => T_PRIVATE, + T_PROTECTED => T_PROTECTED, + T_PUBLIC => T_PUBLIC, + T_READONLY => T_READONLY, + T_REQUIRE => T_REQUIRE, + T_REQUIRE_ONCE => T_REQUIRE_ONCE, + T_RETURN => T_RETURN, + T_STATIC => T_STATIC, + T_SWITCH => T_SWITCH, + T_THROW => T_THROW, + T_TRAIT => T_TRAIT, + T_TRY => T_TRY, + T_USE => T_USE, + T_VAR => T_VAR, + T_WHILE => T_WHILE, + T_YIELD => T_YIELD, + T_YIELD_FROM => T_YIELD_FROM, + ]; + /** * Given a token, returns the name of the token. diff --git a/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc b/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc new file mode 100644 index 0000000000..9506a35c6e --- /dev/null +++ b/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc @@ -0,0 +1,208 @@ + 'a', + 2 => 'b', + /* testMatchDefaultIsKeyword */ default => 'default', +}; + +$closure = /* testFnIsKeyword */ fn () => 'string'; + +function () { + /* testYieldIsKeyword */ yield $f; + /* testYieldFromIsKeyword */ yield from someFunction(); +}; + +/* testDeclareIsKeyword */ declare(ticks=1): +/* testEndDeclareIsKeyword */ enddeclare; + +if (true /* testAndIsKeyword */ and false /* testOrIsKeyword */ or null /* testXorIsKeyword */ xor 0) { + +} + +$anonymousClass = new /* testAnonymousClassIsKeyword */ class {}; +$anonymousClass2 = new class /* testExtendsInAnonymousClassIsKeyword */ extends SomeParent {}; +$anonymousClass3 = new class /* testImplementsInAnonymousClassIsKeyword */ implements SomeInterface {}; + +class Foo extends /* testNamespaceInNameIsKeyword */ namespace\Exception +{} diff --git a/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php b/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php new file mode 100644 index 0000000000..4c200fbc30 --- /dev/null +++ b/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php @@ -0,0 +1,469 @@ + + * @copyright 2020 Squiz Pty Ltd (ABN 77 084 670 600) + * @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer; + +use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest; +use PHP_CodeSniffer\Util\Tokens; + +class ContextSensitiveKeywordsTest extends AbstractMethodUnitTest +{ + + + /** + * Test that context sensitive keyword is tokenized as string when it should be string. + * + * @param string $testMarker The comment which prefaces the target token in the test file. + * + * @dataProvider dataStrings + * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize + * + * @return void + */ + public function testStrings($testMarker) + { + $tokens = self::$phpcsFile->getTokens(); + + $token = $this->getTargetToken($testMarker, (Tokens::$contextSensitiveKeywords + [T_STRING])); + + $this->assertSame(T_STRING, $tokens[$token]['code']); + $this->assertSame('T_STRING', $tokens[$token]['type']); + + }//end testStrings() + + + /** + * Data provider. + * + * @see testStrings() + * + * @return array + */ + public function dataStrings() + { + return [ + ['/* testAbstract */'], + ['/* testArray */'], + ['/* testAs */'], + ['/* testBreak */'], + ['/* testCallable */'], + ['/* testCase */'], + ['/* testCatch */'], + ['/* testClass */'], + ['/* testClone */'], + ['/* testConst */'], + ['/* testContinue */'], + ['/* testDeclare */'], + ['/* testDefault */'], + ['/* testDo */'], + ['/* testEcho */'], + ['/* testElse */'], + ['/* testElseIf */'], + ['/* testEndDeclare */'], + ['/* testEndFor */'], + ['/* testEndForeach */'], + ['/* testEndIf */'], + ['/* testEndSwitch */'], + ['/* testEndWhile */'], + ['/* testExit */'], + ['/* testExtends */'], + ['/* testFinal */'], + ['/* testFinally */'], + ['/* testFn */'], + ['/* testFor */'], + ['/* testForeach */'], + ['/* testFunction */'], + ['/* testGlobal */'], + ['/* testGoto */'], + ['/* testIf */'], + ['/* testImplements */'], + ['/* testInclude */'], + ['/* testIncludeOnce */'], + ['/* testInstanceOf */'], + ['/* testInsteadOf */'], + ['/* testInterface */'], + ['/* testList */'], + ['/* testMatch */'], + ['/* testNamespace */'], + ['/* testNew */'], + ['/* testParent */'], + ['/* testPrint */'], + ['/* testPrivate */'], + ['/* testProtected */'], + ['/* testPublic */'], + ['/* testReadonly */'], + ['/* testRequire */'], + ['/* testRequireOnce */'], + ['/* testReturn */'], + ['/* testSelf */'], + ['/* testStatic */'], + ['/* testSwitch */'], + ['/* testThrows */'], + ['/* testTrait */'], + ['/* testTry */'], + ['/* testUse */'], + ['/* testVar */'], + ['/* testWhile */'], + ['/* testYield */'], + ['/* testYieldFrom */'], + ['/* testAnd */'], + ['/* testOr */'], + ['/* testXor */'], + + ['/* testKeywordAfterNamespaceShouldBeString */'], + ['/* testNamespaceNameIsString1 */'], + ['/* testNamespaceNameIsString2 */'], + ['/* testNamespaceNameIsString3 */'], + ]; + + }//end dataStrings() + + + /** + * Test that context sensitive keyword is tokenized as keyword when it should be keyword. + * + * @param string $testMarker The comment which prefaces the target token in the test file. + * @param string $expectedTokenType The expected token type. + * + * @dataProvider dataKeywords + * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize + * + * @return void + */ + public function testKeywords($testMarker, $expectedTokenType) + { + $tokens = self::$phpcsFile->getTokens(); + + $token = $this->getTargetToken($testMarker, (Tokens::$contextSensitiveKeywords + [T_ANON_CLASS, T_MATCH_DEFAULT, T_PARENT, T_SELF, T_STRING])); + + $this->assertSame(constant($expectedTokenType), $tokens[$token]['code']); + $this->assertSame($expectedTokenType, $tokens[$token]['type']); + + }//end testKeywords() + + + /** + * Data provider. + * + * @see testKeywords() + * + * @return array + */ + public function dataKeywords() + { + return [ + [ + '/* testNamespaceIsKeyword */', + 'T_NAMESPACE', + ], + [ + '/* testAbstractIsKeyword */', + 'T_ABSTRACT', + ], + [ + '/* testClassIsKeyword */', + 'T_CLASS', + ], + [ + '/* testExtendsIsKeyword */', + 'T_EXTENDS', + ], + [ + '/* testImplementsIsKeyword */', + 'T_IMPLEMENTS', + ], + [ + '/* testUseIsKeyword */', + 'T_USE', + ], + [ + '/* testInsteadOfIsKeyword */', + 'T_INSTEADOF', + ], + [ + '/* testAsIsKeyword */', + 'T_AS', + ], + [ + '/* testConstIsKeyword */', + 'T_CONST', + ], + [ + '/* testPrivateIsKeyword */', + 'T_PRIVATE', + ], + [ + '/* testProtectedIsKeyword */', + 'T_PROTECTED', + ], + [ + '/* testPublicIsKeyword */', + 'T_PUBLIC', + ], + [ + '/* testVarIsKeyword */', + 'T_VAR', + ], + [ + '/* testStaticIsKeyword */', + 'T_STATIC', + ], + [ + '/* testReadonlyIsKeyword */', + 'T_READONLY', + ], + [ + '/* testFinalIsKeyword */', + 'T_FINAL', + ], + [ + '/* testFunctionIsKeyword */', + 'T_FUNCTION', + ], + [ + '/* testCallableIsKeyword */', + 'T_CALLABLE', + ], + [ + '/* testSelfIsKeyword */', + 'T_SELF', + ], + [ + '/* testParentIsKeyword */', + 'T_PARENT', + ], + [ + '/* testReturnIsKeyword */', + 'T_RETURN', + ], + + [ + '/* testInterfaceIsKeyword */', + 'T_INTERFACE', + ], + [ + '/* testTraitIsKeyword */', + 'T_TRAIT', + ], + + [ + '/* testNewIsKeyword */', + 'T_NEW', + ], + [ + '/* testInstanceOfIsKeyword */', + 'T_INSTANCEOF', + ], + [ + '/* testCloneIsKeyword */', + 'T_CLONE', + ], + + [ + '/* testIfIsKeyword */', + 'T_IF', + ], + [ + '/* testElseIfIsKeyword */', + 'T_ELSEIF', + ], + [ + '/* testElseIsKeyword */', + 'T_ELSE', + ], + [ + '/* testEndIfIsKeyword */', + 'T_ENDIF', + ], + + [ + '/* testForIsKeyword */', + 'T_FOR', + ], + [ + '/* testEndForIsKeyword */', + 'T_ENDFOR', + ], + + [ + '/* testForeachIsKeyword */', + 'T_FOREACH', + ], + [ + '/* testEndForeachIsKeyword */', + 'T_ENDFOREACH', + ], + + [ + '/* testSwitchIsKeyword */', + 'T_SWITCH', + ], + [ + '/* testCaseIsKeyword */', + 'T_CASE', + ], + [ + '/* testDefaultIsKeyword */', + 'T_DEFAULT', + ], + [ + '/* testEndSwitchIsKeyword */', + 'T_ENDSWITCH', + ], + [ + '/* testBreakIsKeyword */', + 'T_BREAK', + ], + [ + '/* testContinueIsKeyword */', + 'T_CONTINUE', + ], + + [ + '/* testDoIsKeyword */', + 'T_DO', + ], + [ + '/* testWhileIsKeyword */', + 'T_WHILE', + ], + [ + '/* testEndWhileIsKeyword */', + 'T_ENDWHILE', + ], + + [ + '/* testTryIsKeyword */', + 'T_TRY', + ], + [ + '/* testThrowIsKeyword */', + 'T_THROW', + ], + [ + '/* testCatchIsKeyword */', + 'T_CATCH', + ], + [ + '/* testFinallyIsKeyword */', + 'T_FINALLY', + ], + + [ + '/* testGlobalIsKeyword */', + 'T_GLOBAL', + ], + [ + '/* testEchoIsKeyword */', + 'T_ECHO', + ], + [ + '/* testPrintIsKeyword */', + 'T_PRINT', + ], + [ + '/* testDieIsKeyword */', + 'T_EXIT', + ], + [ + '/* testExitIsKeyword */', + 'T_EXIT', + ], + + [ + '/* testIncludeIsKeyword */', + 'T_INCLUDE', + ], + [ + '/* testIncludeOnceIsKeyword */', + 'T_INCLUDE_ONCE', + ], + [ + '/* testRequireIsKeyword */', + 'T_REQUIRE', + ], + [ + '/* testRequireOnceIsKeyword */', + 'T_REQUIRE_ONCE', + ], + + [ + '/* testListIsKeyword */', + 'T_LIST', + ], + [ + '/* testGotoIsKeyword */', + 'T_GOTO', + ], + [ + '/* testMatchIsKeyword */', + 'T_MATCH', + ], + [ + '/* testMatchDefaultIsKeyword */', + 'T_MATCH_DEFAULT', + ], + [ + '/* testFnIsKeyword */', + 'T_FN', + ], + + [ + '/* testYieldIsKeyword */', + 'T_YIELD', + ], + [ + '/* testYieldFromIsKeyword */', + 'T_YIELD_FROM', + ], + + [ + '/* testDeclareIsKeyword */', + 'T_DECLARE', + ], + [ + '/* testEndDeclareIsKeyword */', + 'T_ENDDECLARE', + ], + + [ + '/* testAndIsKeyword */', + 'T_LOGICAL_AND', + ], + [ + '/* testOrIsKeyword */', + 'T_LOGICAL_OR', + ], + [ + '/* testXorIsKeyword */', + 'T_LOGICAL_XOR', + ], + + [ + '/* testAnonymousClassIsKeyword */', + 'T_ANON_CLASS', + ], + [ + '/* testExtendsInAnonymousClassIsKeyword */', + 'T_EXTENDS', + ], + [ + '/* testImplementsInAnonymousClassIsKeyword */', + 'T_IMPLEMENTS', + ], + [ + '/* testNamespaceInNameIsKeyword */', + 'T_NAMESPACE', + ], + ]; + + }//end dataKeywords() + + +}//end class From c73f456fb650e2d1ec3796dce41d08b09fa27c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaroslav=20Hansl=C3=ADk?= Date: Tue, 21 Dec 2021 10:24:31 +0100 Subject: [PATCH 2/2] Removed dead code --- src/Tokenizers/PHP.php | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index 7ddfa35350..0d37929ff2 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -1786,22 +1786,6 @@ function return types. We want to keep the parenthesis map clean, && is_array($tokens[$x]) === false && $tokens[$x] === ':' ) { - $allowed = [ - T_STRING => T_STRING, - T_NAME_FULLY_QUALIFIED => T_NAME_FULLY_QUALIFIED, - T_NAME_RELATIVE => T_NAME_RELATIVE, - T_NAME_QUALIFIED => T_NAME_QUALIFIED, - T_ARRAY => T_ARRAY, - T_CALLABLE => T_CALLABLE, - T_SELF => T_SELF, - T_PARENT => T_PARENT, - T_NAMESPACE => T_NAMESPACE, - T_STATIC => T_STATIC, - T_NS_SEPARATOR => T_NS_SEPARATOR, - ]; - - $allowed += Util\Tokens::$emptyTokens; - // Find the start of the return type. for ($x += 1; $x < $numTokens; $x++) { if (is_array($tokens[$x]) === true