diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index 3fc67b0c81..5b9962b69d 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -566,6 +566,17 @@ protected function tokenize($string) $lastNotEmptyToken = ($newStackPtr - 1); } + // Get the next non-empty token. + $nextNonEmptyToken = null; + for ($i = ($stackPtr + 1); $i < $numTokens; $i++) { + if (is_array($tokens[$i]) === false + || isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === false + ) { + $nextNonEmptyToken = $i; + break; + } + } + /* If we are using \r\n newline characters, the \r and \n are sometimes split over two tokens. This normally occurs after comments. We need @@ -601,15 +612,22 @@ protected function tokenize($string) echo PHP_EOL; } + if (PHP_VERSION_ID < 802000 && $token[0] === T_STRING && strtolower($token[1]) === 'readonly') { + $token[0] = T_READONLY; + } + /* - Tokenize context sensitive keyword as string when it should be string. + Tokenize context-sensitive keyword as string when it should be string. */ if ($tokenIsArray === true && isset(Util\Tokens::$contextSensitiveKeywords[$token[0]]) === true && (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true - || $finalTokens[$lastNotEmptyToken]['content'] === '&') + || $finalTokens[$lastNotEmptyToken]['content'] === '&' + || (isset($nextNonEmptyToken) === true && $tokens[$nextNonEmptyToken] === '(')) ) { + $preserveKeyword = true; + if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) { $preserveKeyword = false; @@ -649,8 +667,6 @@ protected function tokenize($string) }//end if if ($finalTokens[$lastNotEmptyToken]['content'] === '&') { - $preserveKeyword = true; - for ($i = ($lastNotEmptyToken - 1); $i >= 0; $i--) { if (isset(Util\Tokens::$emptyTokens[$finalTokens[$i]['code']]) === true) { continue; @@ -664,6 +680,61 @@ protected function tokenize($string) } } + if ($preserveKeyword === true + && $token[0] === T_READONLY + && isset($nextNonEmptyToken) === true + && $tokens[$nextNonEmptyToken] === '(' + ) { + $foundProperty = false; + $foundDNFCloseParen = false; + $foundDNFPipe = false; + + for ($i = ($nextNonEmptyToken + 1); $i < $numTokens; $i++) { + if ($foundDNFCloseParen === false && $tokens[$i] === ')') { + $foundDNFCloseParen = true; + continue; + } + + if ($foundDNFCloseParen === true && $foundDNFPipe === false && $tokens[$i] === '|') { + $foundDNFPipe = true; + continue; + } + + $stopTokens = [ + '{', + '}', + '=', + ';', + '|', + ':', + ',', + '(', + ')', + ]; + if (in_array($tokens[$i], $stopTokens, true) === true) { + // We have finished our search. + break; + } + + if ($foundDNFPipe !== true) { + continue; + } + + if (is_array($tokens[$i]) === false) { + continue; + } + + if ($tokens[$i][0] === T_VARIABLE) { + $foundProperty = true; + break; + } + }//end for + + if ($foundProperty === false) { + $preserveKeyword = false; + } + }//end if + if ($preserveKeyword === false) { if (PHP_CODESNIFFER_VERBOSITY > 1) { $type = Util\Tokens::tokenName($token[0]); @@ -1012,18 +1083,9 @@ protected function tokenize($string) && $token[0] === T_STRING && strtolower($token[1]) === 'enum' ) { - // Get the next non-empty token. - for ($i = ($stackPtr + 1); $i < $numTokens; $i++) { - if (is_array($tokens[$i]) === false - || isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === false - ) { - break; - } - } - - if (isset($tokens[$i]) === true - && is_array($tokens[$i]) === true - && $tokens[$i][0] === T_STRING + if (isset($tokens[$nextNonEmptyToken]) === true + && is_array($tokens[$nextNonEmptyToken]) === true + && $tokens[$nextNonEmptyToken][0] === T_STRING ) { // Modify $tokens directly so we can use it later when converting enum "case". $tokens[$stackPtr][0] = T_ENUM; @@ -1230,18 +1292,9 @@ protected function tokenize($string) && ($token[0] === T_STRING || preg_match('`^[a-zA-Z_\x80-\xff]`', $token[1]) === 1) ) { - // Get the next non-empty token. - for ($i = ($stackPtr + 1); $i < $numTokens; $i++) { - if (is_array($tokens[$i]) === false - || isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === false - ) { - break; - } - } - - if (isset($tokens[$i]) === true - && is_array($tokens[$i]) === false - && $tokens[$i] === ':' + if (isset($tokens[$nextNonEmptyToken]) === true + && is_array($tokens[$nextNonEmptyToken]) === false + && $tokens[$nextNonEmptyToken] === ':' ) { // Get the previous non-empty token. for ($j = ($stackPtr - 1); $j > 0; $j--) { @@ -1287,17 +1340,8 @@ protected function tokenize($string) && strtolower($token[1]) === 'readonly' && isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === false ) { - // Get the next non-whitespace token. - for ($i = ($stackPtr + 1); $i < $numTokens; $i++) { - if (is_array($tokens[$i]) === false - || $tokens[$i][0] !== T_WHITESPACE - ) { - break; - } - } - - if (isset($tokens[$i]) === false - || $tokens[$i] !== '(' + if (isset($tokens[$nextNonEmptyToken]) === false + || $tokens[$nextNonEmptyToken] !== '(' ) { $finalTokens[$newStackPtr] = [ 'code' => T_READONLY, diff --git a/tests/Core/Tokenizer/BackfillReadonlyTest.inc b/tests/Core/Tokenizer/BackfillReadonlyTest.inc index eaf0b4b3cc..a93a2d96f2 100644 --- a/tests/Core/Tokenizer/BackfillReadonlyTest.inc +++ b/tests/Core/Tokenizer/BackfillReadonlyTest.inc @@ -95,6 +95,63 @@ echo ClassName::READONLY; /* testReadonlyUsedAsFunctionCallWithSpaceBetweenKeywordAndParens */ $var = readonly /* comment */ (); +// These test cases are inspired by +// https://github.com/php/php-src/commit/08b75395838b4b42a41e3c70684fa6c6b113eee0 +class Dnf +{ + /* testDNFPropertyReadonlyPublicABC */ + readonly public A|(B&C) $a; + /* testDNFPropertyReadonlyPublicBCA */ + readonly public (B&C)|A $b; + + /* testDNFPropertyReadonlyProtectedABC */ + readonly protected A|(B&C) $c; + /* testDNFPropertyReadonlyProtectedBCA */ + readonly protected (B&C)|A $d; + + /* testDNFPropertyReadonlyPrivateABC */ + readonly private A|(B&C) $e; + /* testDNFPropertyReadonlyPrivateBCA */ + readonly private (B&C)|A $f; + + /* testDNFPropertyReadonlyABC */ + readonly A|(B&C) $g; + /* testDNFPropertyReadonlyBCA */ + readonly (B&C)|A $h; + + /* testDNFPropertyPublicReadonlyABC */ + public readonly A|(B&C) $i; + /* testDNFPropertyPublicReadonlyBCA */ + public readonly (B&C)|A $j; + + /* testDNFPropertyProtectedReadonlyABC */ + protected readonly A|(B&C) $k; + /* testDNFPropertyProtectedReadonlyBCA */ + protected readonly (B&C)|A $l; + + /* testDNFPropertyPrivateReadonlyABC */ + private readonly A|(B&C) $m; + /* testDNFPropertyPrivateReadonlyBCA */ + private readonly (B&C)|A $n; + + /* testDNFPropertyPrivateReadonlyB_space_CA */ + private readonly (B & C)|A $o; + /* testDNFPropertyPrivateReadonlyBC_space_A */ + private readonly (B&C) | A $p; + /* testDNFPropertyPrivateReadonlyB_space_C_space_A */ + private readonly (B & C) | A $q; + + /* testDNFMethodAB */ + public function readonly (A&B $param): void {} + + public function __construct( + /* testDNFPropertyPromotionABC */ + private readonly A|(B&C) $a1, + /* testDNFPropertyPromotionBCA */ + private readonly (B&C)|A $b1, + ) {} +} + /* testParseErrorLiveCoding */ // This must be the last test in the file. readonly diff --git a/tests/Core/Tokenizer/BackfillReadonlyTest.php b/tests/Core/Tokenizer/BackfillReadonlyTest.php index dddc18ebc2..1a2fbe33ba 100644 --- a/tests/Core/Tokenizer/BackfillReadonlyTest.php +++ b/tests/Core/Tokenizer/BackfillReadonlyTest.php @@ -148,7 +148,79 @@ public function dataReadonly() 'readonly', ], [ - '/* testReadonlyUsedAsFunctionCallWithSpaceBetweenKeywordAndParens */', + '/* testDNFPropertyReadonlyPublicABC */', + 'readonly', + ], + [ + '/* testDNFPropertyReadonlyPublicBCA */', + 'readonly', + ], + [ + '/* testDNFPropertyReadonlyProtectedABC */', + 'readonly', + ], + [ + '/* testDNFPropertyReadonlyProtectedBCA */', + 'readonly', + ], + [ + '/* testDNFPropertyReadonlyPrivateABC */', + 'readonly', + ], + [ + '/* testDNFPropertyReadonlyPrivateBCA */', + 'readonly', + ], + [ + '/* testDNFPropertyReadonlyABC */', + 'readonly', + ], + [ + '/* testDNFPropertyReadonlyBCA */', + 'readonly', + ], + [ + '/* testDNFPropertyPublicReadonlyABC */', + 'readonly', + ], + [ + '/* testDNFPropertyPublicReadonlyBCA */', + 'readonly', + ], + [ + '/* testDNFPropertyProtectedReadonlyABC */', + 'readonly', + ], + [ + '/* testDNFPropertyProtectedReadonlyBCA */', + 'readonly', + ], + [ + '/* testDNFPropertyPrivateReadonlyABC */', + 'readonly', + ], + [ + '/* testDNFPropertyPrivateReadonlyBCA */', + 'readonly', + ], + [ + '/* testDNFPropertyPrivateReadonlyB_space_CA */', + 'readonly', + ], + [ + '/* testDNFPropertyPrivateReadonlyBC_space_A */', + 'readonly', + ], + [ + '/* testDNFPropertyPrivateReadonlyB_space_C_space_A */', + 'readonly', + ], + [ + '/* testDNFPropertyPromotionABC */', + 'readonly', + ], + [ + '/* testDNFPropertyPromotionBCA */', 'readonly', ], [ @@ -224,10 +296,18 @@ public function dataNotReadonly() '/* testReadonlyAsFunctionCall */', 'readonly', ], + [ + '/* testReadonlyUsedAsFunctionCallWithSpaceBetweenKeywordAndParens */', + 'readonly', + ], [ '/* testClassConstantFetchWithReadonlyAsConstantName */', 'READONLY', ], + [ + '/* testDNFMethodAB */', + 'readonly', + ], ]; }//end dataNotReadonly()