From 868be7e76267567084017daee49dadff7e9d3b40 Mon Sep 17 00:00:00 2001 From: Alessandro Chitolina Date: Wed, 3 Feb 2021 09:43:37 +0100 Subject: [PATCH] handled nested attributes tokenization --- src/Tokenizers/PHP.php | 39 ++++++-- src/Tokenizers/Tokenizer.php | 41 ++++++--- tests/Core/Tokenizer/AttributesTest.inc | 4 + tests/Core/Tokenizer/AttributesTest.php | 117 +++++++++++++++++++++++- 4 files changed, 177 insertions(+), 24 deletions(-) diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index c46d8ab4ef..f5b6cdea01 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -927,7 +927,7 @@ protected function tokenize($string) && $token[0] === T_ATTRIBUTE ) { // Go looking for the close bracket. - $bracketCloser = $this->findCloser($tokens, ($stackPtr + 1), '[', ']'); + $bracketCloser = $this->findCloser($tokens, ($stackPtr + 1), ['[', '#['], ']'); $newToken = []; $newToken['code'] = T_ATTRIBUTE; @@ -3126,20 +3126,24 @@ public static function resolveSimpleToken($token) * Finds a "closer" token (closing parenthesis or square bracket for example) * Handle parenthesis balancing while searching for closing token * - * @param array $tokens The list of tokens to iterate searching the closing token (as returned by token_get_all) - * @param int $start The starting position - * @param string $openerChar The opening character - * @param string $closerChar The closing character + * @param array $tokens The list of tokens to iterate searching the closing token (as returned by token_get_all) + * @param int $start The starting position + * @param string|string[] $openerTokens The opening character + * @param string $closerChar The closing character * * @return int|null The position of the closing token, if found. NULL otherwise. */ - private function findCloser(array &$tokens, $start, $openerChar, $closerChar) + private function findCloser(array &$tokens, $start, $openerTokens, $closerChar) { - $numTokens = count($tokens); - $stack = [0]; - $closer = null; + $numTokens = count($tokens); + $stack = [0]; + $closer = null; + $openerTokens = (array) $openerTokens; + for ($x = $start; $x < $numTokens; $x++) { - if ($tokens[$x] === $openerChar) { + if (in_array($tokens[$x], $openerTokens, true) === true + || (is_array($tokens[$x]) === true && in_array($tokens[$x][1], $openerTokens, true) === true) + ) { $stack[] = $x; } else if ($tokens[$x] === $closerChar) { array_pop($stack); @@ -3171,6 +3175,21 @@ private function parsePhpAttribute(array &$tokens, $stackPtr) $commentBody = substr($token[1], 2); $subTokens = @token_get_all(' $subToken) { + if (is_array($subToken) === true + && $subToken[0] === T_COMMENT + && strpos($subToken[1], '#[') === 0 + ) { + $reparsed = $this->parsePhpAttribute($subTokens, $i); + if ($reparsed !== null) { + array_splice($subTokens, $i, 1, $reparsed); + } else { + $subToken[0] = T_ATTRIBUTE; + } + } + } + array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]); // Go looking for the close bracket. diff --git a/src/Tokenizers/Tokenizer.php b/src/Tokenizers/Tokenizer.php index 30156f22c1..a229d6979e 100644 --- a/src/Tokenizers/Tokenizer.php +++ b/src/Tokenizers/Tokenizer.php @@ -741,24 +741,39 @@ private function createTokenMap() $this->tokens[$opener]['parenthesis_closer'] = $i; }//end if } else if ($this->tokens[$i]['code'] === T_ATTRIBUTE) { - $found = null; - $numTokens = count($this->tokens); - for ($x = ($i + 1); $x < $numTokens; $x++) { - if ($this->tokens[$x]['code'] === T_ATTRIBUTE_END) { - $found = $x; - break; - } + $openers[] = $i; + if (PHP_CODESNIFFER_VERBOSITY > 1) { + echo str_repeat("\t", count($openers)); + echo "=> Found attribute opener at $i".PHP_EOL; } $this->tokens[$i]['attribute_opener'] = $i; - $this->tokens[$i]['attribute_closer'] = $found; + $this->tokens[$i]['attribute_closer'] = null; + } else if ($this->tokens[$i]['code'] === T_ATTRIBUTE_END) { + $numOpeners = count($openers); + if ($numOpeners !== 0) { + $opener = array_pop($openers); + if (isset($this->tokens[$opener]['attribute_opener']) === true) { + $this->tokens[$opener]['attribute_closer'] = $i; - if ($found !== null) { - for ($x = ($i + 1); $x <= $found; ++$x) { - $this->tokens[$x]['attribute_opener'] = $i; - $this->tokens[$x]['attribute_closer'] = $found; + if (PHP_CODESNIFFER_VERBOSITY > 1) { + echo str_repeat("\t", (count($openers) + 1)); + echo "=> Found attribute closer at $i for $opener".PHP_EOL; + } + + for ($x = ($opener + 1); $x <= $i; ++$x) { + if (isset($this->tokens[$x]['attribute_closer']) === true) { + continue; + } + + $this->tokens[$x]['attribute_opener'] = $opener; + $this->tokens[$x]['attribute_closer'] = $i; + } + } else if (PHP_CODESNIFFER_VERBOSITY > 1) { + echo str_repeat("\t", (count($openers) + 1)); + echo "=> Found unowned attribute closer at $i for $opener".PHP_EOL; } - } + }//end if }//end if /* diff --git a/tests/Core/Tokenizer/AttributesTest.inc b/tests/Core/Tokenizer/AttributesTest.inc index e620ec40cb..9b7b869d13 100644 --- a/tests/Core/Tokenizer/AttributesTest.inc +++ b/tests/Core/Tokenizer/AttributesTest.inc @@ -64,6 +64,10 @@ function multiple_attributes_on_parameter_test(#[ParamAttribute, AttributeWithPa #[Boo\QualifiedName, \Foo\FullyQualifiedName('foo')] function fqcn_attrebute_test() {} +/* testNestedAttributes */ +#[Boo\QualifiedName(fn (#[AttributeOne('boo')] $value) => (string) $value)] +function nested_attributes_test() {} + /* testMultilineAttributesOnParameter */ function multiline_attributes_on_parameter_test(#[ AttributeWithParams( diff --git a/tests/Core/Tokenizer/AttributesTest.php b/tests/Core/Tokenizer/AttributesTest.php index 5da1ed523f..0358d63965 100644 --- a/tests/Core/Tokenizer/AttributesTest.php +++ b/tests/Core/Tokenizer/AttributesTest.php @@ -294,6 +294,7 @@ public function testAttributeAndLineComment() * @param string $testMarker The comment which prefaces the target token in the test file. * @param int $position The token position (starting from T_FUNCTION) of T_ATTRIBUTE token. * @param int $length The number of tokens between opener and closer. + * @param array $tokenCodes The codes of tokens inside the attributes. * * @dataProvider dataAttributeOnParameters * @@ -303,7 +304,7 @@ public function testAttributeAndLineComment() * * @return void */ - public function testAttributeOnParameters($testMarker, $position, $length) + public function testAttributeOnParameters($testMarker, $position, $length, array $tokenCodes) { $tokens = self::$phpcsFile->getTokens(); @@ -312,6 +313,7 @@ public function testAttributeOnParameters($testMarker, $position, $length) $this->assertSame(T_ATTRIBUTE, $tokens[$attribute]['code']); $this->assertArrayHasKey('attribute_closer', $tokens[$attribute]); + $this->assertSame(($attribute + $length), $tokens[$attribute]['attribute_closer']); $closer = $tokens[$attribute]['attribute_closer']; @@ -322,6 +324,18 @@ public function testAttributeOnParameters($testMarker, $position, $length) $this->assertSame(T_VARIABLE, $tokens[($closer + 4)]['code']); $this->assertSame('$param', $tokens[($closer + 4)]['content']); + $map = array_map( + function ($token) use ($attribute, $length) { + $this->assertArrayHasKey('attribute_closer', $token); + $this->assertSame(($attribute + $length), $token['attribute_closer']); + + return $token['code']; + }, + array_slice($tokens, ($attribute + 1), ($length - 1)) + ); + + $this->assertSame($tokenCodes, $map); + }//end testAttributeOnParameters() @@ -339,16 +353,42 @@ public function dataAttributeOnParameters() '/* testSingleAttributeOnParameter */', 4, 2, + [T_STRING], ], [ '/* testMultipleAttributesOnParameter */', 4, 10, + [ + T_STRING, + T_COMMA, + T_WHITESPACE, + T_STRING, + T_OPEN_PARENTHESIS, + T_COMMENT, + T_WHITESPACE, + T_CONSTANT_ENCAPSED_STRING, + T_CLOSE_PARENTHESIS, + ], ], [ '/* testMultilineAttributesOnParameter */', 4, 13, + [ + T_WHITESPACE, + T_WHITESPACE, + T_STRING, + T_OPEN_PARENTHESIS, + T_WHITESPACE, + T_WHITESPACE, + T_CONSTANT_ENCAPSED_STRING, + T_WHITESPACE, + T_WHITESPACE, + T_CLOSE_PARENTHESIS, + T_WHITESPACE, + T_WHITESPACE, + ], ], ]; @@ -376,4 +416,79 @@ public function testInvalidAttribute() }//end testInvalidAttribute() + /** + * Test that nested attributes are parsed correctly. + * + * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize + * @covers PHP_CodeSniffer\Tokenizers\PHP::findCloser + * @covers PHP_CodeSniffer\Tokenizers\PHP::parsePhpAttribute + * + * @return void + */ + public function testNestedAttributes() + { + $tokens = self::$phpcsFile->getTokens(); + $tokenCodes = [ + T_STRING, + T_NS_SEPARATOR, + T_STRING, + T_OPEN_PARENTHESIS, + T_FN, + T_WHITESPACE, + T_OPEN_PARENTHESIS, + T_ATTRIBUTE, + T_STRING, + T_OPEN_PARENTHESIS, + T_CONSTANT_ENCAPSED_STRING, + T_CLOSE_PARENTHESIS, + T_ATTRIBUTE_END, + T_WHITESPACE, + T_VARIABLE, + T_CLOSE_PARENTHESIS, + T_WHITESPACE, + T_FN_ARROW, + T_WHITESPACE, + T_STRING_CAST, + T_WHITESPACE, + T_VARIABLE, + T_CLOSE_PARENTHESIS, + ]; + + $attribute = $this->getTargetToken('/* testNestedAttributes */', T_ATTRIBUTE); + $this->assertArrayHasKey('attribute_closer', $tokens[$attribute]); + + $closer = $tokens[$attribute]['attribute_closer']; + $this->assertSame(($attribute + 24), $closer); + + $this->assertSame(T_ATTRIBUTE_END, $tokens[$closer]['code']); + + $this->assertSame($tokens[$attribute]['attribute_opener'], $tokens[$closer]['attribute_opener']); + $this->assertSame($tokens[$attribute]['attribute_closer'], $tokens[$closer]['attribute_closer']); + + $test = function (array $tokens, $length) use ($attribute) { + foreach ($tokens as $token) { + $this->assertArrayHasKey('attribute_closer', $token); + $this->assertSame(($attribute + $length), $token['attribute_closer']); + } + }; + + $test(array_slice($tokens, ($attribute + 1), 7), 24); + + // Length here is 8 (nested attribute offset) + 5 (real length). + $test(array_slice($tokens, ($attribute + 8), 6), 8 + 5); + + $test(array_slice($tokens, ($attribute + 14), 11), 24); + + $map = array_map( + static function ($token) { + return $token['code']; + }, + array_slice($tokens, ($attribute + 1), 23) + ); + + $this->assertSame($tokenCodes, $map); + + }//end testNestedAttributes() + + }//end class