Skip to content

Commit

Permalink
handled nested attributes tokenization
Browse files Browse the repository at this point in the history
  • Loading branch information
@alekitto
alekitto committed Mar 19, 2021
1 parent 2f92a4d commit 868be7e
Show file tree
Hide file tree
Showing 4 changed files with 177 additions and 24 deletions.
39 changes: 29 additions & 10 deletions src/Tokenizers/PHP.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -927,7 +927,7 @@ protected function tokenize($string)
&& $token[0] === T_ATTRIBUTE
) {
// Go looking for the close bracket.
$bracketCloser = $this->findCloser($tokens, ($stackPtr + 1), '[', ']');
$bracketCloser = $this->findCloser($tokens, ($stackPtr + 1), ['[', '#['], ']');

$newToken = [];
$newToken['code'] = T_ATTRIBUTE;
Expand Down Expand Up @@ -3126,20 +3126,24 @@ public static function resolveSimpleToken($token)
* Finds a "closer" token (closing parenthesis or square bracket for example)
* Handle parenthesis balancing while searching for closing token
*
* @param array $tokens The list of tokens to iterate searching the closing token (as returned by token_get_all)
* @param int $start The starting position
* @param string $openerChar The opening character
* @param string $closerChar The closing character
* @param array $tokens The list of tokens to iterate searching the closing token (as returned by token_get_all)
* @param int $start The starting position
* @param string|string[] $openerTokens The opening character
* @param string $closerChar The closing character
*
* @return int|null The position of the closing token, if found. NULL otherwise.
*/
private function findCloser(array &$tokens, $start, $openerChar, $closerChar)
private function findCloser(array &$tokens, $start, $openerTokens, $closerChar)
{
$numTokens = count($tokens);
$stack = [0];
$closer = null;
$numTokens = count($tokens);
$stack = [0];
$closer = null;
$openerTokens = (array) $openerTokens;

for ($x = $start; $x < $numTokens; $x++) {
if ($tokens[$x] === $openerChar) {
if (in_array($tokens[$x], $openerTokens, true) === true
|| (is_array($tokens[$x]) === true && in_array($tokens[$x][1], $openerTokens, true) === true)
) {
$stack[] = $x;
} else if ($tokens[$x] === $closerChar) {
array_pop($stack);
Expand Down Expand Up @@ -3171,6 +3175,21 @@ private function parsePhpAttribute(array &$tokens, $stackPtr)

$commentBody = substr($token[1], 2);
$subTokens = @token_get_all('<?php '.$commentBody);

foreach ($subTokens as $i => $subToken) {
if (is_array($subToken) === true
&& $subToken[0] === T_COMMENT
&& strpos($subToken[1], '#[') === 0
) {
$reparsed = $this->parsePhpAttribute($subTokens, $i);
if ($reparsed !== null) {
array_splice($subTokens, $i, 1, $reparsed);
} else {
$subToken[0] = T_ATTRIBUTE;
}
}
}

array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]);

// Go looking for the close bracket.
Expand Down
41 changes: 28 additions & 13 deletions src/Tokenizers/Tokenizer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -741,24 +741,39 @@ private function createTokenMap()
$this->tokens[$opener]['parenthesis_closer'] = $i;
}//end if
} else if ($this->tokens[$i]['code'] === T_ATTRIBUTE) {
$found = null;
$numTokens = count($this->tokens);
for ($x = ($i + 1); $x < $numTokens; $x++) {
if ($this->tokens[$x]['code'] === T_ATTRIBUTE_END) {
$found = $x;
break;
}
$openers[] = $i;
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo str_repeat("\t", count($openers));
echo "=> Found attribute opener at $i".PHP_EOL;
}

$this->tokens[$i]['attribute_opener'] = $i;
$this->tokens[$i]['attribute_closer'] = $found;
$this->tokens[$i]['attribute_closer'] = null;
} else if ($this->tokens[$i]['code'] === T_ATTRIBUTE_END) {
$numOpeners = count($openers);
if ($numOpeners !== 0) {
$opener = array_pop($openers);
if (isset($this->tokens[$opener]['attribute_opener']) === true) {
$this->tokens[$opener]['attribute_closer'] = $i;

if ($found !== null) {
for ($x = ($i + 1); $x <= $found; ++$x) {
$this->tokens[$x]['attribute_opener'] = $i;
$this->tokens[$x]['attribute_closer'] = $found;
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo str_repeat("\t", (count($openers) + 1));
echo "=> Found attribute closer at $i for $opener".PHP_EOL;
}

for ($x = ($opener + 1); $x <= $i; ++$x) {
if (isset($this->tokens[$x]['attribute_closer']) === true) {
continue;
}

$this->tokens[$x]['attribute_opener'] = $opener;
$this->tokens[$x]['attribute_closer'] = $i;
}
} else if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo str_repeat("\t", (count($openers) + 1));
echo "=> Found unowned attribute closer at $i for $opener".PHP_EOL;
}
}
}//end if
}//end if

/*
Expand Down
4 changes: 4 additions & 0 deletions tests/Core/Tokenizer/AttributesTest.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ function multiple_attributes_on_parameter_test(#[ParamAttribute, AttributeWithPa
#[Boo\QualifiedName, \Foo\FullyQualifiedName('foo')]
function fqcn_attrebute_test() {}

/* testNestedAttributes */
#[Boo\QualifiedName(fn (#[AttributeOne('boo')] $value) => (string) $value)]
function nested_attributes_test() {}

/* testMultilineAttributesOnParameter */
function multiline_attributes_on_parameter_test(#[
AttributeWithParams(
Expand Down
117 changes: 116 additions & 1 deletion tests/Core/Tokenizer/AttributesTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,7 @@ public function testAttributeAndLineComment()
* @param string $testMarker The comment which prefaces the target token in the test file.
* @param int $position The token position (starting from T_FUNCTION) of T_ATTRIBUTE token.
* @param int $length The number of tokens between opener and closer.
* @param array $tokenCodes The codes of tokens inside the attributes.
*
* @dataProvider dataAttributeOnParameters
*
Expand All @@ -303,7 +304,7 @@ public function testAttributeAndLineComment()
*
* @return void
*/
public function testAttributeOnParameters($testMarker, $position, $length)
public function testAttributeOnParameters($testMarker, $position, $length, array $tokenCodes)
{
$tokens = self::$phpcsFile->getTokens();

Expand All @@ -312,6 +313,7 @@ public function testAttributeOnParameters($testMarker, $position, $length)

$this->assertSame(T_ATTRIBUTE, $tokens[$attribute]['code']);
$this->assertArrayHasKey('attribute_closer', $tokens[$attribute]);

$this->assertSame(($attribute + $length), $tokens[$attribute]['attribute_closer']);

$closer = $tokens[$attribute]['attribute_closer'];
Expand All @@ -322,6 +324,18 @@ public function testAttributeOnParameters($testMarker, $position, $length)
$this->assertSame(T_VARIABLE, $tokens[($closer + 4)]['code']);
$this->assertSame('$param', $tokens[($closer + 4)]['content']);

$map = array_map(
function ($token) use ($attribute, $length) {
$this->assertArrayHasKey('attribute_closer', $token);
$this->assertSame(($attribute + $length), $token['attribute_closer']);

return $token['code'];
},
array_slice($tokens, ($attribute + 1), ($length - 1))
);

$this->assertSame($tokenCodes, $map);

}//end testAttributeOnParameters()


Expand All @@ -339,16 +353,42 @@ public function dataAttributeOnParameters()
'/* testSingleAttributeOnParameter */',
4,
2,
[T_STRING],
],
[
'/* testMultipleAttributesOnParameter */',
4,
10,
[
T_STRING,
T_COMMA,
T_WHITESPACE,
T_STRING,
T_OPEN_PARENTHESIS,
T_COMMENT,
T_WHITESPACE,
T_CONSTANT_ENCAPSED_STRING,
T_CLOSE_PARENTHESIS,
],
],
[
'/* testMultilineAttributesOnParameter */',
4,
13,
[
T_WHITESPACE,
T_WHITESPACE,
T_STRING,
T_OPEN_PARENTHESIS,
T_WHITESPACE,
T_WHITESPACE,
T_CONSTANT_ENCAPSED_STRING,
T_WHITESPACE,
T_WHITESPACE,
T_CLOSE_PARENTHESIS,
T_WHITESPACE,
T_WHITESPACE,
],
],
];

Expand Down Expand Up @@ -376,4 +416,79 @@ public function testInvalidAttribute()
}//end testInvalidAttribute()


/**
* Test that nested attributes are parsed correctly.
*
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
* @covers PHP_CodeSniffer\Tokenizers\PHP::findCloser
* @covers PHP_CodeSniffer\Tokenizers\PHP::parsePhpAttribute
*
* @return void
*/
public function testNestedAttributes()
{
$tokens = self::$phpcsFile->getTokens();
$tokenCodes = [
T_STRING,
T_NS_SEPARATOR,
T_STRING,
T_OPEN_PARENTHESIS,
T_FN,
T_WHITESPACE,
T_OPEN_PARENTHESIS,
T_ATTRIBUTE,
T_STRING,
T_OPEN_PARENTHESIS,
T_CONSTANT_ENCAPSED_STRING,
T_CLOSE_PARENTHESIS,
T_ATTRIBUTE_END,
T_WHITESPACE,
T_VARIABLE,
T_CLOSE_PARENTHESIS,
T_WHITESPACE,
T_FN_ARROW,
T_WHITESPACE,
T_STRING_CAST,
T_WHITESPACE,
T_VARIABLE,
T_CLOSE_PARENTHESIS,
];

$attribute = $this->getTargetToken('/* testNestedAttributes */', T_ATTRIBUTE);
$this->assertArrayHasKey('attribute_closer', $tokens[$attribute]);

$closer = $tokens[$attribute]['attribute_closer'];
$this->assertSame(($attribute + 24), $closer);

$this->assertSame(T_ATTRIBUTE_END, $tokens[$closer]['code']);

$this->assertSame($tokens[$attribute]['attribute_opener'], $tokens[$closer]['attribute_opener']);
$this->assertSame($tokens[$attribute]['attribute_closer'], $tokens[$closer]['attribute_closer']);

$test = function (array $tokens, $length) use ($attribute) {
foreach ($tokens as $token) {
$this->assertArrayHasKey('attribute_closer', $token);
$this->assertSame(($attribute + $length), $token['attribute_closer']);
}
};

$test(array_slice($tokens, ($attribute + 1), 7), 24);

// Length here is 8 (nested attribute offset) + 5 (real length).
$test(array_slice($tokens, ($attribute + 8), 6), 8 + 5);

$test(array_slice($tokens, ($attribute + 14), 11), 24);

$map = array_map(
static function ($token) {
return $token['code'];
},
array_slice($tokens, ($attribute + 1), 23)
);

$this->assertSame($tokenCodes, $map);

}//end testNestedAttributes()


}//end class

0 comments on commit 868be7e

Please sign in to comment.