Skip to content

Commit

Permalink
T_FN tokens now have scope openers and closers (ref #2523)
Browse files Browse the repository at this point in the history 
This changed the way the T_FN token was backfilled as it now needs to set before processAdditional kicks in so it also works in PHP 7.4. Scope information is added in processAdditional because the way the opener and closer is calulated is too specific to put into the generic scope mapping code.
  • Loading branch information
@gsherwood
gsherwood committed Nov 6, 2019
1 parent 033b431 commit 51afb54
Show file tree
Hide file tree
Showing 4 changed files with 354 additions and 36 deletions.
5 changes: 4 additions & 1 deletion package.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@ http://pear.php.net/dtd/package-2.0.xsd">
<notes>
- The PHP 7.4 T_FN token has been made available for older versions
-- T_FN represents the fn string used for arrow functions
-- The token is associated with the opening and closing parenthesis of the statement
-- The double arrow becomes the scope opener
-- The token after the statement (normally a semicolon) becomes the scope closer
-- The token is also associated with the opening and closing parenthesis of the statement
-- Any functions named "fn" will cause have a T_FN token for the function name, but have no scope information
- File::getMethodParameters() now supports arrow functions
- File::getMethodProperties() now supports arrow functions
- Generic.CodeAnalysis.EmptyPhpStatement now reports unnecessary semicolons after control structure closing braces
Expand Down
114 changes: 98 additions & 16 deletions src/Tokenizers/PHP.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1217,6 +1217,28 @@ function return types. We want to keep the parenthesis map clean,
continue;
}

/*
Backfill the T_FN token for PHP versions < 7.4.
*/

if ($tokenIsArray === true
&& $token[0] === T_STRING
&& strtolower($token[1]) === 'fn'
) {
$finalTokens[$newStackPtr] = [
'content' => $token[1],
'code' => T_FN,
'type' => 'T_FN',
];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* token $stackPtr changed from T_STRING to T_FN".PHP_EOL;
}

$newStackPtr++;
continue;
}

/*
PHP doesn't assign a token to goto labels, so we have to.
These are just string tokens with a single colon after them. Double
Expand Down Expand Up @@ -1606,9 +1628,7 @@ protected function processAdditional()
}//end if

continue;
} else if ($this->tokens[$i]['code'] === T_STRING
&& strtolower($this->tokens[$i]['content']) === 'fn'
) {
} else if ($this->tokens[$i]['code'] === T_FN) {
// Possible arrow function.
for ($x = ($i + 1); $i < $numTokens; $x++) {
if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) {
Expand All @@ -1618,22 +1638,84 @@ protected function processAdditional()
}

if ($this->tokens[$x]['code'] === T_OPEN_PARENTHESIS) {
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$line = $this->tokens[$i]['line'];
echo "\t* token $i on line $line changed from T_STRING to T_FN".PHP_EOL;
$ignore = Util\Tokens::$emptyTokens;
$ignore += [
T_STRING => T_STRING,
T_COLON => T_COLON,
];

$closer = $this->tokens[$x]['parenthesis_closer'];
for ($arrow = ($closer + 1); $arrow < $numTokens; $arrow++) {
if (isset($ignore[$this->tokens[$arrow]['code']]) === false) {
break;
}
}

$this->tokens[$i]['code'] = T_FN;
$this->tokens[$i]['type'] = 'T_FN';
$this->tokens[$i]['parenthesis_owner'] = $i;
$this->tokens[$i]['parenthesis_opener'] = $x;
$this->tokens[$i]['parenthesis_closer'] = $this->tokens[$x]['parenthesis_closer'];
if ($this->tokens[$arrow]['code'] === T_DOUBLE_ARROW) {
$endTokens = [
T_COLON => true,
T_COMMA => true,
T_DOUBLE_ARROW => true,
T_SEMICOLON => true,
T_CLOSE_PARENTHESIS => true,
T_CLOSE_SQUARE_BRACKET => true,
T_CLOSE_CURLY_BRACKET => true,
T_CLOSE_SHORT_ARRAY => true,
T_OPEN_TAG => true,
T_CLOSE_TAG => true,
];

$opener = $this->tokens[$i]['parenthesis_opener'];
$closer = $this->tokens[$i]['parenthesis_closer'];
$this->tokens[$opener]['parenthesis_owner'] = $i;
$this->tokens[$closer]['parenthesis_owner'] = $i;
}
for ($scopeCloser = ($arrow + 1); $scopeCloser < $numTokens; $scopeCloser++) {
if (isset($endTokens[$this->tokens[$scopeCloser]['code']]) === true) {
break;
}

if (isset($this->tokens[$scopeCloser]['scope_closer']) === true) {
// We minus 1 here in case the closer can be shared with us.
$scopeCloser = ($this->tokens[$scopeCloser]['scope_closer'] - 1);
continue;
}

if (isset($this->tokens[$scopeCloser]['parenthesis_closer']) === true) {
$scopeCloser = $this->tokens[$scopeCloser]['parenthesis_closer'];
continue;
}

if (isset($this->tokens[$scopeCloser]['bracket_closer']) === true) {
$scopeCloser = $this->tokens[$scopeCloser]['bracket_closer'];
continue;
}
}//end for

if ($scopeCloser !== $numTokens) {
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$line = $this->tokens[$i]['line'];
echo "\t* token $i on line $line changed from T_STRING to T_FN".PHP_EOL;
}

$this->tokens[$i]['code'] = T_FN;
$this->tokens[$i]['type'] = 'T_FN';
$this->tokens[$i]['scope_condition'] = $i;
$this->tokens[$i]['scope_opener'] = $arrow;
$this->tokens[$i]['scope_closer'] = $scopeCloser;
$this->tokens[$i]['parenthesis_owner'] = $i;
$this->tokens[$i]['parenthesis_opener'] = $x;
$this->tokens[$i]['parenthesis_closer'] = $this->tokens[$x]['parenthesis_closer'];

$this->tokens[$arrow]['scope_condition'] = $i;
$this->tokens[$arrow]['scope_opener'] = $arrow;
$this->tokens[$arrow]['scope_closer'] = $scopeCloser;
$this->tokens[$scopeCloser]['scope_condition'] = $i;
$this->tokens[$scopeCloser]['scope_opener'] = $arrow;
$this->tokens[$scopeCloser]['scope_closer'] = $scopeCloser;

$opener = $this->tokens[$i]['parenthesis_opener'];
$closer = $this->tokens[$i]['parenthesis_closer'];
$this->tokens[$opener]['parenthesis_owner'] = $i;
$this->tokens[$closer]['parenthesis_owner'] = $i;
}//end if
}//end if
}//end if
} else if ($this->tokens[$i]['code'] === T_OPEN_SQUARE_BRACKET) {
if (isset($this->tokens[$i]['bracket_closer']) === false) {
continue;
Expand Down
21 changes: 21 additions & 0 deletions tests/Core/Tokenizer/BackfillFnTokenTest.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,24 @@ $fn1 = fn /* comment here */ ($x) => $x + $y;

/* testFunctionName */
function fn() {}

/* testNested */
$fn = fn($x) => fn($y) => $x * $y + $z;

/* testFunctionCall */
$extended = fn($c) => $callable($factory($c), $c);

/* testClosure */
$extended = fn($c) => $callable(function() {
for ($x = 1; $x < 10; $x++) {
echo $x;
}

echo 'done';
}, $c);

$result = array_map(
/* testReturnType */
static fn(int $number) : int => $number + 1,
$numbers
);
Loading

0 comments on commit 51afb54

Please sign in to comment.