Skip to content

Commit

Permalink
PHP 8.0: "undo" namespaced names as single token
Browse files Browse the repository at this point in the history
As per the proposal in 3041.

This effectively "undoes" the new PHP 8.0 tokenization of identifier names for PHPCS 3.x.

Includes extensive unit tests to ensure the correct re-tokenization as well as that the rest of the tokenization is not adversely affected by this change.

Includes preventing `function ...` within a group use statement from breaking the retokenization.

Includes fixing the nullable tokenization when combined with any of the new PHP 8 identifier name tokens.
  • Loading branch information
jrfnl committed Sep 2, 2020
1 parent 1371c59 commit ba63323
Show file tree
Hide file tree
Showing 5 changed files with 1,562 additions and 9 deletions.
6 changes: 6 additions & 0 deletions package.xml
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<file baseinstalldir="" name="StableCommentWhitespaceTest.php" role="test" />
<file baseinstalldir="" name="StableCommentWhitespaceWinTest.inc" role="test" />
<file baseinstalldir="" name="StableCommentWhitespaceWinTest.php" role="test" />
<file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.inc" role="test" />
<file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.php" role="test" />
</dir>
<file baseinstalldir="" name="AbstractMethodUnitTest.php" role="test" />
<file baseinstalldir="" name="AllTests.php" role="test" />
Expand Down Expand Up @@ -1997,6 +1999,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
</filelist>
Expand Down Expand Up @@ -2058,6 +2062,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
<ignore name="bin/phpcs.bat" />
Expand Down
109 changes: 100 additions & 9 deletions src/Tokenizers/PHP.php
Original file line number Diff line number Diff line change
Expand Up @@ -815,6 +815,81 @@ protected function tokenize($string)
continue;
}//end if

/*
As of PHP 8.0 fully qualified, partially qualified and namespace relative
identifier names are tokenized differently.
This "undoes" the new tokenization so the tokenization will be the same in
in PHP 5, 7 and 8.
*/

if (PHP_VERSION_ID >= 80000
&& $tokenIsArray === true
&& ($token[0] === T_NAME_QUALIFIED
|| $token[0] === T_NAME_FULLY_QUALIFIED
|| $token[0] === T_NAME_RELATIVE)
) {
$name = $token[1];

if ($token[0] === T_NAME_FULLY_QUALIFIED) {
$newToken = [];
$newToken['code'] = T_NS_SEPARATOR;
$newToken['type'] = 'T_NS_SEPARATOR';
$newToken['content'] = '\\';
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;

$name = ltrim($name, '\\');
}

if ($token[0] === T_NAME_RELATIVE) {
$newToken = [];
$newToken['code'] = T_NAMESPACE;
$newToken['type'] = 'T_NAMESPACE';
$newToken['content'] = substr($name, 0, 9);
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;

$newToken = [];
$newToken['code'] = T_NS_SEPARATOR;
$newToken['type'] = 'T_NS_SEPARATOR';
$newToken['content'] = '\\';
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;

$name = substr($name, 10);
}

$parts = explode('\\', $name);
$partCount = count($parts);
$lastPart = ($partCount - 1);

foreach ($parts as $i => $part) {
$newToken = [];
$newToken['code'] = T_STRING;
$newToken['type'] = 'T_STRING';
$newToken['content'] = $part;
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;

if ($i !== $lastPart) {
$newToken = [];
$newToken['code'] = T_NS_SEPARATOR;
$newToken['type'] = 'T_NS_SEPARATOR';
$newToken['content'] = '\\';
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;
}
}

if (PHP_CODESNIFFER_VERBOSITY > 1) {
$type = Util\Tokens::tokenName($token[0]);
$content = Util\Common::prepareForOutput($token[1]);
echo "\t\t* token $stackPtr split into individual tokens; was: $type => $content".PHP_EOL;
}

continue;
}//end if

/*
Before PHP 7.0, the "yield from" was tokenized as
T_YIELD, T_WHITESPACE and T_STRING. So look for
Expand Down Expand Up @@ -1131,7 +1206,7 @@ protected function tokenize($string)
* Check if the next non-empty token is one of the tokens which can be used
* in type declarations. If not, it's definitely a ternary.
* At this point, the only token types which need to be taken into consideration
* as potential type declarations are T_STRING, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR.
* as potential type declarations are identifier names, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR.
*/

$lastRelevantNonEmpty = null;
Expand All @@ -1148,6 +1223,9 @@ protected function tokenize($string)
}

if ($tokenType === T_STRING
|| $tokenType === T_NAME_FULLY_QUALIFIED
|| $tokenType === T_NAME_RELATIVE
|| $tokenType === T_NAME_QUALIFIED
|| $tokenType === T_ARRAY
|| $tokenType === T_NS_SEPARATOR
) {
Expand All @@ -1159,7 +1237,10 @@ protected function tokenize($string)
&& isset($lastRelevantNonEmpty) === false)
|| ($lastRelevantNonEmpty === T_ARRAY
&& $tokenType === '(')
|| ($lastRelevantNonEmpty === T_STRING
|| (($lastRelevantNonEmpty === T_STRING
|| $lastRelevantNonEmpty === T_NAME_FULLY_QUALIFIED
|| $lastRelevantNonEmpty === T_NAME_RELATIVE
|| $lastRelevantNonEmpty === T_NAME_QUALIFIED)
&& ($tokenType === T_DOUBLE_COLON
|| $tokenType === '('
|| $tokenType === ':'))
Expand Down Expand Up @@ -1304,6 +1385,10 @@ protected function tokenize($string)
tokenized as T_STRING even if it appears to be a different token,
such as when writing code like: function default(): foo
so go forward and change the token type before it is processed.
Note: this should not be done for `function Level\Name` within a
group use statement for the PHP 8 identifier name tokens as it
would interfere with the re-tokenization of those.
*/

if ($tokenIsArray === true
Expand All @@ -1321,7 +1406,10 @@ protected function tokenize($string)
}
}

if ($x < $numTokens && is_array($tokens[$x]) === true) {
if ($x < $numTokens
&& is_array($tokens[$x]) === true
&& $tokens[$x][0] !== T_NAME_QUALIFIED
) {
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$oldType = Util\Tokens::tokenName($tokens[$x][0]);
echo "\t\t* token $x changed from $oldType to T_STRING".PHP_EOL;
Expand Down Expand Up @@ -1377,12 +1465,15 @@ function return types. We want to keep the parenthesis map clean,
&& $tokens[$x] === ':'
) {
$allowed = [
T_STRING => T_STRING,
T_ARRAY => T_ARRAY,
T_CALLABLE => T_CALLABLE,
T_SELF => T_SELF,
T_PARENT => T_PARENT,
T_NS_SEPARATOR => T_NS_SEPARATOR,
T_STRING => T_STRING,
T_NAME_FULLY_QUALIFIED => T_NAME_FULLY_QUALIFIED,
T_NAME_RELATIVE => T_NAME_RELATIVE,
T_NAME_QUALIFIED => T_NAME_QUALIFIED,
T_ARRAY => T_ARRAY,
T_CALLABLE => T_CALLABLE,
T_SELF => T_SELF,
T_PARENT => T_PARENT,
T_NS_SEPARATOR => T_NS_SEPARATOR,
];

$allowed += Util\Tokens::$emptyTokens;
Expand Down
12 changes: 12 additions & 0 deletions src/Util/Tokens.php
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,18 @@
define('T_NULLSAFE_OBJECT_OPERATOR', 'PHPCS_T_NULLSAFE_OBJECT_OPERATOR');
}

if (defined('T_NAME_QUALIFIED') === false) {
define('T_NAME_QUALIFIED', 'PHPCS_T_NAME_QUALIFIED');
}

if (defined('T_NAME_FULLY_QUALIFIED') === false) {
define('T_NAME_FULLY_QUALIFIED', 'PHPCS_T_NAME_FULLY_QUALIFIED');
}

if (defined('T_NAME_RELATIVE') === false) {
define('T_NAME_RELATIVE', 'PHPCS_T_NAME_RELATIVE');
}

// Tokens used for parsing doc blocks.
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');
Expand Down
147 changes: 147 additions & 0 deletions tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
<?php

/* testNamespaceDeclaration */
namespace Package;

/* testNamespaceDeclarationWithLevels */
namespace Vendor\SubLevel\Domain;

/* testUseStatement */
use ClassName;

/* testUseStatementWithLevels */
use Vendor\Level\Domain;

/* testFunctionUseStatement */
use function function_name;

/* testFunctionUseStatementWithLevels */
use function Vendor\Level\function_in_ns;

/* testConstantUseStatement */
use const CONSTANT_NAME;

/* testConstantUseStatementWithLevels */
use const Vendor\Level\OTHER_CONSTANT;

/* testMultiUseUnqualified */
use UnqualifiedClassName,
/* testMultiUsePartiallyQualified */
Sublevel\PartiallyClassName;

/* testGroupUseStatement */
use Vendor\Level\{
AnotherDomain,
function function_grouped,
const CONSTANT_GROUPED,
Sub\YetAnotherDomain,
function SubLevelA\function_grouped_too,
const SubLevelB\CONSTANT_GROUPED_TOO,
};

/* testClassName */
class MyClass
/* testExtendedFQN */
extends \Vendor\Level\FQN
/* testImplementsRelative */
implements namespace\Name,
/* testImplementsFQN */
\Fully\Qualified,
/* testImplementsUnqualified */
Unqualified,
/* testImplementsPartiallyQualified */
Sub\Level\Name
{
/* testFunctionName */
public function function_name(
/* testTypeDeclarationRelative */
?namespace\Name|object $paramA,

/* testTypeDeclarationFQN */
\Fully\Qualified\Name $paramB,

/* testTypeDeclarationUnqualified */
Unqualified|false $paramC,

/* testTypeDeclarationPartiallyQualified */
?Sublevel\Name $paramD,

/* testReturnTypeFQN */
) : ?\Name {

try {
/* testFunctionCallRelative */
echo NameSpace\function_name();

/* testFunctionCallFQN */
echo \Vendor\Package\function_name();

/* testFunctionCallUnqualified */
echo function_name();

/* testFunctionPartiallyQualified */
echo Level\function_name();

/* testCatchRelative */
} catch (namespace\SubLevel\Exception $e) {

/* testCatchFQN */
} catch (\Exception $e) {

/* testCatchUnqualified */
} catch (Exception $e) {

/* testCatchPartiallyQualified */
} catch (Level\Exception $e) {
}

/* testNewRelative */
$obj = new namespace\ClassName();

/* testNewFQN */
$obj = new \Vendor\ClassName();

/* testNewUnqualified */
$obj = new ClassName;

/* testNewPartiallyQualified */
$obj = new Level\ClassName;

/* testDoubleColonRelative */
$value = namespace\ClassName::property;

/* testDoubleColonFQN */
$value = \ClassName::static_function();

/* testDoubleColonUnqualified */
$value = ClassName::CONSTANT_NAME;

/* testDoubleColonPartiallyQualified */
$value = Level\ClassName::CONSTANT_NAME['key'];

/* testInstanceOfRelative */
$is = $obj instanceof namespace\ClassName;

/* testInstanceOfFQN */
if ($obj instanceof \Full\ClassName) {}

/* testInstanceOfUnqualified */
if ($a === $b && $obj instanceof ClassName && true) {}

/* testInstanceOfPartiallyQualified */
$is = $obj instanceof Partially\ClassName;
}
}

/* testInvalidInPHP8Whitespace */
namespace \ Sublevel
\ function_name();

/* testInvalidInPHP8Comments */
$value = \Fully
// phpcs:ignore Stnd.Cat.Sniff -- for reasons
\Qualified
/* comment */
\Name
// comment
:: function_name();
Loading

0 comments on commit ba63323

Please sign in to comment.