From bdf5c493d9a7be338b7e5d3077504dd192a38792 Mon Sep 17 00:00:00 2001 From: jrfnl Date: Fri, 29 Jul 2022 16:58:57 +0200 Subject: [PATCH] Tokenizer: apply tab replacement to heredoc/nowdoc closers Since PHP 7.3, heredoc/nowdoc closers may be indented. This indent can use either tabs or spaces and the indent is included in the `T_END_HEREDOC`/`T_END_NOWDOC` token contents as received from the PHP native tokenizer. However, the PHPCS `Tokenizer` did no execute tab replacement on these token leading to unexpected `'content'` and incorrect `'length'` values in the `File::$tokens` array, which in turn could lead to incorrect sniff results and incorrect fixes. This commit adds the `T_END_HEREDOC`/`T_END_NOWDOC` tokens to the array of tokens for which to do tab replacement to make them more consistent with the rest of PHPCS. I also considered splitting the token into a `T_WHITESPACE` token and the `T_END_HEREDOC`/`T_END_NOWDOC` token, but that could potentially break sniffs which expect the `T_END_HEREDOC`/`T_END_NOWDOC` token directly after the last `T_HEREDOC`/`T_NOWDOC` token. The current fix does not contain that risk. Includes unit tests safeguarding this change. The tests will only run on PHP 7.3+ as flexible heredoc/nowdocs don't tokenize correctly in PHP < 7.3. --- package.xml | 6 + src/Tokenizers/Tokenizer.php | 2 + .../Tokenizer/HeredocNowdocCloserTest.inc | 43 +++++ .../Tokenizer/HeredocNowdocCloserTest.php | 150 ++++++++++++++++++ 4 files changed, 201 insertions(+) create mode 100644 tests/Core/Tokenizer/HeredocNowdocCloserTest.inc create mode 100644 tests/Core/Tokenizer/HeredocNowdocCloserTest.php diff --git a/package.xml b/package.xml index 2c593d512a..44a50d158f 100644 --- a/package.xml +++ b/package.xml @@ -143,6 +143,8 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + @@ -2116,6 +2118,8 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + @@ -2220,6 +2224,8 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + diff --git a/src/Tokenizers/Tokenizer.php b/src/Tokenizers/Tokenizer.php index c79323ccd3..0e00bf7f22 100644 --- a/src/Tokenizers/Tokenizer.php +++ b/src/Tokenizers/Tokenizer.php @@ -194,6 +194,8 @@ private function createPositionMap() T_DOUBLE_QUOTED_STRING => true, T_HEREDOC => true, T_NOWDOC => true, + T_END_HEREDOC => true, + T_END_NOWDOC => true, T_INLINE_HTML => true, ]; diff --git a/tests/Core/Tokenizer/HeredocNowdocCloserTest.inc b/tests/Core/Tokenizer/HeredocNowdocCloserTest.inc new file mode 100644 index 0000000000..a800980b8e --- /dev/null +++ b/tests/Core/Tokenizer/HeredocNowdocCloserTest.inc @@ -0,0 +1,43 @@ + + * @copyright 2020 Squiz Pty Ltd (ABN 77 084 670 600) + * @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer; + +use PHP_CodeSniffer\Config; +use PHP_CodeSniffer\Ruleset; +use PHP_CodeSniffer\Files\DummyFile; +use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest; + +/** + * Heredoc/nowdoc closer token test. + * + * @requires PHP 7.3 + */ +class HeredocNowdocCloserTest extends AbstractMethodUnitTest +{ + + + /** + * Initialize & tokenize \PHP_CodeSniffer\Files\File with code from the test case file. + * + * {@internal This is a near duplicate of the original method. Only difference is that + * tab replacement is enabled for this test.} + * + * @return void + */ + public static function setUpBeforeClass() + { + $config = new Config(); + $config->standards = ['PSR1']; + $config->tabWidth = 4; + + $ruleset = new Ruleset($config); + + // Default to a file with the same name as the test class. Extension is property based. + $relativeCN = str_replace(__NAMESPACE__, '', get_called_class()); + $relativePath = str_replace('\\', DIRECTORY_SEPARATOR, $relativeCN); + $pathToTestFile = realpath(__DIR__).$relativePath.'.'.static::$fileExtension; + + // Make sure the file gets parsed correctly based on the file type. + $contents = 'phpcs_input_file: '.$pathToTestFile.PHP_EOL; + $contents .= file_get_contents($pathToTestFile); + + self::$phpcsFile = new DummyFile($contents, $ruleset, $config); + self::$phpcsFile->process(); + + }//end setUpBeforeClass() + + + /** + * Verify that leading (indent) whitespace in a heredoc/nowdoc closer token get the tab replacement treatment. + * + * @param string $testMarker The comment prefacing the target token. + * @param array $expected Expectations for the token array. + * + * @dataProvider dataHeredocNowdocCloserTabReplacement + * @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createPositionMap + * + * @return void + */ + public function testHeredocNowdocCloserTabReplacement($testMarker, $expected) + { + $tokens = self::$phpcsFile->getTokens(); + + $closer = $this->getTargetToken($testMarker, [T_END_HEREDOC, T_END_NOWDOC]); + + foreach ($expected as $key => $value) { + if ($key === 'orig_content' && $value === null) { + $this->assertArrayNotHasKey($key, $tokens[$closer], "Unexpected 'orig_content' key found in the token array."); + continue; + } + + $this->assertArrayHasKey($key, $tokens[$closer], "Key $key not found in the token array."); + $this->assertSame($value, $tokens[$closer][$key], "Value for key $key does not match expectation."); + } + + }//end testHeredocNowdocCloserTabReplacement() + + + /** + * Data provider. + * + * @see testHeredocNowdocCloserTabReplacement() + * + * @return array + */ + public function dataHeredocNowdocCloserTabReplacement() + { + return [ + [ + 'testMarker' => '/* testHeredocCloserNoIndent */', + 'expected' => [ + 'length' => 3, + 'content' => 'EOD', + 'orig_content' => null, + ], + ], + [ + 'testMarker' => '/* testNowdocCloserNoIndent */', + 'expected' => [ + 'length' => 3, + 'content' => 'EOD', + 'orig_content' => null, + ], + ], + [ + 'testMarker' => '/* testHeredocCloserSpaceIndent */', + 'expected' => [ + 'length' => 7, + 'content' => ' END', + 'orig_content' => null, + ], + ], + [ + 'testMarker' => '/* testNowdocCloserSpaceIndent */', + 'expected' => [ + 'length' => 8, + 'content' => ' END', + 'orig_content' => null, + ], + ], + [ + 'testMarker' => '/* testHeredocCloserTabIndent */', + 'expected' => [ + 'length' => 8, + 'content' => ' END', + 'orig_content' => ' END', + ], + ], + [ + 'testMarker' => '/* testNowdocCloserTabIndent */', + 'expected' => [ + 'length' => 7, + 'content' => ' END', + 'orig_content' => ' END', + ], + ], + ]; + + }//end dataHeredocNowdocCloserTabReplacement() + + +}//end class