-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Tokenizer: apply tab replacement to heredoc/nowdoc closers
Since PHP 7.3, heredoc/nowdoc closers may be indented. This indent can use either tabs or spaces and the indent is included in the `T_END_HEREDOC`/`T_END_NOWDOC` token contents as received from the PHP native tokenizer. However, the PHPCS `Tokenizer` did no execute tab replacement on these token leading to unexpected `'content'` and incorrect `'length'` values in the `File::$tokens` array, which in turn could lead to incorrect sniff results and incorrect fixes. This commit adds the `T_END_HEREDOC`/`T_END_NOWDOC` tokens to the array of tokens for which to do tab replacement to make them more consistent with the rest of PHPCS. I also considered splitting the token into a `T_WHITESPACE` token and the `T_END_HEREDOC`/`T_END_NOWDOC` token, but that could potentially break sniffs which expect the `T_END_HEREDOC`/`T_END_NOWDOC` token directly after the last `T_HEREDOC`/`T_NOWDOC` token. The current fix does not contain that risk. Includes unit tests safeguarding this change. The tests will only run on PHP 7.3+ as flexible heredoc/nowdocs don't tokenize correctly in PHP < 7.3.
- Loading branch information
Showing
4 changed files
with
201 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| <?php | ||
|
|
||
| /* testHeredocCloserNoIndent */ | ||
| $heredoc = <<<EOD | ||
| some text | ||
| some text | ||
| some text | ||
| EOD; | ||
|
|
||
| /* testNowdocCloserNoIndent */ | ||
| $nowdoc = <<<'EOD' | ||
| some text | ||
| some text | ||
| some text | ||
| EOD; | ||
|
|
||
| /* testHeredocCloserSpaceIndent */ | ||
| $heredoc = <<<END | ||
| a | ||
| b | ||
| c | ||
| END; | ||
|
|
||
| /* testNowdocCloserSpaceIndent */ | ||
| $nowdoc = <<<'END' | ||
| a | ||
| b | ||
| c | ||
| END; | ||
|
|
||
| /* testHeredocCloserTabIndent */ | ||
| $heredoc = <<<"END" | ||
| a | ||
| b | ||
| c | ||
| END; | ||
|
|
||
| /* testNowdocCloserTabIndent */ | ||
| $nowdoc = <<<'END' | ||
| a | ||
| b | ||
| c | ||
| END; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,150 @@ | ||
| <?php | ||
| /** | ||
| * Tests the tokenization of goto declarations and statements. | ||
| * | ||
| * @author Juliette Reinders Folmer <[email protected]> | ||
| * @copyright 2020 Squiz Pty Ltd (ABN 77 084 670 600) | ||
| * @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence | ||
| */ | ||
|
|
||
| namespace PHP_CodeSniffer\Tests\Core\Tokenizer; | ||
|
|
||
| use PHP_CodeSniffer\Config; | ||
| use PHP_CodeSniffer\Ruleset; | ||
| use PHP_CodeSniffer\Files\DummyFile; | ||
| use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest; | ||
|
|
||
| /** | ||
| * Heredoc/nowdoc closer token test. | ||
| * | ||
| * @requires PHP 7.3 | ||
| */ | ||
| class HeredocNowdocCloserTest extends AbstractMethodUnitTest | ||
| { | ||
|
|
||
|
|
||
| /** | ||
| * Initialize & tokenize \PHP_CodeSniffer\Files\File with code from the test case file. | ||
| * | ||
| * {@internal This is a near duplicate of the original method. Only difference is that | ||
| * tab replacement is enabled for this test.} | ||
| * | ||
| * @return void | ||
| */ | ||
| public static function setUpBeforeClass() | ||
| { | ||
| $config = new Config(); | ||
| $config->standards = ['PSR1']; | ||
| $config->tabWidth = 4; | ||
|
|
||
| $ruleset = new Ruleset($config); | ||
|
|
||
| // Default to a file with the same name as the test class. Extension is property based. | ||
| $relativeCN = str_replace(__NAMESPACE__, '', get_called_class()); | ||
| $relativePath = str_replace('\\', DIRECTORY_SEPARATOR, $relativeCN); | ||
| $pathToTestFile = realpath(__DIR__).$relativePath.'.'.static::$fileExtension; | ||
|
|
||
| // Make sure the file gets parsed correctly based on the file type. | ||
| $contents = 'phpcs_input_file: '.$pathToTestFile.PHP_EOL; | ||
| $contents .= file_get_contents($pathToTestFile); | ||
|
|
||
| self::$phpcsFile = new DummyFile($contents, $ruleset, $config); | ||
| self::$phpcsFile->process(); | ||
|
|
||
| }//end setUpBeforeClass() | ||
|
|
||
|
|
||
| /** | ||
| * Verify that leading (indent) whitespace in a heredoc/nowdoc closer token get the tab replacement treatment. | ||
| * | ||
| * @param string $testMarker The comment prefacing the target token. | ||
| * @param array $expected Expectations for the token array. | ||
| * | ||
| * @dataProvider dataHeredocNowdocCloserTabReplacement | ||
| * @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createPositionMap | ||
| * | ||
| * @return void | ||
| */ | ||
| public function testHeredocNowdocCloserTabReplacement($testMarker, $expected) | ||
| { | ||
| $tokens = self::$phpcsFile->getTokens(); | ||
|
|
||
| $closer = $this->getTargetToken($testMarker, [T_END_HEREDOC, T_END_NOWDOC]); | ||
|
|
||
| foreach ($expected as $key => $value) { | ||
| if ($key === 'orig_content' && $value === null) { | ||
| $this->assertArrayNotHasKey($key, $tokens[$closer], "Unexpected 'orig_content' key found in the token array."); | ||
| continue; | ||
| } | ||
|
|
||
| $this->assertArrayHasKey($key, $tokens[$closer], "Key $key not found in the token array."); | ||
| $this->assertSame($value, $tokens[$closer][$key], "Value for key $key does not match expectation."); | ||
| } | ||
|
|
||
| }//end testHeredocNowdocCloserTabReplacement() | ||
|
|
||
|
|
||
| /** | ||
| * Data provider. | ||
| * | ||
| * @see testHeredocNowdocCloserTabReplacement() | ||
| * | ||
| * @return array | ||
| */ | ||
| public function dataHeredocNowdocCloserTabReplacement() | ||
| { | ||
| return [ | ||
| [ | ||
| 'testMarker' => '/* testHeredocCloserNoIndent */', | ||
| 'expected' => [ | ||
| 'length' => 3, | ||
| 'content' => 'EOD', | ||
| 'orig_content' => null, | ||
| ], | ||
| ], | ||
| [ | ||
| 'testMarker' => '/* testNowdocCloserNoIndent */', | ||
| 'expected' => [ | ||
| 'length' => 3, | ||
| 'content' => 'EOD', | ||
| 'orig_content' => null, | ||
| ], | ||
| ], | ||
| [ | ||
| 'testMarker' => '/* testHeredocCloserSpaceIndent */', | ||
| 'expected' => [ | ||
| 'length' => 7, | ||
| 'content' => ' END', | ||
| 'orig_content' => null, | ||
| ], | ||
| ], | ||
| [ | ||
| 'testMarker' => '/* testNowdocCloserSpaceIndent */', | ||
| 'expected' => [ | ||
| 'length' => 8, | ||
| 'content' => ' END', | ||
| 'orig_content' => null, | ||
| ], | ||
| ], | ||
| [ | ||
| 'testMarker' => '/* testHeredocCloserTabIndent */', | ||
| 'expected' => [ | ||
| 'length' => 8, | ||
| 'content' => ' END', | ||
| 'orig_content' => ' END', | ||
| ], | ||
| ], | ||
| [ | ||
| 'testMarker' => '/* testNowdocCloserTabIndent */', | ||
| 'expected' => [ | ||
| 'length' => 7, | ||
| 'content' => ' END', | ||
| 'orig_content' => ' END', | ||
| ], | ||
| ], | ||
| ]; | ||
|
|
||
| }//end dataHeredocNowdocCloserTabReplacement() | ||
|
|
||
|
|
||
| }//end class |