diff --git a/.gitattributes b/.gitattributes index 56dcc5dc74..5456688d96 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,3 +3,7 @@ package.xml export-ignore phpunit.xml.dist export-ignore php5-testingConfig.ini export-ignore php7-testingConfig.ini export-ignore + +# Declare files that should always have CRLF line endings on checkout. +*WinTest.inc text eol=crlf +*WinTest.php text eol=crlf diff --git a/package.xml b/package.xml index fcdb419187..a80b1b59e2 100644 --- a/package.xml +++ b/package.xml @@ -120,6 +120,10 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + + + @@ -1986,6 +1990,10 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + + + @@ -2041,6 +2049,10 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + + + diff --git a/phpcs.xml.dist b/phpcs.xml.dist index 48cbdd8ff7..f7f32e6be4 100644 --- a/phpcs.xml.dist +++ b/phpcs.xml.dist @@ -143,7 +143,12 @@ - tests/bootstrap.php + tests/bootstrap\.php + + + + + tests/Core/Tokenizer/StableCommentWhitespaceWinTest\.php diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index 1c30b15179..61f9c2f798 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -566,6 +566,47 @@ protected function tokenize($string) continue; } + /* + PHP 8 tokenizes a new line after a slash comment to the next whitespace token. + */ + + if (PHP_VERSION_ID >= 80000 + && $tokenIsArray === true + && ($token[0] === T_COMMENT && strpos($token[1], '//') === 0) + && isset($tokens[($stackPtr + 1)]) === true + && is_array($tokens[($stackPtr + 1)]) === true + && $tokens[($stackPtr + 1)][0] === T_WHITESPACE + ) { + $nextToken = $tokens[($stackPtr + 1)]; + + // If the next token is a single new line, merge it into the comment token + // and set to it up to be skipped. + if ($nextToken[1] === "\n" || $nextToken[1] === "\r\n" || $nextToken[1] === "\n\r") { + $token[1] .= $nextToken[1]; + $tokens[($stackPtr + 1)] = null; + + if (PHP_CODESNIFFER_VERBOSITY > 1) { + echo "\t\t* merged newline after comment into comment token $stackPtr".PHP_EOL; + } + } else { + // This may be a whitespace token consisting of multiple new lines. + if (strpos($nextToken[1], "\r\n") === 0) { + $token[1] .= "\r\n"; + $tokens[($stackPtr + 1)][1] = substr($nextToken[1], 2); + } else if (strpos($nextToken[1], "\n\r") === 0) { + $token[1] .= "\n\r"; + $tokens[($stackPtr + 1)][1] = substr($nextToken[1], 2); + } else if (strpos($nextToken[1], "\n") === 0) { + $token[1] .= "\n"; + $tokens[($stackPtr + 1)][1] = substr($nextToken[1], 1); + } + + if (PHP_CODESNIFFER_VERBOSITY > 1) { + echo "\t\t* stripped first newline after comment and added it to comment token $stackPtr".PHP_EOL; + } + }//end if + }//end if + /* If this is a double quoted string, PHP will tokenize the whole thing which causes problems with the scope map when braces are diff --git a/tests/Core/Tokenizer/StableCommentWhitespaceTest.inc b/tests/Core/Tokenizer/StableCommentWhitespaceTest.inc new file mode 100644 index 0000000000..6d92b9cfd3 --- /dev/null +++ b/tests/Core/Tokenizer/StableCommentWhitespaceTest.inc @@ -0,0 +1,119 @@ + + + * @copyright 2020 Squiz Pty Ltd (ABN 77 084 670 600) + * @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer; + +use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest; +use PHP_CodeSniffer\Util\Tokens; + +class StableCommentWhitespaceTest extends AbstractMethodUnitTest +{ + + + /** + * Test that comment tokenization with new lines at the end of the comment is stable. + * + * @param string $testMarker The comment prefacing the test. + * @param array $expectedTokens The tokenization expected. + * + * @dataProvider dataCommentTokenization + * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize + * + * @return void + */ + public function testCommentTokenization($testMarker, $expectedTokens) + { + $tokens = self::$phpcsFile->getTokens(); + $comment = $this->getTargetToken($testMarker, Tokens::$commentTokens); + + foreach ($expectedTokens as $key => $tokenInfo) { + $this->assertSame(constant($tokenInfo['type']), $tokens[$comment]['code']); + $this->assertSame($tokenInfo['type'], $tokens[$comment]['type']); + $this->assertSame($tokenInfo['content'], $tokens[$comment]['content']); + + ++$comment; + } + + }//end testCommentTokenization() + + + /** + * Data provider. + * + * @see testCommentTokenization() + * + * @return array + */ + public function dataCommentTokenization() + { + return [ + [ + '/* testSingleLineSlashComment */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineSlashCommentTrailing */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineSlashAnnotation */', + [ + [ + 'type' => 'T_PHPCS_DISABLE', + 'content' => '// phpcs:disable Stnd.Cat +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineSlashComment */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment1 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment2 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment3 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineSlashCommentWithIndent */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment1 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment2 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment3 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineSlashCommentWithAnnotationStart */', + [ + [ + 'type' => 'T_PHPCS_IGNORE', + 'content' => '// phpcs:ignore Stnd.Cat +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment2 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment3 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineSlashCommentWithAnnotationMiddle */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment1 +', + ], + [ + 'type' => 'T_PHPCS_IGNORE', + 'content' => '// @phpcs:ignore Stnd.Cat +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment3 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineSlashCommentWithAnnotationEnd */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment1 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment2 +', + ], + [ + 'type' => 'T_PHPCS_IGNORE', + 'content' => '// phpcs:ignore Stnd.Cat +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineStarComment */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '/* Single line star comment */', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineStarCommentTrailing */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '/* Comment */', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineStarAnnotation */', + [ + [ + 'type' => 'T_PHPCS_IGNORE', + 'content' => '/* phpcs:ignore Stnd.Cat */', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineStarComment */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '/* Comment1 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => ' * Comment2 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => ' * Comment3 */', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineStarCommentWithIndent */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '/* Comment1 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => ' * Comment2 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => ' * Comment3 */', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineStarCommentWithAnnotationStart */', + [ + [ + 'type' => 'T_PHPCS_IGNORE', + 'content' => '/* @phpcs:ignore Stnd.Cat +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => ' * Comment2 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => ' * Comment3 */', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineStarCommentWithAnnotationMiddle */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '/* Comment1 +', + ], + [ + 'type' => 'T_PHPCS_IGNORE', + 'content' => ' * phpcs:ignore Stnd.Cat +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => ' * Comment3 */', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineStarCommentWithAnnotationEnd */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '/* Comment1 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => ' * Comment2 +', + ], + [ + 'type' => 'T_PHPCS_IGNORE', + 'content' => ' * phpcs:ignore Stnd.Cat */', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + + [ + '/* testSingleLineDocblockComment */', + [ + [ + 'type' => 'T_DOC_COMMENT_OPEN_TAG', + 'content' => '/**', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STRING', + 'content' => 'Comment ', + ], + [ + 'type' => 'T_DOC_COMMENT_CLOSE_TAG', + 'content' => '*/', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineDocblockCommentTrailing */', + [ + [ + 'type' => 'T_DOC_COMMENT_OPEN_TAG', + 'content' => '/**', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STRING', + 'content' => 'Comment ', + ], + [ + 'type' => 'T_DOC_COMMENT_CLOSE_TAG', + 'content' => '*/', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineDocblockAnnotation */', + [ + [ + 'type' => 'T_DOC_COMMENT_OPEN_TAG', + 'content' => '/**', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_PHPCS_IGNORE', + 'content' => 'phpcs:ignore Stnd.Cat.Sniff ', + ], + [ + 'type' => 'T_DOC_COMMENT_CLOSE_TAG', + 'content' => '*/', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + + [ + '/* testMultiLineDocblockComment */', + [ + [ + 'type' => 'T_DOC_COMMENT_OPEN_TAG', + 'content' => '/**', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STRING', + 'content' => 'Comment1', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STRING', + 'content' => 'Comment2', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_TAG', + 'content' => '@tag', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STRING', + 'content' => 'Comment', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_CLOSE_TAG', + 'content' => '*/', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineDocblockCommentWithIndent */', + [ + [ + 'type' => 'T_DOC_COMMENT_OPEN_TAG', + 'content' => '/**', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STRING', + 'content' => 'Comment1', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STRING', + 'content' => 'Comment2', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_TAG', + 'content' => '@tag', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STRING', + 'content' => 'Comment', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_CLOSE_TAG', + 'content' => '*/', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineDocblockCommentWithAnnotation */', + [ + [ + 'type' => 'T_DOC_COMMENT_OPEN_TAG', + 'content' => '/**', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STRING', + 'content' => 'Comment', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_PHPCS_IGNORE', + 'content' => 'phpcs:ignore Stnd.Cat', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_TAG', + 'content' => '@tag', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STRING', + 'content' => 'Comment', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_CLOSE_TAG', + 'content' => '*/', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineDocblockCommentWithTagAnnotation */', + [ + [ + 'type' => 'T_DOC_COMMENT_OPEN_TAG', + 'content' => '/**', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STRING', + 'content' => 'Comment', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_PHPCS_IGNORE', + 'content' => '@phpcs:ignore Stnd.Cat', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_TAG', + 'content' => '@tag', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_STRING', + 'content' => 'Comment', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_DOC_COMMENT_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_DOC_COMMENT_CLOSE_TAG', + 'content' => '*/', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineSlashCommentNoNewLineAtEnd */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '// Slash ', + ], + [ + 'type' => 'T_CLOSE_TAG', + 'content' => '?> +', + ], + ], + ], + [ + '/* testCommentAtEndOfFile */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '/* Comment', + ], + ], + ], + ]; + + }//end dataCommentTokenization() + + +}//end class diff --git a/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc b/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc new file mode 100644 index 0000000000..8ca0003924 --- /dev/null +++ b/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc @@ -0,0 +1,43 @@ + + + * @copyright 2020 Squiz Pty Ltd (ABN 77 084 670 600) + * @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer; + +use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest; +use PHP_CodeSniffer\Util\Tokens; + +class StableCommentWhitespaceWinTest extends AbstractMethodUnitTest +{ + + + /** + * Test that comment tokenization with new lines at the end of the comment is stable. + * + * @param string $testMarker The comment prefacing the test. + * @param array $expectedTokens The tokenization expected. + * + * @dataProvider dataCommentTokenization + * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize + * + * @return void + */ + public function testCommentTokenization($testMarker, $expectedTokens) + { + $tokens = self::$phpcsFile->getTokens(); + $comment = $this->getTargetToken($testMarker, Tokens::$commentTokens); + + foreach ($expectedTokens as $key => $tokenInfo) { + $this->assertSame(constant($tokenInfo['type']), $tokens[$comment]['code']); + $this->assertSame($tokenInfo['type'], $tokens[$comment]['type']); + $this->assertSame($tokenInfo['content'], $tokens[$comment]['content']); + + ++$comment; + } + + }//end testCommentTokenization() + + + /** + * Data provider. + * + * @see testCommentTokenization() + * + * @return array + */ + public function dataCommentTokenization() + { + return [ + [ + '/* testSingleLineSlashComment */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineSlashCommentTrailing */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineSlashAnnotation */', + [ + [ + 'type' => 'T_PHPCS_DISABLE', + 'content' => '// phpcs:disable Stnd.Cat +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineSlashComment */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment1 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment2 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment3 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineSlashCommentWithIndent */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment1 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment2 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment3 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineSlashCommentWithAnnotationStart */', + [ + [ + 'type' => 'T_PHPCS_IGNORE', + 'content' => '// phpcs:ignore Stnd.Cat +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment2 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment3 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineSlashCommentWithAnnotationMiddle */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment1 +', + ], + [ + 'type' => 'T_PHPCS_IGNORE', + 'content' => '// @phpcs:ignore Stnd.Cat +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment3 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineSlashCommentWithAnnotationEnd */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment1 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// Comment2 +', + ], + [ + 'type' => 'T_PHPCS_IGNORE', + 'content' => '// phpcs:ignore Stnd.Cat +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineSlashCommentNoNewLineAtEnd */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '// Slash ', + ], + [ + 'type' => 'T_CLOSE_TAG', + 'content' => '?> +', + ], + ], + ], + [ + '/* testCommentAtEndOfFile */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '/* Comment', + ], + ], + ], + ]; + + }//end dataCommentTokenization() + + +}//end class