diff --git a/.gitattributes b/.gitattributes
index 56dcc5dc74..5456688d96 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -3,3 +3,7 @@ package.xml export-ignore
phpunit.xml.dist export-ignore
php5-testingConfig.ini export-ignore
php7-testingConfig.ini export-ignore
+
+# Declare files that should always have CRLF line endings on checkout.
+*WinTest.inc text eol=crlf
+*WinTest.php text eol=crlf
diff --git a/package.xml b/package.xml
index fcdb419187..a80b1b59e2 100644
--- a/package.xml
+++ b/package.xml
@@ -120,6 +120,10 @@ http://pear.php.net/dtd/package-2.0.xsd">
+
+
+
+
@@ -1986,6 +1990,10 @@ http://pear.php.net/dtd/package-2.0.xsd">
+
+
+
+
@@ -2041,6 +2049,10 @@ http://pear.php.net/dtd/package-2.0.xsd">
+
+
+
+
diff --git a/phpcs.xml.dist b/phpcs.xml.dist
index 48cbdd8ff7..f7f32e6be4 100644
--- a/phpcs.xml.dist
+++ b/phpcs.xml.dist
@@ -143,7 +143,12 @@
- tests/bootstrap.php
+ tests/bootstrap\.php
+
+
+
+
+ tests/Core/Tokenizer/StableCommentWhitespaceWinTest\.php
diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
index 1c30b15179..61f9c2f798 100644
--- a/src/Tokenizers/PHP.php
+++ b/src/Tokenizers/PHP.php
@@ -566,6 +566,47 @@ protected function tokenize($string)
continue;
}
+ /*
+ PHP 8 tokenizes a new line after a slash comment to the next whitespace token.
+ */
+
+ if (PHP_VERSION_ID >= 80000
+ && $tokenIsArray === true
+ && ($token[0] === T_COMMENT && strpos($token[1], '//') === 0)
+ && isset($tokens[($stackPtr + 1)]) === true
+ && is_array($tokens[($stackPtr + 1)]) === true
+ && $tokens[($stackPtr + 1)][0] === T_WHITESPACE
+ ) {
+ $nextToken = $tokens[($stackPtr + 1)];
+
+ // If the next token is a single new line, merge it into the comment token
+ // and set to it up to be skipped.
+ if ($nextToken[1] === "\n" || $nextToken[1] === "\r\n" || $nextToken[1] === "\n\r") {
+ $token[1] .= $nextToken[1];
+ $tokens[($stackPtr + 1)] = null;
+
+ if (PHP_CODESNIFFER_VERBOSITY > 1) {
+ echo "\t\t* merged newline after comment into comment token $stackPtr".PHP_EOL;
+ }
+ } else {
+ // This may be a whitespace token consisting of multiple new lines.
+ if (strpos($nextToken[1], "\r\n") === 0) {
+ $token[1] .= "\r\n";
+ $tokens[($stackPtr + 1)][1] = substr($nextToken[1], 2);
+ } else if (strpos($nextToken[1], "\n\r") === 0) {
+ $token[1] .= "\n\r";
+ $tokens[($stackPtr + 1)][1] = substr($nextToken[1], 2);
+ } else if (strpos($nextToken[1], "\n") === 0) {
+ $token[1] .= "\n";
+ $tokens[($stackPtr + 1)][1] = substr($nextToken[1], 1);
+ }
+
+ if (PHP_CODESNIFFER_VERBOSITY > 1) {
+ echo "\t\t* stripped first newline after comment and added it to comment token $stackPtr".PHP_EOL;
+ }
+ }//end if
+ }//end if
+
/*
If this is a double quoted string, PHP will tokenize the whole
thing which causes problems with the scope map when braces are
diff --git a/tests/Core/Tokenizer/StableCommentWhitespaceTest.inc b/tests/Core/Tokenizer/StableCommentWhitespaceTest.inc
new file mode 100644
index 0000000000..6d92b9cfd3
--- /dev/null
+++ b/tests/Core/Tokenizer/StableCommentWhitespaceTest.inc
@@ -0,0 +1,119 @@
+
+
+ * @copyright 2020 Squiz Pty Ltd (ABN 77 084 670 600)
+ * @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+
+use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest;
+use PHP_CodeSniffer\Util\Tokens;
+
+class StableCommentWhitespaceTest extends AbstractMethodUnitTest
+{
+
+
+ /**
+ * Test that comment tokenization with new lines at the end of the comment is stable.
+ *
+ * @param string $testMarker The comment prefacing the test.
+ * @param array $expectedTokens The tokenization expected.
+ *
+ * @dataProvider dataCommentTokenization
+ * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
+ *
+ * @return void
+ */
+ public function testCommentTokenization($testMarker, $expectedTokens)
+ {
+ $tokens = self::$phpcsFile->getTokens();
+ $comment = $this->getTargetToken($testMarker, Tokens::$commentTokens);
+
+ foreach ($expectedTokens as $key => $tokenInfo) {
+ $this->assertSame(constant($tokenInfo['type']), $tokens[$comment]['code']);
+ $this->assertSame($tokenInfo['type'], $tokens[$comment]['type']);
+ $this->assertSame($tokenInfo['content'], $tokens[$comment]['content']);
+
+ ++$comment;
+ }
+
+ }//end testCommentTokenization()
+
+
+ /**
+ * Data provider.
+ *
+ * @see testCommentTokenization()
+ *
+ * @return array
+ */
+ public function dataCommentTokenization()
+ {
+ return [
+ [
+ '/* testSingleLineSlashComment */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testSingleLineSlashCommentTrailing */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testSingleLineSlashAnnotation */',
+ [
+ [
+ 'type' => 'T_PHPCS_DISABLE',
+ 'content' => '// phpcs:disable Stnd.Cat
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineSlashComment */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment1
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment2
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment3
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineSlashCommentWithIndent */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment1
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment2
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment3
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineSlashCommentWithAnnotationStart */',
+ [
+ [
+ 'type' => 'T_PHPCS_IGNORE',
+ 'content' => '// phpcs:ignore Stnd.Cat
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment2
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment3
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineSlashCommentWithAnnotationMiddle */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment1
+',
+ ],
+ [
+ 'type' => 'T_PHPCS_IGNORE',
+ 'content' => '// @phpcs:ignore Stnd.Cat
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment3
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineSlashCommentWithAnnotationEnd */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment1
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment2
+',
+ ],
+ [
+ 'type' => 'T_PHPCS_IGNORE',
+ 'content' => '// phpcs:ignore Stnd.Cat
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testSingleLineStarComment */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '/* Single line star comment */',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testSingleLineStarCommentTrailing */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '/* Comment */',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testSingleLineStarAnnotation */',
+ [
+ [
+ 'type' => 'T_PHPCS_IGNORE',
+ 'content' => '/* phpcs:ignore Stnd.Cat */',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineStarComment */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '/* Comment1
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => ' * Comment2
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => ' * Comment3 */',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineStarCommentWithIndent */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '/* Comment1
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => ' * Comment2
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => ' * Comment3 */',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineStarCommentWithAnnotationStart */',
+ [
+ [
+ 'type' => 'T_PHPCS_IGNORE',
+ 'content' => '/* @phpcs:ignore Stnd.Cat
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => ' * Comment2
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => ' * Comment3 */',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineStarCommentWithAnnotationMiddle */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '/* Comment1
+',
+ ],
+ [
+ 'type' => 'T_PHPCS_IGNORE',
+ 'content' => ' * phpcs:ignore Stnd.Cat
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => ' * Comment3 */',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineStarCommentWithAnnotationEnd */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '/* Comment1
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => ' * Comment2
+',
+ ],
+ [
+ 'type' => 'T_PHPCS_IGNORE',
+ 'content' => ' * phpcs:ignore Stnd.Cat */',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+
+ [
+ '/* testSingleLineDocblockComment */',
+ [
+ [
+ 'type' => 'T_DOC_COMMENT_OPEN_TAG',
+ 'content' => '/**',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STRING',
+ 'content' => 'Comment ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_CLOSE_TAG',
+ 'content' => '*/',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testSingleLineDocblockCommentTrailing */',
+ [
+ [
+ 'type' => 'T_DOC_COMMENT_OPEN_TAG',
+ 'content' => '/**',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STRING',
+ 'content' => 'Comment ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_CLOSE_TAG',
+ 'content' => '*/',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testSingleLineDocblockAnnotation */',
+ [
+ [
+ 'type' => 'T_DOC_COMMENT_OPEN_TAG',
+ 'content' => '/**',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_PHPCS_IGNORE',
+ 'content' => 'phpcs:ignore Stnd.Cat.Sniff ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_CLOSE_TAG',
+ 'content' => '*/',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+
+ [
+ '/* testMultiLineDocblockComment */',
+ [
+ [
+ 'type' => 'T_DOC_COMMENT_OPEN_TAG',
+ 'content' => '/**',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STRING',
+ 'content' => 'Comment1',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STRING',
+ 'content' => 'Comment2',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_TAG',
+ 'content' => '@tag',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STRING',
+ 'content' => 'Comment',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_CLOSE_TAG',
+ 'content' => '*/',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineDocblockCommentWithIndent */',
+ [
+ [
+ 'type' => 'T_DOC_COMMENT_OPEN_TAG',
+ 'content' => '/**',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STRING',
+ 'content' => 'Comment1',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STRING',
+ 'content' => 'Comment2',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_TAG',
+ 'content' => '@tag',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STRING',
+ 'content' => 'Comment',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_CLOSE_TAG',
+ 'content' => '*/',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineDocblockCommentWithAnnotation */',
+ [
+ [
+ 'type' => 'T_DOC_COMMENT_OPEN_TAG',
+ 'content' => '/**',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STRING',
+ 'content' => 'Comment',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_PHPCS_IGNORE',
+ 'content' => 'phpcs:ignore Stnd.Cat',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_TAG',
+ 'content' => '@tag',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STRING',
+ 'content' => 'Comment',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_CLOSE_TAG',
+ 'content' => '*/',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineDocblockCommentWithTagAnnotation */',
+ [
+ [
+ 'type' => 'T_DOC_COMMENT_OPEN_TAG',
+ 'content' => '/**',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STRING',
+ 'content' => 'Comment',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_PHPCS_IGNORE',
+ 'content' => '@phpcs:ignore Stnd.Cat',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STAR',
+ 'content' => '*',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_TAG',
+ 'content' => '@tag',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_STRING',
+ 'content' => 'Comment',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => '
+',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_DOC_COMMENT_CLOSE_TAG',
+ 'content' => '*/',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testSingleLineSlashCommentNoNewLineAtEnd */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Slash ',
+ ],
+ [
+ 'type' => 'T_CLOSE_TAG',
+ 'content' => '?>
+',
+ ],
+ ],
+ ],
+ [
+ '/* testCommentAtEndOfFile */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '/* Comment',
+ ],
+ ],
+ ],
+ ];
+
+ }//end dataCommentTokenization()
+
+
+}//end class
diff --git a/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc b/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc
new file mode 100644
index 0000000000..8ca0003924
--- /dev/null
+++ b/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc
@@ -0,0 +1,43 @@
+
+
+ * @copyright 2020 Squiz Pty Ltd (ABN 77 084 670 600)
+ * @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+
+use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest;
+use PHP_CodeSniffer\Util\Tokens;
+
+class StableCommentWhitespaceWinTest extends AbstractMethodUnitTest
+{
+
+
+ /**
+ * Test that comment tokenization with new lines at the end of the comment is stable.
+ *
+ * @param string $testMarker The comment prefacing the test.
+ * @param array $expectedTokens The tokenization expected.
+ *
+ * @dataProvider dataCommentTokenization
+ * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
+ *
+ * @return void
+ */
+ public function testCommentTokenization($testMarker, $expectedTokens)
+ {
+ $tokens = self::$phpcsFile->getTokens();
+ $comment = $this->getTargetToken($testMarker, Tokens::$commentTokens);
+
+ foreach ($expectedTokens as $key => $tokenInfo) {
+ $this->assertSame(constant($tokenInfo['type']), $tokens[$comment]['code']);
+ $this->assertSame($tokenInfo['type'], $tokens[$comment]['type']);
+ $this->assertSame($tokenInfo['content'], $tokens[$comment]['content']);
+
+ ++$comment;
+ }
+
+ }//end testCommentTokenization()
+
+
+ /**
+ * Data provider.
+ *
+ * @see testCommentTokenization()
+ *
+ * @return array
+ */
+ public function dataCommentTokenization()
+ {
+ return [
+ [
+ '/* testSingleLineSlashComment */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testSingleLineSlashCommentTrailing */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testSingleLineSlashAnnotation */',
+ [
+ [
+ 'type' => 'T_PHPCS_DISABLE',
+ 'content' => '// phpcs:disable Stnd.Cat
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineSlashComment */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment1
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment2
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment3
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineSlashCommentWithIndent */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment1
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment2
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => ' ',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment3
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineSlashCommentWithAnnotationStart */',
+ [
+ [
+ 'type' => 'T_PHPCS_IGNORE',
+ 'content' => '// phpcs:ignore Stnd.Cat
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment2
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment3
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineSlashCommentWithAnnotationMiddle */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment1
+',
+ ],
+ [
+ 'type' => 'T_PHPCS_IGNORE',
+ 'content' => '// @phpcs:ignore Stnd.Cat
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment3
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testMultiLineSlashCommentWithAnnotationEnd */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment1
+',
+ ],
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Comment2
+',
+ ],
+ [
+ 'type' => 'T_PHPCS_IGNORE',
+ 'content' => '// phpcs:ignore Stnd.Cat
+',
+ ],
+ [
+ 'type' => 'T_WHITESPACE',
+ 'content' => '
+',
+ ],
+ ],
+ ],
+ [
+ '/* testSingleLineSlashCommentNoNewLineAtEnd */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '// Slash ',
+ ],
+ [
+ 'type' => 'T_CLOSE_TAG',
+ 'content' => '?>
+',
+ ],
+ ],
+ ],
+ [
+ '/* testCommentAtEndOfFile */',
+ [
+ [
+ 'type' => 'T_COMMENT',
+ 'content' => '/* Comment',
+ ],
+ ],
+ ],
+ ];
+
+ }//end dataCommentTokenization()
+
+
+}//end class