Skip to content

Commit

Permalink
PHP 8.0 | Stabilize comment tokenization for hash comments
Browse files Browse the repository at this point in the history 
Follow up on 3027 which handled this for slash-style comments.

Hash-style comments were not addressed in the earlier change. This oversight has now been fixed.

Includes additional unit tests.

Fixes 3069
  • Loading branch information
@jrfnl
jrfnl committed Aug 31, 2020
1 parent d33a6a9 commit 0be1542
Show file tree
Hide file tree
Showing 5 changed files with 246 additions and 2 deletions.
4 changes: 2 additions & 2 deletions src/Tokenizers/PHP.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -567,12 +567,12 @@ protected function tokenize($string)
}

/*
PHP 8 tokenizes a new line after a slash comment to the next whitespace token.
PHP 8 tokenizes a new line after a slash and hash comment to the next whitespace token.
*/

if (PHP_VERSION_ID >= 80000
&& $tokenIsArray === true
&& ($token[0] === T_COMMENT && strpos($token[1], '//') === 0)
&& ($token[0] === T_COMMENT && (strpos($token[1], '//') === 0 || strpos($token[1], '#') === 0))
&& isset($tokens[($stackPtr + 1)]) === true
&& is_array($tokens[($stackPtr + 1)]) === true
&& $tokens[($stackPtr + 1)][0] === T_WHITESPACE
Expand Down
20 changes: 20 additions & 0 deletions tests/Core/Tokenizer/StableCommentWhitespaceTest.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,9 +111,29 @@ $prop = 123; /** Comment */
* @tag Comment
*/

/* testSingleLineHashComment */
# Comment

/* testSingleLineHashCommentTrailing */
echo 'a'; # Comment

/* testMultiLineHashComment */
# Comment1
# Comment2
# Comment3

/* testMultiLineHashCommentWithIndent */
# Comment1
# Comment2
# Comment3

/* testSingleLineSlashCommentNoNewLineAtEnd */
// Slash ?>
<?php

/* testSingleLineHashCommentNoNewLineAtEnd */
# Hash ?>
<?php

/* testCommentAtEndOfFile */
/* Comment
102 changes: 102 additions & 0 deletions tests/Core/Tokenizer/StableCommentWhitespaceTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -919,6 +919,94 @@ public function dataCommentTokenization()
[
'type' => 'T_WHITESPACE',
'content' => '
',
],
],
],
[
'/* testSingleLineHashComment */',
[
[
'type' => 'T_COMMENT',
'content' => '# Comment
',
],
[
'type' => 'T_WHITESPACE',
'content' => '
',
],
],
],
[
'/* testSingleLineHashCommentTrailing */',
[
[
'type' => 'T_COMMENT',
'content' => '# Comment
',
],
[
'type' => 'T_WHITESPACE',
'content' => '
',
],
],
],
[
'/* testMultiLineHashComment */',
[
[
'type' => 'T_COMMENT',
'content' => '# Comment1
',
],
[
'type' => 'T_COMMENT',
'content' => '# Comment2
',
],
[
'type' => 'T_COMMENT',
'content' => '# Comment3
',
],
[
'type' => 'T_WHITESPACE',
'content' => '
',
],
],
],
[
'/* testMultiLineHashCommentWithIndent */',
[
[
'type' => 'T_COMMENT',
'content' => '# Comment1
',
],
[
'type' => 'T_WHITESPACE',
'content' => ' ',
],
[
'type' => 'T_COMMENT',
'content' => '# Comment2
',
],
[
'type' => 'T_WHITESPACE',
'content' => ' ',
],
[
'type' => 'T_COMMENT',
'content' => '# Comment3
',
],
[
'type' => 'T_WHITESPACE',
'content' => '
',
],
],
Expand All @@ -933,6 +1021,20 @@ public function dataCommentTokenization()
[
'type' => 'T_CLOSE_TAG',
'content' => '?>
',
],
],
],
[
'/* testSingleLineHashCommentNoNewLineAtEnd */',
[
[
'type' => 'T_COMMENT',
'content' => '# Hash ',
],
[
'type' => 'T_CLOSE_TAG',
'content' => '?>
',
],
],
Expand Down
20 changes: 20 additions & 0 deletions tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,5 +39,25 @@ echo 'a'; // Comment
// Slash ?>
<?php

/* testSingleLineHashComment */
# Comment

/* testSingleLineHashCommentTrailing */
echo 'a'; # Comment

/* testMultiLineHashComment */
# Comment1
# Comment2
# Comment3

/* testMultiLineHashCommentWithIndent */
# Comment1
# Comment2
# Comment3

/* testSingleLineHashCommentNoNewLineAtEnd */
# Hash ?>
<?php

/* testCommentAtEndOfFile */
/* Comment
102 changes: 102 additions & 0 deletions tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,108 @@ public function dataCommentTokenization()
[
'type' => 'T_CLOSE_TAG',
'content' => '?>
',
],
],
],
[
'/* testSingleLineHashComment */',
[
[
'type' => 'T_COMMENT',
'content' => '# Comment
',
],
[
'type' => 'T_WHITESPACE',
'content' => '
',
],
],
],
[
'/* testSingleLineHashCommentTrailing */',
[
[
'type' => 'T_COMMENT',
'content' => '# Comment
',
],
[
'type' => 'T_WHITESPACE',
'content' => '
',
],
],
],
[
'/* testMultiLineHashComment */',
[
[
'type' => 'T_COMMENT',
'content' => '# Comment1
',
],
[
'type' => 'T_COMMENT',
'content' => '# Comment2
',
],
[
'type' => 'T_COMMENT',
'content' => '# Comment3
',
],
[
'type' => 'T_WHITESPACE',
'content' => '
',
],
],
],
[
'/* testMultiLineHashCommentWithIndent */',
[
[
'type' => 'T_COMMENT',
'content' => '# Comment1
',
],
[
'type' => 'T_WHITESPACE',
'content' => ' ',
],
[
'type' => 'T_COMMENT',
'content' => '# Comment2
',
],
[
'type' => 'T_WHITESPACE',
'content' => ' ',
],
[
'type' => 'T_COMMENT',
'content' => '# Comment3
',
],
[
'type' => 'T_WHITESPACE',
'content' => '
',
],
],
],
[
'/* testSingleLineHashCommentNoNewLineAtEnd */',
[
[
'type' => 'T_COMMENT',
'content' => '# Hash ',
],
[
'type' => 'T_CLOSE_TAG',
'content' => '?>
',
],
],
Expand Down

0 comments on commit 0be1542

Please sign in to comment.