Skip to content

Commit

Permalink
Merge pull request #244 from calebdw/nowdoc
Browse files Browse the repository at this point in the history 
fix: nowdoc end tag parsing
  • Loading branch information
@calebdw
calebdw authored May 19, 2024
2 parents 27afeb0 + 510dc4f commit 17cca5f
Show file tree
Hide file tree
Showing 10 changed files with 135,495 additions and 134,001 deletions.
36 changes: 14 additions & 22 deletions common/define-grammar.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1391,7 +1391,10 @@ module.exports = function defineGrammar(dialect) {
_interpolated_string_body_heredoc: $ => repeat1(
choice(
$.escape_sequence,
seq($.variable_name, alias($.encapsed_string_chars_after_variable_heredoc, $.string_content)),
seq(
$.variable_name,
alias($.encapsed_string_chars_after_variable_heredoc, $.string_content),
),
alias($.encapsed_string_chars_heredoc, $.string_content),
$._simple_string_part,
$._complex_string_part,
Expand Down Expand Up @@ -1426,32 +1429,24 @@ module.exports = function defineGrammar(dialect) {

heredoc: $ => seq(
token('<<<'),
field('identifier', choice(
$.heredoc_start,
seq('"', $.heredoc_start, token.immediate('"')),
)),
optional('"'),
field('identifier', $.heredoc_start),
optional(token.immediate('"')),
choice(
seq(
field('value', $.heredoc_body),
$._new_line,
field('end_tag', $.heredoc_end),
),
seq(
field('value', optional($.heredoc_body)),
field('end_tag', $.heredoc_end),
),
field('value', optional($.heredoc_body)),
),
field('end_tag', $.heredoc_end),
),

_new_line: _ => /\r?\n|\r/,

nowdoc_body: $ => seq($._new_line,
choice(
repeat1(
$.nowdoc_string,
),
alias('', $.nowdoc_string),
),
nowdoc_body: $ => seq(
$._new_line,
repeat1($.nowdoc_string),
),

nowdoc: $ => seq(
Expand All @@ -1463,13 +1458,10 @@ module.exports = function defineGrammar(dialect) {
seq(
field('value', $.nowdoc_body),
$._new_line,
field('end_tag', $.heredoc_end),
),
seq(
field('value', optional($.nowdoc_body)),
field('end_tag', $.heredoc_end),
),
field('value', optional($.nowdoc_body)),
),
field('end_tag', $.heredoc_end),
),

_interpolated_execution_operator_body: $ => repeat1(
Expand Down
16 changes: 4 additions & 12 deletions common/scanner.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ static String scan_heredoc_word(TSLexer *lexer) {
return result;
}

static inline bool scan_nowdoc_string(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
static inline bool scan_nowdoc_string(Scanner *scanner, TSLexer *lexer) {
bool has_consumed_content = false;
if (scanner->heredocs.size == 0) {
return false;
Expand All @@ -199,7 +199,6 @@ static inline bool scan_nowdoc_string(Scanner *scanner, TSLexer *lexer, const bo

bool end_tag_matched = false;
String heredoc_tag = array_back(&scanner->heredocs)->word;
String word = (String)array_new();

for (uint32_t i = 0; i < heredoc_tag.size; i++) {
if (lexer->lookahead != heredoc_tag.contents[i]) {
Expand All @@ -208,17 +207,10 @@ static inline bool scan_nowdoc_string(Scanner *scanner, TSLexer *lexer, const bo
advance(lexer);
has_consumed_content = true;

array_push(&word, heredoc_tag.contents[i]);
end_tag_matched = (i == heredoc_tag.size - 1 && (iswspace(lexer->lookahead) || lexer->lookahead == ';' ||
lexer->lookahead == ',' || lexer->lookahead == ')'));
}

if (valid_symbols[HEREDOC_END] && string_eq(&word, &heredoc_tag)) {
lexer->result_symbol = HEREDOC_END;
lexer->mark_end(lexer);
array_delete(&word);
return true;
}
array_delete(&word);

if (end_tag_matched) {
// There may be an arbitrary amount of white space after the end tag
while (iswspace(lexer->lookahead) && lexer->lookahead != '\r' && lexer->lookahead != '\n') {
Expand Down Expand Up @@ -450,7 +442,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {

if (valid_symbols[NOWDOC_STRING]) {
lexer->result_symbol = NOWDOC_STRING;
return scan_nowdoc_string(scanner, lexer, valid_symbols);
return scan_nowdoc_string(scanner, lexer);
}

if (valid_symbols[HEREDOC_END]) {
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@
"scripts": {
"install": "node-gyp-build",
"prebuildify": "prebuildify --napi --strip",
"generate": "for dir in php php_only; do cd $dir && tree-sitter generate --no-bindings && cd ..; done",
"lint": "eslint common/define-grammar.js",
"parse": "tree-sitter parse",
"test": "tree-sitter test"
Expand Down
186 changes: 74 additions & 112 deletions php/src/grammar.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7351,39 +7351,41 @@
"value": "<<<"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "\""
},
{
"type": "BLANK"
}
]
},
{
"type": "FIELD",
"name": "identifier",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "heredoc_start"
},
{
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\""
},
{
"type": "SYMBOL",
"name": "heredoc_start"
},
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "STRING",
"value": "\""
}
}
]
}
]
"type": "SYMBOL",
"name": "heredoc_start"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "STRING",
"value": "\""
}
},
{
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
Expand All @@ -7401,47 +7403,34 @@
{
"type": "SYMBOL",
"name": "_new_line"
},
{
"type": "FIELD",
"name": "end_tag",
"content": {
"type": "SYMBOL",
"name": "heredoc_end"
}
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "FIELD",
"name": "value",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "heredoc_body"
},
{
"type": "BLANK"
}
]
}
},
{
"type": "FIELD",
"name": "end_tag",
"content": {
"type": "FIELD",
"name": "value",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "heredoc_end"
"name": "heredoc_body"
},
{
"type": "BLANK"
}
}
]
]
}
}
]
},
{
"type": "FIELD",
"name": "end_tag",
"content": {
"type": "SYMBOL",
"name": "heredoc_end"
}
}
]
},
Expand All @@ -7457,25 +7446,11 @@
"name": "_new_line"
},
{
"type": "CHOICE",
"members": [
{
"type": "REPEAT1",
"content": {
"type": "SYMBOL",
"name": "nowdoc_string"
}
},
{
"type": "ALIAS",
"content": {
"type": "STRING",
"value": ""
},
"named": true,
"value": "nowdoc_string"
}
]
"type": "REPEAT1",
"content": {
"type": "SYMBOL",
"name": "nowdoc_string"
}
}
]
},
Expand Down Expand Up @@ -7525,47 +7500,34 @@
{
"type": "SYMBOL",
"name": "_new_line"
},
{
"type": "FIELD",
"name": "end_tag",
"content": {
"type": "SYMBOL",
"name": "heredoc_end"
}
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "FIELD",
"name": "value",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "nowdoc_body"
},
{
"type": "BLANK"
}
]
}
},
{
"type": "FIELD",
"name": "end_tag",
"content": {
"type": "FIELD",
"name": "value",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "heredoc_end"
"name": "nowdoc_body"
},
{
"type": "BLANK"
}
}
]
]
}
}
]
},
{
"type": "FIELD",
"name": "end_tag",
"content": {
"type": "SYMBOL",
"name": "heredoc_end"
}
}
]
},
Expand Down
6 changes: 1 addition & 5 deletions php/src/node-types.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2396,13 +2396,9 @@
]
},
"identifier": {
"multiple": true,
"multiple": false,
"required": true,
"types": [
{
"type": "\"",
"named": false
},
{
"type": "heredoc_start",
"named": true
Expand Down
Loading

0 comments on commit 17cca5f

Please sign in to comment.