Fix endAttributes assignment

The end attributes previously were always assigned from the last read token, which does not necessarily correspond to the last token in the reduced rule. In particular this occurs if the parser read a new token and based on that lookahead decided to reduce a rule. The behavior was only correct if the newly read token was first shifted and then the rule was reduced. This is fixed by buffering the endAttributes of the new token in a temporary variable and only assigning them once the token is shifted.
nikic · Nov 20, 2012 · cdbad02 · cdbad02
1 parent b0c8787
commit cdbad02
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 4 deletions.
diff --git a/grammar/kmyacc.php.parser b/grammar/kmyacc.php.parser
@@ -194,8 +194,11 @@ class #(-p)
                 $yyn = self::$yydefault[$state];
             } else {
                 if ($tokenId === self::TOKEN_NONE) {
-                    // fetch the next token id from the lexer and fetch additional info by-ref
-                    $origTokenId = $this->lexer->getNextToken($tokenValue, $startAttributes, $endAttributes);
+                    // Fetch the next token id from the lexer and fetch additional info by-ref.
+                    // The end attributes are fetched into a temporary variable and only set once the token is really
+                    // shifted (not during read). Otherwise you would sometimes get off-by-one errors, when a rule is
+                    // reduced after a token was read but not yet shifted.
+                    $origTokenId = $this->lexer->getNextToken($tokenValue, $startAttributes, $nextEndAttributes);
 
                     // map the lexer token id to the internally used token id's
                     $tokenId = $origTokenId >= 0 && $origTokenId < self::TOKEN_MAP_SIZE
@@ -241,6 +244,7 @@ class #(-p)
                         $stateStack[$this->stackPos]     = $state = $yyn;
                         $this->yyastk[$this->stackPos]   = $tokenValue;
                         $attributeStack[$this->stackPos] = $startAttributes;
+                        $endAttributes = $nextEndAttributes;
                         $tokenId = self::TOKEN_NONE;
 
                         if ($yyn < self::YYNLSTATES)

diff --git a/lib/PHPParser/Parser.php b/lib/PHPParser/Parser.php
@@ -972,8 +972,11 @@ public function parse($code) {
                 $yyn = self::$yydefault[$state];
             } else {
                 if ($tokenId === self::TOKEN_NONE) {
-                    // fetch the next token id from the lexer and fetch additional info by-ref
-                    $origTokenId = $this->lexer->getNextToken($tokenValue, $startAttributes, $endAttributes);
+                    // Fetch the next token id from the lexer and fetch additional info by-ref.
+                    // The end attributes are fetched into a temporary variable and only set once the token is really
+                    // shifted (not during read). Otherwise you would sometimes get off-by-one errors, when a rule is
+                    // reduced after a token was read but not yet shifted.
+                    $origTokenId = $this->lexer->getNextToken($tokenValue, $startAttributes, $nextEndAttributes);
 
                     // map the lexer token id to the internally used token id's
                     $tokenId = $origTokenId >= 0 && $origTokenId < self::TOKEN_MAP_SIZE
@@ -1011,6 +1014,7 @@ public function parse($code) {
                         $stateStack[$this->stackPos]     = $state = $yyn;
                         $this->yyastk[$this->stackPos]   = $tokenValue;
                         $attributeStack[$this->stackPos] = $startAttributes;
+                        $endAttributes = $nextEndAttributes;
                         $tokenId = self::TOKEN_NONE;
 
                         if ($yyn < self::YYNLSTATES)