From a4d4cc37242b1639bd77e7242d2be8e8c5b4d289 Mon Sep 17 00:00:00 2001
From: Greg Sherwood <gsherwood@squiz.net>
Date: Wed, 22 Jan 2020 08:47:15 +1100
Subject: [PATCH] Run the tokenizer as normal after the T_FN backfill

The backfill already modified the original tokens array so that future passes would detect the token as T_FN, but the return type check needs to process the T_FN token itself, so continue processing and let the T_FN token get added as normal.
---
 src/Tokenizers/PHP.php                       | 42 +++++++-------------
 tests/Core/Tokenizer/BackfillFnTokenTest.inc |  3 ++
 tests/Core/Tokenizer/BackfillFnTokenTest.php |  1 +
 3 files changed, 19 insertions(+), 27 deletions(-)

diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
index 1722498dfd..94eae962f6 100644
--- a/src/Tokenizers/PHP.php
+++ b/src/Tokenizers/PHP.php
@@ -1208,6 +1208,21 @@ protected function tokenize($string)
                 continue;
             }
 
+            /*
+                Backfill the T_FN token for PHP versions < 7.4.
+            */
+
+            if ($tokenIsArray === true
+                && $token[0] === T_STRING
+                && strtolower($token[1]) === 'fn'
+            ) {
+                // Modify the original token stack so that
+                // future checks (like looking for T_NULLABLE) can
+                // detect the T_FN token more easily.
+                $tokens[$stackPtr][0] = T_FN;
+                $token[0] = T_FN;
+            }
+
             /*
                 The string-like token after a function keyword should always be
                 tokenized as T_STRING even if it appears to be a different token,
@@ -1357,33 +1372,6 @@ function return types. We want to keep the parenthesis map clean,
                 continue;
             }
 
-            /*
-                Backfill the T_FN token for PHP versions < 7.4.
-            */
-
-            if ($tokenIsArray === true
-                && $token[0] === T_STRING
-                && strtolower($token[1]) === 'fn'
-            ) {
-                // Modify the original token stack so that
-                // future checks (like looking for T_NULLABLE) can
-                // detect the T_FN token more easily.
-                $tokens[$stackPtr][0] = T_FN;
-
-                $finalTokens[$newStackPtr] = [
-                    'content' => $token[1],
-                    'code'    => T_FN,
-                    'type'    => 'T_FN',
-                ];
-
-                if (PHP_CODESNIFFER_VERBOSITY > 1) {
-                    echo "\t\t* token $stackPtr changed from T_STRING to T_FN".PHP_EOL;
-                }
-
-                $newStackPtr++;
-                continue;
-            }
-
             /*
                 PHP doesn't assign a token to goto labels, so we have to.
                 These are just string tokens with a single colon after them. Double
diff --git a/tests/Core/Tokenizer/BackfillFnTokenTest.inc b/tests/Core/Tokenizer/BackfillFnTokenTest.inc
index 0bff299369..3490b94cf5 100644
--- a/tests/Core/Tokenizer/BackfillFnTokenTest.inc
+++ b/tests/Core/Tokenizer/BackfillFnTokenTest.inc
@@ -67,5 +67,8 @@ fn(parent $a) : parent => $a;
 /* testCallableReturnType */
 fn(callable $a) : callable => $a;
 
+/* testArrayReturnType */
+fn(array $a) : array => $a;
+
 /* testTernary */
 $fn = fn($a) => $a ? /* testTernaryThen */ fn() : string => 'a' : /* testTernaryElse */ fn() : string => 'b';
diff --git a/tests/Core/Tokenizer/BackfillFnTokenTest.php b/tests/Core/Tokenizer/BackfillFnTokenTest.php
index 4132335680..b23ed62a5d 100644
--- a/tests/Core/Tokenizer/BackfillFnTokenTest.php
+++ b/tests/Core/Tokenizer/BackfillFnTokenTest.php
@@ -474,6 +474,7 @@ public function testKeywordReturnTypes()
             'Self',
             'Parent',
             'Callable',
+            'Array',
         ];
 
         foreach ($testMarkers as $marker) {