From 411d0c3f0ba403ca7cbda65b444a60cd2ddd5789 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Danie=CC=88l=20de=20Kok?= <me@danieldk.eu>
Date: Tue, 18 Jun 2024 11:56:46 +0200
Subject: [PATCH] Idefics2: sync added image tokens with transformers

Before this change, the number of reserved image tokens was not the
same as the number of images. Fixes #2029.
---
 .../test_flash_idefics2_next_load.json        | 2578 +++++++++--------
 .../test_flash_idefics2_next_simple.json      |   20 +-
 .../test_flash_idefics2_two_images.json       |   38 +-
 .../models/pali_gemma.py                      |    4 +-
 .../models/vlm_causal_lm.py                   |   17 +-
 5 files changed, 1410 insertions(+), 1247 deletions(-)

diff --git a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_load.json b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_load.json
index 4bc90896786..3682b56df38 100644
--- a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_load.json
+++ b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_load.json
@@ -12,12 +12,12 @@
         },
         {
           "id": 1247,
-          "logprob": -5.2421875,
+          "logprob": -5.2382812,
           "text": "User"
         },
         {
           "id": 28747,
-          "logprob": -6.9570312,
+          "logprob": -6.9609375,
           "text": ":"
         },
         {
@@ -37,7 +37,7 @@
         },
         {
           "id": 32001,
-          "logprob": -19.484375,
+          "logprob": -19.46875,
           "text": "<image>"
         },
         {
@@ -57,7 +57,7 @@
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -20.21875,
           "text": "<image>"
         },
         {
@@ -67,7 +67,7 @@
         },
         {
           "id": 32001,
-          "logprob": -19.828125,
+          "logprob": -19.84375,
           "text": "<image>"
         },
         {
@@ -97,12 +97,12 @@
         },
         {
           "id": 32001,
-          "logprob": -21.296875,
+          "logprob": -21.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.015625,
+          "logprob": -21.03125,
           "text": "<image>"
         },
         {
@@ -117,7 +117,7 @@
         },
         {
           "id": 32001,
-          "logprob": -19.0625,
+          "logprob": -19.046875,
           "text": "<image>"
         },
         {
@@ -127,22 +127,22 @@
         },
         {
           "id": 32001,
-          "logprob": -23.625,
+          "logprob": -23.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.40625,
+          "logprob": -20.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.421875,
+          "logprob": -20.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.84375,
+          "logprob": -20.875,
           "text": "<image>"
         },
         {
@@ -152,12 +152,12 @@
         },
         {
           "id": 32001,
-          "logprob": -19.984375,
+          "logprob": -19.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.21875,
+          "logprob": -18.234375,
           "text": "<image>"
         },
         {
@@ -167,7 +167,7 @@
         },
         {
           "id": 32001,
-          "logprob": -21.203125,
+          "logprob": -21.1875,
           "text": "<image>"
         },
         {
@@ -182,7 +182,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.984375,
+          "logprob": -19.015625,
           "text": "<image>"
         },
         {
@@ -197,17 +197,17 @@
         },
         {
           "id": 32001,
-          "logprob": -18.0,
+          "logprob": -17.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.828125,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.9375,
+          "logprob": -17.921875,
           "text": "<image>"
         },
         {
@@ -217,17 +217,17 @@
         },
         {
           "id": 32001,
-          "logprob": -18.640625,
+          "logprob": -18.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.125,
+          "logprob": -20.140625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.4375,
+          "logprob": -19.421875,
           "text": "<image>"
         },
         {
@@ -247,7 +247,7 @@
         },
         {
           "id": 32001,
-          "logprob": -17.4375,
+          "logprob": -17.421875,
           "text": "<image>"
         },
         {
@@ -257,7 +257,7 @@
         },
         {
           "id": 32001,
-          "logprob": -23.015625,
+          "logprob": -23.0,
           "text": "<image>"
         },
         {
@@ -282,7 +282,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.40625,
+          "logprob": -18.421875,
           "text": "<image>"
         },
         {
@@ -292,17 +292,17 @@
         },
         {
           "id": 32001,
-          "logprob": -18.34375,
+          "logprob": -18.328125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.140625,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.671875,
+          "logprob": -18.65625,
           "text": "<image>"
         },
         {
@@ -317,7 +317,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.1875,
+          "logprob": -18.15625,
           "text": "<image>"
         },
         {
@@ -332,52 +332,62 @@
         },
         {
           "id": 32001,
-          "logprob": -18.96875,
+          "logprob": -18.953125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.71875,
+          "logprob": -18.703125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.9375,
+          "logprob": -17.921875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -3.015625,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.796875,
+          "text": "<fake_token_around_image>"
+        },
         {
           "id": 32001,
-          "logprob": -17.921875,
+          "logprob": -22.09375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.125,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.296875,
+          "logprob": -20.046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.25,
+          "logprob": -17.078125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.96875,
+          "logprob": -17.734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.546875,
+          "logprob": -19.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.921875,
+          "logprob": -19.5625,
           "text": "<image>"
         },
         {
@@ -387,532 +397,542 @@
         },
         {
           "id": 32001,
-          "logprob": -19.546875,
+          "logprob": -19.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.1875,
+          "logprob": -19.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.953125,
+          "logprob": -20.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.828125,
+          "logprob": -15.8046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.375,
+          "logprob": -16.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.171875,
+          "logprob": -19.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.03125,
+          "logprob": -18.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.71875,
+          "logprob": -21.546875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.65625,
+          "logprob": -20.484375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.453125,
+          "logprob": -19.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.484375,
+          "logprob": -19.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.875,
+          "logprob": -16.671875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.65625,
+          "logprob": -20.390625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.296875,
+          "logprob": -21.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.546875,
+          "logprob": -20.359375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.78125,
+          "logprob": -20.703125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.96875,
+          "logprob": -17.71875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.5,
+          "logprob": -20.28125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4140625,
+          "logprob": -15.2421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.046875,
+          "logprob": -18.78125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.109375,
+          "logprob": -17.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.7265625,
+          "logprob": -15.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.5625,
+          "logprob": -18.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.734375,
+          "logprob": -20.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.359375,
+          "logprob": -20.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.421875,
+          "logprob": -18.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.984375,
+          "logprob": -17.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.265625,
+          "logprob": -17.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.78125,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.046875,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.4375,
+          "logprob": -18.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.78125,
+          "logprob": -19.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.671875,
+          "logprob": -19.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -14.2421875,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.0,
+          "logprob": -18.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.40625,
+          "logprob": -18.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.59375,
+          "logprob": -17.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.671875,
+          "logprob": -20.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.265625,
+          "logprob": -19.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.578125,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.234375,
+          "logprob": -18.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.484375,
+          "logprob": -17.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.84375,
+          "logprob": -16.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.296875,
+          "logprob": -18.078125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.8671875,
+          "logprob": -15.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.375,
+          "logprob": -18.15625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.765625,
+          "logprob": -18.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.609375,
+          "logprob": -18.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.515625,
+          "logprob": -17.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.25,
+          "logprob": -16.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.640625,
+          "logprob": -19.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -14.8515625,
+          "logprob": -14.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.875,
+          "logprob": -20.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.28125,
+          "logprob": -20.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.453125,
+          "logprob": -18.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.25,
+          "logprob": -18.15625,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -3.3222656,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -21.375,
+          "text": "<fake_token_around_image>"
+        },
         {
           "id": 32001,
-          "logprob": -19.203125,
+          "logprob": -25.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.71875,
+          "logprob": -18.359375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.390625,
+          "logprob": -19.234375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.984375,
+          "logprob": -17.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.390625,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.5,
+          "logprob": -20.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.296875,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.4375,
+          "logprob": -16.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.015625,
+          "logprob": -19.640625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.359375,
+          "logprob": -18.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.375,
+          "logprob": -16.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.8125,
+          "logprob": -15.7421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.953125,
+          "logprob": -16.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.515625,
+          "logprob": -17.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.875,
+          "logprob": -16.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.109375,
+          "logprob": -18.78125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.265625,
+          "logprob": -18.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.234375,
+          "logprob": -16.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.28125,
+          "logprob": -18.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.25,
+          "logprob": -19.703125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.25,
+          "logprob": -19.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.765625,
+          "logprob": -20.390625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.609375,
+          "logprob": -19.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.359375,
+          "logprob": -20.046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.90625,
+          "logprob": -19.28125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.28125,
+          "logprob": -19.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.5,
+          "logprob": -19.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.0625,
+          "logprob": -19.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.125,
+          "logprob": -17.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.40625,
+          "logprob": -16.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.546875,
+          "logprob": -15.8828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.84375,
+          "logprob": -19.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.484375,
+          "logprob": -20.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.265625,
+          "logprob": -18.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.0625,
+          "logprob": -15.359375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -19.734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.9453125,
+          "logprob": -15.3984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.0625,
+          "logprob": -20.640625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.515625,
+          "logprob": -20.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.375,
+          "logprob": -19.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.796875,
+          "logprob": -17.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.03125,
+          "logprob": -15.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.671875,
+          "logprob": -18.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.15625,
+          "logprob": -19.671875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -19.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.84375,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.78125,
+          "logprob": -19.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.234375,
+          "logprob": -18.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.078125,
+          "logprob": -17.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.28125,
+          "logprob": -18.859375,
           "text": "<image>"
         },
         {
@@ -922,102 +942,112 @@
         },
         {
           "id": 32001,
-          "logprob": -16.59375,
+          "logprob": -16.171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.65625,
+          "logprob": -16.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.453125,
+          "logprob": -18.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.9375,
+          "logprob": -19.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.703125,
+          "logprob": -16.28125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.15625,
+          "logprob": -17.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.46875,
+          "logprob": -16.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.796875,
+          "logprob": -19.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.34375,
+          "logprob": -16.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.3125,
+          "logprob": -16.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.203125,
+          "logprob": -20.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.921875,
+          "logprob": -18.5,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.09375,
+          "logprob": -17.984375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -2.7539062,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.296875,
+          "text": "<fake_token_around_image>"
+        },
         {
           "id": 32001,
-          "logprob": -18.6875,
+          "logprob": -23.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.625,
+          "logprob": -22.078125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.375,
+          "logprob": -20.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.765625,
+          "logprob": -18.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.46875,
+          "logprob": -18.078125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.546875,
+          "logprob": -20.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.875,
+          "logprob": -18.40625,
           "text": "<image>"
         },
         {
@@ -1027,247 +1057,247 @@
         },
         {
           "id": 32001,
-          "logprob": -21.09375,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.5625,
+          "logprob": -19.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.15625,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.171875,
+          "logprob": -16.046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.671875,
+          "logprob": -17.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.84375,
+          "logprob": -18.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.75,
+          "logprob": -20.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.8125,
+          "logprob": -21.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.96875,
+          "logprob": -19.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.046875,
+          "logprob": -18.671875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.78125,
+          "logprob": -19.640625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.421875,
+          "logprob": -19.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.21875,
+          "logprob": -20.921875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.515625,
+          "logprob": -21.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.609375,
+          "logprob": -19.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.71875,
+          "logprob": -20.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.046875,
+          "logprob": -18.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.1875,
+          "logprob": -19.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.1875,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.828125,
+          "logprob": -19.390625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.359375,
+          "logprob": -17.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.75,
+          "logprob": -17.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.90625,
+          "logprob": -18.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.765625,
+          "logprob": -18.5625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.453125,
+          "logprob": -20.15625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.890625,
+          "logprob": -19.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.015625,
+          "logprob": -15.7734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.90625,
+          "logprob": -18.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.953125,
+          "logprob": -15.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.46875,
+          "logprob": -21.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.984375,
+          "logprob": -19.734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.875,
+          "logprob": -19.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.859375,
+          "logprob": -18.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.046875,
+          "logprob": -15.9609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.375,
+          "logprob": -20.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.140625,
+          "logprob": -19.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.140625,
+          "logprob": -20.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.6875,
+          "logprob": -21.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.453125,
+          "logprob": -21.171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -19.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.78125,
+          "logprob": -17.5625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.65625,
+          "logprob": -19.328125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.078125,
+          "logprob": -17.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.109375,
+          "logprob": -16.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -18.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.453125,
+          "logprob": -20.09375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.0625,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.734375,
+          "logprob": -16.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.21875,
+          "logprob": -18.90625,
           "text": "<image>"
         },
         {
@@ -1277,72 +1307,82 @@
         },
         {
           "id": 32001,
-          "logprob": -20.015625,
+          "logprob": -19.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.796875,
+          "logprob": -17.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.3125,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.390625,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.28125,
+          "logprob": -18.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.59375,
+          "logprob": -18.5625,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -3.0898438,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.09375,
+          "text": "<fake_token_around_image>"
+        },
         {
           "id": 32001,
-          "logprob": -18.8125,
+          "logprob": -25.5625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.09375,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.890625,
+          "logprob": -20.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.09375,
+          "logprob": -17.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.75,
+          "logprob": -18.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.90625,
+          "logprob": -18.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.375,
+          "logprob": -21.5,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.640625,
+          "logprob": -16.78125,
           "text": "<image>"
         },
         {
@@ -1352,22 +1392,22 @@
         },
         {
           "id": 32001,
-          "logprob": -21.40625,
+          "logprob": -21.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.8828125,
+          "logprob": -15.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.1171875,
+          "logprob": -15.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.0625,
+          "logprob": -17.15625,
           "text": "<image>"
         },
         {
@@ -1377,7 +1417,7 @@
         },
         {
           "id": 32001,
-          "logprob": -21.40625,
+          "logprob": -21.46875,
           "text": "<image>"
         },
         {
@@ -1387,57 +1427,57 @@
         },
         {
           "id": 32001,
-          "logprob": -20.75,
+          "logprob": -20.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.25,
+          "logprob": -16.296875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.46875,
+          "logprob": -19.484375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.59375,
+          "logprob": -21.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.421875,
+          "logprob": -22.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.9375,
+          "logprob": -20.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.671875,
+          "logprob": -19.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.890625,
+          "logprob": -21.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.921875,
+          "logprob": -16.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.5,
+          "logprob": -17.53125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.90625,
+          "logprob": -17.9375,
           "text": "<image>"
         },
         {
@@ -1452,37 +1492,37 @@
         },
         {
           "id": 32001,
-          "logprob": -16.40625,
+          "logprob": -16.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.453125,
+          "logprob": -18.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -20.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.28125,
+          "logprob": -22.296875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.515625,
+          "logprob": -18.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4296875,
+          "logprob": -15.3984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.765625,
+          "logprob": -19.734375,
           "text": "<image>"
         },
         {
@@ -1492,57 +1532,57 @@
         },
         {
           "id": 32001,
-          "logprob": -21.46875,
+          "logprob": -21.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.875,
+          "logprob": -18.84375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.859375,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.078125,
+          "logprob": -17.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.4375,
+          "logprob": -16.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.015625,
+          "logprob": -21.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.234375,
+          "logprob": -21.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.140625,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.484375,
+          "logprob": -21.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.015625,
+          "logprob": -20.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.84375,
+          "logprob": -18.78125,
           "text": "<image>"
         },
         {
@@ -1552,137 +1592,137 @@
         },
         {
           "id": 32001,
-          "logprob": -16.890625,
+          "logprob": -16.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.65625,
+          "logprob": -16.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -13.6328125,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4140625,
+          "logprob": -15.3984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.546875,
+          "logprob": -17.5,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.859375,
+          "logprob": -21.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.65625,
+          "logprob": -15.6171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.484375,
+          "logprob": -16.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.359375,
+          "logprob": -16.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.9375,
+          "logprob": -19.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.875,
+          "logprob": -17.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.453125,
+          "logprob": -17.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.390625,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -19.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.9921875,
+          "logprob": -15.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -2.0429688,
+          "logprob": -2.1738281,
           "text": "<fake_token_around_image>"
         },
         {
           "id": 12018,
-          "logprob": -12.03125,
+          "logprob": -11.859375,
           "text": "Write"
         },
         {
           "id": 528,
-          "logprob": -10.25,
+          "logprob": -10.0,
           "text": "me"
         },
         {
           "id": 264,
-          "logprob": -0.10437012,
+          "logprob": -0.09326172,
           "text": "a"
         },
         {
           "id": 2485,
-          "logprob": -4.5742188,
+          "logprob": -4.4648438,
           "text": "short"
         },
         {
           "id": 2838,
-          "logprob": -0.2277832,
+          "logprob": -0.2626953,
           "text": "story"
         },
         {
           "id": 32002,
-          "logprob": -10.84375,
+          "logprob": -11.0390625,
           "text": "<end_of_utterance>"
         },
         {
           "id": 259,
-          "logprob": -20.1875,
+          "logprob": -20.125,
           "text": " "
         },
         {
           "id": 13,
-          "logprob": -8.7578125,
+          "logprob": -8.046875,
           "text": "\n"
         },
         {
           "id": 7226,
-          "logprob": -10.421875,
+          "logprob": -10.5,
           "text": "Ass"
         },
         {
           "id": 11143,
-          "logprob": -13.640625,
+          "logprob": -13.4921875,
           "text": "istant"
         },
         {
           "id": 28747,
-          "logprob": -0.005619049,
+          "logprob": -0.003282547,
           "text": ":"
         }
       ],
@@ -1690,61 +1730,61 @@
       "tokens": [
         {
           "id": 330,
-          "logprob": -0.12939453,
+          "logprob": -0.08648682,
           "special": false,
           "text": " A"
         },
         {
           "id": 13088,
-          "logprob": -0.6660156,
+          "logprob": -0.71191406,
           "special": false,
           "text": " chicken"
         },
         {
           "id": 349,
-          "logprob": -0.29638672,
+          "logprob": -0.33032227,
           "special": false,
           "text": " is"
         },
         {
           "id": 6398,
-          "logprob": -0.05960083,
+          "logprob": -0.051239014,
           "special": false,
           "text": " sitting"
         },
         {
           "id": 356,
-          "logprob": -0.26953125,
+          "logprob": -0.35473633,
           "special": false,
           "text": " on"
         },
         {
           "id": 264,
-          "logprob": -0.1427002,
+          "logprob": -0.1239624,
           "special": false,
           "text": " a"
         },
         {
           "id": 17972,
-          "logprob": -0.040649414,
+          "logprob": -0.038116455,
           "special": false,
           "text": " pile"
         },
         {
           "id": 302,
-          "logprob": -0.0002708435,
+          "logprob": -0.000187397,
           "special": false,
           "text": " of"
         },
         {
           "id": 2445,
-          "logprob": -0.09429932,
+          "logprob": -0.072265625,
           "special": false,
           "text": " money"
         },
         {
           "id": 28723,
-          "logprob": -0.006931305,
+          "logprob": -0.0047912598,
           "special": false,
           "text": "."
         }
@@ -1766,12 +1806,12 @@
         },
         {
           "id": 1247,
-          "logprob": -5.234375,
+          "logprob": -5.2421875,
           "text": "User"
         },
         {
           "id": 28747,
-          "logprob": -6.9648438,
+          "logprob": -6.9570312,
           "text": ":"
         },
         {
@@ -1811,7 +1851,7 @@
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -20.21875,
           "text": "<image>"
         },
         {
@@ -1821,12 +1861,12 @@
         },
         {
           "id": 32001,
-          "logprob": -19.828125,
+          "logprob": -19.84375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -23.25,
+          "logprob": -23.265625,
           "text": "<image>"
         },
         {
@@ -1851,27 +1891,27 @@
         },
         {
           "id": 32001,
-          "logprob": -21.296875,
+          "logprob": -21.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.015625,
+          "logprob": -21.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.4375,
+          "logprob": -20.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.015625,
+          "logprob": -16.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.0625,
+          "logprob": -19.046875,
           "text": "<image>"
         },
         {
@@ -1886,17 +1926,17 @@
         },
         {
           "id": 32001,
-          "logprob": -20.40625,
+          "logprob": -20.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.421875,
+          "logprob": -20.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.84375,
+          "logprob": -20.875,
           "text": "<image>"
         },
         {
@@ -1906,12 +1946,12 @@
         },
         {
           "id": 32001,
-          "logprob": -19.984375,
+          "logprob": -19.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.21875,
+          "logprob": -18.234375,
           "text": "<image>"
         },
         {
@@ -1921,7 +1961,7 @@
         },
         {
           "id": 32001,
-          "logprob": -21.203125,
+          "logprob": -21.1875,
           "text": "<image>"
         },
         {
@@ -1936,7 +1976,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.984375,
+          "logprob": -19.03125,
           "text": "<image>"
         },
         {
@@ -1951,17 +1991,17 @@
         },
         {
           "id": 32001,
-          "logprob": -18.0,
+          "logprob": -17.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.828125,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.9375,
+          "logprob": -17.921875,
           "text": "<image>"
         },
         {
@@ -1971,12 +2011,12 @@
         },
         {
           "id": 32001,
-          "logprob": -18.640625,
+          "logprob": -18.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.125,
+          "logprob": -20.140625,
           "text": "<image>"
         },
         {
@@ -2001,7 +2041,7 @@
         },
         {
           "id": 32001,
-          "logprob": -17.4375,
+          "logprob": -17.421875,
           "text": "<image>"
         },
         {
@@ -2011,7 +2051,7 @@
         },
         {
           "id": 32001,
-          "logprob": -23.015625,
+          "logprob": -23.0,
           "text": "<image>"
         },
         {
@@ -2046,7 +2086,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.34375,
+          "logprob": -18.328125,
           "text": "<image>"
         },
         {
@@ -2056,22 +2096,22 @@
         },
         {
           "id": 32001,
-          "logprob": -18.671875,
+          "logprob": -18.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.875,
+          "logprob": -18.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.0,
+          "logprob": -15.9921875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.1875,
+          "logprob": -18.15625,
           "text": "<image>"
         },
         {
@@ -2081,7 +2121,7 @@
         },
         {
           "id": 32001,
-          "logprob": -20.140625,
+          "logprob": -20.15625,
           "text": "<image>"
         },
         {
@@ -2091,417 +2131,417 @@
         },
         {
           "id": 32001,
-          "logprob": -18.71875,
+          "logprob": -18.703125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.9375,
+          "logprob": -17.921875,
           "text": "<image>"
         },
         {
-          "id": 32001,
-          "logprob": -17.90625,
-          "text": "<image>"
+          "id": 32000,
+          "logprob": -3.015625,
+          "text": "<fake_token_around_image>"
         },
         {
-          "id": 32001,
-          "logprob": -19.125,
-          "text": "<image>"
+          "id": 32000,
+          "logprob": -18.796875,
+          "text": "<fake_token_around_image>"
         },
         {
           "id": 32001,
-          "logprob": -20.296875,
+          "logprob": -22.09375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.25,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.96875,
+          "logprob": -20.046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.546875,
+          "logprob": -17.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.921875,
+          "logprob": -17.71875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.3125,
+          "logprob": -19.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.546875,
+          "logprob": -19.5625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.1875,
+          "logprob": -16.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.953125,
+          "logprob": -19.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.828125,
+          "logprob": -19.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.375,
+          "logprob": -20.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.171875,
+          "logprob": -15.8046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.03125,
+          "logprob": -16.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.71875,
+          "logprob": -19.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.65625,
+          "logprob": -18.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.453125,
+          "logprob": -21.546875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.484375,
+          "logprob": -20.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.875,
+          "logprob": -19.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.65625,
+          "logprob": -19.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.296875,
+          "logprob": -16.671875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.546875,
+          "logprob": -20.390625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.78125,
+          "logprob": -21.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.96875,
+          "logprob": -20.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.5,
+          "logprob": -20.703125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4140625,
+          "logprob": -17.71875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.046875,
+          "logprob": -20.28125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.109375,
+          "logprob": -15.2421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.7265625,
+          "logprob": -18.78125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.5625,
+          "logprob": -17.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.734375,
+          "logprob": -15.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.359375,
+          "logprob": -18.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.421875,
+          "logprob": -20.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.984375,
+          "logprob": -20.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.265625,
+          "logprob": -18.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.78125,
+          "logprob": -17.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.046875,
+          "logprob": -17.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.4375,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.78125,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.671875,
+          "logprob": -18.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -14.2421875,
+          "logprob": -19.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.0,
+          "logprob": -19.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.40625,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.59375,
+          "logprob": -18.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.671875,
+          "logprob": -18.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.265625,
+          "logprob": -17.390625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.578125,
+          "logprob": -20.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.234375,
+          "logprob": -19.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.484375,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.84375,
+          "logprob": -18.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.28125,
+          "logprob": -17.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.8671875,
+          "logprob": -16.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.375,
+          "logprob": -18.078125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.765625,
+          "logprob": -15.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.609375,
+          "logprob": -18.15625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.515625,
+          "logprob": -18.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.25,
+          "logprob": -18.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.640625,
+          "logprob": -17.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -14.8515625,
+          "logprob": -16.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.875,
+          "logprob": -19.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.28125,
+          "logprob": -14.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.453125,
+          "logprob": -20.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.25,
+          "logprob": -20.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.203125,
+          "logprob": -18.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.703125,
+          "logprob": -18.171875,
           "text": "<image>"
         },
         {
-          "id": 32001,
-          "logprob": -19.390625,
-          "text": "<image>"
+          "id": 32000,
+          "logprob": -3.3222656,
+          "text": "<fake_token_around_image>"
         },
         {
-          "id": 32001,
-          "logprob": -17.984375,
-          "text": "<image>"
+          "id": 32000,
+          "logprob": -21.375,
+          "text": "<fake_token_around_image>"
         },
         {
           "id": 32001,
-          "logprob": -17.390625,
+          "logprob": -25.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.5,
+          "logprob": -18.359375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.296875,
+          "logprob": -19.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.4375,
+          "logprob": -17.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.015625,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.359375,
+          "logprob": -20.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.375,
+          "logprob": -17.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.8125,
+          "logprob": -16.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.953125,
+          "logprob": -19.640625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.515625,
+          "logprob": -18.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.875,
+          "logprob": -16.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.109375,
+          "logprob": -15.7421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.265625,
+          "logprob": -16.265625,
           "text": "<image>"
         },
         {
@@ -2511,267 +2551,297 @@
         },
         {
           "id": 32001,
-          "logprob": -19.28125,
+          "logprob": -16.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -18.78125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.25,
+          "logprob": -18.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.765625,
+          "logprob": -16.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.609375,
+          "logprob": -18.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.359375,
+          "logprob": -19.703125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.90625,
+          "logprob": -19.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.28125,
+          "logprob": -20.390625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.5,
+          "logprob": -19.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.0625,
+          "logprob": -20.046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.125,
+          "logprob": -19.28125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.40625,
+          "logprob": -19.84375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.546875,
+          "logprob": -19.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.84375,
+          "logprob": -19.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.484375,
+          "logprob": -17.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.265625,
+          "logprob": -16.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.0625,
+          "logprob": -15.8828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -19.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.9453125,
+          "logprob": -20.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.0625,
+          "logprob": -18.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.515625,
+          "logprob": -15.3671875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.375,
+          "logprob": -19.734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.796875,
+          "logprob": -15.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.03125,
+          "logprob": -20.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.671875,
+          "logprob": -20.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.15625,
+          "logprob": -19.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -17.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.84375,
+          "logprob": -15.9453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.78125,
+          "logprob": -18.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.234375,
+          "logprob": -19.671875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.078125,
+          "logprob": -19.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.28125,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.09375,
+          "logprob": -19.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.59375,
+          "logprob": -18.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.65625,
+          "logprob": -17.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.4375,
+          "logprob": -18.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.9375,
+          "logprob": -17.078125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.703125,
+          "logprob": -16.171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.15625,
+          "logprob": -16.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.46875,
+          "logprob": -18.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.796875,
+          "logprob": -19.484375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.34375,
+          "logprob": -16.296875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.3125,
+          "logprob": -17.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.203125,
+          "logprob": -16.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.90625,
+          "logprob": -19.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.09375,
+          "logprob": -16.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.6875,
+          "logprob": -16.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.625,
+          "logprob": -20.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.375,
+          "logprob": -18.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -2.7539062,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.296875,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.765625,
+          "logprob": -22.09375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.46875,
+          "logprob": -20.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.546875,
+          "logprob": -18.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.875,
+          "logprob": -18.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.390625,
           "text": "<image>"
         },
         {
@@ -2781,212 +2851,212 @@
         },
         {
           "id": 32001,
-          "logprob": -21.09375,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.5625,
+          "logprob": -19.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.15625,
+          "logprob": -18.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.171875,
+          "logprob": -16.046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.671875,
+          "logprob": -17.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.84375,
+          "logprob": -18.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.75,
+          "logprob": -20.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.8125,
+          "logprob": -21.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.96875,
+          "logprob": -19.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.046875,
+          "logprob": -18.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.78125,
+          "logprob": -19.640625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.421875,
+          "logprob": -19.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.21875,
+          "logprob": -20.921875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.515625,
+          "logprob": -21.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.609375,
+          "logprob": -19.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.71875,
+          "logprob": -20.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.046875,
+          "logprob": -18.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.171875,
+          "logprob": -19.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.1875,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.828125,
+          "logprob": -19.390625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.359375,
+          "logprob": -17.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.75,
+          "logprob": -17.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.90625,
+          "logprob": -18.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.765625,
+          "logprob": -18.5625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.453125,
+          "logprob": -20.15625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.890625,
+          "logprob": -19.78125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.015625,
+          "logprob": -15.7734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.90625,
+          "logprob": -18.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.953125,
+          "logprob": -15.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.46875,
+          "logprob": -21.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.984375,
+          "logprob": -19.734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.890625,
+          "logprob": -19.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.875,
+          "logprob": -18.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.046875,
+          "logprob": -15.9609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.375,
+          "logprob": -20.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.140625,
+          "logprob": -19.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.140625,
+          "logprob": -20.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.703125,
+          "logprob": -21.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.453125,
+          "logprob": -21.171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -19.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.765625,
+          "logprob": -17.5625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.65625,
+          "logprob": -19.328125,
           "text": "<image>"
         },
         {
@@ -2996,32 +3066,32 @@
         },
         {
           "id": 32001,
-          "logprob": -17.109375,
+          "logprob": -16.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -18.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.453125,
+          "logprob": -20.09375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.0625,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.734375,
+          "logprob": -16.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.21875,
+          "logprob": -18.90625,
           "text": "<image>"
         },
         {
@@ -3031,72 +3101,82 @@
         },
         {
           "id": 32001,
-          "logprob": -20.015625,
+          "logprob": -19.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.796875,
+          "logprob": -17.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.3125,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.390625,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.28125,
+          "logprob": -18.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.59375,
+          "logprob": -18.5625,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -3.0898438,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.09375,
+          "text": "<fake_token_around_image>"
+        },
         {
           "id": 32001,
-          "logprob": -18.8125,
+          "logprob": -25.5625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.09375,
+          "logprob": -18.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.890625,
+          "logprob": -20.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.09375,
+          "logprob": -17.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.75,
+          "logprob": -18.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.90625,
+          "logprob": -18.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.375,
+          "logprob": -21.484375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.640625,
+          "logprob": -16.78125,
           "text": "<image>"
         },
         {
@@ -3106,22 +3186,22 @@
         },
         {
           "id": 32001,
-          "logprob": -21.40625,
+          "logprob": -21.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.875,
+          "logprob": -15.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.1171875,
+          "logprob": -15.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.078125,
+          "logprob": -17.15625,
           "text": "<image>"
         },
         {
@@ -3131,7 +3211,7 @@
         },
         {
           "id": 32001,
-          "logprob": -21.40625,
+          "logprob": -21.46875,
           "text": "<image>"
         },
         {
@@ -3141,57 +3221,57 @@
         },
         {
           "id": 32001,
-          "logprob": -20.75,
+          "logprob": -20.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.25,
+          "logprob": -16.296875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.453125,
+          "logprob": -19.484375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.59375,
+          "logprob": -21.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.421875,
+          "logprob": -22.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.9375,
+          "logprob": -20.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.671875,
+          "logprob": -19.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.890625,
+          "logprob": -21.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.921875,
+          "logprob": -16.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.5,
+          "logprob": -17.53125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.890625,
+          "logprob": -17.9375,
           "text": "<image>"
         },
         {
@@ -3206,77 +3286,77 @@
         },
         {
           "id": 32001,
-          "logprob": -16.40625,
+          "logprob": -16.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.453125,
+          "logprob": -18.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.21875,
+          "logprob": -20.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.28125,
+          "logprob": -22.296875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.515625,
+          "logprob": -18.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4296875,
+          "logprob": -15.3984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.765625,
+          "logprob": -19.734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -14.6484375,
+          "logprob": -14.640625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.46875,
+          "logprob": -21.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.875,
+          "logprob": -18.84375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.859375,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.078125,
+          "logprob": -17.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.4375,
+          "logprob": -16.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.015625,
+          "logprob": -21.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.234375,
+          "logprob": -21.21875,
           "text": "<image>"
         },
         {
@@ -3286,157 +3366,157 @@
         },
         {
           "id": 32001,
-          "logprob": -21.484375,
+          "logprob": -21.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.015625,
+          "logprob": -20.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.84375,
+          "logprob": -18.78125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.421875,
+          "logprob": -16.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.890625,
+          "logprob": -16.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.65625,
+          "logprob": -16.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -13.640625,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.421875,
+          "logprob": -15.3984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.546875,
+          "logprob": -17.5,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.875,
+          "logprob": -21.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.65625,
+          "logprob": -15.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.484375,
+          "logprob": -16.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.359375,
+          "logprob": -16.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.953125,
+          "logprob": -19.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.875,
+          "logprob": -17.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.453125,
+          "logprob": -17.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.390625,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -19.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.9921875,
+          "logprob": -15.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -2.0429688,
+          "logprob": -2.1777344,
           "text": "<fake_token_around_image>"
         },
         {
           "id": 12018,
-          "logprob": -12.03125,
+          "logprob": -11.8671875,
           "text": "Write"
         },
         {
           "id": 528,
-          "logprob": -10.2578125,
+          "logprob": -10.0078125,
           "text": "me"
         },
         {
           "id": 264,
-          "logprob": -0.10418701,
+          "logprob": -0.093444824,
           "text": "a"
         },
         {
           "id": 2485,
-          "logprob": -4.5664062,
+          "logprob": -4.4648438,
           "text": "short"
         },
         {
           "id": 2838,
-          "logprob": -0.22741699,
+          "logprob": -0.26293945,
           "text": "story"
         },
         {
           "id": 32002,
-          "logprob": -10.8515625,
+          "logprob": -11.046875,
           "text": "<end_of_utterance>"
         },
         {
           "id": 259,
-          "logprob": -20.203125,
+          "logprob": -20.125,
           "text": " "
         },
         {
           "id": 13,
-          "logprob": -8.7421875,
+          "logprob": -8.0234375,
           "text": "\n"
         },
         {
           "id": 7226,
-          "logprob": -10.4140625,
+          "logprob": -10.5078125,
           "text": "Ass"
         },
         {
           "id": 11143,
-          "logprob": -13.6328125,
+          "logprob": -13.5,
           "text": "istant"
         },
         {
           "id": 28747,
-          "logprob": -0.005580902,
+          "logprob": -0.0031471252,
           "text": ":"
         }
       ],
@@ -3444,61 +3524,61 @@
       "tokens": [
         {
           "id": 330,
-          "logprob": -0.1295166,
+          "logprob": -0.08660889,
           "special": false,
           "text": " A"
         },
         {
           "id": 13088,
-          "logprob": -0.6669922,
+          "logprob": -0.7089844,
           "special": false,
           "text": " chicken"
         },
         {
           "id": 349,
-          "logprob": -0.29711914,
+          "logprob": -0.32861328,
           "special": false,
           "text": " is"
         },
         {
           "id": 6398,
-          "logprob": -0.059936523,
+          "logprob": -0.051513672,
           "special": false,
           "text": " sitting"
         },
         {
           "id": 356,
-          "logprob": -0.27124023,
+          "logprob": -0.3569336,
           "special": false,
           "text": " on"
         },
         {
           "id": 264,
-          "logprob": -0.140625,
+          "logprob": -0.1239624,
           "special": false,
           "text": " a"
         },
         {
           "id": 17972,
-          "logprob": -0.04058838,
+          "logprob": -0.038146973,
           "special": false,
           "text": " pile"
         },
         {
           "id": 302,
-          "logprob": -0.00027012825,
+          "logprob": -0.00018715858,
           "special": false,
           "text": " of"
         },
         {
           "id": 2445,
-          "logprob": -0.09503174,
+          "logprob": -0.071777344,
           "special": false,
           "text": " money"
         },
         {
           "id": 28723,
-          "logprob": -0.006942749,
+          "logprob": -0.004798889,
           "special": false,
           "text": "."
         }
@@ -3520,12 +3600,12 @@
         },
         {
           "id": 1247,
-          "logprob": -5.2460938,
+          "logprob": -5.2382812,
           "text": "User"
         },
         {
           "id": 28747,
-          "logprob": -6.9570312,
+          "logprob": -6.9609375,
           "text": ":"
         },
         {
@@ -3565,7 +3645,7 @@
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -20.21875,
           "text": "<image>"
         },
         {
@@ -3605,17 +3685,17 @@
         },
         {
           "id": 32001,
-          "logprob": -21.296875,
+          "logprob": -21.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.015625,
+          "logprob": -21.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.4375,
+          "logprob": -20.421875,
           "text": "<image>"
         },
         {
@@ -3625,7 +3705,7 @@
         },
         {
           "id": 32001,
-          "logprob": -19.0625,
+          "logprob": -19.046875,
           "text": "<image>"
         },
         {
@@ -3635,22 +3715,22 @@
         },
         {
           "id": 32001,
-          "logprob": -23.625,
+          "logprob": -23.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.40625,
+          "logprob": -20.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.421875,
+          "logprob": -20.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.84375,
+          "logprob": -20.875,
           "text": "<image>"
         },
         {
@@ -3660,12 +3740,12 @@
         },
         {
           "id": 32001,
-          "logprob": -19.984375,
+          "logprob": -19.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.21875,
+          "logprob": -18.234375,
           "text": "<image>"
         },
         {
@@ -3675,7 +3755,7 @@
         },
         {
           "id": 32001,
-          "logprob": -21.21875,
+          "logprob": -21.1875,
           "text": "<image>"
         },
         {
@@ -3690,7 +3770,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.984375,
+          "logprob": -19.015625,
           "text": "<image>"
         },
         {
@@ -3705,12 +3785,12 @@
         },
         {
           "id": 32001,
-          "logprob": -18.0,
+          "logprob": -17.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.84375,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
@@ -3725,17 +3805,17 @@
         },
         {
           "id": 32001,
-          "logprob": -18.640625,
+          "logprob": -18.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.125,
+          "logprob": -20.140625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.4375,
+          "logprob": -19.421875,
           "text": "<image>"
         },
         {
@@ -3750,12 +3830,12 @@
         },
         {
           "id": 32001,
-          "logprob": -23.203125,
+          "logprob": -23.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.4375,
+          "logprob": -17.421875,
           "text": "<image>"
         },
         {
@@ -3765,7 +3845,7 @@
         },
         {
           "id": 32001,
-          "logprob": -23.015625,
+          "logprob": -23.0,
           "text": "<image>"
         },
         {
@@ -3790,7 +3870,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.40625,
+          "logprob": -18.421875,
           "text": "<image>"
         },
         {
@@ -3800,7 +3880,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.34375,
+          "logprob": -18.328125,
           "text": "<image>"
         },
         {
@@ -3810,7 +3890,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.671875,
+          "logprob": -18.65625,
           "text": "<image>"
         },
         {
@@ -3825,7 +3905,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.1875,
+          "logprob": -18.15625,
           "text": "<image>"
         },
         {
@@ -3835,57 +3915,67 @@
         },
         {
           "id": 32001,
-          "logprob": -20.140625,
+          "logprob": -20.15625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.96875,
+          "logprob": -18.953125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.71875,
+          "logprob": -18.703125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.9375,
+          "logprob": -17.921875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -3.015625,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.796875,
+          "text": "<fake_token_around_image>"
+        },
         {
           "id": 32001,
-          "logprob": -17.921875,
+          "logprob": -22.09375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.125,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.296875,
+          "logprob": -20.046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.25,
+          "logprob": -17.078125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.96875,
+          "logprob": -17.734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.546875,
+          "logprob": -19.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.921875,
+          "logprob": -19.5625,
           "text": "<image>"
         },
         {
@@ -3895,532 +3985,542 @@
         },
         {
           "id": 32001,
-          "logprob": -19.546875,
+          "logprob": -19.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.1875,
+          "logprob": -19.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.953125,
+          "logprob": -20.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.8359375,
+          "logprob": -15.8046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.375,
+          "logprob": -16.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.1875,
+          "logprob": -19.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.03125,
+          "logprob": -18.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.71875,
+          "logprob": -21.546875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.65625,
+          "logprob": -20.484375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.453125,
+          "logprob": -19.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.484375,
+          "logprob": -19.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.875,
+          "logprob": -16.671875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.65625,
+          "logprob": -20.390625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.296875,
+          "logprob": -21.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.546875,
+          "logprob": -20.359375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.78125,
+          "logprob": -20.703125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.96875,
+          "logprob": -17.71875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.5,
+          "logprob": -20.28125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4140625,
+          "logprob": -15.2421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.046875,
+          "logprob": -18.78125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.109375,
+          "logprob": -17.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.7265625,
+          "logprob": -15.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.5625,
+          "logprob": -18.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.734375,
+          "logprob": -20.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.359375,
+          "logprob": -20.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.421875,
+          "logprob": -18.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.984375,
+          "logprob": -17.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.265625,
+          "logprob": -17.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.78125,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.046875,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.4375,
+          "logprob": -18.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.78125,
+          "logprob": -19.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.671875,
+          "logprob": -19.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -14.25,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.0,
+          "logprob": -18.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.40625,
+          "logprob": -18.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.59375,
+          "logprob": -17.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.671875,
+          "logprob": -20.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.265625,
+          "logprob": -19.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.578125,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.234375,
+          "logprob": -18.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.484375,
+          "logprob": -17.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.84375,
+          "logprob": -16.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.28125,
+          "logprob": -18.078125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.8671875,
+          "logprob": -15.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.375,
+          "logprob": -18.15625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.765625,
+          "logprob": -18.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.609375,
+          "logprob": -18.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.515625,
+          "logprob": -17.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.25,
+          "logprob": -16.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.640625,
+          "logprob": -19.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -14.8515625,
+          "logprob": -14.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.875,
+          "logprob": -20.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.265625,
+          "logprob": -20.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.453125,
+          "logprob": -18.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.25,
+          "logprob": -18.15625,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -3.3222656,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -21.375,
+          "text": "<fake_token_around_image>"
+        },
         {
           "id": 32001,
-          "logprob": -19.203125,
+          "logprob": -25.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.71875,
+          "logprob": -18.359375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.390625,
+          "logprob": -19.234375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.0,
+          "logprob": -17.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.390625,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.5,
+          "logprob": -20.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.296875,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.4375,
+          "logprob": -16.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.015625,
+          "logprob": -19.640625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.359375,
+          "logprob": -18.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.375,
+          "logprob": -16.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.8125,
+          "logprob": -15.7421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.953125,
+          "logprob": -16.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.515625,
+          "logprob": -17.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.875,
+          "logprob": -16.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.109375,
+          "logprob": -18.78125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.265625,
+          "logprob": -18.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.234375,
+          "logprob": -16.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.28125,
+          "logprob": -18.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -19.703125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.25,
+          "logprob": -19.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.765625,
+          "logprob": -20.390625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.609375,
+          "logprob": -19.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.34375,
+          "logprob": -20.046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.921875,
+          "logprob": -19.28125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.28125,
+          "logprob": -19.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.515625,
+          "logprob": -19.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.0625,
+          "logprob": -19.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.125,
+          "logprob": -17.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.40625,
+          "logprob": -16.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.546875,
+          "logprob": -15.8828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.84375,
+          "logprob": -19.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.484375,
+          "logprob": -20.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.265625,
+          "logprob": -18.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.0625,
+          "logprob": -15.359375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -19.734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.9453125,
+          "logprob": -15.3984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.046875,
+          "logprob": -20.640625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.515625,
+          "logprob": -20.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.375,
+          "logprob": -19.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.796875,
+          "logprob": -17.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.03125,
+          "logprob": -15.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.671875,
+          "logprob": -18.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.15625,
+          "logprob": -19.671875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -19.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.84375,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.78125,
+          "logprob": -19.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.234375,
+          "logprob": -18.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.078125,
+          "logprob": -17.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.28125,
+          "logprob": -18.859375,
           "text": "<image>"
         },
         {
@@ -4430,102 +4530,112 @@
         },
         {
           "id": 32001,
-          "logprob": -16.59375,
+          "logprob": -16.171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.65625,
+          "logprob": -16.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.453125,
+          "logprob": -18.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.9375,
+          "logprob": -19.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.703125,
+          "logprob": -16.28125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.15625,
+          "logprob": -17.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.46875,
+          "logprob": -16.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.796875,
+          "logprob": -19.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.34375,
+          "logprob": -16.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.3125,
+          "logprob": -16.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.1875,
+          "logprob": -20.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.921875,
+          "logprob": -18.5,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.09375,
+          "logprob": -17.984375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -2.7539062,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.296875,
+          "text": "<fake_token_around_image>"
+        },
         {
           "id": 32001,
-          "logprob": -18.6875,
+          "logprob": -23.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.625,
+          "logprob": -22.078125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.359375,
+          "logprob": -20.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.765625,
+          "logprob": -18.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.46875,
+          "logprob": -18.078125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.546875,
+          "logprob": -20.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.875,
+          "logprob": -18.40625,
           "text": "<image>"
         },
         {
@@ -4535,247 +4645,247 @@
         },
         {
           "id": 32001,
-          "logprob": -21.09375,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.5625,
+          "logprob": -19.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.15625,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.171875,
+          "logprob": -16.046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.671875,
+          "logprob": -17.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.859375,
+          "logprob": -18.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.734375,
+          "logprob": -20.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.8125,
+          "logprob": -21.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.96875,
+          "logprob": -19.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.046875,
+          "logprob": -18.671875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.78125,
+          "logprob": -19.640625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.421875,
+          "logprob": -19.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.21875,
+          "logprob": -20.921875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.515625,
+          "logprob": -21.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.609375,
+          "logprob": -19.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.734375,
+          "logprob": -20.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.046875,
+          "logprob": -18.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.1875,
+          "logprob": -19.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.1875,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.828125,
+          "logprob": -19.390625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.359375,
+          "logprob": -17.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.75,
+          "logprob": -17.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.90625,
+          "logprob": -18.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.765625,
+          "logprob": -18.5625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.453125,
+          "logprob": -20.15625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.890625,
+          "logprob": -19.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.015625,
+          "logprob": -15.7734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.90625,
+          "logprob": -18.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.953125,
+          "logprob": -15.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.46875,
+          "logprob": -21.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.984375,
+          "logprob": -19.734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.875,
+          "logprob": -19.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.859375,
+          "logprob": -18.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.046875,
+          "logprob": -15.9609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.375,
+          "logprob": -20.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.140625,
+          "logprob": -19.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.140625,
+          "logprob": -20.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.703125,
+          "logprob": -21.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.453125,
+          "logprob": -21.171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -19.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.765625,
+          "logprob": -17.5625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.65625,
+          "logprob": -19.328125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.078125,
+          "logprob": -17.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.109375,
+          "logprob": -16.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -18.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.453125,
+          "logprob": -20.09375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.0625,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.734375,
+          "logprob": -16.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.21875,
+          "logprob": -18.90625,
           "text": "<image>"
         },
         {
@@ -4785,72 +4895,82 @@
         },
         {
           "id": 32001,
-          "logprob": -20.015625,
+          "logprob": -19.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.796875,
+          "logprob": -17.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.3125,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.390625,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.28125,
+          "logprob": -18.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.59375,
+          "logprob": -18.5625,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -3.0898438,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.09375,
+          "text": "<fake_token_around_image>"
+        },
         {
           "id": 32001,
-          "logprob": -18.8125,
+          "logprob": -25.5625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.09375,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.890625,
+          "logprob": -20.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.09375,
+          "logprob": -17.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.75,
+          "logprob": -18.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.921875,
+          "logprob": -18.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.375,
+          "logprob": -21.5,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.640625,
+          "logprob": -16.78125,
           "text": "<image>"
         },
         {
@@ -4860,22 +4980,22 @@
         },
         {
           "id": 32001,
-          "logprob": -21.40625,
+          "logprob": -21.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.875,
+          "logprob": -15.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.1171875,
+          "logprob": -15.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.078125,
+          "logprob": -17.15625,
           "text": "<image>"
         },
         {
@@ -4885,7 +5005,7 @@
         },
         {
           "id": 32001,
-          "logprob": -21.40625,
+          "logprob": -21.46875,
           "text": "<image>"
         },
         {
@@ -4895,57 +5015,57 @@
         },
         {
           "id": 32001,
-          "logprob": -20.75,
+          "logprob": -20.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.25,
+          "logprob": -16.296875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.453125,
+          "logprob": -19.484375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.59375,
+          "logprob": -21.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.421875,
+          "logprob": -22.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.9375,
+          "logprob": -20.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.671875,
+          "logprob": -19.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.890625,
+          "logprob": -21.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.921875,
+          "logprob": -16.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.5,
+          "logprob": -17.53125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.90625,
+          "logprob": -17.9375,
           "text": "<image>"
         },
         {
@@ -4960,37 +5080,37 @@
         },
         {
           "id": 32001,
-          "logprob": -16.40625,
+          "logprob": -16.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.453125,
+          "logprob": -18.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -20.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.28125,
+          "logprob": -22.296875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.53125,
+          "logprob": -18.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4296875,
+          "logprob": -15.3984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.765625,
+          "logprob": -19.734375,
           "text": "<image>"
         },
         {
@@ -5000,197 +5120,197 @@
         },
         {
           "id": 32001,
-          "logprob": -21.46875,
+          "logprob": -21.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.875,
+          "logprob": -18.84375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.859375,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.078125,
+          "logprob": -17.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.421875,
+          "logprob": -16.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.0,
+          "logprob": -21.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.234375,
+          "logprob": -21.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.140625,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.484375,
+          "logprob": -21.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.015625,
+          "logprob": -20.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.84375,
+          "logprob": -18.78125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.421875,
+          "logprob": -16.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.890625,
+          "logprob": -16.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.65625,
+          "logprob": -16.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -13.640625,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4140625,
+          "logprob": -15.3984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.546875,
+          "logprob": -17.5,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.859375,
+          "logprob": -21.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.65625,
+          "logprob": -15.6171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.484375,
+          "logprob": -16.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.359375,
+          "logprob": -16.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.953125,
+          "logprob": -19.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.875,
+          "logprob": -17.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.4375,
+          "logprob": -17.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.390625,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -19.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.9921875,
+          "logprob": -15.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -2.0429688,
+          "logprob": -2.1738281,
           "text": "<fake_token_around_image>"
         },
         {
           "id": 12018,
-          "logprob": -12.0390625,
+          "logprob": -11.859375,
           "text": "Write"
         },
         {
           "id": 528,
-          "logprob": -10.25,
+          "logprob": -10.0,
           "text": "me"
         },
         {
           "id": 264,
-          "logprob": -0.10443115,
+          "logprob": -0.09326172,
           "text": "a"
         },
         {
           "id": 2485,
-          "logprob": -4.5742188,
+          "logprob": -4.4648438,
           "text": "short"
         },
         {
           "id": 2838,
-          "logprob": -0.22729492,
+          "logprob": -0.2626953,
           "text": "story"
         },
         {
           "id": 32002,
-          "logprob": -10.84375,
+          "logprob": -11.0390625,
           "text": "<end_of_utterance>"
         },
         {
           "id": 259,
-          "logprob": -20.1875,
+          "logprob": -20.125,
           "text": " "
         },
         {
           "id": 13,
-          "logprob": -8.7578125,
+          "logprob": -8.046875,
           "text": "\n"
         },
         {
           "id": 7226,
-          "logprob": -10.4140625,
+          "logprob": -10.5,
           "text": "Ass"
         },
         {
           "id": 11143,
-          "logprob": -13.6328125,
+          "logprob": -13.4921875,
           "text": "istant"
         },
         {
           "id": 28747,
-          "logprob": -0.0056533813,
+          "logprob": -0.003282547,
           "text": ":"
         }
       ],
@@ -5198,61 +5318,61 @@
       "tokens": [
         {
           "id": 330,
-          "logprob": -0.12963867,
+          "logprob": -0.08648682,
           "special": false,
           "text": " A"
         },
         {
           "id": 13088,
-          "logprob": -0.6660156,
+          "logprob": -0.71191406,
           "special": false,
           "text": " chicken"
         },
         {
           "id": 349,
-          "logprob": -0.29516602,
+          "logprob": -0.33032227,
           "special": false,
           "text": " is"
         },
         {
           "id": 6398,
-          "logprob": -0.060028076,
+          "logprob": -0.051239014,
           "special": false,
           "text": " sitting"
         },
         {
           "id": 356,
-          "logprob": -0.27075195,
+          "logprob": -0.35473633,
           "special": false,
           "text": " on"
         },
         {
           "id": 264,
-          "logprob": -0.1427002,
+          "logprob": -0.1239624,
           "special": false,
           "text": " a"
         },
         {
           "id": 17972,
-          "logprob": -0.04067993,
+          "logprob": -0.038116455,
           "special": false,
           "text": " pile"
         },
         {
           "id": 302,
-          "logprob": -0.000269413,
+          "logprob": -0.00018763542,
           "special": false,
           "text": " of"
         },
         {
           "id": 2445,
-          "logprob": -0.09387207,
+          "logprob": -0.072265625,
           "special": false,
           "text": " money"
         },
         {
           "id": 28723,
-          "logprob": -0.0069236755,
+          "logprob": -0.0047912598,
           "special": false,
           "text": "."
         }
@@ -5274,12 +5394,12 @@
         },
         {
           "id": 1247,
-          "logprob": -5.2421875,
+          "logprob": -5.2382812,
           "text": "User"
         },
         {
           "id": 28747,
-          "logprob": -6.9570312,
+          "logprob": -6.9609375,
           "text": ":"
         },
         {
@@ -5319,7 +5439,7 @@
         },
         {
           "id": 32001,
-          "logprob": -20.25,
+          "logprob": -20.21875,
           "text": "<image>"
         },
         {
@@ -5359,12 +5479,12 @@
         },
         {
           "id": 32001,
-          "logprob": -21.296875,
+          "logprob": -21.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.015625,
+          "logprob": -21.03125,
           "text": "<image>"
         },
         {
@@ -5379,7 +5499,7 @@
         },
         {
           "id": 32001,
-          "logprob": -19.078125,
+          "logprob": -19.046875,
           "text": "<image>"
         },
         {
@@ -5389,22 +5509,22 @@
         },
         {
           "id": 32001,
-          "logprob": -23.625,
+          "logprob": -23.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.40625,
+          "logprob": -20.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.421875,
+          "logprob": -20.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.84375,
+          "logprob": -20.875,
           "text": "<image>"
         },
         {
@@ -5419,7 +5539,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.21875,
+          "logprob": -18.234375,
           "text": "<image>"
         },
         {
@@ -5429,7 +5549,7 @@
         },
         {
           "id": 32001,
-          "logprob": -21.203125,
+          "logprob": -21.1875,
           "text": "<image>"
         },
         {
@@ -5444,7 +5564,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.984375,
+          "logprob": -19.015625,
           "text": "<image>"
         },
         {
@@ -5459,17 +5579,17 @@
         },
         {
           "id": 32001,
-          "logprob": -18.0,
+          "logprob": -17.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.84375,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.9375,
+          "logprob": -17.921875,
           "text": "<image>"
         },
         {
@@ -5479,17 +5599,17 @@
         },
         {
           "id": 32001,
-          "logprob": -18.640625,
+          "logprob": -18.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.125,
+          "logprob": -20.140625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.4375,
+          "logprob": -19.421875,
           "text": "<image>"
         },
         {
@@ -5504,12 +5624,12 @@
         },
         {
           "id": 32001,
-          "logprob": -23.203125,
+          "logprob": -23.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.4375,
+          "logprob": -17.421875,
           "text": "<image>"
         },
         {
@@ -5519,7 +5639,7 @@
         },
         {
           "id": 32001,
-          "logprob": -23.015625,
+          "logprob": -23.0,
           "text": "<image>"
         },
         {
@@ -5544,7 +5664,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.40625,
+          "logprob": -18.421875,
           "text": "<image>"
         },
         {
@@ -5554,17 +5674,17 @@
         },
         {
           "id": 32001,
-          "logprob": -18.34375,
+          "logprob": -18.328125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.140625,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.671875,
+          "logprob": -18.65625,
           "text": "<image>"
         },
         {
@@ -5579,7 +5699,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.1875,
+          "logprob": -18.15625,
           "text": "<image>"
         },
         {
@@ -5594,52 +5714,62 @@
         },
         {
           "id": 32001,
-          "logprob": -18.96875,
+          "logprob": -18.953125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.71875,
+          "logprob": -18.703125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.9375,
+          "logprob": -17.921875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -3.015625,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.796875,
+          "text": "<fake_token_around_image>"
+        },
         {
           "id": 32001,
-          "logprob": -17.90625,
+          "logprob": -22.09375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.125,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.296875,
+          "logprob": -20.046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.25,
+          "logprob": -17.078125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.96875,
+          "logprob": -17.734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.546875,
+          "logprob": -19.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.921875,
+          "logprob": -19.5625,
           "text": "<image>"
         },
         {
@@ -5649,532 +5779,542 @@
         },
         {
           "id": 32001,
-          "logprob": -19.546875,
+          "logprob": -19.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.1875,
+          "logprob": -19.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.953125,
+          "logprob": -15.8046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.8359375,
+          "logprob": -16.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.375,
+          "logprob": -19.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.171875,
+          "logprob": -18.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.03125,
+          "logprob": -21.546875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.71875,
+          "logprob": -20.484375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.65625,
+          "logprob": -19.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.453125,
+          "logprob": -19.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.484375,
+          "logprob": -16.671875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.875,
+          "logprob": -20.390625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.65625,
+          "logprob": -21.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.296875,
+          "logprob": -20.359375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.546875,
+          "logprob": -20.703125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.78125,
+          "logprob": -17.71875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.96875,
+          "logprob": -20.28125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.5,
+          "logprob": -15.2421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4140625,
+          "logprob": -18.78125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.046875,
+          "logprob": -17.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.109375,
+          "logprob": -15.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.7265625,
+          "logprob": -18.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.5625,
+          "logprob": -20.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.734375,
+          "logprob": -20.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.359375,
+          "logprob": -18.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.421875,
+          "logprob": -17.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.984375,
+          "logprob": -17.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.265625,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.78125,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.046875,
+          "logprob": -18.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.4375,
+          "logprob": -19.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.78125,
+          "logprob": -19.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.671875,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -14.2421875,
+          "logprob": -18.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.015625,
+          "logprob": -18.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.40625,
+          "logprob": -17.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.59375,
+          "logprob": -20.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.671875,
+          "logprob": -19.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.265625,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.578125,
+          "logprob": -18.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.25,
+          "logprob": -17.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.46875,
+          "logprob": -16.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.078125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.84375,
+          "logprob": -15.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.296875,
+          "logprob": -18.15625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.8671875,
+          "logprob": -18.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.375,
+          "logprob": -18.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.765625,
+          "logprob": -17.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.609375,
+          "logprob": -16.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.515625,
+          "logprob": -19.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.25,
+          "logprob": -14.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.640625,
+          "logprob": -20.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -14.8515625,
+          "logprob": -20.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.875,
+          "logprob": -18.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.28125,
+          "logprob": -18.15625,
           "text": "<image>"
         },
         {
-          "id": 32001,
-          "logprob": -18.453125,
-          "text": "<image>"
+          "id": 32000,
+          "logprob": -3.3222656,
+          "text": "<fake_token_around_image>"
         },
         {
-          "id": 32001,
-          "logprob": -18.25,
-          "text": "<image>"
+          "id": 32000,
+          "logprob": -21.375,
+          "text": "<fake_token_around_image>"
         },
         {
           "id": 32001,
-          "logprob": -19.203125,
+          "logprob": -25.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.71875,
+          "logprob": -18.359375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.390625,
+          "logprob": -19.234375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.984375,
+          "logprob": -17.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.390625,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.5,
+          "logprob": -20.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.296875,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.4375,
+          "logprob": -16.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.015625,
+          "logprob": -19.640625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.359375,
+          "logprob": -18.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.375,
+          "logprob": -16.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.8125,
+          "logprob": -15.7421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.9375,
+          "logprob": -16.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.515625,
+          "logprob": -17.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.875,
+          "logprob": -16.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.09375,
+          "logprob": -18.78125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.265625,
+          "logprob": -18.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.234375,
+          "logprob": -16.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.28125,
+          "logprob": -18.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.25,
+          "logprob": -19.703125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.25,
+          "logprob": -19.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.765625,
+          "logprob": -20.390625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.609375,
+          "logprob": -19.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.359375,
+          "logprob": -20.046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.90625,
+          "logprob": -19.28125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.28125,
+          "logprob": -19.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.5,
+          "logprob": -19.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.0625,
+          "logprob": -19.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.125,
+          "logprob": -17.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.40625,
+          "logprob": -16.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.546875,
+          "logprob": -15.8828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.84375,
+          "logprob": -19.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.484375,
+          "logprob": -20.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.265625,
+          "logprob": -18.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.0625,
+          "logprob": -15.359375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -19.734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.9453125,
+          "logprob": -15.3984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.046875,
+          "logprob": -20.640625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.515625,
+          "logprob": -20.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.375,
+          "logprob": -19.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.796875,
+          "logprob": -17.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.03125,
+          "logprob": -15.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.671875,
+          "logprob": -18.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.15625,
+          "logprob": -19.671875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -19.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.84375,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.78125,
+          "logprob": -19.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.234375,
+          "logprob": -18.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.078125,
+          "logprob": -17.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.28125,
+          "logprob": -18.859375,
           "text": "<image>"
         },
         {
@@ -6184,102 +6324,112 @@
         },
         {
           "id": 32001,
-          "logprob": -16.59375,
+          "logprob": -16.171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.65625,
+          "logprob": -16.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.4375,
+          "logprob": -18.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.9375,
+          "logprob": -19.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.703125,
+          "logprob": -16.28125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.15625,
+          "logprob": -17.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.46875,
+          "logprob": -16.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.796875,
+          "logprob": -19.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.359375,
+          "logprob": -16.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.3125,
+          "logprob": -16.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.203125,
+          "logprob": -20.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.921875,
+          "logprob": -18.5,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.09375,
+          "logprob": -17.984375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -2.7539062,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.296875,
+          "text": "<fake_token_around_image>"
+        },
         {
           "id": 32001,
-          "logprob": -18.6875,
+          "logprob": -23.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.625,
+          "logprob": -22.078125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.375,
+          "logprob": -20.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.765625,
+          "logprob": -18.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.46875,
+          "logprob": -18.078125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.546875,
+          "logprob": -20.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.875,
+          "logprob": -18.40625,
           "text": "<image>"
         },
         {
@@ -6289,247 +6439,247 @@
         },
         {
           "id": 32001,
-          "logprob": -21.09375,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.5625,
+          "logprob": -19.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.15625,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.171875,
+          "logprob": -16.046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.671875,
+          "logprob": -17.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.859375,
+          "logprob": -18.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.734375,
+          "logprob": -20.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.8125,
+          "logprob": -21.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.96875,
+          "logprob": -19.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.046875,
+          "logprob": -18.671875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.78125,
+          "logprob": -19.640625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.421875,
+          "logprob": -19.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.21875,
+          "logprob": -20.921875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.515625,
+          "logprob": -21.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.625,
+          "logprob": -19.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.734375,
+          "logprob": -20.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.046875,
+          "logprob": -18.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.1875,
+          "logprob": -19.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.1875,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.828125,
+          "logprob": -19.390625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.359375,
+          "logprob": -17.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.75,
+          "logprob": -17.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.90625,
+          "logprob": -18.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.765625,
+          "logprob": -18.5625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.453125,
+          "logprob": -20.15625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.890625,
+          "logprob": -19.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.015625,
+          "logprob": -15.7734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.90625,
+          "logprob": -18.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.953125,
+          "logprob": -15.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.46875,
+          "logprob": -21.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.984375,
+          "logprob": -19.734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.875,
+          "logprob": -19.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.859375,
+          "logprob": -18.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.046875,
+          "logprob": -15.9609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.375,
+          "logprob": -20.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.140625,
+          "logprob": -19.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.140625,
+          "logprob": -20.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.6875,
+          "logprob": -21.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.453125,
+          "logprob": -21.171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.1875,
+          "logprob": -19.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.765625,
+          "logprob": -17.5625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.65625,
+          "logprob": -19.328125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.078125,
+          "logprob": -17.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.109375,
+          "logprob": -16.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -18.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.4375,
+          "logprob": -20.09375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.0625,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.734375,
+          "logprob": -16.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.21875,
+          "logprob": -18.90625,
           "text": "<image>"
         },
         {
@@ -6539,72 +6689,82 @@
         },
         {
           "id": 32001,
-          "logprob": -20.015625,
+          "logprob": -19.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.796875,
+          "logprob": -17.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.3125,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.390625,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.28125,
+          "logprob": -18.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.59375,
+          "logprob": -18.5625,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -3.0898438,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.09375,
+          "text": "<fake_token_around_image>"
+        },
         {
           "id": 32001,
-          "logprob": -18.8125,
+          "logprob": -25.5625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.09375,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.890625,
+          "logprob": -20.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.09375,
+          "logprob": -17.9375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.75,
+          "logprob": -18.796875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.90625,
+          "logprob": -18.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.375,
+          "logprob": -21.5,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.640625,
+          "logprob": -16.78125,
           "text": "<image>"
         },
         {
@@ -6614,22 +6774,22 @@
         },
         {
           "id": 32001,
-          "logprob": -21.40625,
+          "logprob": -21.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.875,
+          "logprob": -15.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.1171875,
+          "logprob": -15.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.078125,
+          "logprob": -17.15625,
           "text": "<image>"
         },
         {
@@ -6639,7 +6799,7 @@
         },
         {
           "id": 32001,
-          "logprob": -21.40625,
+          "logprob": -21.46875,
           "text": "<image>"
         },
         {
@@ -6649,57 +6809,57 @@
         },
         {
           "id": 32001,
-          "logprob": -20.75,
+          "logprob": -20.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.25,
+          "logprob": -16.296875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.453125,
+          "logprob": -19.484375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.59375,
+          "logprob": -21.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.421875,
+          "logprob": -22.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.9375,
+          "logprob": -20.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.671875,
+          "logprob": -19.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.890625,
+          "logprob": -21.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.921875,
+          "logprob": -16.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.5,
+          "logprob": -17.53125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.90625,
+          "logprob": -17.9375,
           "text": "<image>"
         },
         {
@@ -6709,42 +6869,42 @@
         },
         {
           "id": 32001,
-          "logprob": -18.734375,
+          "logprob": -18.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.40625,
+          "logprob": -16.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.453125,
+          "logprob": -18.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -20.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.28125,
+          "logprob": -22.296875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.515625,
+          "logprob": -18.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4296875,
+          "logprob": -15.3984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.765625,
+          "logprob": -19.734375,
           "text": "<image>"
         },
         {
@@ -6754,197 +6914,197 @@
         },
         {
           "id": 32001,
-          "logprob": -21.46875,
+          "logprob": -21.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.875,
+          "logprob": -18.84375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.859375,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.078125,
+          "logprob": -17.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.4375,
+          "logprob": -16.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.015625,
+          "logprob": -21.0625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.234375,
+          "logprob": -21.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.140625,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.484375,
+          "logprob": -21.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.015625,
+          "logprob": -20.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.84375,
+          "logprob": -18.78125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.421875,
+          "logprob": -16.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.890625,
+          "logprob": -16.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.65625,
+          "logprob": -16.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -13.640625,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4140625,
+          "logprob": -15.3984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.546875,
+          "logprob": -17.5,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.859375,
+          "logprob": -21.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.65625,
+          "logprob": -15.6171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.484375,
+          "logprob": -16.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.359375,
+          "logprob": -16.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.9375,
+          "logprob": -19.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.875,
+          "logprob": -17.90625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.453125,
+          "logprob": -17.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.390625,
+          "logprob": -20.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -19.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.9921875,
+          "logprob": -15.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -2.0429688,
+          "logprob": -2.1738281,
           "text": "<fake_token_around_image>"
         },
         {
           "id": 12018,
-          "logprob": -12.03125,
+          "logprob": -11.859375,
           "text": "Write"
         },
         {
           "id": 528,
-          "logprob": -10.25,
+          "logprob": -10.0,
           "text": "me"
         },
         {
           "id": 264,
-          "logprob": -0.10437012,
+          "logprob": -0.09326172,
           "text": "a"
         },
         {
           "id": 2485,
-          "logprob": -4.578125,
+          "logprob": -4.4648438,
           "text": "short"
         },
         {
           "id": 2838,
-          "logprob": -0.22924805,
+          "logprob": -0.2626953,
           "text": "story"
         },
         {
           "id": 32002,
-          "logprob": -10.84375,
+          "logprob": -11.0390625,
           "text": "<end_of_utterance>"
         },
         {
           "id": 259,
-          "logprob": -20.171875,
+          "logprob": -20.125,
           "text": " "
         },
         {
           "id": 13,
-          "logprob": -8.765625,
+          "logprob": -8.046875,
           "text": "\n"
         },
         {
           "id": 7226,
-          "logprob": -10.4140625,
+          "logprob": -10.5,
           "text": "Ass"
         },
         {
           "id": 11143,
-          "logprob": -13.640625,
+          "logprob": -13.4921875,
           "text": "istant"
         },
         {
           "id": 28747,
-          "logprob": -0.005744934,
+          "logprob": -0.003282547,
           "text": ":"
         }
       ],
@@ -6952,61 +7112,61 @@
       "tokens": [
         {
           "id": 330,
-          "logprob": -0.12976074,
+          "logprob": -0.08648682,
           "special": false,
           "text": " A"
         },
         {
           "id": 13088,
-          "logprob": -0.66308594,
+          "logprob": -0.71191406,
           "special": false,
           "text": " chicken"
         },
         {
           "id": 349,
-          "logprob": -0.29541016,
+          "logprob": -0.33032227,
           "special": false,
           "text": " is"
         },
         {
           "id": 6398,
-          "logprob": -0.05996704,
+          "logprob": -0.051239014,
           "special": false,
           "text": " sitting"
         },
         {
           "id": 356,
-          "logprob": -0.27075195,
+          "logprob": -0.35473633,
           "special": false,
           "text": " on"
         },
         {
           "id": 264,
-          "logprob": -0.14160156,
+          "logprob": -0.1239624,
           "special": false,
           "text": " a"
         },
         {
           "id": 17972,
-          "logprob": -0.040863037,
+          "logprob": -0.038116455,
           "special": false,
           "text": " pile"
         },
         {
           "id": 302,
-          "logprob": -0.00027036667,
+          "logprob": -0.000187397,
           "special": false,
           "text": " of"
         },
         {
           "id": 2445,
-          "logprob": -0.093322754,
+          "logprob": -0.072265625,
           "special": false,
           "text": " money"
         },
         {
           "id": 28723,
-          "logprob": -0.006931305,
+          "logprob": -0.004787445,
           "special": false,
           "text": "."
         }
diff --git a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_simple.json b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_simple.json
index a3b18d0adf2..da2ac897143 100644
--- a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_simple.json
+++ b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_simple.json
@@ -8,61 +8,61 @@
     "tokens": [
       {
         "id": 330,
-        "logprob": -0.13000488,
+        "logprob": -0.08660889,
         "special": false,
         "text": " A"
       },
       {
         "id": 13088,
-        "logprob": -0.6713867,
+        "logprob": -0.7089844,
         "special": false,
         "text": " chicken"
       },
       {
         "id": 349,
-        "logprob": -0.2980957,
+        "logprob": -0.32885742,
         "special": false,
         "text": " is"
       },
       {
         "id": 6398,
-        "logprob": -0.060638428,
+        "logprob": -0.05126953,
         "special": false,
         "text": " sitting"
       },
       {
         "id": 356,
-        "logprob": -0.27319336,
+        "logprob": -0.35229492,
         "special": false,
         "text": " on"
       },
       {
         "id": 264,
-        "logprob": -0.140625,
+        "logprob": -0.12561035,
         "special": false,
         "text": " a"
       },
       {
         "id": 17972,
-        "logprob": -0.040405273,
+        "logprob": -0.038085938,
         "special": false,
         "text": " pile"
       },
       {
         "id": 302,
-        "logprob": -0.0002708435,
+        "logprob": -0.00018656254,
         "special": false,
         "text": " of"
       },
       {
         "id": 2445,
-        "logprob": -0.095336914,
+        "logprob": -0.07293701,
         "special": false,
         "text": " money"
       },
       {
         "id": 28723,
-        "logprob": -0.0068359375,
+        "logprob": -0.004852295,
         "special": false,
         "text": "."
       }
diff --git a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_two_images.json b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_two_images.json
index 86c95b29525..bf2dc5a1ffe 100644
--- a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_two_images.json
+++ b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_two_images.json
@@ -8,115 +8,115 @@
     "tokens": [
       {
         "id": 415,
-        "logprob": -0.04421997,
+        "logprob": -0.039886475,
         "special": false,
         "text": " The"
       },
       {
         "id": 12072,
-        "logprob": -0.13500977,
+        "logprob": -0.1430664,
         "special": false,
         "text": " cow"
       },
       {
         "id": 349,
-        "logprob": -0.06750488,
+        "logprob": -0.056488037,
         "special": false,
         "text": " is"
       },
       {
         "id": 6328,
-        "logprob": -0.6352539,
+        "logprob": -0.6855469,
         "special": false,
         "text": " standing"
       },
       {
         "id": 356,
-        "logprob": -0.16186523,
+        "logprob": -0.1685791,
         "special": false,
         "text": " on"
       },
       {
         "id": 272,
-        "logprob": -0.5078125,
+        "logprob": -0.50097656,
         "special": false,
         "text": " the"
       },
       {
         "id": 10305,
-        "logprob": -0.017913818,
+        "logprob": -0.017303467,
         "special": false,
         "text": " beach"
       },
       {
         "id": 304,
-        "logprob": -1.5205078,
+        "logprob": -1.3564453,
         "special": false,
         "text": " and"
       },
       {
         "id": 272,
-        "logprob": -0.029174805,
+        "logprob": -0.017868042,
         "special": false,
         "text": " the"
       },
       {
         "id": 13088,
-        "logprob": -0.003479004,
+        "logprob": -0.0027103424,
         "special": false,
         "text": " chicken"
       },
       {
         "id": 349,
-        "logprob": -0.0035095215,
+        "logprob": -0.003156662,
         "special": false,
         "text": " is"
       },
       {
         "id": 6398,
-        "logprob": -0.3088379,
+        "logprob": -0.37304688,
         "special": false,
         "text": " sitting"
       },
       {
         "id": 356,
-        "logprob": -0.027755737,
+        "logprob": -0.034576416,
         "special": false,
         "text": " on"
       },
       {
         "id": 264,
-        "logprob": -0.31884766,
+        "logprob": -0.29418945,
         "special": false,
         "text": " a"
       },
       {
         "id": 17972,
-        "logprob": -0.047943115,
+        "logprob": -0.042877197,
         "special": false,
         "text": " pile"
       },
       {
         "id": 302,
-        "logprob": -0.0002925396,
+        "logprob": -0.00028443336,
         "special": false,
         "text": " of"
       },
       {
         "id": 2445,
-        "logprob": -0.02935791,
+        "logprob": -0.023223877,
         "special": false,
         "text": " money"
       },
       {
         "id": 28723,
-        "logprob": -0.031219482,
+        "logprob": -0.018157959,
         "special": false,
         "text": "."
       },
       {
         "id": 32002,
-        "logprob": -0.00034475327,
+        "logprob": -0.00018393993,
         "special": true,
         "text": "<end_of_utterance>"
       },
diff --git a/server/text_generation_server/models/pali_gemma.py b/server/text_generation_server/models/pali_gemma.py
index e883ce02e18..a167e4679a5 100644
--- a/server/text_generation_server/models/pali_gemma.py
+++ b/server/text_generation_server/models/pali_gemma.py
@@ -39,7 +39,9 @@ def batch_tokenized_inputs(
                     # TODO do_convert_RGB should be on by default ?
                     image = image.convert("RGB")
                     image_input = processor.image_processor(image, return_tensors="pt")
-                    full_text += image_text_replacement(image_input, config, image_id)
+                    full_text += image_text_replacement(
+                        processor, image_input, config, image_id
+                    )
                     image_inputs.append(image_input)
                 else:
                     raise RuntimeError(f"Invalid chunk type {chunk_type}")
diff --git a/server/text_generation_server/models/vlm_causal_lm.py b/server/text_generation_server/models/vlm_causal_lm.py
index 8b5819d17cd..a02a209108c 100644
--- a/server/text_generation_server/models/vlm_causal_lm.py
+++ b/server/text_generation_server/models/vlm_causal_lm.py
@@ -39,15 +39,14 @@ def get_anyres_image_grid_shape(image_size, grid_pinpoints, patch_size):
     return height // patch_size, width // patch_size
 
 
-def image_text_replacement(image_input, config, image_id) -> str:
+def image_text_replacement(processor, image_input, config, image_id) -> str:
     if config.model_type == "idefics2":
         # TODO technically depends on image splitting which is not implemented.
-        num_features = 320
-        return (
-            "<fake_token_around_image>"
-            + "<image>" * num_features
-            + "<fake_token_around_image>"
-        )
+        image_seq_len = 64
+        image_str = f"<fake_token_around_image>{'<image>' * image_seq_len}<fake_token_around_image>"
+        if processor.image_processor.do_image_splitting:
+            image_str *= 5
+        return image_str
     elif config.model_type == "llava_next":
         height, width = image_input["image_sizes"][image_id]
         num_features = get_number_of_features(height, width, config)
@@ -168,7 +167,9 @@ def batch_tokenized_inputs(
                 if chunk_type == "text":
                     full_text += chunk.text
                 elif chunk_type == "image":
-                    full_text += image_text_replacement(image_inputs, config, image_id)
+                    full_text += image_text_replacement(
+                        processor, image_inputs, config, image_id
+                    )
                     image_id += 1
 
             batch_inputs.append(full_text)