Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix LLaVA-NeXT handling of non-square images #2097

Closed
wants to merge 2 commits into from
Closed

Fix LLaVA-NeXT handling of non-square images #2097

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9ce4552
Idefics2: sync added image tokens with transformers
danieldk Jun 20, 2024
e7b1d5e
Fix LLaVA-NeXT handling of non-square images
danieldk Jun 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2,578 changes: 1,369 additions & 1,209 deletions integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_load.json
View file
Open in desktop

Large diffs are not rendered by default.

20 changes: 10 additions & 10 deletions integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_simple.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,61 +8,61 @@
"tokens": [
{
"id": 330,
"logprob": -0.13000488,
"logprob": -0.08660889,
"special": false,
"text": " A"
},
{
"id": 13088,
"logprob": -0.6713867,
"logprob": -0.7089844,
"special": false,
"text": " chicken"
},
{
"id": 349,
"logprob": -0.2980957,
"logprob": -0.32885742,
"special": false,
"text": " is"
},
{
"id": 6398,
"logprob": -0.060638428,
"logprob": -0.05126953,
"special": false,
"text": " sitting"
},
{
"id": 356,
"logprob": -0.27319336,
"logprob": -0.35229492,
"special": false,
"text": " on"
},
{
"id": 264,
"logprob": -0.140625,
"logprob": -0.12561035,
"special": false,
"text": " a"
},
{
"id": 17972,
"logprob": -0.040405273,
"logprob": -0.038085938,
"special": false,
"text": " pile"
},
{
"id": 302,
"logprob": -0.0002708435,
"logprob": -0.00018656254,
"special": false,
"text": " of"
},
{
"id": 2445,
"logprob": -0.095336914,
"logprob": -0.07293701,
"special": false,
"text": " money"
},
{
"id": 28723,
"logprob": -0.0068359375,
"logprob": -0.004852295,
"special": false,
"text": "."
}
Expand Down
38 changes: 19 additions & 19 deletions integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_two_images.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,115 +8,115 @@
"tokens": [
{
"id": 415,
"logprob": -0.04421997,
"logprob": -0.039886475,
"special": false,
"text": " The"
},
{
"id": 12072,
"logprob": -0.13500977,
"logprob": -0.1430664,
"special": false,
"text": " cow"
},
{
"id": 349,
"logprob": -0.06750488,
"logprob": -0.056488037,
"special": false,
"text": " is"
},
{
"id": 6328,
"logprob": -0.6352539,
"logprob": -0.6855469,
"special": false,
"text": " standing"
},
{
"id": 356,
"logprob": -0.16186523,
"logprob": -0.1685791,
"special": false,
"text": " on"
},
{
"id": 272,
"logprob": -0.5078125,
"logprob": -0.50097656,
"special": false,
"text": " the"
},
{
"id": 10305,
"logprob": -0.017913818,
"logprob": -0.017303467,
"special": false,
"text": " beach"
},
{
"id": 304,
"logprob": -1.5205078,
"logprob": -1.3564453,
"special": false,
"text": " and"
},
{
"id": 272,
"logprob": -0.029174805,
"logprob": -0.017868042,
"special": false,
"text": " the"
},
{
"id": 13088,
"logprob": -0.003479004,
"logprob": -0.0027103424,
"special": false,
"text": " chicken"
},
{
"id": 349,
"logprob": -0.0035095215,
"logprob": -0.003156662,
"special": false,
"text": " is"
},
{
"id": 6398,
"logprob": -0.3088379,
"logprob": -0.37304688,
"special": false,
"text": " sitting"
},
{
"id": 356,
"logprob": -0.027755737,
"logprob": -0.034576416,
"special": false,
"text": " on"
},
{
"id": 264,
"logprob": -0.31884766,
"logprob": -0.29418945,
"special": false,
"text": " a"
},
{
"id": 17972,
"logprob": -0.047943115,
"logprob": -0.042877197,
"special": false,
"text": " pile"
},
{
"id": 302,
"logprob": -0.0002925396,
"logprob": -0.00028443336,
"special": false,
"text": " of"
},
{
"id": 2445,
"logprob": -0.02935791,
"logprob": -0.023223877,
"special": false,
"text": " money"
},
{
"id": 28723,
"logprob": -0.031219482,
"logprob": -0.018157959,
"special": false,
"text": "."
},
{
"id": 32002,
"logprob": -0.00034475327,
"logprob": -0.00018393993,
"special": true,
"text": "<end_of_utterance>"
},
Expand Down
12 changes: 8 additions & 4 deletions router/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,12 @@ fn get_unpadded_features(
let current_aspect_ratio: f64 = current_width as f64 / current_height as f64;
let (current_height, current_width) = if aspect_ratio > current_aspect_ratio {
let new_height = (height * current_width) / width;
(new_height, current_width)
let padding = (current_height - new_height) / 2;
(current_height - (2 * padding), current_width)
} else {
let new_width = (width * current_height) / height;
(current_height, new_width)
let padding = (current_width - new_width) / 2;
(current_height, current_width - (2 * padding))
};

let unpadded_features = current_height * current_width;
Expand All @@ -88,7 +90,9 @@ impl LlavaNext {
let patch_size = self.vision_config.patch_size;
assert!(image_size % patch_size == 0);
let npatches = image_size / patch_size;
let (num_patch_height, num_patch_width) =
// Dimensions are intentionally swapped to be bug-compatible with
// upstream: https://github.com/LLaVA-VL/LLaVA-NeXT/issues/59
let (num_patch_width, num_patch_height) =
get_anyres_image_grid_shape(height, width, &self.image_grid_pinpoints, image_size);

let (unpadded_features, newline_features) =
Expand All @@ -112,7 +116,7 @@ pub struct Idefics2 {}

impl Idefics2 {
pub fn get_number_of_features(&self, _height: usize, _width: usize) -> usize {
320
64
}
}

Expand Down
19 changes: 19 additions & 0 deletions router/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,25 @@ impl HubTokenizerConfig {
}
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "processor_class")]
pub enum HubPreprocessorConfig {
Idefics2Processor(Idefics2Preprocessor),
}

impl HubPreprocessorConfig {
pub fn from_file<P: AsRef<std::path::Path>>(filename: P) -> Option<Self> {
let content = std::fs::read_to_string(filename).ok()?;
serde_json::from_str(&content).ok()
}
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Idefics2Preprocessor {
#[serde(default)]
do_image_splitting: bool,
}

#[derive(Debug, Clone, Deserialize, Default)]
pub struct HubProcessorConfig {
pub chat_template: Option<ChatTemplateVersions>,
Expand Down
12 changes: 11 additions & 1 deletion router/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ use std::io::BufReader;
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::path::{Path, PathBuf};
use text_generation_router::config::Config;
use text_generation_router::{server, HubModelInfo, HubProcessorConfig, HubTokenizerConfig};
use text_generation_router::{
server, HubModelInfo, HubPreprocessorConfig, HubProcessorConfig, HubTokenizerConfig,
};
use thiserror::Error;
use tokenizers::Tokenizer;
use tower_http::cors::AllowOrigin;
Expand Down Expand Up @@ -209,13 +211,15 @@ async fn main() -> Result<(), RouterError> {
tokenizer_filename,
config_filename,
tokenizer_config_filename,
preprocessor_config_filename,
processor_config_filename,
model_info,
) = match api {
Type::None => (
Some(local_path.join("tokenizer.json")),
Some(local_path.join("config.json")),
Some(local_path.join("tokenizer_config.json")),
Some(local_path.join("preprocessor_config.json")),
Some(local_path.join("processor_config.json")),
None,
),
Expand All @@ -232,6 +236,7 @@ async fn main() -> Result<(), RouterError> {
};
let config_filename = api_repo.get("config.json").await.ok();
let tokenizer_config_filename = api_repo.get("tokenizer_config.json").await.ok();
let preprocessor_config_filename = api_repo.get("preprocessor_config.json").await.ok();
let processor_config_filename = api_repo.get("processor_config.json").await.ok();

let model_info = if let Some(model_info) = get_model_info(&api_repo).await {
Expand All @@ -244,6 +249,7 @@ async fn main() -> Result<(), RouterError> {
tokenizer_filename,
config_filename,
tokenizer_config_filename,
preprocessor_config_filename,
processor_config_filename,
model_info,
)
Expand All @@ -258,6 +264,7 @@ async fn main() -> Result<(), RouterError> {
repo.get("tokenizer.json"),
repo.get("config.json"),
repo.get("tokenizer_config.json"),
repo.get("preprocessor_config.json"),
repo.get("processor_config.json"),
None,
)
Expand Down Expand Up @@ -295,6 +302,8 @@ async fn main() -> Result<(), RouterError> {
HubTokenizerConfig::default()
});

let preprocessor_config =
preprocessor_config_filename.and_then(HubPreprocessorConfig::from_file);
let processor_config = processor_config_filename
.and_then(HubProcessorConfig::from_file)
.unwrap_or_default();
Expand Down Expand Up @@ -356,6 +365,7 @@ async fn main() -> Result<(), RouterError> {
ngrok_authtoken,
ngrok_edge,
tokenizer_config,
preprocessor_config,
processor_config,
messages_api_enabled,
disable_grammar_support,
Expand Down
8 changes: 5 additions & 3 deletions router/src/server.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ use crate::kserve::{
use crate::validation::ValidationError;
use crate::{
BestOfSequence, Details, ErrorResponse, FinishReason, GenerateParameters, GenerateRequest,
GenerateResponse, GrammarType, HubModelInfo, HubProcessorConfig, HubTokenizerConfig, Info,
Message, PrefillToken, SimpleToken, StreamDetails, StreamResponse, Token, TokenizeResponse,
Usage, Validation,
GenerateResponse, GrammarType, HubModelInfo, HubPreprocessorConfig, HubProcessorConfig,
HubTokenizerConfig, Info, Message, PrefillToken, SimpleToken, StreamDetails, StreamResponse,
Token, TokenizeResponse, Usage, Validation,
};
use crate::{
ChatCompletion, ChatCompletionChoice, ChatCompletionChunk, ChatCompletionComplete,
Expand Down Expand Up @@ -1421,6 +1421,7 @@ pub async fn run(
_ngrok_authtoken: Option<String>,
_ngrok_edge: Option<String>,
tokenizer_config: HubTokenizerConfig,
preprocessor_config: Option<HubPreprocessorConfig>,
processor_config: HubProcessorConfig,
messages_api_enabled: bool,
grammar_support: bool,
Expand Down Expand Up @@ -1634,6 +1635,7 @@ pub async fn run(
validation_workers,
tokenizer,
config,
preprocessor_config,
max_best_of,
max_stop_sequences,
max_top_n_tokens,
Expand Down
Loading
Loading