diff --git a/apps/desktop/dist/.keep b/apps/desktop/dist/.keep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/server/bleep/src/llm/call.rs b/server/bleep/src/llm/call.rs index 56aa658493..a1f053b961 100644 --- a/server/bleep/src/llm/call.rs +++ b/server/bleep/src/llm/call.rs @@ -80,10 +80,16 @@ struct OpenAiRequest { pub async fn llm_call( req: api::LLMRequest, ) -> anyhow::Result>> { + // print req.messages.messages + for message in &req.messages.messages { + println!("OpenAiMessage: {:?}", message); + } + let model = match req.model.as_deref() { Some(model) => model.to_owned(), - None => "gpt-4-turbo".into(), + None => "gpt-4o".into(), }; + //filter out Null values from messages let builder = { let request = OpenAiRequest { @@ -104,7 +110,7 @@ pub async fn llm_call( reqwest::Client::new() .post("https://api.openai.com/v1/chat/completions") - .bearer_auth(req.openai_key) + .bearer_auth(req.openai_key.clone()) .json(&request) }; @@ -116,6 +122,17 @@ pub async fn llm_call( Some(Ok(reqwest_eventsource::Event::Open)) => {} Some(Err(reqwest_eventsource::Error::InvalidStatusCode(status, _))) => { error!("{}", &status); + if status == 429 { + response.close(); + println!("Rate limit exceeded, try again after 5s"); + tokio::time::sleep(Duration::from_secs(5)).await; + let openai_key = req.openai_key.clone(); + return Box::pin(llm_call(api::LLMRequest { + openai_key, + ..req.clone() + })).await; + } + return Err(api::Error::BadOpenAiRequest.into()); } Some(Err(e)) => { @@ -163,14 +180,27 @@ pub async fn llm_call( Some(ChatChoice { ref mut delta, .. }) => { // The first message contains a redundant `role` field. We remove it. delta.remove("role"); - if delta.is_empty() { - return Ok(None); - } if delta.len() == 2 { delta.remove("content"); } + if delta.contains_key("content") { + if delta.get_key_value("content").unwrap().1.is_null() { + delta.remove("content"); + } + } + + if delta.contains_key("refusal") { + if delta.get_key_value("refusal").unwrap().1.is_null() { + delta.remove("refusal"); + } + } + + if delta.is_empty() { + return Ok(None); + } + let delta = serde_json::from_value(delta.clone().into()).map_err(|e| { error!(?delta, "{}", e); api::Error::BadOpenAiRequest diff --git a/server/bleep/src/llm/client.rs b/server/bleep/src/llm/client.rs index 3c4449e9c3..ed487c8e28 100644 --- a/server/bleep/src/llm/client.rs +++ b/server/bleep/src/llm/client.rs @@ -73,7 +73,7 @@ pub mod api { pub functions: Vec, } - #[derive(Debug, serde::Serialize, serde::Deserialize)] + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct LLMRequest { pub openai_key: String, pub messages: Messages,