From d86f1fd2c3692f5fe015dd65b99f1ceb35ac28ca Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 9 Nov 2023 06:37:32 -0800 Subject: [PATCH 1/2] OpenAI API: stop streaming on client disconnect (closes #4521) --- extensions/openai/script.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/extensions/openai/script.py b/extensions/openai/script.py index 94e4160e18..40574a72af 100644 --- a/extensions/openai/script.py +++ b/extensions/openai/script.py @@ -91,6 +91,10 @@ async def generator(): async with streaming_semaphore: response = OAIcompletions.stream_completions(to_dict(request_data), is_legacy=is_legacy) for resp in response: + disconnected = await request.is_disconnected() + if disconnected: + break + yield {"data": json.dumps(resp)} return EventSourceResponse(generator()) # SSE streaming @@ -110,6 +114,10 @@ async def generator(): async with streaming_semaphore: response = OAIcompletions.stream_chat_completions(to_dict(request_data), is_legacy=is_legacy) for resp in response: + disconnected = await request.is_disconnected() + if disconnected: + break + yield {"data": json.dumps(resp)} return EventSourceResponse(generator()) # SSE streaming From 372d712921c6f9fa334c9aa38b88a001afeae6af Mon Sep 17 00:00:00 2001 From: Ashley Kleynhans Date: Thu, 9 Nov 2023 19:51:50 +0200 Subject: [PATCH 2/2] Fix deprecated API (#4539) --- extensions/api/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/api/util.py b/extensions/api/util.py index 206f259771..2c7e73fc5d 100644 --- a/extensions/api/util.py +++ b/extensions/api/util.py @@ -74,8 +74,8 @@ def build_parameters(body, chat=False): if str(character) == "None": character = "Assistant" - name1, name2, _, greeting, context, _ = load_character_memoized(character, str(body.get('your_name', shared.settings['name1'])), '', instruct=False) - name1_instruct, name2_instruct, _, _, context_instruct, turn_template = load_character_memoized(instruction_template, '', '', instruct=True) + name1, name2, _, greeting, context, _, _ = load_character_memoized(character, str(body.get('your_name', shared.settings['name1'])), '', instruct=False) + name1_instruct, name2_instruct, _, _, context_instruct, turn_template, _ = load_character_memoized(instruction_template, '', '', instruct=True) generate_params.update({ 'mode': str(body.get('mode', 'chat')), 'name1': str(body.get('name1', name1)),