From 3c0b51c5ba6c42080b358ec1ef0590eb9c417c83 Mon Sep 17 00:00:00 2001 From: Gautam <102940730+GautamR-Samagra@users.noreply.github.com> Date: Fri, 3 Nov 2023 09:40:15 +0530 Subject: [PATCH 01/19] Update config.json --- config.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/config.json b/config.json index 2312db3..4e6162a 100644 --- a/config.json +++ b/config.json @@ -1,5 +1,14 @@ { "models": [ + { + "serviceName": "convo_starter", + "modelBasePath": "src/text_classification/convo_starter_orgbot/local/.", + "apiBasePath": "text_classification/convo_starter_orgbot/local/", + "containerPort": 8000, + "environment": { }, + "nginx": [], + "build": true + }, { "serviceName": "search_tf", "modelBasePath": "src/search/tf_search/local/.", From 737b64bdc81970dc7a08afc2ccd5ec8b61b45c2b Mon Sep 17 00:00:00 2001 From: Gautam <102940730+GautamR-Samagra@users.noreply.github.com> Date: Mon, 6 Nov 2023 12:17:07 +0530 Subject: [PATCH 02/19] Update config.json --- config.json | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/config.json b/config.json index 4e6162a..823e73e 100644 --- a/config.json +++ b/config.json @@ -7,7 +7,7 @@ "containerPort": 8000, "environment": { }, "nginx": [], - "build": true + "build": false }, { "serviceName": "search_tf", @@ -16,13 +16,12 @@ "containerPort": 8000, "environment": { }, "nginx": ["client_max_body_size 100M;", "proxy_read_timeout 600;", "proxy_connect_timeout 600;", "proxy_send_timeout 600;"], - "build": true + "build": false }, - { - "serviceName": "embedding_instructor_gpu", - "modelBasePath": "src/embeddings/instructor_gpu/local/.", - "apiBasePath": "embeddings/instructor_gpu/local/", + "serviceName": "embedding_instructor", + "modelBasePath": "src/embeddings/instructor/local/.", + "apiBasePath": "/embeddings/instructor/local", "containerPort": 8000, "environment": { "OPENAI_API_KEY": "${OPENAI_API_KEY}", @@ -30,7 +29,7 @@ }, "nginx": ["client_max_body_size 100M;", "proxy_read_timeout 600;", "proxy_connect_timeout 600;", "proxy_send_timeout 600;"], "constraints": ["node.labels.node_vm_type==gpu"], - "build": false + "build": true }, { "serviceName": "asr_whisper_en", @@ -186,18 +185,6 @@ "constraints": ["node.labels.node_vm_type==gpu"], "build": false }, - { - "serviceName": "embedding_instructor", - "modelBasePath": "src/embeddings/instructor/local/.", - "apiBasePath": "/embeddings/instructor/local", - "containerPort": 8000, - "environment": { - "OPENAI_API_KEY": "${OPENAI_API_KEY}" - }, - "nginx": ["client_max_body_size 100M;", "proxy_read_timeout 600;", "proxy_connect_timeout 600;", "proxy_send_timeout 600;"], - "constraints": ["node.labels.node_vm_type==gpu"], - "build": false - }, { "serviceName": "llm_openai_gpt3", "modelBasePath": "src/llm/openai/chatgpt3/.", @@ -208,7 +195,7 @@ }, "nginx": [], "constraints": ["node.labels.node_vm_type==gpu"], - "build": true + "build": false } ] } From 4facfdc3a096d2a8291536bf4a30d1b52fa3ccbd Mon Sep 17 00:00:00 2001 From: Gautam <102940730+GautamR-Samagra@users.noreply.github.com> Date: Mon, 6 Nov 2023 12:53:25 +0530 Subject: [PATCH 03/19] Update config.json --- config.json | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/config.json b/config.json index 823e73e..10b1e32 100644 --- a/config.json +++ b/config.json @@ -80,16 +80,6 @@ "nginx": [], "build": false }, - { - "serviceName": "flow_classification", - "modelBasePath": "src/text_classification/flow_classification/local/.", - "apiBasePath": "text_classification/flow_classification/local/", - "containerPort": 8000, - "environment": {}, - "nginx": [], - "constraints": ["node.role==worker"], - "build": false - }, { "serviceName": "text_translation_azure_dict", "modelBasePath": "src/text_translation/azure_dict/remote/.", From c27be7240189defa7f9ce59119c5420e917b1c28 Mon Sep 17 00:00:00 2001 From: Amruth-Vamshi <130033232+Amruth-Vamshi@users.noreply.github.com> Date: Tue, 7 Nov 2023 10:59:27 +0530 Subject: [PATCH 04/19] Update config.json --- config.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/config.json b/config.json index 10b1e32..300be57 100644 --- a/config.json +++ b/config.json @@ -186,6 +186,17 @@ "nginx": [], "constraints": ["node.labels.node_vm_type==gpu"], "build": false + }, + { + "serviceName": "llm_openai_gpt4", + "modelBasePath": "src/llm/openai/chatgpt4/.", + "apiBasePath": "/llm/openai/chatgpt4", + "containerPort": 8000, + "environment": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}" + }, + "nginx": [], + "build": false } ] } From b8db57a2e6ab8dfd49269919d2789350c97f1789 Mon Sep 17 00:00:00 2001 From: Amruth-Vamshi <130033232+Amruth-Vamshi@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:00:15 +0530 Subject: [PATCH 05/19] Update config.json --- config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json b/config.json index 300be57..08980f0 100644 --- a/config.json +++ b/config.json @@ -196,7 +196,7 @@ "OPENAI_API_KEY": "${OPENAI_API_KEY}" }, "nginx": [], - "build": false + "build": true } ] } From 502b2ffa1ded588b4bdf5f3966b12f5f24995612 Mon Sep 17 00:00:00 2001 From: Amruth-Vamshi <130033232+Amruth-Vamshi@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:04:27 +0530 Subject: [PATCH 06/19] update GPT4 model --- config.json | 2 +- src/llm/openai/chatgpt4/model.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json b/config.json index 08980f0..3fae0c1 100644 --- a/config.json +++ b/config.json @@ -29,7 +29,7 @@ }, "nginx": ["client_max_body_size 100M;", "proxy_read_timeout 600;", "proxy_connect_timeout 600;", "proxy_send_timeout 600;"], "constraints": ["node.labels.node_vm_type==gpu"], - "build": true + "build": false }, { "serviceName": "asr_whisper_en", diff --git a/src/llm/openai/chatgpt4/model.py b/src/llm/openai/chatgpt4/model.py index 90b0e9e..2f19bff 100644 --- a/src/llm/openai/chatgpt4/model.py +++ b/src/llm/openai/chatgpt4/model.py @@ -22,7 +22,7 @@ async def inference(self, request: ModelRequest): openai.api_key, timeout=20000, payload={ - "model": "gpt-4", + "model": "gpt-4-1106-preview", "temperature": 0, "messages": [{"role":"user","content" : request.prompt}], }, From 26a2de6074e92f940cd6ded25cbc743ec51ff459 Mon Sep 17 00:00:00 2001 From: Amruth-Vamshi <130033232+Amruth-Vamshi@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:13:36 +0530 Subject: [PATCH 07/19] Update requirements.txt --- src/llm/openai/chatgpt4/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llm/openai/chatgpt4/requirements.txt b/src/llm/openai/chatgpt4/requirements.txt index 753c69a..9012b18 100644 --- a/src/llm/openai/chatgpt4/requirements.txt +++ b/src/llm/openai/chatgpt4/requirements.txt @@ -1,7 +1,7 @@ aiohttp==3.8.4 -quart==0.18.3 +quart async-cache==1.1.1 requests openai openai_async -tenacity \ No newline at end of file +tenacity From 167e0c6a8fe906bf4fb27b7dc8c2ac399a262db6 Mon Sep 17 00:00:00 2001 From: Amruth-Vamshi <130033232+Amruth-Vamshi@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:19:18 +0530 Subject: [PATCH 08/19] updated GPT4 model --- src/llm/openai/chatgpt4/model.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/llm/openai/chatgpt4/model.py b/src/llm/openai/chatgpt4/model.py index 2f19bff..6afad88 100644 --- a/src/llm/openai/chatgpt4/model.py +++ b/src/llm/openai/chatgpt4/model.py @@ -15,7 +15,6 @@ def __new__(cls, context): cls.instance = super(Model, cls).__new__(cls) return cls.instance - @AsyncTTL(time_to_live=600000, maxsize=1024) @retry(wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(6)) async def inference(self, request: ModelRequest): response = await openai_async.chat_complete( @@ -24,11 +23,7 @@ async def inference(self, request: ModelRequest): payload={ "model": "gpt-4-1106-preview", "temperature": 0, - "messages": [{"role":"user","content" : request.prompt}], + "messages": request.prompt, }, ) - try: - ans = response.json()["choices"][0]["message"]["content"] - return {"ans":ans} - except: - return response.json() + return response.json() From 4684dd8a87861ba5c5fc2aa4abef619228d91291 Mon Sep 17 00:00:00 2001 From: Amruth-Vamshi <130033232+Amruth-Vamshi@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:47:37 +0530 Subject: [PATCH 09/19] updated openAI embeddings. --- src/embeddings/openai/remote/model.py | 53 +++++++------------------ src/embeddings/openai/remote/request.py | 11 +++-- 2 files changed, 21 insertions(+), 43 deletions(-) diff --git a/src/embeddings/openai/remote/model.py b/src/embeddings/openai/remote/model.py index 3d6b13f..dafcdee 100644 --- a/src/embeddings/openai/remote/model.py +++ b/src/embeddings/openai/remote/model.py @@ -1,52 +1,27 @@ -import os -import openai -from openai.embeddings_utils import get_embedding -from cache import AsyncTTL -from request import ModelRequest -import numpy as np -import pandas as pd -import tiktoken -import ast -from sklearn.metrics.pairwise import cosine_similarity - -openai.api_key = os.getenv("OPENAI_API_KEY") +from openai import OpenAI class Model: - embedding_df = None embedding_model = "text-embedding-ada-002" - embedding_encoding = "cl100k_base" # this the encoding for text-embedding-ada-002 - max_tokens = 8000 # the maximum for text-embedding-ada-002 is 8191 def __new__(cls, context): cls.context = context if not hasattr(cls, 'instance'): - cls.embedding_df = pd.read_csv('src/embeddings/openai/remote/akai.csv') - cls.embedding_df['embedding'] = cls.embedding_df['embedding'].apply(ast.literal_eval) + cls.instance = client = OpenAI( + api_key=os.getenv("OPENAI_API_KEY"), + ) cls.instance = super(Model, cls).__new__(cls) return cls.instance - @AsyncTTL(time_to_live=600000, maxsize=1024) async def inference(self, request: ModelRequest): - print("request.prompt", request.prompt) - new_prompt_embedding = get_embedding(request.prompt, engine=self.embedding_model) - similarity_scores = cosine_similarity( - [new_prompt_embedding], np.stack(self.embedding_df['embedding'], axis=0))[0] - most_similar_indices = np.argsort(similarity_scores)[::-1] - most_similar_prompts = self.embedding_df.loc[most_similar_indices, ['combined_prompt', 'combined_content']] - most_similar_prompts['similarity_score'] = np.sort(similarity_scores)[::-1] - similar_content = most_similar_prompts.iloc[0:20] - sim_cutoff_range = np.max(similar_content['similarity_score']) - request.similarity_score_range - similar_content_df = similar_content.loc[similar_content['similarity_score'] >= sim_cutoff_range, :] - similar_content_df1 = similar_content_df.drop(columns='similarity_score') - similar_content_dict = similar_content_df1.to_dict('records') - # modified_content_dict = remove_content_tags_from_dic(similar_content_dict) - print("similar_content_dict", similar_content_dict) - return (similar_content_dict) + # Modify this function according to model requirements such that inputs and output remains the same + query = request.query - async def create_embeddings(self, embedding_df): - encoding = tiktoken.get_encoding(self.embedding_encoding) - embedding_df["n_tokens"] = embedding_df.combined_prompt.apply(lambda x: len(encoding.encode(x))) - embedding_df["embedding"] = embedding_df.combined_prompt.apply( - lambda x: get_embedding(x, engine=self.embedding_model)) - return embedding_df \ No newline at end of file + if(query != None): + embedding = client.embeddings.create( + input=query, + model=self.embedding_model, + ).data[0].embedding + return embedding + + return "Invalid input" diff --git a/src/embeddings/openai/remote/request.py b/src/embeddings/openai/remote/request.py index f05ba1a..23dc54e 100644 --- a/src/embeddings/openai/remote/request.py +++ b/src/embeddings/openai/remote/request.py @@ -1,11 +1,14 @@ import json +import pandas as pd class ModelRequest(): - def __init__(self, prompt, similarity_score_range=0): - self.prompt = prompt - self.similarity_score_range = similarity_score_range + def __init__(self, query=None, df = pd.DataFrame(), query_type = None): + # Url to download csv file + self.query = query # String + self.query_type = query_type + self.df = df def to_json(self): return json.dumps(self, default=lambda o: o.__dict__, - sort_keys=True, indent=4) + sort_keys=True, indent=4) \ No newline at end of file From af3da2f0c0b8cd1e8c993b010e18d03688fa1086 Mon Sep 17 00:00:00 2001 From: Amruth-Vamshi <130033232+Amruth-Vamshi@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:50:22 +0530 Subject: [PATCH 10/19] Update config.json --- config.json | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/config.json b/config.json index 3fae0c1..a58e74b 100644 --- a/config.json +++ b/config.json @@ -31,6 +31,17 @@ "constraints": ["node.labels.node_vm_type==gpu"], "build": false }, + { + "serviceName": "embedding_openai", + "modelBasePath": "src/embeddings/openai/remote/.", + "apiBasePath": "/embeddings/openai/remote", + "containerPort": 8000, + "environment": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}" + }, + "nginx": ["client_max_body_size 100M;", "proxy_read_timeout 600;", "proxy_connect_timeout 600;", "proxy_send_timeout 600;"], + "build": true + }, { "serviceName": "asr_whisper_en", "modelBasePath": "src/asr/whisper_en/local/.", @@ -196,7 +207,7 @@ "OPENAI_API_KEY": "${OPENAI_API_KEY}" }, "nginx": [], - "build": true + "build": false } ] } From b6e93bcdea2f58823ccc021309dd3414dbfb10a5 Mon Sep 17 00:00:00 2001 From: Amruth-Vamshi <130033232+Amruth-Vamshi@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:57:59 +0530 Subject: [PATCH 11/19] Fixed import --- src/embeddings/openai/remote/model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/embeddings/openai/remote/model.py b/src/embeddings/openai/remote/model.py index dafcdee..02f8d25 100644 --- a/src/embeddings/openai/remote/model.py +++ b/src/embeddings/openai/remote/model.py @@ -1,3 +1,4 @@ +from request import ModelRequest from openai import OpenAI From 19a134080171fa6d97f0a3e4bbc45e353a0b9004 Mon Sep 17 00:00:00 2001 From: Amruth-Vamshi <130033232+Amruth-Vamshi@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:03:34 +0530 Subject: [PATCH 12/19] Update requirements.txt --- src/embeddings/openai/remote/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/embeddings/openai/remote/requirements.txt b/src/embeddings/openai/remote/requirements.txt index 6b7961f..81dce00 100644 --- a/src/embeddings/openai/remote/requirements.txt +++ b/src/embeddings/openai/remote/requirements.txt @@ -1,9 +1,9 @@ aiohttp==3.8.4 -quart==0.18.3 +quart async-cache==1.1.1 requests openai numpy pandas tiktoken -sklearn \ No newline at end of file +sklearn From a7710f4f153490ccc811df61e5234531a3629b79 Mon Sep 17 00:00:00 2001 From: Amruth-Vamshi <130033232+Amruth-Vamshi@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:06:13 +0530 Subject: [PATCH 13/19] Update requirements.txt --- src/embeddings/openai/remote/requirements.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/embeddings/openai/remote/requirements.txt b/src/embeddings/openai/remote/requirements.txt index 81dce00..e8dc93a 100644 --- a/src/embeddings/openai/remote/requirements.txt +++ b/src/embeddings/openai/remote/requirements.txt @@ -3,7 +3,3 @@ quart async-cache==1.1.1 requests openai -numpy -pandas -tiktoken -sklearn From 8c58443e2e1dd67650755b374af99e029ffc8143 Mon Sep 17 00:00:00 2001 From: Amruth-Vamshi <130033232+Amruth-Vamshi@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:08:37 +0530 Subject: [PATCH 14/19] Update model.py --- src/embeddings/openai/remote/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/embeddings/openai/remote/model.py b/src/embeddings/openai/remote/model.py index 02f8d25..2e4785e 100644 --- a/src/embeddings/openai/remote/model.py +++ b/src/embeddings/openai/remote/model.py @@ -23,6 +23,6 @@ async def inference(self, request: ModelRequest): input=query, model=self.embedding_model, ).data[0].embedding - return embedding + return [embedding] return "Invalid input" From d4c83188551ae0a8a2840bf1ece43d48232652d3 Mon Sep 17 00:00:00 2001 From: Amruth-Vamshi <130033232+Amruth-Vamshi@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:18:29 +0530 Subject: [PATCH 15/19] Update request.py --- src/embeddings/openai/remote/request.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/embeddings/openai/remote/request.py b/src/embeddings/openai/remote/request.py index 23dc54e..346b5b5 100644 --- a/src/embeddings/openai/remote/request.py +++ b/src/embeddings/openai/remote/request.py @@ -1,6 +1,4 @@ import json -import pandas as pd - class ModelRequest(): def __init__(self, query=None, df = pd.DataFrame(), query_type = None): @@ -11,4 +9,4 @@ def __init__(self, query=None, df = pd.DataFrame(), query_type = None): def to_json(self): return json.dumps(self, default=lambda o: o.__dict__, - sort_keys=True, indent=4) \ No newline at end of file + sort_keys=True, indent=4) From 155f6b0f37d384b9ec42f221dc5803fbabbbac1c Mon Sep 17 00:00:00 2001 From: Amruth-Vamshi <130033232+Amruth-Vamshi@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:23:56 +0530 Subject: [PATCH 16/19] Update request.py --- src/embeddings/openai/remote/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/embeddings/openai/remote/request.py b/src/embeddings/openai/remote/request.py index 346b5b5..7807ffd 100644 --- a/src/embeddings/openai/remote/request.py +++ b/src/embeddings/openai/remote/request.py @@ -1,7 +1,7 @@ import json class ModelRequest(): - def __init__(self, query=None, df = pd.DataFrame(), query_type = None): + def __init__(self, query=None, df = None, query_type = None): # Url to download csv file self.query = query # String self.query_type = query_type From 19150c72808ee5285c814994a5f4cdf72f96dad9 Mon Sep 17 00:00:00 2001 From: Amruth-Vamshi <130033232+Amruth-Vamshi@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:30:25 +0530 Subject: [PATCH 17/19] Update model.py --- src/embeddings/openai/remote/model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/embeddings/openai/remote/model.py b/src/embeddings/openai/remote/model.py index 2e4785e..b6ea8ba 100644 --- a/src/embeddings/openai/remote/model.py +++ b/src/embeddings/openai/remote/model.py @@ -1,6 +1,6 @@ from request import ModelRequest from openai import OpenAI - +import os class Model: embedding_model = "text-embedding-ada-002" @@ -8,7 +8,7 @@ class Model: def __new__(cls, context): cls.context = context if not hasattr(cls, 'instance'): - cls.instance = client = OpenAI( + cls.client = OpenAI( api_key=os.getenv("OPENAI_API_KEY"), ) cls.instance = super(Model, cls).__new__(cls) @@ -19,7 +19,7 @@ async def inference(self, request: ModelRequest): query = request.query if(query != None): - embedding = client.embeddings.create( + embedding = self.client.embeddings.create( input=query, model=self.embedding_model, ).data[0].embedding From 96d60d38aa843926264ad8059a630d726cc118fb Mon Sep 17 00:00:00 2001 From: Gautam <102940730+GautamR-Samagra@users.noreply.github.com> Date: Tue, 7 Nov 2023 17:17:03 +0530 Subject: [PATCH 18/19] Update model.py --- src/llm/openai/chatgpt4/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llm/openai/chatgpt4/model.py b/src/llm/openai/chatgpt4/model.py index 6afad88..55f93bc 100644 --- a/src/llm/openai/chatgpt4/model.py +++ b/src/llm/openai/chatgpt4/model.py @@ -21,7 +21,7 @@ async def inference(self, request: ModelRequest): openai.api_key, timeout=20000, payload={ - "model": "gpt-4-1106-preview", + "model": "gpt-4", "temperature": 0, "messages": request.prompt, }, From dc38d3a07015ba8ca3686c8bfe8ed0253873c122 Mon Sep 17 00:00:00 2001 From: Gautam <102940730+GautamR-Samagra@users.noreply.github.com> Date: Tue, 7 Nov 2023 17:24:57 +0530 Subject: [PATCH 19/19] Update config.json --- config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json b/config.json index a58e74b..c690c42 100644 --- a/config.json +++ b/config.json @@ -40,7 +40,7 @@ "OPENAI_API_KEY": "${OPENAI_API_KEY}" }, "nginx": ["client_max_body_size 100M;", "proxy_read_timeout 600;", "proxy_connect_timeout 600;", "proxy_send_timeout 600;"], - "build": true + "build": false }, { "serviceName": "asr_whisper_en", @@ -207,7 +207,7 @@ "OPENAI_API_KEY": "${OPENAI_API_KEY}" }, "nginx": [], - "build": false + "build": true } ] }