Skip to content

Commit

Permalink
Merge pull request #233 from nulib/deploy/staging
Browse files Browse the repository at this point in the history
Deploy to production
  • Loading branch information
bmquinn authored Jul 22, 2024
2 parents 5c6322c + 6c84bdc commit 6fb9ca2
Show file tree
Hide file tree
Showing 8 changed files with 113 additions and 78 deletions.
2 changes: 1 addition & 1 deletion chat/src/handlers/chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
RESPONSE_TYPES = {
"base": ["answer", "ref"],
"debug": ["answer", "attributes", "azure_endpoint", "deployment_name", "is_superuser", "k", "openai_api_version", "prompt", "question", "ref", "temperature", "text_key", "token_counts"],
"log": ["answer", "is_superuser", "k", "openai_api_version", "prompt", "question", "ref", "temperature", "token_counts"]
"log": ["answer", "deployment_name", "is_superuser", "k", "openai_api_version", "prompt", "question", "ref", "source_documents", "temperature", "token_counts"]
}

def handler(event, context):
Expand Down
8 changes: 4 additions & 4 deletions chat/src/handlers/opensearch_neural_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,19 @@ def __init__(
self.text_field = text_field

def similarity_search(
self, query: str, k: int = 10, subquery: Any = None, **kwargs: Any
self, query: str, k: int = 10, **kwargs: Any
) -> List[Document]:
"""Return docs most similar to the embedding vector."""
docs_with_scores = self.similarity_search_with_score(
query, k, subquery, **kwargs
query, k, **kwargs
)
return [doc[0] for doc in docs_with_scores]

def similarity_search_with_score(
self, query: str, k: int = 10, subquery: Any = None, **kwargs: Any
self, query: str, k: int = 10, **kwargs: Any
) -> List[Tuple[Document, float]]:
"""Return docs most similar to query."""
dsl = hybrid_query(query=query, model_id=self.model_id, vector_field=self.vector_field, k=k, subquery=subquery, **kwargs)
dsl = hybrid_query(query=query, model_id=self.model_id, vector_field=self.vector_field, k=k, **kwargs)
response = self.client.search(index=self.index, body=dsl, params={"search_pipeline": self.search_pipeline} if self.search_pipeline else None)
documents_with_scores = [
(
Expand Down
23 changes: 7 additions & 16 deletions chat/src/helpers/hybrid_query.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,22 +11,18 @@ def filter(query: dict):
}
}

def hybrid_query(query: str, model_id: str, vector_field: str = "embedding", k: int = 10, subquery: Any = None, **kwargs: Any):
if subquery:
weights = [0.5, 0.3, 0.2]
else:
weights = [0.7, 0.3]

def hybrid_query(query: str, model_id: str, vector_field: str = "embedding", k: int = 10, **kwargs: Any):
result = {
"size": k,
"query": {
"hybrid": {
"queries": [
filter({
"query_string": {
"default_operator": "AND",
"fields": ["title^5", "all_controlled_labels", "all_ids^5"],
"query": query
"default_operator": "AND",
"fields": ["all_titles^5", "all_controlled_labels", "all_ids^5"],
"query": query,
"analyzer": "english"
}
}),
filter({
Expand All @@ -47,7 +43,7 @@ def hybrid_query(query: str, model_id: str, vector_field: str = "embedding", k:
"normalization-processor": {
"combination": {
"parameters": {
"weights": weights
"weights": [0.25, 0.75]
},
"technique": "arithmetic_mean"
},
Expand All @@ -60,12 +56,7 @@ def hybrid_query(query: str, model_id: str, vector_field: str = "embedding", k:
}
}

if subquery:
result["query"]["hybrid"]["queries"].append(filter(subquery))

for key, value in kwargs.items():
result[key] = value

return result


19 changes: 19 additions & 0 deletions chat/src/helpers/metrics.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,24 @@
import tiktoken

def debug_response(config, response, original_question):
source_urls = [doc["api_link"] for doc in original_question.get("source_documents", [])]

return {
"answer": response,
"attributes": config.attributes,
"azure_endpoint": config.azure_endpoint,
"deployment_name": config.deployment_name,
"is_superuser": config.api_token.is_superuser(),
"k": config.k,
"openai_api_version": config.openai_api_version,
"prompt": config.prompt_text,
"question": config.question,
"ref": config.ref,
"source_documents": source_urls,
"temperature": config.temperature,
"text_key": config.text_key,
"token_counts": token_usage(config, response, original_question),
}

def token_usage(config, response, original_question):
data = {
Expand Down
30 changes: 2 additions & 28 deletions chat/src/helpers/response.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from helpers.metrics import token_usage
from helpers.metrics import debug_response
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnableLambda, RunnablePassthrough

Expand All @@ -16,23 +16,6 @@ def __init__(self, config):
self.store = {}

def debug_response_passthrough(self):
def debug_response(config, response, original_question):
return {
"answer": response,
"attributes": config.attributes,
"azure_endpoint": config.azure_endpoint,
"deployment_name": config.deployment_name,
"is_superuser": config.api_token.is_superuser(),
"k": config.k,
"openai_api_version": config.openai_api_version,
"prompt": config.prompt_text,
"question": config.question,
"ref": config.ref,
"temperature": config.temperature,
"text_key": config.text_key,
"token_counts": token_usage(config, response, original_question),
}

return RunnableLambda(lambda x: debug_response(self.config, x, self.original_question))

def original_question_passthrough(self):
Expand All @@ -56,16 +39,7 @@ def get_and_send_original_question(docs):

def prepare_response(self):
try:
subquery = {
"match": {
"all_titles": {
"query": self.config.question,
"operator": "AND",
"analyzer": "english"
}
}
}
retriever = self.config.opensearch.as_retriever(search_type="similarity", search_kwargs={"k": self.config.k, "subquery": subquery, "_source": {"excludes": ["embedding"]}})
retriever = self.config.opensearch.as_retriever(search_type="similarity", search_kwargs={"k": self.config.k, "_source": {"excludes": ["embedding"]}})
chain = (
{"context": retriever, "question": RunnablePassthrough()}
| self.original_question_passthrough()
Expand Down
10 changes: 4 additions & 6 deletions chat/test/helpers/test_hybrid_query.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,17 @@

class TestFunction(TestCase):
def test_hybrid_query(self):
subquery = { "term": { "title": { "value": "The Title" } } }
dsl = hybrid_query("Question?", "MODEL_ID", k=10, subquery=subquery)
dsl = hybrid_query("Question?", "MODEL_ID", k=10)
subject = dsl["query"]["hybrid"]["queries"]

checks = [
(lambda x: x["query_string"]["query"], "Question?"),
(lambda x: x["neural"]["embedding"]["model_id"], "MODEL_ID"),
(lambda x: x["term"]["title"]["value"], "The Title")
(lambda x: x["neural"]["embedding"]["model_id"], "MODEL_ID")
]

self.assertEqual(len(subject), 3)
self.assertEqual(len(subject), 2)

for i in range(3):
for i in range(2):
lookup, expected = checks[i]
queries = subject[i]["bool"]["must"]
self.assertEqual(lookup(queries[0]), expected)
Expand Down
95 changes: 72 additions & 23 deletions chat/test/helpers/test_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,53 +5,102 @@
sys.path.append('./src')

from unittest import TestCase, mock
from helpers.metrics import count_tokens, token_usage
from helpers.metrics import count_tokens, debug_response, token_usage
from event_config import EventConfig



@mock.patch.dict(
os.environ,
{
"AZURE_OPENAI_RESOURCE_NAME": "test",
"WEAVIATE_URL": "http://test",
"WEAVIATE_API_KEY": "test"
},
)
class TestMetrics(TestCase):
def test_token_usage(self):
original_question = {
"question": "What is your name?",
"source_documents": [],
@mock.patch.dict(
os.environ,
{
"AZURE_OPENAI_RESOURCE_NAME": "test",
"WEAVIATE_URL": "http://test",
"WEAVIATE_API_KEY": "test"
},
)
def setUp(self):
self.question = "What is your name?"
self.original_question = {
"question": self.question,
"source_documents": [
{
"accession_number": "SourceDoc:1",
"api_link": "https://api.dc.library.northwestern.edu/api/v2/works/881e8cae-67be-4e04-9970-7eafb52b2c5c",
"canonical_link": "https://dc.library.northwestern.edu/items/881e8cae-67be-4e04-9970-7eafb52b2c5c",
"title": "Source Document One!"
},
{
"accession_number": "SourceDoc:2",
"api_link": "https://api.dc.library.northwestern.edu/api/v2/works/ac0b2a0d-8f80-420a-b1a1-63b6ac2299f1",
"canonical_link": "https://dc.library.northwestern.edu/items/ac0b2a0d-8f80-420a-b1a1-63b6ac2299f1",
"title": "Source Document Two!"
},
{
"accession_number": "SourceDoc:3",
"api_link": "https://api.dc.library.northwestern.edu/api/v2/works/11569bb5-1b89-4fa9-bdfb-2caf2ded5aa5",
"canonical_link": "https://dc.library.northwestern.edu/items/11569bb5-1b89-4fa9-bdfb-2caf2ded5aa5",
"title": "Source Document Three!"
},
{
"accession_number": "SourceDoc:4",
"api_link": "https://api.dc.library.northwestern.edu/api/v2/works/211eeeca-d56e-4c6e-9123-1612d72258f9",
"canonical_link": "https://dc.library.northwestern.edu/items/211eeeca-d56e-4c6e-9123-1612d72258f9",
"title": "Source Document Four!"
},
{
"accession_number": "SourceDoc:5",
"api_link": "https://api.dc.library.northwestern.edu/api/v2/works/10e45e7a-8011-4ac5-97df-efa6a5439d0e",
"canonical_link": "https://dc.library.northwestern.edu/items/10e45e7a-8011-4ac5-97df-efa6a5439d0e",
"title": "Source Document Five!"
}
],
}
event = {
self.event = {
"body": json.dumps({
"deployment_name": "test",
"index": "test",
"k": 1,
"k": 5,
"openai_api_version": "2019-05-06",
"prompt": "This is a test prompt.",
"question": original_question,
"question": self.question,
"ref": "test",
"temperature": 0.5,
"text_key": "text",
"auth": "test123"
})
}
config = EventConfig(event=event)

response = {
self.config = EventConfig(event=self.event)
self.response = {
"output_text": "This is a test response.",
}

def test_debug_response(self):
result = debug_response(self.config, self.response, self.original_question)

self.assertEqual(result["k"], 5)
self.assertEqual(result["question"], self.question)
self.assertEqual(result["ref"], "test")
self.assertEqual(
result["source_documents"],
[
"https://api.dc.library.northwestern.edu/api/v2/works/881e8cae-67be-4e04-9970-7eafb52b2c5c",
"https://api.dc.library.northwestern.edu/api/v2/works/ac0b2a0d-8f80-420a-b1a1-63b6ac2299f1",
"https://api.dc.library.northwestern.edu/api/v2/works/11569bb5-1b89-4fa9-bdfb-2caf2ded5aa5",
"https://api.dc.library.northwestern.edu/api/v2/works/211eeeca-d56e-4c6e-9123-1612d72258f9",
"https://api.dc.library.northwestern.edu/api/v2/works/10e45e7a-8011-4ac5-97df-efa6a5439d0e"
]
)

result = token_usage(config, response, original_question)
def test_token_usage(self):
result = token_usage(self.config, self.response, self.original_question)

expected_result = {
"answer": 12,
"prompt": 314,
"question": 15,
"source_documents": 1,
"total": 342
"question": 5,
"source_documents": 527,
"total": 858
}

self.assertEqual(result, expected_result)
Expand Down
4 changes: 4 additions & 0 deletions template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1159,3 +1159,7 @@ Resources:
AuthorizationType: NONE
RouteKey: GET /docs/v2/{proxy+}
Target: !Sub "integrations/${docsIntegration}"
Outputs:
Endpoint:
Description: "The base API endpoint for the stack"
Value: !Sub "https://${CustomDomainHost}.${CustomDomainZone}/api/v2"

0 comments on commit 6fb9ca2

Please sign in to comment.