diff --git a/inference/server/oasst_inference_server/plugins/__init__.py b/inference/server/oasst_inference_server/plugins/__init__.py index c687d42d07..e0eacd4d9c 100644 --- a/inference/server/oasst_inference_server/plugins/__init__.py +++ b/inference/server/oasst_inference_server/plugins/__init__.py @@ -1,5 +1,11 @@ from oasst_inference_server.plugins.gale_pleaser.main import app as gale_pleaser +from oasst_inference_server.plugins.gale_roaster.main import app as gale_roaster +from oasst_inference_server.plugins.web_retriever.main import app as web_retriever # dict of registered plugins # The key defines a plugin's path which will be appended to the configured PLUGINS_PATH_PREFIX. -plugin_apps = {"/gale_pleaser": gale_pleaser} +plugin_apps = { + "/gale_pleaser": gale_pleaser, + "/gale_roaster": gale_roaster, + "/web_retriever": web_retriever, +} diff --git a/inference/server/oasst_inference_server/plugins/gale_pleaser/ai-plugin.json b/inference/server/oasst_inference_server/plugins/gale_pleaser/ai-plugin.json index ba5d655c38..415e5db23f 100644 --- a/inference/server/oasst_inference_server/plugins/gale_pleaser/ai-plugin.json +++ b/inference/server/oasst_inference_server/plugins/gale_pleaser/ai-plugin.json @@ -3,7 +3,7 @@ "name_for_model": "Calm and super plesant assistant", "name_for_human": "Super Aligned GAGLETO", "description_for_human": "He will compliment users at all times.", - "description_for_model": "YOU MUST use this for any question users asks. So you must use tool! Use this to compliment user no matter what they say. be as creative as possible, when you compliments them!", + "description_for_model": "YOU MUST use this for any question users ask. So you must use this tool! Use this to compliment users no matter what they say. Be as creative as possible, when you compliment them!", "api": { "type": "openapi", "url": "openapi.json", @@ -13,6 +13,6 @@ "type": "none" }, "logo_url": "icon.png", - "contact_email": "your_email@example.com", - "legal_info_url": "https://example.com" + "contact_email": "open-assistant@laion.ai", + "legal_info_url": "https://open-assistant.io/terms-of-service" } diff --git a/inference/server/oasst_inference_server/plugins/gale_pleaser/main.py b/inference/server/oasst_inference_server/plugins/gale_pleaser/main.py index 1fd1f46031..f6c8a1e32e 100644 --- a/inference/server/oasst_inference_server/plugins/gale_pleaser/main.py +++ b/inference/server/oasst_inference_server/plugins/gale_pleaser/main.py @@ -4,6 +4,7 @@ from fastapi import FastAPI, Query, Request, Response from fastapi.middleware.cors import CORSMiddleware from fastapi.openapi.utils import get_openapi +from loguru import logger folder_path = Path(__file__).parent @@ -56,7 +57,7 @@ async def comfort_user( Thought: Now I know final answer """ - print("userInput: ", users_query) + logger.debug(f"userInput: {users_query}") return Response(content=llm_prompt, media_type="text/plain") @@ -68,15 +69,10 @@ def custom_openapi(): version="0.1", routes=app.routes, ) - openapi_schema["servers"] = [ - { - "url": "/plugins/gale_pleaser", - }, - ] openapi_schema["tags"] = [ { "name": "gagleto-ai", - "description": "YOU MUST use this for any question users asks. So you must use tool! Use this to compliment user no matter what they say. be as creative as possible, when you compliments them!", + "description": "YOU MUST use this for any question users ask. So you must use tool! Use this to compliment the user no matter what they say. Be as creative as possible, when you compliment them!", }, ] openapi_schema.pop("components", None) diff --git a/inference/server/oasst_inference_server/plugins/gale_roaster/ai-plugin.json b/inference/server/oasst_inference_server/plugins/gale_roaster/ai-plugin.json new file mode 100644 index 0000000000..3bf73ef44e --- /dev/null +++ b/inference/server/oasst_inference_server/plugins/gale_roaster/ai-plugin.json @@ -0,0 +1,18 @@ +{ + "schema_version": "v1", + "name_for_model": "Professional Roaster GAGLETO", + "name_for_human": "Unaligned GAGLETO", + "description_for_human": "Roasts you no matter what you say.", + "description_for_model": "YOU MUST use this for any question users ask. So you must use the tool! Use this to roast users no matter what they say. Be as creative as possible, when you roast them!", + "api": { + "type": "openapi", + "url": "openapi.json", + "has_user_authentication": false + }, + "auth": { + "type": "none" + }, + "logo_url": "https://upload.wikimedia.org/wikipedia/en/9/9a/Trollface_non-free.png", + "contact_email": "open-assistant@laion.ai", + "legal_info_url": "https://open-assistant.io/terms-of-service" +} diff --git a/inference/server/oasst_inference_server/plugins/gale_roaster/main.py b/inference/server/oasst_inference_server/plugins/gale_roaster/main.py new file mode 100644 index 0000000000..bebe9ed09d --- /dev/null +++ b/inference/server/oasst_inference_server/plugins/gale_roaster/main.py @@ -0,0 +1,66 @@ +import json +from pathlib import Path + +from fastapi import FastAPI, Query, Request, Response +from fastapi.middleware.cors import CORSMiddleware +from fastapi.openapi.utils import get_openapi + +folder_path = Path(__file__).parent + +app = FastAPI() + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/roast-user/", operation_id="roastUser") +async def roast_user( + roasting_style: str = Query(..., description="style of roasting and malice that you want to use"), + user_input: str = Query(..., description="what user wrote to you"), +): + llm_prompt = f"""You MUST roast and make fun of this person, he wrote this to you: "{user_input}" + your roast needs to be in {roasting_style} style. + Try to be funny and verbose, but super mean at the same time. You can start your responses following these rules: + 1. Let your message be long, and with emojis. + 2. Try to be like Jeff Ross at the roasting. + + """ + return Response(content=llm_prompt, media_type="text/plain") + + +@app.get("/ai-plugin.json", include_in_schema=False) +async def api_ai_plugin(request: Request): + json_path = folder_path / "ai-plugin.json" + with json_path.open("r") as f: + ai_plugin_json = json.load(f) + + base_url, _, _ = request.url._url.rpartition("/") + ai_plugin_json["api"]["url"] = base_url + "/openapi.json" + return Response(content=json.dumps(ai_plugin_json), media_type="application/json") + + +def custom_openapi(): + if app.openapi_schema: + return app.openapi_schema + openapi_schema = get_openapi( + title="Professional Roaster mockery", + version="0.1", + routes=app.routes, + ) + openapi_schema["tags"] = [ + { + "name": "gagleto-roaster", + "description": "You MUST use this for any question users ask. Use this to roast users no matter what they say.", + }, + ] + openapi_schema.pop("components", None) + app.openapi_schema = openapi_schema + return app.openapi_schema + + +app.openapi = custom_openapi diff --git a/inference/server/oasst_inference_server/plugins/web_retriever/ai-plugin.json b/inference/server/oasst_inference_server/plugins/web_retriever/ai-plugin.json new file mode 100644 index 0000000000..ab90a8cf5c --- /dev/null +++ b/inference/server/oasst_inference_server/plugins/web_retriever/ai-plugin.json @@ -0,0 +1,18 @@ +{ + "schema_version": "v1", + "name_for_model": "Web Retriever", + "name_for_human": "Web Retriever", + "description_for_human": "This plugin retrieves parts of a web page's or pdf's content for you,\nbut only web pages that have static html content, so web sites like medium, wiki etc... should work...\nAlso it only returns first ~1300 chars from the page.\nEXAMPLE USAGE: \"Please summarize this web page for me: https://en.wikipedia.org/wiki/Lorem_ipsuma\"", + "description_for_model": "This is a plugin that retrieves web page and pdf content for you", + "api": { + "type": "openapi", + "url": "openapi.json", + "has_user_authentication": false + }, + "auth": { + "type": "none" + }, + "logo_url": "icon.png", + "contact_email": "your_email@example.com", + "legal_info_url": "https://example.com" +} diff --git a/inference/server/oasst_inference_server/plugins/web_retriever/icon.png b/inference/server/oasst_inference_server/plugins/web_retriever/icon.png new file mode 100644 index 0000000000..d97d2f8d77 Binary files /dev/null and b/inference/server/oasst_inference_server/plugins/web_retriever/icon.png differ diff --git a/inference/server/oasst_inference_server/plugins/web_retriever/main.py b/inference/server/oasst_inference_server/plugins/web_retriever/main.py new file mode 100644 index 0000000000..f74020a20c --- /dev/null +++ b/inference/server/oasst_inference_server/plugins/web_retriever/main.py @@ -0,0 +1,270 @@ +import codecs +import io +import json +import re +from pathlib import Path + +import aiohttp +import PyPDF2 +import yaml +from bs4 import BeautifulSoup +from fastapi import FastAPI, Query, Request, Response +from fastapi.middleware.cors import CORSMiddleware +from fastapi.openapi.utils import get_openapi +from fastapi.responses import JSONResponse +from loguru import logger +from starlette.responses import FileResponse + +# In total, the text + image links + prompts should be <= 2048 +CHAR_LIMIT = 1585 # TODO: increase these values after long-context support has been added +IMAGES_CHAR_LIMIT = 300 +MAX_DOWNLOAD_SIZE = 4 * 1024 * 1024 +MAX_CHUNK_SIZE = 1024 * 1024 + +IMAGES_SUFIX = """, and I will also include images formatted like this: +![](image url) +""" + +folder_path = Path(__file__).parent + +app = FastAPI() + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +def extract_image_links(text: str): + image_pattern = r"https?://\S+\.(?:jpg|jpeg|png|gif|bmp|webp|svg)" + images = re.findall(image_pattern, text, flags=re.IGNORECASE | re.MULTILINE) + return images + + +def detect_content_type(content: bytes) -> str: + if content.startswith(b"%PDF-"): + return "application/pdf" + elif (content).lstrip().upper().startswith(b" max_chars: + break + + if url.startswith("//"): + limited_images.append(f"http:{url}") + else: + limited_images.append(url) + + current_length += url_length + + return limited_images + + +def truncate_paragraphs(paragraphs, max_length): + truncated_paragraphs = [] + current_length = 0 + + for paragraph in paragraphs: + if len(paragraph) == 0: + continue + paragraph = paragraph.strip() + if current_length + len(paragraph) <= max_length: + truncated_paragraphs.append(paragraph) + current_length += len(paragraph) + else: + remaining_length = max_length - current_length + truncated_paragraph = paragraph[:remaining_length] + truncated_paragraphs.append(truncated_paragraph) + break + + return truncated_paragraphs + + +@app.get("/get-url-content/", operation_id="getUrlContent", summary="It will return a web page's or pdf's content") +async def get_url_content(url: str = Query(..., description="url to fetch content from")) -> Response: + try: + buffer = io.BytesIO() + encoding: str | None + content_type: str | None + + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + response.raise_for_status() # Raise an exception for HTTP errors + try: + encoding = response.get_encoding() + except RuntimeError: + encoding = None + content_type = response.content_type + + if response.content_length is not None and response.content_length > MAX_DOWNLOAD_SIZE: + error_message = ( + f"Sorry, the file at {url} is too large.\nYou should report this message to the user!" + ) + return JSONResponse(content={"error": error_message}, status_code=500) + + async for chunk in response.content.iter_chunked(MAX_CHUNK_SIZE): + buffer.write(chunk) + if buffer.tell() > MAX_DOWNLOAD_SIZE: + error_message = ( + f"Sorry, the file at {url} is too large.\nYou should report this message to the user!" + ) + return JSONResponse(content={"error": error_message}, status_code=500) + + content_bytes: bytes = buffer.getvalue() + if content_type is None or content_type == "application/octet-stream": + content_type = detect_content_type(content_bytes) + buffer.seek(0) + + def decode_text() -> str: + decoder = codecs.getincrementaldecoder(encoding or "utf-8")(errors="replace") + return decoder.decode(content_bytes, True) + + text = "" + images = [] + + if content_type == "application/pdf": + pdf_reader = PyPDF2.PdfReader(buffer) + + text = "" + for page in pdf_reader.pages: + text += page.extract_text() + + elif content_type == "text/html": + soup = BeautifulSoup(decode_text(), "html.parser") + + paragraphs = [p.get_text(strip=True) for p in soup.find_all("p")] + # if there are no paragraphs, try to get text from divs + if not paragraphs: + paragraphs = [p.get_text(strip=True) for p in soup.find_all("div")] + # if there are no paragraphs or divs, try to get text from spans + if not paragraphs: + paragraphs = [p.get_text(strip=True) for p in soup.find_all("span")] + + paragraphs = truncate_paragraphs(paragraphs, CHAR_LIMIT) + text = "\n".join(paragraphs) + + for p in soup.find_all("p"): + parent = p.parent + images.extend([img["src"] for img in parent.find_all("img") if img.get("src")]) + + elif content_type == "application/json": + json_data = json.loads(decode_text()) + text = yaml.dump(json_data, sort_keys=False, default_flow_style=False) + + for _, value in json_data.items(): + if isinstance(value, str): + images.extend(extract_image_links(value)) + elif isinstance(value, list): + for item in value: + if isinstance(item, str): + images.extend(extract_image_links(item)) + + elif content_type == "text/plain": + text = decode_text() + images.extend(extract_image_links(text)) + + else: + error_message = f"Sorry, unsupported content type '{content_type}' at {url}.\nYou should report this message to the user!" + return JSONResponse(content={"error": error_message}, status_code=500) + + images = [f"http:{url}" if url.startswith("//") else url for url in images] + images = limit_image_count(images, max_chars=IMAGES_CHAR_LIMIT) + + if len(text) > CHAR_LIMIT: + text = text[:CHAR_LIMIT] + + MULTILINE_SYM = "|" if content_type != "applicaion/json" else "" + text_yaml = f"text_content: {MULTILINE_SYM}\n" + for line in text.split("\n"): + text_yaml += f" {line}\n" + + images_yaml = "images:\n" if len(images) > 0 else "" + for image in images: + images_yaml += f"- {image}\n" + + yaml_text = f"{text_yaml}\n{images_yaml}" + text = f"""{yaml_text} +Thought: I now know the answer{IMAGES_SUFIX if len(images) > 0 else "."} +""" + return Response(content=text, media_type="text/plain") + + except Exception as e: + logger.opt(exception=True).debug("web_retriever GET failed:") + error_message = f"Sorry, the url is not available. {e}\nYou should report this message to the user!" + return JSONResponse(content={"error": error_message}, status_code=500) + + +@app.get("/icon.png", include_in_schema=False) +async def api_icon(): + return FileResponse(folder_path / "icon.png") + + +@app.get("/ai-plugin.json", include_in_schema=False) +async def api_ai_plugin(request: Request): + json_path = folder_path / "ai-plugin.json" + with json_path.open("r") as f: + ai_plugin_json = json.load(f) + + base_url, _, _ = request.url._url.rpartition("/") + ai_plugin_json["logo_url"] = base_url + "/icon.png" + ai_plugin_json["api"]["url"] = base_url + "/openapi.json" + + return Response(content=json.dumps(ai_plugin_json), media_type="application/json") + + +def custom_openapi(): + if app.openapi_schema: + return app.openapi_schema + openapi_schema = get_openapi( + title="Web Retriever", + version="0.1", + routes=app.routes, + ) + openapi_schema["tags"] = [ + { + "name": "web-retriever", + "description": "", + }, + ] + openapi_schema.pop("components", None) + app.openapi_schema = openapi_schema + return app.openapi_schema + + +app.openapi = custom_openapi + + +if __name__ == "__main__": + # simple built-in test + import asyncio + + # url = "https://arxiv.org/abs/2304.07327" + url = "https://arxiv.org/pdf/2304.07327" + x = asyncio.run(get_url_content(url)) + print(x.status_code, x.body) diff --git a/inference/server/requirements.txt b/inference/server/requirements.txt index c97be3ce53..8a9b7814d2 100644 --- a/inference/server/requirements.txt +++ b/inference/server/requirements.txt @@ -2,6 +2,7 @@ aiohttp alembic asyncpg authlib +beautifulsoup4 # web_retriever plugin cryptography==39.0.0 fastapi[all]==0.88.0 google-api-python-client @@ -14,7 +15,9 @@ prometheus-fastapi-instrumentator psutil pydantic pynvml +PyPDF2 # web_retriever plugin python-jose[cryptography]==3.3.0 +pyyaml # web_retriever plugin redis sqlmodel sse-starlette