From c98a043353c9d21ba1e05a7e6f6a37385dbb3d8d Mon Sep 17 00:00:00 2001 From: Nightshade14 Date: Mon, 18 Nov 2024 01:06:38 -0500 Subject: [PATCH] fix: remove unnecessary files This PR indicates the existence of an API made to just manipulate GMAIL operations and nothing else. --- .python-version | 1 - README.md | 17 +-- assistant.py | 203 ------------------------------ server.py | 175 -------------------------- slack.py | 23 ---- src/db/__init__.py | 0 src/db/models.py | 39 ------ src/gpt/__init__.py | 0 src/gpt/gpt.py | 28 ----- src/slack/__init__.py | 0 src/slack/slack.py | 18 --- {test => tests}/__init__.py | 0 {test => tests}/test_gmail.py | 0 {test => tests}/test_gmail_api.py | 0 14 files changed, 1 insertion(+), 503 deletions(-) delete mode 100644 .python-version delete mode 100644 assistant.py delete mode 100644 server.py delete mode 100644 slack.py delete mode 100644 src/db/__init__.py delete mode 100644 src/db/models.py delete mode 100644 src/gpt/__init__.py delete mode 100644 src/gpt/gpt.py delete mode 100644 src/slack/__init__.py delete mode 100644 src/slack/slack.py rename {test => tests}/__init__.py (100%) rename {test => tests}/test_gmail.py (100%) rename {test => tests}/test_gmail_api.py (100%) diff --git a/.python-version b/.python-version deleted file mode 100644 index e4fba21..0000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.12 diff --git a/README.md b/README.md index 22faa36..c2d5b85 100644 --- a/README.md +++ b/README.md @@ -52,21 +52,6 @@ Note: One can run tools like ruff and pytest independently or can run them throu Note: Request to keep all `communication` in the Google Chats Project Group. -## Trello Setup - -- Create a Trello account. -- Go to https://trello.com/power-ups/admin to create a new Power-up. -- Check the new power-up, click `API key` on the left side panel. -- Copy the API key and visit https://trello.com/1/authorize?expiration=1day&name=yourAppName&scope=read,write&response_type=token&key=your_api_key, replace `your_api_key` with your own api key. The expiration and the name can be changed if needed. -- Create a .env file, input your API key and token: - -``` -TRELLO_API_KEY=your_api_key -TRELLO_OAUTH_TOKEN=your_token -``` - -- Run TrelloManager.py and you will see the test result. - # License -mAIgic has a MIT-style license, as found in the [LICENSE](LICENSE) file. +mAIgic has a MIT-style license, as found in the [LICENSE](LICENSE) file. \ No newline at end of file diff --git a/assistant.py b/assistant.py deleted file mode 100644 index 3a55af2..0000000 --- a/assistant.py +++ /dev/null @@ -1,203 +0,0 @@ -import json -from pprint import pformat -from typing import Any -from src.db.models import DB, Chat, Item, ItemType -from src.gmail.client import Gmail -from src.gmail.models import GMailMessage -import logging - -from src.gpt.gpt import GPT, Line, Role -from pydantic import BaseModel - -LOGGER = logging.getLogger(__file__) - - -GMAIL_PRIMER = Line( - role=Role.SYSTEM, - # FIXME: this content needs to be tailored to the custome - content=""" - I am Kamen Yotov. - My emails are kamen@yotov.org, kyotov@gmail.com, ky12@nyu.edu. - - When I first receive an email, I want you to analyze it respond with the analysis in json as follows: - - * "author" - * "time_received" in iso format - * "categories" - ** "urgent": bool - ** "important": bool - ** "spam": bool - * "summary" : summary of the content - * "action": proposed next action - - Then answer my follow up questions in markdown supported by slack api. - """, -) - - -class ItemKey(BaseModel): - type: ItemType - id: str - - @classmethod - def from_item(cls, item: Item) -> "ItemKey": - return ItemKey(type=item.type, id=item.id) - - -class Conversation: - def __init__(self, db: DB, gpt: GPT, item_key: ItemKey) -> None: - self.db = db - self.gpt = gpt - self.item_key = item_key - - self.item = ( - db.session.query(Item) - .where((Item.type == item_key.type) & (Item.id == item_key.id)) - .one() - ) - - self.thread = list( - db.session.query(Chat) - .where((Chat.type == item_key.type) & (Chat.id == item_key.id)) - .order_by(Chat.seq) - .all() - ) - - def primer_lines(self) -> list[Line]: - return [ - GMAIL_PRIMER, - Line(role=Role.USER, content=self.item.content), - ] - - def handle_user_reply(self, message: str = "") -> str: - lines = self.primer_lines() + [ - Line(role=x.role, content=x.content) for x in self.thread - ] - - if len(lines) == 2: # this conversation has not started yet! - assert message == "" # there is no message by user yet! - else: - assert len(lines) >= 3, lines # there should be at least 1 reply ... - assert lines[-1].role == Role.ASSISTANT # ... by the assistant - - assert message - lines.append(Line(role=Role.USER, content=message)) - - r = self.gpt.complete(lines) - - # TODO: this is where we need to differentiate if it is a tool/function call... - # for now assert it is not! - assert len(r.choices) == 1 - choice = r.choices[0] - assert choice.finish_reason == "stop" - result = choice.message.content - assert result - - self.db.session.add( - Chat( - type=self.item.type, - id=self.item.id, - role=Role.USER, - content=message, - ) - ) - self.db.session.add( - Chat( - type=self.item.type, - id=self.item.id, - role=Role.ASSISTANT, - content=result, - ) - ) - self.db.session.commit() - - return result - - -class Assistant: - def __init__(self) -> None: - # self.db = DB() - # self.gpt = GPT() - pass - - def fetch_emails(self) -> int: - try: - gmail = Gmail() - db = DB() - new_count = 0 - # TODO(kamen): figure out how to query for new emails only? - # maybe based on date of the last email we know about? - # just ask for emails that are newer than that date? - for message in gmail.query(): - if ( - found := db.session.query(Item) - .where((Item.type == ItemType.gmail) & (Item.id == message["id"])) - .first() - ): - LOGGER.debug("skipping %s already in database", found.id) - else: - mm = gmail.get_message(message["id"]) - m = GMailMessage(mm) - db.session.add( - Item(type=ItemType.gmail, id=mm["id"], content=m.as_md()) - ) - db.session.commit() - LOGGER.debug("added %s to database", mm["id"]) - new_count += 1 - return new_count - except Exception: - LOGGER.exception("fetch_emails failed") - raise - - def handle_one(self) -> dict[str, Any]: - try: - db = DB() - gpt = GPT() - - if ( - new_item := db.session.query(Item) - .where(Item.slack_thread.is_(None)) - .first() - ): - convo = Conversation(db, gpt, ItemKey.from_item(new_item)) - - LOGGER.info("handling %s", new_item.id) - - result = convo.handle_user_reply() - result = result.removeprefix("```json").removesuffix("```") - result = json.loads(result) - result["id"] = new_item.id - - LOGGER.info(pformat(result)) - return result - except: - LOGGER.exception("") - raise - - def handle_reply(self, thread_ts: str, reply: str) -> str: - try: - db = DB() - gpt = GPT() - - if ( - item := db.session.query(Item) - .where(Item.slack_thread == thread_ts) - .first() - ): - convo = Conversation(db, gpt, ItemKey.from_item(item)) - - LOGGER.info("handling reply on %s for %s", thread_ts, item.id) - result = convo.handle_user_reply(reply) - LOGGER.info(pformat(result)) - return result - else: - raise RuntimeError(f"item not found for {thread_ts}") - except Exception as error_msg: - LOGGER.exception(str(error_msg)) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.DEBUG) - a = Assistant() - a.fetch_emails() - a.handle_one() diff --git a/server.py b/server.py deleted file mode 100644 index ed141c5..0000000 --- a/server.py +++ /dev/null @@ -1,175 +0,0 @@ -import asyncio -from datetime import datetime -import enum -import logging -import os -from concurrent.futures import Future, ProcessPoolExecutor -from pprint import pformat -from queue import Queue -import re -from typing import TypeVar - -from slack_bolt.adapter.socket_mode.aiohttp import AsyncSocketModeHandler -from slack_bolt.async_app import AsyncApp - -from assistant import Assistant -from src.db.models import DB, Item - -logging.basicConfig(level=logging.INFO) -LOGGER = logging.getLogger(__file__) - -app = AsyncApp(token=os.environ["SLACK_BOT_TOKEN"]) - - -class OP(enum.Enum): - fetch = enum.auto() - handle_one = enum.auto() - - -T = TypeVar("T") - - -class Server: - pool: ProcessPoolExecutor - - def __init__(self) -> None: - self.assistant = Assistant() - app.message()(self.message_hello) - - self.queue = Queue() - - async def await_future(self, future: Future[T]) -> T: - done = asyncio.Event() - future.add_done_callback(lambda _: done.set()) - await done.wait() - return future.result() - - async def _slack_app(self): - handler = AsyncSocketModeHandler(app, os.environ["SLACK_APP_TOKEN"]) - await handler.start_async() - - def run(self): - with ProcessPoolExecutor() as pool: - self.pool = pool - asyncio.run(self._slack_app()) - - def fix_md_for_slack(self, md_text: str) -> str: - # Convert **bold** to *bold* (Slack syntax) - slack_text = re.sub(r"\*\*(.*?)\*\*", r"*\1*", md_text) - - # Convert *italic* to _italic_ (Slack syntax) - slack_text = re.sub(r"\*(.*?)\*", r"_\1_", slack_text) - - # Convert [text](link) to (Slack link syntax) - slack_text = re.sub(r"\[(.*?)\]\((.*?)\)", r"<\2|\1>", slack_text) - - return slack_text - - async def message_hello(self, message, ack): - await ack() - - if thread_ts := message.get("thread_ts"): - text = message["text"] - future = self.pool.submit(self.assistant.handle_reply, thread_ts, text) - response = await self.await_future(future) - LOGGER.info("response: %s", response) - - r = await app.client.chat_postMessage( - channel=os.getenv("SLACK_CHANNEL_ID"),\ - thread_ts=thread_ts, - blocks=[ - { - "type": "context", - "elements": [ - { - "type": "mrkdwn", - "text": self.fix_md_for_slack(response), - }, - ], - }, - ], - unfurl_links=False, - unfurl_media=False, - ) - r.validate() - - match message["text"]: - - case str(value) if "fetch" in value: - future = self.pool.submit(self.assistant.fetch_emails) - found = await self.await_future(future) - LOGGER.info("%s new emails found", found) - - case str(value) if "handle_one" in value: - future = self.pool.submit(self.assistant.handle_one) - result = await self.await_future(future) - - time = datetime.fromisoformat(result["time_received"]) - - r = await app.client.chat_postMessage( - channel=os.getenv("SLACK_CHANNEL_ID"), - text=f"Handling `{result['id']}`...", - unfurl_links=False, - unfurl_media=False, - ) - r.validate() - ts = r.get("ts") - - r = await app.client.chat_update( - channel=os.getenv("SLACK_CHANNEL_ID"), - ts=ts, - blocks=[ - { - "type": "context", - "elements": [ - { - "type": "mrkdwn", - "text": f"`From:` {result['author']}", - }, - ], - }, - { - "type": "context", - "elements": [ - { - "type": "mrkdwn", - "text": f"`When:` ", - }, - ], - }, - {"type": "divider"}, - { - "type": "context", - "elements": [ - { - "type": "mrkdwn", - "text": f"{result['summary']}", - }, - ], - }, - { - "type": "context", - "elements": [ - { - "type": "mrkdwn", - "text": f"{result['action']}", - }, - ], - }, - ], - ) - r.validate() - - db = DB() - item = db.session.query(Item).where(Item.id == result["id"]).one() - item.slack_channel = "C01CAH729TK" - item.slack_thread = r.get("ts") - db.session.commit() - - LOGGER.info("\n" + pformat(message)) - - -if __name__ == "__main__": - Server().run() - -# https://mail.google.com/mail/u/0/#inbox/19245a9bd60a75f6 diff --git a/slack.py b/slack.py deleted file mode 100644 index 9755f74..0000000 --- a/slack.py +++ /dev/null @@ -1,23 +0,0 @@ -from slack_sdk import WebClient -import os - - -def main(): - client = WebClient( - token=os.getenv("SLACK_BOT_TOKEN") - ) - - # r = client.users_list() - # r.validate() - # pprint(r.data) - - # r = client.conversations_open(channel="U01C91V6QNQ") - # r.validate() - - r = client.chat_postMessage(channel=os.getenv("SLACK_CHANNEL_ID"), - text="hello!") - r.validate() - - -if __name__ == "__main__": - main() diff --git a/src/db/__init__.py b/src/db/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/db/models.py b/src/db/models.py deleted file mode 100644 index c35f4af..0000000 --- a/src/db/models.py +++ /dev/null @@ -1,39 +0,0 @@ -from sqlalchemy import Integer, create_engine, Column, String, Enum -import enum -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker - - -Base = declarative_base() - - -class ItemType(enum.StrEnum): - gmail = enum.auto() - - -class Item(Base): - __tablename__ = "items" - - type = Column(Enum(ItemType), nullable=False, primary_key=True) - id = Column(String, primary_key=True) - slack_channel = Column(String) - slack_thread = Column(String) - content = Column(String) - - -class Chat(Base): - __tablename__ = "chats" - - seq = Column(Integer, primary_key=True, autoincrement=True) - type = Column(Enum(ItemType), nullable=False) # this and the next one is a foreign key to Item - id = Column(String) - role = Column(String) - content = Column(String) - - -class DB: - def __init__(self) -> None: - self.engine = create_engine("sqlite:///maigic.db", echo=False) - Base.metadata.create_all(self.engine) - self.Session = sessionmaker(bind=self.engine) - self.session = self.Session() diff --git a/src/gpt/__init__.py b/src/gpt/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/gpt/gpt.py b/src/gpt/gpt.py deleted file mode 100644 index 246b4b9..0000000 --- a/src/gpt/gpt.py +++ /dev/null @@ -1,28 +0,0 @@ -from openai import OpenAI -from pydantic import BaseModel -import enum -from openai.types.chat import ChatCompletion - - -class Role(enum.StrEnum): - SYSTEM = "system" - USER = "user" - ASSISTANT = "assistant" - FUNCTION = "function" - - -class Line(BaseModel): - role: Role - content: str - - -class GPT: - def __init__(self) -> None: - self.client = OpenAI() - - def complete(self, log: list[Line]) -> ChatCompletion: - return self.client.chat.completions.create( - # model="gpt-4", - model="gpt-4o-mini", - messages=[{"role": x.role, "content": x.content} for x in log], - ) diff --git a/src/slack/__init__.py b/src/slack/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/slack/slack.py b/src/slack/slack.py deleted file mode 100644 index 5b3fe4c..0000000 --- a/src/slack/slack.py +++ /dev/null @@ -1,18 +0,0 @@ -import os - -from slack_sdk import WebClient - - -class Slack: - def __init__(self) -> None: - # replaced client with self.client as client was not used anywhere - self.client = WebClient(token=os.getenv("SLACK_BOT_TOKEN")) - - def send(self, thread: str | None, text: str) -> None: - r = self.client.chat_postMessage( - channel=os.getenc("SLACK_CHANNEL_ID"), - thread=thread, - text=text, - ) - r.validate() - return r diff --git a/test/__init__.py b/tests/__init__.py similarity index 100% rename from test/__init__.py rename to tests/__init__.py diff --git a/test/test_gmail.py b/tests/test_gmail.py similarity index 100% rename from test/test_gmail.py rename to tests/test_gmail.py diff --git a/test/test_gmail_api.py b/tests/test_gmail_api.py similarity index 100% rename from test/test_gmail_api.py rename to tests/test_gmail_api.py