From 060276a258b5a6c3bf8e7e4b724738a4d4a646bf Mon Sep 17 00:00:00 2001 From: Frankbz Date: Thu, 31 Oct 2024 22:10:21 -0400 Subject: [PATCH] new: Gmail implementation --- .circleci/config.yml | 54 +++++++++++++++++ .gitignore | 18 +++--- README.md | 73 +++++++++++++++++++++- assistant.py | 4 +- config/.env | 14 +++++ pyproject.toml | 2 + src/__init__.py | 1 + src/gmail/API.py | 34 +++++++++++ src/gmail/client.py | 105 ++++++++++++++++++++++++++++++-- src/gmail/models.py | 2 +- src/slack/slack.py | 3 +- test/__init__.py | 1 + test/test_gmail.py | 134 +++++++++++++++++++++++++++++++++++++++++ test/test_gmail_api.py | 35 +++++++++++ uv.lock | 49 +++++++++++++++ 15 files changed, 511 insertions(+), 18 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 config/.env create mode 100644 src/gmail/API.py create mode 100644 test/__init__.py create mode 100644 test/test_gmail.py create mode 100644 test/test_gmail_api.py diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..687abed --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,54 @@ +version: 2.1 + +executors: + my-docker-executor: + docker: + - image: cimg/python:3.11.10 + +jobs: + build: + executor: my-docker-executor + steps: + - checkout + + - run: + name: Get uv via curl and install it + command: | + curl -LsSf https://astral.sh/uv/install.sh | sh + + - persist_to_workspace: + root: /home/circleci/ + paths: + - .cargo + + test: + executor: my-docker-executor + steps: + - checkout + + - attach_workspace: + at: /home/circleci/ + + - run: + name: Set-up dependencies path + command: echo 'export PATH=$HOME/.cargo/bin:$PATH' >> $BASH_ENV + + - run: + name: Set-up project dependencies + command: uv sync + + - run: + name: Format python files with ruff + command: uv run ruff check . --fix + + - run: + name: Run test files + command: uv run pytest + +workflows: + build_and_test: + jobs: + - build + - test: + requires: + - build diff --git a/.gitignore b/.gitignore index 9291fde..284d87c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ -/*.json -/email_examples -__pycache__ -/maigic.db -/secrets.sh -/secret.sh -/secrets -/credentials.json \ No newline at end of file +**/*.json +**/email_examples +**/__pycache__ +**/maigic.db +**/secrets.sh +**/secret.sh +**/secrets +**/credentials.json +**/.pytest_cache +**/.ruff_cache \ No newline at end of file diff --git a/README.md b/README.md index c8de4d7..22faa36 100644 --- a/README.md +++ b/README.md @@ -1 +1,72 @@ -Collaboration doc for this initiative: https://docs.google.com/document/d/1duhM5Ufkq_doBnMvHo65S7cplmUJw4KqQvDRfZlmj7s/edit +# mAIgic + +This project will organize life by tracking messages and information that need follow-ups and reminding the user to take follow-ups. It will also provide a search based on individuals. + +The project aims to leverage AI to track and remind for follow up messages for systems like Slack, Trello, Whatsapp, Gmail, Google Docs comments, etc. + +# Technical tools + +- Programming Language: Python +- Project dependency management tool: uv +- Project linter: ruff +- Project tester: pytest +- Continuous Integration (CI) tool: circleCI + +# Instructions + +## Setup + +- Install `uv` on your system with command with `curl -LsSf https://astral.sh/uv/install.sh | sh`. +- `cd` into the cloned repository and execute `uv sync` (This will install all the dependencies). +- Activate the environment `venv` by executing `source ./.venv/bin/activate`. + +## Processes + +- To check the formatting standard with ruff (linter) execute either `uv run ruff check .` or `ruff check .` from the root directory of the project. +- To test the code, execute either `uv run pytest .` or `pytest .` from the root directory of the project. + +Note: One can run tools like ruff and pytest independently or can run them through `uv`. + +## Commit Style Guide + +- Request to `strictly adhere` to the suggested commit style guide. +- The project follows Udacity's [Commit Style Guide](https://udacity.github.io/git-styleguide/). +- Reason: + - It is simple, elegant, concise and effective. + - It does not have many rules that could create confusion but yet have just enough to keep things working smoothly through clear and concise communication. + +## GitHub Workflow + +- Members of same team can preferably `clone` the repository. +- Make sure to push new changes to `dev` remote branch. +- Create a `Pull Request` and the changes would be reviewed and merged to the `main` remote branch. `Review` includes code, code quality, code standards, commit style guides, and Pull Request descriptions. Consistent code standards and documentation would be aided by `ruff`. +- `main` branch serves as production branch which would accumulate new changes, features and bug-fixes from `dev` branch. +- Would appreciate if you open `issues` whenever you come across any. Issues can be bugs, proposed features, performance / code improvement proposals, etc. + +## Requesting access to project and CI space + +- Send your`github username` to become collaborators to the project. +- Send your `email id` used to `register with circleCI` to get access to the circleCI organization to manage CI workflows and triggers. You will receive an invitation in the provided email's inbox to join the circleCI organization. +- Currently, the `magic2` CircleCI project is attached to this project. +- Collaboration doc for this initiative: https://docs.google.com/document/d/1duhM5Ufkq_doBnMvHo65S7cplmUJw4KqQvDRfZlmj7s/edit + +Note: Request to keep all `communication` in the Google Chats Project Group. + +## Trello Setup + +- Create a Trello account. +- Go to https://trello.com/power-ups/admin to create a new Power-up. +- Check the new power-up, click `API key` on the left side panel. +- Copy the API key and visit https://trello.com/1/authorize?expiration=1day&name=yourAppName&scope=read,write&response_type=token&key=your_api_key, replace `your_api_key` with your own api key. The expiration and the name can be changed if needed. +- Create a .env file, input your API key and token: + +``` +TRELLO_API_KEY=your_api_key +TRELLO_OAUTH_TOKEN=your_token +``` + +- Run TrelloManager.py and you will see the test result. + +# License + +mAIgic has a MIT-style license, as found in the [LICENSE](LICENSE) file. diff --git a/assistant.py b/assistant.py index b0dc374..3a55af2 100644 --- a/assistant.py +++ b/assistant.py @@ -192,8 +192,8 @@ def handle_reply(self, thread_ts: str, reply: str) -> str: return result else: raise RuntimeError(f"item not found for {thread_ts}") - except: - LOGGER.exception("") + except Exception as error_msg: + LOGGER.exception(str(error_msg)) if __name__ == "__main__": diff --git a/config/.env b/config/.env new file mode 100644 index 0000000..046736b --- /dev/null +++ b/config/.env @@ -0,0 +1,14 @@ +context = 'I am Satyam Chatrola. + My personal email id is satyamchatrola14@gmail.com. + + When I first receive an email, I want you to analyze it respond with the analysis in json as follows: + + * "author" + * "time_received" in iso format + ** "urgent": bool + ** "important": bool + ** "spam": bool + * "summary" : summary of the content + * "action": proposed next action + + Then answer my follow up questions in markdown supported by slack api.' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 71a0d47..2a18e85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,8 @@ dependencies = [ "aiohttp>=3.10.8", "asyncio>=3.4.3", "humanreadable>=0.4.0", + "pytest>=8.3.3", + "pytest-mock>=3.14.0", ] [tool.uv] diff --git a/src/__init__.py b/src/__init__.py index e69de29..5c21d1e 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -0,0 +1 @@ +# init file diff --git a/src/gmail/API.py b/src/gmail/API.py new file mode 100644 index 0000000..62b16d1 --- /dev/null +++ b/src/gmail/API.py @@ -0,0 +1,34 @@ +from typing import Optional, Generator +from datetime import datetime +from .client import Gmail + +# Create a single instance of Gmail +_gmail = Gmail() + + +def query( + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None, + max_results: int = 10000, +) -> Generator[dict, None, None]: + """Query Gmail messages with filters.""" + return _gmail.query(start_date, end_date, max_results) + + +def get_message(message_id: str) -> dict: + """Fetch a specific email message by ID.""" + return _gmail.get_message(message_id) + + +def get_labels() -> list[dict]: + """Fetch all labels in the user's mailbox.""" + return _gmail.get_labels() + + +def search(query: str, max_results: int = 100) -> Generator[dict, None, None]: + """Search emails using Gmail's query syntax.""" + return _gmail.search(query, max_results) + + +# Export only the public functions +__all__ = ["query", "get_message", "get_labels", "search"] diff --git a/src/gmail/client.py b/src/gmail/client.py index cf8c398..e272d9c 100644 --- a/src/gmail/client.py +++ b/src/gmail/client.py @@ -1,19 +1,41 @@ import os.path from typing import ClassVar +from datetime import datetime +from typing import Optional, Generator, Any from google.auth.transport.requests import Request from google.oauth2.credentials import Credentials from google_auth_oauthlib.flow import InstalledAppFlow from googleapiclient.discovery import build import os +import logging + +LOGGER = logging.getLogger(__name__) class Gmail: + """A class to interact with Gmail using the Gmail API. + + This class provides methods to authenticate with Gmail and perform + various operations like fetching and filtering emails. + + """ + SCOPES: ClassVar[list[str]] = ["https://www.googleapis.com/auth/gmail.readonly"] def __init__(self) -> None: + """Initialize Gmail client with authenticated credentials.""" self._client = self.authenticate() - def authenticate(self): + def authenticate(self) -> Any: + """Authenticate with Gmail API using OAuth 2.0. + + Returns: + An authenticated Gmail API service object. + + Raises: + FileNotFoundError: If credentials file is not found. + ValueError: If authentication fails. + """ creds = None if os.path.exists("token.json"): creds = Credentials.from_authorized_user_file("token.json", self.SCOPES) @@ -29,17 +51,90 @@ def authenticate(self): token.write(creds.to_json()) return build("gmail", "v1", credentials=creds, cache_discovery=False) - def query(self): + # we can add start_date, end_date to fetch commands in slack bot + def query( + self, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None, + max_results: int = 10000, + ) -> Generator[dict, None, None]: + """Query Gmail messages with filters. + + Args: + start_date: Only return messages after this date + end_date: Only return messages before this date + max_results: Maximum number of messages to return + + Yields: + Dict containing message metadata + + Search operators used: + after:YYYY/MM/DD - Messages after date + before:YYYY/MM/DD - Messages before date + """ + query_parts = [] + + if start_date: + query_parts.append(f"after:{start_date.strftime('%Y/%m/%d')}") + + if end_date: + query_parts.append(f"before:{end_date.strftime('%Y/%m/%d')}") + + query_string = " AND ".join(query_parts) if query_parts else "" + results = ( self._client.users() .messages() - .list(userId="me", labelIds=["INBOX"], maxResults=10000) + .list(userId="me", maxResults=max_results, q=query_string) .execute() ) - messages = results.get("messages", []) + messages = results.get("messages", []) for message in messages: yield message - def get_message(self, message_id: str): + def get_message(self, message_id: str) -> dict: + """Fetch a specific email message by ID. + + Args: + message_id: The ID of the message to fetch. + + Returns: + Dict containing the message details. + + """ return self._client.users().messages().get(userId="me", id=message_id).execute() + + def get_labels(self) -> list[dict]: + """Fetch all labels in the user's mailbox. + + Returns: + List of dictionaries containing label information. + """ + results = self._client.users().labels().list(userId="me").execute() + return results.get("labels", []) + + def search( + self, + query: str, + max_results: int = 100, + ) -> Generator[dict, None, None]: + """Search emails using Gmail's query syntax. + + Args: + query: Gmail search query string. + max_results: Maximum number of results to return. + + Yields: + Dict containing message information. + """ + results = ( + self._client.users() + .messages() + .list(userId="me", maxResults=max_results, q=query) + .execute() + ) + + messages = results.get("messages", []) + for message in messages: + yield message diff --git a/src/gmail/models.py b/src/gmail/models.py index b1b0643..97addb3 100644 --- a/src/gmail/models.py +++ b/src/gmail/models.py @@ -41,7 +41,7 @@ def body(self) -> str: if " None: - client = WebClient(token=os.getenv("SLACK_BOT_TOKEN")) + # replaced client with self.client as client was not used anywhere + self.client = WebClient(token=os.getenv("SLACK_BOT_TOKEN")) def send(self, thread: str | None, text: str) -> None: r = self.client.chat_postMessage( diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/test/__init__.py @@ -0,0 +1 @@ + diff --git a/test/test_gmail.py b/test/test_gmail.py new file mode 100644 index 0000000..7b2d9e2 --- /dev/null +++ b/test/test_gmail.py @@ -0,0 +1,134 @@ +# pytest test/test_gmail.py -v +import pytest +from datetime import datetime +from unittest.mock import Mock, patch +from src.gmail.client import Gmail + + +@pytest.fixture +def mock_gmail_service(): + """Fixture to mock the Gmail API service""" + with patch("src.gmail.client.build") as mock_build: + mock_service = Mock() + mock_build.return_value = mock_service + yield mock_service + + +@pytest.fixture +def gmail_client(mock_gmail_service): + """Fixture to create a Gmail client with mocked authentication""" + with patch("src.gmail.client.Credentials") as mock_creds: + mock_creds.from_authorized_user_file.return_value = Mock(valid=True) + return Gmail() + + +def test_query_method(gmail_client, mock_gmail_service): + # Mock response data + mock_messages = { + "messages": [ + {"id": "123", "threadId": "thread123"}, + {"id": "456", "threadId": "thread456"}, + ] + } + + # Setup the mock chain + mock_service = mock_gmail_service + mock_service.users.return_value.messages.return_value.list.return_value.execute.return_value = mock_messages + + # Test the query method + start_date = datetime(2024, 10, 30) + end_date = datetime(2024, 10, 31) + messages = list( + gmail_client.query(start_date=start_date, end_date=end_date, max_results=5) + ) + + # Assertions + assert len(messages) == 2 + assert messages[0]["id"] == "123" + assert messages[1]["id"] == "456" + + # Verify the correct query parameters were used + mock_service.users.return_value.messages.return_value.list.assert_called_with( + userId="me", maxResults=5, q="after:2024/10/30 AND before:2024/10/31" + ) + + +def test_get_message(gmail_client, mock_gmail_service): + # Mock response data + mock_message = { + "id": "123", + "payload": { + "headers": [ + {"name": "Subject", "value": "Test Subject"}, + {"name": "From", "value": "sender@example.com"}, + ] + }, + } + + # Setup the mock + mock_service = mock_gmail_service + mock_service.users.return_value.messages.return_value.get.return_value.execute.return_value = mock_message + + # Test get_message + result = gmail_client.get_message("123") + + # Assertions + assert result == mock_message + assert result["payload"]["headers"][0]["value"] == "Test Subject" + + # Verify the correct parameters were used + mock_service.users.return_value.messages.return_value.get.assert_called_with( + userId="me", id="123" + ) + + +def test_search_method(gmail_client, mock_gmail_service): + # Mock response data + mock_search_results = { + "messages": [ + {"id": "789", "threadId": "thread789"}, + {"id": "012", "threadId": "thread012"}, + ] + } + + # Setup the mock + mock_service = mock_gmail_service + mock_service.users.return_value.messages.return_value.list.return_value.execute.return_value = mock_search_results + + # Test search method + search_query = "test query" + results = list(gmail_client.search(search_query, max_results=2)) + + # Assertions + assert len(results) == 2 + assert results[0]["id"] == "789" + assert results[1]["id"] == "012" + + # Verify the correct parameters were used + mock_service.users.return_value.messages.return_value.list.assert_called_with( + userId="me", maxResults=2, q=search_query + ) + + +def test_get_labels(gmail_client, mock_gmail_service): + # Mock response data + mock_labels = { + "labels": [{"id": "INBOX", "name": "INBOX"}, {"id": "SENT", "name": "SENT"}] + } + + # Setup the mock + mock_service = mock_gmail_service + mock_service.users.return_value.labels.return_value.list.return_value.execute.return_value = mock_labels + + # Test get_labels + results = gmail_client.get_labels() + + # Assertions + assert len(results) == 2 + assert results[0]["name"] == "INBOX" + assert results[1]["name"] == "SENT" + + # Verify the correct parameters were used + mock_service.users.return_value.labels.return_value.list.assert_called_with( + userId="me" + ) diff --git a/test/test_gmail_api.py b/test/test_gmail_api.py new file mode 100644 index 0000000..963be2a --- /dev/null +++ b/test/test_gmail_api.py @@ -0,0 +1,35 @@ +# In order to test Google Auth, actual message in your Gmail and API. You can +# replace specific info (like start/end date or message_id) in this file. +# Then compare the output with the actual output in your Gmail. + +from datetime import datetime +from src.gmail.API import query, get_message, get_labels, search + +## Query Messages Example +# Get messages from a specific date range +start_date = datetime(2024, 10, 31) +end_date = datetime(2024, 11, 1) +messages = query(start_date=start_date, end_date=end_date, max_results=5) +for msg in messages: + print(f"Message ID: {msg['id']}, Thread ID: {msg['threadId']}") + +## Get Single Message Example +# Fetch details of a specific message +message_id = "192e80bb14e565c5" +message = get_message(message_id) +print(f"Subject: {message.get('subject')}") +print(f"Snippet: {message.get('snippet')}") + +## Get Labels Example +# Fetch all Gmail labels +labels = get_labels() +for label in labels: + print(f"Label Name: {label['name']}, Label ID: {label['id']}") + +## Search Example +# Search for specific emails +search_query = "aaa" +search_results = search(query=search_query, max_results=5) +for result in search_results: + message_details = get_message(result["id"]) + print(f"Found message: {message_details.get('snippet')}") diff --git a/uv.lock b/uv.lock index cf29625..1670d2d 100644 --- a/uv.lock +++ b/uv.lock @@ -514,6 +514,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442 }, ] +[[package]] +name = "iniconfig" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", size = 4646 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", size = 5892 }, +] + [[package]] name = "jiter" version = "0.5.0" @@ -550,6 +559,8 @@ dependencies = [ { name = "humanreadable" }, { name = "openai" }, { name = "pydantic" }, + { name = "pytest" }, + { name = "pytest-mock" }, { name = "slack-bolt" }, { name = "slack-sdk" }, { name = "sqlalchemy" }, @@ -577,6 +588,8 @@ requires-dist = [ { name = "humanreadable", specifier = ">=0.4.0" }, { name = "openai", specifier = ">=1.50.2" }, { name = "pydantic", specifier = ">=2.9.2" }, + { name = "pytest", specifier = ">=8.3.3" }, + { name = "pytest-mock", specifier = ">=3.14.0" }, { name = "slack-bolt", specifier = ">=1.20.1" }, { name = "slack-sdk", specifier = ">=3.33.1" }, { name = "sqlalchemy", specifier = ">=2.0.35" }, @@ -736,6 +749,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb", size = 18439 }, ] +[[package]] +name = "pluggy" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", size = 67955 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556 }, +] + [[package]] name = "proto-plus" version = "1.24.0" @@ -850,6 +872,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/0c/0e3c05b1c87bb6a1c76d281b0f35e78d2d80ac91b5f8f524cebf77f51049/pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c", size = 104100 }, ] +[[package]] +name = "pytest" +version = "8.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/6c/62bbd536103af674e227c41a8f3dcd022d591f6eed5facb5a0f31ee33bbc/pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181", size = 1442487 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/77/7440a06a8ead44c7757a64362dd22df5760f9b12dc5f11b6188cd2fc27a0/pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2", size = 342341 }, +] + +[[package]] +name = "pytest-mock" +version = "3.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/90/a955c3ab35ccd41ad4de556596fa86685bf4fc5ffcc62d22d856cfd4e29a/pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0", size = 32814 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/3b/b26f90f74e2986a82df6e7ac7e319b8ea7ccece1caec9f8ab6104dc70603/pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f", size = 9863 }, +] + [[package]] name = "requests" version = "2.32.3"