From 6e507bd8c0512835153410eff82d6b82bee44edc Mon Sep 17 00:00:00 2001 From: jtyoung84 <104453205+jtyoung84@users.noreply.github.com> Date: Mon, 30 Oct 2023 09:55:49 -0700 Subject: [PATCH] Feat 134 smartsheet client (#150) * feat: adds smartsheet client * feat: makes get async * feat: drop support for python 3.7 --- .github/workflows/ci.yml | 2 +- pyproject.toml | 5 +- src/aind_metadata_service/labtracks/client.py | 7 +- src/aind_metadata_service/server.py | 5 + .../sharepoint/nsb2023/mapping.py | 18 +- .../smartsheet/__init__.py | 1 + .../smartsheet/client.py | 75 ++++ tests/labtracks/test_response_handler.py | 4 +- tests/resources/smartsheet/test_sheet.json | 359 ++++++++++++++++++ tests/smartsheet/__init__.py | 1 + tests/smartsheet/test_client.py | 90 +++++ 11 files changed, 549 insertions(+), 18 deletions(-) create mode 100644 src/aind_metadata_service/smartsheet/__init__.py create mode 100644 src/aind_metadata_service/smartsheet/client.py create mode 100644 tests/resources/smartsheet/test_sheet.json create mode 100644 tests/smartsheet/__init__.py create mode 100644 tests/smartsheet/test_client.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84139d61..de8af6f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.7', '3.8', '3.9', '3.10' ] + python-version: [ '3.8', '3.9', '3.10', '3.11' ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/pyproject.toml b/pyproject.toml index c3b3c216..51f11cc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ license = {text = "MIT"} authors = [ {name = "Allen Institute for Neural Dynamics"} ] -requires-python = ">=3.7" +requires-python = ">=3.8" classifiers = [ "Programming Language :: Python :: 3" ] @@ -36,6 +36,7 @@ server = [ 'pydantic<2.0', 'pyodbc', 'office365-rest-python-client==2.4.1', + 'smartsheet-python-sdk==3.0.2', 'fastapi', 'uvicorn[standard]', 'python-dateutil' @@ -54,7 +55,7 @@ readme = {file = ["README.md"]} [tool.black] line-length = 79 -target_version = ['py36'] +target_version = ['py38'] exclude = ''' ( diff --git a/src/aind_metadata_service/labtracks/client.py b/src/aind_metadata_service/labtracks/client.py index 2330e899..c18d8e6d 100644 --- a/src/aind_metadata_service/labtracks/client.py +++ b/src/aind_metadata_service/labtracks/client.py @@ -349,8 +349,11 @@ def _map_housing( room_id = None if room_id is None or int(room_id) < 0 else room_id cage_id = None if cage_id is None or int(cage_id) < 0 else cage_id - return Housing.construct(room_id=room_id, - cage_id=cage_id) if room_id is not None or cage_id is not None else None + return ( + Housing.construct(room_id=room_id, cage_id=cage_id) + if room_id is not None or cage_id is not None + else None + ) def map_response_to_subject(self, results: List[dict]) -> List[Subject]: """ diff --git a/src/aind_metadata_service/server.py b/src/aind_metadata_service/server.py index 4a0c970b..e7faf674 100644 --- a/src/aind_metadata_service/server.py +++ b/src/aind_metadata_service/server.py @@ -14,10 +14,15 @@ SharePointClient, SharepointSettings, ) +from aind_metadata_service.smartsheet.client import SmartsheetSettings +# TODO: Move client instantiation when the server starts instead of creating +# one for each request? sharepoint_settings = SharepointSettings() labtracks_settings = LabTracksSettings() +smartsheet_settings = SmartsheetSettings() + app = FastAPI() app.add_middleware( diff --git a/src/aind_metadata_service/sharepoint/nsb2023/mapping.py b/src/aind_metadata_service/sharepoint/nsb2023/mapping.py index c0c1a7b2..0e9dc602 100644 --- a/src/aind_metadata_service/sharepoint/nsb2023/mapping.py +++ b/src/aind_metadata_service/sharepoint/nsb2023/mapping.py @@ -2955,8 +2955,7 @@ def burr_hole_info(self, burr_hole_num: int) -> BurrHoleInfo: alternating_current=self.aind_inj1_alternating_time, inj_duration=self.aind_inj1_ionto_time, inj_volume=self._map_burr_hole_volume( - vol=self.aind_inj1volperdepth, - dv=coordinate_depth + vol=self.aind_inj1volperdepth, dv=coordinate_depth ), fiber_implant_depth=self.aind_fiber_implant1_dv, ) @@ -2979,8 +2978,7 @@ def burr_hole_info(self, burr_hole_num: int) -> BurrHoleInfo: alternating_current=self.aind_inj2_alternating_time, inj_duration=self.aind_inj2_ionto_time, inj_volume=self._map_burr_hole_volume( - vol=self.aind_inj2volperdepth, - dv=coordinate_depth + vol=self.aind_inj2volperdepth, dv=coordinate_depth ), fiber_implant_depth=self.aind_fiber_implant2_dv, ) @@ -3003,8 +3001,7 @@ def burr_hole_info(self, burr_hole_num: int) -> BurrHoleInfo: alternating_current=self.aind_inj3_alternating_time, inj_duration=self.aind_inj3_ionto_time, inj_volume=self._map_burr_hole_volume( - vol=self.aind_inj3volperdepth, - dv=coordinate_depth + vol=self.aind_inj3volperdepth, dv=coordinate_depth ), fiber_implant_depth=self.aind_fiber_implant3_d_x00, ) @@ -3027,8 +3024,7 @@ def burr_hole_info(self, burr_hole_num: int) -> BurrHoleInfo: alternating_current=self.aind_inj4_alternating_time, inj_duration=self.aind_inj4_ionto_time, inj_volume=self._map_burr_hole_volume( - vol=self.aind_inj4volperdepth, - dv=coordinate_depth + vol=self.aind_inj4volperdepth, dv=coordinate_depth ), fiber_implant_depth=self.aind_fiber_implant4_d_x00, ) @@ -3051,8 +3047,7 @@ def burr_hole_info(self, burr_hole_num: int) -> BurrHoleInfo: alternating_current=self.aind_inj5_alternating_time, inj_duration=self.aind_inj5_ionto_time, inj_volume=self._map_burr_hole_volume( - vol=self.aind_inj5volperdepth, - dv=coordinate_depth + vol=self.aind_inj5volperdepth, dv=coordinate_depth ), fiber_implant_depth=self.aind_fiber_implant5_d_x00, ) @@ -3075,8 +3070,7 @@ def burr_hole_info(self, burr_hole_num: int) -> BurrHoleInfo: alternating_current=self.aind_inj6_alternating_time, inj_duration=self.aind_inj6_ionto_time, inj_volume=self._map_burr_hole_volume( - vol=self.aind_inj6volperdepth, - dv=coordinate_depth + vol=self.aind_inj6volperdepth, dv=coordinate_depth ), fiber_implant_depth=self.aind_fiber_implant6_d_x00, ) diff --git a/src/aind_metadata_service/smartsheet/__init__.py b/src/aind_metadata_service/smartsheet/__init__.py new file mode 100644 index 00000000..7e8b28ad --- /dev/null +++ b/src/aind_metadata_service/smartsheet/__init__.py @@ -0,0 +1 @@ +"""Package to retrieve information from Smartsheet""" diff --git a/src/aind_metadata_service/smartsheet/client.py b/src/aind_metadata_service/smartsheet/client.py new file mode 100644 index 00000000..c416345a --- /dev/null +++ b/src/aind_metadata_service/smartsheet/client.py @@ -0,0 +1,75 @@ +"""Module to instantiate a client to connect to Smartsheet and provide helpful +methods to retrieve data.""" + +import logging +from typing import Optional + +from pydantic import BaseSettings, Extra, Field, SecretStr +from smartsheet import Smartsheet + +from aind_metadata_service import __version__ + + +class SmartsheetSettings(BaseSettings): + """Configuration class. Mostly a wrapper around smartsheet.Smartsheet + class constructor arguments.""" + + access_token: SecretStr = Field( + ..., description="API token can be created in Smartsheet UI" + ) + sheet_id: int = Field( + ..., + description=( + "Sheet ID to query. Can be found in Smartsheet under " + "File | Properties." + ), + ) + user_agent: Optional[str] = Field( + default=f"AIND_Metadata_Service/{__version__}", + description=( + "The user agent to use when making requests. " + "Helps identify requests coming from this app." + ), + ) + max_connections: int = Field( + default=8, description="Maximum connection pool size." + ) + + class Config: + """Set env prefix and forbid extra fields.""" + + env_prefix = "SMARTSHEET_" + extra = Extra.forbid + + +class SmartSheetClient: + """Main client to connect to a Smartsheet sheet. Requires an API token + and the sheet id.""" + + def __init__(self, smartsheet_settings: SmartsheetSettings): + """ + Class constructor + Parameters + ---------- + smartsheet_settings : SmartsheetSettings + """ + self.smartsheet_settings = smartsheet_settings + self.smartsheet_client = Smartsheet( + user_agent=self.smartsheet_settings.user_agent, + max_connections=self.smartsheet_settings.max_connections, + access_token=( + self.smartsheet_settings.access_token.get_secret_value() + ), + ) + + async def get_sheet(self) -> dict: + """Retrieve the sheet defined by the settings sheet_id.""" + try: + smartsheet_response = self.smartsheet_client.Sheets.get_sheet( + self.smartsheet_settings.sheet_id + ) + smartsheet_json = smartsheet_response.to_json() + return smartsheet_json + except Exception as e: + logging.error(repr(e)) + raise Exception(e) diff --git a/tests/labtracks/test_response_handler.py b/tests/labtracks/test_response_handler.py index 5e9f7b55..cb4400fa 100644 --- a/tests/labtracks/test_response_handler.py +++ b/tests/labtracks/test_response_handler.py @@ -241,7 +241,9 @@ def test_map_housing(self): housing3 = Housing.construct(cage_id="1234") housing4 = Housing.construct(room_id="000", cage_id="1234") - subject_housing1 = self.rh._map_housing(room_id="-99999999999", cage_id=None) + subject_housing1 = self.rh._map_housing( + room_id="-99999999999", cage_id=None + ) subject_housing2 = self.rh._map_housing(room_id="000", cage_id=None) subject_housing3 = self.rh._map_housing(room_id=None, cage_id="1234") subject_housing4 = self.rh._map_housing(room_id="000", cage_id="1234") diff --git a/tests/resources/smartsheet/test_sheet.json b/tests/resources/smartsheet/test_sheet.json new file mode 100644 index 00000000..3fcaf6fa --- /dev/null +++ b/tests/resources/smartsheet/test_sheet.json @@ -0,0 +1,359 @@ +{ + "accessLevel": "EDITOR_SHARE", + "columns": [ + { + "id": 8808571546324868, + "index": 0, + "options": [ + "Specimen Procedures", + "Surgical Procedures", + "Ephys Techniques", + "Ophys Techniques", + "Imaging Techniques" + ], + "title": "Protocol Type", + "type": "PICKLIST", + "validation": true, + "version": 0, + "width": 150 + }, + { + "id": 6725511871942532, + "index": 1, + "title": "Procedure name", + "type": "TEXT_NUMBER", + "validation": false, + "version": 0, + "width": 150 + }, + { + "id": 6394619537346436, + "index": 2, + "primary": true, + "title": "Protocol name", + "type": "TEXT_NUMBER", + "validation": false, + "version": 0, + "width": 444 + }, + { + "id": 4142819723661188, + "index": 3, + "title": "DOI", + "type": "TEXT_NUMBER", + "validation": false, + "version": 0, + "width": 225 + }, + { + "id": 8646419351031684, + "index": 4, + "title": "Version", + "type": "TEXT_NUMBER", + "validation": false, + "version": 0, + "width": 96 + }, + { + "id": 223584756649860, + "index": 5, + "title": "Protocol collection", + "type": "CHECKBOX", + "validation": true, + "version": 0, + "width": 75 + } + ], + "createdAt": "2023-09-21T19:21:18+00:00Z", + "dependenciesEnabled": false, + "effectiveAttachmentOptions": [ + "LINK", + "GOOGLE_DRIVE", + "DROPBOX", + "EVERNOTE", + "EGNYTE", + "BOX_COM", + "FILE", + "ONEDRIVE" + ], + "ganttEnabled": false, + "hasSummaryFields": false, + "id": 7478444220698500, + "modifiedAt": "2023-10-02T21:53:15+00:00Z", + "name": "Published Protocols", + "permalink": "https://app.smartsheet.com/sheets/3XQgWWrXW3mh46xmXCw5Q9GfqQmmP4xwF9Cjfqg1", + "resourceManagementEnabled": false, + "rows": [ + { + "cells": [ + { + "columnId": 8808571546324868, + "displayValue": "Specimen Procedures", + "value": "Specimen Procedures" + }, + { + "columnId": 6725511871942532, + "displayValue": "Immunolabeling", + "value": "Immunolabeling" + }, + { + "columnId": 6394619537346436, + "displayValue": "Immunolabeling of a Whole Mouse Brain", + "value": "Immunolabeling of a Whole Mouse Brain" + }, + { + "columnId": 4142819723661188, + "displayValue": "dx.doi.org/10.17504/protocols.io.ewov1okwylr2/v1", + "value": "dx.doi.org/10.17504/protocols.io.ewov1okwylr2/v1" + }, + { + "columnId": 8646419351031684, + "displayValue": "1", + "value": 1.0 + }, + { + "columnId": 223584756649860 + } + ], + "createdAt": "2023-09-21T19:31:40+00:00Z", + "expanded": true, + "id": 5216174748766084, + "modifiedAt": "2023-10-02T20:20:50+00:00Z", + "rowNumber": 1 + }, + { + "cells": [ + { + "columnId": 8808571546324868, + "displayValue": "Specimen Procedures", + "value": "Specimen Procedures" + }, + { + "columnId": 6725511871942532, + "displayValue": "Delipidation", + "value": "Delipidation" + }, + { + "columnId": 6394619537346436, + "displayValue": "Tetrahydrofuran and Dichloromethane Delipidation of a Whole Mouse Brain", + "value": "Tetrahydrofuran and Dichloromethane Delipidation of a Whole Mouse Brain" + }, + { + "columnId": 4142819723661188, + "displayValue": "dx.doi.org/10.17504/protocols.io.36wgqj1kxvk5/v1", + "value": "dx.doi.org/10.17504/protocols.io.36wgqj1kxvk5/v1" + }, + { + "columnId": 8646419351031684, + "displayValue": "1", + "value": 1.0 + }, + { + "columnId": 223584756649860 + } + ], + "createdAt": "2023-09-21T19:31:40+00:00Z", + "expanded": true, + "id": 2964374935080836, + "modifiedAt": "2023-10-02T20:20:50+00:00Z", + "rowNumber": 2, + "siblingId": 5216174748766084 + }, + { + "cells": [ + { + "columnId": 8808571546324868, + "displayValue": "Specimen Procedures", + "value": "Specimen Procedures" + }, + { + "columnId": 6725511871942532, + "displayValue": "Delipidation", + "value": "Delipidation" + }, + { + "columnId": 6394619537346436, + "displayValue": "Aqueous (SBiP) Delipidation of a Whole Mouse Brain", + "value": "Aqueous (SBiP) Delipidation of a Whole Mouse Brain" + }, + { + "columnId": 4142819723661188, + "displayValue": "dx.doi.org/10.17504/protocols.io.n2bvj81mwgk5/v1", + "value": "dx.doi.org/10.17504/protocols.io.n2bvj81mwgk5/v1" + }, + { + "columnId": 8646419351031684, + "displayValue": "1", + "value": 1.0 + }, + { + "columnId": 223584756649860 + } + ], + "createdAt": "2023-09-21T19:31:40+00:00Z", + "expanded": true, + "id": 7467974562451332, + "modifiedAt": "2023-10-02T20:20:50+00:00Z", + "rowNumber": 3, + "siblingId": 2964374935080836 + }, + { + "cells": [ + { + "columnId": 8808571546324868 + }, + { + "columnId": 6725511871942532 + }, + { + "columnId": 6394619537346436, + "displayValue": "Whole Mouse Brain Delipidation, Immunolabeling, and Expansion Microscopy", + "value": "Whole Mouse Brain Delipidation, Immunolabeling, and Expansion Microscopy" + }, + { + "columnId": 4142819723661188, + "displayValue": "dx.doi.org/10.17504/protocols.io.n92ldpwjxl5b/v1", + "value": "dx.doi.org/10.17504/protocols.io.n92ldpwjxl5b/v1" + }, + { + "columnId": 8646419351031684, + "displayValue": "1", + "value": 1.0 + }, + { + "columnId": 223584756649860, + "value": true + } + ], + "createdAt": "2023-09-21T19:31:40+00:00Z", + "expanded": true, + "id": 1838475028238212, + "modifiedAt": "2023-09-21T19:31:40+00:00Z", + "rowNumber": 4, + "siblingId": 7467974562451332 + }, + { + "cells": [ + { + "columnId": 8808571546324868, + "displayValue": "Surgical Procedures", + "value": "Surgical Procedures" + }, + { + "columnId": 6725511871942532, + "displayValue": "Injection Nanoject", + "value": "Injection Nanoject" + }, + { + "columnId": 6394619537346436, + "displayValue": "Injection of Viral Tracers by Nanoject V.3", + "value": "Injection of Viral Tracers by Nanoject V.3" + }, + { + "columnId": 4142819723661188, + "displayValue": "dx.doi.org/10.17504/protocols.io.bgpujvnw", + "value": "dx.doi.org/10.17504/protocols.io.bgpujvnw" + }, + { + "columnId": 8646419351031684, + "displayValue": "3", + "value": 3.0 + }, + { + "columnId": 223584756649860 + } + ], + "createdAt": "2023-09-21T19:31:40+00:00Z", + "expanded": true, + "id": 6342074655608708, + "modifiedAt": "2023-10-02T20:20:50+00:00Z", + "rowNumber": 5, + "siblingId": 1838475028238212 + }, + { + "cells": [ + { + "columnId": 8808571546324868, + "displayValue": "Surgical Procedures", + "value": "Surgical Procedures" + }, + { + "columnId": 6725511871942532, + "displayValue": "Injection Iontophoresis", + "value": "Injection Iontophoresis" + }, + { + "columnId": 6394619537346436, + "displayValue": "Stereotaxic Surgery for Delivery of Tracers by Iontophoresis V.3", + "value": "Stereotaxic Surgery for Delivery of Tracers by Iontophoresis V.3" + }, + { + "columnId": 4142819723661188, + "displayValue": "dx.doi.org/10.17504/protocols.io.bgpvjvn6", + "value": "dx.doi.org/10.17504/protocols.io.bgpvjvn6" + }, + { + "columnId": 8646419351031684, + "displayValue": "3", + "value": 3.0 + }, + { + "columnId": 223584756649860 + } + ], + "createdAt": "2023-09-21T19:31:40+00:00Z", + "expanded": true, + "id": 4090274841923460, + "modifiedAt": "2023-10-02T20:20:50+00:00Z", + "rowNumber": 6, + "siblingId": 6342074655608708 + }, + { + "cells": [ + { + "columnId": 8808571546324868, + "displayValue": "Surgical Procedures", + "value": "Surgical Procedures" + }, + { + "columnId": 6725511871942532, + "displayValue": "Perfusion", + "value": "Perfusion" + }, + { + "columnId": 6394619537346436, + "displayValue": "Mouse Cardiac Perfusion Fixation and Brain Collection V.5", + "value": "Mouse Cardiac Perfusion Fixation and Brain Collection V.5" + }, + { + "columnId": 4142819723661188, + "displayValue": "dx.doi.org/10.17504/protocols.io.bg5vjy66", + "value": "dx.doi.org/10.17504/protocols.io.bg5vjy66" + }, + { + "columnId": 8646419351031684, + "displayValue": "5", + "value": 5.0 + }, + { + "columnId": 223584756649860 + } + ], + "createdAt": "2023-09-21T19:31:40+00:00Z", + "expanded": true, + "id": 8593874469293956, + "modifiedAt": "2023-10-02T20:20:50+00:00Z", + "rowNumber": 7, + "siblingId": 4090274841923460 + } + ], + "totalRowCount": 7, + "userPermissions": { + "summaryPermissions": "READ_WRITE" + }, + "userSettings": { + "criticalPathEnabled": false, + "displaySummaryTasks": true + }, + "version": 6 +} diff --git a/tests/smartsheet/__init__.py b/tests/smartsheet/__init__.py new file mode 100644 index 00000000..4abf460a --- /dev/null +++ b/tests/smartsheet/__init__.py @@ -0,0 +1 @@ +"""Tests Smartsheet client package""" diff --git a/tests/smartsheet/test_client.py b/tests/smartsheet/test_client.py new file mode 100644 index 00000000..0b22df29 --- /dev/null +++ b/tests/smartsheet/test_client.py @@ -0,0 +1,90 @@ +"""Module to test Smartsheet client class""" + +import json +import os +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from aind_metadata_service.smartsheet.client import ( + SmartSheetClient, + SmartsheetSettings, +) + +TEST_DIR = Path(os.path.dirname(os.path.realpath(__file__))) / ".." +EXAMPLE_PATH = TEST_DIR / "resources" / "smartsheet" / "test_sheet.json" + + +class TestSmartsheetSettings(unittest.TestCase): + """Class to test methods for SmartsheetSettings.""" + + EXAMPLE_ENV_VAR1 = { + "SMARTSHEET_ACCESS_TOKEN": "abc-123", + "SMARTSHEET_SHEET_ID": "123456789", + "SMARTSHEET_MAX_CONNECTIONS": "16", + } + + @patch.dict(os.environ, EXAMPLE_ENV_VAR1, clear=True) + def test_settings_set_from_env_vars(self): + """Tests that the settings can be set from env vars.""" + + settings1 = SmartsheetSettings() + settings2 = SmartsheetSettings(user_agent="Agent0", max_connections=8) + self.assertEqual("abc-123", settings1.access_token.get_secret_value()) + self.assertEqual(123456789, settings1.sheet_id) + self.assertTrue("AIND_Metadata_Service" in settings1.user_agent) + self.assertEqual(16, settings1.max_connections) + self.assertEqual("abc-123", settings2.access_token.get_secret_value()) + self.assertEqual(123456789, settings2.sheet_id) + self.assertEqual("Agent0", settings2.user_agent) + self.assertEqual(8, settings2.max_connections) + + +class TestSmartsheetClient(unittest.IsolatedAsyncioTestCase): + """Class to test methods for SmartsheetClient.""" + + @classmethod + def setUpClass(cls): + """Load json files before running tests.""" + with open(EXAMPLE_PATH, "r") as f: + contents = json.load(f) + cls.example_sheet = contents + + @patch("smartsheet.sheets.Sheets.get_sheet") + async def test_get_sheet_success(self, mock_get_sheet: MagicMock): + """Tests successful sheet return response""" + mock_get_sheet.return_value.to_json.return_value = self.example_sheet + settings = SmartsheetSettings( + access_token="abc-123", sheet_id=7478444220698500 + ) + client = SmartSheetClient(smartsheet_settings=settings) + sheet = await client.get_sheet() + self.assertEqual("Published Protocols", sheet["name"]) + self.assertEqual(7478444220698500, sheet["id"]) + self.assertEqual(6, sheet["version"]) + self.assertEqual(7, sheet["totalRowCount"]) + + @patch("smartsheet.sheets.Sheets.get_sheet") + @patch("logging.error") + async def test_get_sheet_error( + self, mock_log_error: MagicMock, mock_get_sheet: MagicMock + ): + """Tests sheet return error response""" + mock_get_sheet.side_effect = MagicMock( + side_effect=Exception("Error connecting to server") + ) + settings = SmartsheetSettings( + access_token="abc-123", sheet_id=7478444220698500 + ) + client = SmartSheetClient(smartsheet_settings=settings) + with self.assertRaises(Exception) as e: + _ = await client.get_sheet() + + self.assertEqual("Error connecting to server", str(e.exception)) + mock_log_error.assert_called_once_with( + "Exception('Error connecting to server')" + ) + + +if __name__ == "__main__": + unittest.main()