From 63c64497fdd446c7ac1c658d0cf9f24cf7b55fc1 Mon Sep 17 00:00:00 2001 From: thomasyu888 Date: Thu, 20 Jan 2022 19:03:11 +0800 Subject: [PATCH 1/4] Add .gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1c07fe3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__ +*.egg-info +dist +build +.vscode From 5568e7127f740fac124e2ee017b557db455188a0 Mon Sep 17 00:00:00 2001 From: thomasyu888 Date: Thu, 20 Jan 2022 19:11:09 +0800 Subject: [PATCH 2/4] Return a list of synapse ids instead of dataframe --- synapsemonitor/monitor.py | 63 ++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index a4100a7..463a1c7 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -68,39 +68,57 @@ def _render_fileview( def _find_modified_entities_fileview( syn: Synapse, syn_id: str, days: int = 1 -) -> pd.DataFrame: - """Performs query to find modified entities in id and render columns - These modified entities include newly uploaded ones +) -> list: + """Finds entities scoped in a fileview modified in the past N number of days Args: syn: Synapse connection - view_id: Synapse View Id - epochtime: Epoch time in milliseconds + syn_id: Synapse Fileview Id + days: N number of days Returns: - Dataframe with updated entities + List of synapse ids """ # Update the view # _force_update_view(syn, view_id) query = ( - "select id, name, currentVersion, modifiedOn, modifiedBy, " - f"createdOn, projectId, type from {syn_id} where " + f"select id from {syn_id} where " f"modifiedOn > unix_timestamp(NOW() - INTERVAL {days} DAY)*1000" ) results = syn.tableQuery(query) resultsdf = results.asDataFrame() - return _render_fileview(syn, viewdf=resultsdf) + return resultsdf['id'].tolist() def _find_modified_entities_file( syn: Synapse, syn_id: str, days: int = 1 -) -> pd.DataFrame: +) -> list: + """Determines if entity was modified in the past N number of days + + Args: + syn: Synapse connection + syn_id: Synapse File Id + days: N number of days + + Returns: + List of synapse ids + """ raise NotImplementedError("Files not supported yet") def _find_modified_entities_container( syn: Synapse, syn_id: str, days: int = 1 -) -> pd.DataFrame: +) -> list: + """Finds entities in a folder or project modified in the past N number of days + + Args: + syn: Synapse connection + syn_id: Synapse Folder or Project Id + days: N number of days + + Returns: + List of synapse ids + """ raise NotImplementedError("Projects and folders not supported yet") @@ -135,8 +153,17 @@ def _get_user_ids(syn: Synapse, users: list = None): return user_ids -def find_modified_entities(syn: Synapse, syn_id: str, days: int) -> pd.DataFrame: - """Find modified entities based on the type of the input""" +def find_modified_entities(syn: Synapse, syn_id: str, days: int) -> list: + """Find modified entities based on the type of the input + + Args: + syn: Synapse connection + syn_id: Synapse Entity Id + days: N number of days + + Returns: + List of synapse ids + """ entity = syn.get(syn_id, downloadFile=False) if isinstance(entity, synapseclient.EntityViewSchema): return _find_modified_entities_fileview(syn=syn, syn_id=syn_id, days=days) @@ -170,9 +197,9 @@ def monitoring( Dataframe with files modified within last N days """ # get dataframe of files - filesdf = find_modified_entities(syn=syn, syn_id=syn_id, days=days) + modified_entities = find_modified_entities(syn=syn, syn_id=syn_id, days=days) # Filter out projects and folders - print(f"Total number of entities = {len(filesdf.index)}") + print(f"Total number of entities = {len(modified_entities)}") # get user ids user_ids = _get_user_ids(syn, users) @@ -180,11 +207,11 @@ def monitoring( # TODO: Add function to beautify email message # Prepare and send Message - if not filesdf.empty: + if not modified_entities: syn.sendMessage( user_ids, email_subject, - filesdf.to_html(index=False), + ", ".join(modified_entities), contentType="text/html", ) - return filesdf + return modified_entities From b295fce35352fbbcba617d40f31e647308ff5bc9 Mon Sep 17 00:00:00 2001 From: thomasyu888 Date: Thu, 20 Jan 2022 19:15:36 +0800 Subject: [PATCH 3/4] Fix tests --- synapsemonitor/monitor.py | 2 +- test/test_monitor.py | 25 ++++++++++++------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index 463a1c7..cecd6f8 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -207,7 +207,7 @@ def monitoring( # TODO: Add function to beautify email message # Prepare and send Message - if not modified_entities: + if modified_entities: syn.sendMessage( user_ids, email_subject, diff --git a/test/test_monitor.py b/test/test_monitor.py index ee21125..b3b0f14 100644 --- a/test/test_monitor.py +++ b/test/test_monitor.py @@ -52,22 +52,21 @@ def test__find_modified_entities_fileview(self): with patch.object(self.syn, "tableQuery", return_value=self.table_query_results) as patch_q,\ patch.object(self.table_query_results, "asDataFrame", - return_value=self.query_resultsdf) as patch_asdf,\ - patch.object(monitor, "_render_fileview", - return_value=self.expecteddf) as patch_render: - resultdf = monitor._find_modified_entities_fileview( + return_value=self.query_resultsdf) as patch_asdf: + # patch.object(monitor, "_render_fileview", + # return_value=self.expecteddf) as patch_render: + modified_list = monitor._find_modified_entities_fileview( self.syn, "syn44444", days=2 ) patch_q.assert_called_once_with( - "select id, name, currentVersion, modifiedOn, modifiedBy, " - "createdOn, projectId, type from syn44444 where " + "select id from syn44444 where " "modifiedOn > unix_timestamp(NOW() - INTERVAL 2 DAY)*1000" ) patch_asdf.assert_called_once_with() - assert resultdf.equals(self.expecteddf) - patch_render.assert_called_once_with( - self.syn, viewdf=self.query_resultsdf - ) + assert modified_list == ["syn23333"] + # patch_render.assert_called_once_with( + # self.syn, viewdf=self.query_resultsdf + # ) def test__get_user_ids_none(): @@ -148,9 +147,9 @@ def setup_method(self): def test_monitoring_fail_integration(self): """Test all monitoring functions are called""" - returndf = pd.DataFrame({"test": ["foo"]}) + modified_list = ["syn2222", "syn33333"] with patch.object(monitor, "find_modified_entities", - return_value=returndf) as patch_find,\ + return_value=modified_list) as patch_find,\ patch.object(monitor, "_get_user_ids", return_value=[111]) as patch_get_user,\ patch.object(self.syn, "sendMessage") as patch_send: @@ -159,5 +158,5 @@ def test_monitoring_fail_integration(self): patch_find.assert_called_once_with(syn=self.syn, syn_id="syn12345", days=15) patch_get_user.assert_called_once_with(self.syn, ["2222", "fooo"]) patch_send.assert_called_once_with([111], "new subject", - returndf.to_html(index=False), + "syn2222, syn33333", contentType='text/html') From adf0a942bb4eddf41e3069a98835631928c60dfb Mon Sep 17 00:00:00 2001 From: thomasyu888 Date: Thu, 20 Jan 2022 19:16:26 +0800 Subject: [PATCH 4/4] Run black --- synapsemonitor/monitor.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index cecd6f8..6de8ff4 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -66,9 +66,7 @@ def _render_fileview( return viewdf -def _find_modified_entities_fileview( - syn: Synapse, syn_id: str, days: int = 1 -) -> list: +def _find_modified_entities_fileview(syn: Synapse, syn_id: str, days: int = 1) -> list: """Finds entities scoped in a fileview modified in the past N number of days Args: @@ -87,12 +85,10 @@ def _find_modified_entities_fileview( ) results = syn.tableQuery(query) resultsdf = results.asDataFrame() - return resultsdf['id'].tolist() + return resultsdf["id"].tolist() -def _find_modified_entities_file( - syn: Synapse, syn_id: str, days: int = 1 -) -> list: +def _find_modified_entities_file(syn: Synapse, syn_id: str, days: int = 1) -> list: """Determines if entity was modified in the past N number of days Args: @@ -106,9 +102,7 @@ def _find_modified_entities_file( raise NotImplementedError("Files not supported yet") -def _find_modified_entities_container( - syn: Synapse, syn_id: str, days: int = 1 -) -> list: +def _find_modified_entities_container(syn: Synapse, syn_id: str, days: int = 1) -> list: """Finds entities in a folder or project modified in the past N number of days Args: