From 95f932936eb9c2c8537787597754ee88137d13a2 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Mon, 31 Jan 2022 19:12:55 -0800 Subject: [PATCH 01/27] fix parameter names and formatting to cli --- synapsemonitor/__main__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/synapsemonitor/__main__.py b/synapsemonitor/__main__.py index ed4de21..398a06c 100644 --- a/synapsemonitor/__main__.py +++ b/synapsemonitor/__main__.py @@ -20,13 +20,13 @@ def monitor_cli(syn, args): email_action = actions.EmailAction( syn=syn, - syn_id=args.view_id, + syn_id=args.synapse_id, email_subject=args.email_subject, users=args.users, days=args.days, ) action_results = actions.synapse_action(action_cls=email_action) - filesdf = pd.DataFrame({"syn_id": action_results}) + ids = pd.DataFrame({"syn_id": action_results}) if args.output: pd.DataFrame(ids).to_csv(args.output, index=False, header=False) else: @@ -107,7 +107,6 @@ def build_parser(): parser_monitor.add_argument( "--log", "-l", - metavar="level", type=str, choices=["debug", "info", "warning", "error"], default="error", From 9d94a324dc964e5cca54cdb890234ff286fe0cc5 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Mon, 31 Jan 2022 19:13:54 -0800 Subject: [PATCH 02/27] monitor contents of a container --- synapsemonitor/monitor.py | 52 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index a3a37cf..d060625 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -7,6 +7,7 @@ import pandas as pd import synapseclient from synapseclient import EntityViewSchema, EntityViewType, Synapse +from synapseutils import walk def create_file_view( @@ -113,6 +114,47 @@ def _find_modified_entities_file(syn: Synapse, syn_id: str, days: int = 1) -> li return [] +def _traverse( + syn: Synapse, synid_root: str, include_types: list = ["folder", "file", "project"] +) -> list: + """Traverse Synapse entity hierarchy to gather all descendant + entities of a root entity. + Args: + syn: Synapse connection + synid_root: Synapse ID of root entity. + include_types: Must be a list of entity types (ie. [“folder”,”file”]) + which can be found here: + http://docs.synapse.org/rest/org/sagebionetworks/repo/model/EntityType.html + Returns: + List of descendant Synapse IDs and root Synapse ID + """ + + synid_desc = [] + exclude_folder = True + + # full traverse depends on examining folder entities, even if not requested + include_types_mod = include_types.copy() + for include_type in include_types: + if include_type == "folder": + exclude_folder = False + if exclude_folder: + include_types_mod.append("folder") + + synid_children = syn.getChildren(parent=synid_root, includeTypes=include_types_mod) + print(synid_children) + for synid_child in synid_children: + synid_desc = synid_desc + _traverse( + syn=syn, synid_root=synid_child["id"], include_types=include_types + ) + + # only return folder entities if requested + entity = syn.get(synid_root, downloadFile=False) + if not (exclude_folder and entity["concreteType"].split(".")[-1] == "Folder"): + synid_desc.append(synid_root) + + return synid_desc + + def _find_modified_entities_container(syn: Synapse, syn_id: str, days: int = 1) -> list: """Finds entities in a folder or project modified in the past N number of days @@ -124,7 +166,15 @@ def _find_modified_entities_container(syn: Synapse, syn_id: str, days: int = 1) Returns: List of synapse ids """ - raise NotImplementedError("Projects and folders not supported yet") + syn_id_mod = [] + syn_id_children = _traverse(syn, syn_id) + + for syn_id_child in syn_id_children: + syn_id_res = _find_modified_entities_file(syn, syn_id_child, days) + if syn_id_res is not None: + syn_id_mod.extend(syn_id_res) + + return syn_id_mod def _force_update_view(syn: Synapse, view_id: str): From a3aaf37cb10cd5c375bbd882f1c8d0616036147e Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Mon, 31 Jan 2022 19:16:02 -0800 Subject: [PATCH 03/27] remove test for unimplemented function --- test/test_monitor.py | 157 ++++++++++++++++++++----------------------- 1 file changed, 73 insertions(+), 84 deletions(-) diff --git a/test/test_monitor.py b/test/test_monitor.py index 4afcc11..575b2f9 100644 --- a/test/test_monitor.py +++ b/test/test_monitor.py @@ -6,7 +6,7 @@ import pandas as pd import pytest -from synapseclient import EntityViewSchema, Project, Folder, File, Entity +from synapseclient import Synapse, EntityViewSchema, Project, Folder, File, Entity from synapsemonitor import monitor import synapsemonitor @@ -14,47 +14,56 @@ class TestModifiedEntitiesFileView: """Test modifying entities""" + def setup_method(self): self.syn = Mock() self.table_query_results = Mock() query_results = { - "id": ["syn23333"], "name": ["test"], - 'currentVersion': [2], 'modifiedOn': [1000000000], - 'createdOn': [1000000000], - 'modifiedBy': [333333], 'type': "file", - 'projectId': ['syn55555'] + "id": ["syn23333"], + "name": ["test"], + "currentVersion": [2], + "modifiedOn": [1000000000], + "createdOn": [1000000000], + "modifiedBy": [333333], + "type": "file", + "projectId": ["syn55555"], } self.query_resultsdf = pd.DataFrame(query_results) - self.expecteddf = pd.DataFrame({ - "id": ["syn23333"], "name": ["test"], - 'currentVersion': [2], - 'modifiedOn': ["1970-01-12 05:46:40-08:00"], - 'createdOn': ["1970-01-12 05:46:40-08:00"], - 'modifiedBy': ["user"], 'type': "file", - 'projectId': 'syn55555' - }) - self.expecteddf['createdOn'] = self.expecteddf['createdOn'].astype( - 'datetime64[ns, US/Pacific]' + self.expecteddf = pd.DataFrame( + { + "id": ["syn23333"], + "name": ["test"], + "currentVersion": [2], + "modifiedOn": ["1970-01-12 05:46:40-08:00"], + "createdOn": ["1970-01-12 05:46:40-08:00"], + "modifiedBy": ["user"], + "type": "file", + "projectId": "syn55555", + } + ) + self.expecteddf["createdOn"] = self.expecteddf["createdOn"].astype( + "datetime64[ns, US/Pacific]" ) - self.expecteddf['modifiedOn'] = self.expecteddf['modifiedOn'].astype( - 'datetime64[ns, US/Pacific]' + self.expecteddf["modifiedOn"] = self.expecteddf["modifiedOn"].astype( + "datetime64[ns, US/Pacific]" ) def test__render_fileview(self): """Test rendering of file view""" - with patch.object(self.syn, "getUserProfile", - return_value={"userName": "user"}) as patch_get: - rendereddf = monitor._render_fileview(self.syn, - self.query_resultsdf) + with patch.object( + self.syn, "getUserProfile", return_value={"userName": "user"} + ) as patch_get: + rendereddf = monitor._render_fileview(self.syn, self.query_resultsdf) patch_get.assert_called_once_with(333333) assert rendereddf.equals(self.expecteddf) def test__find_modified_entities_fileview(self): """Patch finding modified entities""" - with patch.object(self.syn, "tableQuery", - return_value=self.table_query_results) as patch_q,\ - patch.object(self.table_query_results, "asDataFrame", - return_value=self.query_resultsdf) as patch_asdf: + with patch.object( + self.syn, "tableQuery", return_value=self.table_query_results + ) as patch_q, patch.object( + self.table_query_results, "asDataFrame", return_value=self.query_resultsdf + ) as patch_asdf: # patch.object(monitor, "_render_fileview", # return_value=self.expecteddf) as patch_render: modified_list = monitor._find_modified_entities_fileview( @@ -74,14 +83,12 @@ def test__find_modified_entities_fileview(self): def test__find_modified_entities_file_modified(): """Patch finding modified entities no modified""" syn = Mock() - date_mod = datetime.now().replace(tzinfo=tz.tzutc()).strftime( - "%Y-%m-%dT%H:%M:%S.%fZ" + date_mod = ( + datetime.now().replace(tzinfo=tz.tzutc()).strftime("%Y-%m-%dT%H:%M:%S.%fZ") ) entity = File("test", "syn234", modifiedOn=date_mod) with patch.object(syn, "get", return_value=entity) as patch_get: - modified_list = monitor._find_modified_entities_file( - syn, "syn234", days=1 - ) + modified_list = monitor._find_modified_entities_file(syn, "syn234", days=1) patch_get.assert_called_once_with("syn234", downloadFile=False) assert modified_list == ["syn234"] @@ -91,14 +98,10 @@ def test__find_modified_entities_file_none(): syn = Mock() # create a modified date that is in the past old_modified = datetime.now() - timedelta(days=3) - date_mod = old_modified.replace(tzinfo=tz.tzutc()).strftime( - "%Y-%m-%dT%H:%M:%S.%fZ" - ) + date_mod = old_modified.replace(tzinfo=tz.tzutc()).strftime("%Y-%m-%dT%H:%M:%S.%fZ") entity = File("test", "syn234", modifiedOn=date_mod) with patch.object(syn, "get", return_value=entity) as patch_get: - modified_list = monitor._find_modified_entities_file( - syn, "syn234", days=1 - ) + modified_list = monitor._find_modified_entities_file(syn, "syn234", days=1) patch_get.assert_called_once_with("syn234", downloadFile=False) assert modified_list == [] @@ -106,8 +109,9 @@ def test__find_modified_entities_file_none(): def test__get_user_ids_none(): """Test getting logged in user profile when no users specified""" syn = Mock() - with patch.object(syn, "getUserProfile", - return_value={"ownerId": "111"}) as patch_get: + with patch.object( + syn, "getUserProfile", return_value={"ownerId": "111"} + ) as patch_get: user_ids = monitor._get_user_ids(syn, None) patch_get.assert_called_once_with() assert user_ids == ["111"] @@ -116,45 +120,24 @@ def test__get_user_ids_none(): def test__get_user_ids(): """Test getting user profiles ids""" syn = Mock() - with patch.object(syn, "getUserProfile", - return_value={"ownerId": "111"}) as patch_get: + with patch.object( + syn, "getUserProfile", return_value={"ownerId": "111"} + ) as patch_get: user_ids = monitor._get_user_ids(syn, [1, "username"]) patch_get.has_calls([mock.call(1), mock.call("username")]) assert user_ids == ["111", "111"] @pytest.mark.parametrize( - "entity", - [ - Project(id="syn12345", parentId="syn3333"), - Folder(id="syn12345", parentId="syn3333") - ] -) -def test_find_modified_entities_not_implemented(entity): - """Test not implemented entity types""" - syn = Mock() - with pytest.raises(NotImplementedError, match=".not supported yet"),\ - patch.object(syn, "get", return_value=entity): - monitor.find_modified_entities( - syn=syn, syn_id="syn12345", days=1 - ) - - -@pytest.mark.parametrize( - "entity, entity_type", - [ - (Entity(id="syn12345", parentId="syn3333"), "Entity") - ] + "entity, entity_type", [(Entity(id="syn12345", parentId="syn3333"), "Entity")] ) def test_find_modified_entities_unsupported(entity, entity_type): """Test unsupported entity types""" syn = Mock() - with pytest.raises(ValueError, - match=f".+synapseclient.entity.{entity_type}'> not supported"),\ - patch.object(syn, "get", return_value=entity): - monitor.find_modified_entities( - syn=syn, syn_id="syn12345", days=1 - ) + with pytest.raises( + ValueError, match=f".+synapseclient.entity.{entity_type}'> not supported" + ), patch.object(syn, "get", return_value=entity): + monitor.find_modified_entities(syn=syn, syn_id="syn12345", days=1) def test_find_modified_entities_supported(): @@ -162,12 +145,10 @@ def test_find_modified_entities_supported(): entity = EntityViewSchema(id="syn12345", parentId="syn3333") syn = Mock() empty = pd.DataFrame() - with patch.object(syn, "get", return_value=entity) as patch_get,\ - patch.object(monitor, "_find_modified_entities_fileview", - return_value=empty) as patch_mod: - value = monitor.find_modified_entities( - syn=syn, syn_id="syn12345", days=1 - ) + with patch.object(syn, "get", return_value=entity) as patch_get, patch.object( + monitor, "_find_modified_entities_fileview", return_value=empty + ) as patch_mod: + value = monitor.find_modified_entities(syn=syn, syn_id="syn12345", days=1) patch_get.assert_called_once_with("syn12345", downloadFile=False) patch_mod.assert_called_once() assert empty.equals(value) @@ -175,21 +156,29 @@ def test_find_modified_entities_supported(): class TestMonitoring: """Test monitoring function, includes integration test""" + def setup_method(self): self.syn = Mock() def test_monitoring_fail_integration(self): """Test all monitoring functions are called""" modified_list = ["syn2222", "syn33333"] - with patch.object(monitor, "find_modified_entities", - return_value=modified_list) as patch_find,\ - patch.object(monitor, "_get_user_ids", - return_value=[111]) as patch_get_user,\ - patch.object(self.syn, "sendMessage") as patch_send: - monitor.monitoring(self.syn, "syn12345", users=["2222", "fooo"], - email_subject="new subject", days=15) + with patch.object( + monitor, "find_modified_entities", return_value=modified_list + ) as patch_find, patch.object( + monitor, "_get_user_ids", return_value=[111] + ) as patch_get_user, patch.object( + self.syn, "sendMessage" + ) as patch_send: + monitor.monitoring( + self.syn, + "syn12345", + users=["2222", "fooo"], + email_subject="new subject", + days=15, + ) patch_find.assert_called_once_with(syn=self.syn, syn_id="syn12345", days=15) patch_get_user.assert_called_once_with(self.syn, ["2222", "fooo"]) - patch_send.assert_called_once_with([111], "new subject", - "syn2222, syn33333", - contentType='text/html') + patch_send.assert_called_once_with( + [111], "new subject", "syn2222, syn33333", contentType="text/html" + ) From 4e3884785152ff7e00ab8bfcb60ca8256311dacd Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Sat, 5 Feb 2022 16:20:26 -0800 Subject: [PATCH 04/27] remove print statement --- synapsemonitor/monitor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index d060625..c06d457 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -141,7 +141,6 @@ def _traverse( include_types_mod.append("folder") synid_children = syn.getChildren(parent=synid_root, includeTypes=include_types_mod) - print(synid_children) for synid_child in synid_children: synid_desc = synid_desc + _traverse( syn=syn, synid_root=synid_child["id"], include_types=include_types From f9eec2c43e0d114d7aeb06f13263c37842bd7c18 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Sat, 5 Feb 2022 16:20:56 -0800 Subject: [PATCH 05/27] add tests for modified containers --- test/test_monitor.py | 99 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/test/test_monitor.py b/test/test_monitor.py index 575b2f9..f2db7b3 100644 --- a/test/test_monitor.py +++ b/test/test_monitor.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta from dateutil import tz from unittest import mock -from unittest.mock import Mock, patch +from unittest.mock import Mock, patch, MagicMock import pandas as pd import pytest @@ -80,6 +80,103 @@ def test__find_modified_entities_fileview(self): # ) +class TestModifiedContainer: + """Test modifying containers""" + + def setup_method(self): + self.syn = Mock() + self.days = 1 + self.now = datetime.now().replace(tzinfo=tz.tzutc()).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + self.past = (datetime.now().replace(tzinfo=tz.tzutc()) - timedelta(days=self.days+1)).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + self.project = Project(name="test_project", parentId = "syn0", modifiedOn=self.now) + self.folder = Folder(name="test_folder", parentId = "syn1", modifiedOn=self.now) + self.file = File(name="test_file", parentId = "syn2", modifiedOn=self.now) + + + def test__traverse_folder_include(self): + """Traverse folder no children""" + with patch.object(self.syn, "get", + return_value=self.folder) as patch_get,\ + patch.object(self.syn, "getChildren", + return_value=[]) as patch_child: + desc = monitor._traverse(self.syn, self.folder["parentId"]) + patch_get.assert_called() + patch_child.assert_called() + assert desc == [self.folder["parentId"]] + + + def test__traverse_folder_exclude(self): + """Traverse folder no children""" + with patch.object(self.syn, "get", + return_value=self.folder) as patch_get,\ + patch.object(self.syn, "getChildren", + return_value=[]) as patch_child: + desc = monitor._traverse(self.syn, self.folder["parentId"], include_types=["file"]) + patch_get.assert_called() + patch_child.assert_called() + assert desc == [] + + + def test__find_modified_entities_folder_modified(self): + """Find modified entities in a folder""" + with patch.object(self.syn, "get", + return_value=self.folder) as patch_get,\ + patch.object(self.syn, "getChildren", + return_value=[]) as patch_child: + modified_list = monitor._find_modified_entities_container( + self.syn, self.folder["parentId"], days=self.days + ) + patch_get.assert_called() + patch_child.assert_called() + assert modified_list == ["syn1"] + + + def test__find_modified_entities_project_modified(self): + """Find modified entities in a project""" + with patch.object(self.syn, "get", + return_value=self.project) as patch_get,\ + patch.object(self.syn, "getChildren", + return_value=[]) as patch_child: + modified_list = monitor._find_modified_entities_container( + self.syn, self.project["parentId"], days=self.days + ) + patch_get.assert_called() + patch_child.assert_called() + assert modified_list == ["syn0"] + + def test__find_modified_entities_folder_not_modified(self): + """Find no modified entities in a folder""" + + folder = self.folder + folder['modifiedOn'] = self.past + with patch.object(self.syn, "get", + return_value=folder) as patch_get,\ + patch.object(self.syn, "getChildren", + return_value=[]) as patch_child: + modified_list = monitor._find_modified_entities_container( + self.syn, "syn234", days=self.days + ) + patch_get.assert_called() + patch_child.assert_called() + assert modified_list == [] + + + def test__find_modified_entities_project_not_modified(self): + """Find no modified entities in a project""" + project = self.project + project['modifiedOn'] = self.past + with patch.object(self.syn, "get", + return_value=self.project) as patch_get,\ + patch.object(self.syn, "getChildren", + return_value=[]) as patch_child: + modified_list = monitor._find_modified_entities_container( + self.syn, "syn234", days=self.days + ) + patch_get.assert_called() + patch_child.assert_called() + assert modified_list == [] + + def test__find_modified_entities_file_modified(): """Patch finding modified entities no modified""" syn = Mock() From abc17f5753d1d321f2a58558ad9b6957015c0ba2 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Sat, 5 Feb 2022 16:30:46 -0800 Subject: [PATCH 06/27] remove unused modules --- synapsemonitor/monitor.py | 1 - test/test_monitor.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index c06d457..9417bec 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -7,7 +7,6 @@ import pandas as pd import synapseclient from synapseclient import EntityViewSchema, EntityViewType, Synapse -from synapseutils import walk def create_file_view( diff --git a/test/test_monitor.py b/test/test_monitor.py index f2db7b3..713b785 100644 --- a/test/test_monitor.py +++ b/test/test_monitor.py @@ -2,14 +2,13 @@ from datetime import datetime, timedelta from dateutil import tz from unittest import mock -from unittest.mock import Mock, patch, MagicMock +from unittest.mock import Mock, patch import pandas as pd import pytest -from synapseclient import Synapse, EntityViewSchema, Project, Folder, File, Entity +from synapseclient import EntityViewSchema, Project, Folder, File, Entity from synapsemonitor import monitor -import synapsemonitor class TestModifiedEntitiesFileView: From 254a5dc9479a1a6bd0ad658a16e99c15865b6ffd Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Sat, 5 Feb 2022 16:38:56 -0800 Subject: [PATCH 07/27] update help messages in cli --- synapsemonitor/__main__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/synapsemonitor/__main__.py b/synapsemonitor/__main__.py index 398a06c..798f4a3 100644 --- a/synapsemonitor/__main__.py +++ b/synapsemonitor/__main__.py @@ -48,11 +48,12 @@ def create_file_view_cli(syn, args): def build_parser(): """Set up argument parser and returns""" parser = argparse.ArgumentParser( - description="Checks for new/modified entities in a Fileview. A Synapse " - "Fileview can be created to allow users to track entities in a Project " - "or Folder. For more information, head to " + description="Checks for new/modified Synapse entities. " + "If a Project or Folder entity is specified, that entity and all its contents will be monitored. " + "A Synapse File View can be created to allow users to track the contents of Projects " + "or Folders with many entities more efficiently. For more information, head to " "https://docs.synapse.org/articles/views.html. You can use the " - "`create-file-view` function provided in this package to create a File View." + "`create` function provided in this package to create a File View." ) parser.add_argument( "-c", @@ -75,7 +76,7 @@ def build_parser(): "synapse_id", metavar="synapse_id", type=str, - help="Synapse ID of fileview to be monitored.", + help="Synapse ID of entity to be monitored.", ) parser_monitor.add_argument( "--users", From dd141fd1b23c1586ce20e50c01000a18af51b006 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Sat, 5 Feb 2022 16:39:16 -0800 Subject: [PATCH 08/27] lint using black --- synapsemonitor/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapsemonitor/__main__.py b/synapsemonitor/__main__.py index 798f4a3..49521d3 100644 --- a/synapsemonitor/__main__.py +++ b/synapsemonitor/__main__.py @@ -49,7 +49,7 @@ def build_parser(): """Set up argument parser and returns""" parser = argparse.ArgumentParser( description="Checks for new/modified Synapse entities. " - "If a Project or Folder entity is specified, that entity and all its contents will be monitored. " + "If a Project or Folder entity is specified, that entity and all its contents will be monitored. " "A Synapse File View can be created to allow users to track the contents of Projects " "or Folders with many entities more efficiently. For more information, head to " "https://docs.synapse.org/articles/views.html. You can use the " From 013836fb7f1a02b0be734db40e545f6908a14c12 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Mon, 7 Feb 2022 08:39:01 -0800 Subject: [PATCH 09/27] patch all function calls in modified container function --- test/test_monitor.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/test_monitor.py b/test/test_monitor.py index 713b785..688c508 100644 --- a/test/test_monitor.py +++ b/test/test_monitor.py @@ -118,10 +118,10 @@ def test__traverse_folder_exclude(self): def test__find_modified_entities_folder_modified(self): """Find modified entities in a folder""" - with patch.object(self.syn, "get", - return_value=self.folder) as patch_get,\ - patch.object(self.syn, "getChildren", - return_value=[]) as patch_child: + with patch.object(monitor, "_traverse", + return_value=[self.folder["parentId"]]) as patch_get,\ + patch.object(monitor, "_find_modified_entities_file", + return_value=[self.folder["parentId"]]) as patch_child: modified_list = monitor._find_modified_entities_container( self.syn, self.folder["parentId"], days=self.days ) @@ -132,10 +132,10 @@ def test__find_modified_entities_folder_modified(self): def test__find_modified_entities_project_modified(self): """Find modified entities in a project""" - with patch.object(self.syn, "get", - return_value=self.project) as patch_get,\ - patch.object(self.syn, "getChildren", - return_value=[]) as patch_child: + with patch.object(monitor, "_traverse", + return_value=[self.project["parentId"]]) as patch_get,\ + patch.object(monitor, "_find_modified_entities_file", + return_value=[self.project["parentId"]]) as patch_child: modified_list = monitor._find_modified_entities_container( self.syn, self.project["parentId"], days=self.days ) @@ -148,9 +148,9 @@ def test__find_modified_entities_folder_not_modified(self): folder = self.folder folder['modifiedOn'] = self.past - with patch.object(self.syn, "get", - return_value=folder) as patch_get,\ - patch.object(self.syn, "getChildren", + with patch.object(monitor, "_traverse", + return_value=[self.folder["parentId"]]) as patch_get,\ + patch.object(monitor, "_find_modified_entities_file", return_value=[]) as patch_child: modified_list = monitor._find_modified_entities_container( self.syn, "syn234", days=self.days @@ -164,9 +164,9 @@ def test__find_modified_entities_project_not_modified(self): """Find no modified entities in a project""" project = self.project project['modifiedOn'] = self.past - with patch.object(self.syn, "get", - return_value=self.project) as patch_get,\ - patch.object(self.syn, "getChildren", + with patch.object(monitor, "_traverse", + return_value=[self.folder["parentId"]]) as patch_get,\ + patch.object(monitor, "_find_modified_entities_file", return_value=[]) as patch_child: modified_list = monitor._find_modified_entities_container( self.syn, "syn234", days=self.days From 34dd50bfc712f351d157e67561896281af0293e5 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Tue, 8 Feb 2022 08:38:51 -0800 Subject: [PATCH 10/27] refactor _traverse --- synapsemonitor/monitor.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index 9417bec..0ac3670 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -114,7 +114,9 @@ def _find_modified_entities_file(syn: Synapse, syn_id: str, days: int = 1) -> li def _traverse( - syn: Synapse, synid_root: str, include_types: list = ["folder", "file", "project"] + syn: Synapse, + synid_root: str, + include_types: typing.List = ["folder", "file", "project"], ) -> list: """Traverse Synapse entity hierarchy to gather all descendant entities of a root entity. @@ -129,14 +131,10 @@ def _traverse( """ synid_desc = [] - exclude_folder = True # full traverse depends on examining folder entities, even if not requested include_types_mod = include_types.copy() - for include_type in include_types: - if include_type == "folder": - exclude_folder = False - if exclude_folder: + if "folder" not in include_types_mod: include_types_mod.append("folder") synid_children = syn.getChildren(parent=synid_root, includeTypes=include_types_mod) @@ -147,7 +145,10 @@ def _traverse( # only return folder entities if requested entity = syn.get(synid_root, downloadFile=False) - if not (exclude_folder and entity["concreteType"].split(".")[-1] == "Folder"): + if not ( + "folder" not in include_types + and entity["concreteType"].split(".")[-1] == "Folder" + ): synid_desc.append(synid_root) return synid_desc From 057c76825f1efb40319cb20428bab394e0692716 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck <17149604+hhunterzinck@users.noreply.github.com> Date: Tue, 8 Feb 2022 18:14:20 -0800 Subject: [PATCH 11/27] Update synapsemonitor/monitor.py Co-authored-by: Thomas Yu --- synapsemonitor/monitor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index 0ac3670..3ab0012 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -133,9 +133,9 @@ def _traverse( synid_desc = [] # full traverse depends on examining folder entities, even if not requested - include_types_mod = include_types.copy() - if "folder" not in include_types_mod: - include_types_mod.append("folder") + include_types_mod = set(include_types) + include_types_mod.add("folder") + include_types_mod = list(include_types_mod) synid_children = syn.getChildren(parent=synid_root, includeTypes=include_types_mod) for synid_child in synid_children: From dbae3a421638ee99cf3467d5f7d3666d94989698 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Tue, 8 Feb 2022 18:36:23 -0800 Subject: [PATCH 12/27] rewrite boolean for traverse and extend --- synapsemonitor/monitor.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index 3ab0012..bd72cb7 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -139,16 +139,14 @@ def _traverse( synid_children = syn.getChildren(parent=synid_root, includeTypes=include_types_mod) for synid_child in synid_children: - synid_desc = synid_desc + _traverse( + synid_desc.extend(_traverse( syn=syn, synid_root=synid_child["id"], include_types=include_types - ) + )) # only return folder entities if requested entity = syn.get(synid_root, downloadFile=False) - if not ( - "folder" not in include_types - and entity["concreteType"].split(".")[-1] == "Folder" - ): + entity_type = entity["concreteType"].split(".")[-1] + if "folder" in include_types or entity_type != "Folder": synid_desc.append(synid_root) return synid_desc @@ -170,7 +168,7 @@ def _find_modified_entities_container(syn: Synapse, syn_id: str, days: int = 1) for syn_id_child in syn_id_children: syn_id_res = _find_modified_entities_file(syn, syn_id_child, days) - if syn_id_res is not None: + if syn_id_res: syn_id_mod.extend(syn_id_res) return syn_id_mod From 9bbb4fa104f31c6c33c97ebc189bb45c7d4edeb5 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Tue, 8 Feb 2022 18:36:51 -0800 Subject: [PATCH 13/27] lint with black --- synapsemonitor/monitor.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index bd72cb7..fa8b539 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -139,9 +139,11 @@ def _traverse( synid_children = syn.getChildren(parent=synid_root, includeTypes=include_types_mod) for synid_child in synid_children: - synid_desc.extend(_traverse( - syn=syn, synid_root=synid_child["id"], include_types=include_types - )) + synid_desc.extend( + _traverse( + syn=syn, synid_root=synid_child["id"], include_types=include_types + ) + ) # only return folder entities if requested entity = syn.get(synid_root, downloadFile=False) From 90d186ae6b18e4e0311c264d28f709c438b2eaca Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Tue, 8 Feb 2022 19:16:15 -0800 Subject: [PATCH 14/27] fix filtering by type at end of traverse --- synapsemonitor/monitor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index fa8b539..fbe0890 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -145,10 +145,10 @@ def _traverse( ) ) - # only return folder entities if requested + # only requested entity types entity = syn.get(synid_root, downloadFile=False) - entity_type = entity["concreteType"].split(".")[-1] - if "folder" in include_types or entity_type != "Folder": + entity_type = entity["concreteType"].split(".")[-1].lower().replace("entity", "") + if entity_type in include_types: synid_desc.append(synid_root) return synid_desc From 917dccae189486d515d6eba3a9b1e76ab46dba50 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Wed, 9 Feb 2022 08:20:03 -0800 Subject: [PATCH 15/27] only traverse containers to limit recursive calls --- synapsemonitor/monitor.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index fbe0890..3d8862f 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -139,11 +139,16 @@ def _traverse( synid_children = syn.getChildren(parent=synid_root, includeTypes=include_types_mod) for synid_child in synid_children: - synid_desc.extend( - _traverse( - syn=syn, synid_root=synid_child["id"], include_types=include_types + entity = syn.get(synid_child["id"], downloadFile=False) + entity_type = entity["concreteType"].split(".")[-1].lower().replace("entity", "") + if entity_type in ['folder','project']: + synid_desc.extend( + _traverse( + syn=syn, synid_root=synid_child["id"], include_types=include_types + ) ) - ) + else: + synid_desc.append(synid_child["id"]) # only requested entity types entity = syn.get(synid_root, downloadFile=False) From 5c57b3532435e4304374559ec58782a23d7d5a0e Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Wed, 9 Feb 2022 08:20:22 -0800 Subject: [PATCH 16/27] lint with black --- synapsemonitor/monitor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index 3d8862f..c7ce33b 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -140,8 +140,10 @@ def _traverse( synid_children = syn.getChildren(parent=synid_root, includeTypes=include_types_mod) for synid_child in synid_children: entity = syn.get(synid_child["id"], downloadFile=False) - entity_type = entity["concreteType"].split(".")[-1].lower().replace("entity", "") - if entity_type in ['folder','project']: + entity_type = ( + entity["concreteType"].split(".")[-1].lower().replace("entity", "") + ) + if entity_type in ["folder", "project"]: synid_desc.extend( _traverse( syn=syn, synid_root=synid_child["id"], include_types=include_types From ae142cfca5b6b29f5dcd1b190fc60eb7382dbfac Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Wed, 9 Feb 2022 08:33:23 -0800 Subject: [PATCH 17/27] check entity type before adding if not traversing --- synapsemonitor/monitor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index c7ce33b..fc92877 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -150,7 +150,8 @@ def _traverse( ) ) else: - synid_desc.append(synid_child["id"]) + if entity_type in include_types: + synid_desc.append(synid_child["id"]) # only requested entity types entity = syn.get(synid_root, downloadFile=False) From 336f82c048be8e574547fcfbe21c7a17a2dfa718 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Wed, 9 Feb 2022 08:33:52 -0800 Subject: [PATCH 18/27] add tests for traverse on folder and project --- test/test_monitor.py | 52 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/test/test_monitor.py b/test/test_monitor.py index 688c508..b1ee383 100644 --- a/test/test_monitor.py +++ b/test/test_monitor.py @@ -93,7 +93,7 @@ def setup_method(self): def test__traverse_folder_include(self): - """Traverse folder no children""" + """Traverse folder no children including project entity types""" with patch.object(self.syn, "get", return_value=self.folder) as patch_get,\ patch.object(self.syn, "getChildren", @@ -105,7 +105,7 @@ def test__traverse_folder_include(self): def test__traverse_folder_exclude(self): - """Traverse folder no children""" + """Traverse folder no children excluding folder entity types""" with patch.object(self.syn, "get", return_value=self.folder) as patch_get,\ patch.object(self.syn, "getChildren", @@ -116,6 +116,54 @@ def test__traverse_folder_exclude(self): assert desc == [] + def test__traverse_project_include(self): + """Traverse project with no children including project entity types""" + with patch.object(self.syn, "get", + return_value=self.project) as patch_get,\ + patch.object(self.syn, "getChildren", + return_value=[]) as patch_child: + desc = monitor._traverse(self.syn, self.project["parentId"], include_types=["file", "folder", "project"]) + patch_get.assert_called() + patch_child.assert_called() + assert desc == [self.project["parentId"]] + + + def test__traverse_project_exclude(self): + """Traverse project with no children excluding project entity types""" + with patch.object(self.syn, "get", + return_value=self.project) as patch_get,\ + patch.object(self.syn, "getChildren", + return_value=[]) as patch_child: + desc = monitor._traverse(self.syn, self.project["parentId"], include_types=["file", "folder"]) + patch_get.assert_called() + patch_child.assert_called() + assert desc == [] + + + def test__traverse_file_include(self): + """Traverse file including file entity types""" + with patch.object(self.syn, "get", + return_value=self.file) as patch_get,\ + patch.object(self.syn, "getChildren", + return_value=[]) as patch_child: + desc = monitor._traverse(self.syn, self.file["parentId"], include_types=["file", "folder", "project"]) + patch_get.assert_called() + patch_child.assert_called() + assert desc == [self.file["parentId"]] + + + def test__traverse_file_exclude(self): + """Traverse project with no children excluding project entity types""" + with patch.object(self.syn, "get", + return_value=self.file) as patch_get,\ + patch.object(self.syn, "getChildren", + return_value=[]) as patch_child: + desc = monitor._traverse(self.syn, self.file["parentId"], include_types=["folder", "project"]) + patch_get.assert_called() + patch_child.assert_called() + assert desc == [] + + def test__find_modified_entities_folder_modified(self): """Find modified entities in a folder""" with patch.object(monitor, "_traverse", From cda348a2775d45171b04e4ec2b2a1096d4341993 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Wed, 9 Feb 2022 08:37:24 -0800 Subject: [PATCH 19/27] correct id versus parentId specification of synapse entity objects --- test/test_monitor.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/test/test_monitor.py b/test/test_monitor.py index b1ee383..afba99d 100644 --- a/test/test_monitor.py +++ b/test/test_monitor.py @@ -87,9 +87,9 @@ def setup_method(self): self.days = 1 self.now = datetime.now().replace(tzinfo=tz.tzutc()).strftime("%Y-%m-%dT%H:%M:%S.%fZ") self.past = (datetime.now().replace(tzinfo=tz.tzutc()) - timedelta(days=self.days+1)).strftime("%Y-%m-%dT%H:%M:%S.%fZ") - self.project = Project(name="test_project", parentId = "syn0", modifiedOn=self.now) - self.folder = Folder(name="test_folder", parentId = "syn1", modifiedOn=self.now) - self.file = File(name="test_file", parentId = "syn2", modifiedOn=self.now) + self.project = Project(name="test_project", id = "syn0", modifiedOn=self.now, parentId = "syn00") + self.folder = Folder(name="test_folder", id = "syn1", modifiedOn=self.now, parentId = "syn0") + self.file = File(name="test_file", id = "syn2", modifiedOn=self.now, parentId = "syn1") def test__traverse_folder_include(self): @@ -98,10 +98,10 @@ def test__traverse_folder_include(self): return_value=self.folder) as patch_get,\ patch.object(self.syn, "getChildren", return_value=[]) as patch_child: - desc = monitor._traverse(self.syn, self.folder["parentId"]) + desc = monitor._traverse(self.syn, self.folder["id"]) patch_get.assert_called() patch_child.assert_called() - assert desc == [self.folder["parentId"]] + assert desc == [self.folder["id"]] def test__traverse_folder_exclude(self): @@ -110,7 +110,7 @@ def test__traverse_folder_exclude(self): return_value=self.folder) as patch_get,\ patch.object(self.syn, "getChildren", return_value=[]) as patch_child: - desc = monitor._traverse(self.syn, self.folder["parentId"], include_types=["file"]) + desc = monitor._traverse(self.syn, self.folder["id"], include_types=["file"]) patch_get.assert_called() patch_child.assert_called() assert desc == [] @@ -122,10 +122,10 @@ def test__traverse_project_include(self): return_value=self.project) as patch_get,\ patch.object(self.syn, "getChildren", return_value=[]) as patch_child: - desc = monitor._traverse(self.syn, self.project["parentId"], include_types=["file", "folder", "project"]) + desc = monitor._traverse(self.syn, self.project["id"], include_types=["file", "folder", "project"]) patch_get.assert_called() patch_child.assert_called() - assert desc == [self.project["parentId"]] + assert desc == [self.project["id"]] def test__traverse_project_exclude(self): @@ -134,7 +134,7 @@ def test__traverse_project_exclude(self): return_value=self.project) as patch_get,\ patch.object(self.syn, "getChildren", return_value=[]) as patch_child: - desc = monitor._traverse(self.syn, self.project["parentId"], include_types=["file", "folder"]) + desc = monitor._traverse(self.syn, self.project["id"], include_types=["file", "folder"]) patch_get.assert_called() patch_child.assert_called() assert desc == [] @@ -146,10 +146,10 @@ def test__traverse_file_include(self): return_value=self.file) as patch_get,\ patch.object(self.syn, "getChildren", return_value=[]) as patch_child: - desc = monitor._traverse(self.syn, self.file["parentId"], include_types=["file", "folder", "project"]) + desc = monitor._traverse(self.syn, self.file["id"], include_types=["file", "folder", "project"]) patch_get.assert_called() patch_child.assert_called() - assert desc == [self.file["parentId"]] + assert desc == [self.file["id"]] def test__traverse_file_exclude(self): @@ -158,7 +158,7 @@ def test__traverse_file_exclude(self): return_value=self.file) as patch_get,\ patch.object(self.syn, "getChildren", return_value=[]) as patch_child: - desc = monitor._traverse(self.syn, self.file["parentId"], include_types=["folder", "project"]) + desc = monitor._traverse(self.syn, self.file["id"], include_types=["folder", "project"]) patch_get.assert_called() patch_child.assert_called() assert desc == [] @@ -167,11 +167,11 @@ def test__traverse_file_exclude(self): def test__find_modified_entities_folder_modified(self): """Find modified entities in a folder""" with patch.object(monitor, "_traverse", - return_value=[self.folder["parentId"]]) as patch_get,\ + return_value=[self.folder["id"]]) as patch_get,\ patch.object(monitor, "_find_modified_entities_file", - return_value=[self.folder["parentId"]]) as patch_child: + return_value=[self.folder["id"]]) as patch_child: modified_list = monitor._find_modified_entities_container( - self.syn, self.folder["parentId"], days=self.days + self.syn, self.folder["id"], days=self.days ) patch_get.assert_called() patch_child.assert_called() @@ -181,11 +181,11 @@ def test__find_modified_entities_folder_modified(self): def test__find_modified_entities_project_modified(self): """Find modified entities in a project""" with patch.object(monitor, "_traverse", - return_value=[self.project["parentId"]]) as patch_get,\ + return_value=[self.project["id"]]) as patch_get,\ patch.object(monitor, "_find_modified_entities_file", - return_value=[self.project["parentId"]]) as patch_child: + return_value=[self.project["id"]]) as patch_child: modified_list = monitor._find_modified_entities_container( - self.syn, self.project["parentId"], days=self.days + self.syn, self.project["id"], days=self.days ) patch_get.assert_called() patch_child.assert_called() @@ -197,7 +197,7 @@ def test__find_modified_entities_folder_not_modified(self): folder = self.folder folder['modifiedOn'] = self.past with patch.object(monitor, "_traverse", - return_value=[self.folder["parentId"]]) as patch_get,\ + return_value=[self.folder["id"]]) as patch_get,\ patch.object(monitor, "_find_modified_entities_file", return_value=[]) as patch_child: modified_list = monitor._find_modified_entities_container( @@ -213,7 +213,7 @@ def test__find_modified_entities_project_not_modified(self): project = self.project project['modifiedOn'] = self.past with patch.object(monitor, "_traverse", - return_value=[self.folder["parentId"]]) as patch_get,\ + return_value=[self.folder["id"]]) as patch_get,\ patch.object(monitor, "_find_modified_entities_file", return_value=[]) as patch_child: modified_list = monitor._find_modified_entities_container( From de717e64a1aa70207a231bbaae2780c7c3566713 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Thu, 10 Feb 2022 08:48:18 -0800 Subject: [PATCH 20/27] remove unnecessary syn.get --- synapsemonitor/monitor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index fc92877..729e6a1 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -139,9 +139,8 @@ def _traverse( synid_children = syn.getChildren(parent=synid_root, includeTypes=include_types_mod) for synid_child in synid_children: - entity = syn.get(synid_child["id"], downloadFile=False) entity_type = ( - entity["concreteType"].split(".")[-1].lower().replace("entity", "") + synid_child["type"].split(".")[-1].lower().replace("entity", "") ) if entity_type in ["folder", "project"]: synid_desc.extend( From 5511064f05f11ef68f9a33b94b8c48c54b685cfe Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Thu, 10 Feb 2022 08:48:31 -0800 Subject: [PATCH 21/27] add test for traverse folder with file child --- test/test_monitor.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/test_monitor.py b/test/test_monitor.py index afba99d..7d973dc 100644 --- a/test/test_monitor.py +++ b/test/test_monitor.py @@ -90,6 +90,7 @@ def setup_method(self): self.project = Project(name="test_project", id = "syn0", modifiedOn=self.now, parentId = "syn00") self.folder = Folder(name="test_folder", id = "syn1", modifiedOn=self.now, parentId = "syn0") self.file = File(name="test_file", id = "syn2", modifiedOn=self.now, parentId = "syn1") + self.file_child = File(name="test_file", id = "syn2", modifiedOn=self.now, parentId = "syn1", type = "org.sagebionetworks.repo.model.FileEntity") def test__traverse_folder_include(self): @@ -164,6 +165,18 @@ def test__traverse_file_exclude(self): assert desc == [] + def test__traverse_folder_with_file_child(self): + """Traverse project with no children excluding project entity types""" + with patch.object(self.syn, "get", + return_value=self.folder) as patch_get,\ + patch.object(self.syn, "getChildren", + return_value=[self.file_child]) as patch_child: + desc = monitor._traverse(self.syn, self.folder["id"], include_types=["folder", "project", "file"]) + patch_get.assert_called() + patch_child.assert_called() + assert desc == [self.file_child["id"], self.folder["id"]] + + def test__find_modified_entities_folder_modified(self): """Find modified entities in a folder""" with patch.object(monitor, "_traverse", From 07bcd2f7ae3fd7ea0e35a44cd8eb1acc40e264de Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Thu, 10 Feb 2022 08:50:20 -0800 Subject: [PATCH 22/27] lint with black --- synapsemonitor/monitor.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index 729e6a1..84af0a7 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -139,9 +139,7 @@ def _traverse( synid_children = syn.getChildren(parent=synid_root, includeTypes=include_types_mod) for synid_child in synid_children: - entity_type = ( - synid_child["type"].split(".")[-1].lower().replace("entity", "") - ) + entity_type = synid_child["type"].split(".")[-1].lower().replace("entity", "") if entity_type in ["folder", "project"]: synid_desc.extend( _traverse( From 01f007681e8ef287bcce543075b6b0664694add7 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Mon, 14 Feb 2022 15:47:26 -0800 Subject: [PATCH 23/27] change default include types in traverse function --- synapsemonitor/monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index 84af0a7..37ae738 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -116,7 +116,7 @@ def _find_modified_entities_file(syn: Synapse, syn_id: str, days: int = 1) -> li def _traverse( syn: Synapse, synid_root: str, - include_types: typing.List = ["folder", "file", "project"], + include_types: typing.List = ["file"], ) -> list: """Traverse Synapse entity hierarchy to gather all descendant entities of a root entity. From a280afcfb688ac49f1c18805899faf1b1e6d437f Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Mon, 14 Feb 2022 15:48:37 -0800 Subject: [PATCH 24/27] remove check for child of type project --- synapsemonitor/monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index 37ae738..ff7fae9 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -140,7 +140,7 @@ def _traverse( synid_children = syn.getChildren(parent=synid_root, includeTypes=include_types_mod) for synid_child in synid_children: entity_type = synid_child["type"].split(".")[-1].lower().replace("entity", "") - if entity_type in ["folder", "project"]: + if entity_type == "folder": synid_desc.extend( _traverse( syn=syn, synid_root=synid_child["id"], include_types=include_types From ed6dbd5afc1abca2b0f9714512995c5ac88c4c0a Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Mon, 14 Feb 2022 16:39:23 -0800 Subject: [PATCH 25/27] create wrapper function for traverse --- synapsemonitor/monitor.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index ff7fae9..5a01878 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -127,7 +127,7 @@ def _traverse( which can be found here: http://docs.synapse.org/rest/org/sagebionetworks/repo/model/EntityType.html Returns: - List of descendant Synapse IDs and root Synapse ID + List of descendant Synapse IDs without root Synapse ID """ synid_desc = [] @@ -146,11 +146,26 @@ def _traverse( syn=syn, synid_root=synid_child["id"], include_types=include_types ) ) - else: - if entity_type in include_types: - synid_desc.append(synid_child["id"]) + if entity_type in include_types: + synid_desc.append(synid_child["id"]) - # only requested entity types + return synid_desc + + +def _traverse_root(syn: Synapse, synid_root: str, include_types: typing.List = ["file"],) -> list: + """Wrapper for call traverse to include root. + + Args: + syn (Synapse): Synapse connection + synid_root (str): Synapse ID of root entity. + include_types (typing.List, optional): Must be a list of entity types (ie. [“folder”,”file”]) + which can be found here: + http://docs.synapse.org/rest/org/sagebionetworks/repo/model/EntityType.html + + Returns: + list: List of descendant Synapse IDs with root Synapse ID + """ + synid_desc = _traverse(syn, synid_root, include_types) entity = syn.get(synid_root, downloadFile=False) entity_type = entity["concreteType"].split(".")[-1].lower().replace("entity", "") if entity_type in include_types: @@ -171,7 +186,7 @@ def _find_modified_entities_container(syn: Synapse, syn_id: str, days: int = 1) List of synapse ids """ syn_id_mod = [] - syn_id_children = _traverse(syn, syn_id) + syn_id_children = _traverse_root(syn, syn_id) for syn_id_child in syn_id_children: syn_id_res = _find_modified_entities_file(syn, syn_id_child, days) From fcad2dba86e6d3b95128ff873989a334507f0420 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Mon, 14 Feb 2022 16:39:36 -0800 Subject: [PATCH 26/27] update tests for new wrapper function --- test/test_monitor.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/test_monitor.py b/test/test_monitor.py index 7d973dc..1fa31cf 100644 --- a/test/test_monitor.py +++ b/test/test_monitor.py @@ -99,7 +99,7 @@ def test__traverse_folder_include(self): return_value=self.folder) as patch_get,\ patch.object(self.syn, "getChildren", return_value=[]) as patch_child: - desc = monitor._traverse(self.syn, self.folder["id"]) + desc = monitor._traverse_root(self.syn, self.folder["id"], include_types=["file", "folder", "project"]) patch_get.assert_called() patch_child.assert_called() assert desc == [self.folder["id"]] @@ -111,7 +111,7 @@ def test__traverse_folder_exclude(self): return_value=self.folder) as patch_get,\ patch.object(self.syn, "getChildren", return_value=[]) as patch_child: - desc = monitor._traverse(self.syn, self.folder["id"], include_types=["file"]) + desc = monitor._traverse_root(self.syn, self.folder["id"], include_types=["file"]) patch_get.assert_called() patch_child.assert_called() assert desc == [] @@ -123,7 +123,7 @@ def test__traverse_project_include(self): return_value=self.project) as patch_get,\ patch.object(self.syn, "getChildren", return_value=[]) as patch_child: - desc = monitor._traverse(self.syn, self.project["id"], include_types=["file", "folder", "project"]) + desc = monitor._traverse_root(self.syn, self.project["id"], include_types=["file", "folder", "project"]) patch_get.assert_called() patch_child.assert_called() assert desc == [self.project["id"]] @@ -135,7 +135,7 @@ def test__traverse_project_exclude(self): return_value=self.project) as patch_get,\ patch.object(self.syn, "getChildren", return_value=[]) as patch_child: - desc = monitor._traverse(self.syn, self.project["id"], include_types=["file", "folder"]) + desc = monitor._traverse_root(self.syn, self.project["id"], include_types=["file", "folder"]) patch_get.assert_called() patch_child.assert_called() assert desc == [] @@ -147,7 +147,7 @@ def test__traverse_file_include(self): return_value=self.file) as patch_get,\ patch.object(self.syn, "getChildren", return_value=[]) as patch_child: - desc = monitor._traverse(self.syn, self.file["id"], include_types=["file", "folder", "project"]) + desc = monitor._traverse_root(self.syn, self.file["id"], include_types=["file", "folder", "project"]) patch_get.assert_called() patch_child.assert_called() assert desc == [self.file["id"]] @@ -159,7 +159,7 @@ def test__traverse_file_exclude(self): return_value=self.file) as patch_get,\ patch.object(self.syn, "getChildren", return_value=[]) as patch_child: - desc = monitor._traverse(self.syn, self.file["id"], include_types=["folder", "project"]) + desc = monitor._traverse_root(self.syn, self.file["id"], include_types=["folder", "project"]) patch_get.assert_called() patch_child.assert_called() assert desc == [] @@ -171,7 +171,7 @@ def test__traverse_folder_with_file_child(self): return_value=self.folder) as patch_get,\ patch.object(self.syn, "getChildren", return_value=[self.file_child]) as patch_child: - desc = monitor._traverse(self.syn, self.folder["id"], include_types=["folder", "project", "file"]) + desc = monitor._traverse_root(self.syn, self.folder["id"], include_types=["folder", "project", "file"]) patch_get.assert_called() patch_child.assert_called() assert desc == [self.file_child["id"], self.folder["id"]] @@ -179,7 +179,7 @@ def test__traverse_folder_with_file_child(self): def test__find_modified_entities_folder_modified(self): """Find modified entities in a folder""" - with patch.object(monitor, "_traverse", + with patch.object(monitor, "_traverse_root", return_value=[self.folder["id"]]) as patch_get,\ patch.object(monitor, "_find_modified_entities_file", return_value=[self.folder["id"]]) as patch_child: @@ -193,7 +193,7 @@ def test__find_modified_entities_folder_modified(self): def test__find_modified_entities_project_modified(self): """Find modified entities in a project""" - with patch.object(monitor, "_traverse", + with patch.object(monitor, "_traverse_root", return_value=[self.project["id"]]) as patch_get,\ patch.object(monitor, "_find_modified_entities_file", return_value=[self.project["id"]]) as patch_child: @@ -209,7 +209,7 @@ def test__find_modified_entities_folder_not_modified(self): folder = self.folder folder['modifiedOn'] = self.past - with patch.object(monitor, "_traverse", + with patch.object(monitor, "_traverse_root", return_value=[self.folder["id"]]) as patch_get,\ patch.object(monitor, "_find_modified_entities_file", return_value=[]) as patch_child: @@ -225,7 +225,7 @@ def test__find_modified_entities_project_not_modified(self): """Find no modified entities in a project""" project = self.project project['modifiedOn'] = self.past - with patch.object(monitor, "_traverse", + with patch.object(monitor, "_traverse_root", return_value=[self.folder["id"]]) as patch_get,\ patch.object(monitor, "_find_modified_entities_file", return_value=[]) as patch_child: From fd26fa2a780591b177e45aa5a3126f938f366796 Mon Sep 17 00:00:00 2001 From: Haley Hunter-Zinck Date: Mon, 14 Feb 2022 16:39:59 -0800 Subject: [PATCH 27/27] lint with black --- synapsemonitor/monitor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/synapsemonitor/monitor.py b/synapsemonitor/monitor.py index 5a01878..438d9c6 100755 --- a/synapsemonitor/monitor.py +++ b/synapsemonitor/monitor.py @@ -152,8 +152,12 @@ def _traverse( return synid_desc -def _traverse_root(syn: Synapse, synid_root: str, include_types: typing.List = ["file"],) -> list: - """Wrapper for call traverse to include root. +def _traverse_root( + syn: Synapse, + synid_root: str, + include_types: typing.List = ["file"], +) -> list: + """Wrapper for call traverse to include root. Args: syn (Synapse): Synapse connection