feat(directory): Publish Services (#4579)

* feat(directory): publish Services data

* chore: fix tests

* chore: create new version

tools/directory/CHANGELOG.md

@@ -1,6 +1,9 @@
 # Changelog
 
-## Version 1.1.1
+## Version 1.2.0
+- Include publishing of the Services table
+
+## Version 1.1.0
 - Include publishing of the Studies table
 
 ## Version 1.0.0

tools/directory/pyproject.toml

@@ -7,3 +7,11 @@ build-backend = "setuptools.build_meta"
 # See configuration details in https://github.com/pypa/setuptools_scm
 root = "../../"
 version_scheme = "no-guess-dev"
+
+[tool.black]
+exclude = '''
+(
+  setup.py
+  init.py
+)
+'''

tools/directory/setup.cfg

@@ -2,7 +2,7 @@
 commit = False
 tag = False
 tag_name = {new_version}
-current_version = 1.1.0
+current_version = 1.2.0
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)(?P<build>\d+))?
 serialize =
 	{major}.{minor}.{patch}-{release}{build}

tools/directory/setup.py

@@ -10,7 +10,10 @@
 
 if __name__ == "__main__":
     try:
-        setup(version="1.1.0", use_scm_version={"version_scheme": "no-guess-dev"})
+        setup(
+            version="1.2.0",  # bumpversion.sh needs single-quotes # noqa : E261, W291,
+            use_scm_version={"version_scheme": "no-guess-dev"},
+        )
     except:  # noqa
         print(
             "\n\nAn error occurred while building the project, "

tools/directory/src/molgenis_emx2/directory_client/__init__.py

@@ -15,4 +15,4 @@
 finally:
     del version, PackageNotFoundError
 
-__version__ = "1.1.0"
+__version__ = "1.2.0"  # bumpversion.sh needs single-quotes # noqa : E261, W291

tools/directory/src/molgenis_emx2/directory_client/directory.py

@@ -91,6 +91,7 @@ def _init_state(self, nodes: List[Node], report: ErrorReport) -> PublishingState
                 networks=["id", "national_node"],
                 also_known_in=["id", "national_node"],
                 biobanks=["id", "pid", "name", "national_node", "withdrawn"],
+                services=["id", "national_node"],
                 collections=["id", "national_node"],
                 facts=["id", "national_node"],
                 studies=["id", "national_node"],

tools/directory/src/molgenis_emx2/directory_client/directory_client.py

@@ -37,6 +37,7 @@ class AttributesRequest:
     networks: List[str]
     also_known_in: List[str]
     biobanks: List[str]
+    services: List[str]
     studies: List[str]
     collections: List[str]
     facts: List[str]
@@ -177,22 +178,31 @@ def get_quality_info(self) -> QualityInfo:
             table="QualityInfoCollections",
         )
 
+        service_qualities = self.get(
+            table="QualityInfoServices",
+        )
+
         bb_qual = defaultdict(list)
         bb_level = defaultdict(list)
         coll_qual = defaultdict(list)
         coll_level = defaultdict(list)
+        service_qual = defaultdict(list)
         for row in biobank_qualities:
             bb_qual[row["biobank"]].append(row["id"])
             bb_level[row["biobank"]].append(row["assess_level_bio"])
         for row in collection_qualities:
             coll_qual[row["collection"]].append(row["id"])
             coll_level[row["collection"]].append(row["assess_level_col"])
 
+        for row in service_qualities:
+            service_qual[row["service"]].append(row["id"])
+
         return QualityInfo(
             biobanks=bb_qual,
             collections=coll_qual,
             biobank_levels=bb_level,
             collection_levels=coll_level,
+            services=service_qual,
         )
 
     def get_node(self, code: str) -> Node:

tools/directory/src/molgenis_emx2/directory_client/model.py

@@ -17,6 +17,7 @@ class TableType(Enum):
     ALSO_KNOWN = "also_known_in"
     NETWORKS = "networks"
     BIOBANKS = "biobanks"
+    SERVICES = "services"
     STUDIES = "studies"
     COLLECTIONS = "collections"
     FACTS = "facts"
@@ -208,6 +209,7 @@ class Node:
         TableType.NETWORKS: "networkID",
         TableType.ALSO_KNOWN: "akiID",
         TableType.BIOBANKS: "ID",
+        TableType.SERVICES: "serviceID",
         TableType.STUDIES: "studyID",
         TableType.COLLECTIONS: "ID",
         TableType.FACTS: "factID",
@@ -287,14 +289,16 @@ class Source(Enum):
 
 @dataclass
 class DirectoryData(ABC):
-    """Abstract base class for containers storing rows from the seven Directory tables:
-    persons, networks, also_known_in, biobanks, collections, facts and studies."""
+    """Abstract base class for containers storing rows from the Directory tables:
+    persons, also_known_in, networks, biobanks, services, studies, collections,
+    and facts."""
 
     source: Source
     persons: Table
     also_known_in: Table
     networks: Table
     biobanks: Table
+    services: Table
     studies: Table
     collections: Table
     facts: Table
@@ -306,6 +310,7 @@ def __post_init__(self):
             TableType.NETWORKS: self.networks,
             TableType.ALSO_KNOWN: self.also_known_in,
             TableType.BIOBANKS: self.biobanks,
+            TableType.SERVICES: self.services,
             TableType.STUDIES: self.studies,
             TableType.COLLECTIONS: self.collections,
             TableType.FACTS: self.facts,
@@ -318,6 +323,7 @@ def import_order(self) -> List[Table]:
             self.networks,
             self.also_known_in,
             self.biobanks,
+            self.services,
             self.studies,
             self.collections,
             self.facts,
@@ -367,6 +373,7 @@ def merge(self, other_data: DirectoryData):
         self.networks.rows_by_id.update(other_data.networks.rows_by_id)
         self.also_known_in.rows_by_id.update(other_data.also_known_in.rows_by_id)
         self.biobanks.rows_by_id.update(other_data.biobanks.rows_by_id)
+        self.services.rows_by_id.update(other_data.services.rows_by_id)
         self.studies.rows_by_id.update(other_data.studies.rows_by_id)
         self.collections.rows_by_id.update(other_data.collections.rows_by_id)
         self.facts.rows_by_id.update(other_data.facts.rows_by_id)
@@ -385,6 +392,7 @@ def copy_empty(self) -> "MixedData":
             networks=Table.of_empty(TableType.NETWORKS, self.networks.meta),
             also_known_in=Table.of_empty(TableType.ALSO_KNOWN, self.also_known_in.meta),
             biobanks=Table.of_empty(TableType.BIOBANKS, self.biobanks.meta),
+            services=Table.of_empty(TableType.SERVICES, self.services.meta),
             studies=Table.of_empty(TableType.STUDIES, self.studies.meta),
             collections=Table.of_empty(TableType.COLLECTIONS, self.collections.meta),
             facts=Table.of_empty(TableType.FACTS, self.facts.meta),
@@ -394,7 +402,7 @@ def copy_empty(self) -> "MixedData":
 @dataclass(frozen=True)
 class QualityInfo:
     """
-    Stores the quality information for biobanks and collections.
+    Stores the quality information for biobanks, collections and services
     """
 
     biobanks: Dict[str, List[str]]
@@ -409,11 +417,16 @@ class QualityInfo:
     collection_levels: Dict[str, List[str]]
     """Dictionary of collection ids and their assessment levels"""
 
+    services: Dict[str, List[str]]
+    """Dictionary of service ids and their quality ids"""
+
     def get_qualities(self, table_type: TableType) -> Dict[str, List[str]]:
         if table_type == TableType.BIOBANKS:
             return self.biobanks
         elif table_type == TableType.COLLECTIONS:
             return self.collections
+        elif table_type == TableType.SERVICES:
+            return self.services
         else:
             return dict()
 

tools/directory/tests/conftest.py

@@ -27,11 +27,11 @@ def get_data(table_type) -> List[dict]:
 @pytest.fixture
 def node_data() -> NodeData:
     """
-    Reads json files with the six data sources and
+    Reads json files with the eight data sources and
     returns NodeData to test with.
     """
     persons_meta = MagicMock()
-    persons_meta.id = "eu_bbmri_eric_NL_persons"
+    persons_meta.id = "Persons"
     persons_meta.id_attribute = "id"
     persons = Table.of(
         TableType.PERSONS,
@@ -40,7 +40,7 @@ def node_data() -> NodeData:
     )
 
     networks_meta = MagicMock()
-    networks_meta.id = "eu_bbmri_eric_NL_networks"
+    networks_meta.id = "Networks"
     networks_meta.id_attribute = "id"
     networks = Table.of(
         TableType.NETWORKS,
@@ -49,7 +49,7 @@ def node_data() -> NodeData:
     )
 
     also_known_meta = MagicMock()
-    also_known_meta.id = "eu_bbmri_eric_NL_also_known_in"
+    also_known_meta.id = "AlsoKnownIn"
     also_known_meta.id_attribute = "id"
     also_known = Table.of(
         TableType.ALSO_KNOWN,
@@ -58,7 +58,7 @@ def node_data() -> NodeData:
     )
 
     biobanks_meta = MagicMock()
-    biobanks_meta.id = "eu_bbmri_eric_NL_biobanks"
+    biobanks_meta.id = "Biobanks"
     biobanks_meta.id_attribute = "id"
     biobanks_meta.hyperlinks = ["url"]
     biobanks = Table.of(
@@ -67,8 +67,28 @@ def node_data() -> NodeData:
         get_data("biobanks"),
     )
 
+    services_meta = MagicMock()
+    services_meta.id = "Services"
+    services_meta.id_attribute = "id"
+    services_meta.hyperlinks = ["url"]
+    services = Table.of(
+        TableType.SERVICES,
+        services_meta,
+        get_data("services"),
+    )
+
+    studies_meta = MagicMock()
+    studies_meta.id = "Studies"
+    studies_meta.id_attribute = "id"
+    studies_meta.hyperlinks = ["url"]
+    studies = Table.of(
+        TableType.STUDIES,
+        studies_meta,
+        get_data("studies"),
+    )
+
     collection_meta = MagicMock()
-    collection_meta.id = "eu_bbmri_eric_NL_collections"
+    collection_meta.id = "Collections"
     collection_meta.id_attribute = "id"
     collections = Table.of(
         TableType.COLLECTIONS,
@@ -77,7 +97,7 @@ def node_data() -> NodeData:
     )
 
     facts_meta = MagicMock()
-    facts_meta.id = "eu_bbmri_eric_NL_facts"
+    facts_meta.id = "CollectionFacts"
     facts_meta.id_attribute = "id"
     facts = Table.of(
         TableType.FACTS,
@@ -93,6 +113,8 @@ def node_data() -> NodeData:
             TableType.NETWORKS.value: networks,
             TableType.ALSO_KNOWN.value: also_known,
             TableType.BIOBANKS.value: biobanks,
+            TableType.SERVICES.value: services,
+            TableType.STUDIES.value: studies,
             TableType.COLLECTIONS.value: collections,
             TableType.FACTS.value: facts,
         },

tools/directory/tests/resources/services.json

@@ -0,0 +1,11 @@
+[
+  {
+    "id": "bbmri-eric:serviceID:NL_valid-serviceID",
+    "biobank": "bbmri-eric:ID:NL_valid-biobankID-1"
+  },
+  {
+    "id": "bbmri-eric:NL_invalid:serviceID",
+    "biobank": "bbmri-eric:ID:NL_valid:biobankID-2"
+  }
+
+]

tools/directory/tests/resources/studies.json

@@ -0,0 +1,9 @@
+[
+  {
+    "id": "bbmri-eric:studyID:NL_valid-factID"
+  },
+  {
+    "id": "bbmri-eric:NL_invalid:studiesID"
+  }
+
+]

tools/directory/tests/test_model.py

@@ -17,6 +17,8 @@ def test_table_type_order():
         TableType.ALSO_KNOWN,
         TableType.NETWORKS,
         TableType.BIOBANKS,
+        TableType.SERVICES,
+        TableType.STUDIES,
         TableType.COLLECTIONS,
         TableType.FACTS,
     ]
@@ -27,6 +29,8 @@ def test_table_type_base_ids():
     assert TableType.ALSO_KNOWN.base_id == "AlsoKnownIn"
     assert TableType.NETWORKS.base_id == "Networks"
     assert TableType.BIOBANKS.base_id == "Biobanks"
+    assert TableType.SERVICES.base_id == "Services"
+    assert TableType.STUDIES.base_id == "Studies"
     assert TableType.COLLECTIONS.base_id == "Collections"
     assert TableType.FACTS.base_id == "CollectionFacts"
 
@@ -51,7 +55,10 @@ def test_node_staging_id():
 
     assert node.get_staging_id(TableType.PERSONS) == "Persons"
     assert node.get_staging_id(TableType.NETWORKS) == "Networks"
+    assert node.get_staging_id(TableType.ALSO_KNOWN) == "AlsoKnownIn"
     assert node.get_staging_id(TableType.BIOBANKS) == "Biobanks"
+    assert node.get_staging_id(TableType.SERVICES) == "Services"
+    assert node.get_staging_id(TableType.STUDIES) == "Studies"
     assert node.get_staging_id(TableType.COLLECTIONS) == "Collections"
     assert node.get_staging_id(TableType.FACTS) == "CollectionFacts"
 
@@ -61,7 +68,10 @@ def test_node_id_prefix():
 
     assert node.get_id_prefix(TableType.PERSONS) == "bbmri-eric:contactID:BE_"
     assert node.get_id_prefix(TableType.NETWORKS) == "bbmri-eric:networkID:BE_"
+    assert node.get_id_prefix(TableType.ALSO_KNOWN) == "bbmri-eric:akiID:BE_"
     assert node.get_id_prefix(TableType.BIOBANKS) == "bbmri-eric:ID:BE_"
+    assert node.get_id_prefix(TableType.SERVICES) == "bbmri-eric:serviceID:BE_"
+    assert node.get_id_prefix(TableType.STUDIES) == "bbmri-eric:studyID:BE_"
     assert node.get_id_prefix(TableType.COLLECTIONS) == "bbmri-eric:ID:BE_"
     assert node.get_id_prefix(TableType.FACTS) == "bbmri-eric:factID:BE_"
 
@@ -71,7 +81,10 @@ def test_node_eu_id_prefix():
 
     assert node.get_eu_id_prefix(TableType.PERSONS) == "bbmri-eric:contactID:EU_"
     assert node.get_eu_id_prefix(TableType.NETWORKS) == "bbmri-eric:networkID:EU_"
+    assert node.get_eu_id_prefix(TableType.ALSO_KNOWN) == "bbmri-eric:akiID:EU_"
     assert node.get_eu_id_prefix(TableType.BIOBANKS) == "bbmri-eric:ID:EU_"
+    assert node.get_eu_id_prefix(TableType.SERVICES) == "bbmri-eric:serviceID:EU_"
+    assert node.get_eu_id_prefix(TableType.STUDIES) == "bbmri-eric:studyID:EU_"
     assert node.get_eu_id_prefix(TableType.COLLECTIONS) == "bbmri-eric:ID:EU_"
     assert node.get_eu_id_prefix(TableType.FACTS) == "bbmri-eric:factID:EU_"
 
@@ -90,6 +103,8 @@ def test_node_data_order():
     networks = Table.of(TableType.NETWORKS, meta, [{"id": "1"}])
     also_known_in = Table.of(TableType.ALSO_KNOWN, meta, [{"id": "1"}])
     biobanks = Table.of(TableType.BIOBANKS, meta, [{"id": "1"}])
+    services = Table.of(TableType.SERVICES, meta, [{"id": "1"}])
+    studies = Table.of(TableType.STUDIES, meta, [{"id": "1"}])
     collections = Table.of(TableType.COLLECTIONS, meta, [{"id": "1"}])
     facts = Table.of(TableType.FACTS, meta, [{"id": "1"}])
     node = Node("NL", "NL")
@@ -101,6 +116,8 @@ def test_node_data_order():
         networks=networks,
         also_known_in=also_known_in,
         biobanks=biobanks,
+        services=services,
+        studies=studies,
         collections=collections,
         facts=facts,
     )
@@ -110,6 +127,8 @@ def test_node_data_order():
         networks,
         also_known_in,
         biobanks,
+        services,
+        studies,
         collections,
         facts,
     ]