Merge pull request #92 from helxplatform/robokop-v5.0

Robokop v5.0
helxplatform · Nov 29, 2023 · 1413e92 · 1413e92
2 parents 96c75df + 48650b8
commit 1413e92
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 4 deletions.
diff --git a/dags/dug_helpers/dug_utils.py b/dags/dug_helpers/dug_utils.py
@@ -388,6 +388,7 @@ def _search_elements(self, curie, search_term):
                 raise Exception(f"Validation error - Did not find {curie} for"
                                 f"Search term: {search_term}")
             else:
+                del response['total_items']
                 for element_type in response:
                     all_elements_ids = [e['id'] for e in
                                         reduce(lambda x, y: x + y['elements'], response[element_type], [])]
@@ -434,12 +435,15 @@ def crawl_concepts(self, concepts, data_set_name):
                 casting_config = query['casting_config']
                 tranql_source = query['tranql_source']
                 dug_element_type = query['output_dug_type']
-                extracted_dug_elements += crawler.expand_to_dug_element(
+                new_elements =  crawler.expand_to_dug_element(
                     concept=concept,
                     casting_config=casting_config,
                     dug_element_type=dug_element_type,
                     tranql_source=tranql_source
                 )
+                log.debug("extracted:")
+                log.debug(str(list([el.get_searchable_dict() for el in new_elements])))
+                extracted_dug_elements += new_elements
             concept.clean()
             percent_complete = int((counter / total) * 100)
             if percent_complete % 10 == 0:

diff --git a/dags/metadata.yaml b/dags/metadata.yaml
@@ -97,6 +97,18 @@ kgx:
     files:
       - cde/annotated_edges_v4.0.jsonl
       - cde/annotated_nodes_v4.0.jsonl
+  - version: v5.0
+    name: baseline-graph
+    format: jsonl
+    files:
+      - baseline-5.0/edges_v5.0.jsonl
+      - baseline-5.0/nodes_v5.0.jsonl
+  - version: v5.0
+    name: cde-graph
+    format: jsonl
+    files:
+      - cde/annotated_edges_v5.0.jsonl
+      - cde/annotated_nodes_v5.0.jsonl
 dug_inputs:
   versions:
     - name: bdc

diff --git a/dags/roger/config/config.yaml b/dags/roger/config/config.yaml
@@ -16,7 +16,7 @@ annotation_base_data_uri: https://stars.renci.org/var/dug/
 
 kgx:
   biolink_model_version: v3.1.2
-  dataset_version: v4.0
+  dataset_version: v5.0
   merge_db_id: 1
   merge_db_temp_dir: workspace
   data_sets:
@@ -85,7 +85,7 @@ indexing:
         desc: "summary"
         collection_name: "cde_category"
         collection_id: "cde_category"
-        collection_action: "files"
+        action: "files"
 
 elasticsearch:
   host: elasticsearch

diff --git a/requirements.txt b/requirements.txt
@@ -6,7 +6,8 @@ flatten-dict
 redisgraph-bulk-loader==0.12.3
 pytest
 PyYAML
-git+https://github.com/helxplatform/dug@2.12.0
+git+https://github.com/helxplatform/dug@cde-links-2
 orjson
 kg-utils==0.0.6
 bmt==1.1.0
+linkml-runtime==1.6.0