HeLx-UI/TranQL Integration Iteration 2 and Autocompletion Improvements

requirements.txt

@@ -19,8 +19,8 @@ redisgraph==2.2.4
 requests==2.25.1
 requests-cache==0.4.13
 requests-mock==1.5.2
-PLATER-GRAPH==1.9.8
+git+https://github.com/helxplatform/[email protected]
 bmt==0.7.2
 git+https://github.com/ranking-agent/reasoner.git
 git+https://github.com/TranslatorSRI/[email protected]#egg=reasoner-pydantic
-git+https://github.com/TranslatorSRI/[email protected]#egg=reasoner-converter
+git+https://github.com/TranslatorSRI/[email protected]#egg=reasoner-converter

src/tranql/tranql_schema.py

@@ -9,7 +9,11 @@
 import time
 import threading
 from PLATER.services.util.graph_adapter import GraphInterface
-from tranql.util import snake_case
+from tranql.util import snake_case, title_case
+import logging
+
+logger = logging.getLogger(__name__)
+
 
 class NetworkxGraph:
     def __init__(self):
@@ -25,14 +29,19 @@ def get_node (self, identifier, properties=None):
         nodes = self.net.nodes(data=True)
         filtered = [i for i in nodes if i[0] == identifier]
         return filtered[0] if len(filtered) > 0 else None
-    def get_edge (self, start, end, properties=None):
-        result = None
-        for e in self.net.edges:
-            #print (f"-----    {start} {end} | {e[0]} {e[2]}")
-            if e[0] == start and e[1] == end:
-                result = e
-                break
-        return result
+    """ Returns an edge from the graph or None. Note that the edge attr dict returned is mutable.
+    :param predicate: If None, will return the first edge found between start->end if one exists.
+    :return: (start, predicate, end, attr_data) | None
+    """
+    def get_edge (self, start, end, predicate=None, properties=None):
+        try:
+            edges = self.net[start][end]
+            if predicate is None:
+                predicate = list(edges.keys())[0]
+            edge_data = edges[predicate]
+            return (start, predicate, end, edge_data)
+        except:
+            return None
     def get_nodes (self,**kwargs):
         return self.net.nodes(**kwargs)
     def get_edges (self,**kwargs):
@@ -161,7 +170,7 @@ def _create_graph_interface(service_name, redis_conf, tranql_config):
             **redis_connection_details
         )
 
-    def _get_adatpter(self, name):
+    def _get_adapter(self, name):
         if name not in RedisAdapter.registry_adapters:
             raise ValueError(f"Redis backend with name {name} not registered.")
         return RedisAdapter.registry_adapters.get(name)
@@ -174,7 +183,7 @@ def set_adapter(self, name, redis_config, tranql_config):
         )
 
     def get_schema(self, name):
-        gi: GraphInterface = self._get_adatpter(name)
+        gi: GraphInterface = self._get_adapter(name)
         schema = gi.get_schema(force_update=True)
         return schema
 
@@ -337,6 +346,42 @@ def add_layer (self, layer, name=None):
                 for link in links:
                     #print (f" {source_name}->{target_type} [{link}]")
                     self.schema_graph.add_edge (source_name, link, target_type, {"reasoner":[name]})
+        self.decorate_schema(layer, name)
+
+    """
+    Manually perform scoring on the Redis reasoner, since it supports it. If more reasoners support it,
+    this should obviously be changed to be done dynamically on each reasoner and should probably be generalized
+    to attribute decoration rather than just scoring.
+    """
+    def decorate_schema(self, layer, name=None):
+        if name != "redis":
+            return
+
+        def toBiolink(concept):
+            return "biolink:" + title_case(concept)
+
+        redis_adapter = RedisAdapter()
+        adapter = redis_adapter._get_adapter(name)
+        schema_summary = adapter.summary
+        for source_name, targets_list in layer.items ():
+            source_node = self.get_node (node_id=source_name)
+            biolink_source_name = toBiolink(source_name)
+            if not biolink_source_name in schema_summary: continue
+            for target_type, links in targets_list.items ():
+                target_node = self.get_node (node_id=target_type)
+                biolink_target_type = toBiolink(target_type)
+                if not biolink_target_type in schema_summary[biolink_source_name]: continue
+                edge_summary = schema_summary[biolink_source_name][biolink_target_type]
+                total_count = sum(edge_summary.values())
+                if isinstance(links, str):
+                    links = [links]
+                for link in links:
+                    biolink_link = "biolink:" + link
+                    if not biolink_link in edge_summary: continue
+                    (_, _, _, edge_data) = self.schema_graph.get_edge (source_name, target_type, link)
+                    individual_count = edge_summary[biolink_link]
+                    edge_data["score"] = individual_count / total_count
+
 
     def get_edge (self, plan, source_name, source_type, target_name, target_type,
                   predicate, edge_direction):

src/tranql/util.py

@@ -408,6 +408,16 @@ def deep_merge(source, destination, no_list_repeat=True):
     return destination
 
 
+def camel_case(arg: str):
+    """Convert snake_case string to camelCase"""
+    split = arg.split("_")
+    return split[0] + ''.join([i.title() for i in split[1:]])
+
+def title_case(arg: str):
+    """Convert snake_case string to TitleCase"""
+    split = arg.split("_")
+    return ''.join([i.title() for i in arg.split("_")])
+
 def snake_case(arg: str):
     """Convert string to snake_case.
     Non-alphanumeric characters are replaced with _.

src/tranql/web/package.json

@@ -6,6 +6,7 @@
     "@emotion/core": "^10.1.1",
     "@material-ui/core": "^3.9.2",
     "@material-ui/lab": "^3.0.0-alpha.30",
+    "@popperjs/core": "^2.8.0",
     "abortcontroller-polyfill": "^1.7.3",
     "bootstrap": "^4.3.1",
     "classnames": "^2.3.1",
@@ -41,6 +42,7 @@
     "react-json-tree": "^0.11.2",
     "react-notifications": "^1.4.3",
     "react-scripts": "^2.1.8",
+    "react-sizeme": "^3.0.2",
     "react-spinners": "^0.5.3",
     "react-split-pane": "^0.1.92",
     "react-table": "^6.9.2",

src/tranql/web/src/App.js

@@ -398,9 +398,11 @@ class App extends Component {
 
       activeModal : null,
 
-      exampleQueries : require("./static/app_data/example_queries.js")
+      exampleQueries : require("./static/app_data/example_queries.js"),
 
       //showAnswerViewer : true
+      // showAnswerViewerOnLoad: false
+      useLastUsedView: false
     };
 
     // This is a cache that stores results from `this._resolveIdentifiersFromConcept` for use in codemirror tooltips.
@@ -544,6 +546,39 @@ class App extends Component {
     this.setState({
       code: newCode
     });
+
+    /**
+     * Perform editor-related tasks upon query updating.
+     * Perform this on the callback (updated) state, so that the codemirror state is updated.
+     * 
+     */
+    this.setState({}, () => {
+      const editor = this._codemirror;
+      if (!editor) return;
+      if (editor.state.completionActive) {
+        this._codeAutoComplete();
+      }
+      /* Traverse through each token in the editor and perform identifier resolution on it if it's a string representing a curie */
+      const lines = editor.lineCount();
+      for (let lineNumber=0; lineNumber<lines; lineNumber++) {
+        const tokens = editor.getLineTokens(lineNumber);
+        for (let i=0; i<tokens.length; i++) {
+          const currentToken = tokens[i];
+          if (currentToken.type === "string" && currentToken.string.includes(":")) {
+            // It really doesn't need to be a curie here, since it'll just return no results,
+            // but might as well check if there's a colon to avoid making unnecessary API requests.
+            // Also, enable returning cached results (from previous calls), since this will be run on every curie
+            // each time the codemirror query is changed.
+            this._resolveIdentifiersFromCurie(getCurieFromCMToken(currentToken.string)).catch(() => {}, false);
+          }
+        }
+      }
+    });
+    /**
+     * Note that the debounced query execution in embedded mode still occurs in the codemirror's `onChange` callback.
+     * This is because initially loading the ?query param into the graph should execute *immediately*, not on a debounce,
+     * (and there are a couple other issues other than the immediate execution that arise from having it here related the ?query loading). 
+     */
   }
 
   /**
@@ -1012,7 +1047,7 @@ class App extends Component {
         node["id"] = id;
         if (node["category"] !== undefined && node["category"] !== null) {
           const catArr = node["category"];
-          console.log(`CATEGORY=[${catArr}]`);
+          // console.log(`CATEGORY=[${catArr}]`);
           let typeArr = catArr.map(this._categoryToType);
           node["type"] = typeArr;
         }
@@ -1050,9 +1085,7 @@ class App extends Component {
       };
       message.knowledge_graph.edges = newLinkArray;
     };
-    // NOTE: Pretty sure this is the culprit of a bug where the graph is set to the schema on page load,
-    // causing the graph view to display the schema until a query is made overriding it.
-    this._configureMessage (message,false,false);
+    this._configureMessage (message,false,schema);
     this.setState({},() => {
       if (typeof noRenderChain === "undefined") noRenderChain = false;
       if (typeof schema === "undefined") schema = this.state.schemaViewerActive && this.state.schemaViewerEnabled;
@@ -1258,8 +1291,18 @@ class App extends Component {
    * Very similar to `_resolveIdentifiersFromConcept`, except that is designed for use with a curie instead of a name & biolink type.
    * This is made to be used so that whenever a user types a curie directly into the query, it can automatically resolve the English name for them.
    * 
+   * @param {string} curie - A curie, e.g. "MONDO:0005240"
+   * @param {boolean=false} ignoreCache - When false, this method returns previously cached results for `curie`.
+   * 
    */
-  async _resolveIdentifiersFromCurie(curie) {
+  async _resolveIdentifiersFromCurie(curie, ignoreCache=false) {
+    /** If caching is enabled and the curie results are cached, return the cached results */
+    if (!ignoreCache && this._autocompleteResolvedIdentifiers[curie]) {
+      const results = {};
+      results[curie] = this._autocompleteResolvedIdentifiers[curie];
+      return results;
+    }
+
     const nodeNormResult = await (await fetch(
       this.nodeNormalizationURL + '/get_normalized_nodes?' + qs.stringify({ curie })
     )).json();
@@ -1281,6 +1324,7 @@ class App extends Component {
   _updateResolvedIdentifiers(results) {
     // Add results to the cache.
     this._autocompleteResolvedIdentifiers = { ...results, ...this._autocompleteResolvedIdentifiers };
+    localStorage.setItem('autocompleteIdentifiers', JSON.stringify(this._autocompleteResolvedIdentifiers));
     // Update codemirror tooltips with new cached results.
     this._codemirror.state.resolvedIdentifiers = this._autocompleteResolvedIdentifiers;
   }
@@ -1540,7 +1584,8 @@ class App extends Component {
     if (this.fg) {
       // Performing actions such as dragging a node will reset fg.controls().enabled back to true automatically,
       // thus it's necessary to hook into the TrackballControls and make sure that they're immediately re-disabled.
-      if (this.embedded) {
+      // *Currently disabled always.* 
+      if (this.embedded && false) {
         const controls = this.fg.controls();
         controls.enabled = false;
         const _update = controls.update.bind(controls);
@@ -1716,15 +1761,6 @@ class App extends Component {
                   value={this.state.code}
                   onBeforeChange={(editor, data, code) => this._updateCode(code)}
                   onChange={(editor) => {
-                    if (editor.state.completionActive) {
-                      this._codeAutoComplete();
-                    }
-                    const currentToken = editor.getTokenAt(editor.getCursor());
-                    if (currentToken.type === "string" && currentToken.string.includes(":")) {
-                      // It really doesn't need to be a curie here, since it'll just return no results,
-                      // but might as well check if there's a colon to avoid making unnecessary API requests.
-                      this._resolveIdentifiersFromCurie(getCurieFromCMToken(currentToken.string)).catch(() => {})
-                    }
                     if (this.embedded) this._debouncedExecuteQuery();
                   }}
                   options={this.state.codeMirrorOptions}
@@ -2337,8 +2373,20 @@ class App extends Component {
     // `ignoreQueryPrefix` automatically truncates the leading question mark within a query string in order to prevent it from being interpreted as part of it
     const params = qs.parse(window.location.search, { ignoreQueryPrefix : true });
 
+    /**
+     * Parse params into vars:
+     * - q|query
+     * - embedded: full | (graph|simple|true|"") | (false|<any>)
+     * - answer_viewer: (true|"") | (false|<any>) <- DISABLED
+     * - use_last_view: (true|"") | (false|<any>)
+     */
     const tranqlQuery = params.q || params.query;
-    const embedded = params.embed;
+    const {
+      embed: embedded,
+      use_last_view: useLastUsedView
+      // answer_viewer: showAnswerViewer
+    } = params;
+
     if (tranqlQuery !== undefined) {
       this._queryExecOnLoad = tranqlQuery;
     }
@@ -2366,18 +2414,30 @@ class App extends Component {
       this.setState({
         useCache : false
       });
-      // Disable localStorage on embedded websites. Even though settings/cache is disabled,
-      // some other parts of the app also write/read from localStorage, for example:
-      //   when a query is executed, it stores the query in local stoarge under the key `code`.
-      window.localStorage.__proto__ = Object.create({
+      // Maintain localStorage under window.embeddedLocalStorage such that
+      // the App can only use localStorage while embedded when explcitly intended.
+      window.embeddedLocalStorage = window.localStorage;
+      const _localStorage = window.localStorage;
+      // localStorage is a property of window with a getter but no setter, so it has to be fully deleted before setting it to a new value.
+      delete window.localStorage;
+      // Create a skeleton version of localStorage, in which persistent methods are overwritten while preserving the full API.
+      const skeletonStorage = Object.assign({
         setItem: () => {},
         removeItem: () => {},
         key: () => {},
         getItem: () => {},
         removeItem: () => {},
         length: 0
+      }, _localStorage);
+      // Create the new localStorage property with the skeleton storage.
+      Object.defineProperty(window, "localStorage", {
+        get: () => skeletonStorage
       });
     }
+    // Note: this option is only intended for use within an embedded page, since it doesn't make much sense in a normal context.
+    // Note2: *DISABLED*
+    // this.setState({ showAnswerViewerOnLoad: showAnswerViewer === "true" || showAnswerViewer === "" });
+    this.setState({ useLastUsedView: useLastUsedView === "true" || useLastUsedView === "" });
   }
   /**
    * Perform any necessary cleanup before being unmounted
@@ -2404,7 +2464,14 @@ class App extends Component {
     // & hydrate state accordingly
     this._handleQueryString ();
     // Hydrate persistent state from local storage
-    if (!this.embedded) this._hydrateState ();
+    if (!this.embedded) {
+      this._hydrateState ();
+      // This is a class field, not a state variable, so it needs to be manually loaded. It also has to update codemirror state,
+      // which is another reason it needs to be manualy handled.
+      this._updateResolvedIdentifiers(JSON.parse(localStorage.getItem('autocompleteIdentifiers')) || {});
+      // Make sure that the code loaded from localStorage goes through `_updateCode`
+      this.setState({}, () => this._updateCode(this.state.code));
+    }
 
     // Populate the cache viewer
     this._updateCacheViewer ();
@@ -2448,6 +2515,7 @@ class App extends Component {
   render() {
     // Render just the graph if the app is being embedded
     if (this.embedded) return <TranQLEmbedded embedMode={this.embedMode}
+                                              useLastUsedView={this.state.useLastUsedView}
                                               graphLoading={this.state.loading}
                                               graph={this.state.graph}
                                               renderForceGraph={this._renderForceGraph}
@@ -2547,7 +2615,7 @@ class App extends Component {
         {this._renderAnswerViewer()}
         <ReactTooltip place="left"/>
         {this._renderBanner()}
-        <div>
+        <div className="App-body">
           {
             this.state.showCodeMirror ?
               (