From e1e4ba64e5c1800041aaf9aedce6a45b53dc45d0 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 5 Sep 2024 14:17:59 +0100 Subject: [PATCH 1/4] adding white to reserved colours --- src/vfb_connect/cross_server_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vfb_connect/cross_server_tools.py b/src/vfb_connect/cross_server_tools.py index da4d0486..676414fd 100644 --- a/src/vfb_connect/cross_server_tools.py +++ b/src/vfb_connect/cross_server_tools.py @@ -1413,7 +1413,7 @@ def generate_lab_colors(self, num_colors, min_distance=100, verbose=False): rgb_colors = [] # Select the first color - lab_tree = KDTree([(0, 0, 0)]) # Start tree with black + lab_tree = KDTree([(255,255,255),(0, 0, 0)]) # Start tree with black and white # Pick colors that are far apart from each other and from black for lab in lab_colors[0:]: From ea3658c49612d69b18364f6df71eee3d34e9ea15 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 5 Sep 2024 15:01:13 +0100 Subject: [PATCH 2/4] adding retry --- src/vfb_connect/neo/query_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vfb_connect/neo/query_wrapper.py b/src/vfb_connect/neo/query_wrapper.py index 215a27bc..5102606d 100644 --- a/src/vfb_connect/neo/query_wrapper.py +++ b/src/vfb_connect/neo/query_wrapper.py @@ -18,7 +18,7 @@ from vfb_connect.neo.neo4j_tools import chunks, Neo4jConnect, dict_cursor, escape_string # Connect to the VFB SOLR server -vfb_solr = pysolr.Solr('http://solr.virtualflybrain.org/solr/vfb_json/', always_commit=False, timeout=990) +vfb_solr = pysolr.Solr('http://solr.virtualflybrain.org/solr/vfb_json/', always_commit=False, timeout=990, retry_count=10, retry_timeout=10) def batch_query(func): From 92e6e1932d252561c0db0e1935b247638e426d8a Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 5 Sep 2024 15:05:40 +0100 Subject: [PATCH 3/4] adding retry for owl network issues --- src/vfb_connect/owl/owlery_query_tools.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/vfb_connect/owl/owlery_query_tools.py b/src/vfb_connect/owl/owlery_query_tools.py index 330545a7..eea9031f 100644 --- a/src/vfb_connect/owl/owlery_query_tools.py +++ b/src/vfb_connect/owl/owlery_query_tools.py @@ -1,3 +1,4 @@ +from time import sleep import requests import re import json @@ -74,7 +75,12 @@ def query(self, query_type, return_type, payload = {'object': query, 'prefixes': json.dumps(self.curies), 'direct': direct} # print(payload) - r = requests.get(url=owl_endpoint, params=payload) + try: + r = requests.get(url=owl_endpoint, params=payload) + except requests.exceptions.RequestException as e: + print("\033[31mConnection Error:\033[0m " + str(e)) + sleep(15) + return query(query_type, return_type, query, query_by_label, direct, verbose) if verbose: print("Query URL: " + r.url) if r.status_code == 200: From f5716e8902ddcda1bdf3ed33775da5642085ac08 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 5 Sep 2024 15:41:29 +0100 Subject: [PATCH 4/4] adding delay and retry for solr --- src/vfb_connect/neo/query_wrapper.py | 11 +++++++++-- src/vfb_connect/owl/owlery_query_tools.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/vfb_connect/neo/query_wrapper.py b/src/vfb_connect/neo/query_wrapper.py index 5102606d..f42cf7a3 100644 --- a/src/vfb_connect/neo/query_wrapper.py +++ b/src/vfb_connect/neo/query_wrapper.py @@ -4,6 +4,7 @@ import shutil from inspect import getfullargspec from string import Template +from time import sleep from xml.sax import saxutils import pandas as pd import pkg_resources @@ -18,7 +19,7 @@ from vfb_connect.neo.neo4j_tools import chunks, Neo4jConnect, dict_cursor, escape_string # Connect to the VFB SOLR server -vfb_solr = pysolr.Solr('http://solr.virtualflybrain.org/solr/vfb_json/', always_commit=False, timeout=990, retry_count=10, retry_timeout=10) +vfb_solr = pysolr.Solr('http://solr.virtualflybrain.org/solr/vfb_json/', always_commit=False, timeout=990) def batch_query(func): @@ -609,7 +610,13 @@ def _get_Cached_TermInfo(self, short_forms: iter, summary=True, return_dataframe short_forms = list(chain.from_iterable(short_forms)) if any(isinstance(i, list) for i in short_forms) else short_forms print(f"Checking cache for results: short_forms={short_forms}") if verbose else None print(f"Looking for {len(short_forms)} results.") if verbose else None - results = self._serialize_solr_output(vfb_solr.search('*', **{'fl': 'term_info','df': 'id', 'defType': 'edismax', 'q.op': 'OR','rows': len(short_forms)+10,'fq':'{!terms f=id}'+ ','.join(short_forms)})) + try: + result = vfb_solr.search('*', **{'fl': 'term_info','df': 'id', 'defType': 'edismax', 'q.op': 'OR','rows': len(short_forms)+10,'fq':'{!terms f=id}'+ ','.join(short_forms)}) + except Exception as e: + print(f"\033[33mWarning:\033[0m Cache query failed. Error: {e}") + sleep(15) # Sleep for 15 seconds to avoid overloading the server + return self._get_Cached_TermInfo(short_forms, summary=summary, return_dataframe=return_dataframe, verbose=verbose) + results = self._serialize_solr_output(result) print(f"Got {len(results)} results.") if verbose else None if len(short_forms) != len(results): print(f"Warning: Cache didn't return all results. Got {len(results)} out of {len(short_forms)}") if verbose else None diff --git a/src/vfb_connect/owl/owlery_query_tools.py b/src/vfb_connect/owl/owlery_query_tools.py index eea9031f..5aaa678b 100644 --- a/src/vfb_connect/owl/owlery_query_tools.py +++ b/src/vfb_connect/owl/owlery_query_tools.py @@ -71,7 +71,7 @@ def query(self, query_type, return_type, if query_by_label: query = self.labels_2_ids(query) if verbose: - print("Running query: " + query) + print("Running query: " + query) payload = {'object': query, 'prefixes': json.dumps(self.curies), 'direct': direct} # print(payload)