Make transitive closure function use oxigraph store to speed up

helm/charts/sql-core/templates/core-tables.yaml

@@ -150,7 +150,7 @@ spec:
            `ts`
     FROM (
       SELECT *,
-      ROW_NUMBER() OVER (PARTITION BY `id`
+      ROW_NUMBER() OVER (PARTITION BY `id`, `index`
          ORDER BY ts DESC) AS rownum
       FROM `attributes`)
       WHERE rownum = 1 and entityId is NOT NULL;
@@ -232,7 +232,7 @@ spec:
            `ts`
     FROM (
       SELECT *,
-      ROW_NUMBER() OVER (PARTITION BY `id`
+      ROW_NUMBER() OVER (PARTITION BY `id`, `index`
          ORDER BY ts DESC) AS rownum
       FROM `attributes_insert_filter`)
       WHERE rownum = 1 and entityId is NOT NULL;

semantic-model/shacl2flink/create_ngsild_models.py

@@ -107,6 +107,26 @@ def nullify(field):
     return field
 
 
+class StringIndexer:
+    def __init__(self):
+        self.id_to_index_map = {}
+
+    def add_or_get_index(self, id, string):
+        # Initialize the id in the map if it doesn't exist
+        if id not in self.id_to_index_map:
+            self.id_to_index_map[id] = {'string_to_index': {}, 'current_index': 0}
+
+        id_map = self.id_to_index_map[id]
+        # If the string is already known for this id, return its index
+        if string in id_map['string_to_index']:
+            return id_map['string_to_index'][string]
+        # Otherwise, assign a new index, increment the counter, and return it
+        else:
+            id_map['string_to_index'][string] = id_map['current_index']
+            id_map['current_index'] += 1
+            return id_map['string_to_index'][string]
+
+
 def main(shaclfile, knowledgefile, modelfile, output_folder='output'):
     utils.create_output_folder(output_folder)
     with open(os.path.join(output_folder, "ngsild-models.sqlite"), "w")\
@@ -118,6 +138,7 @@ def main(shaclfile, knowledgefile, modelfile, output_folder='output'):
         knowledge = Graph(store="Oxigraph")
         knowledge.parse(knowledgefile)
         attributes_model = model + g + knowledge
+        string_indexer = StringIndexer()
 
         qres = attributes_model.query(attributes_query)
         first = True
@@ -134,7 +155,7 @@ def main(shaclfile, knowledgefile, modelfile, output_folder='output'):
                 current_index = index
                 if isinstance(index, URIRef):
                     try:
-                        current_index = int(utils.strip_class(current_index.toPython()))
+                        current_index = string_indexer.add_or_get_index(id, utils.strip_class(current_index.toPython()))
                     except:
                         current_index = 0
             valueType = nullify(valueType)
@@ -153,7 +174,7 @@ def main(shaclfile, knowledgefile, modelfile, output_folder='output'):
                   name.toPython() +
                   "', '" + nodeType + "', " + valueType + ", " +
                   str(current_index) +
-                  ", '" + type.toPython() + "', 'http://example.com/index/" + str(current_index) +
+                  ", '" + type.toPython() + "', '" + str(index) +
                   "'," + hasValue + ", " +
                   hasObject + ", " + current_timestamp + ")", end='',
                   file=sqlitef)

semantic-model/shacl2flink/lib/bgp_translation_utils.py

@@ -262,8 +262,8 @@ def select_candidates(bounds, triples, graph):
     while len(sorted_triples) > 0:
         candidate = select_candidates(bounds, sorted_triples, graph)
         if candidate is None:
-            raise utils.SparqlValidationFailed("Could not determine the right order \
-of triples")
+            raise utils.SparqlValidationFailed(f"Could not determine the right order \
+of triples {str(triples)}")
         result.append(candidate)
         sorted_triples.remove(candidate)
     return result

semantic-model/shacl2flink/lib/sparql_to_sql.py

@@ -96,7 +96,7 @@
 
 properties = {}
 relationships = {}
-g = Graph()
+g = Graph(store="Oxigraph")
 
 
 def translate_sparql(shaclfile, knowledgefile, sparql_query, target_class, lg):
@@ -616,6 +616,7 @@ def remap_join_constraint_to_where(node):
 def copy_context(ctx):
     # avoid deep copy of graph. it is not needed and creates problems with oxigraph
     graph = ctx['g']
+    del ctx['g']
     ctx_copy = copy.deepcopy(ctx)
     # copy graph manually into the new structure
     ctx_copy['g'] = graph

semantic-model/shacl2flink/lib/utils.py

@@ -602,7 +602,7 @@ def split_statementsets(statementsets, max_map_size):
 # This creates a transitive closure of all OWL.TransitiveProperty elements given in the ontology
 # plus rdfs:subClassOf. In addition is makes sure that every rdfs:Class and owl:Class are reflexive
 def transitive_closure(g):
-    closure_graph = Graph()
+    closure_graph = Graph(store="Oxigraph")
     closure_graph += g
 
     # Ensure rdfs:subClassOf is defined as an OWL.TransitiveProperty if it is not already defined

semantic-model/shacl2flink/tests/sql-tests/kms-constraints/test1/model11.jsonld

@@ -0,0 +1,79 @@
+[
+    {
+        "@context": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld",
+        "id": "urn:plasmacutter:1",
+        "type": "https://industry-fusion.com/types/v0.9/plasmacutter",
+        "https://industry-fusion.com/types/v0.9/state": {
+          "type": "Property",
+          "value": {
+            "@id": "https://industry-fusion.com/types/v0.9/state_OFF"
+          }
+        },
+        "https://industry-fusion.com/types/v0.9/stringState": {
+          "type": "Property",
+          "value": "T\"ST" 
+        },
+        "https://industry-fusion.com/types/v0.9/hasWorkpiece": {
+          "type": "Relationship",
+          "object": "urn:workpiece:1"
+        },
+        "https://industry-fusion.com/types/v0.9/hasFilter": [
+          {
+          "type": "Relationship",
+          "object": "urn:filter:1",
+          "datasetId": "urn:iff:datasetId:filter1"
+          },
+          {
+            "type": "Relationship",
+            "object": "urn:filter:2",
+            "datasetId": "urn:iff:datasetId:filter2"
+            }
+      ]
+    },
+    {
+        "@context": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld",
+        "id": "urn:filter:1",
+        "type": "https://industry-fusion.com/types/v0.9/filter",
+        "https://industry-fusion.com/types/v0.9/state": {
+        "type": "Property",
+        "value": {
+            "@id": "https://industry-fusion.com/types/v0.9/state_PREPARING"
+        }
+        },
+        "https://industry-fusion.com/types/v0.9/stringState": {
+          "type": "Property",
+          "value": "t1,t2" 
+        },
+        "https://industry-fusion.com/types/v0.9/strength": {
+        "type": "Property",
+        "value": 0.5
+        },
+        "https://industry-fusion.com/types/v0.9/hasCartridge": {
+        "type": "Relationship",
+        "object": "urn:filterCartridge:1"
+        }
+    },
+    {
+        "@context": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld",
+        "id": "urn:filter:2",
+        "type": "https://industry-fusion.com/types/v0.9/filter",
+        "https://industry-fusion.com/types/v0.9/state": {
+        "type": "Property",
+        "value": {
+            "@id": "https://industry-fusion.com/types/v0.9/state_PREPARING"
+        }
+        },
+        "https://industry-fusion.com/types/v0.9/stringState": {
+          "type": "Property",
+          "value": "t1,t2" 
+        },
+        "https://industry-fusion.com/types/v0.9/strength": {
+        "type": "Property",
+        "value": 0.5
+        },
+        "https://industry-fusion.com/types/v0.9/hasCartridge": {
+        "type": "Relationship",
+        "object": "urn:filterCartridge:1"
+        }
+    }
+]

semantic-model/shacl2flink/tests/sql-tests/kms-constraints/test1/model11.jsonld_result

@@ -0,0 +1,37 @@
+'urn:filter:1','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge[0])','warning'
+'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge)','ok'
+'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/state)','ok'
+'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/stringState)','ok'
+'urn:filter:1','DatatypeConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
+'urn:filter:1','InConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
+'urn:filter:1','MaxInclusiveConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
+'urn:filter:1','MinInclusiveConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
+'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge)','ok'
+'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
+'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
+'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
+'urn:filter:2','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge[0])','warning'
+'urn:filter:2','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge)','ok'
+'urn:filter:2','CountConstraintComponent(https://industry-fusion.com/types/v0.9/state)','ok'
+'urn:filter:2','CountConstraintComponent(https://industry-fusion.com/types/v0.9/stringState)','ok'
+'urn:filter:2','DatatypeConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
+'urn:filter:2','InConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
+'urn:filter:2','MaxInclusiveConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
+'urn:filter:2','MinInclusiveConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
+'urn:filter:2','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge)','ok'
+'urn:filter:2','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
+'urn:filter:2','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
+'urn:filter:2','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
+'urn:plasmacutter:1','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter[0])','ok'
+'urn:plasmacutter:1','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter[1])','ok'
+'urn:plasmacutter:1','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece[0])','warning'
+'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter)','warning'
+'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece)','ok'
+'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/state)','ok'
+'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/stringState)','ok'
+'urn:plasmacutter:1','DatatypeConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
+'urn:plasmacutter:1','InConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','warning'
+'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter)','ok'
+'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece)','ok'
+'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
+'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'