Add opcua placeholder management in arrays

In OPCUA Object-arrays are manage by so called placeholders. For instance, placeholder Pump-<No> means that all BroseNames like Pump-1, Pump-02, ..., etc are part of the same array. Another way to express placeholders is to embrace the whole expression by <>, e.g. <pump>. This maps all names references from the same Object which have the same types to the same array. Signed-off-by: marcel <[email protected]>
IndustryFusion · Oct 26, 2024 · 1c4ff84 · 1c4ff84
1 parent 3dbe788
commit 1c4ff84
Show file tree

Hide file tree

Showing 11 changed files with 3,107 additions and 1,219 deletions.
diff --git a/semantic-model/opcua/extractType.py b/semantic-model/opcua/extractType.py
@@ -26,6 +26,8 @@
 from lib.shacl import Shacl
 
 
+attribute_prefix = 'has'
+
 query_namespaces = """
 PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
 PREFIX rdf:  <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
@@ -215,15 +217,16 @@ def scan_type_recursive(o, node, instancetype, shapename):
     if str(instancetype) == str(classtype):
         return False
 
-    attributename = urllib.parse.quote(f'has{browse_name}')
+    attributename = urllib.parse.quote(f'{browse_name}')
     rdfutils.get_modelling_rule(g, o, shacl_rule, instancetype)
 
+    placeholder_pattern = None
     decoded_attributename = urllib.parse.unquote(attributename)
     if utils.contains_both_angle_brackets(decoded_attributename):
-        decoded_attributename = utils.normalize_angle_bracket_name(decoded_attributename)
-    attributename = urllib.parse.quote(decoded_attributename)
-    if attributename == 'has':  # full template, ignore it
-        return False
+        decoded_attributename, placeholder_pattern = utils.normalize_angle_bracket_name(decoded_attributename)
+    attributename = urllib.parse.quote(f'{attribute_prefix}{decoded_attributename}')
+    if len(decoded_attributename) == 0:  # full template, ignore it
+        raise Exception(f"Unexpected attributename {attributename}")
     shacl_rule['path'] = entity_namespace[attributename]
 
     if rdfutils.isObjectNodeClass(nodeclass):
@@ -248,7 +251,8 @@ def scan_type_recursive(o, node, instancetype, shapename):
             has_components = True
             shacl_rule['contentclass'] = classtype
             shaclg.create_shacl_property(shapename, shacl_rule['path'], shacl_rule['optional'], shacl_rule['array'],
-                                         False, True, shacl_rule['contentclass'], None, is_subcomponent=True)
+                                         False, True, shacl_rule['contentclass'], None, is_subcomponent=True,
+                                         placeholder_pattern=placeholder_pattern)
     elif rdfutils.isVariableNodeClass(nodeclass):
         stop_scan = check_variable_consistency(instancetype, entity_namespace[attributename], classtype)
         if stop_scan:
@@ -332,36 +336,51 @@ def scan_entitiy_recursive(node, id, instance, node_id, o):
     shacl_rule = {}
     browse_name = next(g.objects(o, basens['hasBrowseName']))
     nodeclass, classtype = rdfutils.get_type(g, o)
-    attributename = urllib.parse.quote(f'has{browse_name}')
+    attributename = urllib.parse.quote(browse_name)
 
-    decoded_attributename = utils.normalize_angle_bracket_name(urllib.parse.unquote(attributename))
-    optional, array = shaclg.get_modelling_rule(entity_namespace[decoded_attributename], URIRef(instance['type']))
+    original_attributename = None
+    decoded_attributename = urllib.parse.unquote(attributename)
+    optional, array, path = shaclg.get_modelling_rule_and_path(decoded_attributename, URIRef(instance['type']),
+                                                               classtype, attribute_prefix)
+    if path is not None:
+        original_attributename = decoded_attributename
+        decoded_attributename = path.removeprefix(entity_namespace)
+        if not path.startswith(str(entity_namespace)):
+            print(f"Warning: Mismatch of {entity_namespace} namespace and {path}")
+    else:
+        decoded_attributename = f'{attribute_prefix}{decoded_attributename}'
     shacl_rule['optional'] = optional
     shacl_rule['array'] = array
     datasetId = None
+
     try:
         is_placeholder = shaclg.is_placeholder(URIRef(instance['type']), entity_namespace[decoded_attributename])
     except:
         is_placeholder = False
     if is_placeholder:
-        datasetId = f'{datasetid_urn}:{attributename}'
-        attributename = urllib.parse.quote(decoded_attributename)
-    shacl_rule['path'] = entity_namespace[attributename]
+        if original_attributename is None:
+            raise Exception(f"No original_attributename given but datasetId neeeded for {decoded_attributename}")
+        datasetId = f'{datasetid_urn}:{original_attributename}'
+    attributename = urllib.parse.quote(decoded_attributename)
 
     if rdfutils.isObjectNodeClass(nodeclass):
         shacl_rule['is_property'] = False
         relid = scan_entity(o, classtype, id, shacl_rule['optional'])
         if relid is not None:
             has_components = True
-            instance[f'{entity_ontology_prefix}:{attributename}'] = {
+            full_attribute_name = f'{entity_ontology_prefix}:{attributename}'
+            if instance.get(full_attribute_name) is None:
+                instance[full_attribute_name] = []
+            attr_instance = {
                 'type': 'Relationship',
                 'object': relid
             }
             if is_placeholder and datasetId is not None:
-                instance[f'{entity_ontology_prefix}:{attributename}']['datasetId'] = datasetId
+                attr_instance['datasetId'] = datasetId
             if debug:
-                instance[f'{entity_ontology_prefix}:{attributename}']['debug'] = \
+                attr_instance['debug'] = \
                     f'{entity_ontology_prefix}:{attributename}'
+            instance[full_attribute_name].append(attr_instance)
             shacl_rule['contentclass'] = classtype
             minshaclg.copy_property_from_shacl(shaclg, instance['type'], entity_namespace[attributename])
         if not shacl_rule['optional']:

diff --git a/semantic-model/opcua/lib/shacl.py b/semantic-model/opcua/lib/shacl.py
@@ -23,21 +23,39 @@
 
 
 query_minmax = """
-SELECT ?mincount ?maxcount WHERE {
-     ?shape sh:targetClass ?targetclass .
-     OPTIONAL{
-        ?shape sh:property [
-            sh:path ?path;
-            sh:maxCount ?maxcount
-      ]
+SELECT ?path ?pattern ?mincount ?maxcount ?localName
+WHERE {
+    ?shape a sh:NodeShape ;
+           sh:property ?property ;
+           sh:targetClass ?targetclass .
+
+    ?property sh:path ?path ;
+        sh:property [ sh:class ?attributeclass ] .
+    OPTIONAL {
+        ?property base:hasPlaceHolderPattern ?pattern .
+    }
+
+    OPTIONAL {
+        ?property sh:maxCount ?maxcount .
     }
-     OPTIONAL{
-        ?shape sh:property [
-            sh:path ?path;
-            sh:minCount ?mincount
-      ]
+
+    OPTIONAL {
+        ?property sh:minCount ?mincount .
     }
+
+    # Extract the local name from the path (after the last occurrence of '/', '#' or ':')
+    BIND(REPLACE(str(?path), '.*[#/](?=[^#/]*$)', '') AS ?localName)
+
+    # Conditional filtering based on whether the pattern exists
+    FILTER (
+        IF(bound(?pattern),
+           regex(?name, ?pattern),  # If pattern exists, use regex
+           ?localName = ?prefixname        # Otherwise, match the local name
+        )
+    )
+  BIND(IF(?localName = ?prefixname, 0, 1) AS ?order)
 }
+ORDER BY ?order
 """
 
 
@@ -60,7 +78,7 @@ def create_shacl_type(self, targetclass):
         return shapename
 
     def create_shacl_property(self, shapename, path, optional, is_array, is_property, is_iri, contentclass, datatype,
-                              is_subcomponent=False):
+                              is_subcomponent=False, placeholder_pattern=None):
         innerproperty = BNode()
         property = BNode()
         maxCount = 1
@@ -80,6 +98,8 @@ def create_shacl_property(self, shapename, path, optional, is_array, is_property
             self.shaclg.add((innerproperty, SH.path, self.ngsildns['hasObject']))
             if is_array:
                 self.shaclg.add((property, self.basens['isPlaceHolder'], Literal(True)))
+            if placeholder_pattern is not None:
+                self.shaclg.add((property, self.basens['hasPlaceHolderPattern'], Literal(placeholder_pattern)))
             if is_subcomponent:
                 self.shaclg.add((property, RDF.type, self.basens['SubComponentRelationship']))
             else:
@@ -128,24 +148,33 @@ def get_shacl_iri_and_contentclass(self, g, node, shacl_rule):
             else:
                 shacl_rule['is_iri'] = False
                 shacl_rule['contentclass'] = None
+                shacl_rule['datatype'] = None
         except:
             shacl_rule['is_iri'] = False
             shacl_rule['contentclass'] = None
+            shacl_rule['datatype'] = None
 
-    def get_modelling_rule(self, path, target_class):
-        bindings = {'targetclass': target_class, 'path': path}
+    def get_modelling_rule_and_path(self, name, target_class, attributeclass, prefix):
+        bindings = {'targetclass': target_class, 'name': Literal(name), 'attributeclass': attributeclass,
+                    'prefixname': Literal(f'{prefix}{name}')}
         optional = True
         array = True
+        path = None
         try:
-            results = list(self.shaclg.query(query_minmax, initBindings=bindings, initNs={'sh': SH}))
+            results = list(self.shaclg.query(query_minmax, initBindings=bindings,
+                                             initNs={'sh': SH, 'base': self.basens}))
             if len(results) > 0:
-                if int(results[0][0]) > 0:
+                if results[0][0] is not None:
+                    path = results[0][0]
+                if int(results[0][2]) > 0:
                     optional = False
-                if int(results[0][1]) <= 1:
+                if int(results[0][3]) <= 1:
                     array = False
+            if len(results) > 1:
+                print("Warning: more than one path match for {path}")
         except:
             pass
-        return optional, array
+        return optional, array, path
 
     def attribute_is_indomain(self, targetclass, attributename):
         property = self._get_property(targetclass, attributename)
@@ -168,6 +197,8 @@ def _get_property(self, targetclass, propertypath):
 
     def is_placeholder(self, targetclass, attributename):
         property = self._get_property(targetclass, attributename)
+        if property is None:
+            return False
         try:
             return bool(next(self.shaclg.objects(property, self.basens['isPlaceHolder'])))
         except:
@@ -191,7 +222,6 @@ def update_shclass_in_property(self, property, shclass):
             pass
 
     def copy_property_from_shacl(self, source_graph, targetclass, propertypath):
-        print(f"woudld copy  {targetclass}=>{propertypath}")
         shape = self.create_shape_if_not_exists(source_graph, targetclass)
         if shape is None:
             return
@@ -210,8 +240,6 @@ def copy_bnode_triples(self, source_graph, bnode, shape):
         for s, p, o in source_graph.get_graph().triples((bnode, None, None)):
             # Add the triple to the target graph
             self.shaclg.add((s, p, o))
-            print(f"Adding: {s}, {p}, {o}")
-
             # If the object is another blank node, recurse into it
             if isinstance(o, BNode):
                 self.copy_bnode_triples(source_graph, o, None)

diff --git a/semantic-model/opcua/lib/utils.py b/semantic-model/opcua/lib/utils.py
@@ -152,11 +152,30 @@ def get_default_value(datatype):
 
 
 def normalize_angle_bracket_name(s):
-    # Remove content inside angle brackets and the brackets themselves
-    no_brackets = re.sub(r'<[^>]*>', '', s)
-    # Strip trailing numbers and non-alphabetic characters
-    normalized = re.sub(r'[^a-zA-Z]+$', '', no_brackets)
-    return normalized
+    # Check if there are any angle brackets in the input string
+    if '<' in s and '>' in s:
+        # Remove everything inside and including the angle brackets
+        no_brackets = re.sub(r'<[^>]*>', '', s)
+
+        # If the result is empty, it means the entire name was in brackets like <Tank>
+        if no_brackets.strip() == '':
+            # Extract the name inside the angle brackets
+            base_name = re.sub(r'[<>]', '', s)
+            # The pattern should match valid BrowseNames
+            pattern = r'[a-zA-Z0-9_-]+'
+        else:
+            # Otherwise, use the part before the angle brackets
+            base_name = no_brackets.strip()
+            # Construct a pattern to match the base name followed by valid BrowseName characters
+            pattern = re.sub(r'<[^>]*>', r'[a-zA-Z0-9_-]+', s)
+    else:
+        # If there are no angle brackets, the base name is just the input string itself
+        base_name = s
+        # Pattern matches exactly the base name
+        pattern = re.escape(s)  # Escape any special characters in the base name
+
+    # Return the cleaned base name and the regular expression pattern
+    return base_name.strip(), pattern
 
 
 def contains_both_angle_brackets(s):

diff --git a/semantic-model/opcua/tests/extractType/test.bash b/semantic-model/opcua/tests/extractType/test.bash
@@ -23,6 +23,7 @@ TESTNODESETS=(
     test_object_overwrite_type.NodeSet2,${TESTURI}AlphaType
     test_variable_enum.NodeSet2,${TESTURI}AlphaType
     test_object_subtypes.NodeSet2,${TESTURI}AlphaType
+    test_object_example.NodeSet2,${TESTURI}AlphaType
     test_object_hierarchies_no_DataValue,${TESTURI}AlphaType
     test_ignore_references.NodeSet2,${TESTURI}AlphaType
     test_references_to_typedefinitions.NodeSet2,${TESTURI}AlphaType

diff --git a/semantic-model/opcua/tests/extractType/test_object_example.NodeSet2.instances b/semantic-model/opcua/tests/extractType/test_object_example.NodeSet2.instances
@@ -0,0 +1,30 @@
+[
+    {
+        "type": "http://my.test/BSubType",
+        "id": "urn:test:AlphaInstance:sub:i2012",
+        "@context": [
+            "http://localhost:8099/context.jsonld"
+        ],
+        "uaentity:hasMyVariable": {
+            "type": "Property",
+            "value": false
+        }
+    },
+    {
+        "type": "http://my.test/AlphaType",
+        "id": "urn:test:AlphaInstance",
+        "@context": [
+            "http://localhost:8099/context.jsonld"
+        ],
+        "uaentity:hasC": {
+            "type": "Property",
+            "value": 0.0
+        },
+        "uaentity:hasB": [
+            {
+                "type": "Relationship",
+                "object": "urn:test:AlphaInstance:sub:i2012"
+            }
+        ]
+    }
+]
diff --git a/semantic-model/opcua/tests/extractType/test_object_example.NodeSet2.shacl b/semantic-model/opcua/tests/extractType/test_object_example.NodeSet2.shacl
@@ -0,0 +1,41 @@
+@prefix base: <https://industryfusion.github.io/contexts/ontology/v0/base/> .
+@prefix ngsi-ld: <https://uri.etsi.org/ngsi-ld/> .
+@prefix sh: <http://www.w3.org/ns/shacl#> .
+@prefix shacl: <http://my.test/shacl/> .
+@prefix test: <http://my.test/> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+shacl:AlphaTypeShape a sh:NodeShape ;
+    sh:property [ a base:SubComponentRelationship ;
+            sh:maxCount 1 ;
+            sh:minCount 0 ;
+            sh:nodeKind sh:BlankNode ;
+            sh:path <http://my.test/entity/hasB> ;
+            sh:property [ sh:class test:BType ;
+                    sh:maxCount 1 ;
+                    sh:minCount 1 ;
+                    sh:nodeKind sh:IRI ;
+                    sh:path ngsi-ld:hasObject ] ],
+        [ sh:maxCount 1 ;
+            sh:minCount 0 ;
+            sh:nodeKind sh:BlankNode ;
+            sh:path <http://my.test/entity/hasC> ;
+            sh:property [ sh:datatype xsd:double ;
+                    sh:maxCount 1 ;
+                    sh:minCount 1 ;
+                    sh:nodeKind sh:Literal ;
+                    sh:path ngsi-ld:hasValue ] ] ;
+    sh:targetClass test:AlphaType .
+
+shacl:BTypeShape a sh:NodeShape ;
+    sh:property [ sh:maxCount 1 ;
+            sh:minCount 0 ;
+            sh:nodeKind sh:BlankNode ;
+            sh:path <http://my.test/entity/hasMyVariable> ;
+            sh:property [ sh:datatype xsd:boolean ;
+                    sh:maxCount 1 ;
+                    sh:minCount 1 ;
+                    sh:nodeKind sh:Literal ;
+                    sh:path ngsi-ld:hasValue ] ] ;
+    sh:targetClass test:BType .
+