Introduce external ontology handling.

With this PR external ontologies can be pulled and combined to create the KMS data.
Every onology contains of a prefix, and three parts: prefix_entities.ttl, prefix_knowledge.ttl
and prefix_shacl.ttl. Different ontologies can be pulled together with a make target:
make ontology2kms ONTOLOGIES="base filter"
would pull in base and filter ontologies and compile a joint kms structure.
The PR provides:
* Syncing of knowledge configmaps between debezium-bridge and semantic model
* Added stakater/Reloader to restart debezium-bridge when new knowledge is deployed
* Extended SPARQL parser to enable consistent inheritance of RDFS Types when creating SHACL constraints and rules
* Tolerating of rdfs:subClassOf* path expression
* Streamlining of Knowledge closure calculation across all tools and add explicit script to calculate
  knowledge closure (create_knowledge_closure.py)
* Add explicit tool to execute SPARQL query (check_sparql_expression.py)
* For the time being - drop the sh:ord based deterministic ordering of fields. It creates confusion but will be needed in future
  to allow forward compatibility. For the time being, deterministic ordering is achieved by lexical ordering by field names.

Related EPIC: IndustryFusion#514
Related User Story: IndustryFusion#515

Signed-off-by: marcel <[email protected]>

helm/airgap-deployment/prepare-airgap.bash

@@ -117,3 +117,4 @@ wget -O- https://github.com/strimzi/strimzi-kafka-operator/releases/download/${S
 ( cd ${OFFLINE_DIR} && rm -rf emqx-operator && git clone https://github.com/emqx/emqx-operator.git && cd emqx-operator && git checkout ${EMQX_OPERATOR_VERSION} )
 ( cd ${OFFLINE_DIR} && rm -rf postgres-operator && git clone https://github.com/zalando/postgres-operator.git && cd postgres-operator && git checkout ${POSTGRES_OPERATOR_VERSION} )
 ( cd ${OFFLINE_DIR} && rm -rf helm-charts && git clone https://github.com/vmware-tanzu/helm-charts.git && cd helm-charts && git checkout ${VELERO_HELM_VERSION})
+( cd ${OFFLINE_DIR} && rm -rf Reloader && git clone https://github.com/stakater/Reloader.git && cd Reloader && git checkout ${RELOADER_HELM_VERSION})

helm/charts/kafka-bridges/templates/alerta-bridge-deployment.yaml

@@ -58,7 +58,5 @@ spec:
             items:
               - key: "config.json"
                 path: "config.json"
-              - key: "knowledge.ttl"
-                path: "knowledge.ttl"
       imagePullSecrets:
         - name: {{ .Values.pullSecretCredentials }}

helm/charts/kafka-bridges/templates/bridge-configmap.yaml

@@ -114,5 +114,4 @@ data:
               "password": "POSTGRES_PASSWORD"
       }
     }
-  knowledge.ttl: |
-{{ .Files.Get "kms/knowledge.ttl" | indent 4 }}
+

helm/charts/kafka-bridges/templates/debezium-bridge-deployment.yaml

@@ -8,6 +8,8 @@ metadata:
   namespace: {{ .Release.Namespace }}
   labels:
     app: debezium-bridge
+  annotations:
+    configmap.reloader.stakater.com/reload: "knowledge"
 spec:
   replicas: {{ .Values.kafkaBridge.debezium.replicaCount }}
   selector:
@@ -45,19 +47,20 @@ spec:
             initialDelaySeconds: 5
           volumeMounts:
             - name: config
-              mountPath: /opt/config
+              mountPath: /opt/config/config.json
+              readOnly: true
+              subPath: config.json
+            - name: knowledge-config
+              mountPath: /opt/config/knowledge.ttl
+              subPath: knowledge.ttl
               readOnly: true
           resources:
       volumes:
         - name: config
           configMap:
-            # Provide the name of the ConfigMap you want to mount.
             name: bridge-configmap
-            # An array of keys from the ConfigMap to create as files
-            items:
-              - key: "config.json"
-                path: "config.json"
-              - key: "knowledge.ttl"
-                path: "knowledge.ttl"
+        - name: knowledge-config
+          configMap:
+            name: knowledge
       imagePullSecrets:
         - name: {{ .Values.pullSecretCredentials }}

helm/charts/kafka-bridges/templates/knowledge-configmap.yaml

@@ -0,0 +1,30 @@
+---
+# yamllint disable rule:line-length
+# yamllint disable rule:braces
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: knowledge
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: knowledge
+data:
+  {{- $configmap := (lookup "v1" "ConfigMap" .Release.Namespace "knowledge") }}
+  {{- if $configmap }}
+  {{- $knowledge := get $configmap.data "knowledge.ttl" }}
+  knowledge.ttl: {{ $knowledge | quote }}
+  {{- else }}
+  knowledge.ttl: |
+        PREFIX rdf:   <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+        PREFIX rdfs:   <http://www.w3.org/2000/01/rdf-schema#>
+        PREFIX iff: <https://industry-fusion.com/types/v0.9/>
+        PREFIX oee: <https://industry-fusion.com/oee/v0.9/>
+        PREFIX owl: <http://www.w3.org/2002/07/owl#>
+
+        ### Test classes
+        iff:cutter_test rdfs:subClassOf iff:machine_test ;
+                a rdfs:Class .
+        iff:plasmacutter_test rdfs:subClassOf iff:cutter_test ;
+                a rdfs:Class .
+        ### End of Test Content
+  {{- end }}

helm/charts/kafka-bridges/templates/ngsild-updates-bridge-deployment.yaml

@@ -62,7 +62,5 @@ spec:
             items:
               - key: "config.json"
                 path: "config.json"
-              - key: "knowledge.ttl"
-                path: "knowledge.ttl"
       imagePullSecrets:
         - name: {{ .Values.pullSecretCredentials }}

helm/charts/kafka-bridges/templates/timescaledb-bridge-deployment.yaml

@@ -64,7 +64,5 @@ spec:
             items:
               - key: "config.json"
                 path: "config.json"
-              - key: "knowledge.ttl"
-                path: "knowledge.ttl"
       imagePullSecrets:
         - name: {{ .Values.pullSecretCredentials }}

helm/common.yaml

@@ -13,4 +13,8 @@ db:
   clusterSvcPostfix: cluster
   svcPort: "5432"
   dbUser: "ngb"
-  oispdbUser: "oisp"
+  oispdbUser: "oisp"
+
+ontology:
+  baseUri: https://industryfusion.github.io/contexts/staging/ontology/v0.1/
+

helm/env.sh

@@ -14,6 +14,7 @@ export STRIMZI_VERSION="0.32.0"
 export OFFLINE=${OFFLINE:-false}
 export OFFLINE_DIR=$(cd $DIRNAME/airgap-deployment; pwd)
 export KEYCLOAK_VERSION=21.1.2
+export RELOADER_HELM_VERSION=v1.0.67
 COMMON_MAIN_REGISTRY=$(yq ".mainRegistry" < $DIRNAME/common.yaml)
 COMMON_EXTERNAL_REGISTRY=$(yq ".externalRegistry" < $DIRNAME/common.yaml)
 COMMON_EXTERNAL_REGISTRY2=$(yq ".externalRegistry2" < $DIRNAME/common.yaml)

helm/install_operators.sh

@@ -118,11 +118,25 @@ fi
 sleep 10
 done
 
+
+printf "\n"
+printf "\033[1mInstalling Reloader Operator\n"
+printf -- "------------------------\033[0m\n"
+if [ "$OFFLINE" = "true" ]; then
+  ( cd ${OFFLINE_DIR}/Reloader && helm upgrade --install --atomic reloader ./deployments/kubernetes/chart/reloader \
+      --set reloader.image.name=${REGISTRY}/stakater/reloader --set reloader.reloadOnCreate=true)
+else
+  helm repo add stakater https://stakater.github.io/stakater-charts
+  helm repo update
+  helm upgrade --install reloader stakater/reloader --version ${RELOADER_HELM_VERSION} --set reloader.reloadOnCreate=true
+fi
+
 printf "\n"
 printf "\033[1mPrepare Velero Helm Chart Repo\n"
 printf -- "------------------------\033[0m\n"
 
 if [ ! "$OFFLINE" = "true" ]; then
   ( cd ${OFFLINE_DIR} && rm -rf helm-charts && git clone https://github.com/vmware-tanzu/helm-charts.git && cd helm-charts && git checkout ${VELERO_HELM_VERSION} )
 fi
+
 printf -- "\033[1mOperators installed successfully.\033[0m\n"