Merge pull request #105 from ConductionNL/feature/fetch-extra-data

Fetch extra data

.github/workflows/pull-request-from-branch-check.yaml

@@ -1,18 +1,18 @@
 name: Main Branch Protection
 
-on:
-  pull_request:
-    branches:
-      - main
+#on:
+#  pull_request:
+#    branches:
+#      - main
 
-jobs:
-  check-branch:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check branch
-        run: |
-          if [[ ${GITHUB_HEAD_REF} != development ]] && [[ ${GITHUB_HEAD_REF} != documentation ]] && ! [[ ${GITHUB_HEAD_REF} =~ ^hotfix/ ]]; 
-          then
-            echo "Error: Pull request must come from 'development', 'documentation' or 'hotfix/' branch"
-            exit 1
-          fi
+#jobs:
+#  check-branch:
+#    runs-on: ubuntu-latest
+#    steps:
+#      - name: Check branch
+#        run: |
+#          if [[ ${GITHUB_HEAD_REF} != development ]] && [[ ${GITHUB_HEAD_REF} != documentation ]] && ! [[ ${GITHUB_HEAD_REF} =~ ^hotfix/ ]]; 
+#          then
+#            echo "Error: Pull request must come from 'development', 'documentation' or 'hotfix/' branch"
+#            exit 1
+#          fi

appinfo/info.xml

@@ -13,7 +13,7 @@ The OpenConnector Nextcloud app provides a ESB-framework to work together in an
 - 🆓 Map and translate API calls
 
     ]]></description>
-	<version>0.1.16</version>
+	<version>0.1.20</version>
 	<licence>agpl</licence>
 	<category>integration</category>
 	<author mail="[email protected]" homepage="https://www.conduction.nl/">Conduction</author>

configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json

@@ -7,17 +7,21 @@
         "description": "d.woo_x005f_beschrijving",
         "summary": "d.woo_x005f_samenvatting",
         "category": "d.woo_x005f_categorie",
-        "published": "d.woo_x005f_publicatiedatum",
-        "modified": "d.vti_x005f_nexttolasttimemodified"
-	},
-	"unset": [],
-	"cast": {
-        "title": "d.woo_x005f_titel",
-        "description": "d.woo_x005f_beschrijving",
-        "summary": "d.woo_x005f_samenvatting",
-        "category": "d.woo_x005f_categorie",
-        "published": "d.woo_x005f_publicatiedatum",
-        "modified": "d.vti_x005f_nexttolasttimemodified"
+        "published": "{% if d['woo_x005f_publicatiedatum']|default %}{{ d['woo_x005f_publicatiedatum']|date('Y-m-d') }}{% endif %}",
+        "modified": "d.vti_x005f_nexttolasttimemodified",
+        "attachments": "{{ '[' }}{% if fileUrls['d']['results']|default %}{% set index = 0 %}{% for file in fileUrls['d']['results'] %}{% if index > 0 %}{{ ', ' }}{% endif %}{{ '{' }}{% if file['Name']|default %} \"title\": \"{{ file['Name'] }}\",{% endif %} \"downloadUrl\": {{ '{' }}{% set uri = file['__metadata']['uri']~'/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"1\"' }}{{ '}' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}",
+        "status": "{% if d['woo_x005f_publicatiedatum']|default %}Published{% else %}Concept{% endif %}",
+        "catalog": ""
+    },
+    "unset": [],
+    "cast": {
+        "title": "unsetIfValue==d.woo_x005f_titel",
+        "description": "unsetIfValue==d.woo_x005f_beschrijving",
+        "summary": "unsetIfValue==d.woo_x005f_samenvatting",
+        "category": "unsetIfValue==d.woo_x005f_categorie",
+        "published": "unsetIfValue==d.woo_x005f_publicatiedatum",
+        "modified": "unsetIfValue==d.vti_x005f_nexttolasttimemodified",
+        "attachments": "jsonToArray"
     },
 	"passThrough": false
-}
+}

configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json

@@ -7,10 +7,17 @@
 		"title": "omschrijving",
 		"summary": "zaaktypeomschrijving",
 		"description": "zaaktypeomschrijving",
-        "published" : "startdatum",
-        "modified" : "{{ 'now'|date(H:i:sTm-d-Y') }}"
+        "published" : "{% if startdatum|default %}{{ startdatum|date('Y-m-d') }}{% endif %}",
+        "modified" : "{{ 'now'|date(H:i:sTm-d-Y') }}",
+        "status": "{% if startdatum|default %}Published{% else %}Concept{% endif %}",
+        "catalog": ""
 	},
 	"unset": [],
-	"cast": [],
+	"cast": {
+		"title": "unsetIfValue==omschrijving",
+		"summary": "unsetIfValue==zaaktypeomschrijving",
+		"description": "unsetIfValue==zaaktypeomschrijving",
+        "published" : "unsetIfValue=="
+    },
 	"passThrough": false
 }

configurations/sharepoint-woo/synchronizations/sharepoint-convenanten-to-publications.json

@@ -7,14 +7,17 @@
 	"sourceHash": "",
 	"sourceTargetMapping": "1",
 	"sourceConfig": {
-		"idPosition": "Properties.__deferred.uri",
-		"resultsPosition": "d.results",
-		"endpoint": "/Web/GetFolderByServerRelativePath(decodedurl='/WOO/Convenanten')/folders",
-        "singleEndpoint": "{{ originId }}",
-        "mergeDataSingleEndpoint": true,
-		"headers": [],
-		"query": []
-	},
+        "idPosition": "UniqueId",
+        "resultsPosition": "d.results",
+        "endpoint": "\/Web\/GetFolderByServerRelativePath(decodedurl='\/WOO\/Convenanten')\/folders",
+        "extraDataConfigs.0.dynamicEndpointLocation": "Properties.__deferred.uri",
+        "extraDataConfigs.0.mergeExtraData": "true",
+        "extraDataConfigs.1.dynamicEndpointLocation": "Files.__deferred.uri",
+        "extraDataConfigs.1.mergeExtraData": "true",
+        "extraDataConfigs.1.keyToSetExtraData": "fileUrls",
+        "headers": [],
+        "query": []
+    },
 	"targetId": "1/1",
 	"targetType": "register/schema"
 }

configurations/sharepoint-woo/synchronizations/sharepoint-woo-verzoeken-to-publications.json

@@ -7,13 +7,16 @@
 	"sourceHash": "",
 	"sourceTargetMapping": "1",
 	"sourceConfig": {
-		"idPosition": "Properties.__deferred.uri",
+        "idPosition": "UniqueId",
 		"resultsPosition": "d.results",
 		"endpoint": "/Web/GetFolderByServerRelativePath(decodedurl='/WOO/Woo-verzoeken en -besluiten')/folders",
-        "singleEndpoint": "{{ originId }}",
-        "mergeDataSingleEndpoint": true,
-		"headers": [],
-		"query": []
+        "extraDataConfigs.0.dynamicEndpointLocation": "Properties.__deferred.uri",
+        "extraDataConfigs.0.mergeExtraData": "true",
+        "extraDataConfigs.1.dynamicEndpointLocation": "Files.__deferred.uri",
+        "extraDataConfigs.1.mergeExtraData": "true",
+        "extraDataConfigs.1.keyToSetExtraData": "fileUrls",
+        "headers": [],
+        "query": []
 	},
 	"targetId": "1/1",
 	"targetType": "register/schema"

lib/Migration/Version1Date20241111144800.php

@@ -101,7 +101,7 @@ public function postSchemaChange(IOutput $output, Closure $schemaClosure, array
 			&& $table->hasColumn('source_id') === true && $table->hasColumn('source_hash') === true
 		) {
 			$this->connection->executeQuery("
-				UPDATE openconnector_synchronization_contracts
+				UPDATE oc_openconnector_synchronization_contracts
 				SET origin_id = source_id, origin_hash = source_hash
 				WHERE source_id IS NOT NULL
 			");

lib/Service/CallService.php

@@ -249,6 +249,8 @@ public function call(
 
 		$time_end = microtime(true);
 
+		$body = $response->getBody()->getContents();
+
 		// Let create the data array
 		$data = [
 			'request' => [
@@ -263,7 +265,8 @@ public function call(
 				'size' => $response->getBody()->getSize(),
 				'remoteIp' => $response->getHeaderLine('X-Real-IP') ?: $response->getHeaderLine('X-Forwarded-For') ?: null,
 				'headers' => $response->getHeaders(),
-				'body' => $response->getBody()->getContents(),
+				'body' => mb_check_encoding(value: $body, encoding: 'UTF-8') !== false ? $body : base64_encode($body),
+				'encoding' => mb_check_encoding(value: $body, encoding: 'UTF-8') !== false ? 'UTF-8' : 'base64',
 			]
 		];
 

lib/Service/SynchronizationService.php

@@ -47,8 +47,11 @@ class SynchronizationService
     private ObjectService $objectService;
     private Source $source;
 
-    const SINGLE_ENDPOINT            = 'singleEndpoint';
-    const MERGE_DATA_SINGLE_ENDPOINT = 'mergeDataSingleEndpoint';
+    const EXTRA_DATA_CONFIGS_LOCATION          = 'extraDataConfigs';
+    const EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION = 'dynamicEndpointLocation';
+    const EXTRA_DATA_STATIC_ENDPOINT_LOCATION  = 'staticEndpoint';
+    const KEY_FOR_EXTRA_DATA_LOCATION          = 'keyToSetExtraData';
+    const MERGE_EXTRA_DATA_OBJECT_LOCATION     = 'mergeExtraData';
 
 
 	public function __construct(
@@ -217,6 +220,69 @@ public function getObjectFromSource(Synchronization $synchronization, string $en
 		return json_decode($response['body'], true);
 	}
 
+    /**
+     * Fetches additional data for a given object based on the synchronization configuration.
+     *
+     * This method retrieves extra data using either a dynamically determined endpoint from the object
+     * or a statically defined endpoint in the configuration. The extra data can be merged with the original
+     * object or returned as-is, based on the provided configuration.
+     *
+     * @param Synchronization $synchronization The synchronization instance containing configuration details.
+     * @param array $extraDataConfig The configuration array specifying how to retrieve and handle the extra data:
+     *      - EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION: The key to retrieve the dynamic endpoint from the object.
+     *      - EXTRA_DATA_STATIC_ENDPOINT_LOCATION: The statically defined endpoint.
+     *      - KEY_FOR_EXTRA_DATA_LOCATION: The key under which the extra data should be returned.
+     *      - MERGE_EXTRA_DATA_OBJECT_LOCATION: Boolean flag indicating whether to merge the extra data with the object.
+     * @param array $object The original object for which extra data needs to be fetched.
+     *
+     * @return array The original object merged with the extra data, or the extra data itself based on the configuration.
+     *
+     * @throws Exception If both dynamic and static endpoint configurations are missing or the endpoint cannot be determined.
+     */
+    private function fetchExtraDataForObject(Synchronization $synchronization, array $extraDataConfig, array $object)
+    {
+        if (isset($extraDataConfig[$this::EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION]) === false && isset($extraDataConfig[$this::EXTRA_DATA_STATIC_ENDPOINT_LOCATION]) === false) {
+            return $object;
+        }
+
+        // Get endpoint from earlier fetched object.
+        if (isset($extraDataConfig[$this::EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION]) === true) {
+            $dotObject = new Dot($object);
+            $endpoint = $dotObject->get($extraDataConfig[$this::EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION] ?? null);
+        }
+
+        // Get endpoint static defined in config.
+        if (isset($extraDataConfig[$this::EXTRA_DATA_STATIC_ENDPOINT_LOCATION]) === true) {
+            $endpoint = $extraDataConfig[$this::EXTRA_DATA_STATIC_ENDPOINT_LOCATION];
+            $endpoint = str_replace(search: '{{ originId }}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint);
+            $endpoint = str_replace(search: '{{originId}}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint);
+        }
+
+        if (!$endpoint) {
+            throw new Exception(
+                sprintf(
+                    'Could not get static or dynamic endpoint, object: %s',
+                    json_encode($object)
+                )
+            );
+        }
+
+        $extraData = $this->getObjectFromSource($synchronization, $endpoint);
+
+        // Set new key if configured.
+        if (isset($extraDataConfig[$this::KEY_FOR_EXTRA_DATA_LOCATION]) === true) {
+            $extraData = [$extraDataConfig[$this::KEY_FOR_EXTRA_DATA_LOCATION] => $extraData];
+        }
+
+        // Merge with earlier fetchde object if configured.
+        if (isset($extraDataConfig[$this::MERGE_EXTRA_DATA_OBJECT_LOCATION]) === true && $extraDataConfig[$this::MERGE_EXTRA_DATA_OBJECT_LOCATION] === true) {
+            return array_merge($object, $extraData);
+        }
+
+        return $extraData;
+    }
+
+
 	/**
 	 * Synchronize a contract
 	 *
@@ -233,25 +299,18 @@ public function getObjectFromSource(Synchronization $synchronization, string $en
 	 */
     public function synchronizeContract(SynchronizationContract $synchronizationContract, Synchronization $synchronization = null, array $object = [], ?bool $isTest = false): SynchronizationContract|Exception|array
 	{
-        $sourceConfig = $synchronization->getSourceConfig();
-		if ($synchronization !== null && isset($sourceConfig[$this::SINGLE_ENDPOINT]) === true) {
-
-			// Update endpoint
-			$endpoint = str_replace(search: '{{ originId }}', replace: $this->getOriginId($synchronization, $object), subject: $sourceConfig[$this::SINGLE_ENDPOINT]);
-			$endpoint = str_replace(search: '{{originId}}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint);
+        $sourceConfig = $this->callService->applyConfigDot($synchronization->getSourceConfig());
 
-			// Get object from source
-            if (isset($sourceConfig[$this::MERGE_DATA_SINGLE_ENDPOINT]) === true && $sourceConfig[$this::MERGE_DATA_SINGLE_ENDPOINT] === true) {
-                $object = array_merge($object, $this->getObjectFromSource(synchronization: $synchronization, endpoint: $endpoint));
-            } else {
-                $object = $this->getObjectFromSource(synchronization: $synchronization, endpoint: $endpoint);
+        // Check if extra data needs to be fetched
+        if (isset($sourceConfig[$this::EXTRA_DATA_CONFIGS_LOCATION]) === true) {
+            foreach ($sourceConfig[$this::EXTRA_DATA_CONFIGS_LOCATION] as $extraDataConfig) {
+                $object = array_merge($object, $this->fetchExtraDataForObject($synchronization, $extraDataConfig, $object));
             }
-		}
+        }
 
 
         // Let create a source hash for the object
         $originHash = md5(serialize($object));
-        $synchronizationContract->setSourceLastChecked(new DateTime());
 
         // Let's prevent pointless updates @todo account for omnidirectional sync, unless the config has been updated since last check then we do want to rebuild and check if the tagert object has changed
         if ($originHash === $synchronizationContract->getOriginHash() && $synchronization->getUpdated() < $synchronizationContract->getSourceLastChecked()) {
@@ -262,6 +321,7 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont
         // The object has changed, oke let do mappig and bla die bla
         $synchronizationContract->setOriginHash($originHash);
         $synchronizationContract->setSourceLastChanged(new DateTime());
+		$synchronizationContract->setSourceLastChecked(new DateTime());
 
 		// Check if object adheres to conditions.
 		// Take note, JsonLogic::apply() returns a range of return types, so checking it with '=== false' or '!== true' does not work properly.

lib/Settings/OpenConnectorAdmin.php

@@ -0,0 +1,43 @@
+<?php
+namespace OCA\OpenConnector\Settings;
+
+use OCP\AppFramework\Http\TemplateResponse;
+use OCP\IConfig;
+use OCP\IL10N;
+use OCP\Settings\ISettings;
+
+class OpenConnectorAdmin implements ISettings {
+	private IL10N $l;
+	private IConfig $config;
+
+	public function __construct(IConfig $config, IL10N $l) {
+		$this->config = $config;
+		$this->l = $l;
+	}
+
+	/**
+	 * @return TemplateResponse
+	 */
+	public function getForm() {
+		$parameters = [
+			'mySetting' => $this->config->getSystemValue('open_connector_setting', true),
+		];
+
+		return new TemplateResponse('openconnector', 'settings/admin', $parameters, '');
+	}
+
+	public function getSection() {
+		return 'openconnector'; // Name of the previously created section.
+	}
+
+	/**
+	 * @return int whether the form should be rather on the top or bottom of
+	 * the admin section. The forms are arranged in ascending order of the
+	 * priority values. It is required to return a value between 0 and 100.
+	 *
+	 * E.g.: 70
+	 */
+	public function getPriority() {
+		return 10;
+	}
+}