Merge branch 'development' into feature/CONNECTOR-144/rate-limits-sync

.github/workflows/pull-request-from-branch-check.yaml

@@ -1,18 +1,18 @@
 name: Main Branch Protection
 
-on:
-  pull_request:
-    branches:
-      - main
+#on:
+#  pull_request:
+#    branches:
+#      - main
 
-jobs:
-  check-branch:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check branch
-        run: |
-          if [[ ${GITHUB_HEAD_REF} != development ]] && [[ ${GITHUB_HEAD_REF} != documentation ]] && ! [[ ${GITHUB_HEAD_REF} =~ ^hotfix/ ]]; 
-          then
-            echo "Error: Pull request must come from 'development', 'documentation' or 'hotfix/' branch"
-            exit 1
-          fi
+#jobs:
+#  check-branch:
+#    runs-on: ubuntu-latest
+#    steps:
+#      - name: Check branch
+#        run: |
+#          if [[ ${GITHUB_HEAD_REF} != development ]] && [[ ${GITHUB_HEAD_REF} != documentation ]] && ! [[ ${GITHUB_HEAD_REF} =~ ^hotfix/ ]]; 
+#          then
+#            echo "Error: Pull request must come from 'development', 'documentation' or 'hotfix/' branch"
+#            exit 1
+#          fi

appinfo/info.xml

@@ -13,7 +13,7 @@ The OpenConnector Nextcloud app provides a ESB-framework to work together in an
 - 🆓 Map and translate API calls
 
     ]]></description>
-	<version>0.1.16</version>
+	<version>0.1.20</version>
 	<licence>agpl</licence>
 	<category>integration</category>
 	<author mail="[email protected]" homepage="https://www.conduction.nl/">Conduction</author>

configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json

@@ -7,17 +7,24 @@
         "description": "d.woo_x005f_beschrijving",
         "summary": "d.woo_x005f_samenvatting",
         "category": "d.woo_x005f_categorie",
-        "published": "d.woo_x005f_publicatiedatum",
-        "modified": "d.vti_x005f_nexttolasttimemodified"
-	},
-	"unset": [],
-	"cast": {
-        "title": "d.woo_x005f_titel",
-        "description": "d.woo_x005f_beschrijving",
-        "summary": "d.woo_x005f_samenvatting",
-        "category": "d.woo_x005f_categorie",
-        "published": "d.woo_x005f_publicatiedatum",
-        "modified": "d.vti_x005f_nexttolasttimemodified"
+        "published": "{% if d['woo_x005f_publicatiedatum']|default %}{{ d['woo_x005f_publicatiedatum']|date('Y-m-d') }}{% endif %}",
+        "modified": "d.vti_x005f_nexttolasttimemodified",
+        "attachments": "[ {% for file in fileUrls %} { {% if file['Name']|default %}\"title\": \"{{ file['Name'] }}\",{% endif %}{% if file['d']['document_x005f_label']|default %}\"labels\": [\"{{ file['d']['document_x005f_label'] }}\"],{% endif %}{% if file['__metadata']['uri']|default %}\"downloadUrl\": {\"accessUrl\": \"{{ file['__metadata']['uri']~'/$value' }}\", \"source\": \"1\"}{% endif %} }{{ loop.last ? '' : ',' }} {% endfor %} ]",
+        "status": "Concept",
+        "catalog": "",
+        "publicationType": ""
+    },
+    "unset": [],
+    "cast": {
+        "title": "unsetIfValue==d.woo_x005f_titel",
+        "description": "unsetIfValue==d.woo_x005f_beschrijving",
+        "summary": "unsetIfValue==d.woo_x005f_samenvatting",
+        "category": "unsetIfValue==d.woo_x005f_categorie",
+        "published": "unsetIfValue==",
+        "modified": "unsetIfValue==d.vti_x005f_nexttolasttimemodified",
+        "attachments": "jsonToArray",
+        "publicationType": "unsetIfValue==",
+        "catalog": "unsetIfValue=="
     },
 	"passThrough": false
-}
+}

configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json

@@ -7,10 +7,20 @@
 		"title": "omschrijving",
 		"summary": "zaaktypeomschrijving",
 		"description": "zaaktypeomschrijving",
-        "published" : "startdatum",
-        "modified" : "{{ 'now'|date(H:i:sTm-d-Y') }}"
+        "published" : "",
+        "modified" : "{{ 'now'|date(H:i:sTm-d-Y') }}",
+        "status": "Concept",
+        "catalog": "",
+        "publicationType": ""
 	},
 	"unset": [],
-	"cast": [],
+	"cast": {
+		"title": "unsetIfValue==omschrijving",
+		"summary": "unsetIfValue==zaaktypeomschrijving",
+		"description": "unsetIfValue==zaaktypeomschrijving",
+        "published" : "unsetIfValue==",
+        "catalog": "unsetIfValue==",
+        "publicationType": "unsetIfValue=="
+    },
 	"passThrough": false
 }

configurations/sharepoint-woo/synchronizations/sharepoint-convenanten-to-publications.json

@@ -7,14 +7,20 @@
 	"sourceHash": "",
 	"sourceTargetMapping": "1",
 	"sourceConfig": {
-		"idPosition": "Properties.__deferred.uri",
-		"resultsPosition": "d.results",
-		"endpoint": "/Web/GetFolderByServerRelativePath(decodedurl='/WOO/Convenanten')/folders",
-        "singleEndpoint": "{{ originId }}",
-        "mergeDataSingleEndpoint": true,
-		"headers": [],
-		"query": []
-	},
+        "idPosition": "UniqueId",
+        "resultsPosition": "d.results",
+        "endpoint": "\/Web\/GetFolderByServerRelativePath(decodedurl='\/WOO\/Convenanten')\/folders",
+        "extraDataConfigs.0.dynamicEndpointLocation": "Properties.__deferred.uri",
+        "extraDataConfigs.0.mergeExtraData": "true",
+        "extraDataConfigs.1.dynamicEndpointLocation": "Files.__deferred.uri",
+        "extraDataConfigs.1.mergeExtraData": "true",
+        "extraDataConfigs.1.keyToSetExtraData": "fileUrls",
+        "extraDataConfigs.1.resultsLocation": "d.results",
+        "extraDataConfigs.1.extraDataConfigPerResult.dynamicEndpointLocation": "Properties.__deferred.uri",
+        "extraDataConfigs.1.extraDataConfigPerResult.mergeExtraData": "true",
+        "headers": [],
+        "query": []
+    },
 	"targetId": "1/1",
 	"targetType": "register/schema"
 }

configurations/sharepoint-woo/synchronizations/sharepoint-woo-verzoeken-to-publications.json

@@ -7,13 +7,19 @@
 	"sourceHash": "",
 	"sourceTargetMapping": "1",
 	"sourceConfig": {
-		"idPosition": "Properties.__deferred.uri",
+        "idPosition": "UniqueId",
 		"resultsPosition": "d.results",
 		"endpoint": "/Web/GetFolderByServerRelativePath(decodedurl='/WOO/Woo-verzoeken en -besluiten')/folders",
-        "singleEndpoint": "{{ originId }}",
-        "mergeDataSingleEndpoint": true,
-		"headers": [],
-		"query": []
+        "extraDataConfigs.0.dynamicEndpointLocation": "Properties.__deferred.uri",
+        "extraDataConfigs.0.mergeExtraData": "true",
+        "extraDataConfigs.1.dynamicEndpointLocation": "Files.__deferred.uri",
+        "extraDataConfigs.1.mergeExtraData": "true",
+        "extraDataConfigs.1.keyToSetExtraData": "fileUrls",
+        "extraDataConfigs.1.resultsLocation": "d.results",
+        "extraDataConfigs.1.extraDataConfigPerResult.dynamicEndpointLocation": "Properties.__deferred.uri",
+        "extraDataConfigs.1.extraDataConfigPerResult.mergeExtraData": "true",
+        "headers": [],
+        "query": []
 	},
 	"targetId": "1/1",
 	"targetType": "register/schema"

lib/Cron/ActionTask.php

@@ -8,7 +8,8 @@
 use OCP\BackgroundJob\TimedJob;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\IJobList;
-use OCP\DB\Exception;
+use OCP\IUserManager;
+use OCP\IUserSession;
 use Psr\Container\ContainerExceptionInterface;
 use Psr\Container\ContainerInterface;
 use Psr\Container\NotFoundExceptionInterface;
@@ -33,7 +34,9 @@ public function __construct(
         JobMapper $jobMapper,
         JobLogMapper $jobLogMapper,
         IJobList $jobList,
-        ContainerInterface $containerInterface
+        ContainerInterface $containerInterface,
+		private IUserSession $userSession,
+		private IUserManager $userManager,
     ) {
         parent::__construct($time);
         $this->jobMapper = $jobMapper;
@@ -58,7 +61,7 @@ public function __construct(
 	 * @param $argument
 	 *
 	 * @return JobLog|void
-	 * @throws Exception
+	 * @throws \OCP\DB\Exception
 	 * @throws ContainerExceptionInterface
 	 * @throws NotFoundExceptionInterface
 	 */
@@ -86,6 +89,11 @@ public function run($argument)
             return;
         }
 
+		if(empty($job->getUserId()) === false && $this->userSession->getUser() === null) {
+			$user = $this->userManager->get($job->getUserId());
+			$this->userSession->setUser($user);
+		}
+
 		$time_start = microtime(true);
 
         $action =  $this->containerInterface->get($job->getJobClass());

lib/Migration/Version1Date20241111144800.php

@@ -101,7 +101,7 @@ public function postSchemaChange(IOutput $output, Closure $schemaClosure, array
 			&& $table->hasColumn('source_id') === true && $table->hasColumn('source_hash') === true
 		) {
 			$this->connection->executeQuery("
-				UPDATE openconnector_synchronization_contracts
+				UPDATE oc_openconnector_synchronization_contracts
 				SET origin_id = source_id, origin_hash = source_hash
 				WHERE source_id IS NOT NULL
 			");

lib/Service/AuthenticationService.php

@@ -230,7 +230,7 @@ public function fetchDecosToken(array $configuration): string
 	private function getRSJWK(array $configuration): ?JWK
 	{
 		$stamp = microtime().getmypid();
-		$filename = "privatekey-$stamp";
+		$filename = "/var/tmp/privatekey-$stamp";
 		file_put_contents($filename, base64_decode($configuration['secret']));
 		$jwk = null;
 		try {

lib/Service/CallService.php

@@ -249,6 +249,8 @@ public function call(
 
 		$time_end = microtime(true);
 
+		$body = $response->getBody()->getContents();
+
 		// Let create the data array
 		$data = [
 			'request' => [
@@ -263,7 +265,8 @@ public function call(
 				'size' => $response->getBody()->getSize(),
 				'remoteIp' => $response->getHeaderLine('X-Real-IP') ?: $response->getHeaderLine('X-Forwarded-For') ?: null,
 				'headers' => $response->getHeaders(),
-				'body' => $response->getBody()->getContents(),
+				'body' => mb_check_encoding(value: $body, encoding: 'UTF-8') !== false ? $body : base64_encode($body),
+				'encoding' => mb_check_encoding(value: $body, encoding: 'UTF-8') !== false ? 'UTF-8' : 'base64',
 			]
 		];
 

lib/Service/SynchronizationService.php

@@ -48,8 +48,11 @@ class SynchronizationService
     private ObjectService $objectService;
     private Source $source;
 
-    const SINGLE_ENDPOINT            = 'singleEndpoint';
-    const MERGE_DATA_SINGLE_ENDPOINT = 'mergeDataSingleEndpoint';
+    const EXTRA_DATA_CONFIGS_LOCATION          = 'extraDataConfigs';
+    const EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION = 'dynamicEndpointLocation';
+    const EXTRA_DATA_STATIC_ENDPOINT_LOCATION  = 'staticEndpoint';
+    const KEY_FOR_EXTRA_DATA_LOCATION          = 'keyToSetExtraData';
+    const MERGE_EXTRA_DATA_OBJECT_LOCATION     = 'mergeExtraData';
 
 
 	public function __construct(
@@ -230,6 +233,81 @@ public function getObjectFromSource(Synchronization $synchronization, string $en
 		return json_decode($response['body'], true);
 	}
 
+    /**
+     * Fetches additional data for a given object based on the synchronization configuration.
+     *
+     * This method retrieves extra data using either a dynamically determined endpoint from the object
+     * or a statically defined endpoint in the configuration. The extra data can be merged with the original
+     * object or returned as-is, based on the provided configuration.
+     *
+     * @param Synchronization $synchronization The synchronization instance containing configuration details.
+     * @param array $extraDataConfig The configuration array specifying how to retrieve and handle the extra data:
+     *      - EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION: The key to retrieve the dynamic endpoint from the object.
+     *      - EXTRA_DATA_STATIC_ENDPOINT_LOCATION: The statically defined endpoint.
+     *      - KEY_FOR_EXTRA_DATA_LOCATION: The key under which the extra data should be returned.
+     *      - MERGE_EXTRA_DATA_OBJECT_LOCATION: Boolean flag indicating whether to merge the extra data with the object.
+     * @param array $object The original object for which extra data needs to be fetched.
+     *
+     * @return array The original object merged with the extra data, or the extra data itself based on the configuration.
+     *
+     * @throws Exception If both dynamic and static endpoint configurations are missing or the endpoint cannot be determined.
+     */
+    private function fetchExtraDataForObject(Synchronization $synchronization, array $extraDataConfig, array $object)
+    {
+        if (isset($extraDataConfig[$this::EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION]) === false && isset($extraDataConfig[$this::EXTRA_DATA_STATIC_ENDPOINT_LOCATION]) === false) {
+            return $object;
+        }
+
+        // Get endpoint from earlier fetched object.
+        if (isset($extraDataConfig[$this::EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION]) === true) {
+            $dotObject = new Dot($object);
+            $endpoint = $dotObject->get($extraDataConfig[$this::EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION] ?? null);
+        }
+
+        // Get endpoint static defined in config.
+        if (isset($extraDataConfig[$this::EXTRA_DATA_STATIC_ENDPOINT_LOCATION]) === true) {
+            $endpoint = $extraDataConfig[$this::EXTRA_DATA_STATIC_ENDPOINT_LOCATION];
+            $endpoint = str_replace(search: '{{ originId }}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint);
+            $endpoint = str_replace(search: '{{originId}}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint);
+        }
+
+        if (!$endpoint) {
+            throw new Exception(
+                sprintf(
+                    'Could not get static or dynamic endpoint, object: %s',
+                    json_encode($object)
+                )
+            );
+        }
+
+        $extraData = $this->getObjectFromSource($synchronization, $endpoint);
+
+        // Temporary fix,
+        if (isset($extraDataConfig['extraDataConfigPerResult']) === true) {
+            $dotObject = new Dot($extraData);
+            $results = $dotObject->get($extraDataConfig['resultsLocation']);
+
+            foreach ($results as $key => $result) {
+                $results[$key] = $this->fetchExtraDataForObject($synchronization, $extraDataConfig['extraDataConfigPerResult'], $result);
+            }
+
+            $extraData = $results;
+        }
+
+        // Set new key if configured.
+        if (isset($extraDataConfig[$this::KEY_FOR_EXTRA_DATA_LOCATION]) === true) {
+            $extraData = [$extraDataConfig[$this::KEY_FOR_EXTRA_DATA_LOCATION] => $extraData];
+        }
+
+        // Merge with earlier fetchde object if configured.
+        if (isset($extraDataConfig[$this::MERGE_EXTRA_DATA_OBJECT_LOCATION]) === true && ($extraDataConfig[$this::MERGE_EXTRA_DATA_OBJECT_LOCATION] === true || $extraDataConfig[$this::MERGE_EXTRA_DATA_OBJECT_LOCATION] === 'true')) {
+            return array_merge($object, $extraData);
+        }
+
+        return $extraData;
+    }
+
+
 	/**
 	 * Synchronize a contract
 	 *
@@ -246,35 +324,34 @@ public function getObjectFromSource(Synchronization $synchronization, string $en
 	 */
     public function synchronizeContract(SynchronizationContract $synchronizationContract, Synchronization $synchronization = null, array $object = [], ?bool $isTest = false): SynchronizationContract|Exception|array
 	{
-        $sourceConfig = $synchronization->getSourceConfig();
-		if ($synchronization !== null && isset($sourceConfig[$this::SINGLE_ENDPOINT]) === true) {
-
-			// Update endpoint
-			$endpoint = str_replace(search: '{{ originId }}', replace: $this->getOriginId($synchronization, $object), subject: $sourceConfig[$this::SINGLE_ENDPOINT]);
-			$endpoint = str_replace(search: '{{originId}}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint);
+        $sourceConfig = $this->callService->applyConfigDot($synchronization->getSourceConfig());
 
-			// Get object from source
-            if (isset($sourceConfig[$this::MERGE_DATA_SINGLE_ENDPOINT]) === true && $sourceConfig[$this::MERGE_DATA_SINGLE_ENDPOINT] === true) {
-                $object = array_merge($object, $this->getObjectFromSource(synchronization: $synchronization, endpoint: $endpoint));
-            } else {
-                $object = $this->getObjectFromSource(synchronization: $synchronization, endpoint: $endpoint);
+        // Check if extra data needs to be fetched
+        if (isset($sourceConfig[$this::EXTRA_DATA_CONFIGS_LOCATION]) === true) {
+            foreach ($sourceConfig[$this::EXTRA_DATA_CONFIGS_LOCATION] as $extraDataConfig) {
+                $object = array_merge($object, $this->fetchExtraDataForObject($synchronization, $extraDataConfig, $object));
             }
+        }
+
+		// @TODO: This should be unset through pre-mapping
+		if(isset($object['d']['vti_x005f_dirlateststamp']) === true) {
+			unset($object['d']['vti_x005f_dirlateststamp']);
 		}
 
 
         // Let create a source hash for the object
         $originHash = md5(serialize($object));
-        $synchronizationContract->setSourceLastChecked(new DateTime());
 
         // Let's prevent pointless updates @todo account for omnidirectional sync, unless the config has been updated since last check then we do want to rebuild and check if the tagert object has changed
         if ($originHash === $synchronizationContract->getOriginHash() && $synchronization->getUpdated() < $synchronizationContract->getSourceLastChecked()) {
             // The object has not changed and the config has not been updated since last check
 			return $synchronizationContract;
         }
-
+		
         // The object has changed, oke let do mappig and bla die bla
         $synchronizationContract->setOriginHash($originHash);
         $synchronizationContract->setSourceLastChanged(new DateTime());
+		$synchronizationContract->setSourceLastChecked(new DateTime());
 
 		// Check if object adheres to conditions.
 		// Take note, JsonLogic::apply() returns a range of return types, so checking it with '=== false' or '!== true' does not work properly.
@@ -300,7 +377,6 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont
             }
         }
 
-
         // set the target hash
         $targetHash = md5(serialize($targetObject));
         $synchronizationContract->setTargetHash($targetHash);