Merge branch 'development' into feature/CONNECTOR-144/rate-limits-sync

appinfo/info.xml

@@ -13,7 +13,7 @@ The OpenConnector Nextcloud app provides a ESB-framework to work together in an
 - 🆓 Map and translate API calls
 
     ]]></description>
-	<version>0.1.20</version>
+	<version>0.1.26</version>
 	<licence>agpl</licence>
 	<category>integration</category>
 	<author mail="[email protected]" homepage="https://www.conduction.nl/">Conduction</author>
@@ -33,6 +33,10 @@ The OpenConnector Nextcloud app provides a ESB-framework to work together in an
         <owncloud max-version="0" min-version="0"/>
 	</dependencies>
 
+	<background-jobs>
+		<job>OCA\OpenConnector\Cron\LogCleanUpTask</job>
+	</background-jobs>
+
 	<navigations>
 		<navigation>
 			<id>openconnector</id>

configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json

@@ -1,26 +1,26 @@
 {
 	"name": "Xxllnc suite to Publication",
-	"description": "",
-	"version": "",
-	"reference": "",
+	"version": "0.0.1",
 	"mapping": {
-		"title": "omschrijving",
-		"summary": "zaaktypeomschrijving",
-		"description": "zaaktypeomschrijving",
-        "published" : "",
-        "modified" : "{{ 'now'|date(H:i:sTm-d-Y') }}",
-        "status": "Concept",
-        "catalog": "",
-        "publicationType": ""
-	},
-	"unset": [],
-	"cast": {
-		"title": "unsetIfValue==omschrijving",
-		"summary": "unsetIfValue==zaaktypeomschrijving",
-		"description": "unsetIfValue==zaaktypeomschrijving",
-        "published" : "unsetIfValue==",
-        "catalog": "unsetIfValue==",
-        "publicationType": "unsetIfValue=="
+        "title": "omschrijving",
+        "summary": "zaaktypeomschrijving",
+        "description": "zaaktypeomschrijving",
+        "category": "{% if zaaktypecode|default %}{% set wooVerzoekenEnBesluiten = ['LP00000431', 'B1873', 'cherry'] %}{% set klachtoordelen = ['LP00000091', 'LP00001132', 'LP00000121', 'B0757', 'LP00000832', 'LP00001096'] %}{% if zaaktypecode in wooVerzoekenEnBesluiten %}{{ 'Woo-verzoeken en -besluiten' }}{% elseif zaaktypecode in klachtoordelen %}{{ 'Klachtoordelen' }}{% endif %}{% endif %}",
+        "published": "startdatum",
+        "modified": "{{ 'now'|date('H:i:sTm-d-Y') }}",
+        "attachments": "[{% if files|default %}{% for file in files %} { {% if file['titel']|default %}\"title\": \"{{ file['titel'] }}\",{% endif %}\"labels\": [\"{{ 'Informatieverzoek' }}\"],{% if file['formaat']|default %}\"extension\": \"{{ file['formaat']|split('/')|last }}\",\"type\": \"{{ file['formaat'] }}\",{% endif %}{% if file['inhoud']|default and file['formaat']|default %}\"accessUrl\": \"data:{{ file['formaat'] }};base64,{{ file.inhoud }}\"{% endif %} }{{ loop.last ? '' : ',' }} {% endfor %}{% endif %}]",
+        "status": "Concept"
+    },
+    "unset": [
+        ""
+    ],
+    "cast": {
+        "title": "unsetIfValue==omschrijving",
+        "summary": "unsetIfValue==zaaktypeomschrijving",
+        "description": "unsetIfValue==zaaktypeomschrijving",
+        "category": "unsetIfValue==",
+        "published": "unsetIfValue==startdatum",
+        "attachments": "jsonToArray"
     },
 	"passThrough": false
 }

configurations/sharepoint-woo/synchronizations/xxllnc-suite-to-publications.json

@@ -7,13 +7,19 @@
 	"sourceHash": "",
 	"sourceTargetMapping": "1",
 	"sourceConfig": {
-		"idPosition": "id",
-		"resultsPosition": "results",
-		"endpoint": "/tlb/zaaksysteem/api/v1/zaken",
-		"headers": [],
-		"query.startdatum__gte": "2024-07-06",
-		"query.einddatum__lt": "2024-08-01"
-	},
+        "idPosition": "identificatie",
+        "resultsPosition": "results",
+        "endpoint": "\/tlb\/zaaksysteem\/api\/v1\/zaken",
+        "headers": [],
+        "query.startdatum__gte": "2024-08-01",
+        "query.einddatum__lt": "2025-01-01",
+        "usesPagination": "false",
+        "extraDataConfigs.0.staticEndpoint": "/tlb/zaaksysteem/api/v1/zaken/{{ originId }}/informatieobjecten",
+        "extraDataConfigs.0.mergeExtraData": "true",
+        "extraDataConfigs.0.keyToSetExtraData": "files",
+        "extraDataConfigs.0.resultsLocation": "results",
+        "extraDataConfigs.0.extraDataConfigPerResult.staticEndpoint": "/tlb/zaaksysteem/api/v1/informatieobjecten/{{ originId }}"
+    },
 	"targetId": "1/1",
 	"targetType": "register/schema"
 }

lib/Cron/ActionTask.php

@@ -113,10 +113,11 @@ public function run($argument)
 
 
         // Update the job
-		$nextRun = new DateTime();
+		$nextRun = new DateTime('now + '.$job->getInterval().' seconds');
 		if (isset($result['nextRun']) === true) {
 			$nextRun = DateTime::createFromFormat('U', $result['nextRun']);
 		}
+		$nextRun->setTime(hour: $nextRun->format('H'), minute: $nextRun->format('i'));
         $job->setLastRun(new DateTime());
         $job->setNextRun($nextRun);
         $this->jobMapper->update($job);
@@ -145,6 +146,8 @@ public function run($argument)
             }
         }
 
+		$this->jobLogMapper->update(entity: $jobLog);
+
         // Let's report back about what we have just done
         return $jobLog;
     }

lib/Cron/LogCleanUpTask.php

@@ -3,13 +3,35 @@
 namespace OCA\OpenConnector\Cron;
 
 use OCA\OpenConnector\Db\CallLogMapper;
+use OCA\OpenConnector\Db\JobLogMapper;
+use OCA\OpenConnector\Db\JobMapper;
+use OCP\BackgroundJob\IJobList;
 use OCP\BackgroundJob\TimedJob;
 use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\IUserManager;
+use OCP\IUserSession;
 
-class LogCleanUpTask
+class LogCleanUpTask extends TimedJob
 {
-    public function doCron(array $arguments, CallLogMapper $callLogMapper){
-        $callLogMapper = new ClearLogs();
+	public function __construct(
+		ITimeFactory $time,
+		private readonly CallLogMapper $callLogMapper,
+	) {
+		parent::__construct($time);
+
+		// Run every 5 minutes
+		$this->setInterval(300);
+
+		// Delay until low-load time
+		//$this->setTimeSensitivity(\OCP\BackgroundJob\IJob::TIME_SENSITIVE);
+		// Or $this->setTimeSensitivity(\OCP\BackgroundJob\IJob::TIME_INSENSITIVE);
+
+		// Only run one instance of this job at a time
+		$this->setAllowParallelRuns(false);
+	}
+
+    public function run(mixed $argument){
+        $this->callLogMapper->clearLogs();
     }
 
 }

lib/Db/CallLogMapper.php

@@ -93,7 +93,7 @@ public function clearLogs(): bool
 
         // Build the delete query
         $qb->delete('openconnector_call_logs')
-           ->where($qb->expr()->lt('expired', $qb->createFunction('NOW()')));
+           ->where($qb->expr()->lt('expires', $qb->createFunction('NOW()')));
 
         // Execute the query and get the number of affected rows
         $result = $qb->execute();

lib/Db/Synchronization.php

@@ -67,6 +67,21 @@ public function __construct() {
 		$this->addType(fieldName:'followUps', type: 'json');
 	}
 
+    /**
+     * Checks through sourceConfig if the source of this sync uses pagination
+     * 
+     * @return bool true if its uses pagination
+     */
+    public function usesPagination(): bool
+    {
+        if (isset($this->sourceConfig['usesPagination']) === true && ($this->sourceConfig['usesPagination'] === false || $this->sourceConfig['usesPagination'] === 'false')) {
+            return false;
+        }
+
+        // By default sources use basic pagination.
+        return true;
+    }
+
 	public function getJsonFields(): array
 	{
 		return array_keys(

lib/Migration/Version1Date20241206095007.php

@@ -0,0 +1,66 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\OpenConnector\Migration;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+/**
+ * FIXME Auto-generated migration step: Please modify to your needs!
+ */
+class Version1Date20241206095007 extends SimpleMigrationStep {
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure(): ISchemaWrapper $schemaClosure
+	 * @param array $options
+	 */
+	public function preSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void {
+	}
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure(): ISchemaWrapper $schemaClosure
+	 * @param array $options
+	 * @return null|ISchemaWrapper
+	 */
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+		/**
+		 * @var ISchemaWrapper $schema
+		 */
+		$schema = $schemaClosure();
+
+		if ($schema->hasTable('openconnector_sources') === true) {
+			$table = $schema->getTable('openconnector_sources');
+
+			if ($table->hasColumn('logRetention') === true) {
+				$table->dropColumn('logRetention');
+				$table->addColumn('log_retention', Types::INTEGER)->setNotnull(false)->setDefault(3600);
+			}
+			if ($table->hasColumn('errorRetention') === true) {
+				$table->dropColumn('errorRetention');
+				$table->addColumn('error_retention', Types::INTEGER)->setNotnull(false)->setDefault(86400);
+			}
+		}
+
+		return $schema;
+	}
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure(): ISchemaWrapper $schemaClosure
+	 * @param array $options
+	 */
+	public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void {
+	}
+}

lib/Service/CallService.php

@@ -138,6 +138,7 @@ public function call(
 			$callLog->setStatusCode(409);
 			$callLog->setStatusMessage("This source is not enabled");
 			$callLog->setCreated(new \DateTime());
+			$callLog->setExpires(new \DateTime('now + '.$source->getErrorRetention().' seconds'));
 
 			$this->callLogMapper->insert($callLog);
 
@@ -152,6 +153,7 @@ public function call(
 			$callLog->setStatusCode(409);
 			$callLog->setStatusMessage("This source has no location");
 			$callLog->setCreated(new \DateTime());
+			$callLog->setExpires(new \DateTime('now + '.$source->getErrorRetention().' seconds'));
 
 			$this->callLogMapper->insert($callLog);
 
@@ -178,6 +180,7 @@ public function call(
 			$callLog->setStatusCode(429); //
 			$callLog->setStatusMessage("The rate limit for this source has been exceeded. Try again later.");
 			$callLog->setCreated(new \DateTime());
+			$callLog->setExpires(new \DateTime('now + '.$source->getErrorRetention().' seconds'));
 
 			$this->callLogMapper->insert($callLog);
 
@@ -282,6 +285,7 @@ public function call(
 		$callLog->setRequest($data['request']);
 		$callLog->setResponse($data['response']);
 		$callLog->setCreated(new \DateTime());
+		$callLog->setExpires(new \DateTime('now + '.($data['response']['statusCode'] < 400 ? $source->getLogRetention() : $source->getErrorRetention()).' seconds'));
 
 		$this->callLogMapper->insert($callLog);
 

lib/Service/SynchronizationService.php

@@ -617,7 +617,7 @@ public function getAllObjectsFromApi(Synchronization $synchronization, ?bool $is
 			$this->synchronizationMapper->update($synchronization);
 		}
 
-		if ($useNextEndpoint === false) {
+		if ($useNextEndpoint === false && $synchronization->usesPagination() === true) {
 			do {
 				$config   = $this->getNextPage(config: $config, sourceConfig: $sourceConfig, currentPage: $currentPage);
 				$callLog = $this->callService->call(source: $source, endpoint: $endpoint, method: 'GET', config: $config);