diff --git a/appinfo/info.xml b/appinfo/info.xml index 768b23e..f04f76a 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -13,7 +13,7 @@ The OpenConnector Nextcloud app provides a ESB-framework to work together in an - 🆓 Map and translate API calls ]]> - 0.1.20 + 0.1.26 agpl integration Conduction @@ -33,6 +33,10 @@ The OpenConnector Nextcloud app provides a ESB-framework to work together in an + + OCA\OpenConnector\Cron\LogCleanUpTask + + openconnector diff --git a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json index 7a2ea21..744823b 100644 --- a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json +++ b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json @@ -1,26 +1,26 @@ { "name": "Xxllnc suite to Publication", - "description": "", - "version": "", - "reference": "", + "version": "0.0.1", "mapping": { - "title": "omschrijving", - "summary": "zaaktypeomschrijving", - "description": "zaaktypeomschrijving", - "published" : "", - "modified" : "{{ 'now'|date(H:i:sTm-d-Y') }}", - "status": "Concept", - "catalog": "", - "publicationType": "" - }, - "unset": [], - "cast": { - "title": "unsetIfValue==omschrijving", - "summary": "unsetIfValue==zaaktypeomschrijving", - "description": "unsetIfValue==zaaktypeomschrijving", - "published" : "unsetIfValue==", - "catalog": "unsetIfValue==", - "publicationType": "unsetIfValue==" + "title": "omschrijving", + "summary": "zaaktypeomschrijving", + "description": "zaaktypeomschrijving", + "category": "{% if zaaktypecode|default %}{% set wooVerzoekenEnBesluiten = ['LP00000431', 'B1873', 'cherry'] %}{% set klachtoordelen = ['LP00000091', 'LP00001132', 'LP00000121', 'B0757', 'LP00000832', 'LP00001096'] %}{% if zaaktypecode in wooVerzoekenEnBesluiten %}{{ 'Woo-verzoeken en -besluiten' }}{% elseif zaaktypecode in klachtoordelen %}{{ 'Klachtoordelen' }}{% endif %}{% endif %}", + "published": "startdatum", + "modified": "{{ 'now'|date('H:i:sTm-d-Y') }}", + "attachments": "[{% if files|default %}{% for file in files %} { {% if file['titel']|default %}\"title\": \"{{ file['titel'] }}\",{% endif %}\"labels\": [\"{{ 'Informatieverzoek' }}\"],{% if file['formaat']|default %}\"extension\": \"{{ file['formaat']|split('/')|last }}\",\"type\": \"{{ file['formaat'] }}\",{% endif %}{% if file['inhoud']|default and file['formaat']|default %}\"accessUrl\": \"data:{{ file['formaat'] }};base64,{{ file.inhoud }}\"{% endif %} }{{ loop.last ? '' : ',' }} {% endfor %}{% endif %}]", + "status": "Concept" + }, + "unset": [ + "" + ], + "cast": { + "title": "unsetIfValue==omschrijving", + "summary": "unsetIfValue==zaaktypeomschrijving", + "description": "unsetIfValue==zaaktypeomschrijving", + "category": "unsetIfValue==", + "published": "unsetIfValue==startdatum", + "attachments": "jsonToArray" }, "passThrough": false } \ No newline at end of file diff --git a/configurations/sharepoint-woo/synchronizations/xxllnc-suite-to-publications.json b/configurations/sharepoint-woo/synchronizations/xxllnc-suite-to-publications.json index 6dfedf8..91922ef 100644 --- a/configurations/sharepoint-woo/synchronizations/xxllnc-suite-to-publications.json +++ b/configurations/sharepoint-woo/synchronizations/xxllnc-suite-to-publications.json @@ -7,13 +7,19 @@ "sourceHash": "", "sourceTargetMapping": "1", "sourceConfig": { - "idPosition": "id", - "resultsPosition": "results", - "endpoint": "/tlb/zaaksysteem/api/v1/zaken", - "headers": [], - "query.startdatum__gte": "2024-07-06", - "query.einddatum__lt": "2024-08-01" - }, + "idPosition": "identificatie", + "resultsPosition": "results", + "endpoint": "\/tlb\/zaaksysteem\/api\/v1\/zaken", + "headers": [], + "query.startdatum__gte": "2024-08-01", + "query.einddatum__lt": "2025-01-01", + "usesPagination": "false", + "extraDataConfigs.0.staticEndpoint": "/tlb/zaaksysteem/api/v1/zaken/{{ originId }}/informatieobjecten", + "extraDataConfigs.0.mergeExtraData": "true", + "extraDataConfigs.0.keyToSetExtraData": "files", + "extraDataConfigs.0.resultsLocation": "results", + "extraDataConfigs.0.extraDataConfigPerResult.staticEndpoint": "/tlb/zaaksysteem/api/v1/informatieobjecten/{{ originId }}" + }, "targetId": "1/1", "targetType": "register/schema" } \ No newline at end of file diff --git a/lib/Cron/ActionTask.php b/lib/Cron/ActionTask.php index f011735..efaa66d 100644 --- a/lib/Cron/ActionTask.php +++ b/lib/Cron/ActionTask.php @@ -113,10 +113,11 @@ public function run($argument) // Update the job - $nextRun = new DateTime(); + $nextRun = new DateTime('now + '.$job->getInterval().' seconds'); if (isset($result['nextRun']) === true) { $nextRun = DateTime::createFromFormat('U', $result['nextRun']); } + $nextRun->setTime(hour: $nextRun->format('H'), minute: $nextRun->format('i')); $job->setLastRun(new DateTime()); $job->setNextRun($nextRun); $this->jobMapper->update($job); @@ -145,6 +146,8 @@ public function run($argument) } } + $this->jobLogMapper->update(entity: $jobLog); + // Let's report back about what we have just done return $jobLog; } diff --git a/lib/Cron/LogCleanUpTask.php b/lib/Cron/LogCleanUpTask.php index 0af048a..0aa7cf3 100644 --- a/lib/Cron/LogCleanUpTask.php +++ b/lib/Cron/LogCleanUpTask.php @@ -3,13 +3,35 @@ namespace OCA\OpenConnector\Cron; use OCA\OpenConnector\Db\CallLogMapper; +use OCA\OpenConnector\Db\JobLogMapper; +use OCA\OpenConnector\Db\JobMapper; +use OCP\BackgroundJob\IJobList; use OCP\BackgroundJob\TimedJob; use OCP\AppFramework\Utility\ITimeFactory; +use OCP\IUserManager; +use OCP\IUserSession; -class LogCleanUpTask +class LogCleanUpTask extends TimedJob { - public function doCron(array $arguments, CallLogMapper $callLogMapper){ - $callLogMapper = new ClearLogs(); + public function __construct( + ITimeFactory $time, + private readonly CallLogMapper $callLogMapper, + ) { + parent::__construct($time); + + // Run every 5 minutes + $this->setInterval(300); + + // Delay until low-load time + //$this->setTimeSensitivity(\OCP\BackgroundJob\IJob::TIME_SENSITIVE); + // Or $this->setTimeSensitivity(\OCP\BackgroundJob\IJob::TIME_INSENSITIVE); + + // Only run one instance of this job at a time + $this->setAllowParallelRuns(false); + } + + public function run(mixed $argument){ + $this->callLogMapper->clearLogs(); } } diff --git a/lib/Db/CallLogMapper.php b/lib/Db/CallLogMapper.php index 49cac82..615b5bb 100644 --- a/lib/Db/CallLogMapper.php +++ b/lib/Db/CallLogMapper.php @@ -93,7 +93,7 @@ public function clearLogs(): bool // Build the delete query $qb->delete('openconnector_call_logs') - ->where($qb->expr()->lt('expired', $qb->createFunction('NOW()'))); + ->where($qb->expr()->lt('expires', $qb->createFunction('NOW()'))); // Execute the query and get the number of affected rows $result = $qb->execute(); diff --git a/lib/Db/Synchronization.php b/lib/Db/Synchronization.php index 3b6ea48..9cc466d 100644 --- a/lib/Db/Synchronization.php +++ b/lib/Db/Synchronization.php @@ -67,6 +67,21 @@ public function __construct() { $this->addType(fieldName:'followUps', type: 'json'); } + /** + * Checks through sourceConfig if the source of this sync uses pagination + * + * @return bool true if its uses pagination + */ + public function usesPagination(): bool + { + if (isset($this->sourceConfig['usesPagination']) === true && ($this->sourceConfig['usesPagination'] === false || $this->sourceConfig['usesPagination'] === 'false')) { + return false; + } + + // By default sources use basic pagination. + return true; + } + public function getJsonFields(): array { return array_keys( diff --git a/lib/Migration/Version1Date20241206095007.php b/lib/Migration/Version1Date20241206095007.php new file mode 100644 index 0000000..d26a0c8 --- /dev/null +++ b/lib/Migration/Version1Date20241206095007.php @@ -0,0 +1,66 @@ +hasTable('openconnector_sources') === true) { + $table = $schema->getTable('openconnector_sources'); + + if ($table->hasColumn('logRetention') === true) { + $table->dropColumn('logRetention'); + $table->addColumn('log_retention', Types::INTEGER)->setNotnull(false)->setDefault(3600); + } + if ($table->hasColumn('errorRetention') === true) { + $table->dropColumn('errorRetention'); + $table->addColumn('error_retention', Types::INTEGER)->setNotnull(false)->setDefault(86400); + } + } + + return $schema; + } + + /** + * @param IOutput $output + * @param Closure(): ISchemaWrapper $schemaClosure + * @param array $options + */ + public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void { + } +} diff --git a/lib/Service/CallService.php b/lib/Service/CallService.php index e40bc95..8db6434 100644 --- a/lib/Service/CallService.php +++ b/lib/Service/CallService.php @@ -138,6 +138,7 @@ public function call( $callLog->setStatusCode(409); $callLog->setStatusMessage("This source is not enabled"); $callLog->setCreated(new \DateTime()); + $callLog->setExpires(new \DateTime('now + '.$source->getErrorRetention().' seconds')); $this->callLogMapper->insert($callLog); @@ -152,6 +153,7 @@ public function call( $callLog->setStatusCode(409); $callLog->setStatusMessage("This source has no location"); $callLog->setCreated(new \DateTime()); + $callLog->setExpires(new \DateTime('now + '.$source->getErrorRetention().' seconds')); $this->callLogMapper->insert($callLog); @@ -178,6 +180,7 @@ public function call( $callLog->setStatusCode(429); // $callLog->setStatusMessage("The rate limit for this source has been exceeded. Try again later."); $callLog->setCreated(new \DateTime()); + $callLog->setExpires(new \DateTime('now + '.$source->getErrorRetention().' seconds')); $this->callLogMapper->insert($callLog); @@ -282,6 +285,7 @@ public function call( $callLog->setRequest($data['request']); $callLog->setResponse($data['response']); $callLog->setCreated(new \DateTime()); + $callLog->setExpires(new \DateTime('now + '.($data['response']['statusCode'] < 400 ? $source->getLogRetention() : $source->getErrorRetention()).' seconds')); $this->callLogMapper->insert($callLog); diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index c52d624..fc5bff9 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -617,7 +617,7 @@ public function getAllObjectsFromApi(Synchronization $synchronization, ?bool $is $this->synchronizationMapper->update($synchronization); } - if ($useNextEndpoint === false) { + if ($useNextEndpoint === false && $synchronization->usesPagination() === true) { do { $config = $this->getNextPage(config: $config, sourceConfig: $sourceConfig, currentPage: $currentPage); $callLog = $this->callService->call(source: $source, endpoint: $endpoint, method: 'GET', config: $config);