From f41f494a665d8009a2d46699a3bf4bca9d3236df Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 20 Nov 2024 09:44:28 +0000 Subject: [PATCH 01/43] Bump version to 0.1.17 --- appinfo/info.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appinfo/info.xml b/appinfo/info.xml index 260f2fe6..d27e4daf 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -13,7 +13,7 @@ The OpenConnector Nextcloud app provides a ESB-framework to work together in an - 🆓 Map and translate API calls ]]> - 0.1.16 + 0.1.17 agpl integration Conduction From d8aadf7dfadbcef182bec3719bcf9539176cfcef Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Thu, 21 Nov 2024 22:04:11 +0100 Subject: [PATCH 02/43] Hotfix/disable-branche-protection Temporarily disable branche protections since it breaks the release workflow --- .../pull-request-from-branch-check.yaml | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pull-request-from-branch-check.yaml b/.github/workflows/pull-request-from-branch-check.yaml index 77d1970f..401de613 100644 --- a/.github/workflows/pull-request-from-branch-check.yaml +++ b/.github/workflows/pull-request-from-branch-check.yaml @@ -1,18 +1,18 @@ name: Main Branch Protection -on: - pull_request: - branches: - - main +#on: +# pull_request: +# branches: +# - main -jobs: - check-branch: - runs-on: ubuntu-latest - steps: - - name: Check branch - run: | - if [[ ${GITHUB_HEAD_REF} != development ]] && [[ ${GITHUB_HEAD_REF} != documentation ]] && ! [[ ${GITHUB_HEAD_REF} =~ ^hotfix/ ]]; - then - echo "Error: Pull request must come from 'development', 'documentation' or 'hotfix/' branch" - exit 1 - fi +#jobs: +# check-branch: +# runs-on: ubuntu-latest +# steps: +# - name: Check branch +# run: | +# if [[ ${GITHUB_HEAD_REF} != development ]] && [[ ${GITHUB_HEAD_REF} != documentation ]] && ! [[ ${GITHUB_HEAD_REF} =~ ^hotfix/ ]]; +# then +# echo "Error: Pull request must come from 'development', 'documentation' or 'hotfix/' branch" +# exit 1 +# fi From be9f15c770358d631abf9ead8b7b5768059aaf01 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 21 Nov 2024 21:09:03 +0000 Subject: [PATCH 03/43] Bump version to 0.1.18 --- appinfo/info.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appinfo/info.xml b/appinfo/info.xml index d27e4daf..6c3f148a 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -13,7 +13,7 @@ The OpenConnector Nextcloud app provides a ESB-framework to work together in an - 🆓 Map and translate API calls ]]> - 0.1.17 + 0.1.18 agpl integration Conduction From 8fc5f258984125ece3756241f404a91a707021c4 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 26 Nov 2024 14:57:15 +0000 Subject: [PATCH 04/43] Bump version to 0.1.19 --- appinfo/info.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appinfo/info.xml b/appinfo/info.xml index e20cc8f2..08df4fc1 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -13,7 +13,7 @@ The OpenConnector Nextcloud app provides a ESB-framework to work together in an - 🆓 Map and translate API calls ]]> - 0.1.18 + 0.1.19 agpl integration Conduction From 960243070cb1a62949ce08ef0d5814f1a3f1d9c3 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Tue, 26 Nov 2024 16:44:39 +0100 Subject: [PATCH 05/43] Add an authentication method for DeCOS auth --- lib/Service/AuthenticationService.php | 29 +++++++++++++++++++++++++++ lib/Twig/AuthenticationExtension.php | 1 + lib/Twig/AuthenticationRuntime.php | 19 ++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/lib/Service/AuthenticationService.php b/lib/Service/AuthenticationService.php index b59e41b4..e433110a 100644 --- a/lib/Service/AuthenticationService.php +++ b/lib/Service/AuthenticationService.php @@ -192,6 +192,35 @@ public function fetchOAuthTokens (array $configuration): string return $result['access_token']; } + /** + * Fetch an access token from the DeCOS non-implementation of OAuth 2.0 + * + * @param array $configuration The configuration of the source. + * + * @return string The access token + * + * @throws \GuzzleHttp\Exception\GuzzleException + */ + public function fetchDecosToken(array $configuration): string + { + $url = $configuration['tokenUrl']; + $tokenLocation = $configuration['tokenLocation']; + unset($configuration['tokenUrl']); + + $callConfig['json'] = $configuration; + + $client = new Client(); + $response = $client->post(uri: $url, options: $callConfig); + + $result = json_decode(json: $response->getBody()->getContents(), associative: true); + + if (isset($tokenLocation) === true) { + return $result[$tokenLocation]; + } + + return $result['token']; + } + /** * Get RSA key for RS and PS (asymmetrical) encryption. * diff --git a/lib/Twig/AuthenticationExtension.php b/lib/Twig/AuthenticationExtension.php index fed2dfb5..a4cc8ec4 100644 --- a/lib/Twig/AuthenticationExtension.php +++ b/lib/Twig/AuthenticationExtension.php @@ -11,6 +11,7 @@ public function getFunctions(): array { return [ new TwigFunction(name: 'oauthToken', callable: [AuthenticationRuntime::class, 'oauthToken']), + new TwigFunction(name: 'decosToken', callable: [AuthenticationRuntime::class, 'decosToken']), new TwigFunction(name: 'jwtToken', callable: [AuthenticationRuntime::class, 'jwtToken']), ]; //return parent::getFunctions(); // TODO: Change the autogenerated stub diff --git a/lib/Twig/AuthenticationRuntime.php b/lib/Twig/AuthenticationRuntime.php index 1655229e..7534bcf9 100644 --- a/lib/Twig/AuthenticationRuntime.php +++ b/lib/Twig/AuthenticationRuntime.php @@ -37,6 +37,25 @@ public function oauthToken(Source $source): string ); } + /** + * Add a decos non-oauth token to the configuration. + * + * @param Source $source + * @return string + * + * @throws GuzzleException + */ + public function decosToken(Source $source): string + { + $configuration = new Dot($source->getConfiguration(), true); + + $authConfig = $configuration->get('authentication'); + + return $this->authService->fetchDecosToken( + configuration: $authConfig + ); + } + /** * Add a jwt token to the configuration. * From 44d31fbf4aa691d571a7d8e815d0ef5ab1ab32c4 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Wed, 27 Nov 2024 17:31:43 +0100 Subject: [PATCH 06/43] Start made to fetch extra data for object --- lib/Service/SynchronizationService.php | 51 +++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index 8f882507..3ba2209e 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -47,6 +47,8 @@ class SynchronizationService private ObjectService $objectService; private Source $source; + const EXTRA_DATA_ENDPOINT_LOCATION = 'extraDataEndpointLocation'; + const MERGE_EXTRA_DATA_OBJECT = 'mergeExtraDataObject'; public function __construct( CallService $callService, @@ -214,6 +216,47 @@ public function getObjectFromSource(Synchronization $synchronization, string $en return json_decode($response['body'], true); } + /** + * Fetches extra data for a given object based on the provided synchronization configuration. + * + * @param Synchronization $synchronization The synchronization instance containing configuration details. + * @param array $object The object for which extra data needs to be fetched. + * + * @return array The original object merged with the extra data or the extra data itself, depending on configuration. + * + * @throws Exception If the endpoint cannot be retrieved from the source configuration and the provided object. + */ + private function fetchExtraDataForObject(Synchronization $synchronization, array $object) + { + $sourceConfig = $synchronization->getSourceConfig(); + + if (isset($sourceConfig[$this::EXTRA_DATA_ENDPOINT_LOCATION]) === false) { + return $object; + } + + $dotObject = new Dot($object); + $endpoint = $dotObject->get($sourceConfig[$this::EXTRA_DATA_ENDPOINT_LOCATION] ?? null); + + if (!$endpoint) { + throw new Exception( + sprintf( + 'Could not get endpoint with extra data location: %s, object: %s', + $sourceConfig[$this::EXTRA_DATA_ENDPOINT_LOCATION], + json_encode($object) + ) + ); + } + + $extraDataObject = $this->getObjectFromSource($synchronization, $endpoint); + + if (isset($sourceConfig[$this::MERGE_EXTRA_DATA_OBJECT]) === true && $sourceConfig[$this::MERGE_EXTRA_DATA_OBJECT] === true) { + return array_merge($extraDataObject, $object); + } + + return $extraDataObject; +} + + /** * Synchronize a contract * @@ -230,11 +273,12 @@ public function getObjectFromSource(Synchronization $synchronization, string $en */ public function synchronizeContract(SynchronizationContract $synchronizationContract, Synchronization $synchronization = null, array $object = [], ?bool $isTest = false): SynchronizationContract|Exception|array { + $sourceConfig = $synchronization->getSourceConfig(); - if ($synchronization !== null && isset($synchronization->getSourceConfig()['singleEndpoint']) === true) { + if ($synchronization !== null && isset($sourceConfig['singleEndpoint']) === true) { // Update endpoint - $endpoint = str_replace(search: '{{ originId }}', replace: $this->getOriginId($synchronization, $object), subject: $synchronization->getSourceConfig()['singleEndpoint']); + $endpoint = str_replace(search: '{{ originId }}', replace: $this->getOriginId($synchronization, $object), subject: $sourceConfig['singleEndpoint']); $endpoint = str_replace(search: '{{originId}}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint); // Get object from source @@ -242,6 +286,9 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont } + // Check if extra data needs to be fetched + $object = $this->fetchExtraDataForObject($synchronization, $object); + // Let create a source hash for the object $originHash = md5(serialize($object)); From fd188f1268bd261040c25b50ca0838282a7b1e60 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Thu, 28 Nov 2024 10:16:52 +0100 Subject: [PATCH 07/43] Merge result from singleEndpoint --- lib/Service/SynchronizationService.php | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index 8f882507..bba34456 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -47,6 +47,9 @@ class SynchronizationService private ObjectService $objectService; private Source $source; + const SINGLE_ENDPOINT = 'singleEndpoint'; + const MERGE_DATA_SINGLE_ENDPOINT = 'mergeDataSingleEndpoint'; + public function __construct( CallService $callService, @@ -230,16 +233,19 @@ public function getObjectFromSource(Synchronization $synchronization, string $en */ public function synchronizeContract(SynchronizationContract $synchronizationContract, Synchronization $synchronization = null, array $object = [], ?bool $isTest = false): SynchronizationContract|Exception|array { - - if ($synchronization !== null && isset($synchronization->getSourceConfig()['singleEndpoint']) === true) { + $sourceConfig = $synchronization->getSourceConfig(); + if ($synchronization !== null && isset($sourceConfig[$this::SINGLE_ENDPOINT]) === true) { // Update endpoint - $endpoint = str_replace(search: '{{ originId }}', replace: $this->getOriginId($synchronization, $object), subject: $synchronization->getSourceConfig()['singleEndpoint']); + $endpoint = str_replace(search: '{{ originId }}', replace: $this->getOriginId($synchronization, $object), subject: $sourceConfig[$this::SINGLE_ENDPOINT]); $endpoint = str_replace(search: '{{originId}}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint); // Get object from source - $object = $this->getObjectFromSource(synchronization: $synchronization, endpoint: $endpoint); - + if (isset($sourceConfig[$this::MERGE_DATA_SINGLE_ENDPOINT]) === true && $sourceConfig[$this::MERGE_DATA_SINGLE_ENDPOINT] === true) { + $object = array_merge($object, $this->getObjectFromSource(synchronization: $synchronization, endpoint: $endpoint)); + } else { + $object = $this->getObjectFromSource(synchronization: $synchronization, endpoint: $endpoint); + } } From d099b23b6c84f2d0b6669e6404763580ad522e93 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Thu, 28 Nov 2024 13:09:36 +0100 Subject: [PATCH 08/43] Mapping update --- .../sharepoint-woo-verzoek-to-publications.json | 16 ++++++++++++++-- .../sharepoint-convenanten-to-publications.json | 4 +++- ...sharepoint-woo-verzoeken-to-publications.json | 4 +++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json index b678deaa..9e299ee7 100644 --- a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json +++ b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json @@ -3,9 +3,21 @@ "description": "", "version": "0.0.1", "mapping": { - "title": "Name" + "title": "d.woo_x005f_titel", + "description": "d.woo_x005f_beschrijving", + "summary": "d.woo_x005f_samenvatting", + "category": "d.woo_x005f_categorie", + "published": "d.woo_x005f_publicatiedatum", + "modified": "d.vti_x005f_nexttolasttimemodified" }, - "unset": [], + "unset": { + "title": "d.woo_x005f_titel", + "description": "d.woo_x005f_beschrijving", + "summary": "d.woo_x005f_samenvatting", + "category": "d.woo_x005f_categorie", + "published": "d.woo_x005f_publicatiedatum", + "modified": "d.vti_x005f_nexttolasttimemodified" + }, "cast": [], "passThrough": false } \ No newline at end of file diff --git a/configurations/sharepoint-woo/synchronizations/sharepoint-convenanten-to-publications.json b/configurations/sharepoint-woo/synchronizations/sharepoint-convenanten-to-publications.json index e39deb85..bc539411 100644 --- a/configurations/sharepoint-woo/synchronizations/sharepoint-convenanten-to-publications.json +++ b/configurations/sharepoint-woo/synchronizations/sharepoint-convenanten-to-publications.json @@ -7,9 +7,11 @@ "sourceHash": "", "sourceTargetMapping": "1", "sourceConfig": { - "idPosition": "UniqueId", + "idPosition": "Properties.__deferred.uri", "resultsPosition": "d.results", "endpoint": "/Web/GetFolderByServerRelativePath(decodedurl='/WOO/Convenanten')/folders", + "singleEndpoint": "{{ originId }}", + "mergeDataSingleEndpoint": true, "headers": [], "query": [] }, diff --git a/configurations/sharepoint-woo/synchronizations/sharepoint-woo-verzoeken-to-publications.json b/configurations/sharepoint-woo/synchronizations/sharepoint-woo-verzoeken-to-publications.json index 7aa65121..f7f3e47d 100644 --- a/configurations/sharepoint-woo/synchronizations/sharepoint-woo-verzoeken-to-publications.json +++ b/configurations/sharepoint-woo/synchronizations/sharepoint-woo-verzoeken-to-publications.json @@ -7,9 +7,11 @@ "sourceHash": "", "sourceTargetMapping": "1", "sourceConfig": { - "idPosition": "UniqueId", + "idPosition": "Properties.__deferred.uri", "resultsPosition": "d.results", "endpoint": "/Web/GetFolderByServerRelativePath(decodedurl='/WOO/Woo-verzoeken en -besluiten')/folders", + "singleEndpoint": "{{ originId }}", + "mergeDataSingleEndpoint": true, "headers": [], "query": [] }, From 54e7eae6335abd983f629dacbb004f8284662f0b Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Thu, 28 Nov 2024 13:34:48 +0100 Subject: [PATCH 09/43] update mapping --- .../mappings/xxllnc-suite-to-publications.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json index 17e38a87..17dd7bd7 100644 --- a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json +++ b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json @@ -5,8 +5,10 @@ "reference": "", "mapping": { "title": "omschrijving", - "summary": "statustoelichting", - "description": "statusomschrijving" + "summary": "zaaktypeomschrijving", + "description": "zaaktypeomschrijving", + "published" : "startdatum", + "modified" : "{{ 'now'|date }}" }, "unset": [], "cast": [], From a8021cc0765b323a14ae42be2204f65d2ad1d2f3 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Thu, 28 Nov 2024 13:37:54 +0100 Subject: [PATCH 10/43] update mapping --- .../sharepoint-woo/mappings/xxllnc-suite-to-publications.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json index 17dd7bd7..c1428201 100644 --- a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json +++ b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json @@ -8,7 +8,7 @@ "summary": "zaaktypeomschrijving", "description": "zaaktypeomschrijving", "published" : "startdatum", - "modified" : "{{ 'now'|date }}" + "modified" : "{{ 'now'|date(H:i:sTm-d-Y') }}" }, "unset": [], "cast": [], From f2ad65e275b8d3f343464e6290d2154c7072fde0 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Thu, 28 Nov 2024 14:12:13 +0100 Subject: [PATCH 11/43] fix cast --- .../mappings/sharepoint-woo-verzoek-to-publications.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json index 9e299ee7..e74ad30a 100644 --- a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json +++ b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json @@ -10,7 +10,8 @@ "published": "d.woo_x005f_publicatiedatum", "modified": "d.vti_x005f_nexttolasttimemodified" }, - "unset": { + "unset": [], + "cast": { "title": "d.woo_x005f_titel", "description": "d.woo_x005f_beschrijving", "summary": "d.woo_x005f_samenvatting", @@ -18,6 +19,5 @@ "published": "d.woo_x005f_publicatiedatum", "modified": "d.vti_x005f_nexttolasttimemodified" }, - "cast": [], "passThrough": false } \ No newline at end of file From 6531b690e75ae031f1b67fdaa366e0f6a78b4d76 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 28 Nov 2024 13:50:44 +0000 Subject: [PATCH 12/43] Bump version to 0.1.20 --- appinfo/info.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appinfo/info.xml b/appinfo/info.xml index 08df4fc1..768b23e0 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -13,7 +13,7 @@ The OpenConnector Nextcloud app provides a ESB-framework to work together in an - 🆓 Map and translate API calls ]]> - 0.1.19 + 0.1.20 agpl integration Conduction From 0b584f8b1899ca1889fac86f3a7f6d280a1b23f6 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Fri, 29 Nov 2024 10:17:34 +0100 Subject: [PATCH 13/43] Fix migration on table name --- lib/Migration/Version1Date20241111144800.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Migration/Version1Date20241111144800.php b/lib/Migration/Version1Date20241111144800.php index 9eb4651d..10a6768c 100644 --- a/lib/Migration/Version1Date20241111144800.php +++ b/lib/Migration/Version1Date20241111144800.php @@ -101,7 +101,7 @@ public function postSchemaChange(IOutput $output, Closure $schemaClosure, array && $table->hasColumn('source_id') === true && $table->hasColumn('source_hash') === true ) { $this->connection->executeQuery(" - UPDATE openconnector_synchronization_contracts + UPDATE oc_openconnector_synchronization_contracts SET origin_id = source_id, origin_hash = source_hash WHERE source_id IS NOT NULL "); From 3dfec5c28c7c66ab0477e12d8b8f9ee79503a4a6 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Fri, 29 Nov 2024 13:51:16 +0100 Subject: [PATCH 14/43] Work with multiple configs for multiple requests --- lib/Service/SynchronizationService.php | 51 +++++++++++++------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index 3ba2209e..9ef22bea 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -47,8 +47,10 @@ class SynchronizationService private ObjectService $objectService; private Source $source; - const EXTRA_DATA_ENDPOINT_LOCATION = 'extraDataEndpointLocation'; - const MERGE_EXTRA_DATA_OBJECT = 'mergeExtraDataObject'; + const EXTRA_DATA_CONFIGS_LOCATION = 'extraDataConfigs'; + const EXTRA_DATA_ENDPOINT_LOCATION = 'endpointLocation'; + const KEY_FOR_EXTRA_DATA_LOCATION = 'key'; + const MERGE_EXTRA_DATA_OBJECT_LOCATION = 'mergeExtraDataKey'; public function __construct( CallService $callService, @@ -220,41 +222,47 @@ public function getObjectFromSource(Synchronization $synchronization, string $en * Fetches extra data for a given object based on the provided synchronization configuration. * * @param Synchronization $synchronization The synchronization instance containing configuration details. + * @param array $extraDataConfig The extra data configuration. * @param array $object The object for which extra data needs to be fetched. * * @return array The original object merged with the extra data or the extra data itself, depending on configuration. * * @throws Exception If the endpoint cannot be retrieved from the source configuration and the provided object. */ - private function fetchExtraDataForObject(Synchronization $synchronization, array $object) + private function fetchExtraDataForObject(Synchronization $synchronization, array $extraDataConfig, array $object) { - $sourceConfig = $synchronization->getSourceConfig(); - if (isset($sourceConfig[$this::EXTRA_DATA_ENDPOINT_LOCATION]) === false) { return $object; } $dotObject = new Dot($object); - $endpoint = $dotObject->get($sourceConfig[$this::EXTRA_DATA_ENDPOINT_LOCATION] ?? null); + $endpoint = $dotObject->get($extraDataConfig[$this::EXTRA_DATA_ENDPOINT_LOCATION] ?? null); + + $endpoint = str_replace(search: '{{ originId }}', replace: $this->getOriginId($synchronization, $object), subject: $extraDataConfig[$this::EXTRA_DATA_ENDPOINT_LOCATION]); + $endpoint = str_replace(search: '{{originId}}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint); if (!$endpoint) { throw new Exception( sprintf( 'Could not get endpoint with extra data location: %s, object: %s', - $sourceConfig[$this::EXTRA_DATA_ENDPOINT_LOCATION], + $extraDataConfig[$this::EXTRA_DATA_ENDPOINT_LOCATION], json_encode($object) ) ); } - $extraDataObject = $this->getObjectFromSource($synchronization, $endpoint); + $extraData = $this->getObjectFromSource($synchronization, $endpoint); - if (isset($sourceConfig[$this::MERGE_EXTRA_DATA_OBJECT]) === true && $sourceConfig[$this::MERGE_EXTRA_DATA_OBJECT] === true) { - return array_merge($extraDataObject, $object); + if (isset($extraDataConfig[$this::KEY_FOR_EXTRA_DATA_LOCATION]) === true) { + $extraData = [$extraDataConfig[$this::KEY_FOR_EXTRA_DATA_LOCATION] => $extraData]; } - return $extraDataObject; -} + if (isset($extraDataConfig[$this::MERGE_EXTRA_DATA_OBJECT_LOCATION]) === true && $extraDataConfig[$this::MERGE_EXTRA_DATA_OBJECT_LOCATION] === true) { + return array_merge($object, $extraData); + } + + return $extraData; + } /** @@ -273,21 +281,14 @@ private function fetchExtraDataForObject(Synchronization $synchronization, array */ public function synchronizeContract(SynchronizationContract $synchronizationContract, Synchronization $synchronization = null, array $object = [], ?bool $isTest = false): SynchronizationContract|Exception|array { - $sourceConfig = $synchronization->getSourceConfig(); - - if ($synchronization !== null && isset($sourceConfig['singleEndpoint']) === true) { - - // Update endpoint - $endpoint = str_replace(search: '{{ originId }}', replace: $this->getOriginId($synchronization, $object), subject: $sourceConfig['singleEndpoint']); - $endpoint = str_replace(search: '{{originId}}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint); - - // Get object from source - $object = $this->getObjectFromSource(synchronization: $synchronization, endpoint: $endpoint); - - } + $sourceConfig = $this->callService->applyConfigDot($synchronization->getSourceConfig()); // Check if extra data needs to be fetched - $object = $this->fetchExtraDataForObject($synchronization, $object); + if (isset($sourceConfig[$this::EXTRA_DATA_CONFIGS_LOCATION]) === true) { + foreach ($sourceConfig[$this::EXTRA_DATA_CONFIGS_LOCATION] as $extraDataConfig) { + $object = array_merge($object, $this->fetchExtraDataForObject($synchronization, $extraDataConfig, $object)); + } + } // Let create a source hash for the object From 423369bfcc6e7213af1ffb4958410edd8b5d6be1 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Fri, 29 Nov 2024 13:53:48 +0100 Subject: [PATCH 15/43] improve key names --- lib/Service/SynchronizationService.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index db052074..aa601d12 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -49,8 +49,8 @@ class SynchronizationService const EXTRA_DATA_CONFIGS_LOCATION = 'extraDataConfigs'; const EXTRA_DATA_ENDPOINT_LOCATION = 'endpointLocation'; - const KEY_FOR_EXTRA_DATA_LOCATION = 'key'; - const MERGE_EXTRA_DATA_OBJECT_LOCATION = 'mergeExtraDataKey'; + const KEY_FOR_EXTRA_DATA_LOCATION = 'keyToSetExtraData'; + const MERGE_EXTRA_DATA_OBJECT_LOCATION = 'mergeExtraData'; public function __construct( From d6379796cf7edfd83ac130a770eb7a534409080b Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Fri, 29 Nov 2024 13:56:47 +0100 Subject: [PATCH 16/43] key improvement --- lib/Service/SynchronizationService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index aa601d12..e9842531 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -48,7 +48,7 @@ class SynchronizationService private Source $source; const EXTRA_DATA_CONFIGS_LOCATION = 'extraDataConfigs'; - const EXTRA_DATA_ENDPOINT_LOCATION = 'endpointLocation'; + const EXTRA_DATA_ENDPOINT_LOCATION = 'endpoint'; const KEY_FOR_EXTRA_DATA_LOCATION = 'keyToSetExtraData'; const MERGE_EXTRA_DATA_OBJECT_LOCATION = 'mergeExtraData'; From 4a2fe5bb40844063438c1b993e57d021cb150511 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Fri, 29 Nov 2024 14:10:22 +0100 Subject: [PATCH 17/43] Fix singleEndpoint and better docblock --- lib/Service/SynchronizationService.php | 49 +++++++++++++++++--------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index e9842531..7ce815d6 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -47,10 +47,11 @@ class SynchronizationService private ObjectService $objectService; private Source $source; - const EXTRA_DATA_CONFIGS_LOCATION = 'extraDataConfigs'; - const EXTRA_DATA_ENDPOINT_LOCATION = 'endpoint'; - const KEY_FOR_EXTRA_DATA_LOCATION = 'keyToSetExtraData'; - const MERGE_EXTRA_DATA_OBJECT_LOCATION = 'mergeExtraData'; + const EXTRA_DATA_CONFIGS_LOCATION = 'extraDataConfigs'; + const EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION = 'dynamicEndpointLocation'; + const EXTRA_DATA_STATIC_ENDPOINT_LOCATION = 'staticEndpoint'; + const KEY_FOR_EXTRA_DATA_LOCATION = 'keyToSetExtraData'; + const MERGE_EXTRA_DATA_OBJECT_LOCATION = 'mergeExtraData'; public function __construct( @@ -220,33 +221,47 @@ public function getObjectFromSource(Synchronization $synchronization, string $en } /** - * Fetches extra data for a given object based on the provided synchronization configuration. + * Fetches additional data for a given object based on the synchronization configuration. + * + * This method retrieves extra data using either a dynamically determined endpoint from the object + * or a statically defined endpoint in the configuration. The extra data can be merged with the original + * object or returned as-is, based on the provided configuration. * * @param Synchronization $synchronization The synchronization instance containing configuration details. - * @param array $extraDataConfig The extra data configuration. - * @param array $object The object for which extra data needs to be fetched. + * @param array $extraDataConfig The configuration array specifying how to retrieve and handle the extra data: + * - EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION: The key to retrieve the dynamic endpoint from the object. + * - EXTRA_DATA_STATIC_ENDPOINT_LOCATION: The statically defined endpoint. + * - KEY_FOR_EXTRA_DATA_LOCATION: The key under which the extra data should be returned. + * - MERGE_EXTRA_DATA_OBJECT_LOCATION: Boolean flag indicating whether to merge the extra data with the object. + * @param array $object The original object for which extra data needs to be fetched. * - * @return array The original object merged with the extra data or the extra data itself, depending on configuration. + * @return array The original object merged with the extra data, or the extra data itself based on the configuration. * - * @throws Exception If the endpoint cannot be retrieved from the source configuration and the provided object. + * @throws Exception If both dynamic and static endpoint configurations are missing or the endpoint cannot be determined. */ private function fetchExtraDataForObject(Synchronization $synchronization, array $extraDataConfig, array $object) { - if (isset($sourceConfig[$this::EXTRA_DATA_ENDPOINT_LOCATION]) === false) { + if (isset($sourceConfig[$this::EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION]) === false && isset($sourceConfig[$this::EXTRA_DATA_STATIC_ENDPOINT_LOCATION]) === false) { return $object; } - $dotObject = new Dot($object); - $endpoint = $dotObject->get($extraDataConfig[$this::EXTRA_DATA_ENDPOINT_LOCATION] ?? null); + // Get endpoint from earlier fetched object. + if (isset($sourceConfig[$this::EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION]) === true) { + $dotObject = new Dot($object); + $endpoint = $dotObject->get($extraDataConfig[$this::EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION] ?? null); + } - $endpoint = str_replace(search: '{{ originId }}', replace: $this->getOriginId($synchronization, $object), subject: $extraDataConfig[$this::EXTRA_DATA_ENDPOINT_LOCATION]); - $endpoint = str_replace(search: '{{originId}}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint); + // Get endpoint static defined in config. + if (isset($sourceConfig[$this::EXTRA_DATA_STATIC_ENDPOINT_LOCATION]) === true) { + $endpoint = $extraDataConfig[$this::EXTRA_DATA_STATIC_ENDPOINT_LOCATION]; + $endpoint = str_replace(search: '{{ originId }}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint); + $endpoint = str_replace(search: '{{originId}}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint); + } if (!$endpoint) { throw new Exception( sprintf( - 'Could not get endpoint with extra data location: %s, object: %s', - $extraDataConfig[$this::EXTRA_DATA_ENDPOINT_LOCATION], + 'Could not get static or dynamic endpoint, object: %s', json_encode($object) ) ); @@ -254,10 +269,12 @@ private function fetchExtraDataForObject(Synchronization $synchronization, array $extraData = $this->getObjectFromSource($synchronization, $endpoint); + // Set new key if configured. if (isset($extraDataConfig[$this::KEY_FOR_EXTRA_DATA_LOCATION]) === true) { $extraData = [$extraDataConfig[$this::KEY_FOR_EXTRA_DATA_LOCATION] => $extraData]; } + // Merge with earlier fetchde object if configured. if (isset($extraDataConfig[$this::MERGE_EXTRA_DATA_OBJECT_LOCATION]) === true && $extraDataConfig[$this::MERGE_EXTRA_DATA_OBJECT_LOCATION] === true) { return array_merge($object, $extraData); } From d8676e5b6e22e253828fe65e94eea30546537d48 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Fri, 29 Nov 2024 14:12:31 +0100 Subject: [PATCH 18/43] Fixes --- lib/Service/SynchronizationService.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index 7ce815d6..e9a4b42e 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -241,18 +241,18 @@ public function getObjectFromSource(Synchronization $synchronization, string $en */ private function fetchExtraDataForObject(Synchronization $synchronization, array $extraDataConfig, array $object) { - if (isset($sourceConfig[$this::EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION]) === false && isset($sourceConfig[$this::EXTRA_DATA_STATIC_ENDPOINT_LOCATION]) === false) { + if (isset($extraDataConfig[$this::EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION]) === false && isset($extraDataConfig[$this::EXTRA_DATA_STATIC_ENDPOINT_LOCATION]) === false) { return $object; } // Get endpoint from earlier fetched object. - if (isset($sourceConfig[$this::EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION]) === true) { + if (isset($extraDataConfig[$this::EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION]) === true) { $dotObject = new Dot($object); $endpoint = $dotObject->get($extraDataConfig[$this::EXTRA_DATA_DYNAMIC_ENDPOINT_LOCATION] ?? null); } // Get endpoint static defined in config. - if (isset($sourceConfig[$this::EXTRA_DATA_STATIC_ENDPOINT_LOCATION]) === true) { + if (isset($extraDataConfig[$this::EXTRA_DATA_STATIC_ENDPOINT_LOCATION]) === true) { $endpoint = $extraDataConfig[$this::EXTRA_DATA_STATIC_ENDPOINT_LOCATION]; $endpoint = str_replace(search: '{{ originId }}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint); $endpoint = str_replace(search: '{{originId}}', replace: $this->getOriginId($synchronization, $object), subject: $endpoint); From 6737ab8e352aceede433c9838ad20e053542a497 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Fri, 29 Nov 2024 16:18:52 +0100 Subject: [PATCH 19/43] Update mapping and config --- ...harepoint-woo-verzoek-to-publications.json | 22 ++++++++++--------- ...harepoint-convenanten-to-publications.json | 19 +++++++++------- ...repoint-woo-verzoeken-to-publications.json | 13 ++++++----- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json index e74ad30a..cd7743b9 100644 --- a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json +++ b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json @@ -8,16 +8,18 @@ "summary": "d.woo_x005f_samenvatting", "category": "d.woo_x005f_categorie", "published": "d.woo_x005f_publicatiedatum", - "modified": "d.vti_x005f_nexttolasttimemodified" - }, - "unset": [], - "cast": { - "title": "d.woo_x005f_titel", - "description": "d.woo_x005f_beschrijving", - "summary": "d.woo_x005f_samenvatting", - "category": "d.woo_x005f_categorie", - "published": "d.woo_x005f_publicatiedatum", - "modified": "d.vti_x005f_nexttolasttimemodified" + "modified": "d.vti_x005f_nexttolasttimemodified", + "attachments": "{{ '[' }}{% if fileUrls['d']['results']|default %}{% set index = 0 %}{% for file in fileUrls['d']['results'] %}{% if index > 0 %}{{ ', ' }}{% endif %}{{ '{' }}{% set uri = file['__metadata']['uri']~'/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"1\"' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}" + }, + "unset": [], + "cast": { + "title": "unsetIfValue==d.woo_x005f_titel", + "description": "unsetIfValue==d.woo_x005f_beschrijving", + "summary": "unsetIfValue==d.woo_x005f_samenvatting", + "category": "unsetIfValue==d.woo_x005f_categorie", + "published": "unsetIfValue==d.woo_x005f_publicatiedatum", + "modified": "unsetIfValue==d.vti_x005f_nexttolasttimemodified", + "attachments": "jsonToArray" }, "passThrough": false } \ No newline at end of file diff --git a/configurations/sharepoint-woo/synchronizations/sharepoint-convenanten-to-publications.json b/configurations/sharepoint-woo/synchronizations/sharepoint-convenanten-to-publications.json index bc539411..030054f2 100644 --- a/configurations/sharepoint-woo/synchronizations/sharepoint-convenanten-to-publications.json +++ b/configurations/sharepoint-woo/synchronizations/sharepoint-convenanten-to-publications.json @@ -7,14 +7,17 @@ "sourceHash": "", "sourceTargetMapping": "1", "sourceConfig": { - "idPosition": "Properties.__deferred.uri", - "resultsPosition": "d.results", - "endpoint": "/Web/GetFolderByServerRelativePath(decodedurl='/WOO/Convenanten')/folders", - "singleEndpoint": "{{ originId }}", - "mergeDataSingleEndpoint": true, - "headers": [], - "query": [] - }, + "idPosition": "UniqueId", + "resultsPosition": "d.results", + "endpoint": "\/Web\/GetFolderByServerRelativePath(decodedurl='\/WOO\/Convenanten')\/folders", + "extraDataConfigs.0.dynamicEndpointLocation": "Properties.__deferred.uri", + "extraDataConfigs.0.mergeExtraData": "true", + "extraDataConfigs.1.dynamicEndpointLocation": "Files.__deferred.uri", + "extraDataConfigs.1.mergeExtraData": "true", + "extraDataConfigs.1.keyToSetExtraData": "fileUrls", + "headers": [], + "query": [] + }, "targetId": "1/1", "targetType": "register/schema" } \ No newline at end of file diff --git a/configurations/sharepoint-woo/synchronizations/sharepoint-woo-verzoeken-to-publications.json b/configurations/sharepoint-woo/synchronizations/sharepoint-woo-verzoeken-to-publications.json index f7f3e47d..e4c66965 100644 --- a/configurations/sharepoint-woo/synchronizations/sharepoint-woo-verzoeken-to-publications.json +++ b/configurations/sharepoint-woo/synchronizations/sharepoint-woo-verzoeken-to-publications.json @@ -7,13 +7,16 @@ "sourceHash": "", "sourceTargetMapping": "1", "sourceConfig": { - "idPosition": "Properties.__deferred.uri", + "idPosition": "UniqueId", "resultsPosition": "d.results", "endpoint": "/Web/GetFolderByServerRelativePath(decodedurl='/WOO/Woo-verzoeken en -besluiten')/folders", - "singleEndpoint": "{{ originId }}", - "mergeDataSingleEndpoint": true, - "headers": [], - "query": [] + "extraDataConfigs.0.dynamicEndpointLocation": "Properties.__deferred.uri", + "extraDataConfigs.0.mergeExtraData": "true", + "extraDataConfigs.1.dynamicEndpointLocation": "Files.__deferred.uri", + "extraDataConfigs.1.mergeExtraData": "true", + "extraDataConfigs.1.keyToSetExtraData": "fileUrls", + "headers": [], + "query": [] }, "targetId": "1/1", "targetType": "register/schema" From cd9d093ca5845d912183238d632aa67b35359ea5 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Fri, 29 Nov 2024 16:40:00 +0100 Subject: [PATCH 20/43] mapping improvements --- .../mappings/sharepoint-woo-verzoek-to-publications.json | 2 +- .../mappings/xxllnc-suite-to-publications.json | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json index cd7743b9..89c58ecb 100644 --- a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json +++ b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json @@ -7,7 +7,7 @@ "description": "d.woo_x005f_beschrijving", "summary": "d.woo_x005f_samenvatting", "category": "d.woo_x005f_categorie", - "published": "d.woo_x005f_publicatiedatum", + "published": "{% if d['woo_x005f_publicatiedatum']|default %}{{ d['woo_x005f_publicatiedatum']|date('Y-m-d') }}{% endif %}", "modified": "d.vti_x005f_nexttolasttimemodified", "attachments": "{{ '[' }}{% if fileUrls['d']['results']|default %}{% set index = 0 %}{% for file in fileUrls['d']['results'] %}{% if index > 0 %}{{ ', ' }}{% endif %}{{ '{' }}{% set uri = file['__metadata']['uri']~'/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"1\"' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}" }, diff --git a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json index c1428201..2a84a6c5 100644 --- a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json +++ b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json @@ -7,10 +7,15 @@ "title": "omschrijving", "summary": "zaaktypeomschrijving", "description": "zaaktypeomschrijving", - "published" : "startdatum", + "published" : "{% if startdatum|default %}{{ startdatum|date('Y-m-d') }}{% endif %}", "modified" : "{{ 'now'|date(H:i:sTm-d-Y') }}" }, "unset": [], - "cast": [], + "cast": { + "title": "unsetIfValue===omschrijving", + "summary": "unsetIfValue===zaaktypeomschrijving", + "description": "unsetIfValue===zaaktypeomschrijving", + "published" : "unsetIfValue===" + }, "passThrough": false } \ No newline at end of file From 60868c1e3068137e1adefca070d94798d2111c82 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Fri, 29 Nov 2024 17:01:57 +0100 Subject: [PATCH 21/43] config fix --- .../sharepoint-woo-verzoek-to-publications.json | 3 ++- .../mappings/xxllnc-suite-to-publications.json | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json index 89c58ecb..914adb38 100644 --- a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json +++ b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json @@ -9,7 +9,8 @@ "category": "d.woo_x005f_categorie", "published": "{% if d['woo_x005f_publicatiedatum']|default %}{{ d['woo_x005f_publicatiedatum']|date('Y-m-d') }}{% endif %}", "modified": "d.vti_x005f_nexttolasttimemodified", - "attachments": "{{ '[' }}{% if fileUrls['d']['results']|default %}{% set index = 0 %}{% for file in fileUrls['d']['results'] %}{% if index > 0 %}{{ ', ' }}{% endif %}{{ '{' }}{% set uri = file['__metadata']['uri']~'/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"1\"' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}" + "attachments": "{{ '[' }}{% if fileUrls['d']['results']|default %}{% set index = 0 %}{% for file in fileUrls['d']['results'] %}{% if index > 0 %}{{ ', ' }}{% endif %}{{ '{' }}{% set uri = file['__metadata']['uri']~'/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"1\"' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}", + "status": "{% if d['woo_x005f_publicatiedatum']|default %}Published{% else %}Concept{% endif %}" }, "unset": [], "cast": { diff --git a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json index 2a84a6c5..206c15cc 100644 --- a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json +++ b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json @@ -8,14 +8,15 @@ "summary": "zaaktypeomschrijving", "description": "zaaktypeomschrijving", "published" : "{% if startdatum|default %}{{ startdatum|date('Y-m-d') }}{% endif %}", - "modified" : "{{ 'now'|date(H:i:sTm-d-Y') }}" + "modified" : "{{ 'now'|date(H:i:sTm-d-Y') }}", + "status": "{% if startdatum|default %}Published{% else %}Concept{% endif %}" }, "unset": [], "cast": { - "title": "unsetIfValue===omschrijving", - "summary": "unsetIfValue===zaaktypeomschrijving", - "description": "unsetIfValue===zaaktypeomschrijving", - "published" : "unsetIfValue===" + "title": "unsetIfValue==omschrijving", + "summary": "unsetIfValue==zaaktypeomschrijving", + "description": "unsetIfValue==zaaktypeomschrijving", + "published" : "unsetIfValue==" }, "passThrough": false } \ No newline at end of file From 6676cebbb539e7db0ba2e606a77ea99c87954fc6 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Fri, 29 Nov 2024 17:36:08 +0100 Subject: [PATCH 22/43] Mapping improvement --- .../mappings/sharepoint-woo-verzoek-to-publications.json | 3 ++- .../sharepoint-woo/mappings/xxllnc-suite-to-publications.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json index 914adb38..75e5e38f 100644 --- a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json +++ b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json @@ -10,7 +10,8 @@ "published": "{% if d['woo_x005f_publicatiedatum']|default %}{{ d['woo_x005f_publicatiedatum']|date('Y-m-d') }}{% endif %}", "modified": "d.vti_x005f_nexttolasttimemodified", "attachments": "{{ '[' }}{% if fileUrls['d']['results']|default %}{% set index = 0 %}{% for file in fileUrls['d']['results'] %}{% if index > 0 %}{{ ', ' }}{% endif %}{{ '{' }}{% set uri = file['__metadata']['uri']~'/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"1\"' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}", - "status": "{% if d['woo_x005f_publicatiedatum']|default %}Published{% else %}Concept{% endif %}" + "status": "{% if d['woo_x005f_publicatiedatum']|default %}Published{% else %}Concept{% endif %}", + "catalog": "" }, "unset": [], "cast": { diff --git a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json index 206c15cc..54ef5361 100644 --- a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json +++ b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json @@ -9,7 +9,8 @@ "description": "zaaktypeomschrijving", "published" : "{% if startdatum|default %}{{ startdatum|date('Y-m-d') }}{% endif %}", "modified" : "{{ 'now'|date(H:i:sTm-d-Y') }}", - "status": "{% if startdatum|default %}Published{% else %}Concept{% endif %}" + "status": "{% if startdatum|default %}Published{% else %}Concept{% endif %}", + "catalog": "" }, "unset": [], "cast": { From 21f5cf711b349ffe3e84394c56fd01cedc57d023 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Fri, 29 Nov 2024 20:53:40 +0100 Subject: [PATCH 23/43] Encode binary strings --- lib/Service/CallService.php | 4 ++- lib/Settings/OpenConnectorAdmin.php | 43 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/lib/Service/CallService.php b/lib/Service/CallService.php index b86d1843..68e4003e 100644 --- a/lib/Service/CallService.php +++ b/lib/Service/CallService.php @@ -249,6 +249,8 @@ public function call( $time_end = microtime(true); + $body = $response->getBody()->getContents(); + // Let create the data array $data = [ 'request' => [ @@ -263,7 +265,7 @@ public function call( 'size' => $response->getBody()->getSize(), 'remoteIp' => $response->getHeaderLine('X-Real-IP') ?: $response->getHeaderLine('X-Forwarded-For') ?: null, 'headers' => $response->getHeaders(), - 'body' => $response->getBody()->getContents(), + 'body' => mb_check_encoding(value: $body, encoding: 'UTF-8') !== false ? $body : base64_encode($body), ] ]; diff --git a/lib/Settings/OpenConnectorAdmin.php b/lib/Settings/OpenConnectorAdmin.php index e69de29b..4e35c3fa 100644 --- a/lib/Settings/OpenConnectorAdmin.php +++ b/lib/Settings/OpenConnectorAdmin.php @@ -0,0 +1,43 @@ +config = $config; + $this->l = $l; + } + + /** + * @return TemplateResponse + */ + public function getForm() { + $parameters = [ + 'mySetting' => $this->config->getSystemValue('open_connector_setting', true), + ]; + + return new TemplateResponse('openconnector', 'settings/admin', $parameters, ''); + } + + public function getSection() { + return 'openconnector'; // Name of the previously created section. + } + + /** + * @return int whether the form should be rather on the top or bottom of + * the admin section. The forms are arranged in ascending order of the + * priority values. It is required to return a value between 0 and 100. + * + * E.g.: 70 + */ + public function getPriority() { + return 10; + } +} From 107934e759027ba1152f7e9ce435be50284cbe31 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Sat, 30 Nov 2024 10:14:25 +0100 Subject: [PATCH 24/43] Encode non-utf-8 response bodies --- lib/Service/CallService.php | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/Service/CallService.php b/lib/Service/CallService.php index 68e4003e..e40bc951 100644 --- a/lib/Service/CallService.php +++ b/lib/Service/CallService.php @@ -266,6 +266,7 @@ public function call( 'remoteIp' => $response->getHeaderLine('X-Real-IP') ?: $response->getHeaderLine('X-Forwarded-For') ?: null, 'headers' => $response->getHeaders(), 'body' => mb_check_encoding(value: $body, encoding: 'UTF-8') !== false ? $body : base64_encode($body), + 'encoding' => mb_check_encoding(value: $body, encoding: 'UTF-8') !== false ? 'UTF-8' : 'base64', ] ]; From 66696a9b322fe163034a9b8c483e79db1b65a901 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Sat, 30 Nov 2024 11:02:36 +0100 Subject: [PATCH 25/43] Fix datetime check --- lib/Service/SynchronizationService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index bba34456..ef9ff3ba 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -251,7 +251,6 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont // Let create a source hash for the object $originHash = md5(serialize($object)); - $synchronizationContract->setSourceLastChecked(new DateTime()); // Let's prevent pointless updates @todo account for omnidirectional sync, unless the config has been updated since last check then we do want to rebuild and check if the tagert object has changed if ($originHash === $synchronizationContract->getOriginHash() && $synchronization->getUpdated() < $synchronizationContract->getSourceLastChecked()) { @@ -262,6 +261,7 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont // The object has changed, oke let do mappig and bla die bla $synchronizationContract->setOriginHash($originHash); $synchronizationContract->setSourceLastChanged(new DateTime()); + $synchronizationContract->setSourceLastChecked(new DateTime()); // Check if object adheres to conditions. // Take note, JsonLogic::apply() returns a range of return types, so checking it with '=== false' or '!== true' does not work properly. From d72aa3698a280ef724d1f988ddaa405e03cbf245 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Sat, 30 Nov 2024 12:23:21 +0100 Subject: [PATCH 26/43] Nested downloadUrl --- .../mappings/sharepoint-woo-verzoek-to-publications.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json index 75e5e38f..e056e72a 100644 --- a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json +++ b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json @@ -9,7 +9,7 @@ "category": "d.woo_x005f_categorie", "published": "{% if d['woo_x005f_publicatiedatum']|default %}{{ d['woo_x005f_publicatiedatum']|date('Y-m-d') }}{% endif %}", "modified": "d.vti_x005f_nexttolasttimemodified", - "attachments": "{{ '[' }}{% if fileUrls['d']['results']|default %}{% set index = 0 %}{% for file in fileUrls['d']['results'] %}{% if index > 0 %}{{ ', ' }}{% endif %}{{ '{' }}{% set uri = file['__metadata']['uri']~'/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"1\"' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}", + "attachments": "{{ '[' }}{% if fileUrls['d']['results']|default %}{% set index = 0 %}{% for file in fileUrls['d']['results'] %}{% if index > 0 %}{{ ', ' }}{% endif %}{{ '{' }}\"downloadUrl\": {{ '{' }}{% set uri = file['__metadata']['uri']~'/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"1\"' }}{{ '}' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}", "status": "{% if d['woo_x005f_publicatiedatum']|default %}Published{% else %}Concept{% endif %}", "catalog": "" }, @@ -24,4 +24,4 @@ "attachments": "jsonToArray" }, "passThrough": false -} \ No newline at end of file +} From b54936841ae41b45026456291d6908db3863889e Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Sat, 30 Nov 2024 16:50:41 +0100 Subject: [PATCH 27/43] add mapping for sharepoint attachment title --- .../mappings/sharepoint-woo-verzoek-to-publications.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json index e056e72a..23b63987 100644 --- a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json +++ b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json @@ -9,7 +9,7 @@ "category": "d.woo_x005f_categorie", "published": "{% if d['woo_x005f_publicatiedatum']|default %}{{ d['woo_x005f_publicatiedatum']|date('Y-m-d') }}{% endif %}", "modified": "d.vti_x005f_nexttolasttimemodified", - "attachments": "{{ '[' }}{% if fileUrls['d']['results']|default %}{% set index = 0 %}{% for file in fileUrls['d']['results'] %}{% if index > 0 %}{{ ', ' }}{% endif %}{{ '{' }}\"downloadUrl\": {{ '{' }}{% set uri = file['__metadata']['uri']~'/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"1\"' }}{{ '}' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}", + "attachments": "{{ '[' }}{% if fileUrls['d']['results']|default %}{% set index = 0 %}{% for file in fileUrls['d']['results'] %}{% if index > 0 %}{{ ', ' }}{% endif %}{{ '{' }}{% if file['Name']|default %} \"title\": \"{{ file['Name'] }}\",{% endif %} \"downloadUrl\": {{ '{' }}{% set uri = file['__metadata']['uri']~'/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"1\"' }}{{ '}' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}", "status": "{% if d['woo_x005f_publicatiedatum']|default %}Published{% else %}Concept{% endif %}", "catalog": "" }, From 99390249797961f933e14fb2470755cd344195c4 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Mon, 2 Dec 2024 11:45:34 +0100 Subject: [PATCH 28/43] fetch extra data foreach result fetched from extra data --- lib/Service/SynchronizationService.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index 5e3f1838..abec6d8c 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -269,6 +269,18 @@ private function fetchExtraDataForObject(Synchronization $synchronization, array $extraData = $this->getObjectFromSource($synchronization, $endpoint); + // Temporary fix, this goes too deep + if ($extraDataConfig['extraDataConfigPerResult']) { + $dotObject = new Dot($extraData); + $results = $dotObject->get($extraDataConfig['resultsLocation']); + + foreach ($results as $key => $result) { + $results[$key] = $this->fetchExtraDataForObject($synchronization, $extraDataConfig['extraDataConfigPerResult'], $result); + } + + $extraData = $results; + } + // Set new key if configured. if (isset($extraDataConfig[$this::KEY_FOR_EXTRA_DATA_LOCATION]) === true) { $extraData = [$extraDataConfig[$this::KEY_FOR_EXTRA_DATA_LOCATION] => $extraData]; From ca92a46fa7cf89e0861ce6eb2ae5879d675b94d4 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Mon, 2 Dec 2024 12:26:11 +0100 Subject: [PATCH 29/43] Mapping fix --- .../mappings/sharepoint-woo-verzoek-to-publications.json | 2 +- lib/Service/SynchronizationService.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json index 23b63987..e38960a5 100644 --- a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json +++ b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json @@ -9,7 +9,7 @@ "category": "d.woo_x005f_categorie", "published": "{% if d['woo_x005f_publicatiedatum']|default %}{{ d['woo_x005f_publicatiedatum']|date('Y-m-d') }}{% endif %}", "modified": "d.vti_x005f_nexttolasttimemodified", - "attachments": "{{ '[' }}{% if fileUrls['d']['results']|default %}{% set index = 0 %}{% for file in fileUrls['d']['results'] %}{% if index > 0 %}{{ ', ' }}{% endif %}{{ '{' }}{% if file['Name']|default %} \"title\": \"{{ file['Name'] }}\",{% endif %} \"downloadUrl\": {{ '{' }}{% set uri = file['__metadata']['uri']~'/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"1\"' }}{{ '}' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}", + "attachments": "{{ '[' }}{% if fileUrls|default %}{% set index = 0 %}{% for file in fileUrls %}{% if index \u003E 0 %}{{ ', ' }}{% endif %}{{ '{' }}{% if file['Name']|default %} \"title\": \"{{ file['Name'] }}\",{% endif %}{% if file['d']['document_x005f_label']|default %}\"labels\": [\"{{ file['d']['document_x005f_label'] }}\"],{% endif %} \"downloadUrl\": {{ '{' }}{% set uri = file['__metadata']['uri']~'\/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"2\"' }}{{ '}' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}", "status": "{% if d['woo_x005f_publicatiedatum']|default %}Published{% else %}Concept{% endif %}", "catalog": "" }, diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index abec6d8c..35b43434 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -269,7 +269,7 @@ private function fetchExtraDataForObject(Synchronization $synchronization, array $extraData = $this->getObjectFromSource($synchronization, $endpoint); - // Temporary fix, this goes too deep + // Temporary fix, if ($extraDataConfig['extraDataConfigPerResult']) { $dotObject = new Dot($extraData); $results = $dotObject->get($extraDataConfig['resultsLocation']); From 943bd66489eb11b688ad4f597e39bd80d8eedbc8 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Mon, 2 Dec 2024 12:29:55 +0100 Subject: [PATCH 30/43] Update sync configs sharepoint --- .../sharepoint-convenanten-to-publications.json | 3 +++ .../sharepoint-woo-verzoeken-to-publications.json | 3 +++ 2 files changed, 6 insertions(+) diff --git a/configurations/sharepoint-woo/synchronizations/sharepoint-convenanten-to-publications.json b/configurations/sharepoint-woo/synchronizations/sharepoint-convenanten-to-publications.json index 030054f2..539175ce 100644 --- a/configurations/sharepoint-woo/synchronizations/sharepoint-convenanten-to-publications.json +++ b/configurations/sharepoint-woo/synchronizations/sharepoint-convenanten-to-publications.json @@ -15,6 +15,9 @@ "extraDataConfigs.1.dynamicEndpointLocation": "Files.__deferred.uri", "extraDataConfigs.1.mergeExtraData": "true", "extraDataConfigs.1.keyToSetExtraData": "fileUrls", + "extraDataConfigs.1.resultsLocation": "d.results", + "extraDataConfigs.1.extraDataConfigPerResult.dynamicEndpointLocation": "Properties.__deferred.uri", + "extraDataConfigs.1.extraDataConfigPerResult.mergeExtraData": "true", "headers": [], "query": [] }, diff --git a/configurations/sharepoint-woo/synchronizations/sharepoint-woo-verzoeken-to-publications.json b/configurations/sharepoint-woo/synchronizations/sharepoint-woo-verzoeken-to-publications.json index e4c66965..88568b92 100644 --- a/configurations/sharepoint-woo/synchronizations/sharepoint-woo-verzoeken-to-publications.json +++ b/configurations/sharepoint-woo/synchronizations/sharepoint-woo-verzoeken-to-publications.json @@ -15,6 +15,9 @@ "extraDataConfigs.1.dynamicEndpointLocation": "Files.__deferred.uri", "extraDataConfigs.1.mergeExtraData": "true", "extraDataConfigs.1.keyToSetExtraData": "fileUrls", + "extraDataConfigs.1.resultsLocation": "d.results", + "extraDataConfigs.1.extraDataConfigPerResult.dynamicEndpointLocation": "Properties.__deferred.uri", + "extraDataConfigs.1.extraDataConfigPerResult.mergeExtraData": "true", "headers": [], "query": [] }, From 7e3309850e212b5035c138194b319155ba9527c0 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Mon, 2 Dec 2024 14:34:50 +0100 Subject: [PATCH 31/43] fix mapping --- .../mappings/sharepoint-woo-verzoek-to-publications.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json index e38960a5..76ae88fb 100644 --- a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json +++ b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json @@ -9,7 +9,7 @@ "category": "d.woo_x005f_categorie", "published": "{% if d['woo_x005f_publicatiedatum']|default %}{{ d['woo_x005f_publicatiedatum']|date('Y-m-d') }}{% endif %}", "modified": "d.vti_x005f_nexttolasttimemodified", - "attachments": "{{ '[' }}{% if fileUrls|default %}{% set index = 0 %}{% for file in fileUrls %}{% if index \u003E 0 %}{{ ', ' }}{% endif %}{{ '{' }}{% if file['Name']|default %} \"title\": \"{{ file['Name'] }}\",{% endif %}{% if file['d']['document_x005f_label']|default %}\"labels\": [\"{{ file['d']['document_x005f_label'] }}\"],{% endif %} \"downloadUrl\": {{ '{' }}{% set uri = file['__metadata']['uri']~'\/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"2\"' }}{{ '}' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}", + "attachments": "{{ '[' }}{% if fileUrls|default %}{% set index = 0 %}{% for file in fileUrls %}{% if index > 0 %}{{ ', ' }}{% endif %}{{ '{' }}{% if file['Name']|default %} \"title\": \"{{ file['Name'] }}\",{% endif %}{% if file['d']['document_x005f_label']|default %}\"labels\": [\"{{ file['d']['document_x005f_label'] }}\"],{% endif %} \"downloadUrl\": {{ '{' }}{% set uri = file['__metadata']['uri']~'\/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"1\"' }}{{ '}' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}", "status": "{% if d['woo_x005f_publicatiedatum']|default %}Published{% else %}Concept{% endif %}", "catalog": "" }, From 8e499600614e3a5fea3b62e6538b704e6d0c37c0 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Mon, 2 Dec 2024 15:42:25 +0100 Subject: [PATCH 32/43] Mapping fix --- .../mappings/sharepoint-woo-verzoek-to-publications.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json index 76ae88fb..f97770ae 100644 --- a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json +++ b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json @@ -9,7 +9,7 @@ "category": "d.woo_x005f_categorie", "published": "{% if d['woo_x005f_publicatiedatum']|default %}{{ d['woo_x005f_publicatiedatum']|date('Y-m-d') }}{% endif %}", "modified": "d.vti_x005f_nexttolasttimemodified", - "attachments": "{{ '[' }}{% if fileUrls|default %}{% set index = 0 %}{% for file in fileUrls %}{% if index > 0 %}{{ ', ' }}{% endif %}{{ '{' }}{% if file['Name']|default %} \"title\": \"{{ file['Name'] }}\",{% endif %}{% if file['d']['document_x005f_label']|default %}\"labels\": [\"{{ file['d']['document_x005f_label'] }}\"],{% endif %} \"downloadUrl\": {{ '{' }}{% set uri = file['__metadata']['uri']~'\/$value' %}{{ '\"downloadUrl\": \"'~uri~'\",\"source\": \"1\"' }}{{ '}' }}{{ '}' }}{% endfor %}{% endif %}{{ ']' }}", + "attachments": "[ {% for file in fileUrls %} { {% if file['Name']|default %}\"title\": \"{{ file['Name'] }}\",{% endif %}{% if file['d']['document_x005f_label']|default %}\"labels\": [\"{{ file['d']['document_x005f_label'] }}\"],{% endif %}{% if file['__metadata']['uri']|default %}\"downloadUrl\": {\"downloadUrl\": \"{{ file['__metadata']['uri']~'/$value' }}\", \"source\": \"1\"}{% endif %} }{{ loop.last ? '' : ',' }} {% endfor %} ]", "status": "{% if d['woo_x005f_publicatiedatum']|default %}Published{% else %}Concept{% endif %}", "catalog": "" }, From 9732fe31e8e468777cddf931358bd5def99dc6fb Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Mon, 2 Dec 2024 15:42:32 +0100 Subject: [PATCH 33/43] Config check fix --- lib/Service/SynchronizationService.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index 35b43434..4e019549 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -287,7 +287,7 @@ private function fetchExtraDataForObject(Synchronization $synchronization, array } // Merge with earlier fetchde object if configured. - if (isset($extraDataConfig[$this::MERGE_EXTRA_DATA_OBJECT_LOCATION]) === true && $extraDataConfig[$this::MERGE_EXTRA_DATA_OBJECT_LOCATION] === true) { + if (isset($extraDataConfig[$this::MERGE_EXTRA_DATA_OBJECT_LOCATION]) === true && ($extraDataConfig[$this::MERGE_EXTRA_DATA_OBJECT_LOCATION] === true || $extraDataConfig[$this::MERGE_EXTRA_DATA_OBJECT_LOCATION] === 'true')) { return array_merge($object, $extraData); } @@ -359,7 +359,6 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont } } - // set the target hash $targetHash = md5(serialize($targetObject)); $synchronizationContract->setTargetHash($targetHash); From f6744e829d186b8838e07d346af5b77953f723d5 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Tue, 3 Dec 2024 12:46:56 +0100 Subject: [PATCH 34/43] check if extra data config is set --- lib/Service/SynchronizationService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index 4e019549..37b40f18 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -270,7 +270,7 @@ private function fetchExtraDataForObject(Synchronization $synchronization, array $extraData = $this->getObjectFromSource($synchronization, $endpoint); // Temporary fix, - if ($extraDataConfig['extraDataConfigPerResult']) { + if (isset($extraDataConfig['extraDataConfigPerResult']) === true) { $dotObject = new Dot($extraData); $results = $dotObject->get($extraDataConfig['resultsLocation']); From b654e1e564b4a0e7d1bfe82fe39b0db6a1cb6439 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Wed, 4 Dec 2024 10:48:35 +0100 Subject: [PATCH 35/43] Write private keys to tmp folder --- lib/Service/AuthenticationService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service/AuthenticationService.php b/lib/Service/AuthenticationService.php index e433110a..f6048746 100644 --- a/lib/Service/AuthenticationService.php +++ b/lib/Service/AuthenticationService.php @@ -230,7 +230,7 @@ public function fetchDecosToken(array $configuration): string private function getRSJWK(array $configuration): ?JWK { $stamp = microtime().getmypid(); - $filename = "privatekey-$stamp"; + $filename = "/var/tmp/privatekey-$stamp"; file_put_contents($filename, base64_decode($configuration['secret'])); $jwk = null; try { From e848688eb89cfd49f3cca38615e103b31f61fc99 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Wed, 4 Dec 2024 10:48:49 +0100 Subject: [PATCH 36/43] Run cronjob as user id if provided --- lib/Cron/ActionTask.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/Cron/ActionTask.php b/lib/Cron/ActionTask.php index a5b3a54d..c5f3f657 100644 --- a/lib/Cron/ActionTask.php +++ b/lib/Cron/ActionTask.php @@ -8,6 +8,8 @@ use OCP\BackgroundJob\TimedJob; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\IJobList; +use OCP\IUserManager; +use OCP\IUserSession; use Psr\Container\ContainerInterface; use Symfony\Component\Uid\Uuid; use DateInterval; @@ -30,7 +32,9 @@ public function __construct( JobMapper $jobMapper, JobLogMapper $jobLogMapper, IJobList $jobList, - ContainerInterface $containerInterface + ContainerInterface $containerInterface, + private IUserSession $userSession, + private IUserManager $userManager, ) { parent::__construct($time); $this->jobMapper = $jobMapper; @@ -80,6 +84,11 @@ public function run($argument) return; } + if(empty($job->getUserId()) === false && $this->userSession->getUser() === null) { + $user = $this->userManager->get($job->getUserId()); + $this->userSession->setUser($user); + } + $time_start = microtime(true); $action = $this->containerInterface->get($job->getJobClass()); From 73074a2d2bd038d35fb133d3bc9333e1eac51c9b Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Wed, 4 Dec 2024 11:16:23 +0100 Subject: [PATCH 37/43] Fix: unset dirlateststamp for now --- lib/Service/SynchronizationService.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index 37b40f18..8a251296 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -320,6 +320,11 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont } } + // @TODO: This should be unset through pre-mapping + if(isset($object['d']['vti_x005f_dirlateststamp']) === true) { + unset($object['d']['vti_x005f_dirlateststamp']); + } + // Let create a source hash for the object $originHash = md5(serialize($object)); @@ -329,6 +334,7 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont // The object has not changed and the config has not been updated since last check return $synchronizationContract; } + var_dump($originHash === $synchronizationContract->getOriginHash(), $originHash, $synchronizationContract->getOriginHash()); // The object has changed, oke let do mappig and bla die bla $synchronizationContract->setOriginHash($originHash); From 7962a301d9d90840b7ef5d70e87682200c00f70c Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Wed, 4 Dec 2024 11:17:27 +0100 Subject: [PATCH 38/43] remove dump --- lib/Service/SynchronizationService.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index 8a251296..0b205ff1 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -334,8 +334,7 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont // The object has not changed and the config has not been updated since last check return $synchronizationContract; } - var_dump($originHash === $synchronizationContract->getOriginHash(), $originHash, $synchronizationContract->getOriginHash()); - + // The object has changed, oke let do mappig and bla die bla $synchronizationContract->setOriginHash($originHash); $synchronizationContract->setSourceLastChanged(new DateTime()); From 8a2651257d35f3fea39bcc42df22136919b9fe51 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Thu, 5 Dec 2024 09:11:07 +0100 Subject: [PATCH 39/43] Updated mapping --- .../sharepoint-woo-verzoek-to-publications.json | 13 ++++++++----- .../mappings/xxllnc-suite-to-publications.json | 11 +++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json index f97770ae..a6cdbade 100644 --- a/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json +++ b/configurations/sharepoint-woo/mappings/sharepoint-woo-verzoek-to-publications.json @@ -9,9 +9,10 @@ "category": "d.woo_x005f_categorie", "published": "{% if d['woo_x005f_publicatiedatum']|default %}{{ d['woo_x005f_publicatiedatum']|date('Y-m-d') }}{% endif %}", "modified": "d.vti_x005f_nexttolasttimemodified", - "attachments": "[ {% for file in fileUrls %} { {% if file['Name']|default %}\"title\": \"{{ file['Name'] }}\",{% endif %}{% if file['d']['document_x005f_label']|default %}\"labels\": [\"{{ file['d']['document_x005f_label'] }}\"],{% endif %}{% if file['__metadata']['uri']|default %}\"downloadUrl\": {\"downloadUrl\": \"{{ file['__metadata']['uri']~'/$value' }}\", \"source\": \"1\"}{% endif %} }{{ loop.last ? '' : ',' }} {% endfor %} ]", - "status": "{% if d['woo_x005f_publicatiedatum']|default %}Published{% else %}Concept{% endif %}", - "catalog": "" + "attachments": "[ {% for file in fileUrls %} { {% if file['Name']|default %}\"title\": \"{{ file['Name'] }}\",{% endif %}{% if file['d']['document_x005f_label']|default %}\"labels\": [\"{{ file['d']['document_x005f_label'] }}\"],{% endif %}{% if file['__metadata']['uri']|default %}\"downloadUrl\": {\"accessUrl\": \"{{ file['__metadata']['uri']~'/$value' }}\", \"source\": \"1\"}{% endif %} }{{ loop.last ? '' : ',' }} {% endfor %} ]", + "status": "Concept", + "catalog": "", + "publicationType": "" }, "unset": [], "cast": { @@ -19,9 +20,11 @@ "description": "unsetIfValue==d.woo_x005f_beschrijving", "summary": "unsetIfValue==d.woo_x005f_samenvatting", "category": "unsetIfValue==d.woo_x005f_categorie", - "published": "unsetIfValue==d.woo_x005f_publicatiedatum", + "published": "unsetIfValue==", "modified": "unsetIfValue==d.vti_x005f_nexttolasttimemodified", - "attachments": "jsonToArray" + "attachments": "jsonToArray", + "publicationType": "unsetIfValue==", + "catalog": "unsetIfValue==" }, "passThrough": false } diff --git a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json index 54ef5361..7a2ea214 100644 --- a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json +++ b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json @@ -7,17 +7,20 @@ "title": "omschrijving", "summary": "zaaktypeomschrijving", "description": "zaaktypeomschrijving", - "published" : "{% if startdatum|default %}{{ startdatum|date('Y-m-d') }}{% endif %}", + "published" : "", "modified" : "{{ 'now'|date(H:i:sTm-d-Y') }}", - "status": "{% if startdatum|default %}Published{% else %}Concept{% endif %}", - "catalog": "" + "status": "Concept", + "catalog": "", + "publicationType": "" }, "unset": [], "cast": { "title": "unsetIfValue==omschrijving", "summary": "unsetIfValue==zaaktypeomschrijving", "description": "unsetIfValue==zaaktypeomschrijving", - "published" : "unsetIfValue==" + "published" : "unsetIfValue==", + "catalog": "unsetIfValue==", + "publicationType": "unsetIfValue==" }, "passThrough": false } \ No newline at end of file From 9061076373252a60fc961231f57c83d6c8aed700 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Thu, 5 Dec 2024 10:34:09 +0100 Subject: [PATCH 40/43] Store JobLogs after updating them --- lib/Cron/ActionTask.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/Cron/ActionTask.php b/lib/Cron/ActionTask.php index c5f3f657..471f186e 100644 --- a/lib/Cron/ActionTask.php +++ b/lib/Cron/ActionTask.php @@ -136,6 +136,8 @@ public function run($argument) } } + $this->jobLogMapper->update(entity: $jobLog); + // Let's report back about what we have just done return $jobLog; } From 45889f518e43131612a3ebfc8cdf4299dbce5f14 Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Thu, 5 Dec 2024 15:12:01 +0100 Subject: [PATCH 41/43] Check if have to use pagination --- lib/Db/Synchronization.php | 10 ++++++++++ lib/Service/SynchronizationService.php | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/Db/Synchronization.php b/lib/Db/Synchronization.php index 208bef2c..1ba4676f 100644 --- a/lib/Db/Synchronization.php +++ b/lib/Db/Synchronization.php @@ -65,6 +65,16 @@ public function __construct() { $this->addType(fieldName:'followUps', type: 'json'); } + public function usesPagination(): bool + { + if (isset($this->sourceConfig['usesPagination']) === true && ($this->sourceConfig['usesPagination'] === false || $this->sourceConfig['usesPagination'] === 'false')) { + return false; + } + + // By default sources use basic pagination. + return true; + } + public function getJsonFields(): array { return array_keys( diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index 0b205ff1..3f51accd 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -554,7 +554,7 @@ public function getAllObjectsFromApi(Synchronization $synchronization, ?bool $is $objects = array_merge($objects, $this->getAllObjectsFromArray($body, $synchronization)); } - if ($useNextEndpoint === false) { + if ($useNextEndpoint === false && $synchronization->usesPagination() === true) { do { $config = $this->getNextPage(config: $config, sourceConfig: $sourceConfig, currentPage: $currentPage); $response = $this->callService->call(source: $source, endpoint: $endpoint, method: 'GET', config: $config)->getResponse(); From fdc12e85565a8bab60af26759088d33b6a409b4f Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Thu, 5 Dec 2024 15:27:29 +0100 Subject: [PATCH 42/43] Update config and docblock --- .../xxllnc-suite-to-publications.json | 36 ++++++++++--------- .../xxllnc-suite-to-publications.json | 20 +++++++---- lib/Db/Synchronization.php | 5 +++ 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json index 7a2ea214..dcd13502 100644 --- a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json +++ b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json @@ -4,23 +4,25 @@ "version": "", "reference": "", "mapping": { - "title": "omschrijving", - "summary": "zaaktypeomschrijving", - "description": "zaaktypeomschrijving", - "published" : "", - "modified" : "{{ 'now'|date(H:i:sTm-d-Y') }}", - "status": "Concept", - "catalog": "", - "publicationType": "" - }, - "unset": [], - "cast": { - "title": "unsetIfValue==omschrijving", - "summary": "unsetIfValue==zaaktypeomschrijving", - "description": "unsetIfValue==zaaktypeomschrijving", - "published" : "unsetIfValue==", - "catalog": "unsetIfValue==", - "publicationType": "unsetIfValue==" + "title": "omschrijving", + "summary": "zaaktypeomschrijving", + "description": "zaaktypeomschrijving", + "category": "{% if zaaktypecode|default %}{% set wooVerzoekenEnBesluiten = ['LP00000431', 'B1873', 'cherry'] %}{% set klachtoordelen = ['LP00000091', 'LP00001132', 'LP00000121', 'B0757', 'LP00000832', 'LP00001096'] %}{% if zaaktypecode in wooVerzoekenEnBesluiten %}{{ 'Woo-verzoeken en -besluiten' }}{% elseif zaaktypecode in klachtoordelen %}{{ 'Klachtoordelen' }}{% endif %}{% endif %}", + "published": "startdatum", + "modified": "{{ 'now'|date('H:i:sTm-d-Y') }}", + "attachments": "[{% if files|default %}{% for file in files %} { {% if file['titel']|default %}\"title\": \"{{ file['titel'] }}\",{% endif %}\"labels\": [\"{{ 'Informatieverzoek' }}\"],{% if file['formaat']|default %}\"extension\": \"{{ file['formaat']|split('/')|last }}\",\"type\": \"{{ file['formaat'] }}\",{% endif %}{% if file['inhoud']|default and file['formaat']|default %}\"accessUrl\": \"data:{{ file['formaat'] }};base64,{{ file.inhoud }}\"{% endif %} }{{ loop.last ? '' : ',' }} {% endfor %}{% endif %}]", + "status": "Concept" + }, + "unset": [ + "" + ], + "cast": { + "title": "unsetIfValue==omschrijving", + "summary": "unsetIfValue==zaaktypeomschrijving", + "description": "unsetIfValue==zaaktypeomschrijving", + "category": "unsetIfValue==", + "published": "unsetIfValue==startdatum", + "attachments": "jsonToArray" }, "passThrough": false } \ No newline at end of file diff --git a/configurations/sharepoint-woo/synchronizations/xxllnc-suite-to-publications.json b/configurations/sharepoint-woo/synchronizations/xxllnc-suite-to-publications.json index 6dfedf86..91922ef1 100644 --- a/configurations/sharepoint-woo/synchronizations/xxllnc-suite-to-publications.json +++ b/configurations/sharepoint-woo/synchronizations/xxllnc-suite-to-publications.json @@ -7,13 +7,19 @@ "sourceHash": "", "sourceTargetMapping": "1", "sourceConfig": { - "idPosition": "id", - "resultsPosition": "results", - "endpoint": "/tlb/zaaksysteem/api/v1/zaken", - "headers": [], - "query.startdatum__gte": "2024-07-06", - "query.einddatum__lt": "2024-08-01" - }, + "idPosition": "identificatie", + "resultsPosition": "results", + "endpoint": "\/tlb\/zaaksysteem\/api\/v1\/zaken", + "headers": [], + "query.startdatum__gte": "2024-08-01", + "query.einddatum__lt": "2025-01-01", + "usesPagination": "false", + "extraDataConfigs.0.staticEndpoint": "/tlb/zaaksysteem/api/v1/zaken/{{ originId }}/informatieobjecten", + "extraDataConfigs.0.mergeExtraData": "true", + "extraDataConfigs.0.keyToSetExtraData": "files", + "extraDataConfigs.0.resultsLocation": "results", + "extraDataConfigs.0.extraDataConfigPerResult.staticEndpoint": "/tlb/zaaksysteem/api/v1/informatieobjecten/{{ originId }}" + }, "targetId": "1/1", "targetType": "register/schema" } \ No newline at end of file diff --git a/lib/Db/Synchronization.php b/lib/Db/Synchronization.php index 1ba4676f..473524b0 100644 --- a/lib/Db/Synchronization.php +++ b/lib/Db/Synchronization.php @@ -65,6 +65,11 @@ public function __construct() { $this->addType(fieldName:'followUps', type: 'json'); } + /** + * Checks through sourceConfig if the source of this sync uses pagination + * + * @return bool true if its uses pagination + */ public function usesPagination(): bool { if (isset($this->sourceConfig['usesPagination']) === true && ($this->sourceConfig['usesPagination'] === false || $this->sourceConfig['usesPagination'] === 'false')) { From 4119fb760053afdd201407549c4f1e2636939ffc Mon Sep 17 00:00:00 2001 From: Barry Brands Date: Thu, 5 Dec 2024 15:56:21 +0100 Subject: [PATCH 43/43] set version in config --- .../sharepoint-woo/mappings/xxllnc-suite-to-publications.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json index dcd13502..744823b1 100644 --- a/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json +++ b/configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json @@ -1,8 +1,6 @@ { "name": "Xxllnc suite to Publication", - "description": "", - "version": "", - "reference": "", + "version": "0.0.1", "mapping": { "title": "omschrijving", "summary": "zaaktypeomschrijving",