Added source hash mapping

lib/Db/Synchronization.php

@@ -16,6 +16,7 @@ class Synchronization extends Entity implements JsonSerializable
 	protected ?string $sourceId = null;	// The id of the source object
 	protected ?string $sourceType = null;	// The type of the source object (e.g. api, database, register/schema.)
 	protected ?string $sourceHash = null;	// The hash of the source object when it was last synced.
+	protected ?string $sourceHashMapping = null;	// The mapping id of the mapping that we map the object to for hashing.
 	protected ?string $sourceTargetMapping = null;	// The mapping of the source object to the target object
 	protected ?array $sourceConfig = null; // The configuration of the object in the source
 	protected ?DateTime $sourceLastChanged = null;	// The last changed date of the source object
@@ -46,6 +47,7 @@ public function __construct() {
 		$this->addType('sourceId', 'string');
 		$this->addType('sourceType', 'string');
 		$this->addType('sourceHash', 'string');
+		$this->addType('sourceHashMapping', 'string');
 		$this->addType('sourceTargetMapping', 'string');
 		$this->addType('sourceConfig', 'json');
 		$this->addType('sourceLastChanged', 'datetime');
@@ -121,6 +123,7 @@ public function jsonSerialize(): array
 			'sourceId' => $this->sourceId,
 			'sourceType' => $this->sourceType,
 			'sourceHash' => $this->sourceHash,
+			'sourceHashMapping' => $this->sourceHashMapping,
 			'sourceTargetMapping' => $this->sourceTargetMapping,
 			'sourceConfig' => $this->sourceConfig,
 			'sourceLastChanged' => isset($this->sourceLastChanged) === true ? $this->sourceLastChanged->format('c') : null,

lib/Migration/Version1Date20241210100000.php

@@ -0,0 +1,62 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\OpenConnector\Migration;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+/**
+ * FIXME Auto-generated migration step: Please modify to your needs!
+ */
+class Version1Date20241210100000 extends SimpleMigrationStep {
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure(): ISchemaWrapper $schemaClosure
+	 * @param array $options
+	 */
+	public function preSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void {
+	}
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure(): ISchemaWrapper $schemaClosure
+	 * @param array $options
+	 * @return null|ISchemaWrapper
+	 */
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+		/**
+		 * @var ISchemaWrapper $schema
+		 */
+		$schema = $schemaClosure();
+
+		if ($schema->hasTable('openconnector_synchronizations') === true) {
+			$table = $schema->getTable('openconnector_synchronizations');
+
+			if ($table->hasColumn('sourceHashMapping') === false) {
+				$table->dropColumn('sourceHashMapping');
+				$table->addColumn('source_hash_mapping', Types::STRING)->setNotnull(false);
+			}
+		}
+
+		return $schema;
+	}
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure(): ISchemaWrapper $schemaClosure
+	 * @param array $options
+	 */
+	public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void {
+	}
+}

lib/Service/SynchronizationService.php

@@ -294,6 +294,57 @@ private function fetchExtraDataForObject(Synchronization $synchronization, array
         return $extraData;
     }
 
+    /**
+     * Fetches multiple extra data entries for an object based on the source configuration.
+     *
+     * This method iterates through a list of extra data configurations, fetches the additional data for each configuration,
+     * and merges it with the original object.
+     *
+     * @param Synchronization $synchronization The synchronization instance containing configuration details.
+     * @param array           $sourceConfig    The source configuration containing extra data retrieval settings.
+     * @param array           $object          The original object for which extra data needs to be fetched.
+     *
+     * @return array The updated object with all fetched extra data merged into it.
+     */
+    private function fetchMultipleExtraData(Synchronization $synchronization, array $sourceConfig, array $object): array
+    {
+        if (isset($sourceConfig[$this::EXTRA_DATA_CONFIGS_LOCATION]) === true) {
+            foreach ($sourceConfig[$this::EXTRA_DATA_CONFIGS_LOCATION] as $extraDataConfig) {
+                $object = array_merge($object, $this->fetchExtraDataForObject($synchronization, $extraDataConfig, $object));
+            }
+        }
+
+        return $object;
+    }
+
+    /**
+     * Maps a given object using a source hash mapping configuration.
+     *
+     * This function retrieves a hash mapping configuration for a synchronization instance, if available,
+     * and applies it to the input object using the mapping service.
+     *
+     * @param Synchronization $synchronization The synchronization instance containing the hash mapping configuration.
+     * @param array           $object          The input object to be mapped.
+     *
+     * @return array The mapped object, or the original object if no mapping is found.
+     */
+    private function mapHashObject(Synchronization $synchronization, array $object): array
+    {
+        if (empty($synchronization->getSourceHashMapping()) === false) {
+            try {
+                $sourceHashMapping = $this->mappingMapper->find(id: $synchronization->getSourceHashMapping());
+            } catch (DoesNotExistException $exception) {
+                return new Exception($exception->getMessage());
+            }
+
+            // Execute mapping if found
+            if ($sourceHashMapping) {
+                return $this->mappingService->executeMapping(mapping: $sourceHashMapping, input: $object);
+            }
+        }
+
+        return $object;
+    }
 
 	/**
 	 * Synchronize a contract
@@ -314,27 +365,19 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont
         $sourceConfig = $this->callService->applyConfigDot($synchronization->getSourceConfig());
 
         // Check if extra data needs to be fetched
-        if (isset($sourceConfig[$this::EXTRA_DATA_CONFIGS_LOCATION]) === true) {
-            foreach ($sourceConfig[$this::EXTRA_DATA_CONFIGS_LOCATION] as $extraDataConfig) {
-                $object = array_merge($object, $this->fetchExtraDataForObject($synchronization, $extraDataConfig, $object));
-            }
-        }
-
-		// @TODO: This should be unset through pre-mapping
-		if(isset($object['d']['vti_x005f_dirlateststamp']) === true) {
-			unset($object['d']['vti_x005f_dirlateststamp']);
-		}
-
+        $object = $this->fetchMultipleExtraData(synchronization: $synchronization, sourceConfig: $sourceConfig, object: $object);
 
+        // Get mapped hash object (some fields can make it look the object has changed even if it hasn't)
+        $hashObject = $this->mapHashObject(synchronization: $synchronization, object: $object);
         // Let create a source hash for the object
-        $originHash = md5(serialize($object));
+        $originHash = md5(serialize($hashObject));
 
         // Let's prevent pointless updates @todo account for omnidirectional sync, unless the config has been updated since last check then we do want to rebuild and check if the tagert object has changed
         if ($originHash === $synchronizationContract->getOriginHash() && $synchronization->getUpdated() < $synchronizationContract->getSourceLastChecked()) {
             // The object has not changed and the config has not been updated since last check
 			return $synchronizationContract;
         }
-		
+
         // The object has changed, oke let do mappig and bla die bla
         $synchronizationContract->setOriginHash($originHash);
         $synchronizationContract->setSourceLastChanged(new DateTime());

src/entities/synchronization/synchronization.mock.ts

@@ -9,6 +9,7 @@ export const mockSynchronizationData = (): TSynchronization[] => [
 		sourceId: 'source1',
 		sourceType: 'api',
 		sourceHash: 'source1',
+		sourceHashMapping: '1',
 		sourceTargetMapping: 'source1',
 		sourceConfig: {},
 		sourceLastChanged: '2023-05-01T12:00:00Z',
@@ -32,6 +33,7 @@ export const mockSynchronizationData = (): TSynchronization[] => [
 		sourceId: 'source2',
 		sourceType: 'api',
 		sourceHash: 'source2',
+		sourceHashMapping: '1',
 		sourceTargetMapping: 'source2',
 		sourceConfig: {},
 		sourceLastChanged: '2023-05-02T12:00:00Z',

src/entities/synchronization/synchronization.ts

@@ -11,6 +11,7 @@ export class Synchronization extends ReadonlyBaseClass implements TSynchronizati
 	public sourceId: string
 	public sourceType: string
 	public sourceHash: string
+	public sourceHashMapping: string
 	public sourceTargetMapping: string
 	public sourceConfig: Record<string, string>
 	public sourceLastChanged: string
@@ -35,6 +36,7 @@ export class Synchronization extends ReadonlyBaseClass implements TSynchronizati
 			sourceId: synchronization.sourceId || '',
 			sourceType: synchronization.sourceType || '',
 			sourceHash: synchronization.sourceHash || '',
+			sourceHashMapping: synchronization.sourceHashMapping || '',
 			sourceTargetMapping: synchronization.sourceTargetMapping || '',
 			sourceConfig: synchronization.sourceConfig || {},
 			sourceLastChanged: synchronization.sourceLastChanged || '',
@@ -63,6 +65,7 @@ export class Synchronization extends ReadonlyBaseClass implements TSynchronizati
 			sourceId: z.string(),
 			sourceType: z.string(),
 			sourceHash: z.string(),
+			sourceHashMapping: z.string(),
 			sourceTargetMapping: z.string(),
 			sourceConfig: z.record(z.string(), z.string()),
 			sourceLastChanged: z.string(),

src/entities/synchronization/synchronization.types.ts

@@ -5,6 +5,7 @@ export type TSynchronization = {
     sourceId: string
     sourceType: string
     sourceHash: string
+    sourceHashMapping: string
     sourceTargetMapping: string
     sourceConfig: Record<string, string>
     sourceLastChanged: string

src/modals/Synchronization/EditSynchronization.vue

@@ -90,10 +90,15 @@ import { synchronizationStore, navigationStore, sourceStore, mappingStore } from
 					:loading="sourcesLoading"
 					input-label="Source ID" />
 
+				<NcSelect v-bind="sourceTargetMappingOptions"
+					v-model="sourceTargetMappingOptions.hashValue"
+					:loading="sourceTargetMappingLoading"
+					input-label="Source hash mapping" />
+
 				<NcSelect v-bind="sourceTargetMappingOptions"
 					v-model="sourceTargetMappingOptions.sourceValue"
 					:loading="sourceTargetMappingLoading"
-					input-label="sourceTargetMapping" />
+					input-label="Source target mapping" />
 
 				<NcTextField :value.sync="synchronizationItem.sourceConfig.idPosition"
 					label="(optional) Position of id in source object" />
@@ -135,7 +140,7 @@ import { synchronizationStore, navigationStore, sourceStore, mappingStore } from
 				<NcSelect v-bind="sourceTargetMappingOptions"
 					v-model="sourceTargetMappingOptions.targetValue"
 					:loading="sourceTargetMappingLoading"
-					input-label="targetSourceMapping" />
+					input-label="Target source mapping" />
 			</form>
 
 			<NcButton v-if="!success"
@@ -208,6 +213,7 @@ export default {
 					headers: {},
 					query: {},
 				},
+				sourceHashMapping: '',
 				sourceTargetMapping: '',
 				targetId: '',
 				targetType: 'register/schema',
@@ -234,6 +240,7 @@ export default {
 			sourceTargetMappingLoading: false, // Indicates if the mappings are loading
 			sourceTargetMappingOptions: { // A list of mappings
 				options: [],
+				hashValue: null,
 				sourceValue: null,
 				targetValue: null,
 			},
@@ -340,12 +347,19 @@ export default {
 				.then(({ entities }) => {
 					const activeSourceMapping = entities.find(mapping => mapping.id.toString() === this.synchronizationItem.sourceTargetMapping.toString())
 					const activeTargetMapping = entities.find(mapping => mapping.id.toString() === this.synchronizationItem.targetSourceMapping.toString())
+					const sourceHashMapping = entities.find(mapping => mapping.id.toString() === this.synchronizationItem.sourceHashMapping.toString())
 
 					this.sourceTargetMappingOptions = {
 						options: entities.map(mapping => ({
 							label: mapping.name,
 							id: mapping.id,
 						})),
+						hashValue: sourceHashMapping
+							? {
+								label: sourceHashMapping.name,
+								id: sourceHashMapping.id,
+							}
+							: null,
 						sourceValue: activeSourceMapping
 							? {
 								label: activeSourceMapping.name,
@@ -528,6 +542,7 @@ export default {
 				...this.synchronizationItem,
 				sourceId: this.sourceOptions.sourceValue?.id || null,
 				sourceType: this.typeOptions.value?.id || null,
+				sourceHashMapping: this.sourceTargetMappingOptions.hashValue?.id || null,
 				sourceTargetMapping: this.sourceTargetMappingOptions.sourceValue?.id || null,
 				targetType: this.targetTypeOptions.value?.id || null,
 				targetId: targetId || null,

src/views/Synchronization/SynchronizationDetails.vue

@@ -88,6 +88,10 @@ import { synchronizationStore, navigationStore, logStore } from '../../store/sto
 						<b>Source Hash:</b>
 						<p>{{ synchronizationStore.synchronizationItem.sourceHash || 'N/A' }}</p>
 					</div>
+					<div class="gridContent gridFullWidth">
+						<b>Source Hash mapping id:</b>
+						<p>{{ synchronizationStore.synchronizationItem.sourceHashMapping || 'N/A' }}</p>
+					</div>
 					<div class="gridContent gridFullWidth">
 						<b>Source Last Changed:</b>
 						<p>{{ synchronizationStore.synchronizationItem.sourceLastChanged || 'N/A' }}</p>