associate DAM and FAL records via DAM instead of FAL

Reason is issue b13#62 - DAM may have multiple records for the same file.
Since we correlate DAM and FAL by file path, we need to associate at
least one DAM record to each migrated FAL record, thus the old way of
using one field sys_file._migrateddamuid was incomplete as it only
allowed associating at most one DAM record to each FAL record.

Note that we may have different metadata for the same file as a
result. Before these changes, only the metadata of the last file
having been migrated was used. Now, all metadata will be processed
which may lead to errors. This commit is only intended to record the
change in columns and does not handle that issue yet.

Classes/Controller/DamMigrationCommandController.php

@@ -52,7 +52,7 @@ public function migrateDamRecordsToStorageCommand($storageUid = 1, $recordLimit
 
 	/**
 	 * Migrates all DAM records to FAL.
-	 * A database field "_migrateddamuid" connects each FAL record to the original DAM record.
+	 * A database field "_migratedfaluid" connects each original DAM record to the matching FAL record.
 	 *
 	 * @param int $storageUid The UID of the storage (default: 1 Do not modify if you are unsure.)
 	 * @param int $recordLimit The amount of records to process in a single run. You can set this value if you have memory constraints.
@@ -373,9 +373,9 @@ public function migrateDamFrontendPluginsCommand() {
 				}
 				else {
 					$newFalRecords = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
-						'uid, _migrateddamuid',
-						'sys_file',
-						'_migrateddamuid IN (' . $plugin['tx_damdownloadlist_records'] . ')'
+						'_migratedfaluid AS uid',
+						'tx_dam',
+						'tx_dam.uid IN (' . $plugin['tx_damdownloadlist_records'] . ') AND _migratedfaluid > 0'
 					);
 
 					// update ctype for dam_frontend_pi2 plugin

Classes/Helper/DatabaseHelper.php

@@ -52,9 +52,9 @@ public static function getInstance() {
 	 */
 	public function getAllMigratedFalRecords() {
 		return $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
-			'uid, _migrateddamuid AS damuid',
-			'sys_file',
-			'_migrateddamuid>0',
+			'sys_file.uid AS uid, tx_dam.uid AS damuid',
+			'tx_dam JOIN sys_file ON sys_file._migratedfaluid = tx_dam.uid',
+			'', // where
 			'',	// group by
 			'', // order by
 			'', // limit

Classes/Service/MigrateCategoryRelationsService.php

@@ -100,8 +100,8 @@ public function execute() {
 	protected function getCategoryRelationsWhereSysCategoryExists() {
 		$rows = $this->database->exec_SELECTgetRows(
 			'MM.*, SM.uid as sys_metadata_uid, SC.uid as sys_category_uid',
-			'tx_dam_mm_cat MM, sys_file SF, sys_category SC, sys_file_metadata SM',
-			'SC._migrateddamcatuid = MM.uid_foreign AND SF._migrateddamuid = MM.uid_local AND SM.file = SF.uid'
+			'tx_dam_mm_cat MM, tx_dam D, sys_category SC, sys_file_metadata SM',
+			'SC._migrateddamcatuid = MM.uid_foreign AND D.uid = MM.uid_local AND SM.file = D._migratedfaluid'
 		);
 		if ($rows === NULL) {
 			throw new \Exception('SQL-Error in getCategoryRelationsWhereSysCategoryExists()', 1382968725);

Classes/Service/MigrateLinksService.php

@@ -188,9 +188,9 @@ protected function findMigratedFileUIDForDAMRecord($mediaUID) {
 
 		// query database
 		$res = $this->database->exec_SELECTgetSingleRow(
-			'uid',
-			'sys_file',
-			'`_migrateddamuid` = ' . $this->database->fullQuoteStr((int)$mediaUID, 'sys_file', false)
+			'_migratedfaluid AS uid',
+			'tx_dam',
+			'tx_dam.uid = ' . $this->database->fullQuoteStr((int)$mediaUID, 'tx_dam', false)
 		);
 
 		// use negative value if not found

Classes/Service/MigrateMetadataService.php

@@ -163,9 +163,9 @@ public function execute() {
 	 */
 	protected function execSelectMigratedSysFilesQuery() {
 		return $this->database->exec_SELECTquery(
-			'DISTINCT m.uid AS metadata_uid, f.uid as file_uid, f._migrateddamuid AS dam_uid, d.*',
+			'DISTINCT m.uid AS metadata_uid, f.uid as file_uid, d.uid AS dam_uid, d.*',
 			'sys_file f, sys_file_metadata m, tx_dam d',
-			'm.file=f.uid AND f._migrateddamuid=d.uid AND f._migrateddamuid > 0'
+			'm.file=f.uid AND d._migratedfaluid=f.uid'
 		);
 	}
 

Classes/Service/MigrateRelationsService.php

@@ -315,9 +315,9 @@ protected function getPidOfForeignRecord(array $damRelation) {
     }
 
 	/**
-	 * After a migration of tx_dam -> sys_file the col _migrateddamuid is
-	 * filled with dam uid Now we can search in dam relations for dam records
-	 * which have already been migrated to sys_file
+	 * After a migration of tx_dam -> sys_file the col _migratedfaluid is
+	 * filled with FAL uid. Now we can search in DAM relations for DAM records
+	 * which have already been migrated to sys_file.
 	 *
 	 * @return \mysqli_result
 	 */
@@ -339,8 +339,10 @@ protected function execSelectDamReferencesWhereSysFileExists() {
 		return $this->database->exec_SELECTquery(
 			$selectFields,
 			'tx_dam_mm_ref
+			JOIN tx_dam ON
+				tx_dam.uid = tx_dam_mm_ref.uid_local
 			JOIN sys_file ON
-				sys_file._migrateddamuid = tx_dam_mm_ref.uid_local
+				sys_file.uid = tx_dam._migratedfaluid
 			JOIN sys_file_metadata ON
 				sys_file.uid = sys_file_metadata.file
 				',

Classes/Service/MigrateRteMediaTagService.php

@@ -59,9 +59,9 @@ public function execute($table, $field) {
 
 		/** @var PreparedStatement $getSysFileUidStatement */
 		$getSysFileUidStatement = $this->database->prepare_SELECTquery(
-			'uid',
-			'sys_file',
-			'_migrateddamuid = :migrateddamuid'
+			'_migratedfaluid AS uid',
+			'tx_dam',
+			'tx_dam.uid = :migrateddamuid AND _migratedfaluid > 0'
 		);
 		foreach ($records as $rec) {
 			$originalContent = $rec[$field];

Classes/Service/MigrateService.php

@@ -30,7 +30,7 @@
 /**
  * Service to Migrate Records
  * Finds all DAM records that have not been migrated yet
- * and adds a DB field "_migrateddamuid" to each FAL record
+ * and adds a DB field "_migratedfaluid" to each DAM record
  * to connect the DAM and FAL DB records
  *
  * currently it only works for files within the fileadmin
@@ -165,7 +165,7 @@ protected function isValidDirectory(array $damRecord) {
 	public function countNotMigratedDamRecordsQuery() {
 		$row = $this->database->exec_SELECTgetSingleRow(
 			'COUNT(*) as total',
-			'tx_dam LEFT JOIN sys_file ON (tx_dam.uid = sys_file._migrateddamuid)',
+			'tx_dam LEFT JOIN sys_file ON (tx_dam._migratedfaluid = sys_file.uid)',
 			'sys_file.uid IS NULL AND
 			 tx_dam.deleted = 0 AND
 			 tx_dam.file_path LIKE "' . $this->storageBasePath . '%" AND
@@ -182,7 +182,7 @@ public function countNotMigratedDamRecordsQuery() {
 	protected function execSelectNotMigratedDamRecordsQuery() {
 		return $this->database->exec_SELECTquery(
 			'tx_dam.*',
-			'tx_dam LEFT JOIN sys_file ON (tx_dam.uid = sys_file._migrateddamuid)',
+			'tx_dam LEFT JOIN sys_file ON (tx_dam._migratedfaluid = sys_file.uid)',
 			'sys_file.uid IS NULL AND
 			 tx_dam.deleted = 0 AND
 			 tx_dam.file_path LIKE "' . $this->storageBasePath . '%" AND
@@ -228,11 +228,11 @@ protected function migrateFileFromDamToFal(array $damRecord, \TYPO3\CMS\Core\Res
 				$this->createArrayForUpdateInsertSysFileRecord($damRecord)
 			);
 
-			// add the migrated uid of the DAM record to the FAL record
+			// add the uid of the FAL record to the original DAM record
 			$this->database->exec_UPDATEquery(
-				'sys_file',
-				'uid = ' . $fileObject->getUid(),
-				array('_migrateddamuid' => $damRecord['uid'])
+				'tx_dam',
+				'uid = ' . $damRecord['uid'],
+				array('_migratedfaluid' => $fileObject->getUid())
 			);
 		}
 	}

Documentation/CommandReference/Index.rst

@@ -147,7 +147,7 @@ dam_falmigration:dammigration:migratedamrecords
 
 **Migrates all DAM records to FAL.**
 
-A database field "_migrateddamuid" connects each FAL record to the original DAM record.
+A database field "_migratedfaluid" connects each original DAM record to the matching FAL record.
 
 
 

Documentation/UserManual/Index.rst

@@ -34,9 +34,9 @@ The available migration tasks can be found under the *extbase* cliKey:
 	                                           "_migrated/dam" is created and files
 	                                           are copied and indexed.
 	  dammigration:migratedamrecords           Migrates all DAM records to FAL. A
-	                                           DB field "_migrateddamuid" connects
-	                                           each FAL record to the original DAM
-	                                           record.
+                                               DB field "_migratedfaluid" connects
+                                               each original DAM record to the
+                                               matching FAL record.
 	  dammigration:migratedammetadata          Migrates DAM metadata to FAL
 	                                           metadata. Searches for all migrated
 	                                           sys_file records that do not have any

ext_tables.sql

@@ -1,11 +1,3 @@
-#
-# Table structure for table 'sys_file'
-#
-CREATE TABLE sys_file (
-	_migrateddamuid int(11) unsigned DEFAULT '0' NOT NULL,
-	KEY migratedRecords (_migrateddamuid)
-);
-
 #
 # Table structure for table 'sys_file_collection'
 #
@@ -26,6 +18,8 @@ CREATE TABLE sys_category (
 #
 CREATE TABLE tx_dam (
 	_missingfile tinyint(4) unsigned DEFAULT '0' NOT NULL,
+	_migratedfaluid int(11) unsigned DEFAULT '0' NOT NULL,
+	KEY migratedRecords (_migratedfaluid),
 	KEY deletedRecords (uid, deleted),
 	KEY missingFiles (_missingfile)
 );