diff --git a/csync/src/csync.c b/csync/src/csync.c index fe6704846c5..7d04708c466 100644 --- a/csync/src/csync.c +++ b/csync/src/csync.c @@ -742,6 +742,7 @@ void csync_file_stat_free(csync_file_stat_t *st) SAFE_FREE(st->directDownloadCookies); SAFE_FREE(st->etag); SAFE_FREE(st->destpath); + SAFE_FREE(st->checksum); SAFE_FREE(st); } } diff --git a/csync/src/csync.h b/csync/src/csync.h index e2ab1aea7ed..892d3e6ac8d 100644 --- a/csync/src/csync.h +++ b/csync/src/csync.h @@ -301,6 +301,12 @@ typedef void (*csync_vio_closedir_hook) (csync_vio_handle_t *dhhandle, typedef int (*csync_vio_stat_hook) (csync_vio_handle_t *dhhandle, void *userdata); +/* compute the checksum of the given \a checksumTypeId for \a path + * and return true if it's the same as \a checksum */ +typedef bool (*csync_checksum_hook) (const char *path, + uint32_t checksumTypeId, const char *checksum, + void *userdata); + /** * @brief Allocate a csync context. * diff --git a/csync/src/csync_private.h b/csync/src/csync_private.h index e8f7c4b93af..291482d13f6 100644 --- a/csync/src/csync_private.h +++ b/csync/src/csync_private.h @@ -96,6 +96,11 @@ struct csync_s { csync_vio_readdir_hook remote_readdir_hook; csync_vio_closedir_hook remote_closedir_hook; void *vio_userdata; + + /* hook for comparing checksums of files during discovery */ + csync_checksum_hook checksum_hook; + void *checksum_userdata; + } callbacks; c_strlist_t *excludes; @@ -192,6 +197,9 @@ struct csync_file_stat_s { char *directDownloadCookies; char remotePerm[REMOTE_PERM_BUF_SIZE+1]; + char *checksum; + uint32_t checksumTypeId; + CSYNC_STATUS error_status; enum csync_instructions_e instruction; /* u32 */ diff --git a/csync/src/csync_statedb.c b/csync/src/csync_statedb.c index 61fc89756f9..38aec0a280f 100644 --- a/csync/src/csync_statedb.c +++ b/csync/src/csync_statedb.c @@ -226,6 +226,8 @@ int csync_statedb_close(CSYNC *ctx) { return rc; } +#define METADATA_COLUMNS "phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId" + // This funciton parses a line from the metadata table into the given csync_file_stat // structure which it is also allocating. // Note that this function calls laso sqlite3_step to actually get the info from db and @@ -286,6 +288,11 @@ static int _csync_file_stat_from_metadata_table( csync_file_stat_t **st, sqlite3 if(column_count > 13) { (*st)->has_ignored_files = sqlite3_column_int(stmt, 13); } + if(column_count > 15 && sqlite3_column_int(stmt, 15)) { + (*st)->checksum = c_strdup( (char*) sqlite3_column_text(stmt, 14)); + (*st)->checksumTypeId = sqlite3_column_int(stmt, 15); + } + } } else { if( rc != SQLITE_DONE ) { @@ -307,7 +314,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx, } if( ctx->statedb.by_hash_stmt == NULL ) { - const char *hash_query = "SELECT * FROM metadata WHERE phash=?1"; + const char *hash_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE phash=?1"; SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, hash_query, strlen(hash_query), &ctx->statedb.by_hash_stmt, NULL)); ctx->statedb.lastReturnValue = rc; @@ -350,7 +357,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_file_id(CSYNC *ctx, } if( ctx->statedb.by_fileid_stmt == NULL ) { - const char *query = "SELECT * FROM metadata WHERE fileid=?1"; + const char *query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE fileid=?1"; SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, query, strlen(query), &ctx->statedb.by_fileid_stmt, NULL)); ctx->statedb.lastReturnValue = rc; @@ -390,7 +397,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_inode(CSYNC *ctx, } if( ctx->statedb.by_inode_stmt == NULL ) { - const char *inode_query = "SELECT * FROM metadata WHERE inode=?1"; + const char *inode_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE inode=?1"; SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, inode_query, strlen(inode_query), &ctx->statedb.by_inode_stmt, NULL)); ctx->statedb.lastReturnValue = rc; @@ -433,7 +440,7 @@ int csync_statedb_get_below_path( CSYNC *ctx, const char *path ) { * In other words, anything that is between path+'/' and path+'0', * (because '0' follows '/' in ascii) */ - const char *below_path_query = "SELECT phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote FROM metadata WHERE path > (?||'/') AND path < (?||'0')"; + const char *below_path_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE path > (?||'/') AND path < (?||'0')"; SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, below_path_query, -1, &stmt, NULL)); ctx->statedb.lastReturnValue = rc; if( rc != SQLITE_OK ) { diff --git a/csync/src/csync_update.c b/csync/src/csync_update.c index e7b215bbdda..28d7d0def27 100644 --- a/csync/src/csync_update.c +++ b/csync/src/csync_update.c @@ -270,25 +270,43 @@ static int _csync_detect_update(CSYNC *ctx, const char *file, ((int64_t) fs->mtime), ((int64_t) tmp->modtime), fs->etag, tmp->etag, (uint64_t) fs->inode, (uint64_t) tmp->inode, (uint64_t) fs->size, (uint64_t) tmp->size, fs->remotePerm, tmp->remotePerm, tmp->has_ignored_files ); - if((ctx->current == REMOTE_REPLICA && !c_streq(fs->etag, tmp->etag )) - || (ctx->current == LOCAL_REPLICA && (!_csync_mtime_equal(fs->mtime, tmp->modtime) - // zero size in statedb can happen during migration - || (tmp->size != 0 && fs->size != tmp->size) + if (ctx->current == REMOTE_REPLICA && !c_streq(fs->etag, tmp->etag)) { + st->instruction = CSYNC_INSTRUCTION_EVAL; + goto out; + } + if (ctx->current == LOCAL_REPLICA && + (!_csync_mtime_equal(fs->mtime, tmp->modtime) + // zero size in statedb can happen during migration + || (tmp->size != 0 && fs->size != tmp->size) #if 0 - || fs->inode != tmp->inode + /* Comparison of the local inode is disabled because people reported problems + * on windows with flacky inode values, see github bug #779 + * + * The inode needs to be observed because: + * $> echo a > a.txt ; echo b > b.txt + * both files have the same mtime + * sync them. + * $> rm a.txt && mv b.txt a.txt + * makes b.txt appearing as a.txt yet a sync is not performed because + * both have the same modtime as mv does not change that. + */ + || fs->inode != tmp->inode #endif - ))) { - /* Comparison of the local inode is disabled because people reported problems - * on windows with flacky inode values, see github bug #779 - * - * The inode needs to be observed because: - * $> echo a > a.txt ; echo b > b.txt - * both files have the same mtime - * sync them. - * $> rm a.txt && mv b.txt a.txt - * makes b.txt appearing as a.txt yet a sync is not performed because - * both have the same modtime as mv does not change that. - */ + )) { + + if (fs->size == tmp->size && tmp->checksumTypeId) { + bool checksumIdentical = false; + if (ctx->callbacks.checksum_hook) { + checksumIdentical = ctx->callbacks.checksum_hook( + file, tmp->checksumTypeId, tmp->checksum, + ctx->callbacks.checksum_userdata); + } + if (checksumIdentical) { + st->instruction = CSYNC_INSTRUCTION_NONE; + st->should_update_metadata = true; + goto out; + } + } st->instruction = CSYNC_INSTRUCTION_EVAL; goto out; } diff --git a/csync/tests/csync_tests/check_csync_update.c b/csync/tests/csync_tests/check_csync_update.c index ced053e4042..c3a7eaf01ff 100644 --- a/csync/tests/csync_tests/check_csync_update.c +++ b/csync/tests/csync_tests/check_csync_update.c @@ -41,6 +41,12 @@ static void statedb_create_metadata_table(sqlite3 *db) "modtime INTEGER(8)," "type INTEGER," "md5 VARCHAR(32)," + "fileid VARCHAR(128)," + "remotePerm VARCHAR(128)," + "filesize BIGINT," + "ignoredChildrenRemote INT," + "contentChecksum TEXT," + "contentChecksumTypeId INTEGER," "PRIMARY KEY(phash));"; rc = sqlite3_exec(db, sql, NULL, NULL, NULL); diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index a0fde8a6c5e..ccf2595a985 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -351,10 +351,7 @@ void PropagateDownloadFileQNAM::start() if (_resumeStart == _item->_size) { qDebug() << "File is already complete, no need to download"; _tmpFile.close(); - - // Unfortunately we lost the checksum header, if any... - QByteArray noChecksumData; - downloadFinished(noChecksumData, noChecksumData); + downloadFinished(); return; } } @@ -537,7 +534,7 @@ void PropagateDownloadFileQNAM::slotGetFinished() // as this is (still) also correct. ValidateChecksumHeader *validator = new ValidateChecksumHeader(this); connect(validator, SIGNAL(validated(QByteArray,QByteArray)), - SLOT(downloadFinished(QByteArray,QByteArray))); + SLOT(downloadFinished())); connect(validator, SIGNAL(validationFailed(QString)), SLOT(slotChecksumFail(QString))); auto checksumHeader = job->reply()->rawHeader(checkSumHeaderC); @@ -621,13 +618,8 @@ static void handleRecallFile(const QString &fn) } } // end namespace -void PropagateDownloadFileQNAM::downloadFinished(const QByteArray& checksumType, const QByteArray& checksum) +void PropagateDownloadFileQNAM::downloadFinished() { - if (!checksumType.isEmpty()) { - _item->_transmissionChecksum = checksum; - _item->_transmissionChecksumType = checksumType; - } - QString fn = _propagator->getFilePath(_item->_file); // In case of file name clash, report an error diff --git a/src/libsync/propagatedownload.h b/src/libsync/propagatedownload.h index 2a8877ed13b..32e05db28ae 100644 --- a/src/libsync/propagatedownload.h +++ b/src/libsync/propagatedownload.h @@ -117,7 +117,7 @@ class PropagateDownloadFileQNAM : public PropagateItemJob { private slots: void slotGetFinished(); void abort() Q_DECL_OVERRIDE; - void downloadFinished(const QByteArray& checksumType, const QByteArray& checksum); + void downloadFinished(); void slotDownloadProgress(qint64,qint64); void slotChecksumFail( const QString& errMsg ); diff --git a/src/libsync/propagateremotemove.cpp b/src/libsync/propagateremotemove.cpp index 33501aa49f4..f41fd13ef5c 100644 --- a/src/libsync/propagateremotemove.cpp +++ b/src/libsync/propagateremotemove.cpp @@ -158,8 +158,8 @@ void PropagateRemoteMove::finalize() SyncJournalFileRecord record(*_item, _propagator->getFilePath(_item->_renameTarget)); record._path = _item->_renameTarget; - record._transmissionChecksum = oldRecord._transmissionChecksum; - record._transmissionChecksumType = oldRecord._transmissionChecksumType; + record._contentChecksum = oldRecord._contentChecksum; + record._contentChecksumType = oldRecord._contentChecksumType; _propagator->_journal->setFileRecord(record); _propagator->_journal->commit("Remote Rename"); diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 2ad23ddba7b..859c499c880 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -216,24 +216,38 @@ void PropagateUploadFileQNAM::start() _stopWatch.start(); - auto supportedChecksumTypes = _propagator->account()->capabilities().supportedChecksumTypes(); - - // If we already have a checksum header and the checksum type is supported - // by the server, we keep that - otherwise recompute. - // - // Note: Currently we *always* recompute because we usually only upload - // files that have changed and thus have a new checksum. But if an earlier - // phase computed a checksum, this is where we would make use of it. - if (!_item->_transmissionChecksumType.isEmpty()) { - if (supportedChecksumTypes.contains(_item->_transmissionChecksumType)) { - // TODO: We could validate the old checksum and thereby determine whether - // an upload is necessary or not. - slotStartUpload(_item->_transmissionChecksumType, _item->_transmissionChecksum); - return; - } + // Compute the content checksum. + auto computeChecksum = new ComputeChecksum(this); + // We currently only do content checksums for the particular .eml case + // This should be done more generally in the future! + if (filePath.endsWith(QLatin1String(".eml"), Qt::CaseInsensitive)) { + computeChecksum->setChecksumType("MD5"); + } else { + computeChecksum->setChecksumType(QByteArray()); + } + + connect(computeChecksum, SIGNAL(done(QByteArray,QByteArray)), + SLOT(slotComputeTransmissionChecksum(QByteArray,QByteArray))); + computeChecksum->start(filePath); +} + +void PropagateUploadFileQNAM::slotComputeTransmissionChecksum(const QByteArray& contentChecksumType, const QByteArray& contentChecksum) +{ + _item->_contentChecksum = contentChecksum; + _item->_contentChecksumType = contentChecksumType; + + _stopWatch.addLapTime(QLatin1String("ContentChecksum")); + _stopWatch.start(); + + // Reuse the content checksum as the transmission checksum if possible + const auto supportedTransmissionChecksums = + _propagator->account()->capabilities().supportedChecksumTypes(); + if (supportedTransmissionChecksums.contains(contentChecksumType)) { + slotStartUpload(contentChecksumType, contentChecksum); + return; } - // Compute a new checksum. + // Compute the transmission checksum. auto computeChecksum = new ComputeChecksum(this); if (uploadChecksumEnabled()) { computeChecksum->setChecksumType(_propagator->account()->capabilities().preferredChecksumType()); @@ -243,19 +257,14 @@ void PropagateUploadFileQNAM::start() connect(computeChecksum, SIGNAL(done(QByteArray,QByteArray)), SLOT(slotStartUpload(QByteArray,QByteArray))); + const QString filePath = _propagator->getFilePath(_item->_file); computeChecksum->start(filePath); } -void PropagateUploadFileQNAM::slotStartUpload(const QByteArray& checksumType, const QByteArray& checksum) +void PropagateUploadFileQNAM::slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum) { - // Store the computed checksum in the database, if different - if (checksumType != _item->_transmissionChecksumType - || checksum != _item->_transmissionChecksum) { - _item->_transmissionChecksum = checksum; - _item->_transmissionChecksumType = checksumType; - _propagator->_journal->updateFileRecordChecksum( - _item->_file, checksum, checksumType); - } + _transmissionChecksum = transmissionChecksum; + _transmissionChecksumType = transmissionChecksumType; const QString fullFilePath = _propagator->getFilePath(_item->_file); @@ -263,7 +272,7 @@ void PropagateUploadFileQNAM::slotStartUpload(const QByteArray& checksumType, co done(SyncFileItem::SoftError, tr("File Removed")); return; } - _stopWatch.addLapTime(QLatin1String("Checksum")); + _stopWatch.addLapTime(QLatin1String("TransmissionChecksum")); time_t prevModtime = _item->_modtime; // the _item value was set in PropagateUploadFileQNAM::start() // but a potential checksum calculation could have taken some time during which the file could @@ -511,9 +520,9 @@ void PropagateUploadFileQNAM::startNextChunk() isFinalChunk = true; } - if (isFinalChunk && !_item->_transmissionChecksumType.isEmpty()) { + if (isFinalChunk && !_transmissionChecksumType.isEmpty()) { headers[checkSumHeaderC] = makeChecksumHeader( - _item->_transmissionChecksumType, _item->_transmissionChecksum); + _transmissionChecksumType, _transmissionChecksum); } if (! device->prepareAndOpen(_propagator->getFilePath(_item->_file), chunkStart, currentChunkSize)) { @@ -724,7 +733,10 @@ void PropagateUploadFileQNAM::slotPutFinished() // performance logging _item->_requestDuration = _stopWatch.stop(); - qDebug() << "*==* duration UPLOAD" << _item->_size << _stopWatch.durationOfLap(QLatin1String("Checksum")) << _item->_requestDuration; + qDebug() << "*==* duration UPLOAD" << _item->_size + << _stopWatch.durationOfLap(QLatin1String("ContentChecksum")) + << _stopWatch.durationOfLap(QLatin1String("TransmissionChecksum")) + << _item->_requestDuration; finalize(*_item); } diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index 9451e12b922..1b4bc74e6f0 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -183,6 +183,9 @@ class PropagateUploadFileQNAM : public PropagateItemJob { // measure the performance of checksum calc and upload Utility::StopWatch _stopWatch; + QByteArray _transmissionChecksum; + QByteArray _transmissionChecksumType; + public: PropagateUploadFileQNAM(OwncloudPropagator* propagator,const SyncFileItemPtr& item) : PropagateItemJob(propagator, item), _startChunk(0), _currentChunk(0), _chunkCount(0), _transferId(0), _finished(false) {} @@ -195,7 +198,8 @@ private slots: void startNextChunk(); void finalize(const SyncFileItem&); void slotJobDestroyed(QObject *job); - void slotStartUpload(const QByteArray& checksumType, const QByteArray& checksum); + void slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum); + void slotComputeTransmissionChecksum(const QByteArray& contentChecksumType, const QByteArray& contentChecksum); private: void startPollJob(const QString& path); diff --git a/src/libsync/propagatorjobs.cpp b/src/libsync/propagatorjobs.cpp index ddb17ef37bf..5b70b097e46 100644 --- a/src/libsync/propagatorjobs.cpp +++ b/src/libsync/propagatorjobs.cpp @@ -209,8 +209,8 @@ void PropagateLocalRename::start() SyncJournalFileRecord record(*_item, targetFile); record._path = _item->_renameTarget; - record._transmissionChecksum = oldRecord._transmissionChecksum; - record._transmissionChecksumType = oldRecord._transmissionChecksumType; + record._contentChecksum = oldRecord._contentChecksum; + record._contentChecksumType = oldRecord._contentChecksumType; if (!_item->_isDirectory) { // Directories are saved at the end _propagator->_journal->setFileRecord(record); diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 624c83bf273..d1e70cff8b6 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -71,6 +71,7 @@ SyncEngine::SyncEngine(AccountPtr account, CSYNC *ctx, const QString& localPath, , _uploadLimit(0) , _downloadLimit(0) , _newBigFolderSizeLimit(-1) + , _checksum_hook(journal) , _anotherSyncNeeded(false) { qRegisterMetaType("SyncFileItem"); @@ -466,10 +467,11 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote ) int re = 0; switch(file->instruction) { - case CSYNC_INSTRUCTION_NONE: + case CSYNC_INSTRUCTION_NONE: { if (remote && item->_should_update_metadata && !item->_isDirectory && item->_instruction == CSYNC_INSTRUCTION_NONE) { - // Update the database now already: New fileid or Etag or RemotePerm + // Update the database now already: New remote fileid or Etag or RemotePerm // Or for files that were detected as "resolved conflict". + // Or a local inode/mtime change (see localMetadataUpdate below) // In case of "resolved conflict": there should have been a conflict because they // both were new, or both had their local mtime or remote etag modified, but the @@ -496,21 +498,28 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote ) _journal->setFileRecordMetadata(SyncJournalFileRecord(*item, filePath)); item->_should_update_metadata = false; + + // Technically we're done with this item. See localMetadataUpdate hack below. + _syncItemMap.remove(key); + } + // Any files that are instruction NONE? + if (!item->_isDirectory && file->other.instruction == CSYNC_INSTRUCTION_NONE) { + _hasNoneFiles = true; } - if (item->_isDirectory && file->should_update_metadata) { - // Because we want to still update etags of directories - dir = SyncFileItem::None; - } else { - // No need to do anything. - if (file->other.instruction == CSYNC_INSTRUCTION_NONE - // Directories with ignored files does not count as 'None' - && (file->type != CSYNC_FTW_TYPE_DIR || !file->has_ignored_files)) { - _hasNoneFiles = true; + // We want to still update etags of directories, other NONE + // items can be ignored. + bool directoryEtagUpdate = item->_isDirectory && file->should_update_metadata; + bool localMetadataUpdate = !remote && file->should_update_metadata; + if (!directoryEtagUpdate) { + if (localMetadataUpdate) { + // Hack, we want a local metadata update to happen, but only if the + // remote tree doesn't ask us to do some kind of propagation. + _syncItemMap.insert(key, item); } - return re; } break; + } case CSYNC_INSTRUCTION_RENAME: dir = !remote ? SyncFileItem::Down : SyncFileItem::Up; item->_renameTarget = renameTarget; @@ -686,6 +695,10 @@ void SyncEngine::startSync() csync_set_userdata(_csync_ctx, this); + // Set up checksumming hook + _csync_ctx->callbacks.checksum_hook = &CSyncChecksumHook::hook; + _csync_ctx->callbacks.checksum_userdata = &_checksum_hook; + _stopWatch.start(); qDebug() << "#### Discovery start #################################################### >>"; diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h index fce1e1cf513..beedf84097c 100644 --- a/src/libsync/syncengine.h +++ b/src/libsync/syncengine.h @@ -37,6 +37,7 @@ #include "syncfilestatus.h" #include "accountfwd.h" #include "discoveryphase.h" +#include "transmissionchecksumvalidator.h" class QProcess; @@ -216,6 +217,9 @@ private slots: // hash containing the permissions on the remote directory QHash _remotePerms; + /// Hook for computing checksums from csync_update + CSyncChecksumHook _checksum_hook; + bool _anotherSyncNeeded; }; diff --git a/src/libsync/syncfileitem.h b/src/libsync/syncfileitem.h index b3901bd8285..1642ab27fae 100644 --- a/src/libsync/syncfileitem.h +++ b/src/libsync/syncfileitem.h @@ -165,8 +165,8 @@ class SyncFileItem { quint64 _inode; QByteArray _fileId; QByteArray _remotePerm; - QByteArray _transmissionChecksum; - QByteArray _transmissionChecksumType; + QByteArray _contentChecksum; + QByteArray _contentChecksumType; QString _directDownloadUrl; QString _directDownloadCookies; diff --git a/src/libsync/syncjournaldb.cpp b/src/libsync/syncjournaldb.cpp index eb52d67e0f7..1fa8c54c99c 100644 --- a/src/libsync/syncjournaldb.cpp +++ b/src/libsync/syncjournaldb.cpp @@ -204,10 +204,13 @@ bool SyncJournalDb::checkConnect() "modtime INTEGER(8)," "type INTEGER," "md5 VARCHAR(32)," /* This is the etag. Called md5 for compatibility */ - // updateDatabaseStructure() will add a fileid column - // updateDatabaseStructure() will add a remotePerm column - // updateDatabaseStructure() will add a transmissionChecksum column - // updateDatabaseStructure() will add a transmissionChecksumTypeId column + // updateDatabaseStructure() will add + // fileid + // remotePerm + // filesize + // ignoredChildrenRemote + // contentChecksum + // contentChecksumTypeId "PRIMARY KEY(phash)" ");"); @@ -358,20 +361,20 @@ bool SyncJournalDb::checkConnect() _getFileRecordQuery.reset(new SqlQuery(_db)); _getFileRecordQuery->prepare( "SELECT path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize," - " ignoredChildrenRemote, transmissionChecksum, checksumtype.name" + " ignoredChildrenRemote, contentChecksum, contentchecksumtype.name" " FROM metadata" - " LEFT JOIN checksumtype ON metadata.transmissionChecksumTypeId == checksumtype.id" + " LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id" " WHERE phash=?1" ); _setFileRecordQuery.reset(new SqlQuery(_db) ); _setFileRecordQuery->prepare("INSERT OR REPLACE INTO metadata " - "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, transmissionChecksum, transmissionChecksumTypeId) " + "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId) " "VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7, ?8 , ?9 , ?10, ?11, ?12, ?13, ?14, ?15, ?16);" ); _setFileRecordChecksumQuery.reset(new SqlQuery(_db) ); _setFileRecordChecksumQuery->prepare( "UPDATE metadata" - " SET transmissionChecksum = ?2, transmissionChecksumTypeId = ?3" + " SET contentChecksum = ?2, contentChecksumTypeId = ?3" " WHERE phash == ?1;"); _getDownloadInfoQuery.reset(new SqlQuery(_db) ); @@ -426,6 +429,9 @@ bool SyncJournalDb::checkConnect() _getChecksumTypeIdQuery.reset(new SqlQuery(_db)); _getChecksumTypeIdQuery->prepare("SELECT id FROM checksumtype WHERE name=?1"); + _getChecksumTypeQuery.reset(new SqlQuery(_db)); + _getChecksumTypeQuery->prepare("SELECT name FROM checksumtype WHERE id=?1"); + _insertChecksumTypeQuery.reset(new SqlQuery(_db)); _insertChecksumTypeQuery->prepare("INSERT OR IGNORE INTO checksumtype (name) VALUES (?1)"); @@ -463,6 +469,7 @@ void SyncJournalDb::close() _setErrorBlacklistQuery.reset(0); _getSelectiveSyncListQuery.reset(0); _getChecksumTypeIdQuery.reset(0); + _getChecksumTypeQuery.reset(0); _insertChecksumTypeQuery.reset(0); _db.close(); @@ -557,23 +564,23 @@ bool SyncJournalDb::updateMetadataTableStructure() commitInternal("update database structure: add ignoredChildrenRemote col"); } - if( columns.indexOf(QLatin1String("transmissionChecksum")) == -1 ) { + if( columns.indexOf(QLatin1String("contentChecksum")) == -1 ) { SqlQuery query(_db); - query.prepare("ALTER TABLE metadata ADD COLUMN transmissionChecksum TEXT;"); + query.prepare("ALTER TABLE metadata ADD COLUMN contentChecksum TEXT;"); if( !query.exec()) { - sqlFail("updateMetadataTableStructure: add transmissionChecksum column", query); + sqlFail("updateMetadataTableStructure: add contentChecksum column", query); re = false; } - commitInternal("update database structure: add transmissionChecksum col"); + commitInternal("update database structure: add contentChecksum col"); } - if( columns.indexOf(QLatin1String("transmissionChecksumTypeId")) == -1 ) { + if( columns.indexOf(QLatin1String("contentChecksumTypeId")) == -1 ) { SqlQuery query(_db); - query.prepare("ALTER TABLE metadata ADD COLUMN transmissionChecksumTypeId INTEGER;"); + query.prepare("ALTER TABLE metadata ADD COLUMN contentChecksumTypeId INTEGER;"); if( !query.exec()) { - sqlFail("updateMetadataTableStructure: add transmissionChecksumTypeId column", query); + sqlFail("updateMetadataTableStructure: add contentChecksumTypeId column", query); re = false; } - commitInternal("update database structure: add transmissionChecksumTypeId col"); + commitInternal("update database structure: add contentChecksumTypeId col"); } @@ -677,7 +684,7 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record ) if( fileId.isEmpty() ) fileId = ""; QString remotePerm (record._remotePerm); if (remotePerm.isEmpty()) remotePerm = QString(); // have NULL in DB (vs empty) - int checksumTypeId = mapChecksumType(record._transmissionChecksumType); + int contentChecksumTypeId = mapChecksumType(record._contentChecksumType); _setFileRecordQuery->reset(); _setFileRecordQuery->bindValue(1, QString::number(phash)); _setFileRecordQuery->bindValue(2, plen); @@ -693,8 +700,8 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record ) _setFileRecordQuery->bindValue(12, remotePerm ); _setFileRecordQuery->bindValue(13, record._fileSize ); _setFileRecordQuery->bindValue(14, record._serverHasIgnoredFiles ? 1:0); - _setFileRecordQuery->bindValue(15, record._transmissionChecksum ); - _setFileRecordQuery->bindValue(16, checksumTypeId ); + _setFileRecordQuery->bindValue(15, record._contentChecksum ); + _setFileRecordQuery->bindValue(16, contentChecksumTypeId ); if( !_setFileRecordQuery->exec() ) { qWarning() << "Error SQL statement setFileRecord: " << _setFileRecordQuery->lastQuery() << " :" @@ -705,7 +712,7 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record ) qDebug() << _setFileRecordQuery->lastQuery() << phash << plen << record._path << record._inode << QString::number(Utility::qDateTimeToTime_t(record._modtime)) << QString::number(record._type) << record._etag << record._fileId << record._remotePerm << record._fileSize << (record._serverHasIgnoredFiles ? 1:0) - << record._transmissionChecksum << record._transmissionChecksumType << checksumTypeId; + << record._contentChecksum << record._contentChecksumType << contentChecksumTypeId; _setFileRecordQuery->reset(); return true; @@ -785,9 +792,9 @@ SyncJournalFileRecord SyncJournalDb::getFileRecord( const QString& filename ) rec._remotePerm = _getFileRecordQuery->baValue(9); rec._fileSize = _getFileRecordQuery->int64Value(10); rec._serverHasIgnoredFiles = (_getFileRecordQuery->intValue(11) > 0); - rec._transmissionChecksum = _getFileRecordQuery->baValue(12); + rec._contentChecksum = _getFileRecordQuery->baValue(12); if( !_getFileRecordQuery->nullValue(13) ) { - rec._transmissionChecksumType = _getFileRecordQuery->baValue(13); + rec._contentChecksumType = _getFileRecordQuery->baValue(13); } } else { QString err = _getFileRecordQuery->error(); @@ -878,8 +885,8 @@ int SyncJournalDb::getFileRecordCount() } bool SyncJournalDb::updateFileRecordChecksum(const QString& filename, - const QByteArray& transmisisonChecksum, - const QByteArray& transmissionChecksumType) + const QByteArray& contentChecksum, + const QByteArray& contentChecksumType) { QMutexLocker locker(&_mutex); @@ -889,12 +896,12 @@ bool SyncJournalDb::updateFileRecordChecksum(const QString& filename, return false; } - int checksumTypeId = mapChecksumType(transmissionChecksumType); + int checksumTypeId = mapChecksumType(contentChecksumType); auto & query = _setFileRecordChecksumQuery; query->reset(); query->bindValue(1, QString::number(phash)); - query->bindValue(2, transmisisonChecksum); + query->bindValue(2, contentChecksum); query->bindValue(3, checksumTypeId); if( !query->exec() ) { @@ -904,8 +911,8 @@ bool SyncJournalDb::updateFileRecordChecksum(const QString& filename, return false; } - qDebug() << query->lastQuery() << phash << transmisisonChecksum - << transmissionChecksumType << checksumTypeId; + qDebug() << query->lastQuery() << phash << contentChecksum + << contentChecksumType << checksumTypeId; query->reset(); return true; @@ -1501,6 +1508,32 @@ void SyncJournalDb::forceRemoteDiscoveryNextSyncLocked() } } + +QByteArray SyncJournalDb::getChecksumType(int checksumTypeId) +{ + QMutexLocker locker(&_mutex); + if( !checkConnect() ) { + return QByteArray(); + } + + // Retrieve the id + auto & query = *_getChecksumTypeQuery; + query.reset(); + query.bindValue(1, checksumTypeId); + if( !query.exec() ) { + qWarning() << "Error SQL statement getChecksumType: " + << query.lastQuery() << " :" + << query.error(); + return 0; + } + + if( !query.next() ) { + qDebug() << "No checksum type mapping found for" << checksumTypeId; + return 0; + } + return query.baValue(0); +} + int SyncJournalDb::mapChecksumType(const QByteArray& checksumType) { if (checksumType.isEmpty()) { diff --git a/src/libsync/syncjournaldb.h b/src/libsync/syncjournaldb.h index b5521ff38a9..a56e41b6091 100644 --- a/src/libsync/syncjournaldb.h +++ b/src/libsync/syncjournaldb.h @@ -47,8 +47,8 @@ class OWNCLOUDSYNC_EXPORT SyncJournalDb : public QObject bool deleteFileRecord( const QString& filename, bool recursively = false ); int getFileRecordCount(); bool updateFileRecordChecksum(const QString& filename, - const QByteArray& transmisisonChecksum, - const QByteArray& transmissionChecksumType); + const QByteArray& contentChecksum, + const QByteArray& contentChecksumType); bool exists(); void walCheckpoint(); @@ -146,6 +146,11 @@ class OWNCLOUDSYNC_EXPORT SyncJournalDb : public QObject */ bool isConnected(); + /** + * Returns the checksum type for an id. + */ + QByteArray getChecksumType(int checksumTypeId); + private: bool updateDatabaseStructure(); bool updateMetadataTableStructure(); @@ -186,6 +191,7 @@ class OWNCLOUDSYNC_EXPORT SyncJournalDb : public QObject QScopedPointer _setErrorBlacklistQuery; QScopedPointer _getSelectiveSyncListQuery; QScopedPointer _getChecksumTypeIdQuery; + QScopedPointer _getChecksumTypeQuery; QScopedPointer _insertChecksumTypeQuery; /* This is the list of paths we called avoidReadFromDbOnNextSync on. diff --git a/src/libsync/syncjournalfilerecord.cpp b/src/libsync/syncjournalfilerecord.cpp index 57081c1e13a..c0bbd6335bf 100644 --- a/src/libsync/syncjournalfilerecord.cpp +++ b/src/libsync/syncjournalfilerecord.cpp @@ -36,8 +36,8 @@ SyncJournalFileRecord::SyncJournalFileRecord(const SyncFileItem &item, const QSt : _path(item._file), _modtime(Utility::qDateTimeFromTime_t(item._modtime)), _type(item._type), _etag(item._etag), _fileId(item._fileId), _fileSize(item._size), _remotePerm(item._remotePerm), _serverHasIgnoredFiles(item._serverHasIgnoredFiles), - _transmissionChecksum(item._transmissionChecksum), - _transmissionChecksumType(item._transmissionChecksumType) + _contentChecksum(item._contentChecksum), + _contentChecksumType(item._contentChecksumType) { // use the "old" inode coming with the item for the case where the // filesystem stat fails. That can happen if the the file was removed @@ -97,8 +97,8 @@ SyncFileItem SyncJournalFileRecord::toSyncFileItem() item._size = _fileSize; item._remotePerm = _remotePerm; item._serverHasIgnoredFiles = _serverHasIgnoredFiles; - item._transmissionChecksum = _transmissionChecksum; - item._transmissionChecksumType = _transmissionChecksumType; + item._contentChecksum = _contentChecksum; + item._contentChecksumType = _contentChecksumType; return item; } @@ -176,8 +176,8 @@ bool operator==(const SyncJournalFileRecord & lhs, && lhs._fileSize == rhs._fileSize && lhs._remotePerm == rhs._remotePerm && lhs._serverHasIgnoredFiles == rhs._serverHasIgnoredFiles - && lhs._transmissionChecksum == rhs._transmissionChecksum - && lhs._transmissionChecksumType == rhs._transmissionChecksumType; + && lhs._contentChecksum == rhs._contentChecksum + && lhs._contentChecksumType == rhs._contentChecksumType; } } diff --git a/src/libsync/syncjournalfilerecord.h b/src/libsync/syncjournalfilerecord.h index 251e96ba5fc..ec2ec9c1664 100644 --- a/src/libsync/syncjournalfilerecord.h +++ b/src/libsync/syncjournalfilerecord.h @@ -55,8 +55,8 @@ class OWNCLOUDSYNC_EXPORT SyncJournalFileRecord qint64 _fileSize; QByteArray _remotePerm; bool _serverHasIgnoredFiles; - QByteArray _transmissionChecksum; - QByteArray _transmissionChecksumType; + QByteArray _contentChecksum; + QByteArray _contentChecksumType; }; bool OWNCLOUDSYNC_EXPORT diff --git a/src/libsync/transmissionchecksumvalidator.cpp b/src/libsync/transmissionchecksumvalidator.cpp index 15923bd2dde..9edd70d0567 100644 --- a/src/libsync/transmissionchecksumvalidator.cpp +++ b/src/libsync/transmissionchecksumvalidator.cpp @@ -77,36 +77,40 @@ QByteArray ComputeChecksum::checksumType() const void ComputeChecksum::start(const QString& filePath) { - const QString csType = checksumType(); - // Calculate the checksum in a different thread first. connect( &_watcher, SIGNAL(finished()), this, SLOT(slotCalculationDone()), Qt::UniqueConnection ); - if( csType == checkSumMD5C ) { - _watcher.setFuture(QtConcurrent::run(FileSystem::calcMd5, filePath)); + _watcher.setFuture(QtConcurrent::run(ComputeChecksum::computeNow, filePath, checksumType())); +} - } else if( csType == checkSumSHA1C ) { - _watcher.setFuture(QtConcurrent::run( FileSystem::calcSha1, filePath)); +QByteArray ComputeChecksum::computeNow(const QString& filePath, const QByteArray& checksumType) +{ + if( checksumType == checkSumMD5C ) { + return FileSystem::calcMd5(filePath); + } else if( checksumType == checkSumSHA1C ) { + return FileSystem::calcSha1(filePath); } #ifdef ZLIB_FOUND - else if( csType == checkSumAdlerC) { - _watcher.setFuture(QtConcurrent::run(FileSystem::calcAdler32, filePath)); + else if( checksumType == checkSumAdlerC) { + return FileSystem::calcAdler32(filePath); } #endif - else { - // for an unknown checksum or no checksum, we're done right now - if( !csType.isEmpty() ) { - qDebug() << "Unknown checksum type:" << csType; - } - emit done(QByteArray(), QByteArray()); + // for an unknown checksum or no checksum, we're done right now + if( !checksumType.isEmpty() ) { + qDebug() << "Unknown checksum type:" << checksumType; } + return QByteArray(); } void ComputeChecksum::slotCalculationDone() { QByteArray checksum = _watcher.future().result(); - emit done(_checksumType, checksum); + if (!checksum.isNull()) { + emit done(_checksumType, checksum); + } else { + emit done(QByteArray(), QByteArray()); + } } @@ -151,4 +155,32 @@ void ValidateChecksumHeader::slotChecksumCalculated(const QByteArray& checksumTy emit validated(checksumType, checksum); } +CSyncChecksumHook::CSyncChecksumHook(SyncJournalDb *journal) + : _journal(journal) +{ +} + +bool CSyncChecksumHook::hook(const char* path, uint32_t checksumTypeId, const char* checksum, void *this_obj) +{ + CSyncChecksumHook* checksumHook = static_cast(this_obj); + return checksumHook->check(QString::fromUtf8(path), checksumTypeId, QByteArray(checksum)); +} + +bool CSyncChecksumHook::check(const QString& path, int checksumTypeId, const QByteArray& checksum) +{ + QByteArray checksumType = _journal->getChecksumType(checksumTypeId); + if (checksumType.isEmpty()) { + qDebug() << "Checksum type" << checksumTypeId << "not found"; + return false; + } + + QByteArray newChecksum = ComputeChecksum::computeNow(path, checksumType); + if (newChecksum.isNull()) { + qDebug() << "Failed to compute checksum" << checksumType << "for" << path; + return false; + } + return newChecksum == checksum; +} + + } diff --git a/src/libsync/transmissionchecksumvalidator.h b/src/libsync/transmissionchecksumvalidator.h index 3a37c6948ca..bee78cd69ab 100644 --- a/src/libsync/transmissionchecksumvalidator.h +++ b/src/libsync/transmissionchecksumvalidator.h @@ -23,6 +23,8 @@ namespace OCC { +class SyncJournalDb; + /// Creates a checksum header from type and value. QByteArray makeChecksumHeader(const QByteArray& checksumType, const QByteArray& checksum); @@ -59,6 +61,11 @@ class OWNCLOUDSYNC_EXPORT ComputeChecksum : public QObject */ void start(const QString& filePath); + /** + * Computes the checksum synchronously. + */ + static QByteArray computeNow(const QString& filePath, const QByteArray& checksumType); + signals: void done(const QByteArray& checksumType, const QByteArray& checksum); @@ -103,4 +110,29 @@ private slots: QByteArray _expectedChecksum; }; +/** + * Hooks checksum computations into csync. + * @ingroup libsync + */ +class OWNCLOUDSYNC_EXPORT CSyncChecksumHook : public QObject +{ + Q_OBJECT +public: + explicit CSyncChecksumHook(SyncJournalDb* journal); + + /** + * Returns true if the checksum for \a path is the same as the one provided. + * + * Called from csync, where a instance of CSyncChecksumHook + * has to be set as userdata. + */ + static bool hook(const char* path, uint32_t checksumTypeId, const char* checksum, + void* this_obj); + + bool check(const QString& path, int checksumTypeId, const QByteArray& checksum); + +private: + SyncJournalDb* _journal; +}; + } diff --git a/test/test_journal.db b/test/test_journal.db index 30905b66c97..2b58c07cb86 100644 Binary files a/test/test_journal.db and b/test/test_journal.db differ diff --git a/test/testsyncjournaldb.h b/test/testsyncjournaldb.h index 758180ec87a..1f9ee6d17d9 100644 --- a/test/testsyncjournaldb.h +++ b/test/testsyncjournaldb.h @@ -59,17 +59,17 @@ private slots: record._fileId = "abcd"; record._remotePerm = "744"; record._fileSize = 213089055; - record._transmissionChecksum = "mychecksum"; - record._transmissionChecksumType = "MD5"; + record._contentChecksum = "mychecksum"; + record._contentChecksumType = "MD5"; QVERIFY(_db.setFileRecord(record)); SyncJournalFileRecord storedRecord = _db.getFileRecord("foo"); QVERIFY(storedRecord == record); // Update checksum - record._transmissionChecksum = "newchecksum"; - record._transmissionChecksumType = "Adler32"; - _db.updateFileRecordChecksum("foo", record._transmissionChecksum, record._transmissionChecksumType); + record._contentChecksum = "newchecksum"; + record._contentChecksumType = "Adler32"; + _db.updateFileRecordChecksum("foo", record._contentChecksum, record._contentChecksumType); storedRecord = _db.getFileRecord("foo"); QVERIFY(storedRecord == record); @@ -97,8 +97,8 @@ private slots: SyncJournalFileRecord record; record._path = "foo-checksum"; record._remotePerm = "744"; - record._transmissionChecksum = "mychecksum"; - record._transmissionChecksumType = "MD5"; + record._contentChecksum = "mychecksum"; + record._contentChecksumType = "MD5"; QVERIFY(_db.setFileRecord(record)); SyncJournalFileRecord storedRecord = _db.getFileRecord("foo-checksum");