Upgrade Checksum algorithms

src/common/checksums.cpp

@@ -21,6 +21,11 @@
 
 #include <QLoggingCategory>
 #include <qtconcurrentrun.h>
+#include <QCryptographicHash>
+
+#ifdef ZLIB_FOUND
+#include <zlib.h>
+#endif
 
 /** \file checksums.cpp
  *
@@ -73,13 +78,62 @@
  * - Adler32 (requires zlib)
  * - MD5
  * - SHA1
+ * - SHA256
+ * - SHA3-256 (requires Qt 5.9)
  *
  */
 
 namespace OCC {
 
 Q_LOGGING_CATEGORY(lcChecksums, "sync.checksums", QtInfoMsg)
 
+#define BUFSIZE qint64(500 * 1024) // 500 KiB
+
+static QByteArray calcCryptoHash( const QString& filename, QCryptographicHash::Algorithm algo )
+{
+     QFile file(filename);
+     QByteArray arr;
+     QCryptographicHash crypto( algo );
+
+     if (file.open(QIODevice::ReadOnly)) {
+         if (crypto.addData(&file)) {
+             arr = crypto.result().toHex();
+         }
+     }
+     return arr;
+ }
+
+QByteArray calcMd5(const QString &filename)
+{
+    return calcCryptoHash(filename, QCryptographicHash::Md5);
+}
+
+QByteArray calcSha1(const QString &filename)
+{
+    return calcCryptoHash(filename, QCryptographicHash::Sha1);
+}
+
+#ifdef ZLIB_FOUND
+QByteArray calcAdler32(const QString &filename)
+{
+    QFile file(filename);
+    const qint64 bufSize = qMin(BUFSIZE, file.size() + 1);
+    QByteArray buf(bufSize, Qt::Uninitialized);
+
+    unsigned int adler = adler32(0L, Z_NULL, 0);
+    if (file.open(QIODevice::ReadOnly)) {
+        qint64 size;
+        while (!file.atEnd()) {
+            size = file.read(buf.data(), bufSize);
+            if (size > 0)
+                adler = adler32(adler, (const Bytef *)buf.data(), size);
+        }
+    }
+
+    return QByteArray::number(adler, 16);
+}
+#endif
+
 QByteArray makeChecksumHeader(const QByteArray &checksumType, const QByteArray &checksum)
 {
     if (checksumType.isEmpty() || checksum.isEmpty())
@@ -94,7 +148,9 @@ QByteArray findBestChecksum(const QByteArray &checksums)
 {
     int i = 0;
     // The order of the searches here defines the preference ordering.
-    if (-1 != (i = checksums.indexOf("SHA1:"))
+    if (-1 != (i = checksums.indexOf("SHA3-256:"))
+        || -1 != (i = checksums.indexOf("SHA256:"))
+        || -1 != (i = checksums.indexOf("SHA1:"))
         || -1 != (i = checksums.indexOf("MD5:"))
         || -1 != (i = checksums.indexOf("Adler32:"))) {
         // Now i is the start of the best checksum
@@ -188,13 +244,20 @@ QByteArray ComputeChecksum::computeNow(const QString &filePath, const QByteArray
     }
 
     if (checksumType == checkSumMD5C) {
-        return FileSystem::calcMd5(filePath);
+        return calcMd5(filePath);
     } else if (checksumType == checkSumSHA1C) {
-        return FileSystem::calcSha1(filePath);
+        return calcSha1(filePath);
+    } else if (checksumType == checkSumSHA2C) {
+        return calcCryptoHash(filePath, QCryptographicHash::Sha256);
     }
+#if QT_VERSION >= QT_VERSION_CHECK(5, 9, 0)
+    else if (checksumType == checkSumSHA3C) {
+        return calcCryptoHash(filePath, QCryptographicHash::Sha3_256);
+    }
+#endif
 #ifdef ZLIB_FOUND
     else if (checksumType == checkSumAdlerC) {
-        return FileSystem::calcAdler32(filePath);
+        return calcAdler32(filePath);
     }
 #endif
     // for an unknown checksum or no checksum, we're done right now

src/common/checksums.h

@@ -19,6 +19,7 @@
 #pragma once
 
 #include "ocsynclib.h"
+#include "config.h"
 
 #include <QObject>
 #include <QByteArray>
@@ -32,6 +33,8 @@ namespace OCC {
  */
 static const char checkSumMD5C[] = "MD5";
 static const char checkSumSHA1C[] = "SHA1";
+static const char checkSumSHA2C[] = "SHA256";
+static const char checkSumSHA3C[] = "SHA3-256";
 static const char checkSumAdlerC[] = "Adler32";
 
 class SyncJournalDb;
@@ -61,6 +64,12 @@ OCSYNC_EXPORT bool uploadChecksumEnabled();
 /// Checks OWNCLOUD_CONTENT_CHECKSUM_TYPE (default: SHA1)
 OCSYNC_EXPORT QByteArray contentChecksumType();
 
+// Exported functions for the tests.
+QByteArray OCSYNC_EXPORT calcMd5(const QString &fileName);
+QByteArray OCSYNC_EXPORT calcSha1(const QString &fileName);
+#ifdef ZLIB_FOUND
+QByteArray OCSYNC_EXPORT calcAdler32(const QString &fileName);
+#endif
 
 /**
  * Computes the checksum of a file.

src/common/filesystembase.cpp

@@ -22,16 +22,11 @@
 #include <QDir>
 #include <QUrl>
 #include <QFile>
-#include <QCryptographicHash>
 #include <QCoreApplication>
 
 #include <sys/stat.h>
 #include <sys/types.h>
 
-#ifdef ZLIB_FOUND
-#include <zlib.h>
-#endif
-
 #ifdef Q_OS_WIN
 #include <windows.h>
 #include <windef.h>
@@ -359,53 +354,6 @@ QString FileSystem::fileSystemForPath(const QString &path)
 }
 #endif
 
-#define BUFSIZE qint64(500 * 1024) // 500 KiB
-
-static QByteArray readToCrypto( const QString& filename, QCryptographicHash::Algorithm algo )
- {
-     QFile file(filename);
-     QByteArray arr;
-     QCryptographicHash crypto( algo );
-
-     if (file.open(QIODevice::ReadOnly)) {
-         if (crypto.addData(&file)) {
-             arr = crypto.result().toHex();
-         }
-     }
-     return arr;
- }
-
-QByteArray FileSystem::calcMd5(const QString &filename)
-{
-    return readToCrypto(filename, QCryptographicHash::Md5);
-}
-
-QByteArray FileSystem::calcSha1(const QString &filename)
-{
-    return readToCrypto(filename, QCryptographicHash::Sha1);
-}
-
-#ifdef ZLIB_FOUND
-QByteArray FileSystem::calcAdler32(const QString &filename)
-{
-    QFile file(filename);
-    const qint64 bufSize = qMin(BUFSIZE, file.size() + 1);
-    QByteArray buf(bufSize, Qt::Uninitialized);
-
-    unsigned int adler = adler32(0L, Z_NULL, 0);
-    if (file.open(QIODevice::ReadOnly)) {
-        qint64 size;
-        while (!file.atEnd()) {
-            size = file.read(buf.data(), bufSize);
-            if (size > 0)
-                adler = adler32(adler, (const Bytef *)buf.data(), size);
-        }
-    }
-
-    return QByteArray::number(adler, 16);
-}
-#endif
-
 bool FileSystem::remove(const QString &fileName, QString *errorString)
 {
 #ifdef Q_OS_WIN

src/common/filesystembase.h

@@ -130,12 +130,6 @@ namespace FileSystem {
     QString fileSystemForPath(const QString &path);
 #endif
 
-    QByteArray OCSYNC_EXPORT calcMd5(const QString &fileName);
-    QByteArray OCSYNC_EXPORT calcSha1(const QString &fileName);
-#ifdef ZLIB_FOUND
-    QByteArray OCSYNC_EXPORT calcAdler32(const QString &fileName);
-#endif
-
     /**
      * Returns true when a file is locked. (Windows only)
      */

src/csync/csync_util.cpp

@@ -194,6 +194,6 @@ time_t oc_httpdate_parse( const char *date ) {
 
 bool csync_is_collision_safe_hash(const QByteArray &checksum_header)
 {
-    return checksum_header.startsWith("SHA1:")
+    return checksum_header.startsWith("SHA")
         || checksum_header.startsWith("MD5:");
 }

test/CMakeLists.txt

@@ -39,7 +39,6 @@ owncloud_add_test(ChecksumValidator "")
 
 owncloud_add_test(ExcludedFiles "")
 
-owncloud_add_test(FileSystem "")
 owncloud_add_test(Utility "")
 owncloud_add_test(SyncEngine "syncenginetestutils.h")
 owncloud_add_test(SyncVirtualFiles "syncenginetestutils.h")

test/testchecksumvalidator.cpp

@@ -15,15 +15,14 @@
 #include "filesystem.h"
 #include "propagatorjobs.h"
 
-
 using namespace OCC;
+using namespace OCC::Utility;
 
     class TestChecksumValidator : public QObject
     {
         Q_OBJECT
-
     private:
-        QString _root;
+        QTemporaryDir _root;
         QString _testfile;
         QString _expectedError;
         QByteArray     _expected;
@@ -48,17 +47,62 @@ using namespace OCC;
          _errorSeen = true;
     }
 
+    static QByteArray shellSum( const QByteArray& cmd, const QString& file )
+    {
+        QProcess md5;
+        QStringList args;
+        args.append(file);
+        md5.start(cmd, args);
+        QByteArray sumShell;
+        qDebug() << "File: "<< file;
+
+        if( md5.waitForFinished()  ) {
+
+            sumShell = md5.readAll();
+            sumShell = sumShell.left( sumShell.indexOf(' '));
+        }
+        return sumShell;
+    }
+
     private slots:
 
     void initTestCase() {
-        _root = QDir::tempPath() + "/" + "test_" + QString::number(qrand());
-        QDir rootDir(_root);
-
-        rootDir.mkpath(_root );
-        _testfile = _root+"/csFile";
+        _testfile = _root.path()+"/csFile";
         Utility::writeRandomFile( _testfile);
     }
 
+    void testMd5Calc()
+    {
+        QString file( _root.path() + "/file_a.bin");
+        QVERIFY(writeRandomFile(file));
+        QFileInfo fi(file);
+        QVERIFY(fi.exists());
+        QByteArray sum = calcMd5(file);
+
+        QByteArray sSum = shellSum("md5sum", file);
+        if (sSum.isEmpty())
+            QSKIP("Couldn't execute md5sum to calculate checksum, executable missing?", SkipSingle);
+
+        QVERIFY(!sum.isEmpty());
+        QCOMPARE(sSum, sum);
+    }
+
+    void testSha1Calc()
+    {
+        QString file( _root.path() + "/file_b.bin");
+        writeRandomFile(file);
+        QFileInfo fi(file);
+        QVERIFY(fi.exists());
+        QByteArray sum = calcSha1(file);
+
+        QByteArray sSum = shellSum("sha1sum", file);
+        if (sSum.isEmpty())
+            QSKIP("Couldn't execute sha1sum to calculate checksum, executable missing?", SkipSingle);
+
+        QVERIFY(!sum.isEmpty());
+        QCOMPARE(sSum, sum);
+    }
+
     void testUploadChecksummingAdler() {
 #ifndef ZLIB_FOUND
         QSKIP("ZLIB not found.", SkipSingle);
@@ -69,7 +113,7 @@ using namespace OCC;
 
         connect(vali, SIGNAL(done(QByteArray,QByteArray)), SLOT(slotUpValidated(QByteArray,QByteArray)));
 
-        _expected = FileSystem::calcAdler32( _testfile );
+        _expected = calcAdler32( _testfile );
         qDebug() << "XX Expected Checksum: " << _expected;
         vali->start(_testfile);
 
@@ -88,7 +132,7 @@ using namespace OCC;
         vali->setChecksumType(_expectedType);
         connect(vali, SIGNAL(done(QByteArray,QByteArray)), this, SLOT(slotUpValidated(QByteArray,QByteArray)));
 
-        _expected = FileSystem::calcMd5( _testfile );
+        _expected = calcMd5( _testfile );
         vali->start(_testfile);
 
         QEventLoop loop;
@@ -105,7 +149,7 @@ using namespace OCC;
         vali->setChecksumType(_expectedType);
         connect(vali, SIGNAL(done(QByteArray,QByteArray)), this, SLOT(slotUpValidated(QByteArray,QByteArray)));
 
-        _expected = FileSystem::calcSha1( _testfile );
+        _expected = calcSha1( _testfile );
 
         vali->start(_testfile);
 
@@ -122,7 +166,7 @@ using namespace OCC;
 #else
         QByteArray adler =  checkSumAdlerC;
         adler.append(":");
-        adler.append(FileSystem::calcAdler32( _testfile ));
+        adler.append(calcAdler32( _testfile ));
         _successDown = false;
 
         ValidateChecksumHeader *vali = new ValidateChecksumHeader(this);