From 8723d0ec52ae83d6460863baf56769ddf5cb034a Mon Sep 17 00:00:00 2001 From: Patrick Servello Date: Sun, 29 Dec 2019 22:39:52 -0600 Subject: [PATCH 1/6] Added SQLite header parsing functionality and associated tests --- include/SQLiteCpp/Database.h | 43 ++++++++++- src/Database.cpp | 133 +++++++++++++++++++++++++++++++++++ tests/Database_test.cpp | 45 +++++++++++- 3 files changed, 219 insertions(+), 2 deletions(-) diff --git a/include/SQLiteCpp/Database.h b/include/SQLiteCpp/Database.h index 1584a6e5..70c35525 100644 --- a/include/SQLiteCpp/Database.h +++ b/include/SQLiteCpp/Database.h @@ -12,7 +12,7 @@ #include #include // definition of nullptr for C++98/C++03 compilers - +#include #include // Forward declarations to avoid inclusion of in a header @@ -53,6 +53,32 @@ const char* getLibVersion() noexcept; // nothrow /// Return SQLite version number using runtime call to the compiled library int getLibVersionNumber() noexcept; // nothrow +// Public structure for representing all fields contained within the SQLite header. +// Official documentation for fields: https://www.sqlite.org/fileformat.html#the_database_header +struct Header { + unsigned char headerStr[16]; + uint16_t pageSizeBytes; + unsigned char fileFormatWriteVersion; + unsigned char fileFormatReadVersion; + unsigned char reservedSpaceBytes; + unsigned char maxEmbeddedPayloadFrac; + unsigned char minEmbeddedPayloadFrac; + unsigned char leafPayloadFrac; + uint32_t fileChangeCounter; + uint32_t databaseSizePages; + uint32_t firstFreelistTrunkPage; + uint32_t totalFreelistPages; + uint32_t schemaCookie; + uint32_t schemaFormatNumber; + uint32_t defaultPageCacheSizeBytes; + uint32_t largestBTreePageNumber; + uint32_t databaseTextEncoding; + uint32_t userVersion; + uint32_t incrementalVaccumMode; + uint32_t applicationId; + uint32_t versionValidFor; + uint32_t sqliteVersion; +}; /** * @brief RAII management of a SQLite Database Connection. @@ -434,6 +460,21 @@ class Database */ static bool isUnencrypted(const std::string& aFilename); + /** + * @brief Parse SQLite header data from a database file. + * + * This function reads the first 100 bytes of a SQLite database file + * and reconstructs groups of individual bytes into the associated fields + * in a Header object. + * + * @param[in] aFilename path/uri to a file + * + * @return Header object containing file data + * + * @throw SQLite::Exception in case of error + */ + static Header getHeaderInfo(const std::string& aFilename); + /** * @brief BackupType for the backup() method */ diff --git a/src/Database.cpp b/src/Database.cpp index 88a35b1e..6544a5ca 100644 --- a/src/Database.cpp +++ b/src/Database.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #ifndef SQLITE_DETERMINISTIC @@ -300,6 +301,138 @@ bool Database::isUnencrypted(const std::string& aFilename) throw exception; } +// Parse header data from a database. +Header Database::getHeaderInfo(const std::string& aFilename) +{ + Header h; + unsigned char buf[100]; + + if (aFilename.length() > 0 ) + { + + std::ifstream fileBuffer(aFilename.c_str(), std::ios::in | std::ios::binary); + + if (fileBuffer.is_open()) + { + fileBuffer.seekg(0, std::ios::beg); + fileBuffer.read(reinterpret_cast(&buf[0]), 100); + fileBuffer.close(); + strncpy(reinterpret_cast(&h.headerStr[0]), reinterpret_cast(buf), 16); + } + + else + { + const SQLite::Exception exception("Error opening file: " + aFilename); + throw exception; + } + + // If the "magic string" can't be found then header is invalid, corrupt or unreadable + if (!strncmp(reinterpret_cast(h.headerStr), "SQLite format 3", 15) == 0) + { + const SQLite::Exception exception("Invalid SQLite header"); + throw exception; + } + + h.pageSizeBytes = (buf[16] << 8) | buf[17]; + h.fileFormatWriteVersion = buf[18]; + h.fileFormatReadVersion = buf[19]; + h.reservedSpaceBytes = buf[20]; + h.maxEmbeddedPayloadFrac = buf[21]; + h.minEmbeddedPayloadFrac = buf[22]; + h.leafPayloadFrac = buf[23]; + + h.fileChangeCounter = + (buf[24] << 24) | + (buf[25] << 16) | + (buf[26] << 8) | + (buf[27] << 0); + + h.databaseSizePages = + (buf[28] << 24) | + (buf[29] << 16) | + (buf[30] << 8) | + (buf[31] << 0); + + h.firstFreelistTrunkPage = + (buf[32] << 24) | + (buf[33] << 16) | + (buf[34] << 8) | + (buf[35] << 0); + + h.totalFreelistPages = + (buf[36] << 24) | + (buf[37] << 16) | + (buf[38] << 8) | + (buf[39] << 0); + + h.schemaCookie = + (buf[40] << 24) | + (buf[41] << 16) | + (buf[42] << 8) | + (buf[43] << 0); + + h.schemaFormatNumber = + (buf[44] << 24) | + (buf[45] << 16) | + (buf[46] << 8) | + (buf[47] << 0); + + h.defaultPageCacheSizeBytes = + (buf[48] << 24) | + (buf[49] << 16) | + (buf[50] << 8) | + (buf[51] << 0); + + h.largestBTreePageNumber = + (buf[52] << 24) | + (buf[53] << 16) | + (buf[54] << 8) | + (buf[55] << 0); + + h.databaseTextEncoding = + (buf[56] << 24) | + (buf[57] << 16) | + (buf[58] << 8) | + (buf[59] << 0); + + h.userVersion = + (buf[60] << 24) | + (buf[61] << 16) | + (buf[62] << 8) | + (buf[63] << 0); + + h.incrementalVaccumMode = + (buf[64] << 24) | + (buf[65] << 16) | + (buf[66] << 8) | + (buf[67] << 0); + + h.applicationId = + (buf[68] << 24) | + (buf[69] << 16) | + (buf[70] << 8) | + (buf[71] << 0); + + h.versionValidFor = + (buf[92] << 24) | + (buf[93] << 16) | + (buf[94] << 8) | + (buf[95] << 0); + + h.sqliteVersion = + (buf[96] << 24) | + (buf[97] << 16) | + (buf[98] << 8) | + (buf[99] << 0); + + return h; + } + + const SQLite::Exception exception("Could not open database, the aFilename parameter was empty."); + throw exception; +} + + // This is a reference implementation of live backup taken from the official sit: // https://www.sqlite.org/backup.html diff --git a/tests/Database_test.cpp b/tests/Database_test.cpp index b9973e3f..1c0e6bcd 100644 --- a/tests/Database_test.cpp +++ b/tests/Database_test.cpp @@ -283,7 +283,7 @@ TEST(Database, execException) EXPECT_THROW(db.exec("INSERT INTO test VALUES (NULL, \"first\", 3)"), SQLite::Exception); EXPECT_EQ(SQLITE_ERROR, db.getErrorCode()); EXPECT_EQ(SQLITE_ERROR, db.getExtendedErrorCode()); - EXPECT_STREQ("no such table: test", db.getErrorMsg()); + EXPECT_STREQ("no such table: test", db.getErrorMsg());remove("test.db3"); // Create a new table db.exec("CREATE TABLE test (id INTEGER PRIMARY KEY, value TEXT, weight INTEGER)"); @@ -314,6 +314,49 @@ TEST(Database, execException) // TODO: test Database::createFunction() // TODO: test Database::loadExtension() +TEST(Database, getHeaderInfo) +{ + remove("test.db3"); + { + // Create a new database + SQLite::Database db("test.db3", SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE); + db.exec("CREATE TABLE test (id INTEGER PRIMARY KEY, value TEXT)"); + + // Set assorted SQLite header values using associated PRAGMA + db.exec("PRAGMA main.user_version = 12345"); + db.exec("PRAGMA main.application_id = 2468"); + + // Parse header fields from test database + SQLite::Header h = SQLite::Database::getHeaderInfo("test.db3"); + + //Test header values expliticly set via PRAGMA statements + EXPECT_EQ(h.userVersion, 12345); + EXPECT_EQ(h.applicationId, 2468); + + //Test header values with expected default values + EXPECT_EQ(h.pageSizeBytes, 4096); + EXPECT_EQ(h.fileFormatWriteVersion,1); + EXPECT_EQ(h.fileFormatReadVersion,1); + EXPECT_EQ(h.reservedSpaceBytes,0); + EXPECT_EQ(h.maxEmbeddedPayloadFrac, 64); + EXPECT_EQ(h.minEmbeddedPayloadFrac, 32); + EXPECT_EQ(h.leafPayloadFrac, 32); + EXPECT_EQ(h.fileChangeCounter, 3); + EXPECT_EQ(h.databaseSizePages, 2); + EXPECT_EQ(h.firstFreelistTrunkPage, 0); + EXPECT_EQ(h.totalFreelistPages, 0); + EXPECT_EQ(h.schemaCookie, 1); + EXPECT_EQ(h.schemaFormatNumber, 4); + EXPECT_EQ(h.defaultPageCacheSizeBytes, 0); + EXPECT_EQ(h.largestBTreePageNumber, 0); + EXPECT_EQ(h.databaseTextEncoding, 1); + EXPECT_EQ(h.incrementalVaccumMode, 0); + EXPECT_EQ(h.versionValidFor, 3); + EXPECT_EQ(h.sqliteVersion, SQLITE_VERSION_NUMBER); + } + remove("test.db3"); +} + #ifdef SQLITE_HAS_CODEC TEST(Database, encryptAndDecrypt) { From 0e88d62733926c97dd30d264013bfaaab61b89be Mon Sep 17 00:00:00 2001 From: Patrick Servello Date: Sun, 29 Dec 2019 22:45:43 -0600 Subject: [PATCH 2/6] Removed unused header file. --- src/Database.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Database.cpp b/src/Database.cpp index 6544a5ca..89a23b89 100644 --- a/src/Database.cpp +++ b/src/Database.cpp @@ -16,7 +16,6 @@ #include #include -#include #include #ifndef SQLITE_DETERMINISTIC From 251388c8a01f71adc463eab9c1105cb89795d2a1 Mon Sep 17 00:00:00 2001 From: Patrick Servello Date: Sun, 29 Dec 2019 22:50:30 -0600 Subject: [PATCH 3/6] Removed an accidental copy pasted remove() statement --- tests/Database_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Database_test.cpp b/tests/Database_test.cpp index 1c0e6bcd..7b7e2162 100644 --- a/tests/Database_test.cpp +++ b/tests/Database_test.cpp @@ -283,7 +283,7 @@ TEST(Database, execException) EXPECT_THROW(db.exec("INSERT INTO test VALUES (NULL, \"first\", 3)"), SQLite::Exception); EXPECT_EQ(SQLITE_ERROR, db.getErrorCode()); EXPECT_EQ(SQLITE_ERROR, db.getExtendedErrorCode()); - EXPECT_STREQ("no such table: test", db.getErrorMsg());remove("test.db3"); + EXPECT_STREQ("no such table: test", db.getErrorMsg()); // Create a new table db.exec("CREATE TABLE test (id INTEGER PRIMARY KEY, value TEXT, weight INTEGER)"); From afe10c398a3ab904710666cb096858c71643f322 Mon Sep 17 00:00:00 2001 From: Patrick Servello Date: Mon, 30 Dec 2019 01:06:40 -0600 Subject: [PATCH 4/6] Replaced stdint with plain old C types for now. Will apply fixed with datatypes to cpp11 branch --- include/SQLiteCpp/Database.h | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/include/SQLiteCpp/Database.h b/include/SQLiteCpp/Database.h index 70c35525..12b03dfe 100644 --- a/include/SQLiteCpp/Database.h +++ b/include/SQLiteCpp/Database.h @@ -12,7 +12,6 @@ #include #include // definition of nullptr for C++98/C++03 compilers -#include #include // Forward declarations to avoid inclusion of in a header @@ -57,27 +56,27 @@ int getLibVersionNumber() noexcept; // nothrow // Official documentation for fields: https://www.sqlite.org/fileformat.html#the_database_header struct Header { unsigned char headerStr[16]; - uint16_t pageSizeBytes; + unsigned int pageSizeBytes; unsigned char fileFormatWriteVersion; unsigned char fileFormatReadVersion; unsigned char reservedSpaceBytes; unsigned char maxEmbeddedPayloadFrac; unsigned char minEmbeddedPayloadFrac; unsigned char leafPayloadFrac; - uint32_t fileChangeCounter; - uint32_t databaseSizePages; - uint32_t firstFreelistTrunkPage; - uint32_t totalFreelistPages; - uint32_t schemaCookie; - uint32_t schemaFormatNumber; - uint32_t defaultPageCacheSizeBytes; - uint32_t largestBTreePageNumber; - uint32_t databaseTextEncoding; - uint32_t userVersion; - uint32_t incrementalVaccumMode; - uint32_t applicationId; - uint32_t versionValidFor; - uint32_t sqliteVersion; + unsigned long fileChangeCounter; + unsigned long databaseSizePages; + unsigned long firstFreelistTrunkPage; + unsigned long totalFreelistPages; + unsigned long schemaCookie; + unsigned long schemaFormatNumber; + unsigned long defaultPageCacheSizeBytes; + unsigned long largestBTreePageNumber; + unsigned long databaseTextEncoding; + unsigned long userVersion; + unsigned long incrementalVaccumMode; + unsigned long applicationId; + unsigned long versionValidFor; + unsigned long sqliteVersion; }; /** From 1acb606f5ca42e5d3d14522506cd0692bc063423 Mon Sep 17 00:00:00 2001 From: Patrick Servello Date: Mon, 30 Dec 2019 02:07:28 -0600 Subject: [PATCH 5/6] Added test scenarios to simulate blank file name, non existant file and a corrupt header --- tests/Database_test.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/Database_test.cpp b/tests/Database_test.cpp index 7b7e2162..80559235 100644 --- a/tests/Database_test.cpp +++ b/tests/Database_test.cpp @@ -16,6 +16,7 @@ #include #include +#include #ifdef SQLITECPP_ENABLE_ASSERT_HANDLER namespace SQLite @@ -318,6 +319,22 @@ TEST(Database, getHeaderInfo) { remove("test.db3"); { + //Call without passing a database file name + EXPECT_THROW(SQLite::Database::getHeaderInfo(""),SQLite::Exception); + + //Call with a non existant database + EXPECT_THROW(SQLite::Database::getHeaderInfo("test.db3"), SQLite::Exception); + + //Simulate a corrupt header by writing garbage to a file + unsigned char badData[100]; + std::ofstream corruptDb; + corruptDb.open("corrupt.db3", std::ios::app | std::ios::binary); + corruptDb.write(reinterpret_cast(&badData), sizeof(badData)); + + EXPECT_THROW(SQLite::Database::getHeaderInfo("corrupt.db3"), SQLite::Exception); + + remove("corrupt.db3"); + // Create a new database SQLite::Database db("test.db3", SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE); db.exec("CREATE TABLE test (id INTEGER PRIMARY KEY, value TEXT)"); From fe1455990608ff74e1181fd54aadd39144918b5d Mon Sep 17 00:00:00 2001 From: Patrick Servello Date: Mon, 30 Dec 2019 06:20:30 -0600 Subject: [PATCH 6/6] Refactored exception flow to match latest tidying, brought casts out of function calls and cleared up invalid header exception message --- src/Database.cpp | 230 ++++++++++++++++++++-------------------- tests/Database_test.cpp | 4 +- 2 files changed, 117 insertions(+), 117 deletions(-) diff --git a/src/Database.cpp b/src/Database.cpp index 89a23b89..a44ec3d4 100644 --- a/src/Database.cpp +++ b/src/Database.cpp @@ -305,130 +305,128 @@ Header Database::getHeaderInfo(const std::string& aFilename) { Header h; unsigned char buf[100]; + char* pBuf = reinterpret_cast(&buf[0]); + char* pHeaderStr = reinterpret_cast(&h.headerStr[0]); - if (aFilename.length() > 0 ) + if (aFilename.empty()) { + throw SQLite::Exception("Could not open database, the aFilename parameter was empty."); + } - std::ifstream fileBuffer(aFilename.c_str(), std::ios::in | std::ios::binary); - - if (fileBuffer.is_open()) - { - fileBuffer.seekg(0, std::ios::beg); - fileBuffer.read(reinterpret_cast(&buf[0]), 100); - fileBuffer.close(); - strncpy(reinterpret_cast(&h.headerStr[0]), reinterpret_cast(buf), 16); - } + std::ifstream fileBuffer(aFilename.c_str(), std::ios::in | std::ios::binary); - else - { - const SQLite::Exception exception("Error opening file: " + aFilename); - throw exception; - } + if (fileBuffer.is_open()) + { + fileBuffer.seekg(0, std::ios::beg); + fileBuffer.read(pBuf, 100); + fileBuffer.close(); + strncpy(pHeaderStr, pBuf, 16); + } - // If the "magic string" can't be found then header is invalid, corrupt or unreadable - if (!strncmp(reinterpret_cast(h.headerStr), "SQLite format 3", 15) == 0) - { - const SQLite::Exception exception("Invalid SQLite header"); - throw exception; - } + else + { + throw SQLite::Exception("Error opening file: " + aFilename); + } - h.pageSizeBytes = (buf[16] << 8) | buf[17]; - h.fileFormatWriteVersion = buf[18]; - h.fileFormatReadVersion = buf[19]; - h.reservedSpaceBytes = buf[20]; - h.maxEmbeddedPayloadFrac = buf[21]; - h.minEmbeddedPayloadFrac = buf[22]; - h.leafPayloadFrac = buf[23]; - - h.fileChangeCounter = - (buf[24] << 24) | - (buf[25] << 16) | - (buf[26] << 8) | - (buf[27] << 0); - - h.databaseSizePages = - (buf[28] << 24) | - (buf[29] << 16) | - (buf[30] << 8) | - (buf[31] << 0); - - h.firstFreelistTrunkPage = - (buf[32] << 24) | - (buf[33] << 16) | - (buf[34] << 8) | - (buf[35] << 0); - - h.totalFreelistPages = - (buf[36] << 24) | - (buf[37] << 16) | - (buf[38] << 8) | - (buf[39] << 0); - - h.schemaCookie = - (buf[40] << 24) | - (buf[41] << 16) | - (buf[42] << 8) | - (buf[43] << 0); - - h.schemaFormatNumber = - (buf[44] << 24) | - (buf[45] << 16) | - (buf[46] << 8) | - (buf[47] << 0); - - h.defaultPageCacheSizeBytes = - (buf[48] << 24) | - (buf[49] << 16) | - (buf[50] << 8) | - (buf[51] << 0); - - h.largestBTreePageNumber = - (buf[52] << 24) | - (buf[53] << 16) | - (buf[54] << 8) | - (buf[55] << 0); - - h.databaseTextEncoding = - (buf[56] << 24) | - (buf[57] << 16) | - (buf[58] << 8) | - (buf[59] << 0); - - h.userVersion = - (buf[60] << 24) | - (buf[61] << 16) | - (buf[62] << 8) | - (buf[63] << 0); - - h.incrementalVaccumMode = - (buf[64] << 24) | - (buf[65] << 16) | - (buf[66] << 8) | - (buf[67] << 0); - - h.applicationId = - (buf[68] << 24) | - (buf[69] << 16) | - (buf[70] << 8) | - (buf[71] << 0); - - h.versionValidFor = - (buf[92] << 24) | - (buf[93] << 16) | - (buf[94] << 8) | - (buf[95] << 0); - - h.sqliteVersion = - (buf[96] << 24) | - (buf[97] << 16) | - (buf[98] << 8) | - (buf[99] << 0); - - return h; + // If the "magic string" can't be found then header is invalid, corrupt or unreadable + if (!strncmp(pHeaderStr, "SQLite format 3", 15) == 0) + { + throw SQLite::Exception("Invalid or encrypted SQLite header"); } - const SQLite::Exception exception("Could not open database, the aFilename parameter was empty."); - throw exception; + h.pageSizeBytes = (buf[16] << 8) | buf[17]; + h.fileFormatWriteVersion = buf[18]; + h.fileFormatReadVersion = buf[19]; + h.reservedSpaceBytes = buf[20]; + h.maxEmbeddedPayloadFrac = buf[21]; + h.minEmbeddedPayloadFrac = buf[22]; + h.leafPayloadFrac = buf[23]; + + h.fileChangeCounter = + (buf[24] << 24) | + (buf[25] << 16) | + (buf[26] << 8) | + (buf[27] << 0); + + h.databaseSizePages = + (buf[28] << 24) | + (buf[29] << 16) | + (buf[30] << 8) | + (buf[31] << 0); + + h.firstFreelistTrunkPage = + (buf[32] << 24) | + (buf[33] << 16) | + (buf[34] << 8) | + (buf[35] << 0); + + h.totalFreelistPages = + (buf[36] << 24) | + (buf[37] << 16) | + (buf[38] << 8) | + (buf[39] << 0); + + h.schemaCookie = + (buf[40] << 24) | + (buf[41] << 16) | + (buf[42] << 8) | + (buf[43] << 0); + + h.schemaFormatNumber = + (buf[44] << 24) | + (buf[45] << 16) | + (buf[46] << 8) | + (buf[47] << 0); + + h.defaultPageCacheSizeBytes = + (buf[48] << 24) | + (buf[49] << 16) | + (buf[50] << 8) | + (buf[51] << 0); + + h.largestBTreePageNumber = + (buf[52] << 24) | + (buf[53] << 16) | + (buf[54] << 8) | + (buf[55] << 0); + + h.databaseTextEncoding = + (buf[56] << 24) | + (buf[57] << 16) | + (buf[58] << 8) | + (buf[59] << 0); + + h.userVersion = + (buf[60] << 24) | + (buf[61] << 16) | + (buf[62] << 8) | + (buf[63] << 0); + + h.incrementalVaccumMode = + (buf[64] << 24) | + (buf[65] << 16) | + (buf[66] << 8) | + (buf[67] << 0); + + h.applicationId = + (buf[68] << 24) | + (buf[69] << 16) | + (buf[70] << 8) | + (buf[71] << 0); + + h.versionValidFor = + (buf[92] << 24) | + (buf[93] << 16) | + (buf[94] << 8) | + (buf[95] << 0); + + h.sqliteVersion = + (buf[96] << 24) | + (buf[97] << 16) | + (buf[98] << 8) | + (buf[99] << 0); + + return h; } diff --git a/tests/Database_test.cpp b/tests/Database_test.cpp index 80559235..7d10238a 100644 --- a/tests/Database_test.cpp +++ b/tests/Database_test.cpp @@ -327,9 +327,11 @@ TEST(Database, getHeaderInfo) //Simulate a corrupt header by writing garbage to a file unsigned char badData[100]; + char* pBadData = reinterpret_cast(&badData[0]); + std::ofstream corruptDb; corruptDb.open("corrupt.db3", std::ios::app | std::ios::binary); - corruptDb.write(reinterpret_cast(&badData), sizeof(badData)); + corruptDb.write(pBadData, 100); EXPECT_THROW(SQLite::Database::getHeaderInfo("corrupt.db3"), SQLite::Exception);