feat: allow SQLite database backend (#1663)

* simplify leaderboard code, fully abstract database

* update exception catching

* update exception catching and sql references, remove ugc from gamemessages

fix deleting model

remove unrelated changes

Update GameMessages.cpp

* remove ugc from gamemessages

* Update GameMessages.cpp

* Update Leaderboard.cpp

* bug fixes

* fix racing leaderboard

* remove extra stuff

* update

* add sqlite

* use a default for optimizations

* update sqlite

* Fix limits on update and delete

* fix bugs

* use definition to switch between databases

* add switch for different backends

* fix include guard and includes

* always build both

* add mysql if block

* Update Database.cpp

* add new options and add check to prevent overriding mysql

* correct config names

* Update README.md

* Update README.md

* merge to 1 sql file for sqlite database

* move to sqlite folder

* add back mysql migrations

* Update README.md

* add migration to correct the folder name or mysql

* yes aron

* updates

* Update CMakeLists.txt

* dont use paths at all, add where check to only update if folder name still exist

check also doesnt check for slashes and assumes one will be there since it will be.

* default dont auto create account

for releases we can change this flag

* default 0

* add times played query

* fix leaderboard not incrementing on a not better score

* add env vars with defaults for docker

* use an "enum"

* default to mariadb

* Update .env.example

.env.example

@@ -7,8 +7,16 @@ NET_VERSION=171022
 ACCOUNT_MANAGER_SECRET=
 # Should be the externally facing IP of your server host
 EXTERNAL_IP=localhost
+
+# The database type that will be used.
+# Acceptable values are `sqlite`, `mysql`, `mariadb`, `maria`.
+# Case insensitive.
+DATABASE_TYPE=mariadb
+SQLITE_DATABASE_PATH=resServer/dlu.sqlite
+
 # Database values
 # Be careful with special characters here. It is more safe to use normal characters and/or numbers.
 MARIADB_USER=darkflame
 MARIADB_PASSWORD=
 MARIADB_DATABASE=darkflame
+SKIP_ACCOUNT_CREATION=1

.gitignore

@@ -6,7 +6,6 @@ docker/configs
 # Third party libraries
 thirdparty/mysql/
 thirdparty/mysql_linux/
-CMakeVariables.txt
 
 # Build folders
 build/

CMakeLists.txt

@@ -175,16 +175,18 @@ foreach(resource_file ${RESOURCE_FILES})
 				list(GET line_split 0 variable_name)
 
 				if(NOT ${parsed_current_file_contents} MATCHES ${variable_name})
-					message(STATUS "Adding missing config option " ${variable_name} " to " ${resource_file})
-					set(line_to_add ${line_to_add} ${line})
+					# For backwards compatibility with older setup versions, dont add this option.
+					if(NOT ${variable_name} MATCHES "database_type")
+						message(STATUS "Adding missing config option " ${variable_name} " to " ${resource_file})
+						set(line_to_add ${line_to_add} ${line})
 
-					foreach(line_to_append ${line_to_add})
-						file(APPEND ${DLU_CONFIG_DIR}/${resource_file} "\n" ${line_to_append})
-					endforeach()
+						foreach(line_to_append ${line_to_add})
+							file(APPEND ${DLU_CONFIG_DIR}/${resource_file} "\n" ${line_to_append})
+						endforeach()
 
-					file(APPEND ${DLU_CONFIG_DIR}/${resource_file} "\n")
+						file(APPEND ${DLU_CONFIG_DIR}/${resource_file} "\n")
+					endif()
 				endif()
-
 				set(line_to_add "")
 			else()
 				set(line_to_add ${line_to_add} ${line})
@@ -214,21 +216,8 @@ foreach(file ${VANITY_FILES})
 endforeach()
 
 # Move our migrations for MasterServer to run
-file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/dlu/)
-file(GLOB SQL_FILES ${CMAKE_SOURCE_DIR}/migrations/dlu/*.sql)
-
-foreach(file ${SQL_FILES})
-	get_filename_component(file ${file} NAME)
-	configure_file(${CMAKE_SOURCE_DIR}/migrations/dlu/${file} ${PROJECT_BINARY_DIR}/migrations/dlu/${file})
-endforeach()
-
-file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/cdserver/)
-file(GLOB SQL_FILES ${CMAKE_SOURCE_DIR}/migrations/cdserver/*.sql)
-
-foreach(file ${SQL_FILES})
-	get_filename_component(file ${file} NAME)
-	configure_file(${CMAKE_SOURCE_DIR}/migrations/cdserver/${file} ${PROJECT_BINARY_DIR}/migrations/cdserver/${file})
-endforeach()
+file(REMOVE_RECURSE ${PROJECT_BINARY_DIR}/migrations)
+file(COPY ${CMAKE_SOURCE_DIR}/migrations DESTINATION ${CMAKE_BINARY_DIR})
 
 # Add system specfic includes for Apple, Windows and Other Unix OS' (including Linux)
 if (APPLE)

README.md

@@ -13,21 +13,32 @@ Darkflame Universe is licensed under AGPLv3, please read [LICENSE](LICENSE). Som
 * You must disclose any changes you make to the code when you distribute it
 * Hosting a server for others counts as distribution
 
-## Disclaimers
-### Setup difficulty
-Throughout the entire build and setup process a level of familiarity with the command line and preferably a Unix-like development environment is greatly advantageous.
-
 ### Hosting a server
 We do not recommend hosting public servers. Darkflame Universe is intended for small scale deployment, for example within a group of friends. It has not been tested for large scale deployment which comes with additional security risks.
 
 ### Supply of resource files
 Darkflame Universe is a server emulator and does not distribute any LEGO® Universe files. A separate game client is required to setup this server emulator and play the game, which we cannot supply. Users are strongly suggested to refer to the safe checksums listed [here](#verifying-your-client-files) to see if a client will work.
 
-## Step by step walkthrough for a single-player server
-If you would like a setup for a single player server only on a Windows machine, use the [Native Windows Setup Guide by HailStorm](https://gist.github.com/HailStorm32/169df65a47a104199b5cc57d10fa57de) and skip this README.
+## Setting up a single player server
+* If you don't know what WSL is, skip this warning.
+	Warning: WSL version 1 does NOT support using sqlite as a database due to how it handles filesystem synchronization.
+	You must use Version 2 if you must run the server under WSL. Not doing so will result in save data loss.
+* Single player installs now no longer require building the server from source or installing development tools.
+* Download the [latest release](https://github.com/DarkflameUniverse/DarkflameServer/releases) and extract the files into a folder inside your client.
+* You should be able to see the folder with the server executables in the same folder as `legouniverse.exe`.
+* Open `sharedconfig.ini` and find the line that says `client_location` and put `..` after it so the line reads `client_location=..`.
+* To run the server, double-click `MasterServer.exe`.
+* You will be asked to create an account the first time you run the server.
+* When shutting down the server, it is highly recommended to click the `MasterServer.exe` window and hold `ctrl` while pressing `c` to stop the server.
+* We are working on a way to make it so when you close the game, the server saves automatically alongside when you open the game, the server starts automatically.
+
+<font size="32">**If you are not planning on hosting a server for others, working in the codebase or wanting to use MariaDB for a database, you can stop reading here.**</font>
+
+If you would like to use a MariaDB as a database instead of the default of sqlite, follow the steps [here](#database-setup).
 
-## Steps to setup server
+# Steps to setup a development environment
 * [Clone this repository](#clone-the-repository)
+* [Setting up a development environment](#setting-up-a-development-environment)
 * [Install dependencies](#install-dependencies)
 * [Database setup](#database-setup)
 * [Build the server](#build-the-server)
@@ -39,6 +50,13 @@ If you would like a setup for a single player server only on a Windows machine,
 * [User Guide](#user-guide)
 * [Docker](#docker)
 
+## Disclaimers
+### Setup difficulty
+Throughout the entire build and setup process a level of familiarity with the command line and preferably a Unix-like development environment is greatly advantageous.
+
+## Step by step walkthrough for building a single-player Windows server from source
+If you would like a setup for a single player server only on a Windows machine built from source, use the [Native Windows Setup Guide by HailStorm](https://gist.github.com/HailStorm32/169df65a47a104199b5cc57d10fa57de) and skip this README.
+
 ## Clone the repository
 If you are on Windows, you will need to download and install git from [here](https://git-scm.com/download/win)
 
@@ -266,8 +284,8 @@ systemctl stop darkflame.service
 journalctl -xeu darkflame.service
 ```
 
-### First admin user
-Run `MasterServer -a` to get prompted to create an admin account. This method is only intended for the system administrator as a means to get started, do NOT use this method to create accounts for other users!
+### First user or adding more users.
+The first time you run `MasterServer`, you will be prompted to create an account. To create more accounts from the command line, `MasterServer -a` to get prompted to create an admin account. This method is only intended for the system administrator as a means to get started, do NOT use this method to create accounts for other users!
 
 ### Account management tool (Nexus Dashboard)
 **If you are just using this server for yourself, you can skip setting up Nexus Dashboard**

dCommon/CMakeLists.txt

@@ -37,7 +37,6 @@ target_include_directories(dCommon
 	"${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase"
 	"${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase/ITables"
 	"${PROJECT_SOURCE_DIR}/dDatabase/CDClientDatabase"
-	"${PROJECT_SOURCE_DIR}/thirdparty/mariadb-connector-cpp/include"
 )
 
 if (UNIX)

dDatabase/GameDatabase/CMakeLists.txt

@@ -8,6 +8,12 @@ foreach(file ${DDATABSE_DATABSES_MYSQL_SOURCES})
 	set(DDATABASE_GAMEDATABASE_SOURCES ${DDATABASE_GAMEDATABASE_SOURCES} "MySQL/${file}")
 endforeach()
 
+add_subdirectory(SQLite)
+
+foreach(file ${DDATABSE_DATABSES_SQLITE_SOURCES})
+	set(DDATABASE_GAMEDATABASE_SOURCES ${DDATABASE_GAMEDATABASE_SOURCES} "SQLite/${file}")
+endforeach()
+
 add_subdirectory(TestSQL)
 
 foreach(file ${DDATABSE_DATABSES_TEST_SQL_SOURCES})
@@ -16,13 +22,14 @@ endforeach()
 
 add_library(dDatabaseGame STATIC ${DDATABASE_GAMEDATABASE_SOURCES})
 target_include_directories(dDatabaseGame PUBLIC "."
-	"ITables" PRIVATE "MySQL" "TestSQL"
+	"ITables" PRIVATE "MySQL" "SQLite" "TestSQL"
 	"${PROJECT_SOURCE_DIR}/dCommon"
 	"${PROJECT_SOURCE_DIR}/dCommon/dEnums"
 )
+
 target_link_libraries(dDatabaseGame
-	PUBLIC MariaDB::ConnCpp
-	INTERFACE dCommon)
+	INTERFACE dCommon
+	PRIVATE sqlite3 MariaDB::ConnCpp)
 
 # Glob together all headers that need to be precompiled
 file(

dDatabase/GameDatabase/Database.cpp

@@ -2,22 +2,46 @@
 #include "Game.h"
 #include "dConfig.h"
 #include "Logger.h"
-#include "MySQLDatabase.h"
 #include "DluAssert.h"
 
+#include "SQLiteDatabase.h"
+#include "MySQLDatabase.h"
+
+#include <ranges>
+
 #pragma warning (disable:4251) //Disables SQL warnings
 
 namespace {
 	GameDatabase* database = nullptr;
 }
 
+std::string Database::GetMigrationFolder() {
+	const std::set<std::string> validMysqlTypes = { "mysql", "mariadb", "maria" };
+	auto databaseType = Game::config->GetValue("database_type");
+	std::ranges::transform(databaseType, databaseType.begin(), ::tolower);
+	if (databaseType == "sqlite") return "sqlite";
+	else if (validMysqlTypes.contains(databaseType)) return "mysql";
+	else {
+		LOG("No database specified, using MySQL");
+		return "mysql";
+	}
+}
+
 void Database::Connect() {
 	if (database) {
 		LOG("Tried to connect to database when it's already connected!");
 		return;
 	}
 
-	database = new MySQLDatabase();
+	const auto databaseType = GetMigrationFolder();
+
+	if (databaseType == "sqlite") database = new SQLiteDatabase();
+	else if (databaseType == "mysql") database = new MySQLDatabase();
+	else {
+		LOG("Invalid database type specified in config, using MySQL");
+		database = new MySQLDatabase();
+	}
+
 	database->Connect();
 }
 

dDatabase/GameDatabase/Database.h

@@ -12,4 +12,6 @@ namespace Database {
 	// Used for assigning a test database as the handler for database logic.
 	// Do not use in production code.
 	void _setDatabase(GameDatabase* const db);
+
+	std::string GetMigrationFolder();
 };

dDatabase/GameDatabase/GameDatabase.h

@@ -26,13 +26,8 @@
 #include "IBehaviors.h"
 #include "IUgcModularBuild.h"
 
-namespace sql {
-	class Statement;
-	class PreparedStatement;
-};
-
 #ifdef _DEBUG
-#  define DLU_SQL_TRY_CATCH_RETHROW(x) do { try { x; } catch (sql::SQLException& ex) { LOG("SQL Error: %s", ex.what()); throw; } } while(0)
+#  define DLU_SQL_TRY_CATCH_RETHROW(x) do { try { x; } catch (std::exception& ex) { LOG("SQL Error: %s", ex.what()); throw; } } while(0)
 #else
 #  define DLU_SQL_TRY_CATCH_RETHROW(x) x
 #endif // _DEBUG
@@ -50,7 +45,6 @@ class GameDatabase :
 	virtual void Connect() = 0;
 	virtual void Destroy(std::string source = "") = 0;
 	virtual void ExecuteCustomQuery(const std::string_view query) = 0;
-	virtual sql::PreparedStatement* CreatePreppedStmt(const std::string& query) = 0;
 	virtual void Commit() = 0;
 	virtual bool GetAutoCommit() = 0;
 	virtual void SetAutoCommit(bool value) = 0;

dDatabase/GameDatabase/ITables/IAccounts.h

@@ -36,6 +36,8 @@ class IAccounts {
 
 	// Update the GameMaster level of an account.
 	virtual void UpdateAccountGmLevel(const uint32_t accountId, const eGameMasterLevel gmLevel) = 0;
+
+	virtual uint32_t GetAccountCount() = 0;
 };
 
 #endif  //!__IACCOUNTS__H__

dDatabase/GameDatabase/MySQL/MySQLDatabase.cpp

@@ -14,6 +14,7 @@ namespace {
 };
 
 void MySQLDatabase::Connect() {
+	LOG("Using MySQL database");
 	driver = sql::mariadb::get_driver_instance();
 
 	// The mariadb connector is *supposed* to handle unix:// and pipe:// prefixes to hostName, but there are bugs where
@@ -67,7 +68,7 @@ void MySQLDatabase::ExecuteCustomQuery(const std::string_view query) {
 
 sql::PreparedStatement* MySQLDatabase::CreatePreppedStmt(const std::string& query) {
 	if (!con) {
-		Connect();
+		Database::Get()->Connect();
 		LOG("Trying to reconnect to MySQL");
 	}
 
@@ -76,7 +77,7 @@ sql::PreparedStatement* MySQLDatabase::CreatePreppedStmt(const std::string& quer
 
 		con = nullptr;
 
-		Connect();
+		Database::Get()->Connect();
 		LOG("Trying to reconnect to MySQL from invalid or closed connection");
 	}
 

dDatabase/GameDatabase/MySQL/MySQLDatabase.h

@@ -30,7 +30,6 @@ class MySQLDatabase : public GameDatabase {
 	void Connect() override;
 	void Destroy(std::string source = "") override;
 
-	sql::PreparedStatement* CreatePreppedStmt(const std::string& query) override;
 	void Commit() override;
 	bool GetAutoCommit() override;
 	void SetAutoCommit(bool value) override;
@@ -125,6 +124,8 @@ class MySQLDatabase : public GameDatabase {
 	void IncrementTimesPlayed(const uint32_t playerId, const uint32_t gameId) override;
 	void InsertUgcBuild(const std::string& modules, const LWOOBJID bigId, const std::optional<uint32_t> characterId) override;
 	void DeleteUgcBuild(const LWOOBJID bigId) override;
+	sql::PreparedStatement* CreatePreppedStmt(const std::string& query);
+	uint32_t GetAccountCount() override;
 private:
 
 	// Generic query functions that can be used for any query.

dDatabase/GameDatabase/MySQL/Tables/Accounts.cpp

@@ -39,3 +39,8 @@ void MySQLDatabase::InsertNewAccount(const std::string_view username, const std:
 void MySQLDatabase::UpdateAccountGmLevel(const uint32_t accountId, const eGameMasterLevel gmLevel) {
 	ExecuteUpdate("UPDATE accounts SET gm_level = ? WHERE id = ?;", static_cast<int32_t>(gmLevel), accountId);
 }
+
+uint32_t MySQLDatabase::GetAccountCount() {
+	auto res = ExecuteSelect("SELECT COUNT(*) as count FROM accounts;");
+	return res->next() ? res->getUInt("count") : 0;
+}

dDatabase/GameDatabase/SQLite/CMakeLists.txt

@@ -0,0 +1,11 @@
+SET(DDATABSE_DATABSES_SQLITE_SOURCES
+	"SQLiteDatabase.cpp"
+)
+
+add_subdirectory(Tables)
+
+foreach(file ${DDATABASES_DATABASES_SQLITE_TABLES_SOURCES})
+	set(DDATABSE_DATABSES_SQLITE_SOURCES ${DDATABSE_DATABSES_SQLITE_SOURCES} "Tables/${file}")
+endforeach()
+
+set(DDATABSE_DATABSES_SQLITE_SOURCES ${DDATABSE_DATABSES_SQLITE_SOURCES} PARENT_SCOPE)