Add support for packed clients (#802)

* First iteration of pack reader and interface

* Fix memory leak and remove logs

* Complete packed asset interface and begin on file loading replacement

* Implement proper BinaryIO error

* Improve AssetMemoryBuffer for reading and implement more reading

* Repair more file loading code and improve how navmeshes are loaded

* Missing checks implementation

* Revert addition of Manifest class and migration changes

* Resolved all feedback.

CMakeLists.txt

@@ -101,6 +101,17 @@ foreach(resource_file ${RESOURCE_FILES})
 	endif()
 endforeach()
 
+# Copy navmesh data on first build and extract it
+if (NOT EXISTS ${PROJECT_BINARY_DIR}/navmeshes/)
+	configure_file(
+		${CMAKE_SOURCE_DIR}/resources/navmeshes.zip ${PROJECT_BINARY_DIR}/navmeshes.zip
+		COPYONLY
+	)
+
+	file(ARCHIVE_EXTRACT INPUT ${PROJECT_BINARY_DIR}/navmeshes.zip)
+	file(REMOVE ${PROJECT_BINARY_DIR}/navmeshes.zip)
+endif()
+
 # Copy vanity files on first build
 set(VANITY_FILES "CREDITS.md" "INFO.md" "TESTAMENT.md" "NPC.xml")
 foreach(file ${VANITY_FILES})
@@ -145,6 +156,7 @@ set(INCLUDED_DIRECTORIES
 	"dGame/dEntity"
 	"dGame/dPropertyBehaviors"
 	"dGame/dUtilities"
+	"dCommon/dClient"
 	"dPhysics"
 	"dNavigation"
 	"dNavigation/dTerrain"

dChatServer/ChatServer.cpp

@@ -12,6 +12,7 @@
 #include "dMessageIdentifiers.h"
 #include "dChatFilter.h"
 #include "Diagnostics.h"
+#include "AssetManager.h"
 
 #include "PlayerContainer.h"
 #include "ChatPacketHandler.h"
@@ -22,6 +23,7 @@ namespace Game {
 	dServer* server;
 	dConfig* config;
 	dChatFilter* chatFilter;
+	AssetManager* assetManager;
 }
 
 //RakNet includes:
@@ -50,6 +52,16 @@ int main(int argc, char** argv) {
 	Game::logger->SetLogToConsole(bool(std::stoi(config.GetValue("log_to_console"))));
 	Game::logger->SetLogDebugStatements(config.GetValue("log_debug_statements") == "1");
 
+	try {
+		std::string client_path = config.GetValue("client_location");
+		if (client_path.empty()) client_path = "./res";
+		Game::assetManager = new AssetManager(config.GetValue("client_location"));
+	} catch (std::runtime_error& ex) {
+		Game::logger->Log("ChatServer", "Got an error while setting up assets: %s", ex.what());
+
+		return EXIT_FAILURE;
+	}
+
 	//Connect to the MySQL Database
 	std::string mysql_host = config.GetValue("mysql_host");
 	std::string mysql_database = config.GetValue("mysql_database");
@@ -87,7 +99,7 @@ int main(int argc, char** argv) {
 
 	Game::server = new dServer(config.GetValue("external_ip"), ourPort, 0, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::Chat);
 
-	Game::chatFilter = new dChatFilter("./res/chatplus_en_us", bool(std::stoi(config.GetValue("dont_generate_dcf"))));
+	Game::chatFilter = new dChatFilter(Game::assetManager->GetResPath().string() + "/chatplus_en_us", bool(std::stoi(config.GetValue("dont_generate_dcf"))));
 
 	//Run it until server gets a kill message from Master:
 	auto t = std::chrono::high_resolution_clock::now();

dCommon/BinaryIO.cpp

@@ -10,7 +10,7 @@ void BinaryIO::WriteString(const std::string& stringToWrite, std::ofstream& outs
 }
 
 //For reading null-terminated strings
-std::string BinaryIO::ReadString(std::ifstream& instream) {
+std::string BinaryIO::ReadString(std::istream& instream) {
 	std::string toReturn;
 	char buffer;
 
@@ -25,7 +25,7 @@ std::string BinaryIO::ReadString(std::ifstream& instream) {
 }
 
 //For reading strings of a specific size
-std::string BinaryIO::ReadString(std::ifstream& instream, size_t size) {
+std::string BinaryIO::ReadString(std::istream& instream, size_t size) {
 	std::string toReturn;
 	char buffer;
 
@@ -37,7 +37,7 @@ std::string BinaryIO::ReadString(std::ifstream& instream, size_t size) {
 	return toReturn;
 }
 
-std::string BinaryIO::ReadWString(std::ifstream& instream) {
+std::string BinaryIO::ReadWString(std::istream& instream) {
 	size_t size;
 	BinaryRead(instream, size);
 	//toReturn.resize(size);

dCommon/BinaryIO.h

@@ -10,16 +10,15 @@ namespace BinaryIO {
 
 	template<typename T>
 	std::istream& BinaryRead(std::istream& stream, T& value) {
-		if (!stream.good())
-			printf("bla");
+		if (!stream.good()) throw std::runtime_error("Failed to read from istream.");
 
 		return stream.read(reinterpret_cast<char*>(&value), sizeof(T));
 	}
 
 	void WriteString(const std::string& stringToWrite, std::ofstream& outstream);
-	std::string ReadString(std::ifstream& instream);
-	std::string ReadString(std::ifstream& instream, size_t size);
-	std::string ReadWString(std::ifstream& instream);
+	std::string ReadString(std::istream& instream);
+	std::string ReadString(std::istream& instream, size_t size);
+	std::string ReadWString(std::istream& instream);
 
 	inline bool DoesFileExist(const std::string& name) {
 		std::ifstream f(name.c_str());

dCommon/BrickByBrickFix.cpp

@@ -49,10 +49,10 @@ uint32_t BrickByBrickFix::TruncateBrokenBrickByBrickXml() {
 				}
 
 				// Ignore the valgrind warning about uninitialized values.  These are discarded later when we know the actual uncompressed size.
-				std::unique_ptr<uint8_t[]> uncompressedChunk(new uint8_t[MAX_SD0_CHUNK_SIZE]);
+				std::unique_ptr<uint8_t[]> uncompressedChunk(new uint8_t[ZCompression::MAX_SD0_CHUNK_SIZE]);
 				int32_t err{};
 				int32_t actualUncompressedSize = ZCompression::Decompress(
-					compressedChunk.get(), chunkSize, uncompressedChunk.get(), MAX_SD0_CHUNK_SIZE, err);
+					compressedChunk.get(), chunkSize, uncompressedChunk.get(), ZCompression::MAX_SD0_CHUNK_SIZE, err);
 
 				if (actualUncompressedSize != -1) {
 					uint32_t previousSize = completeUncompressedModel.size();

dCommon/BrickByBrickFix.h

@@ -17,10 +17,4 @@ namespace BrickByBrickFix {
 	 * @return The number of BrickByBrick models that were updated
 	 */
 	uint32_t UpdateBrickByBrickModelsToSd0();
-
-	/**
-	 * @brief Max size of an inflated sd0 zlib chunk
-	 * 
-	 */
-	constexpr uint32_t MAX_SD0_CHUNK_SIZE = 1024 * 256;
 };

dCommon/CMakeLists.txt

@@ -17,6 +17,12 @@ set(DCOMMON_SOURCES "AMFFormat.cpp"
 		"BrickByBrickFix.cpp"
 )
 
+add_subdirectory(dClient)
+
+foreach(file ${DCOMMON_DCLIENT_SOURCES})
+	set(DCOMMON_SOURCES ${DCOMMON_SOURCES} "dClient/${file}")
+endforeach()
+
 include_directories(${PROJECT_SOURCE_DIR}/dCommon/)
 
 add_library(dCommon STATIC ${DCOMMON_SOURCES})

dCommon/Game.h

@@ -10,6 +10,7 @@ class dChatFilter;
 class dConfig;
 class dLocale;
 class RakPeerInterface;
+class AssetManager;
 struct SystemAddress;
 
 namespace Game {
@@ -22,5 +23,6 @@ namespace Game {
 	extern dLocale* locale;
 	extern std::mt19937 randomEngine;
 	extern RakPeerInterface* chatServer;
+	extern AssetManager* assetManager;
 	extern SystemAddress chatSysAddr;
 }

dCommon/GeneralUtils.cpp

@@ -50,6 +50,7 @@ bool _IsSuffixChar(uint8_t c) {
 
 bool GeneralUtils::_NextUTF8Char(std::string_view& slice, uint32_t& out) {
 	size_t rem = slice.length();
+	if (slice.empty()) return false;
 	const uint8_t* bytes = (const uint8_t*)&slice.front();
 	if (rem > 0) {
 		uint8_t first = bytes[0];

dCommon/ZCompression.h

@@ -8,5 +8,11 @@ namespace ZCompression {
 	int32_t Compress(const uint8_t* abSrc, int32_t nLenSrc, uint8_t* abDst, int32_t nLenDst);
 
 	int32_t Decompress(const uint8_t* abSrc, int32_t nLenSrc, uint8_t* abDst, int32_t nLenDst, int32_t& nErr);
+
+	/**
+	 * @brief Max size of an inflated sd0 zlib chunk
+	 *
+	 */
+	constexpr uint32_t MAX_SD0_CHUNK_SIZE = 1024 * 256;
 }
 

dCommon/dClient/AssetManager.cpp

@@ -0,0 +1,201 @@
+#include "AssetManager.h"
+
+#include <zlib.h>
+
+AssetManager::AssetManager(const std::string& path) {
+	if (!std::filesystem::is_directory(path)) {
+		throw std::runtime_error("Attempted to load asset bundle (" + path + ") however it is not a valid directory.");
+	}
+
+	m_Path = std::filesystem::path(path);
+
+	if (std::filesystem::exists(m_Path / "client") && std::filesystem::exists(m_Path / "versions")) {
+		m_AssetBundleType = eAssetBundleType::Packed;
+
+		m_RootPath = m_Path;
+		m_ResPath = (m_Path / "client" / "res");
+	} else if (std::filesystem::exists(m_Path / ".." / "versions") && std::filesystem::exists(m_Path / "res")) {
+		m_AssetBundleType = eAssetBundleType::Packed;
+
+		m_RootPath = (m_Path / "..");
+		m_ResPath = (m_Path / "res");
+	} else if (std::filesystem::exists(m_Path / "pack") && std::filesystem::exists(m_Path / ".." / ".." / "versions")) {
+		m_AssetBundleType = eAssetBundleType::Packed;
+
+		m_RootPath = (m_Path / ".." / "..");
+		m_ResPath = m_Path;
+	} else if (std::filesystem::exists(m_Path / "res" / "cdclient.fdb") && !std::filesystem::exists(m_Path / "res" / "pack")) {
+		m_AssetBundleType = eAssetBundleType::Unpacked;
+
+		m_ResPath = (m_Path / "res");
+	} else if (std::filesystem::exists(m_Path / "cdclient.fdb") && !std::filesystem::exists(m_Path / "pack")) {
+		m_AssetBundleType = eAssetBundleType::Unpacked;
+
+		m_ResPath = m_Path;
+	}
+
+	if (m_AssetBundleType == eAssetBundleType::None) {
+		throw std::runtime_error("Failed to identify client type, cannot read client data.");
+	}
+
+	switch (m_AssetBundleType) {
+		case eAssetBundleType::Packed: {
+			this->LoadPackIndex();
+
+			this->UnpackRequiredAssets();
+
+			break;
+		}
+	}
+}
+
+void AssetManager::LoadPackIndex() {
+	m_PackIndex = new PackIndex(m_RootPath);
+}
+
+std::filesystem::path AssetManager::GetResPath() {
+	return m_ResPath;
+}
+
+eAssetBundleType AssetManager::GetAssetBundleType() {
+	return m_AssetBundleType;
+}
+
+bool AssetManager::HasFile(const char* name) {
+	auto fixedName = std::string(name);
+	std::transform(fixedName.begin(), fixedName.end(), fixedName.begin(), [](uint8_t c) { return std::tolower(c); });
+	std::replace(fixedName.begin(), fixedName.end(), '/', '\\');
+
+	auto realPathName = fixedName;
+
+	if (fixedName.rfind("client\\res\\", 0) != 0) {
+		fixedName = "client\\res\\" + fixedName;
+	}
+
+	if (std::filesystem::exists(m_ResPath / realPathName)) {
+		return true;
+	}
+
+	uint32_t crc = crc32b(0xFFFFFFFF, (uint8_t*)fixedName.c_str(), fixedName.size());
+	crc = crc32b(crc, (Bytef*)"\0\0\0\0", 4);
+
+	for (const auto& item : this->m_PackIndex->GetPackFileIndices()) {
+		if (item.m_Crc == crc) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+bool AssetManager::GetFile(const char* name, char** data, uint32_t* len) {
+	auto fixedName = std::string(name);
+	std::transform(fixedName.begin(), fixedName.end(), fixedName.begin(), [](uint8_t c) { return std::tolower(c); });
+	std::replace(fixedName.begin(), fixedName.end(), '/', '\\');
+
+	auto realPathName = fixedName;
+
+	if (fixedName.rfind("client\\res\\", 0) != 0) {
+		fixedName = "client\\res\\" + fixedName;
+	}
+
+	if (std::filesystem::exists(m_ResPath / realPathName)) {
+		FILE* file;
+#ifdef _WIN32
+		fopen_s(&file, (m_ResPath / realPathName).string().c_str(), "rb");
+#elif __APPLE__
+		// macOS has 64bit file IO by default
+		file = fopen((m_ResPath / realPathName).string().c_str(), "rb");
+#else
+		file = fopen64((m_ResPath / realPathName).string().c_str(), "rb");
+#endif
+		fseek(file, 0, SEEK_END);
+		*len = ftell(file);
+		*data = (char*)malloc(*len);
+		fseek(file, 0, SEEK_SET);
+		fread(*data, sizeof(uint8_t), *len, file);
+		fclose(file);
+
+		return true;
+	}
+
+	if (this->m_AssetBundleType == eAssetBundleType::Unpacked) return false;
+
+	int32_t packIndex = -1;
+	uint32_t crc = crc32b(0xFFFFFFFF, (uint8_t*)fixedName.c_str(), fixedName.size());
+	crc = crc32b(crc, (Bytef*)"\0\0\0\0", 4);
+
+	for (const auto& item : this->m_PackIndex->GetPackFileIndices()) {
+		if (item.m_Crc == crc) {
+			packIndex = item.m_PackFileIndex;
+			crc = item.m_Crc;
+			break;
+		}
+	}
+
+	if (packIndex == -1 || !crc) {
+		return false;
+	}
+
+	auto packs = this->m_PackIndex->GetPacks();
+	auto* pack = packs.at(packIndex);
+
+	bool success = pack->ReadFileFromPack(crc, data, len);
+
+	return success;
+}
+
+AssetMemoryBuffer AssetManager::GetFileAsBuffer(const char* name) {
+	char* buf;
+	uint32_t len;
+
+	bool success = this->GetFile(name, &buf, &len);
+
+	return AssetMemoryBuffer(buf, len, success);
+}
+
+void AssetManager::UnpackRequiredAssets() {
+	if (std::filesystem::exists(m_ResPath / "cdclient.fdb")) return;
+
+	char* data;
+	uint32_t size;
+
+	bool success = this->GetFile("cdclient.fdb", &data, &size);
+
+	if (!success) {
+		Game::logger->Log("AssetManager", "Failed to extract required files from the packs.");
+
+		delete data;
+
+		return;
+	}
+
+	std::ofstream cdclientOutput(m_ResPath / "cdclient.fdb", std::ios::out | std::ios::binary);
+	cdclientOutput.write(data, size);
+	cdclientOutput.close();
+
+	delete data;
+
+	return;
+}
+
+uint32_t AssetManager::crc32b(uint32_t base, uint8_t* message, size_t l) {
+	size_t i, j;
+	uint32_t crc, msb;
+
+	crc = base;
+	for (i = 0; i < l; i++) {
+		// xor next byte to upper bits of crc
+		crc ^= (((unsigned int)message[i]) << 24);
+		for (j = 0; j < 8; j++) { // Do eight times.
+			msb = crc >> 31;
+			crc <<= 1;
+			crc ^= (0 - msb) & 0x04C11DB7;
+		}
+	}
+	return crc; // don't complement crc on output
+}
+
+AssetManager::~AssetManager() {
+	delete m_PackIndex;
+}