Add file server to networking module (#423)

* Add version table builder class

* Add version tables to Cache

* Slightly better but still not perfect dialogue converter

* Add cache sector loading

* Add file provider to get and store cache sectors

* Add better version table support to Cache

* Add file server support to Network.kt

* Convert protocol to use an array

* Tidy up Main.kt

* Move remaining network processing into LoginServer class

* Tidy up network module packages

* Rename Network to GameServer

* Update cache builder

* Close trade on x-log, add trade messages

* Picking failed message

* Fix statement line wrap width

* Make file server usage optional

* Disable removing bzip2 for now, writing doesn't seem to work

* Reuse PrefetchKeyGeneration

* Make VersionTableBuilder thread safe by using ByteArray instead of BufferWriter

* Update prefetch keys

cache/src/main/kotlin/world/gregs/voidps/cache/Cache.kt

@@ -1,11 +1,17 @@
 package world.gregs.voidps.cache
 
+import java.util.*
+
 interface Cache {
 
+    val versionTable: ByteArray
+
     fun indexCount(): Int
 
     fun indices(): IntArray
 
+    fun sector(index: Int, archive: Int): ByteArray?
+
     fun archives(index: Int): IntArray
 
     fun archiveCount(index: Int): Int
@@ -34,4 +40,12 @@ interface Cache {
 
     fun close()
 
+
+    companion object {
+        fun load(properties: Properties): Cache {
+            val live = properties.getProperty("live").toBoolean()
+            val loader = if (live) MemoryCache else FileCache
+            return loader.load(properties)
+        }
+    }
 }

cache/src/main/kotlin/world/gregs/voidps/cache/CacheDelegate.kt

@@ -2,21 +2,28 @@ package world.gregs.voidps.cache
 
 import com.displee.cache.CacheLibrary
 import com.github.michaelbull.logging.InlineLogger
+import java.math.BigInteger
 
-class CacheDelegate(directory: String) : Cache {
+class CacheDelegate(private val library: CacheLibrary, exponent: BigInteger? = null, modulus: BigInteger? = null) : Cache {
 
-    private val library: CacheLibrary
+    constructor(directory: String, exponent: BigInteger? = null, modulus: BigInteger? = null) : this(timed(directory), exponent, modulus)
 
-    init {
-        val start = System.currentTimeMillis()
-        library = CacheLibrary(directory)
-        logger.info { "Cache read from $directory in ${System.currentTimeMillis() - start}ms" }
+    override val versionTable: ByteArray = if (exponent == null || modulus == null) ByteArray(0) else {
+        library.generateNewUkeys(exponent, modulus)
     }
 
     override fun indexCount() = library.indices().size
 
     override fun indices() = library.indices().map { it.id }.toIntArray()
 
+    override fun sector(index: Int, archive: Int): ByteArray? {
+        return if (index == 255) {
+            library.index255
+        } else {
+            library.index(index)
+        }?.readArchiveSector(archive)?.data
+    }
+
     override fun archives(index: Int) = library.index(index).archiveIds()
 
     override fun archiveCount(index: Int) = library.index(index).archiveIds().size
@@ -63,5 +70,12 @@ class CacheDelegate(directory: String) : Cache {
 
     companion object {
         private val logger = InlineLogger()
+
+        private fun timed(directory: String): CacheLibrary {
+            val start = System.currentTimeMillis()
+            val library = CacheLibrary(directory)
+            logger.info { "Cache read from $directory in ${System.currentTimeMillis() - start}ms" }
+            return library
+        }
     }
 }

cache/src/main/kotlin/world/gregs/voidps/cache/CacheLoader.kt

@@ -1,12 +1,23 @@
 package world.gregs.voidps.cache
 
+import world.gregs.voidps.cache.secure.VersionTableBuilder
 import java.io.File
 import java.io.FileNotFoundException
 import java.io.RandomAccessFile
+import java.math.BigInteger
+import java.util.*
 
 interface CacheLoader {
 
-    fun load(path: String, xteas: Map<Int, IntArray>? = null, threadUsage: Double = 1.0): Cache {
+    fun load(properties: Properties, xteas: Map<Int, IntArray>? = null): Cache {
+        val fileModulus = BigInteger(properties.getProperty("fileModulus"), 16)
+        val filePrivate = BigInteger(properties.getProperty("filePrivate"), 16)
+        val cachePath = properties.getProperty("cachePath")
+        val threadUsage = properties.getProperty("threadUsage", "1.0").toDouble()
+        return load(cachePath, filePrivate, fileModulus, threadUsage = threadUsage)
+    }
+
+    fun load(path: String, exponent: BigInteger? = null, modulus: BigInteger? = null, xteas: Map<Int, IntArray>? = null, threadUsage: Double = 1.0): Cache {
         val mainFile = File(path, "${FileCache.CACHE_FILE_NAME}.dat2")
         if (!mainFile.exists()) {
             throw FileNotFoundException("Main file not found at '${mainFile.absolutePath}'.")
@@ -18,8 +29,19 @@ interface CacheLoader {
         }
         val index255 = RandomAccessFile(index255File, "r")
         val indexCount = index255.length().toInt() / ReadOnlyCache.INDEX_SIZE
-        return load(path, mainFile, main, index255File, index255, indexCount, xteas, threadUsage)
+        val versionTable = if (exponent != null && modulus != null) VersionTableBuilder(exponent, modulus, indexCount) else null
+        return load(path, mainFile, main, index255File, index255, indexCount, versionTable, xteas, threadUsage)
     }
 
-    fun load(path: String, mainFile: File, main: RandomAccessFile, index255File: File, index255: RandomAccessFile, indexCount: Int, xteas: Map<Int, IntArray>? = null, threadUsage: Double = 1.0): Cache
+    fun load(
+        path: String,
+        mainFile: File,
+        main: RandomAccessFile,
+        index255File: File,
+        index255: RandomAccessFile,
+        indexCount: Int,
+        versionTable: VersionTableBuilder? = null,
+        xteas: Map<Int, IntArray>? = null,
+        threadUsage: Double = 1.0
+    ): Cache
 }

cache/src/main/kotlin/world/gregs/voidps/cache/FileCache.kt

@@ -1,28 +1,44 @@
 package world.gregs.voidps.cache
 
 import world.gregs.voidps.cache.compress.DecompressionContext
+import world.gregs.voidps.cache.secure.VersionTableBuilder
+import world.gregs.voidps.cache.secure.Whirlpool
 import java.io.File
 import java.io.RandomAccessFile
+import java.math.BigInteger
 
 /**
  * [Cache] which reads data directly from file
  * Average read speeds, fast loading and low but variable memory usage.
  */
 class FileCache(
     private val main: RandomAccessFile,
+    private val index255: RandomAccessFile,
     private val indexes: Array<RandomAccessFile?>,
     indexCount: Int,
     val xteas: Map<Int, IntArray>?
 ) : ReadOnlyCache(indexCount) {
 
     private val dataCache = object : LinkedHashMap<Int, Array<ByteArray?>>(16, 0.75f, true) {
-        override fun removeEldestEntry(eldest: MutableMap.MutableEntry<Int, Array<ByteArray?>>?): Boolean {
+        override fun removeEldestEntry(eldest: MutableMap.MutableEntry<Int, Array<ByteArray?>>): Boolean {
+            return size > 12
+        }
+    }
+    private val sectorCache = object : LinkedHashMap<Int, ByteArray?>(16, 0.75f, true) {
+        override fun removeEldestEntry(eldest: MutableMap.MutableEntry<Int, ByteArray?>): Boolean {
             return size > 12
         }
     }
     private val length = main.length()
     private val context = DecompressionContext()
 
+    override fun sector(index: Int, archive: Int): ByteArray? {
+        val indexRaf = if (index == 255) index255 else indexes[index] ?: return null
+        return sectorCache.getOrPut(index + (archive shl 6)) {
+            readSector(main, length, indexRaf, index, archive)
+        }
+    }
+
     override fun data(index: Int, archive: Int, file: Int, xtea: IntArray?): ByteArray? {
         val matchingIndex = files.getOrNull(index)?.getOrNull(archive)?.indexOf(file) ?: -1
         if (matchingIndex == -1) {
@@ -31,7 +47,7 @@ class FileCache(
         val hash = index + (archive shl 6)
         val files = dataCache.getOrPut(hash) {
             val indexRaf = indexes[index] ?: return null
-            readFileData(context, main, length, indexRaf, index, archive, xteas) ?: return null
+            fileData(context, main, length, indexRaf, index, archive, xteas) ?: return null
         }
         return files[matchingIndex]
     }
@@ -46,34 +62,37 @@ class FileCache(
     companion object : CacheLoader {
         const val CACHE_FILE_NAME = "main_file_cache"
 
-        operator fun invoke(path: String, xteas: Map<Int, IntArray>? = null): Cache {
-            return load(path, xteas)
+        operator fun invoke(path: String, exponent: BigInteger? = null, modulus: BigInteger? = null, xteas: Map<Int, IntArray>? = null): Cache {
+            return load(path, exponent, modulus, xteas)
         }
 
         /**
          * Create [RandomAccessFile]'s for each index file, load only the archive data into memory
          */
-        override fun load(path: String, mainFile: File, main: RandomAccessFile, index255File: File, index255: RandomAccessFile, indexCount: Int, xteas: Map<Int, IntArray>?, threadUsage: Double): Cache {
+        override fun load(
+            path: String,
+            mainFile: File,
+            main: RandomAccessFile,
+            index255File: File,
+            index255: RandomAccessFile,
+            indexCount: Int,
+            versionTable: VersionTableBuilder?,
+            xteas: Map<Int, IntArray>?,
+            threadUsage: Double
+        ): Cache {
             val length = mainFile.length()
             val context = DecompressionContext()
             val indices = Array(indexCount) { indexId ->
                 val file = File(path, "${CACHE_FILE_NAME}.idx$indexId")
                 if (file.exists()) RandomAccessFile(file, "r") else null
             }
-            val cache = FileCache(main, indices, indexCount, xteas)
+            val whirlpool = Whirlpool()
+            val cache = FileCache(main, index255, indices, indexCount, xteas)
             for (indexId in 0 until indexCount) {
-                cache.readArchiveData(context, main, length, index255, indexId)
+                cache.archiveData(context, main, length, index255, indexId, versionTable, whirlpool)
             }
+            cache.versionTable = versionTable?.build(whirlpool) ?: ByteArray(0)
             return cache
         }
-
-        @JvmStatic
-        fun main(args: Array<String>) {
-            val path = "./data/cache/"
-
-            val start = System.currentTimeMillis()
-            val cache = load(path)
-            println("Loaded cache in ${System.currentTimeMillis() - start}ms")
-        }
     }
 }

cache/src/main/kotlin/world/gregs/voidps/cache/MemoryCache.kt

@@ -3,17 +3,34 @@ package world.gregs.voidps.cache
 import com.github.michaelbull.logging.InlineLogger
 import kotlinx.coroutines.*
 import world.gregs.voidps.cache.compress.DecompressionContext
+import world.gregs.voidps.cache.secure.VersionTableBuilder
+import world.gregs.voidps.cache.secure.Whirlpool
 import java.io.File
 import java.io.RandomAccessFile
+import java.math.BigInteger
 
 /**
  * [Cache] that holds all data in memory
  * Read speeds are as fast, loading is slow and memory usage is high but stable.
  * Loading is done in parallel as it is much slower to load than [FileCache]
+ *
+ * Not much benefit of using this in the live game as file providers cache the
+ * sector data independently for the file server; so the only use after startup is
+ * reading dynamic map regions in MapDefinitions.
+ * It is however very useful for integration tests to speed world resetting.
  */
 class MemoryCache(indexCount: Int) : ReadOnlyCache(indexCount) {
 
     val data: Array<Array<Array<ByteArray?>?>?> = arrayOfNulls(indexCount)
+    val sectors: Array<Array<ByteArray?>?> = arrayOfNulls(indexCount)
+    val index255: Array<ByteArray?> = arrayOfNulls(indexCount)
+
+    override fun sector(index: Int, archive: Int): ByteArray? {
+        if (index == 255) {
+            return index255.getOrNull(archive)
+        }
+        return sectors.getOrNull(index)?.getOrNull(archive)
+    }
 
     override fun data(index: Int, archive: Int, file: Int, xtea: IntArray?): ByteArray? {
         return data.getOrNull(index)?.getOrNull(archive)?.getOrNull(file)
@@ -22,15 +39,25 @@ class MemoryCache(indexCount: Int) : ReadOnlyCache(indexCount) {
     companion object : CacheLoader {
         private val logger = InlineLogger()
 
-        operator fun invoke(path: String, threadUsage: Double = 1.0, xteas: Map<Int, IntArray>? = null): Cache {
-            return load(path, xteas, threadUsage)
+        operator fun invoke(path: String, threadUsage: Double = 1.0, exponent: BigInteger? = null, modulus: BigInteger? = null, xteas: Map<Int, IntArray>? = null): Cache {
+            return load(path, exponent, modulus, xteas, threadUsage) as ReadOnlyCache
         }
 
         /**
          * Load each index in parallel using a percentage of cpu cores
          */
         @OptIn(DelicateCoroutinesApi::class)
-        override fun load(path: String, mainFile: File, main: RandomAccessFile, index255File: File, index255: RandomAccessFile, indexCount: Int, xteas: Map<Int, IntArray>?, threadUsage: Double): Cache {
+        override fun load(
+            path: String,
+            mainFile: File,
+            main: RandomAccessFile,
+            index255File: File,
+            index255: RandomAccessFile,
+            indexCount: Int,
+            versionTable: VersionTableBuilder?,
+            xteas: Map<Int, IntArray>?,
+            threadUsage: Double
+        ): Cache {
             val cache = MemoryCache(indexCount)
             val processors = (Runtime.getRuntime().availableProcessors() * threadUsage).toInt().coerceAtLeast(1)
             newFixedThreadPoolContext(processors, "cache-loader").use { dispatcher ->
@@ -39,12 +66,13 @@ class MemoryCache(indexCount: Int) : ReadOnlyCache(indexCount) {
                         val fileLength = mainFile.length()
                         for (indexId in 0 until indexCount) {
                             launch {
-                                loadIndex(path, indexId, mainFile, fileLength, index255File, xteas, processors, cache)
+                                loadIndex(path, indexId, mainFile, fileLength, index255File, xteas, processors, cache, versionTable)
                             }
                         }
                     }
                 }
             }
+            cache.versionTable = versionTable?.build() ?: ByteArray(0)
             return cache
         }
 
@@ -59,7 +87,8 @@ class MemoryCache(indexCount: Int) : ReadOnlyCache(indexCount) {
             index255File: File,
             xteas: Map<Int, IntArray>?,
             processors: Int,
-            cache: MemoryCache
+            cache: MemoryCache,
+            versionTable: VersionTableBuilder?
         ) {
             val file = File(path, "${FileCache.CACHE_FILE_NAME}.idx$indexId")
             if (!file.exists()) {
@@ -75,10 +104,12 @@ class MemoryCache(indexCount: Int) : ReadOnlyCache(indexCount) {
                     RandomAccessFile(index255File, "r")
                 }
                 val context = DecompressionContext()
-                val highest = cache.readArchiveData(context, main, mainFileLength, index255, indexId)
+                val whirlpool = Whirlpool()
+                val highest = cache.archiveData(context, main, mainFileLength, index255, indexId, versionTable, whirlpool, cache.index255)
                 if (highest == -1) {
                     return
                 }
+                cache.sectors[indexId] = arrayOfNulls(highest + 1)
                 cache.data[indexId] = arrayOfNulls<Array<ByteArray?>?>(highest + 1)
                 coroutineScope {
                     if (processors in 2 until highest) {
@@ -114,7 +145,9 @@ class MemoryCache(indexCount: Int) : ReadOnlyCache(indexCount) {
             val raf = RandomAccessFile(file, "r")
             val main = RandomAccessFile(mainFile, "r")
             for (archiveId in archives) {
-                val archiveFiles = cache.readFileData(context, main, mainFileLength, raf, indexId, archiveId, xteas) ?: continue
+                val archiveFiles = cache.fileData(
+                    context, main, mainFileLength, raf, indexId, archiveId, xteas, cache.sectors
+                ) ?: continue
                 val archiveFileIds = cache.files[indexId]?.get(archiveId) ?: continue
                 val fileId = archiveFileIds.last()
                 val fileCount = cache.fileCounts[indexId]?.getOrNull(archiveId) ?: continue
@@ -126,13 +159,5 @@ class MemoryCache(indexCount: Int) : ReadOnlyCache(indexCount) {
                 cache.data[indexId]!![archiveId] = archiveData
             }
         }
-
-        @JvmStatic
-        fun main(args: Array<String>) {
-            val path = "./data/cache/"
-            var start = System.currentTimeMillis()
-            val cache = load(path, null)
-            println("Loaded cache in ${System.currentTimeMillis() - start}ms")
-        }
     }
 }