-
Notifications
You must be signed in to change notification settings - Fork 21
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add fast file and in memory cache implementations (#422)
* Add lightweight read only cache implementation with array backing * Add more efficient decompression * Add fast random access file cache impl * Ad script to remove bzip2 compression from a cache * Add cache parallel loading * Add script to remove xteas * Add automated script to build a new cache * Fix equipment stats attack rate and examines closes #154 * Move Xteas and binary file into tools module * Remove ActiveCache.kt * Rename decoder loadCache() back to load() * Fix client script and interface encoding * Add middle mouse camera panning cs2 and interface events closes #418 * Update prefetch keys * Fix charge spell unit tests * Add script to check file list against existing hashes
- Loading branch information
Showing
150 changed files
with
3,013 additions
and
1,642 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
25 changes: 25 additions & 0 deletions
25
cache/src/main/kotlin/world/gregs/voidps/cache/CacheLoader.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| package world.gregs.voidps.cache | ||
|
|
||
| import java.io.File | ||
| import java.io.FileNotFoundException | ||
| import java.io.RandomAccessFile | ||
|
|
||
| interface CacheLoader { | ||
|
|
||
| fun load(path: String, xteas: Map<Int, IntArray>? = null, threadUsage: Double = 1.0): Cache { | ||
| val mainFile = File(path, "${FileCache.CACHE_FILE_NAME}.dat2") | ||
| if (!mainFile.exists()) { | ||
| throw FileNotFoundException("Main file not found at '${mainFile.absolutePath}'.") | ||
| } | ||
| val main = RandomAccessFile(mainFile, "r") | ||
| val index255File = File(path, "${FileCache.CACHE_FILE_NAME}.idx255") | ||
| if (!index255File.exists()) { | ||
| throw FileNotFoundException("Checksum file not found at '${index255File.absolutePath}'.") | ||
| } | ||
| val index255 = RandomAccessFile(index255File, "r") | ||
| val indexCount = index255.length().toInt() / ReadOnlyCache.INDEX_SIZE | ||
| return load(path, mainFile, main, index255File, index255, indexCount, xteas, threadUsage) | ||
| } | ||
|
|
||
| fun load(path: String, mainFile: File, main: RandomAccessFile, index255File: File, index255: RandomAccessFile, indexCount: Int, xteas: Map<Int, IntArray>? = null, threadUsage: Double = 1.0): Cache | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
79 changes: 79 additions & 0 deletions
79
cache/src/main/kotlin/world/gregs/voidps/cache/FileCache.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| package world.gregs.voidps.cache | ||
|
|
||
| import world.gregs.voidps.cache.compress.DecompressionContext | ||
| import java.io.File | ||
| import java.io.RandomAccessFile | ||
|
|
||
| /** | ||
| * [Cache] which reads data directly from file | ||
| * Average read speeds, fast loading and low but variable memory usage. | ||
| */ | ||
| class FileCache( | ||
| private val main: RandomAccessFile, | ||
| private val indexes: Array<RandomAccessFile?>, | ||
| indexCount: Int, | ||
| val xteas: Map<Int, IntArray>? | ||
| ) : ReadOnlyCache(indexCount) { | ||
|
|
||
| private val dataCache = object : LinkedHashMap<Int, Array<ByteArray?>>(16, 0.75f, true) { | ||
| override fun removeEldestEntry(eldest: MutableMap.MutableEntry<Int, Array<ByteArray?>>?): Boolean { | ||
| return size > 12 | ||
| } | ||
| } | ||
| private val length = main.length() | ||
| private val context = DecompressionContext() | ||
|
|
||
| override fun data(index: Int, archive: Int, file: Int, xtea: IntArray?): ByteArray? { | ||
| val matchingIndex = files.getOrNull(index)?.getOrNull(archive)?.indexOf(file) ?: -1 | ||
| if (matchingIndex == -1) { | ||
| return null | ||
| } | ||
| val hash = index + (archive shl 6) | ||
| val files = dataCache.getOrPut(hash) { | ||
| val indexRaf = indexes[index] ?: return null | ||
| readFileData(context, main, length, indexRaf, index, archive, xteas) ?: return null | ||
| } | ||
| return files[matchingIndex] | ||
| } | ||
|
|
||
| override fun close() { | ||
| main.close() | ||
| for (file in indexes) { | ||
| file?.close() | ||
| } | ||
| } | ||
|
|
||
| companion object : CacheLoader { | ||
| const val CACHE_FILE_NAME = "main_file_cache" | ||
|
|
||
| operator fun invoke(path: String, xteas: Map<Int, IntArray>? = null): Cache { | ||
| return load(path, xteas) | ||
| } | ||
|
|
||
| /** | ||
| * Create [RandomAccessFile]'s for each index file, load only the archive data into memory | ||
| */ | ||
| override fun load(path: String, mainFile: File, main: RandomAccessFile, index255File: File, index255: RandomAccessFile, indexCount: Int, xteas: Map<Int, IntArray>?, threadUsage: Double): Cache { | ||
| val length = mainFile.length() | ||
| val context = DecompressionContext() | ||
| val indices = Array(indexCount) { indexId -> | ||
| val file = File(path, "${CACHE_FILE_NAME}.idx$indexId") | ||
| if (file.exists()) RandomAccessFile(file, "r") else null | ||
| } | ||
| val cache = FileCache(main, indices, indexCount, xteas) | ||
| for (indexId in 0 until indexCount) { | ||
| cache.readArchiveData(context, main, length, index255, indexId) | ||
| } | ||
| return cache | ||
| } | ||
|
|
||
| @JvmStatic | ||
| fun main(args: Array<String>) { | ||
| val path = "./data/cache/" | ||
|
|
||
| val start = System.currentTimeMillis() | ||
| val cache = load(path) | ||
| println("Loaded cache in ${System.currentTimeMillis() - start}ms") | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.