diff --git a/TEKSteamClient.csproj b/TEKSteamClient.csproj index d149921..536d0e7 100644 --- a/TEKSteamClient.csproj +++ b/TEKSteamClient.csproj @@ -1,7 +1,7 @@ net8.0 - 1.2.0 + 1.3.0 $(BaseVersion) $(BaseVersion)-alpha.$(GITHUB_RUN_NUMBER) Nuclearist diff --git a/src/AppManager.cs b/src/AppManager.cs index 87c73fa..aa63383 100644 --- a/src/AppManager.cs +++ b/src/AppManager.cs @@ -94,7 +94,7 @@ private void PatchAndRelocateChunks(ItemState state, string localPath, DepotMani byte[] buffer = GC.AllocateUninitializedArray(delta.MaxTransferBufferSize); SafeFileHandle? intermediateFileHandle = null; var decoder = new Utils.LZMA.Decoder(); - void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliaryDir, string path, int recursionLevel) + void processDir(in DirectoryEntry.AuxiliaryEntry dir, string path, int recursionLevel) { int index; if (state.ProgressIndexStack.Count > recursionLevel) @@ -104,9 +104,9 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar state.ProgressIndexStack.Add(0); index = 0; } - for (; index < auxiliaryDir.Files.Count; index++) + for (; index < dir.Files.Count; index++) { - var auxiliaryFile = auxiliaryDir.Files[index]; + var auxiliaryFile = dir.Files[index]; if (auxiliaryFile.TransferOperations.Count is 0) continue; if (cancellationToken.IsCancellationRequested) @@ -114,7 +114,7 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar state.ProgressIndexStack[recursionLevel] = index; return; } - var file = dir.Files[auxiliaryFile.Index]; + var file = targetManifest.FileBuffer[auxiliaryFile.Index]; string filePath = Path.Combine(path, file.Name); var attributes = File.GetAttributes(filePath); if (attributes.HasFlag(FileAttributes.ReadOnly)) @@ -143,7 +143,7 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar if (transferOperation is FileEntry.AuxiliaryEntry.ChunkPatchEntry chunkPatch) { var sourceChunk = sourceManifest.ChunkBuffer[patch!.Chunks[chunkPatch.PatchChunkIndex].SourceChunkIndex]; - var targetChunk = file.Chunks[chunkPatch.ChunkIndex]; + var targetChunk = targetManifest.ChunkBuffer[chunkPatch.ChunkIndex]; if (sourceChunk.Offset + sourceChunk.UncompressedSize > fileSize) { state.Status = ItemState.ItemStatus.Corrupted; @@ -233,7 +233,7 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar if (transferOperation is FileEntry.AuxiliaryEntry.ChunkPatchEntry chunkPatch) { var sourceChunk = sourceManifest.ChunkBuffer[patch!.Chunks[chunkPatch.PatchChunkIndex].SourceChunkIndex]; - var targetChunk = file.Chunks[chunkPatch.ChunkIndex]; + var targetChunk = targetManifest.ChunkBuffer[chunkPatch.ChunkIndex]; if (targetChunk.UncompressedSize < sourceChunk.UncompressedSize) { var span = new Span(buffer, 0, targetChunk.UncompressedSize); @@ -266,17 +266,16 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar } state.ProgressIndexStack.RemoveAt(transferOpRecLevel); } - index -= auxiliaryDir.Files.Count; - for (; index < auxiliaryDir.Subdirectories.Count; index++) + index -= dir.Files.Count; + for (; index < dir.Subdirectories.Count; index++) { - var auxiliarySubdir = auxiliaryDir.Subdirectories[index]; - if (auxiliarySubdir.FilesToRemove.HasValue && auxiliarySubdir.FilesToRemove.Value.Count is 0) + var subdir = dir.Subdirectories[index]; + if (subdir.FilesToRemove.HasValue && subdir.FilesToRemove.Value.Count is 0) continue; - var subdir = dir.Subdirectories[auxiliarySubdir.Index]; - processDir(in subdir, in auxiliarySubdir, Path.Combine(path, subdir.Name), recursionLevel + 1); + processDir(in subdir, Path.Combine(path, targetManifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); if (cancellationToken.IsCancellationRequested) { - state.ProgressIndexStack[recursionLevel] = auxiliaryDir.Files.Count + index; + state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; return; } } @@ -292,7 +291,7 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar ProgressInitiated?.Invoke(ProgressType.Percentage, delta.NumTransferOperations, state.DisplayProgress); if (delta.IntermediateFileSize > 0) intermediateFileHandle = File.OpenHandle(Path.Combine(CdnClient.DownloadsDirectory!, $"{state.Id}.screlocpatchcache"), FileMode.OpenOrCreate, FileAccess.ReadWrite, options: FileOptions.SequentialScan); - processDir(in targetManifest.Root, in delta.AuxiliaryTree,localPath, 0); + processDir(in delta.AuxiliaryTree, localPath, 0); intermediateFileHandle?.Dispose(); if (cancellationToken.IsCancellationRequested) { @@ -311,7 +310,7 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar /// Token to monitor for cancellation requests. private void RemoveOldFiles(ItemState state, string localPath, DepotManifest sourceManifest, DepotManifest targetManifest, DepotDelta delta, CancellationToken cancellationToken) { - void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliaryDir, string path, int recursionLevel) + void processDir(in DirectoryEntry.AuxiliaryEntry dir, string path, int recursionLevel) { int index; if (state.ProgressIndexStack.Count > recursionLevel) @@ -321,9 +320,9 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar state.ProgressIndexStack.Add(0); index = 0; } - if (auxiliaryDir.FilesToRemove.HasValue) + if (dir.FilesToRemove.HasValue) { - var filesToRemove = auxiliaryDir.FilesToRemove.Value; + var filesToRemove = dir.FilesToRemove.Value; for (; index < filesToRemove.Count; index++) { if (cancellationToken.IsCancellationRequested) @@ -334,27 +333,26 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar File.Delete(Path.Combine(path, sourceManifest.FileBuffer[filesToRemove[index]].Name)); ProgressUpdated?.Invoke(++state.DisplayProgress); } - index -= auxiliaryDir.Files.Count; + index -= dir.Files.Count; } - for (; index < auxiliaryDir.Subdirectories.Count; index++) + for (; index < dir.Subdirectories.Count; index++) { if (cancellationToken.IsCancellationRequested) { - state.ProgressIndexStack[recursionLevel] = auxiliaryDir.Files.Count + index; + state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; return; } - var auxiliarySubdir = auxiliaryDir.Subdirectories[index]; - if (auxiliarySubdir.FilesToRemove.HasValue && auxiliarySubdir.FilesToRemove.Value.Count is 0) + var subdir = dir.Subdirectories[index]; + if (subdir.FilesToRemove.HasValue && subdir.FilesToRemove.Value.Count is 0) { - Directory.Delete(Path.Combine(path, sourceManifest.DirectoryBuffer[auxiliarySubdir.Index].Name), true); + Directory.Delete(Path.Combine(path, sourceManifest.DirectoryBuffer[subdir.Index].Name), true); ProgressUpdated?.Invoke(++state.DisplayProgress); continue; } - var subdir = dir.Subdirectories[auxiliarySubdir.Index]; - processDir(in subdir, in auxiliarySubdir, Path.Combine(path, subdir.Name), recursionLevel + 1); + processDir(in subdir, Path.Combine(path, targetManifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); if (cancellationToken.IsCancellationRequested) { - state.ProgressIndexStack[recursionLevel] = auxiliaryDir.Files.Count + index; + state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; return; } } @@ -368,7 +366,7 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar } StatusUpdated?.Invoke(Status.RemovingOldFiles); ProgressInitiated?.Invoke(ProgressType.Numeric, delta.NumRemovals, state.DisplayProgress); - processDir(in targetManifest.Root, in delta.AuxiliaryTree, localPath, 0); + processDir(in delta.AuxiliaryTree, localPath, 0); if (cancellationToken.IsCancellationRequested) { state.SaveToFile(); @@ -385,7 +383,7 @@ private void WriteNewData(ItemState state, string localPath, DepotManifest manif { byte[] buffer = GC.AllocateUninitializedArray(0x100000); SafeFileHandle? chunkBufferFileHandle = null; - void writeDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acquisitionDir, string downloadPath, string localPath, int recursionLevel) + void writeDir(in DirectoryEntry.AcquisitionEntry dir, string downloadPath, string localPath, int recursionLevel) { static long countTotalDirSize(in DirectoryEntry dir) { @@ -396,10 +394,10 @@ static long countTotalDirSize(in DirectoryEntry dir) result += countTotalDirSize(in subdir); return result; } - if (acquisitionDir.IsNew) + if (dir.IsNew) { Directory.Move(downloadPath, localPath); - state.DisplayProgress += countTotalDirSize(in dir); + state.DisplayProgress += countTotalDirSize(in manifest.DirectoryBuffer[dir.Index]); ProgressUpdated?.Invoke(state.DisplayProgress); return; } @@ -411,15 +409,15 @@ static long countTotalDirSize(in DirectoryEntry dir) state.ProgressIndexStack.Add(0); index = 0; } - for (; index < acquisitionDir.Files.Count; index++) + for (; index < dir.Files.Count; index++) { if (cancellationToken.IsCancellationRequested) { state.ProgressIndexStack[recursionLevel] = index; return; } - var acquisitonFile = acquisitionDir.Files[index]; - var file = dir.Files[acquisitonFile.Index]; + var acquisitonFile = dir.Files[index]; + var file = manifest.FileBuffer[acquisitonFile.Index]; if (acquisitonFile.Chunks.Count is 0) { string destinationFile = Path.Combine(localPath, file.Name); @@ -466,7 +464,7 @@ static long countTotalDirSize(in DirectoryEntry dir) return; } var acquisitionChunk = acquisitonFile.Chunks[chunkIndex]; - var chunk = file.Chunks[acquisitionChunk.Index]; + var chunk = manifest.ChunkBuffer[acquisitionChunk.Index]; var span = new Span(buffer, 0, chunk.UncompressedSize); RandomAccess.Read(chunkBufferFileHandle!, span, acquisitionChunk.Offset); RandomAccess.Write(fileHandle, span, chunk.Offset); @@ -476,15 +474,15 @@ static long countTotalDirSize(in DirectoryEntry dir) state.ProgressIndexStack.RemoveAt(chunkRecLevel); } } - index -= acquisitionDir.Files.Count; - for (; index < acquisitionDir.Subdirectories.Count; index++) + index -= dir.Files.Count; + for (; index < dir.Subdirectories.Count; index++) { - var acquisitionSubdir = acquisitionDir.Subdirectories[index]; - var subdir = dir.Subdirectories[acquisitionSubdir.Index]; - writeDir(in subdir, in acquisitionSubdir, Path.Combine(downloadPath, subdir.Name), Path.Combine(localPath, subdir.Name), recursionLevel + 1); + var subdir = dir.Subdirectories[index]; + string subdirName = manifest.DirectoryBuffer[subdir.Index].Name; + writeDir(in subdir, Path.Combine(downloadPath, subdirName), Path.Combine(localPath, subdirName), recursionLevel + 1); if (cancellationToken.IsCancellationRequested) { - state.ProgressIndexStack[recursionLevel] = acquisitionDir.Files.Count + index; + state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; return; } } @@ -500,7 +498,7 @@ static long countTotalDirSize(in DirectoryEntry dir) ProgressInitiated?.Invoke(ProgressType.Binary, delta.DownloadCacheSize - delta.IntermediateFileSize, state.DisplayProgress); if (delta.ChunkBufferFileSize > 0) chunkBufferFileHandle = File.OpenHandle(Path.Combine(CdnClient.DownloadsDirectory!, $"{state.Id}.scchunkbuffer"), options: FileOptions.SequentialScan); - writeDir(in manifest.Root, in delta.AcquisitionTree, Path.Combine(CdnClient.DownloadsDirectory!, state.Id.ToString()), localPath, 0); + writeDir(in delta.AcquisitionTree, Path.Combine(CdnClient.DownloadsDirectory!, state.Id.ToString()), localPath, 0); chunkBufferFileHandle?.Dispose(); if (cancellationToken.IsCancellationRequested) { @@ -524,12 +522,14 @@ private DepotDelta ComputeUpdateDelta(DepotManifest sourceManifest, DepotManifes var auxiliaryRoot = new DirectoryEntry.AuxiliaryStaging(0); void processDir(in DirectoryEntry sourceDir, in DirectoryEntry targetDir, DirectoryEntry.AcquisitionStaging acquisitionDir, DirectoryEntry.AuxiliaryStaging auxiliaryDir) { + int targetDirFileOffset = targetDir.Files.Offset; int i = 0, targetOffset = 0; for (; i < sourceDir.Files.Count && i + targetOffset < targetDir.Files.Count; i++) //Range intersecting both directories { int targetIndex = i + targetOffset; var sourceFile = sourceDir.Files[i]; var targetFile = targetDir.Files[targetIndex]; + targetIndex += targetDirFileOffset; int difference = string.Compare(sourceFile.Name, targetFile.Name, StringComparison.Ordinal); if (difference < 0) //File is present in the source directory but has been removed from the target one { @@ -547,12 +547,14 @@ void processDir(in DirectoryEntry sourceDir, in DirectoryEntry targetDir, Direct bool resized = false; var acquisitionFile = new FileEntry.AcquisitionStaging(targetIndex); var auxiliaryFile = new FileEntry.AuxiliaryStaging(targetIndex); + int targetFileChunkOffset = targetFile.Chunks.Offset; int j = 0, targetChunkOffset = 0; for (; j < sourceFile.Chunks.Count && j + targetChunkOffset < targetFile.Chunks.Count; j++) //Range intersecting both files { int targetChunkIndex = j + targetChunkOffset; var sourceChunk = sourceFile.Chunks[j]; var targetChunk = targetFile.Chunks[targetChunkIndex]; + targetChunkIndex += targetFileChunkOffset; int chunkDifference = sourceChunk.Gid.CompareTo(targetChunk.Gid); if (chunkDifference < 0) //Chunk has been removed in target version of the file { @@ -569,7 +571,7 @@ void processDir(in DirectoryEntry sourceDir, in DirectoryEntry targetDir, Direct { SourceChunkIndex = 0, TargetChunkIndex = targetFile.Chunks.Offset + targetChunkIndex, - Data = default + Data = ReadOnlyMemory.Empty }); if (patchChunkIndex >= 0) auxiliaryFile.ChunkPatches.Add(new() @@ -602,11 +604,12 @@ void processDir(in DirectoryEntry sourceDir, in DirectoryEntry targetDir, Direct acquisitionFile.Chunks.Add(new(j)); else { + int chunkIndex = targetFileChunkOffset + j; int patchChunkIndex = Array.BinarySearch(patch.Chunks, new PatchChunkEntry { SourceChunkIndex = 0, - TargetChunkIndex = targetFile.Chunks.Offset + j, - Data = default + TargetChunkIndex = chunkIndex, + Data = ReadOnlyMemory.Empty }); if (patchChunkIndex >= 0) auxiliaryFile.ChunkPatches.Add(new() @@ -614,10 +617,10 @@ void processDir(in DirectoryEntry sourceDir, in DirectoryEntry targetDir, Direct UseIntermediateFile = true, ChunkIndex = j, PatchChunkIndex = patchChunkIndex, - Size = Math.Min(targetFile.Chunks[j].UncompressedSize, sourceManifest.ChunkBuffer[patch.Chunks[patchChunkIndex].SourceChunkIndex].UncompressedSize) + Size = Math.Min(targetManifest.ChunkBuffer[chunkIndex].UncompressedSize, sourceManifest.ChunkBuffer[patch.Chunks[patchChunkIndex].SourceChunkIndex].UncompressedSize) }); else - acquisitionFile.Chunks.Add(new(j)); + acquisitionFile.Chunks.Add(new(chunkIndex)); } } if (acquisitionFile.Chunks.Count > 0) @@ -630,10 +633,10 @@ void processDir(in DirectoryEntry sourceDir, in DirectoryEntry targetDir, Direct { var chunkPatches = auxiliaryFile.ChunkPatches; var relocations = auxiliaryFile.Relocations; - //Batch relocation operations to reduce CPU load when computing weights and number of IO requests + //Batch relocation operations to reduce the number of IO requests and the CPU load when computing weights if (relocations.Count > 0) { - relocations.Sort((a, b) => a.SourceOffset.CompareTo(b.SourceOffset)); + relocations.Sort(); for (int k = 0; k < relocations.Count; k++) { var reloc = relocations[k]; @@ -764,18 +767,21 @@ void processDir(in DirectoryEntry sourceDir, in DirectoryEntry targetDir, Direct auxiliaryDir.FilesToRemove.Add(fileIndex++); } for (int j = i + targetOffset; j < targetDir.Files.Count; j++) //Add remaining files that are unique to the target directory - acquisitionDir.Files.Add(new(j)); + acquisitionDir.Files.Add(new(targetDirFileOffset + j)); static void addSubdir(DirectoryEntry.AcquisitionStaging acquisitionDir, in DirectoryEntry subdir, int subdirIndex) { var acquisitionSubdir = new DirectoryEntry.AcquisitionStaging(subdirIndex, true); acquisitionSubdir.Files.Capacity = subdir.Files.Count; + int subdirFileOffset = subdir.Files.Offset; for (int i = 0; i < subdir.Files.Count; i++) - acquisitionSubdir.Files.Add(new(i)); + acquisitionSubdir.Files.Add(new(subdirFileOffset + i)); acquisitionSubdir.Subdirectories.Capacity = subdir.Subdirectories.Count; + int subdirSubdirOffset = subdir.Subdirectories.Offset; for (int i = 0; i < subdir.Subdirectories.Count; i++) - addSubdir(acquisitionSubdir, subdir.Subdirectories[i], i); + addSubdir(acquisitionSubdir, subdir.Subdirectories[i], subdirSubdirOffset + i); acquisitionDir.Subdirectories.Add(acquisitionSubdir); } + int targetDirSubdirOffset = targetDir.Subdirectories.Offset; i = 0; targetOffset = 0; for (; i < sourceDir.Subdirectories.Count && i + targetOffset < targetDir.Subdirectories.Count; i++) //Range intersecting both directories @@ -783,6 +789,7 @@ static void addSubdir(DirectoryEntry.AcquisitionStaging acquisitionDir, in Direc int targetIndex = i + targetOffset; var sourceSubdir = sourceDir.Subdirectories[i]; var targetSubdir = targetDir.Subdirectories[targetIndex]; + targetIndex += targetDirSubdirOffset; int difference = string.Compare(sourceSubdir.Name, targetSubdir.Name, StringComparison.Ordinal); if (difference < 0) //Subdirectory is present in the source directory but has been removed from the target one { @@ -810,11 +817,11 @@ static void addSubdir(DirectoryEntry.AcquisitionStaging acquisitionDir, in Direc if (numRemainingSubdirs > 0) //Remove remaining subdirectories that are unique to the source directory { int subdirIndex = sourceDir.Subdirectories.Offset + i; - for (int j = 0; j < numRemainingFiles; j++) + for (int j = 0; j < numRemainingSubdirs; j++) auxiliaryDir.Subdirectories.Add(new(subdirIndex++) { FilesToRemove = [] }); } for (int j = i + targetOffset; j < targetDir.Subdirectories.Count; j++) //Add remaining subdirectories that are unique to the target directory - addSubdir(acquisitionDir, targetDir.Subdirectories[j], j); + addSubdir(acquisitionDir, targetDir.Subdirectories[j], targetDirSubdirOffset + j); } processDir(in sourceManifest.Root, in targetManifest.Root, acquisitionRoot, auxiliaryRoot); return new(targetManifest, acquisitionRoot, auxiliaryRoot); @@ -836,12 +843,12 @@ void copyDirToStagingAndCount(in DirectoryEntry directory, DirectoryEntry.Acquis for (int i = 0; i < files.Count; i++) { state.DisplayProgress += files[i].Size; - stagingDir.Files.Add(new(i)); + stagingDir.Files.Add(new(files.Offset + i)); } var subdirs = directory.Subdirectories; for (int i = 0; i < subdirs.Count; i++) { - var stagingSubDir = new DirectoryEntry.AcquisitionStaging(i, true); + var stagingSubDir = new DirectoryEntry.AcquisitionStaging(subdirs.Offset + i, true); copyDirToStagingAndCount(subdirs[i], stagingSubDir); stagingDir.Subdirectories.Add(stagingSubDir); } @@ -875,6 +882,7 @@ void validateDir(in DirectoryEntry directory, DirectoryEntry.AcquisitionStaging index = 0; continueType = 0; } + int dirFileOffset = directory.Files.Offset; for (; index < directory.Files.Count; index++) { if (cancellationToken.IsCancellationRequested) @@ -882,18 +890,19 @@ void validateDir(in DirectoryEntry directory, DirectoryEntry.AcquisitionStaging state.ProgressIndexStack[recursionLevel] = index; return; } + int absFileIndex = dirFileOffset + index; var file = directory.Files[index]; string filePath = Path.Combine(path, file.Name); if (!File.Exists(filePath)) { - stagingDir.Files.Add(new(index)); + stagingDir.Files.Add(new(absFileIndex)); cache.FilesMissing++; state.DisplayProgress += file.Size; ProgressUpdated?.Invoke(state.DisplayProgress); ValidationCounterUpdated?.Invoke(cache.FilesMissing, ValidationCounterType.Missing); continue; } - var stagingFile = continueType is 1 ? (stagingDir.Files.Find(f => f.Index == index) ?? new FileEntry.AcquisitionStaging(index)) : new FileEntry.AcquisitionStaging(index); + var stagingFile = continueType is 1 ? (stagingDir.Files.Find(f => f.Index == absFileIndex) ?? new FileEntry.AcquisitionStaging(absFileIndex)) : new FileEntry.AcquisitionStaging(absFileIndex); using var fileHandle = File.OpenHandle(filePath, options: FileOptions.RandomAccess); int chunkRecLevel = recursionLevel + 1; int chunkIndex; @@ -907,6 +916,7 @@ void validateDir(in DirectoryEntry directory, DirectoryEntry.AcquisitionStaging long fileSize = RandomAccess.GetLength(fileHandle); var chunks = file.Chunks; var span = new Span(buffer); + int fileChunkOffset = chunks.Offset; for (; chunkIndex < chunks.Count; chunkIndex++) { if (cancellationToken.IsCancellationRequested) @@ -915,15 +925,16 @@ void validateDir(in DirectoryEntry directory, DirectoryEntry.AcquisitionStaging state.ProgressIndexStack[recursionLevel] = index; return; } + int absChunkIndex = fileChunkOffset + chunkIndex; var chunk = chunks[chunkIndex]; if (chunk.Offset + chunk.UncompressedSize > fileSize) - stagingFile.Chunks.Add(new(chunkIndex)); + stagingFile.Chunks.Add(new(absChunkIndex)); else { var chunkSpan = span[..chunk.UncompressedSize]; RandomAccess.Read(fileHandle, chunkSpan, chunk.Offset); if (Adler.ComputeChecksum(chunkSpan) != chunk.Checksum) - stagingFile.Chunks.Add(new(chunkIndex)); + stagingFile.Chunks.Add(new(absChunkIndex)); } state.DisplayProgress += chunk.UncompressedSize; ProgressUpdated?.Invoke(state.DisplayProgress); @@ -940,10 +951,12 @@ void validateDir(in DirectoryEntry directory, DirectoryEntry.AcquisitionStaging ValidationCounterUpdated?.Invoke(++cache.FilesMatching, ValidationCounterType.Matching); } index -= directory.Files.Count; + int dirSubdirOffset = directory.Subdirectories.Offset; for (; index < directory.Subdirectories.Count; index++) { + int absSubdirIndex = dirSubdirOffset + index; var subdir = directory.Subdirectories[index]; - var stagingSubdir = continueType is 2 ? (stagingDir.Subdirectories.Find(sd => sd.Index == index) ?? new DirectoryEntry.AcquisitionStaging(index, false)) : new DirectoryEntry.AcquisitionStaging(index, false); + var stagingSubdir = continueType is 2 ? (stagingDir.Subdirectories.Find(sd => sd.Index == absSubdirIndex) ?? new DirectoryEntry.AcquisitionStaging(absSubdirIndex, false)) : new DirectoryEntry.AcquisitionStaging(absSubdirIndex, false); validateDir(in subdir, stagingSubdir, Path.Combine(path, subdir.Name), recursionLevel + 1); if (continueType is 2) continueType = 0; diff --git a/src/CDNClient.cs b/src/CDNClient.cs index 129e47d..2d685d1 100644 --- a/src/CDNClient.cs +++ b/src/CDNClient.cs @@ -152,7 +152,7 @@ internal void DownloadContent(ItemState state, DepotManifest manifest, DepotDelt string? chunkBufferFilePath = null; LimitedUseFileHandle? chunkBufferFileHandle = null; string baseRequestUrl = $"depot/{state.Id.DepotId}/chunk/"; - void downloadDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acquisitionDir, string path, int recursionLevel) + void downloadDir(in DirectoryEntry.AcquisitionEntry dir, string path, int recursionLevel) { int index; if (state.ProgressIndexStack.Count > recursionLevel) @@ -162,10 +162,10 @@ void downloadDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui state.ProgressIndexStack.Add(0); index = 0; } - for (; index < acquisitionDir.Files.Count; index++) + for (; index < dir.Files.Count; index++) { - var acquisitonFile = acquisitionDir.Files[index]; - var file = dir.Files[acquisitonFile.Index]; + var acquisitonFile = dir.Files[index]; + var file = manifest.FileBuffer[acquisitonFile.Index]; if (file.Size is 0) continue; if (linkedCts.IsCancellationRequested) @@ -182,10 +182,10 @@ void downloadDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui state.ProgressIndexStack.Add(0); chunkIndex = 0; } - var chunks = file.Chunks; if (acquisitonFile.Chunks.Count is 0) { string filePath = Path.Combine(path, file.Name); + var chunks = file.Chunks; LimitedUseFileHandle? handle; if (numResumedContexts > 0) { @@ -275,7 +275,7 @@ void downloadDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui return; } var acquisitionChunk = acquisitionChunks[chunkIndex]; - var chunk = chunks[acquisitionChunk.Index]; + var chunk = manifest.ChunkBuffer[acquisitionChunk.Index]; int contextIndex = -1; for (int i = 0; i < contexts.Length; i++) { @@ -332,15 +332,14 @@ void downloadDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui } state.ProgressIndexStack.RemoveAt(chunkRecLevel); } - index -= acquisitionDir.Files.Count; - for (; index < acquisitionDir.Subdirectories.Count; index++) + index -= dir.Files.Count; + for (; index < dir.Subdirectories.Count; index++) { - var acquisitionSubdir = acquisitionDir.Subdirectories[index]; - var subdir = dir.Subdirectories[acquisitionSubdir.Index]; - downloadDir(in subdir, in acquisitionSubdir, Path.Combine(path, subdir.Name), recursionLevel + 1); + var subdir = dir.Subdirectories[index]; + downloadDir(in subdir, Path.Combine(path, manifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); if (linkedCts.IsCancellationRequested) { - state.ProgressIndexStack[recursionLevel] = acquisitionDir.Files.Count + index; + state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; return; } } @@ -411,7 +410,7 @@ void downloadDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui for (int i = 0; i < numResumedContexts; i++) tasks[i] = Task.Factory.StartNew(AcquireChunk, contexts[i], TaskCreationOptions.DenyChildAttach); } - downloadDir(in manifest.Root, in delta.AcquisitionTree, Path.Combine(DownloadsDirectory!, state.Id.ToString()), 0); + downloadDir(in delta.AcquisitionTree, Path.Combine(DownloadsDirectory!, state.Id.ToString()), 0); foreach (var task in tasks) { if (task is null) @@ -458,7 +457,7 @@ void downloadDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui /// Token to monitor for cancellation requests. internal void Preallocate(ItemState state, DepotManifest manifest, DepotDelta delta, CancellationToken cancellationToken) { - void preallocDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acquisitionDir, string path, int recursionLevel) + void preallocDir(in DirectoryEntry.AcquisitionEntry dir, string path, int recursionLevel) { int index; if (state.ProgressIndexStack.Count > recursionLevel) @@ -467,12 +466,12 @@ void preallocDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui { state.ProgressIndexStack.Add(0); index = 0; - if (acquisitionDir.IsNew || acquisitionDir.Files.Any(a => a.Chunks.Count is 0)) + if (dir.IsNew || dir.Files.Any(a => a.Chunks.Count is 0)) Directory.CreateDirectory(path); } - for (; index < acquisitionDir.Files.Count; index++) + for (; index < dir.Files.Count; index++) { - var acquisitonFile = acquisitionDir.Files[index]; + var acquisitonFile = dir.Files[index]; if (acquisitonFile.Chunks.Count is not 0) continue; if (cancellationToken.IsCancellationRequested) @@ -480,21 +479,20 @@ void preallocDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui state.ProgressIndexStack[recursionLevel] = index; return; } - var file = dir.Files[acquisitonFile.Index]; + var file = manifest.FileBuffer[acquisitonFile.Index]; var handle = File.OpenHandle(Path.Combine(path, file.Name), FileMode.Create, FileAccess.Write, preallocationSize: file.Size); RandomAccess.SetLength(handle, file.Size); handle.Dispose(); ProgressUpdated?.Invoke(++state.DisplayProgress); } - index -= acquisitionDir.Files.Count; - for (; index < acquisitionDir.Subdirectories.Count; index++) + index -= dir.Files.Count; + for (; index < dir.Subdirectories.Count; index++) { - var acquisitionSubdir = acquisitionDir.Subdirectories[index]; - var subdir = dir.Subdirectories[acquisitionSubdir.Index]; - preallocDir(in subdir, in acquisitionSubdir, Path.Combine(path, subdir.Name), recursionLevel + 1); + var subdir = dir.Subdirectories[index]; + preallocDir(in subdir, Path.Combine(path, manifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); if (cancellationToken.IsCancellationRequested) { - state.ProgressIndexStack[recursionLevel] = acquisitionDir.Files.Count + index; + state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; return; } } @@ -510,7 +508,7 @@ void preallocDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui if (new DriveInfo(DownloadsDirectory!).AvailableFreeSpace < delta.DownloadCacheSize) throw new SteamNotEnoughDiskSpaceException(delta.DownloadCacheSize); ProgressInitiated?.Invoke(ProgressType.Numeric, delta.NumDownloadFiles, state.DisplayProgress); - preallocDir(in manifest.Root, in delta.AcquisitionTree, Path.Combine(DownloadsDirectory!, state.Id.ToString()), 0); + preallocDir(in delta.AcquisitionTree, Path.Combine(DownloadsDirectory!, state.Id.ToString()), 0); if (cancellationToken.IsCancellationRequested) { state.SaveToFile(); diff --git a/src/CM/WebSocketConnection.cs b/src/CM/WebSocketConnection.cs index 45e58d5..f539d0e 100644 --- a/src/CM/WebSocketConnection.cs +++ b/src/CM/WebSocketConnection.cs @@ -277,7 +277,10 @@ public void Connect() _socket.Dispose(); continue; } - _thread = new Thread(ConnectionLoop); + _thread = new Thread(ConnectionLoop) + { + Name = "CM WebSocket Connection Thread" + }; _thread.Start(); return; } diff --git a/src/Manifest/DepotDelta.cs b/src/Manifest/DepotDelta.cs index 9db7f1a..de88731 100644 --- a/src/Manifest/DepotDelta.cs +++ b/src/Manifest/DepotDelta.cs @@ -28,7 +28,7 @@ void countAcq(in DirectoryEntry dir, DirectoryEntry.AcquisitionStaging acquisiti foreach (var acquisitionFile in acquisitionDir.Files) { numChunks += acquisitionFile.Chunks.Count; - var file = dir.Files[acquisitionFile.Index]; + var file = manifest.FileBuffer[acquisitionFile.Index]; if (acquisitionFile.Chunks.Count is 0) { numDownloadFiles++; @@ -40,7 +40,7 @@ void countAcq(in DirectoryEntry dir, DirectoryEntry.AcquisitionStaging acquisiti for (int i = 0; i < acquisitionFile.Chunks.Count; i++) { int index = acquisitionFile.Chunks[i].Index; - var chunk = file.Chunks[index]; + var chunk = manifest.ChunkBuffer[index]; downloadSize += chunk.CompressedSize; acquisitionFile.Chunks[i] = new(index) { Offset = chunkBufferFileSize }; chunkBufferFileSize += chunk.UncompressedSize; diff --git a/src/Manifest/DirectoryEntry.cs b/src/Manifest/DirectoryEntry.cs index 19f527a..895bac2 100644 --- a/src/Manifest/DirectoryEntry.cs +++ b/src/Manifest/DirectoryEntry.cs @@ -14,7 +14,7 @@ internal readonly struct AcquisitionEntry { /// Indicates whether directory has been added or modifed. public required bool IsNew { get; init; } - /// Index of the directory entry in its parent directory. + /// Index of the directory entry in . public required int Index { get; init; } /// Directory's file entries. public required ArraySegment Files { get; init; } @@ -24,7 +24,7 @@ internal readonly struct AcquisitionEntry /// Structure used in to store auxiliary data like patched chunks and relocations for files or removed items. internal readonly struct AuxiliaryEntry { - /// Index of the directory entry in its parent directory, or in source manifest directory buffer if is empty. + /// Index of the directory entry in . If is empty, the index is for source manifest. public required int Index { get; init; } /// Indexes of the files that must be removed. If empty, the directory with all its contents is removed instead. public ArraySegment? FilesToRemove { get; init; } @@ -37,7 +37,7 @@ internal readonly struct AuxiliaryEntry internal class AcquisitionStaging { /// Creates an empty staging directory entry with specified index. - /// Index of the directory entry in its parent directory. + /// Index of the directory entry in . /// Value indicating whether directory has been added or modifed. public AcquisitionStaging(int index, bool isNew) { @@ -64,7 +64,7 @@ public AcquisitionStaging(ReadOnlySpan buffer, ref int offset) } /// Indicates whether directory has been added or modifed. public bool IsNew { get; internal set; } - /// Index of the directory entry in its parent directory. + /// Index of the directory entry in . public int Index { get; } /// Directory's file entries. public List Files { get; } @@ -90,7 +90,7 @@ public void WriteToBuffer(Span buffer, ref int offset) /// Index of the directory entry in its parent directory. internal class AuxiliaryStaging(int index) { - /// Index of the directory entry in its parent directory, or in source manifest directory buffer if is empty. + /// Index of the directory entry in . If is empty, the index is for source manifest. public int Index { get; } = index; /// Indexes of the files that must be removed. If empty, the directory with all its contents is removed instead. public List? FilesToRemove { get; set; } diff --git a/src/Manifest/FileEntry.cs b/src/Manifest/FileEntry.cs index 8c6b225..a8ac71c 100644 --- a/src/Manifest/FileEntry.cs +++ b/src/Manifest/FileEntry.cs @@ -29,7 +29,7 @@ public enum Flag /// Structure used in to store indexes of files and chunks that must be acquired. internal readonly struct AcquisitionEntry { - /// Index of the file entry in its parent directory. + /// Index of the file entry in . public required int Index { get; init; } /// File's chunks that must be acquired. If empty, the whole file is acquired. public required ArraySegment Chunks { get; init; } @@ -37,7 +37,7 @@ internal readonly struct AcquisitionEntry /// Index of the chunk in its file. public readonly struct ChunkEntry(int index) { - /// Index of the chunk entry in its file. + /// Index of the chunk entry in . public int Index { get; } = index; /// Offset of the chunk data from the beginning of chunk buffer file. public long Offset { get; init; } @@ -46,7 +46,7 @@ public readonly struct ChunkEntry(int index) /// Structure used in to store auxiliary data like patched chunks and relocations. internal readonly struct AuxiliaryEntry { - /// Index of the file entry in its parent directory. + /// Index of the file entry in . public required int Index { get; init; } /// File's chunk patch and relocation entries. public ArraySegment TransferOperations { get; init; } @@ -55,7 +55,7 @@ public interface ITransferOperation { } /// Contains data that is needed to patch a chunk. public class ChunkPatchEntry : ITransferOperation { - /// Index of the target chunk in the file. + /// Index of target chunk entry in . public required int ChunkIndex { get; init; } /// Index of the corresponding patch chunk. public required int PatchChunkIndex { get; init; } @@ -96,7 +96,7 @@ public AcquisitionStaging(ReadOnlySpan buffer, ref int offset) for (int i = 0; i < numChunks; i++) Chunks.Add(new(buffer[offset++])); } - /// Index of the file entry in its parent directory. + /// Index of the file entry in . public int Index { get; } /// File's chunks that must be acquired. If empty, the whole file is acquired. public List Chunks { get; } @@ -115,7 +115,7 @@ public void WriteToBuffer(Span buffer, ref int offset) /// Index of the file entry in its parent directory. internal class AuxiliaryStaging(int index) { - /// Index of the file entry in its parent directory. + /// Index of the file entry in . public int Index { get; } = index; /// File's chunk patch entries. public List ChunkPatches { get; } = []; @@ -130,7 +130,7 @@ public class ChunkPatchEntry : ITransferOperation { /// Indicates whether intermediate file needs to be used as a buffer to avoid overlapping further chunks. public bool UseIntermediateFile { get; set; } - /// Index of the target chunk in the file. + /// Index of target chunk entry in . public required int ChunkIndex { get; init; } /// Index of the corresponding patch chunk. public required int PatchChunkIndex { get; init; } @@ -138,7 +138,7 @@ public class ChunkPatchEntry : ITransferOperation public required long Size { get; init; } } /// Describes data that needs to be moved within the file. - public class RelocationEntry : ITransferOperation + public class RelocationEntry : ITransferOperation, IComparable { /// Indicates whether intermediate file needs to be used as a buffer to avoid overlapping further relocations and chunk patches. public bool UseIntermediateFile { get; set; } @@ -148,6 +148,8 @@ public class RelocationEntry : ITransferOperation public required long TargetOffset { get; init; } /// Size of data bulk. public required long Size { get; set; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int CompareTo(RelocationEntry? other) => other is null ? 1 : SourceOffset.CompareTo(other.SourceOffset); } /// Represents an operation that takes data from certain region of a file and writes data to another region of a file. public class TransferOperation : IComparable