From d58a0bb27bfa8c95ebc5698f5b06caf163487957 Mon Sep 17 00:00:00 2001 From: Nuclearist Date: Thu, 25 Jan 2024 15:57:39 +0300 Subject: [PATCH 1/4] Switched to absolute manifest buffer indexes in Acq and Aux entries --- TEKSteamClient.csproj | 2 +- src/AppManager.cs | 133 ++++++++++++++++++--------------- src/CDNClient.cs | 48 ++++++------ src/CM/WebSocketConnection.cs | 5 +- src/Manifest/DepotDelta.cs | 4 +- src/Manifest/DirectoryEntry.cs | 10 +-- src/Manifest/FileEntry.cs | 18 +++-- 7 files changed, 118 insertions(+), 102 deletions(-) diff --git a/TEKSteamClient.csproj b/TEKSteamClient.csproj index d149921..536d0e7 100644 --- a/TEKSteamClient.csproj +++ b/TEKSteamClient.csproj @@ -1,7 +1,7 @@ net8.0 - 1.2.0 + 1.3.0 $(BaseVersion) $(BaseVersion)-alpha.$(GITHUB_RUN_NUMBER) Nuclearist diff --git a/src/AppManager.cs b/src/AppManager.cs index 87c73fa..aa63383 100644 --- a/src/AppManager.cs +++ b/src/AppManager.cs @@ -94,7 +94,7 @@ private void PatchAndRelocateChunks(ItemState state, string localPath, DepotMani byte[] buffer = GC.AllocateUninitializedArray(delta.MaxTransferBufferSize); SafeFileHandle? intermediateFileHandle = null; var decoder = new Utils.LZMA.Decoder(); - void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliaryDir, string path, int recursionLevel) + void processDir(in DirectoryEntry.AuxiliaryEntry dir, string path, int recursionLevel) { int index; if (state.ProgressIndexStack.Count > recursionLevel) @@ -104,9 +104,9 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar state.ProgressIndexStack.Add(0); index = 0; } - for (; index < auxiliaryDir.Files.Count; index++) + for (; index < dir.Files.Count; index++) { - var auxiliaryFile = auxiliaryDir.Files[index]; + var auxiliaryFile = dir.Files[index]; if (auxiliaryFile.TransferOperations.Count is 0) continue; if (cancellationToken.IsCancellationRequested) @@ -114,7 +114,7 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar state.ProgressIndexStack[recursionLevel] = index; return; } - var file = dir.Files[auxiliaryFile.Index]; + var file = targetManifest.FileBuffer[auxiliaryFile.Index]; string filePath = Path.Combine(path, file.Name); var attributes = File.GetAttributes(filePath); if (attributes.HasFlag(FileAttributes.ReadOnly)) @@ -143,7 +143,7 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar if (transferOperation is FileEntry.AuxiliaryEntry.ChunkPatchEntry chunkPatch) { var sourceChunk = sourceManifest.ChunkBuffer[patch!.Chunks[chunkPatch.PatchChunkIndex].SourceChunkIndex]; - var targetChunk = file.Chunks[chunkPatch.ChunkIndex]; + var targetChunk = targetManifest.ChunkBuffer[chunkPatch.ChunkIndex]; if (sourceChunk.Offset + sourceChunk.UncompressedSize > fileSize) { state.Status = ItemState.ItemStatus.Corrupted; @@ -233,7 +233,7 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar if (transferOperation is FileEntry.AuxiliaryEntry.ChunkPatchEntry chunkPatch) { var sourceChunk = sourceManifest.ChunkBuffer[patch!.Chunks[chunkPatch.PatchChunkIndex].SourceChunkIndex]; - var targetChunk = file.Chunks[chunkPatch.ChunkIndex]; + var targetChunk = targetManifest.ChunkBuffer[chunkPatch.ChunkIndex]; if (targetChunk.UncompressedSize < sourceChunk.UncompressedSize) { var span = new Span(buffer, 0, targetChunk.UncompressedSize); @@ -266,17 +266,16 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar } state.ProgressIndexStack.RemoveAt(transferOpRecLevel); } - index -= auxiliaryDir.Files.Count; - for (; index < auxiliaryDir.Subdirectories.Count; index++) + index -= dir.Files.Count; + for (; index < dir.Subdirectories.Count; index++) { - var auxiliarySubdir = auxiliaryDir.Subdirectories[index]; - if (auxiliarySubdir.FilesToRemove.HasValue && auxiliarySubdir.FilesToRemove.Value.Count is 0) + var subdir = dir.Subdirectories[index]; + if (subdir.FilesToRemove.HasValue && subdir.FilesToRemove.Value.Count is 0) continue; - var subdir = dir.Subdirectories[auxiliarySubdir.Index]; - processDir(in subdir, in auxiliarySubdir, Path.Combine(path, subdir.Name), recursionLevel + 1); + processDir(in subdir, Path.Combine(path, targetManifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); if (cancellationToken.IsCancellationRequested) { - state.ProgressIndexStack[recursionLevel] = auxiliaryDir.Files.Count + index; + state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; return; } } @@ -292,7 +291,7 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar ProgressInitiated?.Invoke(ProgressType.Percentage, delta.NumTransferOperations, state.DisplayProgress); if (delta.IntermediateFileSize > 0) intermediateFileHandle = File.OpenHandle(Path.Combine(CdnClient.DownloadsDirectory!, $"{state.Id}.screlocpatchcache"), FileMode.OpenOrCreate, FileAccess.ReadWrite, options: FileOptions.SequentialScan); - processDir(in targetManifest.Root, in delta.AuxiliaryTree,localPath, 0); + processDir(in delta.AuxiliaryTree, localPath, 0); intermediateFileHandle?.Dispose(); if (cancellationToken.IsCancellationRequested) { @@ -311,7 +310,7 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar /// Token to monitor for cancellation requests. private void RemoveOldFiles(ItemState state, string localPath, DepotManifest sourceManifest, DepotManifest targetManifest, DepotDelta delta, CancellationToken cancellationToken) { - void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliaryDir, string path, int recursionLevel) + void processDir(in DirectoryEntry.AuxiliaryEntry dir, string path, int recursionLevel) { int index; if (state.ProgressIndexStack.Count > recursionLevel) @@ -321,9 +320,9 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar state.ProgressIndexStack.Add(0); index = 0; } - if (auxiliaryDir.FilesToRemove.HasValue) + if (dir.FilesToRemove.HasValue) { - var filesToRemove = auxiliaryDir.FilesToRemove.Value; + var filesToRemove = dir.FilesToRemove.Value; for (; index < filesToRemove.Count; index++) { if (cancellationToken.IsCancellationRequested) @@ -334,27 +333,26 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar File.Delete(Path.Combine(path, sourceManifest.FileBuffer[filesToRemove[index]].Name)); ProgressUpdated?.Invoke(++state.DisplayProgress); } - index -= auxiliaryDir.Files.Count; + index -= dir.Files.Count; } - for (; index < auxiliaryDir.Subdirectories.Count; index++) + for (; index < dir.Subdirectories.Count; index++) { if (cancellationToken.IsCancellationRequested) { - state.ProgressIndexStack[recursionLevel] = auxiliaryDir.Files.Count + index; + state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; return; } - var auxiliarySubdir = auxiliaryDir.Subdirectories[index]; - if (auxiliarySubdir.FilesToRemove.HasValue && auxiliarySubdir.FilesToRemove.Value.Count is 0) + var subdir = dir.Subdirectories[index]; + if (subdir.FilesToRemove.HasValue && subdir.FilesToRemove.Value.Count is 0) { - Directory.Delete(Path.Combine(path, sourceManifest.DirectoryBuffer[auxiliarySubdir.Index].Name), true); + Directory.Delete(Path.Combine(path, sourceManifest.DirectoryBuffer[subdir.Index].Name), true); ProgressUpdated?.Invoke(++state.DisplayProgress); continue; } - var subdir = dir.Subdirectories[auxiliarySubdir.Index]; - processDir(in subdir, in auxiliarySubdir, Path.Combine(path, subdir.Name), recursionLevel + 1); + processDir(in subdir, Path.Combine(path, targetManifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); if (cancellationToken.IsCancellationRequested) { - state.ProgressIndexStack[recursionLevel] = auxiliaryDir.Files.Count + index; + state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; return; } } @@ -368,7 +366,7 @@ void processDir(in DirectoryEntry dir, in DirectoryEntry.AuxiliaryEntry auxiliar } StatusUpdated?.Invoke(Status.RemovingOldFiles); ProgressInitiated?.Invoke(ProgressType.Numeric, delta.NumRemovals, state.DisplayProgress); - processDir(in targetManifest.Root, in delta.AuxiliaryTree, localPath, 0); + processDir(in delta.AuxiliaryTree, localPath, 0); if (cancellationToken.IsCancellationRequested) { state.SaveToFile(); @@ -385,7 +383,7 @@ private void WriteNewData(ItemState state, string localPath, DepotManifest manif { byte[] buffer = GC.AllocateUninitializedArray(0x100000); SafeFileHandle? chunkBufferFileHandle = null; - void writeDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acquisitionDir, string downloadPath, string localPath, int recursionLevel) + void writeDir(in DirectoryEntry.AcquisitionEntry dir, string downloadPath, string localPath, int recursionLevel) { static long countTotalDirSize(in DirectoryEntry dir) { @@ -396,10 +394,10 @@ static long countTotalDirSize(in DirectoryEntry dir) result += countTotalDirSize(in subdir); return result; } - if (acquisitionDir.IsNew) + if (dir.IsNew) { Directory.Move(downloadPath, localPath); - state.DisplayProgress += countTotalDirSize(in dir); + state.DisplayProgress += countTotalDirSize(in manifest.DirectoryBuffer[dir.Index]); ProgressUpdated?.Invoke(state.DisplayProgress); return; } @@ -411,15 +409,15 @@ static long countTotalDirSize(in DirectoryEntry dir) state.ProgressIndexStack.Add(0); index = 0; } - for (; index < acquisitionDir.Files.Count; index++) + for (; index < dir.Files.Count; index++) { if (cancellationToken.IsCancellationRequested) { state.ProgressIndexStack[recursionLevel] = index; return; } - var acquisitonFile = acquisitionDir.Files[index]; - var file = dir.Files[acquisitonFile.Index]; + var acquisitonFile = dir.Files[index]; + var file = manifest.FileBuffer[acquisitonFile.Index]; if (acquisitonFile.Chunks.Count is 0) { string destinationFile = Path.Combine(localPath, file.Name); @@ -466,7 +464,7 @@ static long countTotalDirSize(in DirectoryEntry dir) return; } var acquisitionChunk = acquisitonFile.Chunks[chunkIndex]; - var chunk = file.Chunks[acquisitionChunk.Index]; + var chunk = manifest.ChunkBuffer[acquisitionChunk.Index]; var span = new Span(buffer, 0, chunk.UncompressedSize); RandomAccess.Read(chunkBufferFileHandle!, span, acquisitionChunk.Offset); RandomAccess.Write(fileHandle, span, chunk.Offset); @@ -476,15 +474,15 @@ static long countTotalDirSize(in DirectoryEntry dir) state.ProgressIndexStack.RemoveAt(chunkRecLevel); } } - index -= acquisitionDir.Files.Count; - for (; index < acquisitionDir.Subdirectories.Count; index++) + index -= dir.Files.Count; + for (; index < dir.Subdirectories.Count; index++) { - var acquisitionSubdir = acquisitionDir.Subdirectories[index]; - var subdir = dir.Subdirectories[acquisitionSubdir.Index]; - writeDir(in subdir, in acquisitionSubdir, Path.Combine(downloadPath, subdir.Name), Path.Combine(localPath, subdir.Name), recursionLevel + 1); + var subdir = dir.Subdirectories[index]; + string subdirName = manifest.DirectoryBuffer[subdir.Index].Name; + writeDir(in subdir, Path.Combine(downloadPath, subdirName), Path.Combine(localPath, subdirName), recursionLevel + 1); if (cancellationToken.IsCancellationRequested) { - state.ProgressIndexStack[recursionLevel] = acquisitionDir.Files.Count + index; + state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; return; } } @@ -500,7 +498,7 @@ static long countTotalDirSize(in DirectoryEntry dir) ProgressInitiated?.Invoke(ProgressType.Binary, delta.DownloadCacheSize - delta.IntermediateFileSize, state.DisplayProgress); if (delta.ChunkBufferFileSize > 0) chunkBufferFileHandle = File.OpenHandle(Path.Combine(CdnClient.DownloadsDirectory!, $"{state.Id}.scchunkbuffer"), options: FileOptions.SequentialScan); - writeDir(in manifest.Root, in delta.AcquisitionTree, Path.Combine(CdnClient.DownloadsDirectory!, state.Id.ToString()), localPath, 0); + writeDir(in delta.AcquisitionTree, Path.Combine(CdnClient.DownloadsDirectory!, state.Id.ToString()), localPath, 0); chunkBufferFileHandle?.Dispose(); if (cancellationToken.IsCancellationRequested) { @@ -524,12 +522,14 @@ private DepotDelta ComputeUpdateDelta(DepotManifest sourceManifest, DepotManifes var auxiliaryRoot = new DirectoryEntry.AuxiliaryStaging(0); void processDir(in DirectoryEntry sourceDir, in DirectoryEntry targetDir, DirectoryEntry.AcquisitionStaging acquisitionDir, DirectoryEntry.AuxiliaryStaging auxiliaryDir) { + int targetDirFileOffset = targetDir.Files.Offset; int i = 0, targetOffset = 0; for (; i < sourceDir.Files.Count && i + targetOffset < targetDir.Files.Count; i++) //Range intersecting both directories { int targetIndex = i + targetOffset; var sourceFile = sourceDir.Files[i]; var targetFile = targetDir.Files[targetIndex]; + targetIndex += targetDirFileOffset; int difference = string.Compare(sourceFile.Name, targetFile.Name, StringComparison.Ordinal); if (difference < 0) //File is present in the source directory but has been removed from the target one { @@ -547,12 +547,14 @@ void processDir(in DirectoryEntry sourceDir, in DirectoryEntry targetDir, Direct bool resized = false; var acquisitionFile = new FileEntry.AcquisitionStaging(targetIndex); var auxiliaryFile = new FileEntry.AuxiliaryStaging(targetIndex); + int targetFileChunkOffset = targetFile.Chunks.Offset; int j = 0, targetChunkOffset = 0; for (; j < sourceFile.Chunks.Count && j + targetChunkOffset < targetFile.Chunks.Count; j++) //Range intersecting both files { int targetChunkIndex = j + targetChunkOffset; var sourceChunk = sourceFile.Chunks[j]; var targetChunk = targetFile.Chunks[targetChunkIndex]; + targetChunkIndex += targetFileChunkOffset; int chunkDifference = sourceChunk.Gid.CompareTo(targetChunk.Gid); if (chunkDifference < 0) //Chunk has been removed in target version of the file { @@ -569,7 +571,7 @@ void processDir(in DirectoryEntry sourceDir, in DirectoryEntry targetDir, Direct { SourceChunkIndex = 0, TargetChunkIndex = targetFile.Chunks.Offset + targetChunkIndex, - Data = default + Data = ReadOnlyMemory.Empty }); if (patchChunkIndex >= 0) auxiliaryFile.ChunkPatches.Add(new() @@ -602,11 +604,12 @@ void processDir(in DirectoryEntry sourceDir, in DirectoryEntry targetDir, Direct acquisitionFile.Chunks.Add(new(j)); else { + int chunkIndex = targetFileChunkOffset + j; int patchChunkIndex = Array.BinarySearch(patch.Chunks, new PatchChunkEntry { SourceChunkIndex = 0, - TargetChunkIndex = targetFile.Chunks.Offset + j, - Data = default + TargetChunkIndex = chunkIndex, + Data = ReadOnlyMemory.Empty }); if (patchChunkIndex >= 0) auxiliaryFile.ChunkPatches.Add(new() @@ -614,10 +617,10 @@ void processDir(in DirectoryEntry sourceDir, in DirectoryEntry targetDir, Direct UseIntermediateFile = true, ChunkIndex = j, PatchChunkIndex = patchChunkIndex, - Size = Math.Min(targetFile.Chunks[j].UncompressedSize, sourceManifest.ChunkBuffer[patch.Chunks[patchChunkIndex].SourceChunkIndex].UncompressedSize) + Size = Math.Min(targetManifest.ChunkBuffer[chunkIndex].UncompressedSize, sourceManifest.ChunkBuffer[patch.Chunks[patchChunkIndex].SourceChunkIndex].UncompressedSize) }); else - acquisitionFile.Chunks.Add(new(j)); + acquisitionFile.Chunks.Add(new(chunkIndex)); } } if (acquisitionFile.Chunks.Count > 0) @@ -630,10 +633,10 @@ void processDir(in DirectoryEntry sourceDir, in DirectoryEntry targetDir, Direct { var chunkPatches = auxiliaryFile.ChunkPatches; var relocations = auxiliaryFile.Relocations; - //Batch relocation operations to reduce CPU load when computing weights and number of IO requests + //Batch relocation operations to reduce the number of IO requests and the CPU load when computing weights if (relocations.Count > 0) { - relocations.Sort((a, b) => a.SourceOffset.CompareTo(b.SourceOffset)); + relocations.Sort(); for (int k = 0; k < relocations.Count; k++) { var reloc = relocations[k]; @@ -764,18 +767,21 @@ void processDir(in DirectoryEntry sourceDir, in DirectoryEntry targetDir, Direct auxiliaryDir.FilesToRemove.Add(fileIndex++); } for (int j = i + targetOffset; j < targetDir.Files.Count; j++) //Add remaining files that are unique to the target directory - acquisitionDir.Files.Add(new(j)); + acquisitionDir.Files.Add(new(targetDirFileOffset + j)); static void addSubdir(DirectoryEntry.AcquisitionStaging acquisitionDir, in DirectoryEntry subdir, int subdirIndex) { var acquisitionSubdir = new DirectoryEntry.AcquisitionStaging(subdirIndex, true); acquisitionSubdir.Files.Capacity = subdir.Files.Count; + int subdirFileOffset = subdir.Files.Offset; for (int i = 0; i < subdir.Files.Count; i++) - acquisitionSubdir.Files.Add(new(i)); + acquisitionSubdir.Files.Add(new(subdirFileOffset + i)); acquisitionSubdir.Subdirectories.Capacity = subdir.Subdirectories.Count; + int subdirSubdirOffset = subdir.Subdirectories.Offset; for (int i = 0; i < subdir.Subdirectories.Count; i++) - addSubdir(acquisitionSubdir, subdir.Subdirectories[i], i); + addSubdir(acquisitionSubdir, subdir.Subdirectories[i], subdirSubdirOffset + i); acquisitionDir.Subdirectories.Add(acquisitionSubdir); } + int targetDirSubdirOffset = targetDir.Subdirectories.Offset; i = 0; targetOffset = 0; for (; i < sourceDir.Subdirectories.Count && i + targetOffset < targetDir.Subdirectories.Count; i++) //Range intersecting both directories @@ -783,6 +789,7 @@ static void addSubdir(DirectoryEntry.AcquisitionStaging acquisitionDir, in Direc int targetIndex = i + targetOffset; var sourceSubdir = sourceDir.Subdirectories[i]; var targetSubdir = targetDir.Subdirectories[targetIndex]; + targetIndex += targetDirSubdirOffset; int difference = string.Compare(sourceSubdir.Name, targetSubdir.Name, StringComparison.Ordinal); if (difference < 0) //Subdirectory is present in the source directory but has been removed from the target one { @@ -810,11 +817,11 @@ static void addSubdir(DirectoryEntry.AcquisitionStaging acquisitionDir, in Direc if (numRemainingSubdirs > 0) //Remove remaining subdirectories that are unique to the source directory { int subdirIndex = sourceDir.Subdirectories.Offset + i; - for (int j = 0; j < numRemainingFiles; j++) + for (int j = 0; j < numRemainingSubdirs; j++) auxiliaryDir.Subdirectories.Add(new(subdirIndex++) { FilesToRemove = [] }); } for (int j = i + targetOffset; j < targetDir.Subdirectories.Count; j++) //Add remaining subdirectories that are unique to the target directory - addSubdir(acquisitionDir, targetDir.Subdirectories[j], j); + addSubdir(acquisitionDir, targetDir.Subdirectories[j], targetDirSubdirOffset + j); } processDir(in sourceManifest.Root, in targetManifest.Root, acquisitionRoot, auxiliaryRoot); return new(targetManifest, acquisitionRoot, auxiliaryRoot); @@ -836,12 +843,12 @@ void copyDirToStagingAndCount(in DirectoryEntry directory, DirectoryEntry.Acquis for (int i = 0; i < files.Count; i++) { state.DisplayProgress += files[i].Size; - stagingDir.Files.Add(new(i)); + stagingDir.Files.Add(new(files.Offset + i)); } var subdirs = directory.Subdirectories; for (int i = 0; i < subdirs.Count; i++) { - var stagingSubDir = new DirectoryEntry.AcquisitionStaging(i, true); + var stagingSubDir = new DirectoryEntry.AcquisitionStaging(subdirs.Offset + i, true); copyDirToStagingAndCount(subdirs[i], stagingSubDir); stagingDir.Subdirectories.Add(stagingSubDir); } @@ -875,6 +882,7 @@ void validateDir(in DirectoryEntry directory, DirectoryEntry.AcquisitionStaging index = 0; continueType = 0; } + int dirFileOffset = directory.Files.Offset; for (; index < directory.Files.Count; index++) { if (cancellationToken.IsCancellationRequested) @@ -882,18 +890,19 @@ void validateDir(in DirectoryEntry directory, DirectoryEntry.AcquisitionStaging state.ProgressIndexStack[recursionLevel] = index; return; } + int absFileIndex = dirFileOffset + index; var file = directory.Files[index]; string filePath = Path.Combine(path, file.Name); if (!File.Exists(filePath)) { - stagingDir.Files.Add(new(index)); + stagingDir.Files.Add(new(absFileIndex)); cache.FilesMissing++; state.DisplayProgress += file.Size; ProgressUpdated?.Invoke(state.DisplayProgress); ValidationCounterUpdated?.Invoke(cache.FilesMissing, ValidationCounterType.Missing); continue; } - var stagingFile = continueType is 1 ? (stagingDir.Files.Find(f => f.Index == index) ?? new FileEntry.AcquisitionStaging(index)) : new FileEntry.AcquisitionStaging(index); + var stagingFile = continueType is 1 ? (stagingDir.Files.Find(f => f.Index == absFileIndex) ?? new FileEntry.AcquisitionStaging(absFileIndex)) : new FileEntry.AcquisitionStaging(absFileIndex); using var fileHandle = File.OpenHandle(filePath, options: FileOptions.RandomAccess); int chunkRecLevel = recursionLevel + 1; int chunkIndex; @@ -907,6 +916,7 @@ void validateDir(in DirectoryEntry directory, DirectoryEntry.AcquisitionStaging long fileSize = RandomAccess.GetLength(fileHandle); var chunks = file.Chunks; var span = new Span(buffer); + int fileChunkOffset = chunks.Offset; for (; chunkIndex < chunks.Count; chunkIndex++) { if (cancellationToken.IsCancellationRequested) @@ -915,15 +925,16 @@ void validateDir(in DirectoryEntry directory, DirectoryEntry.AcquisitionStaging state.ProgressIndexStack[recursionLevel] = index; return; } + int absChunkIndex = fileChunkOffset + chunkIndex; var chunk = chunks[chunkIndex]; if (chunk.Offset + chunk.UncompressedSize > fileSize) - stagingFile.Chunks.Add(new(chunkIndex)); + stagingFile.Chunks.Add(new(absChunkIndex)); else { var chunkSpan = span[..chunk.UncompressedSize]; RandomAccess.Read(fileHandle, chunkSpan, chunk.Offset); if (Adler.ComputeChecksum(chunkSpan) != chunk.Checksum) - stagingFile.Chunks.Add(new(chunkIndex)); + stagingFile.Chunks.Add(new(absChunkIndex)); } state.DisplayProgress += chunk.UncompressedSize; ProgressUpdated?.Invoke(state.DisplayProgress); @@ -940,10 +951,12 @@ void validateDir(in DirectoryEntry directory, DirectoryEntry.AcquisitionStaging ValidationCounterUpdated?.Invoke(++cache.FilesMatching, ValidationCounterType.Matching); } index -= directory.Files.Count; + int dirSubdirOffset = directory.Subdirectories.Offset; for (; index < directory.Subdirectories.Count; index++) { + int absSubdirIndex = dirSubdirOffset + index; var subdir = directory.Subdirectories[index]; - var stagingSubdir = continueType is 2 ? (stagingDir.Subdirectories.Find(sd => sd.Index == index) ?? new DirectoryEntry.AcquisitionStaging(index, false)) : new DirectoryEntry.AcquisitionStaging(index, false); + var stagingSubdir = continueType is 2 ? (stagingDir.Subdirectories.Find(sd => sd.Index == absSubdirIndex) ?? new DirectoryEntry.AcquisitionStaging(absSubdirIndex, false)) : new DirectoryEntry.AcquisitionStaging(absSubdirIndex, false); validateDir(in subdir, stagingSubdir, Path.Combine(path, subdir.Name), recursionLevel + 1); if (continueType is 2) continueType = 0; diff --git a/src/CDNClient.cs b/src/CDNClient.cs index 129e47d..2d685d1 100644 --- a/src/CDNClient.cs +++ b/src/CDNClient.cs @@ -152,7 +152,7 @@ internal void DownloadContent(ItemState state, DepotManifest manifest, DepotDelt string? chunkBufferFilePath = null; LimitedUseFileHandle? chunkBufferFileHandle = null; string baseRequestUrl = $"depot/{state.Id.DepotId}/chunk/"; - void downloadDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acquisitionDir, string path, int recursionLevel) + void downloadDir(in DirectoryEntry.AcquisitionEntry dir, string path, int recursionLevel) { int index; if (state.ProgressIndexStack.Count > recursionLevel) @@ -162,10 +162,10 @@ void downloadDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui state.ProgressIndexStack.Add(0); index = 0; } - for (; index < acquisitionDir.Files.Count; index++) + for (; index < dir.Files.Count; index++) { - var acquisitonFile = acquisitionDir.Files[index]; - var file = dir.Files[acquisitonFile.Index]; + var acquisitonFile = dir.Files[index]; + var file = manifest.FileBuffer[acquisitonFile.Index]; if (file.Size is 0) continue; if (linkedCts.IsCancellationRequested) @@ -182,10 +182,10 @@ void downloadDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui state.ProgressIndexStack.Add(0); chunkIndex = 0; } - var chunks = file.Chunks; if (acquisitonFile.Chunks.Count is 0) { string filePath = Path.Combine(path, file.Name); + var chunks = file.Chunks; LimitedUseFileHandle? handle; if (numResumedContexts > 0) { @@ -275,7 +275,7 @@ void downloadDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui return; } var acquisitionChunk = acquisitionChunks[chunkIndex]; - var chunk = chunks[acquisitionChunk.Index]; + var chunk = manifest.ChunkBuffer[acquisitionChunk.Index]; int contextIndex = -1; for (int i = 0; i < contexts.Length; i++) { @@ -332,15 +332,14 @@ void downloadDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui } state.ProgressIndexStack.RemoveAt(chunkRecLevel); } - index -= acquisitionDir.Files.Count; - for (; index < acquisitionDir.Subdirectories.Count; index++) + index -= dir.Files.Count; + for (; index < dir.Subdirectories.Count; index++) { - var acquisitionSubdir = acquisitionDir.Subdirectories[index]; - var subdir = dir.Subdirectories[acquisitionSubdir.Index]; - downloadDir(in subdir, in acquisitionSubdir, Path.Combine(path, subdir.Name), recursionLevel + 1); + var subdir = dir.Subdirectories[index]; + downloadDir(in subdir, Path.Combine(path, manifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); if (linkedCts.IsCancellationRequested) { - state.ProgressIndexStack[recursionLevel] = acquisitionDir.Files.Count + index; + state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; return; } } @@ -411,7 +410,7 @@ void downloadDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui for (int i = 0; i < numResumedContexts; i++) tasks[i] = Task.Factory.StartNew(AcquireChunk, contexts[i], TaskCreationOptions.DenyChildAttach); } - downloadDir(in manifest.Root, in delta.AcquisitionTree, Path.Combine(DownloadsDirectory!, state.Id.ToString()), 0); + downloadDir(in delta.AcquisitionTree, Path.Combine(DownloadsDirectory!, state.Id.ToString()), 0); foreach (var task in tasks) { if (task is null) @@ -458,7 +457,7 @@ void downloadDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui /// Token to monitor for cancellation requests. internal void Preallocate(ItemState state, DepotManifest manifest, DepotDelta delta, CancellationToken cancellationToken) { - void preallocDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acquisitionDir, string path, int recursionLevel) + void preallocDir(in DirectoryEntry.AcquisitionEntry dir, string path, int recursionLevel) { int index; if (state.ProgressIndexStack.Count > recursionLevel) @@ -467,12 +466,12 @@ void preallocDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui { state.ProgressIndexStack.Add(0); index = 0; - if (acquisitionDir.IsNew || acquisitionDir.Files.Any(a => a.Chunks.Count is 0)) + if (dir.IsNew || dir.Files.Any(a => a.Chunks.Count is 0)) Directory.CreateDirectory(path); } - for (; index < acquisitionDir.Files.Count; index++) + for (; index < dir.Files.Count; index++) { - var acquisitonFile = acquisitionDir.Files[index]; + var acquisitonFile = dir.Files[index]; if (acquisitonFile.Chunks.Count is not 0) continue; if (cancellationToken.IsCancellationRequested) @@ -480,21 +479,20 @@ void preallocDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui state.ProgressIndexStack[recursionLevel] = index; return; } - var file = dir.Files[acquisitonFile.Index]; + var file = manifest.FileBuffer[acquisitonFile.Index]; var handle = File.OpenHandle(Path.Combine(path, file.Name), FileMode.Create, FileAccess.Write, preallocationSize: file.Size); RandomAccess.SetLength(handle, file.Size); handle.Dispose(); ProgressUpdated?.Invoke(++state.DisplayProgress); } - index -= acquisitionDir.Files.Count; - for (; index < acquisitionDir.Subdirectories.Count; index++) + index -= dir.Files.Count; + for (; index < dir.Subdirectories.Count; index++) { - var acquisitionSubdir = acquisitionDir.Subdirectories[index]; - var subdir = dir.Subdirectories[acquisitionSubdir.Index]; - preallocDir(in subdir, in acquisitionSubdir, Path.Combine(path, subdir.Name), recursionLevel + 1); + var subdir = dir.Subdirectories[index]; + preallocDir(in subdir, Path.Combine(path, manifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); if (cancellationToken.IsCancellationRequested) { - state.ProgressIndexStack[recursionLevel] = acquisitionDir.Files.Count + index; + state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; return; } } @@ -510,7 +508,7 @@ void preallocDir(in DirectoryEntry dir, in DirectoryEntry.AcquisitionEntry acqui if (new DriveInfo(DownloadsDirectory!).AvailableFreeSpace < delta.DownloadCacheSize) throw new SteamNotEnoughDiskSpaceException(delta.DownloadCacheSize); ProgressInitiated?.Invoke(ProgressType.Numeric, delta.NumDownloadFiles, state.DisplayProgress); - preallocDir(in manifest.Root, in delta.AcquisitionTree, Path.Combine(DownloadsDirectory!, state.Id.ToString()), 0); + preallocDir(in delta.AcquisitionTree, Path.Combine(DownloadsDirectory!, state.Id.ToString()), 0); if (cancellationToken.IsCancellationRequested) { state.SaveToFile(); diff --git a/src/CM/WebSocketConnection.cs b/src/CM/WebSocketConnection.cs index 45e58d5..f539d0e 100644 --- a/src/CM/WebSocketConnection.cs +++ b/src/CM/WebSocketConnection.cs @@ -277,7 +277,10 @@ public void Connect() _socket.Dispose(); continue; } - _thread = new Thread(ConnectionLoop); + _thread = new Thread(ConnectionLoop) + { + Name = "CM WebSocket Connection Thread" + }; _thread.Start(); return; } diff --git a/src/Manifest/DepotDelta.cs b/src/Manifest/DepotDelta.cs index 9db7f1a..de88731 100644 --- a/src/Manifest/DepotDelta.cs +++ b/src/Manifest/DepotDelta.cs @@ -28,7 +28,7 @@ void countAcq(in DirectoryEntry dir, DirectoryEntry.AcquisitionStaging acquisiti foreach (var acquisitionFile in acquisitionDir.Files) { numChunks += acquisitionFile.Chunks.Count; - var file = dir.Files[acquisitionFile.Index]; + var file = manifest.FileBuffer[acquisitionFile.Index]; if (acquisitionFile.Chunks.Count is 0) { numDownloadFiles++; @@ -40,7 +40,7 @@ void countAcq(in DirectoryEntry dir, DirectoryEntry.AcquisitionStaging acquisiti for (int i = 0; i < acquisitionFile.Chunks.Count; i++) { int index = acquisitionFile.Chunks[i].Index; - var chunk = file.Chunks[index]; + var chunk = manifest.ChunkBuffer[index]; downloadSize += chunk.CompressedSize; acquisitionFile.Chunks[i] = new(index) { Offset = chunkBufferFileSize }; chunkBufferFileSize += chunk.UncompressedSize; diff --git a/src/Manifest/DirectoryEntry.cs b/src/Manifest/DirectoryEntry.cs index 19f527a..895bac2 100644 --- a/src/Manifest/DirectoryEntry.cs +++ b/src/Manifest/DirectoryEntry.cs @@ -14,7 +14,7 @@ internal readonly struct AcquisitionEntry { /// Indicates whether directory has been added or modifed. public required bool IsNew { get; init; } - /// Index of the directory entry in its parent directory. + /// Index of the directory entry in . public required int Index { get; init; } /// Directory's file entries. public required ArraySegment Files { get; init; } @@ -24,7 +24,7 @@ internal readonly struct AcquisitionEntry /// Structure used in to store auxiliary data like patched chunks and relocations for files or removed items. internal readonly struct AuxiliaryEntry { - /// Index of the directory entry in its parent directory, or in source manifest directory buffer if is empty. + /// Index of the directory entry in . If is empty, the index is for source manifest. public required int Index { get; init; } /// Indexes of the files that must be removed. If empty, the directory with all its contents is removed instead. public ArraySegment? FilesToRemove { get; init; } @@ -37,7 +37,7 @@ internal readonly struct AuxiliaryEntry internal class AcquisitionStaging { /// Creates an empty staging directory entry with specified index. - /// Index of the directory entry in its parent directory. + /// Index of the directory entry in . /// Value indicating whether directory has been added or modifed. public AcquisitionStaging(int index, bool isNew) { @@ -64,7 +64,7 @@ public AcquisitionStaging(ReadOnlySpan buffer, ref int offset) } /// Indicates whether directory has been added or modifed. public bool IsNew { get; internal set; } - /// Index of the directory entry in its parent directory. + /// Index of the directory entry in . public int Index { get; } /// Directory's file entries. public List Files { get; } @@ -90,7 +90,7 @@ public void WriteToBuffer(Span buffer, ref int offset) /// Index of the directory entry in its parent directory. internal class AuxiliaryStaging(int index) { - /// Index of the directory entry in its parent directory, or in source manifest directory buffer if is empty. + /// Index of the directory entry in . If is empty, the index is for source manifest. public int Index { get; } = index; /// Indexes of the files that must be removed. If empty, the directory with all its contents is removed instead. public List? FilesToRemove { get; set; } diff --git a/src/Manifest/FileEntry.cs b/src/Manifest/FileEntry.cs index 8c6b225..a8ac71c 100644 --- a/src/Manifest/FileEntry.cs +++ b/src/Manifest/FileEntry.cs @@ -29,7 +29,7 @@ public enum Flag /// Structure used in to store indexes of files and chunks that must be acquired. internal readonly struct AcquisitionEntry { - /// Index of the file entry in its parent directory. + /// Index of the file entry in . public required int Index { get; init; } /// File's chunks that must be acquired. If empty, the whole file is acquired. public required ArraySegment Chunks { get; init; } @@ -37,7 +37,7 @@ internal readonly struct AcquisitionEntry /// Index of the chunk in its file. public readonly struct ChunkEntry(int index) { - /// Index of the chunk entry in its file. + /// Index of the chunk entry in . public int Index { get; } = index; /// Offset of the chunk data from the beginning of chunk buffer file. public long Offset { get; init; } @@ -46,7 +46,7 @@ public readonly struct ChunkEntry(int index) /// Structure used in to store auxiliary data like patched chunks and relocations. internal readonly struct AuxiliaryEntry { - /// Index of the file entry in its parent directory. + /// Index of the file entry in . public required int Index { get; init; } /// File's chunk patch and relocation entries. public ArraySegment TransferOperations { get; init; } @@ -55,7 +55,7 @@ public interface ITransferOperation { } /// Contains data that is needed to patch a chunk. public class ChunkPatchEntry : ITransferOperation { - /// Index of the target chunk in the file. + /// Index of target chunk entry in . public required int ChunkIndex { get; init; } /// Index of the corresponding patch chunk. public required int PatchChunkIndex { get; init; } @@ -96,7 +96,7 @@ public AcquisitionStaging(ReadOnlySpan buffer, ref int offset) for (int i = 0; i < numChunks; i++) Chunks.Add(new(buffer[offset++])); } - /// Index of the file entry in its parent directory. + /// Index of the file entry in . public int Index { get; } /// File's chunks that must be acquired. If empty, the whole file is acquired. public List Chunks { get; } @@ -115,7 +115,7 @@ public void WriteToBuffer(Span buffer, ref int offset) /// Index of the file entry in its parent directory. internal class AuxiliaryStaging(int index) { - /// Index of the file entry in its parent directory. + /// Index of the file entry in . public int Index { get; } = index; /// File's chunk patch entries. public List ChunkPatches { get; } = []; @@ -130,7 +130,7 @@ public class ChunkPatchEntry : ITransferOperation { /// Indicates whether intermediate file needs to be used as a buffer to avoid overlapping further chunks. public bool UseIntermediateFile { get; set; } - /// Index of the target chunk in the file. + /// Index of target chunk entry in . public required int ChunkIndex { get; init; } /// Index of the corresponding patch chunk. public required int PatchChunkIndex { get; init; } @@ -138,7 +138,7 @@ public class ChunkPatchEntry : ITransferOperation public required long Size { get; init; } } /// Describes data that needs to be moved within the file. - public class RelocationEntry : ITransferOperation + public class RelocationEntry : ITransferOperation, IComparable { /// Indicates whether intermediate file needs to be used as a buffer to avoid overlapping further relocations and chunk patches. public bool UseIntermediateFile { get; set; } @@ -148,6 +148,8 @@ public class RelocationEntry : ITransferOperation public required long TargetOffset { get; init; } /// Size of data bulk. public required long Size { get; set; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int CompareTo(RelocationEntry? other) => other is null ? 1 : SourceOffset.CompareTo(other.SourceOffset); } /// Represents an operation that takes data from certain region of a file and writes data to another region of a file. public class TransferOperation : IComparable From 2a7c1c8268630fab6a6c0a2650fcff688da04128 Mon Sep 17 00:00:00 2001 From: Nuclearist Date: Fri, 26 Jan 2024 10:40:23 +0300 Subject: [PATCH 2/4] Replaced Path.Combine with Path.Join --- src/AppManager.cs | 58 +++++++++++++++++++++++------------------------ src/CDNClient.cs | 28 +++++++++++------------ 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/AppManager.cs b/src/AppManager.cs index aa63383..ea87e5d 100644 --- a/src/AppManager.cs +++ b/src/AppManager.cs @@ -20,12 +20,12 @@ public AppManager(uint appId, string installationPath, [Optional]string? worksho { AppId = appId; InstallationPath = installationPath; - _scDataPath = Path.Combine(installationPath, "SCData"); - WorkshopContentPath = workshopContentPath ?? Path.Combine(_scDataPath, "Workshop"); + _scDataPath = Path.Join(installationPath, "SCData"); + WorkshopContentPath = workshopContentPath ?? Path.Join(_scDataPath, "Workshop"); CdnClient = new() { - DownloadsDirectory = Path.Combine(_scDataPath, "Downloads"), - ManifestsDirectory = Path.Combine(_scDataPath, "Manifests"), + DownloadsDirectory = Path.Join(_scDataPath, "Downloads"), + ManifestsDirectory = Path.Join(_scDataPath, "Manifests"), CmClient = CmClient }; CdnClient.ProgressInitiated += (type, totalValue, initialValue) => ProgressInitiated?.Invoke(type, totalValue, initialValue); @@ -72,7 +72,7 @@ public AppManager(uint appId, string installationPath, [Optional]string? worksho /// Token to monitor for cancellation requests. private void Commit(ItemState state, DepotManifest? sourceManifest, DepotManifest targetManifest, DepotPatch? patch, DepotDelta delta, CancellationToken cancellationToken) { - string localPath = state.Id.WorkshopItemId is 0 ? InstallationPath : Path.Combine(WorkshopContentPath, state.Id.WorkshopItemId.ToString()); + string localPath = state.Id.WorkshopItemId is 0 ? InstallationPath : Path.Join(WorkshopContentPath, state.Id.WorkshopItemId.ToString()); if (state.Status <= ItemState.ItemStatus.Patching && delta.NumTransferOperations > 0) PatchAndRelocateChunks(state, localPath, sourceManifest!, targetManifest, patch, delta, cancellationToken); if (state.Status <= ItemState.ItemStatus.WritingNewData) @@ -115,7 +115,7 @@ void processDir(in DirectoryEntry.AuxiliaryEntry dir, string path, int recursion return; } var file = targetManifest.FileBuffer[auxiliaryFile.Index]; - string filePath = Path.Combine(path, file.Name); + string filePath = Path.Join(path, file.Name); var attributes = File.GetAttributes(filePath); if (attributes.HasFlag(FileAttributes.ReadOnly)) File.SetAttributes(filePath, attributes & ~FileAttributes.ReadOnly); @@ -272,7 +272,7 @@ void processDir(in DirectoryEntry.AuxiliaryEntry dir, string path, int recursion var subdir = dir.Subdirectories[index]; if (subdir.FilesToRemove.HasValue && subdir.FilesToRemove.Value.Count is 0) continue; - processDir(in subdir, Path.Combine(path, targetManifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); + processDir(in subdir, Path.Join(path, targetManifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); if (cancellationToken.IsCancellationRequested) { state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; @@ -290,7 +290,7 @@ void processDir(in DirectoryEntry.AuxiliaryEntry dir, string path, int recursion StatusUpdated?.Invoke(Status.Patching); ProgressInitiated?.Invoke(ProgressType.Percentage, delta.NumTransferOperations, state.DisplayProgress); if (delta.IntermediateFileSize > 0) - intermediateFileHandle = File.OpenHandle(Path.Combine(CdnClient.DownloadsDirectory!, $"{state.Id}.screlocpatchcache"), FileMode.OpenOrCreate, FileAccess.ReadWrite, options: FileOptions.SequentialScan); + intermediateFileHandle = File.OpenHandle(Path.Join(CdnClient.DownloadsDirectory!, $"{state.Id}.screlocpatchcache"), FileMode.OpenOrCreate, FileAccess.ReadWrite, options: FileOptions.SequentialScan); processDir(in delta.AuxiliaryTree, localPath, 0); intermediateFileHandle?.Dispose(); if (cancellationToken.IsCancellationRequested) @@ -299,7 +299,7 @@ void processDir(in DirectoryEntry.AuxiliaryEntry dir, string path, int recursion throw new OperationCanceledException(cancellationToken); } if (delta.IntermediateFileSize > 0) - File.Delete(Path.Combine(CdnClient.DownloadsDirectory!, $"{state.Id}.screlocpatchcache")); + File.Delete(Path.Join(CdnClient.DownloadsDirectory!, $"{state.Id}.screlocpatchcache")); } /// Deletes files and directories that have been removed from the target manifest. /// State of the item. @@ -330,7 +330,7 @@ void processDir(in DirectoryEntry.AuxiliaryEntry dir, string path, int recursion state.ProgressIndexStack[recursionLevel] = index; return; } - File.Delete(Path.Combine(path, sourceManifest.FileBuffer[filesToRemove[index]].Name)); + File.Delete(Path.Join(path, sourceManifest.FileBuffer[filesToRemove[index]].Name)); ProgressUpdated?.Invoke(++state.DisplayProgress); } index -= dir.Files.Count; @@ -345,11 +345,11 @@ void processDir(in DirectoryEntry.AuxiliaryEntry dir, string path, int recursion var subdir = dir.Subdirectories[index]; if (subdir.FilesToRemove.HasValue && subdir.FilesToRemove.Value.Count is 0) { - Directory.Delete(Path.Combine(path, sourceManifest.DirectoryBuffer[subdir.Index].Name), true); + Directory.Delete(Path.Join(path, sourceManifest.DirectoryBuffer[subdir.Index].Name), true); ProgressUpdated?.Invoke(++state.DisplayProgress); continue; } - processDir(in subdir, Path.Combine(path, targetManifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); + processDir(in subdir, Path.Join(path, targetManifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); if (cancellationToken.IsCancellationRequested) { state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; @@ -420,10 +420,10 @@ static long countTotalDirSize(in DirectoryEntry dir) var file = manifest.FileBuffer[acquisitonFile.Index]; if (acquisitonFile.Chunks.Count is 0) { - string destinationFile = Path.Combine(localPath, file.Name); + string destinationFile = Path.Join(localPath, file.Name); if (File.Exists(destinationFile)) File.Delete(destinationFile); - File.Move(Path.Combine(downloadPath, file.Name), destinationFile); + File.Move(Path.Join(downloadPath, file.Name), destinationFile); if (file.Flags is not 0) { var attributes = (FileAttributes)0; @@ -450,7 +450,7 @@ static long countTotalDirSize(in DirectoryEntry dir) state.ProgressIndexStack.Add(0); chunkIndex = 0; } - string filePath = Path.Combine(localPath, file.Name); + string filePath = Path.Join(localPath, file.Name); var attributes = File.GetAttributes(filePath); if (attributes.HasFlag(FileAttributes.ReadOnly)) File.SetAttributes(filePath, attributes & ~FileAttributes.ReadOnly); @@ -479,7 +479,7 @@ static long countTotalDirSize(in DirectoryEntry dir) { var subdir = dir.Subdirectories[index]; string subdirName = manifest.DirectoryBuffer[subdir.Index].Name; - writeDir(in subdir, Path.Combine(downloadPath, subdirName), Path.Combine(localPath, subdirName), recursionLevel + 1); + writeDir(in subdir, Path.Join(downloadPath, subdirName), Path.Join(localPath, subdirName), recursionLevel + 1); if (cancellationToken.IsCancellationRequested) { state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; @@ -497,19 +497,19 @@ static long countTotalDirSize(in DirectoryEntry dir) StatusUpdated?.Invoke(Status.WritingNewData); ProgressInitiated?.Invoke(ProgressType.Binary, delta.DownloadCacheSize - delta.IntermediateFileSize, state.DisplayProgress); if (delta.ChunkBufferFileSize > 0) - chunkBufferFileHandle = File.OpenHandle(Path.Combine(CdnClient.DownloadsDirectory!, $"{state.Id}.scchunkbuffer"), options: FileOptions.SequentialScan); - writeDir(in delta.AcquisitionTree, Path.Combine(CdnClient.DownloadsDirectory!, state.Id.ToString()), localPath, 0); + chunkBufferFileHandle = File.OpenHandle(Path.Join(CdnClient.DownloadsDirectory!, $"{state.Id}.scchunkbuffer"), options: FileOptions.SequentialScan); + writeDir(in delta.AcquisitionTree, Path.Join(CdnClient.DownloadsDirectory!, state.Id.ToString()), localPath, 0); chunkBufferFileHandle?.Dispose(); if (cancellationToken.IsCancellationRequested) { state.SaveToFile(); throw new OperationCanceledException(cancellationToken); } - string downloadDirPath = Path.Combine(CdnClient.DownloadsDirectory!, state.Id.ToString()); + string downloadDirPath = Path.Join(CdnClient.DownloadsDirectory!, state.Id.ToString()); if (Directory.Exists(downloadDirPath)) Directory.Delete(downloadDirPath, true); if (delta.ChunkBufferFileSize > 0) - File.Delete(Path.Combine(CdnClient.DownloadsDirectory!, $"{state.Id}.scchunkbuffer")); + File.Delete(Path.Join(CdnClient.DownloadsDirectory!, $"{state.Id}.scchunkbuffer")); } /// Computes difference between source and target manifests' contents /// Source manifest to compute difference from. @@ -833,7 +833,7 @@ static void addSubdir(DirectoryEntry.AcquisitionStaging acquisitionDir, in Direc /// Depot delta object containing index tree for missing chunks. private DepotDelta Validate(ItemState state, DepotManifest manifest, CancellationToken cancellationToken) { - string cachePath = Path.Combine(CdnClient.DownloadsDirectory!, $"{manifest.Item}-{manifest.Id}.scvcache"); + string cachePath = Path.Join(CdnClient.DownloadsDirectory!, $"{manifest.Item}-{manifest.Id}.scvcache"); var cache = File.Exists(cachePath) ? new ValidationCache(cachePath) : new(); byte[] buffer = GC.AllocateUninitializedArray(0x100000); void copyDirToStagingAndCount(in DirectoryEntry directory, DirectoryEntry.AcquisitionStaging stagingDir) @@ -892,7 +892,7 @@ void validateDir(in DirectoryEntry directory, DirectoryEntry.AcquisitionStaging } int absFileIndex = dirFileOffset + index; var file = directory.Files[index]; - string filePath = Path.Combine(path, file.Name); + string filePath = Path.Join(path, file.Name); if (!File.Exists(filePath)) { stagingDir.Files.Add(new(absFileIndex)); @@ -957,7 +957,7 @@ void validateDir(in DirectoryEntry directory, DirectoryEntry.AcquisitionStaging int absSubdirIndex = dirSubdirOffset + index; var subdir = directory.Subdirectories[index]; var stagingSubdir = continueType is 2 ? (stagingDir.Subdirectories.Find(sd => sd.Index == absSubdirIndex) ?? new DirectoryEntry.AcquisitionStaging(absSubdirIndex, false)) : new DirectoryEntry.AcquisitionStaging(absSubdirIndex, false); - validateDir(in subdir, stagingSubdir, Path.Combine(path, subdir.Name), recursionLevel + 1); + validateDir(in subdir, stagingSubdir, Path.Join(path, subdir.Name), recursionLevel + 1); if (continueType is 2) continueType = 0; else if (stagingSubdir.IsNew || stagingSubdir.Files.Count > 0 || stagingSubdir.Subdirectories.Count > 0) @@ -971,7 +971,7 @@ void validateDir(in DirectoryEntry directory, DirectoryEntry.AcquisitionStaging state.ProgressIndexStack.RemoveAt(recursionLevel); return; } - string basePath = state.Id.WorkshopItemId is 0 ? InstallationPath : Path.Combine(WorkshopContentPath, state.Id.WorkshopItemId.ToString()); + string basePath = state.Id.WorkshopItemId is 0 ? InstallationPath : Path.Join(WorkshopContentPath, state.Id.WorkshopItemId.ToString()); if (state.Status is not ItemState.ItemStatus.Validating) { state.Status = ItemState.ItemStatus.Validating; @@ -1012,7 +1012,7 @@ public bool Update(ItemIdentifier item, CancellationToken cancellationToken, [Op targetManifestId = CmClient.GetWorkshopItemManifestId(AppId, item.WorkshopItemId); } } - var state = new ItemState(item, Path.Combine(_scDataPath, $"{item}.scitemstate")); + var state = new ItemState(item, Path.Join(_scDataPath, $"{item}.scitemstate")); if (state.CurrentManifestId is 0) return Validate(item, cancellationToken, targetManifestId); //Cannot update when source version is unknown if (state.CurrentManifestId == targetManifestId) @@ -1022,7 +1022,7 @@ public bool Update(ItemIdentifier item, CancellationToken cancellationToken, [Op var patch = CmClient.GetPatchAvailability(AppId, item.DepotId, sourceManifest.Id, targetManifest.Id) ? CdnClient.GetPatch(AppId, item, sourceManifest, targetManifest, cancellationToken) : null; - string deltaFilePath = Path.Combine(CdnClient.DownloadsDirectory!, $"{item}-{sourceManifest.Id}-{targetManifest.Id}.scdelta"); + string deltaFilePath = Path.Join(CdnClient.DownloadsDirectory!, $"{item}-{sourceManifest.Id}-{targetManifest.Id}.scdelta"); DepotDelta delta; if (state.Status < ItemState.ItemStatus.Preallocating) { @@ -1043,7 +1043,7 @@ public bool Update(ItemIdentifier item, CancellationToken cancellationToken, [Op state.SaveToFile(); File.Delete(deltaFilePath); if (patch is not null) - File.Delete(Path.Combine(CdnClient.DownloadsDirectory!, $"{item}-{sourceManifest.Id}-{targetManifest.Id}.scpatch")); + File.Delete(Path.Join(CdnClient.DownloadsDirectory!, $"{item}-{sourceManifest.Id}-{targetManifest.Id}.scpatch")); return false; } /// Verifies item files, downloads and installs missing data. @@ -1067,9 +1067,9 @@ public bool Validate(ItemIdentifier item, CancellationToken cancellationToken, [ manifestId = CmClient.GetWorkshopItemManifestId(AppId, item.WorkshopItemId); } } - var state = new ItemState(item, Path.Combine(_scDataPath, $"{item}.scitemstate")); + var state = new ItemState(item, Path.Join(_scDataPath, $"{item}.scitemstate")); var manifest = CdnClient.GetManifest(AppId, item, manifestId, cancellationToken); - string deltaFilePath = Path.Combine(CdnClient.DownloadsDirectory!, $"{item}-{manifestId}.scdelta"); + string deltaFilePath = Path.Join(CdnClient.DownloadsDirectory!, $"{item}-{manifestId}.scdelta"); DepotDelta delta; if (state.Status <= ItemState.ItemStatus.Validating) { diff --git a/src/CDNClient.cs b/src/CDNClient.cs index 2d685d1..5f0f7df 100644 --- a/src/CDNClient.cs +++ b/src/CDNClient.cs @@ -184,7 +184,7 @@ void downloadDir(in DirectoryEntry.AcquisitionEntry dir, string path, int recurs } if (acquisitonFile.Chunks.Count is 0) { - string filePath = Path.Combine(path, file.Name); + string filePath = Path.Join(path, file.Name); var chunks = file.Chunks; LimitedUseFileHandle? handle; if (numResumedContexts > 0) @@ -336,7 +336,7 @@ void downloadDir(in DirectoryEntry.AcquisitionEntry dir, string path, int recurs for (; index < dir.Subdirectories.Count; index++) { var subdir = dir.Subdirectories[index]; - downloadDir(in subdir, Path.Combine(path, manifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); + downloadDir(in subdir, Path.Join(path, manifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); if (linkedCts.IsCancellationRequested) { state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; @@ -369,11 +369,11 @@ void downloadDir(in DirectoryEntry.AcquisitionEntry dir, string path, int recurs contexts[i] = new(threadSafeProgress, httpClients[i % httpClients.Length], linkedCts.Token); contexts[i].Aes.Key = decryptionKey; } - string dwContextsFilePath = Path.Combine(DownloadsDirectory!, $"{state.Id}.scdwcontexts"); + string dwContextsFilePath = Path.Join(DownloadsDirectory!, $"{state.Id}.scdwcontexts"); ProgressInitiated?.Invoke(ProgressType.Binary, delta.DownloadSize, state.DisplayProgress); if (delta.ChunkBufferFileSize > 0) { - chunkBufferFilePath = Path.Combine(DownloadsDirectory!, $"{state.Id}.scchunkbuffer"); + chunkBufferFilePath = Path.Join(DownloadsDirectory!, $"{state.Id}.scchunkbuffer"); chunkBufferFileHandle = new(File.OpenHandle(chunkBufferFilePath, FileMode.OpenOrCreate, FileAccess.Write, options: FileOptions.RandomAccess | FileOptions.Asynchronous), int.MaxValue); } if (File.Exists(dwContextsFilePath)) @@ -410,7 +410,7 @@ void downloadDir(in DirectoryEntry.AcquisitionEntry dir, string path, int recurs for (int i = 0; i < numResumedContexts; i++) tasks[i] = Task.Factory.StartNew(AcquireChunk, contexts[i], TaskCreationOptions.DenyChildAttach); } - downloadDir(in delta.AcquisitionTree, Path.Combine(DownloadsDirectory!, state.Id.ToString()), 0); + downloadDir(in delta.AcquisitionTree, Path.Join(DownloadsDirectory!, state.Id.ToString()), 0); foreach (var task in tasks) { if (task is null) @@ -480,7 +480,7 @@ void preallocDir(in DirectoryEntry.AcquisitionEntry dir, string path, int recurs return; } var file = manifest.FileBuffer[acquisitonFile.Index]; - var handle = File.OpenHandle(Path.Combine(path, file.Name), FileMode.Create, FileAccess.Write, preallocationSize: file.Size); + var handle = File.OpenHandle(Path.Join(path, file.Name), FileMode.Create, FileAccess.Write, preallocationSize: file.Size); RandomAccess.SetLength(handle, file.Size); handle.Dispose(); ProgressUpdated?.Invoke(++state.DisplayProgress); @@ -489,7 +489,7 @@ void preallocDir(in DirectoryEntry.AcquisitionEntry dir, string path, int recurs for (; index < dir.Subdirectories.Count; index++) { var subdir = dir.Subdirectories[index]; - preallocDir(in subdir, Path.Combine(path, manifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); + preallocDir(in subdir, Path.Join(path, manifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); if (cancellationToken.IsCancellationRequested) { state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; @@ -508,7 +508,7 @@ void preallocDir(in DirectoryEntry.AcquisitionEntry dir, string path, int recurs if (new DriveInfo(DownloadsDirectory!).AvailableFreeSpace < delta.DownloadCacheSize) throw new SteamNotEnoughDiskSpaceException(delta.DownloadCacheSize); ProgressInitiated?.Invoke(ProgressType.Numeric, delta.NumDownloadFiles, state.DisplayProgress); - preallocDir(in delta.AcquisitionTree, Path.Combine(DownloadsDirectory!, state.Id.ToString()), 0); + preallocDir(in delta.AcquisitionTree, Path.Join(DownloadsDirectory!, state.Id.ToString()), 0); if (cancellationToken.IsCancellationRequested) { state.SaveToFile(); @@ -516,14 +516,14 @@ void preallocDir(in DirectoryEntry.AcquisitionEntry dir, string path, int recurs } if (delta.ChunkBufferFileSize > 0) { - var handle = File.OpenHandle(Path.Combine(DownloadsDirectory!, $"{state.Id}.scchunkbuffer"), FileMode.Create, FileAccess.Write, preallocationSize: delta.ChunkBufferFileSize); + var handle = File.OpenHandle(Path.Join(DownloadsDirectory!, $"{state.Id}.scchunkbuffer"), FileMode.Create, FileAccess.Write, preallocationSize: delta.ChunkBufferFileSize); RandomAccess.SetLength(handle, delta.ChunkBufferFileSize); handle.Dispose(); ProgressUpdated?.Invoke(++state.DisplayProgress); } if (delta.IntermediateFileSize > 0) { - var handle = File.OpenHandle(Path.Combine(DownloadsDirectory!, $"{state.Id}.screlocpatchcache"), FileMode.Create, FileAccess.Write, preallocationSize: delta.IntermediateFileSize); + var handle = File.OpenHandle(Path.Join(DownloadsDirectory!, $"{state.Id}.screlocpatchcache"), FileMode.Create, FileAccess.Write, preallocationSize: delta.IntermediateFileSize); RandomAccess.SetLength(handle, delta.IntermediateFileSize); handle.Dispose(); ProgressUpdated?.Invoke(++state.DisplayProgress); @@ -540,7 +540,7 @@ public DepotManifest GetManifest(uint appId, ItemIdentifier item, ulong manifest { if (ManifestsDirectory is not null) { - string filePath = Path.Combine(ManifestsDirectory, $"{item}-{manifestId}.scmanifest"); + string filePath = Path.Join(ManifestsDirectory, $"{item}-{manifestId}.scmanifest"); if (File.Exists(filePath)) { StatusUpdated?.Invoke(Status.LoadingManifest); @@ -579,7 +579,7 @@ public DepotManifest GetManifest(uint appId, ItemIdentifier item, ulong manifest StatusUpdated?.Invoke(Status.LoadingManifest); var result = new DepotManifest(buffer, item); if (ManifestsDirectory is not null) - result.WriteToFile(Path.Combine(ManifestsDirectory, $"{item}-{manifestId}.scmanifest")); + result.WriteToFile(Path.Join(ManifestsDirectory, $"{item}-{manifestId}.scmanifest")); return result; } catch (Exception e) @@ -604,7 +604,7 @@ public DepotPatch GetPatch(uint appId, ItemIdentifier item, DepotManifest source { if (DownloadsDirectory is not null) { - string filePath = Path.Combine(DownloadsDirectory, $"{item}-{sourceManifest.Id}-{targetManifest.Id}.scpatch"); + string filePath = Path.Join(DownloadsDirectory, $"{item}-{sourceManifest.Id}-{targetManifest.Id}.scpatch"); if (File.Exists(filePath)) { StatusUpdated?.Invoke(Status.LoadingPatch); @@ -642,7 +642,7 @@ public DepotPatch GetPatch(uint appId, ItemIdentifier item, DepotManifest source StatusUpdated?.Invoke(Status.LoadingPatch); var result = new DepotPatch(buffer, item, sourceManifest, targetManifest); if (DownloadsDirectory is not null) - result.WriteToFile(Path.Combine(DownloadsDirectory, $"{item}-{sourceManifest.Id}-{targetManifest.Id}.scpatch")); + result.WriteToFile(Path.Join(DownloadsDirectory, $"{item}-{sourceManifest.Id}-{targetManifest.Id}.scpatch")); return result; } catch (Exception e) From 70b523ed3fbef8bc974ce5198aafd84ebba8d122 Mon Sep 17 00:00:00 2001 From: Nuclearist Date: Thu, 1 Feb 2024 10:03:37 +0300 Subject: [PATCH 3/4] Download context draft --- src/CDNClient.cs | 236 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) diff --git a/src/CDNClient.cs b/src/CDNClient.cs index 5f0f7df..bfa2696 100644 --- a/src/CDNClient.cs +++ b/src/CDNClient.cs @@ -743,6 +743,24 @@ public int LoadFromBuffer(ReadOnlySpan buffer, ref nint offset, ref int st return chunksLeft; } } + /// Context containing all the data needed to download a chunk. + private readonly struct ChunkContext + { + /// Size of LZMA-compressed chunk data. + public required int CompressedSize { get; init; } + /// Size of uncompressed chunk data. + public required int UncompressedSize { get; init; } + /// Adler checksum of chunk data. + public required uint Checksum { get; init; } + /// Offset of chunk data from the beginning of containing file. + public required long FileOffset { get; init; } + /// Path to the file to download chunk to. + public required string FilePath { get; init; } + /// Handle for the file to download chunk to. + public required LimitedUseFileHandle FileHandle { get; init; } + /// GID of the chunk. + public required SHA1Hash Gid { get; init; } + } /// File handle wrapper that releases the handle after the last chunk has been written to the file. /// File handle. /// The number of chunks that will be written to the file. @@ -759,6 +777,224 @@ public void Release() Handle.Dispose(); } } + /// Shared context for download threads. + private class DownloadContext + { + /// Initializes a download context for given delta and state and sets up index stack. + public DownloadContext(ItemState state, DepotManifest manifest, DepotDelta delta, string basePath, ProgressUpdatedHandler? handler) + { + _manifest = manifest; + _state = state; + static int getDirTreeDepth(in DirectoryEntry.AcquisitionEntry dir) + { + int maxChildDepth = 0; + foreach (var subdir in dir.Subdirectories) + { + int childDepth = getDirTreeDepth(in subdir); + if (childDepth > maxChildDepth) + maxChildDepth = childDepth; + } + return 1 + maxChildDepth; + } + int dirTreeDepth = getDirTreeDepth(in delta.AcquisitionTree); + _pathTree = new string[dirTreeDepth + 1]; + _pathTree[0] = basePath; + _currentDirTree = new DirectoryEntry.AcquisitionEntry[dirTreeDepth]; + _currentDirTree[0] = delta.AcquisitionTree; + var indexStack = state.ProgressIndexStack; + if (indexStack.Count is 0) + { + bool findFirstChunk(in DirectoryEntry.AcquisitionEntry dir) + { + var files = dir.Files; + for (int i = 0; i < files.Count; i++) + { + var file = files[i]; + if (file.Chunks.Count > 0 || manifest.FileBuffer[file.Index].Chunks.Count > 0) + { + state.ProgressIndexStack.Add(i); + state.ProgressIndexStack.Add(0); + return true; + } + } + int stackIndex = state.ProgressIndexStack.Count; + state.ProgressIndexStack.Add(0); + var subdirs = dir.Subdirectories; + for (int i = 0; i < subdirs.Count; i++) + if (findFirstChunk(subdirs[i])) + { + state.ProgressIndexStack[stackIndex] = i; + return true; + } + state.ProgressIndexStack.RemoveAt(stackIndex); + return false; + } + findFirstChunk(in _currentDirTree[0]); + } + for (int i = 0; i < state.ProgressIndexStack.Count - 2; i++) + { + _currentDirTree[i + 1] = _currentDirTree[i].Subdirectories[state.ProgressIndexStack[i]]; + _pathTree[i + 1] = manifest.DirectoryBuffer[_currentDirTree[i + 1].Index].Name; + } + _currentFile = _currentDirTree[state.ProgressIndexStack.Count - 2].Files[state.ProgressIndexStack[^2]]; + _pathTree[state.ProgressIndexStack.Count - 1] = manifest.FileBuffer[_currentFile.Index].Name; + if (delta.ChunkBufferFileSize > 0) + { + _chunkBufferFilePath = Path.Join(basePath, $"{state.Id}.scchunkbuffer"); + _chunkBufferFileHandle = new(File.OpenHandle(_chunkBufferFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite, options: FileOptions.RandomAccess | FileOptions.Asynchronous), int.MaxValue); + } + if (state.ProgressIndexStack[^1] > 0) + { + if (_currentFile.Chunks.Count is 0) + { + _currentFilePath = Path.Join(_pathTree); + _currentFileHandle = new(File.OpenHandle(_currentFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite, FileOptions.RandomAccess | FileOptions.Asynchronous), + (_currentFile.Chunks.Count is 0 ? manifest.FileBuffer[_currentFile.Index].Chunks.Count : _currentFile.Chunks.Count) - state.ProgressIndexStack[^1]); + } + else + { + _currentFilePath = _chunkBufferFilePath!; + _currentFileHandle = _chunkBufferFileHandle!; + } + } + _progressUpdatedHandler = handler; + } + /// Path to the currently selected file. + private string? _currentFilePath; + /// Entry for the currently selected file. + private FileEntry.AcquisitionEntry _currentFile; + /// Handle for the currently selected file. + private LimitedUseFileHandle? _currentFileHandle; + /// Path to the chunk buffer file. + private readonly string? _chunkBufferFilePath; + /// Array of directory and file names to compose path from. + private readonly string?[] _pathTree; + /// Path to the chunk buffer file. + private readonly DepotManifest _manifest; + /// Path to the chunk buffer file. + private readonly DirectoryEntry.AcquisitionEntry[] _currentDirTree; + /// Item state. + private readonly ItemState _state; + /// Handle for the chunk buffer file. + private readonly LimitedUseFileHandle? _chunkBufferFileHandle; + /// Called when progress value is updated. + private readonly ProgressUpdatedHandler? _progressUpdatedHandler; + /// Submits progress for the previous chunk, gets context for the next chunk or if the are no more chunks and moves index stack to the next chunk. + public ChunkContext GetNextChunk(long previousChunkSize) + { + var indexStack = _state.ProgressIndexStack; + lock (this) + { + if (previousChunkSize > 0) + { + _state.DisplayProgress += previousChunkSize; + _progressUpdatedHandler?.Invoke(_state.DisplayProgress); + } + if (indexStack.Count is 0) + return default; + int chunkIndex = indexStack[^1]; + if (chunkIndex is 0) + { + if (_currentFile.Chunks.Count is 0) + { + _currentFilePath = Path.Join(_pathTree); + _currentFileHandle = new(File.OpenHandle(_currentFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite, FileOptions.RandomAccess | FileOptions.Asynchronous), _manifest.FileBuffer[_currentFile.Index].Chunks.Count); + } + else + { + _currentFilePath = _chunkBufferFilePath!; + _currentFileHandle = _chunkBufferFileHandle!; + } + } + ChunkEntry chunk; + long chunkBufferOffset; + int numChunks; + if (_currentFile.Chunks.Count is 0) + { + var chunks = _manifest.FileBuffer[_currentFile.Index].Chunks; + chunk = chunks[chunkIndex]; + chunkBufferOffset = -1; + numChunks = chunks.Count; + } + else + { + var acquisitionChunk = _currentFile.Chunks[chunkIndex]; + chunk = _manifest.ChunkBuffer[acquisitionChunk.Index]; + chunkBufferOffset = acquisitionChunk.Offset; + numChunks = _currentFile.Chunks.Count; + } + if (++chunkIndex == numChunks) + { + bool contextRestored = false; + int lastDirLevel = indexStack.Count - 2; + bool findNextChunk(in DirectoryEntry.AcquisitionEntry dir, int recursionLevel) + { + if (contextRestored || recursionLevel == lastDirLevel) + { + var files = dir.Files; + for (int i = contextRestored ? 0 : indexStack[recursionLevel] + 1; i < files.Count; i++) + { + var file = files[i]; + if (file.Chunks.Count > 0 || _manifest.FileBuffer[file.Index].Chunks.Count > 0) + { + if (contextRestored) + { + indexStack.Add(i); + indexStack.Add(0); + } + else + { + indexStack[recursionLevel] = i; + indexStack[recursionLevel + 1] = 0; + } + return true; + } + } + if (!contextRestored) + { + indexStack.RemoveRange(recursionLevel, 2); + contextRestored = true; + } + } + if (contextRestored) + indexStack.Add(0); + var subdirs = dir.Subdirectories; + for (int i = contextRestored ? 0 : indexStack[recursionLevel]; i < subdirs.Count; i++) + if (findNextChunk(subdirs[i], recursionLevel + 1)) + { + indexStack[recursionLevel] = i; + return true; + } + indexStack.RemoveAt(recursionLevel); + return false; + } + if (!findNextChunk(in _currentDirTree[0], 0)) + indexStack.Clear(); + for (int i = 0; i < indexStack.Count - 2; i++) + { + _currentDirTree[i + 1] = _currentDirTree[i].Subdirectories[indexStack[i]]; + _pathTree[i + 1] = _manifest.DirectoryBuffer[_currentDirTree[i + 1].Index].Name; + } + _currentFile = _currentDirTree[indexStack.Count - 2].Files[indexStack[^2]]; + _pathTree[indexStack.Count - 1] = _manifest.FileBuffer[_currentFile.Index].Name; + for (int i = indexStack.Count; i < _pathTree.Length; i++) + _pathTree[i] = null; + } + else + indexStack[^1] = chunkIndex; + return new() + { + CompressedSize = chunk.CompressedSize, + UncompressedSize = chunk.UncompressedSize, + Checksum = chunk.Checksum, + FileOffset = chunkBufferOffset >= 0 ? chunkBufferOffset : chunk.Offset, + FilePath = _currentFilePath!, + FileHandle = _currentFileHandle!, + Gid = chunk.Gid + }; + } + } + } /// Thread-safe wrapper for updating progress value. /// Event handler called when progress is updated. /// Depot state object that holds progress value. From f3fc74141048dd03867fc943d3736930f68e083d Mon Sep 17 00:00:00 2001 From: Nuclearist Date: Fri, 2 Feb 2024 21:14:46 +0300 Subject: [PATCH 4/4] Implemented thread-based downloading --- src/CDNClient.cs | 682 ++++++++++++------------------------- src/Manifest/DepotDelta.cs | 2 +- 2 files changed, 227 insertions(+), 457 deletions(-) diff --git a/src/CDNClient.cs b/src/CDNClient.cs index bfa2696..777ddd2 100644 --- a/src/CDNClient.cs +++ b/src/CDNClient.cs @@ -27,24 +27,17 @@ public class CDNClient /// CM client used to get CDN server list and manifest request codes. public required CM.CMClient CmClient { get; init; } /// - /// Number of servers that clients will simultaneously use when downloading depot content. The default value is . - /// The product of and is the number of simultaneous download tasks, scale it - /// accordingly to your network bandwidth and CPU capabilities. + /// Number of servers that clients will simultaneously use when downloading depot content. The default value is Math.Max(Environment.ProcessorCount * 6, 50). + /// It is also the number of download threads, scale it accordingly to your network bandwidth and CPU capabilities. /// - public static int NumDownloadServers { get; set; } = Environment.ProcessorCount; - /// - /// Number of simultaneous download tasks created per server. The default value is 4. - /// The product of and is the number of simultaneous download tasks, scale it - /// accordingly to your network bandwidth and CPU capabilities. - /// - public static int NumRequestsPerServer { get; set; } = 4; + public static int NumDownloadServers { get; set; } = Math.Max(Environment.ProcessorCount * 6, 50); /// Gets CDN server list if necessary. private void CheckServerList() { - if (_servers.Length >= NumDownloadServers) + if (_servers.Length > NumDownloadServers) return; - var servers = new List(NumDownloadServers); - while (servers.Count < NumDownloadServers) + var servers = new List(NumDownloadServers + 1); + while (servers.Count < NumDownloadServers + 1) servers.AddRange(Array.FindAll(CmClient.GetCDNServers(), s => s.Type is "SteamCache" or "CDN" && s.HttpsSupport is "mandatory" or "optional")); servers.Sort((left, right) => { @@ -61,290 +54,130 @@ private void CheckServerList() for (int i = 0; i < servers.Count; i++) _servers[i] = new(string.Concat("https://", servers[i].Host)); } - /// Downloads, decrypts, decompresses and writes chunk specified in the context. - /// An object. - private static async Task AcquireChunk(object? arg) + private static void DownloadThreadProcedure(object? arg) { - var context = (AcquisitionTaskContext)arg!; - byte[] buffer = context.Buffer; - int compressedSize = context.CompressedSize; - int uncompressedSize = context.UncompressedSize; - var cancellationToken = context.CancellationToken; - var aes = context.Aes; - Exception? exception = null; - var progress = context.Progress; + var context = (DownloadThreadContext)arg!; + byte[] buffer = GC.AllocateUninitializedArray(0x400000); + using var aes = Aes.Create(); + aes.Key = DepotDecryptionKeys[context.SharedContext.DepotId]; + var lzmaDecoder = new Utils.LZMA.Decoder(); + var httpClient = context.SharedContext.HttpClients[context.Index]; + var requestUri = new Uri($"depot/{context.SharedContext.DepotId}/chunk/0000000000000000000000000000000000000000", UriKind.Relative); + ref byte uriGid = ref Unsafe.As(ref MemoryMarshal.GetReference(requestUri.ToString().AsSpan()[^40..])); + var token = context.Cts.Token; var downloadBuffer = new Memory(buffer, 0, 0x200000); - for (int i = 0; i < 5; i++) //5 attempts, after which task fails + var bufferSpan = new Span(buffer); + var ivSpan = (ReadOnlySpan)bufferSpan[..16]; + var decryptedIvSpan = bufferSpan.Slice(0x3FFFF0, 16); + var decryptedDataSpan = new Span(buffer, 0x200000, 0x1FFFF0); + bool resumedContext = context.ChunkContext.FilePath is not null; + int fallbackServerIndex = NumDownloadServers; + for (;;) { - cancellationToken.ThrowIfCancellationRequested(); - try - { - //Download encrypted chunk data - var request = new HttpRequestMessage(HttpMethod.Get, context.RequestUri) { Version = HttpVersion.Version20 }; - using var response = await context.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - using var content = response.EnsureSuccessStatusCode().Content; - using var stream = await content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); - int bytesRead; - using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - cts.CancelAfter(60000); - try { bytesRead = await stream.ReadAtLeastAsync(downloadBuffer, compressedSize, false, cts.Token).ConfigureAwait(false); } - catch (OperationCanceledException oce) - { - if (oce.CancellationToken == cancellationToken) - throw; - throw new TimeoutException(); - } - catch (AggregateException ae) when (ae.InnerException is OperationCanceledException oce) - { - if (oce.CancellationToken == cancellationToken) - throw; - throw new TimeoutException(); - } - if (bytesRead != compressedSize) - { - exception = new InvalidDataException($"Downloaded chunk data size doesn't match expected [URL: {context.HttpClient.BaseAddress}/{request.RequestUri}]"); - continue; - } - //Decrypt the data - aes.DecryptEcb(new ReadOnlySpan(buffer, 0, 16), new Span(buffer, 0x3FFFF0, 16), PaddingMode.None); - int decryptedDataSize = aes.DecryptCbc(new ReadOnlySpan(buffer, 16, compressedSize - 16), new ReadOnlySpan(buffer, 0x3FFFF0, 16), new Span(buffer, 0x200000, 0x1FFFF0)); - //Decompress the data - if (!context.LzmaDecoder.Decode(new ReadOnlySpan(buffer, 0x200000, decryptedDataSize), new Span(buffer, 0, uncompressedSize))) - { - exception = new InvalidDataException("LZMA decoding failed"); - continue; - } - if (Adler.ComputeChecksum(new ReadOnlySpan(buffer, 0, uncompressedSize)) != context.Checksum) - { - exception = new InvalidDataException("Adler checksum mismatch"); - continue; - } - exception = null; - } - catch (OperationCanceledException) { throw; } - catch (AggregateException ae) when (ae.InnerException is OperationCanceledException) { throw ae.InnerException; } - catch (Exception e) - { - exception = e; - continue; - } - } - if (exception is not null) - throw exception; - //Write acquired chunk data to the file - var handle = context.FileHandle; - await RandomAccess.WriteAsync(handle.Handle, new ReadOnlyMemory(buffer, 0, uncompressedSize), context.FileOffset, cancellationToken).ConfigureAwait(false); - handle.Release(); - progress.SubmitChunk(compressedSize); - } - /// Downloads depot content chunks. - /// State of the item. - /// The target manifest. - /// Delta object that lists data to be downloaded. - /// Token to monitor for cancellation requests. - internal void DownloadContent(ItemState state, DepotManifest manifest, DepotDelta delta, CancellationToken cancellationToken) - { - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - var contexts = new AcquisitionTaskContext[NumDownloadServers * NumRequestsPerServer]; - var tasks = new Task?[contexts.Length]; - int numResumedContexts = 0; - Exception? exception = null; - string? chunkBufferFilePath = null; - LimitedUseFileHandle? chunkBufferFileHandle = null; - string baseRequestUrl = $"depot/{state.Id.DepotId}/chunk/"; - void downloadDir(in DirectoryEntry.AcquisitionEntry dir, string path, int recursionLevel) - { - int index; - if (state.ProgressIndexStack.Count > recursionLevel) - index = state.ProgressIndexStack[recursionLevel]; + if (resumedContext) + resumedContext = false; else { - state.ProgressIndexStack.Add(0); - index = 0; + context.ChunkContext = context.SharedContext.GetNextChunk(context.ChunkContext.CompressedSize); + if (context.ChunkContext.FilePath is null) + return; } - for (; index < dir.Files.Count; index++) + Unsafe.CopyBlockUnaligned(ref uriGid, ref Unsafe.As(ref MemoryMarshal.GetReference(context.ChunkContext.Gid.ToString().AsSpan())), 80); + int compressedSize = context.ChunkContext.CompressedSize; + int uncompressedSize = context.ChunkContext.UncompressedSize; + Exception? exception = null; + var dataSpan = (ReadOnlySpan)bufferSpan[16..compressedSize]; + var uncompressedDataSpan = bufferSpan[..uncompressedSize]; + for (int i = 0; i < 5; i++) //5 attempts, after which the thread fails { - var acquisitonFile = dir.Files[index]; - var file = manifest.FileBuffer[acquisitonFile.Index]; - if (file.Size is 0) - continue; - if (linkedCts.IsCancellationRequested) - { - state.ProgressIndexStack[recursionLevel] = index; + if (token.IsCancellationRequested) return; - } - int chunkRecLevel = recursionLevel + 1; - int chunkIndex; - if (state.ProgressIndexStack.Count > chunkRecLevel) - chunkIndex = state.ProgressIndexStack[chunkRecLevel]; - else - { - state.ProgressIndexStack.Add(0); - chunkIndex = 0; - } - if (acquisitonFile.Chunks.Count is 0) + try { - string filePath = Path.Join(path, file.Name); - var chunks = file.Chunks; - LimitedUseFileHandle? handle; - if (numResumedContexts > 0) + //Download encrypted chunk data + var request = new HttpRequestMessage(HttpMethod.Get, requestUri) { Version = HttpVersion.Version20 }; + using var response = httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token).GetAwaiter().GetResult(); + using var content = response.EnsureSuccessStatusCode().Content; + using var stream = content.ReadAsStream(token); + int bytesRead; + using var cts = CancellationTokenSource.CreateLinkedTokenSource(token); + cts.CancelAfter(60000); + try { - handle = null; - for (int i = 0; i < numResumedContexts; i++) - if (contexts[i].FilePath == filePath) - { - handle = contexts[i].FileHandle; - break; - } - numResumedContexts = 0; - handle ??= new(File.OpenHandle(filePath, FileMode.OpenOrCreate, FileAccess.Write, options: FileOptions.RandomAccess | FileOptions.Asynchronous), chunks.Count); + var task = stream.ReadAtLeastAsync(downloadBuffer, compressedSize, false, cts.Token); + if (task.IsCompletedSuccessfully) + bytesRead = task.GetAwaiter().GetResult(); + else + bytesRead = task.AsTask().GetAwaiter().GetResult(); } - else - handle = new(File.OpenHandle(filePath, FileMode.OpenOrCreate, FileAccess.Write, options: FileOptions.RandomAccess | FileOptions.Asynchronous), chunks.Count); - for (; chunkIndex < chunks.Count; chunkIndex++) + catch (OperationCanceledException oce) { - if (linkedCts.IsCancellationRequested) - { - state.ProgressIndexStack[chunkRecLevel] = chunkIndex; - state.ProgressIndexStack[recursionLevel] = index; + if (oce.CancellationToken == token) return; - } - var chunk = chunks[chunkIndex]; - int contextIndex = -1; - for (int i = 0; i < contexts.Length; i++) - { - var task = tasks[i]; - if (task is null) - { - contextIndex = i; - break; - } - if (task.IsCompleted) - { - if (task.IsFaulted) - { - exception = task.Exception; - linkedCts.Cancel(); - state.ProgressIndexStack[chunkRecLevel] = chunkIndex; - state.ProgressIndexStack[recursionLevel] = index; - return; - } - contextIndex = i; - break; - } - } - if (contextIndex < 0) - { - try { contextIndex = Task.WaitAny(tasks!, linkedCts.Token); } - catch (OperationCanceledException) - { - state.ProgressIndexStack[chunkRecLevel] = chunkIndex; - state.ProgressIndexStack[recursionLevel] = index; - return; - } - var task = tasks[contextIndex]!; - if (task.IsFaulted) - { - exception = task.Exception; - linkedCts.Cancel(); - state.ProgressIndexStack[chunkRecLevel] = chunkIndex; - state.ProgressIndexStack[recursionLevel] = index; - return; - } - } - var context = contexts[contextIndex]; - context.CompressedSize = chunk.CompressedSize; - context.UncompressedSize = chunk.UncompressedSize; - context.Checksum = chunk.Checksum; - context.FileOffset = chunk.Offset; - context.FilePath = filePath; - context.RequestUri = new(string.Concat(baseRequestUrl, chunk.Gid.ToString()), UriKind.Relative); - context.FileHandle = handle; - tasks[contextIndex] = Task.Factory.StartNew(AcquireChunk, context, TaskCreationOptions.DenyChildAttach).Result; + exception = new TimeoutException(); + continue; } + if (bytesRead != compressedSize) + { + exception = new InvalidDataException($"Downloaded chunk data size doesn't match expected [URL: {httpClient.BaseAddress}/{request.RequestUri}]"); + continue; + } + //Decrypt the data + aes.DecryptEcb(ivSpan, decryptedIvSpan, PaddingMode.None); + int decryptedDataSize = aes.DecryptCbc(dataSpan, decryptedIvSpan, decryptedDataSpan); + //Decompress the data + if (!lzmaDecoder.Decode(decryptedDataSpan[..decryptedDataSize], uncompressedDataSpan)) + { + exception = new InvalidDataException("LZMA decoding failed"); + continue; + } + if (Adler.ComputeChecksum(uncompressedDataSpan) != context.ChunkContext.Checksum) + { + exception = new InvalidDataException("Adler checksum mismatch"); + continue; + } + exception = null; } - else + catch (OperationCanceledException) { return; } + catch (HttpRequestException hre) when (hre.StatusCode > HttpStatusCode.InternalServerError) { - var acquisitionChunks = acquisitonFile.Chunks; - for (; chunkIndex < acquisitionChunks.Count; chunkIndex++) + if (fallbackServerIndex is 0) { - if (linkedCts.IsCancellationRequested) - { - state.ProgressIndexStack[chunkRecLevel] = chunkIndex; - state.ProgressIndexStack[recursionLevel] = index; - return; - } - var acquisitionChunk = acquisitionChunks[chunkIndex]; - var chunk = manifest.ChunkBuffer[acquisitionChunk.Index]; - int contextIndex = -1; - for (int i = 0; i < contexts.Length; i++) - { - var task = tasks[i]; - if (task is null) - { - contextIndex = i; - break; - } - if (task.IsCompleted) - { - if (task.IsFaulted) - { - exception = task.Exception; - linkedCts.Cancel(); - state.ProgressIndexStack[chunkRecLevel] = chunkIndex; - state.ProgressIndexStack[recursionLevel] = index; - return; - } - contextIndex = i; - break; - } - } - if (contextIndex < 0) - { - try - { contextIndex = Task.WaitAny(tasks!, linkedCts.Token); } - catch (OperationCanceledException) - { - state.ProgressIndexStack[chunkRecLevel] = chunkIndex; - state.ProgressIndexStack[recursionLevel] = index; - return; - } - var task = tasks[contextIndex]!; - if (task.IsFaulted) - { - exception = task.Exception; - linkedCts.Cancel(); - state.ProgressIndexStack[chunkRecLevel] = chunkIndex; - state.ProgressIndexStack[recursionLevel] = index; - return; - } - } - var context = contexts[contextIndex]; - context.CompressedSize = chunk.CompressedSize; - context.UncompressedSize = chunk.UncompressedSize; - context.Checksum = chunk.Checksum; - context.FileOffset = acquisitionChunk.Offset; - context.FilePath = chunkBufferFilePath!; - context.RequestUri = new(string.Concat(baseRequestUrl, chunk.Gid.ToString()), UriKind.Relative); - context.FileHandle = chunkBufferFileHandle!; - tasks[contextIndex] = Task.Factory.StartNew(AcquireChunk, context, TaskCreationOptions.DenyChildAttach).Result; + httpClient = context.SharedContext.HttpClients[context.Index]; + fallbackServerIndex = NumDownloadServers; + } + else + { + httpClient = context.SharedContext.HttpClients[fallbackServerIndex]; + if (++fallbackServerIndex == context.SharedContext.HttpClients.Length) + fallbackServerIndex = 0; } } - state.ProgressIndexStack.RemoveAt(chunkRecLevel); - } - index -= dir.Files.Count; - for (; index < dir.Subdirectories.Count; index++) - { - var subdir = dir.Subdirectories[index]; - downloadDir(in subdir, Path.Join(path, manifest.DirectoryBuffer[subdir.Index].Name), recursionLevel + 1); - if (linkedCts.IsCancellationRequested) + catch (Exception e) { - state.ProgressIndexStack[recursionLevel] = dir.Files.Count + index; - return; + exception = e; + continue; } } - state.ProgressIndexStack.RemoveAt(recursionLevel); + if (exception is not null) + { + context.SharedContext.Exception = exception; + context.Cts.Cancel(); + return; + } + //Write acquired chunk data to the file + var handle = context.ChunkContext.FileHandle; + RandomAccess.Write(handle.Handle, uncompressedDataSpan, context.ChunkContext.FileOffset); + handle.Release(); } + } + /// Downloads depot content chunks. + /// State of the item. + /// The target manifest. + /// Delta object that lists data to be downloaded. + /// Token to monitor for cancellation requests. + internal void DownloadContent(ItemState state, DepotManifest manifest, DepotDelta delta, CancellationToken cancellationToken) + { if (state.Status is not ItemState.ItemStatus.Downloading) { state.Status = ItemState.ItemStatus.Downloading; @@ -352,102 +185,72 @@ void downloadDir(in DirectoryEntry.AcquisitionEntry dir, string path, int recurs state.DisplayProgress = 0; } StatusUpdated?.Invoke(Status.Downloading); - if (!DepotDecryptionKeys.TryGetValue(state.Id.DepotId, out var decryptionKey)) + if (!DepotDecryptionKeys.ContainsKey(state.Id.DepotId)) throw new SteamException(SteamException.ErrorType.DepotDecryptionKeyMissing); CheckServerList(); - var threadSafeProgress = new ThreadSafeProgress(ProgressUpdated, state); - var httpClients = new HttpClient[NumDownloadServers]; - for (int i = 0; i < httpClients.Length; i++) - httpClients[i] = new() + var sharedContext = new DownloadContext(state, manifest, delta, DownloadsDirectory!, ProgressUpdated) { HttpClients = new HttpClient[_servers.Length] }; + for (int i = 0; i < _servers.Length; i++) + sharedContext.HttpClients[i] = new() { BaseAddress = _servers[i], DefaultRequestVersion = HttpVersion.Version20, Timeout = TimeSpan.FromSeconds(10) }; + var contexts = new DownloadThreadContext[NumDownloadServers]; + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); for (int i = 0; i < contexts.Length; i++) - { - contexts[i] = new(threadSafeProgress, httpClients[i % httpClients.Length], linkedCts.Token); - contexts[i].Aes.Key = decryptionKey; - } - string dwContextsFilePath = Path.Join(DownloadsDirectory!, $"{state.Id}.scdwcontexts"); - ProgressInitiated?.Invoke(ProgressType.Binary, delta.DownloadSize, state.DisplayProgress); - if (delta.ChunkBufferFileSize > 0) - { - chunkBufferFilePath = Path.Join(DownloadsDirectory!, $"{state.Id}.scchunkbuffer"); - chunkBufferFileHandle = new(File.OpenHandle(chunkBufferFilePath, FileMode.OpenOrCreate, FileAccess.Write, options: FileOptions.RandomAccess | FileOptions.Asynchronous), int.MaxValue); - } - if (File.Exists(dwContextsFilePath)) + contexts[i] = new() + { + Index = i, + Cts = linkedCts, + SharedContext = sharedContext + }; + string chunkContextsFilePath = Path.Join(DownloadsDirectory!, $"{state.Id}.scchcontexts"); + if (File.Exists(chunkContextsFilePath)) { Span buffer; - using (var fileHandle = File.OpenHandle(dwContextsFilePath)) + using (var fileHandle = File.OpenHandle(chunkContextsFilePath)) { buffer = GC.AllocateUninitializedArray((int)RandomAccess.GetLength(fileHandle)); RandomAccess.Read(fileHandle, buffer, 0); } - numResumedContexts = Unsafe.As(ref MemoryMarshal.GetReference(buffer)); - nint offset = 8; - int stringOffset = 8 + numResumedContexts * 32; - for (int i = 0; i < numResumedContexts; i++) - { - int numChunks = contexts[i].LoadFromBuffer(buffer, ref offset, ref stringOffset); - bool fileCreated = false; - if (chunkBufferFilePath is not null && contexts[i].FilePath == chunkBufferFilePath) - { - contexts[i].FileHandle = chunkBufferFileHandle!; - fileCreated = true; - } - else - for (int j = 0; j < i; j++) - if (contexts[j].FilePath == contexts[i].FilePath) - { - contexts[i].FileHandle = contexts[j].FileHandle; - fileCreated = true; - break; - } - if (!fileCreated) - contexts[i].FileHandle = new(File.OpenHandle(contexts[i].FilePath, FileMode.OpenOrCreate, FileAccess.Write, options: FileOptions.RandomAccess | FileOptions.Asynchronous), numChunks); - } - for (int i = 0; i < numResumedContexts; i++) - tasks[i] = Task.Factory.StartNew(AcquireChunk, contexts[i], TaskCreationOptions.DenyChildAttach); - } - downloadDir(in delta.AcquisitionTree, Path.Join(DownloadsDirectory!, state.Id.ToString()), 0); - foreach (var task in tasks) - { - if (task is null) - continue; - if (!task.IsCompleted) - Task.WaitAny([ task ], CancellationToken.None); - if (task.IsFaulted) - exception = task.Exception; + nint offset = 0; + int pathOffset = contexts.Length * 48; + for (int i = 0; i < contexts.Length; i++) + contexts[i].ChunkContext = ChunkContext.LoadFromBuffer(buffer, ref offset, ref pathOffset); } - chunkBufferFileHandle?.Handle?.Dispose(); - foreach (var context in contexts) - context.Dispose(); + ProgressInitiated?.Invoke(ProgressType.Binary, delta.DownloadSize, state.DisplayProgress); + var threads = new Thread[contexts.Length]; + for (int i = 0; i < threads.Length; i++) + threads[i] = new(DownloadThreadProcedure); + for (int i = 0; i < threads.Length; i++) + threads[i].UnsafeStart(contexts[i]); + foreach (var thread in threads) + thread.Join(); + sharedContext.CurrentFileHandle?.Handle.Close(); + sharedContext.ChunkBufferFileHandle?.Handle.Close(); + foreach (var client in sharedContext.HttpClients) + client.Dispose(); if (linkedCts.IsCancellationRequested) { state.SaveToFile(); - int numContextsToSave = 0; int contextsFileSize = 0; - for (int i = 0; i < contexts.Length; i++) - if (!(tasks[i]?.IsCompletedSuccessfully ?? true)) - { - var context = contexts[i]; - numContextsToSave++; - contextsFileSize += 32 + Encoding.UTF8.GetByteCount(context.FilePath) + Encoding.UTF8.GetByteCount(context.FilePath); - } - if (numContextsToSave > 0) + foreach (var context in contexts) + contextsFileSize += 48 + Encoding.UTF8.GetByteCount(context.ChunkContext.FilePath); + Span buffer = new byte[contextsFileSize]; + nint offset = 0; + int pathOffset = contexts.Length * 48; + foreach (var context in contexts) + context.ChunkContext.WriteToBuffer(buffer, ref offset, ref pathOffset); + using var fileHandle = File.OpenHandle(chunkContextsFilePath, FileMode.Create, FileAccess.Write, preallocationSize: buffer.Length); + RandomAccess.Write(fileHandle, buffer, 0); + var exception = sharedContext.Exception; + throw exception switch { - Span buffer = new byte[contextsFileSize + 8]; - Unsafe.As(ref MemoryMarshal.GetReference(buffer)) = numContextsToSave; - nint offset = 8; - int stringOffset = 8 + numContextsToSave * 32; - for (int i = 0; i < contexts.Length; i++) - if (!(tasks[i]?.IsCompletedSuccessfully ?? true)) - contexts[i].WriteToBuffer(buffer, ref offset, ref stringOffset); - using var fileHandle = File.OpenHandle(dwContextsFilePath, FileMode.Create, FileAccess.Write, preallocationSize: buffer.Length); - RandomAccess.Write(fileHandle, buffer, 0); - } - throw exception is null ? new OperationCanceledException(linkedCts.Token) : exception is SteamException ? exception : new SteamException(SteamException.ErrorType.DownloadFailed, exception); + null => new OperationCanceledException(linkedCts.Token), + SteamException => exception, + _ => new SteamException(SteamException.ErrorType.DownloadFailed, exception) + }; } } /// Preallocates all files for the download on the disk. @@ -661,106 +464,71 @@ public DepotPatch GetPatch(uint appId, ItemIdentifier item, DepotManifest source public event ProgressUpdatedHandler? ProgressUpdated; /// Called when client status is updated. public event StatusUpdatedHandler? StatusUpdated; - /// Persistent context for chunk acquisitions tasks. - /// Progress wrapper. - /// HTTP client with base address set to server to download from. - /// Token to monitor for cancellation requests. - private class AcquisitionTaskContext(ThreadSafeProgress progress, HttpClient httpClient, CancellationToken cancellationToken) : IDisposable + /// Context containing all the data needed to download a chunk. + private readonly struct ChunkContext { - /// Buffer for storing downloaded data and intermediate decrypted and decompressed data. - public byte[] Buffer { get; } = GC.AllocateUninitializedArray(0x400000); /// Size of LZMA-compressed chunk data. - public int CompressedSize { get; internal set; } - /// Size of uncompressed chunk data. If -1, chunk won't be decompressed. - public int UncompressedSize { get; internal set; } + public required int CompressedSize { get; init; } + /// Size of uncompressed chunk data. + public required int UncompressedSize { get; init; } /// Adler checksum of chunk data. - public uint Checksum { get; internal set; } + public required uint Checksum { get; init; } /// Offset of chunk data from the beginning of containing file. - public long FileOffset { get; internal set; } - /// Path to the file to write chunk to. - public string FilePath { get; internal set; } = string.Empty; - /// AES decryptor. - public Aes Aes { get; } = Aes.Create(); - /// Token to monitor for cancellation requests. - public CancellationToken CancellationToken { get; } = cancellationToken; - /// LZMA decoder. - public Utils.LZMA.Decoder LzmaDecoder { get; } = new(); - /// HTTP client used to download chunk data. - public HttpClient HttpClient { get; } = httpClient; - /// Handle of the file to write chunk to. - public LimitedUseFileHandle FileHandle { get; internal set; } = null!; - /// Progress wrapper. - public ThreadSafeProgress Progress { get; } = progress; - /// Relative chunk URL. - public Uri RequestUri { get; internal set; } = null!; - public void Dispose() - { - Aes.Dispose(); - HttpClient.Dispose(); - FileHandle?.Handle?.Dispose(); - } + public required long FileOffset { get; init; } + /// Path to the file to download chunk to. + public required string FilePath { get; init; } + /// Handle for the file to download chunk to. + public required LimitedUseFileHandle FileHandle { get; init; } + /// GID of the chunk. + public required SHA1Hash Gid { get; init; } /// Writes context data to a buffer. /// Buffer to write data to. /// Offset into to write context data to. - /// Offset into to write UTF-8 encoded strings to. - public void WriteToBuffer(Span buffer, ref nint offset, ref int stringOffset) + /// Offset into to write file path to. + public void WriteToBuffer(Span buffer, ref nint offset, ref int pathOffset) { ref byte bufferRef = ref MemoryMarshal.GetReference(buffer); nint entryOffset = offset; - Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset)) = CompressedSize; - Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 4)) = UncompressedSize; - Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 8)) = Checksum; - Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 12)) = FileHandle.ChunksLeft; - Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 16)) = FileOffset; - int stringLength = Encoding.UTF8.GetBytes(FilePath, buffer[stringOffset..]); - Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 24)) = stringLength; - stringOffset += stringLength; - stringLength = Encoding.UTF8.GetBytes(RequestUri.ToString(), buffer[stringOffset..]); - Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 28)) = stringLength; - stringOffset += stringLength; - offset += 32; + Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset)) = Gid; + Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 24)) = CompressedSize; + Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 28)) = UncompressedSize; + Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 32)) = Checksum; + int pathLength = Encoding.UTF8.GetBytes(FilePath, buffer[pathOffset..]); + Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 36)) = pathLength; + pathOffset += pathLength; + Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 40)) = FileOffset; + offset += 48; } /// Loads context data from a buffer. /// Buffer to read data from. /// Offset into to read context data from. - /// Offset into to read UTF-8 encoded strings from. - public int LoadFromBuffer(ReadOnlySpan buffer, ref nint offset, ref int stringOffset) + /// Offset into to read file path from. + /// Loaded chunk context. + public static ChunkContext LoadFromBuffer(ReadOnlySpan buffer, ref nint offset, ref int pathOffset) { ref byte bufferRef = ref MemoryMarshal.GetReference(buffer); nint entryOffset = offset; - CompressedSize = Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset)); - UncompressedSize = Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 4)); - Checksum = Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 8)); - int chunksLeft = Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 12)); - FileOffset = Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 16)); - int stringLength = Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 24)); - FilePath = Encoding.UTF8.GetString(buffer.Slice(stringOffset, stringLength)); - stringOffset += stringLength; - stringLength = Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 28)); - RequestUri = new(Encoding.UTF8.GetString(buffer.Slice(stringOffset, stringLength)), UriKind.Relative); - stringOffset += stringLength; - offset += 32; - return chunksLeft; + var gid = new SHA1Hash(buffer.Slice(offset.ToInt32(), 20)); + int compressedSize = Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 24)); + int uncompressedSize = Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 28)); + uint checksum = Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 32)); + int pathLength = Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 36)); + string filePath = Encoding.UTF8.GetString(buffer.Slice(pathOffset, pathLength)); + pathOffset += pathLength; + long fileOffset = Unsafe.As(ref Unsafe.AddByteOffset(ref bufferRef, entryOffset + 40)); + offset += 48; + return new() + { + CompressedSize = compressedSize, + UncompressedSize = uncompressedSize, + Checksum = checksum, + FileOffset = fileOffset, + FilePath = filePath, + FileHandle = new(File.OpenHandle(filePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite, FileOptions.RandomAccess | FileOptions.Asynchronous), 1), + Gid = gid + }; } } - /// Context containing all the data needed to download a chunk. - private readonly struct ChunkContext - { - /// Size of LZMA-compressed chunk data. - public required int CompressedSize { get; init; } - /// Size of uncompressed chunk data. - public required int UncompressedSize { get; init; } - /// Adler checksum of chunk data. - public required uint Checksum { get; init; } - /// Offset of chunk data from the beginning of containing file. - public required long FileOffset { get; init; } - /// Path to the file to download chunk to. - public required string FilePath { get; init; } - /// Handle for the file to download chunk to. - public required LimitedUseFileHandle FileHandle { get; init; } - /// GID of the chunk. - public required SHA1Hash Gid { get; init; } - } /// File handle wrapper that releases the handle after the last chunk has been written to the file. /// File handle. /// The number of chunks that will be written to the file. @@ -798,7 +566,7 @@ static int getDirTreeDepth(in DirectoryEntry.AcquisitionEntry dir) } int dirTreeDepth = getDirTreeDepth(in delta.AcquisitionTree); _pathTree = new string[dirTreeDepth + 1]; - _pathTree[0] = basePath; + _pathTree[0] = Path.Join(basePath, state.Id.ToString()); _currentDirTree = new DirectoryEntry.AcquisitionEntry[dirTreeDepth]; _currentDirTree[0] = delta.AcquisitionTree; var indexStack = state.ProgressIndexStack; @@ -841,20 +609,20 @@ bool findFirstChunk(in DirectoryEntry.AcquisitionEntry dir) if (delta.ChunkBufferFileSize > 0) { _chunkBufferFilePath = Path.Join(basePath, $"{state.Id}.scchunkbuffer"); - _chunkBufferFileHandle = new(File.OpenHandle(_chunkBufferFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite, options: FileOptions.RandomAccess | FileOptions.Asynchronous), int.MaxValue); + ChunkBufferFileHandle = new(File.OpenHandle(_chunkBufferFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite, options: FileOptions.RandomAccess | FileOptions.Asynchronous), int.MaxValue); } if (state.ProgressIndexStack[^1] > 0) { if (_currentFile.Chunks.Count is 0) { _currentFilePath = Path.Join(_pathTree); - _currentFileHandle = new(File.OpenHandle(_currentFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite, FileOptions.RandomAccess | FileOptions.Asynchronous), + CurrentFileHandle = new(File.OpenHandle(_currentFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite, FileOptions.RandomAccess | FileOptions.Asynchronous), (_currentFile.Chunks.Count is 0 ? manifest.FileBuffer[_currentFile.Index].Chunks.Count : _currentFile.Chunks.Count) - state.ProgressIndexStack[^1]); } else { _currentFilePath = _chunkBufferFilePath!; - _currentFileHandle = _chunkBufferFileHandle!; + CurrentFileHandle = ChunkBufferFileHandle!; } } _progressUpdatedHandler = handler; @@ -863,8 +631,6 @@ bool findFirstChunk(in DirectoryEntry.AcquisitionEntry dir) private string? _currentFilePath; /// Entry for the currently selected file. private FileEntry.AcquisitionEntry _currentFile; - /// Handle for the currently selected file. - private LimitedUseFileHandle? _currentFileHandle; /// Path to the chunk buffer file. private readonly string? _chunkBufferFilePath; /// Array of directory and file names to compose path from. @@ -875,10 +641,18 @@ bool findFirstChunk(in DirectoryEntry.AcquisitionEntry dir) private readonly DirectoryEntry.AcquisitionEntry[] _currentDirTree; /// Item state. private readonly ItemState _state; - /// Handle for the chunk buffer file. - private readonly LimitedUseFileHandle? _chunkBufferFileHandle; /// Called when progress value is updated. private readonly ProgressUpdatedHandler? _progressUpdatedHandler; + /// Gets item depot ID. + public uint DepotId => _state.Id.DepotId; + /// Exception thrown by one of the download threads. + public Exception? Exception { get; set; } + /// HTTP clients for all CDN servers. + public required HttpClient[] HttpClients { get; init; } + /// Handle for the chunk buffer file. + public LimitedUseFileHandle? ChunkBufferFileHandle { get; } + /// Handle for the currently selected file. + public LimitedUseFileHandle? CurrentFileHandle { get; private set; } /// Submits progress for the previous chunk, gets context for the next chunk or if the are no more chunks and moves index stack to the next chunk. public ChunkContext GetNextChunk(long previousChunkSize) { @@ -898,12 +672,12 @@ public ChunkContext GetNextChunk(long previousChunkSize) if (_currentFile.Chunks.Count is 0) { _currentFilePath = Path.Join(_pathTree); - _currentFileHandle = new(File.OpenHandle(_currentFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite, FileOptions.RandomAccess | FileOptions.Asynchronous), _manifest.FileBuffer[_currentFile.Index].Chunks.Count); + CurrentFileHandle = new(File.OpenHandle(_currentFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite, FileOptions.RandomAccess | FileOptions.Asynchronous), _manifest.FileBuffer[_currentFile.Index].Chunks.Count); } else { _currentFilePath = _chunkBufferFilePath!; - _currentFileHandle = _chunkBufferFileHandle!; + CurrentFileHandle = ChunkBufferFileHandle!; } } ChunkEntry chunk; @@ -989,26 +763,22 @@ bool findNextChunk(in DirectoryEntry.AcquisitionEntry dir, int recursionLevel) Checksum = chunk.Checksum, FileOffset = chunkBufferOffset >= 0 ? chunkBufferOffset : chunk.Offset, FilePath = _currentFilePath!, - FileHandle = _currentFileHandle!, + FileHandle = CurrentFileHandle!, Gid = chunk.Gid }; } } } - /// Thread-safe wrapper for updating progress value. - /// Event handler called when progress is updated. - /// Depot state object that holds progress value. - private class ThreadSafeProgress(ProgressUpdatedHandler? handler, ItemState state) + /// Individual context for download threads. + private class DownloadThreadContext { - /// Updates progress value by adding chunk size to it. - /// Size of LZMA-compressed chunk data. - public void SubmitChunk(int chunkSize) - { - lock (this) - { - state.DisplayProgress += chunkSize; - handler?.Invoke(state.DisplayProgress); - } - } + /// Thread index, used to select download server. + public required int Index { get; init; } + /// Cancellation token source for all download threads. + public required CancellationTokenSource Cts { get; init; } + /// Current chunk context. + public ChunkContext ChunkContext { get; set; } + /// Shared download context. + public required DownloadContext SharedContext { get; init; } } } \ No newline at end of file diff --git a/src/Manifest/DepotDelta.cs b/src/Manifest/DepotDelta.cs index de88731..69a5642 100644 --- a/src/Manifest/DepotDelta.cs +++ b/src/Manifest/DepotDelta.cs @@ -47,7 +47,7 @@ void countAcq(in DirectoryEntry dir, DirectoryEntry.AcquisitionStaging acquisiti } } foreach (var subdir in acquisitionDir.Subdirectories) - countAcq(dir.Subdirectories[subdir.Index], subdir); + countAcq(in manifest.DirectoryBuffer[subdir.Index], subdir); } countAcq(in manifest.Root, acquisitionTree); ChunkBufferFileSize = chunkBufferFileSize;