feat(storage): speedup metadata tree building by downloading metadata…

… in parallel
sitkoru · Feb 9, 2024 · 23bac7b · 23bac7b
1 parent 3a06b9a
commit 23bac7b
Showing 1 changed file with 26 additions and 10 deletions.
diff --git a/src/Sitko.Core.Storage/Metadata/EmbedStorageMetadataProvider.cs b/src/Sitko.Core.Storage/Metadata/EmbedStorageMetadataProvider.cs
@@ -1,4 +1,5 @@
-using JetBrains.Annotations;
+using System.Collections.Concurrent;
+using JetBrains.Annotations;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Nito.AsyncEx;
@@ -110,18 +111,32 @@ private async Task BuildStorageTreeAsync(CancellationToken cancellationToken = d
             Logger.LogInformation("Start building storage tree");
             treeBuildTaskSource = new TaskCompletionSource<bool>();
             tree = StorageNode.CreateDirectory("/", "/");
-            var items = await Storage.GetAllItemsAsync("/", cancellationToken);
-            foreach (var info in items)
-            {
-                if (info.Path.EndsWith(MetaDataExtension, StringComparison.InvariantCulture))
+            var items = (await Storage.GetAllItemsAsync("/", cancellationToken)).ToDictionary(info => info.Path,
+                info => info);
+            var metadataItems =
+                items.Where(pair => pair.Key.EndsWith(MetaDataExtension, StringComparison.InvariantCulture))
+                    .Select(pair => pair.Value).ToList();
+            var allMetadata = new ConcurrentDictionary<string, StorageItemMetadata?>();
+            Logger.LogInformation("Download {Count} metadata items", metadataItems.Count);
+            await Parallel.ForEachAsync(metadataItems,
+                new ParallelOptions
                 {
-                    continue;
-                }
-
+                    CancellationToken = cancellationToken,
+                    MaxDegreeOfParallelism = Options.CurrentValue.MaxParallelDownloadStreams
+                }, async (info, token) =>
+                {
+                    var filePath = info.Path.Replace(MetaDataExtension, "");
+                    var metadata = await DoGetMetadataAsync(filePath, token);
+                    allMetadata.TryAdd(filePath, metadata);
+                });
+            Logger.LogInformation("Metadata downloaded");
+            foreach (var (path, info) in items.Where(pair =>
+                         !pair.Key.EndsWith(MetaDataExtension, StringComparison.InvariantCulture)))
+            {
                 StorageItemMetadata? metadata = null;
-                if (info.Metadata is null)
+                if (allMetadata.TryGetValue(path, out var itemMetadata) && itemMetadata is not null)
                 {
-                    metadata = await DoGetMetadataAsync(info.Path, cancellationToken);
+                    metadata = itemMetadata;
                 }
 
                 var item = info.GetStorageItem(metadata);
@@ -172,4 +187,5 @@ public class EmbedStorageMetadataModuleOptions<TStorageOptions> : StorageMetadat
     where TStorageOptions : StorageOptions
 {
     public int StorageTreeCacheTimeoutInMinutes { get; set; } = 30;
+    public int MaxParallelDownloadStreams { get; set; } = 8;
 }