Merge pull request #929 from sitkoru/storage/refresh-embed-cache-on-c…

…hanges Invalidate embed storage cache
sitkoru · Feb 8, 2024 · 72795fd · 72795fd
2 parents c08da30 + bd45e3b
commit 72795fd
Show file tree

Hide file tree

Showing 6 changed files with 84 additions and 13 deletions.
diff --git a/src/Sitko.Core.Storage.FileSystem/FileSystemStorageMetadataProvider.cs b/src/Sitko.Core.Storage.FileSystem/FileSystemStorageMetadataProvider.cs
@@ -20,7 +20,7 @@ public FileSystemStorageMetadataProvider(IOptionsMonitor<TStorageOptions> storag
     {
     }
 
-    protected override Task DoDeleteMetadataAsync(string filePath, CancellationToken cancellationToken = default)
+    protected override Task DeleteEmbededMetadataAsync(string filePath, CancellationToken cancellationToken = default)
     {
         var fullPath = Path.Combine(StorageOptions.CurrentValue.StoragePath, filePath);
         var metaDataPath = GetMetaDataPath(fullPath);
@@ -32,7 +32,7 @@ protected override Task DoDeleteMetadataAsync(string filePath, CancellationToken
         return Task.CompletedTask;
     }
 
-    protected override Task DoDeleteAllMetadataAsync(CancellationToken cancellationToken = default) =>
+    protected override Task DeleteAllEmbededMetadataAsync(CancellationToken cancellationToken = default) =>
         Task.CompletedTask;
 
     protected override async Task<StorageItemMetadata?> DoGetMetadataJsonAsync(string path,
@@ -53,7 +53,7 @@ protected override Task DoDeleteAllMetadataAsync(CancellationToken cancellationT
         return null;
     }
 
-    protected override async Task DoSaveMetadataAsync(StorageItem storageItem, StorageItemMetadata? metadata = null,
+    protected override async Task SaveEmbededMetadataAsync(StorageItem storageItem, StorageItemMetadata? metadata = null,
         bool isNew = true,
         CancellationToken cancellationToken = default)
     {

diff --git a/src/Sitko.Core.Storage.Remote/RemoteStorageMetadataProvider.cs b/src/Sitko.Core.Storage.Remote/RemoteStorageMetadataProvider.cs
@@ -17,15 +17,15 @@ public RemoteStorageMetadataProvider(IOptionsMonitor<RemoteStorageMetadataModule
     }
 
     protected override Task
-        DoDeleteMetadataAsync(string filePath, CancellationToken cancellationToken = default) => Task.CompletedTask;
+        DeleteEmbededMetadataAsync(string filePath, CancellationToken cancellationToken = default) => Task.CompletedTask;
 
-    protected override Task DoDeleteAllMetadataAsync(CancellationToken cancellationToken = default) =>
+    protected override Task DeleteAllEmbededMetadataAsync(CancellationToken cancellationToken = default) =>
         Task.CompletedTask;
 
     protected override Task<StorageItemMetadata?> DoGetMetadataJsonAsync(string path,
         CancellationToken cancellationToken = default) => Task.FromResult<StorageItemMetadata?>(null);
 
-    protected override Task DoSaveMetadataAsync(StorageItem storageItem, StorageItemMetadata? metadata = null,
+    protected override Task SaveEmbededMetadataAsync(StorageItem storageItem, StorageItemMetadata? metadata = null,
         bool isNew = true,
         CancellationToken cancellationToken = default) => isNew
         ? Task.CompletedTask

diff --git a/src/Sitko.Core.Storage.S3/S3StorageMetadataProvider.cs b/src/Sitko.Core.Storage.S3/S3StorageMetadataProvider.cs
@@ -17,7 +17,7 @@ public S3StorageMetadataProvider(IOptionsMonitor<S3StorageMetadataModuleOptions<
     {
     }
 
-    protected override async Task DoDeleteMetadataAsync(string filePath,
+    protected override async Task DeleteEmbededMetadataAsync(string filePath,
         CancellationToken cancellationToken = default)
     {
         if (await Storage.IsObjectExistsAsync(filePath, cancellationToken))
@@ -27,10 +27,11 @@ await Storage.DeleteObjectAsync(GetMetaDataPath(filePath),
         }
     }
 
-    protected override Task DoDeleteAllMetadataAsync(CancellationToken cancellationToken = default) =>
+    protected override Task DeleteAllEmbededMetadataAsync(CancellationToken cancellationToken = default) =>
         Task.CompletedTask;
 
-    protected override async Task DoSaveMetadataAsync(StorageItem storageItem, StorageItemMetadata? metadata = null,
+    protected override async Task SaveEmbededMetadataAsync(StorageItem storageItem,
+        StorageItemMetadata? metadata = null,
         bool isNew = true,
         CancellationToken cancellationToken = default)
     {
@@ -61,4 +62,3 @@ await Storage.DoSaveInternalAsync(GetMetaDataPath(storageItem.FilePath),
 
 public class S3StorageMetadataModuleOptions<TStorageOptions> : EmbedStorageMetadataModuleOptions<TStorageOptions>
     where TStorageOptions : StorageOptions;
-
diff --git a/src/Sitko.Core.Storage/Metadata/EmbedStorageMetadataProvider.cs b/src/Sitko.Core.Storage/Metadata/EmbedStorageMetadataProvider.cs
@@ -8,7 +8,7 @@ namespace Sitko.Core.Storage.Metadata;
 
 public abstract class
     EmbedStorageMetadataProvider<TStorage, TStorageOptions, TOptions> : BaseStorageMetadataProvider<TOptions,
-        TStorageOptions>, IEmbedStorageMetadataProvider
+    TStorageOptions>, IEmbedStorageMetadataProvider
     where TStorage : IStorage<TStorageOptions>
     where TOptions : EmbedStorageMetadataModuleOptions<TStorageOptions>
     where TStorageOptions : StorageOptions
@@ -135,11 +135,41 @@ private async Task BuildStorageTreeAsync(CancellationToken cancellationToken = d
             Logger.LogInformation("Done building storage tree");
         }
     }
+
+
+    protected sealed override async Task DoDeleteMetadataAsync(string filePath,
+        CancellationToken cancellationToken = default)
+    {
+        await DeleteEmbededMetadataAsync(filePath, cancellationToken);
+        tree?.RemoveItem(filePath);
+    }
+
+    protected sealed override async Task DoSaveMetadataAsync(StorageItem storageItem,
+        StorageItemMetadata? metadata = null,
+        bool isNew = true,
+        CancellationToken cancellationToken = default)
+    {
+        await SaveEmbededMetadataAsync(storageItem, metadata, isNew, cancellationToken);
+        tree?.AddOrUpdateItem(storageItem);
+    }
+
+    protected sealed override async Task DoDeleteAllMetadataAsync(CancellationToken cancellationToken = default)
+    {
+        await DeleteAllEmbededMetadataAsync(cancellationToken);
+        tree = null;
+    }
+
+    protected abstract Task DeleteEmbededMetadataAsync(string filePath, CancellationToken cancellationToken = default);
+
+    protected abstract Task DeleteAllEmbededMetadataAsync(CancellationToken cancellationToken = default);
+
+    protected abstract Task SaveEmbededMetadataAsync(StorageItem storageItem, StorageItemMetadata? metadata = null,
+        bool isNew = true,
+        CancellationToken cancellationToken = default);
 }
 
 public class EmbedStorageMetadataModuleOptions<TStorageOptions> : StorageMetadataModuleOptions<TStorageOptions>
     where TStorageOptions : StorageOptions
 {
     public int StorageTreeCacheTimeoutInMinutes { get; set; } = 30;
 }
-
diff --git a/src/Sitko.Core.Storage/StorageNode.cs b/src/Sitko.Core.Storage/StorageNode.cs
@@ -120,11 +120,16 @@ public void RemoveItem(string filePath)
     }
 
     private static string PreparePath(string path) => path.Replace("\\", "/").Replace("//", "/");
+
+    public void AddOrUpdateItem(StorageItem storageItem)
+    {
+        RemoveItem(storageItem);
+        AddItem(storageItem);
+    }
 }
 
 public enum StorageNodeType
 {
     Directory,
     StorageItem
 }
-
diff --git a/tests/Sitko.Core.Storage.Tests/BasicTests.cs b/tests/Sitko.Core.Storage.Tests/BasicTests.cs
@@ -131,6 +131,42 @@ public async Task Traverse()
         await CheckFoldersContent(storage, uploaded, metaData);
     }
 
+    [Fact]
+    public async Task TraverseAfterUpdate()
+    {
+        var scope = await GetScopeAsync();
+
+        var storage = scope.GetService<IStorage>();
+
+        Assert.NotNull(storage);
+
+        StorageItem uploaded;
+        const string fileName = "file.txt";
+        var metaData = new FileMetaData();
+        var dir = "upload/dir1/dir2";
+        await using (var file = File.Open("Data/file.txt", FileMode.Open))
+        {
+            uploaded = await storage.SaveAsync(file, fileName, dir, metaData);
+        }
+
+        Assert.NotNull(uploaded);
+
+        var content = await storage.GetDirectoryContentsAsync(dir);
+        content.Should().HaveCount(1);
+
+        const string fileName2 = "file2.txt";
+        var metaData2 = new FileMetaData();
+        await using (var file = File.Open("Data/file.txt", FileMode.Open))
+        {
+            uploaded = await storage.SaveAsync(file, fileName2, dir, metaData2);
+        }
+
+        uploaded.Should().NotBeNull();
+
+        content = await storage.GetDirectoryContentsAsync(dir);
+        content.Should().HaveCount(2);
+    }
+
     protected static async Task CheckFoldersContent(IStorage storage, StorageItem uploaded,
         FileMetaData? metaData)
     {