Upgrade FASTER dependency to v2.6.4 (#344)

* initial commit * added comment * added readCopyOptions * addressed PR feedback * updated CheckpointVersionShift() * updated Netherite version * Update src/DurableTask.Netherite/StorageLayer/Faster/FasterStorageProvider.cs * Update src/DurableTask.Netherite/DurableTask.Netherite.csproj * update to FASTER v2.6.3 * update to FASTER 2.6.4 --------- Co-authored-by: Sebastian Burckhardt <[email protected]>
microsoft · Apr 22, 2024 · 6870713 · 6870713
1 parent 40a1f8c
commit 6870713
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 6 deletions.
diff --git a/src/DurableTask.Netherite/DurableTask.Netherite.csproj b/src/DurableTask.Netherite/DurableTask.Netherite.csproj
@@ -56,9 +56,9 @@
     <PackageReference Include="Microsoft.Azure.EventHubs.Processor" Version="4.3.2" />
     <PackageReference Include="Microsoft.Azure.Storage.Blob" Version="11.2.3" />
     <PackageReference Include="Azure.Storage.Blobs" Version="12.16.0" />
-	<PackageReference Include="Microsoft.FASTER.Core" Version="2.0.23" />
     <PackageReference Include="Microsoft.Azure.DurableTask.Core" Version="2.15.1" />
-	<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+    <PackageReference Include="Microsoft.FASTER.Core" Version="2.6.4" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
     <PackageReference Include="System.Threading.Channels" Version="4.7.1" />
   </ItemGroup>

diff --git a/src/DurableTask.Netherite/StorageLayer/Faster/AzureBlobs/BlobManager.cs b/src/DurableTask.Netherite/StorageLayer/Faster/AzureBlobs/BlobManager.cs
@@ -124,7 +124,7 @@ public LogSettings GetStoreLogSettings(bool useSeparatePageBlobStorage, long upp
                 MutableFraction = tuningParameters?.StoreLogMutableFraction ?? 0.9,
                 SegmentSizeBits = segmentSizeBits,
                 PreallocateLog = false,
-                ReadFlags = ReadFlags.None,
+                ReadCopyOptions = default, // is overridden by the per-session configuration
                 ReadCacheSettings = null, // no read cache
                 MemorySizeBits = memorySizeBits,
             };
@@ -886,7 +886,7 @@ async Task DeleteCheckpointDirectory(IEnumerable<string> blobsToDelete)
 
         #region ILogCommitManager
 
-        void ILogCommitManager.Commit(long beginAddress, long untilAddress, byte[] commitMetadata, long commitNum)
+        void ILogCommitManager.Commit(long beginAddress, long untilAddress, byte[] commitMetadata, long commitNum, bool forceWriteMetadata)
         {
             try
             {
@@ -1482,5 +1482,10 @@ await this.PerformWithRetriesAsync(
                     });
             }
         }
+
+        public void CheckpointVersionShift(long oldVersion, long newVersion)
+        {
+            // no-op
+        }
     }
 }
diff --git a/src/DurableTask.Netherite/StorageLayer/Faster/AzureBlobs/LocalFileCheckpointManager.cs b/src/DurableTask.Netherite/StorageLayer/Faster/AzureBlobs/LocalFileCheckpointManager.cs
@@ -91,5 +91,10 @@ public void OnRecovery(Guid indexToken, Guid logToken)
 
         void IDisposable.Dispose()
             => this.localCheckpointManager.Dispose();
+
+        public void CheckpointVersionShift(long oldVersion, long newVersion)
+        {
+            // no-op
+        }
     }
 }
diff --git a/src/DurableTask.Netherite/StorageLayer/Faster/FasterKV.cs b/src/DurableTask.Netherite/StorageLayer/Faster/FasterKV.cs
@@ -182,7 +182,12 @@ double GetElapsedCompactionMilliseconds()
         ClientSession<Key, Value, EffectTracker, Output, object, IFunctions<Key, Value, EffectTracker, Output, object>> CreateASession(string id, bool isScan)
         {
             var functions = new Functions(this.partition, this, this.cacheTracker, isScan);
-            return this.fht.NewSession(functions, id, readFlags: (isScan ? ReadFlags.None : ReadFlags.CopyReadsToTail));
+
+            ReadCopyOptions readCopyOptions = isScan
+                ? new ReadCopyOptions(ReadCopyFrom.None, ReadCopyTo.None)
+                : new ReadCopyOptions(ReadCopyFrom.AllImmutable, ReadCopyTo.MainLog);
+
+            return this.fht.NewSession(functions, id, default, readCopyOptions);
         }
 
         public IDisposable TrackTemporarySession(ClientSession<Key, Value, EffectTracker, Output, object, IFunctions<Key, Value, EffectTracker, Output, object>> session)

diff --git a/src/DurableTask.Netherite/StorageLayer/Faster/FasterStorageProvider.cs b/src/DurableTask.Netherite/StorageLayer/Faster/FasterStorageProvider.cs
@@ -142,6 +142,7 @@ async Task<bool> IStorageLayer.CreateTaskhubIfNotExistsAsync()
                 this.traceHelper.TraceProgress($"Using existing blob container at {this.cloudBlobContainer.Result.Uri}");
             }
 
+
             var taskHubParameters = new TaskhubParameters()
             {
                 TaskhubName = this.settings.HubName,

diff --git a/test/DurableTask.Netherite.Tests/DurableTask.Netherite.Tests.csproj b/test/DurableTask.Netherite.Tests/DurableTask.Netherite.Tests.csproj
@@ -9,7 +9,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
     <PackageReference Include="xunit" Version="2.4.2" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
-Original file line number
+Diff line change
@@ Expand Up @@
                     this.traceHelper.TraceProgress($"Using existing blob container at {this.cloudBlobContainer.Result.Uri}");
                 }
                 var taskHubParameters = new TaskhubParameters()
                 {
                     TaskhubName = this.settings.HubName,
@@ Expand Down @@