sync dev w/ main

.github/policies/resourceManagement.yml

@@ -57,6 +57,16 @@ configuration:
           reply: This issue has been marked as duplicate and has not had any activity for **1 day**. It will be closed for housekeeping purposes.
       - closeIssue
     eventResponderTasks:
+    - if:
+          - payloadType: Issues
+          - and:
+              - isOpen
+              - not:
+                  and:
+                    - isLabeled
+      then:
+        - addLabel:
+            label: "Needs: Triage :mag:"
     - if:
       - payloadType: Issue_Comment
       - isAction:

samples/TokenCredentialDTFx/TokenCredentialDTFx.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
@@ -8,7 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Azure.Identity" Version="1.7.0" />
+    <PackageReference Include="Azure.Identity" Version="1.10.2" />
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
     <PackageReference Include="Microsoft.Azure.DurableTask.Netherite" Version="1.3.2" />
   </ItemGroup>

src/DurableTask.Netherite/OrchestrationService/PartitionErrorHandler.cs

@@ -61,7 +61,7 @@ public PartitionErrorHandler(int partitionId, ILogger logger, LogLevel logLevelL
             this.host = host;
         }
 
-        public void HandleError(string context, string message, Exception exception, bool terminatePartition, bool isWarning)
+        public void HandleError(string context, string message, Exception? exception, bool terminatePartition, bool isWarning)
         {
             bool isFatal = exception != null && Utils.IsFatal(exception);
 
@@ -94,7 +94,7 @@ public void TerminateNormally()
             }
         }
 
-        void TraceError(bool isWarning, string context, string message, Exception exception, bool terminatePartition)
+        void TraceError(bool isWarning, string context, string message, Exception? exception, bool terminatePartition)
         {
             var logLevel = isWarning ? LogLevel.Warning : LogLevel.Error;
             if (this.logLevelLimit <= logLevel)

src/DurableTask.Netherite/StorageLayer/Faster/FasterKV.cs

@@ -484,7 +484,11 @@ long MinimalLogSize
         public override async Task<long> RunCompactionAsync(long target)
         {
             string id = DateTime.UtcNow.ToString("O"); // for tracing purposes
+
+            this.blobManager.TraceHelper.FasterProgress($"Compaction {id} is requesting to enter semaphore with {maxCompactionThreads.CurrentCount} threads available");
             await maxCompactionThreads.WaitAsync();
+            this.blobManager.TraceHelper.FasterProgress($"Compaction {id} entered semaphore");
+
             try
             {
                 long beginAddressBeforeCompaction = this.Log.BeginAddress;
@@ -499,20 +503,47 @@ public override async Task<long> RunCompactionAsync(long target)
                     target - this.Log.BeginAddress,
                     this.GetElapsedCompactionMilliseconds());
 
+                var tokenSource = new CancellationTokenSource();
+                var timeoutTask = Task.Delay(TimeSpan.FromMinutes(10), tokenSource.Token);
                 var tcs = new TaskCompletionSource<long>(TaskCreationOptions.RunContinuationsAsynchronously);
                 var thread = TrackedThreads.MakeTrackedThread(RunCompaction, $"Compaction.{id}");
                 thread.Start();
+
+                var winner = await Task.WhenAny(tcs.Task, timeoutTask);
+
+                if (winner == timeoutTask)
+                {
+                    // compaction timed out. Terminate partition
+                    var exceptionMessage = $"Compaction {id} time out";
+                    this.partition.ErrorHandler.HandleError(nameof(RunCompactionAsync), exceptionMessage, e: null, terminatePartition: true, reportAsWarning: true);
+
+                    // we need resolve the task to ensure the 'finally' block is executed which frees up another thread to start compating
+                    tcs.TrySetException(new OperationCanceledException(exceptionMessage));
+                }
+                else
+                {
+                    // cancel the timeout task since compaction completed
+                    tokenSource.Cancel();
+                }
+
+                await timeoutTask.ContinueWith(_ => tokenSource.Dispose());
+
+                // return result of compaction task
                 return await tcs.Task;
 
                 void RunCompaction()
                 {
                     try
                     {
+
                         this.blobManager.TraceHelper.FasterProgress($"Compaction {id} started");
 
                         var session = this.CreateASession($"compaction-{id}", true);
+
+                        this.blobManager.TraceHelper.FasterProgress($"Compaction {id} obtained a FASTER session");
                         using (this.TrackTemporarySession(session))
                         {
+                            this.blobManager.TraceHelper.FasterProgress($"Compaction {id} is invoking FASTER's compaction routine");
                             long compactedUntil = session.Compact(target, CompactionType.Scan);
 
                             this.TraceHelper.FasterCompactionProgress(
@@ -525,17 +556,17 @@ void RunCompaction()
                                 this.Log.BeginAddress - beginAddressBeforeCompaction,
                                 this.GetElapsedCompactionMilliseconds());
 
-                            tcs.SetResult(compactedUntil);
+                            tcs.TrySetResult(compactedUntil);
                         }
                     }
                     catch (Exception exception)
                         when (this.terminationToken.IsCancellationRequested && !Utils.IsFatal(exception))
                     {
-                        tcs.SetException(new OperationCanceledException("Partition was terminated.", exception, this.terminationToken));
+                        tcs.TrySetException(new OperationCanceledException("Partition was terminated.", exception, this.terminationToken));
                     }
                     catch (Exception e)
                     {
-                        tcs.SetException(e);
+                        tcs.TrySetException(e);
                     }
                 }
             }

src/DurableTask.Netherite/StorageLayer/Faster/FasterTraceHelper.cs

@@ -128,6 +128,15 @@ public void FasterProgress(string details)
             }
         }
 
+        public void FasterProgress(Func<string> constructString)
+        {
+            if (this.logLevelLimit <= LogLevel.Debug)
+            {
+                var details = constructString();
+                this.FasterProgress(details);
+            }
+        }
+
         public void FasterStorageProgress(string details)
         {
             if (this.logLevelLimit <= LogLevel.Trace)