Fallback behavior to ensure in-proc payload compatibility with dotnet-isolated as the FUNCTIONS_WORKER_RUNTIME value

release_notes.md

@@ -19,3 +19,4 @@
 - Worker termination path updated with sanitized logging (#10367)
 - Avoid redundant DiagnosticEvents error message (#10395)
 - Added logic to shim older versions of the .NET Worker JsonFunctionProvider to ensure backwards compatibility (#10410)
+- Added fallback behavior when FUNCTIONS_WORKER_RUNTIME does not match with metadata from deployed app payload.

src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs

@@ -183,6 +183,14 @@ internal bool ThrowOnMissingFunctionsWorkerRuntime
             }
         }
 
+        internal bool ThrowOnFunctionsWorkerRuntimeMismatchWithMetadataFromPayload
+        {
+            get
+            {
+                return GetFeatureAsBooleanOrDefault(RpcWorkerConstants.ThrowOnFunctionsWorkerRuntimeMismatchWithMetadataFromPayload, false);
+            }
+        }
+
         /// <summary>
         /// Gets feature by name.
         /// </summary>

src/WebJobs.Script/Diagnostics/DiagnosticEventConstants.cs

@@ -31,5 +31,8 @@ internal static class DiagnosticEventConstants
 
         public const string NonHISSecretLoaded = "AZFD0012";
         public const string NonHISSecretLoadedHelpLink = "https://aka.ms/functions-non-his-secrets";
+
+        public const string WorkerRuntimeDoesNotMatchWithFunctionMetadataErrorCode = "AZFD0013";
+        public const string WorkerRuntimeDoesNotMatchWithFunctionMetadataHelpLink = "https://aka.ms/functions-invalid-worker-runtime";
     }
 }

src/WebJobs.Script/Host/ScriptHost.cs

@@ -773,14 +773,49 @@ private void TrySetDirectType(FunctionMetadata metadata)
             }
         }
 
+        // Ensure customer deployed application payload matches with the worker runtime configured for the function app and log a warning if not.
+        // If a customer has "dotnet-isolated" worker runtime configured for the function app, and then they deploy an in-proc app payload, this will warn/error
+        // If there is a mismatch, the method will return false, else true.
+        internal static bool ValidateAndLogRuntimeMismatch(IEnumerable<FunctionMetadata> functionMetadata, string workerRuntime, IOptions<FunctionsHostingConfigOptions> hostingConfigOptions, ILogger logger)
+        {
+            if (functionMetadata != null && functionMetadata.Any() && !Utility.ContainsAnyFunctionMatchingWorkerRuntime(functionMetadata, workerRuntime))
+            {
+                string baseMessage = $"The '{EnvironmentSettingNames.FunctionWorkerRuntime}' is set to '{workerRuntime}', which does not match the worker runtime metadata found in the deployed function app artifacts. See {DiagnosticEventConstants.WorkerRuntimeDoesNotMatchWithFunctionMetadataHelpLink} for more information.";
+
+                if (hostingConfigOptions.Value.ThrowOnFunctionsWorkerRuntimeMismatchWithMetadataFromPayload)
+                {
+                    logger.LogDiagnosticEventError(DiagnosticEventConstants.WorkerRuntimeDoesNotMatchWithFunctionMetadataErrorCode, baseMessage, DiagnosticEventConstants.WorkerRuntimeDoesNotMatchWithFunctionMetadataHelpLink, null);
+                    throw new HostInitializationException(baseMessage);
+                }
+
+                string warningMessage = baseMessage + " The application will continue to run, but may throw an exception in the future.";
+                logger.LogDiagnosticEventWarning(DiagnosticEventConstants.WorkerRuntimeDoesNotMatchWithFunctionMetadataErrorCode, warningMessage, DiagnosticEventConstants.WorkerRuntimeDoesNotMatchWithFunctionMetadataHelpLink, null);
+                return false;
+            }
+
+            return true;
+        }
+
         internal async Task<Collection<FunctionDescriptor>> GetFunctionDescriptorsAsync(IEnumerable<FunctionMetadata> functions, IEnumerable<FunctionDescriptorProvider> descriptorProviders, string workerRuntime, CancellationToken cancellationToken)
         {
             Collection<FunctionDescriptor> functionDescriptors = new Collection<FunctionDescriptor>();
             if (!cancellationToken.IsCancellationRequested)
             {
+                bool throwOnWorkerRuntimeAndPayloadMetadataMismatch = true;
+                // this dotnet isolated specific logic is temporary to ensure in-proc payload compatibility with "dotnet-isolated" as the FUNCTIONS_WORKER_RUNTIME value.
+                if (string.Equals(workerRuntime, RpcWorkerConstants.DotNetIsolatedLanguageWorkerName, StringComparison.OrdinalIgnoreCase))
+                {
+                    bool isDotnetIsolatedRuntimeWithValidPayload = ValidateAndLogRuntimeMismatch(functions, workerRuntime, _hostingConfigOptions, _logger);
+                    if (!isDotnetIsolatedRuntimeWithValidPayload)
+                    {
+                        UpdateFunctionMetadataLanguageForDotnetAssembly(functions, workerRuntime);
+                        throwOnWorkerRuntimeAndPayloadMetadataMismatch = false; // we do not want to throw an exception in this case
+                    }
+                }
+
                 var httpFunctions = new Dictionary<string, HttpTriggerAttribute>();
 
-                Utility.VerifyFunctionsMatchSpecifiedLanguage(functions, workerRuntime, _environment.IsPlaceholderModeEnabled(), _isHttpWorker, cancellationToken);
+                Utility.VerifyFunctionsMatchSpecifiedLanguage(functions, workerRuntime, _environment.IsPlaceholderModeEnabled(), _isHttpWorker, cancellationToken, throwOnMismatch: throwOnWorkerRuntimeAndPayloadMetadataMismatch);
 
                 foreach (FunctionMetadata metadata in functions)
                 {
@@ -819,6 +854,17 @@ internal async Task<Collection<FunctionDescriptor>> GetFunctionDescriptorsAsync(
             return functionDescriptors;
         }
 
+        private static void UpdateFunctionMetadataLanguageForDotnetAssembly(IEnumerable<FunctionMetadata> functions, string workerRuntime)
+        {
+            foreach (var function in functions)
+            {
+                if (function.Language == DotNetScriptTypes.DotNetAssembly)
+                {
+                    function.Language = workerRuntime;
+                }
+            }
+        }
+
         internal static void ValidateFunction(FunctionDescriptor function, Dictionary<string, HttpTriggerAttribute> httpFunctions, IEnvironment environment)
         {
             var httpTrigger = function.HttpTriggerAttribute;

src/WebJobs.Script/Utility.cs

@@ -629,7 +629,7 @@ internal static bool TryReadFunctionConfig(string scriptDir, out string json, IF
             return true;
         }
 
-        internal static void VerifyFunctionsMatchSpecifiedLanguage(IEnumerable<FunctionMetadata> functions, string workerRuntime, bool isPlaceholderMode, bool isHttpWorker, CancellationToken cancellationToken)
+        internal static void VerifyFunctionsMatchSpecifiedLanguage(IEnumerable<FunctionMetadata> functions, string workerRuntime, bool isPlaceholderMode, bool isHttpWorker, CancellationToken cancellationToken, bool throwOnMismatch = true)
         {
             cancellationToken.ThrowIfCancellationRequested();
 
@@ -644,7 +644,10 @@ internal static void VerifyFunctionsMatchSpecifiedLanguage(IEnumerable<FunctionM
                 {
                     throw new HostInitializationException($"Found functions with more than one language. Select a language for your function app by specifying {RpcWorkerConstants.FunctionWorkerRuntimeSettingName} AppSetting");
                 }
-                else
+            }
+            else
+            {
+                if (throwOnMismatch)
                 {
                     throw new HostInitializationException($"Did not find functions with language [{workerRuntime}].");
                 }
@@ -748,10 +751,20 @@ private static bool ContainsFunctionWithWorkerRuntime(IEnumerable<FunctionMetada
             {
                 return functions.Any(f => dotNetLanguages.Any(l => l.Equals(f.Language, StringComparison.OrdinalIgnoreCase)));
             }
+
+            return ContainsAnyFunctionMatchingWorkerRuntime(functions, workerRuntime);
+        }
+
+        /// <summary>
+        /// Inspect the functions metadata to determine if at least one function is of the specified worker runtime.
+        /// </summary>
+        internal static bool ContainsAnyFunctionMatchingWorkerRuntime(IEnumerable<FunctionMetadata> functions, string workerRuntime)
+        {
             if (functions != null && functions.Any())
             {
                 return functions.Any(f => !string.IsNullOrEmpty(f.Language) && f.Language.Equals(workerRuntime, StringComparison.OrdinalIgnoreCase));
             }
+
             return false;
         }
 

src/WebJobs.Script/Workers/Rpc/FunctionRegistration/RpcFunctionInvocationDispatcher.cs

@@ -40,6 +40,9 @@ internal class RpcFunctionInvocationDispatcher : IFunctionInvocationDispatcher
         private readonly Lazy<Task<int>> _maxProcessCount;
         private readonly IOptions<FunctionsHostingConfigOptions> _hostingConfigOptions;
         private readonly IHostMetrics _hostMetrics;
+        private readonly TimeSpan _defaultProcessStartupInterval = TimeSpan.FromSeconds(5);
+        private readonly TimeSpan _defaultProcessRestartInterval = TimeSpan.FromSeconds(5);
+        private readonly TimeSpan _defaultProcessShutdownInterval = TimeSpan.FromSeconds(5);
 
         private IScriptEventManager _eventManager;
         private IWebHostRpcWorkerChannelManager _webHostLanguageWorkerChannelManager;
@@ -308,13 +311,19 @@ public async Task InitializeAsync(IEnumerable<FunctionMetadata> functions, Cance
             }
             else
             {
-                _processStartupInterval = workerConfig.CountOptions.ProcessStartupInterval;
-                _restartWait = workerConfig.CountOptions.ProcessRestartInterval;
-                _shutdownTimeout = workerConfig.CountOptions.ProcessShutdownTimeout;
+                _processStartupInterval = workerConfig?.CountOptions?.ProcessStartupInterval ?? _defaultProcessStartupInterval;
+                _restartWait = workerConfig?.CountOptions.ProcessRestartInterval ?? _defaultProcessRestartInterval;
+                _shutdownTimeout = workerConfig?.CountOptions.ProcessShutdownTimeout ?? _defaultProcessShutdownInterval;
             }
             ErrorEventsThreshold = 3 * await _maxProcessCount.Value;
 
-            if (Utility.IsSupportedRuntime(_workerRuntime, _workerConfigs) || _environment.IsMultiLanguageRuntimeEnvironment())
+            // If the configured worker runtime is "dotnet-isolated" and no worker config is found, we should assume that this was caused by
+            //  1. App did not start in placeholder mode (so that the dotnet-isolated worker config was not initialized because of https://github.com/Azure/azure-functions-dotnet-worker/pull/2552)
+            //  2. App payload deployed is not "dotnet-isolated" (but "in-proc")
+            // In this case, we want to go ahead and initialize a language worker channel for the runtime.
+            var missingWorkerConfigForDotnetIsolated = !_workerConfigs.Any() && _workerRuntime == RpcWorkerConstants.DotNetIsolatedLanguageWorkerName;
+
+            if (missingWorkerConfigForDotnetIsolated || Utility.IsSupportedRuntime(_workerRuntime, _workerConfigs) || _environment.IsMultiLanguageRuntimeEnvironment())
             {
                 State = FunctionInvocationDispatcherState.Initializing;
                 IDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> webhostLanguageWorkerChannels = _webHostLanguageWorkerChannelManager.GetChannels(_workerRuntime);

src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs

@@ -93,5 +93,6 @@ public static class RpcWorkerConstants
         public const string RevertWorkerShutdownBehavior = "REVERT_WORKER_SHUTDOWN_BEHAVIOR";
         public const string ShutdownWebhostWorkerChannelsOnHostShutdown = "ShutdownWebhostWorkerChannelsOnHostShutdown";
         public const string ThrowOnMissingFunctionsWorkerRuntime = "THROW_ON_MISSING_FUNCTIONS_WORKER_RUNTIME";
+        public const string ThrowOnFunctionsWorkerRuntimeMismatchWithMetadataFromPayload = "THROW_ON_FUNCTIONS_WORKER_RUNTIME_MISMATCH_WITH_PAYLOAD";
     }
 }

test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/WebJobsStartupEndToEndTests.cs

@@ -3,6 +3,7 @@
 using System.Net;
 using System.Threading.Tasks;
 using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
 using Microsoft.Extensions.DependencyInjection;
 using Xunit;
 
@@ -45,6 +46,27 @@ public async Task ExternalStartup_Succeeds()
             }
         }
 
+        [Fact]
+        public async Task InProcAppsWorkWithDotnetIsolatedAsFunctionWorkerRuntimeValue()
+        {
+            // test uses an in-proc app, but we are setting "dotnet-isolated" as functions worker runtime value.
+            var fixture = new CSharpPrecompiledEndToEndTestFixture(_projectName, _envVars, functionWorkerRuntime: RpcWorkerConstants.DotNetIsolatedLanguageWorkerName);
+            try
+            {
+                await fixture.InitializeAsync();
+                var client = fixture.Host.HttpClient;
+
+                var response = await client.GetAsync($"api/Function1");
+
+                // The function does all the validation internally.
+                Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+            }
+            finally
+            {
+                await fixture.DisposeAsync();
+            }
+        }
+
         [Fact]
         public async Task ExternalStartup_InvalidOverwrite_StopsHost()
         {