3 new functions: DryRun script execution

Added DryRun Functions and DataServiceFactory

Install-Scripts.sln

@@ -7,7 +7,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SignScripts", "src\Signing\
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MonitoringFunctions", "src\MonitoringFunctions\MonitoringFunctions.csproj", "{BF6E1679-3E2C-4B36-BBA8-A9DAF1114CF3}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MonitoringFunctions.Test", "tests\MonitoringFunctions.Test\MonitoringFunctions.Test.csproj", "{EF881AB7-BFF9-4FAD-998A-7B6A6A93F2F9}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MonitoringFunctions.Test", "tests\MonitoringFunctions.Test\MonitoringFunctions.Test.csproj", "{EF881AB7-BFF9-4FAD-998A-7B6A6A93F2F9}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

src/MonitoringFunctions/DataService/DataServiceFactory.cs

@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using MonitoringFunctions.Providers;
+using System;
+
+namespace MonitoringFunctions
+{
+    internal sealed class DataServiceFactory
+    {
+        public IDataService GetDataService()
+        {
+            string? environment = Environment.GetEnvironmentVariable("AZURE_FUNCTIONS_ENVIRONMENT");
+
+            if(environment == "Development")
+            {
+                return new DummyDataService();
+            }
+            else
+            {
+                return new KustoDataService();
+            }
+        }
+    }
+}

src/MonitoringFunctions/IDataService.cs → src/MonitoringFunctions/DataService/IDataService.cs

@@ -16,5 +16,13 @@ internal interface IDataService : IDisposable
         /// <param name="httpResponse">Http response data to be stored.</param>
         /// <returns>A task, tracking the initiated async operation. Errors should be reported through exceptions.</returns>
         Task ReportUrlAccessAsync(string monitorName, HttpResponseMessage httpResponse, CancellationToken cancellationToken = default);
+
+        /// <summary>
+        /// Stores the details of the <see cref="HttpResponseMessage"/> in the underlying data store.
+        /// </summary>
+        /// <param name="monitorName">Name of the monitor that will be associated with this data.</param>
+        /// <param name="httpResponse">Http response data to be stored.</param>
+        /// <returns>A task, tracking the initiated async operation. Errors should be reported through exceptions.</returns>
+        Task ReportScriptExecutionAsync(string monitorName, string scriptName, string commandLineArgs, string error, CancellationToken cancellationToken = default);
     }
 }

src/MonitoringFunctions/DataService/Kusto/DirectJsonMappingResolver.cs

@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Kusto.Data.Common;
+using Newtonsoft.Json.Serialization;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace MonitoringFunctions.DataService.Kusto
+{
+    /// <summary>
+    /// Creates column mapping for objects whose json property names directly match the column name in the corresponding Kusto table.
+    /// </summary>
+    internal sealed class DirectJsonMappingResolver : IJsonColumnMappingResolver
+    {
+        public IEnumerable<ColumnMapping> GetColumnMappings<TModel>() where TModel : IKustoTableRow
+        {
+            DefaultContractResolver contractResolver = new DefaultContractResolver();
+            JsonObjectContract? contract = contractResolver.ResolveContract(typeof(TModel)) as JsonObjectContract;
+
+            if (contract == null)
+            {
+                throw new ArgumentException($"Failed to resolve contract. Automatic column mapping is not possible with this type {typeof(TModel)}.");
+            }
+
+            foreach (JsonProperty property in contract.Properties.Where(p => !p.Ignored && p.Readable))
+            {
+                yield return new ColumnMapping()
+                {
+                    ColumnName = property.PropertyName,
+                    Properties = new Dictionary<string, string>() { { "Path", $"$.{property.PropertyName}" } }
+                };
+            }
+        }
+    }
+}

src/MonitoringFunctions/DataService/Kusto/IJsonColumnMappingResolver.cs

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Kusto.Data.Common;
+using System.Collections.Generic;
+
+namespace MonitoringFunctions.DataService.Kusto
+{
+    internal interface IJsonColumnMappingResolver
+    {
+        /// <summary>
+        /// Returns Kusto column mapping objects for the given type where each column in Kusto table is mapped
+        /// to a property when data of the given type is serialized to JSON.
+        /// </summary>
+        /// <typeparam name="T">The model class that will be mapped to a Kusto table after serialization.</typeparam>
+        /// <returns>Column mapping for each of the columns in the Kusto table</returns>
+        IEnumerable<ColumnMapping> GetColumnMappings<T>() where T : IKustoTableRow;
+    }
+}

src/MonitoringFunctions/DataService/Kusto/IKustoTableRow.cs

@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace MonitoringFunctions.DataService.Kusto
+{
+    /// <summary>
+    /// This interface marks model classes that can be inserted into a kusto table as a row.
+    /// </summary>
+    internal interface IKustoTableRow
+    {
+
+    }
+}

src/MonitoringFunctions/DataService/Kusto/KustoTable.cs

@@ -0,0 +1,57 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Kusto.Data;
+using Kusto.Data.Common;
+using Kusto.Ingest;
+using Newtonsoft.Json;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace MonitoringFunctions.DataService.Kusto
+{
+    internal sealed class KustoTable<T> where T : IKustoTableRow
+    {
+        private KustoConnectionStringBuilder _connectionStringBuilder;
+
+        private KustoQueuedIngestionProperties _ingestionProperties;
+
+        public KustoTable(KustoConnectionStringBuilder connectionStringBuilder, string databaseName, string tableName, IJsonColumnMappingResolver columnMappingResolver)
+            : this(connectionStringBuilder, databaseName, tableName, columnMappingResolver.GetColumnMappings<T>())
+        {
+
+        }
+
+        public KustoTable(KustoConnectionStringBuilder connectionStringBuilder, string databaseName, string tableName, IEnumerable<ColumnMapping> columnMappings)
+        {
+            _connectionStringBuilder = connectionStringBuilder;
+
+            _ingestionProperties = new KustoQueuedIngestionProperties(databaseName, tableName)
+            {
+                ReportLevel = IngestionReportLevel.FailuresOnly,
+                ReportMethod = IngestionReportMethod.Queue,
+                IngestionMapping = new IngestionMapping()
+                {
+                    IngestionMappingKind = global::Kusto.Data.Ingestion.IngestionMappingKind.Json,
+                    IngestionMappings = columnMappings
+                },
+                Format = DataSourceFormat.json
+            };
+        }
+
+        public async Task InsertRowAsync(T row, CancellationToken cancellationToken = default)
+        {
+            using IKustoQueuedIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(_connectionStringBuilder);
+
+            string serializedData = JsonConvert.SerializeObject(row);
+            byte[] serializedBytes = Encoding.UTF8.GetBytes(serializedData);
+
+            using MemoryStream dataStream = new MemoryStream(serializedBytes);
+
+            // IKustoQueuedIngestClient doesn't support cancellation at the moment. Update the line below if it does in the future.
+            await ingestClient.IngestFromStreamAsync(dataStream, _ingestionProperties, leaveOpen: true);
+        }
+    }
+}

src/MonitoringFunctions/Providers/DummyDataService.cs → src/MonitoringFunctions/DataService/Providers/DummyDataService.cs

@@ -14,9 +14,15 @@ public async Task ReportUrlAccessAsync(string monitorName, HttpResponseMessage h
             await Task.Delay(new Random().Next(200, 4000), cancellationToken).ConfigureAwait(false);
         }
 
+        public async Task ReportScriptExecutionAsync(string monitorName, string scriptName, string commandLineArgs, string error, CancellationToken cancellationToken = default)
+        {
+            await Task.Delay(new Random().Next(200, 4000), cancellationToken).ConfigureAwait(false);
+        }
+
         public void Dispose()
         {
             // Do nothing
         }
+
     }
 }

src/MonitoringFunctions/DataService/Providers/KustoDataService.cs

@@ -0,0 +1,78 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Kusto.Data;
+using MonitoringFunctions.DataService.Kusto;
+using MonitoringFunctions.Models;
+using System;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace MonitoringFunctions
+{
+    internal sealed class KustoDataService : IDataService
+    {
+        private const string ServiceNameAndRegion = "dotnetinstallcluster.eastus2";
+        private const string DatabaseName = "dotnet_install_monitoring_database";
+
+        private readonly KustoTable<HttpRequestLogEntry> _httpRequestLogsTable;
+        private readonly KustoTable<ScriptExecutionLogEntry> _scriptExecutionLogsTable;
+
+        internal KustoDataService()
+        {
+            KustoConnectionStringBuilder kcsb = new KustoConnectionStringBuilder($"https://ingest-{ServiceNameAndRegion}.kusto.windows.net")
+                .WithAadManagedIdentity("system");
+
+            DirectJsonMappingResolver directJsonMappingResolver = new DirectJsonMappingResolver();
+
+            _httpRequestLogsTable = new KustoTable<HttpRequestLogEntry>(kcsb, DatabaseName, "UrlAccessLogs", directJsonMappingResolver);
+            _scriptExecutionLogsTable = new KustoTable<ScriptExecutionLogEntry>(kcsb, DatabaseName, "ScriptExecLogs", directJsonMappingResolver);
+        }
+
+        /// <summary>
+        /// Reports the details of the <see cref="HttpResponseMessage"/> to kusto.
+        /// </summary>
+        /// <param name="monitorName">Name of the monitor generating this data entry.</param>
+        /// <param name="httpResponse">Response to be reported.</param>
+        /// <returns>A task, tracking this async operation.</returns>
+        public async Task ReportUrlAccessAsync(string monitorName, HttpResponseMessage httpResponse, CancellationToken cancellationToken = default)
+        {
+            HttpRequestLogEntry logEntry = new HttpRequestLogEntry()
+            {
+                MonitorName = monitorName,
+                EventTime = DateTime.UtcNow,
+                RequestedUrl = httpResponse.RequestMessage.RequestUri.AbsoluteUri,
+                HttpResponseCode = (int)httpResponse.StatusCode
+            };
+
+            await _httpRequestLogsTable.InsertRowAsync(logEntry, cancellationToken).ConfigureAwait(false);
+        }
+
+        /// <summary>
+        /// Reports the details of a script execution to kusto.
+        /// </summary>
+        /// <param name="monitorName">Name of the monitor generating this data entry.</param>
+        /// <param name="scriptName">Name of the script that was executed.</param>
+        /// <param name="commandLineArgs">Command line arguments passed to the script at the moment of execution.</param>
+        /// <param name="error">Errors that occured during the execution, if any.</param>
+        /// <returns>A task, tracking this async operation.</returns>
+        public async Task ReportScriptExecutionAsync(string monitorName, string scriptName, string commandLineArgs, string error, CancellationToken cancellationToken = default)
+        {
+            ScriptExecutionLogEntry logEntry = new ScriptExecutionLogEntry()
+            {
+                MonitorName = monitorName,
+                EventTime = DateTime.UtcNow,
+                ScriptName = scriptName,
+                CommandLineArgs = commandLineArgs,
+                Error = error
+            };
+
+            await _scriptExecutionLogsTable.InsertRowAsync(logEntry, cancellationToken).ConfigureAwait(false);
+        }
+
+        public void Dispose()
+        {
+            // Do nothing
+        }
+    }
+}

src/MonitoringFunctions/Functions/DryRunUrlChecker.cs

@@ -0,0 +1,126 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.IO;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Extensions.Logging;
+using MonitoringFunctions.Models;
+using Kusto.Cloud.Platform.IO;
+using System.Threading;
+
+namespace MonitoringFunctions.Functions
+{
+    /// <summary>
+    /// Runs the scripts in -DryRun mode and checks weather the generated links are accessible
+    /// </summary>
+    internal static class DryRunUrlChecker
+    {
+        [FunctionName("DryRunLTS")]
+        public static async Task RunLTSAsync([TimerTrigger("0 */30 * * * *")] TimerInfo myTimer, ILogger log)
+        {
+            string monitorName = "dry_run_LTS";
+            string cmdArgs = "-c LTS";
+
+            await ExecuteDryRunCheckAndReportUrlAccessAsync(log, monitorName, cmdArgs).ConfigureAwait(false);
+        }
+
+        [FunctionName("DryRun3_1")]
+        public static async Task Run3_1Async([TimerTrigger("0 */30 * * * *")] TimerInfo myTimer, ILogger log)
+        {
+            string monitorName = "dry_run_3_1";
+            string cmdArgs = "-c 3.1";
+
+            await ExecuteDryRunCheckAndReportUrlAccessAsync(log, monitorName, cmdArgs).ConfigureAwait(false);
+        }
+
+        [FunctionName("DryRun3_0Runtime")]
+        public static async Task Run3_0RuntimeAsync([TimerTrigger("0 */30 * * * *")] TimerInfo myTimer, ILogger log)
+        {
+            string monitorName = "dry_run_3_0_runtime";
+            string cmdArgs = "-c 3.0 -Runtime dotnet";
+
+            await ExecuteDryRunCheckAndReportUrlAccessAsync(log, monitorName, cmdArgs).ConfigureAwait(false);
+        }
+
+        /// <summary>
+        /// Executes the Ps1 script with DryDun switch,
+        /// Parses the output to acquire primary and legacy Urls,
+        /// Tests the primary Url to see if it is available,
+        /// Reports the results to the data service provided.
+        /// </summary>
+        internal static async Task ExecuteDryRunCheckAndReportUrlAccessAsync(ILogger log, string monitorName, string additionalCmdArgs,
+            CancellationToken cancellationToken = default)
+        {
+            string scriptName = "dotnet-install.ps1";
+            string commandLineArgs = $"-DryRun {additionalCmdArgs}";
+
+            using IDataService dataService = new DataServiceFactory().GetDataService();
+
+            // Execute the script;
+            ScriptExecutionResult results = await HelperMethods.ExecuteInstallScriptPs1Async(commandLineArgs).ConfigureAwait(false);
+
+            log.LogInformation($"Ouput stream: {results.Output}");
+
+            if (!string.IsNullOrWhiteSpace(results.Error))
+            {
+                log.LogError($"Error stream: {results.Error}");
+                await dataService.ReportScriptExecutionAsync(monitorName, scriptName, commandLineArgs, results.Error, cancellationToken)
+                    .ConfigureAwait(false);
+                return;
+            }
+
+            // Parse the output
+            ScriptDryRunResult dryRunResults = ParseDryRunOutput(results.Output);
+
+            if (string.IsNullOrWhiteSpace(dryRunResults.PrimaryUrl))
+            {
+                log.LogError($"Primary Url was not found for channel {additionalCmdArgs}");
+                await dataService.ReportScriptExecutionAsync(monitorName, scriptName, commandLineArgs,
+                    "Failed to parse primary url from the following DryRun execution output: " + results.Output
+                    , cancellationToken).ConfigureAwait(false);
+                return;
+            }
+
+            // Validate URL accessibility
+            await HelperMethods.CheckAndReportUrlAccessAsync(log, monitorName, dryRunResults.PrimaryUrl, dataService);
+        }
+
+        /// <summary>
+        /// Parses the output of the script when executed in DryRun mode and finds out Primary and Legacy urls.
+        /// </summary>
+        /// <param name="output">Output of the script execution in DryRun mode</param>
+        /// <returns>Object containing primary and legacy runs</returns>
+        internal static ScriptDryRunResult ParseDryRunOutput(string? output)
+        {
+            string primaryUrlIdentifier = "Primary named payload URL: ";
+            string legacyUrlIdentifier = "Legacy named payload URL: ";
+
+            ScriptDryRunResult result = new ScriptDryRunResult();
+
+            using StringStream stringStream = new StringStream(output);
+            using StreamReader streamReader = new StreamReader(stringStream);
+
+            string? line;
+            while ((line = streamReader.ReadLine()) != null)
+            {
+                // Does this line contain the primary url?
+                int primaryIdIndex = line.IndexOf(primaryUrlIdentifier);
+                if (primaryIdIndex != -1)
+                {
+                    result.PrimaryUrl = line.Substring(primaryIdIndex + primaryUrlIdentifier.Length);
+                }
+                else
+                {
+                    // Does this line contain the legacy url?
+                    int legacyIdIndex = line.IndexOf(legacyUrlIdentifier);
+                    if(legacyIdIndex != -1)
+                    {
+                        result.LegacyUrl = line.Substring(legacyIdIndex + legacyUrlIdentifier.Length);
+                    }
+                }
+            }
+
+            return result;
+        }
+    }
+}