diff --git a/Install-Scripts.sln b/Install-Scripts.sln index a26b2bb22..4ffed47d4 100644 --- a/Install-Scripts.sln +++ b/Install-Scripts.sln @@ -7,7 +7,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SignScripts", "src\Signing\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MonitoringFunctions", "src\MonitoringFunctions\MonitoringFunctions.csproj", "{BF6E1679-3E2C-4B36-BBA8-A9DAF1114CF3}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MonitoringFunctions.Test", "tests\MonitoringFunctions.Test\MonitoringFunctions.Test.csproj", "{EF881AB7-BFF9-4FAD-998A-7B6A6A93F2F9}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MonitoringFunctions.Test", "tests\MonitoringFunctions.Test\MonitoringFunctions.Test.csproj", "{EF881AB7-BFF9-4FAD-998A-7B6A6A93F2F9}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/src/MonitoringFunctions/DataService/DataServiceFactory.cs b/src/MonitoringFunctions/DataService/DataServiceFactory.cs new file mode 100644 index 000000000..ccd61eb2c --- /dev/null +++ b/src/MonitoringFunctions/DataService/DataServiceFactory.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using MonitoringFunctions.Providers; +using System; + +namespace MonitoringFunctions +{ + internal sealed class DataServiceFactory + { + public IDataService GetDataService() + { + string? environment = Environment.GetEnvironmentVariable("AZURE_FUNCTIONS_ENVIRONMENT"); + + if(environment == "Development") + { + return new DummyDataService(); + } + else + { + return new KustoDataService(); + } + } + } +} diff --git a/src/MonitoringFunctions/IDataService.cs b/src/MonitoringFunctions/DataService/IDataService.cs similarity index 57% rename from src/MonitoringFunctions/IDataService.cs rename to src/MonitoringFunctions/DataService/IDataService.cs index d383eb4a1..3854cd5d7 100644 --- a/src/MonitoringFunctions/IDataService.cs +++ b/src/MonitoringFunctions/DataService/IDataService.cs @@ -16,5 +16,13 @@ internal interface IDataService : IDisposable /// Http response data to be stored. /// A task, tracking the initiated async operation. Errors should be reported through exceptions. Task ReportUrlAccessAsync(string monitorName, HttpResponseMessage httpResponse, CancellationToken cancellationToken = default); + + /// + /// Stores the details of the in the underlying data store. + /// + /// Name of the monitor that will be associated with this data. + /// Http response data to be stored. + /// A task, tracking the initiated async operation. Errors should be reported through exceptions. + Task ReportScriptExecutionAsync(string monitorName, string scriptName, string commandLineArgs, string error, CancellationToken cancellationToken = default); } } diff --git a/src/MonitoringFunctions/DataService/Kusto/DirectJsonMappingResolver.cs b/src/MonitoringFunctions/DataService/Kusto/DirectJsonMappingResolver.cs new file mode 100644 index 000000000..25396ed44 --- /dev/null +++ b/src/MonitoringFunctions/DataService/Kusto/DirectJsonMappingResolver.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Kusto.Data.Common; +using Newtonsoft.Json.Serialization; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace MonitoringFunctions.DataService.Kusto +{ + /// + /// Creates column mapping for objects whose json property names directly match the column name in the corresponding Kusto table. + /// + internal sealed class DirectJsonMappingResolver : IJsonColumnMappingResolver + { + public IEnumerable GetColumnMappings() where TModel : IKustoTableRow + { + DefaultContractResolver contractResolver = new DefaultContractResolver(); + JsonObjectContract? contract = contractResolver.ResolveContract(typeof(TModel)) as JsonObjectContract; + + if (contract == null) + { + throw new ArgumentException($"Failed to resolve contract. Automatic column mapping is not possible with this type {typeof(TModel)}."); + } + + foreach (JsonProperty property in contract.Properties.Where(p => !p.Ignored && p.Readable)) + { + yield return new ColumnMapping() + { + ColumnName = property.PropertyName, + Properties = new Dictionary() { { "Path", $"$.{property.PropertyName}" } } + }; + } + } + } +} diff --git a/src/MonitoringFunctions/DataService/Kusto/IJsonColumnMappingResolver.cs b/src/MonitoringFunctions/DataService/Kusto/IJsonColumnMappingResolver.cs new file mode 100644 index 000000000..2c0a5298e --- /dev/null +++ b/src/MonitoringFunctions/DataService/Kusto/IJsonColumnMappingResolver.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Kusto.Data.Common; +using System.Collections.Generic; + +namespace MonitoringFunctions.DataService.Kusto +{ + internal interface IJsonColumnMappingResolver + { + /// + /// Returns Kusto column mapping objects for the given type where each column in Kusto table is mapped + /// to a property when data of the given type is serialized to JSON. + /// + /// The model class that will be mapped to a Kusto table after serialization. + /// Column mapping for each of the columns in the Kusto table + IEnumerable GetColumnMappings() where T : IKustoTableRow; + } +} diff --git a/src/MonitoringFunctions/DataService/Kusto/IKustoTableRow.cs b/src/MonitoringFunctions/DataService/Kusto/IKustoTableRow.cs new file mode 100644 index 000000000..c60360028 --- /dev/null +++ b/src/MonitoringFunctions/DataService/Kusto/IKustoTableRow.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace MonitoringFunctions.DataService.Kusto +{ + /// + /// This interface marks model classes that can be inserted into a kusto table as a row. + /// + internal interface IKustoTableRow + { + + } +} diff --git a/src/MonitoringFunctions/DataService/Kusto/KustoTable.cs b/src/MonitoringFunctions/DataService/Kusto/KustoTable.cs new file mode 100644 index 000000000..8a555cc01 --- /dev/null +++ b/src/MonitoringFunctions/DataService/Kusto/KustoTable.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Kusto.Data; +using Kusto.Data.Common; +using Kusto.Ingest; +using Newtonsoft.Json; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace MonitoringFunctions.DataService.Kusto +{ + internal sealed class KustoTable where T : IKustoTableRow + { + private KustoConnectionStringBuilder _connectionStringBuilder; + + private KustoQueuedIngestionProperties _ingestionProperties; + + public KustoTable(KustoConnectionStringBuilder connectionStringBuilder, string databaseName, string tableName, IJsonColumnMappingResolver columnMappingResolver) + : this(connectionStringBuilder, databaseName, tableName, columnMappingResolver.GetColumnMappings()) + { + + } + + public KustoTable(KustoConnectionStringBuilder connectionStringBuilder, string databaseName, string tableName, IEnumerable columnMappings) + { + _connectionStringBuilder = connectionStringBuilder; + + _ingestionProperties = new KustoQueuedIngestionProperties(databaseName, tableName) + { + ReportLevel = IngestionReportLevel.FailuresOnly, + ReportMethod = IngestionReportMethod.Queue, + IngestionMapping = new IngestionMapping() + { + IngestionMappingKind = global::Kusto.Data.Ingestion.IngestionMappingKind.Json, + IngestionMappings = columnMappings + }, + Format = DataSourceFormat.json + }; + } + + public async Task InsertRowAsync(T row, CancellationToken cancellationToken = default) + { + using IKustoQueuedIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(_connectionStringBuilder); + + string serializedData = JsonConvert.SerializeObject(row); + byte[] serializedBytes = Encoding.UTF8.GetBytes(serializedData); + + using MemoryStream dataStream = new MemoryStream(serializedBytes); + + // IKustoQueuedIngestClient doesn't support cancellation at the moment. Update the line below if it does in the future. + await ingestClient.IngestFromStreamAsync(dataStream, _ingestionProperties, leaveOpen: true); + } + } +} diff --git a/src/MonitoringFunctions/Providers/DummyDataService.cs b/src/MonitoringFunctions/DataService/Providers/DummyDataService.cs similarity index 66% rename from src/MonitoringFunctions/Providers/DummyDataService.cs rename to src/MonitoringFunctions/DataService/Providers/DummyDataService.cs index bcd8b15e6..29f19f9b8 100644 --- a/src/MonitoringFunctions/Providers/DummyDataService.cs +++ b/src/MonitoringFunctions/DataService/Providers/DummyDataService.cs @@ -14,9 +14,15 @@ public async Task ReportUrlAccessAsync(string monitorName, HttpResponseMessage h await Task.Delay(new Random().Next(200, 4000), cancellationToken).ConfigureAwait(false); } + public async Task ReportScriptExecutionAsync(string monitorName, string scriptName, string commandLineArgs, string error, CancellationToken cancellationToken = default) + { + await Task.Delay(new Random().Next(200, 4000), cancellationToken).ConfigureAwait(false); + } + public void Dispose() { // Do nothing } + } } diff --git a/src/MonitoringFunctions/DataService/Providers/KustoDataService.cs b/src/MonitoringFunctions/DataService/Providers/KustoDataService.cs new file mode 100644 index 000000000..0322e92a9 --- /dev/null +++ b/src/MonitoringFunctions/DataService/Providers/KustoDataService.cs @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Kusto.Data; +using MonitoringFunctions.DataService.Kusto; +using MonitoringFunctions.Models; +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace MonitoringFunctions +{ + internal sealed class KustoDataService : IDataService + { + private const string ServiceNameAndRegion = "dotnetinstallcluster.eastus2"; + private const string DatabaseName = "dotnet_install_monitoring_database"; + + private readonly KustoTable _httpRequestLogsTable; + private readonly KustoTable _scriptExecutionLogsTable; + + internal KustoDataService() + { + KustoConnectionStringBuilder kcsb = new KustoConnectionStringBuilder($"https://ingest-{ServiceNameAndRegion}.kusto.windows.net") + .WithAadManagedIdentity("system"); + + DirectJsonMappingResolver directJsonMappingResolver = new DirectJsonMappingResolver(); + + _httpRequestLogsTable = new KustoTable(kcsb, DatabaseName, "UrlAccessLogs", directJsonMappingResolver); + _scriptExecutionLogsTable = new KustoTable(kcsb, DatabaseName, "ScriptExecLogs", directJsonMappingResolver); + } + + /// + /// Reports the details of the to kusto. + /// + /// Name of the monitor generating this data entry. + /// Response to be reported. + /// A task, tracking this async operation. + public async Task ReportUrlAccessAsync(string monitorName, HttpResponseMessage httpResponse, CancellationToken cancellationToken = default) + { + HttpRequestLogEntry logEntry = new HttpRequestLogEntry() + { + MonitorName = monitorName, + EventTime = DateTime.UtcNow, + RequestedUrl = httpResponse.RequestMessage.RequestUri.AbsoluteUri, + HttpResponseCode = (int)httpResponse.StatusCode + }; + + await _httpRequestLogsTable.InsertRowAsync(logEntry, cancellationToken).ConfigureAwait(false); + } + + /// + /// Reports the details of a script execution to kusto. + /// + /// Name of the monitor generating this data entry. + /// Name of the script that was executed. + /// Command line arguments passed to the script at the moment of execution. + /// Errors that occured during the execution, if any. + /// A task, tracking this async operation. + public async Task ReportScriptExecutionAsync(string monitorName, string scriptName, string commandLineArgs, string error, CancellationToken cancellationToken = default) + { + ScriptExecutionLogEntry logEntry = new ScriptExecutionLogEntry() + { + MonitorName = monitorName, + EventTime = DateTime.UtcNow, + ScriptName = scriptName, + CommandLineArgs = commandLineArgs, + Error = error + }; + + await _scriptExecutionLogsTable.InsertRowAsync(logEntry, cancellationToken).ConfigureAwait(false); + } + + public void Dispose() + { + // Do nothing + } + } +} diff --git a/src/MonitoringFunctions/Functions/DryRunUrlChecker.cs b/src/MonitoringFunctions/Functions/DryRunUrlChecker.cs new file mode 100644 index 000000000..3885f5501 --- /dev/null +++ b/src/MonitoringFunctions/Functions/DryRunUrlChecker.cs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.IO; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs; +using Microsoft.Extensions.Logging; +using MonitoringFunctions.Models; +using Kusto.Cloud.Platform.IO; +using System.Threading; + +namespace MonitoringFunctions.Functions +{ + /// + /// Runs the scripts in -DryRun mode and checks weather the generated links are accessible + /// + internal static class DryRunUrlChecker + { + [FunctionName("DryRunLTS")] + public static async Task RunLTSAsync([TimerTrigger("0 */30 * * * *")] TimerInfo myTimer, ILogger log) + { + string monitorName = "dry_run_LTS"; + string cmdArgs = "-c LTS"; + + await ExecuteDryRunCheckAndReportUrlAccessAsync(log, monitorName, cmdArgs).ConfigureAwait(false); + } + + [FunctionName("DryRun3_1")] + public static async Task Run3_1Async([TimerTrigger("0 */30 * * * *")] TimerInfo myTimer, ILogger log) + { + string monitorName = "dry_run_3_1"; + string cmdArgs = "-c 3.1"; + + await ExecuteDryRunCheckAndReportUrlAccessAsync(log, monitorName, cmdArgs).ConfigureAwait(false); + } + + [FunctionName("DryRun3_0Runtime")] + public static async Task Run3_0RuntimeAsync([TimerTrigger("0 */30 * * * *")] TimerInfo myTimer, ILogger log) + { + string monitorName = "dry_run_3_0_runtime"; + string cmdArgs = "-c 3.0 -Runtime dotnet"; + + await ExecuteDryRunCheckAndReportUrlAccessAsync(log, monitorName, cmdArgs).ConfigureAwait(false); + } + + /// + /// Executes the Ps1 script with DryDun switch, + /// Parses the output to acquire primary and legacy Urls, + /// Tests the primary Url to see if it is available, + /// Reports the results to the data service provided. + /// + internal static async Task ExecuteDryRunCheckAndReportUrlAccessAsync(ILogger log, string monitorName, string additionalCmdArgs, + CancellationToken cancellationToken = default) + { + string scriptName = "dotnet-install.ps1"; + string commandLineArgs = $"-DryRun {additionalCmdArgs}"; + + using IDataService dataService = new DataServiceFactory().GetDataService(); + + // Execute the script; + ScriptExecutionResult results = await HelperMethods.ExecuteInstallScriptPs1Async(commandLineArgs).ConfigureAwait(false); + + log.LogInformation($"Ouput stream: {results.Output}"); + + if (!string.IsNullOrWhiteSpace(results.Error)) + { + log.LogError($"Error stream: {results.Error}"); + await dataService.ReportScriptExecutionAsync(monitorName, scriptName, commandLineArgs, results.Error, cancellationToken) + .ConfigureAwait(false); + return; + } + + // Parse the output + ScriptDryRunResult dryRunResults = ParseDryRunOutput(results.Output); + + if (string.IsNullOrWhiteSpace(dryRunResults.PrimaryUrl)) + { + log.LogError($"Primary Url was not found for channel {additionalCmdArgs}"); + await dataService.ReportScriptExecutionAsync(monitorName, scriptName, commandLineArgs, + "Failed to parse primary url from the following DryRun execution output: " + results.Output + , cancellationToken).ConfigureAwait(false); + return; + } + + // Validate URL accessibility + await HelperMethods.CheckAndReportUrlAccessAsync(log, monitorName, dryRunResults.PrimaryUrl, dataService); + } + + /// + /// Parses the output of the script when executed in DryRun mode and finds out Primary and Legacy urls. + /// + /// Output of the script execution in DryRun mode + /// Object containing primary and legacy runs + internal static ScriptDryRunResult ParseDryRunOutput(string? output) + { + string primaryUrlIdentifier = "Primary named payload URL: "; + string legacyUrlIdentifier = "Legacy named payload URL: "; + + ScriptDryRunResult result = new ScriptDryRunResult(); + + using StringStream stringStream = new StringStream(output); + using StreamReader streamReader = new StreamReader(stringStream); + + string? line; + while ((line = streamReader.ReadLine()) != null) + { + // Does this line contain the primary url? + int primaryIdIndex = line.IndexOf(primaryUrlIdentifier); + if (primaryIdIndex != -1) + { + result.PrimaryUrl = line.Substring(primaryIdIndex + primaryUrlIdentifier.Length); + } + else + { + // Does this line contain the legacy url? + int legacyIdIndex = line.IndexOf(legacyUrlIdentifier); + if(legacyIdIndex != -1) + { + result.LegacyUrl = line.Substring(legacyIdIndex + legacyUrlIdentifier.Length); + } + } + } + + return result; + } + } +} diff --git a/src/MonitoringFunctions/Functions/Ps1Downloader.cs b/src/MonitoringFunctions/Functions/Ps1Downloader.cs index bb44a1b63..e9a34194f 100644 --- a/src/MonitoringFunctions/Functions/Ps1Downloader.cs +++ b/src/MonitoringFunctions/Functions/Ps1Downloader.cs @@ -14,7 +14,7 @@ public static class Ps1Downloader [FunctionName("DownloadPs1")] public static async Task RunAsync([TimerTrigger("0 */30 * * * *")]TimerInfo myTimer, ILogger log) { - using IDataService dataService = new KustoDataService(); + using IDataService dataService = new DataServiceFactory().GetDataService(); await HelperMethods.CheckAndReportUrlAccessAsync(log, _monitorName, _url, dataService).ConfigureAwait(false); } } diff --git a/src/MonitoringFunctions/Functions/ShDownloader.cs b/src/MonitoringFunctions/Functions/ShDownloader.cs index 150bcd3e6..07bba88cc 100644 --- a/src/MonitoringFunctions/Functions/ShDownloader.cs +++ b/src/MonitoringFunctions/Functions/ShDownloader.cs @@ -14,7 +14,7 @@ public static class ShDownloader [FunctionName("DownloadSh")] public static async Task RunAsync([TimerTrigger("0 */30 * * * *")]TimerInfo myTimer, ILogger log) { - using IDataService dataService = new KustoDataService(); + using IDataService dataService = new DataServiceFactory().GetDataService(); await HelperMethods.CheckAndReportUrlAccessAsync(log, _monitorName, _url, dataService).ConfigureAwait(false); } } diff --git a/src/MonitoringFunctions/HelperMethods.cs b/src/MonitoringFunctions/HelperMethods.cs index 9df9ca5ac..dd6a58f01 100644 --- a/src/MonitoringFunctions/HelperMethods.cs +++ b/src/MonitoringFunctions/HelperMethods.cs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Diagnostics; using System.Net.Http; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using MonitoringFunctions.Models; [assembly:InternalsVisibleTo("MonitoringFunctions.Test, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] @@ -27,8 +29,9 @@ internal static class HelperMethods /// that is used to report log information. /// Name of this monitor to be included in the logs and in the data sent to Kusto. /// Url that this method will attempt to access. - /// - internal static async Task CheckAndReportUrlAccessAsync(ILogger log, string monitorName, string url, IDataService dataService, CancellationToken cancellationToken = default) + /// A task, tracking the initiated async operation. Errors should be reported through exceptions. + internal static async Task CheckAndReportUrlAccessAsync(ILogger log, string monitorName, string url, IDataService dataService, + CancellationToken cancellationToken = default) { HttpResponseMessage response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); @@ -44,5 +47,32 @@ internal static async Task CheckAndReportUrlAccessAsync(ILogger log, string moni throw new Exception($"Download failed with status code {response.StatusCode}. Monitor: {monitorName}"); } } + + /// + /// Executes dotnet-install.ps1 script with given arguments and returns the content of the output and error streams. + /// + internal static async Task ExecuteInstallScriptPs1Async(string? args = null) + { + ProcessStartInfo processStartInfo = new ProcessStartInfo("powershell", + @"-NoProfile -Command ""[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;" + + @" $ProgressPreference = 'SilentlyContinue'; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing" + + @" 'https://raw.githubusercontent.com/dotnet/install-scripts/master/src/dotnet-install.ps1')))" + + $@" {args}"""); + + processStartInfo.CreateNoWindow = true; + processStartInfo.UseShellExecute = false; + processStartInfo.RedirectStandardOutput = true; + processStartInfo.RedirectStandardError = true; + + Process installScriptProc = Process.Start(processStartInfo); + string consoleOutput = await installScriptProc.StandardOutput.ReadToEndAsync().ConfigureAwait(false); + string consoleError = await installScriptProc.StandardError.ReadToEndAsync().ConfigureAwait(false); + + return new ScriptExecutionResult() + { + Output = consoleOutput, + Error = consoleError + }; + } } } diff --git a/src/MonitoringFunctions/Models/HttpRequestLogEntry.cs b/src/MonitoringFunctions/Models/HttpRequestLogEntry.cs index 9269cdd41..003cd1bde 100644 --- a/src/MonitoringFunctions/Models/HttpRequestLogEntry.cs +++ b/src/MonitoringFunctions/Models/HttpRequestLogEntry.cs @@ -1,14 +1,16 @@ // Copyright (c) Microsoft. All rights reserved. +using MonitoringFunctions.DataService.Kusto; using Newtonsoft.Json; using System; +using System.Diagnostics.CodeAnalysis; namespace MonitoringFunctions.Models { /// /// Represents an http request event made from a function to be inserted into Kusto. /// - internal class HttpRequestLogEntry + internal struct HttpRequestLogEntry : IEquatable, IKustoTableRow { [JsonProperty("monitor_name"), JsonRequired] public string? MonitorName { get; set; } @@ -19,7 +21,20 @@ internal class HttpRequestLogEntry [JsonProperty("requested_url"), JsonRequired] public string? RequestedUrl { get; set; } - [JsonProperty("http_status_code"), JsonRequired] - public int? HttpStatusCode { get; set; } + [JsonProperty("http_response_code"), JsonRequired] + public int? HttpResponseCode { get; set; } + + public bool Equals([AllowNull] HttpRequestLogEntry other) + { + return MonitorName == other.MonitorName && + EventTime == other.EventTime && + RequestedUrl == other.RequestedUrl && + HttpResponseCode == other.HttpResponseCode; + } + + public override string? ToString() + { + return $"HttpRequestLogEntry - MonitorName: {MonitorName}, EventTime: {EventTime}, HttpResponseCode: {HttpResponseCode}, RequestedUrl: {RequestedUrl}"; + } } } diff --git a/src/MonitoringFunctions/Models/ScriptDryRunResult.cs b/src/MonitoringFunctions/Models/ScriptDryRunResult.cs new file mode 100644 index 000000000..8133fb489 --- /dev/null +++ b/src/MonitoringFunctions/Models/ScriptDryRunResult.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; + +namespace MonitoringFunctions.Models +{ + internal struct ScriptDryRunResult : IEquatable + { + public string? PrimaryUrl { get; set; } + public string? LegacyUrl { get; set; } + + public bool Equals([AllowNull] ScriptDryRunResult other) + { + return PrimaryUrl == other.PrimaryUrl && + LegacyUrl == other.LegacyUrl; + } + + public override string? ToString() + { + return $"ScriptDryRunResult - PrimaryUrl: {PrimaryUrl}, LegacyUrl: {LegacyUrl}"; + } + } +} diff --git a/src/MonitoringFunctions/Models/ScriptExecutionLogEntry.cs b/src/MonitoringFunctions/Models/ScriptExecutionLogEntry.cs new file mode 100644 index 000000000..8c6587890 --- /dev/null +++ b/src/MonitoringFunctions/Models/ScriptExecutionLogEntry.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft. All rights reserved. + +using MonitoringFunctions.DataService.Kusto; +using Newtonsoft.Json; +using System; +using System.Diagnostics.CodeAnalysis; + +namespace MonitoringFunctions.Models +{ + /// + /// Represents a script execution event to be inserted into Kusto. + /// + internal struct ScriptExecutionLogEntry : IEquatable, IKustoTableRow + { + [JsonProperty("monitor_name"), JsonRequired] + public string? MonitorName { get; set; } + + [JsonProperty("timestamp"), JsonRequired] + public DateTime EventTime { get; set; } + + [JsonProperty("script_name")] + public string? ScriptName { get; set; } + + [JsonProperty("cmd_args")] + public string? CommandLineArgs { get; set; } + + [JsonProperty("error")] + public string? Error { get; set; } + + public bool Equals([AllowNull] ScriptExecutionLogEntry other) + { + return MonitorName == other.MonitorName && + EventTime == other.EventTime && + ScriptName == other.ScriptName && + CommandLineArgs == other.CommandLineArgs && + Error == Error; + } + + public override string? ToString() + { + return $"ScriptExecutionLogEntry - MonitorName: {MonitorName}, EventTime: {EventTime}, ScriptName: {ScriptName}, CommandLineArgs: {CommandLineArgs}, Error: {Error}"; + } + } +} diff --git a/src/MonitoringFunctions/Models/ScriptExecutionResult.cs b/src/MonitoringFunctions/Models/ScriptExecutionResult.cs new file mode 100644 index 000000000..81cda1951 --- /dev/null +++ b/src/MonitoringFunctions/Models/ScriptExecutionResult.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; + +namespace MonitoringFunctions.Models +{ + /// Stores the contents of output streams of a script execution. + internal struct ScriptExecutionResult : IEquatable + { + /// Contents of the output stream as string + public string? Output { get; set; } + + /// Contents of the error stream as string + public string? Error { get; set; } + + public bool Equals([AllowNull] ScriptExecutionResult other) + { + return Output == other.Output && + Error == other.Error; + } + + public override string? ToString() + { + return $"ScriptExecutionResult - Output: {Output}, Error: {Error}"; + } + } +} diff --git a/src/MonitoringFunctions/Providers/KustoDataService.cs b/src/MonitoringFunctions/Providers/KustoDataService.cs deleted file mode 100644 index 5b92a3663..000000000 --- a/src/MonitoringFunctions/Providers/KustoDataService.cs +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Kusto.Data; -using Kusto.Data.Common; -using Kusto.Ingest; -using MonitoringFunctions.Models; -using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.IO; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; - -namespace MonitoringFunctions -{ - internal sealed class KustoDataService : IDataService - { - private const string ServiceNameAndRegion = "dotnetinstallcluster.eastus2"; - private const string DatabaseName = "dotnet_install_monitoring_database"; - private const string TableName = "DaemonLogs"; - - private static IEnumerable HttpRequestLogColumnMapping { get; } - - static KustoDataService() - { - HttpRequestLogColumnMapping = new ColumnMapping[] - { - new ColumnMapping() - { - ColumnName = "monitor_name", - Properties = new Dictionary() { { "Path", "$.monitor_name" } } - }, - new ColumnMapping() - { - ColumnName = "timestamp", - Properties = new Dictionary() { { "Path", "$.timestamp" } } - }, - new ColumnMapping() - { - ColumnName = "requested_url", - Properties = new Dictionary() { { "Path", "$.requested_url" } } - }, - new ColumnMapping() - { - ColumnName = "http_response_code", - Properties = new Dictionary() { { "Path", "$.http_response_code" } } - } - }; - } - - internal KustoDataService() { } - - /// - /// Saves the details of the to - /// - /// - /// - /// - public async Task ReportUrlAccessAsync(string monitorName, HttpResponseMessage httpResponse, CancellationToken cancellationToken = default) - { - HttpRequestLogEntry logEntry = new HttpRequestLogEntry() - { - MonitorName = monitorName, - EventTime = DateTime.UtcNow, - RequestedUrl = httpResponse.RequestMessage.RequestUri.AbsoluteUri, - HttpStatusCode = (int)httpResponse.StatusCode - }; - - KustoConnectionStringBuilder kcsb = new KustoConnectionStringBuilder($"https://ingest-{ServiceNameAndRegion}.kusto.windows.net") - .WithAadManagedIdentity("system"); - - using IKustoQueuedIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kcsb); - KustoQueuedIngestionProperties ingestProps = new KustoQueuedIngestionProperties(DatabaseName, TableName) - { - ReportLevel = IngestionReportLevel.FailuresOnly, - ReportMethod = IngestionReportMethod.Queue, - IngestionMapping = new IngestionMapping() - { - IngestionMappingKind = Kusto.Data.Ingestion.IngestionMappingKind.Json, - IngestionMappings = HttpRequestLogColumnMapping - }, - Format = DataSourceFormat.json - }; - - using MemoryStream memStream = new MemoryStream(); - using StreamWriter writer = new StreamWriter(memStream); - - writer.WriteLine(JsonConvert.SerializeObject(logEntry)); - - writer.Flush(); - memStream.Seek(0, SeekOrigin.Begin); - - // IKustoQueuedIngestClient doesn't support cancellation at the moment. Update the line below if it does in the future. - await ingestClient.IngestFromStreamAsync(memStream, ingestProps, leaveOpen: true); - } - - public void Dispose() - { - // Do nothing - } - } -} diff --git a/tests/MonitoringFunctions.Test/TestDirectJsonMappingResolver.cs b/tests/MonitoringFunctions.Test/TestDirectJsonMappingResolver.cs new file mode 100644 index 000000000..4b1777010 --- /dev/null +++ b/tests/MonitoringFunctions.Test/TestDirectJsonMappingResolver.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Kusto.Data.Common; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using MonitoringFunctions.DataService.Kusto; +using Newtonsoft.Json; +using System.Collections.Generic; +using System.Linq; + +namespace MonitoringFunctions.Test +{ + /// + /// Contains tests that make sure can correctly create + /// s for models. + /// + [TestClass] + public class TestDirectJsonMappingResolver + { + private class BasicMappingModel : IKustoTableRow + { + public float someFloat { get; set; } + + public string someString { get; set; } + } + + private class CommonMappingModel : IKustoTableRow + { + [JsonProperty("some_float")] + public float someFloat { get; set; } + + public int someInteger { get; set; } + + private string somePrivateString { get; set; } + } + + private class ComplexMappingModel : CommonMappingModel, IKustoTableRow + { + [JsonIgnore] + public int someOtherInteger { get; set; } + + public string privateGetter { private get; set; } + + public int someField; + } + + /// + /// Tests if can correctly create column mappings + /// for a simple type + /// + [TestMethod] + public void TestBasicMappingAsync() + { + string[] expectedColumnNames = new string[] { "someFloat", "someString" }; + CompareMappings(expectedColumnNames); + } + + /// + /// Tests if can correctly create column mappings + /// for a not-so-basic type + /// + [TestMethod] + public void TestCommonMappingAsync() + { + string[] expectedColumnNames = new string[] { "some_float", "someInteger" }; + CompareMappings(expectedColumnNames); + } + + /// + /// Tests if can correctly create column mappings + /// for a complicated type + /// + [TestMethod] + public void TestComplexMappingAsync() + { + string[] expectedColumnNames = new string[] { "some_float", "someInteger", "someField" }; + CompareMappings(expectedColumnNames); + } + + /// + /// Compares the given column names with the column names generated by + /// for the given type. + /// + /// Data class whose column names will be resolved by + /// Column names to compare. Order of the items is unimportant. + private void CompareMappings(string[] expectedColumnNames) where T : IKustoTableRow + { + DirectJsonMappingResolver mappingResolver = new DirectJsonMappingResolver(); + IEnumerable mappingCollection = mappingResolver.GetColumnMappings(); + + List mappedColumnNames = mappingCollection.Select(m => m.ColumnName).ToList(); + + Assert.IsTrue(!mappedColumnNames.Except(expectedColumnNames).Any(), + $"The following columns shouldn't have been mapped, but they were: { string.Join(", ", mappedColumnNames.Except(expectedColumnNames))}"); + + Assert.IsTrue(!expectedColumnNames.Except(mappedColumnNames).Any(), + $"The following columns should have been mapped, but they weren't: { string.Join(", ", expectedColumnNames.Except(mappedColumnNames))}"); + } + } +} diff --git a/tests/MonitoringFunctions.Test/TestHelperMethods.cs b/tests/MonitoringFunctions.Test/TestHelperMethods.cs index 6eaaeeadc..2a607aecb 100644 --- a/tests/MonitoringFunctions.Test/TestHelperMethods.cs +++ b/tests/MonitoringFunctions.Test/TestHelperMethods.cs @@ -2,7 +2,9 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.VisualStudio.TestTools.UnitTesting; +using MonitoringFunctions.Models; using MonitoringFunctions.Providers; +using System; using System.Threading.Tasks; namespace MonitoringFunctions.Test @@ -11,7 +13,7 @@ namespace MonitoringFunctions.Test public class TestHelperMethods { [TestMethod] - public async Task TestCheckUrlAccess() + public async Task TestCheckUrlAccessAsync() { using IDataService dataService = new DummyDataService(); // Test if we can access a highly available website without throwing an exception. @@ -21,7 +23,7 @@ public async Task TestCheckUrlAccess() [DataRow("definitely not a url")] [DataRow("https://0.com")] [TestMethod] - public async Task TestCheckUrlAccessFailure(string url) + public async Task TestCheckUrlAccessFailureAsync(string url) { try { @@ -34,5 +36,43 @@ public async Task TestCheckUrlAccessFailure(string url) // Test passed } } + + [DataRow("-DryRun -c 3.0")] + [DataRow("-DryRun -c release/5.0.1xx-preview7")] + [DataRow("-DryRun -Version LTS")] + [TestMethod] + public async Task TestExecuteInstallScriptPs1Async(string cmdArgs = "-DryRun") + { + try + { + ScriptExecutionResult executionResult = await HelperMethods.ExecuteInstallScriptPs1Async(cmdArgs).ConfigureAwait(false); + + Assert.IsFalse(string.IsNullOrWhiteSpace(executionResult.Output), "Script execution hasn't returned an output."); + Assert.IsTrue(string.IsNullOrWhiteSpace(executionResult.Error), $"Script execution has returned the following error: {executionResult.Error}"); + } + catch (Exception e) + { + Assert.Fail($"Script execution has failed with an exception: {e.ToString()}"); + } + } + + [DataRow("-DryRun -Version -Channel 3.0")] + [DataRow("-Channel 12")] + [DataRow("-switchThatDoesntExist")] + [TestMethod] + public async Task TestExecuteInstallScriptPs1WrongArgsAsync(string cmdArgs = "-switchThatDoesntExist") + { + try + { + ScriptExecutionResult executionResult = await HelperMethods.ExecuteInstallScriptPs1Async(cmdArgs).ConfigureAwait(false); + + Assert.IsFalse(string.IsNullOrWhiteSpace(executionResult.Error), $"Script execution hasn't returned any errors, but it should have." + + $" Used command line arguments: " + cmdArgs); + } + catch (Exception e) + { + Assert.Fail($"Script execution has failed with an exception: {e.ToString()}"); + } + } } }