diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorServiceTests.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorServiceTests.cs index 54911c1e2522..7c54ab21d680 100644 --- a/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorServiceTests.cs +++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorServiceTests.cs @@ -37,7 +37,7 @@ public class AzPredictorServiceTests public AzPredictorServiceTests(ModelFixture fixture) { this._fixture = fixture; - var startHistory = $"{AzPredictorConstants.CommandHistoryPlaceholder}{AzPredictorConstants.CommandConcatenator}{AzPredictorConstants.CommandHistoryPlaceholder}"; + var startHistory = $"{AzPredictorConstants.CommandPlaceholder}{AzPredictorConstants.CommandConcatenator}{AzPredictorConstants.CommandPlaceholder}"; this._suggestionsPredictor = new Predictor(this._fixture.PredictionCollection[startHistory], null); this._commandsPredictor = new Predictor(this._fixture.CommandCollection, null); @@ -66,7 +66,7 @@ public void VerifyUsingSuggestion(string userInput) Assert.NotNull(actual); Assert.NotNull(actual.Item1); Assert.Equal(expected, actual.Item1); - Assert.Equal(PredictionSource.CurrentHistory, actual.Item2); + Assert.Equal(PredictionSource.CurrentCommand, actual.Item2); } /// @@ -83,14 +83,14 @@ public void VerifyUsingCommand(string userInput) Assert.NotNull(actual); Assert.NotNull(actual.Item1); Assert.Equal(expected, actual.Item1); - Assert.Equal(PredictionSource.Commands, actual.Item2); + Assert.Equal(PredictionSource.StaticCommands, actual.Item2); } /// /// Verify that no prediction for the user input, meaning it's not in the prediction list or the command list. /// [Theory] - [InlineData(AzPredictorConstants.CommandHistoryPlaceholder)] + [InlineData(AzPredictorConstants.CommandPlaceholder)] [InlineData("git status")] [InlineData("Get-ChildItem")] [InlineData("new-azresourcegroup -NoExistingParam")] diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorTests.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorTests.cs index 456e2074ece9..89e7b402f973 100644 --- a/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorTests.cs +++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorTests.cs @@ -38,7 +38,7 @@ public sealed class AzPredictorTests public AzPredictorTests(ModelFixture modelFixture) { this._fixture = modelFixture; - var startHistory = $"{AzPredictorConstants.CommandHistoryPlaceholder}{AzPredictorConstants.CommandConcatenator}{AzPredictorConstants.CommandHistoryPlaceholder}"; + var startHistory = $"{AzPredictorConstants.CommandPlaceholder}{AzPredictorConstants.CommandConcatenator}{AzPredictorConstants.CommandPlaceholder}"; this._service = new MockAzPredictorService(startHistory, this._fixture.PredictionCollection[startHistory], this._fixture.CommandCollection); this._telemetryClient = new MockAzPredictorTelemetryClient(); @@ -69,7 +69,7 @@ public void VerifyWithNonSupportedCommand(string historyLine) this._azPredictor.StartEarlyProcessing(history); Assert.True(this._service.IsPredictionRequested); - Assert.Null(this._telemetryClient.RecordedSuggestion); + Assert.NotNull(this._telemetryClient.RecordedSuggestion); } /// diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Mocks/MockAzPredictorService.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Mocks/MockAzPredictorService.cs index 2cf1ccd43c8c..8c9fd8e5ff27 100644 --- a/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Mocks/MockAzPredictorService.cs +++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Mocks/MockAzPredictorService.cs @@ -34,7 +34,7 @@ sealed class MockAzPredictorService : AzPredictorService /// The commands collection public MockAzPredictorService(string history, IList suggestions, IList commands) { - SetHistory(history); + SetPredictionCommand(history); SetCommandsPredictor(commands); SetSuggestionPredictor(history, suggestions); } diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Mocks/MockAzPredictorTelemetryClient.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Mocks/MockAzPredictorTelemetryClient.cs index c1d0e3d69f63..e39067fb5420 100644 --- a/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Mocks/MockAzPredictorTelemetryClient.cs +++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Mocks/MockAzPredictorTelemetryClient.cs @@ -12,6 +12,7 @@ // limitations under the License. // ---------------------------------------------------------------------------------- +using System; using System.Collections.Generic; namespace Microsoft.Azure.PowerShell.Tools.AzPredictor.Test.Mocks @@ -21,26 +22,36 @@ sealed class MockAzPredictorTelemetryClient : ITelemetryClient public class RecordedSuggestionForHistory { public string HistoryLine { get; set; } - public int? SuggestionIndex { get; set; } - public int? FallbackIndex { get; set; } - public IEnumerable TopSuggestions { get; set; } } + /// + public string SessionId { get; } = Guid.NewGuid().ToString(); + + /// + public string CorrelationId { get; private set; } = Guid.NewGuid().ToString(); + public RecordedSuggestionForHistory RecordedSuggestion { get; set; } public int SuggestionAccepted { get; set; } /// - public void OnSuggestionForHistory(string historyLine, int? suggestionIndex, int? fallbackIndex, IEnumerable topSuggestions) + public void OnHistory(string historyLine) { this.RecordedSuggestion = new RecordedSuggestionForHistory() { HistoryLine = historyLine, - SuggestionIndex = suggestionIndex, - FallbackIndex = fallbackIndex, - TopSuggestions = topSuggestions }; } + /// + public void OnRequestPrediction(string command) + { + } + + /// + public void OnRequestPredictionError(string command, Exception e) + { + } + /// public void OnSuggestionAccepted(string acceptedSuggestion) { @@ -48,7 +59,12 @@ public void OnSuggestionAccepted(string acceptedSuggestion) } /// - public void OnGetSuggestion(PredictionSource predictionSource) + public void OnGetSuggestion(string maskedUserInput, IEnumerable> suggestions, bool isCancelled) + { + } + + /// + public void OnGetSuggestionError(Exception e) { } } diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/PredictorTests.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/PredictorTests.cs index dd68a2c3a08f..a52550a8d79a 100644 --- a/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/PredictorTests.cs +++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/PredictorTests.cs @@ -33,7 +33,7 @@ public class PredictorTests public PredictorTests(ModelFixture fixture) { this._fixture = fixture; - var startHistory = $"{AzPredictorConstants.CommandHistoryPlaceholder}{AzPredictorConstants.CommandConcatenator}{AzPredictorConstants.CommandHistoryPlaceholder}"; + var startHistory = $"{AzPredictorConstants.CommandPlaceholder}{AzPredictorConstants.CommandConcatenator}{AzPredictorConstants.CommandPlaceholder}"; this._predictor = new Predictor(this._fixture.PredictionCollection[startHistory], null); } @@ -44,7 +44,7 @@ public PredictorTests(ModelFixture fixture) [InlineData("NEW-AZCONTEXT")] [InlineData("Get-AzStorageAccount ")] // A complete command and we have exact the same on in the model. [InlineData("get-azaccount ")] - [InlineData(AzPredictorConstants.CommandHistoryPlaceholder)] + [InlineData(AzPredictorConstants.CommandPlaceholder)] [InlineData("git status")] [InlineData("Get-ChildItem")] public void GetNullPredictionWithCommandName(string userInput) diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor/Az.Tools.Predictor.csproj b/tools/Az.Tools.Predictor/Az.Tools.Predictor/Az.Tools.Predictor.csproj index 94314366c3b7..f244d19f8d81 100644 --- a/tools/Az.Tools.Predictor/Az.Tools.Predictor/Az.Tools.Predictor.csproj +++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor/Az.Tools.Predictor.csproj @@ -27,6 +27,10 @@ Auto + + DEBUG;TRACE + + diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictor.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictor.cs index 501a8b85f9ed..06e20ed3f637 100644 --- a/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictor.cs +++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictor.cs @@ -77,7 +77,7 @@ public void StartEarlyProcessing(IReadOnlyList history) while (historyLines.Count() < AzPredictorConstants.CommandHistoryCountToProcess) { - historyLines = historyLines.Prepend(AzPredictorConstants.CommandHistoryPlaceholder); + historyLines = historyLines.Prepend(AzPredictorConstants.CommandPlaceholder); } var commandAsts = historyLines.Select((h) => @@ -93,23 +93,13 @@ public void StartEarlyProcessing(IReadOnlyList history) if (!_service.IsSupportedCommand(commandName)) { - return AzPredictorConstants.CommandHistoryPlaceholder; + return AzPredictorConstants.CommandPlaceholder; } return AzPredictor.MaskCommandLine(c); }); - var lastMaskedHistoryLines = maskedHistoryLines.Last(); - - if (lastMaskedHistoryLines != AzPredictorConstants.CommandHistoryPlaceholder) - { - var commandName = (commandAsts.LastOrDefault()?.CommandElements?.FirstOrDefault() as StringConstantExpressionAst)?.Value; - var suggestionIndex = _service.GetRankOfSuggestion(commandName); - var fallbackIndex = _service.GetRankOfFallback(commandName); - var topFiveSuggestion = _service.GetTopNSuggestions(AzPredictor.SuggestionCountForTelemetry); - _telemetryClient.OnSuggestionForHistory(maskedHistoryLines.LastOrDefault(), suggestionIndex, fallbackIndex, topFiveSuggestion); - } - + _telemetryClient.OnHistory(maskedHistoryLines.Last()); _service.RecordHistory(commandAsts); _service.RequestPredictions(maskedHistoryLines); } @@ -131,17 +121,30 @@ public List GetSuggestion(PredictionContext context, Cance // is prefixed with `userInput`, it should be ordered before result that is not prefixed // with `userInput`. - var userInput = context.InputAst.Extent.Text; - var result = _service.GetSuggestion(context.InputAst, cancellationToken); + Tuple result = Tuple.Create(null, PredictionSource.None); - if (result?.Item1 != null) + try { - cancellationToken.ThrowIfCancellationRequested(); - var fullSuggestion = MergeStrings(userInput, result.Item1); - return new List() { new PredictiveSuggestion(fullSuggestion) }; - } + result = _service.GetSuggestion(context.InputAst, cancellationToken); - this._telemetryClient.OnGetSuggestion(result?.Item2 ?? PredictionSource.None); + if (result?.Item1 != null) + { + cancellationToken.ThrowIfCancellationRequested(); + var userInput = context.InputAst.Extent.Text; + var fullSuggestion = MergeStrings(userInput, result.Item1); + return new List() { new PredictiveSuggestion(fullSuggestion) }; + } + } + catch (Exception e) when (!(e is OperationCanceledException)) + { + this._telemetryClient.OnGetSuggestionError(e); + } + finally + { + var maskedCommandLine = MaskCommandLine(context.InputAst.FindAll((ast) => ast is CommandAst, true).LastOrDefault() as CommandAst); + _telemetryClient.OnGetSuggestion(maskedCommandLine, new Tuple[] { result }, + cancellationToken.IsCancellationRequested); + } return null; } @@ -228,7 +231,7 @@ public void OnImport() { var settings = Settings.GetSettings(); var telemetryClient = new AzPredictorTelemetryClient(); - var azPredictorService = new AzPredictorService(settings.ServiceUri); + var azPredictorService = new AzPredictorService(settings.ServiceUri, telemetryClient); var predictor = new AzPredictor(azPredictorService, telemetryClient); SubsystemManager.RegisterSubsystem(predictor); } diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictorConstants.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictorConstants.cs index e4b21c7432a0..2dd3051c2c78 100644 --- a/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictorConstants.cs +++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictorConstants.cs @@ -22,7 +22,7 @@ internal static class AzPredictorConstants /// /// The value to use when the command isn't an Az command. /// - public const string CommandHistoryPlaceholder = "start_of_snippet"; + public const string CommandPlaceholder = "start_of_snippet"; /// /// The value to check to determine if it's an Az command. diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictorService.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictorService.cs index 695bb86b5bcc..e6daff16adbb 100644 --- a/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictorService.cs +++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictorService.cs @@ -37,9 +37,9 @@ private sealed class PredictionRequestBody { public sealed class RequestContext { - public Guid CorrelationId { get; set; } = Guid.Empty; - public Guid SessionId { get; set; } = Guid.Empty; - public Guid SubscriptionId { get; set; } = Guid.Empty; + public string CorrelationId { get; set; } = Guid.Empty.ToString(); + public string SessionId { get; set; } = Guid.Empty.ToString(); + public string SubscriptionId { get; set; } = Guid.Empty.ToString(); public Version VersionNumber{ get; set; } = new Version(1, 0); } @@ -47,28 +47,32 @@ public sealed class RequestContext public string ClientType { get; set; } = "AzurePowerShell"; public RequestContext Context { get; set; } = new RequestContext(); - public PredictionRequestBody(string history) => this.History = history; + public PredictionRequestBody(string command) => this.History = command; }; private static readonly HttpClient _client = new HttpClient(); private readonly string _commandsEndpoint; private readonly string _predictionsEndpoint; - private volatile Tuple _historySuggestions; // The history and the prediction for that. + private volatile Tuple _commandSuggestions; // The command and the prediction for that. private volatile Predictor _commands; - private volatile string _history; + private volatile string CommandForPrediction; private HashSet _commandSet; private CancellationTokenSource _predictionRequestCancellationSource; private ParameterValuePredictor _parameterValuePredictor = new ParameterValuePredictor(); + private readonly ITelemetryClient _telemetryClient; + /// /// The AzPredictor service interacts with the Aladdin service specified in serviceUri. /// At initialization, it requests a list of the popular commands. /// /// The URI of the Aladdin service. - public AzPredictorService(string serviceUri) + /// The telemetry client. + public AzPredictorService(string serviceUri, ITelemetryClient telemetryClient) { this._commandsEndpoint = serviceUri + AzPredictorConstants.CommandsEndpoint; this._predictionsEndpoint = serviceUri + AzPredictorConstants.PredictionsEndpoint; + this._telemetryClient = telemetryClient; RequestCommands(); } @@ -109,25 +113,25 @@ protected virtual void Dispose(bool disposing) /// public Tuple GetSuggestion(Ast input, CancellationToken cancellationToken) { - var historySuggestions = this._historySuggestions; - var history = this._history; + var commandSuggestions = this._commandSuggestions; + var command = this.CommandForPrediction; var predictionSource = PredictionSource.None; - // We've already used _historySuggestions. There is no need to wait the request to complete at this point. + // We've already used _commandSuggestions. There is no need to wait the request to complete at this point. // Cancel it. this._predictionRequestCancellationSource?.Cancel(); - string result = historySuggestions?.Item2?.Query(input, cancellationToken); + string result = commandSuggestions?.Item2?.Query(input, cancellationToken); if (result != null) { - if (string.Equals(history, historySuggestions?.Item1, StringComparison.Ordinal)) + if (string.Equals(command, commandSuggestions?.Item1, StringComparison.Ordinal)) { - predictionSource = PredictionSource.CurrentHistory; + predictionSource = PredictionSource.CurrentCommand; } else { - predictionSource = PredictionSource.PreviousHistory; + predictionSource = PredictionSource.PreviousCommand; } } else @@ -137,7 +141,7 @@ public Tuple GetSuggestion(Ast input, CancellationToke if (result != null) { - predictionSource = PredictionSource.Commands; + predictionSource = PredictionSource.StaticCommands; } } @@ -145,27 +149,47 @@ public Tuple GetSuggestion(Ast input, CancellationToke } /// - public virtual void RequestPredictions(IEnumerable history) + public virtual void RequestPredictions(IEnumerable commands) { - // Even if it's called multiple times, we only need to keep the one for the latest history. + // Even if it's called multiple times, we only need to keep the one for the latest command. this._predictionRequestCancellationSource?.Cancel(); this._predictionRequestCancellationSource = new CancellationTokenSource(); + var cancellationToken = this._predictionRequestCancellationSource.Token; - var localHistory = string.Join(AzPredictorConstants.CommandConcatenator, history); - this._history = localHistory; + + var localCommands= string.Join(AzPredictorConstants.CommandConcatenator, commands); + this._telemetryClient.OnRequestPrediction(localCommands); + this.SetPredictionCommand(localCommands); // We don't need to block on the task. We send the HTTP request and update prediction list at the background. Task.Run(async () => { - var requestBody = JsonConvert.SerializeObject(new PredictionRequestBody(localHistory)); - var httpResponseMessage = await _client.PostAsync(this._predictionsEndpoint, new StringContent(requestBody, Encoding.UTF8, "application/json"), cancellationToken); + try + { + var requestContext = new PredictionRequestBody.RequestContext() + { + SessionId = this._telemetryClient.SessionId, + CorrelationId = this._telemetryClient.CorrelationId, + }; + var requestBody = new PredictionRequestBody(localCommands) + { + Context = requestContext, + }; + + var requestBodyString = JsonConvert.SerializeObject(requestBody); + var httpResponseMessage = await _client.PostAsync(this._predictionsEndpoint, new StringContent(requestBodyString, Encoding.UTF8, "application/json"), cancellationToken); var reply = await httpResponseMessage.Content.ReadAsStringAsync(cancellationToken); var suggestionsList = JsonConvert.DeserializeObject>(reply); - this.SetSuggestionPredictor(localHistory, suggestionsList); - }, - cancellationToken); + this.SetSuggestionPredictor(localCommands, suggestionsList); + } + catch (Exception e) when (!(e is OperationCanceledException)) + { + this._telemetryClient.OnRequestPredictionError(localCommands, e); + } + }, + cancellationToken); } /// @@ -174,27 +198,6 @@ public virtual void RecordHistory(IEnumerable history) history.ForEach((h) => this._parameterValuePredictor.ProcessHistoryCommand(h)); } - /// - public int? GetRankOfSuggestion(string commandName) - { - var historySuggestions = this._historySuggestions; - return historySuggestions?.Item2?.GetCommandPrediction(commandName, isCommandNameComplete: true, cancellationToken:CancellationToken.None).Item2; - } - - /// - public int? GetRankOfFallback(string commandName) - { - var commands = this._commands; - return commands?.GetCommandPrediction(commandName, isCommandNameComplete:true, cancellationToken:CancellationToken.None).Item2; - } - - /// - public IEnumerable GetTopNSuggestions(int n) - { - var historySuggestions = this._historySuggestions; - return historySuggestions?.Item2?.GetTopNPrediction(n); - } - /// public bool IsSupportedCommand(string cmd) => !string.IsNullOrWhiteSpace(cmd) && (_commandSet?.Contains(cmd) == true); @@ -214,11 +217,9 @@ protected virtual void RequestCommands() this.SetCommandsPredictor(commands_reply); // Initialize predictions - var startHistory = $"{AzPredictorConstants.CommandHistoryPlaceholder}{AzPredictorConstants.CommandConcatenator}{AzPredictorConstants.CommandHistoryPlaceholder}"; RequestPredictions(new string[] { - AzPredictorConstants.CommandHistoryPlaceholder, - AzPredictorConstants.CommandHistoryPlaceholder}); - + AzPredictorConstants.CommandPlaceholder, + AzPredictorConstants.CommandPlaceholder}); }); } @@ -230,26 +231,25 @@ protected void SetCommandsPredictor(IList commands) { this._commands = new Predictor(commands, this._parameterValuePredictor); this._commandSet = commands.Select(x => AzPredictorService.GetCommandName(x)).ToHashSet(StringComparer.OrdinalIgnoreCase); // this could be slow - } /// /// Sets the suggestiosn predictor. /// - /// The history that the suggestions are for + /// The commands that the suggestions are for /// The suggestion collection to set the predictor - protected void SetSuggestionPredictor(string history, IList suggestions) + protected void SetSuggestionPredictor(string commands, IList suggestions) { - this._historySuggestions = Tuple.Create(history, new Predictor(suggestions, this._parameterValuePredictor)); + this._commandSuggestions = Tuple.Create(commands, new Predictor(suggestions, this._parameterValuePredictor)); } /// - /// Updates the history for prediction. + /// Updates the command for prediction. /// - /// The value to update the history - protected void SetHistory(string history) + /// The command for the new prediction + protected void SetPredictionCommand(string command) { - this._history = history; + this.CommandForPrediction = command; } /// diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictorTelemetryClient.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictorTelemetryClient.cs index 756c51a2ab5b..8f98cb31687b 100644 --- a/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictorTelemetryClient.cs +++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictorTelemetryClient.cs @@ -16,12 +16,12 @@ using Microsoft.ApplicationInsights.Extensibility; using Microsoft.Azure.Commands.Common.Authentication; using Microsoft.Azure.Commands.Common.Authentication.Abstractions; +using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; -using System.Linq; namespace Microsoft.Azure.PowerShell.Tools.AzPredictor { @@ -40,8 +40,8 @@ private sealed class TelemetrySession : AzureSession /// public TelemetrySession() { - this.DataStore = new DiskDataStore(); - this.ProfileDirectory = Path.Combine( + DataStore = new DiskDataStore(); + ProfileDirectory = Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), AzPredictorConstants.AzureProfileDirectoryName); } @@ -64,9 +64,15 @@ public override SourceLevels AuthenticationTraceSourceLevel } } + private const string TelemetryEventPrefix = "Az.Tools.Predictor"; + + /// + public string SessionId { get; } = Guid.NewGuid().ToString(); + + /// + public string CorrelationId { get; private set; } = Guid.NewGuid().ToString(); + private readonly TelemetryClient _telemetryClient; - private int _accepts; - private readonly string _sessionId; private object lockObject = new object(); private AzurePSDataCollectionProfile _cachedProfile; @@ -75,15 +81,20 @@ private AzurePSDataCollectionProfile DataCollectionProfile { get { + if (_cachedProfile != null) + { + return _cachedProfile; + } + lock (lockObject) { if (_cachedProfile == null) { var controller = DataCollectionController.Create(new TelemetrySession()); - this._cachedProfile = controller.GetProfile(() => { }); + _cachedProfile = controller.GetProfile(() => { }); } - return this._cachedProfile; + return _cachedProfile; } } } @@ -96,47 +107,105 @@ public AzPredictorTelemetryClient() TelemetryConfiguration configuration = TelemetryConfiguration.CreateDefault(); configuration.InstrumentationKey = "7df6ff70-8353-4672-80d6-568517fed090"; // Use Azuer-PowerShell instrumentation key. see https://github.com/Azure/azure-powershell-common/blob/master/src/Common/AzurePSCmdlet.cs _telemetryClient = new TelemetryClient(configuration); - _sessionId = Guid.NewGuid().ToString(); + _telemetryClient.Context.Location.Ip = "0.0.0.0"; + _telemetryClient.Context.Cloud.RoleInstance = "placeholderdon'tuse"; + _telemetryClient.Context.Cloud.RoleName = "placeholderdon'tuse"; } /// - public void OnSuggestionForHistory(string historyLine, - int? suggestionIndex, - int? fallbackIndex, - IEnumerable topSuggestions) + public void OnHistory(string historyLine) { if (!IsDataCollectionAllowed()) { return; } - var currentLog = new Dictionary(); - currentLog["History"] = historyLine; - currentLog["SessionId"] = _sessionId; + var currentLog = new Dictionary() + { + { "History", historyLine }, + { "SessionId", SessionId }, + { "CorrelationId", CorrelationId }, + }; + + _telemetryClient.TrackEvent($"{AzPredictorTelemetryClient.TelemetryEventPrefix}/CommandHistory", currentLog); - if (suggestionIndex.HasValue) +#if DEBUG + Console.WriteLine("Recording CommandHistory"); +#endif + } + + /// + public void OnRequestPrediction(string command) + { + if (!IsDataCollectionAllowed()) { - currentLog["SuggestionIndex"] = suggestionIndex.Value.ToString(CultureInfo.InvariantCulture); + return; } - if (fallbackIndex.HasValue) + CorrelationId = Guid.NewGuid().ToString(); + + var currentLog = new Dictionary() { - currentLog["FallbackIndex"] = fallbackIndex.Value.ToString(CultureInfo.InvariantCulture); - } + { "Command", command }, + { "SessionId", SessionId }, + { "CorrelationId", CorrelationId }, + }; + + _telemetryClient.TrackEvent($"{AzPredictorTelemetryClient.TelemetryEventPrefix}/RequestPrediction", currentLog); + +#if DEBUG + Console.WriteLine("Recording RequestPrediction"); +#endif + } - if (topSuggestions != null) + /// + public void OnRequestPredictionError(string command, Exception e) + { + if (!IsDataCollectionAllowed()) { - currentLog["Top5Suggestions"] = string.Join(',', topSuggestions.Take(5)); + return; } - this._telemetryClient.TrackEvent("ProcessHistory", currentLog); + var currentLog = new Dictionary() + { + { "Command", command }, + { "SessionId", SessionId }, + { "CorrelationId", CorrelationId }, + { "Exception", e.ToString() }, + }; + + _telemetryClient.TrackEvent($"{AzPredictorTelemetryClient.TelemetryEventPrefix}/RequestPredictionError", currentLog); + +#if DEBUG + Console.WriteLine("Recording RequestPredictionError"); +#endif } /// public void OnSuggestionAccepted(string acceptedSuggestion) { - ++this._accepts; + if (!IsDataCollectionAllowed()) + { + return; + } + var properties = new Dictionary() + { + { "AcceptedSuggestion", acceptedSuggestion }, + { "SessionId", SessionId }, + { "CorrelationId", CorrelationId }, + }; + + _telemetryClient.TrackEvent($"{AzPredictorTelemetryClient.TelemetryEventPrefix}/AcceptSuggestion", properties); + +#if DEBUG + Console.WriteLine("Recording AcceptSuggestion"); +#endif + } + + /// + public void OnGetSuggestion(string maskedUserInput, IEnumerable> suggestions, bool isCancelled) + { if (!IsDataCollectionAllowed()) { return; @@ -144,14 +213,22 @@ public void OnSuggestionAccepted(string acceptedSuggestion) var properties = new Dictionary() { - { "AcceptedSuggestion", acceptedSuggestion } + { "UserInput", maskedUserInput }, + { "Suggestion", JsonConvert.SerializeObject(suggestions) }, + { "SessionId", SessionId }, + { "CorrelationId", CorrelationId }, + { "IsCancelled", isCancelled.ToString(CultureInfo.InvariantCulture) }, }; - this._telemetryClient.TrackEvent("AcceptSuggestion", properties); + _telemetryClient.TrackEvent($"{AzPredictorTelemetryClient.TelemetryEventPrefix}/GetSuggestion", properties); + +#if DEBUG + Console.WriteLine("Recording GetSuggestioin"); +#endif } /// - public void OnGetSuggestion(PredictionSource predictionSource) + public void OnGetSuggestionError(Exception e) { if (!IsDataCollectionAllowed()) { @@ -160,10 +237,16 @@ public void OnGetSuggestion(PredictionSource predictionSource) var properties = new Dictionary() { - { "PredictionSource", predictionSource.ToString() } + { "SessionId", SessionId }, + { "CorrelationId", CorrelationId }, + { "Exception", e.ToString() }, }; - this._telemetryClient.TrackEvent("GetSuggestion", properties); + _telemetryClient.TrackEvent($"{AzPredictorTelemetryClient.TelemetryEventPrefix}/GetSuggestionError", properties); + +#if DEBUG + Console.WriteLine("Recording GetSuggestioinError"); +#endif } /// diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor/IAzPredictorService.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor/IAzPredictorService.cs index 1b07b81657ff..c80a7e328a16 100644 --- a/tools/Az.Tools.Predictor/Az.Tools.Predictor/IAzPredictorService.cs +++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor/IAzPredictorService.cs @@ -32,10 +32,10 @@ public interface IAzPredictorService public Tuple GetSuggestion(Ast input, CancellationToken cancellationToken); /// - /// Requests predictions, given a history string. + /// Requests predictions, given a command string. /// - /// A list of history commands - public void RequestPredictions(IEnumerable history); + /// A list of commands + public void RequestPredictions(IEnumerable commands); /// /// Record the history from PSReadLine. @@ -47,20 +47,5 @@ public interface IAzPredictorService /// Return true if command is part of known set of Az cmdlets, false otherwise. /// public bool IsSupportedCommand(string cmd); - - /// - /// For logging purposes, get the rank of the user input in the model suggestions list. - /// - public int? GetRankOfSuggestion(string commandName); - - /// - /// For logging purposes, get the rank of the user input in the fallback commands cache. - /// - public int? GetRankOfFallback(string commandName); - - /// - /// For logging purposes, get the top N suggestions from the model suggestions list. - /// - public IEnumerable GetTopNSuggestions(int n); } } diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor/ITelemetryClient.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor/ITelemetryClient.cs index f2fc95bb6345..3b17cb52610b 100644 --- a/tools/Az.Tools.Predictor/Az.Tools.Predictor/ITelemetryClient.cs +++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor/ITelemetryClient.cs @@ -12,6 +12,7 @@ // limitations under the License. // ---------------------------------------------------------------------------------- +using System; using System.Collections.Generic; namespace Microsoft.Azure.PowerShell.Tools.AzPredictor @@ -22,13 +23,33 @@ namespace Microsoft.Azure.PowerShell.Tools.AzPredictor public interface ITelemetryClient { /// - /// Collects the event for the top 5 predeiction for the history. + /// Gets the correlation id for the telemetry events. /// - /// The history command that triggers the suggestion. - /// The index of the suggestion from the suggestion model. - /// The index in the command list as a fallback. - /// The top suggestions. - public void OnSuggestionForHistory(string historyLine, int? suggestionIndex, int? fallbackIndex, IEnumerable topSuggestions); + public string CorrelationId { get; } + + /// + /// Gets the session id for the telemetry events. + /// + public string SessionId { get; } + + /// + /// Collects the event of the history command. + /// + /// The history command from PSReadLine. + public void OnHistory(string historyLine); + + /// + /// Collects the event when a prediction is requested. + /// + /// The command to that we request the prediction for. + public void OnRequestPrediction(string command); + + /// + /// Collects the event when we fail to get the prediction for the command + /// + /// The command to that we request the prediction for. + /// The exception + public void OnRequestPredictionError(string command, Exception e); /// /// Collects when a suggestion is accepted. @@ -39,7 +60,16 @@ public interface ITelemetryClient /// /// Collects when we return a suggestion /// - /// The source to get the prediction - public void OnGetSuggestion(PredictionSource predictionSource); + /// The user input that the suggestions are for + /// The list of suggestion and its source + /// Indicates whether the caller has cancelled the call to get suggestion. Usually that's because of time out + public void OnGetSuggestion(string maskedUserInput, IEnumerable> suggestions, bool isCancelled); + + /// + /// Collects when an exception is thrown when we return a suggestion. + /// + /// The exception + + public void OnGetSuggestionError(Exception e); } } diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor/PredictionSource.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor/PredictionSource.cs index 1ca16b5698a3..801e0796d18c 100644 --- a/tools/Az.Tools.Predictor/Az.Tools.Predictor/PredictionSource.cs +++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor/PredictionSource.cs @@ -12,11 +12,15 @@ // limitations under the License. // ---------------------------------------------------------------------------------- +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + namespace Microsoft.Azure.PowerShell.Tools.AzPredictor { /// /// An enum for the source where we get the prediction. /// + [JsonConverter(typeof(StringEnumConverter))] public enum PredictionSource { /// @@ -25,18 +29,18 @@ public enum PredictionSource None, /// - /// The prediction is from the command list. + /// The prediction is from the static command list. /// - Commands, + StaticCommands, /// - /// The prediction is from the list for the old history. + /// The prediction is from the list for the older command. /// - PreviousHistory, + PreviousCommand, /// - /// The prediction is from the list for the current history. + /// The prediction is from the list for the currentc command. /// - CurrentHistory + CurrentCommand } } diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor/Predictor.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor/Predictor.cs index 51d9a991dd7a..60978f0e611e 100644 --- a/tools/Az.Tools.Predictor/Az.Tools.Predictor/Predictor.cs +++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor/Predictor.cs @@ -26,7 +26,6 @@ namespace Microsoft.Azure.PowerShell.Tools.AzPredictor /// internal sealed class Predictor { - private readonly IList _raw; private readonly IList _predictions; private readonly ParameterValuePredictor _parameterValuePredictor; @@ -37,7 +36,6 @@ internal sealed class Predictor /// Provide the prediction to the parameter values. public Predictor(IList modelPredictions, ParameterValuePredictor parameterValuePredictor) { - this._raw = modelPredictions; this._parameterValuePredictor = parameterValuePredictor; this._predictions = new List(); @@ -97,16 +95,6 @@ public string Query(Ast input, CancellationToken cancellationToken) } } - /// - /// Gets the top N predictions from the list. - /// - /// The number of prediction from the top. - /// Collection predictions - internal IEnumerable GetTopNPrediction(int topN) - { - return this._raw.Take(topN); - } - /// /// Get the index in the prediction list of the user input /// If the user input does not match any command in predictions, then return -1.