Skip to content

Commit

Permalink
Correlate the suggestion related events (Azure#14041)
Browse files Browse the repository at this point in the history 
* Correlate the suggestion related events.

- Collects the event when our suggestion is displayed.
- Use SuggestionSessionId to correlate the suggestions that are provided
  by us, displayed by PSReadLine, and accepted by the user.
- Collects the client id in the telemetry events to identify if it's
  called by PSReadLine or something else.

* Update the version.

* Use new api.

* Guard against when the display mode is not known.

* Fix the package and a bug

- Use the preview packages that contain the change.
- Fix a bug that the loop infinitely when parsing the parameter values
  and the last parameter is a switch parameter.

* Rename the variable UserId to HashUserId.

* Updated release notes

* Added ChangeLog.md

* Remove the binaries that're added accidentally.

* Fixed typo in changelog

* Fix the broken merge.

* Fix the merge.

* Remove the test version suffix.

* Add a new field to the telemetry.

* Update the psreadline binaries.

* Update README.md

Updated README to reflect on the change of versions.

* Update Az.Tools.Predictor.psd1

Updated version requirements and addressed release notes comments.

* Update the doc.

* Update the link to dotnet 6.0

* Fix the list number.

* Remove .NET 6

Co-authored-by: Damien Caro <[email protected]>
  • Loading branch information
@kceiw @dcaro
kceiw and dcaro authored Feb 26, 2021
1 parent bed48d3 commit 17f780b
Show file tree
Hide file tree
Showing 32 changed files with 1,610 additions and 242 deletions.
3 changes: 1 addition & 2 deletions tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Az.Tools.Predictor.Test.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net5.0</TargetFramework>
<TargetFramework>net6.0</TargetFramework>
<AssemblyName>Microsoft.Azure.PowerShell.Tools.AzPredictor.Test</AssemblyName>
<RootNamespace>Microsoft.Azure.PowerShell.Tools.AzPredictor.Test</RootNamespace>
<IsPackable>false</IsPackable>
Expand All @@ -14,7 +14,6 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="System.Management.Automation" Version="7.1.0-preview.7" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
Expand Down
787 changes: 787 additions & 0 deletions tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorTelemetryTests.cs
View file

Large diffs are not rendered by default.

97 changes: 53 additions & 44 deletions tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
using System.Linq;
using System.Management.Automation.Subsystem;
using System.Threading;
using System.Threading.Tasks;
using Xunit;

namespace Microsoft.Azure.PowerShell.Tools.AzPredictor.Test
Expand All @@ -27,6 +28,7 @@ namespace Microsoft.Azure.PowerShell.Tools.AzPredictor.Test
[Collection("Model collection")]
public sealed class AzPredictorTests
{
private const string AzPredictorClient = "Test";
private readonly ModelFixture _fixture;
private readonly MockAzPredictorTelemetryClient _telemetryClient;
private readonly MockAzPredictorService _service;
Expand Down Expand Up @@ -54,65 +56,66 @@ public AzPredictorTests(ModelFixture modelFixture)
/// Verify we replace unsupported command with <see cref="AzPredictorConstants.CommandPlaceholder"/>.
/// </summary>
[Fact]
public void VerifyRequestPredictionForOneUnsupportedCommandInHistory()
public async Task VerifyRequestPredictionForOneUnsupportedCommandInHistory()
{
IReadOnlyList<string> history = new List<string>()
{
"git status"
};

_telemetryClient.RecordedSuggestion = null;
_service.Commands = null;
_service.History = null;
_service.ResetRequestPredictionTask();

_azPredictor.StartEarlyProcessing(history);
_azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
await _service.RequestPredictionTaskCompletionSource.Task;

Assert.Equal(new List<string>() { AzPredictorConstants.CommandPlaceholder, AzPredictorConstants.CommandPlaceholder }, _service.Commands);
Assert.Equal(AzPredictorConstants.CommandPlaceholder, _telemetryClient.RecordedSuggestion.HistoryLine);
Assert.Null(_service.History);
}

/// <summary>
/// Verify that we masked the supported command in requesting prediction and telemetry.
/// </summary>
[Fact]
public void VerifyRequestPredictionForOneSupportedCommandInHistory()
public async Task VerifyRequestPredictionForOneSupportedCommandInHistory()
{
IReadOnlyList<string> history = new List<string>()
{
"New-AzVM -Name hello -Location WestUS"
};

_telemetryClient.RecordedSuggestion = null;
_service.Commands = null;
_service.History = null;
_service.ResetRequestPredictionTask();

_azPredictor.StartEarlyProcessing(history);
_azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
await _service.RequestPredictionTaskCompletionSource.Task;

string maskedCommand = "New-AzVM -Location *** -Name ***";

Assert.Equal(new List<string>() { AzPredictorConstants.CommandPlaceholder, maskedCommand }, _service.Commands);
Assert.Equal(maskedCommand, _telemetryClient.RecordedSuggestion.HistoryLine);
Assert.Equal(history[0], _service.History.ToString());
}

/// <summary>
/// Verify that we can handle the two supported command in sequences.
/// </summary>
[Fact]
public void VerifyRequestPredictionForTwoSupportedCommandInHistory()
public async Task VerifyRequestPredictionForTwoSupportedCommandInHistory()
{
IReadOnlyList<string> history = new List<string>()
{
"New-AzResourceGroup -Name 'resourceGroup01'",
"New-AzVM -Name:hello -Location:WestUS"
};

_telemetryClient.RecordedSuggestion = null;
_service.Commands = null;
_service.History = null;
_service.ResetRequestPredictionTask();

_azPredictor.StartEarlyProcessing(history);
_azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
await _service.RequestPredictionTaskCompletionSource.Task;

var maskedCommands = new List<string>()
{
Expand All @@ -121,27 +124,27 @@ public void VerifyRequestPredictionForTwoSupportedCommandInHistory()
};

Assert.Equal(maskedCommands, _service.Commands);
Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
Assert.Equal(history[1], _service.History.ToString());
}

/// <summary>
/// Verify that we can handle the two unsupported command in sequences.
/// </summary>
[Fact]
public void VerifyRequestPredictionForTwoUnsupportedCommandInHistory()
public async Task VerifyRequestPredictionForTwoUnsupportedCommandInHistory()
{
IReadOnlyList<string> history = new List<string>()
{
"git status",
@"$a='ResourceGroup01'",
};

_telemetryClient.RecordedSuggestion = null;
_service.Commands = null;
_service.History = null;
_service.ResetRequestPredictionTask();

_azPredictor.StartEarlyProcessing(history);
_azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
await _service.RequestPredictionTaskCompletionSource.Task;

var maskedCommands = new List<string>()
{
Expand All @@ -150,33 +153,37 @@ public void VerifyRequestPredictionForTwoUnsupportedCommandInHistory()
};

Assert.Equal(maskedCommands, _service.Commands);
Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
Assert.Null(_service.History);
}

/// <summary>
/// Verify that we skip the unsupported commands.
/// </summary>
[Fact]
public void VerifyNotTakeUnsupportedCommands()
public async Task VerifyNotTakeUnsupportedCommands()
{
var history = new List<string>()
{
"New-AzResourceGroup -Name:resourceGroup01",
"New-AzVM -Name hello -Location WestUS"
};

_telemetryClient.RecordedSuggestion = null;
_service.Commands = null;
_service.History = null;
_service.ResetRequestPredictionTask();

_azPredictor.StartEarlyProcessing(history);
_azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
await _service.RequestPredictionTaskCompletionSource.Task;

_service.ResetRequestPredictionTask();
history.Add("git status");
_azPredictor.StartEarlyProcessing(history);
_azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
// Don't need to await on _service.RequestPredictionTask, because "git" isn't a supported command and RequestPredictionsAsync isn't called.

_service.ResetRequestPredictionTask();
history.Add(@"$a='NewResourceName'");
_azPredictor.StartEarlyProcessing(history);
_azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
// Don't need to await on _service.RequestPredictionTask, because assignment isn't supported and RequestPredictionsAsync isn't called.

// We don't take the last two unsupported command to request predictions.
// But we send the masked one in telemetry.
Expand All @@ -188,13 +195,14 @@ public void VerifyNotTakeUnsupportedCommands()
};

Assert.Equal(maskedCommands, _service.Commands);
Assert.Equal(AzPredictorConstants.CommandPlaceholder, _telemetryClient.RecordedSuggestion.HistoryLine);
Assert.Equal(history[1], _service.History.ToString());

// When there is a new supported command, we'll use that for prediction.

_service.ResetRequestPredictionTask();
history.Add("Get-AzResourceGroup -Name ResourceGroup01");
_azPredictor.StartEarlyProcessing(history);
_azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
await _service.RequestPredictionTaskCompletionSource.Task;

maskedCommands = new List<string>()
{
Expand All @@ -203,30 +211,32 @@ public void VerifyNotTakeUnsupportedCommands()
};

Assert.Equal(maskedCommands, _service.Commands);
Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
Assert.Equal(history.Last(), _service.History.ToString());
}

/// <summary>
/// Verify that we handle the three supported command in the same order.
/// </summary>
[Fact]
public void VerifyThreeSupportedCommands()
public async Task VerifyThreeSupportedCommands()
{
var history = new List<string>()
{
"New-AzResourceGroup -Name resourceGroup01",
"New-AzVM -Name:hello -Location:WestUS"
};

_telemetryClient.RecordedSuggestion = null;
_service.Commands = null;
_service.History = null;
_service.ResetRequestPredictionTask();

_azPredictor.StartEarlyProcessing(history);
_azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
await _service.RequestPredictionTaskCompletionSource.Task;

_service.ResetRequestPredictionTask();
history.Add("Get-AzResourceGroup -Name resourceGroup01");
_azPredictor.StartEarlyProcessing(history);
_azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
await _service.RequestPredictionTaskCompletionSource.Task;

var maskedCommands = new List<string>()
{
Expand All @@ -235,27 +245,27 @@ public void VerifyThreeSupportedCommands()
};

Assert.Equal(maskedCommands, _service.Commands);
Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
Assert.Equal(history.Last(), _service.History.ToString());
}

/// <summary>
/// Verify that we handle the sequence of one unsupported command and one supported command.
/// </summary>
[Fact]
public void VerifyUnsupportedAndSupportedCommands()
public async Task VerifyUnsupportedAndSupportedCommands()
{
var history = new List<string>()
{
"git status",
"New-AzVM -Name:hello -Location:WestUS"
};

_telemetryClient.RecordedSuggestion = null;
_service.Commands = null;
_service.History = null;
_service.ResetRequestPredictionTask();

_azPredictor.StartEarlyProcessing(history);
_azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
await _service.RequestPredictionTaskCompletionSource.Task;

var maskedCommands = new List<string>()
{
Expand All @@ -264,27 +274,27 @@ public void VerifyUnsupportedAndSupportedCommands()
};

Assert.Equal(maskedCommands, _service.Commands);
Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
Assert.Equal(history.Last(), _service.History.ToString());
}

/// <summary>
/// Verify that we handle the sequence of one supported command and one unsupported command.
/// </summary>
[Fact]
public void VerifySupportedAndUnsupportedCommands()
public async Task VerifySupportedAndUnsupportedCommands()
{
var history = new List<string>()
{
"New-AzVM -Name hello -Location WestUS",
"git status",
};

_telemetryClient.RecordedSuggestion = null;
_service.Commands = null;
_service.History = null;
_service.ResetRequestPredictionTask();

_azPredictor.StartEarlyProcessing(history);
_azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
await _service.RequestPredictionTaskCompletionSource.Task;

var maskedCommands = new List<string>()
{
Expand All @@ -293,7 +303,6 @@ public void VerifySupportedAndUnsupportedCommands()
};

Assert.Equal(maskedCommands, _service.Commands);
Assert.Equal(AzPredictorConstants.CommandPlaceholder, _telemetryClient.RecordedSuggestion.HistoryLine);
Assert.Equal(history.First(), _service.History.ToString());
}

Expand All @@ -307,10 +316,10 @@ public void VerifySuggestion(string userInput)
{
var predictionContext = PredictionContext.Create(userInput);
var expected = this._service.GetSuggestion(predictionContext, 1, 1, CancellationToken.None);
var actual = this._azPredictor.GetSuggestion(predictionContext, CancellationToken.None);
var actual = this._azPredictor.GetSuggestion(AzPredictorTests.AzPredictorClient, predictionContext, CancellationToken.None);

Assert.Equal(expected.Count, actual.Count);
Assert.Equal(expected.PredictiveSuggestions.First().SuggestionText, actual.First().SuggestionText);
Assert.Equal(expected.Count, actual.SuggestionEntries.Count);
Assert.Equal(expected.PredictiveSuggestions.First().SuggestionText, actual.SuggestionEntries.First().SuggestionText);
}

/// <summary>
Expand All @@ -331,9 +340,9 @@ public void VerifySuggestionOnIncompleteCommand()
var expected = "New-AzResourceGroup -Name 'ResourceGroup01' -Location 'Central US' -WhatIf -Tag value1";

var predictionContext = PredictionContext.Create(userInput);
var actual = localAzPredictor.GetSuggestion(predictionContext, CancellationToken.None);
var actual = localAzPredictor.GetSuggestion(AzPredictorTests.AzPredictorClient, predictionContext, CancellationToken.None);

Assert.Equal(expected, actual.First().SuggestionText);
Assert.Equal(expected, actual.SuggestionEntries.First().SuggestionText);
}

/// <summary>
Expand All @@ -348,9 +357,9 @@ public void VerifySuggestionOnIncompleteCommand()
public void VerifyMalFormattedCommandLine(string userInput)
{
var predictionContext = PredictionContext.Create(userInput);
var actual = _azPredictor.GetSuggestion(predictionContext, CancellationToken.None);
var actual = _azPredictor.GetSuggestion(AzPredictorTests.AzPredictorClient, predictionContext, CancellationToken.None);

Assert.Empty(actual);
Assert.Null(actual.SuggestionEntries);
}
}
}
39 changes: 39 additions & 0 deletions tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Mocks/MockAzContext.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// ----------------------------------------------------------------------------------
//
// Copyright Microsoft Corporation
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------------------------------------------------------------------------

using System;

namespace Microsoft.Azure.PowerShell.Tools.AzPredictor.Test.Mocks
{
sealed class MockAzContext : IAzContext
{
public string HashUserId => "TestUserId";

public string MacAddress => "TestMacAddress";

public string OSVersion => "TestOSVersion";

public Version PowerShellVersion => Version.Parse("0.0.0.0");

public Version ModuleVersion => Version.Parse("0.0.0.0");

public Version AzVersion => Version.Parse("0.0.0.0");

public bool IsInternal => true;

public void UpdateContext()
{
}
}
}
Loading

0 comments on commit 17f780b

Please sign in to comment.