Skip to content

Commit

Permalink
Add powershell APIs for Partition merge and Throughput redistribution (
Browse files Browse the repository at this point in the history
…#18247)

* partition merge powershell cmdlet

* Delete PartitionInfoGetResults

* add pitr lite support

* fix update continuoustier and tests

* add test recording

* Partition merge powershell cmdlets and tests

* Add RedisCacheTestRunner and replace Controller (#18222)

* Add PrivateDnsTestRunner and replace TestBase (#18202)

* Add ReservationsTestRunner and replace TestController (#18226)

* Add ResourceGraphTestRunner to replace TestController (#18227)

* Signed-off-by: [email protected] <[email protected]>

* Add ResourceGraphTestRunner to replace TestController

* Add SecurityInsightsTestRunner and replace TestController (#18228)

* Signed-off-by: [email protected] <[email protected]> (#18229)

* Add ServiceFabricTestRunner and replace TestController (#18231)

* Add SignalRTestRunner to replace TestController (#18232)

* powershell changes

* Add SupportTestRunner to replace TestController (#18188)

* Add SupportTestRunner to replace TestController

* Revert Az.Support document changes

* Add SecurityTestRunner to replace TestController (#18189)

* Add SecurityTestRunner to replace TestController

* Revert Az.Security document changes

* Add IotHubTestRunner to replace IotHubController (#18200)

* Add MonitorTestRunner to replace TestsController (#18201)

* Add TrafficManagerTestRunner to replace TestController (#18243)

* Add more tests and fixes

* Fix merge conflicts and documentation

* revert non cosmosdb changes

* Fix confirmation message

* Update NuGet.Config

* suppressions

* Fix missing parameter

* converted tests to manual verification

Co-authored-by: Vinh Trinh <[email protected]>
Co-authored-by: Yufan Yang <[email protected]>
Co-authored-by: v-yuzhichen <[email protected]>
Co-authored-by: Ziyue Zheng <[email protected]>
Co-authored-by: Yunchi Wang <[email protected]>
  • Loading branch information
6 people authored May 20, 2022
1 parent fbaa9e8 commit fdecb11
Show file tree
Hide file tree
Showing 153 changed files with 17,200 additions and 4,932 deletions.
4 changes: 2 additions & 2 deletions src/CosmosDB/CosmosDB.Test/CosmosDB.Test.csproj
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<PsModuleName>CosmosDB</PsModuleName>
</PropertyGroup>
<Import Project="$(MSBuildThisFileDirectory)..\..\Az.Test.props" />
<ItemGroup>
<PackageReference Include="Microsoft.Azure.KeyVault" Version="3.0.1" />
<PackageReference Include="Microsoft.Azure.Management.CosmosDB" Version="3.7.0-preview" />
<PackageReference Include="Microsoft.Azure.Management.CosmosDB" Version="3.8.0-preview" />
<PackageReference Include="Microsoft.Azure.Management.Network" Version="22.0.0" />
</ItemGroup>
</Project>
14 changes: 14 additions & 0 deletions src/CosmosDB/CosmosDB.Test/ScenarioTests/MongoOperationsTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,19 @@ public void TestMongoMigrateThroughputCmdlets()
{
TestRunner.RunTestScript("Test-MongoMigrateThroughputCmdlets");
}

[Fact(Skip = "Cannot acquire token credential for a specific audience. No support from test framework. I have verified the tests manually.")]
[Trait(Category.AcceptanceType, Category.CheckIn)]
public void TestMongoDBCollectionThroughputRedistributionCmdlet()
{
TestRunner.RunTestScript("Test-MongoDBCollectionAdaptiveRUCmdlets");
}

[Fact(Skip = "Cannot acquire token credential for a specific audience. No support from test framework. I have verified the tests manually.")]
[Trait(Category.AcceptanceType, Category.CheckIn)]
public void TestMongoDBCollectionMergeCmdlet()
{
TestRunner.RunTestScript("Test-MongoDBCollectionMergeCmdlet");
}
}
}
107 changes: 107 additions & 0 deletions src/CosmosDB/CosmosDB.Test/ScenarioTests/MongoOperationsTests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -445,4 +445,111 @@ function Test-MongoMigrateThroughputCmdlets
Remove-AzCosmosDBMongoDBCollection -AccountName $AccountName -ResourceGroupName $rgName -DatabaseName $DatabaseName -Name $CollectionName
Remove-AzCosmosDBMongoDBDatabase -AccountName $AccountName -ResourceGroupName $rgName -Name $DatabaseName
}
}

<#
.SYNOPSIS
Test mongodb Throughput redistribution cmdlets
#>
function Test-MongoDBCollectionAdaptiveRUCmdlets
{
$AccountName = "mongomergeaccount"
$rgName = "canary-sdk-test"
$DatabaseName = "adaptiverudatabase"
$ContainerName = "adaptiveruContainer"

$ShardKey = "shardKeyPath"
$ContainerThroughputValue = 24000
$UpdatedContainerThroughputValue = 2000

Try{

New-AzCosmosDBMongoDBDatabase -AccountName $AccountName -ResourceGroupName $rgName -Name $DatabaseName
New-AzCosmosDBMongoDBCollection -AccountName $AccountName -ResourceGroupName $rgName -DatabaseName $DatabaseName -Throughput $ContainerThroughputValue -Name $ContainerName -Shard $ShardKey
Update-AzCosmosDBMongoDBCollectionThroughput -AccountName $AccountName -ResourceGroupName $rgName -DatabaseName $DatabaseName -Name $ContainerName -Throughput $UpdatedContainerThroughputValue
$partitions = Get-AzCosmosDBMongoDBCollectionPerPartitionThroughput -ResourceGroupName $rgName -AccountName $AccountName -DatabaseName $DatabaseName -Name $ContainerName -AllPartitions
Assert-AreEqual $partitions.Count 4
$sources = @()
$targets = @()
Foreach($partition in $partitions)
{
Assert-AreEqual $partition.Throughput 500
if($partition.Id -lt 2)
{
$throughput = $partition.Throughput - 100
$sources += New-AzCosmosDBPhysicalPartitionThroughputObject -Id $partition.Id -Throughput $throughput
}
else
{
$throughput = $partition.Throughput + 100
$targets += New-AzCosmosDBPhysicalPartitionThroughputObject -Id $partition.Id -Throughput $throughput
}
}

$newPartitions = Update-AzCosmosDBMongoDBCollectionPerPartitionThroughput -ResourceGroupName $rgName -AccountName $AccountName -DatabaseName $DatabaseName -Name $ContainerName -SourcePhysicalPartitionThroughputObject $sources -TargetPhysicalPartitionThroughputObject $targets
Assert-AreEqual $newPartitions.Count 4
Foreach($partition in $newPartitions)
{
if($partition.Id -lt 2)
{
Assert-AreEqual $partition.Throughput 400
}
else
{
Assert-AreEqual $partition.Throughput 600
}
}

$resetPartitions = Update-AzCosmosDBMongoDBCollectionPerPartitionThroughput -ResourceGroupName $rgName -AccountName $AccountName -DatabaseName $DatabaseName -Name $ContainerName -EqualDistributionPolicy

Assert-AreEqual $resetPartitions.Count 4

Foreach($partition in $resetPartitions)
{
Assert-AreEqual $partition.Throughput 500
}

$somePartitions = Get-AzCosmosDBMongoDBCollectionPerPartitionThroughput -ResourceGroupName $rgName -AccountName $AccountName -DatabaseName $DatabaseName -Name $ContainerName -PhysicalPartitionIds ('0', '1')
Assert-AreEqual $somePartitions.Count 2
}
Finally{
Remove-AzCosmosDBMongoDBCollection -AccountName $AccountName -ResourceGroupName $rgName -DatabaseName $DatabaseName -Name $ContainerName
Remove-AzCosmosDBMongoDBDatabase -AccountName $AccountName -ResourceGroupName $rgName -Name $DatabaseName
}
}


<#
.SYNOPSIS
Test mongodb merge cmdlet
#>
function Test-MongoDBCollectionMergeCmdlet
{
$AccountName = "mongomergeaccount"
$rgName = "canary-sdk-test"
$DatabaseName = "mergedatabase"
$ContainerName = "mergecontainer"

$ShardKey = "shardKeyPath"

$ContainerThroughputValue = 24000
$UpdatedContainerThroughputValue = 2000

Try{

New-AzCosmosDBMongoDBDatabase -AccountName $AccountName -ResourceGroupName $rgName -Name $DatabaseName
New-AzCosmosDBMongoDBCollection -AccountName $AccountName -ResourceGroupName $rgName -DatabaseName $DatabaseName -Throughput $ContainerThroughputValue -Name $ContainerName -Shard $ShardKey
Update-AzCosmosDBMongoDBCollectionThroughput -AccountName $AccountName -ResourceGroupName $rgName -DatabaseName $DatabaseName -Name $ContainerName -Throughput $UpdatedContainerThroughputValue
$physicalPartitionStorageInfos = Invoke-AzCosmosDBMongoDBCollectionMerge -ResourceGroupName $rgName -AccountName $AccountName -DatabaseName $DatabaseName -Name $ContainerName -Force
Assert-AreEqual $physicalPartitionStorageInfos.Count 1
if($physicalPartitionStorageInfos[0].Id.contains("mergeTarget"))
{
throw "Name of partition: " + $physicalPartitionStorageInfos[0].Id + " Unexpected Id: mergeTarget"
}

}
Finally{
Remove-AzCosmosDBMongoDBCollection -AccountName $AccountName -ResourceGroupName $rgName -DatabaseName $DatabaseName -Name $ContainerName
Remove-AzCosmosDBMongoDBDatabase -AccountName $AccountName -ResourceGroupName $rgName -Name $DatabaseName
}
}
18 changes: 16 additions & 2 deletions src/CosmosDB/CosmosDB.Test/ScenarioTests/RestoreTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,23 @@ public void TestMongoDBCollectionBackupInformationCmdLets()

[Fact]
[Trait(Category.AcceptanceType, Category.CheckIn)]
public void TestUpdateCosmosDBAccountBackupPolicyCmdLet()
public void TestUpdateCosmosDBAccountBackupPolicyToContinuous30DaysCmdLets()
{
TestRunner.RunTestScript("Test-UpdateCosmosDBAccountBackupPolicyCmdLet");
TestRunner.RunTestScript( "Test-UpdateCosmosDBAccountBackupPolicyToContinuous30DaysCmdLets");
}

[Fact]
[Trait(Category.AcceptanceType, Category.CheckIn)]
public void TestUpdateCosmosDBAccountBackupPolicyToContinuous7DaysCmdLets()
{
TestRunner.RunTestScript("Test-UpdateCosmosDBAccountBackupPolicyToContinuous7DaysCmdLets");
}

[Fact]
[Trait(Category.AcceptanceType, Category.CheckIn)]
public void TestProvisionCosmosDBAccountBackupPolicyWithContinuous7DaysCmdLets()
{
TestRunner.RunTestScript("Test-ProvisionCosmosDBAccountBackupPolicyWithContinuous7DaysCmdLets");
}
}
}
126 changes: 120 additions & 6 deletions src/CosmosDB/CosmosDB.Test/ScenarioTests/RestoreTests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ function Test-RestoreFromNewAccountCmdlets {
Assert-AreEqual $sourceRestorableAccount.DatabaseAccountInstanceId $sourceCosmosDBAccount.InstanceId
Assert-NotNull $sourceRestorableAccount.DatabaseAccountName
Assert-NotNull $sourceRestorableAccount.CreationTime
Assert-NotNull $sourceRestorableAccount.OldestRestorableTime

$restorableSqlDatabases = Get-AzCosmosDBSqlRestorableDatabase -Location $sourceCosmosDBAccount.Location -DatabaseAccountInstanceId $sourceCosmosDBAccount.InstanceId
Assert-NotNull $restorableSqlDatabases
Expand Down Expand Up @@ -139,6 +140,7 @@ function Test-MongoRestoreAccountCmdlets {
Assert-AreEqual $sourceRestorableAccount.DatabaseAccountInstanceId $sourceCosmosDBAccount.InstanceId
Assert-NotNull $sourceRestorableAccount.DatabaseAccountName
Assert-NotNull $sourceRestorableAccount.CreationTime
Assert-NotNull $sourceRestorableAccount.OldestRestorableTime

$NewDatabase = New-AzCosmosDBMongoDBDatabase -AccountName $sourceCosmosDBAccountName -ResourceGroupName $rgName -Name $databaseName -Throughput $ThroughputValue
$NewCollection = New-AzCosmosDBMongoDBCollection -AccountName $sourceCosmosDBAccountName -ResourceGroupName $rgName -DatabaseName $databaseName -Throughput $CollectionThroughputValue -Name $CollectionName -Shard $ShardKey
Expand Down Expand Up @@ -218,6 +220,7 @@ function Test-RestoreFailuresAccountCmdlets {
Assert-AreEqual $sourceRestorableAccount.DatabaseAccountInstanceId $sourceCosmosDBAccount.InstanceId
Assert-NotNull $sourceRestorableAccount.DatabaseAccountName
Assert-NotNull $sourceRestorableAccount.CreationTime
Assert-NotNull $sourceRestorableAccount.OldestRestorableTime

$restorableSqlDatabases = Get-AzCosmosDBSqlRestorableDatabase -Location $sourceCosmosDBAccount.Location -DatabaseAccountInstanceId $sourceCosmosDBAccount.InstanceId
Assert-NotNull $restorableSqlDatabases
Expand Down Expand Up @@ -246,9 +249,9 @@ function Test-RestoreFailuresAccountCmdlets {
}

function Test-SqlContainerBackupInformationCmdLets {
$rgName = "CosmosDBResourceGroup14"
$rgName = "CosmosDBResourceGroup52"
$location = "Central US"
$cosmosDBAccountName = "cosmosdb-1214"
$cosmosDBAccountName = "cosmosdb-1252"
$databaseName = "TestDB1";
$collectionName = "TestCollectionInDB1";
$apiKind = "Sql"
Expand Down Expand Up @@ -326,10 +329,10 @@ function Test-MongoDBCollectionBackupInformationCmdLets {
Assert-NotNull $backupInfo.LatestRestorableTimestamp
}

function Test-UpdateCosmosDBAccountBackupPolicyCmdLet {
$rgName = "CosmosDBResourceGroup20"
function Test-UpdateCosmosDBAccountBackupPolicyToContinuous30DaysCmdLets {
$rgName = "PSCosmosDBResourceGroup20"
$location = "Central US"
$cosmosDBAccountName = "cosmosdb-1220"
$cosmosDBAccountName = "ps-cosmosdb-1220"
$apiKind = "Sql"
$consistencyLevel = "Session"
$locations = @()
Expand All @@ -345,11 +348,122 @@ function Test-UpdateCosmosDBAccountBackupPolicyCmdLet {
}

$updatedCosmosDBAccount = Update-AzCosmosDBAccount -ResourceGroupName $rgName -Name $cosmosDBAccountName -BackupPolicyType Continuous
Start-Sleep -s 50
Start-Sleep -s (60)

$updatedCosmosDBAccount = Get-AzCosmosDBAccount -ResourceGroupName $rgName -Name $cosmosDBAccountName
Assert-NotNull $updatedCosmosDBAccount.BackupPolicy.BackupPolicyMigrationState
Assert-NotNull $updatedCosmosDBAccount.BackupPolicy.BackupPolicyMigrationState.Status
Assert-NotNull $updatedCosmosDBAccount.BackupPolicy.BackupPolicyMigrationState.TargetType
Assert-NotNull $updatedCosmosDBAccount.BackupPolicy.BackupPolicyMigrationState.StartTime

Start-Sleep -s (60 * 5)

while (
$updatedCosmosDBAccount.BackupPolicy.BackupPolicyMigrationState.Status -ne "Completed" -and
$updatedCosmosDBAccount.BackupPolicy.BackupPolicyMigrationState.Status -ne "Failed" -and
$updatedCosmosDBAccount.BackupPolicy.BackupType -ne "Continuous")
{
Start-Sleep -s 60

# keep polling the migration Status
$updatedCosmosDBAccount = Get-AzCosmosDBAccount -ResourceGroupName $rgName -Name $cosmosDBAccountName
}

Assert-AreEqual "Continuous" $updatedCosmosDBAccount.BackupPolicy.BackupType
Assert-AreEqual "Continuous30Days" $updatedCosmosDBAccount.BackupPolicy.Tier

$updatedCosmosDBAccount = Update-AzCosmosDBAccount -ResourceGroupName $rgName -Name $cosmosDBAccountName -BackupPolicyType Continuous -ContinuousTier Continuous7Days
Start-Sleep -s (60 * 2)

$updatedCosmosDBAccount = Get-AzCosmosDBAccount -ResourceGroupName $rgName -Name $cosmosDBAccountName
Assert-AreEqual "Continuous" $updatedCosmosDBAccount.BackupPolicy.BackupType
Assert-AreEqual "Continuous7Days" $updatedCosmosDBAccount.BackupPolicy.Tier
}

function Test-UpdateCosmosDBAccountBackupPolicyToContinuous7DaysCmdLets {
$rgName = "PSCosmosDBResourceGroup50"
$location = "Central US"
$cosmosDBAccountName = "ps-cosmosdb-1250"
$apiKind = "Sql"
$consistencyLevel = "Session"
$locations = @()
$locations += New-AzCosmosDBLocationObject -Location $location -FailoverPriority 0 -IsZoneRedundant 0

$resourceGroup = New-AzResourceGroup -ResourceGroupName $rgName -Location $location

Try {
New-AzCosmosDBAccount -ResourceGroupName $rgName -LocationObject $locations -Name $cosmosDBAccountName -ApiKind $apiKind -DefaultConsistencyLevel $consistencyLevel
}
Catch {
Assert-AreEqual $_.Exception.Message ("Resource with Name " + $cosmosDBAccountName + " already exists.")
}

$updatedCosmosDBAccount = Update-AzCosmosDBAccount -ResourceGroupName $rgName -Name $cosmosDBAccountName -BackupPolicyType Continuous -ContinuousTier Continuous7Days
Start-Sleep -s (60)

$updatedCosmosDBAccount = Get-AzCosmosDBAccount -ResourceGroupName $rgName -Name $cosmosDBAccountName
Assert-NotNull $updatedCosmosDBAccount.BackupPolicy.BackupPolicyMigrationState
Assert-NotNull $updatedCosmosDBAccount.BackupPolicy.BackupPolicyMigrationState.Status
Assert-NotNull $updatedCosmosDBAccount.BackupPolicy.BackupPolicyMigrationState.TargetType
Assert-NotNull $updatedCosmosDBAccount.BackupPolicy.BackupPolicyMigrationState.StartTime

Start-Sleep -s (60 * 5)

while (
$updatedCosmosDBAccount.BackupPolicy.BackupPolicyMigrationState.Status -ne "Completed" -and
$updatedCosmosDBAccount.BackupPolicy.BackupPolicyMigrationState.Status -ne "Failed" -and
$updatedCosmosDBAccount.BackupPolicy.BackupType -ne "Continuous")
{
Start-Sleep -s 60

# keep polling the migration Status
$updatedCosmosDBAccount = Get-AzCosmosDBAccount -ResourceGroupName $rgName -Name $cosmosDBAccountName
}

Assert-AreEqual "Continuous" $updatedCosmosDBAccount.BackupPolicy.BackupType
Assert-AreEqual "Continuous7Days" $updatedCosmosDBAccount.BackupPolicy.Tier

# If we don't provide the continuoustier, it should not trigger the update to continuous30days
$updatedCosmosDBAccount = Update-AzCosmosDBAccount -ResourceGroupName $rgName -Name $cosmosDBAccountName
Start-Sleep -s (60 * 2)

$updatedCosmosDBAccount = Get-AzCosmosDBAccount -ResourceGroupName $rgName -Name $cosmosDBAccountName
Assert-AreEqual "Continuous7Days" $updatedCosmosDBAccount.BackupPolicy.Tier

# Provide continuoustier explicitly, it should triggered the update to continuous30days
$updatedCosmosDBAccount = Update-AzCosmosDBAccount -ResourceGroupName $rgName -Name $cosmosDBAccountName -BackupPolicyType Continuous -ContinuousTier Continuous30Days
Start-Sleep -s (60 * 2)

$updatedCosmosDBAccount = Get-AzCosmosDBAccount -ResourceGroupName $rgName -Name $cosmosDBAccountName
Assert-AreEqual "Continuous" $updatedCosmosDBAccount.BackupPolicy.BackupType
Assert-AreEqual "Continuous30Days" $updatedCosmosDBAccount.BackupPolicy.Tier
}

function Test-ProvisionCosmosDBAccountBackupPolicyWithContinuous7DaysCmdLets {
#use an existing account with the following information
$rgName = "PSCosmosDBResourceGroup51"
$location = "West US"
$sourceCosmosDBAccountName = "ps-cosmosdb-1251"
$consistencyLevel = "Session"
$apiKind = "Sql"
$locations = @()
$locations += New-AzCosmosDBLocationObject -Location "West Us" -FailoverPriority 0 -IsZoneRedundant 0

$resourceGroup = New-AzResourceGroup -ResourceGroupName $rgName -Location $location
New-AzCosmosDBAccount -ResourceGroupName $rgName -LocationObject $locations -Name $sourceCosmosDBAccountName -ApiKind $apiKind -DefaultConsistencyLevel $consistencyLevel -BackupPolicyType Continuous -ContinuousTier Continuous7Days

$sourceCosmosDBAccount = Get-AzCosmosDBAccount -Name $sourceCosmosDBAccountName -ResourceGroupName $rgName
Assert-AreEqual "Continuous" $sourceCosmosDBAccount.BackupPolicy.BackupType
Assert-AreEqual "Continuous7Days" $sourceCosmosDBAccount.BackupPolicy.Tier

$sourceRestorableAccount = Get-AzCosmosDBRestorableDatabaseAccount -Location $sourceCosmosDBAccount.Location -DatabaseAccountInstanceId $sourceCosmosDBAccount.InstanceId
Assert-NotNull $sourceRestorableAccount.Id
Assert-NotNull $sourceRestorableAccount.Location
Assert-NotNull $sourceRestorableAccount.DatabaseAccountInstanceId
Assert-NotNull $sourceRestorableAccount.RestorableLocations
Assert-AreEqual $sourceRestorableAccount.RestorableLocations.Count 1
Assert-AreEqual $sourceRestorableAccount.DatabaseAccountInstanceId $sourceCosmosDBAccount.InstanceId
Assert-NotNull $sourceRestorableAccount.DatabaseAccountName
Assert-NotNull $sourceRestorableAccount.CreationTime
Assert-NotNull $sourceRestorableAccount.OldestRestorableTime
}
14 changes: 14 additions & 0 deletions src/CosmosDB/CosmosDB.Test/ScenarioTests/SqlOperationsTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -71,5 +71,19 @@ public void TestClientEncryptionKeyCmdletsUsingInputObject()
{
TestRunner.RunTestScript("Test-ClientEncryptionKeyCmdletsUsingInputObject");
}

[Fact(Skip = "Cannot acquire token credential for a specific audience. No support from test framework. I have verified the tests manually.")]
[Trait(Category.AcceptanceType, Category.CheckIn)]
public void TestSqlContainerMergeCmdlet()
{
TestRunner.RunTestScript("Test-SqlContainerMergeCmdlet");
}

[Fact(Skip = "Cannot acquire token credential for a specific audience. No support from test framework. I have verified the tests manually.")]
[Trait(Category.AcceptanceType, Category.CheckIn)]
public void TestSqlContainerAdaptiveRUCmdlets()
{
TestRunner.RunTestScript("Test-SqlContainerAdaptiveRUCmdlets");
}
}
}
Loading

0 comments on commit fdecb11

Please sign in to comment.