Skip to content

Commit

Permalink
[Storage] Support ORS
Browse files Browse the repository at this point in the history
  • Loading branch information
blueww committed Aug 14, 2020
1 parent 6daf098 commit a4cf769
Show file tree
Hide file tree
Showing 18 changed files with 4,164 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -65,5 +65,12 @@ public void TestStorageBlobServiceProperties()
{
TestController.NewInstance.RunPsTest(_logger, "Test-StorageBlobServiceProperties");
}

[Fact]
[Trait(Category.AcceptanceType, Category.CheckIn)]
public void TestStorageBlobORS()
{
TestController.NewInstance.RunPsTest(_logger, "Test-StorageBlobORS");
}
}
}
104 changes: 104 additions & 0 deletions src/Storage/Storage.Management.Test/ScenarioTests/StorageBlobTests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -437,3 +437,107 @@ function Test-StorageBlobServiceProperties



<#
.SYNOPSIS
Test StorageAccount Object Replication
.DESCRIPTION
SmokeTest
#>
function Test-StorageBlobORS
{
# Setup
$rgname = Get-StorageManagementTestResourceName;

try
{
# Test
$stoname1 = 'sto' + $rgname + 'src';
$stoname2 = 'sto' + $rgname + 'dest';
$stotype = 'Standard_LRS';
$loc = Get-ProviderLocation ResourceManagement;
$kind = 'StorageV2'

Write-Verbose "RGName: $rgname | Loc: $loc"
New-AzResourceGroup -Name $rgname -Location $loc;

$loc = Get-ProviderLocation_Canary ResourceManagement;
New-AzStorageAccount -ResourceGroupName $rgname -Name $stoname1 -Location $loc -Type $stotype -Kind $kind
New-AzStorageAccount -ResourceGroupName $rgname -Name $stoname2 -Location $loc -Type $stotype -Kind $kind

# Enable Blob Enable Changefeed and versioning
Update-AzStorageBlobServiceProperty -ResourceGroupName $rgname -StorageAccountName $stoname1 -EnableChangeFeed $true -IsVersioningEnabled $true
Update-AzStorageBlobServiceProperty -ResourceGroupName $rgname -StorageAccountName $stoname2 -EnableChangeFeed $true -IsVersioningEnabled $true
$property1 = Get-AzStorageBlobServiceProperty -ResourceGroupName $rgname -StorageAccountName $stoname1
Assert-AreEqual $true $property1.ChangeFeed.Enabled
Assert-AreEqual $true $property1.IsVersioningEnabled
$property2 = Get-AzStorageBlobServiceProperty -ResourceGroupName $rgname -StorageAccountName $stoname2
Assert-AreEqual $true $property2.ChangeFeed.Enabled
Assert-AreEqual $true $property2.IsVersioningEnabled

# create containers
Get-AzStorageAccount -ResourceGroupName $rgname -StorageAccountName $stoname1 | New-AzRmStorageContainer -name src
Get-AzStorageAccount -ResourceGroupName $rgname -StorageAccountName $stoname2 | New-AzRmStorageContainer -name dest
Get-AzStorageAccount -ResourceGroupName $rgname -StorageAccountName $stoname1 | New-AzRmStorageContainer -name src1
Get-AzStorageAccount -ResourceGroupName $rgname -StorageAccountName $stoname2 | New-AzRmStorageContainer -name dest1

# create rules
$minCreationTime = "2019-01-01T16:00:00Z"
$rule1 = New-AzStorageObjectReplicationPolicyRule -SourceContainer src1 -DestinationContainer dest1
$rule2 = New-AzStorageObjectReplicationPolicyRule -SourceContainer src -DestinationContainer dest -MinCreationTime $minCreationTime -PrefixMatch a,abc,dd #-Tag t1,t2,t3

# set policy to dest account
$destPolicy = Set-AzStorageObjectReplicationPolicy -ResourceGroupName $rgname -StorageAccountName $stoname2 -PolicyId default -SourceAccount $stoname1 -Rule $rule1,$rule2
$policyID = $destPolicy.PolicyId
Assert-AreEqual $stoname1 $destPolicy.SourceAccount
Assert-AreEqual $stoname2 $destPolicy.DestinationAccount
Assert-AreEqual 2 $destPolicy.Rules.Count
Assert-AreEqual src1 $destPolicy.Rules[0].SourceContainer
Assert-AreEqual dest1 $destPolicy.Rules[0].DestinationContainer
Assert-AreEqual $null $destPolicy.Rules[0].Filters
Assert-AreEqual src $destPolicy.Rules[1].SourceContainer
Assert-AreEqual dest $destPolicy.Rules[1].DestinationContainer
Assert-AreEqual 3 $destPolicy.Rules[1].Filters.PrefixMatch.Count
Assert-AreEqual $minCreationTime ($destPolicy.Rules[1].Filters.MinCreationTime.ToUniversalTime().ToString("s")+"Z")
$destPolicy = Get-AzStorageObjectReplicationPolicy -ResourceGroupName $rgname -StorageAccountName $stoname2 -PolicyId $destPolicy.PolicyId
Assert-AreEqual $policyID $destPolicy.PolicyId
Assert-AreEqual $stoname1 $destPolicy.SourceAccount
Assert-AreEqual $stoname2 $destPolicy.DestinationAccount
Assert-AreEqual 2 $destPolicy.Rules.Count
Assert-AreEqual src1 $destPolicy.Rules[0].SourceContainer
Assert-AreEqual dest1 $destPolicy.Rules[0].DestinationContainer
Assert-AreEqual $null $destPolicy.Rules[0].Filters
Assert-AreEqual src $destPolicy.Rules[1].SourceContainer
Assert-AreEqual dest $destPolicy.Rules[1].DestinationContainer
Assert-AreEqual 3 $destPolicy.Rules[1].Filters.PrefixMatch.Count
Assert-AreEqual $minCreationTime ($destPolicy.Rules[1].Filters.MinCreationTime.ToUniversalTime().ToString("s")+"Z")

#Set policy to source account
Set-AzStorageObjectReplicationPolicy -ResourceGroupName $rgname -StorageAccountName $stoname1 -InputObject $destPolicy
$srcPolicy = Get-AzStorageObjectReplicationPolicy -ResourceGroupName $rgname -StorageAccountName $stoname1
Assert-AreEqual $policyID $srcPolicy.PolicyId
Assert-AreEqual $stoname1 $srcPolicy.SourceAccount
Assert-AreEqual $stoname2 $srcPolicy.DestinationAccount
Assert-AreEqual 2 $srcPolicy.Rules.Count
Assert-AreEqual src1 $srcPolicy.Rules[0].SourceContainer
Assert-AreEqual dest1 $srcPolicy.Rules[0].DestinationContainer
Assert-AreEqual $null $srcPolicy.Rules[0].Filters
Assert-AreEqual src $srcPolicy.Rules[1].SourceContainer
Assert-AreEqual dest $srcPolicy.Rules[1].DestinationContainer
Assert-AreEqual 3 $srcPolicy.Rules[1].Filters.PrefixMatch.Count
Assert-AreEqual $minCreationTime ($srcPolicy.Rules[1].Filters.MinCreationTime.ToUniversalTime().ToString("s")+"Z")

#remove policies
Remove-AzStorageObjectReplicationPolicy -ResourceGroupName $rgname -StorageAccountName $stoname2 -PolicyId $destPolicy.PolicyId
Remove-AzStorageObjectReplicationPolicy -ResourceGroupName $rgname -StorageAccountName $stoname1 -PolicyId $srcPolicy.PolicyId

Remove-AzStorageAccount -Force -ResourceGroupName $rgname -Name $stoname1;
Remove-AzStorageAccount -Force -ResourceGroupName $rgname -Name $stoname2;
}
finally
{
# Cleanup
Clean-ResourceGroup $rgname
}
}


Large diffs are not rendered by default.

4 changes: 3 additions & 1 deletion src/Storage/Storage.Management/Az.Storage.psd1
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,9 @@ CmdletsToExport = 'Get-AzStorageAccount', 'Get-AzStorageAccountKey',
'Remove-AzDataLakeGen2Item', 'Update-AzDataLakeGen2Item',
'Set-AzDataLakeGen2ItemAclObject', 'Get-AzDataLakeGen2ItemContent',
'Invoke-AzStorageAccountFailover',
'Get-AzStorageBlobQueryResult', 'New-AzStorageBlobQueryConfig'
'Get-AzStorageBlobQueryResult', 'New-AzStorageBlobQueryConfig',
'New-AzStorageObjectReplicationPolicyRule', 'Set-AzStorageObjectReplicationPolicy',
'Get-AzStorageObjectReplicationPolicy', 'Remove-AzStorageObjectReplicationPolicy'

# Variables to export from this module
# VariablesToExport = @()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,23 @@ public class UpdateAzStorageBlobServicePropertyCommand : StorageBlobBaseCmdlet
[ValidateNotNull]
public string DefaultServiceVersion { get; set; }

[Parameter(
Mandatory = false,
HelpMessage = "Enable Change Feed logging for the storage account by set to $true, disable Change Feed logging by set to $false.")]
[ValidateNotNullOrEmpty]
public bool EnableChangeFeed
{
get
{
return enableChangeFeed is null ? false : enableChangeFeed.Value;
}
set
{
enableChangeFeed = value;
}
}
private bool? enableChangeFeed = null;

[Parameter(
Mandatory = false,
HelpMessage = "Gets or sets versioning is enabled if set to true.")]
Expand Down Expand Up @@ -128,6 +145,11 @@ public override void ExecuteCmdlet()
{
serviceProperties.DefaultServiceVersion = this.DefaultServiceVersion;
}
if (enableChangeFeed != null)
{
serviceProperties.ChangeFeed = new ChangeFeed();
serviceProperties.ChangeFeed.Enabled = enableChangeFeed;
}
if (isVersioningEnabled != null)
{
serviceProperties.IsVersioningEnabled = isVersioningEnabled;
Expand Down
9 changes: 8 additions & 1 deletion src/Storage/Storage.Management/ChangeLog.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,14 @@
- `Start-AzStorageBlobCopy`
- `Get-AzDataLakeGen2Item`
* Fixed [#12592]: Fix download blob fail when related sub directory not exist.
- `Get-AzStorageBlobContent`
- `Get-AzStorageBlobContent`
* Support Set/Get/Remove Object Replication Policy on Storage accounts
- `New-AzStorageObjectReplicationPolicyRule`
- `Set-AzStorageObjectReplicationPolicy`
- `Get-AzStorageObjectReplicationPolicy`
- `Remove-AzStorageObjectReplicationPolicy`
* Support enable/disable Changefeed on Blob Service of a Storage account
- `Update-AzStorageBlobServiceProperty`

## Version 2.4.0
* Supported create container/blob Sas token with new permission x,t
Expand Down
195 changes: 195 additions & 0 deletions src/Storage/Storage.Management/Models/PSObjectReplicationPolicy.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
// ----------------------------------------------------------------------------------
//
// Copyright Microsoft Corporation
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------------------------------------------------------------------------

using Microsoft.Azure.Management.Storage.Models;
using Microsoft.WindowsAzure.Commands.Common.Attributes;
using System;
using System.Collections.Generic;

namespace Microsoft.Azure.Commands.Management.Storage.Models
{
/// <summary>
/// Wrapper of SDK type ObjectReplicationPolicy
/// </summary>
public class PSObjectReplicationPolicy
{
[Ps1Xml(Label = "ResourceGroupName", Target = ViewControl.Table, Position = 0)]
public string ResourceGroupName { get; }
[Ps1Xml(Label = "StorageAccountName", Target = ViewControl.Table, Position = 1)]
public string StorageAccountName { get; }
public string ResourceId { get; }
public string Name { get; }
public string Type { get; }

[Ps1Xml(Label = "PolicyId", Target = ViewControl.Table, Position = 2)]
public string PolicyId { get; set; }
[Ps1Xml(Label = "EnabledTime", Target = ViewControl.Table, Position = 3)]
public DateTime? EnabledTime { get; }
[Ps1Xml(Label = "SourceAccount", Target = ViewControl.Table, Position = 4)]
public string SourceAccount { get; set; }
[Ps1Xml(Label = "DestinationAccount", Target = ViewControl.Table, Position = 5)]
public string DestinationAccount { get; set; }
[Ps1Xml(Label = "Rules", Target = ViewControl.Table, ScriptBlock = "if (($_.Rules -ne $null) -and ($_.Rules.Count -ne 0)) {'[' + $_.Rules[0].RuleId + ',...]'} else {$null}", Position = 6)]
public PSObjectReplicationPolicyRule[] Rules { get; set; }

public PSObjectReplicationPolicy()
{ }

public PSObjectReplicationPolicy(ObjectReplicationPolicy policy, string ResourceGroupName, string StorageAccountName)
{
this.ResourceGroupName = ResourceGroupName;
this.StorageAccountName = StorageAccountName;
this.ResourceId = policy.Id;
this.Name = policy.Name;
this.Type = policy.Type;
this.PolicyId = policy.PolicyId;
this.EnabledTime = policy.EnabledTime;
this.SourceAccount = policy.SourceAccount;
this.DestinationAccount = policy.DestinationAccount;
this.Rules = PSObjectReplicationPolicyRule.GetPSObjectReplicationPolicyRules(policy.Rules);
}

public ObjectReplicationPolicy ParseObjectReplicationPolicy()
{
ObjectReplicationPolicy policy = new ObjectReplicationPolicy()
{
SourceAccount = this.SourceAccount,
DestinationAccount = this.DestinationAccount,
Rules = PSObjectReplicationPolicyRule.ParseObjectReplicationPolicyRules(this.Rules)
};
return policy;
}

public static PSObjectReplicationPolicy[] GetPSObjectReplicationPolicies(IEnumerable<ObjectReplicationPolicy> policies, string ResourceGroupName, string StorageAccountName)
{
if (policies == null)
{
return null;
}
List<PSObjectReplicationPolicy> pspolicies = new List<PSObjectReplicationPolicy>();
foreach (ObjectReplicationPolicy policy in policies)
{
pspolicies.Add(new PSObjectReplicationPolicy(policy, ResourceGroupName, StorageAccountName));
}
return pspolicies.ToArray();
}
}

/// <summary>
/// Wrapper of SDK type ObjectReplicationPolicyRule
/// </summary>
public class PSObjectReplicationPolicyRule
{
[Ps1Xml(Label = "RuleId", Target = ViewControl.Table, Position = 0)]
public string RuleId { get; set; }
[Ps1Xml(Label = "SourceContainer", Target = ViewControl.Table, Position = 1)]
public string SourceContainer { get; set; }
[Ps1Xml(Label = "DestinationContainer", Target = ViewControl.Table, Position = 2)]
public string DestinationContainer { get; set; }
[Ps1Xml(Label = "Filter.PrefixMatch", Target = ViewControl.Table, ScriptBlock = "if (($_.Filter -ne $null) -and ($_.Filter.PrefixMatch -ne $null) -and ($_.Filter.PrefixMatch.Count -ne 0)) {'[' + ($_.Filter.PrefixMatch -join ', ') + ']'} else {$null}", Position = 3)]
public PSObjectReplicationPolicyFilter Filters { get; set; }

public PSObjectReplicationPolicyRule()
{
}

public PSObjectReplicationPolicyRule(ObjectReplicationPolicyRule rule)
{
this.RuleId = rule.RuleId;
this.SourceContainer = rule.SourceContainer;
this.DestinationContainer = rule.DestinationContainer;
this.Filters = rule.Filters is null ? null : new PSObjectReplicationPolicyFilter(rule.Filters);
}

public ObjectReplicationPolicyRule ParseObjectReplicationPolicyRule()
{
ObjectReplicationPolicyRule rule = new ObjectReplicationPolicyRule();
rule.RuleId = this.RuleId;
rule.SourceContainer = this.SourceContainer;
rule.DestinationContainer = this.DestinationContainer;
rule.Filters = this.Filters is null ? null : this.Filters.ParseObjectReplicationPolicyFilter();
return rule;
}

public static PSObjectReplicationPolicyRule[] GetPSObjectReplicationPolicyRules(IList<ObjectReplicationPolicyRule> rules)
{
if (rules == null)
{
return null;
}
List<PSObjectReplicationPolicyRule> psrules = new List<PSObjectReplicationPolicyRule>();
foreach (ObjectReplicationPolicyRule rule in rules)
{
psrules.Add(new PSObjectReplicationPolicyRule(rule));
}
return psrules.ToArray();
}

public static List<ObjectReplicationPolicyRule> ParseObjectReplicationPolicyRules(PSObjectReplicationPolicyRule[] psrules)
{
if (psrules == null)
{
return null;
}
List<ObjectReplicationPolicyRule> rules = new List<ObjectReplicationPolicyRule>();
foreach (PSObjectReplicationPolicyRule psrule in psrules)
{
rules.Add(psrule.ParseObjectReplicationPolicyRule());
}
return rules;
}
}

/// <summary>
/// Wrapper of SDK type ObjectReplicationPolicyFilter
/// </summary>
public class PSObjectReplicationPolicyFilter
{
public string[] PrefixMatch { get; set; }
public DateTime? MinCreationTime;

public PSObjectReplicationPolicyFilter()
{
}

public PSObjectReplicationPolicyFilter(ObjectReplicationPolicyFilter filter)
{
if (filter != null)
{
this.PrefixMatch = filter.PrefixMatch is null ? null : new List<string>(filter.PrefixMatch).ToArray();
if (string.IsNullOrEmpty(filter.MinCreationTime))
{
this.MinCreationTime = null;
}
else
{
if (filter.MinCreationTime.ToUpper()[filter.MinCreationTime.Length - 1] != 'Z')
{
filter.MinCreationTime = filter.MinCreationTime + "Z";
}
this.MinCreationTime = Convert.ToDateTime(filter.MinCreationTime);
}
}
}
public ObjectReplicationPolicyFilter ParseObjectReplicationPolicyFilter()
{
return new ObjectReplicationPolicyFilter()
{
PrefixMatch = this.PrefixMatch is null ? null : new List<string>(this.PrefixMatch),
//must be in format: 2020-02-19T16:05:00Z
MinCreationTime = this.MinCreationTime is null ? null : this.MinCreationTime.Value.ToUniversalTime().ToString("s") + "Z"
};
}
}
}
Loading

0 comments on commit a4cf769

Please sign in to comment.