resolved merge conflicts for test recordings / versioning

Azure · Apr 18, 2024 · a95f685 · a95f685
2 parents 68ca894 + a0b17e3
commit a95f685
Show file tree

Hide file tree

Showing 714 changed files with 17,265 additions and 13,629 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -64,6 +64,8 @@
 /sdk/eventhub/                                                       @annatisch @kashifkhan @swathipil @l0lawrence
 
 # ServiceLabel: %Storage
+# ServiceOwners: @jalauzon-msft @vincenttran-msft
+
 # PRLabel: %Storage
 /sdk/storage/                                                        @annatisch @jalauzon-msft @vincenttran-msft @weirongw23-msft
 
@@ -873,9 +875,6 @@
 # ServiceLabel: %Service Attention %SQL - Replication & Failover
 #/<NotInRepo>/          @azureSQLGitHub
 
-# ServiceLabel: %Service Attention %Storage
-#/<NotInRepo>/          @xgithubtriage
-
 # ServiceLabel: %Service Attention %Storsimple
 #/<NotInRepo>/          @anoobbacker @ganzee @manuaery @patelkunal
 

diff --git a/eng/common/pipelines/templates/steps/detect-api-changes.yml b/eng/common/pipelines/templates/steps/detect-api-changes.yml
@@ -22,6 +22,7 @@ steps:
         -RepoFullName $(Build.Repository.Name)
         -APIViewUri $(ApiChangeDetectRequestUrl)
         -ArtifactName ${{ parameters.ArtifactName }}
+        -DevopsProject $(System.TeamProject)
       pwsh: true
     displayName: Detect API changes
     condition: and(succeededOrFailed(), eq(variables['Build.Reason'],'PullRequest'))
diff --git a/eng/common/pipelines/templates/steps/policheck.yml b/eng/common/pipelines/templates/steps/policheck.yml
@@ -2,14 +2,20 @@ parameters:
   ExclusionDataBaseFileName: ''
   TargetDirectory: ''
   PublishAnalysisLogs: false
-  PoliCheckBlobSAS: "$(azuresdk-policheck-blob-SAS)"
   ExclusionFilePath: "$(Build.SourcesDirectory)/eng/guardian-tools/policheck/PolicheckExclusions.xml"
 
 steps:
-  - pwsh: |
-      azcopy copy "https://azuresdkartifacts.blob.core.windows.net/policheck/${{ parameters.ExclusionDataBaseFileName }}.mdb?${{ parameters.PoliCheckBlobSAS }}" `
-      "$(Build.BinariesDirectory)"
-    displayName: 'Download PoliCheck Exclusion Database'
+ - task: AzurePowerShell@5
+    displayName: 'Download Policheck Exclusion Database'
+    inputs:
+      azureSubscription: 'Azure SDK Artifacts'
+      ScriptType: 'InlineScript'
+      azurePowerShellVersion: LatestVersion 
+      pwsh: true
+      Inline: |
+        azcopy copy "https://azuresdkartifacts.blob.core.windows.net/policheck/${{ parameters.ExclusionDataBaseFileName }}.mdb" "$(Build.BinariesDirectory)"
+    env: 
+      AZCOPY_AUTO_LOGIN_TYPE: 'PSCRED'
 
   - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@2
     displayName: 'Run PoliCheck'
@@ -33,4 +39,4 @@ steps:
 
   - ${{ if eq(parameters.PublishAnalysisLogs, 'true') }}:
     - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3
-      displayName: 'Publish Security Analysis Logs'
+      displayName: 'Publish Security Analysis Logs'
diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml
@@ -30,7 +30,34 @@ steps:
         # 7.2 behavior for command argument passing. Newer behaviors will result
         # in errors from git.exe.
         $PSNativeCommandArgumentPassing = 'Legacy'
-        
+
+        function Retry()
+        {
+            Run 3 @args
+        }
+
+        function Run()
+        {
+            $retries, $command, $arguments = $args
+            if ($retries -isnot [int]) {
+                $command, $arguments = $args
+                $retries = 0
+            }
+            Write-Host "==>" $command $arguments
+            $attempt = 0
+            $sleep = 5
+
+            while ($true) {
+                $attempt++
+                & $command $arguments
+                if (!$LASTEXITCODE) { return }
+                if ($attempt -gt $retries) { exit $LASTEXITCODE }
+                Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..."
+                Start-Sleep -Seconds $sleep
+                $sleep *= 2
+            }
+        }
+
         function SparseCheckout([Array]$paths, [Hashtable]$repository)
         {
             $dir = $repository.WorkingDirectory
@@ -47,23 +74,18 @@ steps:
               Write-Host "Repository $($repository.Name) is being initialized."
 
               if ($repository.Commitish -match '^refs/pull/\d+/merge$') {
-                Write-Host "git clone --no-checkout --filter=tree:0 -c remote.origin.fetch='+$($repository.Commitish):refs/remotes/origin/$($repository.Commitish)' https://github.com/$($repository.Name) ."
-                git clone --no-checkout --filter=tree:0 -c remote.origin.fetch=''+$($repository.Commitish):refs/remotes/origin/$($repository.Commitish)'' https://github.com/$($repository.Name) .
+                Retry git clone --no-checkout --filter=tree:0 -c remote.origin.fetch=''+$($repository.Commitish):refs/remotes/origin/$($repository.Commitish)'' https://github.com/$($repository.Name) .
               } else {
-                Write-Host "git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) ."
-                git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) .
+                Retry git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) .
               }
 
               # Turn off git GC for sparse checkout. Note: The devops checkout task does this by default
-              Write-Host "git config gc.auto 0"
-              git config gc.auto 0
+              Run git config gc.auto 0
 
-              Write-Host "git sparse-checkout init"
-              git sparse-checkout init
+              Run git sparse-checkout init
 
               # Set non-cone mode otherwise path filters will not work in git >= 2.37.0
               # See https://github.blog/2022-06-27-highlights-from-git-2-37/#tidbits
-              Write-Host "git sparse-checkout set --no-cone '/*' '!/*/' '/eng'"
               git sparse-checkout set --no-cone '/*' '!/*/' '/eng'
             }
 
@@ -82,10 +104,8 @@ steps:
               $commitish = $repository.Commitish -replace '^refs/heads/', ''
 
               # use -- to prevent git from interpreting the commitish as a path
-              Write-Host "git -c advice.detachedHead=false checkout $commitish --"
-
               # This will use the default branch if repo.Commitish is empty
-              git -c advice.detachedHead=false checkout $commitish --
+              Retry git -c advice.detachedHead=false checkout $commitish --
             } else {
               Write-Host "Skipping checkout as repo has already been initialized"
             }

diff --git a/eng/common/scripts/ChangeLog-Operations.ps1 b/eng/common/scripts/ChangeLog-Operations.ps1
@@ -139,27 +139,23 @@ function Confirm-ChangeLogEntry {
     [String]$VersionString,
     [boolean]$ForRelease = $false,
     [Switch]$SantizeEntry,
-    [PSCustomObject]$ChangeLogStatus = $null
+    [PSCustomObject]$ChangeLogStatus = $null,
+    [boolean]$SuppressErrors = $false
   )
 
-  $suppressErrors = $false
   if (!$ChangeLogStatus) {
     $ChangeLogStatus = [PSCustomObject]@{
       IsValid = $false
       Message = ""
     }
   }
-  else {
-    # Do not stop the script on error when status object is passed as param
-    $suppressErrors = $true
-  }
   $changeLogEntries = Get-ChangeLogEntries -ChangeLogLocation $ChangeLogLocation
   $changeLogEntry = $changeLogEntries[$VersionString]
 
   if (!$changeLogEntry) {
     $ChangeLogStatus.Message = "ChangeLog[${ChangeLogLocation}] does not have an entry for version ${VersionString}."
     $ChangeLogStatus.IsValid = $false
-    if (!$suppressErrors) {
+    if (!$SuppressErrors) {
       LogError "$($ChangeLogStatus.Message)"
     }
     return $false
@@ -179,7 +175,7 @@ function Confirm-ChangeLogEntry {
   if ([System.String]::IsNullOrEmpty($changeLogEntry.ReleaseStatus)) {
     $ChangeLogStatus.Message = "Entry does not have a release status. Please ensure the status is set to a date '($CHANGELOG_DATE_FORMAT)' or '$CHANGELOG_UNRELEASED_STATUS' if not yet released. See https://aka.ms/azsdk/guideline/changelogs for more info."
     $ChangeLogStatus.IsValid = $false
-    if (!$suppressErrors) {
+    if (!$SuppressErrors) {
       LogError "$($ChangeLogStatus.Message)"
     }
     return $false
@@ -188,15 +184,15 @@ function Confirm-ChangeLogEntry {
   if ($ForRelease -eq $True)
   {
     LogDebug "Verifying as a release build because ForRelease parameter is set to true"
-    return Confirm-ChangeLogForRelease -changeLogEntry $changeLogEntry -changeLogEntries $changeLogEntries -ChangeLogStatus $ChangeLogStatus
+    return Confirm-ChangeLogForRelease -changeLogEntry $changeLogEntry -changeLogEntries $changeLogEntries -ChangeLogStatus $ChangeLogStatus -SuppressErrors $SuppressErrors
   }
 
   # If the release status is a valid date then verify like its about to be released
   $status = $changeLogEntry.ReleaseStatus.Trim().Trim("()")
   if ($status -as [DateTime])
   {
     LogDebug "Verifying as a release build because the changelog entry has a valid date."
-    return Confirm-ChangeLogForRelease -changeLogEntry $changeLogEntry -changeLogEntries $changeLogEntries -ChangeLogStatus $ChangeLogStatus
+    return Confirm-ChangeLogForRelease -changeLogEntry $changeLogEntry -changeLogEntries $changeLogEntries -ChangeLogStatus $ChangeLogStatus -SuppressErrors $SuppressErrors
   }
 
   $ChangeLogStatus.Message = "ChangeLog[${ChangeLogLocation}] has an entry for version ${VersionString}."
@@ -361,26 +357,24 @@ function Confirm-ChangeLogForRelease {
     $changeLogEntry,
     [Parameter(Mandatory = $true)]
     $changeLogEntries,
-    $ChangeLogStatus = $null
+    $ChangeLogStatus = $null,
+    $SuppressErrors = $false
   )
 
-  $suppressErrors = $false
   if (!$ChangeLogStatus) {
     $ChangeLogStatus = [PSCustomObject]@{
       IsValid = $false
       Message = ""
     }
   }
-  else {
-    $suppressErrors = $true
-  }
+
   $entries = Sort-ChangeLogEntries -changeLogEntries $changeLogEntries
 
   $ChangeLogStatus.IsValid = $true
   if ($changeLogEntry.ReleaseStatus -eq $CHANGELOG_UNRELEASED_STATUS) {
     $ChangeLogStatus.Message = "Entry has no release date set. Please ensure to set a release date with format '$CHANGELOG_DATE_FORMAT'. See https://aka.ms/azsdk/guideline/changelogs for more info."
     $ChangeLogStatus.IsValid = $false
-    if (!$suppressErrors) {
+    if (!$SuppressErrors) {
       LogError "$($ChangeLogStatus.Message)"
     }
   }
@@ -392,7 +386,7 @@ function Confirm-ChangeLogForRelease {
       {
         $ChangeLogStatus.Message = "Date must be in the format $($CHANGELOG_DATE_FORMAT). See https://aka.ms/azsdk/guideline/changelogs for more info."
         $ChangeLogStatus.IsValid = $false
-        if (!$suppressErrors) {
+        if (!$SuppressErrors) {
           LogError "$($ChangeLogStatus.Message)"
         }
       }
@@ -401,15 +395,15 @@ function Confirm-ChangeLogForRelease {
       {
         $ChangeLogStatus.Message = "Invalid date [ $status ]. The date for the changelog being released must be the latest in the file."
         $ChangeLogStatus.IsValid = $false
-        if (!$suppressErrors) {
+        if (!$SuppressErrors) {
           LogError "$($ChangeLogStatus.Message)"
         }
       }
     }
     catch {
         $ChangeLogStatus.Message = "Invalid date [ $status ] passed as status for Version [$($changeLogEntry.ReleaseVersion)]. See https://aka.ms/azsdk/guideline/changelogs for more info."
         $ChangeLogStatus.IsValid = $false
-        if (!$suppressErrors) {
+        if (!$SuppressErrors) {
           LogError "$($ChangeLogStatus.Message)"
         }
     }
@@ -418,7 +412,7 @@ function Confirm-ChangeLogForRelease {
   if ([System.String]::IsNullOrWhiteSpace($changeLogEntry.ReleaseContent)) {
     $ChangeLogStatus.Message = "Entry has no content. Please ensure to provide some content of what changed in this version. See https://aka.ms/azsdk/guideline/changelogs for more info."
     $ChangeLogStatus.IsValid = $false
-    if (!$suppressErrors) {
+    if (!$SuppressErrors) {
       LogError "$($ChangeLogStatus.Message)"
     }
   }
@@ -441,14 +435,14 @@ function Confirm-ChangeLogForRelease {
   {
     $ChangeLogStatus.Message = "The changelog entry has the following sections with no content ($($emptySections -join ', ')). Please ensure to either remove the empty sections or add content to the section."
     $ChangeLogStatus.IsValid = $false
-    if (!$suppressErrors) {
+    if (!$SuppressErrors) {
       LogError "$($ChangeLogStatus.Message)"
     }
   }
   if (!$foundRecommendedSection)
   {
     $ChangeLogStatus.Message = "The changelog entry did not contain any of the recommended sections ($($RecommendedSectionHeaders -join ', ')), please add at least one. See https://aka.ms/azsdk/guideline/changelogs for more info."
-    if (!$suppressErrors) {
+    if (!$SuppressErrors) {
       LogError "$($ChangeLogStatus.Message)"
     }
   }

diff --git a/eng/common/scripts/Create-APIReview.ps1 b/eng/common/scripts/Create-APIReview.ps1
@@ -78,6 +78,7 @@ function Upload-SourceArtifact($filePath, $apiLabel, $releaseStatus, $packageVer
     try
     {
         $Response = Invoke-WebRequest -Method 'POST' -Uri $uri -Body $multipartContent -Headers $headers
+        Write-Host "API review: $($Response.Content)"
         $StatusCode = $Response.StatusCode
     }
     catch
@@ -114,6 +115,7 @@ function Upload-ReviewTokenFile($packageName, $apiLabel, $releaseStatus, $review
     try
     {
         $Response = Invoke-WebRequest -Method 'GET' -Uri $uri -Headers $headers
+        Write-Host "API review: $($Response.Content)"
         $StatusCode = $Response.StatusCode
     }
     catch

diff --git a/eng/common/scripts/Detect-Api-Changes.ps1 b/eng/common/scripts/Detect-Api-Changes.ps1
@@ -15,7 +15,8 @@ Param (
   [string] $APIViewUri,
   [string] $RepoFullName = "",
   [string] $ArtifactName = "packages",
-  [string] $TargetBranch = ("origin/${env:SYSTEM_PULLREQUEST_TARGETBRANCH}" -replace "refs/heads/")
+  [string] $TargetBranch = ("origin/${env:SYSTEM_PULLREQUEST_TARGETBRANCH}" -replace "refs/heads/"),
+  [string] $DevopsProject = "internal"
 )
 
 . (Join-Path $PSScriptRoot common.ps1)
@@ -37,6 +38,7 @@ function Submit-Request($filePath, $packageName)
     $query.Add('pullRequestNumber', $PullRequestNumber)
     $query.Add('packageName', $packageName)
     $query.Add('language', $LanguageShort)
+    $query.Add('project', $DevopsProject)
     $reviewFileFullName = Join-Path -Path $ArtifactPath $packageName $reviewFileName
     if (Test-Path $reviewFileFullName)
     {
@@ -87,6 +89,7 @@ function Log-Input-Params()
     Write-Host "Language: $($Language)"
     Write-Host "Commit SHA: $($CommitSha)"
     Write-Host "Repo Name: $($RepoFullName)"
+    Write-Host "Project: $($DevopsProject)"
 }
 
 Log-Input-Params

diff --git a/eng/common/scripts/Validate-Package.ps1 b/eng/common/scripts/Validate-Package.ps1
@@ -60,7 +60,7 @@ function ValidateChangeLog($changeLogPath, $versionString, $validationStatus)
         Write-Host "Path to change log: [$changeLogFullPath]"        
         if (Test-Path $changeLogFullPath)
         {
-            Confirm-ChangeLogEntry -ChangeLogLocation $changeLogFullPath -VersionString $versionString -ForRelease $true -ChangeLogStatus $ChangeLogStatus
+            Confirm-ChangeLogEntry -ChangeLogLocation $changeLogFullPath -VersionString $versionString -ForRelease $true -ChangeLogStatus $ChangeLogStatus -SuppressErrors $true
             $validationStatus.Status = if ($ChangeLogStatus.IsValid) { "Success" } else { "Failed" }
             $validationStatus.Message = $ChangeLogStatus.Message 
         }

diff --git a/eng/common/testproxy/target_version.txt b/eng/common/testproxy/target_version.txt
@@ -1 +1 @@
-1.0.0-dev.20240314.1
+1.0.0-dev.20240410.1
diff --git a/eng/pipelines/aggregate-reports.yml b/eng/pipelines/aggregate-reports.yml
@@ -28,7 +28,7 @@ stages:
             parameters:
               ScanPath: $(Build.SourcesDirectory)/sdk
 
-          - task: AzureFileCopy@4
+          - task: AzureFileCopy@6
             displayName: 'Upload Dependency Report'
             condition: and(succeededOrFailed(), eq(variables['System.TeamProject'], 'internal'))
             inputs:
@@ -40,7 +40,7 @@ stages:
               blobPrefix: dependencies
               AdditionalArgumentsForBlobCopy: '--exclude-pattern=*data.js*'
 
-          - task: AzureFileCopy@4
+          - task: AzureFileCopy@6
             displayName: 'Upload Dependency Graph'
             condition: and(succeededOrFailed(), eq(variables['System.TeamProject'], 'internal'))
             inputs:

diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml
@@ -30,6 +30,10 @@ extends:
   ${{ else }}:
     template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates
   parameters:
+    ${{ if eq(parameters.oneESTemplateTag, 'canary') }}:
+      # Enable 1es template team to verify validation has been run on canary
+      customBuildTags:
+        - 1ES.PT.Tag-refs/tags/canary
     settings:
       skipBuildTagsForGitHubPullRequests: true
     sdl:

diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml
@@ -152,6 +152,15 @@ stages:
                           echo "Uploaded sdist to devops feed"
                         displayName: 'Publish package to feed: ${{parameters.DevFeedName}}'
 
+                      - template: /eng/common/pipelines/templates/steps/create-apireview.yml
+                        parameters:
+                          ArtifactPath: $(Pipeline.Workspace)/${{parameters.ArtifactName}}
+                          Artifacts: ${{parameters.Artifacts}}
+                          ConfigFileDir: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo
+                          MarkPackageAsShipped: true
+                          ArtifactName: ${{parameters.ArtifactName}}
+                          PackageName: ${{artifact.name}}
+
           - ${{if ne(artifact.skipPublishDocs, 'true')}}:
             - deployment: PublishGitHubIODocs
               displayName: Publish Docs to GitHubIO Blob Storage

diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py
@@ -45,6 +45,7 @@
     "six": "1.12.0",
     "cryptography": "3.3.2",
     "msal": "1.23.0",
+    "azure-storage-file-datalake": "12.2.0",
 }
 
 MAXIMUM_VERSION_GENERIC_OVERRIDES = {}

diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml
@@ -79,12 +79,8 @@ jobs:
         # import env variable
         export AZURE_TEST_RUN_LIVE=$(AZURE_TEST_RUN_LIVE)
         export TENANT_ID=$(ENV-TENANT-ID)
-        export CLIENT_ID=$(ENV-CLIENT-ID)
-        export CLIENT_SECRET=$(ENV-CLIENT-SECRET)
         export SUBSCRIPTION_ID=$(ENV-SUBSCRIPTION-ID)
         export AZURE_TENANT_ID=$(ENV-TENANT-ID)
-        export AZURE_CLIENT_ID=$(ENV-CLIENT-ID)
-        export AZURE_CLIENT_SECRET=$(ENV-CLIENT-SECRET)
         export AZURE_SUBSCRIPTION_ID=$(ENV-SUBSCRIPTION-ID)
         export ISSUE_LINK=$(ISSUE_LINK)
         export SCRIPT_PATH=$script_path