diff --git a/eng/common/pipelines/templates/steps/create-pull-request.yml b/eng/common/pipelines/templates/steps/create-pull-request.yml index 3ccd4c956d6..429edfe4ce5 100644 --- a/eng/common/pipelines/templates/steps/create-pull-request.yml +++ b/eng/common/pipelines/templates/steps/create-pull-request.yml @@ -74,6 +74,7 @@ steps: - task: PowerShell@2 displayName: Tag a Reviewer on PR condition: and(succeeded(), eq(variables['HasChanges'], 'true')) + continueOnError: true inputs: pwsh: true workingDirectory: ${{ parameters.WorkingDirectory }} diff --git a/eng/common/pipelines/templates/steps/get-pr-owners.yml b/eng/common/pipelines/templates/steps/get-pr-owners.yml index a80d5b83b2d..e3739602b22 100644 --- a/eng/common/pipelines/templates/steps/get-pr-owners.yml +++ b/eng/common/pipelines/templates/steps/get-pr-owners.yml @@ -18,12 +18,20 @@ steps: --kusto-database-var KUSTO_DB ` --kusto-table-var KUSTO_TABLE ` --identity "$(Build.QueuedBy)" - $resolvedIdentity = $result[-1] | ConvertFrom-Json - Write-Host $resolvedIdentity + $resolvedIdentity = "" + try { $resolvedIdentity = $result[-1] | ConvertFrom-Json } catch {} - Write-Output "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$($resolvedIdentity.GithubUserName)" + if($resolvedIdentity) { + Write-Host $resolvedIdentity + + Write-Host "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$($resolvedIdentity.GithubUserName)" + } + else { + Write-Host "Unable to locate a github user for identity $(Build.QueuedBy)" + } displayName: 'Resolving Queuing User' + continueOnError: true workingDirectory: $(Build.SourcesDirectory)/tools_repo/tools/notification-configuration/identity-resolver env: APP_ID: $(notification-aad-app-id) @@ -41,6 +49,6 @@ steps: $originalValue = "$(${{ parameters.TargetVariable }})" $result = $(Build.SourcesDirectory)/eng/common/scripts/get-codeowners.ps1 -TargetDirectory /sdk/${{ parameters.ServiceDirectory }}/ -RootDirectory $(Build.SourcesDirectory) if ($result) { - Write-Output "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$originalValue,$result" + Write-Host "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$originalValue,$result" } displayName: Add CodeOwners if Present \ No newline at end of file diff --git a/eng/common/pipelines/templates/steps/publish-blobs.yml b/eng/common/pipelines/templates/steps/publish-blobs.yml index 682cc4d4f7c..72f5fef1ab0 100644 --- a/eng/common/pipelines/templates/steps/publish-blobs.yml +++ b/eng/common/pipelines/templates/steps/publish-blobs.yml @@ -4,6 +4,8 @@ parameters: TargetLanguage: '' BlobName: '' ScriptPath: '' + RepoId: '' + ReleaseTag: '' steps: - pwsh: | @@ -21,6 +23,8 @@ steps: -SASKey "${{ parameters.BlobSASKey }}" -Language "${{ parameters.TargetLanguage }}" -BlobName "${{ parameters.BlobName }}" + -RepoReplaceRegex "(https://github.com/${{ parameters.RepoId }}/(?:blob|tree)/)master(/.*)" + -Tag: "${{ parameters.ReleaseTag }}" pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Copy Docs to Blob diff --git a/eng/common/scripts/add-pullrequest-reviewers.ps1 b/eng/common/scripts/add-pullrequest-reviewers.ps1 index a80d79485f3..3198dcb40d2 100644 --- a/eng/common/scripts/add-pullrequest-reviewers.ps1 +++ b/eng/common/scripts/add-pullrequest-reviewers.ps1 @@ -18,6 +18,32 @@ param( $AuthToken ) +function AddMembers($memberName, $additionSet) { + $headers = @{ + Authorization = "bearer $AuthToken" + } + $uri = "https://api.github.com/repos/$RepoOwner/$RepoName/pulls/$PRNumber/requested_reviewers" + $errorOccurred = $false + + foreach ($id in $additionSet) { + try { + $postResp = @{} + $postResp[$memberName] = @($id) + $postResp = $postResp | ConvertTo-Json + + Write-Host $postResp + $resp = Invoke-RestMethod -Method Post -Headers $headers -Body $postResp -Uri $uri -MaximumRetryCount 3 + $resp | Write-Verbose + } + catch { + Write-Host "Error attempting to add $user `n$_" + $errorOccurred = $true + } + } + + return $errorOccurred +} + # at least one of these needs to be populated if (-not $GitHubUsers -and -not $GitHubTeams) { Write-Host "No user provided for addition, exiting." @@ -27,54 +53,9 @@ if (-not $GitHubUsers -and -not $GitHubTeams) { $userAdditions = @($GitHubUsers.Split(",") | % { $_.Trim() } | ? { return $_ }) $teamAdditions = @($GitHubTeams.Split(",") | % { $_.Trim() } | ? { return $_ }) -$headers = @{ - Authorization = "bearer $AuthToken" -} -$uri = "https://api.github.com/repos/$RepoOwner/$RepoName/pulls/$PRNumber/requested_reviewers" +$errorsOccurredAddingUsers = AddMembers -memberName "reviewers" -additionSet $userAdditions +$errorsOccurredAddingTeams = AddMembers -memberName "team_reviewers" -additionSet $teamAdditions -try { - $resp = Invoke-RestMethod -Headers $headers $uri -MaximumRetryCount 3 -} -catch { - Write-Error "Invoke-RestMethod [$uri] failed with exception:`n$_" +if ($errorsOccurredAddingUsers -or $errorsOccurredAddingTeams) { exit 1 } - -# the response object takes this form: https://developer.github.com/v3/pulls/review_requests/#response-1 -# before we can push a new reviewer, we need to pull the simple Ids out of the complex objects that came back in the response -$userReviewers = @($resp.users | % { return $_.login }) -$teamReviewers = @($resp.teams | % { return $_.slug }) - -if (!$userReviewers) { $modifiedUserReviewers = @() } else { $modifiedUserReviewers = $userReviewers.Clone() } -$modifiedUserReviewers += ($userAdditions | ? { !$modifiedUserReviewers.Contains($_) }) - -if ($teamReviewers) { $modifiedTeamReviewers = @() } else { $modifiedTeamReviewers = $teamReviewers.Clone() } -$modifiedTeamReviewers += ($teamAdditions | ? { !$modifiedTeamReviewers.Contains($_) }) - -$detectedUserDiffs = Compare-Object -ReferenceObject $userReviewers -DifferenceObject $modifiedUserReviewers -$detectedTeamDiffs = Compare-Object -ReferenceObject $teamReviewers -DifferenceObject $modifiedTeamReviewers - -# Compare-Object returns values when there is a difference between the comparied objects. -# we only want to run the update if there IS a difference. -if ($detectedUserDiffs -or $detectedTeamDiffs) { - $postResp = @{} - - if ($modifiedUserReviewers) { $postResp["reviewers"] = $modifiedUserReviewers } - if ($modifiedTeamReviewers) { $postResp["team_reviewers"] = $modifiedTeamReviewers } - - $postResp = $postResp | ConvertTo-Json - - try { - Write-Host $postResp - $resp = Invoke-RestMethod -Method Post -Headers $headers -Body $postResp -Uri $uri -MaximumRetryCount 3 - $resp | Write-Verbose - } - catch { - Write-Error "Unable to update PR reviewers. `n$_" - } -} -else { - $results = $GitHubUsers + $GitHubTeams - Write-Host "Reviewers $results already added. Exiting." - exit(0) -} diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index f17e59700b6..a6d21e170a4 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -7,7 +7,7 @@ $SDIST_PACKAGE_REGEX = "^(?.*)\-(?$([AzureEngSemanticVer function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) { foreach ($pkgInfo in $pkgList) { Write-Host "Creating release $($pkgInfo.Tag)" - + echo "##vso[task.setvariable variable=ReleaseTag;isOutput=true]$($pkgInfo.Tag)" $releaseNotes = "" if ($pkgInfo.ReleaseNotes -ne $null) { $releaseNotes = $pkgInfo.ReleaseNotes @@ -502,7 +502,7 @@ function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $ap else { Write-Host "The artifact SHA $releaseSha does not match that of the currently existing tag." Write-Host "Tag with issues is $tag with commit SHA $tagSha" - + echo "##vso[task.setvariable variable=ReleaseTag;isOutput=true]$tag" $unmatchedTags += $tag } } diff --git a/eng/common/scripts/copy-docs-to-blobstorage.ps1 b/eng/common/scripts/copy-docs-to-blobstorage.ps1 index a0893099293..1cd496d5a40 100644 --- a/eng/common/scripts/copy-docs-to-blobstorage.ps1 +++ b/eng/common/scripts/copy-docs-to-blobstorage.ps1 @@ -7,7 +7,9 @@ param ( $Language, $BlobName, $ExitOnError=1, - $UploadLatest=1 + $UploadLatest=1, + $RepoReplaceRegex, + $Tag ) $Language = $Language.ToLower() @@ -197,6 +199,12 @@ function Upload-Blobs Write-Host "DocDir $($DocDir)" Write-Host "Final Dest $($DocDest)/$($PkgName)/$($DocVersion)" + # Use the step to replace master link to release tag link + Write-Host "Replacing all readme master links with release tag $Tag" + ReplaceLink -scanFolder $DocDir -fileSuffix ".html" -replacement $Tag -customRegex $RepoReplaceRegex + + + Write-Host "Uploading $($PkgName)/$($DocVersion) to $($DocDest)..." & $($AzCopy) cp "$($DocDir)/**" "$($DocDest)/$($PkgName)/$($DocVersion)$($SASKey)" --recursive=true diff --git a/eng/common/scripts/link-replacement.ps1 b/eng/common/scripts/link-replacement.ps1 new file mode 100644 index 00000000000..6bd4e237589 --- /dev/null +++ b/eng/common/scripts/link-replacement.ps1 @@ -0,0 +1,55 @@ +param ( + # url list to verify links. Can either be a http address or a local file request. Local file paths support md and html files. + [string] $sourceDir, + # file that contains a set of links to ignore when verifying + [string] $ignoreLinksFile = "$PSScriptRoot/ignore-links.txt", + # switch that will enable devops specific logging for warnings + [switch] $devOpsLogging = $false, + [string] $branchReplaceRegex = "(https://github.com/.*/(?:blob|tree)/)master(/.*)", + # the substitute branch name or SHA commit + [string] $releaseTag = "" +) + +$regexOptions = [System.Text.RegularExpressions.RegexOptions]"Singleline, IgnoreCase"; + +function ReplaceLink ($scanFolder, $fileSuffix, $replacement, $customRegex) { + $regex = new-object System.Text.RegularExpressions.Regex ($branchReplaceRegex, $regexOptions) + if ($customRegex) { + $regex = new-object System.Text.RegularExpressions.Regex ($customRegex, $regexOptions) + } + if (!$fileSuffix) { + Write-Host "Will scan the entire repo" + $fileSuffix = "" + } + $replacementPattern = "`${2}$replacement`$3" + if (!$replacement) { + Write-Error "Please provide the replacement string to replace with." + } + if ($scanFolder) { + $fileSuffixArray = $fileSuffix -split "," + $url = @() + + foreach ($fileExtension in $fileSuffixArray) { + $fileRegex = "*" + $fileSuffix.Trim() + $urls += Get-ChildItem -Path "$scanFolder/*" -Recurse -Include $fileRegex + } + + if ($urls.Count -eq 0) { + Write-Host "Usage $($MyInvocation.MyCommand.Name) "; + return; + } + $needTochange = @{} + foreach ($url in $urls) { + while ((Get-Content -Path $url) -match $regex) { + Write-Verbose "We have master link matches in page $url" + $needTochange[$url] = $true + (Get-Content -Path $url) -replace $regex, $replacementPattern | Set-Content -Path $url + } + } + foreach ($page in $needTochange.Keys) { + Write-Host "There are replacements in page $page" + } + } +} + +#ReplaceLink -scanFolder $sourceDir -replacement $releaseTag -fileSuffix ".html" \ No newline at end of file diff --git a/eng/common/scripts/update-docs-metadata.ps1 b/eng/common/scripts/update-docs-metadata.ps1 index 42ca30894fd..d1fae46585f 100644 --- a/eng/common/scripts/update-docs-metadata.ps1 +++ b/eng/common/scripts/update-docs-metadata.ps1 @@ -11,12 +11,14 @@ param ( # arguments necessary to power the docs release $DocRepoLocation, # the location on disk where we have cloned the documentation repository $Language, # EG: js, java, dotnet. Used in language for the embedded readme. - $DocRepoContentLocation = "docs-ref-services/" # within the doc repo, where does our readme go? + $DocRepoContentLocation = "docs-ref-services/", # within the doc repo, where does our readme go? ) . (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1) . (Join-Path $PSScriptRoot SemVer.ps1) +$releaseReplaceRegex = "(https://github.com/$RepoId/(?:blob|tree)/)master(/.*)" + function GetMetaData($lang){ switch ($lang) { "java" { @@ -75,6 +77,12 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){ if ($headerContentMatches) { $foundTitle = $headerContentMatches.Matches[0] $fileContent = $pkgInfo.ReadmeContent -replace $foundTitle, "$foundTitle - Version $($pkgInfo.PackageVersion) `n" + # Replace github master link with release tag. + $regex = new-object System.Text.RegularExpressions.Regex ($releaseReplaceRegex, + [System.Text.RegularExpressions.RegexOptions]"Singleline, IgnoreCase") + + $ReplacementPattern = "`${1}$($pkgInfo.Tag)`$2" + $fileContent = $fileContent -replace $regex, $ReplacementPattern } $header = "---`ntitle: $foundTitle`nkeywords: Azure, $lang, SDK, API, $($pkgInfo.PackageId), $service`nauthor: maggiepint`nms.author: magpint`nms.date: $date`nms.topic: article`nms.prod: azure`nms.technology: azure`nms.devlang: $lang`nms.service: $service`n---`n"