Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Sync eng/common directory with azure-sdk-tools for PR 7569 #22284

Merged
merged 4 commits into from
Jan 25, 2024
Merged

Sync eng/common directory with azure-sdk-tools for PR 7569 #22284

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
19c0a5e
Correct the name of JS package folder
raych1 Jan 23, 2024
2ed62b1
Uncomment the package verification
raych1 Jan 23, 2024
2477ec6
Logging more info for troubleshooting
raych1 Jan 23, 2024
c064da4
Get sdkType and directory from the package info
raych1 Jan 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 76 additions & 66 deletions eng/common/scripts/Verify-RestApiSpecLocation.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,110 +43,112 @@ param (
. (Join-Path $PSScriptRoot Helpers/PSModule-Helpers.ps1)

# Check if github token is set
if(-not $GitHubPat) {
if (-not $GitHubPat) {
LogError "GitHubPat is not set. Please set the environment variable GH_TOKEN or pass the GitHubPat parameter."
exit 1
}

Write-Host "The spec used to release SDK should be from the main branch of Azure/azure-rest-api-specs repository."
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName, ArtifactLocation:$ArtifactLocation, PackageInfoDirectory:$PackageInfoDirectory."
Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module

# This function is used to verify the 'require' and 'input-file' settings in autorest.md point to the main branch of Azure/azure-rest-api-specs repository
# input-file may be:
# https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/purview/data-plane/Azure.Analytics.Purview.MetadataPolicies/preview/2021-07-01-preview/purviewMetadataPolicy.json
# or
# https://github.com/Azure/azure-rest-api-specs/blob/0ebd4949e8e1cd9537ca5a07384c7661162cc7a6/specification/purview/data-plane/Azure.Analytics.Purview.Account/preview/2019-11-01-preview/account.json
function Verify-Url([string]$fileUrl) {
if($fileUrl -match "^https://(raw\.githubusercontent\.com|github\.com)/(?<repo>[^/]*/azure-rest-api-specs)(/(blob|tree))?/(?<commit>[^\/]+(\/[^\/]+)*|[0-9a-f]{40})/(?<path>specification/.*)") {
function Verify-Url([string]$fileUrl, [string]$configFilePath) {
if ($fileUrl -match "^https://(raw\.githubusercontent\.com|github\.com)/(?<repo>[^/]*/azure-rest-api-specs)(/(blob|tree))?/(?<commit>[^\/]+(\/[^\/]+)*|[0-9a-f]{40})/(?<path>specification/.*)") {
$repo = $matches['repo']
$commit = $matches['commit']
if($repo -ne "Azure/azure-rest-api-specs") {
LogError "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Invalid repo in the file url: $fileUrl. Repo should be 'Azure/azure-rest-api-specs'."
if ($repo -ne "Azure/azure-rest-api-specs") {
LogError "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Invalid repo in the file url: $fileUrl. Repo should be 'Azure/azure-rest-api-specs' in the config file:$configFilePath."
exit 1
}
# check the commit hash belongs to main branch
Verify-CommitFromMainBranch $commit
Verify-CommitFromMainBranch $commit $configFilePath
}
else{
LogError "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Invalid file url: $fileUrl"
else {
LogError "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Invalid file url: $fileUrl in the config file:$configFilePath. The spec location should point to the main branch of Azure/azure-rest-api-specs repository."
exit 1
}
}

# This function is used to verify the 'repo' and 'commit' settings in tsp-location.yaml point to the main branch of Azure/azure-rest-api-specs repository
function Verify-TspLocation([System.Object]$tspLocationObj) {
function Verify-TspLocation([System.Object]$tspLocationObj, [string]$tspLocationYamlPath) {
$repo = $tspLocationObj["repo"]
$commit = $tspLocationObj["commit"]
if($repo -ne "Azure/azure-rest-api-specs") {
LogError "Invalid repo setting in the tsp-location.yaml: $repo. Repo should be 'Azure/azure-rest-api-specs'. ServiceDir:$ServiceDirectory, PackageName:$PackageName"
if ($repo -ne "Azure/azure-rest-api-specs") {
LogError "Invalid repo setting in the tsp-location.yaml: $repo. Repo should be 'Azure/azure-rest-api-specs'. ServiceDir:$ServiceDirectory, PackageName:$PackageName, tsp-location.yaml path:$tspLocationYamlPath."
exit 1
}

# check the commit hash belongs to main branch
Verify-CommitFromMainBranch $commit
Verify-CommitFromMainBranch $commit $tspLocationYamlPath
}

# This function is used to verify the specific 'commit' belongs to the main branch of Azure/azure-rest-api-specs repository
function Verify-CommitFromMainBranch([string]$commit) {
if($commit -notmatch "^[0-9a-f]{40}$" -and $commit -ne "main") {
LogError "Invalid commit hash or branch name: $commit. Branch name should be 'main' or the commit should be a 40-character SHA-1 hash. ServiceDir:$ServiceDirectory, PackageName:$PackageName"
function Verify-CommitFromMainBranch([string]$commit, [string]$configFilePath) {
if ($commit -notmatch "^[0-9a-f]{40}$" -and $commit -ne "main") {
LogError "Invalid commit hash or branch name: $commit. Branch name should be 'main' or the commit should be a 40-character SHA-1 hash. ServiceDir:$ServiceDirectory, PackageName:$PackageName, please check the config file:$configFilePath."
exit 1
}
if($commit -eq "main") {
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Branch is $commit branch of Azure/azure-rest-api-specs repository."
if ($commit -eq "main") {
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName, Config file:$configFilePath. Branch is $commit branch of Azure/azure-rest-api-specs repository."
return
}
try {
$searchResult = Search-GitHubCommit -AuthToken $GitHubPat -CommitHash $commit -RepoOwner "Azure" -RepoName "azure-rest-api-specs"
if ($searchResult.total_count -lt 1) {
LogError "Commit $commit doesn't exist in 'main' branch of Azure/azure-rest-api-specs repository. ServiceDir:$ServiceDirectory, PackageName:$PackageName"
LogError "Commit $commit doesn't exist in 'main' branch of Azure/azure-rest-api-specs repository. ServiceDir:$ServiceDirectory, PackageName:$PackageName, please check the config file:$configFilePath. The spec used to release SDK should be from the main branch of Azure/azure-rest-api-specs repository."
exit 1
}
else{
else {
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Commit $commit exists in 'main' branch of Azure/azure-rest-api-specs repository."
}
}
catch {
LogError "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Failed to search commit $commit with exception:`n$_ "
LogError "ServiceDir:$ServiceDirectory, PackageName:$PackageName, Config file:$configFilePath. The spec used to release SDK should be from the main branch of Azure/azure-rest-api-specs repository. Failed to search commit $commit with exception:`n$_ "
exit 1
}
}

function Verify-YamlContent([string]$markdownContent) {
function Verify-YamlContent([string]$markdownContent, [string]$configFilePath) {
$splitString = '``` yaml|```yaml|```'
$yamlContent = $markdownContent -split $splitString
foreach($yamlSection in $yamlContent) {
foreach ($yamlSection in $yamlContent) {
if ($yamlSection) {
try {
# remove the lines like: $(tag) == 'package-preview-2023-09'
$yamlSection = $yamlSection -replace '^\s*\$\(.+\)\s*==.+', ''
$yamlobj = ConvertFrom-Yaml -Yaml $yamlSection
if($yamlobj) {
if ($yamlobj) {
$batchValue = $yamlobj["batch"]
$requireValue = $yamlobj["require"]
$inputFileValue = $yamlobj["input-file"]
if ($requireValue) {
LogDebug "ServiceDir:$ServiceDirectory, PackageName:$PackageName. 'require' is set as:$requireValue"
Verify-Url $requireValue
LogDebug "ServiceDir:$ServiceDirectory, PackageName:$PackageName, Config file:$configFilePath. 'require' is set as:$requireValue"
Verify-Url $requireValue $configFilePath
}
elseif ($inputFileValue) {
LogDebug "ServiceDir:$ServiceDirectory, PackageName:$PackageName. 'input-file' is set as:$inputFileValue"
foreach($inputFile in $inputFileValue) {
Verify-Url $inputFile
LogDebug "ServiceDir:$ServiceDirectory, PackageName:$PackageName, Config file:$configFilePath. 'input-file' is set as:$inputFileValue"
foreach ($inputFile in $inputFileValue) {
Verify-Url $inputFile $configFilePath
}
}
elseif ($batchValue) {
# there are some services which use batch mode for sdk generation, e.g. Azure.AI.Language.QuestionAnswering
foreach($batch in $batchValue) {
foreach ($batch in $batchValue) {
$requireValue = $batch["require"]
$inputFileValue = $batch["input-file"]
if ($requireValue) {
LogDebug "ServiceDir:$ServiceDirectory, PackageName:$PackageName. 'require' is set as:$requireValue"
Verify-Url $requireValue
LogDebug "ServiceDir:$ServiceDirectory, PackageName:$PackageName, Config file:$configFilePath. 'require' is set as:$requireValue"
Verify-Url $requireValue $configFilePath
}
elseif ($inputFileValue) {
LogDebug "ServiceDir:$ServiceDirectory, PackageName:$PackageName. 'input-file' is set as:$inputFileValue"
foreach($inputFile in $inputFileValue) {
Verify-Url $inputFile
LogDebug "ServiceDir:$ServiceDirectory, PackageName:$PackageName, Config file:$configFilePath. 'input-file' is set as:$inputFileValue"
foreach ($inputFile in $inputFileValue) {
Verify-Url $inputFile $configFilePath
}
}
}
Expand All @@ -163,19 +165,16 @@ function Verify-YamlContent([string]$markdownContent) {
function Verify-PackageVersion() {
try {
$packages = @{}
if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn"))
{
if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) {
$packages = &$FindArtifactForApiReviewFn $ArtifactLocation $PackageName
}
else
{
else {
LogError "The function for 'FindArtifactForApiReviewFn' was not found.`
Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.`
See https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/common_engsys.md#code-structure"
exit 1
}
if (-not $PackageInfoDirectory)
{
if (-not $PackageInfoDirectory) {
$PackageInfoDirectory = Join-Path -Path $ArtifactLocation "PackageInfo"
if (-not (Test-Path -Path $PackageInfoDirectory)) {
# Call Save-Package-Properties.ps1 script to generate package info json files
Expand All @@ -185,23 +184,19 @@ function Verify-PackageVersion() {
}

$continueValidation = $false
if ($packages)
{
foreach($pkgPath in $packages.Values)
{
if ($packages) {
foreach ($pkgPath in $packages.Values) {
$pkgPropPath = Join-Path -Path $PackageInfoDirectory "$PackageName.json"
if (-Not (Test-Path $pkgPropPath))
{
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Package property file path $($pkgPropPath) is invalid."
continue
if (-Not (Test-Path $pkgPropPath)) {
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Package property file path $($pkgPropPath) is invalid."
continue
}
# Get package info from json file
$pkgInfo = Get-Content $pkgPropPath | ConvertFrom-Json
$version = [AzureEngSemanticVersion]::ParseVersionString($pkgInfo.Version)
if ($null -eq $version)
{
LogError "ServiceDir:$ServiceDirectory, Version info is not available for package $PackageName, because version '$(pkgInfo.Version)' is invalid. Please check if the version follows Azure SDK package versioning guidelines."
exit 1
if ($null -eq $version) {
LogError "ServiceDir:$ServiceDirectory, Version info is not available for package $PackageName, because version '$(pkgInfo.Version)' is invalid. Please check if the version follows Azure SDK package versioning guidelines."
exit 1
}

Write-Host "Version: $($version)"
Expand All @@ -216,23 +211,36 @@ function Verify-PackageVersion() {
$continueValidation = $true
}
}
if($continueValidation -eq $false) {
if ($continueValidation -eq $false) {
Write-Host "ServiceDir:$ServiceDirectory, no package info is found for package $PackageName, the validation of spec location is ignored."
exit 0
}
# Return the package info
return $pkgInfo
}
catch {
LogError "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Failed to retrieve package and validate package version with exception:`n$_ "
exit 1
}
}

try{
try {
# Verify package version is not a prerelease version, only continue the validation if the package is GA version
Verify-PackageVersion
$packageInfo = Verify-PackageVersion
if (-not $packageInfo) {
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. The package info is not available."
exit 0
}
# Get the package directory path
$PackageDirectory = $null
if ($packageInfo.DirectoryPath) {
$PackageDirectory = Join-Path $RepoRoot $packageInfo.DirectoryPath
}

$ServiceDir = Join-Path $RepoRoot 'sdk' $ServiceDirectory
$PackageDirectory = Join-Path $ServiceDir $PackageName
if ((-not $PackageDirectory) -or (-not (Test-Path $PackageDirectory))) {
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName, PackageDirectory:$PackageDirectory. The package directory path is invalid."
exit 0
}
Push-Location $PackageDirectory

# Load tsp-location.yaml if existed
Expand All @@ -243,39 +251,41 @@ try{
if (Test-Path -Path $tspLocationYamlPath) {
# typespec scenario
$tspLocationYaml = Get-Content -Path $tspLocationYamlPath -Raw | ConvertFrom-Yaml
Verify-TspLocation $tspLocationYaml
Verify-TspLocation $tspLocationYaml $tspLocationYamlPath
}
elseif ($Language -eq "dotnet") {
# only dotnet language sdk uses 'autorest.md' to configure the sdk generation
if (Test-Path -Path $autorestMdPath) {
try {
$autorestMdContent = Get-Content -Path $autorestMdPath -Raw
Verify-YamlContent $autorestMdContent
Verify-YamlContent $autorestMdContent $autorestMdPath
}
catch {
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Failed to parse autorest.md file with exception:`n$_ "
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Failed to parse autorest.md file:$autorestMdPath with exception:`n$_ "
}
}
}
elseif ($Language -eq "java" -or $Language -eq "javascript" -or $Language -eq "python" -or $Language -eq "go") {
# for these languages we ignore the validation because they always use the latest spec from main branch to release SDK
# mgmt plane packages: azure-core-management|azure-resourcemanager|azure-resourcemanager-advisor (java), azure-mgmt-devcenter (python), arm-advisor (js), armaad (go)
if($PackageName -match "^(arm|azure-mgmt|azure-resourcemanager|azure-core-management)[-a-z]*$") {
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Ignore the validation for $Language management plane package."
# for these languages we ignore the validation for mgmt plane SDK because they always use the latest spec from main branch to release SDK
if ($packageInfo.SdkType -eq "mgmt") {
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Ignore the validation for $Language management plane SDK release."
exit 0
}
# for these languages they use 'swagger/readme.md' to configure the sdk generation for data plane scenarios
if (Test-Path -Path $swaggerReadmePath) {
try {
$swaggerReadmeContent = Get-Content -Path $swaggerReadmePath -Raw
Verify-YamlContent $swaggerReadmeContent
Verify-YamlContent $swaggerReadmeContent $swaggerReadmePath
}
catch {
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Failed to parse swagger/readme.md file with exception:`n$_ "
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Failed to parse swagger/readme.md file:$swaggerReadmePath with exception:`n$_ "
}
}
}
}
catch {
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName, PackageDirectory:$PackageDirectory. Failed to validate spec location with exception:`n$_ "
}
finally {
Pop-Location
}