From 9d33608b8faa1ea5f6d2b08e9bc4ba35f701ff42 Mon Sep 17 00:00:00 2001 From: Azure PowerShell <65331932+azure-powershell-bot@users.noreply.github.com> Date: Tue, 16 Apr 2024 16:01:42 +0800 Subject: [PATCH] Sync tools folder from main branch to generation branch (#24630) Co-authored-by: azurepowershell --- .azure-pipelines/refresh-autorest.yml | 208 ++++++++++++++++++ .../SDKGeneratedCodeVerify.ps1 | 24 +- 2 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 .azure-pipelines/refresh-autorest.yml diff --git a/.azure-pipelines/refresh-autorest.yml b/.azure-pipelines/refresh-autorest.yml new file mode 100644 index 000000000000..adbec6cafa98 --- /dev/null +++ b/.azure-pipelines/refresh-autorest.yml @@ -0,0 +1,208 @@ +parameters: +- name: targetBranch + displayName: Target branch where the PR will be submitted + type: string + default: main + +- name: scope + displayName: Select target Autorest modules + type: string + default: autorest-all + values: + - autorest-all + - autorest-v3-only + - autorest-v4-only + - autorest-selected + +- name: selectedServices + displayName: Input service names you want to refresh separated by semi-colon (;). Wildcard match is supported. + type: string + default: ' ' + +- name: changeLogMessage + displayName: Input change log message for the Autorest modules + type: string + +pr: none +trigger: none + +jobs: +- job: + displayName: Refresh Autorest Modules + timeoutInMinutes: 360 + pool: pool-windows-2019 + steps: + - template: util/get-github-pat-steps.yml + + - task: NodeTool@0 + displayName: Use Node 14.15.5 + inputs: + versionSpec: 14.15.5 + + - task: Npm@1 + displayName: Install Autorest + inputs: + command: custom + verbose: false + customCommand: install autorest@latest -g + + - task: PowerShell@2 + displayName: Get services list + inputs: + pwsh: true + targetType: inline + workingDirectory: ./src + script: | + $autorestPaths = @() + $autorestModules = Get-ChildItem -Filter "*.Autorest" -Directory -Recurse | Select-Object -ExpandProperty FullName + if ("${{ parameters.scope }}" -eq "autorest-v3-v4") { + $autorestPaths = $autorestModules + } + elseif ("${{ parameters.scope }}" -eq "autorest-selected") { + $selectedModulesList = '${{ parameters.selectedServices }}' + if (![System.String]::IsNullOrWhiteSpace($selectedModulesList)) { + $selectedModulePatterns = $selectedModulesList -split ";" + $autorestPaths = $selectedModulePatterns | Foreach-Object { + $modulePattern = $_.Trim() + $autorestModules | Where-Object { + $modulePath = Split-Path -Path $_ -Parent + $moduleName = (Get-Item -Path $modulePath).Name + $moduleName -like $modulePattern + } + } + } + } + else { + $isV3Only = "${{ parameters.scope }}" -eq "autorest-v3-only" + $isV4Only = "${{ parameters.scope }}" -eq "autorest-v4-only" + $readmeMDs = $autorestModules | Get-ChildItem -Filter "README.md" -File + $readmeMDs | Foreach-Object { + $content = Get-Content -Path $_ -Raw + $pattern = 'use-extension:\s*"@autorest/powershell":\s*"([^"]+)"' + if ($content -match $pattern) { + $version = $Matches[1].Substring(0, 1) + if (($version -eq "3" -and $isV3Only) -or ($version -eq "4" -and $isV4Only)) { + $autorestPaths += $_.Directory.FullName + } + } + elseif ($isV4Only) { + $autorestPaths += $_.Directory.FullName + } + } + } + + $autorestPaths = $autorestPaths | Sort-Object -Unique + Write-Host "##[section]Autorest modules to refresh: `n`t$($autorestPaths -join "`n`t")" + $autorestPathsList = $autorestPaths -join ";" + Write-Host "##vso[task.setvariable variable=selectedAutorestPaths;isreadonly=true]$autorestPathsList" + + - task: PowerShell@2 + displayName: Generate & Build + inputs: + pwsh: true + targetType: inline + script: | + # Autorest and build the modules + '$(selectedAutorestPaths)'.Split(";") | Foreach-Object { + Write-Host "##[group]Start generating module $_" + + Set-Location -Path $_ + + Write-Host "##[section]Start running autorest command." + autorest --max-memory-size=8192 + Write-Host "##[section]Finish running autorest command." + + Write-Host "##[section]Start building the module." + ./build-module.ps1 + Write-Host "##[section]Finish building the module." + + Write-Host "##[endgroup]" + Write-Host + } + + # Backup the modules in the artifacts folder and restore the changes + $modules = '$(selectedAutorestPaths)' -split ";" | Foreach-Object { Split-Path -Path $_ -Parent } | Select-Object -Unique + $modules | Foreach-Object { + Write-Host "##[group]Start backing up module $_" + + Set-Location -Path $_ + $servicePath = Get-Item -Path . + $serviceName = $servicePath.Name + + Write-Host "##[section]Start copying content to artifacts." + New-Item -Path ../../artifacts/src/$serviceName -ItemType Directory + Copy-Item -Path ./* -Destination ../../artifacts/src/$serviceName -Recurse + Write-Host "##[section]Finish copying content to artifacts." + + Set-Location .. + + Write-Host "##[section]Start restoring the changes." + Remove-Item -Path ./$serviceName -Recurse -Force + git checkout $serviceName/ + Write-Host "##[section]Finish restoring the changes." + + Write-Host "##[endgroup]" + Write-Host + } + + Set-Location .. + git checkout -b 'codegen/${{ parameters.scope }}' 'origin/${{ parameters.targetBranch }}' + + - task: PowerShell@2 + displayName: Migrate from generation to target branch + inputs: + pwsh: true + targetType: inline + script: | + Install-Module -Name PowerShellGet -RequiredVersion 2.2.3 -Force + Install-Module -Name platyPS -RequiredVersion 0.14.2 -Force + Install-Module -Name Az.Accounts -Force + Import-Module ./tools/Gen2Master/MoveFromGeneration2Master.ps1 + + $modules = '$(selectedAutorestPaths)' -split ";" | Foreach-Object { Split-Path -Path $_ -Parent } | Select-Object -Unique + $modules | Foreach-Object { + Write-Host "##[group]Start migrating module $_" + + Set-Location -Path $_ + $servicePath = Get-Item -Path . + $serviceName = $servicePath.Name + + Write-Host "##[section]Start migrating from generation to target." + Move-Generation2Master -SourcePath ../../artifacts/src/$serviceName -DestPath . + Write-Host "##[section]Finish migrating from generation to target." + + Write-Host "##[section]Start updating change log for module." + $changeLog = $servicePath | Get-ChildItem -Filter "ChangeLog.md" -File -Recurse | Select-Object -First 1 + $changeLogPath = $changeLog.FullName + $changeLogMessage = '${{ parameters.changeLogMessage }}' + (Get-Content -Path $changeLogPath -Raw) -replace "`n## Upcoming Release", ("`n## Upcoming Release`n" + "* $changeLogMessage") | Set-Content -Path $changeLogPath -NoNewline + Write-Host "##[section]Finish updating change log for module." + + Write-Host "##[endgroup]" + Write-Host + } + + - task: PowerShell@2 + displayName: Create PR to target branch + inputs: + pwsh: true + targetType: inline + script: | + $sourceBranch = '$(Build.SourceBranch)'.Replace("refs/heads/", "") + $headBranch = 'codegen/${{ parameters.scope }}' + $baseBranch = '${{ parameters.targetBranch }}' + git config user.email "65331932+azure-powershell-bot@users.noreply.github.com" + git config user.name "azure-powershell-bot" + git add src + git add tools/CreateMappings_rules.json + git commit -m "Refresh ${{ parameters.scope }} modules from $sourceBranch to $baseBranch" + git remote set-url origin https://azure-powershell-bot:$(GithubToken)@github.com/Azure/azure-powershell.git + git push origin $headBranch --force + + $title = "Refresh ${{ parameters.scope }} modules from $sourceBranch to $baseBranch" + $description = "Refresh ${{ parameters.scope }} modules from $sourceBranch to $baseBranch." + if ("${{ parameters.scope }}" -eq "autorest-selected") { + $description = "$description `nSelected services are: ${{ parameters.selectedServices }}." + } + + ./tools/Github/CreatePR.ps1 -Title $title -HeadBranch $headBranch -BaseBranch $baseBranch -BotAccessToken $(GithubToken) -Description $description diff --git a/tools/StaticAnalysis/GeneratedSdkAnalyzer/SDKGeneratedCodeVerify.ps1 b/tools/StaticAnalysis/GeneratedSdkAnalyzer/SDKGeneratedCodeVerify.ps1 index ba5050299369..501dd72d7f80 100644 --- a/tools/StaticAnalysis/GeneratedSdkAnalyzer/SDKGeneratedCodeVerify.ps1 +++ b/tools/StaticAnalysis/GeneratedSdkAnalyzer/SDKGeneratedCodeVerify.ps1 @@ -86,6 +86,13 @@ try { Write-Host "Preparing Autorest..." npx autorest --reset foreach ($_ in $ChangedSdks) { + # If it is Resources.Management.Sdk, flag and will use tag for sdk generation + $IsResources = $false; + if ($_ -match "Resources.Management.Sdk") + { + $IsResources = $true; + } + # Extract Module Name $ModuleName = "Az." + ($_ -split "\/|\\")[1] @@ -107,7 +114,22 @@ try { if ([regex]::Matches($readMeContent, '\s*powershell\s*:\s*true\s*') -and [regex]::Matches($readMeContent, '\s*isSdkGenerator\s*:\s*true\s*')) { Write-Host "Using autorest powershell v4:`nRe-generating SDK under Generated folder for $ModuleName..." - npx autorest + if ($IsResources) + { + Write-Host "Specific generation for Resources.Management.Sdk" + rm -r Generated/* + npx autorest --use:@autorest/powershell@4.x --tag=package-privatelinks-2020-05 + npx autorest --use:@autorest/powershell@4.x --tag=package-subscriptions-2021-01 + npx autorest --use:@autorest/powershell@4.x --tag=package-features-2021-07 + npx autorest --use:@autorest/powershell@4.x --tag=package-deploymentscripts-2020-10 + npx autorest --use:@autorest/powershell@4.x --tag=package-resources-2021-04 + npx autorest --use:@autorest/powershell@4.x --tag=package-deploymentstacks-2022-08-preview + npx autorest --use:@autorest/powershell@4.x --tag=package-templatespecs-2021-05 + } + else + { + npx autorest + } } elseif ([regex]::Matches($readMeContent, '\s*csharp\s*:\s*true\s*')) {