From 7c27d870bfd62e1ed3c272ba20e4567cbc1cf8fa Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Mon, 5 Jun 2023 18:12:45 -0700 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools for PR 6272 (#35289) * Support localspecrepo if pass in this parameter * Support regen sdk code based on local typespecs * Added reference doc to error message --------- Co-authored-by: raychen --- .../scripts/TypeSpec-Project-Generate.ps1 | 5 +- .../scripts/TypeSpec-Project-Process.ps1 | 93 +++++---- eng/common/scripts/TypeSpec-Project-Sync.ps1 | 182 ++++++++++-------- 3 files changed, 162 insertions(+), 118 deletions(-) diff --git a/eng/common/scripts/TypeSpec-Project-Generate.ps1 b/eng/common/scripts/TypeSpec-Project-Generate.ps1 index c16dad6644950..418221982a365 100644 --- a/eng/common/scripts/TypeSpec-Project-Generate.ps1 +++ b/eng/common/scripts/TypeSpec-Project-Generate.ps1 @@ -38,7 +38,7 @@ function NpmInstallForProject([string]$workingDirectory) { #default to root/eng/emitter-package.json but you can override by writing #Get-${Language}-EmitterPackageJsonPath in your Language-Settings.ps1 - $replacementPackageJson = "$PSScriptRoot/../../emitter-package.json" + $replacementPackageJson = Join-Path $PSScriptRoot "../../emitter-package.json" if (Test-Path "Function:$GetEmitterPackageJsonPathFn") { $replacementPackageJson = &$GetEmitterPackageJsonPathFn } @@ -52,7 +52,7 @@ function NpmInstallForProject([string]$workingDirectory) { Write-Host "Package.json contains '-alpha.' in the version, Creating .npmrc using public/azure-sdk-for-js-test-autorest feed." "registry=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ `n`nalways-auth=true" | Out-File '.npmrc' } - + npm install --no-lock-file if ($LASTEXITCODE) { exit $LASTEXITCODE } } @@ -112,3 +112,4 @@ $shouldCleanUp = !$SaveInputs if ($shouldCleanUp) { Remove-Item $tempFolder -Recurse -Force } +exit 0 diff --git a/eng/common/scripts/TypeSpec-Project-Process.ps1 b/eng/common/scripts/TypeSpec-Project-Process.ps1 index 3bff436dca7fd..87004c9b494e9 100644 --- a/eng/common/scripts/TypeSpec-Project-Process.ps1 +++ b/eng/common/scripts/TypeSpec-Project-Process.ps1 @@ -16,31 +16,20 @@ param ( Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module function CreateUpdate-TspLocation([System.Object]$tspConfig, [string]$TypeSpecProjectDirectory, [string]$CommitHash, [string]$repo, [string]$repoRoot) { - $serviceDir = "" $additionalDirs = @() - - # Parse tspcofig.yaml to get service-dir, additionalDirectories, and package-dir - if ($tspConfig["parameters"] -and $tspConfig["parameters"]["service-dir"]) { - $serviceDir = $tspConfig["parameters"]["service-dir"]["default"]; - } - else { - Write-Error "Missing service-dir in parameters section of tspconfig.yaml." - exit 1 - } if ($tspConfig["parameters"]["dependencies"] -and $tspConfig["parameters"]["dependencies"]["additionalDirectories"]) { $additionalDirs = $tspConfig["parameters"]["dependencies"]["additionalDirectories"]; } - $packageDir = Get-PackageDir $tspConfig - # Create service-dir if not exist - $serviceDir = Join-Path $repoRoot $serviceDir + $serviceDir = Get-ServiceDir $tspConfig $repoRoot if (!(Test-Path -Path $serviceDir)) { New-Item -Path $serviceDir -ItemType Directory | Out-Null Write-Host "created service folder $serviceDir" } # Create package-dir if not exist + $packageDir = Get-PackageDir $tspConfig $packageDir = Join-Path $serviceDir $packageDir if (!(Test-Path -Path $packageDir)) { New-Item -Path $packageDir -ItemType Directory | Out-Null @@ -71,6 +60,20 @@ function CreateUpdate-TspLocation([System.Object]$tspConfig, [string]$TypeSpecPr return $packageDir } +function Get-ServiceDir([System.Object]$tspConfig, [string]$repoRoot) { + $serviceDir = "" + if ($tspConfig["parameters"] -and $tspConfig["parameters"]["service-dir"]) { + $serviceDir = $tspConfig["parameters"]["service-dir"]["default"]; + } + else { + Write-Error "Missing service-dir in parameters section of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema." + exit 1 + } + + # Create service-dir if not exist + $serviceDir = Join-Path $repoRoot $serviceDir + return $serviceDir +} function Get-PackageDir([System.Object]$tspConfig) { $emitterName = "" if (Test-Path "Function:$GetEmitterNameFn") { @@ -85,18 +88,27 @@ function Get-PackageDir([System.Object]$tspConfig) { $packageDir = $tspConfig["options"]["$emitterName"]["package-dir"] } else { - Write-Error "Missing package-dir in $emitterName options of tspconfig.yaml." + Write-Error "Missing package-dir in $emitterName options of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema." exit 1 } return $packageDir } -$repoRootPath = (Join-Path $PSScriptRoot .. .. ..) -$repoRootPath = Resolve-Path $repoRootPath -$repoRootPath = $repoRootPath -replace "\\", "/" -$tspConfigPath = Join-Path $repoRootPath 'tspconfig.yaml' +function Get-TspLocationFolder([System.Object]$tspConfig, [string]$repoRoot) { + $serviceDir = Get-ServiceDir $tspConfig $repoRoot + $packageDir = Get-PackageDir $tspConfig + $packageDir = Join-Path $serviceDir $packageDir + return $packageDir +} + +$sdkRepoRootPath = (Join-Path $PSScriptRoot .. .. ..) +$sdkRepoRootPath = Resolve-Path $sdkRepoRootPath +$sdkRepoRootPath = $sdkRepoRootPath -replace "\\", "/" +$tspConfigPath = Join-Path $sdkRepoRootPath 'tspconfig.yaml' $tmpTspConfigPath = $tspConfigPath $repo = "" +$specRepoRoot = "" +$generateFromLocalTypeSpec = $false # remote url scenario # example url of tspconfig.yaml: https://github.com/Azure/azure-rest-api-specs-pr/blob/724ccc4d7ef7655c0b4d5c5ac4a5513f19bbef35/specification/containerservice/Fleet.Management/tspconfig.yaml if ($TypeSpecProjectDirectory -match '^https://github.com/(?Azure/azure-rest-api-specs(-pr)?)/blob/(?[0-9a-f]{40})/(?.*)/tspconfig.yaml$') { @@ -121,26 +133,26 @@ if ($TypeSpecProjectDirectory -match '^https://github.com/(?Azure/azure-re exit 1 } $TypeSpecProjectDirectory = $TypeSpecProjectDirectory.Replace("\", "/") - if ($TypeSpecProjectDirectory -match "^.*/(?specification/.*)$") { + if ($TypeSpecProjectDirectory -match "(?^.*)/(?specification/.*)$") { $TypeSpecProjectDirectory = $Matches["path"] + $specRepoRoot = $Matches["repoRoot"] } else { Write-Error "$TypeSpecProjectDirectory doesn't have 'specification' in path." exit 1 } - if (!$CommitHash) { - Write-Error "Parameter of Commithash is not provided in the local path scenario." - exit 1 + if (!$CommitHash -or !$RepoUrl) { + Write-Warning "Parameter of Commithash or RepoUrl are not provided along with the local path of tspconfig.yaml, trying to re-generate sdk code based on the local type specs." + $generateFromLocalTypeSpec = $true } - if (!$RepoUrl) { - Write-Error "Parameter of RepoUrl:$RepoUrl is not provided in the local path scenario." - exit 1 - } - if ($RepoUrl -match "^https://github.com/(?[^/]*/azure-rest-api-specs(-pr)?).*") { - $repo = $Matches["repo"] - } - else { - Write-Error "Parameter 'RepoUrl' has incorrect value:$RepoUrl. It should be similar like 'https://github.com/Azure/azure-rest-api-specs'" - exit 1 + + if ($RepoUrl) { + if ($RepoUrl -match "^https://github.com/(?[^/]*/azure-rest-api-specs(-pr)?).*") { + $repo = $Matches["repo"] + } + else { + Write-Error "Parameter 'RepoUrl' has incorrect value:$RepoUrl. It should be similar like 'https://github.com/Azure/azure-rest-api-specs'" + exit 1 + } } } @@ -150,12 +162,23 @@ $tspConfigYaml = Get-Content $tspConfigPath -Raw | ConvertFrom-Yaml if (Test-Path $tmpTspConfigPath) { Remove-Item $tspConfigPath } -# call CreateUpdate-TspLocation function -$sdkProjectFolder = CreateUpdate-TspLocation $tspConfigYaml $TypeSpecProjectDirectory $CommitHash $repo $repoRootPath + +$sdkProjectFolder = "" +if ($generateFromLocalTypeSpec) { + $sdkProjectFolder = Get-TspLocationFolder $tspConfigYaml $sdkRepoRootPath + $tspLocationYamlPath = Join-Path $sdkProjectFolder "tsp-location.yaml" + if (!(Test-Path -Path $tspLocationYamlPath)) { + Write-Error "Failed to find tsp-location.yaml in '$sdkProjectFolder', please make sure to provide CommitHash and RepoUrl parameters along with the local path of tspconfig.yaml in order to create tsp-location.yaml." + exit 1 + } +} else { + # call CreateUpdate-TspLocation function + $sdkProjectFolder = CreateUpdate-TspLocation $tspConfigYaml $TypeSpecProjectDirectory $CommitHash $repo $sdkRepoRootPath +} # call TypeSpec-Project-Sync.ps1 $syncScript = Join-Path $PSScriptRoot TypeSpec-Project-Sync.ps1 -& $syncScript $sdkProjectFolder +& $syncScript $sdkProjectFolder $specRepoRoot if ($LASTEXITCODE) { exit $LASTEXITCODE } # call TypeSpec-Project-Generate.ps1 diff --git a/eng/common/scripts/TypeSpec-Project-Sync.ps1 b/eng/common/scripts/TypeSpec-Project-Sync.ps1 index 0cf2700e68107..0c7427af4ce53 100644 --- a/eng/common/scripts/TypeSpec-Project-Sync.ps1 +++ b/eng/common/scripts/TypeSpec-Project-Sync.ps1 @@ -2,9 +2,11 @@ [CmdletBinding()] param ( - [Parameter(Position=0)] - [ValidateNotNullOrEmpty()] - [string] $ProjectDirectory + [Parameter(Position = 0)] + [ValidateNotNullOrEmpty()] + [string] $ProjectDirectory, + [Parameter(Position = 1)] + [string] $LocalSpecRepoPath ) $ErrorActionPreference = "Stop" @@ -13,115 +15,133 @@ Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module $sparseCheckoutFile = ".git/info/sparse-checkout" function AddSparseCheckoutPath([string]$subDirectory) { - if (!(Test-Path $sparseCheckoutFile) -or !((Get-Content $sparseCheckoutFile).Contains($subDirectory))) { - Write-Output $subDirectory >> .git/info/sparse-checkout - } + if (!(Test-Path $sparseCheckoutFile) -or !((Get-Content $sparseCheckoutFile).Contains($subDirectory))) { + Write-Output $subDirectory >> .git/info/sparse-checkout + } } function CopySpecToProjectIfNeeded([string]$specCloneRoot, [string]$mainSpecDir, [string]$dest, [string[]]$specAdditionalSubDirectories) { - $source = "$specCloneRoot/$mainSpecDir" - Copy-Item -Path $source -Destination $dest -Recurse -Force - Write-Host "Copying spec from $source to $dest" + $source = Join-Path $specCloneRoot $mainSpecDir + Copy-Item -Path $source -Destination $dest -Recurse -Force + Write-Host "Copying spec from $source to $dest" - foreach ($additionalDir in $specAdditionalSubDirectories) { - $source = "$specCloneRoot/$additionalDir" - Write-Host "Copying spec from $source to $dest" - Copy-Item -Path $source -Destination $dest -Recurse -Force - } + foreach ($additionalDir in $specAdditionalSubDirectories) { + $source = Join-Path $specCloneRoot $additionalDir + Write-Host "Copying spec from $source to $dest" + Copy-Item -Path $source -Destination $dest -Recurse -Force + } } function UpdateSparseCheckoutFile([string]$mainSpecDir, [string[]]$specAdditionalSubDirectories) { - AddSparseCheckoutPath $mainSpecDir - foreach ($subDir in $specAdditionalSubDirectories) { - AddSparseCheckoutPath $subDir - } + AddSparseCheckoutPath $mainSpecDir + foreach ($subDir in $specAdditionalSubDirectories) { + Write-Host "Adding $subDir to sparse checkout" + AddSparseCheckoutPath $subDir + } } function GetGitRemoteValue([string]$repo) { - Push-Location $ProjectDirectory - $result = "" - try { - $gitRemotes = (git remote -v) - foreach ($remote in $gitRemotes) { - if ($remote.StartsWith("origin")) { - if ($remote -match 'https://github.com/\S+') { - $result = "https://github.com/$repo.git" - break - } elseif ($remote -match "git@github.com:\S+"){ - $result = "git@github.com:$repo.git" - break - } else { - throw "Unknown git remote format found: $remote" - } - } + Push-Location $ProjectDirectory + $result = "" + try { + $gitRemotes = (git remote -v) + foreach ($remote in $gitRemotes) { + Write-Host "Checking remote $remote" + if ($remote.StartsWith("origin") -or $remote.StartsWith("main")) { + if ($remote -match 'https://(.*)?github.com/\S+') { + $result = "https://github.com/$repo.git" + break } + elseif ($remote -match "(.*)?git@github.com:\S+") { + $result = "git@github.com:$repo.git" + break + } + else { + throw "Unknown git remote format found: $remote" + } + } } - finally { - Pop-Location - } - - return $result + } + finally { + Pop-Location + } + Write-Host "Found git remote $result" + return $result } function InitializeSparseGitClone([string]$repo) { - git clone --no-checkout --filter=tree:0 $repo . - if ($LASTEXITCODE) { exit $LASTEXITCODE } - git sparse-checkout init - if ($LASTEXITCODE) { exit $LASTEXITCODE } - Remove-Item $sparseCheckoutFile -Force + git clone --no-checkout --filter=tree:0 $repo . + if ($LASTEXITCODE) { exit $LASTEXITCODE } + git sparse-checkout init + if ($LASTEXITCODE) { exit $LASTEXITCODE } + Remove-Item $sparseCheckoutFile -Force } function GetSpecCloneDir([string]$projectName) { - Push-Location $ProjectDirectory - try { - $root = git rev-parse --show-toplevel - } - finally { - Pop-Location - } - - $sparseSpecCloneDir = "$root/../sparse-spec/$projectName" - New-Item $sparseSpecCloneDir -Type Directory -Force | Out-Null - $createResult = Resolve-Path $sparseSpecCloneDir - return $createResult + Push-Location $ProjectDirectory + try { + $root = git rev-parse --show-toplevel + } + finally { + Pop-Location + } + + $sparseSpecCloneDir = "$root/../sparse-spec/$projectName" + New-Item $sparseSpecCloneDir -Type Directory -Force | Out-Null + $createResult = Resolve-Path $sparseSpecCloneDir + return $createResult } $typespecConfigurationFile = Resolve-Path "$ProjectDirectory/tsp-location.yaml" Write-Host "Reading configuration from $typespecConfigurationFile" $configuration = Get-Content -Path $typespecConfigurationFile -Raw | ConvertFrom-Yaml -$pieces = $typespecConfigurationFile.Path.Replace("\","/").Split("/") +$pieces = $typespecConfigurationFile.Path.Replace("\", "/").Split("/") $projectName = $pieces[$pieces.Count - 2] $specSubDirectory = $configuration["directory"] -if ( $configuration["repo"] -and $configuration["commit"]) { - $specCloneDir = GetSpecCloneDir $projectName - $gitRemoteValue = GetGitRemoteValue $configuration["repo"] - - Write-Host "Setting up sparse clone for $projectName at $specCloneDir" - - Push-Location $specCloneDir.Path - try { - if (!(Test-Path ".git")) { - InitializeSparseGitClone $gitRemoteValue - } - UpdateSparseCheckoutFile $specSubDirectory $configuration["additionalDirectories"] - git checkout $configuration["commit"] - if ($LASTEXITCODE) { exit $LASTEXITCODE } - } - finally { - Pop-Location +# use local spec repo if provided +if ($LocalSpecRepoPath) { + $specCloneDir = $LocalSpecRepoPath +} +elseif ($configuration["repo"] -and $configuration["commit"]) { + # use sparse clone if repo and commit are provided + $specCloneDir = GetSpecCloneDir $projectName + $gitRemoteValue = GetGitRemoteValue $configuration["repo"] + + Write-Host "from tsplocation.yaml 'repo' is:"$configuration["repo"] + Write-Host "Setting up sparse clone for $projectName at $specCloneDir" + + Push-Location $specCloneDir.Path + try { + if (!(Test-Path ".git")) { + Write-Host "Initializing sparse clone for repo: $gitRemoteValue" + InitializeSparseGitClone $gitRemoteValue } -} elseif ( $configuration["spec-root-dir"] ) { - $specCloneDir = $configuration["spec-root-dir"] + Write-Host "Updating sparse checkout file with directory:$specSubDirectory" + UpdateSparseCheckoutFile $specSubDirectory $configuration["additionalDirectories"] + $commit = $configuration["commit"] + Write-Host "git checkout commit: $commit" + git checkout $configuration["commit"] + if ($LASTEXITCODE) { exit $LASTEXITCODE } + } + finally { + Pop-Location + } +} +else { + # write error if neither local spec repo nor repo and commit are provided + Write-Error "Must contain both 'repo' and 'commit' in tsp-location.yaml or input 'localSpecRepoPath' parameter." + exit 1 } - $tempTypeSpecDir = "$ProjectDirectory/TempTypeSpecFiles" New-Item $tempTypeSpecDir -Type Directory -Force | Out-Null CopySpecToProjectIfNeeded ` - -specCloneRoot $specCloneDir ` - -mainSpecDir $specSubDirectory ` - -dest $tempTypeSpecDir ` - -specAdditionalSubDirectories $configuration["additionalDirectories"] + -specCloneRoot $specCloneDir ` + -mainSpecDir $specSubDirectory ` + -dest $tempTypeSpecDir ` + -specAdditionalSubDirectories $configuration["additionalDirectories"] + +exit 0