From 9ac1618e1cf5060a00bb20800a5f8638702f2970 Mon Sep 17 00:00:00 2001 From: Rob Holt Date: Tue, 30 Jul 2019 10:39:11 -0700 Subject: [PATCH 1/6] Add test --- Tests/Engine/DeadlockRegression.Tests.ps1 | 5 + Tests/Engine/DeadlockTestAssets/complex.psm1 | 839 +++++++++++++++++++ 2 files changed, 844 insertions(+) create mode 100644 Tests/Engine/DeadlockRegression.Tests.ps1 create mode 100644 Tests/Engine/DeadlockTestAssets/complex.psm1 diff --git a/Tests/Engine/DeadlockRegression.Tests.ps1 b/Tests/Engine/DeadlockRegression.Tests.ps1 new file mode 100644 index 000000000..1def6e5ef --- /dev/null +++ b/Tests/Engine/DeadlockRegression.Tests.ps1 @@ -0,0 +1,5 @@ +Describe "Complex file analysis" { + It "Analyzes file successfully" { + Invoke-ScriptAnalyzer -Path "$PSScriptRoot/DeadlockTestAssets/complex.psm1" + } +} \ No newline at end of file diff --git a/Tests/Engine/DeadlockTestAssets/complex.psm1 b/Tests/Engine/DeadlockTestAssets/complex.psm1 new file mode 100644 index 000000000..347a7883f --- /dev/null +++ b/Tests/Engine/DeadlockTestAssets/complex.psm1 @@ -0,0 +1,839 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +#region Private implementation functions + +function GetPowerShellName { + switch ($PSVersionTable.PSEdition) { + 'Core' { + return 'PowerShell Core' + } + + default { + return 'Windows PowerShell' + } + } +} + +function Test-ConfigFile { + param( + [Parameter(Mandatory)] + [string] $Path + ) + + if ($Path.EndsWith('pspackageproject.json') -and (Test-Path $Path -PathType Leaf)) { + return $true + } +} + +function SearchConfigFile { + param( + [Parameter(Mandatory)] + [string] $Path + ) + + $startPath = $Path + + do { + $configPath = Join-Path $startPath 'pspackageproject.json' + + if (-not (Test-Path $configPath)) { + $startPath = Split-Path $startPath + } + else { + return $configPath + } + } while ($newPath -ne '') +} + +function Join-Path2 { + param( + [Parameter(Mandatory)] + [string[]] $Path, + + [Parameter(Mandatory)] + [string] $ChildPath, + + [string[]] $AdditionalChildPath + ) + + $paths = [System.Collections.ArrayList]::new() + + $Path | ForEach-Object { $null = $paths.Add($_) } + $null = $paths.Add($ChildPath) + $AdditionalChildPath | ForEach-Object { $null = $paths.Add($_) } + + [System.IO.Path]::Combine($paths) +} + +function RunPwshCommandInSubprocess { + param( + [string] + $Command + ) + + if (-not $script:pwshPath) { + $script:pwshPath = (Get-Process -Id $PID).Path + } + + & $script:pwshPath -NoProfile -NoLogo -Command $Command +} + +function RunProjectBuild { + param( + [string] + $ProjectRoot + ) + + & "$ProjectRoot/build.ps1" -Clean -Build +} + +function GetHelpPath { + param( + [cultureinfo] + $Culture, + + [string] + $ProjectRoot + ) + + $ProjectRoot = Resolve-Path -Path $ProjectRoot + + $config = Get-PSPackageProjectConfiguration + + $cultureName = $Culture.Name + + return (Join-Path2 $ProjectRoot $config.HelpPath $cultureName) +} + +function GetOutputModulePath { + $config = Get-PSPackageProjectConfiguration + return (Join-Path $config.BuildOutputPath $config.ModuleName) +} + +function HasCmdletHelp { + param( + [string] + $HelpResourcePath + ) + + if (-not (Test-Path -Path $HelpResourcePath)) { + return $false + } + + $files = Get-ChildItem -Path $HelpResourcePath -ErrorAction Ignore | + Where-Object Name -Like '*.md' | + Where-Object Name -NotLike 'about_*' + + return $files.Count -gt 0 +} + +function Initialize-CIYml { + param( + [Parameter(Mandatory)] + [string] $Path + ) + + $boilerplateCIYml = Join-Path2 -Path $PSScriptRoot -ChildPath 'yml' -AdditionalChildPath 'ci_for_init.yml' + $destYmlPath = New-Item (Join-Path -Path $Path -ChildPath '.ci') -ItemType Directory + Copy-Item $boilerplateCIYml -Destination (Join-Path $destYmlPath -ChildPath 'ci.yml') -Force + + $boilerplateTestYml = Join-Path2 -Path $PSScriptRoot -ChildPath 'yml' -AdditionalChildPath 'test_for_init.yml' + Copy-Item $boilerplateTestYml -Destination (Join-Path $destYmlPath -ChildPath 'test.yml') -Force + + $boilerplateReleaseYml = Join-Path2 -Path $PSScriptRoot -ChildPath 'yml' -AdditionalChildPath 'release_for_init.yml' + Copy-Item $boilerplateTestYml -Destination (Join-Path $destYmlPath -ChildPath 'release.yml') -Force +} + +function Test-PSPesterResults +{ + [CmdletBinding()] + param( + [Parameter()] + [string] $TestResultsFile = "pester-tests.xml" + ) + + if (!(Test-Path $TestResultsFile)) { + throw "Test result file '$testResultsFile' not found for $TestArea." + } + + $x = [xml](Get-Content -raw $testResultsFile) + if ([int]$x.'test-results'.failures -gt 0) { + Write-Error "TEST FAILURES" + # switch between methods, SelectNode is not available on dotnet core + if ( "System.Xml.XmlDocumentXPathExtensions" -as [Type] ) { + $failures = [System.Xml.XmlDocumentXPathExtensions]::SelectNodes($x."test-results", './/test-case[@result = "Failure"]') + } + else { + $failures = $x.SelectNodes('.//test-case[@result = "Failure"]') + } + foreach ( $testfail in $failures ) { + Show-PSPesterError -testFailure $testfail + } + } +} + +function Show-PSPesterError { + [CmdletBinding()] + param ( + [Parameter(Mandatory)] + [Xml.XmlElement]$testFailure + ) + + + $description = $testFailure.description + $name = $testFailure.name + $message = $testFailure.failure.message + $stackTrace = $testFailure.failure."stack-trace" + + $fullMsg = "`n{0}`n{1}`n{2}`n{3}`{4}" -f ("Description: " + $description), ("Name: " + $name), "message:", $message, "stack-trace:", $stackTrace + + Write-Error $fullMsg +} + +function Invoke-FunctionalValidation { + param ( $tags = "CI" ) + $config = Get-PSPackageProjectConfiguration + try { + $testResultFile = "result.pester.xml" + $testPath = $config.TestPath + $modStage = "{0}/{1}" -f $config.BuildOutputPath,$config.ModuleName + $command = "import-module ${modStage} -Force -Verbose; Set-Location $testPath; Invoke-Pester -Path . -OutputFile ${testResultFile} -tags '$tags'" + $output = RunPwshCommandInSubprocess -command $command | Foreach-Object { Write-Verbose -Verbose $_ } + return (Join-Path ${testPath} "$testResultFile") + } + catch { + $output | Foreach-Object { Write-Warning "$_" } + Write-Error "Error invoking tests" + } +} + +function Invoke-StaticValidation { + $fault = $false + + $config = Get-PSPackageProjectConfiguration + + Write-Verbose "Running ScriptAnalyzer" -Verbose + $resultPSSA = RunScriptAnalysis -Location $config.BuildOutputPath + + Write-Verbose -Verbose "PSSA result file: $resultPSSA" + + Test-PSPesterResults -TestResultsFile $resultPSSA + + Write-Verbose "Running BinSkim" -Verbose + $resultBinSkim = Invoke-BinSkim -Location (Join-Path2 -Path $config.BuildOutputPath -ChildPath $config.ModuleName) + + Test-PSPesterResults -TestResultsFile $resultBinSkim +} + +function RunScriptAnalysis { + try { + Push-Location + + $pssaParams = @{ + Severity = 'Warning', 'ParseError' + Path = GetOutputModulePath + Recurse = $true + } + + $results = Invoke-ScriptAnalyzer @pssaParams + if ( $results ) { + $xmlPath = ConvertPssaDiagnosticsToNUnit -Diagnostic $results + # send back the xml file path. + $xmlPath + if ($env:TF_BUILD) { + $powershellName = GetPowerShellName + Publish-AzDevOpsTestResult -Path $xmlPath -Title "PSScriptAnalyzer $env:AGENT_OS - $powershellName Results" -Type NUnit + } + } + } + finally { + Pop-Location + } +} + +function ConvertPssaDiagnosticsToNUnit { + param( + [Parameter(ValueFromPipeline)] + [Microsoft.Windows.PowerShell.ScriptAnalyzer.Generic.DiagnosticRecord[]] + $Diagnostic + ) + + $sb = [System.Text.StringBuilder]::new() + $null = $sb.Append("Describe 'PSScriptAnalyzer Diagnostics' { `n") + foreach ($d in $Diagnostic) { + $severity = $d.Severity + $ruleName = $d.RuleName + $message = $d.Message -replace "'", "``" + $description = "[$severity] ${ruleName}: $message" + $null = $sb.Append("It '$ruleName' { `nthrow '$message' }`n") + } + $null = $sb.Append('}') + + $testPath = Join-Path ([System.IO.Path]::GetTempPath()) "pssa.tests.ps1" + $xmlPath = Join-Path ([System.IO.Path]::GetTempPath()) "pssa.xml" + + try { + Set-Content -Path $testPath -Value $sb.ToString() + Invoke-Pester -Script $testPath -OutputFormat NUnitXml -OutputFile $xmlPath + } + finally { + Remove-Item -Path $testPath -Force + } + + return $xmlPath +} + +<# +.SYNOPSIS +Generates help file stubs. + +.DESCRIPTION +Generates stubs for about_*.md help documentation for a given module. + +.PARAMETER ProjectRoot +The repository root directory path. + +.PARAMETER ModuleName +The name of the module to generate help for. + +.PARAMETER Culture +The culture or locale the help is to be generated in/for. +#> +function Initialize-PSPackageProjectHelp { + [CmdletBinding()] + param( + [Parameter(Mandatory)] + [string] + $ProjectRoot, + + [Parameter(Mandatory)] + [string] + $ModuleName, + + [Parameter()] + [cultureinfo] + $Culture = [cultureinfo]::CurrentCulture + ) + + $ProjectRoot = Resolve-Path -Path $ProjectRoot + + $helpResourcePath = GetHelpPath -ProjectRoot $ProjectRoot -Culture $Culture + + $null = New-Item -Path $helpResourcePath -ItemType Directory -ErrorAction Stop + + New-MarkdownAboutHelp -OutputFolder $helpResourcePath -AboutName $ModuleName +} + +#endregion Private implementation functions + +#region Public commands + +function Invoke-PSPackageProjectTest { + param( + [Parameter()] + [ValidateSet("Functional", "StaticAnalysis")] + [string[]] + $Type + ) + + END { + $config = Get-PSPackageProjectConfiguration + if ($Type -contains "Functional" ) { + # this will return a path to the results + $resultFile = Invoke-FunctionalValidation + Test-PSPesterResults -TestResultsFile $resultFile + $powershellName = GetPowerShellName + Publish-AzDevOpsTestResult -Path $resultFile -Title "Functional Tests - $env:AGENT_OS - $powershellName Results" -Type NUnit + } + + if ($Type -contains "StaticAnalysis" ) { + Invoke-StaticValidation + } + } +} + +function Invoke-BinSkim { + [CmdletBinding(DefaultParameterSetName = 'byPath')] + param( + [Parameter(ParameterSetName = 'byPath', Mandatory)] + [string] + $Location, + [Parameter(ParameterSetName = 'byPath')] + [string] + $Filter = '*' + ) + + $testscript = @' +Describe "BinSkim" { + BeforeAll{ + $outputPath = Join-Path -Path ([System.io.path]::GetTempPath()) -ChildPath 'pspackageproject-results.json' + if(Test-Path $outputPath) + { + $results = Get-Content $outputPath | ConvertFrom-Json + } + } + + foreach($file in $results.runs.files.PsObject.Properties.Name) + { + foreach($rule in $results.runs.rules.psobject.properties.name) + { + $fileResults = @($results.runs.results | + Where-Object { + Write-Verbose "$($_.ruleId) -eq $rule" + $_.locations.analysisTarget.uri -eq $File -and $_.ruleId -eq $rule}) + + $message = $null + if($fileResults.Count -ne 0) { + $fileResult = $fileResults[0] + $message = $results.runs.rules.$rule.messageFormats.($fileResult.Level) -f ($fileResult.formattedRuleMessage.arguments) + } + + if($message){ + it "$file should not have errors for " { + throw $message + } + } + } + } +} +'@ + $eligbleFiles = @(Get-ChildItem -Path $Location -Filter $Filter -Recurse -File | Where-Object { $_.Extension -in '.exe','.dll','','.so','.dylib'}) + if($eligbleFiles.Count -ne 0) + { + $sourceName = 'Nuget' + Register-PackageSource -ProviderName NuGet -Name $sourceName -Location https://api.nuget.org/v3/index.json -erroraction ignore + $packageName = 'microsoft.codeanalysis.binskim' + $packageLocation = Join-Path2 -Path ([System.io.path]::GetTempPath()) -ChildPath 'pspackageproject-packages' + if ($IsLinux) { + $binaryName = 'BinSkim' + $rid = 'linux-x64' + } + elseif ($IsWindows -ne $false) { + $binaryName = 'BinSkim.exe' + if ([Environment]::Is64BitOperatingSystem) { + $rid = 'win-x64' + } + else { + $rid = 'win-x86' + } + } + else { + Write-Warning "unsupported platform" + return + } + + Write-Verbose "Finding binskim..." -Verbose + $packageInfo = Find-Package -Name $packageName -Source $sourceName + $dirName = $packageInfo.Name + '.' + $packageInfo.Version + $toolLocation = Join-Path2 -Path $packageLocation -ChildPath $dirName -AdditionalChildPath 'tools', 'netcoreapp2.0', $rid, $binaryName + if (!(test-path -path $toolLocation)) { + Write-Verbose "Installing binskim..." -Verbose + $packageInfo | Install-Package -Destination $packageLocation -Force + } + + if ($IsLinux) { + chmod a+x $toolLocation + } + + $resolvedPath = (Resolve-Path -Path $Location).ProviderPath + $toAnalyze = Join-Path2 -Path $resolvedPath -ChildPath $Filter + + $outputPath = Join-Path2 -Path ([System.io.path]::GetTempPath()) -ChildPath 'pspackageproject-results.json' + Write-Verbose "Running binskim..." -Verbose + & $toolLocation analyze $toAnalyze --output $outputPath --pretty-print --recurse > binskim.log 2>&1 + Write-Verbose "binskim exitcode: $LASTEXITCODE" -Verbose + $PowerShellName = GetPowerShellName + Publish-Artifact -Path ./binskim.log -Name "binskim-log-${env:AGENT_OS}-${PowerShellName}" + + $testsPath = Join-Path2 -Path ([System.io.path]::GetTempPath()) -ChildPath 'pspackageproject' -AdditionalChildPath 'BinSkim', 'binskim.tests.ps1' + + $null = New-Item -ItemType Directory -Path (Split-Path $testsPath) + + $testscript | Out-File $testsPath -Force + + Write-Verbose "Generating test results..." -Verbose + + Invoke-Pester -Script $testsPath -OutputFile ./binskim-results.xml -OutputFormat NUnitXml + + Publish-AzDevOpsTestResult -Path ./binskim-results.xml -Title "BinSkim $env:AGENT_OS - $PowerShellName Results" -Type NUnit + return ./binskim-results.xml + } +} + +function Publish-AzDevOpsTestResult { + param( + [parameter(Mandatory)] + [string] + $Path, + [parameter(Mandatory)] + [string] + $Title, + [string] + $Type = 'NUnit' + ) + + $artifactPath = (Resolve-Path $Path).ProviderPath + + Write-Verbose -Verbose "Uploading $artifactPath" + + # Just do nothing if we are not in AzDevOps + if ($env:TF_BUILD) { + Write-Host "##vso[results.publish type=$Type;mergeResults=true;runTitle=$Title;publishRunAttachments=true;resultFiles=$artifactPath;failTaskOnFailedTests=true;]" + } +} + +<# +.SYNOPSIS +Create or update cmdlet help stubs markdown files. + +.DESCRIPTION +Creates or updates the cmdlet help resource files +for the given module. +The generated help will be stubs, requiring regions with {{ }} +to be filled in. + +.PARAMETER ProjectRoot +The path to the repository root of the module. + +.PARAMETER ModuleName +The name of the module to generate cmdlet help for. + +.PARAMETER Culture +The culture in which cmdlet help should be created. +#> +function Add-PSPackageProjectCmdletHelp { + [CmdletBinding()] + param( + [Parameter(Mandatory)] + [string] + $ProjectRoot, + + [Parameter(Mandatory)] + [string] + $ModuleName, + + [Parameter()] + [cultureinfo] + $Culture + ) + + $ProjectRoot = Resolve-Path -Path $ProjectRoot + + $helpResourcePath = GetHelpPath -ProjectRoot $ProjectRoot -Culture $Culture + + RunProjectBuild -ProjectRoot $ProjectRoot + + $outModulePath = GetOutputModulePath -ProjectRoot $ProjectRoot -ModuleName $ModuleName + + if (-not (HasCmdletHelp -HelpResourcePath $helpResourcePath)) { + New-Item -Path $helpResourcePath -ItemType Directory -ErrorAction Ignore + RunPwshCommandInSubprocess -Command "Import-Module '$outModulePath'; New-MarkdownHelp -Module $ModuleName -OutputFolder '$helpResourcePath'" + return + } + + RunPwshCommandInSubprocess -Command "Import-Module '$outModulePath'; Update-MarkdownHelp -Path '$helpResourcePath'" +} + +<# +.SYNOPSIS +Assembles help files into staging output. + +.DESCRIPTION +Compiles markdown help resources into +PowerShell external help files and places +them into the staging location. + +.PARAMETER ProjectRoot +The path to the project repository root. + +.PARAMETER ModuleName +The name of the module to publish help for. + +.PARAMETER Culture +The locale or culture the help is written for. +#> +function Export-PSPackageProjectHelp { + [CmdletBinding()] + param( + [Parameter(Mandatory)] + [string] + $ProjectRoot, + + [Parameter(Mandatory)] + [string] + $ModuleName, + + [Parameter()] + [cultureinfo] + $Culture = [cultureinfo]::CurrentCulture + ) + + $cultureName = $Culture.Name + + $helpResourcePath = GetHelpPath -ProjectRoot $ProjectRoot -Culture $Culture + + $outModulePath = GetOutputModulePath -ProjectRoot $ProjectRoot -ModuleName $ModuleName + + New-ExternalHelp -Path $helpResourcePath -OutputPath "$outModulePath/$cultureName" -Force +} + +function Invoke-PSPackageProjectBuild { + [CmdletBinding()] + param( + [Parameter()] + [ScriptBlock] + $BuildScript + ) + + Write-Verbose -Verbose "Invoking build script" + + $BuildScript.Invoke() + + New-PSPackageProjectPackage + + Write-Verbose -Verbose "Finished invoking build script" +} + +function New-PSPackageProjectPackage +{ + Write-Verbose -Message "Starting New-PSPackageProjectPackage" -Verbose + $ErrorActionPreference = 'Stop' + $config = Get-PSPackageProjectConfiguration + $modulePath = Join-Path2 -Path $config.BuildOutputPath -ChildPath $config.ModuleName + $sourceName = 'pspackageproject-local-repo' + $packageLocation = Join-Path2 -Path ([System.io.path]::GetTempPath()) -ChildPath $sourceName + $modulesLocation = Join-Path2 -Path $packageLocation -ChildPath 'modules' + + if (Test-Path $modulesLocation) { + Remove-Item $modulesLocation -Recurse -Force -ErrorAction Ignore + } + + $null = New-Item -Path $modulesLocation -Force -ItemType Directory + $scriptsLocation = $modulesLocation + + Write-Verbose -Message "Starting dependency download" -Verbose + + # TODO : dynamically detect module dependecies and save them + Save-Package2 -Name PlatyPs, Pester -Location $modulesLocation + Save-Package2 -Name PSScriptAnalyzer -RequiredVersion '1.18.0' -Location $modulesLocation + + Write-Verbose -Message "Dependency download complete" -Verbose + if (!(Get-PSRepository -Name $sourceName -ErrorAction Ignore)) { + Register-PSRepository -Name $sourceName -SourceLocation $modulesLocation -PublishLocation $modulesLocation + } + + Write-Verbose -Verbose "Starting to publish module: $modulePath" + Publish-Module -Path $modulePath -Repository $sourceName -NuGetApiKey 'fake' -Force + + Write-Verbose -Message "Local package published" -Verbose + + $nupkgPath = (Get-ChildItem -Path $modulesLocation -Filter "$($config.ModuleName)*.nupkg").FullName + Publish-Artifact -Path $nupkgPath -Name nupkg + + Write-Verbose -Message "Starting New-PSPackageProjectPackage" -Verbose +} + +# Wrapper to push artifact +function Publish-Artifact +{ + param( + [Parameter(Mandatory)] + [ValidateScript({Test-Path -Path $_})] + $Path, + [string] + $Name + ) + + $resolvedPath = (Resolve-Path -Path $Path).ProviderPath + + if(!$Name) + { + $artifactName = [system.io.path]::GetFileName($Path) + } + else + { + $artifactName = $Name + } + + if ($env:TF_BUILD) { + # In Azure DevOps + Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$resolvedPath" + } +} + +function Save-Package2 +{ + param( + [string[]] + $Name, + [String] + $Location, + [string] + $RequiredVersion + ) + + if($RequiredVersion) { + Save-Package -Name $Name -Source 'https://www.powershellgallery.com/api/v2' -Path $Location -ProviderName NuGet -RequiredVersion $RequiredVersion + } else { + Save-Package -Name $Name -Source 'https://www.powershellgallery.com/api/v2' -Path $Location -ProviderName NuGet + } + +} + +function Initialize-PSPackageProject { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true, Position = 0)] + [string]$ModuleName, + [string]$ModuleBase = ".", + [switch]$Force + ) + + if ( [System.Management.Automation.WildcardPattern]::ContainsWildcardCharacters($ModuleBase) ) { + throw "Modulebase '${ModuleBase}' contains wildcards" + } + + $ModuleRoot = (Resolve-Path $ModuleBase -ea SilentlyContinue ).Path + if ( $ModuleRoot -and ! $force ) { + throw "'${ModuleRoot}' already exists, use -Force to overwrite" + } + + if ( ! $ModuleRoot ) { + $ModuleRoot = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($ModuleBase) + } + $null = New-Item -ItemType Directory -Path $ModuleRoot + $ModuleInfo = @{ + ModuleName = $ModuleName + ModuleRoot = $ModuleRoot + } + + # Create the help directory + # and populate a couple of files + Initialize-PSPackageProjectHelp -ProjectRoot $ModuleRoot -ModuleName $ModuleName + + # Create the scaffold for .psd1 and .psm1 + $moduleSourceBase = Join-Path $ModuleRoot "src" + $null = New-Item -ItemType Directory -Path $moduleSourceBase + $moduleFileWithoutExtension = Join-Path $moduleSourceBase ${ModuleName} + New-ModuleManifest -Path "${moduleFileWithoutExtension}.psd1" -CmdletsToExport "verb-noun" -RootModule "./${ModuleName}.dll" + $null = New-Item -Type File "${moduleFileWithoutExtension}.psm1" + + # Create a directory for cs sources and create a classlib csproj file with + # System.Management.Automation as a package reference + $moduleCodeBase = Join-Path $moduleSourceBase "code" + $null = New-Item -ItemType Directory -Path $moduleCodeBase + try { + Push-Location $moduleCodeBase + $output = dotnet new classlib -f netstandard2.0 --no-restore --force + $output += dotnet add package PowerShellStandard.Library --no-restore + Move-Item code.csproj "${ModuleName}.csproj" + @" +using System; +using System.Management.Automation; + +namespace ${ModuleName} +{ + [Cmdlet("verb","noun")] + public class Cmdlet1 : PSCmdlet + { + [Parameter(Mandatory=true,Position=0)] + public string Name {get;set;} + + protected override void ProcessRecord() + { + WriteObject(Name); + } + } +} +"@ > Class1.cs + } + finally { + Pop-Location + } + + # make test folder and create a test template + $testDir = Join-Path $moduleRoot 'test' + $testTemplate = Join-Path $testDir "${moduleName}.Tests.ps1" + $null = New-Item -ItemType Directory -Path "${testDir}" + @" +Describe "Test ${moduleName}" -tags CI { + BeforeAll { + } + BeforeEach { + } + AfterEach { + } + AfterAll { + } + It "This is the first test for ${moduleName}" { + `$name = "Hello World" + verb-noun -name `$name | Should -BeExactly `$name + } +} +"@ | Out-File "${testTemplate}" + + # make CI ymls + Initialize-CIYml -Path ${moduleRoot} + + # make build.ps1 + $boilerplateBuildScript = Join-Path -Path $PSScriptRoot -ChildPath 'build_for_init.ps1' + Copy-Item $boilerplateBuildScript -Destination (Join-Path $ModuleRoot -ChildPath 'build.ps1') -Force + + # make pspackageproject.json + $jsonPrj = + @{ + SourcePath = "src" + ModuleName = "${ModuleName}" + TestPath = 'test' + HelpPath = 'help' + BuildOutputPath = 'out' + Culture = [CultureInfo]::CurrentCulture.Name # This needs to be settable + } | ConvertTo-Json + + if ($(${PSVersionTable}.PSEdition) -eq 'Desktop') { + Write-Warning -Message "UTF-8 characters for module name are not supported in Windows PowerShell." + $jsonPrj | Out-File (Join-Path ${moduleRoot} "pspackageproject.json") -Encoding ascii + } + else { + $jsonPrj | Out-File (Join-Path ${moduleRoot} "pspackageproject.json") -Encoding utf8NoBOM + } +} + +function Get-PSPackageProjectConfiguration { + param( + [Parameter()] + [string] $ConfigPath = "." + ) + + $resolvedPath = Resolve-Path $ConfigPath + + $foundConfigFilePath = if (Test-Path $resolvedPath -PathType Container) { + SearchConfigFile -Path $resolvedPath + } + else { + if (Test-ConfigFile -Path $resolvedPath) { + $resolvedPath + } + } + + if (Test-Path $foundConfigFilePath) { + $configObj = Get-Content -Path $foundConfigFilePath | ConvertFrom-Json + + # Populate with full paths + + $projectRoot = Split-Path $foundConfigFilePath + + $configObj.SourcePath = Join-Path $projectRoot -ChildPath $configObj.SourcePath + $configObj.TestPath = Join-Path $projectRoot -ChildPath $configObj.TestPath + $configObj.HelpPath = Join-Path $projectRoot -ChildPath $configObj.HelpPath + $configObj.BuildOutputPath = Join-Path $projectRoot -ChildPath $configObj.BuildOutputPath + + $configObj + } + else { + throw "'pspackageproject.json' not found at: $resolvePath or any or its parent" + } +} + +#endregion Public commands From 8283030b6a26d40cb7be92720b7e4654baae4ec4 Mon Sep 17 00:00:00 2001 From: Rob Holt Date: Tue, 20 Aug 2019 16:53:14 -0700 Subject: [PATCH 2/6] Partial fix --- Rules/UseCmdletCorrectly.cs | 83 ++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/Rules/UseCmdletCorrectly.cs b/Rules/UseCmdletCorrectly.cs index 5293eaea9..ad51b0f37 100644 --- a/Rules/UseCmdletCorrectly.cs +++ b/Rules/UseCmdletCorrectly.cs @@ -7,6 +7,7 @@ using System.Management.Automation; using System.Management.Automation.Language; using Microsoft.Windows.PowerShell.ScriptAnalyzer.Generic; +using System.Collections.Concurrent; #if !CORECLR using System.ComponentModel.Composition; #endif @@ -22,6 +23,24 @@ namespace Microsoft.Windows.PowerShell.ScriptAnalyzer.BuiltinRules #endif public class UseCmdletCorrectly : IScriptRule { + private static readonly ConcurrentDictionary>> s_pkgMgmtMandatoryParameters = + new ConcurrentDictionary>>(new Dictionary>> + { + { "Find-Package", Array.Empty>() }, + { "Find-PackageProvider", Array.Empty>() }, + { "Get-Package", Array.Empty>() }, + { "Get-PackageProvider", Array.Empty>() }, + { "Get-PackageSource", Array.Empty>() }, + { "Import-PackageProvider", new string[][] { new [] { "Name" } } }, + { "Install-Package", new string[][] { new [] { "Name" } } }, + { "Install-PackageProvider", new string[][] { new [] { "Name" } } }, + { "Register-PackageSource", new string[][] { new [] { "ProviderName" } } }, + { "Save-Package", new string[][] { new [] { "Name" }, new [] { "InputObject" } } }, + { "Set-PackageSource", new string[][] { new [] { "Name" }, new [] { "Location" } } }, + { "Uninstall-Package", new string[][] { new [] { "Name" }, new [] { "InputObject" } } }, + { "Unregister-PackageSource", new string[][] { new [] { "Name" }, new [] { "InputObject" } } }, + }); + /// /// AnalyzeScript: Check that cmdlets are invoked with the correct mandatory parameter /// @@ -61,41 +80,42 @@ public IEnumerable AnalyzeScript(Ast ast, string fileName) /// private bool MandatoryParameterExists(CommandAst cmdAst) { - CommandInfo cmdInfo = null; - List mandParams = new List(); - IEnumerable ceAsts = null; - bool returnValue = false; + #region Compares parameter list and mandatory parameter list. - #region Predicates + CommandInfo cmdInfo = Helper.Instance.GetCommandInfoLegacy(cmdAst.GetCommandName()); - // Predicate to find ParameterAsts. - Func foundParamASTs = delegate(CommandElementAst ceAst) + // If we can't resolve the command or it's not a cmdlet, we are done + if (cmdInfo == null || (cmdInfo.CommandType != System.Management.Automation.CommandTypes.Cmdlet)) { - if (ceAst is CommandParameterAst) return true; - return false; - }; - - #endregion + return true; + } - #region Compares parameter list and mandatory parameter list. + // We can't statically analyze splatted variables, so ignore them + if (Helper.Instance.HasSplattedVariable(cmdAst)) + { + return true; + } - cmdInfo = Helper.Instance.GetCommandInfoLegacy(cmdAst.GetCommandName()); - if (cmdInfo == null || (cmdInfo.CommandType != System.Management.Automation.CommandTypes.Cmdlet)) + // Positional parameters could be mandatory, so we assume all is well + if (Helper.Instance.PositionalParameterUsed(cmdAst) && Helper.Instance.IsKnownCmdletFunctionOrExternalScript(cmdAst)) { return true; } - // ignores if splatted variable is used - if (Helper.Instance.HasSplattedVariable(cmdAst)) + // If the command is piped to, this also precludes mandatory parameters + if (cmdAst.Parent is PipelineAst parentPipeline + && parentPipeline.PipelineElements.Count > 1 + && parentPipeline.PipelineElements[0] != cmdAst) { return true; } - // Gets parameters from command elements. - ceAsts = cmdAst.CommandElements.Where(foundParamASTs); + // We now need to look at all explicit parameters in the given command AST + IEnumerable commandParameterAst = cmdAst.CommandElements.OfType(); // Gets mandatory parameters from cmdlet. // If cannot find any mandatory parameter, it's not necessary to do a further check for current cmdlet. + var mandatoryParameters = new List(); try { int noOfParamSets = cmdInfo.ParameterSets.Count; @@ -119,7 +139,7 @@ private bool MandatoryParameterExists(CommandAst cmdAst) if (count >= noOfParamSets) { - mandParams.Add(pm); + mandatoryParameters.Add(pm); } } } @@ -129,28 +149,25 @@ private bool MandatoryParameterExists(CommandAst cmdAst) return true; } - if (mandParams.Count == 0 || (Helper.Instance.IsKnownCmdletFunctionOrExternalScript(cmdAst) && Helper.Instance.PositionalParameterUsed(cmdAst))) + if (mandatoryParameters.Count == 0) { - returnValue = true; + return true; } - else + + // Compares parameter list and mandatory parameter list. + foreach (CommandElementAst commandElementAst in commandParameterAst) { - // Compares parameter list and mandatory parameter list. - foreach (CommandElementAst ceAst in ceAsts) + CommandParameterAst cpAst = (CommandParameterAst)commandElementAst; + if (mandatoryParameters.Count(item => + item.Name.Equals(cpAst.ParameterName, StringComparison.OrdinalIgnoreCase)) > 0) { - CommandParameterAst cpAst = (CommandParameterAst)ceAst; - if (mandParams.Count(item => - item.Name.Equals(cpAst.ParameterName, StringComparison.OrdinalIgnoreCase)) > 0) - { - returnValue = true; - break; - } + return true; } } #endregion - return returnValue; + return false; } /// From 95ad68b330b38a3ed60ce0e69f8e62b05a6ed618 Mon Sep 17 00:00:00 2001 From: Robert Holt Date: Wed, 21 Aug 2019 09:36:04 -0700 Subject: [PATCH 3/6] Implement workaround for PackageManagement cmdlets --- Rules/UseCmdletCorrectly.cs | 63 ++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/Rules/UseCmdletCorrectly.cs b/Rules/UseCmdletCorrectly.cs index ad51b0f37..f21c19e06 100644 --- a/Rules/UseCmdletCorrectly.cs +++ b/Rules/UseCmdletCorrectly.cs @@ -23,22 +23,22 @@ namespace Microsoft.Windows.PowerShell.ScriptAnalyzer.BuiltinRules #endif public class UseCmdletCorrectly : IScriptRule { - private static readonly ConcurrentDictionary>> s_pkgMgmtMandatoryParameters = - new ConcurrentDictionary>>(new Dictionary>> + private static readonly ConcurrentDictionary> s_pkgMgmtMandatoryParameters = + new ConcurrentDictionary>(new Dictionary> { - { "Find-Package", Array.Empty>() }, - { "Find-PackageProvider", Array.Empty>() }, - { "Get-Package", Array.Empty>() }, - { "Get-PackageProvider", Array.Empty>() }, - { "Get-PackageSource", Array.Empty>() }, - { "Import-PackageProvider", new string[][] { new [] { "Name" } } }, - { "Install-Package", new string[][] { new [] { "Name" } } }, - { "Install-PackageProvider", new string[][] { new [] { "Name" } } }, - { "Register-PackageSource", new string[][] { new [] { "ProviderName" } } }, - { "Save-Package", new string[][] { new [] { "Name" }, new [] { "InputObject" } } }, - { "Set-PackageSource", new string[][] { new [] { "Name" }, new [] { "Location" } } }, - { "Uninstall-Package", new string[][] { new [] { "Name" }, new [] { "InputObject" } } }, - { "Unregister-PackageSource", new string[][] { new [] { "Name" }, new [] { "InputObject" } } }, + { "Find-Package", Array.Empty() }, + { "Find-PackageProvider", Array.Empty() }, + { "Get-Package", Array.Empty() }, + { "Get-PackageProvider", Array.Empty() }, + { "Get-PackageSource", Array.Empty() }, + { "Import-PackageProvider", new string[] { "Name" } }, + { "Install-Package", new string[] { "Name" } }, + { "Install-PackageProvider", new string[] { "Name" } }, + { "Register-PackageSource", new string[] { "ProviderName" } }, + { "Save-Package", new string[] { "Name", "InputObject" } }, + { "Set-PackageSource", new string[] { "Name", "Location" } }, + { "Uninstall-Package", new string[] { "Name", "InputObject" } }, + { "Unregister-PackageSource", new string[] { "Name", "InputObject" } }, }); /// @@ -110,8 +110,35 @@ private bool MandatoryParameterExists(CommandAst cmdAst) return true; } - // We now need to look at all explicit parameters in the given command AST - IEnumerable commandParameterAst = cmdAst.CommandElements.OfType(); + // We want to check cmdlets from PackageManagement separately because they experience a deadlock + // when cmdInfo.Parameters or cmdInfo.ParameterSets is accessed. + // See https://github.com/PowerShell/PSScriptAnalyzer/issues/1297 + if (s_pkgMgmtMandatoryParameters.TryGetValue(cmdInfo.Name, out IReadOnlyList pkgMgmtCmdletMandatoryParams)) + { + // If the command has no parameter sets with mandatory parameters, we are done + if (pkgMgmtCmdletMandatoryParams.Count == 0) + { + return true; + } + + // We make the following simplifications here that all apply to the PackageManagement cmdlets: + // - Only one mandatory parameter per parameter set + // - Any part of the parameter prefix is valid + // - There are no parameter sets without mandatory parameters + IEnumerable parameterAsts = cmdAst.CommandElements.OfType(); + foreach (string mandatoryParameter in pkgMgmtCmdletMandatoryParams) + { + foreach (CommandParameterAst parameterAst in parameterAsts) + { + if (mandatoryParameter.StartsWith(parameterAst.ParameterName)) + { + return true; + } + } + } + + return false; + } // Gets mandatory parameters from cmdlet. // If cannot find any mandatory parameter, it's not necessary to do a further check for current cmdlet. @@ -155,7 +182,7 @@ private bool MandatoryParameterExists(CommandAst cmdAst) } // Compares parameter list and mandatory parameter list. - foreach (CommandElementAst commandElementAst in commandParameterAst) + foreach (CommandElementAst commandElementAst in cmdAst.CommandElements.OfType()) { CommandParameterAst cpAst = (CommandParameterAst)commandElementAst; if (mandatoryParameters.Count(item => From dd12dd5a142a311531e17db2283811ef5d0e8b69 Mon Sep 17 00:00:00 2001 From: Robert Holt Date: Wed, 21 Aug 2019 09:42:56 -0700 Subject: [PATCH 4/6] Fix formatting on test file, add copyright --- Tests/Engine/DeadlockRegression.Tests.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Tests/Engine/DeadlockRegression.Tests.ps1 b/Tests/Engine/DeadlockRegression.Tests.ps1 index 1def6e5ef..a330c1953 100644 --- a/Tests/Engine/DeadlockRegression.Tests.ps1 +++ b/Tests/Engine/DeadlockRegression.Tests.ps1 @@ -1,5 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + Describe "Complex file analysis" { It "Analyzes file successfully" { Invoke-ScriptAnalyzer -Path "$PSScriptRoot/DeadlockTestAssets/complex.psm1" } -} \ No newline at end of file +} From 3a9d89446b49002c42dfe1660388c13b2d04dba6 Mon Sep 17 00:00:00 2001 From: Robert Holt Date: Wed, 21 Aug 2019 11:12:29 -0700 Subject: [PATCH 5/6] Fix net452 issues --- Rules/UseCmdletCorrectly.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Rules/UseCmdletCorrectly.cs b/Rules/UseCmdletCorrectly.cs index f21c19e06..57d295722 100644 --- a/Rules/UseCmdletCorrectly.cs +++ b/Rules/UseCmdletCorrectly.cs @@ -26,11 +26,11 @@ public class UseCmdletCorrectly : IScriptRule private static readonly ConcurrentDictionary> s_pkgMgmtMandatoryParameters = new ConcurrentDictionary>(new Dictionary> { - { "Find-Package", Array.Empty() }, - { "Find-PackageProvider", Array.Empty() }, - { "Get-Package", Array.Empty() }, - { "Get-PackageProvider", Array.Empty() }, - { "Get-PackageSource", Array.Empty() }, + { "Find-Package", new string[0] }, + { "Find-PackageProvider", new string[0] }, + { "Get-Package", new string[0] }, + { "Get-PackageProvider", new string[0] }, + { "Get-PackageSource", new string[0] }, { "Import-PackageProvider", new string[] { "Name" } }, { "Install-Package", new string[] { "Name" } }, { "Install-PackageProvider", new string[] { "Name" } }, From 8ba2eef35046da4049a3317b08ed36b3f659cea1 Mon Sep 17 00:00:00 2001 From: Robert Holt Date: Wed, 21 Aug 2019 20:10:59 -0700 Subject: [PATCH 6/6] Add comments to address @JamesWTruher's feedback --- Rules/UseCmdletCorrectly.cs | 3 +++ Tests/Engine/DeadlockTestAssets/complex.psm1 | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/Rules/UseCmdletCorrectly.cs b/Rules/UseCmdletCorrectly.cs index 57d295722..7762538e4 100644 --- a/Rules/UseCmdletCorrectly.cs +++ b/Rules/UseCmdletCorrectly.cs @@ -23,6 +23,9 @@ namespace Microsoft.Windows.PowerShell.ScriptAnalyzer.BuiltinRules #endif public class UseCmdletCorrectly : IScriptRule { + // Cache of the mandatory parameters of cmdlets in PackageManagement + // Key: Cmdlet name + // Value: List of mandatory parameters private static readonly ConcurrentDictionary> s_pkgMgmtMandatoryParameters = new ConcurrentDictionary>(new Dictionary> { diff --git a/Tests/Engine/DeadlockTestAssets/complex.psm1 b/Tests/Engine/DeadlockTestAssets/complex.psm1 index 347a7883f..d0b30d398 100644 --- a/Tests/Engine/DeadlockTestAssets/complex.psm1 +++ b/Tests/Engine/DeadlockTestAssets/complex.psm1 @@ -1,6 +1,11 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +# Copy of https://github.com/TravisEz13/PSPackageProject/blob/master/src/PSPackageProject.psm1 +# References to PackageManagement cmdlets in this file have reproduced deadlocks in PSSA most consistently +# See https://github.com/PowerShell/PSScriptAnalyzer/issues/1297 +# For the specific lines in this file that can cause deadlocks, search NOTE in this file + #region Private implementation functions function GetPowerShellName { @@ -424,11 +429,13 @@ Describe "BinSkim" { } Write-Verbose "Finding binskim..." -Verbose + # NOTE: Deadlock occurs here on Find-Package $packageInfo = Find-Package -Name $packageName -Source $sourceName $dirName = $packageInfo.Name + '.' + $packageInfo.Version $toolLocation = Join-Path2 -Path $packageLocation -ChildPath $dirName -AdditionalChildPath 'tools', 'netcoreapp2.0', $rid, $binaryName if (!(test-path -path $toolLocation)) { Write-Verbose "Installing binskim..." -Verbose + # NOTE: Deadlock occurs here on Install-Package $packageInfo | Install-Package -Destination $packageLocation -Force } @@ -674,8 +681,10 @@ function Save-Package2 ) if($RequiredVersion) { + # NOTE: Deadlock occurs here on Save-Package Save-Package -Name $Name -Source 'https://www.powershellgallery.com/api/v2' -Path $Location -ProviderName NuGet -RequiredVersion $RequiredVersion } else { + # NOTE: Deadlock occurs here on Save-Package Save-Package -Name $Name -Source 'https://www.powershellgallery.com/api/v2' -Path $Location -ProviderName NuGet }