From 81e3ec524cd860d27f9c965eb2405742f44c78fe Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Thu, 18 Apr 2019 07:01:20 -0700 Subject: [PATCH] Enable parallel runs of platyPS per version (#4171) * make building versions of docs in parallel * fix typo in .gitignore update build.ps1 to build the different versions in parallel * install ThreadJob if not available --- .gitignore | 2 +- build.ps1 | 211 +++++++++++++++++++++++++++++++---------------------- 2 files changed, 126 insertions(+), 87 deletions(-) diff --git a/.gitignore b/.gitignore index c82e6ec3437..3dfc0a51dfe 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ _site/ Tools/NuGet/ _site/ maml/ -updateablehelp/ +updatablehelp/ .openpublishing.build.mdproj diff --git a/build.ps1 b/build.ps1 index 60ba83b837f..b3c65d0684f 100644 --- a/build.ps1 +++ b/build.ps1 @@ -5,37 +5,7 @@ param( # Turning off the progress display, by default $global:ProgressPreference = 'SilentlyContinue' -if($ShowProgress){$ProgressPreference = 'Continue'} - -function Get-ContentWithoutHeader { - param( - $path - ) - - $doc = Get-Content $path -Encoding UTF8 - $start = $end = -1 - - # search the first 30 lines for the Yaml header - # no yaml header in our docset will ever be that long - - for ($x = 0; $x -lt 30; $x++) { - if ($doc[$x] -eq '---') { - if ($start -eq -1) { - $start = $x - } else { - if ($end -eq -1) { - $end = $x+1 - break - } - } - } - } - if ($end -gt $start) { - Write-Output ($doc[$end..$($doc.count)] -join "`r`n") - } else { - Write-Output ($doc -join "`r`n") - } - } +if ($ShowProgress) { $ProgressPreference = 'Continue' } [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 @@ -50,86 +20,155 @@ Invoke-WebRequest -Uri $pandocSourceURL -OutFile $pandocZipPath Expand-Archive -Path (Join-Path $pandocDestinationPath "pandoc-$panDocVersion-windows.zip") -DestinationPath $pandocDestinationPath -Force $pandocExePath = Join-Path (Join-Path $pandocDestinationPath "pandoc-$panDocVersion") "pandoc.exe" +# Install ThreadJob if not available +$threadJob = Get-Module ThreadJob -ListAvailable +if ($null -eq $threadjob) { + Install-Module ThreadJob -RequiredVersion 1.1.2 -Scope CurrentUser -Force +} + # Find the reference folder path w.r.t the script $ReferenceDocset = Join-Path $PSScriptRoot 'reference' -# Variable to collect any errors in during processing -$allErrors = @() - # Go through all the directories in the reference folder +$jobs = [System.Collections.Generic.List[object]]::new() Get-ChildItem $ReferenceDocset -Directory -Exclude 'docs-conceptual', 'mapping', 'bread' | ForEach-Object -Process { - $Version = $_.Name - Write-Verbose -Verbose "Version = $Version" + $job = Start-ThreadJob -Name $_.Name -ArgumentList @($SkipCabs,$pandocExePath,$PSScriptRoot,$_) -ScriptBlock { + param($SkipCabs, $pandocExePath, $WorkingDirectory, $DocSet) + + $tempDir = [System.IO.Path]::GetTempPath() + $workingDir = Join-Path $tempDir $DocSet.Name + $workingDir = New-Item -ItemType Directory -Path $workingDir -Force + Set-Location $WorkingDir + + function Get-ContentWithoutHeader { + param( + $path + ) + + $doc = Get-Content $path -Encoding UTF8 + $start = $end = -1 + + # search the first 30 lines for the Yaml header + # no yaml header in our docset will ever be that long + + for ($x = 0; $x -lt 30; $x++) { + if ($doc[$x] -eq '---') { + if ($start -eq -1) { + $start = $x + } + else { + if ($end -eq -1) { + $end = $x + 1 + break + } + } + } + } + if ($end -gt $start) { + Write-Output ($doc[$end..$($doc.count)] -join "`r`n") + } + else { + Write-Output ($doc -join "`r`n") + } + } - $VersionFolder = $_.FullName - Write-Verbose -Verbose "VersionFolder = $VersionFolder" + $Version = $DocSet.Name + Write-Verbose -Verbose "Version = $Version" - # For each of the directories, go through each module folder - Get-ChildItem $VersionFolder -Directory | ForEach-Object -Process { - $ModuleName = $_.Name - Write-Verbose -Verbose "ModuleName = $ModuleName" + $VersionFolder = $DocSet.FullName + Write-Verbose -Verbose "VersionFolder = $VersionFolder" - $ModulePath = Join-Path $VersionFolder $ModuleName - Write-Verbose -Verbose "ModulePath = $ModulePath" + # For each of the directories, go through each module folder + Get-ChildItem $VersionFolder -Directory | ForEach-Object -Process { + $ModuleName = $_.Name + Write-Verbose -Verbose "ModuleName = $ModuleName" - $LandingPage = Join-Path $ModulePath "$ModuleName.md" - Write-Verbose -Verbose "LandingPage = $LandingPage" + $ModulePath = Join-Path $VersionFolder $ModuleName + Write-Verbose -Verbose "ModulePath = $ModulePath" - $MamlOutputFolder = Join-Path "$PSScriptRoot\maml" "$Version\$ModuleName" - Write-Verbose -Verbose "MamlOutputFolder = $MamlOutputFolder" + $LandingPage = Join-Path $ModulePath "$ModuleName.md" + Write-Verbose -Verbose "LandingPage = $LandingPage" - $CabOutputFolder = Join-Path "$PSScriptRoot\updatablehelp" "$Version\$ModuleName" - Write-Verbose -Verbose "CabOutputFolder = $CabOutputFolder" + $MamlOutputFolder = Join-Path "$WorkingDirectory\maml" "$Version\$ModuleName" + Write-Verbose -Verbose "MamlOutputFolder = $MamlOutputFolder" - if (-not (Test-Path $MamlOutputFolder)) { - New-Item $MamlOutputFolder -ItemType Directory -Force > $null - } + $CabOutputFolder = Join-Path "$WorkingDirectory\updatablehelp" "$Version\$ModuleName" + Write-Verbose -Verbose "CabOutputFolder = $CabOutputFolder" - # Process the about topics if any - $AboutFolder = Join-Path $ModulePath "About" - - if (Test-Path $AboutFolder) { - Write-Verbose -Verbose "AboutFolder = $AboutFolder" - Get-ChildItem "$aboutfolder/about_*.md" | ForEach-Object { - $aboutFileFullName = $_.FullName - $aboutFileOutputName = "$($_.BaseName).help.txt" - $aboutFileOutputFullName = Join-Path $MamlOutputFolder $aboutFileOutputName - - $pandocArgs = @( - "--from=gfm", - "--to=plain+multiline_tables+inline_code_attributes", - "--columns=75", - "--output=$aboutFileOutputFullName", - "--quiet" - ) - - Get-ContentWithoutHeader $aboutFileFullName | & $pandocExePath $pandocArgs + if (-not (Test-Path $MamlOutputFolder)) { + New-Item $MamlOutputFolder -ItemType Directory -Force > $null } - } - try { - # For each module, create a single maml help file - # Adding warningaction=stop to throw errors for all warnings, erroraction=stop to make them terminating errors - New-ExternalHelp -Path $ModulePath -OutputPath $MamlOutputFolder -Force -WarningAction Stop -ErrorAction Stop + # Process the about topics if any + $AboutFolder = Join-Path $ModulePath "About" + + if (Test-Path $AboutFolder) { + Write-Verbose -Verbose "AboutFolder = $AboutFolder" + Get-ChildItem "$aboutfolder/about_*.md" | ForEach-Object { + $aboutFileFullName = $_.FullName + $aboutFileOutputName = "$($_.BaseName).help.txt" + $aboutFileOutputFullName = Join-Path $MamlOutputFolder $aboutFileOutputName + + $pandocArgs = @( + "--from=gfm", + "--to=plain+multiline_tables+inline_code_attributes", + "--columns=75", + "--output=$aboutFileOutputFullName", + "--quiet" + ) + + Get-ContentWithoutHeader $aboutFileFullName | & $pandocExePath $pandocArgs + } + } + + try { + # For each module, create a single maml help file + # Adding warningaction=stop to throw errors for all warnings, erroraction=stop to make them terminating errors + New-ExternalHelp -Path $ModulePath -OutputPath $MamlOutputFolder -Force -WarningAction Stop -ErrorAction Stop - # For each module, create update-help help files (cab and helpinfo.xml files) - if (-not $SkipCabs) { - $cabInfo = New-ExternalHelpCab -CabFilesFolder $MamlOutputFolder -LandingPagePath $LandingPage -OutputFolder $CabOutputFolder + # For each module, create update-help help files (cab and helpinfo.xml files) + if (-not $SkipCabs) { + $cabInfo = New-ExternalHelpCab -CabFilesFolder $MamlOutputFolder -LandingPagePath $LandingPage -OutputFolder $CabOutputFolder - # Only output the cab fileinfo object - if ($cabInfo.Count -eq 8) {$cabInfo[-1].FullName} + # Only output the cab fileinfo object + if ($cabInfo.Count -eq 8) { $cabInfo[-1].FullName } + } + } + catch { + Write-Error -Message "PlatyPS failure: $ModuleName -- $Version" -Exception $_ } } - catch { - $allErrors += $_ - Write-Error -Message "PlatyPS failure: $ModuleName -- $Version" -Exception $_ + + Remove-Item $workingDir -Force -ErrorAction SilentlyContinue + } + Write-Verbose -Verbose "Started job for $($_.Name)" + $jobs += $job +} + +$null = $jobs | Wait-Job + +# Variable to collect any errors in during processing +$allErrors = [System.Collections.Generic.List[string]]::new() +foreach ($job in $jobs) { + Write-Verbose -Verbose "$($job.Name) output:" + if ($job.Verbose.Count -gt 0) { + foreach ($verboseMessage in $job.Verbose) { + Write-Verbose -Verbose $verboseMessage } } + if ($job.State -eq "Failed") { + $allErrors += "$($job.Name) failed due to unhandled exception" + } + + if ($job.Error.Count -gt 0) { + $allErrors += "$($job.Name) failed with errors:" + $allErrors += $job.Error.ReadAll() + } } # If the above block, produced any errors, throw and fail the job -if ($allErrors) { +if ($allErrors.Count -gt 0) { $allErrors throw "There are errors during platyPS run!`nPlease fix your markdown to comply with the schema: https://github.com/PowerShell/platyPS/blob/master/platyPS.schema.md" }