From 5b6220007e81d726ca73ec8906edc66691a6662a Mon Sep 17 00:00:00 2001 From: Kaleb Luedtke Date: Fri, 8 Oct 2021 13:36:18 -0500 Subject: [PATCH 1/6] Ensure file names are valid before saving --- Tools/YamlCreate.ps1 | 83 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 16 deletions(-) diff --git a/Tools/YamlCreate.ps1 b/Tools/YamlCreate.ps1 index 1479350060f08d..e4aa764d57c4dd 100644 --- a/Tools/YamlCreate.ps1 +++ b/Tools/YamlCreate.ps1 @@ -293,6 +293,14 @@ Function TestUrlValidity { return $HTTP_Status } +# Checks a file name for validite and returns a boolean value +function Test-ValidFileName { + param([string]$FileName) + $IndexOfInvalidChar = $FileName.IndexOfAny([System.IO.Path]::GetInvalidFileNameChars()) + # IndexOfAny() returns the value -1 to indicate no such character was found + return $IndexOfInvalidChar -eq -1 +} + # Prompts user to enter an Installer URL, Tests the URL to ensure it results in a response code of 200, validates it against the manifest schema # Returns the validated URL which was entered Function Request-Installer-Url { @@ -373,12 +381,26 @@ Function Read-Installer-Values { $start_time = Get-Date Write-Host $NewLine Write-Host 'Downloading URL. This will take a while...' -ForegroundColor Blue - $WebClient = New-Object System.Net.WebClient - $Filename = [System.IO.Path]::GetFileName($InstallerUrl) - $script:dest = "$env:TEMP\$FileName" - try { - $WebClient.DownloadFile($InstallerUrl, $script:dest) + # Download and store the binary, but do not write to a file yet + $download = Invoke-WebRequest -Uri $InstallerUrl + # Attempt to get the file from the headers + $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) + $_Filename = $contentDisposition.FileName + # Validate the headers reurned a valid file name + if (![string]::IsNullOrWhiteSpace($_Filename) -and $(Test-ValidFileName $_Filename)) { + $Filename = $_Filename + } + # If the headers did not return a valid file name, build our own file name + # Attempt to preserve the extension if it exists, otherwise, create our own + else { + $Filename = "$PackageIdentifier v$PackageVersion" + $(if ([System.IO.Path]::HasExtension($_Filename)) { [System.IO.Path]::GetExtension($_Filename) } elseif ([System.IO.Path]::HasExtension($InstallerUrl)) { [System.IO.Path]::GetExtension($InstallerUrl) } else { '.winget-tmp' }) + } + # Write File to disk + $script:dest = Join-Path -Path $env:TEMP -ChildPath $Filename + $file = [System.IO.FileStream]::new($script:dest, [System.IO.FileMode]::Create) + $file.Write($download.Content, 0, $download.RawContentLength) + $file.Close() } catch { # Here we also want to pass the exception through for potential debugging throw [System.Net.WebException]::new('The file could not be downloaded. Try running the script again', $_.Exception) @@ -754,12 +776,26 @@ Function Read-Installer-Values-Minimal { # Request user enter the new Installer URL $_NewInstaller['InstallerUrl'] = Request-Installer-Url - # Download the file at the URL - $WebClient = New-Object System.Net.WebClient - $Filename = [System.IO.Path]::GetFileName($($_NewInstaller.InstallerUrl)) - $script:dest = "$env:TEMP\$Filename" try { - $WebClient.DownloadFile($($_NewInstaller.InstallerUrl), $script:dest) + # Download and store the binary, but do not write to a file yet + $download = Invoke-WebRequest -Uri $_NewInstaller['InstallerUrl'] + # Attempt to get the file from the headers + $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) + $_Filename = $contentDisposition.FileName + # Validate the headers reurned a valid file name + if (![string]::IsNullOrWhiteSpace($_Filename) -and $(Test-ValidFileName $_Filename)) { + $Filename = $_Filename + } + # If the headers did not return a valid file name, build our own file name + # Attempt to preserve the extension if it exists, otherwise, create our own + else { + $Filename = "$PackageIdentifier v$PackageVersion" + $(if ([System.IO.Path]::HasExtension($_Filename)) { [System.IO.Path]::GetExtension($_Filename) } elseif ([System.IO.Path]::HasExtension($InstallerUrl)) { [System.IO.Path]::GetExtension($InstallerUrl) } else { '.winget-tmp' }) + } + # Write File to disk + $script:dest = Join-Path -Path $env:TEMP -ChildPath $Filename + $file = [System.IO.FileStream]::new($script:dest, [System.IO.FileMode]::Create) + $file.Write($download.Content, 0, $download.RawContentLength) + $file.Close() } catch { # Here we also want to pass the exception through for potential debugging throw [System.Net.WebException]::new('The file could not be downloaded. Try running the script again', $_.Exception) @@ -1400,7 +1436,7 @@ Function Enter-PR-Parameters { # If we are removing a manifest, we need to include the reason if ($CommitType -eq 'Remove') { - $PrBodyContentReply = @("## $($script:RemovalReason)";'')+$PrBodyContentReply + $PrBodyContentReply = @("## $($script:RemovalReason)"; '') + $PrBodyContentReply } # Write the PR using a temporary file @@ -2097,12 +2133,27 @@ Switch ($script:Option) { Write-Host $NewLine Write-Host 'Updating Manifest Information. This may take a while...' -ForegroundColor Blue foreach ($_Installer in $script:OldInstallerManifest.Installers) { - # Download the file at the URL - $WebClient = New-Object System.Net.WebClient - $Filename = [System.IO.Path]::GetFileName($($_Installer.InstallerUrl)) - $script:dest = "$env:TEMP\$Filename" try { - $WebClient.DownloadFile($($_Installer.InstallerUrl), $script:dest) + # Download and store the binary, but do not write to a file yet + $download = Invoke-WebRequest -Uri $_Installer.InstallerUrl + # Attempt to get the file from the headers + $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) + $_Filename = $contentDisposition.FileName + + # Validate the headers reurned a valid file name + if (![string]::IsNullOrWhiteSpace($_Filename) -and $(Test-ValidFileName $_Filename)) { + $Filename = $_Filename + } + # If the headers did not return a valid file name, build our own file name + # Attempt to preserve the extension if it exists, otherwise, create our own + else { + $Filename = "$PackageIdentifier v$PackageVersion" + $(if ([System.IO.Path]::HasExtension($_Filename)) { [System.IO.Path]::GetExtension($_Filename) } elseif ([System.IO.Path]::HasExtension($InstallerUrl)) { [System.IO.Path]::GetExtension($InstallerUrl) } else { '.winget-tmp' }) + } + # Write File to disk + $script:dest = Join-Path -Path $env:TEMP -ChildPath $Filename + $file = [System.IO.FileStream]::new($script:dest, [System.IO.FileMode]::Create) + $file.Write($download.Content, 0, $download.RawContentLength) + $file.Close() } catch { # Here we also want to pass the exception through for potential debugging throw [System.Net.WebException]::new('The file could not be downloaded. Try running the script again', $_.Exception) From 79fc1e953568c21eb5d2d26556a7bcb737b80ba5 Mon Sep 17 00:00:00 2001 From: Kaleb Luedtke Date: Fri, 8 Oct 2021 20:26:10 -0500 Subject: [PATCH 2/6] Fix: Catch when content disposition doesn't exist --- Tools/YamlCreate.ps1 | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Tools/YamlCreate.ps1 b/Tools/YamlCreate.ps1 index e4aa764d57c4dd..71a8f7cc017f05 100644 --- a/Tools/YamlCreate.ps1 +++ b/Tools/YamlCreate.ps1 @@ -385,8 +385,10 @@ Function Read-Installer-Values { # Download and store the binary, but do not write to a file yet $download = Invoke-WebRequest -Uri $InstallerUrl # Attempt to get the file from the headers - $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) - $_Filename = $contentDisposition.FileName + try { + $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) + $_Filename = $contentDisposition.FileName + } catch {} # Validate the headers reurned a valid file name if (![string]::IsNullOrWhiteSpace($_Filename) -and $(Test-ValidFileName $_Filename)) { $Filename = $_Filename @@ -780,8 +782,10 @@ Function Read-Installer-Values-Minimal { # Download and store the binary, but do not write to a file yet $download = Invoke-WebRequest -Uri $_NewInstaller['InstallerUrl'] # Attempt to get the file from the headers - $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) - $_Filename = $contentDisposition.FileName + try { + $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) + $_Filename = $contentDisposition.FileName + } catch {} # Validate the headers reurned a valid file name if (![string]::IsNullOrWhiteSpace($_Filename) -and $(Test-ValidFileName $_Filename)) { $Filename = $_Filename @@ -2137,9 +2141,10 @@ Switch ($script:Option) { # Download and store the binary, but do not write to a file yet $download = Invoke-WebRequest -Uri $_Installer.InstallerUrl # Attempt to get the file from the headers - $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) - $_Filename = $contentDisposition.FileName - + try { + $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) + $_Filename = $contentDisposition.FileName + } catch {} # Validate the headers reurned a valid file name if (![string]::IsNullOrWhiteSpace($_Filename) -and $(Test-ValidFileName $_Filename)) { $Filename = $_Filename From 76b77dffb5f7a3f341f5218a5684126e8209d191 Mon Sep 17 00:00:00 2001 From: Kaleb Luedtke Date: Fri, 8 Oct 2021 20:28:50 -0500 Subject: [PATCH 3/6] Add comment fix back in --- Tools/YamlCreate.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/YamlCreate.ps1 b/Tools/YamlCreate.ps1 index 71a8f7cc017f05..4fd5aba8c248e4 100644 --- a/Tools/YamlCreate.ps1 +++ b/Tools/YamlCreate.ps1 @@ -293,7 +293,7 @@ Function TestUrlValidity { return $HTTP_Status } -# Checks a file name for validite and returns a boolean value +# Checks a file name for validity and returns a boolean value function Test-ValidFileName { param([string]$FileName) $IndexOfInvalidChar = $FileName.IndexOfAny([System.IO.Path]::GetInvalidFileNameChars()) From 8f57b2cf9bbd0ba0edc95544b6bef152511f92a1 Mon Sep 17 00:00:00 2001 From: Kaleb Luedtke Date: Fri, 8 Oct 2021 20:53:06 -0500 Subject: [PATCH 4/6] Fix UserAgent not following redirects --- Tools/YamlCreate.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/YamlCreate.ps1 b/Tools/YamlCreate.ps1 index 4fd5aba8c248e4..2b5d0193a58ce8 100644 --- a/Tools/YamlCreate.ps1 +++ b/Tools/YamlCreate.ps1 @@ -383,7 +383,7 @@ Function Read-Installer-Values { Write-Host 'Downloading URL. This will take a while...' -ForegroundColor Blue try { # Download and store the binary, but do not write to a file yet - $download = Invoke-WebRequest -Uri $InstallerUrl + $download = Invoke-WebRequest -Uri $InstallerUrl -UserAgent [Microsoft.PowerShell.Commands.PSUserAgent]::Chrome # Attempt to get the file from the headers try { $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) @@ -780,7 +780,7 @@ Function Read-Installer-Values-Minimal { try { # Download and store the binary, but do not write to a file yet - $download = Invoke-WebRequest -Uri $_NewInstaller['InstallerUrl'] + $download = Invoke-WebRequest -Uri $_NewInstaller['InstallerUrl'] -UserAgent [Microsoft.PowerShell.Commands.PSUserAgent]::Chrome # Attempt to get the file from the headers try { $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) @@ -2139,7 +2139,7 @@ Switch ($script:Option) { foreach ($_Installer in $script:OldInstallerManifest.Installers) { try { # Download and store the binary, but do not write to a file yet - $download = Invoke-WebRequest -Uri $_Installer.InstallerUrl + $download = Invoke-WebRequest -Uri $_Installer.InstallerUrl -UserAgent [Microsoft.PowerShell.Commands.PSUserAgent]::Chrome # Attempt to get the file from the headers try { $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) From bbf5e09ad2b774eadfacfa88eeb53107c965a3fa Mon Sep 17 00:00:00 2001 From: Kaleb Luedtke Date: Sat, 9 Oct 2021 00:51:36 -0500 Subject: [PATCH 5/6] Additional Web Request Parameters --- Tools/YamlCreate.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/YamlCreate.ps1 b/Tools/YamlCreate.ps1 index 2b5d0193a58ce8..81f1e163d49a2d 100644 --- a/Tools/YamlCreate.ps1 +++ b/Tools/YamlCreate.ps1 @@ -383,7 +383,7 @@ Function Read-Installer-Values { Write-Host 'Downloading URL. This will take a while...' -ForegroundColor Blue try { # Download and store the binary, but do not write to a file yet - $download = Invoke-WebRequest -Uri $InstallerUrl -UserAgent [Microsoft.PowerShell.Commands.PSUserAgent]::Chrome + $download = Invoke-WebRequest -Uri $InstallerUrl -UserAgent [Microsoft.PowerShell.Commands.PSUserAgent]::Chrome -DisableKeepAlive -TimeoutSec 30 -UseBasicParsing # Attempt to get the file from the headers try { $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) @@ -780,7 +780,7 @@ Function Read-Installer-Values-Minimal { try { # Download and store the binary, but do not write to a file yet - $download = Invoke-WebRequest -Uri $_NewInstaller['InstallerUrl'] -UserAgent [Microsoft.PowerShell.Commands.PSUserAgent]::Chrome + $download = Invoke-WebRequest -Uri $_NewInstaller['InstallerUrl'] -UserAgent [Microsoft.PowerShell.Commands.PSUserAgent]::Chrome -DisableKeepAlive -TimeoutSec 30 -UseBasicParsing # Attempt to get the file from the headers try { $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) @@ -2139,7 +2139,7 @@ Switch ($script:Option) { foreach ($_Installer in $script:OldInstallerManifest.Installers) { try { # Download and store the binary, but do not write to a file yet - $download = Invoke-WebRequest -Uri $_Installer.InstallerUrl -UserAgent [Microsoft.PowerShell.Commands.PSUserAgent]::Chrome + $download = Invoke-WebRequest -Uri $_Installer.InstallerUrl -UserAgent [Microsoft.PowerShell.Commands.PSUserAgent]::Chrome -DisableKeepAlive -TimeoutSec 30 -UseBasicParsing # Attempt to get the file from the headers try { $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) From 737f52ce317ec7fd8f5995eb508d961fa36a7b51 Mon Sep 17 00:00:00 2001 From: Kaleb Luedtke Date: Mon, 11 Oct 2021 13:07:23 -0500 Subject: [PATCH 6/6] Fix: Min descriptor length + Agent --- Tools/YamlCreate.ps1 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Tools/YamlCreate.ps1 b/Tools/YamlCreate.ps1 index 81f1e163d49a2d..7a13a08bf9e16d 100644 --- a/Tools/YamlCreate.ps1 +++ b/Tools/YamlCreate.ps1 @@ -383,7 +383,7 @@ Function Read-Installer-Values { Write-Host 'Downloading URL. This will take a while...' -ForegroundColor Blue try { # Download and store the binary, but do not write to a file yet - $download = Invoke-WebRequest -Uri $InstallerUrl -UserAgent [Microsoft.PowerShell.Commands.PSUserAgent]::Chrome -DisableKeepAlive -TimeoutSec 30 -UseBasicParsing + $download = Invoke-WebRequest -Uri $InstallerUrl -UserAgent 'winget/1.0' -DisableKeepAlive -TimeoutSec 30 -UseBasicParsing # Attempt to get the file from the headers try { $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) @@ -780,7 +780,7 @@ Function Read-Installer-Values-Minimal { try { # Download and store the binary, but do not write to a file yet - $download = Invoke-WebRequest -Uri $_NewInstaller['InstallerUrl'] -UserAgent [Microsoft.PowerShell.Commands.PSUserAgent]::Chrome -DisableKeepAlive -TimeoutSec 30 -UseBasicParsing + $download = Invoke-WebRequest -Uri $_NewInstaller['InstallerUrl'] -UserAgent 'winget/1.0' -DisableKeepAlive -TimeoutSec 30 -UseBasicParsing # Attempt to get the file from the headers try { $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition']) @@ -2117,10 +2117,10 @@ Switch ($script:Option) { Write-Host -ForegroundColor 'Green' -Object '[Required] Enter the reason for removing this manifest' $script:RemovalReason = Read-Host -Prompt 'Reason' | TrimString # Check the reason for validity. The length requirements are arbitrary, but they have been set to encourage concise yet meaningful reasons - if (String.Validate $script:RemovalReason -MinLength 16 -MaxLength 128 -NotNull) { + if (String.Validate $script:RemovalReason -MinLength 8 -MaxLength 128 -NotNull) { $script:_returnValue = [ReturnValue]::Success() } else { - $script:_returnValue = [ReturnValue]::LengthError(16, 128) + $script:_returnValue = [ReturnValue]::LengthError(8, 128) } } until ($script:_returnValue.StatusCode -eq [ReturnValue]::Success().StatusCode) @@ -2139,7 +2139,7 @@ Switch ($script:Option) { foreach ($_Installer in $script:OldInstallerManifest.Installers) { try { # Download and store the binary, but do not write to a file yet - $download = Invoke-WebRequest -Uri $_Installer.InstallerUrl -UserAgent [Microsoft.PowerShell.Commands.PSUserAgent]::Chrome -DisableKeepAlive -TimeoutSec 30 -UseBasicParsing + $download = Invoke-WebRequest -Uri $_Installer.InstallerUrl -UserAgent 'winget/1.0' -DisableKeepAlive -TimeoutSec 30 -UseBasicParsing # Attempt to get the file from the headers try { $contentDisposition = [System.Net.Mime.ContentDisposition]::new($download.Headers['Content-Disposition'])