From dd0d7dbd504b04663aca72a305f6b00edf83767b Mon Sep 17 00:00:00 2001 From: Derek Brown Date: Thu, 1 Sep 2022 22:45:15 -0700 Subject: [PATCH 01/16] DO NOT MERGE Initial revision. Attempting single run windows container, in hopes of being able to make shared environment script --- build-container.ps1 | 21 +- windows/Dockerfile | 195 +-- ...kerfile-to-powershell-generated-sample.ps1 | 1072 ----------------- windows/dockerfile-to-powershell.py | 606 ---------- windows/helpers.ps1 | 93 ++ windows/{ => helpers}/aws_networking.ps1 | 0 windows/{ => helpers}/install_7zip.ps1 | 8 +- windows/helpers/install_cert.ps1 | 11 + windows/{ => helpers}/install_cmake.ps1 | 11 +- windows/{ => helpers}/install_codeql.ps1 | 10 +- windows/helpers/install_docker.ps1 | 26 + windows/{ => helpers}/install_dotnetcore.ps1 | 16 +- .../install_embedded_pythons.ps1 | 46 +- windows/{ => helpers}/install_gcloud_sdk.ps1 | 11 +- windows/{ => helpers}/install_go.ps1 | 10 +- windows/helpers/install_ibm_mq.ps1 | 10 + windows/{ => helpers}/install_mingit.ps1 | 10 +- windows/{ => helpers}/install_msys.ps1 | 17 +- windows/{ => helpers}/install_net35.ps1 | 7 +- windows/helpers/install_ninja.ps1 | 14 + windows/{ => helpers}/install_nuget.ps1 | 15 +- windows/helpers/install_python.ps1 | 31 + windows/{ => helpers}/install_ruby.ps1 | 16 +- windows/{ => helpers}/install_vcpkg.ps1 | 2 +- windows/{ => helpers}/install_vcpython.ps1 | 9 +- windows/{ => helpers}/install_vstudio.ps1 | 20 +- windows/{ => helpers}/install_wdk.ps1 | 4 +- windows/helpers/install_winget.ps1 | 19 + windows/helpers/install_wix.ps1 | 28 + windows/install_docker.ps1 | 29 - windows/install_ibm_mq.ps1 | 45 - windows/install_ninja.ps1 | 48 - windows/install_python.ps1 | 39 - windows/install_winget.ps1 | 27 - windows/install_wix.ps1 | 37 - windows/master.ps1 | 61 + windows/test.ps1 | 8 + windows/versions.ps1 | 38 + 38 files changed, 440 insertions(+), 2230 deletions(-) delete mode 100644 windows/dockerfile-to-powershell-generated-sample.ps1 delete mode 100755 windows/dockerfile-to-powershell.py create mode 100644 windows/helpers.ps1 rename windows/{ => helpers}/aws_networking.ps1 (100%) rename windows/{ => helpers}/install_7zip.ps1 (54%) create mode 100644 windows/helpers/install_cert.ps1 rename windows/{ => helpers}/install_cmake.ps1 (65%) rename windows/{ => helpers}/install_codeql.ps1 (67%) create mode 100644 windows/helpers/install_docker.ps1 rename windows/{ => helpers}/install_dotnetcore.ps1 (63%) rename windows/{ => helpers}/install_embedded_pythons.ps1 (55%) rename windows/{ => helpers}/install_gcloud_sdk.ps1 (65%) rename windows/{ => helpers}/install_go.ps1 (60%) create mode 100644 windows/helpers/install_ibm_mq.ps1 rename windows/{ => helpers}/install_mingit.ps1 (57%) rename windows/{ => helpers}/install_msys.ps1 (58%) rename windows/{ => helpers}/install_net35.ps1 (96%) create mode 100644 windows/helpers/install_ninja.ps1 rename windows/{ => helpers}/install_nuget.ps1 (50%) create mode 100644 windows/helpers/install_python.ps1 rename windows/{ => helpers}/install_ruby.ps1 (61%) rename windows/{ => helpers}/install_vcpkg.ps1 (79%) rename windows/{ => helpers}/install_vcpython.ps1 (52%) rename windows/{ => helpers}/install_vstudio.ps1 (64%) rename windows/{ => helpers}/install_wdk.ps1 (82%) create mode 100644 windows/helpers/install_winget.ps1 create mode 100644 windows/helpers/install_wix.ps1 delete mode 100644 windows/install_docker.ps1 delete mode 100644 windows/install_ibm_mq.ps1 delete mode 100644 windows/install_ninja.ps1 delete mode 100644 windows/install_python.ps1 delete mode 100644 windows/install_winget.ps1 delete mode 100644 windows/install_wix.ps1 create mode 100644 windows/master.ps1 create mode 100644 windows/test.ps1 create mode 100644 windows/versions.ps1 diff --git a/build-container.ps1 b/build-container.ps1 index 8f0e3a22d..2b5bece80 100644 --- a/build-container.ps1 +++ b/build-container.ps1 @@ -5,7 +5,7 @@ param( ) $ErrorActionPreference = "Stop" - +. .\windows\versions.ps1 $BaseTable = @{ 1809 = "mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-ltsc2019"; 1909 = "mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-1909"; @@ -26,15 +26,26 @@ if ($build -ge 20348) { Write-Host -ForegroundColor Green "Using base image $($BaseTable[$kernelver])" +$arglist = @() if($Tag -eq $null -or $Tag -eq ""){ $Tag ="builder_$($kernelver)_$Arch" } -$buildcommandparams = "-m 4096M --build-arg BASE_IMAGE=$($BaseTable[$kernelver]) --build-arg DD_TARGET_ARCH=$Arch --build-arg WINDOWS_VERSION=$kernelver -t $Tag --file .\windows\Dockerfile ." +$arglist += "build" + +foreach ($h in $SoftwareTable.GetEnumerator()){ + if( -not ($($h.Key) -like "*SHA256")){ + $arglist += "--build-arg" + $arglist += "$($h.Key)=$($h.Value)" + } +} + if( -not $Cache) { - $buildcommandparams = "--no-cache $buildcommandparams" + $arglist += "--no-cache" } -$buildcommand = "build $buildcommandparams" + +$arglist += -split "-m 4096M --build-arg BASE_IMAGE=$($BaseTable[$kernelver]) --build-arg DD_TARGET_ARCH=$Arch --build-arg WINDOWS_VERSION=$kernelver -t $Tag --file .\windows\Dockerfile ." # Write-Host -ForegroundColor Green "Building with the following command:" # Write-Host -ForegroundColor Green "$buildcommand `n" filter timestamp {"$(Get-Date -Format o): $_"} -& docker $buildcommand.split() | timestamp +& docker $arglist | timestamp + diff --git a/windows/Dockerfile b/windows/Dockerfile index 7b46654b3..330a5eb65 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -23,44 +23,19 @@ ENV WINDOWS_VERSION=${WINDOWS_VERSION:-1809} ARG DD_TARGET_ARCH ENV TARGET_ARCH=${DD_TARGET_ARCH:-x64} -ENV GIT_VERSION "2.26.2" -ENV GIT_SHA256 "2dfbb1c46547c70179442a92b8593d592292b8bce2fd02ac4e0051a8072dde8f" -ENV SEVENZIP_VERSION "19.0.0" -ENV SEVENZIP_SHA256 "0f5d4dbbe5e55b7aa31b91e5925ed901fdf46a367491d81381846f05ad54c45e" -ENV VS2017BUILDTOOLS_VERSION "16.8.3.0" -ENV VS2017BUILDTOOLS_SHA256 "ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0" -#ENV VS2017BUILDTOOLS_DOWNLOAD_URL "https://download.visualstudio.microsoft.com/download/pr/d0eac510-174b-4241-b73b-93dc7cc1fbf7/9822b4c851e14d9658babd1533f66f518c6169196e985fe5713b2774128832ae/vs_BuildTools.exe" -ENV VS2017BUILDTOOLS_DOWNLOAD_URL "https://download.visualstudio.microsoft.com/download/pr/9b3476ff-6d0a-4ff8-956d-270147f21cd4/ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0/vs_BuildTools.exe" -ENV GO_VERSION "1.19.5" -ENV GO_SHA256 "167db91a2e40aeb453d3e59d213ecab06f62e1c4a84d13a06ccda1d999961caa" -ENV RUBY_VERSION "2.6.6-1" -ENV RUBY_SHA256 "fbdf77a3e1fa36e25cf0af1303ac76f67dec7a6f739a829784a299702cad1492" -ENV IBM_MQ_VERSION "9.2.4.0" -ENV IBM_MQ_SHA256 "9cea7e1693d051437e78468fd5e915b7b7ed2baf36cdae4936bcf2b760f55daa" -ENV PYTHON_VERSION "3.8.2" -ENV PYTHON_SHA256 "8e400e3f32cdcb746e62e0db4d3ae4cba1f927141ebc4d0d5a4006b0daee8921" -ENV WIX_VERSION "3.11.2" -ENV WIX_SHA256 "32bb76c478fcb356671d4aaf006ad81ca93eea32c22a9401b168fc7471feccd2" -ENV CMAKE_VERSION "3.23.0" -ENV CMAKE_SHA256 "1e772025844f1cc648d28f42090038e5ca5cf72e2889de26d8d05ee25da17061" -ENV MSYS_VERSION "20210725" -ENV MSYS_SHA256 "4013a9d5e51b448343efc24fc6a324cc999bb96b4c01b13a6bd3c661bb5c8a82" -ENV NUGET_VERSION "5.8.0" -ENV NUGET_SHA256 "5c5b9c96165d3283b2cb9e5b65825d343e0e7139b9e70a250b4bb24c2285f3ba" -ENV WINGET_VERSION "1.0.4.0" -ENV WINGET_SHA256 "8a4b727a8ad588c86603136a5d48093b59ed7491f7c5dc9f13725da0900fe6ab" -ENV EMBEDDED_PYTHON_2_VERSION "2.7.17" -ENV EMBEDDED_PYTHON_2_SHA256 "557ea6690c5927360656c003d3114b73adbd755b712a2911975dde813d6d7afb" -ENV EMBEDDED_PYTHON_3_VERSION "3.8.1" -ENV EMBEDDED_PYTHON_3_SHA256 "58563ca60891025923572107e02b8f07439928eb5222dd10466cc92089072c2a" -ENV EMBEDDED_PIP_VERSION "20.3.4" -ENV CODEQL_VERSION "2.10.3" -ENV CODEQL_HASH "46f64e21c74f41210ea3f2c433d1dc622e3eb0690b42373a73fba82122b929a1" -ENV NINJA_VERSION "1.11.0" -ENV NINJA_SHA256 "d0ee3da143211aa447e750085876c9b9d7bcdd637ab5b2c5b41349c617f22f3b" - -ENV CACERTS_VERSION "2023-01-10" -ENV CACERTS_HASH "fb1ecd641d0a02c01bc9036d513cb658bbda62a75e246bedbc01764560a639f0" +ARG GIT_VERSION +ARG SEVENZIP_VERSION +ARG VS2017BUILDTOOLS_VERSION +ARG GO_VERSION +ARG RUBY_VERSION +ARG WIX_VERSION +ARG CMAKE_VERSION +ARG MSYS_VERSION +ARG PYTHON_VERSION +ARG EMBEDDED_PYTHON_2_VERSION +ARG EMBEDDED_PYTHON_3_VERSION +ARG NUGET_VERSION +ARG CODEQL_VERSION LABEL target_agent="Agent 6/7" LABEL target_arch=${TARGET_ARCH} @@ -78,147 +53,23 @@ LABEL embedded_py2_version=${EMBEDDED_PYTHON_2_VERSION} LABEL embedded_py3_version=${EMBEDDED_PYTHON_3_VERSION} LABEL nuget_version=${NUGET_VERSION} LABEL codeql_version=${CODEQL_VERSION} -# Add certificates needed for build & check certificates file hash -# We need to trust the DigiCert High Assurance EV Root CA certificate, which signs python.org, -# to be able to download some Python components during the Agent build. -RUN (New-Object System.Net.WebClient).DownloadFile(\"https://curl.haxx.se/ca/cacert-${ENV:CACERTS_VERSION}.pem\", \"cacert.pem\") -RUN if ((Get-FileHash -Algorithm SHA256 .\cacert.pem).Hash -ne \"$ENV:CACERTS_HASH\") { Write-Host \"Wrong hashsum for cacert.pem: got '$((Get-FileHash -Algorithm SHA256 .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'.\"; exit 1 } -RUN setx SSL_CERT_FILE \"C:\cacert.pem\" - -### Preliminary step: we need both the .NET 3.5 runtime and -### the .NET 4.8 runtime. To do this, we get 4.8 from a base image and we -### manually the install .NET Framework 3.5 runtime using the instructions in -### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile: -### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile - -### The .NET Fx 3.5 is needed for the Visual C++ Compiler for Python 2.7 -### (https://www.microsoft.com/en-us/download/details.aspx?id=44266) -### and to work around a bug in the WiX 3.11 installer -### (https://github.com/wixtoolset/issues/issues/5661). - -# Install .NET Fx 3.5 -COPY ./windows/install_net35.ps1 install_net35.ps1 -RUN Powershell -C .\install_net35.ps1 - -### End of preliminary step - -# Install 7-zip -Copy ./windows/install_7zip.ps1 install_7zip.ps1 -RUN powershell -Command .\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256 - -# Install git -COPY ./windows/install_mingit.ps1 install_mingit.ps1 -RUN powershell -Command .\install_mingit.ps1 -Version $ENV:GIT_VERSION -Sha256 $ENV:GIT_SHA256 - -### HACK: we disable symbolic links when cloning repositories -### to work around a symlink-related failure in the agent-binaries omnibus project -### when copying the datadog-agent project twice. -RUN git config --system core.symlinks false - -# Install VS2017 -COPY ./windows/install_vstudio.ps1 install_vstudio.ps1 -RUN powershell -Command .\install_vstudio.ps1 -Version $ENV:VS2017BUILDTOOLS_VERSION -Sha256 $ENV:VS2017BUILDTOOLS_SHA256 $ENV:VS2017BUILDTOOLS_DOWNLOAD_URL - -# If x64, install the WDK for driver development -COPY ./windows/install_wdk.ps1 install_wdk.ps1 -RUN if ($Env:TARGET_ARCH -eq 'x64') { powershell -Command .\install_wdk.ps1 } - -# Install Wix and update PATH to include it -COPY ./windows/install_wix.ps1 install_wix.ps1 -RUN powershell -Command .\install_wix.ps1 -Version $ENV:WIX_VERSION -Sha256 $ENV:WIX_SHA256 - -# Install dotnet core -COPY ./windows/install_dotnetcore.ps1 install_dotnetcore.ps1 -RUN powershell -Command .\install_dotnetcore.ps1 - -# install nuget -COPY ./windows/install_nuget.ps1 install_nuget.ps1 -RUN powershell -Command .\install_nuget.ps1 -Version $ENV:NUGET_VERSION -Sha256 $ENV:NUGET_SHA256 - -# Install VC compiler for Python 2.7 -COPY ./windows/install_vcpython.ps1 install_vcpython.ps1 -RUN powershell -Command .\install_vcpython.ps1 - -# Install IBM MQ -COPY ./windows/install_ibm_mq.ps1 install_ibm_mq.ps1 -RUN Powershell -C .\install_ibm_mq.ps1 -Version $ENV:IBM_MQ_VERSION -Sha256 $ENV:IBM_MQ_SHA256 -RUN setx MQ_FILE_PATH c:\ibm_mq - -# Install Cmake and update PATH to include it -COPY ./windows/install_cmake.ps1 install_cmake.ps1 -RUN powershell -Command .\install_cmake.ps1 -Version $ENV:CMAKE_VERSION -Sha256 $ENV:CMAKE_SHA256 - -# Install Winget and update PATH to include it -COPY ./windows/install_winget.ps1 install_winget.ps1 -RUN powershell -Command .\install_winget.ps1 -Version $ENV:WINGET_VERSION -Sha256 $ENV:WINGET_SHA256 -# Install golang and set GOPATH to the dev path used in builds & tests -# RUN cinst -y --no-progress golang $ENV:CHOCO_ARCH_FLAG --version $ENV:GO_VERSION +## WORKDIR should automatically create the directory +WORKDIR c:/scripts +COPY ./windows/versions.ps1 versions.ps1 +COPY ./windows/master.ps1 master.ps1 +COPY ./windows/helpers.ps1 helpers.ps1 +COPY ./windows/helpers/*.ps1 c:/scripts/ -# Some go point releases are not available as chocolatey packages, we have to install them directly -COPY ./windows/install_go.ps1 install_go.ps1 -RUN powershell -C .\install_go.ps1 - -RUN setx GOPATH C:\dev\go - -# Install system Python 3 (to use invoke). -# We always install the 64 bit version because vcredist140 won't work otherwise -COPY ./windows/install_python.ps1 install_python.ps1 COPY ./requirements.txt ./requirements-py2.txt / -RUN powershell -C .\install_python.ps1 -Version $ENV:PYTHON_VERSION -Sha256 $ENV:PYTHON_SHA256 - -# Install 64-bit ruby and bundler (for omnibus builds) -COPY ./windows/install_ruby.ps1 install_ruby.ps1 -RUN powershell -C .\install_ruby.ps1 -Version $ENV:RUBY_VERSION -Sha256 $ENV:RUBY_SHA256 - -# Install msys2 system & install 64-bit C/C++ compilation toolchain -copy ./windows/install_msys.ps1 install_msys.ps1 -RUN powershell -C .\install_msys.ps1 -Version $ENV:MSYS_VERSION -Sha256 $ENV:MSYS_SHA256 - -RUN ridk install 3 - -# Downgrade gcc and binutils due to https://github.com/golang/go/issues/46099 -RUN (New-Object System.Net.WebClient).DownloadFile(\"https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst\", \"C:/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst\") -RUN (New-Object System.Net.WebClient).DownloadFile(\"https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst\", \"C:/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst\") -RUN (New-Object System.Net.WebClient).DownloadFile(\"https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst\", \"C:/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst\") -RUN C:\tools\msys64\msys2_shell.cmd -defterm -no-start -c \"pacman --noconfirm -U /c/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst\" - -# Install docker, manifest-tool and notary -COPY ./windows/install_docker.ps1 install_docker.ps1 -RUN powershell -Command .\install_docker.ps1 - -# Install google cloud SDK -COPY ./windows/install_gcloud_sdk.ps1 install_gcloud_sdk.ps1 -RUN powershell -C .\install_gcloud_sdk.ps1 - -# Install embedded pythons (for unit testing) -COPY ./python-packages-versions.txt python-packages-versions.txt -COPY ./windows/install_embedded_pythons.ps1 install_embedded_pythons.ps1 -RUN powershell -C .\install_embedded_pythons.ps1 - -# install vcpkg -COPY ./windows/install_vcpkg.ps1 install_vcpkg.ps1 -RUN powershell -Command .\install_vcpkg.ps1 - -# Install CodeQL for CI execution of CodeQL -COPY ./windows/install_codeql.ps1 install_codeql.ps1 -RUN powershell -C .\install_codeql.ps1 - -# install ninja -COPY ./windows/install_ninja.ps1 install_ninja.ps1 -RUN powershell -C .\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 - -# Add signtool to path -RUN [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\", [System.EnvironmentVariableTarget]::Machine) - -# Set 32-bit flag env var -RUN if ($Env:TARGET_ARCH -eq 'x86') { setx WINDOWS_BUILD_32_BIT 1 } - +COPY ./python-packages-versions.txt /python-packages-versions.txt COPY ./windows/set_cpython_compiler.cmd set_cpython_compiler.cmd -RUN .\set_cpython_compiler.cmd + +RUN .\master.ps1 -TargetContainer COPY ./windows/entrypoint.bat /entrypoint.bat COPY ./windows/aws_networking.ps1 /aws_networking.ps1 ENTRYPOINT ["/entrypoint.bat"] + diff --git a/windows/dockerfile-to-powershell-generated-sample.ps1 b/windows/dockerfile-to-powershell-generated-sample.ps1 deleted file mode 100644 index 9608312ce..000000000 --- a/windows/dockerfile-to-powershell-generated-sample.ps1 +++ /dev/null @@ -1,1072 +0,0 @@ -# Automatically generated by -# dockerfile-to-powershell.py -d Dockerfile -p dockerfile-to-powershell-generated-sample.ps1 -a WINDOWS_VERSION=1809 -a DD_TARGET_ARCH=x64 -# on 01/12/2023 13:31:57 - -# WARNING!!! -# If this script had been generated from ... -# https://github.com/DataDog/datadog-agent-buildimages/blob/main/windows/Dockerfile -# ... which is currently based ... -# on mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-ltsc2019 -# ... image, then you will need to install .NET 4.8 before running it from one of three sources -# * https://dotnet.microsoft.com/en-us/download/dotnet-framework/net48 -# * https://support.microsoft.com/en-us/topic/microsoft-net-framework-4-8-offline-installer-for-windows-9d23f658-3b97-68ab-d013-aa3c3e7495e0 -# * https://github.com/microsoft/dotnet-framework-docker/blob/528e53c63f429860d99d55409b0018d225059f0c/src/runtime/4.8/windowsservercore-ltsc2019/Dockerfile -# -# Similarly, for other Dockerfiles you need to make sure that you prepare Windows machine to -# be matching its Dockerfile's BASE image. -# -# If you try to run this script more than once some commands may fail and break the script. -# It may be addressed better in the future but for now you may need to comment out offending -# commands. If after running generated from the Dockerfile Powershell script the Dockerfile -# had been modified and you wish to apply these changes, then in contrast to applying newly -# generated Powershell script your mileage may vary (and it depends very much on the Dockerfile -# content) ... -# * It may actually just work -# * You may need to comment out everything except the difference between old and new -# Powershell script -# * You may need to explicitly uninstall or remove previously deployed applications, files -# or other artifacts -# -# Usage: .\ -DockerRepoPath -# -DockerRepoPath is required parameter. You need to clone the -# same repo as the one used for building Docker image. -# Specifically it is https://github.com/DataDog/datadog-agent-buildimages -Param( - [Parameter(Mandatory)] - [string] - $DockerRepoPath -) - -$DockerRepoPathExists = Test-Path $DockerRepoPath -if (!$DockerRepoPathExists) { - Write-Host "Provided 'DockerRepoPath' ($DockerRepoPath) path does not exist. Please provide valid argument" - Exit -} - -# Save current location and sync it to .NET current location (it is not automatic and needed in some cases) -# https://stackoverflow.com/questions/11246068/why-dont-net-objects-in-powershell-use-the-current-directory -$origLocation = Get-Location -[Environment]::CurrentDirectory = $origLocation -# Currently Docker run everything in the root. And we have to keep it this way -Set-Location $origLocation - -$totalStopwatch = [system.diagnostics.stopwatch]::StartNew() - -function Update-SessionEnvironment { - foreach ($s in 'Machine','User') { - [Environment]::GetEnvironmentVariables($s).GetEnumerator(). - Where({$_.Key -ne 'PATH'}) | ForEach-Object { - [Environment]::SetEnvironmentVariable($_.Key,$_.Value,'Process') }} - - $env:PATH = ( ('Machine','User').ForEach({ - [Environment]::GetEnvironmentVariable('PATH',$_)}). - Split(';').Where({$_}) | Select-Object -Unique ) -join ';' -} -# Use the Microsoft-provided .NET Runtime 4.8 image as the base image -# because installing it in the image with Chocolatey requires a reboot. -# There is a bug in the February 11, 2020 update that makes commands fail in -# docker containers (see: https://support.microsoft.com/en-us/help/4542617/you-might-encounter-issues-when-using-windows-server-containers-with-t) -# To avoid that, there are two solutions: -# - both the host and the container must have the February 11, 2020 update, or -# - neither the host and the container must have the February 11, 2020 update. -# Since our 1809 windows-docker host image does not have this update, we use a base -# container image that does not have this update either (thus the 20200114 tag). -# On the contrary, since our 1909 windows-docker host image does have this update, -# we use a base container image that does have this update. -# TODO: Once the 1809 windows-docker host image is updated, update the base container image. -# Content: ARG BASE_IMAGE=mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-ltsc2019 - -# Content: FROM ${BASE_IMAGE} -# Base Image: ${BASE_IMAGE} - -# Content: SHELL ["powershell", "-Command"] -# SHELL="powershell -Command" - -# Content: ARG WINDOWS_VERSION -# ARG=1809 - -# Content: ENV WINDOWS_VERSION=${WINDOWS_VERSION:-1809} -[System.Environment]::SetEnvironmentVariable('WINDOWS_VERSION','1809') - -# Content: ARG DD_TARGET_ARCH -# ARG=x64 - -# Content: ENV TARGET_ARCH=${DD_TARGET_ARCH:-x64} -[System.Environment]::SetEnvironmentVariable('TARGET_ARCH','x64') - -# Content: ENV GIT_VERSION "2.26.2" -[System.Environment]::SetEnvironmentVariable('GIT_VERSION','2.26.2') - -# Content: ENV GIT_SHA256 "2dfbb1c46547c70179442a92b8593d592292b8bce2fd02ac4e0051a8072dde8f" -[System.Environment]::SetEnvironmentVariable('GIT_SHA256','2dfbb1c46547c70179442a92b8593d592292b8bce2fd02ac4e0051a8072dde8f') - -# Content: ENV SEVENZIP_VERSION "19.0.0" -[System.Environment]::SetEnvironmentVariable('SEVENZIP_VERSION','19.0.0') - -# Content: ENV SEVENZIP_SHA256 "0f5d4dbbe5e55b7aa31b91e5925ed901fdf46a367491d81381846f05ad54c45e" -[System.Environment]::SetEnvironmentVariable('SEVENZIP_SHA256','0f5d4dbbe5e55b7aa31b91e5925ed901fdf46a367491d81381846f05ad54c45e') - -# Content: ENV VS2017BUILDTOOLS_VERSION "16.8.3.0" -[System.Environment]::SetEnvironmentVariable('VS2017BUILDTOOLS_VERSION','16.8.3.0') - -# Content: ENV VS2017BUILDTOOLS_SHA256 "ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0" -[System.Environment]::SetEnvironmentVariable('VS2017BUILDTOOLS_SHA256','ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0') - -#ENV VS2017BUILDTOOLS_DOWNLOAD_URL "https://download.visualstudio.microsoft.com/download/pr/d0eac510-174b-4241-b73b-93dc7cc1fbf7/9822b4c851e14d9658babd1533f66f518c6169196e985fe5713b2774128832ae/vs_BuildTools.exe" -# Content: ENV VS2017BUILDTOOLS_DOWNLOAD_URL "https://download.visualstudio.microsoft.com/download/pr/9b3476ff-6d0a-4ff8-956d-270147f21cd4/ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0/vs_BuildTools.exe" -[System.Environment]::SetEnvironmentVariable('VS2017BUILDTOOLS_DOWNLOAD_URL','https://download.visualstudio.microsoft.com/download/pr/9b3476ff-6d0a-4ff8-956d-270147f21cd4/ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0/vs_BuildTools.exe') - -# Content: ENV GO_VERSION "1.19.5" -[System.Environment]::SetEnvironmentVariable('GO_VERSION','1.19.5') - -# Content: ENV GO_SHA256 "167db91a2e40aeb453d3e59d213ecab06f62e1c4a84d13a06ccda1d999961caa" -[System.Environment]::SetEnvironmentVariable('GO_SHA256','167db91a2e40aeb453d3e59d213ecab06f62e1c4a84d13a06ccda1d999961caa') - -# Content: ENV RUBY_VERSION "2.6.6-1" -[System.Environment]::SetEnvironmentVariable('RUBY_VERSION','2.6.6-1') - -# Content: ENV RUBY_SHA256 "fbdf77a3e1fa36e25cf0af1303ac76f67dec7a6f739a829784a299702cad1492" -[System.Environment]::SetEnvironmentVariable('RUBY_SHA256','fbdf77a3e1fa36e25cf0af1303ac76f67dec7a6f739a829784a299702cad1492') - -# Content: ENV IBM_MQ_VERSION "9.2.4.0" -[System.Environment]::SetEnvironmentVariable('IBM_MQ_VERSION','9.2.4.0') - -# Content: ENV IBM_MQ_SHA256 "9cea7e1693d051437e78468fd5e915b7b7ed2baf36cdae4936bcf2b760f55daa" -[System.Environment]::SetEnvironmentVariable('IBM_MQ_SHA256','9cea7e1693d051437e78468fd5e915b7b7ed2baf36cdae4936bcf2b760f55daa') - -# Content: ENV PYTHON_VERSION "3.8.2" -[System.Environment]::SetEnvironmentVariable('PYTHON_VERSION','3.8.2') - -# Content: ENV PYTHON_SHA256 "8e400e3f32cdcb746e62e0db4d3ae4cba1f927141ebc4d0d5a4006b0daee8921" -[System.Environment]::SetEnvironmentVariable('PYTHON_SHA256','8e400e3f32cdcb746e62e0db4d3ae4cba1f927141ebc4d0d5a4006b0daee8921') - -# Content: ENV WIX_VERSION "3.11.2" -[System.Environment]::SetEnvironmentVariable('WIX_VERSION','3.11.2') - -# Content: ENV WIX_SHA256 "32bb76c478fcb356671d4aaf006ad81ca93eea32c22a9401b168fc7471feccd2" -[System.Environment]::SetEnvironmentVariable('WIX_SHA256','32bb76c478fcb356671d4aaf006ad81ca93eea32c22a9401b168fc7471feccd2') - -# Content: ENV CMAKE_VERSION "3.23.0" -[System.Environment]::SetEnvironmentVariable('CMAKE_VERSION','3.23.0') - -# Content: ENV CMAKE_SHA256 "1e772025844f1cc648d28f42090038e5ca5cf72e2889de26d8d05ee25da17061" -[System.Environment]::SetEnvironmentVariable('CMAKE_SHA256','1e772025844f1cc648d28f42090038e5ca5cf72e2889de26d8d05ee25da17061') - -# Content: ENV MSYS_VERSION "20210725" -[System.Environment]::SetEnvironmentVariable('MSYS_VERSION','20210725') - -# Content: ENV MSYS_SHA256 "4013a9d5e51b448343efc24fc6a324cc999bb96b4c01b13a6bd3c661bb5c8a82" -[System.Environment]::SetEnvironmentVariable('MSYS_SHA256','4013a9d5e51b448343efc24fc6a324cc999bb96b4c01b13a6bd3c661bb5c8a82') - -# Content: ENV NUGET_VERSION "5.8.0" -[System.Environment]::SetEnvironmentVariable('NUGET_VERSION','5.8.0') - -# Content: ENV NUGET_SHA256 "5c5b9c96165d3283b2cb9e5b65825d343e0e7139b9e70a250b4bb24c2285f3ba" -[System.Environment]::SetEnvironmentVariable('NUGET_SHA256','5c5b9c96165d3283b2cb9e5b65825d343e0e7139b9e70a250b4bb24c2285f3ba') - -# Content: ENV WINGET_VERSION "1.0.4.0" -[System.Environment]::SetEnvironmentVariable('WINGET_VERSION','1.0.4.0') - -# Content: ENV WINGET_SHA256 "8a4b727a8ad588c86603136a5d48093b59ed7491f7c5dc9f13725da0900fe6ab" -[System.Environment]::SetEnvironmentVariable('WINGET_SHA256','8a4b727a8ad588c86603136a5d48093b59ed7491f7c5dc9f13725da0900fe6ab') - -# Content: ENV EMBEDDED_PYTHON_2_VERSION "2.7.17" -[System.Environment]::SetEnvironmentVariable('EMBEDDED_PYTHON_2_VERSION','2.7.17') - -# Content: ENV EMBEDDED_PYTHON_2_SHA256 "557ea6690c5927360656c003d3114b73adbd755b712a2911975dde813d6d7afb" -[System.Environment]::SetEnvironmentVariable('EMBEDDED_PYTHON_2_SHA256','557ea6690c5927360656c003d3114b73adbd755b712a2911975dde813d6d7afb') - -# Content: ENV EMBEDDED_PYTHON_3_VERSION "3.8.1" -[System.Environment]::SetEnvironmentVariable('EMBEDDED_PYTHON_3_VERSION','3.8.1') - -# Content: ENV EMBEDDED_PYTHON_3_SHA256 "58563ca60891025923572107e02b8f07439928eb5222dd10466cc92089072c2a" -[System.Environment]::SetEnvironmentVariable('EMBEDDED_PYTHON_3_SHA256','58563ca60891025923572107e02b8f07439928eb5222dd10466cc92089072c2a') - -# Content: ENV EMBEDDED_PIP_VERSION "20.3.4" -[System.Environment]::SetEnvironmentVariable('EMBEDDED_PIP_VERSION','20.3.4') - -# Content: ENV CODEQL_VERSION "2.10.3" -[System.Environment]::SetEnvironmentVariable('CODEQL_VERSION','2.10.3') - -# Content: ENV CODEQL_HASH "46f64e21c74f41210ea3f2c433d1dc622e3eb0690b42373a73fba82122b929a1" -[System.Environment]::SetEnvironmentVariable('CODEQL_HASH','46f64e21c74f41210ea3f2c433d1dc622e3eb0690b42373a73fba82122b929a1') - -# Content: ENV NINJA_VERSION "1.11.0" -[System.Environment]::SetEnvironmentVariable('NINJA_VERSION','1.11.0') - -# Content: ENV NINJA_SHA256 "d0ee3da143211aa447e750085876c9b9d7bcdd637ab5b2c5b41349c617f22f3b" -[System.Environment]::SetEnvironmentVariable('NINJA_SHA256','d0ee3da143211aa447e750085876c9b9d7bcdd637ab5b2c5b41349c617f22f3b') - -# Content: ENV CACERTS_VERSION "2023-01-10" -[System.Environment]::SetEnvironmentVariable('CACERTS_VERSION','2023-01-10') - -# Content: ENV CACERTS_HASH "fb1ecd641d0a02c01bc9036d513cb658bbda62a75e246bedbc01764560a639f0" -[System.Environment]::SetEnvironmentVariable('CACERTS_HASH','fb1ecd641d0a02c01bc9036d513cb658bbda62a75e246bedbc01764560a639f0') - -# Content: LABEL target_agent="Agent 6/7" -# LABEL resolved: target_agent='Agent 6/7' - -# Content: LABEL target_arch=${TARGET_ARCH} -# LABEL resolved: target_arch='x64' - -# Content: LABEL windows_version=${WINDOWS_VERSION} -# LABEL resolved: windows_version='1809' - -# Content: LABEL git_version=${GIT_VERSION} -# LABEL resolved: git_version='2.26.2' - -# Content: LABEL sevenzip_version=${SEVENZIP_VERSION} -# LABEL resolved: sevenzip_version='19.0.0' - -# Content: LABEL vs2017buildtools_version=${VS2017BUILDTOOLS_VERSION} -# LABEL resolved: vs2017buildtools_version='16.8.3.0' - -# Content: LABEL go_version=${GO_VERSION} -# LABEL resolved: go_version='1.19.5' - -# Content: LABEL ruby_version=${RUBY_VERSION} -# LABEL resolved: ruby_version='2.6.6-1' - -# Content: LABEL wix_version=${WIX_VERSION} -# LABEL resolved: wix_version='3.11.2' - -# Content: LABEL cmake_version=${CMAKE_VERSION} -# LABEL resolved: cmake_version='3.23.0' - -# Content: LABEL msys_version=${MSYS_VERSION} -# LABEL resolved: msys_version='20210725' - -# Content: LABEL system_python_version=${PYTHON_VERSION} -# LABEL resolved: system_python_version='3.8.2' - -# Content: LABEL embedded_py2_version=${EMBEDDED_PYTHON_2_VERSION} -# LABEL resolved: embedded_py2_version='2.7.17' - -# Content: LABEL embedded_py3_version=${EMBEDDED_PYTHON_3_VERSION} -# LABEL resolved: embedded_py3_version='3.8.1' - -# Content: LABEL nuget_version=${NUGET_VERSION} -# LABEL resolved: nuget_version='5.8.0' - -# Content: LABEL codeql_version=${CODEQL_VERSION} -# LABEL resolved: codeql_version='2.10.3' - -# Add certificates needed for build & check certificates file hash -# We need to trust the DigiCert High Assurance EV Root CA certificate, which signs python.org, -# to be able to download some Python components during the Agent build. -# Content: RUN (New-Object System.Net.WebClient).DownloadFile(\"https://curl.haxx.se/ca/cacert-${ENV:CACERTS_VERSION}.pem\", \"cacert.pem\") -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to (New-Object System.Net.WebClient).DownloadFile("https://curl.haxx.se/ca/cacert-${ENV:CACERTS_VERSION}.pem", "cacert.pem") - ... -'@ -(New-Object System.Net.WebClient).DownloadFile("https://curl.haxx.se/ca/cacert-${ENV:CACERTS_VERSION}.pem", "cacert.pem") -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Content: RUN if ((Get-FileHash -Algorithm SHA256 .\cacert.pem).Hash -ne \"$ENV:CACERTS_HASH\") { Write-Host \"Wrong hashsum for cacert.pem: got '$((Get-FileHash -Algorithm SHA256 .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'.\"; exit 1 } -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to if ((Get-FileHash -Algorithm SHA256 .\cacert.pem).Hash -ne "$ENV:CACERTS_HASH") { Write-Host "Wrong hashsum for cacert.pem: got '$((Get-FileHash -Algorithm SHA256 .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'."; exit 1 } - ... -'@ -if ((Get-FileHash -Algorithm SHA256 .\cacert.pem).Hash -ne "$ENV:CACERTS_HASH") { Write-Host "Wrong hashsum for cacert.pem: got '$((Get-FileHash -Algorithm SHA256 .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'."; exit 1 } -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Content: RUN setx SSL_CERT_FILE \"C:\cacert.pem\" -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to setx SSL_CERT_FILE "C:\cacert.pem" - ... -'@ -setx SSL_CERT_FILE "C:\cacert.pem" -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -### Preliminary step: we need both the .NET 3.5 runtime and -### the .NET 4.8 runtime. To do this, we get 4.8 from a base image and we -### manually the install .NET Framework 3.5 runtime using the instructions in -### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile: -### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile -### The .NET Fx 3.5 is needed for the Visual C++ Compiler for Python 2.7 -### (https://www.microsoft.com/en-us/download/details.aspx?id=44266) -### and to work around a bug in the WiX 3.11 installer -### (https://github.com/wixtoolset/issues/issues/5661). -# Install .NET Fx 3.5 -# Content: COPY ./windows/install_net35.ps1 install_net35.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_net35.ps1 install_net35.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_net35.ps1 install_net35.ps1 -Write-Host - -# Content: RUN Powershell -C .\install_net35.ps1 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_net35.ps1 - ... -'@ -.\install_net35.ps1 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -### End of preliminary step -# Install 7-zip -# Content: Copy ./windows/install_7zip.ps1 install_7zip.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_7zip.ps1 install_7zip.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_7zip.ps1 install_7zip.ps1 -Write-Host - -# Content: RUN powershell -Command .\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256 - ... -'@ -.\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install git -# Content: COPY ./windows/install_mingit.ps1 install_mingit.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_mingit.ps1 install_mingit.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_mingit.ps1 install_mingit.ps1 -Write-Host - -# Content: RUN powershell -Command .\install_mingit.ps1 -Version $ENV:GIT_VERSION -Sha256 $ENV:GIT_SHA256 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_mingit.ps1 -Version $ENV:GIT_VERSION -Sha256 $ENV:GIT_SHA256 - ... -'@ -.\install_mingit.ps1 -Version $ENV:GIT_VERSION -Sha256 $ENV:GIT_SHA256 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -### HACK: we disable symbolic links when cloning repositories -### to work around a symlink-related failure in the agent-binaries omnibus project -### when copying the datadog-agent project twice. -# Content: RUN git config --system core.symlinks false -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to git config --system core.symlinks false - ... -'@ -git config --system core.symlinks false -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install VS2017 -# Content: COPY ./windows/install_vstudio.ps1 install_vstudio.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_vstudio.ps1 install_vstudio.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_vstudio.ps1 install_vstudio.ps1 -Write-Host - -# Content: RUN powershell -Command .\install_vstudio.ps1 -Version $ENV:VS2017BUILDTOOLS_VERSION -Sha256 $ENV:VS2017BUILDTOOLS_SHA256 $ENV:VS2017BUILDTOOLS_DOWNLOAD_URL -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_vstudio.ps1 -Version $ENV:VS2017BUILDTOOLS_VERSION -Sha256 $ENV:VS2017BUILDTOOLS_SHA256 $ENV:VS2017BUILDTOOLS_DOWNLOAD_URL - ... -'@ -.\install_vstudio.ps1 -Version $ENV:VS2017BUILDTOOLS_VERSION -Sha256 $ENV:VS2017BUILDTOOLS_SHA256 $ENV:VS2017BUILDTOOLS_DOWNLOAD_URL -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# If x64, install the WDK for driver development -# Content: COPY ./windows/install_wdk.ps1 install_wdk.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_wdk.ps1 install_wdk.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_wdk.ps1 install_wdk.ps1 -Write-Host - -# Content: RUN if ($Env:TARGET_ARCH -eq 'x64') { powershell -Command .\install_wdk.ps1 } -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to if ($Env:TARGET_ARCH -eq 'x64') { powershell -Command .\install_wdk.ps1 } - ... -'@ -if ($Env:TARGET_ARCH -eq 'x64') { powershell -Command .\install_wdk.ps1 } -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install Wix and update PATH to include it -# Content: COPY ./windows/install_wix.ps1 install_wix.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_wix.ps1 install_wix.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_wix.ps1 install_wix.ps1 -Write-Host - -# Content: RUN powershell -Command .\install_wix.ps1 -Version $ENV:WIX_VERSION -Sha256 $ENV:WIX_SHA256 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_wix.ps1 -Version $ENV:WIX_VERSION -Sha256 $ENV:WIX_SHA256 - ... -'@ -.\install_wix.ps1 -Version $ENV:WIX_VERSION -Sha256 $ENV:WIX_SHA256 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install dotnet core -# Content: COPY ./windows/install_dotnetcore.ps1 install_dotnetcore.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_dotnetcore.ps1 install_dotnetcore.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_dotnetcore.ps1 install_dotnetcore.ps1 -Write-Host - -# Content: RUN powershell -Command .\install_dotnetcore.ps1 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_dotnetcore.ps1 - ... -'@ -.\install_dotnetcore.ps1 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# install nuget -# Content: COPY ./windows/install_nuget.ps1 install_nuget.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_nuget.ps1 install_nuget.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_nuget.ps1 install_nuget.ps1 -Write-Host - -# Content: RUN powershell -Command .\install_nuget.ps1 -Version $ENV:NUGET_VERSION -Sha256 $ENV:NUGET_SHA256 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_nuget.ps1 -Version $ENV:NUGET_VERSION -Sha256 $ENV:NUGET_SHA256 - ... -'@ -.\install_nuget.ps1 -Version $ENV:NUGET_VERSION -Sha256 $ENV:NUGET_SHA256 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install VC compiler for Python 2.7 -# Content: COPY ./windows/install_vcpython.ps1 install_vcpython.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_vcpython.ps1 install_vcpython.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_vcpython.ps1 install_vcpython.ps1 -Write-Host - -# Content: RUN powershell -Command .\install_vcpython.ps1 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_vcpython.ps1 - ... -'@ -.\install_vcpython.ps1 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install IBM MQ -# Content: COPY ./windows/install_ibm_mq.ps1 install_ibm_mq.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_ibm_mq.ps1 install_ibm_mq.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_ibm_mq.ps1 install_ibm_mq.ps1 -Write-Host - -# Content: RUN Powershell -C .\install_ibm_mq.ps1 -Version $ENV:IBM_MQ_VERSION -Sha256 $ENV:IBM_MQ_SHA256 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_ibm_mq.ps1 -Version $ENV:IBM_MQ_VERSION -Sha256 $ENV:IBM_MQ_SHA256 - ... -'@ -.\install_ibm_mq.ps1 -Version $ENV:IBM_MQ_VERSION -Sha256 $ENV:IBM_MQ_SHA256 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Content: RUN setx MQ_FILE_PATH c:\ibm_mq -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to setx MQ_FILE_PATH c:\ibm_mq - ... -'@ -setx MQ_FILE_PATH c:\ibm_mq -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install Cmake and update PATH to include it -# Content: COPY ./windows/install_cmake.ps1 install_cmake.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_cmake.ps1 install_cmake.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_cmake.ps1 install_cmake.ps1 -Write-Host - -# Content: RUN powershell -Command .\install_cmake.ps1 -Version $ENV:CMAKE_VERSION -Sha256 $ENV:CMAKE_SHA256 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_cmake.ps1 -Version $ENV:CMAKE_VERSION -Sha256 $ENV:CMAKE_SHA256 - ... -'@ -.\install_cmake.ps1 -Version $ENV:CMAKE_VERSION -Sha256 $ENV:CMAKE_SHA256 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install Winget and update PATH to include it -# Content: COPY ./windows/install_winget.ps1 install_winget.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_winget.ps1 install_winget.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_winget.ps1 install_winget.ps1 -Write-Host - -# Content: RUN powershell -Command .\install_winget.ps1 -Version $ENV:WINGET_VERSION -Sha256 $ENV:WINGET_SHA256 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_winget.ps1 -Version $ENV:WINGET_VERSION -Sha256 $ENV:WINGET_SHA256 - ... -'@ -.\install_winget.ps1 -Version $ENV:WINGET_VERSION -Sha256 $ENV:WINGET_SHA256 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install golang and set GOPATH to the dev path used in builds & tests -# RUN cinst -y --no-progress golang $ENV:CHOCO_ARCH_FLAG --version $ENV:GO_VERSION -# Some go point releases are not available as chocolatey packages, we have to install them directly -# Content: COPY ./windows/install_go.ps1 install_go.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_go.ps1 install_go.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_go.ps1 install_go.ps1 -Write-Host - -# Content: RUN powershell -C .\install_go.ps1 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_go.ps1 - ... -'@ -.\install_go.ps1 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Content: RUN setx GOPATH C:\dev\go -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to setx GOPATH C:\dev\go - ... -'@ -setx GOPATH C:\dev\go -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install system Python 3 (to use invoke). -# We always install the 64 bit version because vcredist140 won't work otherwise -# Content: COPY ./windows/install_python.ps1 install_python.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_python.ps1 install_python.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_python.ps1 install_python.ps1 -Write-Host - -# Content: COPY ./requirements.txt ./requirements-py2.txt / -Write-Host @' -Starting to copy $DockerRepoPath\.\requirements.txt \ ... -'@ -copy $DockerRepoPath\.\requirements.txt \ -Write-Host @' -Starting to copy $DockerRepoPath\.\requirements-py2.txt \ ... -'@ -copy $DockerRepoPath\.\requirements-py2.txt \ -Write-Host - -# Content: RUN powershell -C .\install_python.ps1 -Version $ENV:PYTHON_VERSION -Sha256 $ENV:PYTHON_SHA256 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_python.ps1 -Version $ENV:PYTHON_VERSION -Sha256 $ENV:PYTHON_SHA256 - ... -'@ -.\install_python.ps1 -Version $ENV:PYTHON_VERSION -Sha256 $ENV:PYTHON_SHA256 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install 64-bit ruby and bundler (for omnibus builds) -# Content: COPY ./windows/install_ruby.ps1 install_ruby.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_ruby.ps1 install_ruby.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_ruby.ps1 install_ruby.ps1 -Write-Host - -# Content: RUN powershell -C .\install_ruby.ps1 -Version $ENV:RUBY_VERSION -Sha256 $ENV:RUBY_SHA256 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_ruby.ps1 -Version $ENV:RUBY_VERSION -Sha256 $ENV:RUBY_SHA256 - ... -'@ -.\install_ruby.ps1 -Version $ENV:RUBY_VERSION -Sha256 $ENV:RUBY_SHA256 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install msys2 system & install 64-bit C/C++ compilation toolchain -# Content: copy ./windows/install_msys.ps1 install_msys.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_msys.ps1 install_msys.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_msys.ps1 install_msys.ps1 -Write-Host - -# Content: RUN powershell -C .\install_msys.ps1 -Version $ENV:MSYS_VERSION -Sha256 $ENV:MSYS_SHA256 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_msys.ps1 -Version $ENV:MSYS_VERSION -Sha256 $ENV:MSYS_SHA256 - ... -'@ -.\install_msys.ps1 -Version $ENV:MSYS_VERSION -Sha256 $ENV:MSYS_SHA256 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Content: RUN ridk install 3 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to ridk install 3 - ... -'@ -ridk install 3 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Downgrade gcc and binutils due to https://github.com/golang/go/issues/46099 -# Content: RUN (New-Object System.Net.WebClient).DownloadFile(\"https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst\", \"C:/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst\") -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to (New-Object System.Net.WebClient).DownloadFile("https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst", "C:/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst") - ... -'@ -(New-Object System.Net.WebClient).DownloadFile("https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst", "C:/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst") -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Content: RUN (New-Object System.Net.WebClient).DownloadFile(\"https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst\", \"C:/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst\") -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to (New-Object System.Net.WebClient).DownloadFile("https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst", "C:/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst") - ... -'@ -(New-Object System.Net.WebClient).DownloadFile("https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst", "C:/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst") -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Content: RUN (New-Object System.Net.WebClient).DownloadFile(\"https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst\", \"C:/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst\") -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to (New-Object System.Net.WebClient).DownloadFile("https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst", "C:/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst") - ... -'@ -(New-Object System.Net.WebClient).DownloadFile("https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst", "C:/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst") -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Content: RUN C:\tools\msys64\msys2_shell.cmd -defterm -no-start -c \"pacman --noconfirm -U /c/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst\" -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to C:\tools\msys64\msys2_shell.cmd -defterm -no-start -c "pacman --noconfirm -U /c/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst" - ... -'@ -C:\tools\msys64\msys2_shell.cmd -defterm -no-start -c "pacman --noconfirm -U /c/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst" -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install docker, manifest-tool and notary -# Content: COPY ./windows/install_docker.ps1 install_docker.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_docker.ps1 install_docker.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_docker.ps1 install_docker.ps1 -Write-Host - -# Content: RUN powershell -Command .\install_docker.ps1 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_docker.ps1 - ... -'@ -.\install_docker.ps1 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install google cloud SDK -# Content: COPY ./windows/install_gcloud_sdk.ps1 install_gcloud_sdk.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_gcloud_sdk.ps1 install_gcloud_sdk.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_gcloud_sdk.ps1 install_gcloud_sdk.ps1 -Write-Host - -# Content: RUN powershell -C .\install_gcloud_sdk.ps1 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_gcloud_sdk.ps1 - ... -'@ -.\install_gcloud_sdk.ps1 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install embedded pythons (for unit testing) -# Content: COPY ./python-packages-versions.txt python-packages-versions.txt -Write-Host @' -Starting to copy $DockerRepoPath\.\python-packages-versions.txt python-packages-versions.txt ... -'@ -copy $DockerRepoPath\.\python-packages-versions.txt python-packages-versions.txt -Write-Host - -# Content: COPY ./windows/install_embedded_pythons.ps1 install_embedded_pythons.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_embedded_pythons.ps1 install_embedded_pythons.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_embedded_pythons.ps1 install_embedded_pythons.ps1 -Write-Host - -# Content: RUN powershell -C .\install_embedded_pythons.ps1 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_embedded_pythons.ps1 - ... -'@ -.\install_embedded_pythons.ps1 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# install vcpkg -# Content: COPY ./windows/install_vcpkg.ps1 install_vcpkg.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_vcpkg.ps1 install_vcpkg.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_vcpkg.ps1 install_vcpkg.ps1 -Write-Host - -# Content: RUN powershell -Command .\install_vcpkg.ps1 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_vcpkg.ps1 - ... -'@ -.\install_vcpkg.ps1 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Install CodeQL for CI execution of CodeQL -# Content: COPY ./windows/install_codeql.ps1 install_codeql.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_codeql.ps1 install_codeql.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_codeql.ps1 install_codeql.ps1 -Write-Host - -# Content: RUN powershell -C .\install_codeql.ps1 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_codeql.ps1 - ... -'@ -.\install_codeql.ps1 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# install ninja -# Content: COPY ./windows/install_ninja.ps1 install_ninja.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\install_ninja.ps1 install_ninja.ps1 ... -'@ -copy $DockerRepoPath\.\windows\install_ninja.ps1 install_ninja.ps1 -Write-Host - -# Content: RUN powershell -C .\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 - ... -'@ -.\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Add signtool to path -# Content: RUN [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\", [System.EnvironmentVariableTarget]::Machine) -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64", [System.EnvironmentVariableTarget]::Machine) - ... -'@ -[Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64", [System.EnvironmentVariableTarget]::Machine) -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Set 32-bit flag env var -# Content: RUN if ($Env:TARGET_ARCH -eq 'x86') { setx WINDOWS_BUILD_32_BIT 1 } -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to if ($Env:TARGET_ARCH -eq 'x86') { setx WINDOWS_BUILD_32_BIT 1 } - ... -'@ -if ($Env:TARGET_ARCH -eq 'x86') { setx WINDOWS_BUILD_32_BIT 1 } -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Content: COPY ./windows/set_cpython_compiler.cmd set_cpython_compiler.cmd -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\set_cpython_compiler.cmd set_cpython_compiler.cmd ... -'@ -copy $DockerRepoPath\.\windows\set_cpython_compiler.cmd set_cpython_compiler.cmd -Write-Host - -# Content: RUN .\set_cpython_compiler.cmd -$stopwatch = [system.diagnostics.stopwatch]::StartNew() -Write-Host @' -Starting to .\set_cpython_compiler.cmd - ... -'@ -.\set_cpython_compiler.cmd -# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -# Content: COPY ./windows/entrypoint.bat /entrypoint.bat -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\entrypoint.bat \entrypoint.bat ... -'@ -copy $DockerRepoPath\.\windows\entrypoint.bat \entrypoint.bat -Write-Host - -# Content: COPY ./windows/aws_networking.ps1 /aws_networking.ps1 -Write-Host @' -Starting to copy $DockerRepoPath\.\windows\aws_networking.ps1 \aws_networking.ps1 ... -'@ -copy $DockerRepoPath\.\windows\aws_networking.ps1 \aws_networking.ps1 -Write-Host - -# Content: ENTRYPOINT ["/entrypoint.bat"] - - -$totalStopwatch.Stop() -$timeTaken = $totalStopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "Script executed in $timeTaken" diff --git a/windows/dockerfile-to-powershell.py b/windows/dockerfile-to-powershell.py deleted file mode 100755 index abda78142..000000000 --- a/windows/dockerfile-to-powershell.py +++ /dev/null @@ -1,606 +0,0 @@ -from ast import Not -import sys -import getopt -import re -import io -from datetime import datetime -from dockerfile_parse import DockerfileParser - -# This script imports dockerfile-parse project -# https://github.com/containerbuildsystem/dockerfile-parse -# Before running the script run: python3 -m pip install -r requirements.txt -# -# This python script generates Powershell script which when executed will perform the -# same steps as the Docker when it builds an image from the same Dockerfile. This -# script runs the same commands and effectively simulates Docker image build but -# without Docker. Accordingly when generated Powershell script is run on a clean -# Windows machine or VM it should produce the same final state as built Docker -# container. If, for example, the Dockerfile creates Windows build-ready container -# then the generated Powershell script when run should make Windows machine or VM -# build-ready as well. -# -# This script processes only specified Dockerfile although in the future we can -# attempt to process Dockerfile of the bases image as well (and may be recursively up -# as much as we can) -# -# This script in theory should work with any Windows Dockerfile. However it had been -# tested mostly with -# https://github.com/DataDog/datadog-agent-buildimages/blob/main/windows/Dockerfile. -# It is possible that the Dockerfile modification may break the this python script or -# the generated Powershell. We will try to address future issues as soon as we can. -# -# There are few nuances which you should be aware when looking on the Python or the -# generated Powershell scripts: -# a) This script fasciliate two separate phases of running Dockerized Powershell -# script. n -# b) Generated Powershell script "memorizes" current Powershell directory and -# restores it after each RUN command executed (in case it had been changed). -# It also means that before the script it run one need to make sure to set -# current directory to the one which would be expected by the Docker when -# it build images. E.g. for datadog-agent-buildimages repository current -# directory need to be seto to C:\ because that is where Docker build image -# will run. Note: Unfortunately it is not explicit information but implied -# by some of its RUN and COPY commands. -# -# b) Simulation of Docker build image execution is not literal for RUN commands. -# Instead of spawning a shell or a process to execute run command generated script -# invokes these commands directly. The pros that it is simpler and faster but the -# cons that if the command failed it will break main script execution. In future -# we may either wrap it into error handling block or indeed spawn a separate -# process. -# -# In addition this approach will make impossible to run generated Powershell -# more than once on the same machine because some commands refuse to run if -# certain directories already exist (but offending process can be commented out). -# -# d) Because a command may change an environment variable which can be used in -# subsequent RUN command we generate Powershell call after each RUN command to -# update sessions' environment variables. You will see invocation of -# "Update-SessionEnvironment \" after each RUN command. -# -# e) In addition each command is echoed on the console and each command is -# measured. We also print a reference to the Dockerfile line number where -# Powershell script line was derived from. -# -# f) If you already generated and run Powershell script and corresponding Dockerfile -# had been changed you can still re-generate newer version of the Powershell -# script and try to re-apply it again. However your mileage may vary (and it -# depends very much on the Dockerfile content) ... -# * It may actually just work. -# * You may need to comment out everything except the difference between old -# and new Powershell script. -# * You may need to explicitly uninstall or remove previously deployed -# applications, files or other artifacts - - -# =========================================================== -# -# U T I L I T Y -# -def getCommandLine(): - cmdLine = '' - for arg in sys.argv: - if ' ' in arg: - cmdLine += '"{}" '.format(arg) - else: - cmdLine += "{} ".format(arg) - - return cmdLine - -class Args(object): - pass - -class Context(object): - pass - -def usage(): - print("""Usage: dockerfile-to-powershell.py -Example - dockerfile-to-powershell.py -d .\\Dockerfile -p .\\build.ps1 -a WINDOWS_VERSION=1809 -Required arguments - -d # INPUT - -p # OUTPUT> - -Optional arguments - -a dockerarg1=val1 # Used as Dockerfile ARG instruction - ... - -a dockerargN=valN - """) - - -def parseParams(argv): - dockerFilePath = '' - powershellFilePath = '' - verbose = False - dockerArgs = {} - try: - opts, args = getopt.getopt(argv,'d:p:a:-v') - except getopt.GetoptError: - print('dockerfile-to-powershell.py -d -p [-a dockerarg1=val1 ...][-v]') - sys.exit(1) - - for opt, arg in opts: - if opt == '-d': - dockerFilePath = arg - elif opt == '-p': - powershellFilePath = arg - elif opt == '-a': - # Split and validate - nameValue = arg.split('=') - if len(nameValue) != 2 or len(nameValue[0].strip()) == 0 or len(nameValue[1].strip()) == 0: - print('invalid docker argument "{}"'.format(arg)) - sys.exit(2) - - dockerArgs[nameValue[0].strip().upper()] = nameValue[1].strip() - elif opt == '-v': - verbose = True - - # Validate that both required parameters are provided - if (len(dockerFilePath) == 0 or len(powershellFilePath) == 0): - usage() - sys.exit(1) - - retArgs = Args() - retArgs.dockerFilePath = dockerFilePath - retArgs.powershellFilePath = powershellFilePath - retArgs.dockerArgs = dockerArgs - retArgs.verbose = verbose - - return retArgs - -# Parse Dockerfile into array of instructions. Currently we are using -# https://github.com/containerbuildsystem/dockerfile-parse -def parseDockerFile(dockerFilePath): - try: - with open(dockerFilePath, 'r') as dockerFile: - dockerFileContent = dockerFile.read() - - dockerParser = DockerfileParser() - - # If dockerFilePath is not "Dockerfile" then dockerParser.content will create "Dockerfile" file - # which is not desirable and have to be fixed in future. This behavior is part of the dockerfile - # parser sub-module - dockerParser.content = dockerFileContent - - return dockerParser - except Exception as inst: - errMsg = 'Failed to open "{}" Dockerfile. Please validate that it is accessible'.format(dockerFilePath) - print(errMsg) - sys.exit(1) - - -# Open/Create Powershell file for writing -def createPowershellFile(powershellFilePath): - try: - return io.open(powershellFilePath, 'w', newline="\r\n") - except Exception as inst: - print('Failed to create "{}" Powershell file. Please validate that it is accessible'.format(powershellFile)) - sys.exit(1) - - -# https://docs.docker.com/engine/reference/builder/#environment-replacement -def replaceArgOrEnv(dockerArgs, value): - - # No need to replace? - if value[0] != '$': - return value - - # Find terminating '}' - replacingValue = '' - suffix = '' - if value[1] == '{': - nameValue = value[2:].split('}') - if len(nameValue) == 1: - return value # no termination is found then it is just no-replacement string - - replacingValue = nameValue[0] - suffix = nameValue[1] - else: - replacingValue = value[1:] - - # If we do not have + or - replacement - nameValue = replacingValue.split(':') - if len(nameValue) == 1: - replacedValue = dockerArgs.get(replacingValue) - return replacedValue + suffix if replacedValue != None else '' + suffix - - - # We can have only one ':' - if len(nameValue) != 2: - print('The ENV replacement format "{}" is not supported or understood'.format(value)) - exit(1) - - # We can have only + or minus - minus = nameValue[1][0] == '-' - plus = nameValue[1][0] == '+' - if not minus and not plus: - print('The ENV replacement format "{}" is not supported or understood'.format(value)) - exit(1) - - replacingValue = nameValue[0] - defaultValue = nameValue[1][1:] - - # '-' - if minus: - return dockerArgs.get(replacingValue) + suffix if dockerArgs.get(replacingValue) != None else defaultValue + suffix - - # '+' - return defaultValue + suffix if dockerArgs.get(replacingValue) != None else '' + suffix - - -# =========================================================== -# -# G E N E R A T E -# - -# =========================================================== -# ARG -# -def lineGenerator_ARG(context, powershellFile, args, dockerLine): - generateDockerfileReferenceComment(powershellFile, dockerLine, args.verbose) - - # Check for default value ARG - nameValue = dockerLine['value'].split('=') - if len(nameValue) == 2: - # Add default argument as if it was not provided - argValueNorm = nameValue[0].strip().upper() - if not argValueNorm in args.dockerArgs: - args.dockerArgs[argValueNorm] = nameValue[1].strip() - elif len(nameValue) == 1: - # Get argument and print it as a comment - val = args.dockerArgs.get(nameValue[0].strip()) - if val != None: - powershellFile.write('# ARG={}\n'.format(val)) - else: - powershellFile.write('# ARG=') - else: - print('The ARG instruction format is not supported or understood'.format(dockerLine['content'])) - sys.exit(1) - - powershellFile.write('\n') - -# =========================================================== -# FROM -# -def lineGenerator_FROM(context, powershellFile, args, dockerLine): - generateDockerfileReferenceComment(powershellFile, dockerLine, args.verbose) - - value = dockerLine['value'] - - powershellFile.write('# Base Image: {}\n\n'.format(value)) - - - -# =========================================================== -# SHELL -# -def lineGenerator_SHELL(context, powershellFile, args, dockerLine): - generateDockerfileReferenceComment(powershellFile, dockerLine, args.verbose) - - # Parse and concatenate to sting string a value like this - # ["powershell", "-Command"] - shell = '' - for shellValue in dockerLine['value'][1:-1].split(','): - shellValue = shellValue.strip()[1:-1].strip() - shell = shell + ' ' + shellValue if len(shell) > 0 else shellValue - - args.dockerArgs['SHELL'] = shell - powershellFile.write('# SHELL="{}"\n\n'.format(shell)) - -# =========================================================== -# COMMENT -# -def lineGenerator_COMMENT(context, powershellFile, args, dockerLine): - powershellFile.write(dockerLine['content']) - -# =========================================================== -# ENV -# -def lineGenerator_ENV(context, powershellFile, args, dockerLine): - generateDockerfileReferenceComment(powershellFile, dockerLine, args.verbose) - - nameValue = dockerLine['value'].split(' ') - if len(nameValue) == 1: - nameValue = dockerLine['value'].split('=') - - envName = nameValue[0].strip() - value = nameValue[1].strip() - envValue = replaceArgOrEnv(args.dockerArgs, value) if value[0] != '"' else value[1:-1] - - powershellFile.write("[System.Environment]::SetEnvironmentVariable('{}','{}')\n\n".format(envName, envValue)) - - # add ENV to dockerArgs (because ${xxx} replacement looks similar) - args.dockerArgs[envName.upper()] = envValue - -# =========================================================== -# LABEL -# -def lineGenerator_LABEL(context, powershellFile, args, dockerLine): - generateDockerfileReferenceComment(powershellFile, dockerLine, args.verbose) - - nameValue = dockerLine['value'].split('=') - if len(nameValue) != 2: - print('The LABEL instruction format is not supported or understood'.format(dockerLine['content'])) - sys.exit(1) - - label = nameValue[0].strip() - valueRaw = nameValue[1].strip() - value = replaceArgOrEnv(args.dockerArgs, valueRaw) if valueRaw[0] != '"' else valueRaw[1:-1] - - powershellFile.write("# LABEL resolved: {}='{}'\n\n".format(label, value)) - -# =========================================================== -# RUN -# -def lineGenerator_RUN(context, powershellFile, args, dockerLine): - generateDockerfileReferenceComment(powershellFile, dockerLine, args.verbose) - - # There are two forms of RUN but we are using only one currently - runRaw = dockerLine['content'][4:] - runNoEscapedQuotes = runRaw.replace("\\\"", "\"") - runNoEscapedTick = runNoEscapedQuotes.replace("`", "") - - runLines = runNoEscapedTick.split('\\\n') - - firstRunLine = runLines[0] - shell = args.dockerArgs['SHELL'] - powershellC = 'powershell -c ' - powershellCIdx = firstRunLine.lower().find(powershellC) - powershellCommand = 'powershell -command ' - powershellCommandIdx = firstRunLine.lower().find(powershellCommand) - shellIsPowershell = True if shell.lower().find('powershell -command') == 0 else False - - # Replace Powershell script invokation with simple "Call operator &" - firstLineCmd = '' - if shellIsPowershell: - if powershellCIdx == 0: - firstLineCmd = '{}'.format(firstRunLine[len(powershellC):]) - elif powershellCommandIdx == 0: - firstLineCmd = '{}'.format(firstRunLine[len(powershellCommand):]) - else: - firstLineCmd = '{}'.format(firstRunLine) - else: - firstLineCmd = '{} {}'.format(args.dockerArgs['SHELL'], firstRunLine) - - # Start stopwatch - powershellFile.write('$stopwatch = [system.diagnostics.stopwatch]::StartNew()\n') - - # Show step # - context.currentTaskIndex += 1 - generateProgress(context, powershellFile, args.verbose, firstLineCmd) - - # This will run it - if len(runLines) == 1: - powershellFile.write(firstLineCmd) - else: - powershellFile.write('{}`\n'.format(firstLineCmd)) - for idx in range(1, len(runLines)): - if idx < len(runLines) - 1: - powershellFile.write('{}`\n'.format(runLines[idx])) - else: - powershellFile.write('{}\n'.format(runLines[idx])) - - - # Restore environment - powershellFile.write("""# Restore location and reload environment in case if they were changed -Set-Location $origLocation -# [Environment]::CurrentDirectory = $origLocation ## In some future cases this line may need to be executed -Update-SessionEnvironment -# Measure taken time -$stopwatch.Stop() -$timeTaken = $stopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "... this step executed in $timeTaken`n" - -""") - - -# =========================================================== -# COPY -# -def lineGenerator_COPY(context, powershellFile, args, dockerLine): - generateDockerfileReferenceComment(powershellFile, dockerLine, args.verbose) - - # Remove Dockerfile based specific "--from=xxx" construct. We can do it - # because Docker base image should be accessible in file system during - # Powershell script run - copyNoFrom = re.sub('--from=\S+', '', dockerLine['value']).strip() - nameValues = copyNoFrom.split() - - context.currentTaskIndex += 1 - - dest = nameValues[len(nameValues) - 1].replace('/', '\\') - - # Copy each of the few src - for srcIdx in range(len(nameValues) - 1): - src = nameValues[srcIdx].replace('/', '\\') - - # Absolute path will copy AS IS (it usually part of "--from=xxx") - srcPath = src if len(src) > 2 and src[1] == ':' else '$DockerRepoPath\{}'.format(src) - - # Need to dynamically detect if the source is directory and copy as directory - # currently copy is good only for files - cmd = 'copy {} {}'.format(srcPath, dest) - generateProgress(context, powershellFile, args.verbose, cmd) - powershellFile.write('{}\n'.format(cmd)) - - powershellFile.write('Write-Host\n\n') - - -# =========================================================== -# ENTRYPOINT -# -def lineGenerator_ENTRYPOINT(context, powershellFile, args, dockerLine): - generateDockerfileReferenceComment(powershellFile, dockerLine, args.verbose) - powershellFile.write('\n\n') - -def lineGenerator_CMD(context, powershellFile, args, dockerLine): - generateDockerfileReferenceComment(powershellFile, dockerLine, args.verbose) - powershellFile.write('\n\n') - -def getLineGeneratorsMap(): - lineHandlersMap = { - 'ARG': lineGenerator_ARG, - 'FROM': lineGenerator_FROM, - 'SHELL': lineGenerator_SHELL, - 'COMMENT': lineGenerator_COMMENT, - 'ENV': lineGenerator_ENV, - 'LABEL': lineGenerator_LABEL, - 'RUN': lineGenerator_RUN, - 'COPY': lineGenerator_COPY, - 'ENTRYPOINT': lineGenerator_ENTRYPOINT, - 'CMD': lineGenerator_CMD - } - return lineHandlersMap - -def getTaskCountersMap(): - taskCountersMap = { - 'ARG': 0, - 'FROM': 0, - 'SHELL': 0, - 'COMMENT': 0, - 'ENV': 0, - 'LABEL': 0, - 'RUN': 1, - 'COPY': 1, - 'ENTRYPOINT': 0, - 'CMD': 0 - } - return taskCountersMap - -def generatePowershell(dockerParser, powershellFile, args): - - # Add header to the Powershell file - generatePowershellHeader(powershellFile) - - # Calculate counters - context = Context() - context.currentTaskIndex = 0 - context.totalTask = 0 - taskCountersMap = getTaskCountersMap() - for dockerLine in dockerParser.structure: - context.totalTask += taskCountersMap[dockerLine['instruction']] - - # Process Dockerfile line by line - lineHandlersMap = getLineGeneratorsMap() - for dockerLine in dockerParser.structure: - lineHandlersMap[dockerLine['instruction']](context, powershellFile, args, dockerLine) - - # Generate footer - generateFooter(powershellFile) - - # close the file - powershellFile.close() - -def generatePowershellHeader(powershellFile): - powershellFile.write('# Automatically generated by\n') - powershellFile.write('# {}\n'.format(getCommandLine())) - powershellFile.write('# on {}\n'.format(datetime.now().strftime('%m/%d/%Y %H:%M:%S'))) - - powershellFile.write(""" -# WARNING!!! -# If this script had been generated from ... -# https://github.com/DataDog/datadog-agent-buildimages/blob/main/windows/Dockerfile -# ... which is currently based ... -# on mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-ltsc2019 -# ... image, then you will need to install .NET 4.8 before running it from one of three sources -# * https://dotnet.microsoft.com/en-us/download/dotnet-framework/net48 -# * https://support.microsoft.com/en-us/topic/microsoft-net-framework-4-8-offline-installer-for-windows-9d23f658-3b97-68ab-d013-aa3c3e7495e0 -# * https://github.com/microsoft/dotnet-framework-docker/blob/528e53c63f429860d99d55409b0018d225059f0c/src/runtime/4.8/windowsservercore-ltsc2019/Dockerfile -# -# Similarly, for other Dockerfiles you need to make sure that you prepare Windows machine to -# be matching its Dockerfile's BASE image. -# -# If you try to run this script more than once some commands may fail and break the script. -# It may be addressed better in the future but for now you may need to comment out offending -# commands. If after running generated from the Dockerfile Powershell script the Dockerfile -# had been modified and you wish to apply these changes, then in contrast to applying newly -# generated Powershell script your mileage may vary (and it depends very much on the Dockerfile -# content) ... -# * It may actually just work -# * You may need to comment out everything except the difference between old and new -# Powershell script -# * You may need to explicitly uninstall or remove previously deployed applications, files -# or other artifacts -# -# Usage: .\ -DockerRepoPath -# -DockerRepoPath is required parameter. You need to clone the -# same repo as the one used for building Docker image. -# Specifically it is https://github.com/DataDog/datadog-agent-buildimages -Param( - [Parameter(Mandatory)] - [string] - $DockerRepoPath -) - -$DockerRepoPathExists = Test-Path $DockerRepoPath -if (!$DockerRepoPathExists) { - Write-Host "Provided \'DockerRepoPath\' ($DockerRepoPath) path does not exist. Please provide valid argument" - Exit -} - -# Save current location and sync it to .NET current location (it is not automatic and needed in some cases) -# https://stackoverflow.com/questions/11246068/why-dont-net-objects-in-powershell-use-the-current-directory -$origLocation = Get-Location -[Environment]::CurrentDirectory = $origLocation -# Currently Docker run everything in the root. And we have to keep it this way -Set-Location $origLocation - -$totalStopwatch = [system.diagnostics.stopwatch]::StartNew() - -function Update-SessionEnvironment { - foreach ($s in 'Machine','User') { - [Environment]::GetEnvironmentVariables($s).GetEnumerator(). - Where({$_.Key -ne 'PATH'}) | ForEach-Object { - [Environment]::SetEnvironmentVariable($_.Key,$_.Value,'Process') }} - - $env:PATH = ( ('Machine','User').ForEach({ - [Environment]::GetEnvironmentVariable('PATH',$_)}). - Split(';').Where({$_}) | Select-Object -Unique ) -join ';' -} -""") - -def generateProgress(context, powershellFile, verbose, cmd): - powershellFile.write("Write-Host @'\n") - if verbose: - powershellFile.write('[{}/{}] Starting to {} ...\n'.format(context.currentTaskIndex, context.totalTask, cmd)) - else: - powershellFile.write('Starting to {} ...\n'.format(cmd)) - - powershellFile.write("'@\n") - -def generateFooter(powershellFile): - powershellFile.write("""$totalStopwatch.Stop() -$timeTaken = $totalStopwatch.Elapsed.ToString("dd\.hh\:mm\:ss") -Write-Host "Script executed in $timeTaken" -""") - -def generateDockerfileReferenceComment(powershellFile, dockerLine, verbose): - lines = dockerLine['content'].split('\n') - if verbose: - powershellFile.write('# Line: {}, Content: {}\n'.format((dockerLine['startline'] + 1), lines[0])) - else: - powershellFile.write('# Content: {}\n'.format(lines[0])) - - if len(lines) > 1: - for lineIdx in range(1, len(lines) -1): - powershellFile.write('# {}\n'.format(lines[lineIdx])) - -# =========================================================== -# -# M A I N -# - -def main(argv): - # Get parameters - args = parseParams(argv) - - # Parse Dockerfile - dockerParser = parseDockerFile(args.dockerFilePath) - - # Create Powershell file - powershellFile = createPowershellFile(args.powershellFilePath) - - generatePowershell(dockerParser, powershellFile, args) - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/windows/helpers.ps1 b/windows/helpers.ps1 new file mode 100644 index 000000000..0bdf0b827 --- /dev/null +++ b/windows/helpers.ps1 @@ -0,0 +1,93 @@ +param( + [Parameter(Mandatory = $false)][switch] $Container +) + +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$ErrorActionPreference = 'Stop' + +# Define get-remotefile so that it can be used throughout +function Get-RemoteFile() { + param( + [Parameter(Mandatory = $true)][string] $RemoteFile, + [Parameter(Mandatory = $true)][string] $LocalFile, + [Parameter(Mandatory = $false)][string] $VerifyHash + ) + Write-Host -ForegroundColor Green "Downloading: $RemoteFile" + Write-Host -ForegroundColor Green " To: $LocalFile" + (New-Object System.Net.WebClient).DownloadFile($RemoteFile, $LocalFile) + if ($VerifyHash){ + $dlhash = (Get-FileHash -Algorithm SHA256 $LocalFile).hash.ToLower() + if($dlhash -ne $VerifyHash){ + Write-Host -ForegroundColor Red "Unexpected file hash downloading $LocalFile from $RemoteFile" + Write-Host -ForegroundColor Red "Expected $VerifyHash, got $dlhash" + throw 'Unexpected File Hash' + } + } +} + +function Add-EnvironmentVariable() { + param( + [Parameter(Mandatory = $true)][string] $Variable, + [Parameter(Mandatory = $true)][string] $Value, + [Parameter(Mandatory = $false)][switch] $Local, + [Parameter(Mandatory = $false)][switch] $Global + ) + if($Local) { + [Environment]::SetEnvironmentVariable($Variable, $Value, [System.EnvironmentVariableTarget]::Process) + } + if($Global){ + [Environment]::SetEnvironmentVariable($Variable, $Value, [System.EnvironmentVariableTarget]::Machine) + } +} + +function Add-ToPath() { + param( + [Parameter(Mandatory = $true)][string] $NewPath, + [Parameter(Mandatory = $false)][switch] $Local, + [Parameter(Mandatory = $false)][switch] $Global + ) + if($Local) { + $Env:Path="$Env:Path;$NewPath" + } + if($Global){ + $oldPath=[Environment]::GetEnvironmentVariable("Path", [System.EnvironmentVariableTarget]::Machine) + $target="$oldPath;$NewPath" + [Environment]::SetEnvironmentVariable("Path", $target, [System.EnvironmentVariableTarget]::Machine) + } +} + +function DownloadAndExpandTo{ + param( + [Parameter(Mandatory = $true)][string] $TargetDir, + [Parameter(Mandatory = $true)][string] $SourceURL, + [Parameter(Mandatory = $true)][string] $Sha256 + ) + $tmpOutFile = New-TemporaryFile + + Get-RemoteFile -LocalFile $tmpOutFile -RemoteFile $SourceURL -VerifyHash $Sha256 + + If(!(Test-Path $TargetDir)) + { + md $TargetDir + } + + Start-Process "7z" -ArgumentList "x -o${TargetDir} $tmpOutFile" -Wait + Remove-Item $tmpOutFile +} + +function Reload-Path() { + $newpath = @() + $syspath = [Environment]::GetEnvironmentVariable("Path", [System.EnvironmentVariableTarget]::Machine) + $userpath = [Environment]::GetEnvironmentVariable("Path", [System.EnvironmentVariableTarget]::User) + $existingpath = $Env:PATH + $all = @() + $all += $syspath -split ";" + $all += $userpath -split ";" + $all += $existingpath -split ";" + foreach ($p in $all){ + if ($newpath -notcontains $p){ + $newpath += $p + } + } + $Env:PATH=$newpath -join ";" +} \ No newline at end of file diff --git a/windows/aws_networking.ps1 b/windows/helpers/aws_networking.ps1 similarity index 100% rename from windows/aws_networking.ps1 rename to windows/helpers/aws_networking.ps1 diff --git a/windows/install_7zip.ps1 b/windows/helpers/install_7zip.ps1 similarity index 54% rename from windows/install_7zip.ps1 rename to windows/helpers/install_7zip.ps1 index 5692fb4e9..76b20e9be 100644 --- a/windows/install_7zip.ps1 +++ b/windows/helpers/install_7zip.ps1 @@ -8,18 +8,16 @@ $ErrorActionPreference = 'Stop' # Script directory is $PSScriptRoot -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + $shortenedver = $Version.Replace('.','') $sevenzip="https://www.7-zip.org/a/7z$($shortenedver)-x64.exe" Write-Host -ForegroundColor Green "Installing 7zip $sevenzip" $out = "$($PSScriptRoot)\7zip.exe" -(New-Object System.Net.WebClient).DownloadFile($sevenzip, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$Sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$Sha256'.\"; exit 1 } +Get-RemoteFile -RemoteFile $sevenzip -LocalFile $out -VerifyHash $Sha256 Start-Process 7zip.exe -ArgumentList '/S' -Wait Remove-Item $out -setx PATH "$Env:Path;c:\program files\7-zip" -$Env:Path="$Env:Path;c:\program files\7-zip" +Add-ToPath -NewPath "c:\program files\7-zip" -Local -Global Write-Host -ForegroundColor Green Done with 7zip diff --git a/windows/helpers/install_cert.ps1 b/windows/helpers/install_cert.ps1 new file mode 100644 index 000000000..269a0b289 --- /dev/null +++ b/windows/helpers/install_cert.ps1 @@ -0,0 +1,11 @@ +# +# Install_cert.ps1 +# +# downloads the certificate root file necessary for downloads during the agent build. +# Add certificates needed for build & check certificates file hash +# We need to trust the DigiCert High Assurance EV Root CA certificate, which signs python.org, +# to be able to download some Python components during the Agent build. +# +Get-RemoteFile -RemoteFile "https://curl.haxx.se/ca/cacert-${ENV:CACERTS_VERSION}.pem" -LocalFile "c:\cacert.pem" -VerifyHash $ENV:CACERTS_HASH + +Add-EnvironmentVariable -Variable "SSL_CERT_FILE" -Value "C:\cacert.pem" -Global \ No newline at end of file diff --git a/windows/install_cmake.ps1 b/windows/helpers/install_cmake.ps1 similarity index 65% rename from windows/install_cmake.ps1 rename to windows/helpers/install_cmake.ps1 index b9a9b9d8a..921fa9d61 100644 --- a/windows/install_cmake.ps1 +++ b/windows/helpers/install_cmake.ps1 @@ -3,13 +3,8 @@ param ( [Parameter(Mandatory=$true)][string]$Sha256 ) -# Enabled TLS12 -$ErrorActionPreference = 'Stop' - # Script directory is $PSScriptRoot -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - $shortenedver = $Version.Replace('.','') $splitver = $Version.split(".") $majmin = "$($splitver[0])$($splitver[1])" @@ -19,11 +14,11 @@ $cmakeurl = "https://github.com/Kitware/CMake/releases/download/v$($Version)/cma Write-Host -ForegroundColor Green starting with CMake $out = "$($PSScriptRoot)\cmake.msi" -(New-Object System.Net.WebClient).DownloadFile($cmakeurl, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$Sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$Sha256'.\"; exit 1 } + +Get-RemoteFile -RemoteFile $cmakeurl -LocalFile $out -VerifyHash $Sha256 Start-Process msiexec -ArgumentList "/q /i $($out) ADD_CMAKE_TO_PATH=System" -Wait Remove-Item $out - +Reload-Path Write-Host -ForegroundColor Green Done with CMake \ No newline at end of file diff --git a/windows/install_codeql.ps1 b/windows/helpers/install_codeql.ps1 similarity index 67% rename from windows/install_codeql.ps1 rename to windows/helpers/install_codeql.ps1 index 6dda52df4..5c0eb6a67 100644 --- a/windows/install_codeql.ps1 +++ b/windows/helpers/install_codeql.ps1 @@ -8,12 +8,7 @@ $out = 'codeql.zip' Write-Host -ForegroundColor Green "Downloading $codeqlzip to $out" -(New-Object System.Net.WebClient).DownloadFile($codeqlzip, $out) - -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$ENV:CODEQL_HASH") { - Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$ENV:CODEQL_HASH'.\"; - exit 1 -} +Get-RemoteFile -RemoteFile $codeqlzip -LocalFile $out -VerifyHash $ENV:CODEQL_HASH Write-Host -ForegroundColor Green "Extracting $out to c:\" @@ -23,7 +18,6 @@ Write-Host -ForegroundColor Green "Removing temporary file $out" Remove-Item $out -setx PATH "$Env:Path;c:\CodeQL;" -$Env:Path="$Env:Path;c:\CodeQL;" +Add-ToPath -NewPath "c:\CodeQL" -Global Write-Host -ForegroundColor Green "Installed CodeQL $ENV:CODEQL_VERSION" diff --git a/windows/helpers/install_docker.ps1 b/windows/helpers/install_docker.ps1 new file mode 100644 index 000000000..bb9357c04 --- /dev/null +++ b/windows/helpers/install_docker.ps1 @@ -0,0 +1,26 @@ +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' + +mkdir C:\Docker + +# Docker CLI builds maintained by a Docker engineer +$dockerVersion = "19.03.3" +$out = "C:\Docker\docker.exe" +$sha256 = "2d6ff967c717a38dd41f5ad418396bdeb84642fe04985b30925e38f593d386da" + +Get-RemoteFile -RemoteFile "https://github.com/StefanScherer/docker-cli-builder/releases/download/$dockerVersion/docker.exe" -LocalFile $out -VerifyHash $sha256 + +# Install manifest-tool +$manifestVersion = "v1.0.1" +$out = "C:\Docker\manifest-tool.exe" +$sha256 = "41c08bc1052534f07282eae1f2998e542734b53e79e8d84e4f989ac1c27b2861" +Get-RemoteFile -RemoteFile "https://github.com/estesp/manifest-tool/releases/download/$manifestVersion/manifest-tool-windows-amd64.exe" -LocalFile $out -VerifyHash $sha256 + +# Install notary +$notaryVersion = "v0.6.1" +$out = "C:\Docker\notary.exe" +$sha256 = "9d736f9b569b6a6a3de30cbfa3c60a764acdd445cf4ced760efa9d370bcad64f" +Get-RemoteFile -RemoteFile "https://github.com/theupdateframework/notary/releases/download/$notaryVersion/notary-Windows-amd64.exe" -LocalFile $out -VerifyHash $sha256 + +# Add Docker to path +Add-ToPath -NewPath "c:\Docker" -Local -Global diff --git a/windows/install_dotnetcore.ps1 b/windows/helpers/install_dotnetcore.ps1 similarity index 63% rename from windows/install_dotnetcore.ps1 rename to windows/helpers/install_dotnetcore.ps1 index 249c30afe..150ae0e42 100644 --- a/windows/install_dotnetcore.ps1 +++ b/windows/helpers/install_dotnetcore.ps1 @@ -1,10 +1,4 @@ - -# Enabled TLS12 -$ErrorActionPreference = 'Stop' - -# Script directory is $PSScriptRoot - -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +. .\helpers.ps1 $Version = "6.0.100" $url = "https://download.visualstudio.microsoft.com/download/pr/0f71eaf1-ce85-480b-8e11-c3e2725b763a/9044bfd1c453e2215b6f9a0c224d20fe/dotnet-sdk-6.0.100-win-x64.exe" Write-Host -ForegroundColor Green "Installing dotnetcore from $($Url)" @@ -12,17 +6,19 @@ Write-Host -ForegroundColor Green "Installing dotnetcore from $($Url)" $out = "$($PSScriptRoot)\dotnetcoresdk.exe" $sha256 = "1fcf6b9efd37d25e75b426cd8430eb3c006092bee07d748967f1dbfc3f9a0190" +Get-RemoteFile -RemoteFile $url -LocalFile $out -VerifyHash $sha256 Write-Host -ForegroundColor Green Downloading $Url to $out -(New-Object System.Net.WebClient).DownloadFile($Url, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$sha256'.\"; exit 1 } + # Skip extraction of XML docs - generally not useful within an image/container - helps performance -setx NUGET_XMLDOC_MODE skip +Add-EnvironmentVariable -Variable "NUGET_XMLDOC_NODE" -Value "skip" -Global -Local +#setx NUGET_XMLDOC_MODE skip start-process -FilePath $out -ArgumentList "/install /quiet /norestart" -wait Remove-Item $out +Reload-Path # Trigger first run experience by running arbitrary cmd dotnet help diff --git a/windows/install_embedded_pythons.ps1 b/windows/helpers/install_embedded_pythons.ps1 similarity index 55% rename from windows/install_embedded_pythons.ps1 rename to windows/helpers/install_embedded_pythons.ps1 index b490d8f34..4b6a77319 100644 --- a/windows/install_embedded_pythons.ps1 +++ b/windows/helpers/install_embedded_pythons.ps1 @@ -15,40 +15,6 @@ $ErrorActionPreference = 'Stop' -function DownloadFile{ - param( - [Parameter(Mandatory = $true)][string] $TargetFile, - [Parameter(Mandatory = $true)][string] $SourceURL, - [Parameter(Mandatory = $true)][string] $Sha256 - ) - $ErrorActionPreference = 'Stop' - $ProgressPreference = 'SilentlyContinue' - [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - - Write-Host -ForegroundColor Green "Downloading $SourceUrl to $TargetFile" - (New-Object System.Net.WebClient).DownloadFile($SourceURL, $TargetFile) - if ((Get-FileHash -Algorithm SHA256 $TargetFile).Hash -ne "$Sha256") { Write-Host \"Wrong hashsum for ${TargetFile}: got '$((Get-FileHash -Algorithm SHA256 $TargetFile).Hash)', expected '$Sha256'.\"; exit 1 } -} - -function DownloadAndExpandTo{ - param( - [Parameter(Mandatory = $true)][string] $TargetDir, - [Parameter(Mandatory = $true)][string] $SourceURL, - [Parameter(Mandatory = $true)][string] $Sha256 - ) - $tmpOutFile = New-TemporaryFile - - DownloadFile -TargetFile $tmpOutFile -SourceURL $SourceURL -Sha256 $Sha256 - - If(!(Test-Path $TargetDir)) - { - md $TargetDir - } - - Start-Process "7z" -ArgumentList "x -o${TargetDir} $tmpOutFile" -Wait - Remove-Item $tmpOutFile -} - $py2 = "https://s3.amazonaws.com/dd-agent-omnibus/python-windows-${Env:EMBEDDED_PYTHON_2_VERSION}-amd64.zip" $py3 = "https://s3.amazonaws.com/dd-agent-omnibus/python-windows-${Env:EMBEDDED_PYTHON_3_VERSION}-amd64.zip" @@ -59,19 +25,19 @@ DownloadAndExpandTo -TargetDir $py2Target -SourceURL $py2 -Sha256 "$Env:EMBEDDED DownloadAndExpandTo -TargetDir $py3Target -SourceURL $py3 -Sha256 "$Env:EMBEDDED_PYTHON_3_SHA256" -setx TEST_EMBEDDED_PY2 $py2Target -setx TEST_EMBEDDED_PY3 $py3Target +Add-EnvironmentVariable -Variable "TEST_EMBEDDED_PY2" -Value $py2Target -Global +Add-EnvironmentVariable -Variable "TEST_EMBEDDED_PY3" -Value $py3Target -Global # Read DD_PIP_VERSION{,_PY3} and DD_SETUPTOOLS_VERSION{,_PY3} to variables -Get-Content .\python-packages-versions.txt | Where-Object { $_.Trim() -ne '' } | Where-Object { $_.Trim() -notlike "#*" } | Foreach-Object{ +Get-Content \python-packages-versions.txt | Where-Object { $_.Trim() -ne '' } | Where-Object { $_.Trim() -notlike "#*" } | Foreach-Object{ $var = $_.Split('=') - [System.Environment]::SetEnvironmentVariable($var[0], $var[1]) + Add-EnvironmentVariable -Variable $var[0] -Value $var[1] -Local } # Python 2 $py2getpip = "https://raw.githubusercontent.com/pypa/get-pip/38e54e5de07c66e875c11a1ebbdb938854625dd8/public/2.7/get-pip.py" $py2getpipsha256 = "40ee07eac6674b8d60fce2bbabc148cf0e2f1408c167683f110fd608b8d6f416" -DownloadFile -TargetFile "get-pip.py" -SourceURL $py2getpip -Sha256 $py2getpipsha256 +Get-RemoteFile -LocalFile "get-pip.py" -RemoteFile $py2getpip -VerifyHash $py2getpipsha256 & "$py2Target\python" get-pip.py pip==${Env:DD_PIP_VERSION} If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } & "$py2Target\python" -m pip install -r ../requirements-py2.txt @@ -80,7 +46,7 @@ If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } # Python 3 $py3getpip = "https://raw.githubusercontent.com/pypa/get-pip/38e54e5de07c66e875c11a1ebbdb938854625dd8/public/get-pip.py" $py3getpipsha256 = "e235c437e5c7d7524fbce3880ca39b917a73dc565e0c813465b7a7a329bb279a" -DownloadFile -TargetFile "get-pip.py" -SourceURL $py3getpip -Sha256 $py3getpipsha256 +Get-RemoteFile -LocalFile "get-pip.py" -RemoteFile $py3getpip -VerifyHash $py3getpipsha256 & "$py3Target\python" get-pip.py pip==${Env:DD_PIP_VERSION_PY3} If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } & "$py3Target\python" -m pip install -r ../requirements.txt diff --git a/windows/install_gcloud_sdk.ps1 b/windows/helpers/install_gcloud_sdk.ps1 similarity index 65% rename from windows/install_gcloud_sdk.ps1 rename to windows/helpers/install_gcloud_sdk.ps1 index ac943425d..68ccb9c34 100644 --- a/windows/install_gcloud_sdk.ps1 +++ b/windows/helpers/install_gcloud_sdk.ps1 @@ -9,20 +9,19 @@ $out = 'gsdk.zip' $sha256 = "c9b283c9db4ed472111ccf32e6689fd467daf18ce3a77b8e601f9c646a83d86b" Write-Host -ForegroundColor Green Downloading $gsdk to $out -(New-Object System.Net.WebClient).DownloadFile($gsdk, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$sha256'.\"; exit 1 } + +Get-RemoteFile -RemoteFile $gsdk -LocalFile $out -VerifyHash $sha256 Get-ChildItem $out # write file size to make sure it worked Write-Host -ForegroundColor Green "File size is $((get-item $out).length)" -Expand-Archive gsdk.zip -DestinationPath C: +Expand-Archive gsdk.zip -DestinationPath C:\ Remove-Item $out # add to path -$pwd = pwd -setx PATH "$Env:Path;$pwd\google-cloud-sdk\bin" -$Env:Path="$Env:Path;$pwd\google-cloud-sdk\bin" +#$pwd = pwd +Add-ToPath -NewPath "c:\google-cloud-sdk\bin" -Local -Global Write-Host -ForegroundColor Green Done Installing Google Cloud SDK diff --git a/windows/install_go.ps1 b/windows/helpers/install_go.ps1 similarity index 60% rename from windows/install_go.ps1 rename to windows/helpers/install_go.ps1 index d0ba3b411..f154a9032 100644 --- a/windows/install_go.ps1 +++ b/windows/helpers/install_go.ps1 @@ -9,8 +9,7 @@ $out = 'go.zip' Write-Host -ForegroundColor Green "Downloading $gozip to $out" -(New-Object System.Net.WebClient).DownloadFile($gozip, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$ENV:GO_SHA256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$ENV:GO_SHA256'.\"; exit 1 } +Get-RemoteFile -RemoteFile $gozip -LocalFile $out -VerifyHash $ENV:GO_SHA256 Write-Host -ForegroundColor Green "Extracting $out to c:\" @@ -20,10 +19,9 @@ Write-Host -ForegroundColor Green "Removing temporary file $out" Remove-Item $out -setx GOROOT c:\go -$Env:GOROOT="c:\go" -setx PATH "$Env:Path;c:\go\bin;" -$Env:Path="$Env:Path;c:\go\bin;" +Add-EnvironmentVariable -Variable GOROOT -VALUE "c:\go" -Local -Global +Add-EnvironmentVariable -Variable GOPATH -VALUE "c:\dev\go" -Global +Add-ToPath -NewPath "c:\go" -Local -Global Write-Host -ForegroundColor Green "Installed go $ENV:GO_VERSION" diff --git a/windows/helpers/install_ibm_mq.ps1 b/windows/helpers/install_ibm_mq.ps1 new file mode 100644 index 000000000..d27b7fe80 --- /dev/null +++ b/windows/helpers/install_ibm_mq.ps1 @@ -0,0 +1,10 @@ +param ( + [Parameter(Mandatory=$true)][string]$Version, + [Parameter(Mandatory=$true)][string]$Sha256 +) + +$source = "https://s3.amazonaws.com/dd-agent-omnibus/ibm-mq-backup/$($Version)-IBM-MQC-Redist-Win64.zip" +$target = "c:\ibm_mq" + +DownloadAndExpandTo -TargetDir $target -SourceURL $source -Sha256 $Sha256 +Add-EnvironmentVariable -Variable MQ_FILE_PATH -VALUE "c:\ibm_mq" -Global \ No newline at end of file diff --git a/windows/install_mingit.ps1 b/windows/helpers/install_mingit.ps1 similarity index 57% rename from windows/install_mingit.ps1 rename to windows/helpers/install_mingit.ps1 index 5ff2c8192..6c09d0222 100644 --- a/windows/install_mingit.ps1 +++ b/windows/helpers/install_mingit.ps1 @@ -4,26 +4,22 @@ param ( ) # Enabled TLS12 -$ErrorActionPreference = 'Stop' -# Script directory is $PSScriptRoot -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +# Script directory is $PSScriptRoot $mingit = "https://github.com/git-for-windows/git/releases/download/v$($Version).windows.1/MinGit-$($Version)-64-bit.zip" Write-Host -ForegroundColor Green Installing MinGit $out = "$($PSScriptRoot)\mingit.zip" -(New-Object System.Net.WebClient).DownloadFile($mingit, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$Sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$Sha256'.\"; exit 1 } +Get-RemoteFile -RemoteFile $mingit -LocalFile $out -VerifyHash $Sha256 md c:\devtools\git & '7z' x -oc:\devtools\git $out Remove-Item $out # set path locally so we can initialize git config -setx PATH "$Env:Path;c:\devtools\git\cmd;c:\devtools\git\usr\bin" -$Env:Path="$Env:Path;c:\devtools\git\cmd;c:\devtools\git\usr\bin" +Add-ToPath -NewPath "c:\devtools\git\cmd;c:\devtools\git\usr\bin" -Local -Global & 'git.exe' config --global user.name "Croissant Builder" & 'git.exe' config --global user.email "croissant@datadoghq.com" diff --git a/windows/install_msys.ps1 b/windows/helpers/install_msys.ps1 similarity index 58% rename from windows/install_msys.ps1 rename to windows/helpers/install_msys.ps1 index 72e5f8ea0..c5c06620c 100644 --- a/windows/install_msys.ps1 +++ b/windows/helpers/install_msys.ps1 @@ -24,20 +24,14 @@ function Invoke-Msys2Shell($Arguments) { Write-Host "Invoking msys2 shell command:" $params.ArgumentList Start-Process @params } -# Enabled TLS12 -$ErrorActionPreference = 'Stop' - -# Script directory is $PSScriptRoot - -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 # https://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20200629.tar.xz $msyszip = "https://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-$($Version).tar.xz" Write-Host -ForegroundColor Green starting with MSYS $out = "$($PSScriptRoot)\msys.tar.xz" -(New-Object System.Net.WebClient).DownloadFile($msyszip, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$Sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$Sha256'.\"; exit 1 } + +Get-RemoteFile -RemoteFile $msyszip -LocalFile $out -VerifyHash $Sha256 # uncompress the tar-xz into a tar $msystar = "msys.tar" @@ -50,4 +44,11 @@ Remove-Item $msystar ## invoke the first-run shell Invoke-Msys2Shell +ridk install 3 +# Downgrade gcc and binutils due to https://github.com/golang/go/issues/46099 +Get-RemoteFile -RemoteFile "https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst" -LocalFile "C:/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst" +Get-RemoteFile -RemoteFile "https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst" -LocalFile "C:/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst" +Get-RemoteFile -RemoteFile "https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst" -LocalFile "C:/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst" +& C:\tools\msys64\msys2_shell.cmd -defterm -no-start -c "pacman --noconfirm -U /c/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst" + Write-Host -ForegroundColor Green Done with MSYS diff --git a/windows/install_net35.ps1 b/windows/helpers/install_net35.ps1 similarity index 96% rename from windows/install_net35.ps1 rename to windows/helpers/install_net35.ps1 index df0993f0e..fffb1d15d 100644 --- a/windows/install_net35.ps1 +++ b/windows/helpers/install_net35.ps1 @@ -92,9 +92,10 @@ Write-Host curl.exe -fSLo $out $UpgradeTable[$kernelver]["patch"] curl.exe -fSLo $out $UpgradeTable[$kernelver]["patch"] if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$sha256'.\"; exit 1 } + mkdir patch expand patch.msu patch -F:* remove-item -force patch.msu -Write-Host DISM /Online /Quiet /Add-Package /PackagePath:C:\patch\$($UpgradeTable[$kernelver]["expandedpatch"]) -DISM /Online /Quiet /Add-Package /PackagePath:C:\patch\$($UpgradeTable[$kernelver]["expandedpatch"]) -remove-item -force -recurse patch \ No newline at end of file +Write-Host DISM /Online /Quiet /Add-Package /PackagePath:$($PSScriptRoot)\$($UpgradeTable[$kernelver]["expandedpatch"]) +DISM /Online /Quiet /Add-Package /PackagePath:$($PSScriptRoot)\$($UpgradeTable[$kernelver]["expandedpatch"]) +remove-item -force -recurse patch diff --git a/windows/helpers/install_ninja.ps1 b/windows/helpers/install_ninja.ps1 new file mode 100644 index 000000000..d9704b932 --- /dev/null +++ b/windows/helpers/install_ninja.ps1 @@ -0,0 +1,14 @@ +param ( + [Parameter(Mandatory=$true)][string]$Version, + [Parameter(Mandatory=$true)][string]$Sha256 +) + +$ErrorActionPreference = 'Stop' + + +$source="https://github.com/ninja-build/ninja/releases/download/v${Version}/ninja-win.zip" +$target = "c:\ninja-build" + +DownloadAndExpandTo -TargetDir $target -SourceURL $source -Sha256 $Sha256 +Add-ToPath -NewPath "c:\ninja-build" -Global +Write-Host -ForegroundColor Green Done with ninja-build diff --git a/windows/install_nuget.ps1 b/windows/helpers/install_nuget.ps1 similarity index 50% rename from windows/install_nuget.ps1 rename to windows/helpers/install_nuget.ps1 index 321fa7bb9..9af85a54e 100644 --- a/windows/install_nuget.ps1 +++ b/windows/helpers/install_nuget.ps1 @@ -3,26 +3,19 @@ param ( [Parameter(Mandatory=$true)][string]$Sha256 ) -# Enabled TLS12 -$ErrorActionPreference = 'Stop' - -# Script directory is $PSScriptRoot - -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - # https://dist.nuget.org/win-x86-commandline/v5.7.0/nuget.exe $nugetexe="https://dist.nuget.org/win-x86-commandline/v$($Version)/nuget.exe" Write-Host -ForegroundColor Green "Downloading nuget" $out = "$($PSScriptRoot)\nuget.exe" -(New-Object System.Net.WebClient).DownloadFile($nugetexe, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$Sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$Sha256'.\"; exit 1 } + +Get-RemoteFile -RemoteFile $nugetexe -LocalFile $out -VerifyHash $Sha256 # just put it in it's own directory mkdir \nuget Copy-Item $out \nuget\nuget.exe Remove-Item $out -setx PATH "$Env:Path;c:\nuget" -$Env:Path="$Env:Path;c:\nuget" +Add-ToPath -NewPath "c:\nuget" -Local -Global + Write-Host -ForegroundColor Green Done with Nuget diff --git a/windows/helpers/install_python.ps1 b/windows/helpers/install_python.ps1 new file mode 100644 index 000000000..8d80ad678 --- /dev/null +++ b/windows/helpers/install_python.ps1 @@ -0,0 +1,31 @@ +param ( + [Parameter(Mandatory=$true)][string]$Version, + [Parameter(Mandatory=$true)][string]$Sha256 +) + + +# https://www.python.org/ftp/python/3.9.1/python-3.9.1-amd64.exe +$pyexe = "https://www.python.org/ftp/python/$($Version)/python-$($Version)-amd64.exe" + +Write-Host -ForegroundColor Green starting with Python +$out = "$($PSScriptRoot)\python.exe" + +Get-RemoteFile -RemoteFile $pyexe -LocalFile $out -VerifyHash $Sha256 + +Write-Host -ForegroundColor Green Done downloading Python, installing + +Start-Process $out -ArgumentList '/quiet InstallAllUsers=1' -Wait + +Add-ToPath "c:\program files\Python38;c:\Program files\python38\scripts" -Global -Local +Remove-Item $out + +$getpipurl = "https://raw.githubusercontent.com/pypa/get-pip/38e54e5de07c66e875c11a1ebbdb938854625dd8/public/get-pip.py" +$getpipsha256 = "e235c437e5c7d7524fbce3880ca39b917a73dc565e0c813465b7a7a329bb279a" +$target = "get-pip.py" + +Get-RemoteFile -RemoteFile $getpipurl -LocalFile $target -VerifyHash $getpipsha256 + +python get-pip.py pip==${Env:DD_PIP_VERSION_PY3} +python -m pip install -r /requirements.txt + +Write-Host -ForegroundColor Green Done with Python diff --git a/windows/install_ruby.ps1 b/windows/helpers/install_ruby.ps1 similarity index 61% rename from windows/install_ruby.ps1 rename to windows/helpers/install_ruby.ps1 index cc9cfd1dc..028c6b249 100644 --- a/windows/install_ruby.ps1 +++ b/windows/helpers/install_ruby.ps1 @@ -3,12 +3,6 @@ param ( [Parameter(Mandatory=$true)][string]$Sha256 ) -# Enabled TLS12 -$ErrorActionPreference = 'Stop' - -# Script directory is $PSScriptRoot - -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 # https://github.com/oneclick/rubyinstaller2/releases/download/RubyInstaller-2.7.4-1/rubyinstaller-2.7.4-1-x64.exe @@ -16,13 +10,15 @@ $rubyexe = "https://github.com/oneclick/rubyinstaller2/releases/download/RubyIns Write-Host -ForegroundColor Green starting with Ruby $out = "$($PSScriptRoot)\rubyinstaller.exe" -(New-Object System.Net.WebClient).DownloadFile($rubyexe, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$Sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$Sha256'.\"; exit 1 } + +Get-RemoteFile -RemoteFile $rubyexe -LocalFile $out -VerifyHash $Sha256 + Write-Host -ForegroundColor Green Done downloading Ruby, installing Start-Process $out -ArgumentList '/verysilent /dir="c:\tools\ruby" /tasks="assocfiles,noridkinstall,modpath"' -Wait -$Env:PATH="$Env:PATH;c:\tools\ruby\bin" -setx RIDK ((Get-Command ridk).Path) + +Add-ToPath -NewPath "c:\tools\ruby\bin" -Local +Add-EnvironmentVariable -Variable RIDK -Value ((Get-Command ridk).Path) -Global Start-Process gem -ArgumentList 'install bundler -v 2.3.24' -Wait diff --git a/windows/install_vcpkg.ps1 b/windows/helpers/install_vcpkg.ps1 similarity index 79% rename from windows/install_vcpkg.ps1 rename to windows/helpers/install_vcpkg.ps1 index 7eb600629..4e90a8f9a 100644 --- a/windows/install_vcpkg.ps1 +++ b/windows/helpers/install_vcpkg.ps1 @@ -15,6 +15,6 @@ Pop-Location Remove-Item -Recurse -Force C:\vcpkg-tool Remove-Item -Recurse -Force C:\vcpkg-build -[Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";C:\vcpkg\", [System.EnvironmentVariableTarget]::Machine) +Add-ToPath -NewPath "C:\vcpkg" -Global .\vcpkg\vcpkg.exe --version .\vcpkg\vcpkg.exe integrate install diff --git a/windows/install_vcpython.ps1 b/windows/helpers/install_vcpython.ps1 similarity index 52% rename from windows/install_vcpython.ps1 rename to windows/helpers/install_vcpython.ps1 index 5b9776b6c..a65c52365 100644 --- a/windows/install_vcpython.ps1 +++ b/windows/helpers/install_vcpython.ps1 @@ -1,9 +1,3 @@ -# Enabled TLS12 -$ErrorActionPreference = 'Stop' - -# Script directory is $PSScriptRoot - -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 $vcpmsi = 'https://s3.amazonaws.com/dd-agent-omnibus/VCForPython27.msi' @@ -12,8 +6,7 @@ Write-host -ForegroundColor Green Downloading VC++ for Python $out = "$($PSScriptRoot)\vcp.msi" $sha256 = "070474db76a2e625513a5835df4595df9324d820f9cc97eab2a596dcbc2f5cbf" -(New-Object System.Net.WebClient).DownloadFile($vcpmsi, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$sha256'.\"; exit 1 } +Get-RemoteFile -RemoteFile $vcpmsi -LocalFile $out -VerifyHash $sha256 Write-host -ForegroundColor Green VC++ for Python downloaded, installing... Start-Process msiexec -ArgumentList '/q /i vcp.msi' -Wait diff --git a/windows/install_vstudio.ps1 b/windows/helpers/install_vstudio.ps1 similarity index 64% rename from windows/install_vstudio.ps1 rename to windows/helpers/install_vstudio.ps1 index 7521cdca3..b68c8bceb 100644 --- a/windows/install_vstudio.ps1 +++ b/windows/helpers/install_vstudio.ps1 @@ -6,20 +6,13 @@ param ( [Parameter(Mandatory=$false)][switch]$NoQuiet ) -# Enabled TLS12 -$ErrorActionPreference = 'Stop' - -# Script directory is $PSScriptRoot - -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 Write-Host -ForegroundColor Green "Installing Visual Studio $($Version) from $($Url)" $out = "$($PSScriptRoot)\vs_buildtools.exe" Write-Host -ForegroundColor Green Downloading $Url to $out -(New-Object System.Net.WebClient).DownloadFile($Url, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$Sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$Sha256'.\"; exit 1 } +Get-RemoteFile -RemoteFile $Url -LocalFile $out -VerifyHash $Sha256 # write file size to make sure it worked Write-Host -ForegroundColor Green "File size is $((get-item $out).length)" @@ -60,16 +53,9 @@ $processparams = @{ } Start-Process @processparams +Add-EnvironmentVariable -Variable VSTUDIO_ROOT -Value $InstallRoot -Global -Local -setx VSTUDIO_ROOT "$InstallRoot" - -# add SDK added above to path for signtool -# C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64 -[Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${env:ProgramFiles(x86)}\Windows Kits\10\bin\10.0.18362.0\x64", [System.EnvironmentVariableTarget]::Machine) - -# add SDK added above to path for signtool -# C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64 -[Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${env:ProgramFiles(x86)}\Windows Kits\10\bin\10.0.18362.0\x64", [System.EnvironmentVariableTarget]::Machine) +Add-ToPath -NewPath "${env:ProgramFiles(x86)}\Windows Kits\10\bin\10.0.18362.0\x64" -Global Remove-Item $out Write-Host -ForegroundColor Green Done with Visual Studio diff --git a/windows/install_wdk.ps1 b/windows/helpers/install_wdk.ps1 similarity index 82% rename from windows/install_wdk.ps1 rename to windows/helpers/install_wdk.ps1 index 18785dbf9..a0e277711 100644 --- a/windows/install_wdk.ps1 +++ b/windows/helpers/install_wdk.ps1 @@ -10,8 +10,8 @@ Write-Host -ForegroundColor Green Installing WDK $out = 'wdksetup.exe' $sha256 = "c35057cb294096c63bbea093e5024a5fb4120103b20c13fa755c92f227b644e5" Write-Host -ForegroundColor Green Downloading $wdk to $out -(New-Object System.Net.WebClient).DownloadFile($wdk, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$sha256'.\"; exit 1 } + +Get-RemoteFile -RemoteFile $wdk -LocalFile $out -VerifyHash $sha256 Get-ChildItem $out # write file size to make sure it worked diff --git a/windows/helpers/install_winget.ps1 b/windows/helpers/install_winget.ps1 new file mode 100644 index 000000000..5ba1ab3d9 --- /dev/null +++ b/windows/helpers/install_winget.ps1 @@ -0,0 +1,19 @@ +param ( + [Parameter(Mandatory=$true)][string]$Version, + [Parameter(Mandatory=$true)][string]$Sha256 +) + + +$wingetexe="https://github.com/microsoft/winget-create/releases/download/v$($Version)/wingetcreate.exe" + +Write-Host -ForegroundColor Green "Downloading winget" +$out = "$($PSScriptRoot)\wingetcreate.exe" + +Get-RemoteFile -RemoteFile $wingetexe -LocalFile $out -VerifyHash $Sha256 + +# just put it in it's own directory +mkdir \winget +Copy-Item $out \winget\wingetcreate.exe +Remove-Item $out +Add-ToPath -NewPath "c:\winget" -Local -Global +Write-Host -ForegroundColor Green Done with Winget diff --git a/windows/helpers/install_wix.ps1 b/windows/helpers/install_wix.ps1 new file mode 100644 index 000000000..7ef4322ab --- /dev/null +++ b/windows/helpers/install_wix.ps1 @@ -0,0 +1,28 @@ +param ( + [Parameter(Mandatory=$true)][string]$Version, + [Parameter(Mandatory=$true)][string]$Sha256 +) + + +$shortenedver = $Version.Replace('.','') +$splitver = $Version.split(".") +$majmin = "$($splitver[0])$($splitver[1])" + +# https://github.com/wixtoolset/wix3/releases/download/wix3112rtm/wix311.exe +$wixzip = "https://github.com/wixtoolset/wix3/releases/download/wix$($shortenedver)rtm/wix$($majmin).exe" + +Write-Host -ForegroundColor Green starting with WiX +$out = "$($PSScriptRoot)\wix.exe" + +Get-RemoteFile -RemoteFile $wixzip -LocalFile $out -VerifyHash $Sha256 + +Write-Host -ForegroundColor Green Done downloading wix, installing +#Start-Process wix.exe -ArgumentList '/quiet' -Wait +Start-Process $out -ArgumentList '/q' -Wait + +Add-ToPath -NewPath "${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin\" -Global +Add-EnvironmentVariable -Variable "WIX" -Value "C:\Program Files (x86)\WiX Toolset v3.11\" -Global + +Remove-Item $out + +Write-Host -ForegroundColor Green Done with WiX \ No newline at end of file diff --git a/windows/install_docker.ps1 b/windows/install_docker.ps1 deleted file mode 100644 index 0c6e00633..000000000 --- a/windows/install_docker.ps1 +++ /dev/null @@ -1,29 +0,0 @@ -$ErrorActionPreference = 'Stop' -$ProgressPreference = 'SilentlyContinue' - -mkdir C:\Docker - -# Docker CLI builds maintained by a Docker engineer -$dockerVersion = "19.03.3" -$out = "C:\Docker\docker.exe" -$sha256 = "2d6ff967c717a38dd41f5ad418396bdeb84642fe04985b30925e38f593d386da" -Invoke-WebRequest -Uri "https://github.com/StefanScherer/docker-cli-builder/releases/download/$dockerVersion/docker.exe" -OutFile $out -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$sha256'.\"; exit 1 } - -# Install manifest-tool -$manifestVersion = "v1.0.1" -$out = "C:\Docker\manifest-tool.exe" -$sha256 = "41c08bc1052534f07282eae1f2998e542734b53e79e8d84e4f989ac1c27b2861" -Invoke-WebRequest -Uri "https://github.com/estesp/manifest-tool/releases/download/$manifestVersion/manifest-tool-windows-amd64.exe" -OutFile $out -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$sha256'.\"; exit 1 } - -# Install notary -$notaryVersion = "v0.6.1" -$out = "C:\Docker\notary.exe" -$sha256 = "9d736f9b569b6a6a3de30cbfa3c60a764acdd445cf4ced760efa9d370bcad64f" -Invoke-WebRequest -Uri "https://github.com/theupdateframework/notary/releases/download/$notaryVersion/notary-Windows-amd64.exe" -OutFile $out -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$sha256'.\"; exit 1 } - -# Add Docker to path -setx PATH "$Env:Path;C:\Docker" -$Env:Path="$Env:Path;C:\Docker" diff --git a/windows/install_ibm_mq.ps1 b/windows/install_ibm_mq.ps1 deleted file mode 100644 index a256c86f7..000000000 --- a/windows/install_ibm_mq.ps1 +++ /dev/null @@ -1,45 +0,0 @@ -param ( - [Parameter(Mandatory=$true)][string]$Version, - [Parameter(Mandatory=$true)][string]$Sha256 -) - -$ErrorActionPreference = 'Stop' - -function DownloadFile{ - param( - [Parameter(Mandatory = $true)][string] $TargetFile, - [Parameter(Mandatory = $true)][string] $SourceURL, - [Parameter(Mandatory = $true)][string] $Sha256 - ) - $ErrorActionPreference = 'Stop' - $ProgressPreference = 'SilentlyContinue' - [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - - Write-Host -ForegroundColor Green "Downloading $SourceUrl to $TargetFile" - (New-Object System.Net.WebClient).DownloadFile($SourceURL, $TargetFile) - if ((Get-FileHash -Algorithm SHA256 $TargetFile).Hash -ne "$Sha256") { Write-Host \"Wrong hashsum for ${TargetFile}: got '$((Get-FileHash -Algorithm SHA256 $TargetFile).Hash)', expected '$Sha256'.\"; exit 1 } -} - -function DownloadAndExpandTo{ - param( - [Parameter(Mandatory = $true)][string] $TargetDir, - [Parameter(Mandatory = $true)][string] $SourceURL, - [Parameter(Mandatory = $true)][string] $Sha256 - ) - $tmpOutFile = New-TemporaryFile - - DownloadFile -TargetFile $tmpOutFile -SourceURL $SourceURL -Sha256 $Sha256 - - If(!(Test-Path $TargetDir)) - { - md $TargetDir - } - - Start-Process "7z" -ArgumentList "x -o${TargetDir} $tmpOutFile" -Wait - Remove-Item $tmpOutFile -} - -$source = "https://s3.amazonaws.com/dd-agent-omnibus/ibm-mq-backup/$($Version)-IBM-MQC-Redist-Win64.zip" -$target = "c:\ibm_mq" - -DownloadAndExpandTo -TargetDir $target -SourceURL $source -Sha256 $Sha256 diff --git a/windows/install_ninja.ps1 b/windows/install_ninja.ps1 deleted file mode 100644 index 5f15bff73..000000000 --- a/windows/install_ninja.ps1 +++ /dev/null @@ -1,48 +0,0 @@ -param ( - [Parameter(Mandatory=$true)][string]$Version, - [Parameter(Mandatory=$true)][string]$Sha256 -) - -$ErrorActionPreference = 'Stop' - -function DownloadFile{ - param( - [Parameter(Mandatory = $true)][string] $TargetFile, - [Parameter(Mandatory = $true)][string] $SourceURL, - [Parameter(Mandatory = $true)][string] $Sha256 - ) - $ErrorActionPreference = 'Stop' - $ProgressPreference = 'SilentlyContinue' - [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - - Write-Host -ForegroundColor Green "Downloading $SourceUrl to $TargetFile" - (New-Object System.Net.WebClient).DownloadFile($SourceURL, $TargetFile) - if ((Get-FileHash -Algorithm SHA256 $TargetFile).Hash -ne "$Sha256") { Write-Host \"Wrong hashsum for ${TargetFile}: got '$((Get-FileHash -Algorithm SHA256 $TargetFile).Hash)', expected '$Sha256'.\"; exit 1 } -} - -function DownloadAndExpandTo{ - param( - [Parameter(Mandatory = $true)][string] $TargetDir, - [Parameter(Mandatory = $true)][string] $SourceURL, - [Parameter(Mandatory = $true)][string] $Sha256 - ) - $tmpOutFile = New-TemporaryFile - - DownloadFile -TargetFile $tmpOutFile -SourceURL $SourceURL -Sha256 $Sha256 - - If(!(Test-Path $TargetDir)) - { - md $TargetDir - } - - Start-Process "7z" -ArgumentList "x -o${TargetDir} $tmpOutFile" -Wait - Remove-Item $tmpOutFile -} - -$source="https://github.com/ninja-build/ninja/releases/download/v${Version}/ninja-win.zip" -$target = "c:\ninja-build" - -DownloadAndExpandTo -TargetDir $target -SourceURL $source -Sha256 $Sha256 -setx PATH "$Env:Path;c:\ninja-build" -$Env:Path="$Env:Path;c:\ninja-build" -Write-Host -ForegroundColor Green Done with ninja-build diff --git a/windows/install_python.ps1 b/windows/install_python.ps1 deleted file mode 100644 index 4a1b743c8..000000000 --- a/windows/install_python.ps1 +++ /dev/null @@ -1,39 +0,0 @@ -param ( - [Parameter(Mandatory=$true)][string]$Version, - [Parameter(Mandatory=$true)][string]$Sha256 -) - -# Enabled TLS12 -$ErrorActionPreference = 'Stop' - -# Script directory is $PSScriptRoot - -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - - -# https://www.python.org/ftp/python/3.9.1/python-3.9.1-amd64.exe -$pyexe = "https://www.python.org/ftp/python/$($Version)/python-$($Version)-amd64.exe" - -Write-Host -ForegroundColor Green starting with Python -$out = "$($PSScriptRoot)\python.exe" -(New-Object System.Net.WebClient).DownloadFile($pyexe, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$Sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$Sha256'.\"; exit 1 } - -Write-Host -ForegroundColor Green Done downloading wix, installing - -Start-Process $out -ArgumentList '/quiet InstallAllUsers=1' -Wait - -setx PATH "$($Env:PATH);c:\program files\Python38;c:\Program files\python38\scripts" -$Env:PATH="$($Env:PATH);c:\program files\Python38;c:\Program files\python38\scripts" -Remove-Item $out - -$getpipurl = "https://raw.githubusercontent.com/pypa/get-pip/38e54e5de07c66e875c11a1ebbdb938854625dd8/public/get-pip.py" -$getpipsha256 = "e235c437e5c7d7524fbce3880ca39b917a73dc565e0c813465b7a7a329bb279a" -$target = "get-pip.py" -curl $getpipurl -o $target -if ((Get-FileHash -Algorithm SHA256 $target).Hash -ne "$getpipsha256") { Write-Host \"Wrong hashsum for ${target}: got '$((Get-FileHash -Algorithm SHA256 $target).Hash)', expected '$getpipsha256'.\"; exit 1 } - -python get-pip.py pip==${Env:DD_PIP_VERSION_PY3} -python -m pip install -r ../requirements.txt - -Write-Host -ForegroundColor Green Done with Python diff --git a/windows/install_winget.ps1 b/windows/install_winget.ps1 deleted file mode 100644 index 09303f326..000000000 --- a/windows/install_winget.ps1 +++ /dev/null @@ -1,27 +0,0 @@ -param ( - [Parameter(Mandatory=$true)][string]$Version, - [Parameter(Mandatory=$true)][string]$Sha256 -) - -# Enabled TLS12 -$ErrorActionPreference = 'Stop' - -# Script directory is $PSScriptRoot - -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - - -$wingetexe="https://github.com/microsoft/winget-create/releases/download/v$($Version)/wingetcreate.exe" - -Write-Host -ForegroundColor Green "Downloading winget" -$out = "$($PSScriptRoot)\wingetcreate.exe" -(New-Object System.Net.WebClient).DownloadFile($wingetexe, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$Sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$Sha256'.\"; exit 1 } - -# just put it in it's own directory -mkdir \winget -Copy-Item $out \winget\wingetcreate.exe -Remove-Item $out -setx PATH "$Env:Path;c:\winget" -$Env:Path="$Env:Path;c:\winget" -Write-Host -ForegroundColor Green Done with Winget diff --git a/windows/install_wix.ps1 b/windows/install_wix.ps1 deleted file mode 100644 index cd671b81d..000000000 --- a/windows/install_wix.ps1 +++ /dev/null @@ -1,37 +0,0 @@ -param ( - [Parameter(Mandatory=$true)][string]$Version, - [Parameter(Mandatory=$true)][string]$Sha256 -) - -# Enabled TLS12 -$ErrorActionPreference = 'Stop' - -# Script directory is $PSScriptRoot - -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - -$shortenedver = $Version.Replace('.','') -$splitver = $Version.split(".") -$majmin = "$($splitver[0])$($splitver[1])" - -# https://github.com/wixtoolset/wix3/releases/download/wix3112rtm/wix311.exe -$wixzip = "https://github.com/wixtoolset/wix3/releases/download/wix$($shortenedver)rtm/wix$($majmin).exe" - -Write-Host -ForegroundColor Green starting with WiX -$out = "$($PSScriptRoot)\wix.exe" -(New-Object System.Net.WebClient).DownloadFile($wixzip, $out) -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$Sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$Sha256'.\"; exit 1 } - -Write-Host -ForegroundColor Green Done downloading wix, installing -#Start-Process wix.exe -ArgumentList '/quiet' -Wait -Start-Process $out -ArgumentList '/q' -Wait - -#Copy-Item c:\devtools\wix\sdk\inc -Destination c:\devtools\wix\sdk\vs2017\inc -Recurse -[Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin\", [System.EnvironmentVariableTarget]::Machine) -#setx PATH "$Env:Path;C:\devtools\wix" -#$Env:Path="$Env:Path;c:\devtools\wix" -#setx WIX "C:\Program Files (x86)\WiX Toolset v3.11\" -[Environment]::SetEnvironmentVariable("WIX", "C:\Program Files (x86)\WiX Toolset v3.11\") -Remove-Item $out - -Write-Host -ForegroundColor Green Done with WiX \ No newline at end of file diff --git a/windows/master.ps1 b/windows/master.ps1 new file mode 100644 index 000000000..c1988ec2e --- /dev/null +++ b/windows/master.ps1 @@ -0,0 +1,61 @@ +param( + [Parameter(Mandatory = $false)][switch] $TargetContainer +) + +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$ErrorActionPreference = 'Stop' + +# this includes the master versions variables +. .\versions.ps1 + +# this includes the helper functions +. .\helpers.ps1 + +# set the environment variables from the versions file + +foreach ($h in $SoftwareTable.GetEnumerator()){ + $key = $($h.Key) + $val = $($h.Value) + [Environment]::SetEnvironmentVariable($key, $val, [System.EnvironmentVariableTarget]::Process) +} + +.\Install_cert.ps1 + +## only do this if building the container +if($TargetContainer){ + Write-Host "Container Flag Set $TargetContainer" + .\install_net35.ps1 +} else { + Write-Host "Container flag not set $TargetContainer" +} + +.\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256 +.\install_mingit.ps1 -Version $ENV:GIT_VERSION -Sha256 $ENV:GIT_SHA256 + +### HACK: we disable symbolic links when cloning repositories +### to work around a symlink-related failure in the agent-binaries omnibus project +### when copying the datadog-agent project twice. +& git config --system core.symlinks false +.\install_vstudio.ps1 -Version $ENV:VS2017BUILDTOOLS_VERSION -Sha256 $ENV:VS2017BUILDTOOLS_SHA256 $ENV:VS2017BUILDTOOLS_DOWNLOAD_URL +.\install_wdk.ps1 +.\install_wix.ps1 -Version $ENV:WIX_VERSION -Sha256 $ENV:WIX_SHA256 +.\install_dotnetcore.ps1 +.\install_nuget.ps1 -Version $ENV:NUGET_VERSION -Sha256 $ENV:NUGET_SHA256 +.\install_vcpython.ps1 +.\install_ibm_mq.ps1 -Version $ENV:IBM_MQ_VERSION -Sha256 $ENV:IBM_MQ_SHA256 +.\install_cmake.ps1 -Version $ENV:CMAKE_VERSION -Sha256 $ENV:CMAKE_SHA256 +.\install_winget.ps1 -Version $ENV:WINGET_VERSION -Sha256 $ENV:WINGET_SHA256 +.\install_go.ps1 +$Env:PATH -split ";" +.\install_python.ps1 -Version $ENV:PYTHON_VERSION -Sha256 $ENV:PYTHON_SHA256 +.\install_ruby.ps1 -Version $ENV:RUBY_VERSION -Sha256 $ENV:RUBY_SHA256 +.\install_msys.ps1 -Version $ENV:MSYS_VERSION -Sha256 $ENV:MSYS_SHA256 +.\install_docker.ps1 +.\install_gcloud_sdk.ps1 +.\install_embedded_pythons.ps1 +.\install_vcpkg.ps1 +.\install_codeql.ps1 +.\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 +## # Add signtool to path +Add-ToPath -NewPath "${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\" -Global +& .\set_cpython_compiler.cmd \ No newline at end of file diff --git a/windows/test.ps1 b/windows/test.ps1 new file mode 100644 index 000000000..9af2d1110 --- /dev/null +++ b/windows/test.ps1 @@ -0,0 +1,8 @@ +param( + [Parameter(Mandatory = $false)][switch] $Container +) +if($Container){ + Write-Host "Container Flag Set" +} else { + Write-Host "Container flag not set" +} diff --git a/windows/versions.ps1 b/windows/versions.ps1 new file mode 100644 index 000000000..99b51a195 --- /dev/null +++ b/windows/versions.ps1 @@ -0,0 +1,38 @@ +$SoftwareTable = @{ + "GIT_VERSION"="2.26.2"; + "GIT_SHA256"="2dfbb1c46547c70179442a92b8593d592292b8bce2fd02ac4e0051a8072dde8f"; + "SEVENZIP_VERSION"="19.0.0"; + "SEVENZIP_SHA256"="0f5d4dbbe5e55b7aa31b91e5925ed901fdf46a367491d81381846f05ad54c45e"; + "VS2017BUILDTOOLS_VERSION"="16.8.3.0"; + "VS2017BUILDTOOLS_SHA256"="ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0"; + "VS2017BUILDTOOLS_DOWNLOAD_URL"="https://download.visualstudio.microsoft.com/download/pr/9b3476ff-6d0a-4ff8-956d-270147f21cd4/ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0/vs_BuildTools.exe"; + "GO_VERSION"="1.19.5"; + "GO_SHA25"="6167db91a2e40aeb453d3e59d213ecab06f62e1c4a84d13a06ccda1d999961caa"; + "RUBY_VERSION"="2.6.6-1"; + "RUBY_SHA256"="fbdf77a3e1fa36e25cf0af1303ac76f67dec7a6f739a829784a299702cad1492"; + "IBM_MQ_VERSION"="9.2.4.0"; + "IBM_MQ_SHA256"="9cea7e1693d051437e78468fd5e915b7b7ed2baf36cdae4936bcf2b760f55daa"; + "PYTHON_VERSION"="3.8.2"; + "PYTHON_SHA256"="8e400e3f32cdcb746e62e0db4d3ae4cba1f927141ebc4d0d5a4006b0daee8921"; + "WIX_VERSION"="3.11.2"; + "WIX_SHA256"="32bb76c478fcb356671d4aaf006ad81ca93eea32c22a9401b168fc7471feccd2"; + "CMAKE_VERSION"="3.23.0"; + "CMAKE_SHA256"="1e772025844f1cc648d28f42090038e5ca5cf72e2889de26d8d05ee25da17061"; + "MSYS_VERSION"="20210725"; + "MSYS_SHA256"="4013a9d5e51b448343efc24fc6a324cc999bb96b4c01b13a6bd3c661bb5c8a82"; + "NUGET_VERSION"="5.8.0"; + "NUGET_SHA256"="5c5b9c96165d3283b2cb9e5b65825d343e0e7139b9e70a250b4bb24c2285f3ba"; + "WINGET_VERSION"="1.0.4.0"; + "WINGET_SHA256"="8a4b727a8ad588c86603136a5d48093b59ed7491f7c5dc9f13725da0900fe6ab"; + "EMBEDDED_PYTHON_2_VERSION"="2.7.17"; + "EMBEDDED_PYTHON_2_SHA256"="557ea6690c5927360656c003d3114b73adbd755b712a2911975dde813d6d7afb"; + "EMBEDDED_PYTHON_3_VERSION"="3.8.1"; + "EMBEDDED_PYTHON_3_SHA256"="58563ca60891025923572107e02b8f07439928eb5222dd10466cc92089072c2a"; + "EMBEDDED_PIP_VERSION"="20.3.4"; + "CODEQL_VERSION"="2.10.3"; + "CODEQL_HASH"="46f64e21c74f41210ea3f2c433d1dc622e3eb0690b42373a73fba82122b929a1"; + "NINJA_VERSION"="1.11.0"; + "NINJA_SHA256"="d0ee3da143211aa447e750085876c9b9d7bcdd637ab5b2c5b41349c617f22f3b"; + "CACERTS_VERSION"="2023-01-10"; + "CACERTS_HASH"="fb1ecd641d0a02c01bc9036d513cb658bbda62a75e246bedbc01764560a639f0"; +} From 88705e0e774ad221f4c77488af4f73e49eb89f01 Mon Sep 17 00:00:00 2001 From: Derek Brown Date: Thu, 1 Sep 2022 23:49:52 -0700 Subject: [PATCH 02/16] fix path location (vcpkg) --- windows/helpers/install_vcpkg.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/windows/helpers/install_vcpkg.ps1 b/windows/helpers/install_vcpkg.ps1 index 4e90a8f9a..010ffbe6b 100644 --- a/windows/helpers/install_vcpkg.ps1 +++ b/windows/helpers/install_vcpkg.ps1 @@ -16,5 +16,5 @@ Remove-Item -Recurse -Force C:\vcpkg-tool Remove-Item -Recurse -Force C:\vcpkg-build Add-ToPath -NewPath "C:\vcpkg" -Global -.\vcpkg\vcpkg.exe --version -.\vcpkg\vcpkg.exe integrate install +c:\vcpkg\vcpkg.exe --version +c:\vcpkg\vcpkg.exe integrate install From 33bf25a222c9615e2d7be737ebf3fd34d158bbe1 Mon Sep 17 00:00:00 2001 From: Derek Brown Date: Fri, 2 Sep 2022 00:17:19 -0700 Subject: [PATCH 03/16] temp stop before vcpkg for debugging --- windows/master.ps1 | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/windows/master.ps1 b/windows/master.ps1 index c1988ec2e..54d8252c9 100644 --- a/windows/master.ps1 +++ b/windows/master.ps1 @@ -46,16 +46,15 @@ if($TargetContainer){ .\install_cmake.ps1 -Version $ENV:CMAKE_VERSION -Sha256 $ENV:CMAKE_SHA256 .\install_winget.ps1 -Version $ENV:WINGET_VERSION -Sha256 $ENV:WINGET_SHA256 .\install_go.ps1 -$Env:PATH -split ";" .\install_python.ps1 -Version $ENV:PYTHON_VERSION -Sha256 $ENV:PYTHON_SHA256 .\install_ruby.ps1 -Version $ENV:RUBY_VERSION -Sha256 $ENV:RUBY_SHA256 .\install_msys.ps1 -Version $ENV:MSYS_VERSION -Sha256 $ENV:MSYS_SHA256 .\install_docker.ps1 .\install_gcloud_sdk.ps1 .\install_embedded_pythons.ps1 -.\install_vcpkg.ps1 -.\install_codeql.ps1 -.\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 +#.\install_vcpkg.ps1 +#.\install_codeql.ps1 +#.\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 ## # Add signtool to path -Add-ToPath -NewPath "${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\" -Global -& .\set_cpython_compiler.cmd \ No newline at end of file +#Add-ToPath -NewPath "${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\" -Global +#& .\set_cpython_compiler.cmd \ No newline at end of file From 49cf0da837d59d3b436f4c42265160006179b6b9 Mon Sep 17 00:00:00 2001 From: Derek Brown Date: Fri, 2 Sep 2022 01:01:32 -0700 Subject: [PATCH 04/16] fix path error in networking script --- windows/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows/Dockerfile b/windows/Dockerfile index 330a5eb65..579c4c188 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -68,7 +68,7 @@ COPY ./windows/set_cpython_compiler.cmd set_cpython_compiler.cmd RUN .\master.ps1 -TargetContainer COPY ./windows/entrypoint.bat /entrypoint.bat -COPY ./windows/aws_networking.ps1 /aws_networking.ps1 +COPY ./windows/scripts/aws_networking.ps1 /aws_networking.ps1 ENTRYPOINT ["/entrypoint.bat"] From f16a973f2e66dd10ce16c9b484633921e794990a Mon Sep 17 00:00:00 2001 From: Derek Brown Date: Fri, 2 Sep 2022 10:23:50 -0700 Subject: [PATCH 05/16] fix vcpkg --- windows/Dockerfile | 2 +- windows/helpers/install_msys.ps1 | 2 ++ windows/helpers/install_vcpkg.ps1 | 4 ++-- windows/master.ps1 | 12 +++++++----- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/windows/Dockerfile b/windows/Dockerfile index 579c4c188..e885d2a79 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -68,7 +68,7 @@ COPY ./windows/set_cpython_compiler.cmd set_cpython_compiler.cmd RUN .\master.ps1 -TargetContainer COPY ./windows/entrypoint.bat /entrypoint.bat -COPY ./windows/scripts/aws_networking.ps1 /aws_networking.ps1 +COPY ./windows/helpers/aws_networking.ps1 /aws_networking.ps1 ENTRYPOINT ["/entrypoint.bat"] diff --git a/windows/helpers/install_msys.ps1 b/windows/helpers/install_msys.ps1 index c5c06620c..f31b2cde6 100644 --- a/windows/helpers/install_msys.ps1 +++ b/windows/helpers/install_msys.ps1 @@ -51,4 +51,6 @@ Get-RemoteFile -RemoteFile "https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64- Get-RemoteFile -RemoteFile "https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst" -LocalFile "C:/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst" & C:\tools\msys64\msys2_shell.cmd -defterm -no-start -c "pacman --noconfirm -U /c/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst" +Remove-Item c:\*.zst + Write-Host -ForegroundColor Green Done with MSYS diff --git a/windows/helpers/install_vcpkg.ps1 b/windows/helpers/install_vcpkg.ps1 index 010ffbe6b..33cbeea6c 100644 --- a/windows/helpers/install_vcpkg.ps1 +++ b/windows/helpers/install_vcpkg.ps1 @@ -1,7 +1,6 @@ $ErrorActionPreference = "Stop" - # Do not use '--depth 1' since vcpkg needs to browse its git history for dependency retrieval -git clone --branch 2022.03.10 https://github.com/microsoft/vcpkg +git clone --branch 2022.03.10 https://github.com/microsoft/vcpkg c:\vcpkg git clone https://github.com/microsoft/vcpkg-tool --branch 2022-03-30 C:\vcpkg-tool @@ -9,6 +8,7 @@ mkdir C:\vcpkg-build Push-Location C:\vcpkg-build cmake -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_BASE_VERSION=2022-03-30 C:\vcpkg-tool cmd /C "%VSTUDIO_ROOT%\VC\Auxiliary\Build\vcvars64.bat && msbuild /p:Configuration=Release vcpkg.sln" + Move-Item C:\vcpkg-build\Release\vcpkg.exe c:\vcpkg\ Pop-Location diff --git a/windows/master.ps1 b/windows/master.ps1 index 54d8252c9..3428f34be 100644 --- a/windows/master.ps1 +++ b/windows/master.ps1 @@ -52,9 +52,11 @@ if($TargetContainer){ .\install_docker.ps1 .\install_gcloud_sdk.ps1 .\install_embedded_pythons.ps1 -#.\install_vcpkg.ps1 -#.\install_codeql.ps1 -#.\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 +.\install_vcpkg.ps1 +.\install_codeql.ps1 +.\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 ## # Add signtool to path -#Add-ToPath -NewPath "${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\" -Global -#& .\set_cpython_compiler.cmd \ No newline at end of file +Add-ToPath -NewPath "${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\" -Global +& .\set_cpython_compiler.cmd +Remove-Item -Recurse -Force c:\tmp\* +Remove-Item -Recurse -Force $Env:TEMP\* From 7240b48e55e5015968d54b34ff4ba50f521a6f9a Mon Sep 17 00:00:00 2001 From: db Date: Fri, 2 Sep 2022 22:33:08 +0000 Subject: [PATCH 06/16] fix go entry in path --- windows/helpers/install_go.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows/helpers/install_go.ps1 b/windows/helpers/install_go.ps1 index f154a9032..ddb94b7a6 100644 --- a/windows/helpers/install_go.ps1 +++ b/windows/helpers/install_go.ps1 @@ -21,7 +21,7 @@ Remove-Item $out Add-EnvironmentVariable -Variable GOROOT -VALUE "c:\go" -Local -Global Add-EnvironmentVariable -Variable GOPATH -VALUE "c:\dev\go" -Global -Add-ToPath -NewPath "c:\go" -Local -Global +Add-ToPath -NewPath "c:\go\bin" -Local -Global Write-Host -ForegroundColor Green "Installed go $ENV:GO_VERSION" From f31cf934a47b195b36424309982edc6ea3567c42 Mon Sep 17 00:00:00 2001 From: Derek Brown Date: Sat, 3 Sep 2022 17:12:19 -0700 Subject: [PATCH 07/16] make work on local system --- windows/Dockerfile | 2 +- windows/helpers.ps1 | 47 ++++++++++++ windows/helpers/install_7zip.ps1 | 16 ++++- windows/helpers/install_cmake.ps1 | 9 +++ windows/helpers/install_codeql.ps1 | 12 +++- windows/helpers/install_docker.ps1 | 24 +++++-- windows/helpers/install_dotnetcore.ps1 | 12 +++- windows/helpers/install_embedded_pythons.ps1 | 7 ++ windows/helpers/install_gcloud_sdk.ps1 | 20 ++++-- windows/helpers/install_go.ps1 | 30 ++++++-- windows/helpers/install_ibm_mq.ps1 | 13 +++- windows/helpers/install_mingit.ps1 | 75 ++++++++++++++------ windows/helpers/install_msys.ps1 | 13 +++- windows/helpers/install_net35.ps1 | 49 +++++++++---- windows/helpers/install_ninja.ps1 | 10 ++- windows/helpers/install_nuget.ps1 | 9 +++ windows/helpers/install_python.ps1 | 12 +++- windows/helpers/install_ruby.ps1 | 11 ++- windows/helpers/install_vcpkg.ps1 | 14 +++- windows/helpers/install_vcpython.ps1 | 7 ++ windows/helpers/install_vstudio.ps1 | 48 +++++++++---- windows/helpers/install_wdk.ps1 | 14 +++- windows/helpers/install_winget.ps1 | 10 +++ windows/helpers/install_wix.ps1 | 11 ++- windows/master.ps1 | 53 +++++++------- windows/test.ps1 | 16 +++-- windows/versions.ps1 | 6 ++ 27 files changed, 437 insertions(+), 113 deletions(-) diff --git a/windows/Dockerfile b/windows/Dockerfile index e885d2a79..851dcf04b 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -59,7 +59,7 @@ WORKDIR c:/scripts COPY ./windows/versions.ps1 versions.ps1 COPY ./windows/master.ps1 master.ps1 COPY ./windows/helpers.ps1 helpers.ps1 -COPY ./windows/helpers/*.ps1 c:/scripts/ +COPY ./windows/helpers/*.ps1 c:/scripts/helpers/ COPY ./requirements.txt ./requirements-py2.txt / COPY ./python-packages-versions.txt /python-packages-versions.txt diff --git a/windows/helpers.ps1 b/windows/helpers.ps1 index 0bdf0b827..24c7b8162 100644 --- a/windows/helpers.ps1 +++ b/windows/helpers.ps1 @@ -5,6 +5,53 @@ param( [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 $ErrorActionPreference = 'Stop' +$RegRootPath = "HKLM:\SOFTWARE\DatadogDeveloper" + +function Get-VersionKey() { + param( + [Parameter(Mandatory = $true)][string] $Component, + [Parameter(Mandatory = $true)][string] $Keyname + ) + $keypath = $RegRootPath + "\$($Component)" + $value = Get-ItemPropertyValue -path $keypath -name $Keyname -ErrorAction SilentlyContinue + if($? -eq $false){ + # key not present + return $null + } + return $value +} +# returns two booleans; first is whether currently installed, second +# is whether targetvalue == current value +function Get-InstallUpgradeStatus() { + param( + [Parameter(Mandatory = $true)][string] $Component, + [Parameter(Mandatory = $true)][string] $Keyname, + [Parameter(Mandatory = $true)][string] $TargetValue + ) + $v = Get-VersionKey -Component $Component -Keyname $Keyname + if($null -eq $v){ + Write-Host -ForegroundColor DarkMagenta "Component $Component $Keyname not found" + return $false, $false + } + if($v -eq $TargetValue){ + return $true, $true + } + Write-Host -ForegroundColor DarkMagenta "Component found, but $v not equal $TargetValue" + return $true, $false +} + +function Set-InstalledVersionKey() { + param( + [Parameter(Mandatory = $true)][string] $Component, + [Parameter(Mandatory = $true)][string] $Keyname, + [Parameter(Mandatory = $true)][string] $TargetValue + ) + $keypath = $RegRootPath + "\$($Component)" + if(!(test-path $keypath)){ + New-Item $keypath -Force + } + New-ItemProperty -Path $keypath -Name $Keyname -Value $TargetValue -PropertyType String -Force +} # Define get-remotefile so that it can be used throughout function Get-RemoteFile() { param( diff --git a/windows/helpers/install_7zip.ps1 b/windows/helpers/install_7zip.ps1 index 76b20e9be..0c6185984 100644 --- a/windows/helpers/install_7zip.ps1 +++ b/windows/helpers/install_7zip.ps1 @@ -6,10 +6,18 @@ param ( # Enabled TLS12 $ErrorActionPreference = 'Stop' -# Script directory is $PSScriptRoot +## check to see if this version is installed +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "sevenzip" -Keyname "version" -TargetValue $Version +if($isInstalled -and $isCurrent){ + Write-Host "SevenZip already installed and current. Skipping" + return +} +# assuming that exe installer would properly handle upgrade if we ever needed to +# so not installed is the same as not current +# Script directory is $PSScriptRoot $shortenedver = $Version.Replace('.','') $sevenzip="https://www.7-zip.org/a/7z$($shortenedver)-x64.exe" @@ -17,7 +25,11 @@ Write-Host -ForegroundColor Green "Installing 7zip $sevenzip" $out = "$($PSScriptRoot)\7zip.exe" Get-RemoteFile -RemoteFile $sevenzip -LocalFile $out -VerifyHash $Sha256 -Start-Process 7zip.exe -ArgumentList '/S' -Wait +Start-Process $out -ArgumentList '/S' -Wait Remove-Item $out Add-ToPath -NewPath "c:\program files\7-zip" -Local -Global + +## Write the version key out to the registry +Set-InstalledVersionKey -Component "sevenzip" -Keyname "Version" -TargetValue $Version + Write-Host -ForegroundColor Green Done with 7zip diff --git a/windows/helpers/install_cmake.ps1 b/windows/helpers/install_cmake.ps1 index 921fa9d61..fa7eed42c 100644 --- a/windows/helpers/install_cmake.ps1 +++ b/windows/helpers/install_cmake.ps1 @@ -9,6 +9,14 @@ $shortenedver = $Version.Replace('.','') $splitver = $Version.split(".") $majmin = "$($splitver[0])$($splitver[1])" +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "CMake" -Keyname "version" -TargetValue $Version +if($isInstalled -and $isCurrent) { + Write-Host -ForegroundColor Green "CMake up to date" + return +} +## presumably cmake msi can handle an upgrade, so there's no further difference +## between install and upgrade + # https://github.com/Kitware/CMake/releases/download/v3.23.0/cmake-3.23.0-windows-x86_64.msi $cmakeurl = "https://github.com/Kitware/CMake/releases/download/v$($Version)/cmake-$($Version)-windows-x86_64.msi" @@ -21,4 +29,5 @@ Start-Process msiexec -ArgumentList "/q /i $($out) ADD_CMAKE_TO_PATH=System" -Wa Remove-Item $out Reload-Path +Set-InstalledVersionKey -Component "CMake" -Keyname "version" -TargetValue $Version Write-Host -ForegroundColor Green Done with CMake \ No newline at end of file diff --git a/windows/helpers/install_codeql.ps1 b/windows/helpers/install_codeql.ps1 index 5c0eb6a67..7efe9b4b1 100644 --- a/windows/helpers/install_codeql.ps1 +++ b/windows/helpers/install_codeql.ps1 @@ -1,10 +1,18 @@ $ErrorActionPreference = 'Stop' $ProgressPreference = 'SilentlyContinue' +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "codeql" -Keyname "version" -TargetValue $ENV:CODEQL_VERSION +if($isInstalled -and $isCurrent) { + Write-Host -ForegroundColor Green "Codeql up to date" + return +} +if($isInstalled -and -not $isCurrent){ + Remove-Item -Recurse -Force c:\CodeQL -ErrorAction SilentlyContinue +} Write-Host -ForegroundColor Green "Installing CodeQL $ENV:CODEQL_VERSION" # https://github.com/github/codeql-cli-binaries/releases/download/v2.10.3/codeql-win64.zip $codeqlzip = "https://github.com/github/codeql-cli-binaries/releases/download/v$ENV:CODEQL_VERSION/codeql-win64.zip" -$out = 'codeql.zip' +$out = "$($PSScriptRoot)\codeql.zip" Write-Host -ForegroundColor Green "Downloading $codeqlzip to $out" @@ -19,5 +27,5 @@ Write-Host -ForegroundColor Green "Removing temporary file $out" Remove-Item $out Add-ToPath -NewPath "c:\CodeQL" -Global - +Set-InstalledVersionKey -Component "codeql" -Keyname "version" -TargetValue $ENV:CODEQL_VERSION Write-Host -ForegroundColor Green "Installed CodeQL $ENV:CODEQL_VERSION" diff --git a/windows/helpers/install_docker.ps1 b/windows/helpers/install_docker.ps1 index bb9357c04..d52d23d0e 100644 --- a/windows/helpers/install_docker.ps1 +++ b/windows/helpers/install_docker.ps1 @@ -1,26 +1,38 @@ $ErrorActionPreference = 'Stop' $ProgressPreference = 'SilentlyContinue' -mkdir C:\Docker +if(!(test-path c:\docker)){ + mkdir C:\Docker +} # Docker CLI builds maintained by a Docker engineer $dockerVersion = "19.03.3" $out = "C:\Docker\docker.exe" $sha256 = "2d6ff967c717a38dd41f5ad418396bdeb84642fe04985b30925e38f593d386da" - -Get-RemoteFile -RemoteFile "https://github.com/StefanScherer/docker-cli-builder/releases/download/$dockerVersion/docker.exe" -LocalFile $out -VerifyHash $sha256 +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "docker-cli" -Keyname "version" -TargetValue $dockerVersion +if(-not $isInstalled -or -not $isCurrent) { + Get-RemoteFile -RemoteFile "https://github.com/StefanScherer/docker-cli-builder/releases/download/$dockerVersion/docker.exe" -LocalFile $out -VerifyHash $sha256 + Set-InstalledVersionKey -Component "docker-cli" -Keyname "version" -TargetValue $dockerVersion +} # Install manifest-tool $manifestVersion = "v1.0.1" $out = "C:\Docker\manifest-tool.exe" $sha256 = "41c08bc1052534f07282eae1f2998e542734b53e79e8d84e4f989ac1c27b2861" -Get-RemoteFile -RemoteFile "https://github.com/estesp/manifest-tool/releases/download/$manifestVersion/manifest-tool-windows-amd64.exe" -LocalFile $out -VerifyHash $sha256 +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "docker-manifest" -Keyname "version" -TargetValue $manifestVersion +if(-not $isInstalled -or -not $isCurrent) { + Get-RemoteFile -RemoteFile "https://github.com/estesp/manifest-tool/releases/download/$manifestVersion/manifest-tool-windows-amd64.exe" -LocalFile $out -VerifyHash $sha256 + Set-InstalledVersionKey -Component "docker-manifest" -Keyname "version" -TargetValue $manifestVersion +} # Install notary $notaryVersion = "v0.6.1" $out = "C:\Docker\notary.exe" $sha256 = "9d736f9b569b6a6a3de30cbfa3c60a764acdd445cf4ced760efa9d370bcad64f" -Get-RemoteFile -RemoteFile "https://github.com/theupdateframework/notary/releases/download/$notaryVersion/notary-Windows-amd64.exe" -LocalFile $out -VerifyHash $sha256 - +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "docker-notary" -Keyname "version" -TargetValue $notaryVersion +if(-not $isInstalled -or -not $isCurrent) { + Get-RemoteFile -RemoteFile "https://github.com/theupdateframework/notary/releases/download/$notaryVersion/notary-Windows-amd64.exe" -LocalFile $out -VerifyHash $sha256 + Set-InstalledVersionKey -Component "docker-notary" -Keyname "version" -TargetValue $notaryVersion +} # Add Docker to path Add-ToPath -NewPath "c:\Docker" -Local -Global diff --git a/windows/helpers/install_dotnetcore.ps1 b/windows/helpers/install_dotnetcore.ps1 index 150ae0e42..ac371f689 100644 --- a/windows/helpers/install_dotnetcore.ps1 +++ b/windows/helpers/install_dotnetcore.ps1 @@ -1,6 +1,16 @@ . .\helpers.ps1 $Version = "6.0.100" $url = "https://download.visualstudio.microsoft.com/download/pr/0f71eaf1-ce85-480b-8e11-c3e2725b763a/9044bfd1c453e2215b6f9a0c224d20fe/dotnet-sdk-6.0.100-win-x64.exe" + +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "dotnetcore" -Keyname "DownloadFile" -TargetValue $url +if($isInstalled -and $isCurrent){ + Write-Host -ForegroundColor Green ".NET Core Up to date" + return +} +if(-not $isCurrent){ + Write-Host -ForegroundColor Yellow "Attempting to update .NET Core" + ## presumably executable knows how to handle upgrade +} Write-Host -ForegroundColor Green "Installing dotnetcore from $($Url)" $out = "$($PSScriptRoot)\dotnetcoresdk.exe" @@ -21,5 +31,5 @@ Remove-Item $out Reload-Path # Trigger first run experience by running arbitrary cmd dotnet help - +Set-InstalledVersionKey -Component "dotnetcore" -Keyname "DownloadFile" -TargetValue $url Write-Host -ForegroundColor Green Done with DotNet Core SDK $Version diff --git a/windows/helpers/install_embedded_pythons.ps1 b/windows/helpers/install_embedded_pythons.ps1 index 4b6a77319..d94249ff9 100644 --- a/windows/helpers/install_embedded_pythons.ps1 +++ b/windows/helpers/install_embedded_pythons.ps1 @@ -15,6 +15,13 @@ $ErrorActionPreference = 'Stop' +if($Env:DD_DEV_TARGET -ne "Container") { + # I think this is actually necessary on server OSes. Come back to this + # on server OSes. + Write-Host -ForegroundColor Green "Skipping embedded pythons on local install" + return +} + $py2 = "https://s3.amazonaws.com/dd-agent-omnibus/python-windows-${Env:EMBEDDED_PYTHON_2_VERSION}-amd64.zip" $py3 = "https://s3.amazonaws.com/dd-agent-omnibus/python-windows-${Env:EMBEDDED_PYTHON_3_VERSION}-amd64.zip" diff --git a/windows/helpers/install_gcloud_sdk.ps1 b/windows/helpers/install_gcloud_sdk.ps1 index 68ccb9c34..b2e4c88bf 100644 --- a/windows/helpers/install_gcloud_sdk.ps1 +++ b/windows/helpers/install_gcloud_sdk.ps1 @@ -3,11 +3,19 @@ $ProgressPreference = 'SilentlyContinue' [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 Write-Host -ForegroundColor Green Installing Google Cloud SDK -$version = "315.0.0" +$version = $ENV:GCLOUD_SDK_VERSION $gsdk = "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-$version-windows-x86_64.zip" -$out = 'gsdk.zip' -$sha256 = "c9b283c9db4ed472111ccf32e6689fd467daf18ce3a77b8e601f9c646a83d86b" - +$out = "$($PSScriptRoot)\gsdk.zip" +$sha256 = $ENV:GCLOUD_SDK_SHA256 + +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "GCloudSDK" -Keyname "version" -TargetValue $version +if($isInstalled) { + if($isCurrent){ + return + } else { + Remove-item -Recurse -Force c:\google-cloud-sdk -ErrorAction SilentlyContinue + } +} Write-Host -ForegroundColor Green Downloading $gsdk to $out Get-RemoteFile -RemoteFile $gsdk -LocalFile $out -VerifyHash $sha256 @@ -17,11 +25,11 @@ Get-ChildItem $out # write file size to make sure it worked Write-Host -ForegroundColor Green "File size is $((get-item $out).length)" -Expand-Archive gsdk.zip -DestinationPath C:\ +Expand-Archive $out -DestinationPath C:\ -ProgressPreference Silen Remove-Item $out # add to path #$pwd = pwd Add-ToPath -NewPath "c:\google-cloud-sdk\bin" -Local -Global - +Set-InstalledVersionKey -Component "GCloudSDK" -Keyname "version" -TargetValue $version Write-Host -ForegroundColor Green Done Installing Google Cloud SDK diff --git a/windows/helpers/install_go.ps1 b/windows/helpers/install_go.ps1 index ddb94b7a6..d9b7543b6 100644 --- a/windows/helpers/install_go.ps1 +++ b/windows/helpers/install_go.ps1 @@ -5,23 +5,43 @@ Write-Host -ForegroundColor Green "Installing go $ENV:GO_VERSION" $gozip = "https://dl.google.com/go/go$ENV:GO_VERSION.windows-amd64.zip" -$out = 'go.zip' - +$out = "$($PSScriptRoot)\go.zip" + +## +## because we want to allow multiple versions of GO, we need to handle +## the version check a bit differently. Get all the installed versions +## and see if this one is present +## +$installedVers = get-childitem -Path "hklm:\Software\DatadogDeveloper\go" -ErrorAction SilentlyContinue | foreach-object { $_.name | split-path -leaf } +if($installedVers -and $installedVers.Contains($ENV:GO_VERSION)) { + Write-Host -ForegroundColor Green "Go version $ENV:GO_VERSION already installed" + return +} Write-Host -ForegroundColor Green "Downloading $gozip to $out" Get-RemoteFile -RemoteFile $gozip -LocalFile $out -VerifyHash $ENV:GO_SHA256 +## set up proper output directory +$godir = "c:\go\$ENV:GO_VERSION" + Write-Host -ForegroundColor Green "Extracting $out to c:\" -Start-Process "7z" -ArgumentList 'x -oc:\ go.zip' -Wait +if(!(test-path c:\go)){ + mkdir c:\go +} +Start-Process "7z" -ArgumentList "x -o$($godir) $out" -Wait Write-Host -ForegroundColor Green "Removing temporary file $out" Remove-Item $out -Add-EnvironmentVariable -Variable GOROOT -VALUE "c:\go" -Local -Global +Add-EnvironmentVariable -Variable GOROOT -VALUE "$($GODIR)\go" -Local -Global Add-EnvironmentVariable -Variable GOPATH -VALUE "c:\dev\go" -Global -Add-ToPath -NewPath "c:\go\bin" -Local -Global +Add-ToPath -NewPath "$($GODIR)\go\bin" -Local -Global +if(!(test-path "$RegRootPath\Go\$($ENV:GO_VERSION)")){ + New-Item "$RegRootPath\Go\$($ENV:GO_VERSION)" -Force +} +New-ItemProperty -Path "$RegRootPath\Go\$($ENV:GO_VERSION)" -Name "goroot" -Value "$($godir)" -PropertyType String Write-Host -ForegroundColor Green "Installed go $ENV:GO_VERSION" diff --git a/windows/helpers/install_ibm_mq.ps1 b/windows/helpers/install_ibm_mq.ps1 index d27b7fe80..0e642eb24 100644 --- a/windows/helpers/install_ibm_mq.ps1 +++ b/windows/helpers/install_ibm_mq.ps1 @@ -6,5 +6,16 @@ param ( $source = "https://s3.amazonaws.com/dd-agent-omnibus/ibm-mq-backup/$($Version)-IBM-MQC-Redist-Win64.zip" $target = "c:\ibm_mq" +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "ibmmq" -Keyname "version" -TargetValue $Version +if($isInstalled -and $isCurrent){ + Write-Host -ForegroundColor Green "IBM-MQ already installed" + return +} +if($isInstalled){ + # just delete the existing installation so the new one will replace it + Remove-Item -Recurse -Force $target -ErrorAction SilentlyContinue +} DownloadAndExpandTo -TargetDir $target -SourceURL $source -Sha256 $Sha256 -Add-EnvironmentVariable -Variable MQ_FILE_PATH -VALUE "c:\ibm_mq" -Global \ No newline at end of file +Add-EnvironmentVariable -Variable MQ_FILE_PATH -VALUE "c:\ibm_mq" -Global + +Set-InstalledVersionKey -Component "ibmmq" -Keyname "version" -TargetValue $Version \ No newline at end of file diff --git a/windows/helpers/install_mingit.ps1 b/windows/helpers/install_mingit.ps1 index 6c09d0222..2eca3fa86 100644 --- a/windows/helpers/install_mingit.ps1 +++ b/windows/helpers/install_mingit.ps1 @@ -3,24 +3,57 @@ param ( [Parameter(Mandatory=$true)][string]$Sha256 ) -# Enabled TLS12 - - -# Script directory is $PSScriptRoot - -$mingit = "https://github.com/git-for-windows/git/releases/download/v$($Version).windows.1/MinGit-$($Version)-64-bit.zip" - -Write-Host -ForegroundColor Green Installing MinGit -$out = "$($PSScriptRoot)\mingit.zip" -Get-RemoteFile -RemoteFile $mingit -LocalFile $out -VerifyHash $Sha256 - -md c:\devtools\git -& '7z' x -oc:\devtools\git $out - -Remove-Item $out -# set path locally so we can initialize git config -Add-ToPath -NewPath "c:\devtools\git\cmd;c:\devtools\git\usr\bin" -Local -Global -& 'git.exe' config --global user.name "Croissant Builder" -& 'git.exe' config --global user.email "croissant@datadoghq.com" - -Write-Host -ForegroundColor Green Done with Git +## +## mingit is used in the container version; We'll install the full WinGit +## in developer mode + +function InstallMinGit() { + # Script directory is $PSScriptRoot + + $mingit = "https://github.com/git-for-windows/git/releases/download/v$($Version).windows.1/MinGit-$($Version)-64-bit.zip" + + Write-Host -ForegroundColor Green Installing MinGit + $out = "$($PSScriptRoot)\mingit.zip" + Get-RemoteFile -RemoteFile $mingit -LocalFile $out -VerifyHash $Sha256 + + md c:\devtools\git + & '7z' x -oc:\devtools\git $out + + Remove-Item $out + # set path locally so we can initialize git config + Add-ToPath -NewPath "c:\devtools\git\cmd;c:\devtools\git\usr\bin" -Local -Global + & 'git.exe' config --global user.name "Croissant Builder" + & 'git.exe' config --global user.email "croissant@datadoghq.com" + + Write-Host -ForegroundColor Green Done with Git +} + +function InstallWinGit() { + + $wingit = $Env:WINGIT_URL + $Sha256 = $Env:WINGIT_SHA256 + $isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "git" -Keyname "version" -TargetValue $wingit + if($isInstalled -and $isCurrent){ + Write-Host -ForegroundColor Green "WinGit already installed" + return + } + Write-Host -ForegroundColor Green "Installing WinGit" + $out = "$($PSScriptRoot)\wingit.exe" + Get-RemoteFile -RemoteFile $wingit -LocalFile $out -VerifyHash $Sha256 + + Start-Process $out -ArgumentList "/VERYSILENT" -Wait -NoNewWindow + Remove-Item $out + # set path locally so we can initialize git config + Reload-Path + Set-InstalledVersionKey -Component "git" -Keyname "Version" -TargetValue $wingit + Write-Host -ForegroundColor Green Done with Git +} + +if($Env:DD_DEV_TARGET -eq "Container") { + InstallMinGit + return +} else { + InstallWinGit + return +} +return diff --git a/windows/helpers/install_msys.ps1 b/windows/helpers/install_msys.ps1 index f31b2cde6..9512f2cc5 100644 --- a/windows/helpers/install_msys.ps1 +++ b/windows/helpers/install_msys.ps1 @@ -24,7 +24,15 @@ function Invoke-Msys2Shell($Arguments) { Write-Host "Invoking msys2 shell command:" $params.ArgumentList Start-Process @params } - +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "msys" -Keyname "version" -TargetValue $Version +if($isInstalled -and $isCurrent) { + Write-Host -ForegroundColor Green "MSys up to date" + return +} +if($isInstalled -and -not $isCurrent) { + Write-Host -ForegroundColor Yellow "upgrading MSYS" + Remove-Item -Recurse -Force "$($InstallPath)\msys64" +} # https://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20200629.tar.xz $msyszip = "https://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-$($Version).tar.xz" @@ -52,5 +60,6 @@ Get-RemoteFile -RemoteFile "https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64- & C:\tools\msys64\msys2_shell.cmd -defterm -no-start -c "pacman --noconfirm -U /c/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst" Remove-Item c:\*.zst - +ridk enable +Set-InstalledVersionKey -Component "msys" -Keyname "version" -TargetValue $Version Write-Host -ForegroundColor Green Done with MSYS diff --git a/windows/helpers/install_net35.ps1 b/windows/helpers/install_net35.ps1 index fffb1d15d..4a8388c66 100644 --- a/windows/helpers/install_net35.ps1 +++ b/windows/helpers/install_net35.ps1 @@ -4,6 +4,13 @@ $ErrorActionPreference = "Stop" $UpgradeTable = @{ + 1607 = @{ + netfxzip = "https://dotnetbinaries.blob.core.windows.net/dockerassets/microsoft-windows-netfx3-ltsc2016.zip"; + netfxsha256 = "303866ec4f396fda465d5c8c563d44b4aa884c60dbe6b20d3ee755b604c4b8cb"; + patch="http://download.windowsupdate.com/d/msdownload/update/software/secu/2020/02/windows10.0-kb4537764-x64_93e41ada5a984e3749ecd87bc5515bdc48cefb4d.msu"; + patchsha256="b94ef8fa977c6e8255d8b50a19ac6512de725da5eae2f3ba96db6ace1a64e244"; + expandedpatch="windows10.0-kb4537764-x64.cab" + }; 1809 = @{ ## Taken from the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile: ## https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile @@ -62,6 +69,18 @@ $UpgradeTable = @{ expandedpatch = "windows10.0-kb5005538-x64-ndp48.cab" } } +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "netfx35" -Keyname "version" -TargetValue "1" +if($isInstalled){ + Write-Host -ForegroundColor Green "NetFX 3.51 already installed" + return +} +if($Env:DD_DEV_TARGET -ne "Container") { + $osInfo = Get-CimInstance -classname win32_operatingsystem + if($osinfo.ProductType -eq "1"){ + Write-Host -ForegroundColor Green "Skipping .NET35 on local workstation install" + return + } +} $kernelver = [int](get-itemproperty -path "hklm:software\microsoft\windows nt\currentversion" -name releaseid).releaseid $build = [System.Environment]::OSVersion.version.build $productname = (get-itemproperty -path "hklm:software\microsoft\windows nt\currentversion" -n productname).productname @@ -73,29 +92,33 @@ if ($build -ge 20348) { $Env:DOTNET_RUNNING_IN_CONTAINER="true" -$out = "microsoft-windows-netfx3.zip" +$out = "$($PSScriptRoot)\microsoft-windows-netfx3.zip" $sha256 = $UpgradeTable[$kernelver]["netfxsha256"] Write-Host curl -fSLo $out $UpgradeTable[$kernelver]["netfxzip"] -curl.exe -fSLo $out $UpgradeTable[$kernelver]["netfxzip"] -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$sha256'.\"; exit 1 } +Get-RemoteFile -RemoteFile $UpgradeTable[$kernelver]["netfxzip"] -LocalFile $out -VerifyHash $sha256 -tar -zxf $out +expand-archive -Path $out -DestinationPath . remove-item -force $out -DISM /Online /Quiet /Add-Package /PackagePath:.\microsoft-windows-netfx3-ondemand-package~31bf3856ad364e35~amd64~~.cab -remove-item microsoft-windows-netfx3-ondemand-package~31bf3856ad364e35~amd64~~.cab +$cabfile = "microsoft-windows-netfx3-ondemand-package~31bf3856ad364e35~amd64~~.cab" +if($kernelver -eq 1607){ + $cabfile = "microsoft-windows-netfx3-ondemand-package.cab" +} +DISM /Online /Quiet /Add-Package /PackagePath:$($cabfile) +remove-item $cabfile Remove-Item -Force -Recurse ${Env:TEMP}\* -ErrorAction SilentlyContinue -$out = "patch.msu" +$out = "$($PSScriptRoot)\patch.msu" $sha256 = $UpgradeTable[$kernelver]["patchsha256"] Write-Host curl.exe -fSLo $out $UpgradeTable[$kernelver]["patch"] -curl.exe -fSLo $out $UpgradeTable[$kernelver]["patch"] -if ((Get-FileHash -Algorithm SHA256 $out).Hash -ne "$sha256") { Write-Host \"Wrong hashsum for ${out}: got '$((Get-FileHash -Algorithm SHA256 $out).Hash)', expected '$sha256'.\"; exit 1 } +Get-RemoteFile -RemoteFile $UpgradeTable[$kernelver]["patch"] -LocalFile $out -VerifyHash $sha256 mkdir patch -expand patch.msu patch -F:* -remove-item -force patch.msu -Write-Host DISM /Online /Quiet /Add-Package /PackagePath:$($PSScriptRoot)\$($UpgradeTable[$kernelver]["expandedpatch"]) -DISM /Online /Quiet /Add-Package /PackagePath:$($PSScriptRoot)\$($UpgradeTable[$kernelver]["expandedpatch"]) +expand "$($PSScriptRoot)\patch.msu" patch -F:* +remove-item -force "$($PSScriptRoot)\patch.msu" +Write-Host DISM /Online /Quiet /Add-Package /PackagePath:$($UpgradeTable[$kernelver]["expandedpatch"]) +DISM /Online /Quiet /Add-Package /PackagePath:$($UpgradeTable[$kernelver]["expandedpatch"]) remove-item -force -recurse patch + +Set-InstalledVersionKey -Component "netfx35" -Keyname "version" -TargetValue "1" \ No newline at end of file diff --git a/windows/helpers/install_ninja.ps1 b/windows/helpers/install_ninja.ps1 index d9704b932..20285a8a0 100644 --- a/windows/helpers/install_ninja.ps1 +++ b/windows/helpers/install_ninja.ps1 @@ -5,10 +5,18 @@ param ( $ErrorActionPreference = 'Stop' - +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "ninja" -Keyname "version" -TargetValue $Version +if($isInstalled -and $isCurrent) { + Write-Host -ForegroundColor Green "Ninja up to date" + return +} +if($isInstalled -and -not $isCurrent){ + Remove-Item -Recurse -Force c:\ninja-build -ErrorAction SilentlyContinue +} $source="https://github.com/ninja-build/ninja/releases/download/v${Version}/ninja-win.zip" $target = "c:\ninja-build" DownloadAndExpandTo -TargetDir $target -SourceURL $source -Sha256 $Sha256 Add-ToPath -NewPath "c:\ninja-build" -Global +Set-InstalledVersionKey -Component "ninja" -Keyname "version" -TargetValue $Version Write-Host -ForegroundColor Green Done with ninja-build diff --git a/windows/helpers/install_nuget.ps1 b/windows/helpers/install_nuget.ps1 index 9af85a54e..d339d1fff 100644 --- a/windows/helpers/install_nuget.ps1 +++ b/windows/helpers/install_nuget.ps1 @@ -7,6 +7,14 @@ param ( $nugetexe="https://dist.nuget.org/win-x86-commandline/v$($Version)/nuget.exe" +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "NuGet" -Keyname "version" -TargetValue $Version +if($isInstalled -and $isCurrent) { + Write-Host -ForegroundColor Green "NuGet up to date" + return +} +if(-not $isInstalled) { + Remove-Item -Force "c:\nuget\nuget.exe" -ErrorAction SilentlyContinue +} Write-Host -ForegroundColor Green "Downloading nuget" $out = "$($PSScriptRoot)\nuget.exe" @@ -19,3 +27,4 @@ Remove-Item $out Add-ToPath -NewPath "c:\nuget" -Local -Global Write-Host -ForegroundColor Green Done with Nuget +Set-InstalledVersionKey -Component "NuGet" -Keyname "version" -TargetValue $Version \ No newline at end of file diff --git a/windows/helpers/install_python.ps1 b/windows/helpers/install_python.ps1 index 8d80ad678..ef32aca7f 100644 --- a/windows/helpers/install_python.ps1 +++ b/windows/helpers/install_python.ps1 @@ -7,6 +7,13 @@ param ( # https://www.python.org/ftp/python/3.9.1/python-3.9.1-amd64.exe $pyexe = "https://www.python.org/ftp/python/$($Version)/python-$($Version)-amd64.exe" +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "Python" -Keyname "version" -TargetValue $Version +if($isInstalled -and $isCurrent) { + Write-Host -ForegroundColor Green "Python up to date" + return +} +## presumably installer exe knows how to handle upgrades + Write-Host -ForegroundColor Green starting with Python $out = "$($PSScriptRoot)\python.exe" @@ -21,11 +28,12 @@ Remove-Item $out $getpipurl = "https://raw.githubusercontent.com/pypa/get-pip/38e54e5de07c66e875c11a1ebbdb938854625dd8/public/get-pip.py" $getpipsha256 = "e235c437e5c7d7524fbce3880ca39b917a73dc565e0c813465b7a7a329bb279a" -$target = "get-pip.py" +$target = "$($PSScriptRoot)\get-pip.py" Get-RemoteFile -RemoteFile $getpipurl -LocalFile $target -VerifyHash $getpipsha256 -python get-pip.py pip==${Env:DD_PIP_VERSION_PY3} +python "$($PSScriptRoot)\get-pip.py" pip==${Env:DD_PIP_VERSION_PY3} python -m pip install -r /requirements.txt +Set-InstalledVersionKey -Component "Python" -Keyname "version" -TargetValue $Version Write-Host -ForegroundColor Green Done with Python diff --git a/windows/helpers/install_ruby.ps1 b/windows/helpers/install_ruby.ps1 index 028c6b249..3ef59da46 100644 --- a/windows/helpers/install_ruby.ps1 +++ b/windows/helpers/install_ruby.ps1 @@ -6,6 +6,15 @@ param ( # https://github.com/oneclick/rubyinstaller2/releases/download/RubyInstaller-2.7.4-1/rubyinstaller-2.7.4-1-x64.exe +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "ruby" -Keyname "version" -TargetValue $Version +if($isInstalled -and $isCurrent) { + Write-Host -ForegroundColor Green "Ruby up to date" + return +} +if($isInstalled -and -not $isCurrent) { + Write-Host -ForegroundColor Yellow "Ruby out of date, but not upgrading (yet)" + return +} $rubyexe = "https://github.com/oneclick/rubyinstaller2/releases/download/RubyInstaller-$($Version)/rubyinstaller-$($Version)-x64.exe" Write-Host -ForegroundColor Green starting with Ruby @@ -23,5 +32,5 @@ Start-Process gem -ArgumentList 'install bundler -v 2.3.24' -Wait Remove-Item $out - +Set-InstalledVersionKey -Component "ruby" -Keyname "version" -TargetValue $Version Write-Host -ForegroundColor Green Done with Ruby diff --git a/windows/helpers/install_vcpkg.ps1 b/windows/helpers/install_vcpkg.ps1 index 33cbeea6c..b6142aeb8 100644 --- a/windows/helpers/install_vcpkg.ps1 +++ b/windows/helpers/install_vcpkg.ps1 @@ -1,6 +1,17 @@ $ErrorActionPreference = "Stop" +$branch = "2022.03.10" + +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "vcpkg" -Keyname "version" -TargetValue $branch +if($isInstalled -and $isCurrent){ + Write-Host -ForegroundColor Green "VCPkg up to date" + return +} +if(-not $isCurrent) { + Remove-Item -recurse -force c:\vcpkg -ErrorAction SilentlyContinue + Write-Host -ForegroundColor Green "Upgrading VCPkg" +} # Do not use '--depth 1' since vcpkg needs to browse its git history for dependency retrieval -git clone --branch 2022.03.10 https://github.com/microsoft/vcpkg c:\vcpkg +git clone --branch $branch https://github.com/microsoft/vcpkg c:\vcpkg git clone https://github.com/microsoft/vcpkg-tool --branch 2022-03-30 C:\vcpkg-tool @@ -18,3 +29,4 @@ Remove-Item -Recurse -Force C:\vcpkg-build Add-ToPath -NewPath "C:\vcpkg" -Global c:\vcpkg\vcpkg.exe --version c:\vcpkg\vcpkg.exe integrate install +Set-InstalledVersionKey -Component "vcpkg" -Keyname "version" -TargetValue $branch \ No newline at end of file diff --git a/windows/helpers/install_vcpython.ps1 b/windows/helpers/install_vcpython.ps1 index a65c52365..71160e2c8 100644 --- a/windows/helpers/install_vcpython.ps1 +++ b/windows/helpers/install_vcpython.ps1 @@ -6,9 +6,16 @@ Write-host -ForegroundColor Green Downloading VC++ for Python $out = "$($PSScriptRoot)\vcp.msi" $sha256 = "070474db76a2e625513a5835df4595df9324d820f9cc97eab2a596dcbc2f5cbf" +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "vcpython" -Keyname "DownloadFile" -TargetValue $vcpmsi +if($isInstalled){ + Write-Host -ForegroundColor Green "VC++ for Python 2.7 already installed" + return +} Get-RemoteFile -RemoteFile $vcpmsi -LocalFile $out -VerifyHash $sha256 Write-host -ForegroundColor Green VC++ for Python downloaded, installing... Start-Process msiexec -ArgumentList '/q /i vcp.msi' -Wait Remove-Item $out + +Set-InstalledVersionKey -Component "vcpython" -Keyname "DownloadFile" -TargetValue $vcpmsi Write-Host -ForegroundColor Green Done with Visual C++ for Python diff --git a/windows/helpers/install_vstudio.ps1 b/windows/helpers/install_vstudio.ps1 index b68c8bceb..659ba1245 100644 --- a/windows/helpers/install_vstudio.ps1 +++ b/windows/helpers/install_vstudio.ps1 @@ -1,22 +1,9 @@ param ( - [Parameter(Mandatory=$true)][string]$Version, - [Parameter(Mandatory=$true)][string]$Sha256, - [Parameter(Mandatory=$true)][string]$Url, [Parameter(Mandatory=$false)][string]$InstallRoot="c:\devtools\vstudio", [Parameter(Mandatory=$false)][switch]$NoQuiet ) -Write-Host -ForegroundColor Green "Installing Visual Studio $($Version) from $($Url)" - -$out = "$($PSScriptRoot)\vs_buildtools.exe" - -Write-Host -ForegroundColor Green Downloading $Url to $out -Get-RemoteFile -RemoteFile $Url -LocalFile $out -VerifyHash $Sha256 - -# write file size to make sure it worked -Write-Host -ForegroundColor Green "File size is $((get-item $out).length)" - $VSPackages = @( "Microsoft.VisualStudio.Workload.ManagedDesktop", "Microsoft.VisualStudio.Workload.NetCoreTools", @@ -40,6 +27,40 @@ $VSPackages = @( # "Microsoft.VisualStudio.Component.Windows10SDK.19041" ) +$VSPackagesDesktop = @( + "Microsoft.VisualStudio.Workload.CoreEditor", + "Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core", + "Microsoft.VisualStudio.Component.IntelliCode" +) + +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "vstudio" -Keyname "DownloadFile" -TargetValue $Env:VS2019INSTALLER_DOWNLOAD_URL + +if($isInstalled){ + if(-not $isCurrent){ + Write-Host -ForegroundColor Yellow "Not attempting to upgrade Visual Studio" + } else { + Write-Host -ForegroundColor Green "Visual Studio already installed" + } + return +} + +if($Env:DD_DEV_TARGET -eq "Container") { + $Sha256 = $Env:VS2017BUILDTOOLS_SHA256 + $Url = $Env:VS2017BUILDTOOLS_DOWNLOAD_URL +} else { + $Sha256 = $Env:VS2019INSTALLER_SHA256 + $Url = $Env:VS2019INSTALLER_DOWNLOAD_URL + $VSPackages += $VSPackagesDesktop +} +Write-Host -ForegroundColor Green "Installing Visual Studio from $($Url)" + +$out = "$($PSScriptRoot)\vs_buildtools.exe" + +Write-Host -ForegroundColor Green Downloading $Url to $out +Get-RemoteFile -RemoteFile $Url -LocalFile $out -VerifyHash $Sha256 + +# write file size to make sure it worked +Write-Host -ForegroundColor Green "File size is $((get-item $out).length)" $VSPackageListParam = $VSPackages -join " --add " $ArgList = "--wait --norestart --nocache --installPath `"$($InstallRoot)`" --add $VSPackageListParam" if(-not $NoQuiet){ @@ -58,4 +79,5 @@ Add-EnvironmentVariable -Variable VSTUDIO_ROOT -Value $InstallRoot -Global -Loca Add-ToPath -NewPath "${env:ProgramFiles(x86)}\Windows Kits\10\bin\10.0.18362.0\x64" -Global Remove-Item $out +Set-InstalledVersionKey -Component vstudio -KeyName "DownloadFile" -TargetValue $Url Write-Host -ForegroundColor Green Done with Visual Studio diff --git a/windows/helpers/install_wdk.ps1 b/windows/helpers/install_wdk.ps1 index a0e277711..e7431ccfd 100644 --- a/windows/helpers/install_wdk.ps1 +++ b/windows/helpers/install_wdk.ps1 @@ -6,8 +6,15 @@ $ProgressPreference = 'SilentlyContinue' ## $wdk ='https://go.microsoft.com/fwlink/?linkid=2026156' $wdk = 'https://go.microsoft.com/fwlink/?linkid=2085767' ## 1903 WDK link +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "wdk" -Keyname "DownloadFile" -TargetValue $wdk +if($isInstalled) { + if(-not $isCurrent){ + Write-Host -ForegroundColor Yellow "Not attempting to upgrade WDK" + } + return +} Write-Host -ForegroundColor Green Installing WDK -$out = 'wdksetup.exe' +$out = "$($PSScriptRoot)\wdksetup.exe" $sha256 = "c35057cb294096c63bbea093e5024a5fb4120103b20c13fa755c92f227b644e5" Write-Host -ForegroundColor Green Downloading $wdk to $out @@ -21,7 +28,9 @@ Write-Host -ForegroundColor Green "File size is $((get-item $out).length)" Start-Process $out -ArgumentList '/q' -Wait #install WDK.vsix (hack) -mkdir \tmp +if(!(test-path c:\tmp)){ + mkdir c:\tmp +} # copy the vsix file out of the installed directory copy-item -Path "C:\Program Files (x86)\Windows Kits\10\Vsix\VS2019\WDK.vsix" -Destination c:\tmp @@ -31,4 +40,5 @@ Copy-Item 'c:\tmp\$MSBuild\Microsoft\*' -Destination "C:\devtools\vstudio\MSBuil remove-item -Force -Path $out remove-item -Force -Path c:\tmp\wdk.vsix #.`clean_tmps.ps1 +Set-InstalledVersionKey -Component "wdk" -KeyName "DownloadFile" -TargetValue $wdk Write-Host -ForegroundColor Green Done with WDK \ No newline at end of file diff --git a/windows/helpers/install_winget.ps1 b/windows/helpers/install_winget.ps1 index 5ba1ab3d9..add531fa7 100644 --- a/windows/helpers/install_winget.ps1 +++ b/windows/helpers/install_winget.ps1 @@ -6,6 +6,15 @@ param ( $wingetexe="https://github.com/microsoft/winget-create/releases/download/v$($Version)/wingetcreate.exe" +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "winget" -Keyname "version" -TargetValue $Version +if($isInstalled -and $isCurrent) { + Write-Host -ForegroundColor Green "winget up to date" + return +} + +if($installed -and -not $isCurrent){ + Remove-Item -Force \winget\wingetcreate.exe -ErrorAction SilentlyContinue +} Write-Host -ForegroundColor Green "Downloading winget" $out = "$($PSScriptRoot)\wingetcreate.exe" @@ -16,4 +25,5 @@ mkdir \winget Copy-Item $out \winget\wingetcreate.exe Remove-Item $out Add-ToPath -NewPath "c:\winget" -Local -Global +Set-InstalledVersionKey -Component "winget" -Keyname "version" -TargetValue $Version Write-Host -ForegroundColor Green Done with Winget diff --git a/windows/helpers/install_wix.ps1 b/windows/helpers/install_wix.ps1 index 7ef4322ab..be3b3df57 100644 --- a/windows/helpers/install_wix.ps1 +++ b/windows/helpers/install_wix.ps1 @@ -11,6 +11,15 @@ $majmin = "$($splitver[0])$($splitver[1])" # https://github.com/wixtoolset/wix3/releases/download/wix3112rtm/wix311.exe $wixzip = "https://github.com/wixtoolset/wix3/releases/download/wix$($shortenedver)rtm/wix$($majmin).exe" +$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "wix" -Keyname "DownloadFile" -TargetValue $wixzip +if($isInstalled) { + if(-not $isCurrent){ + Write-Host -ForegroundColor Yellow "Not attempting to upgrade WiX" + } + return +} + + Write-Host -ForegroundColor Green starting with WiX $out = "$($PSScriptRoot)\wix.exe" @@ -24,5 +33,5 @@ Add-ToPath -NewPath "${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin\" -Global Add-EnvironmentVariable -Variable "WIX" -Value "C:\Program Files (x86)\WiX Toolset v3.11\" -Global Remove-Item $out - +Set-InstalledVersionKey -Component "wix" -Keyname "DownloadFile" -TargetValue $wixzip Write-Host -ForegroundColor Green Done with WiX \ No newline at end of file diff --git a/windows/master.ps1 b/windows/master.ps1 index 3428f34be..30c57e2e3 100644 --- a/windows/master.ps1 +++ b/windows/master.ps1 @@ -19,44 +19,47 @@ foreach ($h in $SoftwareTable.GetEnumerator()){ [Environment]::SetEnvironmentVariable($key, $val, [System.EnvironmentVariableTarget]::Process) } -.\Install_cert.ps1 +.\helpers\install_cert.ps1 ## only do this if building the container if($TargetContainer){ + $Env:DD_DEV_TARGET="Container" Write-Host "Container Flag Set $TargetContainer" - .\install_net35.ps1 } else { Write-Host "Container flag not set $TargetContainer" } -.\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256 -.\install_mingit.ps1 -Version $ENV:GIT_VERSION -Sha256 $ENV:GIT_SHA256 +.\helpers\install_net35.ps1 +.\helpers\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256 +.\helpers\install_mingit.ps1 -Version $ENV:GIT_VERSION -Sha256 $ENV:GIT_SHA256 ### HACK: we disable symbolic links when cloning repositories ### to work around a symlink-related failure in the agent-binaries omnibus project ### when copying the datadog-agent project twice. & git config --system core.symlinks false -.\install_vstudio.ps1 -Version $ENV:VS2017BUILDTOOLS_VERSION -Sha256 $ENV:VS2017BUILDTOOLS_SHA256 $ENV:VS2017BUILDTOOLS_DOWNLOAD_URL -.\install_wdk.ps1 -.\install_wix.ps1 -Version $ENV:WIX_VERSION -Sha256 $ENV:WIX_SHA256 -.\install_dotnetcore.ps1 -.\install_nuget.ps1 -Version $ENV:NUGET_VERSION -Sha256 $ENV:NUGET_SHA256 -.\install_vcpython.ps1 -.\install_ibm_mq.ps1 -Version $ENV:IBM_MQ_VERSION -Sha256 $ENV:IBM_MQ_SHA256 -.\install_cmake.ps1 -Version $ENV:CMAKE_VERSION -Sha256 $ENV:CMAKE_SHA256 -.\install_winget.ps1 -Version $ENV:WINGET_VERSION -Sha256 $ENV:WINGET_SHA256 -.\install_go.ps1 -.\install_python.ps1 -Version $ENV:PYTHON_VERSION -Sha256 $ENV:PYTHON_SHA256 -.\install_ruby.ps1 -Version $ENV:RUBY_VERSION -Sha256 $ENV:RUBY_SHA256 -.\install_msys.ps1 -Version $ENV:MSYS_VERSION -Sha256 $ENV:MSYS_SHA256 -.\install_docker.ps1 -.\install_gcloud_sdk.ps1 -.\install_embedded_pythons.ps1 -.\install_vcpkg.ps1 -.\install_codeql.ps1 -.\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 +.\helpers\install_vstudio.ps1 #-Version $ENV:VS2017BUILDTOOLS_VERSION -Sha256 $ENV:VS2017BUILDTOOLS_SHA256 $ENV:VS2017BUILDTOOLS_DOWNLOAD_URL +.\helpers\install_wdk.ps1 +.\helpers\install_wix.ps1 -Version $ENV:WIX_VERSION -Sha256 $ENV:WIX_SHA256 +.\helpers\install_dotnetcore.ps1 +.\helpers\install_nuget.ps1 -Version $ENV:NUGET_VERSION -Sha256 $ENV:NUGET_SHA256 +.\helpers\install_vcpython.ps1 +.\helpers\install_ibm_mq.ps1 -Version $ENV:IBM_MQ_VERSION -Sha256 $ENV:IBM_MQ_SHA256 +.\helpers\install_cmake.ps1 -Version $ENV:CMAKE_VERSION -Sha256 $ENV:CMAKE_SHA256 +.\helpers\install_winget.ps1 -Version $ENV:WINGET_VERSION -Sha256 $ENV:WINGET_SHA256 +.\helpers\install_go.ps1 +.\helpers\install_python.ps1 -Version $ENV:PYTHON_VERSION -Sha256 $ENV:PYTHON_SHA256 +.\helpers\install_ruby.ps1 -Version $ENV:RUBY_VERSION -Sha256 $ENV:RUBY_SHA256 +.\helpers\install_msys.ps1 -Version $ENV:MSYS_VERSION -Sha256 $ENV:MSYS_SHA256 +.\helpers\install_docker.ps1 +.\helpers\install_gcloud_sdk.ps1 +.\helpers\install_embedded_pythons.ps1 +.\helpers\install_vcpkg.ps1 +.\helpers\install_codeql.ps1 +.\helpers\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 ## # Add signtool to path Add-ToPath -NewPath "${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\" -Global & .\set_cpython_compiler.cmd -Remove-Item -Recurse -Force c:\tmp\* -Remove-Item -Recurse -Force $Env:TEMP\* +if($TargetContainer){ + Remove-Item -Recurse -Force c:\tmp\* + Remove-Item -Recurse -Force $Env:TEMP\* +} diff --git a/windows/test.ps1 b/windows/test.ps1 index 9af2d1110..e8b817a4c 100644 --- a/windows/test.ps1 +++ b/windows/test.ps1 @@ -1,8 +1,10 @@ -param( - [Parameter(Mandatory = $false)][switch] $Container -) -if($Container){ - Write-Host "Container Flag Set" -} else { - Write-Host "Container flag not set" +. .\helpers.ps1 +. .\versions.ps1 + +foreach ($h in $SoftwareTable.GetEnumerator()){ + $key = $($h.Key) + $val = $($h.Value) + [Environment]::SetEnvironmentVariable($key, $val, [System.EnvironmentVariableTarget]::Process) } + +.\helpers\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256 \ No newline at end of file diff --git a/windows/versions.ps1 b/windows/versions.ps1 index 99b51a195..cdb710d15 100644 --- a/windows/versions.ps1 +++ b/windows/versions.ps1 @@ -1,11 +1,15 @@ $SoftwareTable = @{ "GIT_VERSION"="2.26.2"; "GIT_SHA256"="2dfbb1c46547c70179442a92b8593d592292b8bce2fd02ac4e0051a8072dde8f"; + "WINGIT_URL"="https://github.com/git-for-windows/git/releases/download/v2.37.3.windows.1/Git-2.37.3-64-bit.exe"; + "WINGIT_SHA256"="b0442f1b8ea40b6f94ef9a611121d2c204f6aa7f29c54315d2ce59876c3d134e"; "SEVENZIP_VERSION"="19.0.0"; "SEVENZIP_SHA256"="0f5d4dbbe5e55b7aa31b91e5925ed901fdf46a367491d81381846f05ad54c45e"; "VS2017BUILDTOOLS_VERSION"="16.8.3.0"; "VS2017BUILDTOOLS_SHA256"="ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0"; "VS2017BUILDTOOLS_DOWNLOAD_URL"="https://download.visualstudio.microsoft.com/download/pr/9b3476ff-6d0a-4ff8-956d-270147f21cd4/ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0/vs_BuildTools.exe"; + "VS2019INSTALLER_DOWNLOAD_URL"="https://download.visualstudio.microsoft.com/download/pr/3a7354bc-d2e4-430f-92d0-9abd031b5ee5/d9fc228ea71a98adc7bc5f5d8e8800684c647e955601ed721fcb29f74ace7536/vs_Community.exe"; + "VS2019INSTALLER_SHA256"="d9fc228ea71a98adc7bc5f5d8e8800684c647e955601ed721fcb29f74ace7536"; "GO_VERSION"="1.19.5"; "GO_SHA25"="6167db91a2e40aeb453d3e59d213ecab06f62e1c4a84d13a06ccda1d999961caa"; "RUBY_VERSION"="2.6.6-1"; @@ -33,6 +37,8 @@ $SoftwareTable = @{ "CODEQL_HASH"="46f64e21c74f41210ea3f2c433d1dc622e3eb0690b42373a73fba82122b929a1"; "NINJA_VERSION"="1.11.0"; "NINJA_SHA256"="d0ee3da143211aa447e750085876c9b9d7bcdd637ab5b2c5b41349c617f22f3b"; + "GCLOUD_SDK_VERSION"="315.0.0"; + "GCLOUD_SDK_SHA256"="c9b283c9db4ed472111ccf32e6689fd467daf18ce3a77b8e601f9c646a83d86b"; "CACERTS_VERSION"="2023-01-10"; "CACERTS_HASH"="fb1ecd641d0a02c01bc9036d513cb658bbda62a75e246bedbc01764560a639f0"; } From 1402d59a23762b8ae02014f24a35fb67f7aef56a Mon Sep 17 00:00:00 2001 From: Derek Brown Date: Sat, 3 Sep 2022 17:39:46 -0700 Subject: [PATCH 08/16] fix typo in gsdk --- windows/helpers/install_gcloud_sdk.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows/helpers/install_gcloud_sdk.ps1 b/windows/helpers/install_gcloud_sdk.ps1 index b2e4c88bf..81f5c2614 100644 --- a/windows/helpers/install_gcloud_sdk.ps1 +++ b/windows/helpers/install_gcloud_sdk.ps1 @@ -25,7 +25,7 @@ Get-ChildItem $out # write file size to make sure it worked Write-Host -ForegroundColor Green "File size is $((get-item $out).length)" -Expand-Archive $out -DestinationPath C:\ -ProgressPreference Silen +Expand-Archive $out -DestinationPath C:\ Remove-Item $out # add to path From 6dc53bc3142beb6412d267fbd5977bb0e8a959de Mon Sep 17 00:00:00 2001 From: Derek Brown Date: Sat, 3 Sep 2022 23:34:23 -0700 Subject: [PATCH 09/16] debugging statement for vcpkg --- windows/helpers/install_vcpkg.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/windows/helpers/install_vcpkg.ps1 b/windows/helpers/install_vcpkg.ps1 index b6142aeb8..9f5eee3dd 100644 --- a/windows/helpers/install_vcpkg.ps1 +++ b/windows/helpers/install_vcpkg.ps1 @@ -18,6 +18,7 @@ git clone https://github.com/microsoft/vcpkg-tool --branch 2022-03-30 C:\vcpkg-t mkdir C:\vcpkg-build Push-Location C:\vcpkg-build cmake -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_BASE_VERSION=2022-03-30 C:\vcpkg-tool +Get-Command cmd -all cmd /C "%VSTUDIO_ROOT%\VC\Auxiliary\Build\vcvars64.bat && msbuild /p:Configuration=Release vcpkg.sln" Move-Item C:\vcpkg-build\Release\vcpkg.exe c:\vcpkg\ From 8bb691bfbdf4a1df6892dac0d2a2aaa7ede80875 Mon Sep 17 00:00:00 2001 From: Derek Brown Date: Sun, 4 Sep 2022 00:03:19 -0700 Subject: [PATCH 10/16] remove ridk enable; fixes calls to cmd --- windows/helpers/install_msys.ps1 | 1 - windows/helpers/install_vcpkg.ps1 | 1 - 2 files changed, 2 deletions(-) diff --git a/windows/helpers/install_msys.ps1 b/windows/helpers/install_msys.ps1 index 9512f2cc5..34a07cb27 100644 --- a/windows/helpers/install_msys.ps1 +++ b/windows/helpers/install_msys.ps1 @@ -60,6 +60,5 @@ Get-RemoteFile -RemoteFile "https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64- & C:\tools\msys64\msys2_shell.cmd -defterm -no-start -c "pacman --noconfirm -U /c/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst" Remove-Item c:\*.zst -ridk enable Set-InstalledVersionKey -Component "msys" -Keyname "version" -TargetValue $Version Write-Host -ForegroundColor Green Done with MSYS diff --git a/windows/helpers/install_vcpkg.ps1 b/windows/helpers/install_vcpkg.ps1 index 9f5eee3dd..b6142aeb8 100644 --- a/windows/helpers/install_vcpkg.ps1 +++ b/windows/helpers/install_vcpkg.ps1 @@ -18,7 +18,6 @@ git clone https://github.com/microsoft/vcpkg-tool --branch 2022-03-30 C:\vcpkg-t mkdir C:\vcpkg-build Push-Location C:\vcpkg-build cmake -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_BASE_VERSION=2022-03-30 C:\vcpkg-tool -Get-Command cmd -all cmd /C "%VSTUDIO_ROOT%\VC\Auxiliary\Build\vcvars64.bat && msbuild /p:Configuration=Release vcpkg.sln" Move-Item C:\vcpkg-build\Release\vcpkg.exe c:\vcpkg\ From 2dccd55ad7c07e6e0b56755b154fc1dc5a48007f Mon Sep 17 00:00:00 2001 From: Derek Brown Date: Mon, 5 Sep 2022 10:46:57 -0700 Subject: [PATCH 11/16] Working both locally and in container --- windows/Dockerfile | 4 +- windows/helpers.ps1 | 56 +++++++- windows/helpers/install_net35.ps1 | 2 +- windows/helpers/install_nuget.ps1 | 6 +- windows/helpers/install_winget.ps1 | 6 +- windows/install-all.ps1 | 91 +++++++++++++ windows/master.ps1 | 65 ---------- windows/modules/DDDeveloper/DDDeveloper.psd1 | 129 +++++++++++++++++++ windows/modules/DDDeveloper/go/go.psm1 | 43 +++++++ windows/modules/DDDeveloper/main.psm1 | 71 ++++++++++ windows/modules/DDDeveloper/prompt.ps1 | 40 ++++++ 11 files changed, 437 insertions(+), 76 deletions(-) create mode 100644 windows/install-all.ps1 delete mode 100644 windows/master.ps1 create mode 100644 windows/modules/DDDeveloper/DDDeveloper.psd1 create mode 100644 windows/modules/DDDeveloper/go/go.psm1 create mode 100644 windows/modules/DDDeveloper/main.psm1 create mode 100644 windows/modules/DDDeveloper/prompt.ps1 diff --git a/windows/Dockerfile b/windows/Dockerfile index 851dcf04b..49e137e7d 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -57,7 +57,7 @@ LABEL codeql_version=${CODEQL_VERSION} ## WORKDIR should automatically create the directory WORKDIR c:/scripts COPY ./windows/versions.ps1 versions.ps1 -COPY ./windows/master.ps1 master.ps1 +COPY ./windows/install-all.ps1 install-all.ps1 COPY ./windows/helpers.ps1 helpers.ps1 COPY ./windows/helpers/*.ps1 c:/scripts/helpers/ @@ -65,7 +65,7 @@ COPY ./requirements.txt ./requirements-py2.txt / COPY ./python-packages-versions.txt /python-packages-versions.txt COPY ./windows/set_cpython_compiler.cmd set_cpython_compiler.cmd -RUN .\master.ps1 -TargetContainer +RUN .\install-all.ps1 -TargetContainer COPY ./windows/entrypoint.bat /entrypoint.bat COPY ./windows/helpers/aws_networking.ps1 /aws_networking.ps1 diff --git a/windows/helpers.ps1 b/windows/helpers.ps1 index 24c7b8162..4d73eb3f1 100644 --- a/windows/helpers.ps1 +++ b/windows/helpers.ps1 @@ -83,7 +83,11 @@ function Add-EnvironmentVariable() { [Environment]::SetEnvironmentVariable($Variable, $Value, [System.EnvironmentVariableTarget]::Process) } if($Global){ - [Environment]::SetEnvironmentVariable($Variable, $Value, [System.EnvironmentVariableTarget]::Machine) + if($TargetContainer){ + [Environment]::SetEnvironmentVariable($Variable, $Value, [System.EnvironmentVariableTarget]::Machine) + } else { + $GlobalEnvVariables.EnvironmentVars[$($Variable)] = $Value + } } } @@ -97,9 +101,15 @@ function Add-ToPath() { $Env:Path="$Env:Path;$NewPath" } if($Global){ - $oldPath=[Environment]::GetEnvironmentVariable("Path", [System.EnvironmentVariableTarget]::Machine) - $target="$oldPath;$NewPath" - [Environment]::SetEnvironmentVariable("Path", $target, [System.EnvironmentVariableTarget]::Machine) + if($TargetContainer){ + $oldPath=[Environment]::GetEnvironmentVariable("Path", [System.EnvironmentVariableTarget]::Machine) + $target="$oldPath;$NewPath" + [Environment]::SetEnvironmentVariable("Path", $target, [System.EnvironmentVariableTarget]::Machine) + } else { + if ($GlobalEnvVariables.PathEntries -notcontains $NewPath){ + $GlobalEnvVariables.PathEntries += $NewPath + } + } } } @@ -137,4 +147,42 @@ function Reload-Path() { } } $Env:PATH=$newpath -join ";" +} + +function Get-VariableFile { + $targetdir = "$($Env:USERPROFILE)\.ddbuild" + if(!(test-path $targetdir)){ + $null = New-Item -path $targetdir -ItemType Directory + } + + $targetfile = "$targetdir\environment.json" + return $targetfile +} +function Read-Variables() { + $varfile = Get-VariableFile + Write-Host -ForegroundColor Magenta "varfile [$varfile]" + if(! (test-path $varfile -PathType Leaf)){ + Write-Host -ForegroundColor Yellow "$varfile does not exist" + return + } + $fromfile = Get-content $varfile | ConvertFrom-Json + + $GlobalEnvVariables.PathEntries = $fromfile.PathEntries + ## add to the local path in case we need things for adding/upgrading + foreach ($e in $GlobalEnvVariables.PathEntries) { + Add-ToPath -NewPath $e -Local + } + # need to walk the hash table manually. + $fromfile.EnvironmentVars.psobject.properties | foreach { + $GlobalEnvVariables.EnvironmentVars[$_.Name] = $_.Value + + ## set the variable for this shell so that anything that's required + ## for update/upgrade is in place + Add-EnvironmentVariable -Variable $_.Name -Value $_.Value -Local + } +} +function Write-Variables() { + $targetfile = Get-VariableFile + $GlobalEnvVariables | convertto-json | set-content -path $targetfile + } \ No newline at end of file diff --git a/windows/helpers/install_net35.ps1 b/windows/helpers/install_net35.ps1 index 4a8388c66..4a0c44e80 100644 --- a/windows/helpers/install_net35.ps1 +++ b/windows/helpers/install_net35.ps1 @@ -77,7 +77,7 @@ if($isInstalled){ if($Env:DD_DEV_TARGET -ne "Container") { $osInfo = Get-CimInstance -classname win32_operatingsystem if($osinfo.ProductType -eq "1"){ - Write-Host -ForegroundColor Green "Skipping .NET35 on local workstation install" + & dism /online /enable-feature /FeatureName:Netfx3 /all return } } diff --git a/windows/helpers/install_nuget.ps1 b/windows/helpers/install_nuget.ps1 index d339d1fff..18c787323 100644 --- a/windows/helpers/install_nuget.ps1 +++ b/windows/helpers/install_nuget.ps1 @@ -21,8 +21,10 @@ $out = "$($PSScriptRoot)\nuget.exe" Get-RemoteFile -RemoteFile $nugetexe -LocalFile $out -VerifyHash $Sha256 # just put it in it's own directory -mkdir \nuget -Copy-Item $out \nuget\nuget.exe +if(! (test-path "c:\nuget")){ + mkdir c:\nuget +} +Copy-Item $out c:\nuget\nuget.exe Remove-Item $out Add-ToPath -NewPath "c:\nuget" -Local -Global diff --git a/windows/helpers/install_winget.ps1 b/windows/helpers/install_winget.ps1 index add531fa7..1219be29b 100644 --- a/windows/helpers/install_winget.ps1 +++ b/windows/helpers/install_winget.ps1 @@ -21,8 +21,10 @@ $out = "$($PSScriptRoot)\wingetcreate.exe" Get-RemoteFile -RemoteFile $wingetexe -LocalFile $out -VerifyHash $Sha256 # just put it in it's own directory -mkdir \winget -Copy-Item $out \winget\wingetcreate.exe +if(! (test-path c:\winget)){ + mkdir c:\winget +} +Copy-Item $out c:\winget\wingetcreate.exe Remove-Item $out Add-ToPath -NewPath "c:\winget" -Local -Global Set-InstalledVersionKey -Component "winget" -Keyname "version" -TargetValue $Version diff --git a/windows/install-all.ps1 b/windows/install-all.ps1 new file mode 100644 index 000000000..5b9c44004 --- /dev/null +++ b/windows/install-all.ps1 @@ -0,0 +1,91 @@ +param( + [Parameter(Mandatory = $false)][switch] $TargetContainer +) + +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$ErrorActionPreference = 'Stop' + +# this includes the master versions variables +. .\versions.ps1 + +# this includes the helper functions +. .\helpers.ps1 + +# set the environment variables from the versions file + +# Global Variables for saving env variables and path additions on the local build +$GlobalEnvVariables = [PSCustomObject]@{ + EnvironmentVars = @{} + PathEntries = @() +} + + +foreach ($h in $SoftwareTable.GetEnumerator()){ + $key = $($h.Key) + $val = $($h.Value) + [Environment]::SetEnvironmentVariable($key, $val, [System.EnvironmentVariableTarget]::Process) +} + +.\helpers\install_cert.ps1 + +## only do this if building the container +if($TargetContainer){ + $Env:DD_DEV_TARGET="Container" + Write-Host "Container Flag Set $TargetContainer" +} else { + Write-Host "Container flag not set $TargetContainer" + Read-Variables +} + +try { + .\helpers\install_net35.ps1 + .\helpers\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256 + .\helpers\install_mingit.ps1 -Version $ENV:GIT_VERSION -Sha256 $ENV:GIT_SHA256 + + ### HACK: we disable symbolic links when cloning repositories + ### to work around a symlink-related failure in the agent-binaries omnibus project + ### when copying the datadog-agent project twice. + & git config --system core.symlinks false + .\helpers\install_vstudio.ps1 #-Version $ENV:VS2017BUILDTOOLS_VERSION -Sha256 $ENV:VS2017BUILDTOOLS_SHA256 $ENV:VS2017BUILDTOOLS_DOWNLOAD_URL + .\helpers\install_wdk.ps1 + .\helpers\install_wix.ps1 -Version $ENV:WIX_VERSION -Sha256 $ENV:WIX_SHA256 + .\helpers\install_dotnetcore.ps1 + .\helpers\install_nuget.ps1 -Version $ENV:NUGET_VERSION -Sha256 $ENV:NUGET_SHA256 + .\helpers\install_vcpython.ps1 + .\helpers\install_ibm_mq.ps1 -Version $ENV:IBM_MQ_VERSION -Sha256 $ENV:IBM_MQ_SHA256 + .\helpers\install_cmake.ps1 -Version $ENV:CMAKE_VERSION -Sha256 $ENV:CMAKE_SHA256 + .\helpers\install_winget.ps1 -Version $ENV:WINGET_VERSION -Sha256 $ENV:WINGET_SHA256 + .\helpers\install_go.ps1 + .\helpers\install_python.ps1 -Version $ENV:PYTHON_VERSION -Sha256 $ENV:PYTHON_SHA256 + .\helpers\install_ruby.ps1 -Version $ENV:RUBY_VERSION -Sha256 $ENV:RUBY_SHA256 + .\helpers\install_msys.ps1 -Version $ENV:MSYS_VERSION -Sha256 $ENV:MSYS_SHA256 + .\helpers\install_docker.ps1 + .\helpers\install_gcloud_sdk.ps1 + .\helpers\install_embedded_pythons.ps1 + .\helpers\install_vcpkg.ps1 + .\helpers\install_codeql.ps1 + .\helpers\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 + ## # Add signtool to path + Add-ToPath -NewPath "${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\" -Global + & .\set_cpython_compiler.cmd +} +catch { + Write-Host -ForegroundColor Red "Error installing software" + Write-Host -ForegroundColor Red "$_.ScriptStackTrace" + exit -1 +} +finally { + + if(!$TargetContainer){ + Write-Variables + $moduleTarget = "$($Env:USERPROFILE)\Documents\WindowsPowerShell\Modules\DDDeveloper" + if(! (test-path $moduleTarget)){ + mkdir $moduleTarget + } + xcopy /y/e/s "$($PSScriptRoot)\modules\DDDeveloper\*" $moduleTarget + import-module -force DDDeveloper + } + Remove-Item -Recurse -Force c:\tmp\* -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force $Env:TEMP\* -ErrorAction SilentlyContinue + +} diff --git a/windows/master.ps1 b/windows/master.ps1 deleted file mode 100644 index 30c57e2e3..000000000 --- a/windows/master.ps1 +++ /dev/null @@ -1,65 +0,0 @@ -param( - [Parameter(Mandatory = $false)][switch] $TargetContainer -) - -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -$ErrorActionPreference = 'Stop' - -# this includes the master versions variables -. .\versions.ps1 - -# this includes the helper functions -. .\helpers.ps1 - -# set the environment variables from the versions file - -foreach ($h in $SoftwareTable.GetEnumerator()){ - $key = $($h.Key) - $val = $($h.Value) - [Environment]::SetEnvironmentVariable($key, $val, [System.EnvironmentVariableTarget]::Process) -} - -.\helpers\install_cert.ps1 - -## only do this if building the container -if($TargetContainer){ - $Env:DD_DEV_TARGET="Container" - Write-Host "Container Flag Set $TargetContainer" -} else { - Write-Host "Container flag not set $TargetContainer" -} - -.\helpers\install_net35.ps1 -.\helpers\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256 -.\helpers\install_mingit.ps1 -Version $ENV:GIT_VERSION -Sha256 $ENV:GIT_SHA256 - -### HACK: we disable symbolic links when cloning repositories -### to work around a symlink-related failure in the agent-binaries omnibus project -### when copying the datadog-agent project twice. -& git config --system core.symlinks false -.\helpers\install_vstudio.ps1 #-Version $ENV:VS2017BUILDTOOLS_VERSION -Sha256 $ENV:VS2017BUILDTOOLS_SHA256 $ENV:VS2017BUILDTOOLS_DOWNLOAD_URL -.\helpers\install_wdk.ps1 -.\helpers\install_wix.ps1 -Version $ENV:WIX_VERSION -Sha256 $ENV:WIX_SHA256 -.\helpers\install_dotnetcore.ps1 -.\helpers\install_nuget.ps1 -Version $ENV:NUGET_VERSION -Sha256 $ENV:NUGET_SHA256 -.\helpers\install_vcpython.ps1 -.\helpers\install_ibm_mq.ps1 -Version $ENV:IBM_MQ_VERSION -Sha256 $ENV:IBM_MQ_SHA256 -.\helpers\install_cmake.ps1 -Version $ENV:CMAKE_VERSION -Sha256 $ENV:CMAKE_SHA256 -.\helpers\install_winget.ps1 -Version $ENV:WINGET_VERSION -Sha256 $ENV:WINGET_SHA256 -.\helpers\install_go.ps1 -.\helpers\install_python.ps1 -Version $ENV:PYTHON_VERSION -Sha256 $ENV:PYTHON_SHA256 -.\helpers\install_ruby.ps1 -Version $ENV:RUBY_VERSION -Sha256 $ENV:RUBY_SHA256 -.\helpers\install_msys.ps1 -Version $ENV:MSYS_VERSION -Sha256 $ENV:MSYS_SHA256 -.\helpers\install_docker.ps1 -.\helpers\install_gcloud_sdk.ps1 -.\helpers\install_embedded_pythons.ps1 -.\helpers\install_vcpkg.ps1 -.\helpers\install_codeql.ps1 -.\helpers\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 -## # Add signtool to path -Add-ToPath -NewPath "${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\" -Global -& .\set_cpython_compiler.cmd -if($TargetContainer){ - Remove-Item -Recurse -Force c:\tmp\* - Remove-Item -Recurse -Force $Env:TEMP\* -} diff --git a/windows/modules/DDDeveloper/DDDeveloper.psd1 b/windows/modules/DDDeveloper/DDDeveloper.psd1 new file mode 100644 index 000000000..aea132074 --- /dev/null +++ b/windows/modules/DDDeveloper/DDDeveloper.psd1 @@ -0,0 +1,129 @@ +# +# Module manifest for module 'DDDeveloper' +# +# Generated by: db +# +# Generated on: 9/1/2022 +# + +@{ + + # Script module or binary module file associated with this manifest. + # RootModule = '' + + # Version number of this module. + ModuleVersion = '1.0.0.0' + + # Supported PSEditions + # CompatiblePSEditions = @() + + # ID used to uniquely identify this module + GUID = 'f38fadde-15cb-48ed-a64e-5f3738cea028' + + # Author of this module + Author = 'db' + + # Company or vendor of this module + CompanyName = 'Unknown' + + # Copyright statement for this module + Copyright = '(c) 2021 db. All rights reserved.' + + # Description of the functionality provided by this module + Description = 'Windows Development Tools and Helpers' + + # Minimum version of the Windows PowerShell engine required by this module + PowerShellVersion = '5.1.14393.5127' + + # Name of the Windows PowerShell host required by this module + # PowerShellHostName = '' + + # Minimum version of the Windows PowerShell host required by this module + # PowerShellHostVersion = '' + + # Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. + # DotNetFrameworkVersion = '' + + # Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. + # CLRVersion = '' + + # Processor architecture (None, X86, Amd64) required by this module + # ProcessorArchitecture = '' + + # Modules that must be imported into the global environment prior to importing this module + # RequiredModules = @() + + # Assemblies that must be loaded prior to importing this module + # RequiredAssemblies = @() + + # Script files (.ps1) that are run in the caller's environment prior to importing this module. + # ScriptsToProcess = @() + + # Type files (.ps1xml) to be loaded when importing this module + # TypesToProcess = @() + + # Format files (.ps1xml) to be loaded when importing this module + # FormatsToProcess = @() + + # Modules to import as nested modules of the module specified in RootModule/ModuleToProcess + NestedModules = @('.\go\go.psm1', + '.\main.psm1' + ) + + # Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. + FunctionsToExport = @('Set-GoVersion', 'Get-GoVersions', 'Get-GoRootDir', + 'Use-Buildenv' + ) + + + # Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. + CmdletsToExport = @('Get-Which') + + # Variables to export from this module + VariablesToExport = @() + + # Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. + AliasesToExport = @('which') + + # DSC resources to export from this module + # DscResourcesToExport = @() + + # List of all modules packaged with this module + # ModuleList = @() + + # List of all files packaged with this module + # FileList = @() + + # Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. + PrivateData = @{ + + PSData = @{ + + # Tags applied to this module. These help with module discovery in online galleries. + # Tags = @() + + # A URL to the license for this module. + # LicenseUri = '' + + # A URL to the main website for this project. + # ProjectUri = '' + + # A URL to an icon representing this module. + # IconUri = '' + + # ReleaseNotes of this module + # ReleaseNotes = '' + + } # End of PSData hashtable + + } # End of PrivateData hashtable + + # HelpInfo URI of this module + # HelpInfoURI = '' + + # Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. + # DefaultCommandPrefix = '' + + } + + \ No newline at end of file diff --git a/windows/modules/DDDeveloper/go/go.psm1 b/windows/modules/DDDeveloper/go/go.psm1 new file mode 100644 index 000000000..115f71afe --- /dev/null +++ b/windows/modules/DDDeveloper/go/go.psm1 @@ -0,0 +1,43 @@ +#returns installed go versions in order of latest to oldest. Returns a list of strings +function Get-GoVersions { + $rawversions = get-childitem -path "hklm:\Software\DatadogDeveloper\go" | foreach-object { new-object System.Version($_.name | split-path -leaf)} + $versions = @() + $res = $rawversions | sort -Descending | ForEach-Object { $versions += $_.ToString() } + return ,$versions +} +function Get-GoRootDir { + param ( + [Parameter(Mandatory = $true)][string] $GoVer + ) + $path = get-itempropertyvalue -path HKLM:Software\DatadogDeveloper\go\$GoVer -name goroot + if($? -eq $false){ + return $null + } + return $path +} + +function Set-GoVersion { + param ( + [Parameter(Mandatory = $true)][string] $GoVer + ) + $goversions = Get-GoVersions + $goversion = "" + if (! [bool]$GoVer) { + $goversion = $goversions[0] + } else { + if ($goversions -match $GoVer){ + $goversion = $GoVer + } else { + Write-Host -ForegroundColor Red "Could not find Go version $GoVer" + return + } + } + Write-Host "Chose version $goversion" + $gorootdir = Get-GoRootDir $goversion + if ($null -eq $gorootdir) { + Write-Host -ForegroundColor Red "Couldn't find goroot" + return + } + $Env:GOROOT="$gorootdir\go" + $Env:GOPATH="c:\go" +} \ No newline at end of file diff --git a/windows/modules/DDDeveloper/main.psm1 b/windows/modules/DDDeveloper/main.psm1 new file mode 100644 index 000000000..5a1774700 --- /dev/null +++ b/windows/modules/DDDeveloper/main.psm1 @@ -0,0 +1,71 @@ +function Use-BuildEnv { + param ( + [Parameter(Mandatory = $false)][string] $GoVer + ) + + # put python in front of `working path`. Windows 10 includes a dummy python in the path + # to tell you to go get it from the store, which means that the binary can't be found + + $targetdir = "$($Env:USERPROFILE)\.ddbuild" + if(!(test-path $targetdir)){ + mkdir $targetdir + } + $targetfile = "$targetdir\environment.json" + $varsToSet = Get-Content $targetfile | convertfrom-json + + # EnvironmentVars = @{} + # PathEntries = @() + #} + ## first set all the variables we were asked to set + $varsToSet.EnvironmentVars.psobject.properties | foreach { + [Environment]::SetEnvironmentVariable($_.Name, $_.Value, [System.EnvironmentVariableTarget]::Process) + } + + ## save the starting path. This allows multiple invocations in the same shell + if($null -eq $Env:ORIGPATH) { + $Env:ORIGPATH = $ENV:PATH + } + + $newPathEntries = $Env:ORIGPATH -split ";" + + ## now walk all the new entries + foreach($e in $varsToSet.PathEntries) { + if ([string]::IsNullOrWhiteSpace($e)){ + continue; + } + + if($newPathEntries -notcontains $e) { + ## + ## if this is the `go` path, skip it because we're going to manually add it later + if($e -contains "\go\"){ + continue + } + ## need to hack this a bit. Ideally all the path entries would be behind the default + ## windows entries; but windows10/11 includes a python "stub", so we need to put python + ## before the windows entries + if($e -contains "python"){ + $newPathEntries = @($e) + $newPathEntries + } else { + $newPathEntries += $e + } + } + } + + ## append the proper go paths. + if(!$GoVer) { + $GoVer = Get-Goversions + } + Set-GoVersion $GoVer + + $newPathEntries += "$Env:GOROOT\bin" + $newPathEntries += "$Env:GOPATH\bin" + + $Env:PATH=$newPathEntries -join ";" + $Env:BUILDENV="Agent-Build" + + # enable RIDK in this shell + & $Env:RIDK enable + + # load the developer prompt + . $PSScriptRoot\prompt.ps1 +} \ No newline at end of file diff --git a/windows/modules/DDDeveloper/prompt.ps1 b/windows/modules/DDDeveloper/prompt.ps1 new file mode 100644 index 000000000..d725e2d0c --- /dev/null +++ b/windows/modules/DDDeveloper/prompt.ps1 @@ -0,0 +1,40 @@ +function global:prompt { + $identity = [Security.Principal.WindowsIdentity]::GetCurrent() + $principal = [Security.Principal.WindowsPrincipal] $identity + + $uname = "$($identity.Name)" + if($principal.IsInRole([Security.Principal.WindowsBuiltInRole]"Administrator")) { + $uname = "[ADMIN]: $($identity.Name)" + } + $title = $uname + $promptprompt = $uname + if( $Env:PROMPTTYPE -ne $null) { + $promptprompt = "{ $($Env:PROMPTTYPE) } $($uname) " + } + + if($null -ne $Env:GOROOT){ + Write-Host -ForegroundColor Red " GOROOT $Env:GOROOT " -NoNewline + $title = "$title GOROOT:$Env:GOROOT" + } + if($null -ne $Env:GOPATH) { + Write-Host -ForegroundColor Green " GOPATH $Env:GOPATH " -NoNewline + $title = "$title GOPATH:$Env:GOPATH" + } + $host.ui.RawUI.WindowTitle = $title + + $pathline = $null + if($null -ne $Env:BUILDENV){ + $pathline = "$Env:BUILDENV - $((Get-Location).Path)" + } else { + $pathline = "$((Get-Location).Path)" + } + $gb = $null + $gb = git.exe rev-parse --abbrev-ref HEAD 2> $null + + if($?){ + Write-Host -ForegroundColor Cyan "git-branch $gb `n" -NoNewline + } + Write-Host -ForegroundColor Yellow "`n$pathline `n" -NoNewline + Write-Host -ForegroundColor Magenta "$promptprompt >> " -NoNewline + return " " +} \ No newline at end of file From fddb9d97dce4fd3fa877de65dcfc578485ba01c8 Mon Sep 17 00:00:00 2001 From: Derek Brown Date: Mon, 5 Sep 2022 12:50:14 -0700 Subject: [PATCH 12/16] clean up python handling fix error in go path finding Attempt to catch msys install failure :Add some debugging for 2022ltsc Move mingit after vstudio fix path to patch --- windows/helpers.ps1 | 6 +++++- windows/helpers/install_mingit.ps1 | 24 +++++++++++++++++------- windows/helpers/install_msys.ps1 | 13 +++++++++++-- windows/helpers/install_net35.ps1 | 3 ++- windows/helpers/install_python.ps1 | 1 + windows/helpers/install_vstudio.ps1 | 3 +++ windows/install-all.ps1 | 6 +----- windows/modules/DDDeveloper/main.psm1 | 10 ++++++---- 8 files changed, 46 insertions(+), 20 deletions(-) diff --git a/windows/helpers.ps1 b/windows/helpers.ps1 index 4d73eb3f1..3b55786d3 100644 --- a/windows/helpers.ps1 +++ b/windows/helpers.ps1 @@ -98,7 +98,11 @@ function Add-ToPath() { [Parameter(Mandatory = $false)][switch] $Global ) if($Local) { - $Env:Path="$Env:Path;$NewPath" + if( $NewPath -like "*python*"){ + $Env:Path="$NewPath;$Env:PATH" + } else { + $Env:Path="$Env:Path;$NewPath" + } } if($Global){ if($TargetContainer){ diff --git a/windows/helpers/install_mingit.ps1 b/windows/helpers/install_mingit.ps1 index 2eca3fa86..de2ca62fc 100644 --- a/windows/helpers/install_mingit.ps1 +++ b/windows/helpers/install_mingit.ps1 @@ -16,7 +16,9 @@ function InstallMinGit() { $out = "$($PSScriptRoot)\mingit.zip" Get-RemoteFile -RemoteFile $mingit -LocalFile $out -VerifyHash $Sha256 - md c:\devtools\git + if(! (test-path "c:\devtools\git")){ + md c:\devtools\git + } & '7z' x -oc:\devtools\git $out Remove-Item $out @@ -26,6 +28,7 @@ function InstallMinGit() { & 'git.exe' config --global user.email "croissant@datadoghq.com" Write-Host -ForegroundColor Green Done with Git + return $true } function InstallWinGit() { @@ -35,7 +38,7 @@ function InstallWinGit() { $isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "git" -Keyname "version" -TargetValue $wingit if($isInstalled -and $isCurrent){ Write-Host -ForegroundColor Green "WinGit already installed" - return + return $false } Write-Host -ForegroundColor Green "Installing WinGit" $out = "$($PSScriptRoot)\wingit.exe" @@ -47,13 +50,20 @@ function InstallWinGit() { Reload-Path Set-InstalledVersionKey -Component "git" -Keyname "Version" -TargetValue $wingit Write-Host -ForegroundColor Green Done with Git + return $true } - +$installed = $false if($Env:DD_DEV_TARGET -eq "Container") { - InstallMinGit - return + $installed = InstallMinGit + } else { - InstallWinGit - return + $installed = InstallWinGit +} +if($installed){ + ### HACK: we disable symbolic links when cloning repositories + ### to work around a symlink-related failure in the agent-binaries omnibus project + ### when copying the datadog-agent project twice. + & git config --system core.symlinks false + } return diff --git a/windows/helpers/install_msys.ps1 b/windows/helpers/install_msys.ps1 index 34a07cb27..2e71565ab 100644 --- a/windows/helpers/install_msys.ps1 +++ b/windows/helpers/install_msys.ps1 @@ -19,10 +19,12 @@ function Invoke-Msys2Shell($Arguments) { FilePath = Join-Path $basepath msys2_shell.cmd NoNewWindow = $true Wait = $true +# PassThru = $true ArgumentList = "-defterm", "-no-start", "-c", "`"$Arguments`"" } Write-Host "Invoking msys2 shell command:" $params.ArgumentList - Start-Process @params + $p = Start-Process @params + return $lastExitCode } $isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "msys" -Keyname "version" -TargetValue $Version if($isInstalled -and $isCurrent) { @@ -50,9 +52,16 @@ Remove-Item $out Remove-Item $msystar ## invoke the first-run shell -Invoke-Msys2Shell +$mshell = Invoke-Msys2Shell +Write-Host -ForegroundColor Yellow "Invoke-Msys2Shell return code $mshell" +if ( $mshell -ne "0") { + throw "Invoke MSYS returned $mshell" +} ridk install 3 +If ($lastExitCode -ne "0") { + throw "ridk install 3 returned $lastExitCode" +} # Downgrade gcc and binutils due to https://github.com/golang/go/issues/46099 Get-RemoteFile -RemoteFile "https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst" -LocalFile "C:/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst" Get-RemoteFile -RemoteFile "https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst" -LocalFile "C:/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst" diff --git a/windows/helpers/install_net35.ps1 b/windows/helpers/install_net35.ps1 index 4a0c44e80..1b14826b9 100644 --- a/windows/helpers/install_net35.ps1 +++ b/windows/helpers/install_net35.ps1 @@ -78,6 +78,7 @@ if($Env:DD_DEV_TARGET -ne "Container") { $osInfo = Get-CimInstance -classname win32_operatingsystem if($osinfo.ProductType -eq "1"){ & dism /online /enable-feature /FeatureName:Netfx3 /all + Set-InstalledVersionKey -Component "netfx35" -Keyname "version" -TargetValue "1" return } } @@ -118,7 +119,7 @@ mkdir patch expand "$($PSScriptRoot)\patch.msu" patch -F:* remove-item -force "$($PSScriptRoot)\patch.msu" Write-Host DISM /Online /Quiet /Add-Package /PackagePath:$($UpgradeTable[$kernelver]["expandedpatch"]) -DISM /Online /Quiet /Add-Package /PackagePath:$($UpgradeTable[$kernelver]["expandedpatch"]) +DISM /Online /Quiet /Add-Package /PackagePath:"patch\$($UpgradeTable[$kernelver]["expandedpatch"])" remove-item -force -recurse patch Set-InstalledVersionKey -Component "netfx35" -Keyname "version" -TargetValue "1" \ No newline at end of file diff --git a/windows/helpers/install_python.ps1 b/windows/helpers/install_python.ps1 index ef32aca7f..e78036096 100644 --- a/windows/helpers/install_python.ps1 +++ b/windows/helpers/install_python.ps1 @@ -24,6 +24,7 @@ Write-Host -ForegroundColor Green Done downloading Python, installing Start-Process $out -ArgumentList '/quiet InstallAllUsers=1' -Wait Add-ToPath "c:\program files\Python38;c:\Program files\python38\scripts" -Global -Local + Remove-Item $out $getpipurl = "https://raw.githubusercontent.com/pypa/get-pip/38e54e5de07c66e875c11a1ebbdb938854625dd8/public/get-pip.py" diff --git a/windows/helpers/install_vstudio.ps1 b/windows/helpers/install_vstudio.ps1 index 659ba1245..bdb9e0158 100644 --- a/windows/helpers/install_vstudio.ps1 +++ b/windows/helpers/install_vstudio.ps1 @@ -72,7 +72,10 @@ $processparams = @{ Wait = $true ArgumentList = $ArgList } +$st = get-date +Write-Host "Calling Start-Process $st" Start-Process @processparams +Write-Host "start-process done $st $(get-date)" Add-EnvironmentVariable -Variable VSTUDIO_ROOT -Value $InstallRoot -Global -Local diff --git a/windows/install-all.ps1 b/windows/install-all.ps1 index 5b9c44004..b261ba1d1 100644 --- a/windows/install-all.ps1 +++ b/windows/install-all.ps1 @@ -40,13 +40,9 @@ if($TargetContainer){ try { .\helpers\install_net35.ps1 .\helpers\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256 + .\helpers\install_vstudio.ps1 #-Version $ENV:VS2017BUILDTOOLS_VERSION -Sha256 $ENV:VS2017BUILDTOOLS_SHA256 $ENV:VS2017BUILDTOOLS_DOWNLOAD_URL .\helpers\install_mingit.ps1 -Version $ENV:GIT_VERSION -Sha256 $ENV:GIT_SHA256 - ### HACK: we disable symbolic links when cloning repositories - ### to work around a symlink-related failure in the agent-binaries omnibus project - ### when copying the datadog-agent project twice. - & git config --system core.symlinks false - .\helpers\install_vstudio.ps1 #-Version $ENV:VS2017BUILDTOOLS_VERSION -Sha256 $ENV:VS2017BUILDTOOLS_SHA256 $ENV:VS2017BUILDTOOLS_DOWNLOAD_URL .\helpers\install_wdk.ps1 .\helpers\install_wix.ps1 -Version $ENV:WIX_VERSION -Sha256 $ENV:WIX_SHA256 .\helpers\install_dotnetcore.ps1 diff --git a/windows/modules/DDDeveloper/main.psm1 b/windows/modules/DDDeveloper/main.psm1 index 5a1774700..3f85cbf41 100644 --- a/windows/modules/DDDeveloper/main.psm1 +++ b/windows/modules/DDDeveloper/main.psm1 @@ -37,13 +37,13 @@ function Use-BuildEnv { if($newPathEntries -notcontains $e) { ## ## if this is the `go` path, skip it because we're going to manually add it later - if($e -contains "\go\"){ + if($e -like "*\go\*"){ continue } ## need to hack this a bit. Ideally all the path entries would be behind the default ## windows entries; but windows10/11 includes a python "stub", so we need to put python ## before the windows entries - if($e -contains "python"){ + if($e -like "*python*"){ $newPathEntries = @($e) + $newPathEntries } else { $newPathEntries += $e @@ -52,10 +52,12 @@ function Use-BuildEnv { } ## append the proper go paths. + $useGoVersion = $GoVer if(!$GoVer) { - $GoVer = Get-Goversions + $versions = Get-Goversions + $useGoVersion = $versions[0] } - Set-GoVersion $GoVer + Set-GoVersion $useGoVersion $newPathEntries += "$Env:GOROOT\bin" $newPathEntries += "$Env:GOPATH\bin" From 09af9a97527e26b3daa5bbb8d75f659053afb689 Mon Sep 17 00:00:00 2001 From: Branden Clark Date: Wed, 1 Feb 2023 13:04:30 -0500 Subject: [PATCH 13/16] Break install-all into phases to allow for some Docker layers (#346) Change env var helpers to write to USER env instead of SYSTEM env --- windows/Dockerfile | 9 ++- windows/helpers.ps1 | 8 +-- windows/helpers/{ => phase1}/install_7zip.ps1 | 0 .../helpers/{ => phase1}/install_cmake.ps1 | 0 .../{ => phase1}/install_dotnetcore.ps1 | 0 .../helpers/{ => phase1}/install_mingit.ps1 | 0 .../helpers/{ => phase1}/install_net35.ps1 | 0 .../helpers/{ => phase1}/install_nuget.ps1 | 0 .../helpers/{ => phase1}/install_vcpkg.ps1 | 0 .../helpers/{ => phase1}/install_vcpython.ps1 | 0 .../helpers/{ => phase1}/install_vstudio.ps1 | 2 - windows/helpers/{ => phase1}/install_wdk.ps1 | 0 windows/helpers/{ => phase1}/install_wix.ps1 | 0 .../helpers/{ => phase2}/install_docker.ps1 | 0 .../{ => phase2}/install_embedded_pythons.ps1 | 0 .../{ => phase2}/install_gcloud_sdk.ps1 | 0 windows/helpers/{ => phase2}/install_msys.ps1 | 0 .../helpers/{ => phase2}/install_python.ps1 | 0 windows/helpers/{ => phase2}/install_ruby.ps1 | 0 .../helpers/{ => phase3}/install_codeql.ps1 | 0 windows/helpers/{ => phase3}/install_go.ps1 | 0 .../helpers/{ => phase3}/install_ibm_mq.ps1 | 0 .../helpers/{ => phase3}/install_ninja.ps1 | 0 .../helpers/{ => phase3}/install_winget.ps1 | 0 windows/helpers/phase4/.gitkeep.ps1 | 0 windows/install-all.ps1 | 72 ++++++++++++------- 26 files changed, 58 insertions(+), 33 deletions(-) rename windows/helpers/{ => phase1}/install_7zip.ps1 (100%) rename windows/helpers/{ => phase1}/install_cmake.ps1 (100%) rename windows/helpers/{ => phase1}/install_dotnetcore.ps1 (100%) rename windows/helpers/{ => phase1}/install_mingit.ps1 (100%) rename windows/helpers/{ => phase1}/install_net35.ps1 (100%) rename windows/helpers/{ => phase1}/install_nuget.ps1 (100%) rename windows/helpers/{ => phase1}/install_vcpkg.ps1 (100%) rename windows/helpers/{ => phase1}/install_vcpython.ps1 (100%) rename windows/helpers/{ => phase1}/install_vstudio.ps1 (96%) rename windows/helpers/{ => phase1}/install_wdk.ps1 (100%) rename windows/helpers/{ => phase1}/install_wix.ps1 (100%) rename windows/helpers/{ => phase2}/install_docker.ps1 (100%) rename windows/helpers/{ => phase2}/install_embedded_pythons.ps1 (100%) rename windows/helpers/{ => phase2}/install_gcloud_sdk.ps1 (100%) rename windows/helpers/{ => phase2}/install_msys.ps1 (100%) rename windows/helpers/{ => phase2}/install_python.ps1 (100%) rename windows/helpers/{ => phase2}/install_ruby.ps1 (100%) rename windows/helpers/{ => phase3}/install_codeql.ps1 (100%) rename windows/helpers/{ => phase3}/install_go.ps1 (100%) rename windows/helpers/{ => phase3}/install_ibm_mq.ps1 (100%) rename windows/helpers/{ => phase3}/install_ninja.ps1 (100%) rename windows/helpers/{ => phase3}/install_winget.ps1 (100%) create mode 100644 windows/helpers/phase4/.gitkeep.ps1 diff --git a/windows/Dockerfile b/windows/Dockerfile index 49e137e7d..3a419e756 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -65,7 +65,14 @@ COPY ./requirements.txt ./requirements-py2.txt / COPY ./python-packages-versions.txt /python-packages-versions.txt COPY ./windows/set_cpython_compiler.cmd set_cpython_compiler.cmd -RUN .\install-all.ps1 -TargetContainer +COPY ./windows/helpers/phase1/*.ps1 c:/scripts/helpers/phase1/ +RUN .\install-all.ps1 -TargetContainer -Phase 1 +COPY ./windows/helpers/phase2/*.ps1 c:/scripts/helpers/phase2/ +RUN .\install-all.ps1 -TargetContainer -Phase 2 +COPY ./windows/helpers/phase3/*.ps1 c:/scripts/helpers/phase3/ +RUN .\install-all.ps1 -TargetContainer -Phase 3 +COPY ./windows/helpers/phase4/*.ps1 c:/scripts/helpers/phase4/ +RUN .\install-all.ps1 -TargetContainer -Phase 4 COPY ./windows/entrypoint.bat /entrypoint.bat COPY ./windows/helpers/aws_networking.ps1 /aws_networking.ps1 diff --git a/windows/helpers.ps1 b/windows/helpers.ps1 index 3b55786d3..ed7b79df2 100644 --- a/windows/helpers.ps1 +++ b/windows/helpers.ps1 @@ -84,7 +84,7 @@ function Add-EnvironmentVariable() { } if($Global){ if($TargetContainer){ - [Environment]::SetEnvironmentVariable($Variable, $Value, [System.EnvironmentVariableTarget]::Machine) + [Environment]::SetEnvironmentVariable($Variable, $Value, [System.EnvironmentVariableTarget]::User) } else { $GlobalEnvVariables.EnvironmentVars[$($Variable)] = $Value } @@ -106,9 +106,9 @@ function Add-ToPath() { } if($Global){ if($TargetContainer){ - $oldPath=[Environment]::GetEnvironmentVariable("Path", [System.EnvironmentVariableTarget]::Machine) + $oldPath=[Environment]::GetEnvironmentVariable("Path", [System.EnvironmentVariableTarget]::User) $target="$oldPath;$NewPath" - [Environment]::SetEnvironmentVariable("Path", $target, [System.EnvironmentVariableTarget]::Machine) + [Environment]::SetEnvironmentVariable("Path", $target, [System.EnvironmentVariableTarget]::User) } else { if ($GlobalEnvVariables.PathEntries -notcontains $NewPath){ $GlobalEnvVariables.PathEntries += $NewPath @@ -189,4 +189,4 @@ function Write-Variables() { $targetfile = Get-VariableFile $GlobalEnvVariables | convertto-json | set-content -path $targetfile -} \ No newline at end of file +} diff --git a/windows/helpers/install_7zip.ps1 b/windows/helpers/phase1/install_7zip.ps1 similarity index 100% rename from windows/helpers/install_7zip.ps1 rename to windows/helpers/phase1/install_7zip.ps1 diff --git a/windows/helpers/install_cmake.ps1 b/windows/helpers/phase1/install_cmake.ps1 similarity index 100% rename from windows/helpers/install_cmake.ps1 rename to windows/helpers/phase1/install_cmake.ps1 diff --git a/windows/helpers/install_dotnetcore.ps1 b/windows/helpers/phase1/install_dotnetcore.ps1 similarity index 100% rename from windows/helpers/install_dotnetcore.ps1 rename to windows/helpers/phase1/install_dotnetcore.ps1 diff --git a/windows/helpers/install_mingit.ps1 b/windows/helpers/phase1/install_mingit.ps1 similarity index 100% rename from windows/helpers/install_mingit.ps1 rename to windows/helpers/phase1/install_mingit.ps1 diff --git a/windows/helpers/install_net35.ps1 b/windows/helpers/phase1/install_net35.ps1 similarity index 100% rename from windows/helpers/install_net35.ps1 rename to windows/helpers/phase1/install_net35.ps1 diff --git a/windows/helpers/install_nuget.ps1 b/windows/helpers/phase1/install_nuget.ps1 similarity index 100% rename from windows/helpers/install_nuget.ps1 rename to windows/helpers/phase1/install_nuget.ps1 diff --git a/windows/helpers/install_vcpkg.ps1 b/windows/helpers/phase1/install_vcpkg.ps1 similarity index 100% rename from windows/helpers/install_vcpkg.ps1 rename to windows/helpers/phase1/install_vcpkg.ps1 diff --git a/windows/helpers/install_vcpython.ps1 b/windows/helpers/phase1/install_vcpython.ps1 similarity index 100% rename from windows/helpers/install_vcpython.ps1 rename to windows/helpers/phase1/install_vcpython.ps1 diff --git a/windows/helpers/install_vstudio.ps1 b/windows/helpers/phase1/install_vstudio.ps1 similarity index 96% rename from windows/helpers/install_vstudio.ps1 rename to windows/helpers/phase1/install_vstudio.ps1 index bdb9e0158..90c8d673a 100644 --- a/windows/helpers/install_vstudio.ps1 +++ b/windows/helpers/phase1/install_vstudio.ps1 @@ -22,9 +22,7 @@ $VSPackages = @( "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "Microsoft.VisualStudio.Component.VC.Runtimes.x86.x64.Spectre", "Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81", -# "Microsoft.VisualStudio.Component.Windows10SDK.17763", "Microsoft.VisualStudio.Component.Windows10SDK.18362" -# "Microsoft.VisualStudio.Component.Windows10SDK.19041" ) $VSPackagesDesktop = @( diff --git a/windows/helpers/install_wdk.ps1 b/windows/helpers/phase1/install_wdk.ps1 similarity index 100% rename from windows/helpers/install_wdk.ps1 rename to windows/helpers/phase1/install_wdk.ps1 diff --git a/windows/helpers/install_wix.ps1 b/windows/helpers/phase1/install_wix.ps1 similarity index 100% rename from windows/helpers/install_wix.ps1 rename to windows/helpers/phase1/install_wix.ps1 diff --git a/windows/helpers/install_docker.ps1 b/windows/helpers/phase2/install_docker.ps1 similarity index 100% rename from windows/helpers/install_docker.ps1 rename to windows/helpers/phase2/install_docker.ps1 diff --git a/windows/helpers/install_embedded_pythons.ps1 b/windows/helpers/phase2/install_embedded_pythons.ps1 similarity index 100% rename from windows/helpers/install_embedded_pythons.ps1 rename to windows/helpers/phase2/install_embedded_pythons.ps1 diff --git a/windows/helpers/install_gcloud_sdk.ps1 b/windows/helpers/phase2/install_gcloud_sdk.ps1 similarity index 100% rename from windows/helpers/install_gcloud_sdk.ps1 rename to windows/helpers/phase2/install_gcloud_sdk.ps1 diff --git a/windows/helpers/install_msys.ps1 b/windows/helpers/phase2/install_msys.ps1 similarity index 100% rename from windows/helpers/install_msys.ps1 rename to windows/helpers/phase2/install_msys.ps1 diff --git a/windows/helpers/install_python.ps1 b/windows/helpers/phase2/install_python.ps1 similarity index 100% rename from windows/helpers/install_python.ps1 rename to windows/helpers/phase2/install_python.ps1 diff --git a/windows/helpers/install_ruby.ps1 b/windows/helpers/phase2/install_ruby.ps1 similarity index 100% rename from windows/helpers/install_ruby.ps1 rename to windows/helpers/phase2/install_ruby.ps1 diff --git a/windows/helpers/install_codeql.ps1 b/windows/helpers/phase3/install_codeql.ps1 similarity index 100% rename from windows/helpers/install_codeql.ps1 rename to windows/helpers/phase3/install_codeql.ps1 diff --git a/windows/helpers/install_go.ps1 b/windows/helpers/phase3/install_go.ps1 similarity index 100% rename from windows/helpers/install_go.ps1 rename to windows/helpers/phase3/install_go.ps1 diff --git a/windows/helpers/install_ibm_mq.ps1 b/windows/helpers/phase3/install_ibm_mq.ps1 similarity index 100% rename from windows/helpers/install_ibm_mq.ps1 rename to windows/helpers/phase3/install_ibm_mq.ps1 diff --git a/windows/helpers/install_ninja.ps1 b/windows/helpers/phase3/install_ninja.ps1 similarity index 100% rename from windows/helpers/install_ninja.ps1 rename to windows/helpers/phase3/install_ninja.ps1 diff --git a/windows/helpers/install_winget.ps1 b/windows/helpers/phase3/install_winget.ps1 similarity index 100% rename from windows/helpers/install_winget.ps1 rename to windows/helpers/phase3/install_winget.ps1 diff --git a/windows/helpers/phase4/.gitkeep.ps1 b/windows/helpers/phase4/.gitkeep.ps1 new file mode 100644 index 000000000..e69de29bb diff --git a/windows/install-all.ps1 b/windows/install-all.ps1 index b261ba1d1..777a5b3b2 100644 --- a/windows/install-all.ps1 +++ b/windows/install-all.ps1 @@ -1,5 +1,6 @@ param( - [Parameter(Mandatory = $false)][switch] $TargetContainer + [Parameter(Mandatory = $false)][switch] $TargetContainer, + [Parameter(Mandatory = $false)][string] $Phase = "0" ) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 @@ -38,32 +39,51 @@ if($TargetContainer){ } try { - .\helpers\install_net35.ps1 - .\helpers\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256 - .\helpers\install_vstudio.ps1 #-Version $ENV:VS2017BUILDTOOLS_VERSION -Sha256 $ENV:VS2017BUILDTOOLS_SHA256 $ENV:VS2017BUILDTOOLS_DOWNLOAD_URL - .\helpers\install_mingit.ps1 -Version $ENV:GIT_VERSION -Sha256 $ENV:GIT_SHA256 + # Each phase is its own layer in the Dockerfile, to make it easier to iterate on changes + # by making use of Docker's layer cache. The phases are roughly organized by dependencies, + # install time, and frequency of updates, with longer install times and less frequently + # changed items in the earlier phases. + # + # Phase 4 is empty by default. Before starting work on updating an item move the script to Phase 4. + # + if ($Phase -eq 0 -or $Phase -eq 1) { + .\helpers\phase1\install_net35.ps1 + .\helpers\phase1\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256 + .\helpers\phase1\install_mingit.ps1 -Version $ENV:GIT_VERSION -Sha256 $ENV:GIT_SHA256 + .\helpers\phase1\install_vstudio.ps1 + .\helpers\phase1\install_wdk.ps1 + .\helpers\phase1\install_wix.ps1 -Version $ENV:WIX_VERSION -Sha256 $ENV:WIX_SHA256 + .\helpers\phase1\install_dotnetcore.ps1 + .\helpers\phase1\install_nuget.ps1 -Version $ENV:NUGET_VERSION -Sha256 $ENV:NUGET_SHA256 + .\helpers\phase1\install_vcpython.ps1 + .\helpers\phase1\install_cmake.ps1 -Version $ENV:CMAKE_VERSION -Sha256 $ENV:CMAKE_SHA256 + # # vcpkg depends on cmake + .\helpers\phase1\install_vcpkg.ps1 + } + + if ($Phase -eq 0 -or $Phase -eq 2) { + .\helpers\phase2\install_docker.ps1 + .\helpers\phase2\install_ruby.ps1 -Version $ENV:RUBY_VERSION -Sha256 $ENV:RUBY_SHA256 + # msys depends on ruby + .\helpers\phase2\install_msys.ps1 -Version $ENV:MSYS_VERSION -Sha256 $ENV:MSYS_SHA256 + .\helpers\phase2\install_python.ps1 -Version $ENV:PYTHON_VERSION -Sha256 $ENV:PYTHON_SHA256 + .\helpers\phase2\install_gcloud_sdk.ps1 + .\helpers\phase2\install_embedded_pythons.ps1 + } - .\helpers\install_wdk.ps1 - .\helpers\install_wix.ps1 -Version $ENV:WIX_VERSION -Sha256 $ENV:WIX_SHA256 - .\helpers\install_dotnetcore.ps1 - .\helpers\install_nuget.ps1 -Version $ENV:NUGET_VERSION -Sha256 $ENV:NUGET_SHA256 - .\helpers\install_vcpython.ps1 - .\helpers\install_ibm_mq.ps1 -Version $ENV:IBM_MQ_VERSION -Sha256 $ENV:IBM_MQ_SHA256 - .\helpers\install_cmake.ps1 -Version $ENV:CMAKE_VERSION -Sha256 $ENV:CMAKE_SHA256 - .\helpers\install_winget.ps1 -Version $ENV:WINGET_VERSION -Sha256 $ENV:WINGET_SHA256 - .\helpers\install_go.ps1 - .\helpers\install_python.ps1 -Version $ENV:PYTHON_VERSION -Sha256 $ENV:PYTHON_SHA256 - .\helpers\install_ruby.ps1 -Version $ENV:RUBY_VERSION -Sha256 $ENV:RUBY_SHA256 - .\helpers\install_msys.ps1 -Version $ENV:MSYS_VERSION -Sha256 $ENV:MSYS_SHA256 - .\helpers\install_docker.ps1 - .\helpers\install_gcloud_sdk.ps1 - .\helpers\install_embedded_pythons.ps1 - .\helpers\install_vcpkg.ps1 - .\helpers\install_codeql.ps1 - .\helpers\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 - ## # Add signtool to path - Add-ToPath -NewPath "${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\" -Global - & .\set_cpython_compiler.cmd + if ($Phase -eq 0 -or $Phase -eq 3) { + .\helpers\phase3\install_ibm_mq.ps1 -Version $ENV:IBM_MQ_VERSION -Sha256 $ENV:IBM_MQ_SHA256 + .\helpers\phase3\install_winget.ps1 -Version $ENV:WINGET_VERSION -Sha256 $ENV:WINGET_SHA256 + .\helpers\phase3\install_go.ps1 + .\helpers\phase3\install_codeql.ps1 + .\helpers\phase3\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256 + ## # Add signtool to path + Add-ToPath -NewPath "${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\" -Global + & .\set_cpython_compiler.cmd + } + + if ($Phase -eq 0 -or $Phase -eq 4) { + } } catch { Write-Host -ForegroundColor Red "Error installing software" From dccc332160e31b85c80955c54cea83dbd7630752 Mon Sep 17 00:00:00 2001 From: Branden Clark Date: Mon, 6 Feb 2023 20:46:00 -0500 Subject: [PATCH 14/16] Apply suggestions from code review Co-authored-by: Julien Lebot --- windows/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/windows/Dockerfile b/windows/Dockerfile index 3a419e756..181b31e66 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -56,9 +56,7 @@ LABEL codeql_version=${CODEQL_VERSION} ## WORKDIR should automatically create the directory WORKDIR c:/scripts -COPY ./windows/versions.ps1 versions.ps1 -COPY ./windows/install-all.ps1 install-all.ps1 -COPY ./windows/helpers.ps1 helpers.ps1 +COPY ./windows/*.ps1 . COPY ./windows/helpers/*.ps1 c:/scripts/helpers/ COPY ./requirements.txt ./requirements-py2.txt / From 0d6478ef657dc776a252d028494beab1738a625d Mon Sep 17 00:00:00 2001 From: Branden Clark Date: Mon, 6 Feb 2023 21:16:44 -0500 Subject: [PATCH 15/16] fix dockerfile syntax --- windows/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows/Dockerfile b/windows/Dockerfile index 181b31e66..606e5d577 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -56,7 +56,7 @@ LABEL codeql_version=${CODEQL_VERSION} ## WORKDIR should automatically create the directory WORKDIR c:/scripts -COPY ./windows/*.ps1 . +COPY ./windows/*.ps1 ./ COPY ./windows/helpers/*.ps1 c:/scripts/helpers/ COPY ./requirements.txt ./requirements-py2.txt / From e91cd343e784ab87934a1964bead535c3700d9b2 Mon Sep 17 00:00:00 2001 From: Branden Clark Date: Fri, 10 Feb 2023 15:20:39 -0500 Subject: [PATCH 16/16] Fix go SHA256 Ensure Get-RemoteFile can handle empty env vars --- windows/helpers.ps1 | 2 +- windows/versions.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/windows/helpers.ps1 b/windows/helpers.ps1 index ed7b79df2..b306f534b 100644 --- a/windows/helpers.ps1 +++ b/windows/helpers.ps1 @@ -62,7 +62,7 @@ function Get-RemoteFile() { Write-Host -ForegroundColor Green "Downloading: $RemoteFile" Write-Host -ForegroundColor Green " To: $LocalFile" (New-Object System.Net.WebClient).DownloadFile($RemoteFile, $LocalFile) - if ($VerifyHash){ + if ($PSBoundParameters.ContainsKey("VerifyHash")){ $dlhash = (Get-FileHash -Algorithm SHA256 $LocalFile).hash.ToLower() if($dlhash -ne $VerifyHash){ Write-Host -ForegroundColor Red "Unexpected file hash downloading $LocalFile from $RemoteFile" diff --git a/windows/versions.ps1 b/windows/versions.ps1 index cdb710d15..252d2c2fd 100644 --- a/windows/versions.ps1 +++ b/windows/versions.ps1 @@ -11,7 +11,7 @@ $SoftwareTable = @{ "VS2019INSTALLER_DOWNLOAD_URL"="https://download.visualstudio.microsoft.com/download/pr/3a7354bc-d2e4-430f-92d0-9abd031b5ee5/d9fc228ea71a98adc7bc5f5d8e8800684c647e955601ed721fcb29f74ace7536/vs_Community.exe"; "VS2019INSTALLER_SHA256"="d9fc228ea71a98adc7bc5f5d8e8800684c647e955601ed721fcb29f74ace7536"; "GO_VERSION"="1.19.5"; - "GO_SHA25"="6167db91a2e40aeb453d3e59d213ecab06f62e1c4a84d13a06ccda1d999961caa"; + "GO_SHA256"="167db91a2e40aeb453d3e59d213ecab06f62e1c4a84d13a06ccda1d999961caa"; "RUBY_VERSION"="2.6.6-1"; "RUBY_SHA256"="fbdf77a3e1fa36e25cf0af1303ac76f67dec7a6f739a829784a299702cad1492"; "IBM_MQ_VERSION"="9.2.4.0";