Skip to content

Commit

Permalink
Refactor tools to allow building container and setting up dev machine (…
Browse files Browse the repository at this point in the history 
…#276)

* DO NOT MERGE

Initial revision.
Attempting single run windows container, in hopes of being able to make
shared environment script

* fix path location (vcpkg)

* temp stop before vcpkg for debugging

* fix path error in networking script

* fix vcpkg

* fix go entry in path

* make work on local system

* fix typo in gsdk

* debugging statement for vcpkg

* remove ridk enable; fixes calls to cmd

* Working both locally and in container

* clean up python handling

fix error in go path finding

Attempt to catch msys install failure

:Add some debugging for 2022ltsc

Move mingit after vstudio

fix path to patch

* Break install-all into phases to allow for some Docker layers (#346)

Change env var helpers to write to USER env instead of SYSTEM env

* Apply suggestions from code review

Co-authored-by: Julien Lebot <[email protected]>

* fix dockerfile syntax

* Fix go SHA256

Ensure Get-RemoteFile can handle empty env vars

---------

Co-authored-by: Branden Clark <[email protected]>
Co-authored-by: Julien Lebot <[email protected]>
  • Loading branch information
@derekwbrown @clarkb7 @julien-lebot
3 people authored Feb 16, 2023
1 parent ea14b3e commit 8c1c831
Show file tree
Hide file tree
Showing 50 changed files with 1,375 additions and 2,429 deletions.
21 changes: 16 additions & 5 deletions build-container.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ param(
)

$ErrorActionPreference = "Stop"

. .\windows\versions.ps1
$BaseTable = @{
1809 = "mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-ltsc2019";
1909 = "mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-1909";
Expand All @@ -26,15 +26,26 @@ if ($build -ge 20348) {

Write-Host -ForegroundColor Green "Using base image $($BaseTable[$kernelver])"

$arglist = @()
if($Tag -eq $null -or $Tag -eq ""){
$Tag ="builder_$($kernelver)_$Arch"
}
$buildcommandparams = "-m 4096M --build-arg BASE_IMAGE=$($BaseTable[$kernelver]) --build-arg DD_TARGET_ARCH=$Arch --build-arg WINDOWS_VERSION=$kernelver -t $Tag --file .\windows\Dockerfile ."
$arglist += "build"

foreach ($h in $SoftwareTable.GetEnumerator()){
if( -not ($($h.Key) -like "*SHA256")){
$arglist += "--build-arg"
$arglist += "$($h.Key)=$($h.Value)"
}
}

if( -not $Cache) {
$buildcommandparams = "--no-cache $buildcommandparams"
$arglist += "--no-cache"
}
$buildcommand = "build $buildcommandparams"

$arglist += -split "-m 4096M --build-arg BASE_IMAGE=$($BaseTable[$kernelver]) --build-arg DD_TARGET_ARCH=$Arch --build-arg WINDOWS_VERSION=$kernelver -t $Tag --file .\windows\Dockerfile ."
# Write-Host -ForegroundColor Green "Building with the following command:"
# Write-Host -ForegroundColor Green "$buildcommand `n"
filter timestamp {"$(Get-Date -Format o): $_"}
& docker $buildcommand.split() | timestamp
& docker $arglist | timestamp

202 changes: 29 additions & 173 deletions windows/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,44 +23,19 @@ ENV WINDOWS_VERSION=${WINDOWS_VERSION:-1809}
ARG DD_TARGET_ARCH
ENV TARGET_ARCH=${DD_TARGET_ARCH:-x64}

ENV GIT_VERSION "2.26.2"
ENV GIT_SHA256 "2dfbb1c46547c70179442a92b8593d592292b8bce2fd02ac4e0051a8072dde8f"
ENV SEVENZIP_VERSION "19.0.0"
ENV SEVENZIP_SHA256 "0f5d4dbbe5e55b7aa31b91e5925ed901fdf46a367491d81381846f05ad54c45e"
ENV VS2017BUILDTOOLS_VERSION "16.8.3.0"
ENV VS2017BUILDTOOLS_SHA256 "ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0"
#ENV VS2017BUILDTOOLS_DOWNLOAD_URL "https://download.visualstudio.microsoft.com/download/pr/d0eac510-174b-4241-b73b-93dc7cc1fbf7/9822b4c851e14d9658babd1533f66f518c6169196e985fe5713b2774128832ae/vs_BuildTools.exe"
ENV VS2017BUILDTOOLS_DOWNLOAD_URL "https://download.visualstudio.microsoft.com/download/pr/9b3476ff-6d0a-4ff8-956d-270147f21cd4/ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0/vs_BuildTools.exe"
ENV GO_VERSION "1.19.5"
ENV GO_SHA256 "167db91a2e40aeb453d3e59d213ecab06f62e1c4a84d13a06ccda1d999961caa"
ENV RUBY_VERSION "2.6.6-1"
ENV RUBY_SHA256 "fbdf77a3e1fa36e25cf0af1303ac76f67dec7a6f739a829784a299702cad1492"
ENV IBM_MQ_VERSION "9.2.4.0"
ENV IBM_MQ_SHA256 "9cea7e1693d051437e78468fd5e915b7b7ed2baf36cdae4936bcf2b760f55daa"
ENV PYTHON_VERSION "3.8.2"
ENV PYTHON_SHA256 "8e400e3f32cdcb746e62e0db4d3ae4cba1f927141ebc4d0d5a4006b0daee8921"
ENV WIX_VERSION "3.11.2"
ENV WIX_SHA256 "32bb76c478fcb356671d4aaf006ad81ca93eea32c22a9401b168fc7471feccd2"
ENV CMAKE_VERSION "3.23.0"
ENV CMAKE_SHA256 "1e772025844f1cc648d28f42090038e5ca5cf72e2889de26d8d05ee25da17061"
ENV MSYS_VERSION "20210725"
ENV MSYS_SHA256 "4013a9d5e51b448343efc24fc6a324cc999bb96b4c01b13a6bd3c661bb5c8a82"
ENV NUGET_VERSION "5.8.0"
ENV NUGET_SHA256 "5c5b9c96165d3283b2cb9e5b65825d343e0e7139b9e70a250b4bb24c2285f3ba"
ENV WINGET_VERSION "1.0.4.0"
ENV WINGET_SHA256 "8a4b727a8ad588c86603136a5d48093b59ed7491f7c5dc9f13725da0900fe6ab"
ENV EMBEDDED_PYTHON_2_VERSION "2.7.17"
ENV EMBEDDED_PYTHON_2_SHA256 "557ea6690c5927360656c003d3114b73adbd755b712a2911975dde813d6d7afb"
ENV EMBEDDED_PYTHON_3_VERSION "3.8.1"
ENV EMBEDDED_PYTHON_3_SHA256 "58563ca60891025923572107e02b8f07439928eb5222dd10466cc92089072c2a"
ENV EMBEDDED_PIP_VERSION "20.3.4"
ENV CODEQL_VERSION "2.10.3"
ENV CODEQL_HASH "46f64e21c74f41210ea3f2c433d1dc622e3eb0690b42373a73fba82122b929a1"
ENV NINJA_VERSION "1.11.0"
ENV NINJA_SHA256 "d0ee3da143211aa447e750085876c9b9d7bcdd637ab5b2c5b41349c617f22f3b"

ENV CACERTS_VERSION "2023-01-10"
ENV CACERTS_HASH "fb1ecd641d0a02c01bc9036d513cb658bbda62a75e246bedbc01764560a639f0"
ARG GIT_VERSION
ARG SEVENZIP_VERSION
ARG VS2017BUILDTOOLS_VERSION
ARG GO_VERSION
ARG RUBY_VERSION
ARG WIX_VERSION
ARG CMAKE_VERSION
ARG MSYS_VERSION
ARG PYTHON_VERSION
ARG EMBEDDED_PYTHON_2_VERSION
ARG EMBEDDED_PYTHON_3_VERSION
ARG NUGET_VERSION
ARG CODEQL_VERSION

LABEL target_agent="Agent 6/7"
LABEL target_arch=${TARGET_ARCH}
Expand All @@ -78,147 +53,28 @@ LABEL embedded_py2_version=${EMBEDDED_PYTHON_2_VERSION}
LABEL embedded_py3_version=${EMBEDDED_PYTHON_3_VERSION}
LABEL nuget_version=${NUGET_VERSION}
LABEL codeql_version=${CODEQL_VERSION}
# Add certificates needed for build & check certificates file hash
# We need to trust the DigiCert High Assurance EV Root CA certificate, which signs python.org,
# to be able to download some Python components during the Agent build.
RUN (New-Object System.Net.WebClient).DownloadFile(\"https://curl.haxx.se/ca/cacert-${ENV:CACERTS_VERSION}.pem\", \"cacert.pem\")
RUN if ((Get-FileHash -Algorithm SHA256 .\cacert.pem).Hash -ne \"$ENV:CACERTS_HASH\") { Write-Host \"Wrong hashsum for cacert.pem: got '$((Get-FileHash -Algorithm SHA256 .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'.\"; exit 1 }
RUN setx SSL_CERT_FILE \"C:\cacert.pem\"

### Preliminary step: we need both the .NET 3.5 runtime and
### the .NET 4.8 runtime. To do this, we get 4.8 from a base image and we
### manually the install .NET Framework 3.5 runtime using the instructions in
### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile:
### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile

### The .NET Fx 3.5 is needed for the Visual C++ Compiler for Python 2.7
### (https://www.microsoft.com/en-us/download/details.aspx?id=44266)
### and to work around a bug in the WiX 3.11 installer
### (https://github.com/wixtoolset/issues/issues/5661).

# Install .NET Fx 3.5
COPY ./windows/install_net35.ps1 install_net35.ps1
RUN Powershell -C .\install_net35.ps1

### End of preliminary step

# Install 7-zip
Copy ./windows/install_7zip.ps1 install_7zip.ps1
RUN powershell -Command .\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256

# Install git
COPY ./windows/install_mingit.ps1 install_mingit.ps1
RUN powershell -Command .\install_mingit.ps1 -Version $ENV:GIT_VERSION -Sha256 $ENV:GIT_SHA256

### HACK: we disable symbolic links when cloning repositories
### to work around a symlink-related failure in the agent-binaries omnibus project
### when copying the datadog-agent project twice.
RUN git config --system core.symlinks false

# Install VS2017
COPY ./windows/install_vstudio.ps1 install_vstudio.ps1
RUN powershell -Command .\install_vstudio.ps1 -Version $ENV:VS2017BUILDTOOLS_VERSION -Sha256 $ENV:VS2017BUILDTOOLS_SHA256 $ENV:VS2017BUILDTOOLS_DOWNLOAD_URL

# If x64, install the WDK for driver development
COPY ./windows/install_wdk.ps1 install_wdk.ps1
RUN if ($Env:TARGET_ARCH -eq 'x64') { powershell -Command .\install_wdk.ps1 }

# Install Wix and update PATH to include it
COPY ./windows/install_wix.ps1 install_wix.ps1
RUN powershell -Command .\install_wix.ps1 -Version $ENV:WIX_VERSION -Sha256 $ENV:WIX_SHA256

# Install dotnet core
COPY ./windows/install_dotnetcore.ps1 install_dotnetcore.ps1
RUN powershell -Command .\install_dotnetcore.ps1

# install nuget
COPY ./windows/install_nuget.ps1 install_nuget.ps1
RUN powershell -Command .\install_nuget.ps1 -Version $ENV:NUGET_VERSION -Sha256 $ENV:NUGET_SHA256

# Install VC compiler for Python 2.7
COPY ./windows/install_vcpython.ps1 install_vcpython.ps1
RUN powershell -Command .\install_vcpython.ps1

# Install IBM MQ
COPY ./windows/install_ibm_mq.ps1 install_ibm_mq.ps1
RUN Powershell -C .\install_ibm_mq.ps1 -Version $ENV:IBM_MQ_VERSION -Sha256 $ENV:IBM_MQ_SHA256
RUN setx MQ_FILE_PATH c:\ibm_mq

# Install Cmake and update PATH to include it
COPY ./windows/install_cmake.ps1 install_cmake.ps1
RUN powershell -Command .\install_cmake.ps1 -Version $ENV:CMAKE_VERSION -Sha256 $ENV:CMAKE_SHA256

# Install Winget and update PATH to include it
COPY ./windows/install_winget.ps1 install_winget.ps1
RUN powershell -Command .\install_winget.ps1 -Version $ENV:WINGET_VERSION -Sha256 $ENV:WINGET_SHA256

# Install golang and set GOPATH to the dev path used in builds & tests
# RUN cinst -y --no-progress golang $ENV:CHOCO_ARCH_FLAG --version $ENV:GO_VERSION
## WORKDIR should automatically create the directory
WORKDIR c:/scripts
COPY ./windows/*.ps1 ./
COPY ./windows/helpers/*.ps1 c:/scripts/helpers/

# Some go point releases are not available as chocolatey packages, we have to install them directly
COPY ./windows/install_go.ps1 install_go.ps1
RUN powershell -C .\install_go.ps1

RUN setx GOPATH C:\dev\go

# Install system Python 3 (to use invoke).
# We always install the 64 bit version because vcredist140 won't work otherwise
COPY ./windows/install_python.ps1 install_python.ps1
COPY ./requirements.txt ./requirements-py2.txt /
RUN powershell -C .\install_python.ps1 -Version $ENV:PYTHON_VERSION -Sha256 $ENV:PYTHON_SHA256

# Install 64-bit ruby and bundler (for omnibus builds)
COPY ./windows/install_ruby.ps1 install_ruby.ps1
RUN powershell -C .\install_ruby.ps1 -Version $ENV:RUBY_VERSION -Sha256 $ENV:RUBY_SHA256

# Install msys2 system & install 64-bit C/C++ compilation toolchain
copy ./windows/install_msys.ps1 install_msys.ps1
RUN powershell -C .\install_msys.ps1 -Version $ENV:MSYS_VERSION -Sha256 $ENV:MSYS_SHA256

RUN ridk install 3

# Downgrade gcc and binutils due to https://github.com/golang/go/issues/46099
RUN (New-Object System.Net.WebClient).DownloadFile(\"https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst\", \"C:/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst\")
RUN (New-Object System.Net.WebClient).DownloadFile(\"https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst\", \"C:/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst\")
RUN (New-Object System.Net.WebClient).DownloadFile(\"https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst\", \"C:/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst\")
RUN C:\tools\msys64\msys2_shell.cmd -defterm -no-start -c \"pacman --noconfirm -U /c/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst\"

# Install docker, manifest-tool and notary
COPY ./windows/install_docker.ps1 install_docker.ps1
RUN powershell -Command .\install_docker.ps1

# Install google cloud SDK
COPY ./windows/install_gcloud_sdk.ps1 install_gcloud_sdk.ps1
RUN powershell -C .\install_gcloud_sdk.ps1

# Install embedded pythons (for unit testing)
COPY ./python-packages-versions.txt python-packages-versions.txt
COPY ./windows/install_embedded_pythons.ps1 install_embedded_pythons.ps1
RUN powershell -C .\install_embedded_pythons.ps1

# install vcpkg
COPY ./windows/install_vcpkg.ps1 install_vcpkg.ps1
RUN powershell -Command .\install_vcpkg.ps1

# Install CodeQL for CI execution of CodeQL
COPY ./windows/install_codeql.ps1 install_codeql.ps1
RUN powershell -C .\install_codeql.ps1

# install ninja
COPY ./windows/install_ninja.ps1 install_ninja.ps1
RUN powershell -C .\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256

# Add signtool to path
RUN [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\", [System.EnvironmentVariableTarget]::Machine)

# Set 32-bit flag env var
RUN if ($Env:TARGET_ARCH -eq 'x86') { setx WINDOWS_BUILD_32_BIT 1 }

COPY ./python-packages-versions.txt /python-packages-versions.txt
COPY ./windows/set_cpython_compiler.cmd set_cpython_compiler.cmd
RUN .\set_cpython_compiler.cmd

COPY ./windows/helpers/phase1/*.ps1 c:/scripts/helpers/phase1/
RUN .\install-all.ps1 -TargetContainer -Phase 1
COPY ./windows/helpers/phase2/*.ps1 c:/scripts/helpers/phase2/
RUN .\install-all.ps1 -TargetContainer -Phase 2
COPY ./windows/helpers/phase3/*.ps1 c:/scripts/helpers/phase3/
RUN .\install-all.ps1 -TargetContainer -Phase 3
COPY ./windows/helpers/phase4/*.ps1 c:/scripts/helpers/phase4/
RUN .\install-all.ps1 -TargetContainer -Phase 4

COPY ./windows/entrypoint.bat /entrypoint.bat
COPY ./windows/aws_networking.ps1 /aws_networking.ps1
COPY ./windows/helpers/aws_networking.ps1 /aws_networking.ps1


ENTRYPOINT ["/entrypoint.bat"]

Loading

0 comments on commit 8c1c831

Please sign in to comment.