Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor tools to allow building container and setting up dev machine #276

Merged
merged 16 commits into from
Feb 16, 2023
Merged

Refactor tools to allow building container and setting up dev machine #276

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
dd0d7db
DO NOT MERGE
derekwbrown Sep 2, 2022
88705e0
fix path location (vcpkg)
derekwbrown Sep 2, 2022
33bf25a
temp stop before vcpkg for debugging
derekwbrown Sep 2, 2022
49cf0da
fix path error in networking script
derekwbrown Sep 2, 2022
f16a973
fix vcpkg
derekwbrown Sep 2, 2022
7240b48
fix go entry in path
derekwbrown Sep 2, 2022
f31cf93
make work on local system
derekwbrown Sep 4, 2022
1402d59
fix typo in gsdk
derekwbrown Sep 4, 2022
6dc53bc
debugging statement for vcpkg
derekwbrown Sep 4, 2022
8bb691b
remove ridk enable; fixes calls to cmd
derekwbrown Sep 4, 2022
2dccd55
Working both locally and in container
derekwbrown Sep 5, 2022
fddb9d9
clean up python handling
derekwbrown Sep 5, 2022
09af9a9
Break install-all into phases to allow for some Docker layers (#346)
clarkb7 Feb 1, 2023
dccc332
Apply suggestions from code review
clarkb7 Feb 7, 2023
0d6478e
fix dockerfile syntax
clarkb7 Feb 7, 2023
e91cd34
Fix go SHA256
clarkb7 Feb 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 16 additions & 5 deletions build-container.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ param(
)

$ErrorActionPreference = "Stop"

. .\windows\versions.ps1
$BaseTable = @{
1809 = "mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-ltsc2019";
1909 = "mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-1909";
Expand All @@ -26,15 +26,26 @@ if ($build -ge 20348) {

Write-Host -ForegroundColor Green "Using base image $($BaseTable[$kernelver])"

$arglist = @()
if($Tag -eq $null -or $Tag -eq ""){
$Tag ="builder_$($kernelver)_$Arch"
}
$buildcommandparams = "-m 4096M --build-arg BASE_IMAGE=$($BaseTable[$kernelver]) --build-arg DD_TARGET_ARCH=$Arch --build-arg WINDOWS_VERSION=$kernelver -t $Tag --file .\windows\Dockerfile ."
$arglist += "build"

foreach ($h in $SoftwareTable.GetEnumerator()){
if( -not ($($h.Key) -like "*SHA256")){
$arglist += "--build-arg"
$arglist += "$($h.Key)=$($h.Value)"
}
}

if( -not $Cache) {
$buildcommandparams = "--no-cache $buildcommandparams"
$arglist += "--no-cache"
}
$buildcommand = "build $buildcommandparams"

$arglist += -split "-m 4096M --build-arg BASE_IMAGE=$($BaseTable[$kernelver]) --build-arg DD_TARGET_ARCH=$Arch --build-arg WINDOWS_VERSION=$kernelver -t $Tag --file .\windows\Dockerfile ."
# Write-Host -ForegroundColor Green "Building with the following command:"
# Write-Host -ForegroundColor Green "$buildcommand `n"
filter timestamp {"$(Get-Date -Format o): $_"}
& docker $buildcommand.split() | timestamp
& docker $arglist | timestamp

202 changes: 29 additions & 173 deletions windows/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,44 +23,19 @@ ENV WINDOWS_VERSION=${WINDOWS_VERSION:-1809}
ARG DD_TARGET_ARCH
ENV TARGET_ARCH=${DD_TARGET_ARCH:-x64}

ENV GIT_VERSION "2.26.2"
ENV GIT_SHA256 "2dfbb1c46547c70179442a92b8593d592292b8bce2fd02ac4e0051a8072dde8f"
ENV SEVENZIP_VERSION "19.0.0"
ENV SEVENZIP_SHA256 "0f5d4dbbe5e55b7aa31b91e5925ed901fdf46a367491d81381846f05ad54c45e"
ENV VS2017BUILDTOOLS_VERSION "16.8.3.0"
ENV VS2017BUILDTOOLS_SHA256 "ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0"
#ENV VS2017BUILDTOOLS_DOWNLOAD_URL "https://download.visualstudio.microsoft.com/download/pr/d0eac510-174b-4241-b73b-93dc7cc1fbf7/9822b4c851e14d9658babd1533f66f518c6169196e985fe5713b2774128832ae/vs_BuildTools.exe"
ENV VS2017BUILDTOOLS_DOWNLOAD_URL "https://download.visualstudio.microsoft.com/download/pr/9b3476ff-6d0a-4ff8-956d-270147f21cd4/ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0/vs_BuildTools.exe"
ENV GO_VERSION "1.19.5"
ENV GO_SHA256 "167db91a2e40aeb453d3e59d213ecab06f62e1c4a84d13a06ccda1d999961caa"
ENV RUBY_VERSION "2.6.6-1"
ENV RUBY_SHA256 "fbdf77a3e1fa36e25cf0af1303ac76f67dec7a6f739a829784a299702cad1492"
ENV IBM_MQ_VERSION "9.2.4.0"
ENV IBM_MQ_SHA256 "9cea7e1693d051437e78468fd5e915b7b7ed2baf36cdae4936bcf2b760f55daa"
ENV PYTHON_VERSION "3.8.2"
ENV PYTHON_SHA256 "8e400e3f32cdcb746e62e0db4d3ae4cba1f927141ebc4d0d5a4006b0daee8921"
ENV WIX_VERSION "3.11.2"
ENV WIX_SHA256 "32bb76c478fcb356671d4aaf006ad81ca93eea32c22a9401b168fc7471feccd2"
ENV CMAKE_VERSION "3.23.0"
ENV CMAKE_SHA256 "1e772025844f1cc648d28f42090038e5ca5cf72e2889de26d8d05ee25da17061"
ENV MSYS_VERSION "20210725"
ENV MSYS_SHA256 "4013a9d5e51b448343efc24fc6a324cc999bb96b4c01b13a6bd3c661bb5c8a82"
ENV NUGET_VERSION "5.8.0"
ENV NUGET_SHA256 "5c5b9c96165d3283b2cb9e5b65825d343e0e7139b9e70a250b4bb24c2285f3ba"
ENV WINGET_VERSION "1.0.4.0"
ENV WINGET_SHA256 "8a4b727a8ad588c86603136a5d48093b59ed7491f7c5dc9f13725da0900fe6ab"
ENV EMBEDDED_PYTHON_2_VERSION "2.7.17"
ENV EMBEDDED_PYTHON_2_SHA256 "557ea6690c5927360656c003d3114b73adbd755b712a2911975dde813d6d7afb"
ENV EMBEDDED_PYTHON_3_VERSION "3.8.1"
ENV EMBEDDED_PYTHON_3_SHA256 "58563ca60891025923572107e02b8f07439928eb5222dd10466cc92089072c2a"
ENV EMBEDDED_PIP_VERSION "20.3.4"
ENV CODEQL_VERSION "2.10.3"
ENV CODEQL_HASH "46f64e21c74f41210ea3f2c433d1dc622e3eb0690b42373a73fba82122b929a1"
ENV NINJA_VERSION "1.11.0"
ENV NINJA_SHA256 "d0ee3da143211aa447e750085876c9b9d7bcdd637ab5b2c5b41349c617f22f3b"

ENV CACERTS_VERSION "2023-01-10"
ENV CACERTS_HASH "fb1ecd641d0a02c01bc9036d513cb658bbda62a75e246bedbc01764560a639f0"
ARG GIT_VERSION
ARG SEVENZIP_VERSION
ARG VS2017BUILDTOOLS_VERSION
ARG GO_VERSION
ARG RUBY_VERSION
ARG WIX_VERSION
ARG CMAKE_VERSION
ARG MSYS_VERSION
ARG PYTHON_VERSION
ARG EMBEDDED_PYTHON_2_VERSION
ARG EMBEDDED_PYTHON_3_VERSION
ARG NUGET_VERSION
ARG CODEQL_VERSION

LABEL target_agent="Agent 6/7"
LABEL target_arch=${TARGET_ARCH}
Expand All @@ -78,147 +53,28 @@ LABEL embedded_py2_version=${EMBEDDED_PYTHON_2_VERSION}
LABEL embedded_py3_version=${EMBEDDED_PYTHON_3_VERSION}
LABEL nuget_version=${NUGET_VERSION}
LABEL codeql_version=${CODEQL_VERSION}
# Add certificates needed for build & check certificates file hash
# We need to trust the DigiCert High Assurance EV Root CA certificate, which signs python.org,
# to be able to download some Python components during the Agent build.
RUN (New-Object System.Net.WebClient).DownloadFile(\"https://curl.haxx.se/ca/cacert-${ENV:CACERTS_VERSION}.pem\", \"cacert.pem\")
RUN if ((Get-FileHash -Algorithm SHA256 .\cacert.pem).Hash -ne \"$ENV:CACERTS_HASH\") { Write-Host \"Wrong hashsum for cacert.pem: got '$((Get-FileHash -Algorithm SHA256 .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'.\"; exit 1 }
RUN setx SSL_CERT_FILE \"C:\cacert.pem\"

### Preliminary step: we need both the .NET 3.5 runtime and
### the .NET 4.8 runtime. To do this, we get 4.8 from a base image and we
### manually the install .NET Framework 3.5 runtime using the instructions in
### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile:
### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile

### The .NET Fx 3.5 is needed for the Visual C++ Compiler for Python 2.7
### (https://www.microsoft.com/en-us/download/details.aspx?id=44266)
### and to work around a bug in the WiX 3.11 installer
### (https://github.com/wixtoolset/issues/issues/5661).

# Install .NET Fx 3.5
COPY ./windows/install_net35.ps1 install_net35.ps1
RUN Powershell -C .\install_net35.ps1

### End of preliminary step

# Install 7-zip
Copy ./windows/install_7zip.ps1 install_7zip.ps1
RUN powershell -Command .\install_7zip.ps1 -Version $ENV:SEVENZIP_VERSION -Sha256 $ENV:SEVENZIP_SHA256

# Install git
COPY ./windows/install_mingit.ps1 install_mingit.ps1
RUN powershell -Command .\install_mingit.ps1 -Version $ENV:GIT_VERSION -Sha256 $ENV:GIT_SHA256

### HACK: we disable symbolic links when cloning repositories
### to work around a symlink-related failure in the agent-binaries omnibus project
### when copying the datadog-agent project twice.
RUN git config --system core.symlinks false

# Install VS2017
COPY ./windows/install_vstudio.ps1 install_vstudio.ps1
RUN powershell -Command .\install_vstudio.ps1 -Version $ENV:VS2017BUILDTOOLS_VERSION -Sha256 $ENV:VS2017BUILDTOOLS_SHA256 $ENV:VS2017BUILDTOOLS_DOWNLOAD_URL

# If x64, install the WDK for driver development
COPY ./windows/install_wdk.ps1 install_wdk.ps1
RUN if ($Env:TARGET_ARCH -eq 'x64') { powershell -Command .\install_wdk.ps1 }

# Install Wix and update PATH to include it
COPY ./windows/install_wix.ps1 install_wix.ps1
RUN powershell -Command .\install_wix.ps1 -Version $ENV:WIX_VERSION -Sha256 $ENV:WIX_SHA256

# Install dotnet core
COPY ./windows/install_dotnetcore.ps1 install_dotnetcore.ps1
RUN powershell -Command .\install_dotnetcore.ps1

# install nuget
COPY ./windows/install_nuget.ps1 install_nuget.ps1
RUN powershell -Command .\install_nuget.ps1 -Version $ENV:NUGET_VERSION -Sha256 $ENV:NUGET_SHA256

# Install VC compiler for Python 2.7
COPY ./windows/install_vcpython.ps1 install_vcpython.ps1
RUN powershell -Command .\install_vcpython.ps1

# Install IBM MQ
COPY ./windows/install_ibm_mq.ps1 install_ibm_mq.ps1
RUN Powershell -C .\install_ibm_mq.ps1 -Version $ENV:IBM_MQ_VERSION -Sha256 $ENV:IBM_MQ_SHA256
RUN setx MQ_FILE_PATH c:\ibm_mq

# Install Cmake and update PATH to include it
COPY ./windows/install_cmake.ps1 install_cmake.ps1
RUN powershell -Command .\install_cmake.ps1 -Version $ENV:CMAKE_VERSION -Sha256 $ENV:CMAKE_SHA256

# Install Winget and update PATH to include it
COPY ./windows/install_winget.ps1 install_winget.ps1
RUN powershell -Command .\install_winget.ps1 -Version $ENV:WINGET_VERSION -Sha256 $ENV:WINGET_SHA256

# Install golang and set GOPATH to the dev path used in builds & tests
# RUN cinst -y --no-progress golang $ENV:CHOCO_ARCH_FLAG --version $ENV:GO_VERSION
## WORKDIR should automatically create the directory
WORKDIR c:/scripts
COPY ./windows/*.ps1 ./
COPY ./windows/helpers/*.ps1 c:/scripts/helpers/

# Some go point releases are not available as chocolatey packages, we have to install them directly
COPY ./windows/install_go.ps1 install_go.ps1
RUN powershell -C .\install_go.ps1

RUN setx GOPATH C:\dev\go

# Install system Python 3 (to use invoke).
# We always install the 64 bit version because vcredist140 won't work otherwise
COPY ./windows/install_python.ps1 install_python.ps1
COPY ./requirements.txt ./requirements-py2.txt /
RUN powershell -C .\install_python.ps1 -Version $ENV:PYTHON_VERSION -Sha256 $ENV:PYTHON_SHA256

# Install 64-bit ruby and bundler (for omnibus builds)
COPY ./windows/install_ruby.ps1 install_ruby.ps1
RUN powershell -C .\install_ruby.ps1 -Version $ENV:RUBY_VERSION -Sha256 $ENV:RUBY_SHA256

# Install msys2 system & install 64-bit C/C++ compilation toolchain
copy ./windows/install_msys.ps1 install_msys.ps1
RUN powershell -C .\install_msys.ps1 -Version $ENV:MSYS_VERSION -Sha256 $ENV:MSYS_SHA256

RUN ridk install 3

# Downgrade gcc and binutils due to https://github.com/golang/go/issues/46099
RUN (New-Object System.Net.WebClient).DownloadFile(\"https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst\", \"C:/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst\")
RUN (New-Object System.Net.WebClient).DownloadFile(\"https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst\", \"C:/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst\")
RUN (New-Object System.Net.WebClient).DownloadFile(\"https://s3.amazonaws.com/dd-agent-omnibus/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst\", \"C:/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst\")
RUN C:\tools\msys64\msys2_shell.cmd -defterm -no-start -c \"pacman --noconfirm -U /c/mingw-w64-x86_64-binutils-2.35.1-2-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-libs-10.2.0-11-any.pkg.tar.zst /c/mingw-w64-x86_64-gcc-10.2.0-11-any.pkg.tar.zst\"

# Install docker, manifest-tool and notary
COPY ./windows/install_docker.ps1 install_docker.ps1
RUN powershell -Command .\install_docker.ps1

# Install google cloud SDK
COPY ./windows/install_gcloud_sdk.ps1 install_gcloud_sdk.ps1
RUN powershell -C .\install_gcloud_sdk.ps1

# Install embedded pythons (for unit testing)
COPY ./python-packages-versions.txt python-packages-versions.txt
COPY ./windows/install_embedded_pythons.ps1 install_embedded_pythons.ps1
RUN powershell -C .\install_embedded_pythons.ps1

# install vcpkg
COPY ./windows/install_vcpkg.ps1 install_vcpkg.ps1
RUN powershell -Command .\install_vcpkg.ps1

# Install CodeQL for CI execution of CodeQL
COPY ./windows/install_codeql.ps1 install_codeql.ps1
RUN powershell -C .\install_codeql.ps1

# install ninja
COPY ./windows/install_ninja.ps1 install_ninja.ps1
RUN powershell -C .\install_ninja.ps1 -Version $ENV:NINJA_VERSION -Sha256 $ENV:NINJA_SHA256

# Add signtool to path
RUN [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\", [System.EnvironmentVariableTarget]::Machine)

# Set 32-bit flag env var
RUN if ($Env:TARGET_ARCH -eq 'x86') { setx WINDOWS_BUILD_32_BIT 1 }

COPY ./python-packages-versions.txt /python-packages-versions.txt
COPY ./windows/set_cpython_compiler.cmd set_cpython_compiler.cmd
RUN .\set_cpython_compiler.cmd

COPY ./windows/helpers/phase1/*.ps1 c:/scripts/helpers/phase1/
RUN .\install-all.ps1 -TargetContainer -Phase 1
COPY ./windows/helpers/phase2/*.ps1 c:/scripts/helpers/phase2/
RUN .\install-all.ps1 -TargetContainer -Phase 2
COPY ./windows/helpers/phase3/*.ps1 c:/scripts/helpers/phase3/
RUN .\install-all.ps1 -TargetContainer -Phase 3
COPY ./windows/helpers/phase4/*.ps1 c:/scripts/helpers/phase4/
RUN .\install-all.ps1 -TargetContainer -Phase 4

COPY ./windows/entrypoint.bat /entrypoint.bat
COPY ./windows/aws_networking.ps1 /aws_networking.ps1
COPY ./windows/helpers/aws_networking.ps1 /aws_networking.ps1


ENTRYPOINT ["/entrypoint.bat"]

Loading