diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ad6e11974b350..202c143c2da56 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -570,6 +570,7 @@ /tasks/sds.py @DataDog/agent-processing-and-routing /tasks/systray.py @DataDog/windows-agent /tasks/winbuildscripts/ @DataDog/windows-agent +/tasks/winbuild.py @DataDog/windows-agent /tasks/windows_resources.py @DataDog/windows-agent /tasks/components.py @DataDog/agent-shared-components /tasks/components_templates @DataDog/agent-shared-components diff --git a/.gitlab/package_build/installer.yml b/.gitlab/package_build/installer.yml index 443b2fe52318b..6ab10c1a19e62 100644 --- a/.gitlab/package_build/installer.yml +++ b/.gitlab/package_build/installer.yml @@ -220,7 +220,7 @@ windows-installer-amd64: -e USE_S3_CACHING="$USE_S3_CACHING" -e API_KEY_ORG2=${API_KEY_ORG2} registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\buildinstaller.bat + powershell -C "c:\mnt\tasks\winbuildscripts\Build-InstallerPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1" after_script: - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' artifacts: diff --git a/.gitlab/package_build/windows.yml b/.gitlab/package_build/windows.yml index 16734f170f1da..cb57ebafa617f 100644 --- a/.gitlab/package_build/windows.yml +++ b/.gitlab/package_build/windows.yml @@ -32,14 +32,13 @@ -e S3_OMNIBUS_CACHE_BUCKET="$S3_OMNIBUS_CACHE_BUCKET" -e USE_S3_CACHING="$USE_S3_CACHING" -e INTEGRATION_WHEELS_CACHE_BUCKET="$INTEGRATION_WHEELS_CACHE_BUCKET" - -e GO_VERSION_CHECK="true" -e BUNDLE_MIRROR__RUBYGEMS__ORG=${BUNDLE_MIRROR__RUBYGEMS__ORG} -e PIP_INDEX_URL=${PIP_INDEX_URL} -e API_KEY_ORG2=${API_KEY_ORG2} -e OMNIBUS_GIT_CACHE_DIR=${Env:TEMP}/${CI_PIPELINE_ID}/omnibus-git-cache -e AGENT_FLAVOR=${AGENT_FLAVOR} registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\buildwin.bat + powershell -C "c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1" - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - get-childitem omnibus\pkg - !reference [.upload_sbom_artifacts_windows] @@ -89,7 +88,7 @@ windows_zip_agent_binaries_x64-a7: needs: ["go_mod_tidy_check", "go_deps"] variables: ARCH: "x64" - OMNIBUS_TARGET: agent_binaries + OMNIBUS_TARGET: agent-binaries before_script: - set RELEASE_VERSION $RELEASE_VERSION_7 script: @@ -122,7 +121,7 @@ windows_zip_agent_binaries_x64-a7: -e PIP_INDEX_URL=${PIP_INDEX_URL} -e API_KEY_ORG2=${API_KEY_ORG2} registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\buildwin.bat + powershell -C "c:\mnt\tasks\winbuildscripts\Build-OmnibusTarget.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1" - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - get-childitem omnibus\pkg - !reference [.upload_sbom_artifacts_windows] diff --git a/Dockerfiles/agent/windows/README.md b/Dockerfiles/agent/windows/README.md index e33eeda9bd4e0..7c966fd0294c2 100644 --- a/Dockerfiles/agent/windows/README.md +++ b/Dockerfiles/agent/windows/README.md @@ -6,7 +6,7 @@ How to build the Agent docker image From the root of the repository, run the following command: ``` -docker run --rm -it -v "${pwd}:c:\mnt" -e OMNIBUS_TARGET=main -e MAJOR_VERSION=7 -e RELEASE_VERSION=nightly -e PY_RUNTIMES=3 datadog/agent-buildimages-windows_x64:1809 c:\mnt\tasks\winbuildscripts\buildwin.bat +docker run --rm -it -v "${pwd}:c:\mnt" -e OMNIBUS_TARGET=main -e MAJOR_VERSION=7 -e RELEASE_VERSION=nightly-a7 -e PY_RUNTIMES=3 datadog/agent-buildimages-windows_x64:1809 powershell -C "c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1" ``` The build artifacts will be in `omnibus\pkg`. diff --git a/docs/dev/agent_omnibus.md b/docs/dev/agent_omnibus.md index 68ff17b6e3724..421303cbfc173 100644 --- a/docs/dev/agent_omnibus.md +++ b/docs/dev/agent_omnibus.md @@ -90,7 +90,7 @@ Start a Powershell prompt and navigate to your local clone of the `datadog-agent Run the following command: ```powershell -docker run -v "$(Get-Location):c:\mnt" -e OMNIBUS_TARGET=main -e RELEASE_VERSION=nightly -e MAJOR_VERSION=7 -e TARGET_ARCH=x64 datadog/agent-buildimages-windows_x64:1809 c:\mnt\tasks\winbuildscripts\buildwin.bat +docker run -v "$(Get-Location):c:\mnt" -e OMNIBUS_TARGET=main -e RELEASE_VERSION=nightly-a7 -e MAJOR_VERSION=7 -e TARGET_ARCH=x64 datadog/agent-buildimages-windows_x64:1809 powershell -C "c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1" ``` Downloading the Docker image may take some time in the first run. @@ -100,7 +100,7 @@ Alternatively here's a small Powershell script to facilitate using the docker im param ( [int]$MAJOR_VERSION=7, $TARGET_ARCH="x64", - $RELEASE_VERSION="nightly", + $RELEASE_VERSION="nightly-a7", [bool]$RM_CONTAINER=$true, [bool]$DEBUG=$false ) @@ -113,7 +113,7 @@ $opts = "-e OMNIBUS_TARGET=main -e RELEASE_VERSION=$RELEASE_VERSION -e MAJOR_VER if ($DEBUG) { $opts += " -e DEBUG_CUSTOMACTION=yes " } -$cmd += " -m 8192M -v ""$(Get-Location):c:\mnt"" $opts datadog/agent-buildimages-windows_x64:1809 c:\mnt\tasks\winbuildscripts\buildwin.bat" +$cmd += " -m 8192M -v ""$(Get-Location):c:\mnt"" $opts datadog/agent-buildimages-windows_x64:1809 powershell -C ""c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1""" Write-Host $cmd Invoke-Expression -Command $cmd ``` diff --git a/tasks/__init__.py b/tasks/__init__.py index 44adcefb1dcad..d59a06de7463b 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -63,6 +63,7 @@ trace_agent, vim, vscode, + winbuild, worktree, ) from tasks.build_tags import audit_tag_impact, print_default_build_tags @@ -212,6 +213,7 @@ ns.add_collection(collector) ns.add_collection(invoke_unit_tests) ns.add_collection(debug) +ns.add_collection(winbuild) ns.add_collection(worktree) ns.configure( { diff --git a/tasks/winbuild.py b/tasks/winbuild.py new file mode 100644 index 0000000000000..66f0cb02b68a1 --- /dev/null +++ b/tasks/winbuild.py @@ -0,0 +1,71 @@ +import os +import shutil + +from invoke.tasks import task + +from tasks.flavor import AgentFlavor +from tasks.libs.common.utils import get_version +from tasks.msi import build as build_agent_msi +from tasks.msi import build_installer as build_installer_msi +from tasks.omnibus import build as omnibus_build + +# Output directory for package files +OUTPUT_PATH = os.path.join(os.getcwd(), "omnibus", "pkg") +# Omnibus stores files here, e.g. C:\opt\datadog-agent, C:\opt\dataog-installer +OPT_SOURCE_DIR = os.path.join('C:\\', 'opt') + + +@task +def agent_package( + ctx, + flavor=AgentFlavor.base.name, + release_version="nightly-a7", + skip_deps=False, +): + # Build agent + omnibus_build( + ctx, + flavor=flavor, + release_version=release_version, + skip_deps=skip_deps, + ) + + # Package Agent into MSI + build_agent_msi(ctx, release_version=release_version) + + # Package MSI into OCI + if AgentFlavor[flavor] == AgentFlavor.base: + ctx.run('powershell -C "./tasks/winbuildscripts/Generate-OCIPackage.ps1 -package datadog-agent"') + + +@task +def installer_package( + ctx, + release_version="nightly-a7", + skip_deps=False, +): + # Build installer + omnibus_build( + ctx, + release_version=release_version, + skip_deps=skip_deps, + target_project="installer", + ) + + # Package Insaller into MSI + build_installer_msi(ctx) + + # Package MSI into OCI + ctx.run('powershell -C "./tasks/winbuildscripts/Generate-OCIPackage.ps1 -package datadog-installer"') + + # Copy installer.exe to the output dir so it can be deployed as the bootstrapper + agent_version = get_version( + ctx, + include_git=True, + url_safe=True, + include_pipeline_id=True, + ) + shutil.copy2( + os.path.join(OPT_SOURCE_DIR, "datadog-installer\\datadog-installer.exe"), + os.path.join(OUTPUT_PATH, f"datadog-installer-{agent_version}-1-x86_64.exe"), + ) diff --git a/tasks/winbuildscripts/Build-AgentPackages.ps1 b/tasks/winbuildscripts/Build-AgentPackages.ps1 new file mode 100644 index 0000000000000..34c2188c8249b --- /dev/null +++ b/tasks/winbuildscripts/Build-AgentPackages.ps1 @@ -0,0 +1,83 @@ +<# +.SYNOPSIS +Builds the Datadog Agent packages for Windows. Builds everything with omnibus and packages the output into MSI, ZIP, and OCI. + +.DESCRIPTION +This script builds the Datadog Agent packages for Windows, with options to configure the build environment. + +.PARAMETER ReleaseVersion +Specifies the release version of the build. Default is the value of the environment variable RELEASE_VERSION. + +.PARAMETER Flavor +Specifies the flavor of the agent. Default is the value of the environment variable AGENT_FLAVOR. + +.PARAMETER BuildOutOfSource +Specifies whether to build out of source. Default is $false. + +Use this option in the CI to keep the job directory clean and avoid conflicts/stale data. +Use this option in Hyper-V based containers to improve build performance. + +.PARAMETER InstallDeps +Specifies whether to install dependencies (python requirements, go deps, etc.). Default is $true. + +.PARAMETER CheckGoVersion +Specifies whether to check the Go version. If not provided, it defaults to the value of the environment variable GO_VERSION_CHECK or $true if the environment variable is not set. + +.EXAMPLE +.\Build-AgentPackages.ps1 -InstallDeps $false + +.EXAMPLE +.\Build-AgentPackages.ps1 -BuildOutOfSource $true -InstallDeps $true -Flavor "fips" -CheckGoVersion $true + +.NOTES +This script should be run from the root of the repository. + +#> +param( + [bool] $BuildOutOfSource = $false, + [nullable[bool]] $CheckGoVersion, + [bool] $InstallDeps = $true, + [string] $ReleaseVersion = $env:RELEASE_VERSION, + [string] $Flavor = $env:AGENT_FLAVOR +) + +. "$PSScriptRoot\common.ps1" + +Invoke-BuildScript ` + -BuildOutOfSource $BuildOutOfSource ` + -InstallDeps $InstallDeps ` + -CheckGoVersion $CheckGoVersion ` + -Command { + $inv_args = @( + "--skip-deps" + ) + if ($ReleaseVersion) { + $inv_args += "--release-version" + $inv_args += $ReleaseVersion + $env:RELEASE_VERSION=$ReleaseVersion + } + + if ($Flavor) { + $inv_args += "--flavor" + $inv_args += $Flavor + $env:AGENT_FLAVOR=$Flavor + } + + Write-Host "inv -e winbuild.agent-package $inv_args" + inv -e winbuild.agent-package @inv_args + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to build the agent package" + exit 1 + } + + # Show the contents of the output package directories for debugging purposes + Get-ChildItem -Path C:\omnibus-ruby\pkg\ + Get-ChildItem -Path "C:\opt\datadog-agent\bin\agent\" + Get-ChildItem -Path ".\omnibus\pkg\" + + if ($BuildOutOfSource) { + # Copy the resulting package to the mnt directory + mkdir C:\mnt\omnibus\pkg -Force -ErrorAction Stop | Out-Null + Copy-Item -Path ".\omnibus\pkg\*" -Destination "C:\mnt\omnibus\pkg" -Force -ErrorAction Stop + } +} diff --git a/tasks/winbuildscripts/Build-InstallerPackages.ps1 b/tasks/winbuildscripts/Build-InstallerPackages.ps1 new file mode 100644 index 0000000000000..bf381a8ef5f8a --- /dev/null +++ b/tasks/winbuildscripts/Build-InstallerPackages.ps1 @@ -0,0 +1,72 @@ +<# +.SYNOPSIS +Builds the Datadog Installer packages for Windows. Builds everything with omnibus and packages the output into MSI, ZIP, and OCI. + +.DESCRIPTION +This script builds the Datadog Installer packages for Windows, with options to configure the build environment. + +.PARAMETER BuildOutOfSource +Specifies whether to build out of source. Default is $false. + +Use this option in the CI to keep the job directory clean and avoid conflicts/stale data. +Use this option in Hyper-V based containers to improve build performance. + +.PARAMETER InstallDeps +Specifies whether to install dependencies (python requirements, go deps, etc.). Default is $true. + +.PARAMETER ReleaseVersion +Specifies the release version of the build. Default is the value of the environment variable RELEASE_VERSION. + +.PARAMETER CheckGoVersion +Specifies whether to check the Go version. If not provided, it defaults to the value of the environment variable GO_VERSION_CHECK or $true if the environment variable is not set. + +.EXAMPLE +.\Build-InstallerPackages.ps1 -InstallDeps $false + +.EXAMPLE +.\Build-InstallerPackages.ps1 -BuildOutOfSource $true -InstallDeps $true -CheckGoVersion $true + +.NOTES +This script should be run from the root of the repository. + +#> +param( + [bool] $BuildOutOfSource = $false, + [nullable[bool]] $CheckGoVersion, + [bool] $InstallDeps = $true, + [string] $ReleaseVersion = $env:RELEASE_VERSION +) + +. "$PSScriptRoot\common.ps1" + +Invoke-BuildScript ` + -BuildOutOfSource $BuildOutOfSource ` + -InstallDeps $InstallDeps ` + -CheckGoVersion $CheckGoVersion ` + -Command { + $inv_args = @( + "--skip-deps" + ) + if ($ReleaseVersion) { + $inv_args += "--release-version" + $inv_args += $ReleaseVersion + } + + Write-Host "inv -e winbuild.installer-package $inv_args" + inv -e winbuild.installer-package @inv_args + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to build the agent package" + exit 1 + } + + # Show the contents of the output package directories for debugging purposes + Get-ChildItem -Path C:\omnibus-ruby\pkg\ + Get-ChildItem -Path C:\opt\datadog-installer + Get-ChildItem -Path ".\omnibus\pkg\" + + if ($BuildOutOfSource) { + # Copy the resulting package to the mnt directory + mkdir C:\mnt\omnibus\pkg -Force -ErrorAction Stop | Out-Null + Copy-Item -Path ".\omnibus\pkg\*" -Destination "C:\mnt\omnibus\pkg" -Force -ErrorAction Stop + } +} diff --git a/tasks/winbuildscripts/Build-OmnibusTarget.ps1 b/tasks/winbuildscripts/Build-OmnibusTarget.ps1 new file mode 100644 index 0000000000000..50dfc63f30e23 --- /dev/null +++ b/tasks/winbuildscripts/Build-OmnibusTarget.ps1 @@ -0,0 +1,90 @@ +<# +.SYNOPSIS +Builds an Omnibus project for Windows. + +.DESCRIPTION +This script builds an Omnibus Project for Windows, with options to configure the build environment. + +.PARAMETER ReleaseVersion +Specifies the release version of the build. Default is the value of the environment variable RELEASE_VERSION. + +.PARAMETER Flavor +Specifies the flavor of the agent. Default is the value of the environment variable AGENT_FLAVOR. + +.PARAMETER BuildOutOfSource +Specifies whether to build out of source. Default is $false. + +Use this option in the CI to keep the job directory clean and avoid conflicts/stale data. +Use this option in Hyper-V based containers to improve build performance. + +.PARAMETER InstallDeps +Specifies whether to install dependencies (python requirements, go deps, etc.). Default is $true. + +.PARAMETER CheckGoVersion +Specifies whether to check the Go version. If not provided, it defaults to the value of the environment variable GO_VERSION_CHECK or $true if the environment variable is not set. + +.PARAMETER TargetProject +Specifies the target project for the build. This parameter is mandatory and defaults to the value of the environment variable OMNIBUS_TARGET. + +.EXAMPLE +.\Build-OmnibusTarget.ps1 -InstallDeps $false + +.EXAMPLE +.\Build-OmnibusTarget.ps1 -BuildOutOfSource $true -InstallDeps $true -Flavor "fips" -CheckGoVersion $true + +.NOTES +This script should be run from the root of the repository. + +#> +param( + [bool] $BuildOutOfSource = $false, + [nullable[bool]] $CheckGoVersion, + [bool] $InstallDeps = $true, + [string] $ReleaseVersion = $env:RELEASE_VERSION, + [string] $TargetProject = $env:OMNIBUS_TARGET +) + +. "$PSScriptRoot\common.ps1" + +Invoke-BuildScript ` + -BuildOutOfSource $BuildOutOfSource ` + -InstallDeps $InstallDeps ` + -CheckGoVersion $CheckGoVersion ` + -Command { + $inv_args = @( + "--skip-deps" + ) + if ($ReleaseVersion) { + $inv_args += "--release-version" + $inv_args += $ReleaseVersion + $env:RELEASE_VERSION=$ReleaseVersion + } + + if ($TargetProject) { + $inv_args += "--target-project" + $inv_args += $TargetProject + $env:OMNIBUS_TARGET=$TargetProject + } else { + Write-Error "Target project is required" + Write-Error "To build the (default) Agent package, use Build-AgentPackages.ps1 instead" + exit 1 + } + + Write-Host "inv -e omnibus.build $inv_args" + inv -e omnibus.build @inv_args + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to build the agent package" + exit 1 + } + + # Show the contents of the output package directories for debugging purposes + Get-ChildItem -Path C:\omnibus-ruby\pkg\ + Get-ChildItem -Path "C:\opt\datadog-agent\bin\agent\" + Get-ChildItem -Path ".\omnibus\pkg\" + + if ($BuildOutOfSource) { + # Copy the resulting package to the mnt directory + mkdir C:\mnt\omnibus\pkg -Force -ErrorAction Stop | Out-Null + Copy-Item -Path ".\omnibus\pkg\*" -Destination "C:\mnt\omnibus\pkg" -Force -ErrorAction Stop + } +} diff --git a/tasks/winbuildscripts/Generate-OCIPackage.ps1 b/tasks/winbuildscripts/Generate-OCIPackage.ps1 index ee48f4badc9fe..8b64223788b3e 100644 --- a/tasks/winbuildscripts/Generate-OCIPackage.ps1 +++ b/tasks/winbuildscripts/Generate-OCIPackage.ps1 @@ -1,11 +1,10 @@ Param( [Parameter(Mandatory=$true)] [string] $package, - [string] $version + [string] $version, + [string] $omnibusOutput = "$(Get-Location)\omnibus\pkg\" ) -$omnibusOutput = "$($Env:REPO_ROOT)\omnibus\pkg\" - if (-not (Test-Path C:\tools\datadog-package.exe)) { Write-Host "Downloading datadog-package.exe" (New-Object System.Net.WebClient).DownloadFile("https://dd-agent-omnibus.s3.amazonaws.com/datadog-package.exe", "C:\\tools\\datadog-package.exe") @@ -26,10 +25,14 @@ if (Test-Path $omnibusOutput\$packageName) { # datadog-package takes a folder as input and will package everything in that, so copy the msi to its own folder Remove-Item -Recurse -Force C:\oci-pkg -ErrorAction SilentlyContinue -New-Item -ItemType Directory C:\oci-pkg +New-Item -ItemType Directory C:\oci-pkg | Out-Null Copy-Item (Get-ChildItem $omnibusOutput\${package}-${version}-x86_64.msi).FullName -Destination C:\oci-pkg\${package}-${version}-x86_64.msi # The argument --archive-path ".\omnibus\pkg\datadog-agent-${version}.tar.gz" is currently broken and has no effects & C:\tools\datadog-package.exe create --package $package --os windows --arch amd64 --archive --version $version C:\oci-pkg +if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to create OCI package" + exit 1 +} Move-Item ${package}-${version}-windows-amd64.tar $omnibusOutput\$packageName diff --git a/tasks/winbuildscripts/buildinstaller.bat b/tasks/winbuildscripts/buildinstaller.bat deleted file mode 100644 index bd41ccad5dc2a..0000000000000 --- a/tasks/winbuildscripts/buildinstaller.bat +++ /dev/null @@ -1,59 +0,0 @@ -if not exist c:\mnt\ goto nomntdir -@echo on -@echo c:\mnt found, continuing -@echo PARAMS %* -@echo RELEASE_VERSION %RELEASE_VERSION% - -set BUILD_ROOT=c:\buildroot -set REPO_ROOT=%BUILD_ROOT%\datadog-agent -mkdir %REPO_ROOT% -if not exist %REPO_ROOT% exit /b 2 -cd %REPO_ROOT% || exit /b 3 -xcopy /e/s/h/q c:\mnt\*.* || exit /b 4 - -call %BUILD_ROOT%\tasks\winbuildscripts\extract-modcache.bat %REPO_ROOT% modcache - -set OMNIBUS_BUILD=omnibus.build -@rem OMNIBUS_TARGET is also used in the C# code to only produce the .cmd for the Datadog Installer (instead of for both the Agent installer and the Datadog installer). -@rem It's not strictly needed, as we will only invoke the .cmd for the Datadog Installer in the invoke task build-installer, but it's a good practice to be consistent. -set OMNIBUS_TARGET=installer -set OMNIBUS_ARGS=%OMNIBUS_ARGS% --target-project %OMNIBUS_TARGET% -@rem Have to use arcane syntax to store AGENT_VERSION, see https://ss64.com/nt/for_cmd.html -FOR /F "tokens=*" %%g IN ('inv agent.version --url-safe --major-version 7') do (SET AGENT_VERSION=%%g) - -if DEFINED GOMODCACHE set OMNIBUS_ARGS=%OMNIBUS_ARGS% --go-mod-cache %GOMODCACHE% -if DEFINED USE_S3_CACHING set OMNIBUS_ARGS=%OMNIBUS_ARGS% %USE_S3_CACHING% - -SET PATH=%PATH%;%GOPATH%/bin -REM AGENT_MSI_OUTDIR is always overridden in msi.py - -@echo GOPATH %GOPATH% -@echo PATH %PATH% -@echo VSTUDIO_ROOT %VSTUDIO_ROOT% - -call ridk enable -pip3 install -r requirements.txt - -@echo "inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --release-version %RELEASE_VERSION%" -inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --release-version %RELEASE_VERSION% || exit /b 1 -inv -e msi.build-installer || exit /b 2 - -Powershell -C "./tasks/winbuildscripts/Generate-OCIPackage.ps1 -package 'datadog-installer'" - -REM show output package directories (for debugging) -dir \omnibus-ruby\pkg\ -dir C:\opt\datadog-installer\ -dir %REPO_ROOT%\omnibus\pkg\ - -REM copy resulting packages to expected location for collection by gitlab. -if not exist c:\mnt\omnibus\pkg\ mkdir c:\mnt\omnibus\pkg\ || exit /b 5 -copy %REPO_ROOT%\omnibus\pkg\* c:\mnt\omnibus\pkg\ || exit /b 6 -REM Save the installer.exe for bootstrapping -copy C:\opt\datadog-installer\datadog-installer.exe c:\mnt\omnibus\pkg\datadog-installer-%AGENT_VERSION%-1-x86_64.exe || exit /b 7 - -goto :EOF - -:nomntdir -@echo directory not mounted, parameters incorrect -exit /b 1 -goto :EOF diff --git a/tasks/winbuildscripts/buildlocal.bat b/tasks/winbuildscripts/buildlocal.bat deleted file mode 100644 index c2aaab3b4bfa8..0000000000000 --- a/tasks/winbuildscripts/buildlocal.bat +++ /dev/null @@ -1,14 +0,0 @@ -@echo RELEASE_VERSION %RELEASE_VERSION% -@echo MAJOR_VERSION %MAJOR_VERSION% - -REM set up variables for local build. -REM assumes attempting to build A7/x64 nightly -REM assumes target directory is mounted in the container -REM (vs. copied in as in CI build) -if NOT DEFINED RELEASE_VERSION set RELEASE_VERSION=nightly -if NOT DEFINED MAJOR_VERSION set MAJOR_VERSION=7 -if NOT DEFINED CI_JOB_ID set CI_JOB_ID=1 -if NOT DEFINED TARGET_ARCH set TARGET_ARCH=x64 - -call %~dp0dobuild.bat -if not %ERRORLEVEL% == 0 @echo "Build failed %ERRORLEVEL%" diff --git a/tasks/winbuildscripts/buildwin.bat b/tasks/winbuildscripts/buildwin.bat deleted file mode 100644 index e23c72d579db4..0000000000000 --- a/tasks/winbuildscripts/buildwin.bat +++ /dev/null @@ -1,39 +0,0 @@ -if not exist c:\mnt\ goto nomntdir -@echo on -@echo c:\mnt found, continuing - -set BUILD_ROOT=c:\buildroot -mkdir %BUILD_ROOT%\datadog-agent -if not exist %BUILD_ROOT%\datadog-agent exit /b 2 -cd %BUILD_ROOT%\datadog-agent || exit /b 3 -xcopy /e/s/h/q c:\mnt\*.* || exit /b 4 - -call %BUILD_ROOT%\datadog-agent\tasks\winbuildscripts\extract-modcache.bat %BUILD_ROOT%\datadog-agent modcache - -REM -REM after copying files in from the host, execute the build -REM using `dobuild.bat` -REM -call %BUILD_ROOT%\datadog-agent\tasks\winbuildscripts\dobuild.bat %* -if not %ERRORLEVEL% == 0 exit /b %ERRORLEVEL% - -REM show output package directories (for debugging) -dir \omnibus-ruby\pkg\ - -dir %BUILD_ROOT%\datadog-agent\omnibus\pkg\ - -REM copy resulting packages to expected location for collection by gitlab. -if not exist c:\mnt\omnibus\pkg\ mkdir c:\mnt\omnibus\pkg\ || exit /b 5 -copy %BUILD_ROOT%\datadog-agent\omnibus\pkg\* c:\mnt\omnibus\pkg\ || exit /b 6 - -REM show output binary directories (for debugging) -dir C:\opt\datadog-agent\bin\agent\ - -goto :EOF - -:nomntdir -@echo directory not mounted, parameters incorrect -exit /b 1 -goto :EOF - - diff --git a/tasks/winbuildscripts/common.ps1 b/tasks/winbuildscripts/common.ps1 new file mode 100644 index 0000000000000..35ef863d57535 --- /dev/null +++ b/tasks/winbuildscripts/common.ps1 @@ -0,0 +1,242 @@ +<# +.SYNOPSIS +Copies files from C:\mnt into C:\buildroot\datadog-agent and sets the current directory to the buildroot. + +.PARAMETER buildroot +Specifies the root directory where the files will be copied to. The default value is "c:\buildroot". + +.NOTES +This function is typically used in the context of building and running scripts within a containerized environment to +ensure that the files are copied to the container filesystem before running the build scripts. This is useful for +keeping the source clean and can provide speed improvements for hyper-v based containers. +See also, issue with job cross-contamination due to improperly cancelled jobs: https://datadoghq.atlassian.net/browse/CIEXE-143 +#> +function Enter-BuildRoot() { + param( + [string] $buildroot = "c:\buildroot" + ) + if (-Not (Test-Path -Path "c:\mnt")) { + Write-Error "C:\mnt directory not mounted, parameters incorrect" + exit 1 + } + + # copy to buildroot + mkdir -Force "$buildroot\datadog-agent" -ErrorAction Stop | Out-Null + if (-Not (Test-Path -Path "$buildroot\datadog-agent")) { + Write-Error "Failed to create buildroot directory" + exit 2 + } + + # copy the repository into the container filesystem + Write-Host "Switching to buildroot $buildroot\datadog-agent" + Push-Location "$buildroot\datadog-agent" -ErrorAction Stop -StackName AgentBuildRoot + xcopy /e/s/h/q c:\mnt\*.* +} + +<# +.SYNOPSIS +Leaves the buildroot directory and returns to the original working directory. +#> +function Exit-BuildRoot() { + Write-Host "Leaving buildroot" + Pop-Location -StackName AgentBuildRoot +} + +<# +.SYNOPSIS +Expands the Go module cache from an archive file. + +.DESCRIPTION +This function expands the Go module cache from an archive file located in the specified root directory. +It extracts the contents of the archive file into the Go module cache directory defined by the GOMODCACHE environment variable. + +.PARAMETER root +The root directory where the tar.xz file is located. Defaults to the current location. + +.PARAMETER modcache +The base name (without extension) of the file to be expanded. Expected values are `modcache` and `modcache_tools`. + +.NOTES +If the GOMODCACHE environment variable is not set, the function will skip the expansion process. + +.EXAMPLE +Expand-ModCache -modcache "modcache" +This will expand the modcache file located at "\modcache.tar.xz" into the Go module cache directory. + +#> +function Expand-ModCache() { + param( + [string] $root = (Get-Location).Path, + [ValidateSet('modcache', 'modcache_tools')] + [string] $modcache + ) + + $MODCACHE_ROOT = $root + $MODCACHE_FILE_ROOT = $modcache + $MODCACHE_XZ_FILE = Join-Path $MODCACHE_ROOT "$MODCACHE_FILE_ROOT.tar.xz" + $MODCACHE_TAR_FILE = Join-Path $MODCACHE_ROOT "$MODCACHE_FILE_ROOT.tar" + + if (-not $env:GOMODCACHE) { + Write-Host "GOMODCACHE environment variable not set, skipping expansion of mod cache files" + return + } + + Write-Host "MODCACHE_XZ_FILE $MODCACHE_XZ_FILE MODCACHE_TAR_FILE $MODCACHE_TAR_FILE GOMODCACHE $env:GOMODCACHE" + if (Test-Path $MODCACHE_XZ_FILE) { + Write-Host "Extracting modcache file $MODCACHE_XZ_FILE" + & 7z.exe x $MODCACHE_XZ_FILE -o"$MODCACHE_ROOT" -bt + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to extract $MODCACHE_XZ_FILE" + exit 1 + } + Get-ChildItem $MODCACHE_TAR_FILE + # Use -aoa to allow overwriting existing files + # This shouldn't have any negative impact: since modules are + # stored per version and hash, files that get replaced will + # get replaced by the same files + & 7z.exe x $MODCACHE_TAR_FILE -o"$env:GOMODCACHE\cache" -aoa -bt + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to extract $MODCACHE_XZ_FILE" + exit 1 + } + Write-Host "Modcache extracted" + } else { + Write-Host "Modcache XZ file $MODCACHE_XZ_FILE not found, dependencies will be downloaded" + } + + if (Test-Path $MODCACHE_XZ_FILE) { + Write-Host "Deleting modcache tar.xz $MODCACHE_XZ_FILE" + Remove-Item -Force $MODCACHE_XZ_FILE + } + if (Test-Path $MODCACHE_TAR_FILE) { + Write-Host "Deleting modcache tar $MODCACHE_TAR_FILE" + Remove-Item -Force $MODCACHE_TAR_FILE + } +} + +function Install-Deps() { + Write-Host "Installing python requirements" + pip3.exe install -r .\requirements.txt + Write-Host "Installing go dependencies" + inv -e deps +} + +function Enable-DevEnv() { + # Add go bin to PATH for golangci-lint and other go tools + if (-Not $env:GOPATH) { + Write-Host "GOPATH not set, setting to C:\dev\go" + $env:GOPATH = "C:\dev\go" + } + $env:PATH = "$env:GOPATH\bin;$env:PATH" + + # Enable ruby/msys environment, for mingw, make, etc. + ridk enable +} + +<# +.SYNOPSIS +Converts a string value to a boolean value based on specific conditions. + +.DESCRIPTION +This function takes a string input and a default boolean value. +- If the input string is null or empty, it returns the default boolean value. +- If the input string is "true", "yes", or "1" (case insensitive), it returns $true. +- Otherwise, it returns $false. +#> +function Convert-StringToBool() { + param( + [string] $Value, + [bool] $DefaultValue + ) + + if ([string]::IsNullOrEmpty($Value)) { + return $DefaultValue + } + + if ($Value.ToLower() -eq "true") { + return $true + } + + if ($Value.ToLower() -eq "yes" -or $Value -eq "1") { + return $true + } + + return $false +} + +<# +.SYNOPSIS +Invokes a build script with optional parameters for build environment configuration. + +.DESCRIPTION +The Invoke-BuildScript function sets up the build environment, optionally installs dependencies, checks the Go version, and executes a provided script block. It supports building out of source and restores the original working directory upon completion. + +.PARAMETER buildroot +Specifies the root directory for the build. Default is "c:\buildroot". + +.PARAMETER BuildOutOfSource +Specifies whether to build out of source. Default is $false. + +Use this option in the CI to keep the job directory clean and avoid conflicts/stale data. +Use this option in Hyper-V based containers to improve build performance. + +.PARAMETER InstallDeps +Specifies whether to install dependencies (python requirements, go deps, etc.). Default is $true. + +.PARAMETER CheckGoVersion +Specifies whether to check the Go version. If not provided, it defaults to the value of the environment variable GO_VERSION_CHECK or $true if the environment variable is not set. + +.PARAMETER Command +A script block containing the commands to execute as part of the build process. + +.EXAMPLE +Invoke-BuildScript -buildroot "c:\mybuild" -BuildOutOfSource $true -Command { ./build.ps1 } + +#> +function Invoke-BuildScript { + param( + [string] $buildroot = "c:\buildroot", + [bool] $BuildOutOfSource = $false, + [bool] $InstallDeps = $true, + [nullable[bool]] $CheckGoVersion, + [ScriptBlock] $Command = {$null} + ) + + try { + if ($null -eq $CheckGoVersion) { + $CheckGoVersion = Convert-StringToBool -Value $env:GO_VERSION_CHECK -default $true + } + + if ($BuildOutOfSource) { + Enter-BuildRoot + } + + Expand-ModCache -modcache modcache + + Enable-DevEnv + + if ($InstallDeps) { + Install-Deps + } + + if ($CheckGoVersion) { + inv -e check-go-version + if ($LASTEXITCODE -ne 0) { + Write-Error "Go version check failed" + exit 1 + } + } + + # Execute the provided ScriptBlock/Command + & $Command + + } finally { + # This finally block is executed regardless of whether the try block completes successfully, throws an exception, + # or uses `exit` to terminate the script. + + if ($BuildOutOfSource) { + # Restore the original working directory + Exit-BuildRoot + } + } +} diff --git a/tasks/winbuildscripts/dobuild.bat b/tasks/winbuildscripts/dobuild.bat deleted file mode 100644 index 0ed92222f1f71..0000000000000 --- a/tasks/winbuildscripts/dobuild.bat +++ /dev/null @@ -1,62 +0,0 @@ -@echo PARAMS %* -@echo RELEASE_VERSION %RELEASE_VERSION% -@echo MAJOR_VERSION %MAJOR_VERSION% -@echo GO_VERSION_CHECK %GO_VERSION_CHECK% - -if NOT DEFINED RELEASE_VERSION set RELEASE_VERSION=%~1 -if NOT DEFINED MAJOR_VERSION set MAJOR_VERSION=%~2 -if NOT DEFINED GO_VERSION_CHECK set GO_VERSION_CHECK=%~4 - -set OMNIBUS_BUILD=omnibus.build - -if "%OMNIBUS_TARGET%" == "" set OMNIBUS_TARGET=main -if "%OMNIBUS_TARGET%" == "iot" set OMNIBUS_ARGS=--flavor iot -if "%OMNIBUS_TARGET%" == "dogstatsd" set OMNIBUS_ARGS=--target-project dogstatsd -if "%OMNIBUS_TARGET%" == "agent_binaries" set OMNIBUS_ARGS=%OMNIBUS_ARGS% --target-project agent-binaries -if "%AGENT_FLAVOR%" == "fips" set OMNIBUS_ARGS=--flavor fips - -if DEFINED GOMODCACHE set OMNIBUS_ARGS=%OMNIBUS_ARGS% --go-mod-cache %GOMODCACHE% -if DEFINED USE_S3_CACHING set OMNIBUS_ARGS=%OMNIBUS_ARGS% %USE_S3_CACHING% - -set REPO_ROOT=%~p0\..\.. -pushd . -cd %REPO_ROOT% || exit /b 100 - -SET PATH=%PATH%;%GOPATH%\bin - -@echo GOPATH %GOPATH% -@echo PATH %PATH% -@echo VSTUDIO_ROOT %VSTUDIO_ROOT% - -REM Section to pre-install libyajl2 gem with fix for gcc10 compatibility -Powershell -C "ridk enable; ./tasks/winbuildscripts/libyajl2_install.ps1" - - -if "%TARGET_ARCH%" == "x64" ( - @echo IN x64 BRANCH - call ridk enable -) - - -pip3 install -r requirements.txt || exit /b 120 - -inv -e deps || exit /b 130 -if "%GO_VERSION_CHECK%" == "true" ( - inv -e check-go-version || exit /b 140 -) - -@echo "inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --release-version %RELEASE_VERSION%" -inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --release-version %RELEASE_VERSION% || exit /b 150 - -REM only build MSI for main targets for now. -if "%OMNIBUS_TARGET%" == "main" ( - @echo "inv -e msi.build --release-version %RELEASE_VERSION% - inv -e msi.build --release-version %RELEASE_VERSION% || exit /b 160 -) - -REM Build the OCI package for the Agent 7 only. -if %MAJOR_VERSION% == 7 ( - Powershell -C "./tasks/winbuildscripts/Generate-OCIPackage.ps1 -package 'datadog-agent'" -) - -popd diff --git a/tasks/winbuildscripts/libyajl2_install.ps1 b/tasks/winbuildscripts/libyajl2_install.ps1 deleted file mode 100644 index a978c004ffcbd..0000000000000 --- a/tasks/winbuildscripts/libyajl2_install.ps1 +++ /dev/null @@ -1,26 +0,0 @@ -pushd . -New-Item -Force -Path c:\tmp -ItemType Directory -cd \tmp - -# Clone the repo; recursive is needed to pick up the submodule inside -git clone --depth 1 --branch 1.2.0 --recursive https://github.com/DataDog/libyajl2-gem libyajl2 -cd libyajl2 - -# Remove existing rake install, we don't need it and its version is too high -gem uninstall -x rake - -# We don't need the development_extras group, we're not running tests -bundle config set --local without 'development_extras' - -# Install dev dependencies - maybe --path should be used to do a local install, but unsure how this will affect the next commands -bundle install - -# Prepare the repo -rake prep -# Install the gem -gem build ./libyajl2.gemspec -gem install ./libyajl2-1.2.0.gem - -# Cleanup -popd -Remove-Item -Recurse -Force c:\tmp\libyajl2 \ No newline at end of file