Skip to content

Commit

Permalink
Deprecate Windows .bat scripts (DataDog#31732)
Browse files Browse the repository at this point in the history
  • Loading branch information
clarkb7 authored Dec 6, 2024
1 parent 4b5c8b9 commit 858cb1c
Show file tree
Hide file tree
Showing 17 changed files with 576 additions and 213 deletions.
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,7 @@
/tasks/sds.py @DataDog/agent-processing-and-routing
/tasks/systray.py @DataDog/windows-agent
/tasks/winbuildscripts/ @DataDog/windows-agent
/tasks/winbuild.py @DataDog/windows-agent
/tasks/windows_resources.py @DataDog/windows-agent
/tasks/components.py @DataDog/agent-shared-components
/tasks/components_templates @DataDog/agent-shared-components
Expand Down
2 changes: 1 addition & 1 deletion .gitlab/package_build/installer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ windows-installer-amd64:
-e USE_S3_CACHING="$USE_S3_CACHING"
-e API_KEY_ORG2=${API_KEY_ORG2}
registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES}
c:\mnt\tasks\winbuildscripts\buildinstaller.bat
powershell -C "c:\mnt\tasks\winbuildscripts\Build-InstallerPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1"
after_script:
- '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"'
artifacts:
Expand Down
7 changes: 3 additions & 4 deletions .gitlab/package_build/windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,13 @@
-e S3_OMNIBUS_CACHE_BUCKET="$S3_OMNIBUS_CACHE_BUCKET"
-e USE_S3_CACHING="$USE_S3_CACHING"
-e INTEGRATION_WHEELS_CACHE_BUCKET="$INTEGRATION_WHEELS_CACHE_BUCKET"
-e GO_VERSION_CHECK="true"
-e BUNDLE_MIRROR__RUBYGEMS__ORG=${BUNDLE_MIRROR__RUBYGEMS__ORG}
-e PIP_INDEX_URL=${PIP_INDEX_URL}
-e API_KEY_ORG2=${API_KEY_ORG2}
-e OMNIBUS_GIT_CACHE_DIR=${Env:TEMP}/${CI_PIPELINE_ID}/omnibus-git-cache
-e AGENT_FLAVOR=${AGENT_FLAVOR}
registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES}
c:\mnt\tasks\winbuildscripts\buildwin.bat
powershell -C "c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1"
- If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
- get-childitem omnibus\pkg
- !reference [.upload_sbom_artifacts_windows]
Expand Down Expand Up @@ -89,7 +88,7 @@ windows_zip_agent_binaries_x64-a7:
needs: ["go_mod_tidy_check", "go_deps"]
variables:
ARCH: "x64"
OMNIBUS_TARGET: agent_binaries
OMNIBUS_TARGET: agent-binaries
before_script:
- set RELEASE_VERSION $RELEASE_VERSION_7
script:
Expand Down Expand Up @@ -122,7 +121,7 @@ windows_zip_agent_binaries_x64-a7:
-e PIP_INDEX_URL=${PIP_INDEX_URL}
-e API_KEY_ORG2=${API_KEY_ORG2}
registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES}
c:\mnt\tasks\winbuildscripts\buildwin.bat
powershell -C "c:\mnt\tasks\winbuildscripts\Build-OmnibusTarget.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1"
- If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
- get-childitem omnibus\pkg
- !reference [.upload_sbom_artifacts_windows]
Expand Down
2 changes: 1 addition & 1 deletion Dockerfiles/agent/windows/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ How to build the Agent docker image
From the root of the repository, run the following command:

```
docker run --rm -it -v "${pwd}:c:\mnt" -e OMNIBUS_TARGET=main -e MAJOR_VERSION=7 -e RELEASE_VERSION=nightly -e PY_RUNTIMES=3 datadog/agent-buildimages-windows_x64:1809 c:\mnt\tasks\winbuildscripts\buildwin.bat
docker run --rm -it -v "${pwd}:c:\mnt" -e OMNIBUS_TARGET=main -e MAJOR_VERSION=7 -e RELEASE_VERSION=nightly-a7 -e PY_RUNTIMES=3 datadog/agent-buildimages-windows_x64:1809 powershell -C "c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1"
```

The build artifacts will be in `omnibus\pkg`.
Expand Down
6 changes: 3 additions & 3 deletions docs/dev/agent_omnibus.md
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ Start a Powershell prompt and navigate to your local clone of the `datadog-agent
Run the following command:

```powershell
docker run -v "$(Get-Location):c:\mnt" -e OMNIBUS_TARGET=main -e RELEASE_VERSION=nightly -e MAJOR_VERSION=7 -e TARGET_ARCH=x64 datadog/agent-buildimages-windows_x64:1809 c:\mnt\tasks\winbuildscripts\buildwin.bat
docker run -v "$(Get-Location):c:\mnt" -e OMNIBUS_TARGET=main -e RELEASE_VERSION=nightly-a7 -e MAJOR_VERSION=7 -e TARGET_ARCH=x64 datadog/agent-buildimages-windows_x64:1809 powershell -C "c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1"
```

Downloading the Docker image may take some time in the first run.
Expand All @@ -100,7 +100,7 @@ Alternatively here's a small Powershell script to facilitate using the docker im
param (
[int]$MAJOR_VERSION=7,
$TARGET_ARCH="x64",
$RELEASE_VERSION="nightly",
$RELEASE_VERSION="nightly-a7",
[bool]$RM_CONTAINER=$true,
[bool]$DEBUG=$false
)
Expand All @@ -113,7 +113,7 @@ $opts = "-e OMNIBUS_TARGET=main -e RELEASE_VERSION=$RELEASE_VERSION -e MAJOR_VER
if ($DEBUG) {
$opts += " -e DEBUG_CUSTOMACTION=yes "
}
$cmd += " -m 8192M -v ""$(Get-Location):c:\mnt"" $opts datadog/agent-buildimages-windows_x64:1809 c:\mnt\tasks\winbuildscripts\buildwin.bat"
$cmd += " -m 8192M -v ""$(Get-Location):c:\mnt"" $opts datadog/agent-buildimages-windows_x64:1809 powershell -C ""c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1"""
Write-Host $cmd
Invoke-Expression -Command $cmd
```
Expand Down
2 changes: 2 additions & 0 deletions tasks/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
trace_agent,
vim,
vscode,
winbuild,
worktree,
)
from tasks.build_tags import audit_tag_impact, print_default_build_tags
Expand Down Expand Up @@ -212,6 +213,7 @@
ns.add_collection(collector)
ns.add_collection(invoke_unit_tests)
ns.add_collection(debug)
ns.add_collection(winbuild)
ns.add_collection(worktree)
ns.configure(
{
Expand Down
71 changes: 71 additions & 0 deletions tasks/winbuild.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import os
import shutil

from invoke.tasks import task

from tasks.flavor import AgentFlavor
from tasks.libs.common.utils import get_version
from tasks.msi import build as build_agent_msi
from tasks.msi import build_installer as build_installer_msi
from tasks.omnibus import build as omnibus_build

# Output directory for package files
OUTPUT_PATH = os.path.join(os.getcwd(), "omnibus", "pkg")
# Omnibus stores files here, e.g. C:\opt\datadog-agent, C:\opt\dataog-installer
OPT_SOURCE_DIR = os.path.join('C:\\', 'opt')


@task
def agent_package(
ctx,
flavor=AgentFlavor.base.name,
release_version="nightly-a7",
skip_deps=False,
):
# Build agent
omnibus_build(
ctx,
flavor=flavor,
release_version=release_version,
skip_deps=skip_deps,
)

# Package Agent into MSI
build_agent_msi(ctx, release_version=release_version)

# Package MSI into OCI
if AgentFlavor[flavor] == AgentFlavor.base:
ctx.run('powershell -C "./tasks/winbuildscripts/Generate-OCIPackage.ps1 -package datadog-agent"')


@task
def installer_package(
ctx,
release_version="nightly-a7",
skip_deps=False,
):
# Build installer
omnibus_build(
ctx,
release_version=release_version,
skip_deps=skip_deps,
target_project="installer",
)

# Package Insaller into MSI
build_installer_msi(ctx)

# Package MSI into OCI
ctx.run('powershell -C "./tasks/winbuildscripts/Generate-OCIPackage.ps1 -package datadog-installer"')

# Copy installer.exe to the output dir so it can be deployed as the bootstrapper
agent_version = get_version(
ctx,
include_git=True,
url_safe=True,
include_pipeline_id=True,
)
shutil.copy2(
os.path.join(OPT_SOURCE_DIR, "datadog-installer\\datadog-installer.exe"),
os.path.join(OUTPUT_PATH, f"datadog-installer-{agent_version}-1-x86_64.exe"),
)
83 changes: 83 additions & 0 deletions tasks/winbuildscripts/Build-AgentPackages.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
<#
.SYNOPSIS
Builds the Datadog Agent packages for Windows. Builds everything with omnibus and packages the output into MSI, ZIP, and OCI.
.DESCRIPTION
This script builds the Datadog Agent packages for Windows, with options to configure the build environment.
.PARAMETER ReleaseVersion
Specifies the release version of the build. Default is the value of the environment variable RELEASE_VERSION.
.PARAMETER Flavor
Specifies the flavor of the agent. Default is the value of the environment variable AGENT_FLAVOR.
.PARAMETER BuildOutOfSource
Specifies whether to build out of source. Default is $false.
Use this option in the CI to keep the job directory clean and avoid conflicts/stale data.
Use this option in Hyper-V based containers to improve build performance.
.PARAMETER InstallDeps
Specifies whether to install dependencies (python requirements, go deps, etc.). Default is $true.
.PARAMETER CheckGoVersion
Specifies whether to check the Go version. If not provided, it defaults to the value of the environment variable GO_VERSION_CHECK or $true if the environment variable is not set.
.EXAMPLE
.\Build-AgentPackages.ps1 -InstallDeps $false
.EXAMPLE
.\Build-AgentPackages.ps1 -BuildOutOfSource $true -InstallDeps $true -Flavor "fips" -CheckGoVersion $true
.NOTES
This script should be run from the root of the repository.
#>
param(
[bool] $BuildOutOfSource = $false,
[nullable[bool]] $CheckGoVersion,
[bool] $InstallDeps = $true,
[string] $ReleaseVersion = $env:RELEASE_VERSION,
[string] $Flavor = $env:AGENT_FLAVOR
)

. "$PSScriptRoot\common.ps1"

Invoke-BuildScript `
-BuildOutOfSource $BuildOutOfSource `
-InstallDeps $InstallDeps `
-CheckGoVersion $CheckGoVersion `
-Command {
$inv_args = @(
"--skip-deps"
)
if ($ReleaseVersion) {
$inv_args += "--release-version"
$inv_args += $ReleaseVersion
$env:RELEASE_VERSION=$ReleaseVersion
}

if ($Flavor) {
$inv_args += "--flavor"
$inv_args += $Flavor
$env:AGENT_FLAVOR=$Flavor
}

Write-Host "inv -e winbuild.agent-package $inv_args"
inv -e winbuild.agent-package @inv_args
if ($LASTEXITCODE -ne 0) {
Write-Error "Failed to build the agent package"
exit 1
}

# Show the contents of the output package directories for debugging purposes
Get-ChildItem -Path C:\omnibus-ruby\pkg\
Get-ChildItem -Path "C:\opt\datadog-agent\bin\agent\"
Get-ChildItem -Path ".\omnibus\pkg\"

if ($BuildOutOfSource) {
# Copy the resulting package to the mnt directory
mkdir C:\mnt\omnibus\pkg -Force -ErrorAction Stop | Out-Null
Copy-Item -Path ".\omnibus\pkg\*" -Destination "C:\mnt\omnibus\pkg" -Force -ErrorAction Stop
}
}
72 changes: 72 additions & 0 deletions tasks/winbuildscripts/Build-InstallerPackages.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
<#
.SYNOPSIS
Builds the Datadog Installer packages for Windows. Builds everything with omnibus and packages the output into MSI, ZIP, and OCI.
.DESCRIPTION
This script builds the Datadog Installer packages for Windows, with options to configure the build environment.
.PARAMETER BuildOutOfSource
Specifies whether to build out of source. Default is $false.
Use this option in the CI to keep the job directory clean and avoid conflicts/stale data.
Use this option in Hyper-V based containers to improve build performance.
.PARAMETER InstallDeps
Specifies whether to install dependencies (python requirements, go deps, etc.). Default is $true.
.PARAMETER ReleaseVersion
Specifies the release version of the build. Default is the value of the environment variable RELEASE_VERSION.
.PARAMETER CheckGoVersion
Specifies whether to check the Go version. If not provided, it defaults to the value of the environment variable GO_VERSION_CHECK or $true if the environment variable is not set.
.EXAMPLE
.\Build-InstallerPackages.ps1 -InstallDeps $false
.EXAMPLE
.\Build-InstallerPackages.ps1 -BuildOutOfSource $true -InstallDeps $true -CheckGoVersion $true
.NOTES
This script should be run from the root of the repository.
#>
param(
[bool] $BuildOutOfSource = $false,
[nullable[bool]] $CheckGoVersion,
[bool] $InstallDeps = $true,
[string] $ReleaseVersion = $env:RELEASE_VERSION
)

. "$PSScriptRoot\common.ps1"

Invoke-BuildScript `
-BuildOutOfSource $BuildOutOfSource `
-InstallDeps $InstallDeps `
-CheckGoVersion $CheckGoVersion `
-Command {
$inv_args = @(
"--skip-deps"
)
if ($ReleaseVersion) {
$inv_args += "--release-version"
$inv_args += $ReleaseVersion
}

Write-Host "inv -e winbuild.installer-package $inv_args"
inv -e winbuild.installer-package @inv_args
if ($LASTEXITCODE -ne 0) {
Write-Error "Failed to build the agent package"
exit 1
}

# Show the contents of the output package directories for debugging purposes
Get-ChildItem -Path C:\omnibus-ruby\pkg\
Get-ChildItem -Path C:\opt\datadog-installer
Get-ChildItem -Path ".\omnibus\pkg\"

if ($BuildOutOfSource) {
# Copy the resulting package to the mnt directory
mkdir C:\mnt\omnibus\pkg -Force -ErrorAction Stop | Out-Null
Copy-Item -Path ".\omnibus\pkg\*" -Destination "C:\mnt\omnibus\pkg" -Force -ErrorAction Stop
}
}
Loading

0 comments on commit 858cb1c

Please sign in to comment.