From c5d3e9ea94f638bea7f2b3e17a07e1a85889285a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Tue, 12 Sep 2023 17:10:38 +0200 Subject: [PATCH 01/45] omnibus: conditionally enable git cache --- omnibus/omnibus.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/omnibus/omnibus.rb b/omnibus/omnibus.rb index 443342902ca65..626b6d43aec4c 100644 --- a/omnibus/omnibus.rb +++ b/omnibus/omnibus.rb @@ -37,4 +37,10 @@ s3_instance_profile true end end -use_git_caching false + +if not ENV.has_key?("OMNIBUS_GIT_CACHE_DIR") + use_git_caching false +else + use_git_caching true + git_cache_dir ENV["OMNIBUS_GIT_CACHE_DIR"] +end From dfadb8a868393cb693a442af92cea19ba0d9e566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Thu, 21 Sep 2023 16:04:52 +0200 Subject: [PATCH 02/45] omnibus: always build datadog projects/software They are expected to almost always change and wouldn keep invalidating the cache. Not caching those will allow us to not regenerate the cache when there's no need to, which saves a few minutes it takes to recreate the cache bundle and upload it to s3 --- omnibus/config/software/datadog-agent-finalize.rb | 2 ++ omnibus/config/software/datadog-agent-integrations-py2.rb | 2 ++ omnibus/config/software/datadog-agent-integrations-py3.rb | 2 ++ omnibus/config/software/datadog-agent.rb | 2 ++ omnibus/config/software/datadog-security-agent-policies.rb | 2 ++ omnibus/config/software/snmp-traps.rb | 1 - omnibus/config/software/system-probe.rb | 2 ++ 7 files changed, 12 insertions(+), 1 deletion(-) diff --git a/omnibus/config/software/datadog-agent-finalize.rb b/omnibus/config/software/datadog-agent-finalize.rb index ab57604c33a54..8f66fb3d91ed9 100644 --- a/omnibus/config/software/datadog-agent-finalize.rb +++ b/omnibus/config/software/datadog-agent-finalize.rb @@ -14,6 +14,8 @@ skip_transitive_dependency_licensing true +always_build true + build do license :project_license diff --git a/omnibus/config/software/datadog-agent-integrations-py2.rb b/omnibus/config/software/datadog-agent-integrations-py2.rb index f5eaefde2148d..7a6076fd9722d 100644 --- a/omnibus/config/software/datadog-agent-integrations-py2.rb +++ b/omnibus/config/software/datadog-agent-integrations-py2.rb @@ -22,6 +22,8 @@ source git: 'https://github.com/DataDog/integrations-core.git' +always_build true + integrations_core_version = ENV['INTEGRATIONS_CORE_VERSION'] if integrations_core_version.nil? || integrations_core_version.empty? integrations_core_version = 'master' diff --git a/omnibus/config/software/datadog-agent-integrations-py3.rb b/omnibus/config/software/datadog-agent-integrations-py3.rb index 3f7db69574577..dda8984b952c3 100644 --- a/omnibus/config/software/datadog-agent-integrations-py3.rb +++ b/omnibus/config/software/datadog-agent-integrations-py3.rb @@ -22,6 +22,8 @@ source git: 'https://github.com/DataDog/integrations-core.git' +always_build true + gcc_version = ENV['GCC_VERSION'] if gcc_version.nil? || gcc_version.empty? gcc_version = '10.4.0' diff --git a/omnibus/config/software/datadog-agent.rb b/omnibus/config/software/datadog-agent.rb index 923eb12660d04..19c48f9298a3e 100644 --- a/omnibus/config/software/datadog-agent.rb +++ b/omnibus/config/software/datadog-agent.rb @@ -18,6 +18,8 @@ source path: '..' relative_path 'src/github.com/DataDog/datadog-agent' +always_build true + build do license :project_license diff --git a/omnibus/config/software/datadog-security-agent-policies.rb b/omnibus/config/software/datadog-security-agent-policies.rb index 181c9aab15e88..8e20b797aa799 100644 --- a/omnibus/config/software/datadog-security-agent-policies.rb +++ b/omnibus/config/software/datadog-security-agent-policies.rb @@ -20,6 +20,8 @@ end default_version policies_version +always_build true + build do license "Apache-2.0" license_file "./LICENSE" diff --git a/omnibus/config/software/snmp-traps.rb b/omnibus/config/software/snmp-traps.rb index bb13fc542fd3b..a08b4e4217e1b 100644 --- a/omnibus/config/software/snmp-traps.rb +++ b/omnibus/config/software/snmp-traps.rb @@ -5,7 +5,6 @@ :sha256 => "04fb9d43754c2656edf35f08fbad11ba8dc20d52654962933f3dd8f4d463b42c", :target_filename => "dd_traps_db.json.gz" - build do # The dir for confs if osx_target? diff --git a/omnibus/config/software/system-probe.rb b/omnibus/config/software/system-probe.rb index 6a042e080e7f9..de1ba341d3d51 100644 --- a/omnibus/config/software/system-probe.rb +++ b/omnibus/config/software/system-probe.rb @@ -8,6 +8,8 @@ source path: '..' relative_path 'src/github.com/DataDog/datadog-agent' +always_build true + build do license :project_license From a3d09bd8edbccd7cfcaaee2cf0095ae7be3c9a86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Tue, 19 Sep 2023 11:26:51 +0200 Subject: [PATCH 03/45] tasks: omnibus_build: add support for omnibus git caching --- .gitlab-ci.yml | 1 + tasks/agent.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6c8b1c0ad91d0..93d86c61342de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -151,6 +151,7 @@ variables: ## build to succeed with S3 caching disabled. S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable USE_S3_CACHING: --omnibus-s3-cache + OMNIBUS_GIT_CACHE_DIR: /tmp/omnibus-git-cache ## comment out the line below to disable integration wheels cache INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus S3_DD_AGENT_OMNIBUS_LLVM_URI: s3://dd-agent-omnibus/llvm diff --git a/tasks/agent.py b/tasks/agent.py index 3257fcd64d930..84b982c1915be 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -38,6 +38,8 @@ from tasks.ssm import get_pfx_pass, get_signing_cert from tasks.windows_resources import build_messagetable, build_rc, versioninfo_vars +from .release import _get_release_json_value + # constants BIN_DIR = os.path.join(".", "bin") BIN_PATH = os.path.join(BIN_DIR, "agent") @@ -319,8 +321,8 @@ def refresh_assets(_, build_tags, development=True, flavor=AgentFlavor.base.name # Ensure the config folders are not world writable os.chmod(check_dir, mode=0o755) - ## add additional windows-only corechecks, only on windows. Otherwise the check loader - ## on linux will throw an error because the module is not found, but the config is. + # add additional windows-only corechecks, only on windows. Otherwise the check loader + # on linux will throw an error because the module is not found, but the config is. if sys.platform == 'win32': for check in WINDOWS_CORECHECKS: check_dir = os.path.join(dist_folder, f"conf.d/{check}.d/") @@ -930,6 +932,27 @@ def omnibus_build( with timed(quiet=True) as bundle_elapsed: bundle_install_omnibus(ctx, gem_path, env) + omnibus_cache_dir = os.environ.get('OMNIBUS_GIT_CACHE_DIR') + use_omnibus_git_cache = omnibus_cache_dir is not None + if use_omnibus_git_cache: + omnibus_cache_dir += '/opt/datadog-agent' + remote_cache_name = os.environ.get('CI_JOB_NAME_SLUG') + # We don't want to update the cache when not running on a CI + # Individual developers are still able to leverage the cache by providing + # the OMNIBUS_GIT_CACHE_DIR env variable, but they won't pull from the CI + # generated one. + use_remote_cache = remote_cache_name is not None + base_branch = _get_release_json_value("base_branch") + if use_remote_cache: + cache_state = None + git_cache_url = f"s3://{os.environ['S3_OMNIBUS_CACHE_BUCKET']}/builds/{base_branch}/{remote_cache_name}" + bundle_path = "/tmp/omnibus-git-cache-bundle" + with timed(quiet=True) as restore_cache: + # Allow failure in case the cache was evicted + if ctx.run(f"aws s3 cp --only-show-errors {git_cache_url} {bundle_path}", warn=True): + ctx.run(f"git clone --mirror {bundle_path} {omnibus_cache_dir}") + cache_state = ctx.run(f"git -C {omnibus_cache_dir} tag -l").stdout + with timed(quiet=True) as omnibus_elapsed: omnibus_run_task( ctx=ctx, @@ -942,6 +965,23 @@ def omnibus_build( host_distribution=host_distribution, ) + if use_omnibus_git_cache and use_remote_cache: + with timed(quiet=True) as update_cache: + if base_branch == os.environ['CI_COMMIT_BRANCH']: + # Purge the cache manually as omnibus will stick to not restoring a tag when + # a mismatch is detected, but will keep the old cached tags. + # Do this before checking for tag differences, in order to remove staled tags + # in case they were included in the bundle in a previous build + # Allow the command to fail since an empty cache will cause a git reflog failure + stale_tags = ctx.run(f'git -C {omnibus_cache_dir} tag --no-merged', warn=True).stdout + for _, tag in enumerate(stale_tags.split(os.linesep)): + ctx.run(f'git -C {omnibus_cache_dir} tag -d {tag}') + if ctx.run(f"git -C {omnibus_cache_dir} tag -l").stdout != cache_state: + ctx.run(f"git -C {omnibus_cache_dir} bundle create {bundle_path} --tags") + ctx.run(f"aws s3 cp --only-show-errors {bundle_path} {git_cache_url}") + else: + print("Not updating omnibus cache from a feature branch") + # Delete the temporary pip.conf file once the build is done os.remove(pip_config_file) @@ -950,6 +990,10 @@ def omnibus_build( print(f"Deps: {deps_elapsed.duration}") print(f"Bundle: {bundle_elapsed.duration}") print(f"Omnibus: {omnibus_elapsed.duration}") + if use_omnibus_git_cache and use_remote_cache: + print(f"Restoring omnibus cache: {restore_cache.duration}") + print(f"Updating omnibus cache: {update_cache.duration}") + _send_build_metrics(ctx, omnibus_elapsed.duration) From 86533368e705d0cb471897ead74d91606fbb0bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Wed, 18 Oct 2023 10:00:33 +0200 Subject: [PATCH 04/45] always update omnibus cache so that we can measure the results until it's merged & further worked on --- tasks/agent.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tasks/agent.py b/tasks/agent.py index 84b982c1915be..21996349c8caf 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -967,20 +967,20 @@ def omnibus_build( if use_omnibus_git_cache and use_remote_cache: with timed(quiet=True) as update_cache: - if base_branch == os.environ['CI_COMMIT_BRANCH']: - # Purge the cache manually as omnibus will stick to not restoring a tag when - # a mismatch is detected, but will keep the old cached tags. - # Do this before checking for tag differences, in order to remove staled tags - # in case they were included in the bundle in a previous build - # Allow the command to fail since an empty cache will cause a git reflog failure - stale_tags = ctx.run(f'git -C {omnibus_cache_dir} tag --no-merged', warn=True).stdout - for _, tag in enumerate(stale_tags.split(os.linesep)): - ctx.run(f'git -C {omnibus_cache_dir} tag -d {tag}') - if ctx.run(f"git -C {omnibus_cache_dir} tag -l").stdout != cache_state: - ctx.run(f"git -C {omnibus_cache_dir} bundle create {bundle_path} --tags") - ctx.run(f"aws s3 cp --only-show-errors {bundle_path} {git_cache_url}") - else: - print("Not updating omnibus cache from a feature branch") + # if base_branch == os.environ['CI_COMMIT_BRANCH'] or True: + # Purge the cache manually as omnibus will stick to not restoring a tag when + # a mismatch is detected, but will keep the old cached tags. + # Do this before checking for tag differences, in order to remove staled tags + # in case they were included in the bundle in a previous build + # Allow the command to fail since an empty cache will cause a git reflog failure + stale_tags = ctx.run(f'git -C {omnibus_cache_dir} tag --no-merged', warn=True).stdout + for _, tag in enumerate(stale_tags.split(os.linesep)): + ctx.run(f'git -C {omnibus_cache_dir} tag -d {tag}') + if ctx.run(f"git -C {omnibus_cache_dir} tag -l").stdout != cache_state: + ctx.run(f"git -C {omnibus_cache_dir} bundle create {bundle_path} --tags") + ctx.run(f"aws s3 cp --only-show-errors {bundle_path} {git_cache_url}") + # else: + # print("Not updating omnibus cache from a feature branch") # Delete the temporary pip.conf file once the build is done os.remove(pip_config_file) From ac458d2ff211275c559588a76131d0c23008a1b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Tue, 7 Nov 2023 08:25:00 +0100 Subject: [PATCH 05/45] tasks: generate a cache key to fetch omnibus cache --- tasks/agent.py | 151 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 133 insertions(+), 18 deletions(-) diff --git a/tasks/agent.py b/tasks/agent.py index 21996349c8caf..66bce0c533927 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -4,6 +4,7 @@ import ast import glob +import hashlib import json import os import platform @@ -38,8 +39,6 @@ from tasks.ssm import get_pfx_pass, get_signing_cert from tasks.windows_resources import build_messagetable, build_rc, versioninfo_vars -from .release import _get_release_json_value - # constants BIN_DIR = os.path.join(".", "bin") BIN_PATH = os.path.join(BIN_DIR, "agent") @@ -856,6 +855,122 @@ def _send_build_metrics(ctx, overall_duration): print(r.text) +def _get_build_images(ctx): + tags = ctx.run("grep -E 'DATADOG_AGENT_.*BUILDIMAGES:' .gitlab-ci.yml | cut -d ':' -f 2").stdout + return map(lambda t: t.strip(), tags.splitlines()) + + +def _get_environment_for_cache() -> dict: + """ + Compute a hash from the environment after excluding irrelevant/insecure + environment variables to ensure we don't omit a variable + """ + + def env_filter(item): + key = item[0] + excluded_prefixes = [ + 'AGENT_', + 'ARTIFACTORY_', + 'ARTIFACTORY_', + 'AWS_', + 'BUILDENV_', + 'CI_', + 'CLUSTER_AGENT_', + 'DATADOG_AGENT_', + 'DD_', + 'DEB_', + 'DESTINATION_', + 'DOCKER_', + 'FF_', + 'GITLAB_', + 'GIT_', + 'K8S_', + 'KERNEL_MATRIX_TESTING_', + 'KUBERNETES_', + 'OMNIBUS_', + 'POD_', + 'RELEASE_VERSION', + 'RPM_', + 'S3_', + 'TEST_INFRA_', + 'USE_', + 'VAULT_', + 'WINDOWS_', + ] + excluded_suffixes = [ + '_SHA256', + '_VERSION', + ] + excluded_values = [ + "AVAILABILITY_ZONE", + "BENCHMARKS_CI_IMAGE", + "BUCKET_BRANCH", + "BUNDLER_VERSION", + "CHANNEL", + "CI", + "CONSUL_HTTP_ADDR", + "DOGSTATSD_BINARIES_DIR", + "EXPERIMENTS_EVALUATION_ADDRESS", + "GCC_VERSION", + "GCE_METADATA_HOST", + "GENERAL_ARTIFACTS_CACHE_BUCKET_URL", + "GET_SOURCES_ATTEMPTS", + "HOME", + "HOSTNAME", + "HOST_IP", + "INTEGRATION_WHEELS_CACHE_BUCKET", + "KITCHEN_INFRASTRUCTURE_FLAKES_RETRY", + "MACOS_S3_BUCKET", + "MESSAGE", + "OLDPWD", + "PROCESS_S3_BUCKET", + "PWD", + "PYTHON_RUNTIMES", + "RUNNER_TEMP_PROJECT_DIR", + "RUSTC_SHA256", + "RUST_VERSION", + "SHLVL", + "STATIC_BINARIES_DIR", + "STATSD_URL", + "SYSTEM_PROBE_BINARIES_DIR", + "TRACE_AGENT_URL", + "USE_CACHING_PROXY_PYTHON", + "USE_CACHING_PROXY_RUBY", + "USE_S3_CACHING", + "WIN_S3_BUCKET", + "_", + "build_before", + ] + for p in excluded_prefixes: + if key.startswith(p): + return False + for s in excluded_suffixes: + if key.endswith(s): + return False + if key in excluded_values: + return False + return True + + return dict(filter(env_filter, os.environ.items())) + + +def _omnibus_compute_cache_key(ctx): + h = hashlib.sha1() + print('Computing cache key') + omnibus_last_commit = ctx.run('git log -n 1 --pretty=format:%H omnibus/').stdout + h.update(str.encode(omnibus_last_commit)) + print(f'\tLast omnibus commit is {omnibus_last_commit}') + buildimages_hash = _get_build_images(ctx) + for img_hash in buildimages_hash: + h.update(str.encode(img_hash)) + environment = _get_environment_for_cache() + for k, v in environment.items(): + print(f'\tUsing environment variable {k} to compute cache key') + h.update(str.encode(f'{k}={v}')) + # FIXME: include omnibus-ruby and omnibus-software version once they are pinned + return h.hexdigest() + + # hardened-runtime needs to be set to False to build on MacOS < 10.13.6, as the -o runtime option is not supported. @task( help={ @@ -942,16 +1057,20 @@ def omnibus_build( # the OMNIBUS_GIT_CACHE_DIR env variable, but they won't pull from the CI # generated one. use_remote_cache = remote_cache_name is not None - base_branch = _get_release_json_value("base_branch") if use_remote_cache: cache_state = None - git_cache_url = f"s3://{os.environ['S3_OMNIBUS_CACHE_BUCKET']}/builds/{base_branch}/{remote_cache_name}" + cache_key = _omnibus_compute_cache_key(ctx) + print(f'Cache key: {cache_key}') + git_cache_url = f"s3://{os.environ['S3_OMNIBUS_CACHE_BUCKET']}/builds/{cache_key}/{remote_cache_name}" bundle_path = "/tmp/omnibus-git-cache-bundle" with timed(quiet=True) as restore_cache: # Allow failure in case the cache was evicted if ctx.run(f"aws s3 cp --only-show-errors {git_cache_url} {bundle_path}", warn=True): + print(f'Successfully restored cache {cache_key}') ctx.run(f"git clone --mirror {bundle_path} {omnibus_cache_dir}") cache_state = ctx.run(f"git -C {omnibus_cache_dir} tag -l").stdout + else: + print(f'Failed to restore cache from key {cache_key}') with timed(quiet=True) as omnibus_elapsed: omnibus_run_task( @@ -965,22 +1084,18 @@ def omnibus_build( host_distribution=host_distribution, ) - if use_omnibus_git_cache and use_remote_cache: - with timed(quiet=True) as update_cache: - # if base_branch == os.environ['CI_COMMIT_BRANCH'] or True: - # Purge the cache manually as omnibus will stick to not restoring a tag when - # a mismatch is detected, but will keep the old cached tags. - # Do this before checking for tag differences, in order to remove staled tags - # in case they were included in the bundle in a previous build - # Allow the command to fail since an empty cache will cause a git reflog failure - stale_tags = ctx.run(f'git -C {omnibus_cache_dir} tag --no-merged', warn=True).stdout - for _, tag in enumerate(stale_tags.split(os.linesep)): - ctx.run(f'git -C {omnibus_cache_dir} tag -d {tag}') - if ctx.run(f"git -C {omnibus_cache_dir} tag -l").stdout != cache_state: + if use_omnibus_git_cache: + stale_tags = ctx.run(f'git -C {omnibus_cache_dir} tag --no-merged', warn=True).stdout + # Purge the cache manually as omnibus will stick to not restoring a tag when + # a mismatch is detected, but will keep the old cached tags. + # Do this before checking for tag differences, in order to remove staled tags + # in case they were included in the bundle in a previous build + for _, tag in enumerate(stale_tags.split(os.linesep)): + ctx.run(f'git -C {omnibus_cache_dir} tag -d {tag}') + if use_remote_cache and ctx.run(f"git -C {omnibus_cache_dir} tag -l").stdout != cache_state: + with timed(quiet=True) as update_cache: ctx.run(f"git -C {omnibus_cache_dir} bundle create {bundle_path} --tags") ctx.run(f"aws s3 cp --only-show-errors {bundle_path} {git_cache_url}") - # else: - # print("Not updating omnibus cache from a feature branch") # Delete the temporary pip.conf file once the build is done os.remove(pip_config_file) From 37abbc5302965841c3b853947cb2d5daf5d90666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Tue, 9 Jan 2024 13:45:38 +0100 Subject: [PATCH 06/45] currate env variables filter --- tasks/agent.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tasks/agent.py b/tasks/agent.py index 66bce0c533927..7e38505487b7f 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -856,7 +856,9 @@ def _send_build_metrics(ctx, overall_duration): def _get_build_images(ctx): - tags = ctx.run("grep -E 'DATADOG_AGENT_.*BUILDIMAGES:' .gitlab-ci.yml | cut -d ':' -f 2").stdout + # We intentionally include both build images & their test suffixes in the pattern + # as a test image and the merged version shouldn't share their cache + tags = ctx.run("grep -E 'DATADOG_AGENT_.*BUILDIMAGES' .gitlab-ci.yml | cut -d ':' -f 2", hide='stdout').stdout return map(lambda t: t.strip(), tags.splitlines()) @@ -871,7 +873,6 @@ def env_filter(item): excluded_prefixes = [ 'AGENT_', 'ARTIFACTORY_', - 'ARTIFACTORY_', 'AWS_', 'BUILDENV_', 'CI_', @@ -919,13 +920,20 @@ def env_filter(item): "HOSTNAME", "HOST_IP", "INTEGRATION_WHEELS_CACHE_BUCKET", + "IRBRC", "KITCHEN_INFRASTRUCTURE_FLAKES_RETRY", + "LESSCLOSE", + "LESSOPEN", + "LC_CTYPE", + "LS_COLORS", "MACOS_S3_BUCKET", "MESSAGE", "OLDPWD", "PROCESS_S3_BUCKET", "PWD", "PYTHON_RUNTIMES", + "RUN_ALL_BUILDS", + "RUN_KITCHEN_TESTS", "RUNNER_TEMP_PROJECT_DIR", "RUSTC_SHA256", "RUST_VERSION", @@ -951,13 +959,13 @@ def env_filter(item): return False return True - return dict(filter(env_filter, os.environ.items())) + return dict(filter(env_filter, sorted(os.environ.items()))) def _omnibus_compute_cache_key(ctx): - h = hashlib.sha1() print('Computing cache key') - omnibus_last_commit = ctx.run('git log -n 1 --pretty=format:%H omnibus/').stdout + h = hashlib.sha1() + omnibus_last_commit = ctx.run('git log -n 1 --pretty=format:%H omnibus/', hide='stdout').stdout h.update(str.encode(omnibus_last_commit)) print(f'\tLast omnibus commit is {omnibus_last_commit}') buildimages_hash = _get_build_images(ctx) @@ -968,7 +976,9 @@ def _omnibus_compute_cache_key(ctx): print(f'\tUsing environment variable {k} to compute cache key') h.update(str.encode(f'{k}={v}')) # FIXME: include omnibus-ruby and omnibus-software version once they are pinned - return h.hexdigest() + cache_key = h.hexdigest() + print(f'Cache key: {cache_key}') + return cache_key # hardened-runtime needs to be set to False to build on MacOS < 10.13.6, as the -o runtime option is not supported. @@ -1060,7 +1070,6 @@ def omnibus_build( if use_remote_cache: cache_state = None cache_key = _omnibus_compute_cache_key(ctx) - print(f'Cache key: {cache_key}') git_cache_url = f"s3://{os.environ['S3_OMNIBUS_CACHE_BUCKET']}/builds/{cache_key}/{remote_cache_name}" bundle_path = "/tmp/omnibus-git-cache-bundle" with timed(quiet=True) as restore_cache: From bae0667a612cd1065a3b75ace509d766ceef43ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Mon, 4 Mar 2024 16:13:25 +0100 Subject: [PATCH 07/45] attempt to stop hardcoding install dir --- tasks/agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/agent.py b/tasks/agent.py index 7e38505487b7f..cca49a858c4fc 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -1060,7 +1060,7 @@ def omnibus_build( omnibus_cache_dir = os.environ.get('OMNIBUS_GIT_CACHE_DIR') use_omnibus_git_cache = omnibus_cache_dir is not None if use_omnibus_git_cache: - omnibus_cache_dir += '/opt/datadog-agent' + # omnibus_cache_dir += '/opt/datadog-agent' remote_cache_name = os.environ.get('CI_JOB_NAME_SLUG') # We don't want to update the cache when not running on a CI # Individual developers are still able to leverage the cache by providing From 45c6717e7e4e171bfa5c544bbff8157396aa7c39 Mon Sep 17 00:00:00 2001 From: Pythyu Date: Fri, 15 Mar 2024 15:14:17 +0100 Subject: [PATCH 08/45] feat(omnibus-cache): configure cache for remote updater OCI packages --- .gitlab-ci.yml | 4 ++-- .gitlab/package_build/deb.yml | 2 ++ .gitlab/package_build/remote_updater.yml | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 98826df08e2f6..54e6d4cc7dd4e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -155,7 +155,7 @@ variables: ## build to succeed with S3 caching disabled. S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable USE_S3_CACHING: --omnibus-s3-cache - OMNIBUS_GIT_CACHE_DIR: /tmp/omnibus-git-cache + OMNIBUS_GIT_CACHE_DIR: /omnibus/opt/datadog-agent/ ## comment out the line below to disable integration wheels cache INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus S3_DD_AGENT_OMNIBUS_LLVM_URI: s3://dd-agent-omnibus/llvm @@ -207,7 +207,7 @@ variables: RUN_E2E_TESTS: "auto" # Should be "off", "auto" or "on" it will change the trigger condition for new-e2e tests on branch != main # skip known flaky tests by default GO_TEST_SKIP_FLAKE: "true" - + # List of parameters retrieved from AWS SSM # They must be defined as environment variables in the GitLab CI/CD settings, to ease rotation if needed AGENT_QA_PROFILE_SSM_NAME: ci.datadog-agent.agent-qa-profile # agent-ci-experience diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml index 4e2e6ff546a5e..43e6ad3bc9aa8 100644 --- a/.gitlab/package_build/deb.yml +++ b/.gitlab/package_build/deb.yml @@ -292,6 +292,7 @@ updater_deb-amd64: DESTINATION_DBG_DEB: "datadog-updater-dbg_7_amd64.deb" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 + - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages updater_deb-arm64: extends: .updater_build_common_deb @@ -307,6 +308,7 @@ updater_deb-arm64: DESTINATION_DBG_DEB: "datadog-updater-dbg_7_arm64.deb" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 + - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages agentless_scanner_deb-x64: rules: diff --git a/.gitlab/package_build/remote_updater.yml b/.gitlab/package_build/remote_updater.yml index c6ee8f8c17420..399e54aa78234 100644 --- a/.gitlab/package_build/remote_updater.yml +++ b/.gitlab/package_build/remote_updater.yml @@ -53,6 +53,7 @@ agent_remote_updater-x64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 + - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages # build Agent package for remote_updater-x64 agent_remote_updater-arm64-a7: @@ -77,3 +78,4 @@ agent_remote_updater-arm64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 + - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages From 6caa677d2dc302a1abbabad27982e931039d1877 Mon Sep 17 00:00:00 2001 From: Pythyu Date: Fri, 15 Mar 2024 17:40:05 +0100 Subject: [PATCH 09/45] feat(omnibus-cache): configure cache for remote updater OCI packages --- .gitlab-ci.yml | 2 +- .gitlab/package_build/deb.yml | 4 ++-- .gitlab/package_build/remote_updater.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 54e6d4cc7dd4e..c27c6b55685bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -155,7 +155,7 @@ variables: ## build to succeed with S3 caching disabled. S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable USE_S3_CACHING: --omnibus-s3-cache - OMNIBUS_GIT_CACHE_DIR: /omnibus/opt/datadog-agent/ + OMNIBUS_GIT_CACHE_DIR: /tmp/omnibus-git-cache/opt/datadog-agent/ ## comment out the line below to disable integration wheels cache INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus S3_DD_AGENT_OMNIBUS_LLVM_URI: s3://dd-agent-omnibus/llvm diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml index 43e6ad3bc9aa8..a88a75c2e17e7 100644 --- a/.gitlab/package_build/deb.yml +++ b/.gitlab/package_build/deb.yml @@ -292,7 +292,7 @@ updater_deb-amd64: DESTINATION_DBG_DEB: "datadog-updater-dbg_7_amd64.deb" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_DIR=/tmp/omnibus-git-cache/opt/datadog-packages updater_deb-arm64: extends: .updater_build_common_deb @@ -308,7 +308,7 @@ updater_deb-arm64: DESTINATION_DBG_DEB: "datadog-updater-dbg_7_arm64.deb" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_DIR=/tmp/omnibus-git-cache/opt/datadog-packages agentless_scanner_deb-x64: rules: diff --git a/.gitlab/package_build/remote_updater.yml b/.gitlab/package_build/remote_updater.yml index 399e54aa78234..45a45f4ee524f 100644 --- a/.gitlab/package_build/remote_updater.yml +++ b/.gitlab/package_build/remote_updater.yml @@ -53,7 +53,7 @@ agent_remote_updater-x64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_DIR=/tmp/omnibus-git-cache/opt/datadog-packages # build Agent package for remote_updater-x64 agent_remote_updater-arm64-a7: @@ -78,4 +78,4 @@ agent_remote_updater-arm64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_DIR=/tmp/omnibus-git-cache/opt/datadog-packages From 7d35a233ea42501573fbd814ddddbc8080b60e30 Mon Sep 17 00:00:00 2001 From: Pythyu Date: Mon, 18 Mar 2024 14:24:31 +0100 Subject: [PATCH 10/45] feat(test): use omnibus basedir in cache --- .gitlab-ci.yml | 2 +- .gitlab/package_build/deb.yml | 4 ++-- .gitlab/package_build/remote_updater.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c27c6b55685bb..54e6d4cc7dd4e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -155,7 +155,7 @@ variables: ## build to succeed with S3 caching disabled. S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable USE_S3_CACHING: --omnibus-s3-cache - OMNIBUS_GIT_CACHE_DIR: /tmp/omnibus-git-cache/opt/datadog-agent/ + OMNIBUS_GIT_CACHE_DIR: /omnibus/opt/datadog-agent/ ## comment out the line below to disable integration wheels cache INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus S3_DD_AGENT_OMNIBUS_LLVM_URI: s3://dd-agent-omnibus/llvm diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml index a88a75c2e17e7..43e6ad3bc9aa8 100644 --- a/.gitlab/package_build/deb.yml +++ b/.gitlab/package_build/deb.yml @@ -292,7 +292,7 @@ updater_deb-amd64: DESTINATION_DBG_DEB: "datadog-updater-dbg_7_amd64.deb" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_DIR=/tmp/omnibus-git-cache/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages updater_deb-arm64: extends: .updater_build_common_deb @@ -308,7 +308,7 @@ updater_deb-arm64: DESTINATION_DBG_DEB: "datadog-updater-dbg_7_arm64.deb" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_DIR=/tmp/omnibus-git-cache/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages agentless_scanner_deb-x64: rules: diff --git a/.gitlab/package_build/remote_updater.yml b/.gitlab/package_build/remote_updater.yml index 45a45f4ee524f..399e54aa78234 100644 --- a/.gitlab/package_build/remote_updater.yml +++ b/.gitlab/package_build/remote_updater.yml @@ -53,7 +53,7 @@ agent_remote_updater-x64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_DIR=/tmp/omnibus-git-cache/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages # build Agent package for remote_updater-x64 agent_remote_updater-arm64-a7: @@ -78,4 +78,4 @@ agent_remote_updater-arm64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_DIR=/tmp/omnibus-git-cache/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages From 74d010a53e4b685821bfa9e8486306e67b0219a6 Mon Sep 17 00:00:00 2001 From: Pythyu Date: Mon, 18 Mar 2024 15:10:38 +0100 Subject: [PATCH 11/45] feat(test): use two different omnibus cache variable --- .gitlab-ci.yml | 3 ++- .gitlab/package_build/deb.yml | 4 ++-- .gitlab/package_build/remote_updater.yml | 4 ++-- tasks/agent.py | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 54e6d4cc7dd4e..052bbb2aad146 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -155,7 +155,8 @@ variables: ## build to succeed with S3 caching disabled. S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable USE_S3_CACHING: --omnibus-s3-cache - OMNIBUS_GIT_CACHE_DIR: /omnibus/opt/datadog-agent/ + OMNIBUS_GIT_CACHE_DIR: /tmp/omnibus-git-cache + OMNIBUS_GIT_CACHE_SUFFIX: /opt/datadog-agent ## comment out the line below to disable integration wheels cache INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus S3_DD_AGENT_OMNIBUS_LLVM_URI: s3://dd-agent-omnibus/llvm diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml index 43e6ad3bc9aa8..e2c728ed85407 100644 --- a/.gitlab/package_build/deb.yml +++ b/.gitlab/package_build/deb.yml @@ -292,7 +292,7 @@ updater_deb-amd64: DESTINATION_DBG_DEB: "datadog-updater-dbg_7_amd64.deb" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages updater_deb-arm64: extends: .updater_build_common_deb @@ -308,7 +308,7 @@ updater_deb-arm64: DESTINATION_DBG_DEB: "datadog-updater-dbg_7_arm64.deb" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages agentless_scanner_deb-x64: rules: diff --git a/.gitlab/package_build/remote_updater.yml b/.gitlab/package_build/remote_updater.yml index 399e54aa78234..436282a0ed939 100644 --- a/.gitlab/package_build/remote_updater.yml +++ b/.gitlab/package_build/remote_updater.yml @@ -53,7 +53,7 @@ agent_remote_updater-x64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages # build Agent package for remote_updater-x64 agent_remote_updater-arm64-a7: @@ -78,4 +78,4 @@ agent_remote_updater-arm64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_DIR=/omnibus/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages diff --git a/tasks/agent.py b/tasks/agent.py index 7411e5e133c5f..09344cf244c19 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -1063,7 +1063,7 @@ def omnibus_build( omnibus_cache_dir = os.environ.get('OMNIBUS_GIT_CACHE_DIR') use_omnibus_git_cache = omnibus_cache_dir is not None if use_omnibus_git_cache: - # omnibus_cache_dir += '/opt/datadog-agent' + omnibus_cache_dir += os.environ.get('OMNIBUS_GIT_CACHE_SUFFIX') remote_cache_name = os.environ.get('CI_JOB_NAME_SLUG') # We don't want to update the cache when not running on a CI # Individual developers are still able to leverage the cache by providing From 134d81db03328f5069d30a6665f81ee3e0597596 Mon Sep 17 00:00:00 2001 From: Pythyu Date: Mon, 18 Mar 2024 17:13:15 +0100 Subject: [PATCH 12/45] feat(test): updater omnibus cache suffix --- .gitlab/package_build/deb.yml | 4 ++-- .gitlab/package_build/remote_updater.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml index e2c728ed85407..a15eca4d6e975 100644 --- a/.gitlab/package_build/deb.yml +++ b/.gitlab/package_build/deb.yml @@ -292,7 +292,7 @@ updater_deb-amd64: DESTINATION_DBG_DEB: "datadog-updater-dbg_7_amd64.deb" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent updater_deb-arm64: extends: .updater_build_common_deb @@ -308,7 +308,7 @@ updater_deb-arm64: DESTINATION_DBG_DEB: "datadog-updater-dbg_7_arm64.deb" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent agentless_scanner_deb-x64: rules: diff --git a/.gitlab/package_build/remote_updater.yml b/.gitlab/package_build/remote_updater.yml index 436282a0ed939..a74fb4bd5cb52 100644 --- a/.gitlab/package_build/remote_updater.yml +++ b/.gitlab/package_build/remote_updater.yml @@ -53,7 +53,7 @@ agent_remote_updater-x64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent # build Agent package for remote_updater-x64 agent_remote_updater-arm64-a7: @@ -78,4 +78,4 @@ agent_remote_updater-arm64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages + - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent From cea95fa6c6d52e39b3e5246da3249c44ff90cb83 Mon Sep 17 00:00:00 2001 From: Pythyu Date: Tue, 19 Mar 2024 13:43:42 +0100 Subject: [PATCH 13/45] feat(updater): added package version into cache path --- .gitlab/package_build/deb.yml | 4 ++-- .gitlab/package_build/remote_updater.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml index a15eca4d6e975..a446356a5c547 100644 --- a/.gitlab/package_build/deb.yml +++ b/.gitlab/package_build/deb.yml @@ -292,7 +292,7 @@ updater_deb-amd64: DESTINATION_DBG_DEB: "datadog-updater-dbg_7_amd64.deb" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent + - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent/$(inv agent.version -u)-1 updater_deb-arm64: extends: .updater_build_common_deb @@ -308,7 +308,7 @@ updater_deb-arm64: DESTINATION_DBG_DEB: "datadog-updater-dbg_7_arm64.deb" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent + - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent/$(inv agent.version -u)-1 agentless_scanner_deb-x64: rules: diff --git a/.gitlab/package_build/remote_updater.yml b/.gitlab/package_build/remote_updater.yml index a74fb4bd5cb52..77430b08be9e9 100644 --- a/.gitlab/package_build/remote_updater.yml +++ b/.gitlab/package_build/remote_updater.yml @@ -53,7 +53,7 @@ agent_remote_updater-x64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent + - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent/$(inv agent.version -u)-1 # build Agent package for remote_updater-x64 agent_remote_updater-arm64-a7: @@ -78,4 +78,4 @@ agent_remote_updater-arm64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent + - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent/$(inv agent.version -u)-1 From 53185cf1234353d3cf2c14706e32257cdaefd4dc Mon Sep 17 00:00:00 2001 From: Pythyu Date: Tue, 19 Mar 2024 15:57:03 +0100 Subject: [PATCH 14/45] feat(updater): replace OMNIBUS_GIT_CACHE_SUFFIX with an agent.omnibus-build argument --- .gitlab-ci.yml | 1 - .gitlab/package_build/deb.yml | 4 ++-- .gitlab/package_build/remote_updater.yml | 6 +++--- tasks/agent.py | 3 ++- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 052bbb2aad146..ec6da543a8e85 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -156,7 +156,6 @@ variables: S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable USE_S3_CACHING: --omnibus-s3-cache OMNIBUS_GIT_CACHE_DIR: /tmp/omnibus-git-cache - OMNIBUS_GIT_CACHE_SUFFIX: /opt/datadog-agent ## comment out the line below to disable integration wheels cache INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus S3_DD_AGENT_OMNIBUS_LLVM_URI: s3://dd-agent-omnibus/llvm diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml index a446356a5c547..1226fdd5e67fd 100644 --- a/.gitlab/package_build/deb.yml +++ b/.gitlab/package_build/deb.yml @@ -292,7 +292,7 @@ updater_deb-amd64: DESTINATION_DBG_DEB: "datadog-updater-dbg_7_amd64.deb" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent/$(inv agent.version -u)-1 + updater_deb-arm64: extends: .updater_build_common_deb @@ -308,7 +308,7 @@ updater_deb-arm64: DESTINATION_DBG_DEB: "datadog-updater-dbg_7_arm64.deb" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent/$(inv agent.version -u)-1 + agentless_scanner_deb-x64: rules: diff --git a/.gitlab/package_build/remote_updater.yml b/.gitlab/package_build/remote_updater.yml index 77430b08be9e9..20e9763d768a8 100644 --- a/.gitlab/package_build/remote_updater.yml +++ b/.gitlab/package_build/remote_updater.yml @@ -18,7 +18,7 @@ - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf # NOTE: for now, we consider "ociru" to be a "redhat_target" in omnibus/lib/ostools.rb # if we ever start building on a different platform, that might need to change - - inv -e agent.omnibus-build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --host-distribution=ociru + - inv -e agent.omnibus-build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --host-distribution=ociru --install-directory="$INSTALL_DIR" - ls -la $OMNIBUS_PACKAGE_DIR - !reference [.upload_sbom_artifacts] variables: @@ -53,7 +53,7 @@ agent_remote_updater-x64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent/$(inv agent.version -u)-1 + # build Agent package for remote_updater-x64 agent_remote_updater-arm64-a7: @@ -78,4 +78,4 @@ agent_remote_updater-arm64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 - - export OMNIBUS_GIT_CACHE_SUFFIX=/opt/datadog-packages/datadog-agent/$(inv agent.version -u)-1 + diff --git a/tasks/agent.py b/tasks/agent.py index 09344cf244c19..62801b15cd395 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -1010,6 +1010,7 @@ def omnibus_build( python_mirror=None, pip_config_file="pip.conf", host_distribution=None, + install_directory="/opt/datadog-agent" ): """ Build the Agent packages with Omnibus Installer. @@ -1063,7 +1064,7 @@ def omnibus_build( omnibus_cache_dir = os.environ.get('OMNIBUS_GIT_CACHE_DIR') use_omnibus_git_cache = omnibus_cache_dir is not None if use_omnibus_git_cache: - omnibus_cache_dir += os.environ.get('OMNIBUS_GIT_CACHE_SUFFIX') + omnibus_cache_dir += install_directory remote_cache_name = os.environ.get('CI_JOB_NAME_SLUG') # We don't want to update the cache when not running on a CI # Individual developers are still able to leverage the cache by providing From 5a1e18a2b03434f99088df751498f0f92b4fd12d Mon Sep 17 00:00:00 2001 From: Pythyu Date: Tue, 19 Mar 2024 16:25:37 +0100 Subject: [PATCH 15/45] feat(agent.py): black python formatting --- tasks/agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/agent.py b/tasks/agent.py index 62801b15cd395..4a0589c3b6200 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -1010,7 +1010,7 @@ def omnibus_build( python_mirror=None, pip_config_file="pip.conf", host_distribution=None, - install_directory="/opt/datadog-agent" + install_directory="/opt/datadog-agent", ): """ Build the Agent packages with Omnibus Installer. From 913de28569bd92c86207e1bce890b7312da5b1d7 Mon Sep 17 00:00:00 2001 From: Pythyu Date: Wed, 20 Mar 2024 12:35:20 +0100 Subject: [PATCH 16/45] feat(agent.py): lint-python use generator instead of map --- tasks/agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/agent.py b/tasks/agent.py index 4a0589c3b6200..4aed7bd72d694 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -862,7 +862,7 @@ def _get_build_images(ctx): # We intentionally include both build images & their test suffixes in the pattern # as a test image and the merged version shouldn't share their cache tags = ctx.run("grep -E 'DATADOG_AGENT_.*BUILDIMAGES' .gitlab-ci.yml | cut -d ':' -f 2", hide='stdout').stdout - return map(lambda t: t.strip(), tags.splitlines()) + return (t.strip() for t in tags.splitlines()) def _get_environment_for_cache() -> dict: From 5fd51a23232db965d824995e7f753c692a53aa38 Mon Sep 17 00:00:00 2001 From: Pythyu Date: Wed, 20 Mar 2024 16:19:31 +0100 Subject: [PATCH 17/45] feat(PR): apply suggestions --- .gitlab/package_build/deb.yml | 2 - .gitlab/package_build/remote_updater.yml | 2 - omnibus/config/software/snmp-traps.rb | 1 + tasks/agent.py | 129 +---------------------- tasks/libs/omnibus_cache.py | 128 ++++++++++++++++++++++ 5 files changed, 131 insertions(+), 131 deletions(-) create mode 100644 tasks/libs/omnibus_cache.py diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml index 1226fdd5e67fd..4e2e6ff546a5e 100644 --- a/.gitlab/package_build/deb.yml +++ b/.gitlab/package_build/deb.yml @@ -293,7 +293,6 @@ updater_deb-amd64: before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 - updater_deb-arm64: extends: .updater_build_common_deb rules: !reference [.on_a7] @@ -309,7 +308,6 @@ updater_deb-arm64: before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 - agentless_scanner_deb-x64: rules: !reference [.on_a7] diff --git a/.gitlab/package_build/remote_updater.yml b/.gitlab/package_build/remote_updater.yml index 20e9763d768a8..ad5e13974145b 100644 --- a/.gitlab/package_build/remote_updater.yml +++ b/.gitlab/package_build/remote_updater.yml @@ -54,7 +54,6 @@ agent_remote_updater-x64-a7: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 - # build Agent package for remote_updater-x64 agent_remote_updater-arm64-a7: extends: .agent_build_common_remote_updater @@ -78,4 +77,3 @@ agent_remote_updater-arm64-a7: before_script: - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 - diff --git a/omnibus/config/software/snmp-traps.rb b/omnibus/config/software/snmp-traps.rb index a08b4e4217e1b..bb13fc542fd3b 100644 --- a/omnibus/config/software/snmp-traps.rb +++ b/omnibus/config/software/snmp-traps.rb @@ -5,6 +5,7 @@ :sha256 => "04fb9d43754c2656edf35f08fbad11ba8dc20d52654962933f3dd8f4d463b42c", :target_filename => "dd_traps_db.json.gz" + build do # The dir for confs if osx_target? diff --git a/tasks/agent.py b/tasks/agent.py index 4aed7bd72d694..2d6efd0120bd5 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -33,6 +33,7 @@ load_release_versions, timed, ) +from tasks.libs.omnibus_cache import omnibus_compute_cache_key from tasks.rtloader import clean as rtloader_clean from tasks.rtloader import install as rtloader_install from tasks.rtloader import make as rtloader_make @@ -858,132 +859,6 @@ def _send_build_metrics(ctx, overall_duration): print(r.text) -def _get_build_images(ctx): - # We intentionally include both build images & their test suffixes in the pattern - # as a test image and the merged version shouldn't share their cache - tags = ctx.run("grep -E 'DATADOG_AGENT_.*BUILDIMAGES' .gitlab-ci.yml | cut -d ':' -f 2", hide='stdout').stdout - return (t.strip() for t in tags.splitlines()) - - -def _get_environment_for_cache() -> dict: - """ - Compute a hash from the environment after excluding irrelevant/insecure - environment variables to ensure we don't omit a variable - """ - - def env_filter(item): - key = item[0] - excluded_prefixes = [ - 'AGENT_', - 'ARTIFACTORY_', - 'AWS_', - 'BUILDENV_', - 'CI_', - 'CLUSTER_AGENT_', - 'DATADOG_AGENT_', - 'DD_', - 'DEB_', - 'DESTINATION_', - 'DOCKER_', - 'FF_', - 'GITLAB_', - 'GIT_', - 'K8S_', - 'KERNEL_MATRIX_TESTING_', - 'KUBERNETES_', - 'OMNIBUS_', - 'POD_', - 'RELEASE_VERSION', - 'RPM_', - 'S3_', - 'TEST_INFRA_', - 'USE_', - 'VAULT_', - 'WINDOWS_', - ] - excluded_suffixes = [ - '_SHA256', - '_VERSION', - ] - excluded_values = [ - "AVAILABILITY_ZONE", - "BENCHMARKS_CI_IMAGE", - "BUCKET_BRANCH", - "BUNDLER_VERSION", - "CHANNEL", - "CI", - "CONSUL_HTTP_ADDR", - "DOGSTATSD_BINARIES_DIR", - "EXPERIMENTS_EVALUATION_ADDRESS", - "GCC_VERSION", - "GCE_METADATA_HOST", - "GENERAL_ARTIFACTS_CACHE_BUCKET_URL", - "GET_SOURCES_ATTEMPTS", - "HOME", - "HOSTNAME", - "HOST_IP", - "INTEGRATION_WHEELS_CACHE_BUCKET", - "IRBRC", - "KITCHEN_INFRASTRUCTURE_FLAKES_RETRY", - "LESSCLOSE", - "LESSOPEN", - "LC_CTYPE", - "LS_COLORS", - "MACOS_S3_BUCKET", - "MESSAGE", - "OLDPWD", - "PROCESS_S3_BUCKET", - "PWD", - "PYTHON_RUNTIMES", - "RUN_ALL_BUILDS", - "RUN_KITCHEN_TESTS", - "RUNNER_TEMP_PROJECT_DIR", - "RUSTC_SHA256", - "RUST_VERSION", - "SHLVL", - "STATIC_BINARIES_DIR", - "STATSD_URL", - "SYSTEM_PROBE_BINARIES_DIR", - "TRACE_AGENT_URL", - "USE_CACHING_PROXY_PYTHON", - "USE_CACHING_PROXY_RUBY", - "USE_S3_CACHING", - "WIN_S3_BUCKET", - "_", - "build_before", - ] - for p in excluded_prefixes: - if key.startswith(p): - return False - for s in excluded_suffixes: - if key.endswith(s): - return False - if key in excluded_values: - return False - return True - - return dict(filter(env_filter, sorted(os.environ.items()))) - - -def _omnibus_compute_cache_key(ctx): - print('Computing cache key') - h = hashlib.sha1() - omnibus_last_commit = ctx.run('git log -n 1 --pretty=format:%H omnibus/', hide='stdout').stdout - h.update(str.encode(omnibus_last_commit)) - print(f'\tLast omnibus commit is {omnibus_last_commit}') - buildimages_hash = _get_build_images(ctx) - for img_hash in buildimages_hash: - h.update(str.encode(img_hash)) - environment = _get_environment_for_cache() - for k, v in environment.items(): - print(f'\tUsing environment variable {k} to compute cache key') - h.update(str.encode(f'{k}={v}')) - # FIXME: include omnibus-ruby and omnibus-software version once they are pinned - cache_key = h.hexdigest() - print(f'Cache key: {cache_key}') - return cache_key - - # hardened-runtime needs to be set to False to build on MacOS < 10.13.6, as the -o runtime option is not supported. @task( help={ @@ -1073,7 +948,7 @@ def omnibus_build( use_remote_cache = remote_cache_name is not None if use_remote_cache: cache_state = None - cache_key = _omnibus_compute_cache_key(ctx) + cache_key = omnibus_compute_cache_key(ctx) git_cache_url = f"s3://{os.environ['S3_OMNIBUS_CACHE_BUCKET']}/builds/{cache_key}/{remote_cache_name}" bundle_path = "/tmp/omnibus-git-cache-bundle" with timed(quiet=True) as restore_cache: diff --git a/tasks/libs/omnibus_cache.py b/tasks/libs/omnibus_cache.py new file mode 100644 index 0000000000000..ec8e1575156bc --- /dev/null +++ b/tasks/libs/omnibus_cache.py @@ -0,0 +1,128 @@ +import hashlib +import os + + +def _get_build_images(ctx): + # We intentionally include both build images & their test suffixes in the pattern + # as a test image and the merged version shouldn't share their cache + tags = ctx.run("grep -E 'DATADOG_AGENT_.*BUILDIMAGES' .gitlab-ci.yml | cut -d ':' -f 2", hide='stdout').stdout + return (t.strip() for t in tags.splitlines()) + + +def _get_environment_for_cache() -> dict: + """ + Compute a hash from the environment after excluding irrelevant/insecure + environment variables to ensure we don't omit a variable + """ + + def env_filter(item): + key = item[0] + excluded_prefixes = [ + 'AGENT_', + 'ARTIFACTORY_', + 'AWS_', + 'BUILDENV_', + 'CI_', + 'CLUSTER_AGENT_', + 'DATADOG_AGENT_', + 'DD_', + 'DEB_', + 'DESTINATION_', + 'DOCKER_', + 'FF_', + 'GITLAB_', + 'GIT_', + 'K8S_', + 'KERNEL_MATRIX_TESTING_', + 'KUBERNETES_', + 'OMNIBUS_', + 'POD_', + 'RELEASE_VERSION', + 'RPM_', + 'S3_', + 'TEST_INFRA_', + 'USE_', + 'VAULT_', + 'WINDOWS_', + ] + excluded_suffixes = [ + '_SHA256', + '_VERSION', + ] + excluded_values = [ + "AVAILABILITY_ZONE", + "BENCHMARKS_CI_IMAGE", + "BUCKET_BRANCH", + "BUNDLER_VERSION", + "CHANNEL", + "CI", + "CONSUL_HTTP_ADDR", + "DOGSTATSD_BINARIES_DIR", + "EXPERIMENTS_EVALUATION_ADDRESS", + "GCC_VERSION", + "GCE_METADATA_HOST", + "GENERAL_ARTIFACTS_CACHE_BUCKET_URL", + "GET_SOURCES_ATTEMPTS", + "HOME", + "HOSTNAME", + "HOST_IP", + "INTEGRATION_WHEELS_CACHE_BUCKET", + "IRBRC", + "KITCHEN_INFRASTRUCTURE_FLAKES_RETRY", + "LESSCLOSE", + "LESSOPEN", + "LC_CTYPE", + "LS_COLORS", + "MACOS_S3_BUCKET", + "MESSAGE", + "OLDPWD", + "PROCESS_S3_BUCKET", + "PWD", + "PYTHON_RUNTIMES", + "RUN_ALL_BUILDS", + "RUN_KITCHEN_TESTS", + "RUNNER_TEMP_PROJECT_DIR", + "RUSTC_SHA256", + "RUST_VERSION", + "SHLVL", + "STATIC_BINARIES_DIR", + "STATSD_URL", + "SYSTEM_PROBE_BINARIES_DIR", + "TRACE_AGENT_URL", + "USE_CACHING_PROXY_PYTHON", + "USE_CACHING_PROXY_RUBY", + "USE_S3_CACHING", + "WIN_S3_BUCKET", + "_", + "build_before", + ] + for p in excluded_prefixes: + if key.startswith(p): + return False + for s in excluded_suffixes: + if key.endswith(s): + return False + if key in excluded_values: + return False + return True + + return dict(filter(env_filter, sorted(os.environ.items()))) + + +def omnibus_compute_cache_key(ctx): + print('Computing cache key') + h = hashlib.sha1() + omnibus_last_commit = ctx.run('git log -n 1 --pretty=format:%H omnibus/', hide='stdout').stdout + h.update(str.encode(omnibus_last_commit)) + print(f'\tLast omnibus commit is {omnibus_last_commit}') + buildimages_hash = _get_build_images(ctx) + for img_hash in buildimages_hash: + h.update(str.encode(img_hash)) + environment = _get_environment_for_cache() + for k, v in environment.items(): + print(f'\tUsing environment variable {k} to compute cache key') + h.update(str.encode(f'{k}={v}')) + # FIXME: include omnibus-ruby and omnibus-software version once they are pinned + cache_key = h.hexdigest() + print(f'Cache key: {cache_key}') + return cache_key From ee0a2e6c2ab5975fcd24bb1b1a9759c761d3680e Mon Sep 17 00:00:00 2001 From: Pythyu Date: Wed, 20 Mar 2024 16:20:12 +0100 Subject: [PATCH 18/45] feat(PR): apply suggestions --- tasks/libs/omnibus_cache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tasks/libs/omnibus_cache.py b/tasks/libs/omnibus_cache.py index ec8e1575156bc..59c071f5423f5 100644 --- a/tasks/libs/omnibus_cache.py +++ b/tasks/libs/omnibus_cache.py @@ -59,7 +59,6 @@ def env_filter(item): "CONSUL_HTTP_ADDR", "DOGSTATSD_BINARIES_DIR", "EXPERIMENTS_EVALUATION_ADDRESS", - "GCC_VERSION", "GCE_METADATA_HOST", "GENERAL_ARTIFACTS_CACHE_BUCKET_URL", "GET_SOURCES_ATTEMPTS", From 1e44837bcc6b250fdd017e762ef77ef230224c91 Mon Sep 17 00:00:00 2001 From: Pythyu Date: Tue, 2 Apr 2024 11:54:03 +0200 Subject: [PATCH 19/45] feat(linter): fix flake8 unused python import --- tasks/agent.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tasks/agent.py b/tasks/agent.py index 2d6efd0120bd5..31a6b1530adb2 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -4,7 +4,6 @@ import ast import glob -import hashlib import json import os import platform From e8d83e2788e7a27931df1c7f653a34892200f2c3 Mon Sep 17 00:00:00 2001 From: Pythyu Date: Thu, 4 Apr 2024 18:03:26 +0200 Subject: [PATCH 20/45] feat(CODEOWNERS): update codeowners for agent-build-and-release to own omnibus_cache.py --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index af7c851da15c0..1fea90fd8c9d3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -499,6 +499,7 @@ /tasks/components.py @DataDog/agent-shared-components /tasks/components_templates @DataDog/agent-shared-components /tasks/updater.py @DataDog/fleet +/tasks/libs/omnibus_cache.py @DataDog/agent-build-and-releases /test/ @DataDog/agent-developer-tools /test/benchmarks/ @DataDog/agent-metrics-logs /test/benchmarks/kubernetes_state/ @DataDog/container-integrations From 16d0588e624e0195c32c8919db04b54a792f0e6a Mon Sep 17 00:00:00 2001 From: Pythyu Date: Fri, 5 Apr 2024 10:14:24 +0200 Subject: [PATCH 21/45] feat(PR): apply suggestion, enable cache on windows --- .gitlab/package_build/windows.yml | 1 + tasks/agent.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/package_build/windows.yml b/.gitlab/package_build/windows.yml index a0e8019824ceb..e95aa4db418ac 100644 --- a/.gitlab/package_build/windows.yml +++ b/.gitlab/package_build/windows.yml @@ -21,6 +21,7 @@ -e CI_JOB_NAME_SLUG=${CI_JOB_NAME_SLUG} -e CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} -e OMNIBUS_TARGET=${OMNIBUS_TARGET} + -e OMNIBUS_GIT_CACHE_DIR="/tmp/omnibus-git-cache" -e WINDOWS_BUILDER=true -e RELEASE_VERSION="$RELEASE_VERSION" -e MAJOR_VERSION="$AGENT_MAJOR_VERSION" diff --git a/tasks/agent.py b/tasks/agent.py index 31a6b1530adb2..30815e1bb24c9 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -938,7 +938,7 @@ def omnibus_build( omnibus_cache_dir = os.environ.get('OMNIBUS_GIT_CACHE_DIR') use_omnibus_git_cache = omnibus_cache_dir is not None if use_omnibus_git_cache: - omnibus_cache_dir += install_directory + omnibus_cache_dir = os.path.join(omnibus_cache_dir, install_directory) remote_cache_name = os.environ.get('CI_JOB_NAME_SLUG') # We don't want to update the cache when not running on a CI # Individual developers are still able to leverage the cache by providing From 2c8b4c7173035d3b64b2e0fd6d7e15e52cfa709e Mon Sep 17 00:00:00 2001 From: Pythyu Date: Fri, 5 Apr 2024 10:49:08 +0200 Subject: [PATCH 22/45] fix(path): os path join fix --- tasks/agent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tasks/agent.py b/tasks/agent.py index 30815e1bb24c9..957153e03d3c6 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -938,6 +938,8 @@ def omnibus_build( omnibus_cache_dir = os.environ.get('OMNIBUS_GIT_CACHE_DIR') use_omnibus_git_cache = omnibus_cache_dir is not None if use_omnibus_git_cache: + if install_directory[0] == "/": + install_directory = install_directory[1:] omnibus_cache_dir = os.path.join(omnibus_cache_dir, install_directory) remote_cache_name = os.environ.get('CI_JOB_NAME_SLUG') # We don't want to update the cache when not running on a CI From 93aab4345836560aa91b108d378562127eac0e4b Mon Sep 17 00:00:00 2001 From: Pythyu Date: Fri, 5 Apr 2024 13:25:25 +0200 Subject: [PATCH 23/45] feat(windows): correct windows path --- .gitlab/package_build/windows.yml | 2 +- tasks/winbuildscripts/dobuild.bat | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab/package_build/windows.yml b/.gitlab/package_build/windows.yml index e95aa4db418ac..5000086ba3ff2 100644 --- a/.gitlab/package_build/windows.yml +++ b/.gitlab/package_build/windows.yml @@ -21,7 +21,7 @@ -e CI_JOB_NAME_SLUG=${CI_JOB_NAME_SLUG} -e CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} -e OMNIBUS_TARGET=${OMNIBUS_TARGET} - -e OMNIBUS_GIT_CACHE_DIR="/tmp/omnibus-git-cache" + -e OMNIBUS_GIT_CACHE_DIR="C:\TEMP\omnibus-git-cache" -e WINDOWS_BUILDER=true -e RELEASE_VERSION="$RELEASE_VERSION" -e MAJOR_VERSION="$AGENT_MAJOR_VERSION" diff --git a/tasks/winbuildscripts/dobuild.bat b/tasks/winbuildscripts/dobuild.bat index 14858337cc83e..0ddc7cf6086f0 100644 --- a/tasks/winbuildscripts/dobuild.bat +++ b/tasks/winbuildscripts/dobuild.bat @@ -11,6 +11,7 @@ if NOT DEFINED GO_VERSION_CHECK set GO_VERSION_CHECK=%~4 set OMNIBUS_BUILD=agent.omnibus-build set OMNIBUS_ARGS=--python-runtimes "%PY_RUNTIMES%" +set INSTALL_DIR=opt\datadog-agent if "%OMNIBUS_TARGET%" == "iot" set OMNIBUS_ARGS=--flavor iot if "%OMNIBUS_TARGET%" == "dogstatsd" set OMNIBUS_BUILD=dogstatsd.omnibus-build && set OMNIBUS_ARGS= @@ -52,7 +53,7 @@ if "%GO_VERSION_CHECK%" == "true" ( ) @echo "inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --major-version %MAJOR_VERSION% --release-version %RELEASE_VERSION%" -inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --major-version %MAJOR_VERSION% --release-version %RELEASE_VERSION% || exit /b 105 +inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --major-version %MAJOR_VERSION% --release-version %RELEASE_VERSION% --install-directory=%INSTALL_DIR% || exit /b 105 REM only build MSI for main targets for now. if "%OMNIBUS_TARGET%" == "main" ( From d90f60996ab566ea10c5c6d42b9f56bf3c33df81 Mon Sep 17 00:00:00 2001 From: Pythyu Date: Fri, 5 Apr 2024 15:11:57 +0200 Subject: [PATCH 24/45] fix(omnibus_cache): use aws.cmd on windows --- tasks/agent.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tasks/agent.py b/tasks/agent.py index 957153e03d3c6..a965dc68055ee 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -922,6 +922,8 @@ def omnibus_build( elif agent_binaries: target_project = "agent-binaries" + aws_cmd = "aws.cmd" if sys.platform == 'win32' else "aws" + # Get the python_mirror from the PIP_INDEX_URL environment variable if it is not passed in the args python_mirror = python_mirror or os.environ.get("PIP_INDEX_URL") @@ -954,7 +956,7 @@ def omnibus_build( bundle_path = "/tmp/omnibus-git-cache-bundle" with timed(quiet=True) as restore_cache: # Allow failure in case the cache was evicted - if ctx.run(f"aws s3 cp --only-show-errors {git_cache_url} {bundle_path}", warn=True): + if ctx.run(f"{aws_cmd} s3 cp --only-show-errors {git_cache_url} {bundle_path}", warn=True): print(f'Successfully restored cache {cache_key}') ctx.run(f"git clone --mirror {bundle_path} {omnibus_cache_dir}") cache_state = ctx.run(f"git -C {omnibus_cache_dir} tag -l").stdout @@ -984,7 +986,7 @@ def omnibus_build( if use_remote_cache and ctx.run(f"git -C {omnibus_cache_dir} tag -l").stdout != cache_state: with timed(quiet=True) as update_cache: ctx.run(f"git -C {omnibus_cache_dir} bundle create {bundle_path} --tags") - ctx.run(f"aws s3 cp --only-show-errors {bundle_path} {git_cache_url}") + ctx.run(f"{aws_cmd} s3 cp --only-show-errors {bundle_path} {git_cache_url}") # Delete the temporary pip.conf file once the build is done os.remove(pip_config_file) From 3fe2d171299bdb6a7fc260fce4e8c4425af8538e Mon Sep 17 00:00:00 2001 From: Pythyu Date: Fri, 5 Apr 2024 16:26:40 +0200 Subject: [PATCH 25/45] fix(omnibus_cache): use correct git cache bundle path for windows --- tasks/agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/agent.py b/tasks/agent.py index 30ac0cb54ef82..402feb4011eea 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -955,7 +955,7 @@ def omnibus_build( cache_state = None cache_key = omnibus_compute_cache_key(ctx) git_cache_url = f"s3://{os.environ['S3_OMNIBUS_CACHE_BUCKET']}/builds/{cache_key}/{remote_cache_name}" - bundle_path = "/tmp/omnibus-git-cache-bundle" + bundle_path = "/tmp/omnibus-git-cache-bundle" if sys.platform != 'win32' else "C:\TEMP\omnibus-git-cache-bundle" with timed(quiet=True) as restore_cache: # Allow failure in case the cache was evicted if ctx.run(f"{aws_cmd} s3 cp --only-show-errors {git_cache_url} {bundle_path}", warn=True): From 838fd95cdd8ee3bc26945d075743960d34a90f71 Mon Sep 17 00:00:00 2001 From: Pythyu Date: Fri, 5 Apr 2024 16:27:00 +0200 Subject: [PATCH 26/45] fix(omnibus_cache): use correct git cache bundle path for windows --- tasks/agent.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tasks/agent.py b/tasks/agent.py index 402feb4011eea..5f09edf793599 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -955,7 +955,9 @@ def omnibus_build( cache_state = None cache_key = omnibus_compute_cache_key(ctx) git_cache_url = f"s3://{os.environ['S3_OMNIBUS_CACHE_BUCKET']}/builds/{cache_key}/{remote_cache_name}" - bundle_path = "/tmp/omnibus-git-cache-bundle" if sys.platform != 'win32' else "C:\TEMP\omnibus-git-cache-bundle" + bundle_path = ( + "/tmp/omnibus-git-cache-bundle" if sys.platform != 'win32' else "C:\TEMP\omnibus-git-cache-bundle" + ) with timed(quiet=True) as restore_cache: # Allow failure in case the cache was evicted if ctx.run(f"{aws_cmd} s3 cp --only-show-errors {git_cache_url} {bundle_path}", warn=True): From 978a516e6b9d3b6faef8ad53489df5add88ebf68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 12 Apr 2024 14:34:51 +0200 Subject: [PATCH 27/45] fix conflict resolution mistake --- .gitlab/package_build/installer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/package_build/installer.yml b/.gitlab/package_build/installer.yml index c9eff97f5fd4a..94fb2ba8a496b 100644 --- a/.gitlab/package_build/installer.yml +++ b/.gitlab/package_build/installer.yml @@ -94,7 +94,7 @@ datadog-agent-oci-arm64-a7: - rm -rf $OMNIBUS_PACKAGE_DIR/* # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. # Thus, we move the artifacts at the end in a gitlab-friendly dir. - - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project="installer" + - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project="installer" --install-directory="$INSTALL_DIR" - ls -la $OMNIBUS_PACKAGE_DIR - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-installer-*-${PACKAGE_ARCH}.tar.xz $S3_ARTIFACTS_URI/$DESTINATION_FILE - !reference [.upload_sbom_artifacts] From 031ca3e462a4cd6cec330d80aafbed8656a5ee7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 12 Apr 2024 16:22:41 +0200 Subject: [PATCH 28/45] omnibus cache: ignore ssh related env variables --- tasks/libs/common/omnibus.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index dbbdc2fa36365..07bc7a119dc11 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -45,6 +45,7 @@ def env_filter(item): 'RELEASE_VERSION', 'RPM_', 'S3_', + 'SSH_', 'TEST_INFRA_', 'USE_', 'VAULT_', From 5a8eef13da49abf85e7b9b5cd75a2376818dd2b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 12 Apr 2024 16:31:30 +0200 Subject: [PATCH 29/45] correctly dispatch the install dir --- tasks/libs/common/omnibus.py | 14 ++++++++++++++ tasks/omnibus.py | 11 +++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index 07bc7a119dc11..9f6c01e37e4eb 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -237,3 +237,17 @@ def send_build_metrics(ctx, overall_duration): else: print(f'Failed to send build metrics to DataDog: {r.status_code}') print(r.text) + + +def install_dir_for_product(product): + if product == "agent": + folder = 'datadog-agent' + elif product == 'dogstatsd': + folder = 'datadog-dogstatsd' + elif product == 'agentless-scanner': + folder = os.path.join('datadog', 'agentless-scanner') + else: + raise NotImplementedError(f'Unknown product {product}') + if sys.platform == 'win32': + return os.path.join('C:', 'opt', folder) + return os.path.join('/opt', folder) diff --git a/tasks/omnibus.py b/tasks/omnibus.py index a4bac2a3c5260..9ca98c757b427 100644 --- a/tasks/omnibus.py +++ b/tasks/omnibus.py @@ -5,7 +5,12 @@ from tasks.flavor import AgentFlavor from tasks.go import deps -from tasks.libs.common.omnibus import omnibus_compute_cache_key, send_build_metrics, should_retry_bundle_install +from tasks.libs.common.omnibus import ( + install_dir_for_product, + omnibus_compute_cache_key, + send_build_metrics, + should_retry_bundle_install, +) from tasks.libs.common.utils import get_version, load_release_versions, timed from tasks.ssm import get_pfx_pass, get_signing_cert @@ -167,7 +172,7 @@ def build( python_mirror=None, pip_config_file="pip.conf", host_distribution=None, - install_directory="/opt/datadog-agent", + install_directory=None, target_project=None, ): """ @@ -225,6 +230,8 @@ def build( omnibus_cache_dir = os.environ.get('OMNIBUS_GIT_CACHE_DIR') use_omnibus_git_cache = omnibus_cache_dir is not None if use_omnibus_git_cache: + if install_directory is None: + install_directory = install_dir_for_product(target_project) if install_directory[0] == "/": install_directory = install_directory[1:] omnibus_cache_dir = os.path.join(omnibus_cache_dir, install_directory) From 32d6afec6217dffb1f1ddfc21f0a4c4a59b270f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 12 Apr 2024 16:47:48 +0200 Subject: [PATCH 30/45] remove unneeded path sanitization os.path.join will handle this --- tasks/omnibus.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tasks/omnibus.py b/tasks/omnibus.py index 9ca98c757b427..6aaba2ff4f27f 100644 --- a/tasks/omnibus.py +++ b/tasks/omnibus.py @@ -232,8 +232,6 @@ def build( if use_omnibus_git_cache: if install_directory is None: install_directory = install_dir_for_product(target_project) - if install_directory[0] == "/": - install_directory = install_directory[1:] omnibus_cache_dir = os.path.join(omnibus_cache_dir, install_directory) remote_cache_name = os.environ.get('CI_JOB_NAME_SLUG') # We don't want to update the cache when not running on a CI From d505a223e65ae26802415be9e3f93b24454ae43d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 12 Apr 2024 16:48:22 +0200 Subject: [PATCH 31/45] debug --- tasks/omnibus.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tasks/omnibus.py b/tasks/omnibus.py index 6aaba2ff4f27f..6b1dd6b401cfc 100644 --- a/tasks/omnibus.py +++ b/tasks/omnibus.py @@ -233,6 +233,7 @@ def build( if install_directory is None: install_directory = install_dir_for_product(target_project) omnibus_cache_dir = os.path.join(omnibus_cache_dir, install_directory) + print('omnibus cache dir: ', omnibus_cache_dir) remote_cache_name = os.environ.get('CI_JOB_NAME_SLUG') # We don't want to update the cache when not running on a CI # Individual developers are still able to leverage the cache by providing From 48da5a2b37e86fdcf086367e717747498f4a9b05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 12 Apr 2024 17:14:05 +0200 Subject: [PATCH 32/45] fixes & debug --- tasks/libs/common/omnibus.py | 14 ++++++-------- tasks/omnibus.py | 5 +++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index 9f6c01e37e4eb..37a9f2a948db5 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -239,15 +239,13 @@ def send_build_metrics(ctx, overall_duration): print(r.text) -def install_dir_for_product(product): - if product == "agent": +def install_dir_for_project(project): + if project == "agent" or project == "iot-agent": folder = 'datadog-agent' - elif product == 'dogstatsd': + elif project == 'dogstatsd': folder = 'datadog-dogstatsd' - elif product == 'agentless-scanner': + elif project == 'agentless-scanner': folder = os.path.join('datadog', 'agentless-scanner') else: - raise NotImplementedError(f'Unknown product {product}') - if sys.platform == 'win32': - return os.path.join('C:', 'opt', folder) - return os.path.join('/opt', folder) + raise NotImplementedError(f'Unknown project {project}') + return os.path.join('opt', folder) diff --git a/tasks/omnibus.py b/tasks/omnibus.py index 6b1dd6b401cfc..62e72c2715428 100644 --- a/tasks/omnibus.py +++ b/tasks/omnibus.py @@ -6,7 +6,7 @@ from tasks.flavor import AgentFlavor from tasks.go import deps from tasks.libs.common.omnibus import ( - install_dir_for_product, + install_dir_for_project, omnibus_compute_cache_key, send_build_metrics, should_retry_bundle_install, @@ -231,8 +231,9 @@ def build( use_omnibus_git_cache = omnibus_cache_dir is not None if use_omnibus_git_cache: if install_directory is None: - install_directory = install_dir_for_product(target_project) + install_directory = install_dir_for_project(target_project) omnibus_cache_dir = os.path.join(omnibus_cache_dir, install_directory) + print('install dir:', install_directory) print('omnibus cache dir: ', omnibus_cache_dir) remote_cache_name = os.environ.get('CI_JOB_NAME_SLUG') # We don't want to update the cache when not running on a CI From 4cfe1910dcf9ea5f4480f1a5b22588ea7457f1aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 12 Apr 2024 17:50:29 +0200 Subject: [PATCH 33/45] don't provide an empty install path to the non-OCI installer builds --- .gitlab/package_build/installer.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab/package_build/installer.yml b/.gitlab/package_build/installer.yml index 94fb2ba8a496b..ace043293264d 100644 --- a/.gitlab/package_build/installer.yml +++ b/.gitlab/package_build/installer.yml @@ -94,7 +94,7 @@ datadog-agent-oci-arm64-a7: - rm -rf $OMNIBUS_PACKAGE_DIR/* # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. # Thus, we move the artifacts at the end in a gitlab-friendly dir. - - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project="installer" --install-directory="$INSTALL_DIR" + - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project="installer" ${INSTALL_DIR_PARAM} - ls -la $OMNIBUS_PACKAGE_DIR - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-installer-*-${PACKAGE_ARCH}.tar.xz $S3_ARTIFACTS_URI/$DESTINATION_FILE - !reference [.upload_sbom_artifacts] @@ -143,6 +143,8 @@ installer-amd64-oci: before_script: - source /root/.bashrc - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/$(inv agent.version -u)-1 + - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR + installer-arm64-oci: extends: installer-arm64 @@ -151,3 +153,4 @@ installer-arm64-oci: before_script: - source /root/.bashrc - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/$(inv agent.version -u)-1 + - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR From 22456272da5b2e1a7da2df797fd17680cafa5e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 12 Apr 2024 18:15:11 +0200 Subject: [PATCH 34/45] provide a mapping for the non-OCI installer --- tasks/libs/common/omnibus.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index 37a9f2a948db5..efbe039ed9614 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -246,6 +246,8 @@ def install_dir_for_project(project): folder = 'datadog-dogstatsd' elif project == 'agentless-scanner': folder = os.path.join('datadog', 'agentless-scanner') + elif project == 'installer': + folder = 'datadog-installer' else: raise NotImplementedError(f'Unknown project {project}') return os.path.join('opt', folder) From b6a0c5c50cc20685168ca69ed2d8ed08cf9ca037 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 12 Apr 2024 18:34:15 +0200 Subject: [PATCH 35/45] fix typo --- .gitlab/package_build/installer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/package_build/installer.yml b/.gitlab/package_build/installer.yml index ace043293264d..6cf8e3d928dc7 100644 --- a/.gitlab/package_build/installer.yml +++ b/.gitlab/package_build/installer.yml @@ -143,7 +143,7 @@ installer-amd64-oci: before_script: - source /root/.bashrc - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/$(inv agent.version -u)-1 - - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR + - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR" installer-arm64-oci: @@ -153,4 +153,4 @@ installer-arm64-oci: before_script: - source /root/.bashrc - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/$(inv agent.version -u)-1 - - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR + - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR" From 1313603738c28b3aa33430971438a2e8319a46fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Mon, 15 Apr 2024 11:00:01 +0200 Subject: [PATCH 36/45] reintroduce needed sanitization --- tasks/omnibus.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tasks/omnibus.py b/tasks/omnibus.py index 62e72c2715428..5653e5ef4d150 100644 --- a/tasks/omnibus.py +++ b/tasks/omnibus.py @@ -142,6 +142,9 @@ def get_omnibus_env( env['DEPLOY_AGENT'] = os.environ.get('DEPLOY_AGENT') if 'PACKAGE_ARCH' in os.environ: env['PACKAGE_ARCH'] = os.environ.get('PACKAGE_ARCH') + if 'INSTALL_DIR' in os.environ: + print('Forwarding INSTALL_DIR') + env['INSTALL_DIR'] = os.environ.get('INSTALL_DIR') return env @@ -232,9 +235,11 @@ def build( if use_omnibus_git_cache: if install_directory is None: install_directory = install_dir_for_project(target_project) + if install_directory[0] == "/": + # Is the path starts with a /, it's considered the new root for the joined path + # which effectively drops whatever was in omnibus_cache_dir + install_directory = install_directory[1:] omnibus_cache_dir = os.path.join(omnibus_cache_dir, install_directory) - print('install dir:', install_directory) - print('omnibus cache dir: ', omnibus_cache_dir) remote_cache_name = os.environ.get('CI_JOB_NAME_SLUG') # We don't want to update the cache when not running on a CI # Individual developers are still able to leverage the cache by providing From 424e169bf94d753f72302effcb3eed1a23f74fb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Mon, 15 Apr 2024 12:38:07 +0200 Subject: [PATCH 37/45] add more patterns to environment exclusion list --- tasks/libs/common/omnibus.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index efbe039ed9614..fee4fb6630553 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -24,27 +24,36 @@ def env_filter(item): key = item[0] excluded_prefixes = [ 'AGENT_', + 'API_KEY_', + 'APP_KEY_', 'ARTIFACTORY_', 'AWS_', 'BUILDENV_', 'CI_', + 'CHOCOLATEY_', 'CLUSTER_AGENT_', 'DATADOG_AGENT_', 'DD_', 'DEB_', 'DESTINATION_', 'DOCKER_', + 'E2E_TESTS_', + 'EMISSARY_', + 'EXECUTOR_', 'FF_', 'GITLAB_', 'GIT_', 'K8S_', + 'KITCHEN_', 'KERNEL_MATRIX_TESTING_', 'KUBERNETES_', + 'MACOS_GITHUB_', 'OMNIBUS_', 'POD_', 'RELEASE_VERSION', 'RPM_', 'S3_', + 'SMP_', 'SSH_', 'TEST_INFRA_', 'USE_', From 86a84d0653087ad298ad2973b8267de8f5e1664d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Mon, 15 Apr 2024 13:25:46 +0200 Subject: [PATCH 38/45] add a comment explainig the cache dir & install path Also, this will not cause the cache to be invalidated and can be used to test if everything still works --- tasks/omnibus.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tasks/omnibus.py b/tasks/omnibus.py index 5653e5ef4d150..fc8f4223b59c9 100644 --- a/tasks/omnibus.py +++ b/tasks/omnibus.py @@ -233,6 +233,12 @@ def build( omnibus_cache_dir = os.environ.get('OMNIBUS_GIT_CACHE_DIR') use_omnibus_git_cache = omnibus_cache_dir is not None if use_omnibus_git_cache: + # The cache will be written in the provided cache dir (see omnibus.rb) but + # the git repository itself will be located in a subfolder that replicates + # the install_dir hierarchy + # For instance if git_cache_dir is set to "/git/cache/dir" and install_dir is + # set to /a/b/c, the cache git repository will be located in + # /git/cache/dir/a/b/c/.git if install_directory is None: install_directory = install_dir_for_project(target_project) if install_directory[0] == "/": From 3092fc18d59f637b2677bdc49848674211d2c799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Mon, 15 Apr 2024 16:09:10 +0200 Subject: [PATCH 39/45] filter more env variables --- tasks/libs/common/omnibus.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index fee4fb6630553..993c43d7dce02 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -43,6 +43,7 @@ def env_filter(item): 'FF_', 'GITLAB_', 'GIT_', + 'JIRA_', 'K8S_', 'KITCHEN_', 'KERNEL_MATRIX_TESTING_', @@ -52,6 +53,7 @@ def env_filter(item): 'POD_', 'RELEASE_VERSION', 'RPM_', + 'RUN_', 'S3_', 'SMP_', 'SSH_', @@ -69,17 +71,21 @@ def env_filter(item): "BENCHMARKS_CI_IMAGE", "BUCKET_BRANCH", "BUNDLER_VERSION", + "CHANGELOG_COMMIT_SHA_SSM_NAME", + "CLANG_LLVM_VER", "CHANNEL", "CI", - "CONSUL_HTTP_ADDR", + "COMPUTERNAME" "CONSUL_HTTP_ADDR", "DOGSTATSD_BINARIES_DIR", "EXPERIMENTS_EVALUATION_ADDRESS", "GCE_METADATA_HOST", "GENERAL_ARTIFACTS_CACHE_BUCKET_URL", "GET_SOURCES_ATTEMPTS", + "GO_TEST_SKIP_FLAKE", "HOME", "HOSTNAME", "HOST_IP", + "INSTALL_SCRIPT_API_KEY_SSM_NAME", "INTEGRATION_WHEELS_CACHE_BUCKET", "IRBRC", "KITCHEN_INFRASTRUCTURE_FLAKES_RETRY", @@ -93,8 +99,7 @@ def env_filter(item): "PROCESS_S3_BUCKET", "PWD", "PYTHON_RUNTIMES", - "RUN_ALL_BUILDS", - "RUN_KITCHEN_TESTS", + "RESTORE_CACHE_ATTEMPTS", "RUNNER_TEMP_PROJECT_DIR", "RUSTC_SHA256", "RUST_VERSION", @@ -106,7 +111,12 @@ def env_filter(item): "USE_CACHING_PROXY_PYTHON", "USE_CACHING_PROXY_RUBY", "USE_S3_CACHING", + "USERDOMAIN", + "USERNAME", + "USERPROFILE", + "VCPKG_BLOB_SAS_URL_SSM_NAME", "WIN_S3_BUCKET", + "WINGET_PAT_SSM_NAME", "_", "build_before", ] From af4473b70abc95027d5ff70f51581ed0adcc5843 Mon Sep 17 00:00:00 2001 From: Alex Lopez Date: Tue, 16 Apr 2024 10:39:22 +0200 Subject: [PATCH 40/45] simplify install directory sanitization --- tasks/omnibus.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tasks/omnibus.py b/tasks/omnibus.py index fc8f4223b59c9..cd16a6b7dc4f6 100644 --- a/tasks/omnibus.py +++ b/tasks/omnibus.py @@ -241,10 +241,9 @@ def build( # /git/cache/dir/a/b/c/.git if install_directory is None: install_directory = install_dir_for_project(target_project) - if install_directory[0] == "/": - # Is the path starts with a /, it's considered the new root for the joined path - # which effectively drops whatever was in omnibus_cache_dir - install_directory = install_directory[1:] + # Is the path starts with a /, it's considered the new root for the joined path + # which effectively drops whatever was in omnibus_cache_dir + install_directory = install_directory.lstrip('/') omnibus_cache_dir = os.path.join(omnibus_cache_dir, install_directory) remote_cache_name = os.environ.get('CI_JOB_NAME_SLUG') # We don't want to update the cache when not running on a CI From bdf7369dae7e72243f202f4721e0589da7233291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Tue, 16 Apr 2024 10:41:56 +0200 Subject: [PATCH 41/45] remove old fixme --- tasks/libs/common/omnibus.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index 993c43d7dce02..0701c7299c2af 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -146,7 +146,6 @@ def omnibus_compute_cache_key(ctx): for k, v in environment.items(): print(f'\tUsing environment variable {k} to compute cache key') h.update(str.encode(f'{k}={v}')) - # FIXME: include omnibus-ruby and omnibus-software version once they are pinned cache_key = h.hexdigest() print(f'Cache key: {cache_key}') return cache_key From e9d2dad5654ce4e374fd726eae3fd5a0241b3eee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Tue, 16 Apr 2024 11:20:17 +0200 Subject: [PATCH 42/45] use omnibus commits in cache key --- tasks/libs/common/omnibus.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index 0701c7299c2af..3cff24e90128e 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -5,6 +5,7 @@ from datetime import datetime import requests +from release import _get_release_json_value def _get_build_images(ctx): @@ -14,6 +15,11 @@ def _get_build_images(ctx): return (t.strip() for t in tags.splitlines()) +def _get_omnibus_commits(field): + release_version = os.environ['RELEASE_VERSION_7'] + return _get_release_json_value(f'{release_version}::{field}') + + def _get_environment_for_cache() -> dict: """ Compute a hash from the environment after excluding irrelevant/insecure @@ -142,6 +148,10 @@ def omnibus_compute_cache_key(ctx): buildimages_hash = _get_build_images(ctx) for img_hash in buildimages_hash: h.update(str.encode(img_hash)) + omnibus_ruby_commit = _get_omnibus_commits('OMNIBUS_RUBY_VERSION') + omnibus_software_commit = _get_omnibus_commits('OMNIBUS_SOFTWARE_VERSION') + h.update(str.encode(omnibus_ruby_commit)) + h.update(str.encode(omnibus_software_commit)) environment = _get_environment_for_cache() for k, v in environment.items(): print(f'\tUsing environment variable {k} to compute cache key') From 3d78dd79939d51f34ef56307ece66180fee4585a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Tue, 16 Apr 2024 13:25:50 +0200 Subject: [PATCH 43/45] handle both RELEASE_VERSION and RELEASE_VERSION_7 --- tasks/libs/common/omnibus.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index 3cff24e90128e..69bc0e69b9aaf 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -16,7 +16,10 @@ def _get_build_images(ctx): def _get_omnibus_commits(field): - release_version = os.environ['RELEASE_VERSION_7'] + if 'RELEASE_VERSION' in os.environ: + release_version = os.environ['RELEASE_VERSION'] + else: + release_version = os.environ['RELEASE_VERSION_7'] return _get_release_json_value(f'{release_version}::{field}') From 590bd4c42866e7cad38cbf259f22256ec40e2ee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Tue, 16 Apr 2024 13:36:08 +0200 Subject: [PATCH 44/45] remove unneeded windows task parameter --- tasks/winbuildscripts/dobuild.bat | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tasks/winbuildscripts/dobuild.bat b/tasks/winbuildscripts/dobuild.bat index fe62060b07b05..863046d82a3f2 100644 --- a/tasks/winbuildscripts/dobuild.bat +++ b/tasks/winbuildscripts/dobuild.bat @@ -11,7 +11,6 @@ if NOT DEFINED GO_VERSION_CHECK set GO_VERSION_CHECK=%~4 set OMNIBUS_BUILD=omnibus.build set OMNIBUS_ARGS=--python-runtimes "%PY_RUNTIMES%" -set INSTALL_DIR=opt\datadog-agent if "%OMNIBUS_TARGET%" == "iot" set OMNIBUS_ARGS=--flavor iot if "%OMNIBUS_TARGET%" == "dogstatsd" set OMNIBUS_ARGS=--target-project dogstatsd @@ -53,7 +52,7 @@ if "%GO_VERSION_CHECK%" == "true" ( ) @echo "inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --major-version %MAJOR_VERSION% --release-version %RELEASE_VERSION%" -inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --major-version %MAJOR_VERSION% --release-version %RELEASE_VERSION% --install-directory=%INSTALL_DIR% || exit /b 105 +inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --major-version %MAJOR_VERSION% --release-version %RELEASE_VERSION% || exit /b 105 REM only build MSI for main targets for now. if "%OMNIBUS_TARGET%" == "main" ( From 98b6a706a29cf2f9d0e91290ececc914b2e220e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Wed, 17 Apr 2024 13:22:48 +0200 Subject: [PATCH 45/45] display omnibus commits sha1 --- tasks/libs/common/omnibus.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index 69bc0e69b9aaf..22d59f897fe49 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -153,6 +153,8 @@ def omnibus_compute_cache_key(ctx): h.update(str.encode(img_hash)) omnibus_ruby_commit = _get_omnibus_commits('OMNIBUS_RUBY_VERSION') omnibus_software_commit = _get_omnibus_commits('OMNIBUS_SOFTWARE_VERSION') + print(f'Omnibus ruby commit: {omnibus_ruby_commit}') + print(f'Omnibus software commit: {omnibus_software_commit}') h.update(str.encode(omnibus_ruby_commit)) h.update(str.encode(omnibus_software_commit)) environment = _get_environment_for_cache()