Enable omnibus build cache

.gitlab-ci.yml

@@ -154,6 +154,7 @@ variables:
   ## build to succeed with S3 caching disabled.
   S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable
   USE_S3_CACHING: --omnibus-s3-cache
+  OMNIBUS_GIT_CACHE_DIR: /tmp/omnibus-git-cache
   ## comment out the line below to disable integration wheels cache
   INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus
   S3_DD_AGENT_OMNIBUS_LLVM_URI: s3://dd-agent-omnibus/llvm

.gitlab/package_build/remote_updater.yml

@@ -18,7 +18,7 @@
     - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf
     # NOTE: for now, we consider "ociru" to be a "redhat_target" in omnibus/lib/ostools.rb
     # if we ever start building on a different platform, that might need to change
-    - inv -e agent.omnibus-build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR  ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --host-distribution=ociru
+    - inv -e agent.omnibus-build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR  ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --host-distribution=ociru --install-directory="$INSTALL_DIR"
     - ls -la $OMNIBUS_PACKAGE_DIR
     - !reference [.upload_sbom_artifacts]
   variables:

omnibus/config/software/datadog-agent-finalize.rb

@@ -14,6 +14,8 @@
 
 skip_transitive_dependency_licensing true
 
+always_build true
+
 build do
     license :project_license
 

omnibus/config/software/datadog-agent-integrations-py2.rb

@@ -22,6 +22,8 @@
 
 source git: 'https://github.com/DataDog/integrations-core.git'
 
+always_build true
+
 integrations_core_version = ENV['INTEGRATIONS_CORE_VERSION']
 if integrations_core_version.nil? || integrations_core_version.empty?
   integrations_core_version = 'master'

omnibus/config/software/datadog-agent-integrations-py3.rb

@@ -22,6 +22,8 @@
 
 source git: 'https://github.com/DataDog/integrations-core.git'
 
+always_build true
+
 integrations_core_version = ENV['INTEGRATIONS_CORE_VERSION']
 if integrations_core_version.nil? || integrations_core_version.empty?
   integrations_core_version = 'master'

omnibus/config/software/datadog-agent.rb

@@ -18,6 +18,8 @@
 source path: '..'
 relative_path 'src/github.com/DataDog/datadog-agent'
 
+always_build true
+
 build do
   license :project_license
 

omnibus/config/software/datadog-security-agent-policies.rb

@@ -20,6 +20,8 @@
 end
 default_version policies_version
 
+always_build true
+
 build do
   license "Apache-2.0"
   license_file "./LICENSE"

omnibus/config/software/system-probe.rb

@@ -8,6 +8,8 @@
 source path: '..'
 relative_path 'src/github.com/DataDog/datadog-agent'
 
+always_build true
+
 build do
   license :project_license
 

omnibus/omnibus.rb

@@ -37,4 +37,10 @@
     s3_instance_profile true
   end
 end
-use_git_caching false
+
+if not ENV.has_key?("OMNIBUS_GIT_CACHE_DIR")
+  use_git_caching false
+else
+  use_git_caching true
+  git_cache_dir ENV["OMNIBUS_GIT_CACHE_DIR"]
+end

tasks/agent.py

@@ -32,6 +32,7 @@
     load_release_versions,
     timed,
 )
+from tasks.libs.omnibus_cache import omnibus_compute_cache_key
 from tasks.rtloader import clean as rtloader_clean
 from tasks.rtloader import install as rtloader_install
 from tasks.rtloader import make as rtloader_make
@@ -322,8 +323,8 @@ def refresh_assets(_, build_tags, development=True, flavor=AgentFlavor.base.name
         # Ensure the config folders are not world writable
         os.chmod(check_dir, mode=0o755)
 
-    ## add additional windows-only corechecks, only on windows. Otherwise the check loader
-    ## on linux will throw an error because the module is not found, but the config is.
+    # add additional windows-only corechecks, only on windows. Otherwise the check loader
+    # on linux will throw an error because the module is not found, but the config is.
     if sys.platform == 'win32':
         for check in WINDOWS_CORECHECKS:
             check_dir = os.path.join(dist_folder, f"conf.d/{check}.d/")
@@ -883,6 +884,7 @@ def omnibus_build(
     python_mirror=None,
     pip_config_file="pip.conf",
     host_distribution=None,
+    install_directory="/opt/datadog-agent",
 ):
     """
     Build the Agent packages with Omnibus Installer.
@@ -933,6 +935,30 @@ def omnibus_build(
     with timed(quiet=True) as bundle_elapsed:
         bundle_install_omnibus(ctx, gem_path, env)
 
+    omnibus_cache_dir = os.environ.get('OMNIBUS_GIT_CACHE_DIR')
+    use_omnibus_git_cache = omnibus_cache_dir is not None
+    if use_omnibus_git_cache:
+        omnibus_cache_dir += install_directory
+        remote_cache_name = os.environ.get('CI_JOB_NAME_SLUG')
+        # We don't want to update the cache when not running on a CI
+        # Individual developers are still able to leverage the cache by providing
+        # the OMNIBUS_GIT_CACHE_DIR env variable, but they won't pull from the CI
+        # generated one.
+        use_remote_cache = remote_cache_name is not None
+        if use_remote_cache:
+            cache_state = None
+            cache_key = omnibus_compute_cache_key(ctx)
+            git_cache_url = f"s3://{os.environ['S3_OMNIBUS_CACHE_BUCKET']}/builds/{cache_key}/{remote_cache_name}"
+            bundle_path = "/tmp/omnibus-git-cache-bundle"
+            with timed(quiet=True) as restore_cache:
+                # Allow failure in case the cache was evicted
+                if ctx.run(f"aws s3 cp --only-show-errors {git_cache_url} {bundle_path}", warn=True):
+                    print(f'Successfully restored cache {cache_key}')
+                    ctx.run(f"git clone --mirror {bundle_path} {omnibus_cache_dir}")
+                    cache_state = ctx.run(f"git -C {omnibus_cache_dir} tag -l").stdout
+                else:
+                    print(f'Failed to restore cache from key {cache_key}')
+
     with timed(quiet=True) as omnibus_elapsed:
         omnibus_run_task(
             ctx=ctx,
@@ -945,6 +971,19 @@ def omnibus_build(
             host_distribution=host_distribution,
         )
 
+    if use_omnibus_git_cache:
+        stale_tags = ctx.run(f'git -C {omnibus_cache_dir} tag --no-merged', warn=True).stdout
+        # Purge the cache manually as omnibus will stick to not restoring a tag when
+        # a mismatch is detected, but will keep the old cached tags.
+        # Do this before checking for tag differences, in order to remove staled tags
+        # in case they were included in the bundle in a previous build
+        for _, tag in enumerate(stale_tags.split(os.linesep)):
+            ctx.run(f'git -C {omnibus_cache_dir} tag -d {tag}')
+        if use_remote_cache and ctx.run(f"git -C {omnibus_cache_dir} tag -l").stdout != cache_state:
+            with timed(quiet=True) as update_cache:
+                ctx.run(f"git -C {omnibus_cache_dir} bundle create {bundle_path} --tags")
+                ctx.run(f"aws s3 cp --only-show-errors {bundle_path} {git_cache_url}")
+
     # Delete the temporary pip.conf file once the build is done
     os.remove(pip_config_file)
 
@@ -953,6 +992,10 @@ def omnibus_build(
         print(f"Deps:    {deps_elapsed.duration}")
     print(f"Bundle:  {bundle_elapsed.duration}")
     print(f"Omnibus: {omnibus_elapsed.duration}")
+    if use_omnibus_git_cache and use_remote_cache:
+        print(f"Restoring omnibus cache: {restore_cache.duration}")
+        print(f"Updating omnibus cache: {update_cache.duration}")
+
     _send_build_metrics(ctx, omnibus_elapsed.duration)
 
 

tasks/libs/omnibus_cache.py

@@ -0,0 +1,127 @@
+import hashlib
+import os
+
+
+def _get_build_images(ctx):
+    # We intentionally include both build images & their test suffixes in the pattern
+    # as a test image and the merged version shouldn't share their cache
+    tags = ctx.run("grep -E 'DATADOG_AGENT_.*BUILDIMAGES' .gitlab-ci.yml | cut -d ':' -f 2", hide='stdout').stdout
+    return (t.strip() for t in tags.splitlines())
+
+
+def _get_environment_for_cache() -> dict:
+    """
+    Compute a hash from the environment after excluding irrelevant/insecure
+    environment variables to ensure we don't omit a variable
+    """
+
+    def env_filter(item):
+        key = item[0]
+        excluded_prefixes = [
+            'AGENT_',
+            'ARTIFACTORY_',
+            'AWS_',
+            'BUILDENV_',
+            'CI_',
+            'CLUSTER_AGENT_',
+            'DATADOG_AGENT_',
+            'DD_',
+            'DEB_',
+            'DESTINATION_',
+            'DOCKER_',
+            'FF_',
+            'GITLAB_',
+            'GIT_',
+            'K8S_',
+            'KERNEL_MATRIX_TESTING_',
+            'KUBERNETES_',
+            'OMNIBUS_',
+            'POD_',
+            'RELEASE_VERSION',
+            'RPM_',
+            'S3_',
+            'TEST_INFRA_',
+            'USE_',
+            'VAULT_',
+            'WINDOWS_',
+        ]
+        excluded_suffixes = [
+            '_SHA256',
+            '_VERSION',
+        ]
+        excluded_values = [
+            "AVAILABILITY_ZONE",
+            "BENCHMARKS_CI_IMAGE",
+            "BUCKET_BRANCH",
+            "BUNDLER_VERSION",
+            "CHANNEL",
+            "CI",
+            "CONSUL_HTTP_ADDR",
+            "DOGSTATSD_BINARIES_DIR",
+            "EXPERIMENTS_EVALUATION_ADDRESS",
+            "GCE_METADATA_HOST",
+            "GENERAL_ARTIFACTS_CACHE_BUCKET_URL",
+            "GET_SOURCES_ATTEMPTS",
+            "HOME",
+            "HOSTNAME",
+            "HOST_IP",
+            "INTEGRATION_WHEELS_CACHE_BUCKET",
+            "IRBRC",
+            "KITCHEN_INFRASTRUCTURE_FLAKES_RETRY",
+            "LESSCLOSE",
+            "LESSOPEN",
+            "LC_CTYPE",
+            "LS_COLORS",
+            "MACOS_S3_BUCKET",
+            "MESSAGE",
+            "OLDPWD",
+            "PROCESS_S3_BUCKET",
+            "PWD",
+            "PYTHON_RUNTIMES",
+            "RUN_ALL_BUILDS",
+            "RUN_KITCHEN_TESTS",
+            "RUNNER_TEMP_PROJECT_DIR",
+            "RUSTC_SHA256",
+            "RUST_VERSION",
+            "SHLVL",
+            "STATIC_BINARIES_DIR",
+            "STATSD_URL",
+            "SYSTEM_PROBE_BINARIES_DIR",
+            "TRACE_AGENT_URL",
+            "USE_CACHING_PROXY_PYTHON",
+            "USE_CACHING_PROXY_RUBY",
+            "USE_S3_CACHING",
+            "WIN_S3_BUCKET",
+            "_",
+            "build_before",
+        ]
+        for p in excluded_prefixes:
+            if key.startswith(p):
+                return False
+        for s in excluded_suffixes:
+            if key.endswith(s):
+                return False
+        if key in excluded_values:
+            return False
+        return True
+
+    return dict(filter(env_filter, sorted(os.environ.items())))
+
+
+def omnibus_compute_cache_key(ctx):
+    print('Computing cache key')
+    h = hashlib.sha1()
+    omnibus_last_commit = ctx.run('git log -n 1 --pretty=format:%H omnibus/', hide='stdout').stdout
+    h.update(str.encode(omnibus_last_commit))
+    print(f'\tLast omnibus commit is {omnibus_last_commit}')
+    buildimages_hash = _get_build_images(ctx)
+    for img_hash in buildimages_hash:
+        h.update(str.encode(img_hash))
+    environment = _get_environment_for_cache()
+    for k, v in environment.items():
+        print(f'\tUsing environment variable {k} to compute cache key')
+        h.update(str.encode(f'{k}={v}'))
+    # FIXME: include omnibus-ruby and omnibus-software version once they are pinned
+    cache_key = h.hexdigest()
+    print(f'Cache key: {cache_key}')
+    return cache_key