Enable omnibus build cache (#20117)

Co-authored-by: alopezz <[email protected]> Co-authored-by: Pythyu <[email protected]>
DataDog · Apr 17, 2024 · d3f3164 · d3f3164
1 parent 00f9f17
commit d3f3164
Show file tree

Hide file tree

Showing 14 changed files with 268 additions and 6 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -523,6 +523,7 @@
 /tasks/components.py                    @DataDog/agent-shared-components
 /tasks/components_templates             @DataDog/agent-shared-components
 /tasks/updater.py                       @DataDog/fleet
+/tasks/libs/omnibus_cache.py            @DataDog/agent-build-and-releases
 /tasks/installer.py                     @DataDog/fleet
 /test/                                  @DataDog/agent-developer-tools
 /test/benchmarks/                       @DataDog/agent-metrics-logs

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -149,6 +149,7 @@ variables:
   ## build to succeed with S3 caching disabled.
   S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable
   USE_S3_CACHING: --omnibus-s3-cache
+  OMNIBUS_GIT_CACHE_DIR: /tmp/omnibus-git-cache
   ## comment out the line below to disable integration wheels cache
   INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus
   S3_DD_AGENT_OMNIBUS_LLVM_URI: s3://dd-agent-omnibus/llvm

diff --git a/.gitlab/package_build/installer.yml b/.gitlab/package_build/installer.yml
@@ -21,7 +21,7 @@
     - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf
     # NOTE: for now, we consider "ociru" to be a "redhat_target" in omnibus/lib/ostools.rb
     # if we ever start building on a different platform, that might need to change
-    - inv -e omnibus.build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR  ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --host-distribution=ociru
+    - inv -e omnibus.build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR  ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --host-distribution=ociru --install-directory="$INSTALL_DIR"
     - ls -la $OMNIBUS_PACKAGE_DIR
     - !reference [.upload_sbom_artifacts]
   variables:
@@ -94,7 +94,7 @@ datadog-agent-oci-arm64-a7:
     - rm -rf $OMNIBUS_PACKAGE_DIR/*
     # Artifacts and cache must live within project directory but we run omnibus in a neutral directory.
     # Thus, we move the artifacts at the end in a gitlab-friendly dir.
-    - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project="installer"
+    - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project="installer" ${INSTALL_DIR_PARAM}
     - ls -la $OMNIBUS_PACKAGE_DIR
     - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-installer-*-${PACKAGE_ARCH}.tar.xz $S3_ARTIFACTS_URI/$DESTINATION_FILE
     - !reference [.upload_sbom_artifacts]
@@ -143,6 +143,8 @@ installer-amd64-oci:
   before_script:
     - source /root/.bashrc
     - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/$(inv agent.version -u)-1
+    - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR"
+
 
 installer-arm64-oci:
   extends: installer-arm64
@@ -151,3 +153,4 @@ installer-arm64-oci:
   before_script:
     - source /root/.bashrc
     - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/$(inv agent.version -u)-1
+    - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR"
diff --git a/.gitlab/package_build/windows.yml b/.gitlab/package_build/windows.yml
@@ -21,6 +21,7 @@
       -e CI_JOB_NAME_SLUG=${CI_JOB_NAME_SLUG}
       -e CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME}
       -e OMNIBUS_TARGET=${OMNIBUS_TARGET}
+      -e OMNIBUS_GIT_CACHE_DIR="C:\TEMP\omnibus-git-cache"
       -e WINDOWS_BUILDER=true
       -e RELEASE_VERSION="$RELEASE_VERSION"
       -e MAJOR_VERSION="$AGENT_MAJOR_VERSION"

diff --git a/omnibus/config/software/datadog-agent-finalize.rb b/omnibus/config/software/datadog-agent-finalize.rb
@@ -14,6 +14,8 @@
 
 skip_transitive_dependency_licensing true
 
+always_build true
+
 build do
     license :project_license
 

diff --git a/omnibus/config/software/datadog-agent-integrations-py2.rb b/omnibus/config/software/datadog-agent-integrations-py2.rb
@@ -21,6 +21,8 @@
 
 source git: 'https://github.com/DataDog/integrations-core.git'
 
+always_build true
+
 integrations_core_version = ENV['INTEGRATIONS_CORE_VERSION']
 if integrations_core_version.nil? || integrations_core_version.empty?
   integrations_core_version = 'master'

diff --git a/omnibus/config/software/datadog-agent-integrations-py3.rb b/omnibus/config/software/datadog-agent-integrations-py3.rb
@@ -21,6 +21,8 @@
 
 source git: 'https://github.com/DataDog/integrations-core.git'
 
+always_build true
+
 integrations_core_version = ENV['INTEGRATIONS_CORE_VERSION']
 if integrations_core_version.nil? || integrations_core_version.empty?
   integrations_core_version = 'master'

diff --git a/omnibus/config/software/datadog-agent.rb b/omnibus/config/software/datadog-agent.rb
@@ -18,6 +18,8 @@
 source path: '..'
 relative_path 'src/github.com/DataDog/datadog-agent'
 
+always_build true
+
 build do
   license :project_license
 

diff --git a/omnibus/config/software/datadog-security-agent-policies.rb b/omnibus/config/software/datadog-security-agent-policies.rb
@@ -20,6 +20,8 @@
 end
 default_version policies_version
 
+always_build true
+
 build do
   license "Apache-2.0"
   license_file "./LICENSE"

diff --git a/omnibus/config/software/system-probe.rb b/omnibus/config/software/system-probe.rb
@@ -8,6 +8,8 @@
 source path: '..'
 relative_path 'src/github.com/DataDog/datadog-agent'
 
+always_build true
+
 build do
   license :project_license
 

diff --git a/omnibus/omnibus.rb b/omnibus/omnibus.rb
@@ -37,4 +37,10 @@
     s3_instance_profile true
   end
 end
-use_git_caching false
+
+if not ENV.has_key?("OMNIBUS_GIT_CACHE_DIR")
+  use_git_caching false
+else
+  use_git_caching true
+  git_cache_dir ENV["OMNIBUS_GIT_CACHE_DIR"]
+end
diff --git a/tasks/agent.py b/tasks/agent.py
@@ -322,8 +322,8 @@ def refresh_assets(_, build_tags, development=True, flavor=AgentFlavor.base.name
         # Ensure the config folders are not world writable
         os.chmod(check_dir, mode=0o755)
 
-    ## add additional windows-only corechecks, only on windows. Otherwise the check loader
-    ## on linux will throw an error because the module is not found, but the config is.
+    # add additional windows-only corechecks, only on windows. Otherwise the check loader
+    # on linux will throw an error because the module is not found, but the config is.
     if sys.platform == 'win32':
         for check in WINDOWS_CORECHECKS:
             check_dir = os.path.join(dist_folder, f"conf.d/{check}.d/")

diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py
@@ -1,9 +1,169 @@
+import hashlib
 import json
 import os
 import sys
 from datetime import datetime
 
 import requests
+from release import _get_release_json_value
+
+
+def _get_build_images(ctx):
+    # We intentionally include both build images & their test suffixes in the pattern
+    # as a test image and the merged version shouldn't share their cache
+    tags = ctx.run("grep -E 'DATADOG_AGENT_.*BUILDIMAGES' .gitlab-ci.yml | cut -d ':' -f 2", hide='stdout').stdout
+    return (t.strip() for t in tags.splitlines())
+
+
+def _get_omnibus_commits(field):
+    if 'RELEASE_VERSION' in os.environ:
+        release_version = os.environ['RELEASE_VERSION']
+    else:
+        release_version = os.environ['RELEASE_VERSION_7']
+    return _get_release_json_value(f'{release_version}::{field}')
+
+
+def _get_environment_for_cache() -> dict:
+    """
+    Compute a hash from the environment after excluding irrelevant/insecure
+    environment variables to ensure we don't omit a variable
+    """
+
+    def env_filter(item):
+        key = item[0]
+        excluded_prefixes = [
+            'AGENT_',
+            'API_KEY_',
+            'APP_KEY_',
+            'ARTIFACTORY_',
+            'AWS_',
+            'BUILDENV_',
+            'CI_',
+            'CHOCOLATEY_',
+            'CLUSTER_AGENT_',
+            'DATADOG_AGENT_',
+            'DD_',
+            'DEB_',
+            'DESTINATION_',
+            'DOCKER_',
+            'E2E_TESTS_',
+            'EMISSARY_',
+            'EXECUTOR_',
+            'FF_',
+            'GITLAB_',
+            'GIT_',
+            'JIRA_',
+            'K8S_',
+            'KITCHEN_',
+            'KERNEL_MATRIX_TESTING_',
+            'KUBERNETES_',
+            'MACOS_GITHUB_',
+            'OMNIBUS_',
+            'POD_',
+            'RELEASE_VERSION',
+            'RPM_',
+            'RUN_',
+            'S3_',
+            'SMP_',
+            'SSH_',
+            'TEST_INFRA_',
+            'USE_',
+            'VAULT_',
+            'WINDOWS_',
+        ]
+        excluded_suffixes = [
+            '_SHA256',
+            '_VERSION',
+        ]
+        excluded_values = [
+            "AVAILABILITY_ZONE",
+            "BENCHMARKS_CI_IMAGE",
+            "BUCKET_BRANCH",
+            "BUNDLER_VERSION",
+            "CHANGELOG_COMMIT_SHA_SSM_NAME",
+            "CLANG_LLVM_VER",
+            "CHANNEL",
+            "CI",
+            "COMPUTERNAME" "CONSUL_HTTP_ADDR",
+            "DOGSTATSD_BINARIES_DIR",
+            "EXPERIMENTS_EVALUATION_ADDRESS",
+            "GCE_METADATA_HOST",
+            "GENERAL_ARTIFACTS_CACHE_BUCKET_URL",
+            "GET_SOURCES_ATTEMPTS",
+            "GO_TEST_SKIP_FLAKE",
+            "HOME",
+            "HOSTNAME",
+            "HOST_IP",
+            "INSTALL_SCRIPT_API_KEY_SSM_NAME",
+            "INTEGRATION_WHEELS_CACHE_BUCKET",
+            "IRBRC",
+            "KITCHEN_INFRASTRUCTURE_FLAKES_RETRY",
+            "LESSCLOSE",
+            "LESSOPEN",
+            "LC_CTYPE",
+            "LS_COLORS",
+            "MACOS_S3_BUCKET",
+            "MESSAGE",
+            "OLDPWD",
+            "PROCESS_S3_BUCKET",
+            "PWD",
+            "PYTHON_RUNTIMES",
+            "RESTORE_CACHE_ATTEMPTS",
+            "RUNNER_TEMP_PROJECT_DIR",
+            "RUSTC_SHA256",
+            "RUST_VERSION",
+            "SHLVL",
+            "STATIC_BINARIES_DIR",
+            "STATSD_URL",
+            "SYSTEM_PROBE_BINARIES_DIR",
+            "TRACE_AGENT_URL",
+            "USE_CACHING_PROXY_PYTHON",
+            "USE_CACHING_PROXY_RUBY",
+            "USE_S3_CACHING",
+            "USERDOMAIN",
+            "USERNAME",
+            "USERPROFILE",
+            "VCPKG_BLOB_SAS_URL_SSM_NAME",
+            "WIN_S3_BUCKET",
+            "WINGET_PAT_SSM_NAME",
+            "_",
+            "build_before",
+        ]
+        for p in excluded_prefixes:
+            if key.startswith(p):
+                return False
+        for s in excluded_suffixes:
+            if key.endswith(s):
+                return False
+        if key in excluded_values:
+            return False
+        return True
+
+    return dict(filter(env_filter, sorted(os.environ.items())))
+
+
+def omnibus_compute_cache_key(ctx):
+    print('Computing cache key')
+    h = hashlib.sha1()
+    omnibus_last_commit = ctx.run('git log -n 1 --pretty=format:%H omnibus/', hide='stdout').stdout
+    h.update(str.encode(omnibus_last_commit))
+    print(f'\tLast omnibus commit is {omnibus_last_commit}')
+    buildimages_hash = _get_build_images(ctx)
+    for img_hash in buildimages_hash:
+        h.update(str.encode(img_hash))
+    omnibus_ruby_commit = _get_omnibus_commits('OMNIBUS_RUBY_VERSION')
+    omnibus_software_commit = _get_omnibus_commits('OMNIBUS_SOFTWARE_VERSION')
+    print(f'Omnibus ruby commit: {omnibus_ruby_commit}')
+    print(f'Omnibus software commit: {omnibus_software_commit}')
+    h.update(str.encode(omnibus_ruby_commit))
+    h.update(str.encode(omnibus_software_commit))
+    environment = _get_environment_for_cache()
+    for k, v in environment.items():
+        print(f'\tUsing environment variable {k} to compute cache key')
+        h.update(str.encode(f'{k}={v}'))
+    cache_key = h.hexdigest()
+    print(f'Cache key: {cache_key}')
+    return cache_key
 
 
 def should_retry_bundle_install(res):
@@ -110,3 +270,17 @@ def send_build_metrics(ctx, overall_duration):
     else:
         print(f'Failed to send build metrics to DataDog: {r.status_code}')
         print(r.text)
+
+
+def install_dir_for_project(project):
+    if project == "agent" or project == "iot-agent":
+        folder = 'datadog-agent'
+    elif project == 'dogstatsd':
+        folder = 'datadog-dogstatsd'
+    elif project == 'agentless-scanner':
+        folder = os.path.join('datadog', 'agentless-scanner')
+    elif project == 'installer':
+        folder = 'datadog-installer'
+    else:
+        raise NotImplementedError(f'Unknown project {project}')
+    return os.path.join('opt', folder)