add caching of minimized BTFs (#25283)

* add caching of minimized BTFs * add --region * fix brackets * use abs path for download * don't need brackets for command
DataDog · Dec 4, 2024 · b676822 · b676822
1 parent 9e41c11
commit b676822
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 0 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -139,6 +139,7 @@ variables:
   S3_CP_OPTIONS: --only-show-errors --region us-east-1 --sse AES256
   S3_CP_CMD: aws s3 cp $S3_CP_OPTIONS
   S3_ARTIFACTS_URI: s3://dd-ci-artefacts-build-stable/$CI_PROJECT_NAME/$CI_PIPELINE_ID
+  S3_PROJECT_ARTIFACTS_URI: s3://dd-ci-artefacts-build-stable/$CI_PROJECT_NAME
   S3_PERMANENT_ARTIFACTS_URI: s3://dd-ci-persistent-artefacts-build-stable/$CI_PROJECT_NAME
   S3_SBOM_STORAGE_URI: s3://sbom-root-us1-ddbuild-io/$CI_PROJECT_NAME/$CI_PIPELINE_ID
   S3_RELEASE_ARTIFACTS_URI: s3://dd-release-artifacts/$CI_PROJECT_NAME/$CI_PIPELINE_ID

diff --git a/.gitlab/binary_build/system_probe.yml b/.gitlab/binary_build/system_probe.yml
@@ -23,6 +23,7 @@
     expire_in: 2 weeks
     paths:
       - $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz
+      - $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz.sum
 
 build_system-probe-x64:
   stage: binary_build

diff --git a/.gitlab/package_deps_build/package_deps_build.yml b/.gitlab/package_deps_build/package_deps_build.yml
@@ -11,12 +11,26 @@
   tags: ["arch:amd64"]
   script:
     - cd $CI_PROJECT_DIR
+    - export BTFS_ETAG=$(aws s3api head-object --region us-east-1 --bucket dd-agent-omnibus --key btfs/$BTFHUB_ARCHIVE_BRANCH/btfs-$ARCH.tar --query ETag --output text | tr -d \")
+    - export OUTPUTS_HASH=$(sha256sum sysprobe-build-outputs.tar.xz.sum | cut -d' ' -f1)
+    - export MIN_BTFS_FILENAME=minimized-btfs-$BTFS_ETAG-$OUTPUTS_HASH.tar.xz
+    - |
+      # if running all builds, or this is a release branch, skip the cache check
+      if [[ "$RUN_ALL_BUILDS" != "true" && ! $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/ ]]; then
+        if aws s3api head-object --region us-east-1 --bucket dd-ci-artefacts-build-stable --key $CI_PROJECT_NAME/btfs/$MIN_BTFS_FILENAME; then
+          $S3_CP_CMD $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME $CI_PROJECT_DIR/minimized-btfs.tar.xz
+          echo "cached minimized BTFs exist"
+          exit 0
+        fi
+      fi
+    # cache does not exist, download processed BTFs and minimize
     - $S3_CP_CMD $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-$ARCH.tar .
     - tar -xf btfs-$ARCH.tar
     - tar -xf sysprobe-build-outputs.tar.xz
     - inv -e system-probe.generate-minimized-btfs --source-dir "$CI_PROJECT_DIR/btfs-$ARCH" --output-dir "$CI_PROJECT_DIR/minimized-btfs" --input-bpf-programs "$CI_PROJECT_DIR/pkg/ebpf/bytecode/build/co-re"
     - cd minimized-btfs
     - tar -cJf $CI_PROJECT_DIR/minimized-btfs.tar.xz *
+    - $S3_CP_CMD $CI_PROJECT_DIR/minimized-btfs.tar.xz $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME
   variables:
     KUBERNETES_MEMORY_REQUEST: "6Gi"
     KUBERNETES_MEMORY_LIMIT: "12Gi"

diff --git a/tasks/system_probe.py b/tasks/system_probe.py
@@ -1871,6 +1871,7 @@ def save_build_outputs(ctx, destfile):
 
     absdest = os.path.abspath(destfile)
     count = 0
+    outfiles = []
     with tempfile.TemporaryDirectory() as stagedir:
         with open("compile_commands.json", "r") as compiledb:
             for outputitem in json.load(compiledb):
@@ -1885,8 +1886,13 @@ def save_build_outputs(ctx, destfile):
                 outdir = os.path.join(stagedir, filedir)
                 ctx.run(f"mkdir -p {outdir}")
                 ctx.run(f"cp {outputitem['output']} {outdir}/")
+                outfiles.append(outputitem['output'])
                 count += 1
 
         if count == 0:
             raise Exit(message="no build outputs captured")
         ctx.run(f"tar -C {stagedir} -cJf {absdest} .")
+
+    outfiles.sort()
+    for outfile in outfiles:
+        ctx.run(f"sha256sum {outfile} >> {absdest}.sum")