From 16d36595fe1464dd7bd7f0b33df361ed58a03252 Mon Sep 17 00:00:00 2001 From: Bryce Kahle Date: Tue, 30 Apr 2024 14:36:41 -0700 Subject: [PATCH 1/5] add caching of minimized BTFs --- .gitlab-ci.yml | 1 + .gitlab/binary_build/system_probe.yml | 1 + .gitlab/package_deps_build/package_deps_build.yml | 14 ++++++++++++++ tasks/system_probe.py | 6 ++++++ 4 files changed, 22 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d80100b0a80cc..e4cad813fd801 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -140,6 +140,7 @@ variables: S3_CP_OPTIONS: --no-progress --region us-east-1 --sse AES256 S3_CP_CMD: aws s3 cp $S3_CP_OPTIONS S3_ARTIFACTS_URI: s3://dd-ci-artefacts-build-stable/$CI_PROJECT_NAME/$CI_PIPELINE_ID + S3_PROJECT_ARTIFACTS_URI: s3://dd-ci-artefacts-build-stable/$CI_PROJECT_NAME S3_PERMANENT_ARTIFACTS_URI: s3://dd-ci-persistent-artefacts-build-stable/$CI_PROJECT_NAME S3_SBOM_STORAGE_URI: s3://sbom-root-us1-ddbuild-io/$CI_PROJECT_NAME/$CI_PIPELINE_ID S3_RELEASE_ARTIFACTS_URI: s3://dd-release-artifacts/$CI_PROJECT_NAME/$CI_PIPELINE_ID diff --git a/.gitlab/binary_build/system_probe.yml b/.gitlab/binary_build/system_probe.yml index cf68478a4e94a..756a7d736c306 100644 --- a/.gitlab/binary_build/system_probe.yml +++ b/.gitlab/binary_build/system_probe.yml @@ -24,6 +24,7 @@ expire_in: 2 weeks paths: - $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz + - $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz.sum build_system-probe-x64: stage: binary_build diff --git a/.gitlab/package_deps_build/package_deps_build.yml b/.gitlab/package_deps_build/package_deps_build.yml index 85f3b1a1d09f9..7fc8a54efe738 100644 --- a/.gitlab/package_deps_build/package_deps_build.yml +++ b/.gitlab/package_deps_build/package_deps_build.yml @@ -11,12 +11,26 @@ tags: ["arch:amd64"] script: - cd $CI_PROJECT_DIR + - export BTFS_ETAG=$(aws s3api head-object --bucket dd-agent-omnibus --key btfs/$BTFHUB_ARCHIVE_BRANCH/btfs-$ARCH.tar --query ETag --output text | tr -d \") + - export OUTPUTS_HASH=$(sha256sum sysprobe-build-outputs.tar.xz.sum | cut -d' ' -f1) + - export MIN_BTFS_FILENAME=minimized-btfs-$BTFS_ETAG-$OUTPUTS_HASH.tar.xz + - | + # if running all builds, or this is a release branch, skip the cache check + if [ "$RUN_ALL_BUILDS" != "true" && ! $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/ ]; then + if [ aws s3api head-object --bucket dd-ci-artefacts-build-stable --key $CI_PROJECT_NAME/btfs/$MIN_BTFS_FILENAME ]; then + $S3_CP_CMD $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME minimized-btfs.tar.xz + echo "cached minimized BTFs exist" + exit 0 + fi + fi + # cache does not exist, download processed BTFs and minimize - $S3_CP_CMD $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-$ARCH.tar . - tar -xf btfs-$ARCH.tar - tar -xf sysprobe-build-outputs.tar.xz - inv -e system-probe.generate-minimized-btfs --source-dir "$CI_PROJECT_DIR/btfs-$ARCH" --output-dir "$CI_PROJECT_DIR/minimized-btfs" --bpf-programs "$CI_PROJECT_DIR/pkg/ebpf/bytecode/build/co-re" - cd minimized-btfs - tar -cJf $CI_PROJECT_DIR/minimized-btfs.tar.xz * + - $S3_CP_CMD $CI_PROJECT_DIR/minimized-btfs.tar.xz $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME variables: KUBERNETES_MEMORY_REQUEST: "32Gi" KUBERNETES_MEMORY_LIMIT: "32Gi" diff --git a/tasks/system_probe.py b/tasks/system_probe.py index d5916c66312d5..7c2497070e7d1 100644 --- a/tasks/system_probe.py +++ b/tasks/system_probe.py @@ -1780,6 +1780,7 @@ def save_build_outputs(ctx, destfile): absdest = os.path.abspath(destfile) count = 0 + outfiles = [] with tempfile.TemporaryDirectory() as stagedir: with open("compile_commands.json") as compiledb: for outputitem in json.load(compiledb): @@ -1794,8 +1795,13 @@ def save_build_outputs(ctx, destfile): outdir = os.path.join(stagedir, filedir) ctx.run(f"mkdir -p {outdir}") ctx.run(f"cp {outputitem['output']} {outdir}/") + outfiles.append(outputitem['output']) count += 1 if count == 0: raise Exit(message="no build outputs captured") ctx.run(f"tar -C {stagedir} -cJf {absdest} .") + + outfiles.sort() + for outfile in outfiles: + ctx.run(f"sha256sum {outfile} >> {absdest}.sum") From 0c8121fab04e5e93b36583d645f5735323812497 Mon Sep 17 00:00:00 2001 From: Bryce Kahle Date: Tue, 30 Apr 2024 14:42:48 -0700 Subject: [PATCH 2/5] add --region --- .gitlab/package_deps_build/package_deps_build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/package_deps_build/package_deps_build.yml b/.gitlab/package_deps_build/package_deps_build.yml index 7fc8a54efe738..942f5f33ab60b 100644 --- a/.gitlab/package_deps_build/package_deps_build.yml +++ b/.gitlab/package_deps_build/package_deps_build.yml @@ -11,13 +11,13 @@ tags: ["arch:amd64"] script: - cd $CI_PROJECT_DIR - - export BTFS_ETAG=$(aws s3api head-object --bucket dd-agent-omnibus --key btfs/$BTFHUB_ARCHIVE_BRANCH/btfs-$ARCH.tar --query ETag --output text | tr -d \") + - export BTFS_ETAG=$(aws s3api head-object --region us-east-1 --bucket dd-agent-omnibus --key btfs/$BTFHUB_ARCHIVE_BRANCH/btfs-$ARCH.tar --query ETag --output text | tr -d \") - export OUTPUTS_HASH=$(sha256sum sysprobe-build-outputs.tar.xz.sum | cut -d' ' -f1) - export MIN_BTFS_FILENAME=minimized-btfs-$BTFS_ETAG-$OUTPUTS_HASH.tar.xz - | # if running all builds, or this is a release branch, skip the cache check if [ "$RUN_ALL_BUILDS" != "true" && ! $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/ ]; then - if [ aws s3api head-object --bucket dd-ci-artefacts-build-stable --key $CI_PROJECT_NAME/btfs/$MIN_BTFS_FILENAME ]; then + if [ aws s3api head-object --region us-east-1 --bucket dd-ci-artefacts-build-stable --key $CI_PROJECT_NAME/btfs/$MIN_BTFS_FILENAME ]; then $S3_CP_CMD $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME minimized-btfs.tar.xz echo "cached minimized BTFs exist" exit 0 From 2ba9659ad8b445243db1574133fe9cbe05b982dc Mon Sep 17 00:00:00 2001 From: Bryce Kahle Date: Tue, 30 Apr 2024 14:54:43 -0700 Subject: [PATCH 3/5] fix brackets --- .gitlab/package_deps_build/package_deps_build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/package_deps_build/package_deps_build.yml b/.gitlab/package_deps_build/package_deps_build.yml index 942f5f33ab60b..805bb076fb5e8 100644 --- a/.gitlab/package_deps_build/package_deps_build.yml +++ b/.gitlab/package_deps_build/package_deps_build.yml @@ -16,8 +16,8 @@ - export MIN_BTFS_FILENAME=minimized-btfs-$BTFS_ETAG-$OUTPUTS_HASH.tar.xz - | # if running all builds, or this is a release branch, skip the cache check - if [ "$RUN_ALL_BUILDS" != "true" && ! $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/ ]; then - if [ aws s3api head-object --region us-east-1 --bucket dd-ci-artefacts-build-stable --key $CI_PROJECT_NAME/btfs/$MIN_BTFS_FILENAME ]; then + if [[ "$RUN_ALL_BUILDS" != "true" && ! $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/ ]]; then + if [[ aws s3api head-object --region us-east-1 --bucket dd-ci-artefacts-build-stable --key $CI_PROJECT_NAME/btfs/$MIN_BTFS_FILENAME ]]; then $S3_CP_CMD $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME minimized-btfs.tar.xz echo "cached minimized BTFs exist" exit 0 From afdf517eac65cfd3a50072273c9e98adae01261f Mon Sep 17 00:00:00 2001 From: Bryce Kahle Date: Tue, 30 Apr 2024 14:56:46 -0700 Subject: [PATCH 4/5] use abs path for download --- .gitlab/package_deps_build/package_deps_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/package_deps_build/package_deps_build.yml b/.gitlab/package_deps_build/package_deps_build.yml index 805bb076fb5e8..a6fa969cb676e 100644 --- a/.gitlab/package_deps_build/package_deps_build.yml +++ b/.gitlab/package_deps_build/package_deps_build.yml @@ -18,7 +18,7 @@ # if running all builds, or this is a release branch, skip the cache check if [[ "$RUN_ALL_BUILDS" != "true" && ! $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/ ]]; then if [[ aws s3api head-object --region us-east-1 --bucket dd-ci-artefacts-build-stable --key $CI_PROJECT_NAME/btfs/$MIN_BTFS_FILENAME ]]; then - $S3_CP_CMD $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME minimized-btfs.tar.xz + $S3_CP_CMD $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME $CI_PROJECT_DIR/minimized-btfs.tar.xz echo "cached minimized BTFs exist" exit 0 fi From e6c682cb4ff0be5941162b4b4341128f63b5026c Mon Sep 17 00:00:00 2001 From: Bryce Kahle Date: Tue, 30 Apr 2024 15:13:22 -0700 Subject: [PATCH 5/5] don't need brackets for command --- .gitlab/package_deps_build/package_deps_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/package_deps_build/package_deps_build.yml b/.gitlab/package_deps_build/package_deps_build.yml index a6fa969cb676e..1899b69c64be8 100644 --- a/.gitlab/package_deps_build/package_deps_build.yml +++ b/.gitlab/package_deps_build/package_deps_build.yml @@ -17,7 +17,7 @@ - | # if running all builds, or this is a release branch, skip the cache check if [[ "$RUN_ALL_BUILDS" != "true" && ! $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/ ]]; then - if [[ aws s3api head-object --region us-east-1 --bucket dd-ci-artefacts-build-stable --key $CI_PROJECT_NAME/btfs/$MIN_BTFS_FILENAME ]]; then + if aws s3api head-object --region us-east-1 --bucket dd-ci-artefacts-build-stable --key $CI_PROJECT_NAME/btfs/$MIN_BTFS_FILENAME; then $S3_CP_CMD $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME $CI_PROJECT_DIR/minimized-btfs.tar.xz echo "cached minimized BTFs exist" exit 0