split dogstatsd build and packaging (#26045)

* split dogstatsd build and packaging

* remove reference to deleted file

* use the same rules for both dogstatsd binaries jobs

* rename packaging variable

* don't setup package script if we're not packaging

.gitlab/binary_build/linux.yml

@@ -53,9 +53,8 @@ build_dogstatsd-binary_x64:
 
 build_dogstatsd-binary_arm64:
   rules:
-    - !reference [.on_all_builds]
-    - !reference [.on_packaging_change]
-    - !reference [.on_go-version_change]
+    - !reference [.except_mergequeue]
+    - when: on_success
   stage: binary_build
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES
   tags: ["arch:arm64"]

.gitlab/package_build/include.yml

@@ -10,11 +10,8 @@
   - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
 
 include:
-  - .gitlab/package_build/deb.yml
   - .gitlab/package_build/heroku.yml
   - .gitlab/package_build/dmg.yml
   - .gitlab/package_build/installer.yml
-  - .gitlab/package_build/rpm.yml
-  - .gitlab/package_build/suse_rpm.yml
   - .gitlab/package_build/windows.yml
   - .gitlab/package_build/linux.yml

.gitlab/package_build/linux.yml

@@ -134,3 +134,43 @@ iot-agent-armhf:
     # if we use too many compression threads or a too agressive level
     FORCED_PACKAGE_COMPRESSION_LEVEL: 5
 
+.dogstatsd_build_common:
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
+  stage: package_build
+  script:
+    - source /root/.bashrc
+    - echo "About to build for $RELEASE_VERSION"
+    - !reference [.setup_ruby_mirror_linux]
+    - !reference [.setup_python_mirror_linux]
+    - !reference [.retrieve_linux_go_deps]
+    - !reference [.cache_omnibus_ruby_deps, setup]
+    # remove artifacts from previous pipelines that may come from the cache
+    - rm -rf $OMNIBUS_PACKAGE_DIR/*
+    - inv -e omnibus.build --release-version $RELEASE_VERSION_7 --major-version 7 --python-runtimes 3 --base-dir $OMNIBUS_BASE_DIR  ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project dogstatsd
+    - ls -la $OMNIBUS_PACKAGE_DIR
+    - !reference [.upload_sbom_artifacts]
+  variables:
+    KUBERNETES_CPU_REQUEST: 16
+    KUBERNETES_MEMORY_REQUEST: "32Gi"
+    KUBERNETES_MEMORY_LIMIT: "32Gi"
+  artifacts:
+    expire_in: 2 weeks
+    paths:
+      - $OMNIBUS_PACKAGE_DIR
+  cache:
+    - !reference [.cache_omnibus_ruby_deps, cache]
+
+dogstatsd-x64:
+  extends: .dogstatsd_build_common
+  needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"]
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+
+dogstatsd-arm64:
+  extends: .dogstatsd_build_common
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES
+  tags: ["arch:arm64"]
+  needs: ["go_mod_tidy_check", "build_dogstatsd-binary_arm64", "go_deps"]
+

.gitlab/packaging/deb.yml

@@ -177,4 +177,35 @@ iot_agent_deb-armhf:
     DD_PKG_ARCH: "arm64"
     FORCED_PACKAGE_COMPRESSION_LEVEL: 5
 
+dogstatsd_deb-x64:
+  extends: .package_deb_common
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
+  tags: ["arch:amd64"]
+  needs: ["dogstatsd-x64"]
+  variables:
+    DD_PROJECT: dogstatsd
+    PACKAGE_ARCH: amd64
+    DD_PKG_ARCH: "x86_64"
+    DESTINATION_DEB: "datadog-dogstatsd_amd64.deb"
+    RELEASE_VERSION: $RELEASE_VERSION_7
+    AGENT_MAJOR_VERSION: 7
+
+dogstatsd_deb-arm64:
+  extends: .package_deb_common
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
+  tags: ["arch:arm64"]
+  needs: ["dogstatsd-arm64"]
+  variables:
+    DD_PROJECT: dogstatsd
+    PACKAGE_ARCH: arm64
+    DD_PKG_ARCH: "arm64"
+    DESTINATION_DEB: "datadog-dogstatsd_arm64.deb"
+    RELEASE_VERSION: $RELEASE_VERSION_7
+    AGENT_MAJOR_VERSION: 7
 

.gitlab/packaging/rpm.yml

@@ -259,3 +259,26 @@ iot_agent_suse-x64:
   cache:
     - !reference [.cache_omnibus_ruby_deps, cache]
 
+dogstatsd_rpm-x64:
+  extends: .package_rpm_common
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  needs: ["dogstatsd-x64"]
+  variables:
+    PACKAGE_ARCH: x86_64
+    DD_PKG_ARCH: "x86_64"
+    DD_PROJECT: dogstatsd
+    RELEASE_VERSION: $RELEASE_VERSION_7
+    AGENT_MAJOR_VERSION: 7
+
+dogstatsd_suse-x64:
+  extends: .package_suse_rpm_common
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  needs: ["dogstatsd-x64"]
+  variables:
+    PACKAGE_ARCH: x86_64
+    DD_PKG_ARCH: "x86_64"
+    DD_PROJECT: dogstatsd
+    RELEASE_VERSION: $RELEASE_VERSION_7
+    AGENT_MAJOR_VERSION: 7

omnibus/config/projects/dogstatsd.rb

@@ -79,8 +79,38 @@
 # Generic package information
 # ------------------------------------
 
+if ENV["OMNIBUS_PACKAGE_ARTIFACT_DIR"]
+  dependency "package-artifact"
+  do_package = true
+else
+  do_package = false
+  # ------------------------------------
+  # Dependencies
+  # ------------------------------------
+
+  # creates required build directories
+  dependency 'datadog-agent-prepare'
+
+  # version manifest file
+  dependency 'version-manifest'
+
+  # Dogstatsd
+  dependency 'datadog-dogstatsd'
+
+  # this dependency puts few files out of the omnibus install dir and move them
+  # in the final destination. This way such files will be listed in the packages
+  # manifest and owned by the package manager. This is the only point in the build
+  # process where we operate outside the omnibus install dir, thus the need of
+  # the `extra_package_file` directive.
+  # This must be the last dependency in the project.
+
+  dependency 'datadog-dogstatsd-finalize'
+end
+
+
 # .deb specific flags
 package :deb do
+  skip_packager !do_package
   vendor 'Datadog <[email protected]>'
   epoch 1
   license 'Apache License Version 2.0'
@@ -99,6 +129,7 @@
 
 # .rpm specific flags
 package :rpm do
+  skip_packager !do_package
   vendor 'Datadog <[email protected]>'
   epoch 1
   dist_tag ''
@@ -123,7 +154,9 @@
 end
 
 package :xz do
-  skip_packager true
+  skip_packager do_package
+  compression_threads COMPRESSION_THREADS
+  compression_level COMPRESSION_LEVEL
 end
 
 package :msi do
@@ -167,34 +200,17 @@
   pkg_position '10, 10'
 end
 
-# ------------------------------------
-# Dependencies
-# ------------------------------------
-
-# creates required build directories
-dependency 'datadog-agent-prepare'
-
-# version manifest file
-dependency 'version-manifest'
-
-# Dogstatsd
-dependency 'datadog-dogstatsd'
-
-# this dependency puts few files out of the omnibus install dir and move them
-# in the final destination. This way such files will be listed in the packages
-# manifest and owned by the package manager. This is the only point in the build
-# process where we operate outside the omnibus install dir, thus the need of
-# the `extra_package_file` directive.
-# This must be the last dependency in the project.
-
-dependency 'datadog-dogstatsd-finalize'
-
 # package scripts
 if linux_target?
-  if debian_target?
-    package_scripts_path "#{Omnibus::Config.project_root}/package-scripts/dogstatsd-deb"
+  if !do_package
+    extra_package_file "#{Omnibus::Config.project_root}/package-scripts/dogstatsd-deb"
+    extra_package_file "#{Omnibus::Config.project_root}/package-scripts/dogstatsd-rpm"
   else
-    package_scripts_path "#{Omnibus::Config.project_root}/package-scripts/dogstatsd-rpm"
+    if debian_target?
+      package_scripts_path "#{Omnibus::Config.project_root}/package-scripts/dogstatsd-deb"
+    else
+      package_scripts_path "#{Omnibus::Config.project_root}/package-scripts/dogstatsd-rpm"
+    end
   end
 end