diff --git a/.circleci/README.md b/.circleci/README.md deleted file mode 100644 index 78dfeebaa200f1..00000000000000 --- a/.circleci/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# CircleCI - -CircleCI is used to run unit tests on Unix env. - -## Upgrading Golang version - -/!\ Disclaimer: the datadog/agent-buildimages-circleci-runner image should never be used for anything else than CircleCI tests /!\ - -This image is now built alongside other images in [agent-buildimages](https://github.com/DataDog/datadog-agent-buildimages) repository. Change of Golang version must occur in this repository. - -Once you have created a new image by building a new version of agent-buildimages, you can test your modification with the associated invoke task: - -```bash -invoke -e pipeline.update-buildimages --image-tag v12345678-c0mm1t5 -``` -This will update the configuration of circleci and gitlab to use the __test version__ of these images. -Once your test is successful, you can either move the `_test_version` from files or invoke -```bash -invoke -e pipeline.update-buildimages --image-tag v12345678-c0mm1t5 --no-test-version -``` - -If everything is green, get a review and merge the PR. diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 09d93d64823c87..00000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,356 +0,0 @@ -version: 2 - -# This file uses YAML anchors to deduplicate steps -# see https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/ -# and https://learnxinyminutes.com/docs/yaml/ - -experimental: - # For some reason filtering out notifications keeps being undocumented and - # marked as experimental but as of today, it's still working. - notify: - branches: - only: - - main - -templates: - job_template: &job_template - docker: - - image: gcr.io/datadoghq/agent-circleci-runner:v23168861-fcc48431 - environment: - USE_SYSTEM_LIBS: "1" - working_directory: /go/src/github.com/DataDog/datadog-agent - - step_templates: - - restore_cache: &restore_deps - keys: - # The first match will be used. Doing that so new branches - # use the default branch's cache but don't pollute it back. - # - # If incremental dep fails, increase the cache gen number - # in restore_deps AND save_deps - # See https://github.com/DataDog/datadog-agent/pull/2384 - - gen19-godeps-{{ checksum ".circleci/config.yml" }}-{{ .Branch }}-{{ .Revision }} - - gen19-godeps-{{ checksum ".circleci/config.yml" }}-{{ .Branch }}- - - gen19-godeps-{{ checksum ".circleci/config.yml" }}-main- - - save_cache: &save_deps - key: gen19-godeps-{{ checksum ".circleci/config.yml" }}-{{ .Branch }}-{{ .Revision }} - - restore_cache: &restore_source - keys: - # Cache retrieval is faster than full git checkout - - v5-repo-{{ .Revision }} - - save_cache: &save_source - key: v5-repo-{{ .Revision }} - -jobs: - checkout_code: - docker: - - image: alpine/git:latest - working_directory: /go/src/github.com/DataDog/datadog-agent - steps: - - checkout - - save_cache: - <<: *save_source - paths: - - /go/src/github.com/DataDog/datadog-agent - - dependencies: - <<: *job_template - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - name: check go version - command: | - inv -e check-go-version - - run: - name: grab go deps - command: | - inv -e deps - - run: - name: build rtloader - command: | - inv rtloader.make --install-prefix=/go/src/github.com/DataDog/datadog-agent/dev - inv rtloader.install - - run: - name: lint rtloader - command: | - inv rtloader.format --raise-if-changed - - run: - name: test rtloader - command: | - inv rtloader.test - - run: - name: install go tooling - command: | - inv -e install-tools - - save_cache: - <<: *save_deps - paths: - - /go/pkg/mod - - /go/bin - - /go/src/github.com/DataDog/datadog-agent/dev - - /usr/local/bin - - unit_tests: - <<: *job_template - resource_class: xlarge - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - name: run unit tests - no_output_timeout: 20m - command: inv -e test --rerun-fails=2 --python-runtimes 3 --coverage --race --profile --cpus 8 - - run: - name: upload code coverage results - # Never fail on coverage upload - command: inv -e codecov || true - - integration_tests: - <<: *job_template - resource_class: large - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - setup_remote_docker - - run: - name: run integration tests - command: inv -e integration-tests --race --remote-docker - - release_note: - <<: *job_template - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - command: inv -e lint-releasenote - name: run PR check for release note - - team_label: - <<: *job_template - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - command: inv -e lint-teamassignment - name: run PR check for team assignment labels - - milestone: - <<: *job_template - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - command: inv -e lint-milestone - name: run PR check for milestone assignment - - go_linting: - <<: *job_template - resource_class: xlarge - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - name: run go linters - no_output_timeout: 20m - command: inv -e lint-go - - licenses_linting: - <<: *job_template - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - name: run license linting - command: inv -e lint-licenses - - copyrights_linting: - <<: *job_template - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - name: run copyrights linting - command: inv -e lint-copyrights - - filename_linting: - <<: *job_template - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - name: run filename linting - command: inv -e lint-filenames - - shell_linting: - <<: *job_template - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - name: Install shellcheck - command: inv -e install-shellcheck - - run: - name: Print shellcheck version - command: shellcheck --version - - run: - name: Run shellcheck - #Excludes: - #SC2028: echo may not expand escape sequences. Use printf. - #SC2059: Don't use variables in the printf format string. Use printf "..%s.." "$foo". - command: shellcheck --severity=info -e SC2059 -e SC2028 --shell=bash ./cmd/**/*.sh ./omnibus/package-scripts/*/* - - python_linting: - <<: *job_template - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - name: setting env vars for click - command: | - echo 'export LC_ALL="C.UTF-8"' >> $BASH_ENV - echo 'export LANG="C.UTF-8"' >> $BASH_ENV - - run: - name: lint python files - command: inv -e lint-python - - component_linting: - <<: *job_template - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - name: run component linting - command: inv -e lint-components lint-fxutil-oneshot-test - - codeowner_linting: - <<: *job_template - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - name: run CODEOWNER linting - command: inv -e github.lint-codeowner - - docker_tests: - <<: *job_template - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - setup_remote_docker - - run: - name: run docker image tests - command: inv -e docker.test - - run: - name: run docker image integration tests - command: inv -e docker.integration-tests - - build_binaries: - <<: *job_template - resource_class: large - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - name: build dogstatsd - command: inv -e dogstatsd.build --static - - run: - name: build agent - command: inv -e agent.build --exclude-rtloader - - build_iot_agent: - <<: *job_template - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - name: build iot agent - command: inv -e agent.build --flavor iot - - run: - name: test iot agent - command: DD_HOSTNAME=test-circleci-hostname ./bin/agent/agent -c ./bin/agent/dist check cpu - - documentation_generation: - <<: *job_template - steps: - - restore_cache: *restore_source - - restore_cache: *restore_deps - - run: - name: generate doxygen documentation - command: inv -e rtloader.generate-doc - -workflows: - version: 2 - test_and_build: - jobs: - - checkout_code - - dependencies: - requires: - - checkout_code - - unit_tests: - requires: - - dependencies - - integration_tests: - requires: - - dependencies - - release_note: - filters: - branches: - ignore: - - main - requires: - - dependencies - - team_label: - filters: - branches: - ignore: - - main - requires: - - dependencies - - milestone: - filters: - branches: - ignore: - - main - requires: - - dependencies - - go_linting: - requires: - - dependencies - - licenses_linting: - requires: - - dependencies - - copyrights_linting: - requires: - - dependencies - - filename_linting: - requires: - - dependencies - - shell_linting: - requires: - - dependencies - - python_linting: - requires: - - dependencies - - component_linting: - requires: - - dependencies - - codeowner_linting: - requires: - - dependencies - - docker_tests: - requires: - - dependencies - - documentation_generation: - requires: - - dependencies - - build_binaries: - requires: - - unit_tests - - integration_tests - - docker_tests - - build_iot_agent: - requires: - - unit_tests - - integration_tests - - docker_tests diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 7822a4f1ad18ee..00000000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: "Code Scanning - Action" - -on: - push: - branches: - - main - - 7.[0-9][0-9].x - - mq-working-branch-* - pull_request: - branches: - - main - - 7.[0-9][0-9].x - -jobs: - CodeQL-Build: - runs-on: ubuntu-20.04 - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Setup Python3 - uses: actions/setup-python@v4 - with: - python-version: "3.11.5" - cache: "pip" - - run: pip3 install -r requirements.txt - - - name: Setup env variables - run: | - echo "CODEQL_PYTHON=$(which python3)" >> $GITHUB_ENV - echo "$GOPATH/bin" >> $GITHUB_PATH - echo "CGO_LDFLAGS= -L${GITHUB_WORKSPACE}/rtloader/build/rtloader -ldl " >> $GITHUB_ENV - echo "CGO_CFLAGS= -I${GITHUB_WORKSPACE}/rtloader/include -I${GITHUB_WORKSPACE}/rtloader/common " >> $GITHUB_ENV - - - uses: actions/setup-go@v3 - with: - go-version-file: ".go-version" - - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: go, javascript, python, cpp - setup-python-dependencies: false - # Defining a fixed CodeQL bundle version - tools: https://github.com/github/codeql-action/releases/download/codeql-bundle-20230207/codeql-bundle-linux64.tar.gz - - - name: Set Swap Space - uses: pierotofy/set-swap-space@49819abfb41bd9b44fb781159c033dba90353a7c - with: - swap-size-gb: 10 - - - name: Build DataDog agent - run: | - invoke install-tools - invoke deps - invoke agent.build --build-exclude=systemd - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/cws-btfhub-sync.yml b/.github/workflows/cws-btfhub-sync.yml deleted file mode 100644 index 0aee1b35c2265e..00000000000000 --- a/.github/workflows/cws-btfhub-sync.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: "CWS BTFHub constants sync" - -on: - workflow_dispatch: - schedule: - - cron: '30 4 * * 1-5' # at 4:30 UTC every week-day - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - name: Cleanup runner - run: | - sudo rm -rf /usr/local/lib/android >/dev/null 2>&1 - docker rmi $(docker image ls -aq) >/dev/null 2>&1 - - - name: Checkout datadog-agent repository - uses: actions/checkout@v4 - - - name: Checkout btfhub-archive repository - uses: actions/checkout@v4 - with: - repository: aquasecurity/btfhub-archive - path: dev/dist/archive - - - name: Install python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - cache: 'pip' - - run: pip install -r requirements.txt - - - name: Install go - uses: actions/setup-go@v4 - with: - go-version-file: '.go-version' - - - name: Install go deps - run: | - inv -e deps - - - name: Compute branch name - id: branch-name - run: | - echo "BRANCH_NAME=cws/constants-sync-$(date +%s)" >> $GITHUB_OUTPUT - - - name: Sync constants - run: | - inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive - - - uses: stefanzweifel/git-auto-commit-action@v4 - id: commit-creator - with: - commit_message: "CWS: sync BTFhub constants" - branch: ${{ steps.branch-name.outputs.BRANCH_NAME }} - create_branch: true - skip_fetch: true - skip_checkout: true - - - name: Create Pull Request - uses: actions/github-script@v7 - if: steps.commit-creator.outputs.changes_detected == 'true' - with: - script: | - const { repo, owner } = context.repo; - const result = await github.rest.pulls.create({ - title: 'CWS: sync BTFHub constants', - owner, - repo, - head: '${{ steps.branch-name.outputs.BRANCH_NAME }}', - base: 'main', - body: [ - '### What does this PR do?', - 'This PR syncs the BTFHub constants used by CWS', - '_This PR was autogenerated_' - ].join('\n') - }); - github.rest.issues.update({ - owner, - repo, - issue_number: result.data.number, - milestone: 22, // triage - }); - github.rest.issues.addLabels({ - owner, - repo, - issue_number: result.data.number, - labels: ['changelog/no-changelog', 'qa/skip-qa', 'team/agent-security'] - }); diff --git a/.github/workflows/do-not-merge.yml b/.github/workflows/do-not-merge.yml deleted file mode 100644 index 13886c696f6798..00000000000000 --- a/.github/workflows/do-not-merge.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Fails a pull request if a label contains `do-not-merge` -# credit: https://github.com/jessesquires/gh-workflows/blob/main/.github/workflows/do-not-merge.yml - -name: Do Not Merge - -on: - pull_request: - types: [synchronize, opened, reopened, labeled, unlabeled] - push: - branches: - - mq-working-branch-* - -jobs: - do-not-merge: - if: ${{ contains(github.event.*.labels.*.name, 'do-not-merge/hold') || contains(github.event.*.labels.*.name, 'do-not-merge/WIP') }} - name: Check Do Not Merge - runs-on: ubuntu-latest - steps: - - name: check for label - run: | - echo "Pull request is labeled as 'do-not-merge'" - echo "This workflow fails so that the pull request cannot be merged" - exit 1 diff --git a/.github/workflows/go_mod_tidy.yml b/.github/workflows/go_mod_tidy.yml deleted file mode 100644 index 574f2aefff8f9a..00000000000000 --- a/.github/workflows/go_mod_tidy.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: "Run Go Mod Tidy And Generate Licenses" -on: - pull_request: - types: - - labeled - push: - branches: - - mq-working-branch-* - workflow_dispatch: - inputs: - pr_number: - description: "PR number" - required: true - type: number -permissions: - contents: write -jobs: - mod_tidy_and_generate_licenses: - if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.actor == 'dependabot[bot]' && contains(github.event.pull_request.labels.*.name, 'dependencies-go')) }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - - name: Checkout PR - # run only if triggered manually, otherwise we are already on the right branch and we won't have `pr_number` - if: ${{ github.event_name == 'workflow_dispatch' }} - run: gh pr checkout ${{ github.event.inputs.pr_number }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Install go - uses: actions/setup-go@v3 - with: - go-version-file: ".go-version" - - name: Install python - uses: actions/setup-python@v4 - with: - python-version: "3.9.12" - cache: "pip" - - name: Install python requirements.txt - run: python3 -m pip install -r requirements.txt - - name: Go mod tidy - run: inv -e tidy-all - - name: Update LICENSE-3rdparty.csv - if: ${{ github.event_name == 'workflow_dispatch' || !contains(github.event.pull_request.labels.*.name, 'dependencies-go-tools') }} - run: | - inv -e install-tools - inv -e generate-licenses - - name: Update mocks - if: ${{ github.event_name == 'workflow_dispatch' || !contains(github.event.pull_request.labels.*.name, 'dependencies-go-tools') }} - run: inv -e security-agent.gen-mocks # generate both security agent and process mocks - - uses: stefanzweifel/git-auto-commit-action@v4 - id: autocommit - with: - commit_message: Auto-generate go.sum and LICENSE-3rdparty.csv changes - - name: changes - run: | - echo "Changes detected: ${{ steps.autocommit.outputs.changes_detected }}" diff --git a/.github/workflows/gohai.yml b/.github/workflows/gohai.yml deleted file mode 100644 index b0cba50233a734..00000000000000 --- a/.github/workflows/gohai.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: "Gohai Test" - -# Only run the tests if pkg/gohai was changed -on: - push: - branches: - - main - - 7.[0-9][0-9].x - - mq-working-branch-* - paths: - - "pkg/gohai/**" - pull_request: - paths: - - "pkg/gohai/**" - -jobs: - gohai_test: - strategy: - matrix: - # Use oldest and latest available ubuntu, macos and windows - os: - [ - ubuntu-20.04, - ubuntu-latest, - macos-11, - macos-latest, - windows-2019, - windows-latest, - ] - # Run tests with both the agent's version and gohai's pinned version - go-file: [.go-version, pkg/gohai/go.mod] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: ${{ matrix.go-file }} - - name: Test - run: cd pkg/gohai && go test ./... diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml deleted file mode 100644 index 091c422b7a1e0e..00000000000000 --- a/.github/workflows/labeler.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "PR labeler" - -on: - pull_request: - branches: - - main - - 7.[0-9][0-9].x - - mq-working-branch-* - -jobs: - label: - permissions: - contents: read - pull-requests: write - runs-on: ubuntu-latest - steps: - - uses: actions/labeler@v4 - with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" - configuration-path: .github/labeler.yml - # currently doesn't work - sync-labels: true diff --git a/.github/workflows/serverless-benchmarks.yml b/.github/workflows/serverless-benchmarks.yml new file mode 100644 index 00000000000000..2e8b75677bf735 --- /dev/null +++ b/.github/workflows/serverless-benchmarks.yml @@ -0,0 +1,73 @@ +name: "Serverless Benchmarks" + +on: + push: + paths: + - 'cmd/serverless/**' + - 'pkg/serverless/**' + - '.github/workflows/serverless-benchmarks.yml' + +env: + DD_API_KEY: must-be-set + +jobs: + run: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: stable + + - name: Install benchstat + run: | + go install golang.org/x/perf/cmd/benchstat@latest + + - name: Checkout datadog-agent repository + uses: actions/checkout@v3 + with: + path: go/src/github.com/DataDog/datadog-agent + + - name: Checkout datadog-agent base branch + run: | + cd go/src/github.com/DataDog/datadog-agent + git fetch origin $GITHUB_BASE_REF --depth 1 + git checkout $GITHUB_BASE_REF + + - name: Previous benchmark results + run: | + cd go/src/github.com/DataDog/datadog-agent + go test -tags=test -run='^$' -bench=StartEndInvocation -count=4 -benchtime=10x -timeout=60m \ + ./pkg/serverless/... | tee previous.txt + + - name: Checkout datadog-agent pr branch + run: | + cd go/src/github.com/DataDog/datadog-agent + git fetch origin $GITHUB_SHA --depth 1 + git checkout $GITHUB_SHA + + - name: Current benchmark results + run: | + cd go/src/github.com/DataDog/datadog-agent + go test -tags=test -run='^$' -bench=StartEndInvocation -count=4 -benchtime=10x -timeout=60m \ + ./pkg/serverless/... | tee current.txt + + - name: Analyze results + id: analyze + run: | + cd go/src/github.com/DataDog/datadog-agent + OUTPUT=$(benchstat -row /event previous.txt current.txt) + #OUTPUT=$(cat output.txt) + echo "analyze=$OUTPUT" >> $GITHUB_OUTPUT + + - name: Post comment + uses: marocchino/sticky-pull-request-comment@v2.5.0 + with: + hide_and_recreate: true + hide_classify: "RESOLVED" + message: | + ## Serverless Benchmark Results + + ``` + ${{ steps.analyze.outputs.analyze }} + ``` diff --git a/.github/workflows/serverless-binary-size.yml b/.github/workflows/serverless-binary-size.yml deleted file mode 100644 index f90a0bc99777f0..00000000000000 --- a/.github/workflows/serverless-binary-size.yml +++ /dev/null @@ -1,141 +0,0 @@ -name: "Serverless Binary Size" - -on: - pull_request: - push: - branches: - - mq-working-branch-* - -env: - SIZE_ALLOWANCE: fromJSON(1000000) # 1 MB - -jobs: - comment: - runs-on: ubuntu-latest - steps: - - name: Checkout datadog-agent repository - uses: actions/checkout@v3 - with: - path: go/src/github.com/DataDog/datadog-agent - - - name: Checkout datadog-agent base branch - run: | - cd go/src/github.com/DataDog/datadog-agent - git fetch origin $GITHUB_BASE_REF --depth 1 - git checkout $GITHUB_BASE_REF - - - name: Checkout the datadog-lambda-extension repository - uses: actions/checkout@v3 - with: - repository: DataDog/datadog-lambda-extension - path: go/src/github.com/DataDog/datadog-lambda-extension - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Previous binary size and dependencies - id: previous - run: | - cd go/src/github.com/DataDog/datadog-lambda-extension - - OUTPUT=$(./scripts/visualize_size.sh size) - echo "binary size before merging this pull request is $OUTPUT" - echo "result=$OUTPUT" >> $GITHUB_OUTPUT - - echo "deps<> $GITHUB_OUTPUT - ./scripts/visualize_size.sh list_symbols | awk '{print $2}' | head -n 100 >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Checkout datadog-agent pr branch - run: | - cd go/src/github.com/DataDog/datadog-agent - git fetch origin $GITHUB_SHA --depth 1 - git checkout $GITHUB_SHA - - - name: Current binary size and dependencies - id: current - run: | - cd go/src/github.com/DataDog/datadog-lambda-extension - - OUTPUT=$(./scripts/visualize_size.sh size) - echo "binary size after merging this pull request will be $OUTPUT" - echo "result=$OUTPUT" >> $GITHUB_OUTPUT - - echo "deps<> $GITHUB_OUTPUT - ./scripts/visualize_size.sh list_symbols | awk '{print $2}' | head -n 100 >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Compare sizes - id: compare - run: | - OUTPUT=$(( ${{ steps.current.outputs.result }} - ${{ steps.previous.outputs.result }} )) - echo "diff=$OUTPUT" >> $GITHUB_OUTPUT - - OUTPUT=$(( $OUTPUT / 100000 )) - echo "coldstart=$OUTPUT" >> $GITHUB_OUTPUT - - ### Steps below only run if size diff > SIZE_ALLOWANCE ### - - - name: Install graphviz - uses: ts-graphviz/setup-graphviz@v1 - if: steps.compare.outputs.diff > env.SIZE_ALLOWANCE - - - name: Install digraph - if: steps.compare.outputs.diff > env.SIZE_ALLOWANCE - run: | - GOPATH=$(pwd)/go go install golang.org/x/tools/cmd/digraph@latest - - - name: List new dependencies - id: deps - if: steps.compare.outputs.diff > env.SIZE_ALLOWANCE - run: | - echo "deps<> $GITHUB_OUTPUT - for dep in $(echo "${{ steps.current.outputs.deps }}"); do - if ! echo "${{ steps.previous.outputs.deps }}" | grep -w -q "$dep"; then - echo "$dep" >> $GITHUB_OUTPUT - fi - done - echo "EOF" >> $GITHUB_OUTPUT - - - name: Create dependency graphs - if: steps.compare.outputs.diff > env.SIZE_ALLOWANCE - run: | - export PATH=$(pwd)/go/bin:$PATH - cd go/src/github.com/DataDog/datadog-lambda-extension - mkdir graphs - for dep in $(echo "${{ steps.deps.outputs.deps }}"); do - PACKAGE=$dep ./scripts/visualize_size.sh graph - mv .layers/output.svg graphs/$(echo $dep | tr '/' '-').svg - done - - - name: Archive dependency graphs - uses: actions/upload-artifact@v3 - if: steps.compare.outputs.diff > env.SIZE_ALLOWANCE - with: - name: dependency-graphs - path: go/src/github.com/DataDog/datadog-lambda-extension/graphs - - - name: Post comment - uses: marocchino/sticky-pull-request-comment@v2.5.0 - if: steps.compare.outputs.diff > env.SIZE_ALLOWANCE - with: - hide_and_recreate: true - hide_classify: "RESOLVED" - message: | - :warning::rotating_light: Warning, this pull request increases the binary size of serverless extension by ${{ steps.compare.outputs.diff }} bytes. Each MB of binary size increase means about 10ms of additional cold start time, so this pull request would increase cold start time by ${{ steps.compare.outputs.coldstart }}ms. - - If you have questions, we are happy to help, come visit us in the [#serverless](https://dd.slack.com/archives/CBWDFKWV8) slack channel and provide a link to this comment. - -
- Debug info - - These dependencies were added to the serverless extension by this pull request: - - ``` - ${{ steps.deps.outputs.deps }} - ``` - - View dependency graphs for each added dependency in the [artifacts section](https://github.com/DataDog/datadog-agent/actions/runs/${{ github.run_id }}#artifacts) of the github action. - - We suggest you consider adding the `!serverless` build tag to remove any new dependencies not needed in the serverless extension. -
diff --git a/.github/workflows/serverless-integration.yml b/.github/workflows/serverless-integration.yml deleted file mode 100644 index 071af865e5f28e..00000000000000 --- a/.github/workflows/serverless-integration.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: "Serverless Integration Tests" - -on: - push: - paths: - - 'cmd/serverless/**' - - 'pkg/serverless/**' - - 'test/integration/serverless/**' - - '.github/workflows/serverless-integration.yml' - -jobs: - test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - architecture: [amd64, arm64] - suite: [metric, log, trace, appsec] - name: ${{ matrix.suite }} on ${{ matrix.architecture }} - steps: - - name: Checkout datadog-agent repository - uses: actions/checkout@v3 - with: - path: go/src/github.com/DataDog/datadog-agent - - - name: Set up Node 14 - uses: actions/setup-node@v3 - with: - node-version: 14 - - - name: Install Serverless Framework - run: sudo yarn global add serverless@^3.36.0 --prefix /usr/local - - - name: Checkout the datadog-lambda-extension repository - uses: actions/checkout@v3 - with: - repository: DataDog/datadog-lambda-extension - path: go/src/github.com/DataDog/datadog-lambda-extension - - - name: Set up QEMU - id: qemu - uses: docker/setup-qemu-action@v2 - with: - image: tonistiigi/binfmt:latest - platforms: amd64,arm64 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Create raw logs directory - id: rawlogs - run: |- - DIR=$(mktemp -d) - echo "dir=${DIR}" >> $GITHUB_OUTPUT - - - name: Run tests if AWS credentials are available - id: test - uses: nick-fields/retry@v2 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.SERVERLESS_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.SERVERLESS_AWS_SECRET_ACCESS_KEY }} - with: - timeout_minutes: 45 - max_attempts: 2 - command: | - RAWLOGS_DIR="${{ steps.rawlogs.outputs.dir }}/${{ matrix.architecture }}" - cd go/src/github.com/DataDog/datadog-agent - ARCHITECTURE=${{ matrix.architecture }} RAWLOGS_DIR=$RAWLOGS_DIR \ - ./test/integration/serverless/run.sh ${{ matrix.suite }} - - - name: Archive raw logs - if: always() - uses: actions/upload-artifact@v3 - with: - name: rawlogs - path: ${{ steps.rawlogs.outputs.dir }} diff --git a/.github/workflows/slapr.yml b/.github/workflows/slapr.yml deleted file mode 100644 index 45feb5dbf325a6..00000000000000 --- a/.github/workflows/slapr.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -name: Slack emoji PR updates -on: - pull_request_review: - types: [submitted] - pull_request: - types: [closed] - -jobs: - run_slapr_datadog_agent: - runs-on: ubuntu-latest - strategy: - matrix: - channel_variables: - - SLACK_AGENT_PLATFORM_ID - - SLACK_AGENT_PLATFORM_REVIEWS_ID - - SLACK_OPENTELEMETRY_ID - - SLACK_AGENT_METRICS_LOGS_ID - - SLACK_CONTAINER_INTEGRATIONS_ID - - SLACK_AGENT_SHARED_COMPONENTS_ID - steps: - - uses: DataDog/slapr@master - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - SLACK_CHANNEL_ID: "${{ secrets[matrix.channel_variables] }}" - SLACK_API_TOKEN: "${{ secrets.SLACK_API_TOKEN }}" - SLAPR_BOT_USER_ID: "${{ secrets.SLAPR_BOT_USER_ID }}" - SLAPR_EMOJI_REVIEW_STARTED: "review_started" - SLAPR_EMOJI_APPROVED: "gh-approved" - SLAPR_EMOJI_CHANGES_REQUESTED: "request-changes" - SLAPR_EMOJI_MERGED: "merged" - SLAPR_EMOJI_CLOSED: "closed" diff --git a/.github/workflows/windows-linters.yml b/.github/workflows/windows-linters.yml deleted file mode 100644 index 593bb03b9dd948..00000000000000 --- a/.github/workflows/windows-linters.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: "Windows linters" - -on: - push: - branches: - - main - - 7.[0-9][0-9].x - - mq-working-branch-* - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - windows-lint: - runs-on: windows-2019 # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md - steps: - - name: Checkout datadog-agent repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 #needed for 'git describe' to work - - - name: Install python - uses: actions/setup-python@v4 - with: - python-version: "3.9.5" - cache: "pip" - - run: | - python -m pip install -r requirements.txt - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - - name: Install go - uses: actions/setup-go@v3 - with: - go-version-file: ".go-version" - - - name: Install Dotnet - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '7.0.x' - - - name: Set up runner - run: | - $ErrorActionPreference = 'Stop'; - # install dependencies - inv -e install-tools - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - inv -e deps - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - # Run this first as it's quite quick compared to Go - - name: Run Dotnet linter - shell: bash # using bash so we don't have to check $lastExitCode all the time - run: | - dotnet format --verify-no-changes .\\tools\\windows\\DatadogAgentInstaller - - - name: Run Go linters - shell: bash # using bash so we don't have to check $lastExitCode all the time - run: | - export PATH="/c/msys64/mingw64/bin:/c/msys64/usr/bin/:$PATH" # prepend msys, otherwise make from mingw gets used - echo $PATH - powershell.exe -Command ./tasks/winbuildscripts/pre-go-build.ps1 - inv -e rtloader.format --raise-if-changed - inv -e lint-go --timeout 30 - diff --git a/.github/workflows/windows-unittests.yml b/.github/workflows/windows-unittests.yml deleted file mode 100644 index 75c8ead72c9d19..00000000000000 --- a/.github/workflows/windows-unittests.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Windows unit tests" - -on: - push: - branches: - - main - - 7.[0-9][0-9].x - - mq-working-branch-* - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - windows-unit-tests: - runs-on: windows-2022 # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md - steps: - - name: Checkout datadog-agent repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 #needed for 'git describe' to work - - - name: Install python - uses: actions/setup-python@v4 - with: - python-version: "3.9.5" - cache: "pip" - - run: | - python -m pip install -r requirements.txt - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - - name: Install go - uses: actions/setup-go@v3 - with: - go-version-file: ".go-version" - - - name: Set up runner - run: | - $ErrorActionPreference = 'Stop'; - # install dependencies - inv -e install-tools - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - inv -e deps - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - # create 'ddagentuser' user to test the secrets feature on windows - $Password = ConvertTo-SecureString "dummyPW_:-gch6Rejae9" -AsPlainText -Force - New-LocalUser -Name "ddagentuser" -Description "Test user for the secrets feature on windows." -Password $Password - - - name: Run tests - shell: bash # using bash so we don't have to check $lastExitCode all the time - run: | - export PATH="/c/msys64/mingw64/bin:/c/msys64/usr/bin/:$PATH" # prepend msys, otherwise make from mingw gets used - echo $PATH - powershell.exe -Command ./tasks/winbuildscripts/pre-go-build.ps1 - # FIXME: skipping rtloader tests because they fail with a DLL-not-found error - # inv -e rtloader.test - inv -e test --rerun-fails=2 --python-runtimes 3 --coverage --profile --python-home-3=$pythonLocation --timeout=600 - - - name: Upload Codecov results - uses: codecov/codecov-action@v3 - with: - flags: windows - name: codecov-windows diff --git a/.gitlab/JOBOWNERS b/.gitlab/JOBOWNERS deleted file mode 100644 index a5226210254668..00000000000000 --- a/.gitlab/JOBOWNERS +++ /dev/null @@ -1,90 +0,0 @@ -* @DataDog/agent-platform - -# Deps build -build_clang_* @DataDog/ebpf-platform -build_processed_btfhub_archive @DataDog/ebpf-platform - -# Source test -# Notifications are handled separately for more fine-grained control on go tests -tests_* @DataDog/multiple -tests_ebpf* @DataDog/ebpf-platform -security_go_generate_check @DataDog/agent-security -prepare_ebpf_functional_tests* @DataDog/ebpf-platform - -# Golang dependency list generation (currently disabled) -golang_deps_generate @DataDog/agent-shared-components -# Golang dependency diff generation -golang_deps_diff @DataDog/ebpf-platform -golang_deps_commenter @DataDog/ebpf-platform - -# Binary build -build_system-probe* @DataDog/ebpf-platform -cluster_agent_cloudfoundry-build* @Datadog/platform-integrations -cluster_agent-build* @DataDog/container-integrations -cws_instrumentation-build* @DataDog/agent-security -build_serverless* @DataDog/serverless - -# Package deps build -generate_minimized_btfs_* @DataDog/ebpf-platform - -# Kitchen tests -kitchen_windows* @DataDog/windows-agent - -# Image build -docker_build* @DataDog/container-integrations - -# Image scan -scan_* @DataDog/container-integrations -dca_scan_* @DataDog/container-integrations - -# Check deploy -# Do not notify on these jobs, they're expected to fail on duplicate -# deploy pipelines -check_already_deployed_version_* @DataDog/do-not-notify - -# Dev container deploy -dca_dev_branch* @DataDog/container-integrations -dca_dev_master* @DataDog/container-integrations -cws_instrumentation_dev_branch* @DataDog/agent-security -dev_branch* @DataDog/container-integrations -dev_master* @DataDog/container-integrations -dev_nightly* @DataDog/container-integrations - -# Internal image deploy -docker_trigger_internal* @DataDog/container-integrations -docker_trigger_cluster_agent_internal* @DataDog/container-integrations -docker_trigger_cws_instrumentation_internal* @DataDog/agent-security - -# Internal kubernetes deploy -internal_kubernetes_deploy* @DataDog/container-integrations - -# Deploy -deploy_containers* @DataDog/container-integrations - -# Functional test -kitchen_*_system_probe* @DataDog/ebpf-platform -kitchen_*_security_agent* @DataDog/agent-security -kitchen_*_process_agent* @DataDog/processes -pull_test_dockers* @DataDog/universal-service-monitoring -cleanup_kitchen_functional_test @DataDog/ebpf-platform @DataDog/agent-security -serverless_cold_start_performance-deb_x64 @DataDog/serverless - -# E2E -k8s-e2e-* @DataDog/container-integrations -k8s-e2e-otlp-* @DataDog/opentelemetry -k8s-e2e-cws-* @DataDog/agent-security -k8s-e2e-cspm-* @DataDog/agent-security - -# New E2E -new-e2e-containers* @DataDog/container-integrations -new-e2e-agent-subcommands* @DataDog/agent-shared-components -new-e2e-language-detection* @DataDog/processes -new-e2e-agent-platform* @DataDog/agent-platform -new-e2e-npm* @DataDog/Networks - -# Kernel version testing -package_dependencies* @DataDog/ebpf-platform -kernel_matrix_testing* @DataDog/ebpf-platform - -# Single machine performance -single_machine_performance* @DataDog/single-machine-performance diff --git a/.gitlab/benchmarks/benchmarks.yml b/.gitlab/benchmarks/benchmarks.yml deleted file mode 100644 index 4fae53f59348bc..00000000000000 --- a/.gitlab/benchmarks/benchmarks.yml +++ /dev/null @@ -1,23 +0,0 @@ -benchmark: - stage: benchmarks - # This base image is created here: https://gitlab.ddbuild.io/DataDog/apm-reliability/relenv-microbenchmarking-platform - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/relenv-microbenchmarking-platform:trace-agent - timeout: 1h - rules: - !reference [.manual] # TODO: run this benchmark automatically when the trace-agent changes - interruptible: true - # tags: ["runner:apm-k8s-tweaked-metal"] # TODO: Commented out until we have the metal runners available in this repo - tags: ["runner:main"] - script: - - ./test/benchmarks/apm_scripts/capture-hardware-software-info.sh - - ./test/benchmarks/apm_scripts/run-benchmarks.sh - # - ./test/benchmarks/apm_scripts/upload-results-to-s3.sh #commented out until we have merged our permissions changes - - ./test/benchmarks/apm_scripts/analyze-results.sh - - ./test/benchmarks/apm_scripts/post-pr-comment.sh - artifacts: - name: "reports" - paths: - - reports/ - expire_in: 3 months - variables: - FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true" diff --git a/.gitlab/benchmarks/macrobenchmarks.yml b/.gitlab/benchmarks/macrobenchmarks.yml deleted file mode 100644 index 760cda552b7cd8..00000000000000 --- a/.gitlab/benchmarks/macrobenchmarks.yml +++ /dev/null @@ -1,90 +0,0 @@ -variables: - BENCHMARKS_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:trace-agent - -.trace_agent_benchmarks: - stage: benchmarks - tags: ["team:apm-k8s-tweaked-metal-datadog-agent", "specific:true"] - timeout: 1h - rules: - - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/ - when: never - - if: $CI_PIPELINE_SOURCE == "schedule" - when: always - - when: manual - # If you have a problem with Gitlab cache, see Troubleshooting section in Benchmarking Platform docs - image: $BENCHMARKS_CI_IMAGE - needs: ["setup_agent_version"] - script: - - git clone --branch trace-agent https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform - - ./steps/run-benchmarks.sh - artifacts: - name: "artifacts" - when: always - paths: - - platform/artifacts/ - expire_in: 3 months - variables: - FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true" # Important tweak for stability of benchmarks - - # Uncomment to force k8s memory limits for CI job container. - # KUBERNETES_MEMORY_REQUEST: "4Gi" - # KUBERNETES_MEMORY_LIMIT: "4Gi" - - # Workaround: Currently we're not running the benchmarks on every PR, but GitHub still shows them as pending. - # By marking the benchmarks as allow_failure, this should go away. (This workaround should be removed once the - # benchmarks get changed to run on every PR) - allow_failure: true - -trace-agent-v04-2cpus-normal_load-fixed_sps: - extends: .trace_agent_benchmarks - variables: - TRACE_AGENT_ENDPOINT: v04 - TRACE_AGENT_CPUS: 40-41 - DD_APM_MAX_CPU_PERCENT: 0 - DD_APM_MAX_MEMORY: 0 - DD_BENCHMARKS_CONFIGURATION: trace-agent-v04-2cpus-normal_load-fixed_sps - SCENARIOS: > - normal_load-10traces210spans-65ksps| - normal_load-500traces8617spans-65ksps| - normal_load-3885traces3891spans-65ksps - -trace-agent-v04-2cpus-stress_load-fixed_sps: - extends: .trace_agent_benchmarks - when: manual - variables: - TRACE_AGENT_ENDPOINT: v04 - TRACE_AGENT_CPUS: 40-41 - DD_APM_MAX_CPU_PERCENT: 0 - DD_APM_MAX_MEMORY: 0 - DD_BENCHMARKS_CONFIGURATION: trace-agent-v04-2cpus-stress_load-fixed_sps - SCENARIOS: > - stress_load-10traces210spans-525ksps| - stress_load-500traces8617spans-525ksps| - stress_load-3885traces3891spans-525ksps - -trace-agent-v05-2cpus-normal_load-fixed_sps: - extends: .trace_agent_benchmarks - variables: - TRACE_AGENT_ENDPOINT: v05 - TRACE_AGENT_CPUS: 40-41 - DD_APM_MAX_CPU_PERCENT: 0 - DD_APM_MAX_MEMORY: 0 - DD_BENCHMARKS_CONFIGURATION: trace-agent-v05-2cpus-normal_load-fixed_sps - SCENARIO: > - normal_load-10traces210spans-65ksps| - normal_load-500traces8617spans-65ksps| - normal_load-3885traces3891spans-65ksps - -trace-agent-v05-2cpus-stress_load-fixed_sps: - extends: .trace_agent_benchmarks - when: manual - variables: - TRACE_AGENT_ENDPOINT: v05 - TRACE_AGENT_CPUS: 40-41 - DD_APM_MAX_CPU_PERCENT: 0 - DD_APM_MAX_MEMORY: 0 - DD_BENCHMARKS_CONFIGURATION: trace-agent-v05-2cpus-stress_load-fixed_sps - SCENARIOS: > - stress_load-10traces210spans-525ksps| - stress_load-500traces8617spans-525ksps| - stress_load-3885traces3891spans-525ksps diff --git a/.gitlab/binary_build.yml b/.gitlab/binary_build.yml deleted file mode 100644 index b4f168984bb7ef..00000000000000 --- a/.gitlab/binary_build.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# binary_build stage -# Contains jobs which build various go binaries (dogstatsd, IoT agent, cluster-agent, cluster-agent-cloudfoundry). - -include: - - /.gitlab/binary_build/cluster_agent_cloudfoundry.yml - - /.gitlab/binary_build/cluster_agent.yml - - /.gitlab/binary_build/cws_instrumentation.yml - - /.gitlab/binary_build/linux.yml - - /.gitlab/binary_build/system_probe.yml - - /.gitlab/binary_build/windows.yml - - /.gitlab/binary_build/serverless.yml diff --git a/.gitlab/binary_build/cluster_agent.yml b/.gitlab/binary_build/cluster_agent.yml deleted file mode 100644 index 7cc098f2da64a4..00000000000000 --- a/.gitlab/binary_build/cluster_agent.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -.cluster_agent-build_common: - stage: binary_build - needs: ["go_mod_tidy_check"] - script: - - inv check-go-version - - inv -e cluster-agent.build --release-version "$RELEASE_VERSION_7" - - $S3_CP_CMD $CI_PROJECT_DIR/$CLUSTER_AGENT_BINARIES_DIR/datadog-cluster-agent $S3_ARTIFACTS_URI/datadog-cluster-agent.$ARCH - - $S3_CP_CMD $CI_PROJECT_DIR/Dockerfiles/cluster-agent/datadog-cluster.yaml $S3_ARTIFACTS_URI/datadog-cluster.yaml - - $S3_CP_CMD $CI_PROJECT_DIR/Dockerfiles/cluster-agent/security-agent-policies $S3_ARTIFACTS_URI/security-agent-policies --recursive - variables: - KUBERNETES_MEMORY_REQUEST: "8Gi" - KUBERNETES_MEMORY_LIMIT: "8Gi" - -cluster_agent-build_amd64: - extends: .cluster_agent-build_common - rules: - !reference [.on_tag_or_a7] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "go_deps"] - variables: - ARCH: amd64 - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - -cluster_agent-build_arm64: - extends: .cluster_agent-build_common - rules: - !reference [.on_tag_or_a7] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_mod_tidy_check", "go_deps"] - variables: - ARCH: arm64 - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] diff --git a/.gitlab/binary_build/cluster_agent_cloudfoundry.yml b/.gitlab/binary_build/cluster_agent_cloudfoundry.yml deleted file mode 100644 index 34892c1b8d2e74..00000000000000 --- a/.gitlab/binary_build/cluster_agent_cloudfoundry.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -cluster_agent_cloudfoundry-build_amd64: - rules: - !reference [.on_a7] - stage: binary_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "go_deps"] - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR - variables: - ARCH: amd64 - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - script: - - inv check-go-version - - inv -e cluster-agent-cloudfoundry.build - - cd $CI_PROJECT_DIR/$CLUSTER_AGENT_CLOUDFOUNDRY_BINARIES_DIR - - mkdir -p $OMNIBUS_PACKAGE_DIR - - export PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7) - - tar cf $OMNIBUS_PACKAGE_DIR/datadog-cluster-agent-cloudfoundry-$PACKAGE_VERSION-$ARCH.tar.xz datadog-cluster-agent-cloudfoundry diff --git a/.gitlab/binary_build/cws_instrumentation.yml b/.gitlab/binary_build/cws_instrumentation.yml deleted file mode 100644 index fb5d9993f54ccc..00000000000000 --- a/.gitlab/binary_build/cws_instrumentation.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -.cws_instrumentation-build_common: - stage: binary_build - needs: ["go_mod_tidy_check"] - script: - - inv check-go-version - - inv -e cws-instrumentation.build - - $S3_CP_CMD $CI_PROJECT_DIR/$CWS_INSTRUMENTATION_BINARIES_DIR/cws-instrumentation $S3_ARTIFACTS_URI/cws-instrumentation.$ARCH - -cws_instrumentation-build_amd64: - extends: .cws_instrumentation-build_common - rules: - !reference [.on_tag_or_a7] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "go_deps"] - variables: - ARCH: amd64 - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - -cws_instrumentation-build_arm64: - extends: .cws_instrumentation-build_common - rules: - !reference [.on_tag_or_a7] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_mod_tidy_check", "go_deps"] - variables: - ARCH: arm64 - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] diff --git a/.gitlab/binary_build/linux.yml b/.gitlab/binary_build/linux.yml deleted file mode 100644 index 5ed346fee529c0..00000000000000 --- a/.gitlab/binary_build/linux.yml +++ /dev/null @@ -1,99 +0,0 @@ ---- - -build_dogstatsd_static-binary_x64: - stage: binary_build - rules: - !reference [.on_a7] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["lint_linux-x64", "tests_deb-x64-py3", "go_deps"] - variables: - ARCH: amd64 - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - script: - - inv check-go-version - - inv -e dogstatsd.build --static --major-version 7 - - $S3_CP_CMD $CI_PROJECT_DIR/$STATIC_BINARIES_DIR/dogstatsd $S3_ARTIFACTS_URI/static/dogstatsd.$ARCH - -build_dogstatsd_static-binary_arm64: - stage: binary_build - rules: - !reference [.on_a7] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] - needs: ["lint_linux-arm64", "tests_deb-arm64-py3", "go_deps"] - variables: - ARCH: arm64 - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - script: - - inv check-go-version - - inv -e dogstatsd.build --static --major-version 7 - - $S3_CP_CMD $CI_PROJECT_DIR/$STATIC_BINARIES_DIR/dogstatsd $S3_ARTIFACTS_URI/static/dogstatsd.$ARCH - -build_dogstatsd-binary_x64: - stage: binary_build - rules: - !reference [.on_a7] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["lint_linux-x64", "tests_deb-x64-py3", "go_deps"] - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - script: - - inv check-go-version - - inv -e dogstatsd.build --major-version 7 - - $S3_CP_CMD $CI_PROJECT_DIR/$DOGSTATSD_BINARIES_DIR/dogstatsd $S3_ARTIFACTS_URI/dogstatsd/dogstatsd - -build_dogstatsd-binary_arm64: - rules: - !reference [.on_all_builds_a7] - stage: binary_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] - needs: ["lint_linux-arm64", "tests_deb-arm64-py3", "go_deps"] - variables: - ARCH: arm64 - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - script: - - inv check-go-version - - inv -e dogstatsd.build --major-version 7 - - $S3_CP_CMD $CI_PROJECT_DIR/$DOGSTATSD_BINARIES_DIR/dogstatsd $S3_ARTIFACTS_URI/dogstatsd/dogstatsd.$ARCH - -# IoT Agent builds to make sure the build is not broken because of build flags -build_iot_agent-binary_x64: - stage: binary_build - rules: - !reference [.on_a7] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["lint_linux-x64", "tests_deb-x64-py3", "go_deps"] - before_script: - - !reference [.retrieve_linux_go_deps] - script: - - source /root/.bashrc - - inv check-go-version - - inv -e agent.build --flavor iot --major-version 7 - - $S3_CP_CMD $CI_PROJECT_DIR/$AGENT_BINARIES_DIR/agent $S3_ARTIFACTS_URI/iot/agent - -build_iot_agent-binary_arm64: - rules: - !reference [.on_all_builds_a7] - stage: binary_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:arm64"] - needs: ["lint_linux-arm64", "tests_deb-arm64-py3", "go_deps"] - variables: - ARCH: arm64 - before_script: - - !reference [.retrieve_linux_go_deps] - script: - - source /root/.bashrc - - inv check-go-version - - inv -e agent.build --flavor iot --major-version 7 diff --git a/.gitlab/binary_build/serverless.yml b/.gitlab/binary_build/serverless.yml deleted file mode 100644 index 8861528211fab2..00000000000000 --- a/.gitlab/binary_build/serverless.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -.build_serverless_common: - stage: binary_build - rules: - - !reference [.except_mergequeue] - - when: on_success - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - script: - - inv check-go-version - - cd cmd/serverless && go build -ldflags="-w -s" -a -v -tags "serverless otlp" -o $BINARY_NAME - -build_serverless-deb_x64: - extends: .build_serverless_common - variables: - BINARY_NAME: datadog-agent-x64 - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_deps"] - artifacts: - expire_in: 1 day - paths: - - $CI_PROJECT_DIR/cmd/serverless - -build_serverless-deb_arm64: - extends: .build_serverless_common - variables: - BINARY_NAME: datadog-agent-arm64 - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_deps"] diff --git a/.gitlab/binary_build/system_probe.yml b/.gitlab/binary_build/system_probe.yml deleted file mode 100644 index db27cdf1ef1862..00000000000000 --- a/.gitlab/binary_build/system_probe.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -.system-probe_build_common: - rules: - - !reference [.except_mergequeue] - - when: on_success - before_script: - - !reference [.retrieve_linux_go_deps] - # HACK: empty the build directory cache (that can come from previous runs) - # to not have remainders of previous runs, which can make our deps bootstrapping logic fail. - # TODO: remove this once we switch to k8s runners, they won't have this problem - - find "$CI_BUILDS_DIR" ! -path '*DataDog/datadog-agent*' -depth # -delete implies -depth - - find "$CI_BUILDS_DIR" ! -path '*DataDog/datadog-agent*' -delete || true # Allow failure, we can't remove parent folders of datadog-agent - script: - - inv check-go-version - - inv -e system-probe.build --strip-object-files - # fail if references to glibc >= 2.18 - - objdump -p $CI_PROJECT_DIR/$SYSTEM_PROBE_BINARIES_DIR/system-probe | egrep 'GLIBC_2\.(1[8-9]|[2-9][0-9])' && exit 1 - - inv -e system-probe.save-build-outputs $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz - variables: - KUBERNETES_MEMORY_REQUEST: "6Gi" - KUBERNETES_MEMORY_LIMIT: "12Gi" - artifacts: - expire_in: 2 weeks - paths: - - $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz - -build_system-probe-x64: - stage: binary_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_deps"] - extends: .system-probe_build_common - variables: - ARCH: amd64 - -build_system-probe-arm64: - stage: binary_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - needs: ["go_deps"] - tags: ["arch:arm64"] - extends: .system-probe_build_common - variables: - ARCH: arm64 diff --git a/.gitlab/binary_build/windows.yml b/.gitlab/binary_build/windows.yml deleted file mode 100644 index 903744a3594165..00000000000000 --- a/.gitlab/binary_build/windows.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -# Entrypoint for the Windows Docker image -build_windows_container_entrypoint: - rules: - - !reference [.except_mergequeue] - - when: on_success - stage: binary_build - tags: ["runner:windows-docker", "windowsversion:1809"] - needs: ["lint_windows-x64", "tests_windows-x64"] - variables: - ARCH: "x64" - script: - - $ErrorActionPreference = "Stop" - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - - if (Test-Path build-out) { remove-item -recurse -force build-out } - - > - docker run --rm - -m 8192M - -v "$(Get-Location):c:\mnt" - -e CI_JOB_ID=${CI_JOB_ID} - -e CI_PIPELINE_ID=${CI_PIPELINE_ID} - -e CI_PROJECT_NAME=${CI_PROJECT_NAME} - -e WINDOWS_BUILDER=true - -e AWS_NETWORKING=true - -e TARGET_ARCH="$ARCH" - 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\Dockerfiles\agent\windows\entrypoint\build.bat - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - get-childitem build-out\${CI_JOB_ID} - - copy build-out\${CI_JOB_ID}\*.exe ./entrypoint.exe - - remove-item -recurse -force build-out\${CI_JOB_ID} - artifacts: - expire_in: 2 weeks - paths: - - entrypoint.exe diff --git a/.gitlab/cancel-prev-pipelines.yml b/.gitlab/cancel-prev-pipelines.yml deleted file mode 100644 index 5506c0aaa69301..00000000000000 --- a/.gitlab/cancel-prev-pipelines.yml +++ /dev/null @@ -1,20 +0,0 @@ -cancel-prev-pipelines: - stage: .pre - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - rules: # this should only run on dev branches - - if: $CI_PIPELINE_SOURCE =~ /^schedule.*$/ - when: never - - if: $CI_COMMIT_TAG - when: never - - !reference [.except_main_or_release_branch] - - if: $CI_COMMIT_MESSAGE =~ /.*\[skip cancel\].*/ - when: never - - if: $CI_COMMIT_REF_NAME =~ /.*-skip-cancel$/ - when: never - - when: on_success - script: - - source /root/.bashrc - - set +x - - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text) - - inv pipeline.auto-cancel-previous-pipelines diff --git a/.gitlab/check_deploy.yml b/.gitlab/check_deploy.yml deleted file mode 100644 index 66484d81e854ce..00000000000000 --- a/.gitlab/check_deploy.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -# check_deploy stage -# Contains jobs which which check if the version of the Agent built in the pipeline -# alteady exists (in the apt staging repo), or if the release branch is "none". -# In both cases, these jobs fail in order to prevent the pipeline from deploying artifacts. - -# Check that the current version hasn't already been deployed (we don't want to -# overwrite a public package). To update an erroneous package, first remove it -# from our S3 bucket. -check_already_deployed_version_6: - rules: !reference [.on_deploy_a6] - stage: check_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - dependencies: ["agent_deb-x64-a6", "agent_deb-arm64-a6"] - before_script: - - ls $OMNIBUS_PACKAGE_DIR - script: - - cd $OMNIBUS_PACKAGE_DIR && /deploy_scripts/fail_deb_is_pkg_already_exists.sh datadog-agent_6*_amd64.deb - - cd $OMNIBUS_PACKAGE_DIR && /deploy_scripts/fail_deb_is_pkg_already_exists.sh datadog-agent_6*_arm64.deb - -check_already_deployed_version_7: - rules: !reference [.on_deploy_a7] - stage: check_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - dependencies: ["agent_deb-x64-a7", "agent_deb-arm64-a7"] - before_script: - - ls $OMNIBUS_PACKAGE_DIR - script: - - cd $OMNIBUS_PACKAGE_DIR && /deploy_scripts/fail_deb_is_pkg_already_exists.sh datadog-agent_7*_amd64.deb - - cd $OMNIBUS_PACKAGE_DIR && /deploy_scripts/fail_deb_is_pkg_already_exists.sh datadog-agent_7*_arm64.deb - -# If we trigger a build only pipeline we stop here. -check_if_build_only: - rules: !reference [.on_deploy] - stage: check_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - dependencies: [] - script: - - if [ "$BUCKET_BRANCH" == "none" ]; then echo "Stopping pipeline"; exit 1; fi diff --git a/.gitlab/choco_build.yml b/.gitlab/choco_build.yml deleted file mode 100644 index dc38ae48328b43..00000000000000 --- a/.gitlab/choco_build.yml +++ /dev/null @@ -1,50 +0,0 @@ ---- -# choco_build stage -# Contains jobs which build the chocolatey Agent package. - -# Not currently used in the pipeline. -windows_choco_offline_7_x64: - rules: - !reference [.on_a7_manual] - stage: choco_build - tags: ["runner:windows-docker", "windowsversion:1809"] - needs: ["windows_msi_and_bosh_zip_x64-a7"] - variables: - ARCH: "x64" - script: - - $ErrorActionPreference = "Stop" - - Get-ChildItem omnibus\pkg - - copy omnibus\pkg\*.msi .\chocolatey\tools-offline\ - - docker run --rm -v "$(Get-Location):c:\mnt" 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\chocopack.bat offline - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - copy build-out\*.nupkg omnibus\pkg - artifacts: - expire_in: 2 weeks - paths: - - omnibus/pkg - -# The online version of the choco job fetches the msi package from S3 so -# it is run only once the msi package is pushed -windows_choco_online_7_x64: - rules: - !reference [.on_deploy_stable_or_rc_tag_on_beta_repo_branch_a7] - stage: choco_build - tags: ["runner:windows-docker", "windowsversion:1809"] - needs: ["deploy_staging_windows_tags-7"] - variables: - ARCH: "x64" - script: - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - - $ErrorActionPreference = "Stop" - - if (Test-Path .omnibus) { remove-item -recurse -force .omnibus } - - if (Test-Path build-out) { remove-item -recurse -force build-out } - - mkdir omnibus\pkg - - docker run --rm -v "$(Get-Location):c:\mnt" 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\chocopack.bat online - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - copy build-out\*.nupkg omnibus\pkg - artifacts: - expire_in: 2 weeks - paths: - - omnibus/pkg - # Sometimes Chocolatey is flakey - retry: 2 diff --git a/.gitlab/choco_deploy.yml b/.gitlab/choco_deploy.yml deleted file mode 100644 index 02e8e528911eb4..00000000000000 --- a/.gitlab/choco_deploy.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -# choco_build stage -# Contains a job which deploys the chocolatey Agent package. - -publish_choco_7_x64: - rules: - !reference [.on_deploy_stable_or_rc_tag_on_beta_repo_branch_a7_manual_on_stable] - stage: choco_deploy - tags: ["runner:windows-docker", "windowsversion:1809"] - needs: ["windows_choco_online_7_x64"] - variables: - ARCH: "x64" - before_script: - - $chocolateyApiKey = (aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.chocolatey_api_key --with-decryption --query "Parameter.Value" --out text) - script: - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - - $ErrorActionPreference = "Stop" - - Get-ChildItem omnibus\pkg - - if (Test-Path nupkg) { remove-item -recurse -force nupkg } - - mkdir nupkg - - copy omnibus\pkg\*.nupkg nupkg\ - - Get-ChildItem nupkg - - > - docker run --rm - -v "$(Get-Location):c:\mnt" - -e CHOCOLATEY_API_KEY=${chocolateyApiKey} - 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\chocopush.bat - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } diff --git a/.gitlab/container_build.yml b/.gitlab/container_build.yml deleted file mode 100644 index 9a7dc425c29475..00000000000000 --- a/.gitlab/container_build.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# container_build stage -# Contains jobs to build container images of the Agent. - -include: - - /.gitlab/container_build/docker_linux.yml - - /.gitlab/container_build/docker_windows.yml diff --git a/.gitlab/container_build/docker_linux.yml b/.gitlab/container_build/docker_linux.yml deleted file mode 100644 index ce8f88743b1ea2..00000000000000 --- a/.gitlab/container_build/docker_linux.yml +++ /dev/null @@ -1,246 +0,0 @@ ---- -.docker_build_job_definition: - stage: container_build - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - aws s3 sync --only-show-errors $S3_ARTIFACTS_URI $BUILD_CONTEXT - - TAG_SUFFIX=${TAG_SUFFIX:-} - - BUILD_ARG=${BUILD_ARG:-} - - ECR_RELEASE_SUFFIX=${CI_COMMIT_TAG+-release} - - TARGET_TAG=${IMAGE}${ECR_RELEASE_SUFFIX}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}$TAG_SUFFIX-$ARCH - # DockerHub login for build to limit rate limit when pulling base images - - DOCKER_REGISTRY_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_LOGIN_SSM_KEY --with-decryption --query "Parameter.Value" --out text) - - aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_PWD_SSM_KEY --with-decryption --query "Parameter.Value" --out text | docker login --username "$DOCKER_REGISTRY_LOGIN" --password-stdin "$DOCKER_REGISTRY_URL" - # Build image - - docker buildx build --no-cache --push --pull --platform linux/$ARCH --build-arg CIBUILD=true --build-arg GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} $BUILD_ARG --file $BUILD_CONTEXT/Dockerfile --tag ${TARGET_TAG} $BUILD_CONTEXT - # Squash image - - crane flatten -t ${TARGET_TAG} ${TARGET_TAG} - # Workaround for temporary network failures - retry: 2 - -.docker_build_job_definition_amd64: - extends: .docker_build_job_definition - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - ARCH: amd64 - -.docker_build_job_definition_arm64: - extends: .docker_build_job_definition - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:arm64"] - variables: - ARCH: arm64 - -# build agent6 py2 image -docker_build_agent6: - extends: .docker_build_job_definition_amd64 - rules: - !reference [.on_a6] - needs: - - job: agent_deb-x64-a6 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - TAG_SUFFIX: -6 - BUILD_ARG: --target test --build-arg PYTHON_VERSION=2 --build-arg DD_AGENT_ARTIFACT=datadog-agent_6*_amd64.deb - -docker_build_agent6_arm64: - extends: .docker_build_job_definition_arm64 - rules: - !reference [.on_all_builds_a6] - needs: - - job: agent_deb-arm64-a6 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - TAG_SUFFIX: -6 - BUILD_ARG: --target test --build-arg PYTHON_VERSION=2 --build-arg DD_AGENT_ARTIFACT=datadog-agent_6*arm64.deb - -# build agent6 py2 jmx image -docker_build_agent6_jmx: - extends: .docker_build_job_definition_amd64 - rules: - !reference [.on_a6] - needs: - - job: agent_deb-x64-a6 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - BUILD_ARTIFACT_GLOB: datadog-agent_6*_amd64.deb - TAG_SUFFIX: -6-jmx - BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg PYTHON_VERSION=2 --build-arg DD_AGENT_ARTIFACT=datadog-agent_6*_amd64.deb - -# build agent6 py2 jmx image -docker_build_agent6_jmx_arm64: - extends: .docker_build_job_definition_arm64 - rules: - !reference [.on_all_builds_a6] - needs: - - job: agent_deb-arm64-a6 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - BUILD_ARTIFACT_GLOB: datadog-agent_6*arm64.deb - TAG_SUFFIX: -6-jmx - BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg PYTHON_VERSION=2 --build-arg DD_AGENT_ARTIFACT=datadog-agent_6*arm64.deb - -# TESTING ONLY: This image is for internal testing purposes, not customer facing. -# build agent6 jmx unified image (including python3) -docker_build_agent6_py2py3_jmx: - extends: .docker_build_job_definition_amd64 - rules: - !reference [.on_a6] - needs: - - job: agent_deb-x64-a6 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - TAG_SUFFIX: -6-py2py3-jmx - BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg DD_AGENT_ARTIFACT=datadog-agent_6*_amd64.deb - -# build agent7 image -docker_build_agent7: - extends: .docker_build_job_definition_amd64 - rules: - !reference [.on_a7] - needs: - - job: agent_deb-x64-a7 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - TAG_SUFFIX: -7 - BUILD_ARG: --target test --build-arg PYTHON_VERSION=3 --build-arg DD_AGENT_ARTIFACT=datadog-agent_7*_amd64.deb - -single_machine_performance-amd64-a7: - extends: .docker_publish_job_definition - stage: container_build - rules: - !reference [.on_a7] - needs: - - docker_build_agent7 - variables: - IMG_REGISTRIES: internal-aws-smp - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-amd64 - IMG_DESTINATIONS: 08450328-agent:${CI_COMMIT_SHA}-7-amd64 - -docker_build_agent7_arm64: - extends: .docker_build_job_definition_arm64 - rules: - !reference [.on_a7] - needs: - - job: agent_deb-arm64-a7 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - TAG_SUFFIX: -7 - BUILD_ARG: --target test --build-arg PYTHON_VERSION=3 --build-arg DD_AGENT_ARTIFACT=datadog-agent_7*_arm64.deb - -# build agent7 jmx image -docker_build_agent7_jmx: - extends: .docker_build_job_definition_amd64 - rules: - !reference [.on_a7] - needs: - - job: agent_deb-x64-a7 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - TAG_SUFFIX: -7-jmx - BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg PYTHON_VERSION=3 --build-arg DD_AGENT_ARTIFACT=datadog-agent_7*_amd64.deb - -docker_build_agent7_jmx_arm64: - extends: .docker_build_job_definition_arm64 - rules: - !reference [.on_a7] - needs: - - job: agent_deb-arm64-a7 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - TAG_SUFFIX: -7-jmx - BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg PYTHON_VERSION=3 --build-arg DD_AGENT_ARTIFACT=datadog-agent_7*_arm64.deb - -# build the cluster-agent image -docker_build_cluster_agent_amd64: - extends: .docker_build_job_definition_amd64 - rules: - !reference [.on_tag_or_a7] - needs: - - job: cluster_agent-build_amd64 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/cluster-agent - BUILD_CONTEXT: Dockerfiles/cluster-agent - before_script: - - cp -Rvf Dockerfiles/agent/nosys-seccomp Dockerfiles/cluster-agent/ - -docker_build_cluster_agent_arm64: - extends: .docker_build_job_definition_arm64 - rules: - !reference [.on_tag_or_a7] - needs: - - job: cluster_agent-build_arm64 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/cluster-agent - BUILD_CONTEXT: Dockerfiles/cluster-agent - before_script: - - cp -Rvf Dockerfiles/agent/nosys-seccomp Dockerfiles/cluster-agent/ - -# build the cws-instrumentation image -docker_build_cws_instrumentation_amd64: - extends: .docker_build_job_definition_amd64 - rules: - !reference [.on_tag_or_a7] - needs: - - job: cws_instrumentation-build_amd64 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/cws-instrumentation - BUILD_CONTEXT: Dockerfiles/cws-instrumentation - -docker_build_cws_instrumentation_arm64: - extends: .docker_build_job_definition_arm64 - rules: - !reference [.on_tag_or_a7] - needs: - - job: cws_instrumentation-build_arm64 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/cws-instrumentation - BUILD_CONTEXT: Dockerfiles/cws-instrumentation - -# build the dogstatsd image -docker_build_dogstatsd_amd64: - extends: .docker_build_job_definition_amd64 - rules: - !reference [.on_a7] - needs: - - job: build_dogstatsd_static-binary_x64 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/dogstatsd - BUILD_CONTEXT: Dockerfiles/dogstatsd/alpine - -# build the dogstatsd image -docker_build_dogstatsd_arm64: - extends: .docker_build_job_definition_arm64 - rules: - !reference [.on_a7] - needs: - - job: build_dogstatsd_static-binary_arm64 - artifacts: false - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/dogstatsd - BUILD_CONTEXT: Dockerfiles/dogstatsd/alpine diff --git a/.gitlab/container_build/docker_windows.yml b/.gitlab/container_build/docker_windows.yml deleted file mode 100644 index 29ff403c47cbf5..00000000000000 --- a/.gitlab/container_build/docker_windows.yml +++ /dev/null @@ -1,104 +0,0 @@ ---- -.docker_build_windows_job_definition: - stage: container_build - before_script: - - $ErrorActionPreference = "Stop" - - mkdir ci-scripts - - | - @" - Set-PSDebug -Trace 1 - `$ErrorActionPreference = "Stop" - # ECR Login - `$AWS_ECR_PASSWORD = aws ecr get-login-password --region us-east-1 - docker login --username AWS --password "`${AWS_ECR_PASSWORD}" 486234852809.dkr.ecr.us-east-1.amazonaws.com - If (`$lastExitCode -ne "0") { throw "Previous command returned `$lastExitCode" } - # DockerHub login - `$DOCKER_REGISTRY_LOGIN = aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.${DOCKER_REGISTRY_LOGIN_SSM_KEY} --with-decryption --query "Parameter.Value" --out text - `$DOCKER_REGISTRY_PWD = aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.${DOCKER_REGISTRY_PWD_SSM_KEY} --with-decryption --query "Parameter.Value" --out text - docker login --username "`${DOCKER_REGISTRY_LOGIN}" --password "`${DOCKER_REGISTRY_PWD}" "${DOCKER_REGISTRY_URL}" - If (`$lastExitCode -ne "0") { throw "Previous command returned `$lastExitCode" } - "@ | out-file ci-scripts/docker-login.ps1 - # Workaround for temporary network failures - retry: 2 - -.docker_build_agent_windows_common: - extends: - - .docker_build_windows_job_definition - variables: - IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - script: - - $ECR_RELEASE_SUFFIX="$(If ($CI_COMMIT_TAG) { `"-release`" } else { `"`" })" - - $TARGET_TAG="${IMAGE}${ECR_RELEASE_SUFFIX}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}${TAG_SUFFIX}-win${VARIANT}${SERVERCORE}-amd64" - - $ErrorActionPreference = "Stop" - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - - cp ${OMNIBUS_PACKAGE_DIR}/${AGENT_ZIP} ${BUILD_CONTEXT}/datadog-agent-latest.amd64.zip - - cp entrypoint.exe ${BUILD_CONTEXT}/entrypoint.exe - - # Much faster but doesn't exist in build container - # - & 'C:\Program Files\7-Zip\7z.exe' x .\datadog-agent-latest.amd64.zip -o"Datadog Agent" - - pushd ${BUILD_CONTEXT} - - Expand-Archive datadog-agent-latest.amd64.zip - - Remove-Item datadog-agent-latest.amd64.zip - - Get-ChildItem -Path datadog-agent-* | Rename-Item -NewName "Datadog Agent" - - popd - - - get-childitem ${BUILD_CONTEXT} - # Docker setup - - cat ci-scripts/docker-login.ps1 - - > - docker run --rm - -w C:\mnt - -e AWS_NETWORKING=true - -e SIGN_WINDOWS_DD_WCS=true - -e CI_PIPELINE_ID=${CI_PIPELINE_ID} - -e CI_PROJECT_NAME=${CI_PROJECT_NAME} - -v "$(Get-Location):C:\mnt" - -v \\.\pipe\docker_engine:\\.\pipe\docker_engine 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_${Env:VARIANT}_x64${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - powershell - -C C:\mnt\ci-scripts\docker-login.ps1 - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - powershell -Command "docker build --no-cache --build-arg GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} ${BUILD_ARG} --pull --file ${BUILD_CONTEXT}/windows/amd64/Dockerfile --tag ${TARGET_TAG} ${BUILD_CONTEXT}" - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - docker push ${TARGET_TAG} - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - docker rmi ${TARGET_TAG} - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - -.docker_build_agent7_windows_common: - extends: - - .docker_build_agent_windows_common - rules: - !reference [.on_a7] - needs: ["windows_msi_and_bosh_zip_x64-a7", "build_windows_container_entrypoint"] - variables: - AGENT_ZIP: "datadog-agent-7*-x86_64.zip" - BUILD_ARG: "--build-arg BASE_IMAGE=mcr.microsoft.com/powershell:nanoserver-${VARIANT} --build-arg WITH_JMX=${WITH_JMX} --build-arg VARIANT=${VARIANT} --build-arg INSTALL_INFO=nano-${VARIANT}" - -.docker_build_agent6_windows_common: - extends: - - .docker_build_agent_windows_common - rules: - !reference [.on_a6] - needs: ["windows_msi_x64-a6", "build_windows_container_entrypoint"] - variables: - AGENT_ZIP: "datadog-agent-6*-x86_64.zip" - BUILD_ARG: "--build-arg BASE_IMAGE=mcr.microsoft.com/powershell:nanoserver-${VARIANT} --build-arg WITH_JMX=${WITH_JMX} --build-arg VARIANT=${VARIANT} --build-arg INSTALL_INFO=nano-${VARIANT}" - -.docker_build_agent6_windows_servercore_common: - extends: - - .docker_build_agent6_windows_common - variables: - BUILD_ARG: "--build-arg BASE_IMAGE=mcr.microsoft.com/powershell:windowsservercore-${VARIANT} --build-arg WITH_JMX=${WITH_JMX} --build-arg VARIANT=${VARIANT} --build-arg INSTALL_INFO=core-${VARIANT}" - SERVERCORE: "-servercore" - -.docker_build_agent7_windows_servercore_common: - extends: - - .docker_build_agent7_windows_common - variables: - BUILD_ARG: "--build-arg BASE_IMAGE=mcr.microsoft.com/powershell:windowsservercore-${VARIANT} --build-arg WITH_JMX=${WITH_JMX} --build-arg VARIANT=${VARIANT} --build-arg INSTALL_INFO=core-${VARIANT}" - SERVERCORE: "-servercore" - -include: - - /.gitlab/container_build/docker_windows_agent6.yml - - /.gitlab/container_build/docker_windows_agent7.yml diff --git a/.gitlab/container_build/docker_windows_agent6.yml b/.gitlab/container_build/docker_windows_agent6.yml deleted file mode 100644 index e2ef18af2bcc2f..00000000000000 --- a/.gitlab/container_build/docker_windows_agent6.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -# Python 2 does not run on nanoserver -docker_build_agent6_windows1809_core: - extends: - - .docker_build_agent6_windows_servercore_common - tags: ["runner:windows-docker", "windowsversion:1809"] - variables: - VARIANT: 1809 - TAG_SUFFIX: -6 - WITH_JMX: "false" - -docker_build_agent6_windows2022_core: - extends: - - .docker_build_agent6_windows_servercore_common - tags: ["runner:windows-docker", "windowsversion:2022"] - variables: - VARIANT: ltsc2022 - TAG_SUFFIX: -6 - WITH_JMX: "false" diff --git a/.gitlab/container_build/docker_windows_agent7.yml b/.gitlab/container_build/docker_windows_agent7.yml deleted file mode 100644 index e8bf47cbba812f..00000000000000 --- a/.gitlab/container_build/docker_windows_agent7.yml +++ /dev/null @@ -1,74 +0,0 @@ ---- -docker_build_agent7_windows1809: - extends: - - .docker_build_agent7_windows_common - tags: ["runner:windows-docker", "windowsversion:1809"] - variables: - VARIANT: 1809 - TAG_SUFFIX: -7 - WITH_JMX: "false" - -docker_build_agent7_windows1809_jmx: - extends: - - .docker_build_agent7_windows_common - tags: ["runner:windows-docker", "windowsversion:1809"] - variables: - VARIANT: 1809 - TAG_SUFFIX: -7-jmx - WITH_JMX: "true" - -docker_build_agent7_windows2022_jmx: - extends: - - .docker_build_agent7_windows_common - tags: ["runner:windows-docker", "windowsversion:2022"] - needs: ["windows_msi_and_bosh_zip_x64-a7", "build_windows_container_entrypoint"] - variables: - VARIANT: ltsc2022 - TAG_SUFFIX: -7-jmx - WITH_JMX: "true" - -docker_build_agent7_windows2022: - extends: - - .docker_build_agent7_windows_common - tags: ["runner:windows-docker", "windowsversion:2022"] - variables: - VARIANT: ltsc2022 - TAG_SUFFIX: "-7" - WITH_JMX: "false" - -docker_build_agent7_windows1809_core: - extends: - - .docker_build_agent7_windows_servercore_common - tags: ["runner:windows-docker", "windowsversion:1809"] - variables: - VARIANT: 1809 - TAG_SUFFIX: -7 - WITH_JMX: "false" - -docker_build_agent7_windows1809_core_jmx: - extends: - - .docker_build_agent7_windows_servercore_common - tags: ["runner:windows-docker", "windowsversion:1809"] - variables: - VARIANT: 1809 - TAG_SUFFIX: -7-jmx - WITH_JMX: "true" - -docker_build_agent7_windows2022_core: - extends: - - .docker_build_agent7_windows_servercore_common - tags: ["runner:windows-docker", "windowsversion:2022"] - variables: - VARIANT: ltsc2022 - TAG_SUFFIX: "-7" - WITH_JMX: "false" - -docker_build_agent7_windows2022_core_jmx: - extends: - - .docker_build_agent7_windows_servercore_common - tags: ["runner:windows-docker", "windowsversion:2022"] - needs: ["windows_msi_and_bosh_zip_x64-a7", "build_windows_container_entrypoint"] - variables: - VARIANT: ltsc2022 - TAG_SUFFIX: -7-jmx - WITH_JMX: "true" diff --git a/.gitlab/container_scan.yml b/.gitlab/container_scan.yml deleted file mode 100644 index 464aa047dfe5e1..00000000000000 --- a/.gitlab/container_scan.yml +++ /dev/null @@ -1,120 +0,0 @@ ---- -# container_scan stage -# Contains jobs to deploy container images of the Agent to specific Dockerhub repos -# (datadog/agent-scan and datadog/dogstatsd-scan) to be scanned. - -# push every night to docker hub agent-scan repo -scan_nightly-dogstatsd: - extends: .docker_publish_job_definition - stage: container_scan - rules: - !reference [.on_deploy_nightly_repo_branch_a7] - needs: - - docker_build_dogstatsd_amd64 - variables: - IMG_REGISTRIES: dev - IMG_SOURCES: ${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64 - IMG_DESTINATIONS: dogstatsd-scan:${BUCKET_BRANCH} - -scan_nightly-a6: - extends: .docker_publish_job_definition - stage: container_scan - rules: - !reference [.on_deploy_nightly_repo_branch_a6] - needs: - - docker_build_agent6 - - docker_build_agent6_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-amd64 - IMG_DESTINATIONS: agent-scan:${BUCKET_BRANCH}-py2 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-amd64 - IMG_DESTINATIONS: agent-scan:${BUCKET_BRANCH}-py2-jmx - -scan_nightly-a7: - extends: .docker_publish_job_definition - stage: container_scan - rules: - !reference [.on_deploy_nightly_repo_branch_a7] - needs: - - docker_build_agent7 - - docker_build_agent7_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-amd64 - IMG_DESTINATIONS: agent-scan:${BUCKET_BRANCH}-py3 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-amd64 - IMG_DESTINATIONS: agent-scan:${BUCKET_BRANCH}-py3-jmx - -dca_scan_nightly: - extends: .docker_publish_job_definition - stage: container_scan - rules: - !reference [.on_deploy_nightly_repo_branch_a7] - needs: ["docker_build_cluster_agent_amd64"] - variables: - IMG_REGISTRIES: dev - IMG_SOURCES: ${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64 - IMG_DESTINATIONS: cluster-agent-scan:${BUCKET_BRANCH} - -# push on master to docker hub agent-scan repo -scan_master-dogstatsd: - extends: .docker_publish_job_definition - stage: container_scan - rules: - !reference [.on_main_a7] - needs: - - docker_build_dogstatsd_amd64 - variables: - IMG_REGISTRIES: dev - IMG_SOURCES: ${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64 - IMG_DESTINATIONS: dogstatsd-scan:master - -scan_master-a6: - extends: .docker_publish_job_definition - stage: container_scan - rules: - !reference [.on_main_a6] - needs: - - docker_build_agent6 - - docker_build_agent6_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-amd64 - IMG_DESTINATIONS: agent-scan:master-py2 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-amd64 - IMG_DESTINATIONS: agent-scan:master-py2-jmx - -scan_master-a7: - extends: .docker_publish_job_definition - stage: container_scan - rules: - !reference [.on_main_a7] - needs: - - docker_build_agent7 - - docker_build_agent7_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-amd64 - IMG_DESTINATIONS: agent-scan:master-py3 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-amd64 - IMG_DESTINATIONS: agent-scan:master-py3-jmx - -dca_scan_master: - extends: .docker_publish_job_definition - stage: container_scan - rules: - !reference [.on_main_a7] - needs: ["docker_build_cluster_agent_amd64"] - variables: - IMG_REGISTRIES: dev - IMG_SOURCES: ${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64 - IMG_DESTINATIONS: cluster-agent-scan:master diff --git a/.gitlab/deploy_common.yml b/.gitlab/deploy_common.yml deleted file mode 100644 index d892376b8eecd7..00000000000000 --- a/.gitlab/deploy_common.yml +++ /dev/null @@ -1,74 +0,0 @@ ---- -.deploy_packages_deb: - resource_group: deb_bucket - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - before_script: - - ls $OMNIBUS_PACKAGE_DIR - script: - - $S3_CP_CMD --recursive --exclude "*" --include "*_${MAJOR_VERSION}.*${PACKAGE_ARCH}.deb" "$OMNIBUS_PACKAGE_DIR" "$S3_RELEASE_ARTIFACTS_URI/deb/${PACKAGE_ARCH}/" || true - -.deploy_packages_deb-6: - extends: .deploy_packages_deb - stage: deploy_packages - rules: - !reference [.on_deploy_a6] - variables: - MAJOR_VERSION: 6 - -.deploy_packages_deb-7: - extends: .deploy_packages_deb - stage: deploy_packages - rules: - !reference [.on_deploy_a7] - variables: - MAJOR_VERSION: 7 - -.deploy_packages_rpm: - resource_group: rpm_bucket - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - variables: - ARTIFACTS_PREFIX: "" - before_script: - - ls $OMNIBUS_PACKAGE_DIR - script: - - $S3_CP_CMD --recursive --exclude "*" --include "*-${MAJOR_VERSION}.*${PACKAGE_ARCH}.rpm" "$OMNIBUS_PACKAGE_DIR" "$S3_RELEASE_ARTIFACTS_URI/${ARTIFACTS_PREFIX}rpm/${PACKAGE_ARCH}/" || true - -.deploy_packages_rpm-6: - extends: .deploy_packages_rpm - stage: deploy_packages - rules: - !reference [.on_deploy_a6] - variables: - MAJOR_VERSION: 6 - -.deploy_packages_rpm-7: - extends: .deploy_packages_rpm - stage: deploy_packages - rules: - !reference [.on_deploy_a7] - variables: - MAJOR_VERSION: 7 - -.deploy_packages_suse_rpm: - extends: .deploy_packages_rpm - variables: - ARTIFACTS_PREFIX: suse_ - OMNIBUS_PACKAGE_DIR: $OMNIBUS_PACKAGE_DIR_SUSE - -.deploy_packages_suse_rpm-6: - extends: .deploy_packages_suse_rpm - stage: deploy_packages - rules: - !reference [.on_deploy_a6] - variables: - MAJOR_VERSION: 6 - -.deploy_packages_suse_rpm-7: - extends: .deploy_packages_suse_rpm - stage: deploy_packages - rules: - !reference [.on_deploy_a7] - variables: - MAJOR_VERSION: 7 diff --git a/.gitlab/deploy_containers.yml b/.gitlab/deploy_containers.yml deleted file mode 100644 index 0611597cde3754..00000000000000 --- a/.gitlab/deploy_containers.yml +++ /dev/null @@ -1,51 +0,0 @@ ---- -# deploy containers stage -# Contains jobs which create child pipelines to deploy Agent 6 & 7 to staging repositories and to Dockerhub / GCR. - -# -# Agent v6 -# - -deploy_containers-a6: - stage: deploy_containers - rules: - !reference [.on_deploy_a6] - variables: - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - BUCKET_BRANCH: $BUCKET_BRANCH - trigger: - include: .gitlab/deploy_containers/deploy_containers_a6.yml - -deploy_containers-a6-on-failure: - stage: deploy_containers - rules: - !reference [.on_deploy_a6_failure] - variables: - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - BUCKET_BRANCH: $BUCKET_BRANCH - FORCE_MANUAL: "true" - trigger: - include: .gitlab/deploy_containers/deploy_containers_a6.yml - - -deploy_containers-a7: - stage: deploy_containers - rules: - !reference [.on_deploy_a7] - variables: - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - BUCKET_BRANCH: $BUCKET_BRANCH - trigger: - include: .gitlab/deploy_containers/deploy_containers_a7.yml - - -deploy_containers-a7-on-failure: - stage: deploy_containers - rules: - !reference [.on_deploy_a7_failure] - variables: - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - BUCKET_BRANCH: $BUCKET_BRANCH - FORCE_MANUAL: "true" - trigger: - include: .gitlab/deploy_containers/deploy_containers_a7.yml diff --git a/.gitlab/deploy_containers/conditions.yml b/.gitlab/deploy_containers/conditions.yml deleted file mode 100644 index 591105eaba91b3..00000000000000 --- a/.gitlab/deploy_containers/conditions.yml +++ /dev/null @@ -1,88 +0,0 @@ - -# Rule to make job manual when running deploy pipelines, or automatic and on success on RC pipelines -.manual_on_deploy_auto_on_rc: - - if: $BUCKET_BRANCH != "beta" && $BUCKET_BRANCH != "stable" - when: manual - allow_failure: true - variables: - AGENT_REPOSITORY: agent-dev - DSD_REPOSITORY: dogstatsd-dev - IMG_REGISTRIES: dev - - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$/ && $FORCE_MANUAL != "true" - when: on_success - variables: - AGENT_REPOSITORY: agent - DSD_REPOSITORY: dogstatsd - IMG_REGISTRIES: public - - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$/ && $FORCE_MANUAL == "true" - when: manual - allow_failure: true - variables: - AGENT_REPOSITORY: agent - DSD_REPOSITORY: dogstatsd - IMG_REGISTRIES: public - - when: manual - allow_failure: true - variables: - AGENT_REPOSITORY: agent - DSD_REPOSITORY: dogstatsd - IMG_REGISTRIES: public - -# Rule for job that are triggered on_success on RC pipelines -.on_rc: - - if: $FORCE_MANUAL == "true" && $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$/ - when: manual - - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$/ - when: on_success - -# Rule for job that can be triggered manually on final build, deploy to prod repository on stable branch deploy, else to dev repository -.on_final: - - if: $BRANCH_BUCKET == "beta" - when: never - - if: $BUCKET_BRANCH != "beta" && $BUCKET_BRANCH != "stable" - when: manual - allow_failure: true - variables: - AGENT_REPOSITORY: agent-dev - DSD_REPOSITORY: dogstatsd-dev - IMG_REGISTRIES: dev - - when: manual - allow_failure: true - variables: - AGENT_REPOSITORY: agent - DSD_REPOSITORY: dogstatsd - IMG_REGISTRIES: public - -# Rule to deploy to our internal repository, on stable branch deploy -.on_internal_final: - - if: $BRANCH_BUCKET == "beta" - when: never - - if: $BUCKET_BRANCH != "beta" && $BUCKET_BRANCH != "stable" - when: never - - when: manual - allow_failure: true - variables: - AGENT_REPOSITORY: ci/datadog-agent/agent-release - CLUSTER_AGENT_REPOSITORY: ci/datadog-agent/cluster-agent-release - DSD_REPOSITORY: ci/datadog-agent/dogstatsd-release - CWS_INSTRUMENTATION_REPOSITORY: ci/datadog-agent/cws-instrumentation-release - IMG_REGISTRIES: internal-aws-ddbuild - -# Rule to deploy to our internal repository on RC -.on_internal_rc: - - if: $FORCE_MANUAL == "true" && $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$/ - when: manual - variables: - AGENT_REPOSITORY: ci/datadog-agent/agent-release - CLUSTER_AGENT_REPOSITORY: ci/datadog-agent/cluster-agent-release - DSD_REPOSITORY: ci/datadog-agent/dogstatsd-release - CWS_INSTRUMENTATION_REPOSITORY: ci/datadog-agent/cws-instrumentation-release - IMG_REGISTRIES: internal-aws-ddbuild - - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$/ - when: on_success - variables: - AGENT_REPOSITORY: ci/datadog-agent/agent-release - CLUSTER_AGENT_REPOSITORY: ci/datadog-agent/cluster-agent-release - DSD_REPOSITORY: ci/datadog-agent/dogstatsd-release - CWS_INSTRUMENTATION_REPOSITORY: ci/datadog-agent/cws-instrumentation-release - IMG_REGISTRIES: internal-aws-ddbuild diff --git a/.gitlab/deploy_containers/deploy_containers_a6.yml b/.gitlab/deploy_containers/deploy_containers_a6.yml deleted file mode 100644 index 4717bfb310ce64..00000000000000 --- a/.gitlab/deploy_containers/deploy_containers_a6.yml +++ /dev/null @@ -1,70 +0,0 @@ ---- -# deploy containers stage -# Contains jobs which deploy Agent 6 & 7 to staging repositories and to Dockerhub / GCR. - -stages: - - deploy_containers - -include: - - /.gitlab/docker_common/publish_job_templates.yml - - /.gitlab/deploy_containers/conditions.yml - -# -# Image tagging & manifest publication -# - -# -# Agent v6 -# -.deploy_containers-a6-base: - extends: .docker_publish_job_definition - stage: deploy_containers - dependencies: [] - before_script: - - source /root/.bashrc - - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 6 --url-safe --pipeline-id $PARENT_PIPELINE_ID)"; fi - - export IMG_SOURCES="${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6${JMX}-amd64,${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6${JMX}-arm64" - - export IMG_DESTINATIONS="${AGENT_REPOSITORY}:${VERSION}${JMX}" - parallel: - matrix: - - JMX: - - "" - - "-jmx" - - -deploy_containers-a6: - extends: .deploy_containers-a6-base - rules: - !reference [.manual_on_deploy_auto_on_rc] - - - -deploy_containers-a6-rc: - extends: .deploy_containers-a6-base - variables: - AGENT_REPOSITORY: agent - DSD_REPOSITORY: dogstatsd - IMG_REGISTRIES: public - rules: - !reference [.on_rc] - variables: - VERSION: 6-rc - - -# -# Latest publication -# - -deploy_containers_latest-a6: - extends: .docker_publish_job_definition - stage: deploy_containers - rules: - !reference [.on_final] - dependencies: [] - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-amd64,${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-arm64 - IMG_DESTINATIONS: ${AGENT_REPOSITORY}:6,${AGENT_REPOSITORY}:latest-py2 - - IMG_SOURCES: ${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-amd64,${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-arm64 - IMG_DESTINATIONS: ${AGENT_REPOSITORY}:6-jmx,${AGENT_REPOSITORY}:latest-py2-jmx - diff --git a/.gitlab/deploy_containers/deploy_containers_a7.yml b/.gitlab/deploy_containers/deploy_containers_a7.yml deleted file mode 100644 index 9f2ff0fbe8e82c..00000000000000 --- a/.gitlab/deploy_containers/deploy_containers_a7.yml +++ /dev/null @@ -1,132 +0,0 @@ ---- -# deploy containers stage -# Contains jobs which deploy Agent 6 & 7 to staging repositories and to Dockerhub / GCR. - -stages: - - deploy_containers - -include: - - /.gitlab/docker_common/publish_job_templates.yml - - /.gitlab/deploy_containers/conditions.yml - -# -# Image tagging & manifest publication -# -# -# Agent v7 -# -.deploy_containers-a7-base: - extends: .docker_publish_job_definition - stage: deploy_containers - dependencies: [] - before_script: - - source /root/.bashrc - - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)"; fi - - export IMG_BASE_SRC="${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" - - export IMG_LINUX_SOURCES="${IMG_BASE_SRC}-7${JMX}-amd64,${IMG_BASE_SRC}-7${JMX}-arm64" - - export IMG_WINDOWS_SOURCES="${IMG_BASE_SRC}-7${JMX}-win1809${FLAVOR}-amd64,${IMG_BASE_SRC}-7${JMX}-winltsc2022${FLAVOR}-amd64" - - if [[ "$FLAVOR" == "-linux" ]]; then export IMG_SOURCES="${IMG_LINUX_SOURCES}"; elif [[ "$FLAVOR" == "-servercore" ]]; then export IMG_SOURCES="${IMG_WINDOWS_SOURCES}"; else export IMG_SOURCES="${IMG_LINUX_SOURCES},${IMG_WINDOWS_SOURCES}"; fi - - export IMG_DESTINATIONS="${AGENT_REPOSITORY}:${VERSION}${FLAVOR}${JMX}" - - -.deploy_containers-a7_external: - extends: .deploy_containers-a7-base - parallel: - matrix: - - JMX: - - "" - - "-jmx" - FLAVOR: - - "" - - "-servercore" - - "-linux" - - -deploy_containers-a7: - extends: .deploy_containers-a7_external - rules: - !reference [.manual_on_deploy_auto_on_rc] - -deploy_containers-a7-rc: - extends: .deploy_containers-a7_external - rules: - !reference [.on_rc] - variables: - VERSION: 7-rc - -deploy_containers-dogstatsd: - extends: .docker_publish_job_definition - stage: deploy_containers - rules: - !reference [.manual_on_deploy_auto_on_rc] - dependencies: [] - before_script: - - source /root/.bashrc - - export VERSION="$(inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)" - - export IMG_SOURCES="${SRC_DSD}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64" - - export IMG_DESTINATIONS="${DSD_REPOSITORY}:${VERSION}" - -deploy_containers-a7_internal: - extends: .deploy_containers-a7-base - rules: - !reference [.on_internal_final] - variables: - JMX: "-jmx" - - -deploy_containers-a7_internal-rc: - extends: .deploy_containers-a7-base - rules: - !reference [.on_internal_rc] - variables: - VERSION: 7-rc - - -# -# Latest publication -# - -deploy_containers_latest-a7: - extends: .docker_publish_job_definition - stage: deploy_containers - rules: - !reference [.on_final] - dependencies: [] - parallel: - matrix: - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-amd64,%BASE%-arm64,%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: ${AGENT_REPOSITORY}:7,${AGENT_REPOSITORY}:latest - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-amd64,%BASE%-arm64,%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: ${AGENT_REPOSITORY}:7-jmx,${AGENT_REPOSITORY}:latest-jmx - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: ${AGENT_REPOSITORY}:7-servercore,${AGENT_REPOSITORY}:latest-servercore - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: ${AGENT_REPOSITORY}:7-servercore-jmx,${AGENT_REPOSITORY}:latest-servercore-jmx - - -deploy_containers_latest-a7_internal: - extends: .docker_publish_job_definition - stage: deploy_containers - rules: - !reference [.on_internal_final] - dependencies: [] - parallel: - matrix: - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-amd64,%BASE%-arm64,%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: ${AGENT_REPOSITORY}:7-jmx - - -deploy_containers_latest-dogstatsd: - extends: .docker_publish_job_definition - stage: deploy_containers - rules: - !reference [.on_final] - dependencies: [] - variables: - IMG_SOURCES: ${SRC_DSD}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64 - IMG_DESTINATIONS: ${DSD_REPOSITORY}:7,${DSD_REPOSITORY}:latest diff --git a/.gitlab/deploy_cws_instrumentation.yml b/.gitlab/deploy_cws_instrumentation.yml deleted file mode 100644 index 5df8d00c79cd3b..00000000000000 --- a/.gitlab/deploy_cws_instrumentation.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -include: - - /.gitlab/docker_common/publish_job_templates.yml - -# -# CWS Instrumentation image tagging & manifest publication -# - -.deploy_containers-cws-instrumentation-base: - extends: .docker_publish_job_definition - stage: deploy_cws_instrumentation - dependencies: [] - before_script: - - source /root/.bashrc - - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 7 --url-safe)"; fi - - if [[ "$CWS_INSTRUMENTATION_REPOSITORY" == "" ]]; then export CWS_INSTRUMENTATION_REPOSITORY="cws-instrumentation"; fi - - export IMG_BASE_SRC="${SRC_CWS_INSTRUMENTATION}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" - - export IMG_SOURCES="${IMG_BASE_SRC}-amd64,${IMG_BASE_SRC}-arm64" - - export IMG_DESTINATIONS="${CWS_INSTRUMENTATION_REPOSITORY}:${VERSION}" - - -deploy_containers-cws-instrumentation-rc: - extends: .deploy_containers-cws-instrumentation-base - rules: - !reference [.on_deploy_a7_rc] - variables: - VERSION: rc - - -deploy_containers-cws-instrumentation-latest: - extends: .deploy_containers-cws-instrumentation-base - rules: - !reference [.on_deploy_a7_manual_final] - variables: - VERSION: latest diff --git a/.gitlab/deploy_dca.yml b/.gitlab/deploy_dca.yml deleted file mode 100644 index 2ebc7df49b6424..00000000000000 --- a/.gitlab/deploy_dca.yml +++ /dev/null @@ -1,55 +0,0 @@ ---- -include: - - /.gitlab/docker_common/publish_job_templates.yml - -# -# DCA image tagging & manifest publication -# - -.deploy_containers-dca-base: - extends: .docker_publish_job_definition - stage: deploy_dca - needs: - - job: "docker_build_cluster_agent_amd64" - artifacts: false - - job: "docker_build_cluster_agent_arm64" - artifacts: false - before_script: - - source /root/.bashrc - - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 7 --url-safe)"; fi - - if [[ "$CLUSTER_AGENT_REPOSITORY" == "" ]]; then export CLUSTER_AGENT_REPOSITORY="cluster-agent"; fi - - export IMG_BASE_SRC="${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" - - export IMG_SOURCES="${IMG_BASE_SRC}-amd64,${IMG_BASE_SRC}-arm64" - - export IMG_DESTINATIONS="${CLUSTER_AGENT_REPOSITORY}:${VERSION}" - -deploy_containers-dca: - extends: .deploy_containers-dca-base - rules: !reference [.on_deploy_a7_manual_auto_on_rc] - -deploy_containers-dca-rc: - extends: .deploy_containers-dca-base - rules: !reference [.on_deploy_a7_rc] - variables: - VERSION: rc - -deploy_containers-dca-latest: - extends: .deploy_containers-dca-base - rules: !reference [.on_deploy_a7_manual_final] - variables: - VERSION: latest - -deploy_containers-dca_internal: - extends: .deploy_containers-dca-base - rules: !reference [.on_deploy_a7_internal_manual_final] - -deploy_containers-dca_internal-rc: - extends: .deploy_containers-dca-base - rules: !reference [.on_deploy_a7_internal_rc] - variables: - VERSION: rc - -deploy_containers-dca_internal-latest: - extends: .deploy_containers-dca-base - rules: !reference [.on_deploy_a7_internal_manual_final] - variables: - VERSION: latest diff --git a/.gitlab/deploy_packages.yml b/.gitlab/deploy_packages.yml deleted file mode 100644 index 09ee08274eeb1f..00000000000000 --- a/.gitlab/deploy_packages.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# deploy_packages stage -# Contains jobs which deploy Agent 6 & 7 to staging repositories. -# Jobs are expected to depend on the underlying build job and -# start as soon as possible. - -include: - - /.gitlab/deploy_packages/cluster_agent_cloudfoundry.yml - - /.gitlab/deploy_packages/nix.yml - - /.gitlab/deploy_packages/windows.yml - - /.gitlab/deploy_packages/winget.yml diff --git a/.gitlab/deploy_packages/cluster_agent_cloudfoundry.yml b/.gitlab/deploy_packages/cluster_agent_cloudfoundry.yml deleted file mode 100644 index 1c965a021d7deb..00000000000000 --- a/.gitlab/deploy_packages/cluster_agent_cloudfoundry.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -deploy_cluster_agent_cloudfoundry: - rules: - !reference [.on_deploy_a7] - stage: deploy_packages - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - needs: ["cluster_agent_cloudfoundry-build_amd64"] - before_script: - - ls $OMNIBUS_PACKAGE_DIR - script: - - $S3_CP_CMD --recursive --exclude "*" --include "datadog-cluster-agent-cloudfoundry-*.tar.xz" $OMNIBUS_PACKAGE_DIR $S3_DSD6_URI/linux/cluster-agent-cloudfoundry/ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 diff --git a/.gitlab/deploy_packages/nix.yml b/.gitlab/deploy_packages/nix.yml deleted file mode 100644 index b3b0ae67cf6ada..00000000000000 --- a/.gitlab/deploy_packages/nix.yml +++ /dev/null @@ -1,189 +0,0 @@ ---- -# -# Agent v6 -# -deploy_packages_deb-x64-6: - extends: .deploy_packages_deb-6 - needs: [ agent_deb-x64-a6 ] - variables: - PACKAGE_ARCH: amd64 - -deploy_packages_deb-arm64-6: - extends: .deploy_packages_deb-6 - needs: [ agent_deb-arm64-a6 ] - variables: - PACKAGE_ARCH: arm64 - -deploy_packages_heroku_deb-x64-6: - extends: .deploy_packages_deb-6 - needs: [ agent_heroku_deb-x64-a6 ] - variables: - PACKAGE_ARCH: amd64 - -deploy_packages_rpm-x64-6: - extends: .deploy_packages_rpm-6 - needs: [ agent_rpm-x64-a6 ] - variables: - PACKAGE_ARCH: x86_64 - -deploy_packages_rpm-arm64-6: - extends: .deploy_packages_rpm-6 - needs: [ agent_rpm-arm64-a6 ] - variables: - PACKAGE_ARCH: aarch64 - -# NOTE: no SuSE ARM builds currently. -deploy_packages_suse_rpm-x64-6: - extends: .deploy_packages_suse_rpm-6 - needs: [ agent_suse-x64-a6 ] - variables: - PACKAGE_ARCH: x86_64 - -# -# Agent v7 -# -deploy_packages_deb-x64-7: - extends: .deploy_packages_deb-7 - needs: [ agent_deb-x64-a7 ] - variables: - PACKAGE_ARCH: amd64 - -deploy_packages_deb-arm64-7: - extends: .deploy_packages_deb-7 - needs: [ agent_deb-arm64-a7 ] - variables: - PACKAGE_ARCH: arm64 - -deploy_packages_heroku_deb-x64-7: - extends: .deploy_packages_deb-7 - needs: [ agent_heroku_deb-x64-a7 ] - variables: - PACKAGE_ARCH: amd64 - -deploy_packages_iot_deb-x64-7: - extends: .deploy_packages_deb-7 - needs: [ iot_agent_deb-x64 ] - variables: - PACKAGE_ARCH: amd64 - -deploy_packages_iot_deb-arm64-7: - extends: .deploy_packages_deb-7 - needs: [ iot_agent_deb-arm64 ] - variables: - PACKAGE_ARCH: arm64 - -deploy_packages_iot_deb-armhf-7: - extends: .deploy_packages_deb-7 - needs: [ iot_agent_deb-armhf ] - variables: - PACKAGE_ARCH: armhf - -deploy_packages_dogstatsd_deb-x64-7: - extends: .deploy_packages_deb-7 - needs: [ dogstatsd_deb-x64 ] - variables: - PACKAGE_ARCH: amd64 - -deploy_packages_dogstatsd_deb-arm64-7: - extends: .deploy_packages_deb-7 - needs: [ dogstatsd_deb-arm64 ] - variables: - PACKAGE_ARCH: arm64 - -deploy_packages_rpm-x64-7: - extends: .deploy_packages_rpm-7 - needs: [ agent_rpm-x64-a7 ] - variables: - PACKAGE_ARCH: x86_64 - -deploy_packages_rpm-arm64-7: - extends: .deploy_packages_rpm-7 - needs: [ agent_rpm-arm64-a7 ] - variables: - PACKAGE_ARCH: aarch64 - -deploy_packages_iot_rpm-x64-7: - extends: .deploy_packages_rpm-7 - needs: [ iot_agent_rpm-x64 ] - variables: - PACKAGE_ARCH: x86_64 - -deploy_packages_iot_rpm-arm64-7: - extends: .deploy_packages_rpm-7 - needs: [ iot_agent_rpm-arm64 ] - variables: - PACKAGE_ARCH: aarch64 - -deploy_packages_iot_rpm-armhf-7: - extends: .deploy_packages_rpm-7 - needs: [ iot_agent_rpm-armhf ] - variables: - PACKAGE_ARCH: armv7hl - -deploy_packages_dogstatsd_rpm-x64-7: - extends: .deploy_packages_rpm-7 - needs: [ dogstatsd_rpm-x64 ] - variables: - PACKAGE_ARCH: x86_64 - -deploy_packages_suse_rpm-x64-7: - extends: .deploy_packages_suse_rpm-7 - needs: [ agent_suse-x64-a7 ] - variables: - PACKAGE_ARCH: x86_64 - -deploy_packages_suse_rpm-arm64-7: - extends: .deploy_packages_suse_rpm-7 - needs: [ agent_suse-arm64-a7 ] - variables: - PACKAGE_ARCH: aarch64 - -deploy_packages_iot_suse_rpm-x64-7: - extends: .deploy_packages_suse_rpm-7 - needs: [ iot_agent_suse-x64 ] - variables: - PACKAGE_ARCH: x86_64 - -deploy_packages_dogstatsd_suse_rpm-x64-7: - extends: .deploy_packages_suse_rpm-7 - needs: [ dogstatsd_suse-x64 ] - variables: - PACKAGE_ARCH: x86_64 - -deploy_packages_dmg-x64-a7: - rules: - !reference [.on_deploy_a7] - stage: deploy_packages - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - needs: ["agent_dmg-x64-a7"] - before_script: - - ls $OMNIBUS_PACKAGE_DIR - script: - - $S3_CP_CMD --recursive --exclude "*" --include "datadog-agent-7*.dmg" $OMNIBUS_PACKAGE_DIR $S3_RELEASE_ARTIFACTS_URI/dmg/x86_64/ || true - -# deploy dogstatsd x64, non-static binary to staging bucket -deploy_staging_dsd: - rules: - !reference [.on_deploy_a7] - stage: deploy_packages - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - needs: ["build_dogstatsd-binary_x64"] - script: - - $S3_CP_CMD $S3_ARTIFACTS_URI/dogstatsd/dogstatsd ./dogstatsd - - export PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7) - - $S3_CP_CMD ./dogstatsd $S3_DSD6_URI/linux/dogstatsd-$PACKAGE_VERSION --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 - -# deploy iot-agent x64 binary to staging bucket -deploy_staging_iot_agent: - rules: - !reference [.on_deploy_a7] - stage: deploy_packages - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - needs: ["build_iot_agent-binary_x64"] - script: - - $S3_CP_CMD $S3_ARTIFACTS_URI/iot/agent ./agent - - export PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7) - - $S3_CP_CMD ./agent $S3_DSD6_URI/linux/iot/agent-$PACKAGE_VERSION --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 diff --git a/.gitlab/deploy_packages/windows.yml b/.gitlab/deploy_packages/windows.yml deleted file mode 100644 index 4733429c08f36c..00000000000000 --- a/.gitlab/deploy_packages/windows.yml +++ /dev/null @@ -1,69 +0,0 @@ ---- -# -# Agent v6 -# -deploy_packages_windows-x64-6: - rules: - !reference [.on_deploy_a6] - stage: deploy_packages - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - needs: ["windows_msi_x64-a6"] - before_script: - - ls $OMNIBUS_PACKAGE_DIR - script: - - $S3_CP_CMD - --recursive - --exclude "*" - --include "datadog-agent-6*.msi" - --include "datadog-agent-6*.debug.zip" - $OMNIBUS_PACKAGE_DIR $S3_RELEASE_ARTIFACTS_URI/msi/x86_64/ || true - -# -# Agent v7 -# -deploy_packages_windows-x64-7: - rules: - !reference [.on_deploy_a7] - stage: deploy_packages - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - needs: ["windows_msi_and_bosh_zip_x64-a7"] - before_script: - - ls $OMNIBUS_PACKAGE_DIR - script: - - $S3_CP_CMD - --recursive - --exclude "*" - --include "datadog-agent-7*.msi" - --include "datadog-agent-7*.debug.zip" - $OMNIBUS_PACKAGE_DIR $S3_RELEASE_ARTIFACTS_URI/msi/x86_64/ || true - -deploy_staging_windows_tags-7: - rules: - !reference [.on_deploy_stable_or_beta_repo_branch_a7] - stage: deploy_packages - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - needs: ["windows_msi_and_bosh_zip_x64-a7", "windows_zip_agent_binaries_x64-a7"] - before_script: - - ls $OMNIBUS_PACKAGE_DIR - script: - # used for cloudfoundry bosh - - $S3_CP_CMD - --recursive - --exclude "*" - --include "datadog-agent-7.*.zip" - $OMNIBUS_PACKAGE_DIR - $S3_DSD6_URI/windows/agent7/bosh/ - --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers - full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 - # used for cloudfoundry buildpack and azure-app-services - - $S3_CP_CMD - --recursive - --exclude "*" - --include "agent-binaries-7.*.zip" - $OMNIBUS_PACKAGE_DIR $S3_DSD6_URI/windows/agent7/buildpack/ - --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers - full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 - diff --git a/.gitlab/deploy_packages/winget.yml b/.gitlab/deploy_packages/winget.yml deleted file mode 100644 index a7176a3a6c696f..00000000000000 --- a/.gitlab/deploy_packages/winget.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -# deploy_packages stage -# Contains a job which deploys the Winget Agent package. - -publish_winget_7_x64: - dependencies: [] - rules: - !reference [.on_deploy_stable_or_beta_repo_branch_a7_manual] - stage: deploy_packages - tags: ["runner:windows-docker", "windowsversion:1809"] - variables: - ARCH: "x64" - before_script: - - $wingetPat = (aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.winget_pat --with-decryption --query "Parameter.Value" --out text) - script: - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - - $ErrorActionPreference = "Stop" - - > - docker run --rm - -v "$(Get-Location):c:\mnt" - -e WINGET_GITHUB_ACCESS_TOKEN=${wingetPat} - -e GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} - 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - Powershell - -C "C:\mnt\tasks\winbuildscripts\Update-Winget.ps1" - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } diff --git a/.gitlab/deps_build.yml b/.gitlab/deps_build.yml deleted file mode 100644 index 1f38d0d654f085..00000000000000 --- a/.gitlab/deps_build.yml +++ /dev/null @@ -1,130 +0,0 @@ ---- -# deps_build stage -# Contains jobs which build various libraries needed for system-probe. - -.build_clang_common: - rules: - !reference [.manual] - stage: deps_build - script: - # use tmpdir to prevent git remote capture by clang build - - mkdir /tmp/clangbuild && cd /tmp/clangbuild - - CLANG_MAJOR_VER=$(echo "${CLANG_LLVM_VER}" | cut -d '.' -f 1) - - CLANG_SHA256SUM=6e912133bcf56e9cfe6a346fa7e5c52c2cde3e4e48b7a6cc6fcc7c75047da45f - - LLVM_SHA256SUM=7d9a8405f557cefc5a21bf5672af73903b64749d9bc3a50322239f56f34ffddf - - wget ${GENERAL_ARTIFACTS_CACHE_BUCKET_URL}/llvm/clang-${CLANG_LLVM_VER}.src.tar.xz -O clang.src.tar.xz - - wget ${GENERAL_ARTIFACTS_CACHE_BUCKET_URL}/llvm/llvm-${CLANG_LLVM_VER}.src.tar.xz -O llvm.src.tar.xz - - echo -e "${LLVM_SHA256SUM} llvm.src.tar.xz\n${CLANG_SHA256SUM} clang.src.tar.xz" | sha256sum --check - - - mkdir clang && tar xf clang.src.tar.xz --strip-components=1 --no-same-owner -C clang - - mkdir llvm && tar xf llvm.src.tar.xz --strip-components=1 --no-same-owner -C llvm - - mkdir build && cd build - # copy llvm cmake modules into the correct directory to ensure discovery - # Build runs into this issue if the correct modules cannot be discovered: - # https://github.com/llvm/llvm-project/issues/53281 - - cp ../llvm/Modules/* ../llvm/cmake/modules || echo "LLVM common cmake utils not found in directory 'Modules'" - # patch references to glibc >= 2.18 - - patch -ruN ../llvm/CMakeLists.txt < $CI_PROJECT_DIR/pkg/ebpf/c/cmakelists.patch - - patch -ruN ../llvm/lib/Analysis/ConstantFolding.cpp < $CI_PROJECT_DIR/pkg/ebpf/c/constant_folding.patch - - | - cmake -DLLVM_ENABLE_PROJECTS=clang \ - -DLLVM_TARGETS_TO_BUILD="BPF" \ - -DCMAKE_INSTALL_PREFIX=$DATADOG_AGENT_EMBEDDED_PATH \ - -G "Ninja" \ - -DCMAKE_BUILD_TYPE=MinSizeRel \ - -DLLVM_BUILD_TOOLS=ON \ - -DLLVM_ENABLE_TERMINFO=OFF \ - -DLLVM_INCLUDE_EXAMPLES=OFF \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_INCLUDE_BENCHMARKS=OFF \ - -DLLVM_STATIC_LINK_CXX_STDLIB=ON \ - -DLLVM_ENABLE_BINDINGS=OFF \ - -DLLVM_PARALLEL_COMPILE_JOBS=4 \ - -DLLVM_PARALLEL_LINK_JOBS=4 \ - -DLLVM_ENABLE_UNWIND_TABLES=OFF \ - ../llvm - - cmake --build . --target install - # fail if references to glibc >= 2.18 - - objdump -p $DATADOG_AGENT_EMBEDDED_PATH/bin/clang-${CLANG_MAJOR_VER} | egrep 'GLIBC_2\.(1[8-9]|[2-9][0-9])' && exit 1 - - objdump -p $DATADOG_AGENT_EMBEDDED_PATH/bin/llc | egrep 'GLIBC_2\.(1[8-9]|[2-9][0-9])' && exit 1 - # fail if clang or llc targets anything other than bpf - - $DATADOG_AGENT_EMBEDDED_PATH/bin/clang-${CLANG_MAJOR_VER} -print-targets > clang_targets.txt - - diff $CI_PROJECT_DIR/pkg/ebpf/c/expected_targets.txt clang_targets.txt - - $DATADOG_AGENT_EMBEDDED_PATH/bin/llc --version | tail -n +7 > llc_targets.txt # remove first 6 lines of llc version output; we only care about the registered targets - - diff $CI_PROJECT_DIR/pkg/ebpf/c/expected_targets.txt llc_targets.txt - - $S3_CP_CMD $DATADOG_AGENT_EMBEDDED_PATH/bin/clang-${CLANG_MAJOR_VER} $S3_PERMANENT_ARTIFACTS_URI/clang-${CLANG_LLVM_VER}.${ARCH} - - $S3_CP_CMD $DATADOG_AGENT_EMBEDDED_PATH/bin/llc $S3_PERMANENT_ARTIFACTS_URI/llc-${CLANG_LLVM_VER}.${ARCH} - - $S3_CP_CMD $DATADOG_AGENT_EMBEDDED_PATH/bin/clang-${CLANG_MAJOR_VER} $S3_DD_AGENT_OMNIBUS_LLVM_URI/clang-${CLANG_LLVM_VER}.${ARCH} --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers - - $S3_CP_CMD $DATADOG_AGENT_EMBEDDED_PATH/bin/llc $S3_DD_AGENT_OMNIBUS_LLVM_URI/llc-${CLANG_LLVM_VER}.${ARCH} --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers - - mkdir -p $CI_PROJECT_DIR/.tmp - - cp $DATADOG_AGENT_EMBEDDED_PATH/bin/clang-${CLANG_MAJOR_VER} $CI_PROJECT_DIR/.tmp/clang - - cp $DATADOG_AGENT_EMBEDDED_PATH/bin/llc $CI_PROJECT_DIR/.tmp/llc - timeout: 2h 00m - artifacts: - expire_in: 2 weeks - paths: - - $CI_PROJECT_DIR/.tmp/clang - - $CI_PROJECT_DIR/.tmp/llc - -build_clang_x64: - extends: .build_clang_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:amd64"] - variables: - ARCH: amd64 - -build_clang_arm64: - extends: .build_clang_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:arm64"] - variables: - ARCH: arm64 - -build_processed_btfhub_archive: - rules: - - !reference [.except_mergequeue] - - !reference [.on_scheduled_main_or_manual] - stage: deps_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/btf-gen$DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BTF_GEN_BUILDIMAGES - tags: ["arch:amd64"] - script: - - git clone https://github.com/aquasecurity/btfhub-archive.git - - cd btfhub-archive - # Flatten btfhub-archive directories & separate arm/x64 btfs into separate directories - - dirs=("amzn/2" "centos/7" "centos/8" "debian/9" "debian/10" "fedora/28" "fedora/29" "fedora/30" "fedora/31" "ol/7" "ol/8" "rhel/7" "rhel/8") - - | - for dir in ${dirs[@]}; do - platform=${dir%%/*} - mkdir -p "btfs-amd64/${platform}" "btfs-arm64/${platform}" - eval "mv ${dir}/x86_64/*.btf.tar.xz btfs-amd64/${platform}/" - eval "mv ${dir}/arm64/*.btf.tar.xz btfs-arm64/${platform}/" - done - # Handle the these directories separately because they don't have an arm64 version - - x64dirs=("amzn/2018" "fedora/24" "fedora/25" "fedora/26" "fedora/27" "sles/12.3" "sles/12.5" "sles/15.3") - - | - for xdir in ${x64dirs[@]}; do - platform=${xdir%%/*} - mkdir -p "btfs-amd64/${platform}" - eval "mv ${xdir}/x86_64/*.btf.tar.xz btfs-amd64/${platform}" - done - # Handle ubuntu separately because we want to keep the btfs separated by ubuntu version - - ubuntu_dirs=("ubuntu/16.04" "ubuntu/18.04" "ubuntu/20.04") - - | - for udir in ${ubuntu_dirs[@]}; do - mkdir -p "btfs-amd64/${udir}" "btfs-arm64/${udir}" - eval "mv ${udir}/x86_64/*.btf.tar.xz btfs-amd64/${udir}/" - eval "mv ${udir}/arm64/*.btf.tar.xz btfs-arm64/${udir}/" - done - # Clean up platform names to match the names we get at runtime from gopsutil - - mv btfs-amd64/amzn btfs-amd64/amazon - - mv btfs-arm64/amzn btfs-arm64/amazon - - mv btfs-amd64/ol btfs-amd64/oracle - - mv btfs-arm64/ol btfs-arm64/oracle - - mv btfs-amd64/rhel btfs-amd64/redhat - - mv btfs-arm64/rhel btfs-arm64/redhat - # Store results in S3 - - tar -czf btfs-arm64.tar.gz btfs-arm64 - - tar -czf btfs-amd64.tar.gz btfs-amd64 - - $S3_CP_CMD btfs-arm64.tar.gz $S3_PERMANENT_ARTIFACTS_URI/btfs-arm64.tar.gz - - $S3_CP_CMD btfs-amd64.tar.gz $S3_PERMANENT_ARTIFACTS_URI/btfs-amd64.tar.gz - - $S3_CP_CMD btfs-arm64.tar.gz $S3_DD_AGENT_OMNIBUS_BTFS_URI/btfs-arm64.tar.gz --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers - - $S3_CP_CMD btfs-amd64.tar.gz $S3_DD_AGENT_OMNIBUS_BTFS_URI/btfs-amd64.tar.gz --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers diff --git a/.gitlab/deps_fetch.yml b/.gitlab/deps_fetch.yml deleted file mode 100644 index 974463a65591dd..00000000000000 --- a/.gitlab/deps_fetch.yml +++ /dev/null @@ -1,71 +0,0 @@ ---- -# deps_fetch stage -# Contains jobs which fetch go dependencies, in order -# to reuse them in further jobs that need them. - -.retrieve_linux_go_deps: - - mkdir -p $GOPATH/pkg/mod && tar xzf modcache.tar.gz -C $GOPATH/pkg/mod - - rm -f modcache.tar.gz - -.retrieve_linux_go_tools_deps: - - mkdir -p $GOPATH/pkg/mod && tar xzf modcache_tools.tar.gz -C $GOPATH/pkg/mod - - rm -f modcache_tools.tar.gz - -.cache_policy: - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - variables: - POLICY: pull-push - - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH - variables: - POLICY: pull - -go_deps: - stage: deps_fetch - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["setup_agent_version"] - extends: .cache_policy - script: - # If the cache already contains the dependencies, don't redownload them - # but still provide the artifact that's expected for the other jobs to run - - if [ -f modcache.tar.gz ]; then exit 0; fi - - source /root/.bashrc - - inv -e deps --verbose - - cd $GOPATH/pkg/mod/ && tar czf $CI_PROJECT_DIR/modcache.tar.gz . - artifacts: - expire_in: 1 day - paths: - - $CI_PROJECT_DIR/modcache.tar.gz - cache: - - key: - files: - - ./**/go.mod - prefix: "go_deps" - paths: - - modcache.tar.gz - retry: 1 - -go_tools_deps: - stage: deps_fetch - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["setup_agent_version"] - extends: .cache_policy - script: - - if [ -f modcache_tools.tar.gz ]; then exit 0; fi - - source /root/.bashrc - - inv -e download-tools - - cd $GOPATH/pkg/mod/ && tar czf $CI_PROJECT_DIR/modcache_tools.tar.gz . - artifacts: - expire_in: 1 day - paths: - - $CI_PROJECT_DIR/modcache_tools.tar.gz - cache: - - key: - files: - - ./**/go.mod - prefix: "go_tools_deps" - paths: - - modcache_tools.tar.gz - retry: 1 diff --git a/.gitlab/dev_container_deploy.yml b/.gitlab/dev_container_deploy.yml deleted file mode 100644 index 82f26e5e83a51e..00000000000000 --- a/.gitlab/dev_container_deploy.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# dev_container_deploy stage -# Contains jobs to container Docker images of the Agent -# (in the datadog/agent-dev | datadog/dogstatsd-dev Dockerhub repos). - -include: - - /.gitlab/dev_container_deploy/docker_linux.yml - - /.gitlab/dev_container_deploy/docker_windows.yml diff --git a/.gitlab/dev_container_deploy/docker_linux.yml b/.gitlab/dev_container_deploy/docker_linux.yml deleted file mode 100644 index b10cd105adfa04..00000000000000 --- a/.gitlab/dev_container_deploy/docker_linux.yml +++ /dev/null @@ -1,326 +0,0 @@ ---- -include: - - /.gitlab/docker_common/publish_job_templates.yml - -dev_branch-a6: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_a6_manual] - needs: - - docker_build_agent6 - - docker_build_agent6_jmx - - docker_build_agent6_py2py3_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-amd64 - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG},agent-dev:${CI_COMMIT_REF_SLUG}-py2 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-amd64 - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-jmx,agent-dev:${CI_COMMIT_REF_SLUG}-py2-jmx - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-py2py3-jmx-amd64 - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py2py3-jmx - -dev_branch-dogstatsd: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_a7_manual] - needs: - - docker_build_dogstatsd_amd64 - - docker_build_dogstatsd_arm64 - variables: - IMG_REGISTRIES: dev - IMG_SOURCES: ${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64 - IMG_DESTINATIONS: dogstatsd-dev:${CI_COMMIT_REF_SLUG} - -dev_branch_multiarch-a6: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_all_builds_a6_manual] - needs: - - docker_build_agent6 - - docker_build_agent6_arm64 - - docker_build_agent6_jmx - - docker_build_agent6_jmx_arm64 - - docker_build_agent6_py2py3_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-arm64 - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG},agent-dev:${CI_COMMIT_REF_SLUG}-py2 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-arm64 - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-jmx,agent-dev:${CI_COMMIT_REF_SLUG}-py2-jmx - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-py2py3-jmx-amd64 - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py2py3-jmx - -dev_branch_multiarch-a7: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_a7_manual] - needs: - - docker_build_agent7 - - docker_build_agent7_arm64 - - docker_build_agent7_jmx - - docker_build_agent7_jmx_arm64 - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-arm64 - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-arm64 - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx - -dev_branch_multiarch-dogstatsd: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_a7_manual] - needs: - - docker_build_dogstatsd_amd64 - - docker_build_dogstatsd_arm64 - variables: - IMG_REGISTRIES: dev - IMG_SOURCES: ${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64 - IMG_DESTINATIONS: dogstatsd-dev:${CI_COMMIT_REF_SLUG} - -dev_master-a6: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_main_a6] - needs: - - docker_build_agent6 - - docker_build_agent6_arm64 - - docker_build_agent6_jmx - - docker_build_agent6_jmx_arm64 - - docker_build_agent6_py2py3_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-arm64 - IMG_DESTINATIONS: agent-dev:master,agent-dev:master-py2 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-arm64 - IMG_DESTINATIONS: agent-dev:master-jmx,agent-dev:master-py2-jmx - -dev_master-a7: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_main_a7] - needs: - - docker_build_agent7 - - docker_build_agent7_arm64 - - docker_build_agent7_jmx - - docker_build_agent7_jmx_arm64 - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-arm64 - IMG_DESTINATIONS: agent-dev:master-py3 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-arm64 - IMG_DESTINATIONS: agent-dev:master-py3-jmx - -dev_master-dogstatsd: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_main_a7] - needs: - - docker_build_dogstatsd_amd64 - - docker_build_dogstatsd_arm64 - variables: - IMG_REGISTRIES: dev - IMG_SOURCES: ${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64 - IMG_DESTINATIONS: dogstatsd-dev:master - -dca_dev_branch: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_a7_manual] - needs: - - docker_build_cluster_agent_amd64 - variables: - IMG_REGISTRIES: dev - IMG_SOURCES: ${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64 - IMG_DESTINATIONS: cluster-agent-dev:${CI_COMMIT_REF_SLUG} - -dca_dev_branch_multiarch: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_all_builds_a7_manual] - needs: - - docker_build_cluster_agent_amd64 - - docker_build_cluster_agent_arm64 - variables: - IMG_REGISTRIES: dev - IMG_SOURCES: ${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64 - IMG_DESTINATIONS: cluster-agent-dev:${CI_COMMIT_REF_SLUG} - -dca_dev_master: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_main_a7] - needs: - - docker_build_cluster_agent_amd64 - variables: - IMG_REGISTRIES: dev - IMG_SOURCES: ${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64 - IMG_DESTINATIONS: cluster-agent-dev:master - -cws_instrumentation_dev_branch_multiarch: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_all_builds_a7_manual] - needs: - - docker_build_cws_instrumentation_amd64 - - docker_build_cws_instrumentation_arm64 - variables: - IMG_REGISTRIES: dev - IMG_SOURCES: ${SRC_CWS_INSTRUMENTATION}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_CWS_INSTRUMENTATION}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64 - IMG_DESTINATIONS: cws-instrumentation-dev:${CI_COMMIT_REF_SLUG} - -# deploys nightlies to agent-dev -dev_nightly_docker_hub-a6: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_deploy_nightly_repo_branch_a6] - needs: - - docker_build_agent6 - - docker_build_agent6_arm64 - - docker_build_agent6_jmx - - docker_build_agent6_jmx_arm64 - - docker_build_agent6_py2py3_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-arm64 - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA},agent-dev:nightly-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-py2,agent-dev:nightly-${CI_COMMIT_REF_SLUG}-py2 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-arm64 - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-jmx,agent-dev:nightly-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-py2-jmx,agent-dev:nightly-${CI_COMMIT_REF_SLUG}-py2-jmx - -# deploys nightlies to agent-dev -dev_nightly-a7: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_deploy_nightly_repo_branch_a7] - needs: - - docker_build_agent7 - - docker_build_agent7_arm64 - - docker_build_agent7_jmx - - docker_build_agent7_jmx_arm64 - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-arm64 - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-py3,agent-dev:nightly-${CI_COMMIT_REF_SLUG}-py3 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-arm64 - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-py3-jmx,agent-dev:nightly-${CI_COMMIT_REF_SLUG}-py3-jmx - -# deploy nightly build to single-machine-performance -single_machine_performance-nightly-amd64-a7: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_scheduled_main] - needs: - - docker_build_agent7 - variables: - IMG_REGISTRIES: internal-aws-smp - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-amd64 - IMG_DESTINATIONS: 08450328-agent:nightly-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA}-7-amd64 - -# deploys nightlies to agent-dev -dev_nightly-dogstatsd: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_deploy_nightly_repo_branch_a7] - needs: - - docker_build_dogstatsd_amd64 - - docker_build_dogstatsd_arm64 - variables: - IMG_REGISTRIES: dev - IMG_SOURCES: ${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64 - IMG_DESTINATIONS: dogstatsd-dev:nightly-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA},dogstatsd-dev:nightly-${CI_COMMIT_REF_SLUG} - -# push images to `datadog-agent-qa` ECR for the end-to-end tests defined in `e2e.yml` -.qa_agent: - extends: .docker_publish_job_definition - stage: dev_container_deploy - needs: - - docker_build_agent7 - - docker_build_agent7_arm64 - - docker_build_agent7_windows1809 - - docker_build_agent7_windows2022 - variables: - IMG_REGISTRIES: agent-qa - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-arm64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-win1809-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-winltsc2022-amd64 - IMG_DESTINATIONS: agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - -.qa_dca: - extends: .docker_publish_job_definition - stage: dev_container_deploy - needs: - - docker_build_cluster_agent_amd64 - - docker_build_cluster_agent_arm64 - variables: - IMG_REGISTRIES: agent-qa - IMG_SOURCES: ${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64 - IMG_DESTINATIONS: cluster-agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - -.qa_dogstatsd: - extends: .docker_publish_job_definition - stage: dev_container_deploy - needs: - - docker_build_dogstatsd_amd64 - - docker_build_dogstatsd_arm64 - variables: - IMG_REGISTRIES: agent-qa - IMG_SOURCES: ${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64 - IMG_DESTINATIONS: dogstatsd:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - -qa_main_agent: - extends: .qa_agent - rules: - !reference [.on_main_and_no_skip_e2e] - -qa_branch_agent: - extends: .qa_agent - rules: - !reference [.on_dev_branch_manual] - -qa_main_dca: - extends: .qa_dca - rules: - !reference [.on_main_and_no_skip_e2e] - -qa_branch_dca: - extends: .qa_dca - rules: - !reference [.on_dev_branch_manual] - -qa_main_dogstatsd: - extends: .qa_dogstatsd - rules: - !reference [.on_main_and_no_skip_e2e] - -qa_branch_dogstatsd: - extends: .qa_dogstatsd - rules: - !reference [.on_dev_branch_manual] diff --git a/.gitlab/dev_container_deploy/docker_windows.yml b/.gitlab/dev_container_deploy/docker_windows.yml deleted file mode 100644 index 90891df9654fba..00000000000000 --- a/.gitlab/dev_container_deploy/docker_windows.yml +++ /dev/null @@ -1,144 +0,0 @@ ---- -include: - - /.gitlab/docker_common/publish_job_templates.yml - -dev_branch-a7-windows: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_a7_manual] - needs: - - docker_build_agent7_windows1809 - - docker_build_agent7_windows1809_jmx - - docker_build_agent7_windows1809_core - - docker_build_agent7_windows1809_core_jmx - - docker_build_agent7_windows2022 - - docker_build_agent7_windows2022_jmx - - docker_build_agent7_windows2022_core - - docker_build_agent7_windows2022_core_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-win - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx-win - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-win-servercore - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx-win-servercore - -dev_branch-a6-windows: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_a6_manual] - needs: - - docker_build_agent6_windows1809_core - - docker_build_agent6_windows2022_core - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py2-win-servercore - -dev_master-a7-windows: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_main_a7] - needs: - - docker_build_agent7_windows1809 - - docker_build_agent7_windows1809_jmx - - docker_build_agent7_windows1809_core - - docker_build_agent7_windows1809_core_jmx - - docker_build_agent7_windows2022 - - docker_build_agent7_windows2022_jmx - - docker_build_agent7_windows2022_core - - docker_build_agent7_windows2022_core_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:master-py3-win - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:master-py3-jmx-win - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:master-py3-win-servercore - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:master-py3-jmx-win-servercore - -dev_master-a6-windows: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_main_a6] - needs: - - docker_build_agent6_windows1809_core - - docker_build_agent6_windows2022_core - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:master-py2-win-servercore - -dev_nightly-a7-windows: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_deploy_nightly_repo_branch_a7] - needs: - - docker_build_agent7_windows1809 - - docker_build_agent7_windows1809_jmx - - docker_build_agent7_windows1809_core - - docker_build_agent7_windows1809_core_jmx - - docker_build_agent7_windows2022 - - docker_build_agent7_windows2022_jmx - - docker_build_agent7_windows2022_core - - docker_build_agent7_windows2022_core_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_SHORT_SHA}-py3-win - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_SHORT_SHA}-py3-jmx-win - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_SHORT_SHA}-py3-win-servercore - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_SHORT_SHA}-py3-jmx-win-servercore - -dev_nightly-a6-windows: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_deploy_nightly_repo_branch_a6] - needs: - - docker_build_agent6_windows1809_core - - docker_build_agent6_windows2022_core - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_SHORT_SHA}-py2-win-servercore diff --git a/.gitlab/do_not_merge.yml b/.gitlab/do_not_merge.yml deleted file mode 100644 index 877698f1d2881c..00000000000000 --- a/.gitlab/do_not_merge.yml +++ /dev/null @@ -1,28 +0,0 @@ -do-not-merge: - stage: check_merge - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - rules: # this should only run on dev branches - - if: $CI_PIPELINE_SOURCE =~ /^schedule.*$/ - when: never - - if: $CI_COMMIT_TAG - when: never - - !reference [.except_main_or_release_branch] - - when: always - needs: [] - script: - - | - if [ ! -z "$DATADOG_AGENT_BUILDIMAGES_SUFFIX" ] || - [ ! -z "$DATADOG_AGENT_WINBUILDIMAGES_SUFFIX" ] || - [ ! -z "$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX" ] || - [ ! -z "$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX" ] || - [ ! -z "$DATADOG_AGENT_KERNEL_MATRIX_TESTING_BUILDIMAGES_SUFFIX" ] || - [ ! -z "$DATADOG_AGENT_NIKOS_BUILDIMAGES_SUFFIX" ] || - [ ! -z "$DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX" ] || - [ ! -z "$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX" ]; then - echo "Pull request uses non-empty BUILDIMAGES_SUFFIX variable" - echo "This workflow fails so that the pull request cannot be merged" - exit 1 - fi - - inv -e release.check-omnibus-branches || exit 1 - - exit 0 diff --git a/.gitlab/docker_common/publish_job_templates.yml b/.gitlab/docker_common/publish_job_templates.yml deleted file mode 100644 index b1a7ec7c190f87..00000000000000 --- a/.gitlab/docker_common/publish_job_templates.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -.docker_variables: &docker_variables - SRC_AGENT: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/agent - SRC_DSD: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/dogstatsd - SRC_DCA: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/cluster-agent - SRC_CWS_INSTRUMENTATION: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/cws-instrumentation - -.docker_publish_job_definition: - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - <<: *docker_variables - IMG_VARIABLES: "" - IMG_SIGNING: "" - script: # We can't use the 'trigger' keyword on manual jobs, otherwise they can't be run if the pipeline fails and is retried - - source /root/.bashrc - - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text) - - ECR_RELEASE_SUFFIX="${CI_COMMIT_TAG+-release}" - - IMG_VARIABLES="$(sed -E "s#(${SRC_AGENT}|${SRC_DSD}|${SRC_DCA}|${SRC_CWS_INSTRUMENTATION})#\1${ECR_RELEASE_SUFFIX}#g" <<<"$IMG_VARIABLES")" - - IMG_SOURCES="$(sed -E "s#(${SRC_AGENT}|${SRC_DSD}|${SRC_DCA}|${SRC_CWS_INSTRUMENTATION})#\1${ECR_RELEASE_SUFFIX}#g" <<<"$IMG_SOURCES")" - - inv pipeline.trigger-child-pipeline --project-name "DataDog/public-images" --git-ref "main" --variables "IMG_VARIABLES,IMG_REGISTRIES,IMG_SOURCES,IMG_DESTINATIONS,IMG_SIGNING" diff --git a/.gitlab/e2e.yml b/.gitlab/e2e.yml deleted file mode 100644 index a31da4bbf49a26..00000000000000 --- a/.gitlab/e2e.yml +++ /dev/null @@ -1,267 +0,0 @@ ---- -# e2e stage -# Contains jobs which runs e2e tests on our Docker images. - -.k8s_e2e_template: - stage: e2e - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - dependencies: [] - variables: - LANG: C.UTF-8 - before_script: - - export DOCKER_REGISTRY_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_LOGIN_SSM_KEY --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_REGISTRY_PWD=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_PWD_SSM_KEY --with-decryption --query "Parameter.Value" --out text) - - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key --with-decryption --query "Parameter.Value" --out text) - -k8s-e2e-dev: - extends: .k8s_e2e_template - rules: !reference [.on_dev_branch_manual] - # Note: pupernetes-dev requires the below jobs to work. However, - # we can't explicitly define the dependencies because a job cannot depend on other manual jobs. - # Adding the following lines would result in pipelines remaining in a "Running" state forever, - # as the pupernetes-dev job waits for manual jobs that may never be triggered. - # needs: - # - dev_branch_docker_hub-a6 - # - dev_branch_docker_hub-a7 - # We still want to make the job available as soon as possible. In this case, since it's manual - # and requires other manual jobs, it's reasonable make it available from the beginning and let - # engineers trigger the correct sequence of jobs when needed. - needs: [] - script: - - inv -e e2e-tests --agent-image=datadog/agent-dev:${CI_COMMIT_REF_SLUG}-py2 --dca-image=datadog/cluster-agent-dev:${CI_COMMIT_REF_SLUG} --argo-workflow=default - - inv -e e2e-tests --agent-image=datadog/agent-dev:${CI_COMMIT_REF_SLUG}-py3 --dca-image=datadog/cluster-agent-dev:${CI_COMMIT_REF_SLUG} --argo-workflow=default - -k8s-e2e-main: - extends: .k8s_e2e_template - allow_failure: true # temporary while investigating - rules: !reference [.on_main] - # needs: - # - dev_master-a6 - # - dev_master-a7 - script: - - inv -e e2e-tests --agent-image=datadog/agent-dev:master-py2 --dca-image=datadog/cluster-agent-dev:master --argo-workflow=default - - inv -e e2e-tests --agent-image=datadog/agent-dev:master-py3 --dca-image=datadog/cluster-agent-dev:master --argo-workflow=default - -k8s-e2e-tags-6: - extends: .k8s_e2e_template - rules: !reference [.on_deploy_stable_or_beta_repo_branch_a6_manual] - script: - - AGENT_VERSION=$(inv agent.version --major-version 6) - - DCA_VERSION=$(inv -e cluster-agent.version) - - inv -e e2e-tests --agent-image=datadog/agent:${AGENT_VERSION} --dca-image=datadog/cluster-agent:${DCA_VERSION} --argo-workflow=default - -k8s-e2e-tags-7: - extends: .k8s_e2e_template - rules: !reference [.on_deploy_stable_or_beta_repo_branch_a7_manual] - script: - - AGENT_VERSION=$(inv agent.version --major-version 7) - - DCA_VERSION=$(inv -e cluster-agent.version) - - inv -e e2e-tests --agent-image=datadog/agent:${AGENT_VERSION} --dca-image=datadog/cluster-agent:${DCA_VERSION} --argo-workflow=default - -.k8s-e2e-cws-cspm-init: - - set +x - - export DATADOG_AGENT_SITE=datadoghq.com - - export DATADOG_AGENT_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.e2e_tests_api_key --with-decryption --query "Parameter.Value" --out text) - - export DATADOG_AGENT_APP_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.e2e_tests_app_key --with-decryption --query "Parameter.Value" --out text) - - export DATADOG_AGENT_RC_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.e2e_tests_rc_key --with-decryption --query "Parameter.Value" --out text) - -k8s-e2e-cws-dev: - extends: .k8s_e2e_template - rules: !reference [.on_dev_branch_manual] - needs: [] - script: - - !reference [.k8s-e2e-cws-cspm-init] - - inv -e e2e-tests --agent-image=datadog/agent-dev:${CI_COMMIT_REF_SLUG}-py3 --dca-image=datadog/cluster-agent-dev:${CI_COMMIT_REF_SLUG} --argo-workflow=cws - -k8s-e2e-cws-main: - extends: .k8s_e2e_template - rules: !reference [.on_main] - # needs: - # - dev_master-a6 - # - dev_master-a7 - retry: 1 - script: - - !reference [.k8s-e2e-cws-cspm-init] - - inv -e e2e-tests --agent-image=datadog/agent-dev:master-py3 --dca-image=datadog/cluster-agent-dev:master --argo-workflow=cws - -k8s-e2e-cspm-dev: - extends: .k8s_e2e_template - rules: !reference [.on_dev_branch_manual] - needs: [] - script: - - !reference [.k8s-e2e-cws-cspm-init] - - inv -e e2e-tests --agent-image=datadog/agent-dev:${CI_COMMIT_REF_SLUG}-py3 --dca-image=datadog/cluster-agent-dev:${CI_COMMIT_REF_SLUG} --argo-workflow=cspm - -k8s-e2e-cspm-main: - extends: .k8s_e2e_template - rules: !reference [.on_main] - # needs: - # - dev_master-a6 - # - dev_master-a7 - retry: 1 - script: - - !reference [.k8s-e2e-cws-cspm-init] - - inv -e e2e-tests --agent-image=datadog/agent-dev:master-py3 --dca-image=datadog/cluster-agent-dev:master --argo-workflow=cspm - -k8s-e2e-otlp-dev: - extends: .k8s_e2e_template - rules: !reference [.on_dev_branch_manual] - needs: [] # See note on k8s-e2e-dev above - script: - - inv -e e2e-tests --agent-image=datadog/agent-dev:${CI_COMMIT_REF_SLUG}-py3 --dca-image=datadog/cluster-agent-dev:${CI_COMMIT_REF_SLUG} --argo-workflow=otlp - -k8s-e2e-otlp-main: - extends: .k8s_e2e_template - rules: !reference [.on_main] - # needs: - # - dev_master-a6 - # - dev_master-a7 - script: - - inv -e e2e-tests --agent-image=datadog/agent-dev:master-py3 --dca-image=datadog/cluster-agent-dev:master --argo-workflow=otlp - -.new_e2e_template: - stage: e2e - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES - tags: ["arch:amd64"] - before_script: - # Setup AWS Credentials - - mkdir -p ~/.aws - - aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.agent-qa-profile --with-decryption --query "Parameter.Value" --out text >> ~/.aws/config - - export AWS_PROFILE=agent-qa-ci - # Now all `aws` commands target the agent-qa profile - - aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.ssh_public_key --with-decryption --query "Parameter.Value" --out text > $E2E_PUBLIC_KEY_PATH - - aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.ssh_key --with-decryption --query "Parameter.Value" --out text > $E2E_PRIVATE_KEY_PATH - # Use S3 backend - - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" - variables: - KUBERNETES_MEMORY_REQUEST: 12Gi - KUBERNETES_MEMORY_LIMIT: 16Gi - E2E_PUBLIC_KEY_PATH: /tmp/agent-qa-ssh-key.pub - E2E_PRIVATE_KEY_PATH: /tmp/agent-qa-ssh-key - E2E_KEY_PAIR_NAME: datadog-agent-ci - E2E_PIPELINE_ID: $CI_PIPELINE_ID - script: - - inv -e new-e2e-tests.run --targets $TARGETS -c ddagent:fullImagePath=669783387624.dkr.ecr.us-east-1.amazonaws.com/agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} -c ddagent:clusterAgentFullImagePath=669783387624.dkr.ecr.us-east-1.amazonaws.com/cluster-agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} -c dddogstatsd:fullImagePath=669783387624.dkr.ecr.us-east-1.amazonaws.com/dogstatsd:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} -c ddagent:imagePullRegistry=669783387624.dkr.ecr.us-east-1.amazonaws.com -c ddagent:imagePullUsername=AWS -c ddagent:imagePullPassword=$(aws ecr get-login-password) --junit-tar "junit-${CI_JOB_NAME}.tgz" ${EXTRA_PARAMS} - artifacts: - expire_in: 2 weeks - when: always - paths: - # This file will be consumed by the `e2e_test_junit_upload` job in next stage to push the report to datadog. - # If you create a new job from this template, do not forget to update the `dependencies` of the `e2e_test_junit_upload` job. - - junit-*.tgz - reports: - junit: test/new-e2e/junit-*.xml - -new-e2e-containers-dev: - extends: .new_e2e_template - # TODO once images are deployed to ECR for dev branches, update - # on_main_and_no_skip_e2e adding on_dev_branch_manual rules - # and move rules to template - rules: !reference [.on_dev_branch_manual] - needs: [] - variables: - TARGETS: ./tests/containers - TEAM: container-integrations - parallel: - matrix: - - EXTRA_PARAMS: --run TestKindSuite - - EXTRA_PARAMS: --run TestEKSSuite - - EXTRA_PARAMS: --run TestECSSuite - - EXTRA_PARAMS: --skip "Test(Kind|EKS|ECS)Suite" - -new-e2e-containers-main: - extends: .new_e2e_template - rules: !reference [.on_main_and_no_skip_e2e] - needs: - - qa_main_agent - - qa_main_dca - - qa_main_dogstatsd - variables: - TARGETS: ./tests/containers - TEAM: container-integrations - parallel: - matrix: - - EXTRA_PARAMS: --run TestKindSuite - - EXTRA_PARAMS: --run TestEKSSuite - - EXTRA_PARAMS: --run TestECSSuite - - EXTRA_PARAMS: --skip "Test(Kind|EKS|ECS)Suite" - # Temporary, until we manage to stabilize those tests. - allow_failure: true - -new-e2e-agent-shared-components-dev: - extends: .new_e2e_template - rules: !reference [.on_dev_branch_manual] - needs: [] - variables: - TARGETS: ./tests/agent-shared-components - TEAM: agent-shared-components - -new-e2e-agent-shared-components-main: - extends: .new_e2e_template - rules: !reference [.on_main_and_no_skip_e2e] - variables: - TARGETS: ./tests/agent-shared-components - TEAM: agent-shared-components - # Temporary, until we manage to stabilize those tests. - allow_failure: true - -new-e2e-agent-subcommands-dev: - extends: .new_e2e_template - rules: !reference [.on_dev_branch_manual] - needs: [] - variables: - TARGETS: ./tests/agent-subcommands - TEAM: agent-shared-components - -new-e2e-agent-subcommands-main: - extends: .new_e2e_template - rules: !reference [.on_main_and_no_skip_e2e] - variables: - TARGETS: ./tests/agent-subcommands - TEAM: agent-shared-components - # Temporary, until we manage to stabilize those tests. - allow_failure: true - -new-e2e-language-detection-dev: - extends: .new_e2e_template - rules: !reference [.on_dev_branch_manual] - needs: [] - variables: - TARGETS: ./tests/language-detection - TEAM: processes - -new-e2e-language-detection-main: - extends: .new_e2e_template - rules: !reference [.on_main_and_no_skip_e2e] - variables: - TARGETS: ./tests/language-detection - TEAM: processes - -new-e2e-npm-dev: - extends: .new_e2e_template - rules: !reference [.on_dev_branch_manual] - needs: [] - variables: - TARGETS: ./tests/npm - -new-e2e-npm-main: - extends: .new_e2e_template - rules: !reference [.on_main_and_no_skip_e2e] - variables: - TARGETS: ./tests/npm - # Temporary, until we manage to stabilize those tests. - allow_failure: true - -new-e2e-cws-dev: - extends: .new_e2e_template - rules: !reference [.on_dev_branch_manual] - needs: [] - variables: - TARGETS: ./tests/cws - # Temporary, until we manage to stabilize those tests. - allow_failure: true - -# ^ If you create a new job here that extends `.new_e2e_template`, -# /!\ do not forget to add it in the `dependencies` statement of the -# /___\ `e2e_test_junit_upload` job in the `.gitlab/e2e_test_junit_upload.yml` file diff --git a/.gitlab/e2e_test_junit_upload.yml b/.gitlab/e2e_test_junit_upload.yml deleted file mode 100644 index ee901fb168f15c..00000000000000 --- a/.gitlab/e2e_test_junit_upload.yml +++ /dev/null @@ -1,27 +0,0 @@ -e2e_test_junit_upload: - rules: !reference [.on_main] - when: always - stage: e2e_test_junit_upload - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - allow_failure: true - variables: - DD_ENV: ci - dependencies: - # We need to exhaustively list all the `new-e2e-…` jobs that produce junit reports here - # to avoid downloading all the artifacts of all the jobs of all the previous stages. - - new-e2e-containers-main - - new-e2e-agent-shared-components-main - - new-e2e-agent-subcommands-main - - new-e2e-language-detection-main - - new-e2e-agent-platform-install-script-debian-a7-x64 - - new-e2e-agent-platform-install-script-debian-a7-arm64 - - new-e2e-agent-platform-install-script-debian-iot-agent-a7-x86_64 - - new-e2e-agent-platform-install-script-debian-dogstatsd-a7-x86_64 - - new-e2e-agent-platform-install-script-debian-heroku-agent-a7-x86_64 - - new-e2e-npm-main - script: - - set +x - - export DATADOG_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text) - - set -x - - for f in junit-new-e2e-*.tgz; do inv -e junit-upload --tgz-path "$f"; done diff --git a/.gitlab/fakeintake.yml b/.gitlab/fakeintake.yml deleted file mode 100644 index 7380f52a4ffd14..00000000000000 --- a/.gitlab/fakeintake.yml +++ /dev/null @@ -1,60 +0,0 @@ -.on_fakeintake_changes: &on_fakeintake_changes - changes: - - "test/fakeintake/**/*" - - .gitlab/fakeintake.yml - -docker_build_fakeintake: - stage: container_build - rules: - - !reference [.except_mergequeue] - - <<: *on_fakeintake_changes - needs: [] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:20.10-py3 - tags: ["arch:amd64"] - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - TARGET: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/fakeintake:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - DOCKERFILE: test/fakeintake/Dockerfile - PLATFORMS: linux/amd64,linux/arm64 - BUILD_CONTEXT: test/fakeintake - script: - # DockerHub login for build to limit rate limit when pulling base images - - DOCKER_REGISTRY_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_LOGIN_SSM_KEY --with-decryption --query "Parameter.Value" --out text) - - aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_PWD_SSM_KEY --with-decryption --query "Parameter.Value" --out text | docker login --username "$DOCKER_REGISTRY_LOGIN" --password-stdin "$DOCKER_REGISTRY_URL" - - docker buildx build --push --pull --platform ${PLATFORMS} --file ${DOCKERFILE} --tag ${TARGET} $BUILD_CONTEXT - retry: 2 - -publish_fakeintake: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - - !reference [.except_mergequeue] - - <<: *on_fakeintake_changes - if: $CI_COMMIT_BRANCH == "main" - - <<: *on_fakeintake_changes - when: manual - allow_failure: true - needs: - - job: docker_build_fakeintake - optional: false - variables: - IMG_SOURCES: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/fakeintake:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - IMG_DESTINATIONS: fakeintake:v${CI_COMMIT_SHORT_SHA} - IMG_REGISTRIES: public - IMG_SIGNING: "false" - -publish_fakeintake_latest: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - - !reference [.except_mergequeue] - - <<: *on_fakeintake_changes - if: $CI_COMMIT_BRANCH == "main" - needs: - - job: docker_build_fakeintake - optional: false - variables: - IMG_SOURCES: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/fakeintake:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - IMG_DESTINATIONS: fakeintake:latest - IMG_REGISTRIES: public - IMG_SIGNING: "false" diff --git a/.gitlab/functional_test.yml b/.gitlab/functional_test.yml deleted file mode 100644 index 0bfe548bc118dd..00000000000000 --- a/.gitlab/functional_test.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# functional_test stage -# Contains jobs which run kitchen tests on the security-agent and on system-probe - -include: - - /.gitlab/functional_test/security_agent.yml - - /.gitlab/functional_test/serverless.yml - - /.gitlab/functional_test/regression_detector.yml - - /.gitlab/functional_test/workload_checks.yml - - /.gitlab/functional_test/system_probe_windows.yml - - /.gitlab/kernel_version_testing/system_probe.yml - - /.gitlab/functional_test_sysprobe/system_probe.yml diff --git a/.gitlab/functional_test/common.yml b/.gitlab/functional_test/common.yml deleted file mode 100644 index 2d216eb39e2504..00000000000000 --- a/.gitlab/functional_test/common.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -# FIXME: our current Gitlab version doesn't support importing a file more than once -# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987 -# include: -# - /.gitlab/kitchen_common/testing.yml - -.kitchen_test_system_probe: - extends: - - .kitchen_common - - .kitchen_datadog_agent_flavor - rules: - !reference [.on_system_probe_changes_or_manual] - timeout: 3h - variables: - AGENT_MAJOR_VERSION: 7 - DD_PIPELINE_ID: $CI_PIPELINE_ID-fnct - CHEF_VERSION: 14.15.6 diff --git a/.gitlab/functional_test/regression_detector.yml b/.gitlab/functional_test/regression_detector.yml deleted file mode 100644 index 01c0e85357ed75..00000000000000 --- a/.gitlab/functional_test/regression_detector.yml +++ /dev/null @@ -1,127 +0,0 @@ -single-machine-performance-regression_detector: - stage: functional_test - rules: - - !reference [.except_mergequeue] - - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:docker"] - needs: - - job: single_machine_performance-amd64-a7 - artifacts: false - artifacts: - expire_in: 1 weeks - paths: - - submission_metadata # for provenance, debugging - - ${CI_COMMIT_SHA}-baseline_sha # for provenance, debugging - - outputs/report.md # for debugging, also on S3 - - outputs/report.html # for debugging, also on S3 - when: always - variables: - SMP_VERSION: 0.10.0 - LADING_VERSION: 0.19.1 - CPUS: 7 - MEMORY: "30g" - # At present we require two artifacts to exist for the 'baseline' and the - # 'comparison'. We are guaranteed by the structure of the pipeline that - # 'comparison' exists, not so much with 'baseline' as it has to come from main - # merge pipeline run. This is solved in datadog-agent by updating a file in S3 - # with the SHA of the merge base from main. It's solved in Vector by - # building Vector twice for each Regression Detector run. - # - # We allow failure for now. _Unfortunately_ this also means that if the - # Regression Detector finds a performance issue with a PR it will not be - # flagged. - allow_failure: true - script: - # Ensure output files exist for artifact downloads step - - mkdir outputs # Also needed for smp job sync step - - touch outputs/report.md # Will be emitted by smp job sync - - touch outputs/report.html # Will be emitted by smp job sync - # Compute merge base of current commit and `main` - - git fetch origin - - SMP_BASE_BRANCH=$(inv release.get-release-json-value base_branch) - - echo "Looking for merge base for branch ${SMP_BASE_BRANCH}" - - SMP_MERGE_BASE=$(git merge-base ${CI_COMMIT_SHA} origin/${SMP_BASE_BRANCH}) - - echo "Merge base is ${SMP_MERGE_BASE}" - # Setup AWS credentials for single-machine-performance AWS account - - AWS_NAMED_PROFILE="single-machine-performance" - - SMP_ACCOUNT_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.single-machine-performance-account-id --with-decryption --query "Parameter.Value" --out text) - - SMP_ECR_URL=${SMP_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com - - SMP_AGENT_TEAM_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.single-machine-performance-agent-team-id --with-decryption --query "Parameter.Value" --out text) - - SMP_API=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.single-machine-performance-api --with-decryption --query "Parameter.Value" --out text) - - aws configure set aws_access_key_id $(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.single-machine-performance-bot-access-key-id --with-decryption --query "Parameter.Value" --out text) --profile ${AWS_NAMED_PROFILE} - - aws configure set aws_secret_access_key $(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.single-machine-performance-bot-access-key --with-decryption --query "Parameter.Value" --out text) --profile ${AWS_NAMED_PROFILE} - - aws configure set region us-west-2 --profile ${AWS_NAMED_PROFILE} - # Download smp binary and prepare it for use - - aws --profile single-machine-performance s3 cp s3://smp-cli-releases/v${SMP_VERSION}/x86_64-unknown-linux-gnu/smp smp - - chmod +x smp - - BASELINE_SHA="${SMP_MERGE_BASE}" - - echo "Computing baseline..." - - echo "Checking if image exists for commit ${BASELINE_SHA}..." - - while [[ ! $(aws ecr describe-images --profile single-machine-performance --registry-id "${SMP_ACCOUNT_ID}" --repository-name "${SMP_AGENT_TEAM_ID}-agent" --image-ids imageTag="${BASELINE_SHA}-7-amd64") ]]; do echo "No image exists for ${BASELINE_SHA} - checking predecessor of ${BASELINE_SHA} next"; BASELINE_SHA=$(git rev-parse ${BASELINE_SHA}^); echo "Checking if image exists for commit ${BASELINE_SHA}..."; done - - echo "Image exists for commit ${BASELINE_SHA}" - - echo "Baseline SHA is ${BASELINE_SHA}" - - echo -n "${BASELINE_SHA}" > "${CI_COMMIT_SHA}-baseline_sha" - # Copy the baseline SHA to SMP for debugging purposes later - - aws s3 cp --profile single-machine-performance --only-show-errors "${CI_COMMIT_SHA}-baseline_sha" "s3://${SMP_AGENT_TEAM_ID}-smp-artifacts/information/" - - BASELINE_IMAGE=${SMP_ECR_URL}/${SMP_AGENT_TEAM_ID}-agent:${BASELINE_SHA}-7-amd64 - - echo "${BASELINE_SHA} | ${BASELINE_IMAGE}" - - COMPARISON_IMAGE=${SMP_ECR_URL}/${SMP_AGENT_TEAM_ID}-agent:${CI_COMMIT_SHA}-7-amd64 - - echo "${CI_COMMIT_SHA} | ${COMPARISON_IMAGE}" - - RUST_LOG="info,aws_config::profile::credentials=error" - - RUST_LOG_DEBUG="debug,aws_config::profile::credentials=error" - - RUST_LOG="${RUST_LOG}" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API} --aws-named-profile ${AWS_NAMED_PROFILE} - job submit --use-curta - --lading-version ${LADING_VERSION} - --baseline-image ${BASELINE_IMAGE} - --comparison-image ${COMPARISON_IMAGE} - --baseline-sha ${BASELINE_SHA} - --comparison-sha ${CI_COMMIT_SHA} - --target-name datadog-agent - --target-command "/bin/entrypoint.sh" - --target-environment-variables "DD_HOSTNAME=smp-regression,DD_DD_URL=http://127.0.0.1:9092" - --target-config-dir test/regression/ - --target-cpu-allotment ${CPUS} - --target-memory-allotment ${MEMORY} - --submission-metadata submission_metadata - # Wait for job to complete. - - RUST_LOG="${RUST_LOG}" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API} --aws-named-profile ${AWS_NAMED_PROFILE} - job status --use-curta - --wait - --wait-delay-seconds 60 - --submission-metadata submission_metadata - # Now that the job is completed pull the analysis report, output it to stdout. - - RUST_LOG="${RUST_LOG}" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API} --aws-named-profile ${AWS_NAMED_PROFILE} - job sync --use-curta - --submission-metadata submission_metadata - --output-path outputs - # Replace empty lines in the output with lines containing various unicode - # space characters. This avoids - # https://gitlab.com/gitlab-org/gitlab/-/issues/217231. - - cat outputs/report.md | sed "s/^\$/$(echo -ne '\uFEFF\u00A0\u200B')/g" - # Add janky means of installing PR commenter borrowed from - # https://github.com/DataDog/dogweb/blob/45d7fcf035d0d515ebd901919099d4c8bfa82829/docker/docker-builder/Dockerfile#L69-L77 - - apt-get update - - apt-get install -y curl - - curl -OL https://s3.amazonaws.com/dd-package-public/dd-package.deb - - dpkg -i dd-package.deb - - rm dd-package.deb - - apt-get update - - dd-package --bucket binaries.ddbuild.io --package devtools/dd-package-dev --distribution "20.04" - # Kludge from https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4645#note_287636439 to avoid - # doubled output - - echo "" - - "####################### NOTE TO MAINTAINERS #####################################" - - "# Ignore bdist_wheel build error raised when installing 'devtools/pr-commenter' #" - - "# This error is known behavior, and you should see a successful install #" - - "# beneath the error message in the GitLab CI logs #" - - "#################################################################################" - - echo "" - # Pinned version of pr-commenter taken from https://github.com/DataDog/dogweb/blob/b2a891c2a2186b8fecd27ca9b13ebabe5f289a4a/tasks/gitlab/k8s-diff-helper.sh#L22 - - dd-package --bucket binaries.ddbuild.io --package devtools/pr-commenter --distribution "20.04" --version "14692290-a6440fd1" - # Post HTML report to GitHub - - cat outputs/report.html | /usr/local/bin/pr-commenter --for-pr="$CI_COMMIT_REF_NAME" - # Finally, exit 1 if the job signals a regression else 0. - - RUST_LOG="${RUST_LOG}" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API} --aws-named-profile ${AWS_NAMED_PROFILE} - job result --use-curta - --submission-metadata submission_metadata diff --git a/.gitlab/functional_test/security_agent.yml b/.gitlab/functional_test/security_agent.yml deleted file mode 100644 index 3e7207be206b33..00000000000000 --- a/.gitlab/functional_test/security_agent.yml +++ /dev/null @@ -1,180 +0,0 @@ ---- -# FIXME: our current Gitlab version doesn't support importing a file more than once -# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987 -# include: -# - /.gitlab/kitchen_common/testing.yml - -# Expect warning: github.com/DataDog/datadog-agent/pkg/config.LoadCustom:1501 Unknown environment variable: DD_SYSTEM_PROBE_BPF_DIR -.kitchen_test_security_agent: - extends: - - .kitchen_common - - .kitchen_datadog_agent_flavor - rules: - !reference [.manual] - stage: functional_test - retry: 0 - variables: - AGENT_MAJOR_VERSION: 7 - DD_PIPELINE_ID: $CI_PIPELINE_ID-fnct - CHEF_VERSION: 14.15.6 - script: - - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/site-cookbooks/dd-security-agent-check/files/job_env.txt - - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/site-cookbooks/dd-security-agent-check/files/job_env.txt - - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/site-cookbooks/dd-security-agent-check/files/job_env.txt - - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/site-cookbooks/dd-security-agent-check/files/job_env.txt - - tasks/run-test-kitchen.sh security-agent-test $AGENT_MAJOR_VERSION - - popd - - inv system-probe.print-failed-tests --output-dir $DD_AGENT_TESTING_DIR/testjson - artifacts: - expire_in: 2 weeks - when: always - paths: - - $DD_AGENT_TESTING_DIR/kitchen-junit-*.tar.gz - -kitchen_test_security_agent_x64: - extends: - - .kitchen_test_security_agent - - .kitchen_azure_x64 - - .kitchen_azure_location_north_central_us - needs: ["tests_ebpf_x64", "prepare_ebpf_functional_tests_x64"] - variables: - KITCHEN_ARCH: x86_64 - before_script: - - pushd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - parallel: - matrix: - - KITCHEN_PLATFORM: "centos" - KITCHEN_OSVERS: "centos-77" - KITCHEN_CWS_PLATFORM: [host, docker] - - KITCHEN_PLATFORM: "centos" - KITCHEN_OSVERS: "rhel-85,rhel-91" - KITCHEN_CI_MOUNT_PATH: "/mnt/ci" - KITCHEN_CI_ROOT_PATH: "/tmp/ci" - KITCHEN_CWS_PLATFORM: [host] - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-18-04-0,ubuntu-18-04,ubuntu-18-04-3" - KITCHEN_CWS_PLATFORM: [host, docker] - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-20-04,ubuntu-20-04-2,ubuntu-22-04" - KITCHEN_CWS_PLATFORM: [host, docker] - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-22-04" - KITCHEN_CWS_PLATFORM: [ad] - - KITCHEN_PLATFORM: "suse" - KITCHEN_OSVERS: "sles-12,sles-15" - KITCHEN_CWS_PLATFORM: [host] - - KITCHEN_PLATFORM: "suse" - KITCHEN_OSVERS: "opensuse-15-3" - KITCHEN_CWS_PLATFORM: [host] - - KITCHEN_PLATFORM: "debian" - KITCHEN_OSVERS: "debian-10,debian-11" - KITCHEN_CWS_PLATFORM: [host, docker] - - KITCHEN_PLATFORM: "oracle" - KITCHEN_OSVERS: "oracle-7-9" - KITCHEN_CWS_PLATFORM: [host, docker] - -kitchen_test_security_agent_arm64: - extends: - - .kitchen_test_security_agent - - .kitchen_ec2_location_us_east_1 - - .kitchen_ec2 - rules: - !reference [.on_security_agent_changes_or_manual] - needs: [ "tests_ebpf_arm64", "prepare_ebpf_functional_tests_arm64" ] - variables: - KITCHEN_ARCH: arm64 - KITCHEN_EC2_INSTANCE_TYPE: "t4g.xlarge" - KITCHEN_CI_MOUNT_PATH: "/mnt/ci" - KITCHEN_CI_ROOT_PATH: "/tmp/ci" - before_script: - - pushd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - parallel: - matrix: - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-20-04-2,ubuntu-22-04" - KITCHEN_CWS_PLATFORM: [host, docker, ad] - -kitchen_test_security_agent_amazonlinux_x64: - extends: - - .kitchen_test_security_agent - - .kitchen_ec2_location_us_east_1 - - .kitchen_ec2 - rules: - !reference [.on_security_agent_changes_or_manual] - needs: [ "tests_ebpf_x64", "prepare_ebpf_functional_tests_x64" ] - variables: - KITCHEN_ARCH: x86_64 - KITCHEN_EC2_INSTANCE_TYPE: "t3.medium" - KITCHEN_CI_MOUNT_PATH: "/mnt/ci" - KITCHEN_CI_ROOT_PATH: "/tmp/ci" - before_script: - - pushd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - parallel: - matrix: - - KITCHEN_PLATFORM: "amazonlinux" - KITCHEN_OSVERS: "amazonlinux2-4-14,amazonlinux2-5-10" - KITCHEN_CWS_PLATFORM: [host, docker] - - KITCHEN_PLATFORM: "amazonlinux" - KITCHEN_OSVERS: "amazonlinux2022-5-15,amazonlinux2023" - KITCHEN_CWS_PLATFORM: [host, docker] - -kitchen_test_security_agent_x64_ec2: - extends: - - .kitchen_test_security_agent - - .kitchen_ec2_location_us_east_1 - - .kitchen_ec2 - needs: [ "tests_ebpf_x64", "prepare_ebpf_functional_tests_x64" ] - variables: - KITCHEN_ARCH: x86_64 - KITCHEN_EC2_INSTANCE_TYPE: "t3.medium" - KITCHEN_CI_MOUNT_PATH: "/mnt/ci" - KITCHEN_CI_ROOT_PATH: "/tmp/ci" - before_script: - - pushd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - parallel: - matrix: - - KITCHEN_PLATFORM: "centos" - KITCHEN_OSVERS: "rocky-92" - KITCHEN_CWS_PLATFORM: [host] - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-23-10" - KITCHEN_CWS_PLATFORM: [host, docker] - KITCHEN_EC2_DEVICE_NAME: "/dev/sda1" - -kitchen_test_security_agent_amazonlinux_x64_fentry: - extends: - - kitchen_test_security_agent_amazonlinux_x64 - allow_failure: true - parallel: - matrix: - - KITCHEN_PLATFORM: "amazonlinux" - KITCHEN_OSVERS: "amazonlinux2023" - KITCHEN_CWS_PLATFORM: [docker-fentry] # only docker for now because the host test is too slow because of BTF copying - -kitchen_stress_security_agent: - extends: - - .kitchen_common - - .kitchen_datadog_agent_flavor - - .kitchen_azure_x64 - - .kitchen_azure_location_north_central_us - rules: - !reference [.manual] - stage: functional_test - needs: ["tests_ebpf_x64", "prepare_ebpf_functional_tests_x64"] - variables: - AGENT_MAJOR_VERSION: 7 - DD_PIPELINE_ID: $CI_PIPELINE_ID-fnct - before_script: - - pushd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - script: - - tasks/run-test-kitchen.sh security-agent-stress $AGENT_MAJOR_VERSION - parallel: - matrix: - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-20-04" diff --git a/.gitlab/functional_test/serverless.yml b/.gitlab/functional_test/serverless.yml deleted file mode 100644 index 039a7eb0298844..00000000000000 --- a/.gitlab/functional_test/serverless.yml +++ /dev/null @@ -1,17 +0,0 @@ -serverless_cold_start_performance-deb_x64: - stage: functional_test - rules: - - !reference [.except_mergequeue] - - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:docker"] - needs: ["go_deps", "build_serverless-deb_x64"] - before_script: - - !reference [.retrieve_linux_go_deps] - - mkdir -p /tmp/serverless-ci - - cp cmd/serverless/datadog-agent-x64 /tmp/serverless-ci/datadog-agent - script: - - cp test/integration/serverless_perf/* /tmp/serverless-ci - - cd /tmp/serverless-ci # Docker does not like syslinks, that's why it's easier to build the image in /tmp - - docker build -t datadogci/lambda-extension . - - ./compute.sh \ No newline at end of file diff --git a/.gitlab/functional_test/system_probe_windows.yml b/.gitlab/functional_test/system_probe_windows.yml deleted file mode 100644 index 73e3c5a2092211..00000000000000 --- a/.gitlab/functional_test/system_probe_windows.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -# FIXME: our current Gitlab version doesn't support importing a file more than once -# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987 -# include: -# - /.gitlab/kitchen_common/testing.yml -# - /.gitlab/functional_test/common.yml - -kitchen_test_system_probe_windows_x64: - extends: - - .kitchen_agent_a7 - - .kitchen_os_windows - - .kitchen_test_system_probe - - .kitchen_azure_x64 - - .kitchen_azure_location_north_central_us - stage: functional_test - needs: [ "tests_windows_sysprobe_x64" ] - variables: - KITCHEN_ARCH: x86_64 - KITCHEN_OSVERS: "win2016" - CHEF_VERSION: 14.12.9 # newer versions error out during kitchen setup of azure VM - before_script: - - export WINDOWS_DDNPM_DRIVER=$(inv release.get-release-json-value "$RELEASE_VERSION_7::WINDOWS_DDNPM_DRIVER") - - export WINDOWS_DDNPM_VERSION=$(inv release.get-release-json-value "$RELEASE_VERSION_7::WINDOWS_DDNPM_VERSION") - - export WINDOWS_DDNPM_SHASUM=$(inv release.get-release-json-value "$RELEASE_VERSION_7::WINDOWS_DDNPM_SHASUM") - - pushd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - script: - - tasks/run-test-kitchen.sh windows-sysprobe-test $AGENT_MAJOR_VERSION diff --git a/.gitlab/functional_test/workload_checks.yml b/.gitlab/functional_test/workload_checks.yml deleted file mode 100644 index ade86d16ce6673..00000000000000 --- a/.gitlab/functional_test/workload_checks.yml +++ /dev/null @@ -1,60 +0,0 @@ -single-machine-performance-workload-checks: - stage: functional_test - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:docker"] - rules: - !reference [.on_scheduled_main] - needs: - - job: single_machine_performance-nightly-amd64-a7 - artifacts: false - artifacts: - expire_in: 1 weeks - paths: - - submission_metadata # for provenance, debugging - when: always - variables: - SMP_VERSION: 0.10.0 - LADING_VERSION: 0.19.1 - WARMUP_SECONDS: 45 - TOTAL_SAMPLES: 600 - REPLICAS: 5 - allow_failure: true - script: - - git fetch origin - # Setup AWS credentials for single-machine-performance AWS account - - AWS_NAMED_PROFILE="single-machine-performance" - - SMP_ACCOUNT_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.single-machine-performance-account-id --with-decryption --query "Parameter.Value" --out text) - - SMP_ECR_URL=${SMP_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com - - SMP_AGENT_TEAM_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.single-machine-performance-agent-team-id --with-decryption --query "Parameter.Value" --out text) - - SMP_API=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.single-machine-performance-api --with-decryption --query "Parameter.Value" --out text) - - aws configure set aws_access_key_id $(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.single-machine-performance-bot-access-key-id --with-decryption --query "Parameter.Value" --out text) --profile ${AWS_NAMED_PROFILE} - - aws configure set aws_secret_access_key $(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.single-machine-performance-bot-access-key --with-decryption --query "Parameter.Value" --out text) --profile ${AWS_NAMED_PROFILE} - - aws configure set region us-west-2 --profile ${AWS_NAMED_PROFILE} - # Download smp binary and prepare it for use - - aws --profile single-machine-performance s3 cp s3://smp-cli-releases/v${SMP_VERSION}/x86_64-unknown-linux-gnu/smp smp - - chmod +x smp - - CURRENT_DATE=$(date --utc '+%Y_%m_%d') - - TARGET_IMAGE=${SMP_ECR_URL}/${SMP_AGENT_TEAM_ID}-agent:nightly-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA}-7-amd64 - # Copy the TARGET_IMAGE to SMP for debugging purposes later - - RUST_LOG="info,aws_config::profile::credentials=error" - - RUST_LOG_DEBUG="debug,aws_config::profile::credentials=error" - - RUST_BACKTRACE=1 RUST_LOG="${RUST_LOG_DEBUG}" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API} --aws-named-profile ${AWS_NAMED_PROFILE} - job submit-workload - --lading-version ${LADING_VERSION} - --total-samples ${TOTAL_SAMPLES} - --warmup-seconds ${WARMUP_SECONDS} - --replicas ${REPLICAS} - --target-image ${TARGET_IMAGE} - --target-sha ${CI_COMMIT_SHA} - --target-config-dir test/workload-checks - --target-name datadog-agent - --target-command "/bin/entrypoint.sh" - --target-environment-variables "DD_HOSTNAME=smp-workload-checks,DD_DD_URL=http://127.0.0.1:9092,DD_API_KEY=00000001" - --tags smp_status=nightly,client_team="agent",tag_date="${CURRENT_DATE}" - --submission-metadata submission-metadata - # Wait for job to complete. - - RUST_BACKTRACE=1 RUST_LOG="${RUST_LOG_DEBUG}" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API} --aws-named-profile ${AWS_NAMED_PROFILE} - job status - --wait - --wait-delay-seconds 60 - --submission-metadata submission-metadata diff --git a/.gitlab/functional_test_cleanup.yml b/.gitlab/functional_test_cleanup.yml deleted file mode 100644 index 7d2d4bbd866c5d..00000000000000 --- a/.gitlab/functional_test_cleanup.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -# functional_test_cleanup stage -# Contains a job which cleans up kitchen resources created for functional tests. - -# FIXME: our current Gitlab version doesn't support importing a file more than once -# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987 -# include: -# - /.gitlab/kitchen_common/cleanup.yml - -cleanup_kitchen_functional_test: - extends: .kitchen_cleanup_azure_common - rules: - !reference [.manual] - stage: functional_test_cleanup - needs: ["tests_ebpf_x64", "prepare_ebpf_functional_tests_x64"] - variables: - DD_PIPELINE_ID: $CI_PIPELINE_ID-fnct diff --git a/.gitlab/functional_test_junit_upload.yml b/.gitlab/functional_test_junit_upload.yml deleted file mode 100644 index aa5296741062d0..00000000000000 --- a/.gitlab/functional_test_junit_upload.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -# functional_test_junit_upload stage -# Contains jobs which upload the junit output of the kitchen functional tests - -# The system probe and security agent junit upload jobs were split -# to bypass the logical complexity of having many optionally needed jobs. -include: - - /.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml - - /.gitlab/junit_upload/functional_test_junit_upload_security-agent.yml diff --git a/.gitlab/functional_test_sysprobe/system_probe.yml b/.gitlab/functional_test_sysprobe/system_probe.yml deleted file mode 100644 index a3665e23db4e5f..00000000000000 --- a/.gitlab/functional_test_sysprobe/system_probe.yml +++ /dev/null @@ -1,171 +0,0 @@ ---- -# FIXME: our current Gitlab version doesn't support importing a file more than once -# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987 -# include: -# - /.gitlab/kitchen_common/testing.yml -# - /.gitlab/functional_test/common.yml - -.kitchen_test_system_probe_linux: - extends: - - .kitchen_test_system_probe - stage: functional_test_sysprobe - before_script: - - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/job_env.txt - - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/job_env.txt - - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/job_env.txt - - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/job_env.txt - - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/minimized-btfs.tar.xz - - inv system-probe.test-docker-image-list > $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/docker-images.txt - - | - get_docker_secrets() { - if [ -o xtrace ]; then set +x; trap 'set -x' RETURN; fi - export DOCKER_REGISTRY_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_LOGIN_SSM_KEY --with-decryption --query "Parameter.Value" --out text) - export DOCKER_REGISTRY_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_PWD_SSM_KEY --with-decryption --query "Parameter.Value" --out text) - } - get_docker_secrets - - pushd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - script: - - tasks/run-test-kitchen.sh system-probe-test $AGENT_MAJOR_VERSION - - popd - - inv system-probe.print-failed-tests --output-dir $DD_AGENT_TESTING_DIR/testjson - artifacts: - expire_in: 2 weeks - when: always - paths: - - $DD_AGENT_TESTING_DIR/kitchen-junit-*.tar.gz - - $DD_AGENT_TESTING_DIR/testjson - - $CI_PROJECT_DIR/kitchen_logs - -# This dummy job is added here because we want the functional_tests stage to start at the same time as kernel_matrix_testing stage. -# The ebpf-platform team is trying to measure the time from the start of the pipeline to the completion of the kernel_matrix_testing and functional_tests stages, to measure improvement. -# The CI visibility product currently does not provide a way to measure this in the datadog app. -# We are trying to hack around this by having jobs which are triggered at the start of a pipeline. -# For functional tests this is the 'kitchen_test_dummy_job_tmp' and for kernel_matrix_testing it is the 'pull_test_dockers*' jobs. -# This way the stage.duration metric will allow us to measure what we want. -# This will most likely be temporary, until we decide which approach to move forward with for testing in the future. -kitchen_test_dummy_job_tmp: - stage: functional_test_sysprobe - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/btf-gen$DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BTF_GEN_BUILDIMAGES - needs: [] - tags: ["arch:amd64"] - rules: - !reference [ .on_system_probe_changes_or_manual ] - script: - - 'true' - -kitchen_test_system_probe_linux_x64_ec2: - extends: - - .kitchen_test_system_probe_linux - - .kitchen_ec2_location_us_east_1 - - .kitchen_ec2 - needs: [ "tests_ebpf_x64", "prepare_ebpf_functional_tests_x64", "generate_minimized_btfs_x64" ] - retry: 0 - variables: - ARCH: amd64 - KITCHEN_ARCH: x86_64 - KITCHEN_EC2_INSTANCE_TYPE: "t2.xlarge" - KITCHEN_CI_MOUNT_PATH: "/mnt/ci" - KITCHEN_CI_ROOT_PATH: "/tmp/ci" - KITCHEN_DOCKERS: $DD_AGENT_TESTING_DIR/kitchen-dockers-$ARCH - parallel: - matrix: - - KITCHEN_PLATFORM: "amazonlinux" - KITCHEN_OSVERS: "amazonlinux2-4-14" - - KITCHEN_PLATFORM: "amazonlinux" - KITCHEN_OSVERS: "amazonlinux2-5-10" - - KITCHEN_PLATFORM: "amazonlinux" - KITCHEN_OSVERS: "amazonlinux2022-5-15" - - KITCHEN_PLATFORM: "amazonlinux" - KITCHEN_OSVERS: "amazonlinux2023" - - KITCHEN_PLATFORM: "centos" - KITCHEN_OSVERS: "centos-79" - KITCHEN_EC2_DEVICE_NAME: "/dev/sda1" - - KITCHEN_PLATFORM: "centos" - KITCHEN_OSVERS: "rhel-86" - KITCHEN_EC2_DEVICE_NAME: "/dev/sda1" - - KITCHEN_PLATFORM: "debian" - KITCHEN_OSVERS: "debian-10" - - KITCHEN_PLATFORM: "debian" - KITCHEN_OSVERS: "debian-11" - - KITCHEN_PLATFORM: "fedora" - KITCHEN_OSVERS: "fedora-36" - KITCHEN_EC2_DEVICE_NAME: "/dev/sda1" - - KITCHEN_PLATFORM: "fedora" - KITCHEN_OSVERS: "fedora-37" - KITCHEN_EC2_DEVICE_NAME: "/dev/sda1" - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-16-04-4.4" - KITCHEN_EC2_DEVICE_NAME: "/dev/sda1" - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-16-04" - KITCHEN_EC2_DEVICE_NAME: "/dev/sda1" - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-18-04" - KITCHEN_EC2_DEVICE_NAME: "/dev/sda1" - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-20-04" - KITCHEN_EC2_DEVICE_NAME: "/dev/sda1" - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-22-04" - KITCHEN_EC2_DEVICE_NAME: "/dev/sda1" - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-23-04" - KITCHEN_EC2_DEVICE_NAME: "/dev/sda1" - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-23-10" - KITCHEN_EC2_DEVICE_NAME: "/dev/sda1" - -kitchen_test_system_probe_linux_arm64: - extends: - - .kitchen_test_system_probe_linux - - .kitchen_ec2_location_us_east_1 - - .kitchen_ec2 - needs: [ "tests_ebpf_arm64", "prepare_ebpf_functional_tests_arm64", "generate_minimized_btfs_arm64" ] - retry: 0 - variables: - ARCH: arm64 - KITCHEN_ARCH: arm64 - KITCHEN_EC2_INSTANCE_TYPE: "t4g.xlarge" - KITCHEN_CI_MOUNT_PATH: "/mnt/ci" - KITCHEN_CI_ROOT_PATH: "/tmp/ci" - KITCHEN_DOCKERS: $DD_AGENT_TESTING_DIR/kitchen-dockers-$ARCH - parallel: - matrix: - - KITCHEN_PLATFORM: "amazonlinux" - KITCHEN_OSVERS: "amazonlinux2-4-14" - KITCHEN_EC2_DEVICE_NAME: "/dev/xvda" - - KITCHEN_PLATFORM: "amazonlinux" - KITCHEN_OSVERS: "amazonlinux2-5-10" - KITCHEN_EC2_DEVICE_NAME: "/dev/xvda" - - KITCHEN_PLATFORM: "amazonlinux" - KITCHEN_OSVERS: "amazonlinux2022-5-15" - KITCHEN_EC2_DEVICE_NAME: "/dev/xvda" - - KITCHEN_PLATFORM: "amazonlinux" - KITCHEN_OSVERS: "amazonlinux2023" - KITCHEN_EC2_DEVICE_NAME: "/dev/xvda" - - KITCHEN_PLATFORM: "centos" - KITCHEN_OSVERS: "centos-79" - - KITCHEN_PLATFORM: "centos" - KITCHEN_OSVERS: "rhel-86" - - KITCHEN_PLATFORM: "debian" - KITCHEN_OSVERS: "debian-10" - KITCHEN_EC2_DEVICE_NAME: "/dev/xvda" - - KITCHEN_PLATFORM: "debian" - KITCHEN_OSVERS: "debian-11" - KITCHEN_EC2_DEVICE_NAME: "/dev/xvda" - - KITCHEN_PLATFORM: "fedora" - KITCHEN_OSVERS: "fedora-36" - - KITCHEN_PLATFORM: "fedora" - KITCHEN_OSVERS: "fedora-37" - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-18-04" - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-20-04" - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-22-04" - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-23-04" - - KITCHEN_PLATFORM: "ubuntu" - KITCHEN_OSVERS: "ubuntu-23-10" diff --git a/.gitlab/install_script_testing.yml b/.gitlab/install_script_testing.yml deleted file mode 100644 index 2ee2fab6ace3c2..00000000000000 --- a/.gitlab/install_script_testing.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -test_install_script: - stage: install_script_testing - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - script: - - source /root/.bashrc - - set +x - - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text) - - export TESTING_APT_URL=$DEB_TESTING_S3_BUCKET - - export TESTING_YUM_URL=$RPM_TESTING_S3_BUCKET - - export TEST_PIPELINE_ID=$CI_PIPELINE_ID - - inv pipeline.trigger-child-pipeline --project-name "DataDog/agent-linux-install-script" --git-ref "main" --variables "TESTING_APT_URL,TESTING_YUM_URL,TEST_PIPELINE_ID" - needs: ["deploy_deb_testing-a6_x64", "deploy_rpm_testing-a6_x64", "deploy_deb_testing-a7_x64", "deploy_rpm_testing-a7_x64"] - rules: - !reference [.on_deploy] diff --git a/.gitlab/integration_test.yml b/.gitlab/integration_test.yml deleted file mode 100644 index 3d195fbf448d78..00000000000000 --- a/.gitlab/integration_test.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# integration_test stage -# Contains jobs to run integration tests in go binaries - -include: - - /.gitlab/integration_test/dogstatsd.yml - - /.gitlab/integration_test/windows.yml diff --git a/.gitlab/integration_test/dogstatsd.yml b/.gitlab/integration_test/dogstatsd.yml deleted file mode 100644 index 22b8215eac476a..00000000000000 --- a/.gitlab/integration_test/dogstatsd.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -# integration_test stage -# Contains jobs to run integration tests in dogstatsd go binaries - -dogstatsd_x64_size_test: - stage: integration_test - rules: - !reference [.on_a7] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["build_dogstatsd_static-binary_x64"] - before_script: - - source /root/.bashrc - - mkdir -p $STATIC_BINARIES_DIR - - $S3_CP_CMD $S3_ARTIFACTS_URI/static/dogstatsd.amd64 $STATIC_BINARIES_DIR/dogstatsd - script: - - inv -e dogstatsd.size-test --skip-build - -# run benchmarks on deb -# benchmarks-deb_x64: -# stage: integration_test -# image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES -# allow_failure: true # FIXME: this was set to true to temporarily unblock the pipeline -# tags: ["runner:main"] -# script: -# - inv -e bench.aggregator -# # FIXME: in our docker image, non ascii characters printed by the benchmark -# # make invoke traceback. For now, the workaround is to call the benchmarks -# # manually -# - inv -e bench.build-dogstatsd - -# - set +x # make sure we don't output the creds to the build log -# - DD_AGENT_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.dd_agent_api_key --with-decryption --query "Parameter.Value" --out text) - -# # dogstatsd validation - not really benchmarking: gitlab isn't the right place to do this. -# - ./bin/benchmarks/dogstatsd -pps=20000 -dur 30 -ser 5 -branch $DD_REPO_BRANCH_NAME -api-key $DD_AGENT_API_KEY -# artifacts: -# expire_in: 2 weeks -# paths: -# - benchmarks diff --git a/.gitlab/integration_test/windows.yml b/.gitlab/integration_test/windows.yml deleted file mode 100644 index 448247cdf36a27..00000000000000 --- a/.gitlab/integration_test/windows.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -.integration_tests_windows_base: - stage: integration_test - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: ["go_deps", "go_tools_deps"] - tags: ["runner:windows-docker", "windowsversion:1809"] - before_script: - - $vcpkgBlobSaSUrl = (aws ssm get-parameter --region us-east-1 --name ci.datadog-agent-buildimages.vcpkg_blob_sas_url --with-decryption --query "Parameter.Value" --out text) - script: - - $ErrorActionPreference = "Stop" - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - # we pass in CI_JOB_URL and CI_JOB_NAME so that they can be added to additional tags - # inside JUNIT_TAR and then later used by datadog-ci - - !reference [.setup_python_mirror_win] - - > - docker run --rm - -m 16384M - -v "$(Get-Location):c:\mnt" - -e CI_JOB_URL="${CI_JOB_URL}" - -e CI_JOB_NAME="${CI_JOB_NAME}" - -e CI_PIPELINE_ID="${CI_PIPELINE_ID}" - -e CI_PROJECT_NAME="${CI_PROJECT_NAME}" - -e AWS_NETWORKING=true - -e PY_RUNTIMES="$PYTHON_RUNTIMES" - -e GOMODCACHE="c:\modcache" - -e JUNIT_TAR="c:\mnt\junit-${CI_JOB_NAME}.tgz" - -e VCPKG_BINARY_SOURCES="clear;x-azblob,${vcpkgBlobSaSUrl}" - -e PIP_INDEX_URL=${PIP_INDEX_URL} - 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\integrationtests.bat - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - -integration_tests_windows-x64: - extends: .integration_tests_windows_base - variables: - PYTHON_RUNTIMES: 3 - ARCH: "x64" diff --git a/.gitlab/internal_image_deploy.yml b/.gitlab/internal_image_deploy.yml deleted file mode 100644 index a5682a1b2bf0b9..00000000000000 --- a/.gitlab/internal_image_deploy.yml +++ /dev/null @@ -1,89 +0,0 @@ ---- -# internal_image_deploy stage -# Contains jobs to trigger a pipeline in the images repo to create internal Agent images. - -docker_trigger_internal: - stage: internal_image_deploy - rules: - !reference [.on_deploy_a7_internal_or_manual] - needs: - - job: docker_build_agent7_jmx - artifacts: false - - job: docker_build_agent7_jmx_arm64 - artifacts: false - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - DYNAMIC_BUILD_RENDER_RULES: agent-build-only # fake rule to not trigger the ones in the images repo - IMAGE_VERSION: tmpl-v7 - IMAGE_NAME: datadog-agent - RELEASE_TAG: ${CI_COMMIT_REF_SLUG}-jmx - BUILD_TAG: ${CI_COMMIT_REF_SLUG}-jmx - TMPL_SRC_IMAGE: v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx - TMPL_SRC_REPO: ci/datadog-agent/agent - RELEASE_STAGING: "true" - script: - - source /root/.bashrc - - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text) - - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi - - if [ "$BUCKET_BRANCH" = "nightly" ]; then RELEASE_TAG="${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi - - if [ "$BUCKET_BRANCH" = "dev" ]; then RELEASE_TAG="dev-${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi - - inv pipeline.trigger-child-pipeline --project-name "DataDog/images" --git-ref "master" --variables "IMAGE_VERSION,IMAGE_NAME,RELEASE_TAG,BUILD_TAG,TMPL_SRC_IMAGE,TMPL_SRC_REPO,RELEASE_STAGING,RELEASE_PROD,DYNAMIC_BUILD_RENDER_RULES" - - -docker_trigger_cluster_agent_internal: - stage: internal_image_deploy - rules: - !reference [.on_deploy_a7] - needs: - - job: docker_build_cluster_agent_amd64 - artifacts: false - - job: docker_build_cluster_agent_arm64 - artifacts: false - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - DYNAMIC_BUILD_RENDER_RULES: agent-build-only # fake rule to not trigger the ones in the images repo - IMAGE_VERSION: tmpl-v3 - IMAGE_NAME: datadog-cluster-agent - RELEASE_TAG: ${CI_COMMIT_REF_SLUG} - BUILD_TAG: ${CI_COMMIT_REF_SLUG} - TMPL_SRC_IMAGE: v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - TMPL_SRC_REPO: ci/datadog-agent/cluster-agent - RELEASE_STAGING: "true" - RELEASE_PROD: "true" - script: - - source /root/.bashrc - - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text) - - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi - - if [ "$BUCKET_BRANCH" = "nightly" ]; then RELEASE_TAG="${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi - - if [ "$BUCKET_BRANCH" = "dev" ]; then RELEASE_TAG="dev-${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi - - inv pipeline.trigger-child-pipeline --project-name "DataDog/images" --git-ref "master" --variables "IMAGE_VERSION,IMAGE_NAME,RELEASE_TAG,BUILD_TAG,TMPL_SRC_IMAGE,TMPL_SRC_REPO,RELEASE_STAGING,RELEASE_PROD,DYNAMIC_BUILD_RENDER_RULES" - -docker_trigger_cws_instrumentation_internal: - stage: internal_image_deploy - rules: - !reference [.on_deploy_a7] - needs: - - job: docker_build_cws_instrumentation_amd64 - artifacts: false - - job: docker_build_cws_instrumentation_arm64 - artifacts: false - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - DYNAMIC_BUILD_RENDER_RULES: agent-build-only # fake rule to not trigger the ones in the images repo - IMAGE_VERSION: tmpl-v1 - IMAGE_NAME: datadog-cws-instrumentation - RELEASE_TAG: ${CI_COMMIT_REF_SLUG} - BUILD_TAG: ${CI_COMMIT_REF_SLUG} - TMPL_SRC_IMAGE: v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - TMPL_SRC_REPO: ci/datadog-agent/cws-instrumentation - RELEASE_STAGING: "true" - RELEASE_PROD: "true" - script: - - source /root/.bashrc - - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text) - - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi - - if [ "$BUCKET_BRANCH" = "nightly" ]; then RELEASE_TAG="${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi - - inv pipeline.trigger-child-pipeline --project-name "DataDog/images" --git-ref "master" --variables "IMAGE_VERSION,IMAGE_NAME,RELEASE_TAG,BUILD_TAG,TMPL_SRC_IMAGE,TMPL_SRC_REPO,RELEASE_STAGING,RELEASE_PROD,DYNAMIC_BUILD_RENDER_RULES" diff --git a/.gitlab/internal_kubernetes_deploy.yml b/.gitlab/internal_kubernetes_deploy.yml deleted file mode 100644 index 4991b7bd70306d..00000000000000 --- a/.gitlab/internal_kubernetes_deploy.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- -# internal_kubernetes_deploy stage -# Contains jobs to trigger a pipeline in our k8s-datadog-agent-ops repo - -internal_kubernetes_deploy_experimental: - stage: internal_kubernetes_deploy - rules: - - if: $FORCE_K8S_DEPLOYMENT == "true" - when: always - - if: $CI_COMMIT_BRANCH != "main" - when: never - - !reference [.on_deploy_a7] - needs: - - job: docker_trigger_internal - artifacts: false - - job: docker_trigger_cluster_agent_internal - artifacts: false - - job: k8s-e2e-main # Currently only require container Argo workflow - artifacts: false - optional: true - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - OPTION_AUTOMATIC_ROLLOUT: "true" - OPTION_PRE_SCRIPT: "patch-cluster-images-operator.sh env=all-staging ${CI_COMMIT_REF_SLUG}-jmx-${CI_COMMIT_SHORT_SHA} ${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" - SKIP_PLAN_CHECK: "true" - EXPLICIT_WORKFLOWS: "//workflows:deploy_nightly.agents_nightly" - script: - - source /root/.bashrc - - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text) - - inv pipeline.trigger-child-pipeline --project-name "DataDog/k8s-datadog-agent-ops" --git-ref "main" --variables "OPTION_AUTOMATIC_ROLLOUT,EXPLICIT_WORKFLOWS,OPTION_PRE_SCRIPT,SKIP_PLAN_CHECK" --no-follow - -notify-slack: - stage: internal_kubernetes_deploy - rules: - - if: $FORCE_K8S_DEPLOYMENT == "true" - when: always - - if: $CI_COMMIT_BRANCH != "main" - when: never - - !reference [ .on_deploy_a7 ] - image: registry.ddbuild.io/slack-notifier:sdm - tags: ["arch:amd64"] - needs: ["internal_kubernetes_deploy_experimental"] - script: - - python3 -m pip install -r tasks/libs/requirements-notifications.txt - - inv pipeline.changelog ${CI_COMMIT_SHORT_SHA} diff --git a/.gitlab/junit_upload/functional_test_junit_upload_security-agent.yml b/.gitlab/junit_upload/functional_test_junit_upload_security-agent.yml deleted file mode 100644 index 7556de60f50f4f..00000000000000 --- a/.gitlab/junit_upload/functional_test_junit_upload_security-agent.yml +++ /dev/null @@ -1,21 +0,0 @@ -functional_test_junit_upload_security_agent: - stage: functional_test_junit_upload - rules: - - !reference [.except_mergequeue] - - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - allow_failure: true - when: always - needs: - - job: kitchen_test_security_agent_arm64 - optional: true - - job: kitchen_test_security_agent_amazonlinux_x64 - optional: true - variables: - DD_ENV: ci - script: - - set +x - - export DATADOG_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text) - - set -x - - ss=0; for f in $DD_AGENT_TESTING_DIR/kitchen-junit-*.tar.gz; do [[ -e "$f" ]] || continue; inv -e junit-upload --tgz-path $f || ((ss++)); done; exit $ss diff --git a/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml b/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml deleted file mode 100644 index d362441aab6e5e..00000000000000 --- a/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml +++ /dev/null @@ -1,44 +0,0 @@ -functional_test_junit_upload_system_probe: - # Uncomment this to only run junit uploads on main/stable branches - # rules: - # !reference [.on_main_or_release_branch] - stage: functional_test_junit_upload - rules: - - !reference [.except_mergequeue] - - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - allow_failure: true - when: always - needs: - - job: kitchen_test_system_probe_linux_x64_ec2 - optional: true - - job: kitchen_test_system_probe_linux_arm64 - optional: true - variables: - DD_ENV: ci - script: - - set +x - - export DATADOG_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text) - - set -x - - ss=0; for f in $DD_AGENT_TESTING_DIR/kitchen-junit-*.tar.gz; do [[ -e "$f" ]] || continue; inv -e junit-upload --tgz-path $f || ((ss++)); done; exit $ss - -functional_test_junit_upload_system_probe_kmt: - stage: functional_test_junit_upload - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - allow_failure: true - when: always - needs: - - job: kernel_matrix_testing_run_tests_x64 - optional: true - - job: kernel_matrix_testing_run_tests_arm64 - optional: true - variables: - DD_ENV: ci - script: - - set +x - - export DATADOG_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text) - - export JIRA_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.jira_read_api_token --with-decryption --query "Parameter.Value" --out text) - - set -x - - ss=0; for f in $DD_AGENT_TESTING_DIR/junit-*.tar.gz; do [[ -e "$f" ]] || continue; inv -e junit-upload --tgz-path $f || ((ss++)); done; exit $ss diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml deleted file mode 100644 index 1fc2385e2f4c3f..00000000000000 --- a/.gitlab/kernel_version_testing/system_probe.yml +++ /dev/null @@ -1,452 +0,0 @@ -.pull_test_dockers: - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - needs: [] - tags: ["arch:amd64"] - rules: - !reference [ .on_system_probe_changes_or_manual ] - stage: kernel_matrix_testing - script: - # DockerHub login for build to limit rate limit when pulling base images - - DOCKER_REGISTRY_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_LOGIN_SSM_KEY --with-decryption --query "Parameter.Value" --out text) - - aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_PWD_SSM_KEY --with-decryption --query "Parameter.Value" --out text | crane auth login --username "$DOCKER_REGISTRY_LOGIN" --password-stdin "$DOCKER_REGISTRY_URL" - # Pull base images - - mkdir $KMT_DOCKERS - - inv -e system-probe.save-test-dockers --use-crane --output-dir $KMT_DOCKERS --arch $ARCH - artifacts: - expire_in: 1 day - paths: - - $KMT_DOCKERS - variables: - KMT_DOCKERS: $DD_AGENT_TESTING_DIR/kmt-dockers-$ARCH - -pull_test_dockers_x64: - extends: .pull_test_dockers - variables: - ARCH: amd64 - -pull_test_dockers_arm64: - extends: .pull_test_dockers - variables: - ARCH: arm64 - -.shared_filters_and_queries: - - FILTER_TEAM="Name=tag:team,Values=ebpf-platform" - - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi" - - FILTER_STATE="Name=instance-state-name,Values=running" - - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}" - - FILTER_ARCH="Name=tag:arch,Values=${ARCH}" - - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}" - - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId' - - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress' - -.wait_for_instances: - - !reference [.shared_filters_and_queries] - - | - COUNTER=0 - while [[ $(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED $FILTER_STATE $FILTER_PIPELINE --output text --query $QUERY_INSTANCE_IDS | wc -l ) != "2" && $COUNTER -le 40 ]]; do COUNTER=$[$COUNTER +1]; echo "[${COUNTER}] Waiting for instances"; sleep 30; done - # check that 2 instances are ready, or fail - if [ $(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED $FILTER_STATE $FILTER_PIPELINE --output text --query $QUERY_INSTANCE_IDS | wc -l) -ne "2" ]; then - echo "Both instances NOT found" - "false" - fi - echo "Both Instances found" - INSTANCE_IDS=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED $FILTER_STATE $FILTER_PIPELINE --output text --query $QUERY_INSTANCE_IDS | tr '\n' ' ') - aws ec2 wait instance-status-ok --instance-ids $INSTANCE_IDS - sleep 10 - -.write_ssh_key_file: - - set +x - - aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.ssh_key --with-decryption --query "Parameter.Value" --out text > $AWS_EC2_SSH_KEY_FILE - - set -x - # Without the newline ssh silently fails and moves on to try other auth methods - - echo "" >> $AWS_EC2_SSH_KEY_FILE - - chmod 600 $AWS_EC2_SSH_KEY_FILE - -# needs variables: ARCH, INSTANCE_TYPE -.get_instance_ip_by_type: - - INSTANCE_IP=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED $FILTER_STATE $FILTER_PIPELINE "Name=instance-type,Values=${INSTANCE_TYPE}" --output text --query $QUERY_PRIVATE_IPS) - - echo "$ARCH-instance-ip" $INSTANCE_IP - -# needs variables: INSTANCE_IP, AWS_EC2_SSH_KEY_FILE -.setup_ssh_config: - - mkdir -p ~/.ssh && chmod 700 ~/.ssh - - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\n" | tee -a ~/.ssh/config - - chmod 600 ~/.ssh/config - -.package_dependencies: - stage: kernel_matrix_testing - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - allow_failure: true - rules: - !reference [ .on_system_probe_changes_or_manual ] - before_script: - - !reference [.kernel_matrix_testing_new_profile] - - !reference [.write_ssh_key_file] - tags: ["arch:amd64"] - script: - # Build dependencies directory - - mkdir -p $DEPENDENCIES - - pushd $DEPENDENCIES - # copy micro-vm-init.sh - - cp $CI_PROJECT_DIR/test/new-e2e/system-probe/test/micro-vm-init.sh $DEPENDENCIES - # copy over docker images - - cp -R $KMT_DOCKERS $DEPENDENCIES/kmt-docker - # package all the dependencies - - ls -la $DEPENDENCIES - - pushd $DD_AGENT_TESTING_DIR/$ARCH - - tar czvf ../$ARCHIVE_NAME dependencies - - popd - # upload tests - - !reference [.wait_for_instances] - - !reference [.get_instance_ip_by_type] - - !reference [.setup_ssh_config] - - scp $DD_AGENT_TESTING_DIR/$ARCHIVE_NAME metal_instance:/opt/kernel-version-testing/ - variables: - DEPENDENCIES: $DD_AGENT_TESTING_DIR/$ARCH/dependencies - KMT_DOCKERS: $DD_AGENT_TESTING_DIR/kmt-dockers-$ARCH - AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key - -upload_dependencies_x64: - extends: - - .package_dependencies - needs: ["pull_test_dockers_x64"] - variables: - ARCHIVE_NAME: dependencies-x86_64.tar.gz - ARCH: amd64 - INSTANCE_TYPE: m5d.metal - -upload_dependencies_arm64: - extends: - - .package_dependencies - needs: ["pull_test_dockers_arm64"] - variables: - ARCHIVE_NAME: dependencies-arm64.tar.gz - ARCH: arm64 - INSTANCE_TYPE: m6gd.metal - -.upload_system_probe_tests: - stage: kernel_matrix_testing - allow_failure: true - rules: - !reference [ .on_system_probe_changes_or_manual ] - before_script: - - !reference [.retrieve_linux_go_deps] - - !reference [.kernel_matrix_testing_new_profile] - - !reference [.write_ssh_key_file] - script: - # Build dependencies directory - - mkdir -p $DEPENDENCIES - - pushd $DEPENDENCIES - # copy clang and llc bins - - mkdir -p $EMBEDDED_BIN - - cp $CLANG_BPF $EMBEDDED_BIN - - cp $LLC_BPF $EMBEDDED_BIN - - mkdir -p $EMBEDDED_INC - # copy gotestsum and test2json - - mkdir -p $GO_BIN - - cp $GOTESTSUM $GO_BIN - - cp $TEST2JSON $GO_BIN - - mkdir junit - - mkdir testjson - - mkdir pkgjson - # copy system probe tests - - mkdir -p $SYSTEM_PROBE_TESTS - - cp -R $KITCHEN_TESTS $SYSTEM_PROBE_TESTS - - pushd $CI_PROJECT_DIR/test/new-e2e/system-probe/test-runner && go build -o $DEPENDENCIES/test-runner && popd - - pushd $CI_PROJECT_DIR/test/new-e2e/system-probe/test-json-review && go build -o $DEPENDENCIES/test-json-review && popd - - popd - # package all the dependencies - - ls -la $DEPENDENCIES - - pushd $DD_AGENT_TESTING_DIR/$ARCH - - tar czvf ../$ARCHIVE_NAME tests - - popd - # upload tests - - !reference [.wait_for_instances] - - !reference [.get_instance_ip_by_type] - - !reference [.setup_ssh_config] - - scp $DD_AGENT_TESTING_DIR/$ARCHIVE_NAME metal_instance:/opt/kernel-version-testing/ - # build connector - - pushd $CI_PROJECT_DIR/test/new-e2e - - go build -o $CI_PROJECT_DIR/connector-${ARCH} $CI_PROJECT_DIR/test/new-e2e/system-probe/connector/main.go - - popd - # upload connector to metal instance - - scp $CI_PROJECT_DIR/connector-${ARCH} metal_instance:/home/ubuntu/connector - variables: - DEPENDENCIES: $DD_AGENT_TESTING_DIR/$ARCH/tests - EMBEDDED_BIN: opt/datadog-agent/embedded/bin - EMBEDDED_INC: opt/datadog-agent/embedded/include - CLANG_BPF: $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/default/clang-bpf - LLC_BPF: $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/default/llc-bpf - GO_BIN: go/bin - GOTESTSUM: $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/default/gotestsum - TEST2JSON: $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/default/test2json - SYSTEM_PROBE_TESTS: opt/system-probe-tests - KITCHEN_TESTS: $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/default/tests/pkg - AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key - artifacts: - when: always - paths: - - $CI_PROJECT_DIR/connector-${ARCH} - -upload_system_probe_tests_x64: - extends: - - .upload_system_probe_tests - needs: ["go_deps", "prepare_ebpf_functional_tests_x64", "tests_ebpf_x64"] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:amd64"] - variables: - ARCHIVE_NAME: tests-x86_64.tar.gz - ARCH: x86_64 - INSTANCE_TYPE: m5d.metal - -upload_system_probe_tests_arm64: - extends: - - .upload_system_probe_tests - needs: ["go_deps", "prepare_ebpf_functional_tests_arm64", "tests_ebpf_arm64"] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:arm64"] - variables: - ARCHIVE_NAME: tests-arm64.tar.gz - ARCH: arm64 - INSTANCE_TYPE: m6gd.metal - -.upload_minimized_btfs: - stage: kernel_matrix_testing - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:amd64"] - rules: - !reference [ .on_system_probe_changes_or_manual ] - allow_failure: true - script: - # Build dependencies directory - - mkdir -p $DEPENDENCIES - - pushd $DEPENDENCIES - # download and copy btf files - - mkdir -p $BTF_DIR - - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz $BTF_DIR/minimized-btfs.tar.xz - # package all the dependencies - - ls -la $DEPENDENCIES - - pushd $DD_AGENT_TESTING_DIR/$ARCH - - tar czvf ../$ARCHIVE_NAME btfs - - popd - # upload tests - # Switch to new profile after the btfs have been downloaded. Switching before - # causes permission issues. - - !reference [.kernel_matrix_testing_new_profile] - - !reference [.write_ssh_key_file] - - !reference [.wait_for_instances] - - !reference [.get_instance_ip_by_type] - - !reference [.setup_ssh_config] - - scp $DD_AGENT_TESTING_DIR/$ARCHIVE_NAME metal_instance:/opt/kernel-version-testing/ - variables: - DEPENDENCIES: $DD_AGENT_TESTING_DIR/$ARCH/btfs - BTF_DIR: opt/system-probe-tests/pkg/ebpf/bytecode/build/co-re/btf - AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key - -upload_minimized_btfs_x64: - extends: - - .upload_minimized_btfs - needs: ["generate_minimized_btfs_x64"] - variables: - ARCHIVE_NAME: btfs-x86_64.tar.gz - ARCH: x86_64 - INSTANCE_TYPE: m5d.metal - -upload_minimized_btfs_arm64: - extends: - - .upload_minimized_btfs - needs: ["generate_minimized_btfs_arm64"] - variables: - ARCHIVE_NAME: btfs-arm64.tar.gz - ARCH: arm64 - INSTANCE_TYPE: m6gd.metal - -.kernel_matrix_testing_new_profile: - - mkdir -p ~/.aws - - set +x - - aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.agent-qa-profile --with-decryption --query "Parameter.Value" --out text >> ~/.aws/config - - set -x - - export AWS_PROFILE=agent-qa-ci - -.kernel_matrix_testing_setup_env: - extends: - - .kitchen_ec2_location_us_east_1 - stage: kernel_matrix_testing - rules: - !reference [ .on_system_probe_changes_or_manual ] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES - needs: ["go_deps", "go_tools_deps"] - tags: ["arch:amd64"] - variables: - AWS_REGION: us-east-1 - STACK_DIR: $CI_PROJECT_DIR/stack.dir - # The ssh key is created by the pulumi scenario, to be used for creating - # instances in the build-stable account. We reuse this file to ssh into - # the instances in subsequent jobs. - AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key - AWS_EC2_SSH_KEY_NAME: datadog-agent-ci - INFRA_ENV: "aws/agent-qa" - PIPELINE_ID: $CI_PIPELINE_ID - TEAM: "ebpf-platform" - RESOURCE_TAGS: "instance-type:${INSTANCE_TYPE},arch:${ARCH}" - KUBERNETES_MEMORY_REQUEST: "12Gi" - KUBERNETES_MEMORY_LIMIT: "16Gi" - before_script: - - set +x - - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text) - - set -x - - !reference [.retrieve_linux_go_deps] - - !reference [.kernel_matrix_testing_new_profile] - - !reference [.write_ssh_key_file] - script: - - echo "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" > $STACK_DIR - - pulumi login $(cat $STACK_DIR | tr -d '\n') - - inv -e system-probe.start-microvms --provision --vmconfig=$VMCONFIG_PATH $INSTANCE_TYPE_ARG $AMI_ID_ARG --ssh-key-name=$AWS_EC2_SSH_KEY_NAME --ssh-key-path=$AWS_EC2_SSH_KEY_FILE --infra-env=$INFRA_ENV --stack-name=kernel-matrix-testing-$ARCH-$CI_PIPELINE_ID --run-agent - - cat $CI_PROJECT_DIR/stack.output - - pulumi logout - after_script: - - export AWS_PROFILE=agent-qa-ci - - !reference [.shared_filters_and_queries] - - mkdir -p $CI_PROJECT_DIR/libvirt/log/$ARCH $CI_PROJECT_DIR/libvirt/xml/$ARCH - - !reference [.get_instance_ip_by_type] - - ssh -o StrictHostKeyChecking=no -i $AWS_EC2_SSH_KEY_FILE "ubuntu@$INSTANCE_IP" "sudo virsh list --name | grep -v -E '^$' | xargs -I '{}' sh -c \"sudo virsh dumpxml '{}' > /tmp/ddvm-xml-'{}'.txt\"" - - scp -o StrictHostKeyChecking=no -i $AWS_EC2_SSH_KEY_FILE "ubuntu@$INSTANCE_IP:/tmp/ddvm-*.log" $CI_PROJECT_DIR/libvirt/log/$ARCH - - scp -o StrictHostKeyChecking=no -i $AWS_EC2_SSH_KEY_FILE "ubuntu@$INSTANCE_IP:/tmp/ddvm-xml-*" $CI_PROJECT_DIR/libvirt/xml/$ARCH - artifacts: - when: always - paths: - - $CI_PROJECT_DIR/stack.output - - $CI_PROJECT_DIR/libvirt - -kernel_matrix_testing_setup_env_arm64: - extends: - - .kernel_matrix_testing_setup_env - variables: - INSTANCE_TYPE: "m6gd.metal" - INSTANCE_TYPE_ARG: "--instance-type-arm=$INSTANCE_TYPE" - ARCH: arm64 - AMI_ID_ARG: "--arm-ami-id=$KERNEL_MATRIX_TESTING_ARM_AMI_ID" - LibvirtSSHKey: $CI_PROJECT_DIR/libvirt_rsa-arm - VMCONFIG_PATH: $CI_PROJECT_DIR/test/new-e2e/system-probe/config/vmconfig-arm64.json - -kernel_matrix_testing_setup_env_x64: - extends: - - .kernel_matrix_testing_setup_env - variables: - INSTANCE_TYPE: "m5d.metal" - INSTANCE_TYPE_ARG: "--instance-type-x86=$INSTANCE_TYPE" - ARCH: x86_64 - AMI_ID_ARG: "--x86-ami-id=$KERNEL_MATRIX_TESTING_X86_AMI_ID" - LibvirtSSHKey: $CI_PROJECT_DIR/libvirt_rsa-x86 - VMCONFIG_PATH: $CI_PROJECT_DIR/test/new-e2e/system-probe/config/vmconfig-x64.json - -.kernel_matrix_testing_run_tests: - stage: kernel_matrix_testing - allow_failure: true - rules: - !reference [ .on_system_probe_changes_or_manual ] - variables: - AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key - RETRY: 2 - before_script: - - set +x - - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text) - - set -x - - !reference [.kernel_matrix_testing_new_profile] - - !reference [.write_ssh_key_file] - - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt - - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt - - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt - - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt - script: - - INSTANCE_IP=$(grep "$ARCH-instance-ip" $CI_PROJECT_DIR/stack.output | cut -d ' ' -f 2 | tr -d '\n') - - !reference [.shared_filters_and_queries] - - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED $FILTER_PIPELINE "Name=private-ip-address,Values=$INSTANCE_IP" --output text --query $QUERY_INSTANCE_IDS | wc -l ) - - | - if [ $RUNNING_INSTANCES -eq "0" ]; then - echo "These jobs do not permit retries. The go tests are retried a user-specified number of times automatically. In order to re-run the tests, you must trigger the pipeline again" - 'false' - fi - - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | cut -d ' ' -f 2) - - MICRO_VM_NAME=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | cut -d ' ' -f 1) - - GO_VERSION=$(inv go-version) - - !reference [.setup_ssh_config] - # ssh into each micro-vm and run initialization script. This script will also run the tests. - - scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}.txt" - - ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}.txt ${MICRO_VM_IP}:/job_env.txt" - - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'bash /root/fetch_dependencies.sh ${ARCH} && /micro-vm-init.sh ${RETRY}'" - - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE -vm-cmd "${NESTED_VM_CMD}" - - ssh metal_instance "ssh ${MICRO_VM_IP} '/test-json-review'" - after_script: - - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | cut -d ' ' -f 2) - - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}.tar.gz" - - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}.tar.gz" $DD_AGENT_TESTING_DIR/ - - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}.tar.gz" - - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}.tar.gz" $DD_AGENT_TESTING_DIR/ - artifacts: - expire_in: 2 weeks - when: always - paths: - - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG.tar.gz - - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG.tar.gz - -kernel_matrix_testing_run_tests_x64: - extends: - - .kernel_matrix_testing_run_tests - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["kernel_matrix_testing_setup_env_x64", "upload_dependencies_x64", "upload_system_probe_tests_x64", "upload_minimized_btfs_x64"] - timeout: 3h - variables: - ARCH: "x86_64" - parallel: - matrix: - - TAG: ["ubuntu_16.04", "ubuntu_18.04", "ubuntu_20.04", "ubuntu_22.04", "ubuntu_23.10", "amzn_4.14", "amzn_5.4", "amzn_5.10", "fedora_37", "fedora_38", "debian_10", "debian_11", "debian_12", "centos_79"] - -kernel_matrix_testing_run_tests_arm64: - extends: - - .kernel_matrix_testing_run_tests - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:arm64"] - needs: ["kernel_matrix_testing_setup_env_arm64", "upload_dependencies_arm64", "upload_system_probe_tests_arm64", "upload_minimized_btfs_arm64"] - timeout: 3h - variables: - ARCH: "arm64" - parallel: - matrix: - - TAG: ["ubuntu_18.04", "ubuntu_20.04", "ubuntu_22.04", "ubuntu_23.10", "amzn_4.14", "amzn_5.4", "amzn_5.10", "fedora_37", "fedora_38", "debian_10", "debian_11", "debian_12", "centos_79"] - -.kernel_matrix_testing_cleanup: - stage: kernel_matrix_testing - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES - when: always - tags: ["arch:amd64"] - rules: - !reference [ .on_system_probe_changes_or_manual ] - before_script: - - !reference [.kernel_matrix_testing_new_profile] - script: - - !reference [.shared_filters_and_queries] - - aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED $FILTER_PIPELINE $FILTER_ARCH $FILTER_INSTANCE_TYPE --output json --query $QUERY_INSTANCE_IDS | tee -a instance.json - - INSTANCE_ID="$(jq -r '.[0][0]' < instance.json)" - - echo ${INSTANCE_ID} - - | - if [[ "${INSTANCE_ID}" != "" ]]; then - aws ec2 terminate-instances --instance-ids "${INSTANCE_ID}" - fi - -kernel_matrix_testing_cleanup_arm64: - extends: - - .kernel_matrix_testing_cleanup - needs: ["kernel_matrix_testing_setup_env_arm64", "kernel_matrix_testing_run_tests_arm64"] - variables: - ARCH: arm64 - INSTANCE_TYPE: "m6gd.metal" - -kernel_matrix_testing_cleanup_x64: - extends: - - .kernel_matrix_testing_cleanup - needs: ["kernel_matrix_testing_setup_env_x64", "kernel_matrix_testing_run_tests_x64"] - variables: - ARCH: x86_64 - INSTANCE_TYPE: "m5d.metal" diff --git a/.gitlab/kitchen_cleanup.yml b/.gitlab/kitchen_cleanup.yml deleted file mode 100644 index e83a4729492aa3..00000000000000 --- a/.gitlab/kitchen_cleanup.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -# kitchen_cleanup stage -# Contains jobs which clean up kitchen resources created for Agent kitchen tests. - -# FIXME: our current Gitlab version doesn't support importing a file more than once -# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987 -# include: -# - /.gitlab/kitchen_common/cleanup.yml - -kitchen_cleanup_azure-a6: - extends: .kitchen_cleanup_azure_common - rules: - !reference [.on_kitchen_tests_a6_always] - variables: - DD_PIPELINE_ID: $CI_PIPELINE_ID-a6 - -kitchen_cleanup_azure-a7: - extends: .kitchen_cleanup_azure_common - rules: - !reference [.on_default_kitchen_tests_a7_always] - variables: - DD_PIPELINE_ID: $CI_PIPELINE_ID-a7 diff --git a/.gitlab/kitchen_common/cleanup.yml b/.gitlab/kitchen_common/cleanup.yml deleted file mode 100644 index 9422f277b1bffd..00000000000000 --- a/.gitlab/kitchen_common/cleanup.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -.kitchen_cleanup_s3_common: - allow_failure: true - stage: kitchen_cleanup - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - script: - - aws s3 rm s3://$DEB_TESTING_S3_BUCKET/dists/pipeline-$DD_PIPELINE_ID --recursive - - aws s3 rm s3://$RPM_TESTING_S3_BUCKET/testing/pipeline-$DD_PIPELINE_ID --recursive - - aws s3 rm s3://$RPM_TESTING_S3_BUCKET/testing/suse/pipeline-$DD_PIPELINE_ID --recursive - - if [ $AGENT_MAJOR_VERSION == "7" ]; then export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A7; else export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A6; fi - - aws s3 rm s3://$WIN_S3_BUCKET/$WINDOWS_TESTING_S3_BUCKET --recursive - - cd $OMNIBUS_PACKAGE_DIR - # Remove all deb packages for the pipeline in the pool - - for deb in $(ls *amd64.deb); do aws s3 rm s3://$DEB_TESTING_S3_BUCKET/pool/d/da/$deb --recursive; done - # Remove the datadog-signing-keys package for the pipeline in the pool - - aws s3 rm s3://$DEB_TESTING_S3_BUCKET/pool/d/da/datadog-signing-keys_${DD_PIPELINE_ID}.deb - -.kitchen_cleanup_azure_common: - allow_failure: true - stage: kitchen_cleanup - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/dd-agent-testing$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - dependencies: [] - before_script: - script: - - cd $DD_AGENT_TESTING_DIR - - tasks/clean.sh diff --git a/.gitlab/kitchen_common/testing.yml b/.gitlab/kitchen_common/testing.yml deleted file mode 100644 index ef9d92aa5e8dec..00000000000000 --- a/.gitlab/kitchen_common/testing.yml +++ /dev/null @@ -1,250 +0,0 @@ ---- -.kitchen_common: - stage: kitchen_testing - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/dd-agent-testing$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - artifacts: - expire_in: 2 weeks - when: always - paths: - - $CI_PROJECT_DIR/kitchen_logs - retry: 1 - variables: - KUBERNETES_MEMORY_REQUEST: "6Gi" - KUBERNETES_MEMORY_LIMIT: "12Gi" - -.kitchen_common_with_junit: - extends: - - .kitchen_common - after_script: - - echo -n "--tags ci.job.name:${CI_JOB_NAME} --tags arch:${KITCHEN_ARCH} --tags os:${KITCHEN_PLATFORM} upload_option.os_version_from_name" > $CI_PROJECT_DIR/test/kitchen/tags.txt - - echo -n "${CI_JOB_URL}" > $CI_PROJECT_DIR/test/kitchen/job_url.txt - - kitchen_files=$(find $CI_PROJECT_DIR -name "kitchen-rspec-*.xml" -exec basename {} \;) - - find $CI_PROJECT_DIR -name "kitchen-rspec-*.xml" -exec $CI_PROJECT_DIR/test/kitchen/tasks/kitchen_rspec_xml_update.sh {} "${CI_JOB_NAME}" \; - - tar -czvf kitchen-rspec-common-${CI_JOB_NAME}.tar.gz -C $CI_PROJECT_DIR/test/kitchen/ ./job_url.txt ./tags.txt $kitchen_files - artifacts: - expire_in: 2 weeks - when: always - paths: - - $CI_PROJECT_DIR/kitchen_logs - - "**/kitchen-rspec-common-${CI_JOB_NAME}.tar.gz" - -# Kitchen: providers -# --------------- - -# Azure -# --------------- -.kitchen_azure: - variables: - KITCHEN_PROVIDER: azure - -.kitchen_azure_x64: - variables: - KITCHEN_ARCH: x86_64 - extends: - - .kitchen_azure - -# EC2 -# --------------- -.kitchen_ec2: - variables: - KITCHEN_PROVIDER: ec2 - KITCHEN_EC2_IAM_PROFILE_NAME: ci-datadog-agent-e2e-runner - -.kitchen_ec2_x64: - variables: - KITCHEN_ARCH: x86_64 - extends: - - .kitchen_ec2 - -.kitchen_ec2_spot_instances: - extends: .kitchen_ec2 - variables: - KITCHEN_EC2_SPOT_PRICE: on-demand - -.kitchen_ec2_arm64: - variables: - KITCHEN_ARCH: arm64 - KITCHEN_EC2_INSTANCE_TYPE: "t4g.xlarge" - CHEF_VERSION: 14.15.6 - extends: - - .kitchen_ec2 - -# Kitchen: agents -# --------------- - -.kitchen_agent_a6: - extends: .kitchen_common_with_junit - rules: - !reference [.on_kitchen_tests_a6] - variables: - AGENT_MAJOR_VERSION: 6 - DD_PIPELINE_ID: $CI_PIPELINE_ID-a6 - -.kitchen_agent_a7: - extends: .kitchen_common_with_junit - rules: - !reference [.on_kitchen_tests_a7] - variables: - AGENT_MAJOR_VERSION: 7 - DD_PIPELINE_ID: $CI_PIPELINE_ID-a7 - - -# Kitchen: tests -# -------------- - -.kitchen_test_chef: - script: - - tasks/run-test-kitchen.sh chef-test $AGENT_MAJOR_VERSION - -.kitchen_test_step_by_step: - script: - - tasks/run-test-kitchen.sh step-by-step-test $AGENT_MAJOR_VERSION - -.kitchen_test_install_script: - script: - - tasks/run-test-kitchen.sh install-script-test $AGENT_MAJOR_VERSION - -.kitchen_test_upgrade5: - script: - - tasks/run-test-kitchen.sh upgrade5-test $AGENT_MAJOR_VERSION - -.kitchen_test_upgrade6: - script: - - export LAST_STABLE_VERSION=$(cd ../.. && invoke release.get-release-json-value "last_stable::6") - - tasks/run-test-kitchen.sh upgrade6-test $AGENT_MAJOR_VERSION - -.kitchen_test_upgrade7: - script: - - export LAST_STABLE_VERSION=$(cd ../.. && invoke release.get-release-json-value "last_stable::7") - - tasks/run-test-kitchen.sh upgrade7-test $AGENT_MAJOR_VERSION - -.kitchen_test_process_agent: - script: - - tasks/run-test-kitchen.sh process-agent-test $AGENT_MAJOR_VERSION - -# Kitchen: Agent flavor -# ------------------------------- - -.kitchen_datadog_agent_flavor: - variables: - AGENT_FLAVOR: "datadog-agent" - -# For non-main Agent installs, KITCHEN_PLATFORM_SUFFIX needs to be set -# to a unique name. See test/kitchen/drivers/azure_driver.yml for details. -.kitchen_datadog_iot_agent_flavor: - variables: - AGENT_FLAVOR: "datadog-iot-agent" - KITCHEN_PLATFORM_SUFFIX: "iot" - -.kitchen_datadog_heroku_agent_flavor: - variables: - AGENT_FLAVOR: "datadog-heroku-agent" - KITCHEN_PLATFORM_SUFFIX: "heroku" - -.kitchen_datadog_dogstatsd_flavor: - variables: - AGENT_FLAVOR: "datadog-dogstatsd" - KITCHEN_PLATFORM_SUFFIX: "dsd" - -# Kitchen: Azure locations -# ------------------------------- - -.kitchen_azure_location_north_central_us: - variables: - AZURE_LOCATION: "North Central US" - -.kitchen_azure_location_west_central_us: - variables: - AZURE_LOCATION: "West Central US" - -.kitchen_azure_location_central_us: - variables: - AZURE_LOCATION: "Central US" - -.kitchen_azure_location_south_central_us: - variables: - AZURE_LOCATION: "South Central US" - - -# Kitchen: EC2 locations -# ------------------------------- - -.kitchen_ec2_location_us_east_1: - variables: - KITCHEN_EC2_REGION: us-east-1 - KITCHEN_EC2_SUBNET: subnet-05d7c6b1b5cfea811 - KITCHEN_EC2_SG_IDS: sg-019917348cb0eb7e7 - -# Kitchen: Test types (test suite * agent flavor + location in each cloud provider) -# ------------------------------- - -.kitchen_test_chef_agent: - extends: - - .kitchen_test_chef - - .kitchen_datadog_agent_flavor - - .kitchen_azure_location_north_central_us - - .kitchen_ec2_location_us_east_1 - -.kitchen_test_step_by_step_agent: - extends: - - .kitchen_test_step_by_step - - .kitchen_datadog_agent_flavor - - .kitchen_azure_location_central_us - - .kitchen_ec2_location_us_east_1 - -.kitchen_test_install_script_agent: - extends: - - .kitchen_test_install_script - - .kitchen_datadog_agent_flavor - - .kitchen_azure_location_south_central_us - - .kitchen_ec2_location_us_east_1 - -.kitchen_test_install_script_iot_agent: - extends: - - .kitchen_test_install_script - - .kitchen_datadog_iot_agent_flavor - - .kitchen_azure_location_west_central_us - - .kitchen_ec2_location_us_east_1 - -.kitchen_test_install_script_heroku_agent: - extends: - - .kitchen_test_install_script - - .kitchen_datadog_heroku_agent_flavor - - .kitchen_azure_location_west_central_us - - .kitchen_ec2_location_us_east_1 - -.kitchen_test_install_script_dogstatsd: - extends: - - .kitchen_test_install_script - - .kitchen_datadog_dogstatsd_flavor - - .kitchen_azure_location_west_central_us - - .kitchen_ec2_location_us_east_1 - -.kitchen_test_upgrade5_agent: - extends: - - .kitchen_test_upgrade5 - - .kitchen_datadog_agent_flavor - - .kitchen_azure_location_central_us - - .kitchen_ec2_location_us_east_1 - -.kitchen_test_upgrade6_agent: - extends: - - .kitchen_test_upgrade6 - - .kitchen_datadog_agent_flavor - - .kitchen_azure_location_south_central_us - - .kitchen_ec2_location_us_east_1 - -.kitchen_test_upgrade7_agent: - extends: - - .kitchen_test_upgrade7 - - .kitchen_datadog_agent_flavor - - .kitchen_azure_location_south_central_us - - .kitchen_ec2_location_us_east_1 - -.kitchen_test_upgrade7_iot_agent: - extends: - - .kitchen_test_upgrade7 - - .kitchen_datadog_iot_agent_flavor - - .kitchen_azure_location_south_central_us - - .kitchen_ec2_location_us_east_1 diff --git a/.gitlab/kitchen_deploy.yml b/.gitlab/kitchen_deploy.yml deleted file mode 100644 index 05b5e2c0ad5969..00000000000000 --- a/.gitlab/kitchen_deploy.yml +++ /dev/null @@ -1,271 +0,0 @@ ---- -# kitchen_deploy stage -# Contains jobs which deploy Agent package to testing repsoitories that are used in kitchen tests. - -.setup_rpm_signing_key: &setup_rpm_signing_key - - set +x - - RPM_GPG_KEY=$(aws ssm get-parameter --region us-east-1 --name $RPM_GPG_KEY_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - printf -- "$RPM_GPG_KEY" | gpg --import --batch - - export RPM_SIGNING_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name $RPM_SIGNING_PASSPHRASE_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - -.setup_apt_signing_key: &setup_apt_signing_key - - set +x # make sure we don't output the creds to the build log - - - APT_SIGNING_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name $DEB_GPG_KEY_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - APT_SIGNING_KEY_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name $DEB_SIGNING_PASSPHRASE_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - - printf -- "$APT_SIGNING_PRIVATE_KEY" | gpg --import --batch - -.setup_signing_keys_package: &setup_signing_keys_package - # Set up prod apt repo to get the datadog-signing-keys package - - echo 'deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] https://apt.datadoghq.com/ stable 7' > /etc/apt/sources.list.d/datadog.list - - touch /usr/share/keyrings/datadog-archive-keyring.gpg - - chmod a+r /usr/share/keyrings/datadog-archive-keyring.gpg - - curl https://keys.datadoghq.com/DATADOG_APT_KEY_CURRENT.public | gpg --no-default-keyring --keyring /usr/share/keyrings/datadog-archive-keyring.gpg --import --batch - - - apt-get update - # Note: it's fine to overwrite the datadog-signing-keys package during each pipeline run, as the package in the - # pool will remain the same for a given version (the hashsums of the package won't change as long as we - # upload the same package, which is always going to be the case except if we overwrite a datadog-signing-keys package - # in the prod repo). - - apt-get -o "Dir::Cache::archives=$OMNIBUS_PACKAGE_DIR" install --download-only datadog-signing-keys - - # Rename the package name to a name unique to the pipeline & Agent version, to avoid collisions - # between packages in the pool. - - pushd $OMNIBUS_PACKAGE_DIR - - filename=$(ls datadog-signing-keys*.deb); mv $filename datadog-signing-keys_${DD_PIPELINE_ID}.deb - - popd - -# Avoid simultaneous writes on the repo metadata file that made kitchen tests fail before -.deploy_deb_resource_group-a6: &deploy_deb_resource_group-a6 - resource_group: deploy_deb_a6 - -.deploy_deb_resource_group-a7: &deploy_deb_resource_group-a7 - resource_group: deploy_deb_a7 - -.deploy_deb_testing-a6: - stage: kitchen_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - <<: *deploy_deb_resource_group-a6 - variables: - DD_PIPELINE_ID: $CI_PIPELINE_ID-a6 - before_script: - - source /root/.bashrc - - ls $OMNIBUS_PACKAGE_DIR - -deploy_deb_testing-a6_x64: - rules: - !reference [.on_kitchen_tests_a6] - extends: - - .deploy_deb_testing-a6 - needs: ["agent_deb-x64-a6", "agent_heroku_deb-x64-a6", "tests_deb-x64-py2", "lint_linux-x64", "tests_deb-x64-py3"] - script: - - *setup_apt_signing_key - - set +x # make sure we don't output the creds to the build log - - - *setup_signing_keys_package - - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-x86_64" -m 6 -b $DEB_TESTING_S3_BUCKET -a amd64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-*_6*amd64.deb - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-x86_64" -m 6 -b $DEB_TESTING_S3_BUCKET -a x86_64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-*_6*amd64.deb - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-x86_64" -m 6 -b $DEB_TESTING_S3_BUCKET -a amd64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-signing-keys_${DD_PIPELINE_ID}.deb - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-x86_64" -m 6 -b $DEB_TESTING_S3_BUCKET -a x86_64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-signing-keys_${DD_PIPELINE_ID}.deb - -deploy_deb_testing-a6_arm64: - rules: - !reference [.on_all_kitchen_builds_a6] - extends: - - .deploy_deb_testing-a6 - needs: ["agent_deb-arm64-a6", "tests_deb-arm64-py2"] - script: - - *setup_apt_signing_key - - set +x # make sure we don't output the creds to the build log - - - *setup_signing_keys_package - - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-arm64" -m 6 -b $DEB_TESTING_S3_BUCKET -a arm64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-*_6*arm64.deb - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-arm64" -m 6 -b $DEB_TESTING_S3_BUCKET -a arm64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-signing-keys_${DD_PIPELINE_ID}.deb - -.deploy_deb_testing-a7: - stage: kitchen_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - <<: *deploy_deb_resource_group-a7 - variables: - DD_PIPELINE_ID: $CI_PIPELINE_ID-a7 - before_script: - - source /root/.bashrc - - ls $OMNIBUS_PACKAGE_DIR - -deploy_deb_testing-a7_x64: - rules: - !reference [.on_default_kitchen_tests_a7] - extends: - - .deploy_deb_testing-a7 - needs: ["agent_deb-x64-a7", "agent_heroku_deb-x64-a7", "iot_agent_deb-x64", "dogstatsd_deb-x64", "lint_linux-x64", "tests_deb-x64-py3"] - script: - - *setup_apt_signing_key - - set +x # make sure we don't output the creds to the build log - - - *setup_signing_keys_package - - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-x86_64" -m 7 -b $DEB_TESTING_S3_BUCKET -a amd64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-*_7*amd64.deb - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-x86_64" -m 7 -b $DEB_TESTING_S3_BUCKET -a x86_64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-*_7*amd64.deb - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-x86_64" -m 7 -b $DEB_TESTING_S3_BUCKET -a amd64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-signing-keys_${DD_PIPELINE_ID}.deb - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-x86_64" -m 7 -b $DEB_TESTING_S3_BUCKET -a x86_64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-signing-keys_${DD_PIPELINE_ID}.deb - -deploy_deb_testing-a7_arm64: - rules: - !reference [.on_all_kitchen_builds_a7] - extends: - - .deploy_deb_testing-a7 - needs: ["agent_deb-arm64-a7", "lint_linux-arm64", "tests_deb-arm64-py3"] - script: - - *setup_apt_signing_key - - set +x # make sure we don't output the creds to the build log - - - *setup_signing_keys_package - - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-arm64" -m 7 -b $DEB_TESTING_S3_BUCKET -a arm64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-*_7*arm64.deb - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-arm64" -m 7 -b $DEB_TESTING_S3_BUCKET -a arm64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-signing-keys_${DD_PIPELINE_ID}.deb - -.deploy_rpm_testing-a6: - stage: kitchen_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - variables: - DD_PIPELINE_ID: $CI_PIPELINE_ID-a6 - before_script: - - source /root/.bashrc - - ls $OMNIBUS_PACKAGE_DIR - -deploy_rpm_testing-a6_x64: - rules: - !reference [.on_kitchen_tests_a6] - extends: - - .deploy_rpm_testing-a6 - needs: ["agent_rpm-x64-a6", "tests_rpm-x64-py2", "lint_linux-x64", "tests_rpm-x64-py3"] - script: - - *setup_rpm_signing_key - - set +x - - echo "$RPM_SIGNING_PASSPHRASE" | rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "testing/pipeline-$DD_PIPELINE_ID/6/x86_64/" -a "x86_64" --sign --metadata-signing-key $RPM_GPG_KEY_ID $OMNIBUS_PACKAGE_DIR/datadog-*-6.*x86_64.rpm - -deploy_rpm_testing-a6_arm64: - rules: - !reference [.on_all_kitchen_builds_a6] - extends: - - .deploy_rpm_testing-a6 - needs: ["agent_rpm-arm64-a6", "tests_rpm-arm64-py2", "lint_linux-arm64", "tests_rpm-arm64-py3"] - script: - - *setup_rpm_signing_key - - set +x - - echo "$RPM_SIGNING_PASSPHRASE" | rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "testing/pipeline-$DD_PIPELINE_ID/6/aarch64/" -a "aarch64" --sign --metadata-signing-key $RPM_GPG_KEY_ID $OMNIBUS_PACKAGE_DIR/datadog-*-6.*aarch64.rpm - -.deploy_rpm_testing-a7: - stage: kitchen_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - variables: - DD_PIPELINE_ID: $CI_PIPELINE_ID-a7 - before_script: - - source /root/.bashrc - - ls $OMNIBUS_PACKAGE_DIR - -deploy_rpm_testing-a7_x64: - rules: - !reference [.on_default_kitchen_tests_a7] - extends: - - .deploy_rpm_testing-a7 - needs: ["agent_rpm-x64-a7", "iot_agent_rpm-x64", "dogstatsd_rpm-x64", "lint_linux-x64", "tests_rpm-x64-py3"] - script: - - *setup_rpm_signing_key - - set +x - - echo "$RPM_SIGNING_PASSPHRASE" | rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "testing/pipeline-$DD_PIPELINE_ID/7/x86_64/" -a "x86_64" --sign --metadata-signing-key $RPM_GPG_KEY_ID $OMNIBUS_PACKAGE_DIR/datadog-*-7.*x86_64.rpm - -deploy_rpm_testing-a7_arm64: - rules: - !reference [.on_all_kitchen_builds_a7] - extends: - - .deploy_rpm_testing-a7 - needs: ["agent_rpm-arm64-a7", "lint_linux-arm64", "tests_rpm-arm64-py3"] - script: - - *setup_rpm_signing_key - - set +x - - echo "$RPM_SIGNING_PASSPHRASE" | rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "testing/pipeline-$DD_PIPELINE_ID/7/aarch64/" -a "aarch64" --sign --metadata-signing-key $RPM_GPG_KEY_ID $OMNIBUS_PACKAGE_DIR/datadog-*-7.*aarch64.rpm - -deploy_suse_rpm_testing_x64-a6: - rules: - !reference [.on_kitchen_tests_a6] - stage: kitchen_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - needs: ["agent_suse-x64-a6", "tests_rpm-x64-py2", "lint_linux-x64", "tests_rpm-x64-py3"] - variables: - DD_PIPELINE_ID: $CI_PIPELINE_ID-a6 - before_script: - - source /root/.bashrc - - ls $OMNIBUS_PACKAGE_DIR_SUSE - script: - - *setup_rpm_signing_key - - set +x - - echo "$RPM_SIGNING_PASSPHRASE" | rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "suse/testing/pipeline-$DD_PIPELINE_ID/6/x86_64/" -a "x86_64" --sign --metadata-signing-key $RPM_GPG_KEY_ID --repodata-store-public-key $OMNIBUS_PACKAGE_DIR_SUSE/datadog-*-6.*x86_64.rpm - -deploy_suse_rpm_testing_x64-a7: - rules: - !reference [.on_default_kitchen_tests_a7] - stage: kitchen_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - needs: ["agent_suse-x64-a7", "iot_agent_suse-x64", "dogstatsd_suse-x64", "lint_linux-x64", "tests_rpm-x64-py3"] - variables: - DD_PIPELINE_ID: $CI_PIPELINE_ID-a7 - before_script: - - source /root/.bashrc - - ls $OMNIBUS_PACKAGE_DIR_SUSE - script: - - *setup_rpm_signing_key - - set +x - - echo "$RPM_SIGNING_PASSPHRASE" | rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "suse/testing/pipeline-$DD_PIPELINE_ID/7/x86_64/" -a "x86_64" --sign --metadata-signing-key $RPM_GPG_KEY_ID --repodata-store-public-key $OMNIBUS_PACKAGE_DIR_SUSE/datadog-*-7.*x86_64.rpm - -deploy_suse_rpm_testing_arm64-a7: - rules: - !reference [.on_kitchen_tests_a7] - stage: kitchen_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - needs: ["agent_suse-arm64-a7", "lint_linux-arm64", "tests_rpm-arm64-py3"] - variables: - DD_PIPELINE_ID: $CI_PIPELINE_ID-a7 - before_script: - - source /root/.bashrc - - ls $OMNIBUS_PACKAGE_DIR_SUSE - script: - - *setup_rpm_signing_key - - set +x - - echo "$RPM_SIGNING_PASSPHRASE" | rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "suse/testing/pipeline-$DD_PIPELINE_ID/7/aarch64/" -a "aarch64" --sign --metadata-signing-key $RPM_GPG_KEY_ID --repodata-store-public-key $OMNIBUS_PACKAGE_DIR_SUSE/datadog-*-7.*aarch64.rpm - -deploy_windows_testing-a6: - rules: - !reference [.on_kitchen_tests_a6] - stage: kitchen_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - needs: ["lint_windows-x64", "tests_windows-x64", "windows_msi_x64-a6"] - before_script: - - source /root/.bashrc - - ls $OMNIBUS_PACKAGE_DIR - script: - - $S3_CP_CMD --recursive --exclude "*" --include "datadog-agent-6.*.msi" $OMNIBUS_PACKAGE_DIR s3://$WIN_S3_BUCKET/$WINDOWS_TESTING_S3_BUCKET_A6 --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 - -deploy_windows_testing-a7: - rules: - !reference [.on_default_kitchen_tests_a7] - stage: kitchen_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - needs: ["lint_windows-x64", "tests_windows-x64", "windows_msi_and_bosh_zip_x64-a7"] - before_script: - - source /root/.bashrc - - ls $OMNIBUS_PACKAGE_DIR - script: - - $S3_CP_CMD --recursive --exclude "*" --include "datadog-agent-7.*.msi" $OMNIBUS_PACKAGE_DIR s3://$WIN_S3_BUCKET/$WINDOWS_TESTING_S3_BUCKET_A7 --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 diff --git a/.gitlab/kitchen_testing.yml b/.gitlab/kitchen_testing.yml deleted file mode 100644 index b55e87219cacdc..00000000000000 --- a/.gitlab/kitchen_testing.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# kitchen_testing stage -# Contains jobs which run kitchen tests on the Agent packages. - -include: - - /.gitlab/kitchen_testing/amazonlinux.yml - - /.gitlab/kitchen_testing/centos.yml - - /.gitlab/kitchen_testing/debian.yml - - /.gitlab/kitchen_testing/suse.yml - - /.gitlab/kitchen_testing/ubuntu.yml - - /.gitlab/kitchen_testing/windows.yml diff --git a/.gitlab/kitchen_testing/amazonlinux.yml b/.gitlab/kitchen_testing/amazonlinux.yml deleted file mode 100644 index 792d307858a9d6..00000000000000 --- a/.gitlab/kitchen_testing/amazonlinux.yml +++ /dev/null @@ -1,96 +0,0 @@ ---- -# FIXME: our current Gitlab version doesn't support importing a file more than once -# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987 -# include: -# - /.gitlab/kitchen_common/testing.yml - - -# Kitchen: OSes -# ------------- - -.kitchen_os_amazonlinux: - variables: - KITCHEN_PLATFORM: "amazonlinux" - before_script: - - cd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - -# Kitchen: scenarios (os * agent * (cloud + arch)) -# ------------------------------- - -.kitchen_scenario_amazonlinux_a6_x64: - variables: - KITCHEN_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15,amazonlinux2023" - KITCHEN_CWS_SUPPORTED_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15,amazonlinux2023" - DEFAULT_KITCHEN_OSVERS: "amazonlinux2023" - extends: - - .kitchen_agent_a6 - - .kitchen_os_amazonlinux - - .kitchen_ec2_x64 - needs: ["deploy_rpm_testing-a6_x64"] - -.kitchen_scenario_amazonlinux_a7_x64: - variables: - KITCHEN_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15,amazonlinux2023" - KITCHEN_CWS_SUPPORTED_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15,amazonlinux2023" - DEFAULT_KITCHEN_OSVERS: "amazonlinux2023" - extends: - - .kitchen_agent_a7 - - .kitchen_os_amazonlinux - - .kitchen_ec2_x64 - needs: ["deploy_rpm_testing-a7_x64"] - -.kitchen_scenario_amazonlinux_a6_arm64: - variables: - KITCHEN_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15,amazonlinux2023" - KITCHEN_CWS_SUPPORTED_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15,amazonlinux2023" - DEFAULT_KITCHEN_OSVERS: "amazonlinux2023" - extends: - - .kitchen_agent_a6 - - .kitchen_os_amazonlinux - - .kitchen_ec2_arm64 - needs: ["deploy_rpm_testing-a6_arm64"] - -.kitchen_scenario_amazonlinux_a7_arm64: - variables: - KITCHEN_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15,amazonlinux2023" - KITCHEN_CWS_SUPPORTED_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15,amazonlinux2023" - DEFAULT_KITCHEN_OSVERS: "amazonlinux2023" - extends: - - .kitchen_agent_a7 - - .kitchen_os_amazonlinux - - .kitchen_ec2_arm64 - needs: ["deploy_rpm_testing-a7_arm64"] - - # Kitchen: final test matrix (tests * scenarios) -# ---------------------------------------------- - -kitchen_amazonlinux_install_script_agent-a6_x64: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_amazonlinux_a6_x64 - - .kitchen_test_install_script_agent - -kitchen_amazonlinux_install_script_agent-a6_arm64: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_amazonlinux_a6_arm64 - - .kitchen_test_install_script_agent - -kitchen_amazonlinux_install_script_agent-a7_x64: - # Run install script test on branches, on a reduced number of platforms - rules: - !reference [.on_default_kitchen_tests_a7] - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_amazonlinux_a7_x64 - - .kitchen_test_install_script_agent - -kitchen_amazonlinux_install_script_agent-a7_arm64: - rules: - !reference [.on_all_kitchen_builds_a7] - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_amazonlinux_a7_arm64 - - .kitchen_test_install_script_agent diff --git a/.gitlab/kitchen_testing/centos.yml b/.gitlab/kitchen_testing/centos.yml deleted file mode 100644 index 49ae0df9b76d73..00000000000000 --- a/.gitlab/kitchen_testing/centos.yml +++ /dev/null @@ -1,248 +0,0 @@ ---- -# FIXME: our current Gitlab version doesn't support importing a file more than once -# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987 -# include: -# - /.gitlab/kitchen_common/testing.yml - - -# Kitchen: OSes -# ------------- - -# Used for Agent 5 upgrade tests, as Agent 5 RPMs aren't built to be -# installable in FIPS mode. -.kitchen_os_centos_no_support_for_fips: - variables: - KITCHEN_PLATFORM: "centos" - KITCHEN_OSVERS: "centos-610,centos-77,rhel-81" - KITCHEN_CWS_SUPPORTED_OSVERS: "centos-77,rhel-81" - before_script: - - cd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - -.kitchen_os_centos_without_fips: - variables: - KITCHEN_PLATFORM: "centos" - KITCHEN_OSVERS: "centos-610,centos-77,rhel-91" - KITCHEN_CWS_SUPPORTED_OSVERS: "centos-77" - before_script: - - cd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - -.kitchen_os_centos_with_fips: - variables: - KITCHEN_FIPS: "true" - KITCHEN_PLATFORM: "centos" - # For now, we only test FIPS on 8.1 - KITCHEN_OSVERS: "rhel-81" - KITCHEN_CWS_SUPPORTED_OSVERS: "rhel-81" - DEFAULT_KITCHEN_OSVERS: "rhel-81" - before_script: - - export KITCHEN_PLATFORM_SUFFIX="${KITCHEN_PLATFORM_SUFFIX}fips" - - cd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - -.kitchen_os_with_cws: - variables: - KITCHEN_CWS: "true" - -# Kitchen: scenarios (os * agent * (cloud + arch)) -# ------------------------------- - -.kitchen_scenario_centos_no_support_for_fips_a6: - extends: - - .kitchen_agent_a6 - - .kitchen_os_centos_no_support_for_fips - - .kitchen_azure_x64 - needs: ["deploy_rpm_testing-a6_x64"] - -.kitchen_scenario_centos_no_support_for_fips_a7: - extends: - - .kitchen_agent_a7 - - .kitchen_os_centos_no_support_for_fips - - .kitchen_azure_x64 - needs: ["deploy_rpm_testing-a7_x64"] - -.kitchen_scenario_centos_without_fips_a6: - extends: - - .kitchen_os_centos_without_fips - - .kitchen_agent_a6 - - .kitchen_azure_x64 - needs: ["deploy_rpm_testing-a6_x64"] - -.kitchen_scenario_centos_without_fips_a7: - extends: - - .kitchen_agent_a7 - - .kitchen_os_centos_without_fips - - .kitchen_azure_x64 - needs: ["deploy_rpm_testing-a7_x64"] - -.kitchen_scenario_centos_with_fips_a6: - extends: - - .kitchen_agent_a6 - - .kitchen_os_centos_with_fips - - .kitchen_azure_x64 - - .kitchen_os_with_cws - needs: ["deploy_rpm_testing-a6_x64"] - -.kitchen_scenario_centos_with_fips_a7: - extends: - - .kitchen_agent_a7 - - .kitchen_os_centos_with_fips - - .kitchen_azure_x64 - - .kitchen_os_with_cws - needs: ["deploy_rpm_testing-a7_x64"] - -# Kitchen: final test matrix (tests * scenarios) -# ---------------------------------------------- - -# We do install_script, step_by_step, upgrade6, upgrade7 and upgrade7_iot -# with FIPS on for CentOS/RHEL 8, while we do these with FIPS off -# for CentOS/RHEL 6/7. -kitchen_centos_install_script_agent-a6: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_centos_without_fips_a6 - - .kitchen_test_install_script_agent - -kitchen_centos_install_script_agent-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_centos_without_fips_a7 - - .kitchen_test_install_script_agent - -kitchen_centos_fips_install_script_agent-a6: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_centos_with_fips_a6 - - .kitchen_test_install_script_agent - -kitchen_centos_fips_install_script_agent-a7: - # Run install script test on branches, on a reduced number of platforms - rules: - !reference [.on_default_kitchen_tests_a7] - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_centos_with_fips_a7 - - .kitchen_test_install_script_agent - -kitchen_centos_install_script_iot_agent-a7: - extends: - - .kitchen_scenario_centos_without_fips_a7 - - .kitchen_test_install_script_iot_agent - -kitchen_centos_fips_install_script_iot_agent-a7: - extends: - - .kitchen_scenario_centos_with_fips_a7 - - .kitchen_test_install_script_iot_agent - -kitchen_centos_install_script_dogstatsd-a7: - extends: - - .kitchen_scenario_centos_without_fips_a7 - - .kitchen_test_install_script_dogstatsd - -kitchen_centos_fips_install_script_dogstatsd-a7: - extends: - - .kitchen_scenario_centos_with_fips_a7 - - .kitchen_test_install_script_dogstatsd - -# We only want to run step-by-step tests on deploy pipelines, -# which is why they have a different rule (if_deploy_6/7) - -kitchen_centos_step_by_step_agent-a6: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_centos_without_fips_a6 - - .kitchen_test_step_by_step_agent - rules: - !reference [.on_deploy_a6] - -kitchen_centos_fips_step_by_step_agent-a6: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_centos_with_fips_a6 - - .kitchen_test_step_by_step_agent - rules: - !reference [.on_deploy_a6] - -kitchen_centos_step_by_step_agent-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_centos_without_fips_a7 - - .kitchen_test_step_by_step_agent - rules: - !reference [.on_deploy_a7] - -kitchen_centos_fips_step_by_step_agent-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_centos_with_fips_a7 - - .kitchen_test_step_by_step_agent - rules: - !reference [.on_deploy_a7] - -# Agent 5 RPMs won't install on CentOS/RHEL 8 in FIPS mode, so we always -# run upgrade5 on all systems with FIPS off -kitchen_centos_upgrade5_agent-a6: - extends: - - .kitchen_scenario_centos_no_support_for_fips_a6 - - .kitchen_test_upgrade5_agent - -kitchen_centos_upgrade5_agent-a7: - extends: - - .kitchen_scenario_centos_no_support_for_fips_a7 - - .kitchen_test_upgrade5_agent - -kitchen_centos_upgrade6_agent-a6: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_centos_without_fips_a6 - - .kitchen_test_upgrade6_agent - -kitchen_centos_upgrade6_agent-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_centos_without_fips_a7 - - .kitchen_test_upgrade6_agent - -kitchen_centos_upgrade7_agent-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_centos_without_fips_a7 - - .kitchen_test_upgrade7_agent - -kitchen_centos_fips_upgrade6_agent-a6: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_centos_with_fips_a6 - - .kitchen_test_upgrade6_agent - -kitchen_centos_fips_upgrade6_agent-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_centos_with_fips_a7 - - .kitchen_test_upgrade6_agent - -kitchen_centos_fips_upgrade7_agent-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_centos_with_fips_a7 - - .kitchen_test_upgrade7_agent - -kitchen_centos_upgrade7_iot_agent-a7: - extends: - - .kitchen_scenario_centos_without_fips_a7 - - .kitchen_test_upgrade7_iot_agent - -kitchen_centos_fips_upgrade7_iot_agent-a7: - extends: - - .kitchen_scenario_centos_with_fips_a7 - - .kitchen_test_upgrade7_iot_agent - -kitchen_centos_process_agent-a7: - variables: - KITCHEN_OSVERS: "rhel-81" - DEFAULT_KITCHEN_OSVERS: "rhel-81" - extends: - - .kitchen_scenario_centos_no_support_for_fips_a7 - - .kitchen_test_process_agent diff --git a/.gitlab/kitchen_testing/debian.yml b/.gitlab/kitchen_testing/debian.yml deleted file mode 100644 index 6d3b5203a80a35..00000000000000 --- a/.gitlab/kitchen_testing/debian.yml +++ /dev/null @@ -1,146 +0,0 @@ ---- -# FIXME: our current Gitlab version doesn't support importing a file more than once -# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987 -# include: -# - /.gitlab/kitchen_common/testing.yml - - -# Kitchen: OSes -# ------------- - -.kitchen_os_debian: - variables: - KITCHEN_PLATFORM: "debian" - before_script: - - cd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - -# Kitchen: scenarios (os * agent * (cloud + arch)) -# ------------------------------- - -.kitchen_scenario_debian_a6_x64: - variables: - KITCHEN_OSVERS: "debian-9,debian-10,debian-11,debian-12" - KITCHEN_CWS_SUPPORTED_OSVERS: "debian-10,debian-11" - DEFAULT_KITCHEN_OSVERS: "debian-11" - extends: - - .kitchen_agent_a6 - - .kitchen_os_debian - - .kitchen_azure_x64 - needs: ["deploy_deb_testing-a6_x64"] - -.kitchen_scenario_debian_a7_x64: - variables: - KITCHEN_OSVERS: "debian-9,debian-10,debian-11,debian-12" - KITCHEN_CWS_SUPPORTED_OSVERS: "debian-10,debian-11" - DEFAULT_KITCHEN_OSVERS: "debian-11" - extends: - - .kitchen_agent_a7 - - .kitchen_os_debian - - .kitchen_azure_x64 - needs: ["deploy_deb_testing-a7_x64"] - -.kitchen_scenario_debian_a6_arm64: - variables: - KITCHEN_OSVERS: "debian-10" - KITCHEN_CWS_SUPPORTED_OSVERS: "debian-10" - DEFAULT_KITCHEN_OSVERS: "debian-10" - extends: - - .kitchen_agent_a6 - - .kitchen_os_debian - - .kitchen_ec2_arm64 - needs: ["deploy_deb_testing-a6_arm64"] - -.kitchen_scenario_debian_a7_arm64: - variables: - KITCHEN_OSVERS: "debian-10" - KITCHEN_CWS_SUPPORTED_OSVERS: "debian-10" - DEFAULT_KITCHEN_OSVERS: "debian-10" - extends: - - .kitchen_agent_a7 - - .kitchen_os_debian - - .kitchen_ec2_arm64 - needs: ["deploy_deb_testing-a7_arm64"] - -# Kitchen: final test matrix (tests * scenarios) -# ---------------------------------------------- - -kitchen_debian_install_script_agent-a6_x64: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_debian_a6_x64 - - .kitchen_test_install_script_agent - -kitchen_debian_install_script_agent-a6_arm64: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_debian_a6_arm64 - - .kitchen_test_install_script_agent - -kitchen_debian_install_script_heroku_agent-a6: - extends: - - .kitchen_scenario_debian_a6_x64 - - .kitchen_test_install_script_heroku_agent - -# We only want to run step-by-step tests on deploy pipelines, -# which is why they have a different rule (if_deploy_6/7) - -kitchen_debian_step_by_step_agent-a6: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_debian_a6_x64 - - .kitchen_test_step_by_step_agent - rules: - !reference [.on_deploy_a6] - -kitchen_debian_step_by_step_agent-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_debian_a7_x64 - - .kitchen_test_step_by_step_agent - rules: - !reference [.on_deploy_a7] - -kitchen_debian_upgrade5_agent-a6: - extends: - - .kitchen_scenario_debian_a6_x64 - - .kitchen_test_upgrade5_agent - -kitchen_debian_upgrade5_agent-a7: - extends: - - .kitchen_scenario_debian_a7_x64 - - .kitchen_test_upgrade5_agent - -kitchen_debian_upgrade6_agent-a6: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_debian_a6_x64 - - .kitchen_test_upgrade6_agent - -kitchen_debian_upgrade6_agent-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_debian_a7_x64 - - .kitchen_test_upgrade6_agent - -kitchen_debian_upgrade7_agent-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_debian_a7_x64 - - .kitchen_test_upgrade7_agent - -kitchen_debian_upgrade7_iot_agent-a7: - extends: - - .kitchen_scenario_debian_a7_x64 - - .kitchen_test_upgrade7_iot_agent - -kitchen_debian_process_agent-a7: - variables: - KITCHEN_OSVERS: "debian-11" - DEFAULT_KITCHEN_OSVERS: "debian-11" - extends: - - .kitchen_scenario_debian_a7_x64 - - .kitchen_test_process_agent - - diff --git a/.gitlab/kitchen_testing/suse.yml b/.gitlab/kitchen_testing/suse.yml deleted file mode 100644 index d3e66b7e6d82ce..00000000000000 --- a/.gitlab/kitchen_testing/suse.yml +++ /dev/null @@ -1,159 +0,0 @@ ---- -# FIXME: our current Gitlab version doesn't support importing a file more than once -# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987 -# include: -# - /.gitlab/kitchen_common/testing.yml - - -# Kitchen: OSes -# ------------- - -.kitchen_os_suse: - variables: - KITCHEN_PLATFORM: "suse" - KITCHEN_OSVERS: "sles-12,sles-15" - KITCHEN_CWS_SUPPORTED_OSVERS: "sles-12,sles-15" - DEFAULT_KITCHEN_OSVERS: "sles-15" - before_script: - - cd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - -# Kitchen: scenarios (os * agent * (cloud + arch)) -# ------------------------------- - -.kitchen_scenario_suse_x64_a6: - extends: - - .kitchen_agent_a6 - - .kitchen_os_suse - - .kitchen_azure_x64 - needs: ["deploy_suse_rpm_testing_x64-a6"] - -.kitchen_scenario_suse_x64_a7: - extends: - - .kitchen_agent_a7 - - .kitchen_os_suse - - .kitchen_azure_x64 - needs: ["deploy_suse_rpm_testing_x64-a7"] - -.kitchen_scenario_suse_arm64_a7: - variables: - KITCHEN_OSVERS: "sles-15" - DEFAULT_KITCHEN_OSVERS: "sles-15" - extends: - - .kitchen_agent_a7 - - .kitchen_os_suse - - .kitchen_ec2_arm64 - - .kitchen_ec2_location_us_east_1 - needs: ["deploy_suse_rpm_testing_arm64-a7"] - -# Kitchen: final test matrix (tests * scenarios) -# ---------------------------------------------- - -kitchen_suse_install_script_agent_x64-a6: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_suse_x64_a6 - - .kitchen_test_install_script_agent - -kitchen_suse_install_script_agent_x64-a7: - # Run install script test on branches, on a reduced number of platforms - rules: - !reference [.on_default_kitchen_tests_a7] - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_suse_x64_a7 - - .kitchen_test_install_script_agent - -kitchen_suse_install_script_iot_agent_x64-a7: - extends: - - .kitchen_scenario_suse_x64_a7 - - .kitchen_test_install_script_iot_agent - -kitchen_suse_install_script_dogstatsd_x64-a7: - extends: - - .kitchen_scenario_suse_x64_a7 - - .kitchen_test_install_script_dogstatsd - -# We only want to run step-by-step tests on deploy pipelines, -# which is why they have a different rule (if_deploy_6/7) - -kitchen_suse_step_by_step_agent_x64-a6: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_suse_x64_a6 - - .kitchen_test_step_by_step_agent - rules: - !reference [.on_deploy_a6] - -kitchen_suse_step_by_step_agent_x64-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_suse_x64_a7 - - .kitchen_test_step_by_step_agent - rules: - !reference [.on_deploy_a7] - -kitchen_suse_install_script_agent_arm64-a7: - # Run install script test on branches, on a reduced number of platforms - rules: - !reference [.on_kitchen_tests_a7] - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_suse_arm64_a7 - - .kitchen_test_install_script_agent - -# We only want to run step-by-step tests on deploy pipelines, -# which is why they have a different rule (if_deploy_6/7) - -kitchen_suse_step_by_step_agent_arm64-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_suse_arm64_a7 - - .kitchen_test_step_by_step_agent - rules: - !reference [.on_deploy_a7] - - -# The change https://github.com/DataDog/dd-agent/commit/d024c411c56595099761dc4ac4d0133e6fa152bf -# made the SUSE Agent 5 upgrade tests fail when there is no Agent 5 SUSE RPM signed with the new -# key. We disable them temporarily because of this. -# -# kitchen_suse_upgrade5_agent-a6: -# extends: -# - .kitchen_scenario_suse_x64_a6 -# - .kitchen_test_upgrade5_agent -# -# kitchen_suse_upgrade5_agent-a7: -# extends: -# - .kitchen_scenario_suse_x64_a7 -# - .kitchen_test_upgrade5_agent - -kitchen_suse_upgrade6_agent_x64-a6: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_suse_x64_a6 - - .kitchen_test_upgrade6_agent - -kitchen_suse_upgrade6_agent_x64-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_suse_x64_a7 - - .kitchen_test_upgrade6_agent - -kitchen_suse_upgrade7_agent_x64-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_suse_x64_a7 - - .kitchen_test_upgrade7_agent - -kitchen_suse_upgrade7_iot_agent_x64-a7: - extends: - - .kitchen_scenario_suse_x64_a7 - - .kitchen_test_upgrade7_iot_agent - -kitchen_suse_process_agent_x64-a7: - extends: - - .kitchen_scenario_suse_x64_a7 - - .kitchen_test_process_agent - diff --git a/.gitlab/kitchen_testing/ubuntu.yml b/.gitlab/kitchen_testing/ubuntu.yml deleted file mode 100644 index 59967ab8c9f1f2..00000000000000 --- a/.gitlab/kitchen_testing/ubuntu.yml +++ /dev/null @@ -1,181 +0,0 @@ ---- -# FIXME: our current Gitlab version doesn't support importing a file more than once -# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987 -# include: -# - /.gitlab/kitchen_common/testing.yml - -# Kitchen: OSes -# ------------- - -.kitchen_os_ubuntu: - variables: - KITCHEN_PLATFORM: "ubuntu" - before_script: - - cd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - -# Kitchen: scenarios (os * agent * (cloud + arch)) -# ------------------------------- - -.kitchen_scenario_ubuntu_a6_x64: - variables: - KITCHEN_OSVERS: "ubuntu-14-04,ubuntu-16-04,ubuntu-18-04,ubuntu-20-04,ubuntu-22-04" - KITCHEN_CWS_SUPPORTED_OSVERS: "ubuntu-18-04,ubuntu-20-04,ubuntu-22-04" - DEFAULT_KITCHEN_OSVERS: "ubuntu-22-04" - extends: - - .kitchen_agent_a6 - - .kitchen_os_ubuntu - - .kitchen_azure_x64 - needs: ["deploy_deb_testing-a6_x64"] - -.kitchen_scenario_ubuntu_a7_x64: - variables: - KITCHEN_OSVERS: "ubuntu-14-04,ubuntu-16-04,ubuntu-18-04,ubuntu-20-04,ubuntu-22-04" - KITCHEN_CWS_SUPPORTED_OSVERS: "ubuntu-18-04,ubuntu-20-04,ubuntu-22-04" - DEFAULT_KITCHEN_OSVERS: "ubuntu-22-04" - extends: - - .kitchen_agent_a7 - - .kitchen_os_ubuntu - - .kitchen_azure_x64 - needs: ["deploy_deb_testing-a7_x64"] - -.kitchen_scenario_ubuntu_a6_arm64: - variables: - KITCHEN_OSVERS: "ubuntu-18-04,ubuntu-20-04" - KITCHEN_CWS_SUPPORTED_OSVERS: "ubuntu-18-04,ubuntu-20-04" - DEFAULT_KITCHEN_OSVERS: "ubuntu-20-04" - extends: - - .kitchen_agent_a6 - - .kitchen_os_ubuntu - - .kitchen_ec2_arm64 - needs: ["deploy_deb_testing-a6_arm64"] - -.kitchen_scenario_ubuntu_a7_arm64: - variables: - KITCHEN_OSVERS: "ubuntu-18-04,ubuntu-20-04" - KITCHEN_CWS_SUPPORTED_OSVERS: "ubuntu-18-04,ubuntu-20-04" - DEFAULT_KITCHEN_OSVERS: "ubuntu-20-04" - extends: - - .kitchen_agent_a7 - - .kitchen_os_ubuntu - - .kitchen_ec2_arm64 - needs: ["deploy_deb_testing-a7_arm64"] - -# Kitchen: final test matrix (tests * scenarios) -# ---------------------------------------------- - -kitchen_ubuntu_install_script_agent-a6_x64: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_ubuntu_a6_x64 - - .kitchen_test_install_script_agent - -kitchen_ubuntu_install_script_agent-a6_arm64: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_ubuntu_a6_arm64 - - .kitchen_test_install_script_agent - -kitchen_ubuntu_install_script_agent-a7_x64: - # Run install script test on branches, on a reduced number of platforms - rules: - !reference [.on_default_kitchen_tests_a7] - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_ubuntu_a7_x64 - - .kitchen_test_install_script_agent - -kitchen_ubuntu_install_script_agent-a7_arm64: - rules: - !reference [.on_all_kitchen_builds_a7] - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_ubuntu_a7_arm64 - - .kitchen_test_install_script_agent - -kitchen_ubuntu_install_script_iot_agent-a7: - extends: - - .kitchen_scenario_ubuntu_a7_x64 - - .kitchen_test_install_script_iot_agent - -kitchen_ubuntu_install_script_dogstatsd-a7: - extends: - - .kitchen_scenario_ubuntu_a7_x64 - - .kitchen_test_install_script_dogstatsd - -# We only want to run step-by-step tests on deploy pipelines, -# which is why they have a different rule (if_deploy_6/7) - -kitchen_ubuntu_step_by_step_agent-a6_x64: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_ubuntu_a6_x64 - - .kitchen_test_step_by_step_agent - rules: - !reference [.on_deploy_a6] - -kitchen_ubuntu_step_by_step_agent-a6_arm64: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_ubuntu_a6_arm64 - - .kitchen_test_step_by_step_agent - rules: - !reference [.on_deploy_a6] - -kitchen_ubuntu_step_by_step_agent-a7_x64: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_ubuntu_a7_x64 - - .kitchen_test_step_by_step_agent - rules: - !reference [.on_deploy_a7] - -kitchen_ubuntu_step_by_step_agent-a7_arm64: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_ubuntu_a7_arm64 - - .kitchen_test_step_by_step_agent - rules: - !reference [.on_deploy_a7] - -kitchen_ubuntu_upgrade5_agent-a6: - extends: - - .kitchen_scenario_ubuntu_a6_x64 - - .kitchen_test_upgrade5_agent - -kitchen_ubuntu_upgrade5_agent-a7: - extends: - - .kitchen_scenario_ubuntu_a7_x64 - - .kitchen_test_upgrade5_agent - -kitchen_ubuntu_upgrade6_agent-a6: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_ubuntu_a6_x64 - - .kitchen_test_upgrade6_agent - -kitchen_ubuntu_upgrade6_agent-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_ubuntu_a7_x64 - - .kitchen_test_upgrade6_agent - -kitchen_ubuntu_upgrade7_agent-a7: - extends: - - .kitchen_os_with_cws - - .kitchen_scenario_ubuntu_a7_x64 - - .kitchen_test_upgrade7_agent - -kitchen_ubuntu_upgrade7_iot_agent-a7: - extends: - - .kitchen_scenario_ubuntu_a7_x64 - - .kitchen_test_upgrade7_iot_agent - -kitchen_ubuntu_process_agent-a7: - variables: - KITCHEN_OSVERS: "ubuntu-20-04" - DEFAULT_KITCHEN_OSVERS: "ubuntu-20-04" - extends: - - .kitchen_scenario_ubuntu_a7_x64 - - .kitchen_test_process_agent diff --git a/.gitlab/kitchen_testing/windows.yml b/.gitlab/kitchen_testing/windows.yml deleted file mode 100644 index 0fe1a46f663b5d..00000000000000 --- a/.gitlab/kitchen_testing/windows.yml +++ /dev/null @@ -1,187 +0,0 @@ ---- -# FIXME: our current Gitlab version doesn't support importing a file more than once -# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987 -# include: -# - /.gitlab/kitchen_common/testing.yml - -# Kitchen: OSes -# ------------- - -.kitchen_os_windows: - extends: - - .kitchen_azure_x64 - variables: - KITCHEN_PLATFORM: "windows" - KITCHEN_OSVERS: "win2016,win2019,win2019cn,win2022" - DEFAULT_KITCHEN_OSVERS: "win2022" - before_script: # Note: if you are changing this, remember to also change .kitchen_test_windows_installer, which has a copy of this with less TEST_PLATFORMS defined. - - if [ $AGENT_MAJOR_VERSION == "7" ]; then export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A7; else export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A6; fi - - cd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - # Windows kitchen tests are slower and more fragile (lots of WinRM::WinRMAuthorizationError and/or execution expired errors) - # Give them one more chance before failing. - # TODO: understand why they fail more often than Linux jobs on network errors. - retry: 2 - -# Kitchen: tests -# -------------- - -.kitchen_test_windows_installer: - extends: - - .kitchen_azure_x64 - variables: - KITCHEN_PLATFORM: "windows" - KITCHEN_OSVERS: "win2016" - before_script: # Use a smaller set of TEST_PLATFORMS than .kitchen_os_windows - - if [ $AGENT_MAJOR_VERSION == "7" ]; then export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A7; else export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A6; fi - - cd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - script: - - export LAST_STABLE_VERSION=$(cd ../.. && invoke release.get-release-json-value "last_stable::$AGENT_MAJOR_VERSION") - - tasks/run-test-kitchen.sh windows-install-test $AGENT_MAJOR_VERSION - allow_failure: false - -.kitchen_test_windows_npm_driver_base: - extends: - - .kitchen_azure_x64 - before_script: # test all of the kernels to make sure the driver loads and runs properly - - if [ $AGENT_MAJOR_VERSION == "7" ]; then export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A7; else export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A6; fi - - if [ $AGENT_MAJOR_VERSION == "7" ]; then export RELEASE_VERSION=$RELEASE_VERSION_7; else export RELEASE_VERSION=$RELEASE_VERSION_6; fi - - export WINDOWS_DDNPM_DRIVER=$(inv release.get-release-json-value "$RELEASE_VERSION::WINDOWS_DDNPM_DRIVER") - - cd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - script: - - tasks/run-test-kitchen.sh windows-npmdriver $AGENT_MAJOR_VERSION - -.kitchen_test_windows_npm_driver: - extends: - - .kitchen_test_windows_npm_driver_base - variables: - KITCHEN_PLATFORM: "windows" - KITCHEN_OSVERS: "win2016,win2019,win2019cn,win2022" - DEFAULT_KITCHEN_OSVERS: "win2022" - -.kitchen_test_windows_npm_installer: - extends: - - .kitchen_azure_x64 - variables: - KITCHEN_PLATFORM: "windows" - KITCHEN_OSVERS: "win2016" - before_script: # Use only 2016 and 2019 for testing that we upgrade properly and don't install the driver when not specified - - if [ $AGENT_MAJOR_VERSION == "7" ]; then export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A7; else export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A6; fi - - if [ $AGENT_MAJOR_VERSION == "7" ]; then export RELEASE_VERSION=$RELEASE_VERSION_7; else export RELEASE_VERSION=$RELEASE_VERSION_6; fi - - export WINDOWS_DDNPM_DRIVER=$(inv release.get-release-json-value "$RELEASE_VERSION::WINDOWS_DDNPM_DRIVER") - - cd $DD_AGENT_TESTING_DIR - - tasks/kitchen_setup.sh - script: - - export LAST_STABLE_VERSION=$(cd ../.. && invoke release.get-release-json-value "last_stable::$AGENT_MAJOR_VERSION") - - tasks/run-test-kitchen.sh windows-npm-test $AGENT_MAJOR_VERSION - -# Kitchen: Test types (test suite * agent flavor + azure location) -# ------------------------------- - -.kitchen_test_windows_installer_agent: - extends: - - .kitchen_test_windows_installer - - .kitchen_datadog_agent_flavor - - .kitchen_azure_location_north_central_us - -.kitchen_test_windows_installer_driver: - extends: - - .kitchen_test_windows_npm_driver - - .kitchen_datadog_agent_flavor - - .kitchen_azure_location_north_central_us - -.kitchen_test_windows_installer_npm: - extends: - - .kitchen_test_windows_npm_installer - - .kitchen_datadog_agent_flavor - - .kitchen_azure_location_north_central_us - -# Kitchen: scenarios (os * agent) -# ------------------------------- - -.kitchen_scenario_windows_a6: - extends: - - .kitchen_agent_a6 - - .kitchen_os_windows - needs: ["deploy_windows_testing-a6"] - -.kitchen_scenario_windows_a7: - extends: - - .kitchen_agent_a7 - - .kitchen_os_windows - needs: ["deploy_windows_testing-a7"] - -# Kitchen: final test matrix (test types * scenarios) -# ---------------------------------------------- - -kitchen_windows_installer_npm_install_scenarios-a7: - extends: - - .kitchen_scenario_windows_a7 - - .kitchen_test_windows_installer_npm - -kitchen_windows_installer_npm_driver-a7: - # Run NPM driver installer test on branches, on a reduced number of platforms - rules: - !reference [.on_default_kitchen_tests_a7] - extends: - - .kitchen_scenario_windows_a7 - - .kitchen_test_windows_installer_driver - -kitchen_windows_installer_agent-a6: - extends: - - .kitchen_scenario_windows_a6 - - .kitchen_test_windows_installer_agent - -kitchen_windows_installer_agent-a7: - extends: - - .kitchen_scenario_windows_a7 - - .kitchen_test_windows_installer_agent - -kitchen_windows_chef_agent-a6: - extends: - - .kitchen_scenario_windows_a6 - - .kitchen_test_chef_agent - -kitchen_windows_chef_agent-a7: - # Run chef test on branches, on a reduced number of platforms - rules: - !reference [.on_default_kitchen_tests_a7] - extends: - - .kitchen_scenario_windows_a7 - - .kitchen_test_chef_agent - -kitchen_windows_upgrade5_agent-a6: - extends: - - .kitchen_scenario_windows_a6 - - .kitchen_test_upgrade5_agent - -kitchen_windows_upgrade5_agent-a7: - extends: - - .kitchen_scenario_windows_a7 - - .kitchen_test_upgrade5_agent - -kitchen_windows_upgrade6_agent-a6: - extends: - - .kitchen_scenario_windows_a6 - - .kitchen_test_upgrade6_agent - -kitchen_windows_upgrade6_agent-a7: - extends: - - .kitchen_scenario_windows_a7 - - .kitchen_test_upgrade6_agent - -kitchen_windows_upgrade7_agent-a7: - extends: - - .kitchen_scenario_windows_a7 - - .kitchen_test_upgrade7_agent - -kitchen_windows_process_agent-a7: - variables: - KITCHEN_OSVERS: "win2022" - DEFAULT_KITCHEN_OSVERS: "win2022" - extends: - - .kitchen_scenario_windows_a7 - - .kitchen_test_process_agent diff --git a/.gitlab/kitchen_tests_upload.yml b/.gitlab/kitchen_tests_upload.yml deleted file mode 100644 index ef2982f377c00c..00000000000000 --- a/.gitlab/kitchen_tests_upload.yml +++ /dev/null @@ -1,85 +0,0 @@ -kitchen_tests_upload_common: - stage: kitchen_tests_upload - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - allow_failure: true - rules: - - !reference [.except_mergequeue] - - when: always - dependencies: - - kitchen_amazonlinux_install_script_agent-a6_arm64 - - kitchen_amazonlinux_install_script_agent-a6_x64 - - kitchen_amazonlinux_install_script_agent-a7_arm64 - - kitchen_amazonlinux_install_script_agent-a7_x64 - - kitchen_centos_fips_install_script_agent-a6 - - kitchen_centos_fips_install_script_agent-a7 - - kitchen_centos_fips_install_script_dogstatsd-a7 - - kitchen_centos_fips_install_script_iot_agent-a7 - - kitchen_centos_fips_upgrade6_agent-a6 - - kitchen_centos_fips_upgrade6_agent-a7 - - kitchen_centos_fips_upgrade7_agent-a7 - - kitchen_centos_fips_upgrade7_iot_agent-a7 - - kitchen_centos_install_script_agent-a6 - - kitchen_centos_install_script_agent-a7 - - kitchen_centos_install_script_dogstatsd-a7 - - kitchen_centos_install_script_iot_agent-a7 - - kitchen_centos_process_agent-a7 - - kitchen_centos_upgrade5_agent-a6 - - kitchen_centos_upgrade5_agent-a7 - - kitchen_centos_upgrade6_agent-a6 - - kitchen_centos_upgrade6_agent-a7 - - kitchen_centos_upgrade7_agent-a7 - - kitchen_centos_upgrade7_iot_agent-a7 - - kitchen_debian_install_script_agent-a6_arm64 - - kitchen_debian_install_script_agent-a6_x64 - - kitchen_debian_install_script_heroku_agent-a6 - - kitchen_debian_process_agent-a7 - - kitchen_debian_upgrade5_agent-a6 - - kitchen_debian_upgrade5_agent-a7 - - kitchen_debian_upgrade6_agent-a6 - - kitchen_debian_upgrade6_agent-a7 - - kitchen_debian_upgrade7_agent-a7 - - kitchen_debian_upgrade7_iot_agent-a7 - - kitchen_suse_install_script_agent_x64-a6 - - kitchen_suse_install_script_agent_x64-a7 - - kitchen_suse_install_script_dogstatsd_x64-a7 - - kitchen_suse_install_script_iot_agent_x64-a7 - - kitchen_suse_install_script_agent_arm64-a7 - - kitchen_suse_process_agent_x64-a7 - - kitchen_suse_upgrade6_agent_x64-a6 - - kitchen_suse_upgrade6_agent_x64-a7 - - kitchen_suse_upgrade7_agent_x64-a7 - - kitchen_suse_upgrade7_iot_agent_x64-a7 - - kitchen_ubuntu_install_script_agent-a6_arm64 - - kitchen_ubuntu_install_script_agent-a6_x64 - - kitchen_ubuntu_install_script_agent-a7_arm64 - - kitchen_ubuntu_install_script_agent-a7_x64 - - kitchen_ubuntu_install_script_dogstatsd-a7 - - kitchen_ubuntu_install_script_iot_agent-a7 - - kitchen_ubuntu_process_agent-a7 - - kitchen_ubuntu_upgrade5_agent-a6 - - kitchen_ubuntu_upgrade5_agent-a7 - - kitchen_ubuntu_upgrade6_agent-a6 - - kitchen_ubuntu_upgrade6_agent-a7 - - kitchen_ubuntu_upgrade7_agent-a7 - - kitchen_ubuntu_upgrade7_iot_agent-a7 - - kitchen_windows_chef_agent-a6 - - kitchen_windows_chef_agent-a7 - - kitchen_windows_installer_agent-a6 - - kitchen_windows_installer_agent-a7 - - kitchen_windows_installer_npm_driver-a7 - - kitchen_windows_installer_npm_install_scenarios-a7 - - kitchen_windows_process_agent-a7 - - kitchen_windows_upgrade5_agent-a6 - - kitchen_windows_upgrade5_agent-a7 - - kitchen_windows_upgrade6_agent-a6 - - kitchen_windows_upgrade6_agent-a7 - - kitchen_windows_upgrade7_agent-a7 - variables: - DD_ENV: ci - script: - - set +x - - find . -name "kitchen-rspec-common-*.tar.gz" - - export DATADOG_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text) - - set -x - - find . -maxdepth 1 -type f -name "kitchen-rspec-common-*.tar.gz" -exec inv -e junit-upload --tgz-path {} \; diff --git a/.gitlab/maintenance_jobs.yml b/.gitlab/maintenance_jobs.yml deleted file mode 100644 index dfc155470c9bd9..00000000000000 --- a/.gitlab/maintenance_jobs.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# maintenance_jobs stage -# Contains utility jobs to manipulate Docker repositories (Dockerhub & GCR) -# as well as jobs which periodically clean up kitchen resources. - -include: - - /.gitlab/maintenance_jobs/docker.yml - - /.gitlab/maintenance_jobs/kitchen.yml diff --git a/.gitlab/maintenance_jobs/docker.yml b/.gitlab/maintenance_jobs/docker.yml deleted file mode 100644 index 3c6db71714f3cd..00000000000000 --- a/.gitlab/maintenance_jobs/docker.yml +++ /dev/null @@ -1,75 +0,0 @@ ---- -include: - - /.gitlab/docker_common/publish_job_templates.yml - -# -# Use these steps to revert the latest tags to a previous release -# while maintaining content trust signatures -# - Create a pipeline on main with the RELEASE_6 and/or RELEASE_7 env vars -# - in the gitlab pipeline view, trigger the step (in the first column) -# -revert_latest_6: - extends: .docker_publish_job_definition - rules: - !reference [.on_main_manual] - stage: maintenance_jobs - variables: - NEW_LATEST_RELEASE_6: "" # tag name of the non-jmx version, for example "6.21.0" - IMG_REGISTRIES: public - parallel: - matrix: - - IMG_SOURCES: datadog/agent:${NEW_LATEST_RELEASE_6} - IMG_DESTINATIONS: agent:6,agent:latest-py2 - - IMG_SOURCES: datadog/agent:${NEW_LATEST_RELEASE_6}-jmx - IMG_DESTINATIONS: agent:6-jmx,agent:latest-py2-jmx - -revert_latest_7: - extends: .docker_publish_job_definition - rules: - !reference [.on_main_manual] - stage: maintenance_jobs - variables: - NEW_LATEST_RELEASE_7: "" # tag name of the non-jmx version, for example "7.21.0" - IMG_REGISTRIES: public - parallel: - matrix: - - IMG_SOURCES: datadog/agent:${NEW_LATEST_RELEASE_7} - IMG_DESTINATIONS: agent:7,agent:latest - - IMG_SOURCES: datadog/agent:${NEW_LATEST_RELEASE_7}-jmx - IMG_DESTINATIONS: agent:7-jmx,agent:latest-jmx - - IMG_SOURCES: datadog/agent:${NEW_LATEST_RELEASE_7}-servercore - IMG_DESTINATIONS: agent:7-servercore,agent:latest-servercore - - IMG_SOURCES: datadog/agent:${NEW_LATEST_RELEASE_7}-servercore-jmx - IMG_DESTINATIONS: agent:7-servercore-jmx,agent:latest-servercore-jmx - - IMG_SOURCES: datadog/dogstatsd:${NEW_LATEST_RELEASE_7} - IMG_DESTINATIONS: dogstatsd:7,dogstatsd:latest - - IMG_SOURCES: datadog/cluster-agent:${NEW_LATEST_RELEASE_7} - IMG_DESTINATIONS: cluster-agent:latest - -# -# Use this step to delete a tag of a given image -# We call the Docker Hub API because docker cli doesn't support deleting tags -# - Run a pipeline on main with the IMAGE and TAG env vars -# - in the gitlab pipeline view, trigger the step (in the first column) -delete_docker_tag: - rules: - !reference [.on_main_manual] - stage: maintenance_jobs - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker-notary:v2718650-9ce6565-0.6.1-py3 - tags: ["arch:amd64"] - dependencies: [] - variables: - IMAGE: "" # image name, for example "agent" - TAG: "" # tag name, for example "6.9.0" - ORGANIZATION: "datadog" - before_script: - - DOCKER_REGISTRY_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_LOGIN_SSM_KEY --with-decryption --query "Parameter.Value" --out text) - - PASS=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_PWD_SSM_KEY --with-decryption --query "Parameter.Value" --out text) - - !reference [.setup_python_mirror_linux] - - python3 -m pip install -r requirements.txt - - | - export DOCKER_TOKEN=`curl -s -H "Content-Type: application/json" -X POST -d '{"username": "'$DOCKER_REGISTRY_LOGIN'", "password": "'$PASS'"}' https://hub.docker.com/v2/users/login/ | python -c 'import sys, json; print(json.load(sys.stdin)["token"].strip())'` - script: - - if [[ -z "$IMAGE" ]]; then echo "Need an image"; exit 1; fi - - if [[ -z "$TAG" ]]; then echo "Need a tag to delete"; exit 1; fi - - inv -e docker.delete ${ORGANIZATION} ${IMAGE} ${TAG} ${DOCKER_TOKEN} &>/dev/null diff --git a/.gitlab/maintenance_jobs/kitchen.yml b/.gitlab/maintenance_jobs/kitchen.yml deleted file mode 100644 index cf1b9140b6423c..00000000000000 --- a/.gitlab/maintenance_jobs/kitchen.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -# Once a day, before the nightly build, cleans up the artifacts used during kitchen tests which might have been left over -# This can happen when a kitchen test fails and is never retried, since that pipeline's cleanup job won't run -periodic_kitchen_cleanup_s3: - stage: maintenance_jobs - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - rules: !reference [.on_testing_cleanup] - script: - - MAX_AGE_HOURS=24 BUCKET_NAME=$DEB_TESTING_S3_BUCKET BUCKET_PREFIX=pool python3 /deploy_scripts/cleanup_s3.py - - MAX_AGE_HOURS=24 BUCKET_NAME=$DEB_TESTING_S3_BUCKET BUCKET_PREFIX=dists python3 /deploy_scripts/cleanup_s3.py - - MAX_AGE_HOURS=24 BUCKET_NAME=$RPM_TESTING_S3_BUCKET BUCKET_PREFIX=testing/ python3 /deploy_scripts/cleanup_s3.py - - MAX_AGE_HOURS=24 BUCKET_NAME=$RPM_TESTING_S3_BUCKET BUCKET_PREFIX=suse/testing/ python3 /deploy_scripts/cleanup_s3.py - - MAX_AGE_HOURS=24 BUCKET_NAME=$WIN_S3_BUCKET BUCKET_PREFIX=pipelines/A6/ python3 /deploy_scripts/cleanup_s3.py - - MAX_AGE_HOURS=24 BUCKET_NAME=$WIN_S3_BUCKET BUCKET_PREFIX=pipelines/A7/ python3 /deploy_scripts/cleanup_s3.py - -# Kills any VMs that might have been left over by kitchen -# The script only deletes VMs that have been there for >= 4 hours, which is more than the time limit -# for Gitlab jobs (2 hours), so this should never remove a live kitchen test. -periodic_kitchen_cleanup_azure: - stage: maintenance_jobs - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - rules: !reference [.on_main_or_testing_cleanup] - # Note: We're not sure if the cleanup script is safe if run multiple times concurrently, so we limit - # the job to be run one at a time. - resource_group: azure_cleanup - script: - - export ARM_SUBSCRIPTION_ID=`aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.azure_kitchen_subscription_id --with-decryption --query "Parameter.Value" --out text` - - export ARM_CLIENT_ID=`aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.azure_kitchen_client_id --with-decryption --query "Parameter.Value" --out text` - - export ARM_CLIENT_SECRET=`aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.azure_kitchen_client_secret --with-decryption --query "Parameter.Value" --out text` - - export ARM_TENANT_ID=`aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.azure_kitchen_tenant_id --with-decryption --query "Parameter.Value" --out text` - # Remove kitchen resources for all existing test suite prefixes - - RESOURCE_GROUP_PREFIX=kitchen-chef python3 /deploy_scripts/cleanup_azure.py - - RESOURCE_GROUP_PREFIX=kitchen-install-script python3 /deploy_scripts/cleanup_azure.py - - RESOURCE_GROUP_PREFIX=kitchen-upgrade python3 /deploy_scripts/cleanup_azure.py - - RESOURCE_GROUP_PREFIX=kitchen-step-by-step python3 /deploy_scripts/cleanup_azure.py - - RESOURCE_GROUP_PREFIX=kitchen-win python3 /deploy_scripts/cleanup_azure.py - - RESOURCE_GROUP_PREFIX=kitchen-security-agent python3 /deploy_scripts/cleanup_azure.py - - RESOURCE_GROUP_PREFIX=kitchen-system-probe python3 /deploy_scripts/cleanup_azure.py - - RESOURCE_GROUP_PREFIX=kitchen-process-collection python3 /deploy_scripts/cleanup_azure.py - - RESOURCE_GROUP_PREFIX=kitchen-container-collection python3 /deploy_scripts/cleanup_azure.py - -periodic_kitchen_cleanup_ec2: - stage: maintenance_jobs - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS - tags: ["arch:amd64"] - rules: !reference [.on_main_or_testing_cleanup] - script: - # Only run aws ec2 terminate-instances if $ZOMBIES is not empty, otherwise the command fails - # https://docs.aws.amazon.com/cli/latest/reference/ec2/terminate-instances.html#options - - export AWS_DEFAULT_REGION=us-east-1 - - python3 /deploy_scripts/cleanup_ec2.py diff --git a/.gitlab/new-e2e_common/testing.yml b/.gitlab/new-e2e_common/testing.yml deleted file mode 100644 index efacd2d46d2e09..00000000000000 --- a/.gitlab/new-e2e_common/testing.yml +++ /dev/null @@ -1,5 +0,0 @@ -.new-e2e_agent_a7: - rules: - !reference [.on_kitchen_tests_a7] #TODO: Change when migration is complete to another name without 'kitchen' - variables: - AGENT_MAJOR_VERSION: 7 diff --git a/.gitlab/new-e2e_testing.yml b/.gitlab/new-e2e_testing.yml deleted file mode 100644 index 5123c566b4c7fe..00000000000000 --- a/.gitlab/new-e2e_testing.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# new-e2e_testing stage -# Contains jobs which run kitchen tests on the Agent packages. - -include: - - /.gitlab/new-e2e_testing/debian.yml diff --git a/.gitlab/new-e2e_testing/debian.yml b/.gitlab/new-e2e_testing/debian.yml deleted file mode 100644 index 5c4094de0e638b..00000000000000 --- a/.gitlab/new-e2e_testing/debian.yml +++ /dev/null @@ -1,89 +0,0 @@ - -.new-e2e_os_debian: - variables: - E2E_PLATFORM: debian - -.new-e2e_debian_a7_x64: - variables: - E2E_OSVERS: "debian-9,debian-10,debian-11,debian-12" - E2E_CWS_SUPPORTED_OSVERS: "debian-10,debian-11" - E2E_BRANCH_OSVERS: "debian-11" - needs: ["deploy_deb_testing-a7_x64"] - -.new-e2e_debian_a7_arm64: - variables: - E2E_OSVERS: "debian-10" - E2E_CWS_SUPPORTED_OSVERS: "debian-10" - E2E_BRANCH_OSVERS: "debian-10" - needs: ["deploy_deb_testing-a7_arm64"] - -new-e2e-agent-platform-install-script-debian-a7-x64: - stage: kitchen_testing - extends: - - .new_e2e_template - - .new-e2e_os_debian - - .new-e2e_debian_a7_x64 - - .new-e2e_agent_a7 - rules: - !reference [.on_default_new-e2e_tests_a7] - variables: - TARGETS: ./tests/agent-platform/install-script - TEAM: agent-platform - EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --cws-supported-osversion $E2E_CWS_SUPPORTED_OSVERS --arch x86_64 - -new-e2e-agent-platform-install-script-debian-a7-arm64: - stage: kitchen_testing - extends: - - .new_e2e_template - - .new-e2e_os_debian - - .new-e2e_debian_a7_arm64 - - .new-e2e_agent_a7 - rules: - !reference [.on_all_new-e2e_tests_a7] - variables: - TARGETS: ./tests/agent-platform/install-script - TEAM: agent-platform - EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --cws-supported-osversion $E2E_CWS_SUPPORTED_OSVERS --arch arm64 - -new-e2e-agent-platform-install-script-debian-iot-agent-a7-x86_64: - stage: kitchen_testing - extends: - - .new_e2e_template - - .new-e2e_os_debian - - .new-e2e_debian_a7_x64 - - .new-e2e_agent_a7 - rules: - !reference [.on_default_new-e2e_tests_a7] - variables: - TARGETS: ./tests/agent-platform/install-script - TEAM: agent-platform - EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --cws-supported-osversion $E2E_CWS_SUPPORTED_OSVERS --arch x86_64 --flavor datadog-iot-agent - -new-e2e-agent-platform-install-script-debian-dogstatsd-a7-x86_64: - stage: kitchen_testing - extends: - - .new_e2e_template - - .new-e2e_os_debian - - .new-e2e_debian_a7_x64 - - .new-e2e_agent_a7 - variables: - TARGETS: ./tests/agent-platform/install-script - TEAM: agent-platform - EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --cws-supported-osversion $E2E_CWS_SUPPORTED_OSVERS --arch x86_64 --flavor datadog-dogstatsd - -new-e2e-agent-platform-install-script-debian-heroku-agent-a7-x86_64: - stage: kitchen_testing - extends: - - .new_e2e_template - - .new-e2e_os_debian - - .new-e2e_debian_a7_x64 - - .new-e2e_agent_a7 - variables: - TARGETS: ./tests/agent-platform/install-script - TEAM: agent-platform - EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --cws-supported-osversion $E2E_CWS_SUPPORTED_OSVERS --arch x86_64 --flavor datadog-heroku-agent - - - - - diff --git a/.gitlab/notify.yml b/.gitlab/notify.yml deleted file mode 100644 index 5f4453ec71234e..00000000000000 --- a/.gitlab/notify.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- -# notify stage -# Contains jobs which send slack notifications depending on pipeline status. - -include: - - https://gitlab-templates.ddbuild.io/slack-notifier/sdm/template.yml - -notify-on-tagged-success: - extends: .slack-notifier-base - stage: notify - rules: !reference [.on_deploy_stable_or_beta_repo_branch] - dependencies: [] - tags: ["arch:amd64"] - script: | - MESSAGE_TEXT=":host-green: Tagged build <$CI_PIPELINE_URL|$CI_PIPELINE_ID> succeeded. - *$CI_COMMIT_REF_NAME* is available in the staging repositories." - postmessage "#agent-release-sync" "$MESSAGE_TEXT" - -notify: - extends: .slack-notifier-base - stage: notify - rules: !reference [.on_main_or_release_branch_or_deploy_always] - dependencies: [] - tags: ["arch:amd64"] - script: - - set +x - - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_read_api_token --with-decryption --query "Parameter.Value" --out text) - - !reference [.setup_python_mirror_linux] - - python3 -m pip install -r tasks/libs/requirements-notifications.txt - - | - # Do not send notifications if this is a child pipeline of another repo - # The triggering repo should already have its own notification system - if [ "$CI_PIPELINE_SOURCE" != "pipeline" ]; then - if [ "$DEPLOY_AGENT" = "true" ]; then - invoke -e pipeline.notify --notification-type "deploy" - else - invoke -e pipeline.notify --notification-type "merge" - fi - else - echo "This pipeline was triggered by another repository, skipping notification." - fi - -send_pipeline_stats: - stage: notify - # Using a buildimage image with python 3.7+, datadog-api-client and invoke installed - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - when: always - dependencies: [] - script: - - source /root/.bashrc - - set +x - - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_read_api_token --with-decryption --query "Parameter.Value" --out text) - - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text) - - !reference [.setup_python_mirror_linux] - - invoke -e pipeline.send-stats diff --git a/.gitlab/package_build.yml b/.gitlab/package_build.yml deleted file mode 100644 index 858237193d18a6..00000000000000 --- a/.gitlab/package_build.yml +++ /dev/null @@ -1,17 +0,0 @@ -# package_build stage -# Contains jobs which build the Agent packages with omnibus. - ---- -.upload_sbom_artifacts: - - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/version-manifest.json $S3_SBOM_STORAGE_URI/$CI_JOB_NAME/version-manifest.json - -.upload_sbom_artifacts_windows: - - Invoke-Expression "$S3_CP_CMD omnibus\pkg\version-manifest.json $S3_SBOM_STORAGE_URI/$CI_JOB_NAME/version-manifest.json" - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - -include: - - /.gitlab/package_build/deb.yml - - /.gitlab/package_build/dmg.yml - - /.gitlab/package_build/rpm.yml - - /.gitlab/package_build/suse_rpm.yml - - /.gitlab/package_build/windows.yml diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml deleted file mode 100644 index a4c7d11c855d36..00000000000000 --- a/.gitlab/package_build/deb.yml +++ /dev/null @@ -1,242 +0,0 @@ ---- -.setup_deb_signing_key: &setup_deb_signing_key - - set +x - - DEB_GPG_KEY=$(aws ssm get-parameter --region us-east-1 --name $DEB_GPG_KEY_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - printf -- "${DEB_GPG_KEY}" | gpg --import --batch - - export DEB_SIGNING_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name $DEB_SIGNING_PASSPHRASE_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - set -x - -.agent_build_common_deb: - script: - - source /root/.bashrc - - !reference [.setup_ruby_mirror_linux] - - !reference [.setup_python_mirror_linux] - - !reference [.retrieve_linux_go_deps] - - echo "About to build for $RELEASE_VERSION" - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR/* - # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. - # Thus, we move the artifacts at the end in a gitlab-friendly dir. - - *setup_deb_signing_key - - tar -xf $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz - - mkdir -p /tmp/system-probe - - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/clang-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/clang-bpf - - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/llc-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/llc-bpf - - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz /tmp/system-probe/minimized-btfs.tar.xz - - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf - - inv -e agent.omnibus-build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe ${FLAVOR:+--flavor $FLAVOR} - - ls -la $OMNIBUS_PACKAGE_DIR - - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog${FLAVOR:+-$FLAVOR}-agent_*_${PACKAGE_ARCH}.deb $S3_ARTIFACTS_URI/$DESTINATION_DEB - - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog${FLAVOR:+-$FLAVOR}-agent-dbg_*_${PACKAGE_ARCH}.deb $S3_ARTIFACTS_URI/$DESTINATION_DBG_DEB - - !reference [.upload_sbom_artifacts] - variables: - KUBERNETES_MEMORY_REQUEST: "32Gi" - KUBERNETES_MEMORY_LIMIT: "32Gi" - KUBERNETES_CPU_REQUEST: 16 - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR - -agent_deb-x64-a6: - extends: .agent_build_common_deb - rules: - !reference [.on_a6] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] - variables: - AGENT_MAJOR_VERSION: 6 - PYTHON_RUNTIMES: '2,3' - PACKAGE_ARCH: amd64 - DESTINATION_DEB: 'datadog-agent_6_amd64.deb' - DESTINATION_DBG_DEB: 'datadog-agent-dbg_6_amd64.deb' - before_script: - - export RELEASE_VERSION=$RELEASE_VERSION_6 - -agent_deb-x64-a7: - extends: .agent_build_common_deb - rules: - !reference [.on_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] - variables: - AGENT_MAJOR_VERSION: 7 - PYTHON_RUNTIMES: '3' - PACKAGE_ARCH: amd64 - DESTINATION_DEB: 'datadog-agent_7_amd64.deb' - DESTINATION_DBG_DEB: 'datadog-agent-dbg_7_amd64.deb' - before_script: - - export RELEASE_VERSION=$RELEASE_VERSION_7 - -agent_deb-arm64-a6: - extends: .agent_build_common_deb - rules: - !reference [.on_all_builds_a6] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_mod_tidy_check", "build_system-probe-arm64", "go_deps", "generate_minimized_btfs_arm64"] - variables: - AGENT_MAJOR_VERSION: 6 - PYTHON_RUNTIMES: '2,3' - PACKAGE_ARCH: arm64 - DESTINATION_DEB: 'datadog-agent_6_arm64.deb' - DESTINATION_DBG_DEB: 'datadog-agent-dbg_6_arm64.deb' - before_script: - - export RELEASE_VERSION=$RELEASE_VERSION_6 - -agent_deb-arm64-a7: - extends: .agent_build_common_deb - rules: - !reference [.on_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_mod_tidy_check", "build_system-probe-arm64", "go_deps", "generate_minimized_btfs_arm64"] - variables: - AGENT_MAJOR_VERSION: 7 - PYTHON_RUNTIMES: '3' - PACKAGE_ARCH: arm64 - DESTINATION_DEB: 'datadog-agent_7_arm64.deb' - DESTINATION_DBG_DEB: 'datadog-agent-dbg_7_arm64.deb' - before_script: - - export RELEASE_VERSION=$RELEASE_VERSION_7 - -.iot_agent_build_common_deb: - script: - - source /root/.bashrc - - !reference [.setup_ruby_mirror_linux] - - !reference [.setup_python_mirror_linux] - - !reference [.retrieve_linux_go_deps] - - echo "About to build for $RELEASE_VERSION_7" - - echo "Detected host architecture $(uname -m)" - # $DD_TARGET_ARCH is only set by Arm build images, so assume amd64 if not present - - echo "Target architecture ${DD_TARGET_ARCH:=amd64}" - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR/* - # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. - # Thus, we move the artifacts at the end in a gitlab-friendly dir. - - *setup_deb_signing_key - # Use --skip-deps since the deps are installed by `before_script`. - - inv -e agent.omnibus-build --flavor iot --log-level debug --release-version "$RELEASE_VERSION_7" --major-version 7 --base-dir $OMNIBUS_BASE_DIR --skip-deps --go-mod-cache="$GOPATH/pkg/mod" - - ls -la $OMNIBUS_PACKAGE_DIR - - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-iot-agent*_${PACKAGE_ARCH}.deb $S3_ARTIFACTS_URI/$DESTINATION_DEB - - !reference [.upload_sbom_artifacts] - variables: - KUBERNETES_CPU_REQUEST: 8 - KUBERNETES_MEMORY_REQUEST: "16Gi" - KUBERNETES_MEMORY_LIMIT: "16Gi" - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR - -iot_agent_deb-x64: - extends: .iot_agent_build_common_deb - rules: - !reference [.on_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "go_deps"] - variables: - PACKAGE_ARCH: amd64 - DESTINATION_DEB: 'datadog-iot-agent_7_amd64.deb' - -iot_agent_deb-arm64: - extends: .iot_agent_build_common_deb - rules: - !reference [.on_all_builds_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_mod_tidy_check", "go_deps"] - variables: - PACKAGE_ARCH: arm64 - DESTINATION_DEB: 'datadog-iot-agent_7_arm64.deb' - -iot_agent_deb-armhf: - extends: .iot_agent_build_common_deb - rules: - !reference [.on_all_builds_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_armhf$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - # Run with platform:arm64 since no platform:armhf exists and arm64 should be backwards compatible - tags: ["arch:arm64"] - needs: ["go_mod_tidy_check", "go_deps"] - variables: - PACKAGE_ARCH: armhf - DESTINATION_DEB: 'datadog-iot-agent_7_armhf.deb' - -dogstatsd_deb-x64: - rules: - !reference [.on_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"] - variables: - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - script: - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR/* - - !reference [.setup_ruby_mirror_linux] - # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. - # Thus, we move the artifacts at the end in a gitlab-friendly dir. - - *setup_deb_signing_key - # Use --skip-deps since the deps are installed by `before_script`. - - inv -e dogstatsd.omnibus-build --release-version "$RELEASE_VERSION_7" --major-version 7 --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" - - ls -la $OMNIBUS_PACKAGE_DIR - - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd*_amd64.deb $S3_ARTIFACTS_URI/datadog-dogstatsd_amd64.deb - - !reference [.upload_sbom_artifacts] - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR - -dogstatsd_deb-arm64: - rules: - !reference [.on_all_builds_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_mod_tidy_check", "build_dogstatsd-binary_arm64", "go_deps"] - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - script: - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR/* - - !reference [.setup_ruby_mirror_linux] - # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. - # Thus, we move the artifacts at the end in a gitlab-friendly dir. - - *setup_deb_signing_key - # Use --skip-deps since the deps are installed by `before_script`. - - inv -e dogstatsd.omnibus-build --release-version "$RELEASE_VERSION_7" --major-version 7 --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" - - ls -la $OMNIBUS_PACKAGE_DIR - - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd*_arm64.deb $S3_ARTIFACTS_URI/datadog-dogstatsd_arm64.deb - - !reference [.upload_sbom_artifacts] - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR - -agent_heroku_deb-x64-a6: - extends: agent_deb-x64-a6 - variables: - DESTINATION_DEB: 'datadog-heroku-agent_6_amd64.deb' - DESTINATION_DBG_DEB: 'datadog-heroku-agent-dbg_6_amd64.deb' - FLAVOR: heroku - -agent_heroku_deb-x64-a7: - extends: agent_deb-x64-a7 - variables: - DESTINATION_DEB: 'datadog-heroku-agent_7_amd64.deb' - DESTINATION_DBG_DEB: 'datadog-heroku-agent-dbg_7_amd64.deb' - FLAVOR: heroku diff --git a/.gitlab/package_build/dmg.yml b/.gitlab/package_build/dmg.yml deleted file mode 100644 index 71767ae3ce82ab..00000000000000 --- a/.gitlab/package_build/dmg.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -.agent_build_common_dmg: - script: - - echo "About to build for $RELEASE_VERSION" - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR/* - - mkdir -p $OMNIBUS_PACKAGE_DIR - - export GITHUB_KEY_B64=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_key_b64 --with-decryption --query "Parameter.Value" --out text) - - export GITHUB_APP_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_app_id --with-decryption --query "Parameter.Value" --out text) - - export GITHUB_INSTALLATION_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_installation_id --with-decryption --query "Parameter.Value" --out text) - - $S3_CP_CMD $S3_ARTIFACTS_URI/agent-version.cache . - - export VERSION_CACHE_CONTENT=$(cat agent-version.cache | base64 -) - - !reference [.setup_python_mirror_linux] - - python3 -m pip install -r tasks/libs/requirements-github.txt - - inv -e github.trigger-macos-build --datadog-agent-ref "$CI_COMMIT_SHA" --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --destination "$OMNIBUS_PACKAGE_DIR" --version-cache "$VERSION_CACHE_CONTENT" - - !reference [.upload_sbom_artifacts] - timeout: 3h # MacOS builds can take 1h~2h, increase the timeout to avoid timeout flakes - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR - -agent_dmg-x64-a7: - extends: .agent_build_common_dmg - rules: - !reference [.on_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check"] - variables: - AGENT_MAJOR_VERSION: 7 - PYTHON_RUNTIMES: '3' - before_script: - - source /root/.bashrc - - export RELEASE_VERSION=$RELEASE_VERSION_7 diff --git a/.gitlab/package_build/rpm.yml b/.gitlab/package_build/rpm.yml deleted file mode 100644 index a3837afd13b64c..00000000000000 --- a/.gitlab/package_build/rpm.yml +++ /dev/null @@ -1,196 +0,0 @@ ---- -.agent_build_common_rpm: - script: - - echo "About to build for $RELEASE_VERSION" - - !reference [.setup_ruby_mirror_linux] - - !reference [.setup_python_mirror_linux] - - !reference [.retrieve_linux_go_deps] - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR/* - # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. - # Thus, we move the artifacts at the end in a gitlab-friendly dir. - - set +x - - RPM_GPG_KEY=$(aws ssm get-parameter --region us-east-1 --name $RPM_GPG_KEY_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - printf -- "$RPM_GPG_KEY" | gpg --import --batch - - export RPM_SIGNING_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name $RPM_SIGNING_PASSPHRASE_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - set -x - - tar -xf $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz - - mkdir -p /tmp/system-probe - - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/clang-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/clang-bpf - - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/llc-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/llc-bpf - - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz /tmp/system-probe/minimized-btfs.tar.xz - - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf - - inv -e agent.omnibus-build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe - - ls -la $OMNIBUS_PACKAGE_DIR - - !reference [.upload_sbom_artifacts] - variables: - KUBERNETES_CPU_REQUEST: 16 - KUBERNETES_MEMORY_REQUEST: "32Gi" - KUBERNETES_MEMORY_LIMIT: "32Gi" - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR - -# build Agent package for rpm-x64 -agent_rpm-x64-a6: - extends: .agent_build_common_rpm - rules: - !reference [.on_a6] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] - variables: - AGENT_MAJOR_VERSION: 6 - PYTHON_RUNTIMES: '2,3' - PACKAGE_ARCH: amd64 - before_script: - - source /root/.bashrc - - export RELEASE_VERSION=$RELEASE_VERSION_6 - -# build Agent package for rpm-x64 -agent_rpm-x64-a7: - extends: .agent_build_common_rpm - rules: - !reference [.on_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] - variables: - AGENT_MAJOR_VERSION: 7 - PYTHON_RUNTIMES: '3' - PACKAGE_ARCH: amd64 - before_script: - - source /root/.bashrc - - export RELEASE_VERSION=$RELEASE_VERSION_7 - -# build Agent package for rpm-arm64 -agent_rpm-arm64-a6: - extends: .agent_build_common_rpm - rules: - !reference [.on_all_builds_a6] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_mod_tidy_check", "build_system-probe-arm64", "go_deps", "generate_minimized_btfs_arm64"] - variables: - AGENT_MAJOR_VERSION: 6 - PYTHON_RUNTIMES: '2,3' - PACKAGE_ARCH: arm64 - before_script: - - source /root/.bashrc - - export RELEASE_VERSION=$RELEASE_VERSION_6 - -# build Agent package for rpm-arm64 -agent_rpm-arm64-a7: - extends: .agent_build_common_rpm - rules: - !reference [.on_all_builds_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_mod_tidy_check", "build_system-probe-arm64", "go_deps", "generate_minimized_btfs_arm64"] - variables: - AGENT_MAJOR_VERSION: 7 - PYTHON_RUNTIMES: '3' - PACKAGE_ARCH: arm64 - before_script: - - source /root/.bashrc - - export RELEASE_VERSION=$RELEASE_VERSION_7 - -.iot_agent_build_common_rpm: - script: - - echo "About to build iot agent for $RELEASE_VERSION_7" - - source /root/.bashrc - - !reference [.setup_ruby_mirror_linux] - - !reference [.setup_python_mirror_linux] - - !reference [.retrieve_linux_go_deps] - - echo "Detected host architecture $(uname -m)" - # $DD_TARGET_ARCH is only set by Arm build images, so assume amd64 if not present - - echo "Target architecture ${DD_TARGET_ARCH:=amd64}" - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR/* - # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. - # Thus, we move the artifacts at the end in a gitlab-friendly dir. - - set +x - - RPM_GPG_KEY=$(aws ssm get-parameter --region us-east-1 --name $RPM_GPG_KEY_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - printf -- "$RPM_GPG_KEY" | gpg --import --batch - - export RPM_SIGNING_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name $RPM_SIGNING_PASSPHRASE_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - set -x - # Use --skip-deps since the deps are installed by `before_script`. - - inv -e agent.omnibus-build --flavor iot --log-level debug --release-version "$RELEASE_VERSION_7" --major-version 7 --base-dir $OMNIBUS_BASE_DIR --skip-deps --go-mod-cache="$GOPATH/pkg/mod" - - ls -la $OMNIBUS_PACKAGE_DIR - - !reference [.upload_sbom_artifacts] - variables: - KUBERNETES_CPU_REQUEST: 8 - KUBERNETES_MEMORY_REQUEST: "16Gi" - KUBERNETES_MEMORY_LIMIT: "16Gi" - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR - -iot_agent_rpm-x64: - extends: .iot_agent_build_common_rpm - rules: - !reference [.on_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "go_deps"] - -iot_agent_rpm-arm64: - extends: .iot_agent_build_common_rpm - rules: - !reference [.on_all_builds_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_mod_tidy_check", "go_deps"] - -iot_agent_rpm-armhf: - extends: .iot_agent_build_common_rpm - rules: - !reference [.on_all_builds_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_armhf$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - # Run with platform:arm64 since no platform:armhf exists and arm64 should be backwards compatible - tags: ["arch:arm64"] - needs: ["go_mod_tidy_check", "go_deps"] - before_script: - # Ensures uname -m reports armv7l - - export LD_PRELOAD="/usr/local/lib/libfakearmv7l.so" - -dogstatsd_rpm-x64: - rules: - !reference [.on_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"] - variables: - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - script: - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR/* - - !reference [.setup_ruby_mirror_linux] - # Artifacts and cache must live within project directory but we run omnibus - # from the GOPATH (see above). We then call `invoke` passing --base-dir, - # pointing to a gitlab-friendly location. - - set +x - - RPM_GPG_KEY=$(aws ssm get-parameter --region us-east-1 --name $RPM_GPG_KEY_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - printf -- "$RPM_GPG_KEY" | gpg --import --batch - - export RPM_SIGNING_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name $RPM_SIGNING_PASSPHRASE_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - set -x - # Use --skip-deps since the deps are installed by `before_script`. - - inv -e dogstatsd.omnibus-build --release-version "$RELEASE_VERSION_7" --major-version 7 --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" - - ls -la $OMNIBUS_PACKAGE_DIR - - !reference [.upload_sbom_artifacts] - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR diff --git a/.gitlab/package_build/suse_rpm.yml b/.gitlab/package_build/suse_rpm.yml deleted file mode 100644 index 6e295ef1f76e4f..00000000000000 --- a/.gitlab/package_build/suse_rpm.yml +++ /dev/null @@ -1,160 +0,0 @@ ---- -.agent_build_common_suse_rpm: - script: - - echo "About to build for $RELEASE_VERSION" - - !reference [.setup_ruby_mirror_linux] - - !reference [.setup_python_mirror_linux] - - !reference [.retrieve_linux_go_deps] - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR_SUSE/* - # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. - # Thus, we move the artifacts at the end in a gitlab-friendly dir. - - set +x - - RPM_GPG_KEY=$(aws ssm get-parameter --region us-east-1 --name $RPM_GPG_KEY_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - printf -- "$RPM_GPG_KEY" | gpg --import --batch - - export RPM_SIGNING_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name $RPM_SIGNING_PASSPHRASE_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - set -x - - tar -xf $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz - - mkdir -p /tmp/system-probe - - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/clang-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/clang-bpf - - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/llc-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/llc-bpf - - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz /tmp/system-probe/minimized-btfs.tar.xz - - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf - # use --skip-deps since the deps are installed by `before_script` - - inv -e agent.omnibus-build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe ${OMNIBUS_TASK_EXTRA_PARAMS} - - ls -la $OMNIBUS_PACKAGE_DIR - # Copy to a different directory to avoid collisions if a job downloads both the RPM and SUSE RPM artifacts - - mkdir -p $OMNIBUS_PACKAGE_DIR_SUSE && cp $OMNIBUS_PACKAGE_DIR/* $OMNIBUS_PACKAGE_DIR_SUSE - - !reference [.upload_sbom_artifacts] - variables: - KUBERNETES_CPU_REQUEST: 16 - KUBERNETES_MEMORY_REQUEST: "32Gi" - KUBERNETES_MEMORY_LIMIT: "32Gi" - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR_SUSE - -# build Agent package for suse-x64 -agent_suse-x64-a6: - extends: .agent_build_common_suse_rpm - rules: - !reference [.on_a6] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/suse_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] - variables: - AGENT_MAJOR_VERSION: 6 - PYTHON_RUNTIMES: '2,3' - PACKAGE_ARCH: amd64 - before_script: - - source /root/.bashrc - - export RELEASE_VERSION=$RELEASE_VERSION_6 - -# build Agent package for suse-x64 -agent_suse-x64-a7: - extends: .agent_build_common_suse_rpm - rules: - !reference [.on_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/suse_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] - variables: - AGENT_MAJOR_VERSION: 7 - PYTHON_RUNTIMES: '3' - PACKAGE_ARCH: amd64 - before_script: - - source /root/.bashrc - - export RELEASE_VERSION=$RELEASE_VERSION_7 - -# build Agent package for suse-arm64 -# This is a bit hackish and mostly mimics the CentOS7/arm64 build -agent_suse-arm64-a7: - extends: .agent_build_common_suse_rpm - rules: - !reference [.on_all_builds_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_mod_tidy_check", "build_system-probe-arm64", "go_deps", "generate_minimized_btfs_arm64"] - variables: - AGENT_MAJOR_VERSION: 7 - PYTHON_RUNTIMES: '3' - PACKAGE_ARCH: arm64 - OMNIBUS_TASK_EXTRA_PARAMS: '--host-distribution=suse' - before_script: - - source /root/.bashrc - - export RELEASE_VERSION=$RELEASE_VERSION_7 - -iot_agent_suse-x64: - rules: - !reference [.on_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/suse_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "go_deps"] - before_script: - - source /root/.bashrc - script: - - echo "About to build iot agent for $RELEASE_VERSION_7" - - !reference [.setup_ruby_mirror_linux] - - !reference [.setup_python_mirror_linux] - - !reference [.retrieve_linux_go_deps] - - echo "Detected host architecture $(uname -m)" - # $DD_TARGET_ARCH is only set by Arm build images, so assume amd64 if not present - - echo "Target architecture ${DD_TARGET_ARCH:=amd64}" - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR_SUSE/* - # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. - # Thus, we move the artifacts at the end in a gitlab-friendly dir. - - set +x - - RPM_GPG_KEY=$(aws ssm get-parameter --region us-east-1 --name $RPM_GPG_KEY_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - printf -- "$RPM_GPG_KEY" | gpg --import --batch - - export RPM_SIGNING_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name $RPM_SIGNING_PASSPHRASE_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - set -x - # Use --skip-deps since the deps are installed by `before_script`. - - inv -e agent.omnibus-build --flavor iot --log-level debug --release-version "$RELEASE_VERSION_7" --major-version 7 --base-dir $OMNIBUS_BASE_DIR --skip-deps --go-mod-cache="$GOPATH/pkg/mod" - - ls -la $OMNIBUS_PACKAGE_DIR - # Copy to a different directory to avoid collisions if a job downloads both the RPM and SUSE RPM artifacts - - mkdir -p $OMNIBUS_PACKAGE_DIR_SUSE && cp $OMNIBUS_PACKAGE_DIR/* $OMNIBUS_PACKAGE_DIR_SUSE - - !reference [.upload_sbom_artifacts] - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR_SUSE - -dogstatsd_suse-x64: - rules: - !reference [.on_a7] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/suse_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"] - variables: - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - script: - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR_SUSE/* - - !reference [.setup_ruby_mirror_linux] - # Artifacts and cache must live within project directory but we run omnibus - # from the GOPATH (see above). We then call `invoke` passing --base-dir, - # pointing to a gitlab-friendly location. - - set +x - - RPM_GPG_KEY=$(aws ssm get-parameter --region us-east-1 --name $RPM_GPG_KEY_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - printf -- "$RPM_GPG_KEY" | gpg --import --batch - - export RPM_SIGNING_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name $RPM_SIGNING_PASSPHRASE_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - set -x - # Use --skip-deps since the deps are installed by `before_script`. - - inv -e dogstatsd.omnibus-build --release-version "$RELEASE_VERSION_7" --major-version 7 --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" - - ls -la $OMNIBUS_PACKAGE_DIR - # Copy to a different directory to avoid collisions if a job downloads both the RPM and SUSE RPM artifacts - - mkdir -p $OMNIBUS_PACKAGE_DIR_SUSE && cp $OMNIBUS_PACKAGE_DIR/* $OMNIBUS_PACKAGE_DIR_SUSE - - !reference [.upload_sbom_artifacts] - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR_SUSE diff --git a/.gitlab/package_build/windows.yml b/.gitlab/package_build/windows.yml deleted file mode 100644 index c2a5d18f56bc65..00000000000000 --- a/.gitlab/package_build/windows.yml +++ /dev/null @@ -1,129 +0,0 @@ ---- -.windows_msi_base: - stage: package_build - tags: ["runner:windows-docker", "windowsversion:1809"] - needs: ["go_mod_tidy_check", "go_deps"] - script: - - $ErrorActionPreference = 'Stop' - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - - if (Test-Path omnibus\pkg) { remove-item -recurse -force omnibus\pkg } - - mkdir omnibus\pkg - - !reference [.setup_ruby_mirror_win] - - !reference [.setup_python_mirror_win] - - > - docker run --rm - -m 8192M - -v "$(Get-Location):c:\mnt" - -e CI_JOB_ID=${CI_JOB_ID} - -e CI_PIPELINE_ID=${CI_PIPELINE_ID} - -e CI_PROJECT_NAME=${CI_PROJECT_NAME} - -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} - -e OMNIBUS_TARGET=${OMNIBUS_TARGET} - -e WINDOWS_BUILDER=true - -e RELEASE_VERSION="$RELEASE_VERSION" - -e MAJOR_VERSION="$AGENT_MAJOR_VERSION" - -e PY_RUNTIMES="$PYTHON_RUNTIMES" - -e INTEGRATIONS_CORE_VERSION="$INTEGRATIONS_CORE_VERSION" - -e GOMODCACHE="c:\modcache" - -e AWS_NETWORKING=true - -e SIGN_WINDOWS_DD_WCS=true - -e TARGET_ARCH="$ARCH" - -e DEBUG_CUSTOMACTION="$DEBUG_CUSTOMACTION" - -e BUCKET_BRANCH="$BUCKET_BRANCH" - -e S3_OMNIBUS_CACHE_BUCKET="$S3_OMNIBUS_CACHE_BUCKET" - -e USE_S3_CACHING="$USE_S3_CACHING" - -e INTEGRATION_WHEELS_CACHE_BUCKET="$INTEGRATION_WHEELS_CACHE_BUCKET" - -e GO_VERSION_CHECK="true" - -e BUNDLE_MIRROR__RUBYGEMS__ORG=${BUNDLE_MIRROR__RUBYGEMS__ORG} - -e PIP_INDEX_URL=${PIP_INDEX_URL} - 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\buildwin.bat - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - get-childitem omnibus\pkg - - !reference [.upload_sbom_artifacts_windows] - after_script: - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - artifacts: - expire_in: 2 weeks - paths: - - omnibus/pkg - -.windows_main_agent_base: - extends: .windows_msi_base - variables: - OMNIBUS_TARGET: main - -windows_msi_and_bosh_zip_x64-a7: - extends: .windows_main_agent_base - rules: - !reference [.on_a7] - variables: - ARCH: "x64" - AGENT_MAJOR_VERSION: 7 - PYTHON_RUNTIMES: '3' - before_script: - - set RELEASE_VERSION $RELEASE_VERSION_7 - -windows_msi_x64-a6: - extends: .windows_main_agent_base - rules: - !reference [.on_a6] - variables: - ARCH: "x64" - AGENT_MAJOR_VERSION: 6 - PYTHON_RUNTIMES: '2,3' - before_script: - - set RELEASE_VERSION $RELEASE_VERSION_6 - timeout: 3h - -# cloudfoundry IoT build for Windows -windows_zip_agent_binaries_x64-a7: - rules: - !reference [.on_a7] - stage: package_build - tags: ["runner:windows-docker", "windowsversion:1809"] - needs: ["go_mod_tidy_check", "go_deps"] - variables: - ARCH: "x64" - AGENT_MAJOR_VERSION: 7 - OMNIBUS_TARGET: agent_binaries - before_script: - - set RELEASE_VERSION $RELEASE_VERSION_7 - script: - - $ErrorActionPreference = "Stop" - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - - if (Test-Path omnibus\pkg) { remove-item -recurse -force omnibus\pkg } - - mkdir omnibus\pkg - - !reference [.setup_ruby_mirror_win] - - !reference [.setup_python_mirror_win] - - > - docker run --rm - -m 8192M - -v "$(Get-Location):c:\mnt" - -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} - -e CI_PIPELINE_ID=${CI_PIPELINE_ID} - -e CI_PROJECT_NAME=${CI_PROJECT_NAME} - -e OMNIBUS_TARGET=${OMNIBUS_TARGET} - -e WINDOWS_BUILDER=true - -e RELEASE_VERSION="$RELEASE_VERSION" - -e MAJOR_VERSION="$AGENT_MAJOR_VERSION" - -e INTEGRATIONS_CORE_VERSION="$INTEGRATIONS_CORE_VERSION" - -e PY_RUNTIMES="$PYTHON_RUNTIMES" - -e GOMODCACHE="c:\modcache" - -e AWS_NETWORKING=true - -e SIGN_WINDOWS_DD_WCS=true - -e BUCKET_BRANCH="$BUCKET_BRANCH" - -e INTEGRATION_WHEELS_CACHE_BUCKET="$INTEGRATION_WHEELS_CACHE_BUCKET" - -e S3_OMNIBUS_CACHE_BUCKET="$S3_OMNIBUS_CACHE_BUCKET" - -e USE_S3_CACHING="$USE_S3_CACHING" - -e BUNDLE_MIRROR__RUBYGEMS__ORG=${BUNDLE_MIRROR__RUBYGEMS__ORG} - -e PIP_INDEX_URL=${PIP_INDEX_URL} - 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\buildwin.bat - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - get-childitem omnibus\pkg - - !reference [.upload_sbom_artifacts_windows] - artifacts: - expire_in: 2 weeks - paths: - - omnibus/pkg diff --git a/.gitlab/package_deps_build.yml b/.gitlab/package_deps_build.yml deleted file mode 100644 index 6bd9d38bdc1c64..00000000000000 --- a/.gitlab/package_deps_build.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -# package_deps_build stage -# Contains jobs to build dependencies needed for datadog-agent packages - -.generate_minimized_btfs_common: - stage: package_deps_build - rules: - - !reference [.except_mergequeue] - - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/btf-gen$DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BTF_GEN_BUILDIMAGES - tags: ["arch:amd64"] - script: - - cd $CI_PROJECT_DIR - - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/btfs-$ARCH.tar.gz . - - tar -xf btfs-$ARCH.tar.gz - - tar -xf sysprobe-build-outputs.tar.xz - - inv -e system-probe.generate-minimized-btfs --source-dir "$CI_PROJECT_DIR/btfs-$ARCH" --output-dir "$CI_PROJECT_DIR/minimized-btfs" --input-bpf-programs "$CI_PROJECT_DIR/pkg/ebpf/bytecode/build/co-re" - - cd minimized-btfs - - tar -cJf $CI_PROJECT_DIR/minimized-btfs.tar.xz * - variables: - KUBERNETES_MEMORY_REQUEST: "6Gi" - KUBERNETES_MEMORY_LIMIT: "12Gi" - artifacts: - expire_in: 2 weeks - paths: - - $CI_PROJECT_DIR/minimized-btfs.tar.xz - -generate_minimized_btfs_x64: - needs: ["build_system-probe-x64"] - extends: .generate_minimized_btfs_common - variables: - ARCH: amd64 - -generate_minimized_btfs_arm64: - needs: ["build_system-probe-arm64"] - extends: .generate_minimized_btfs_common - variables: - ARCH: arm64 diff --git a/.gitlab/pkg_metrics.yml b/.gitlab/pkg_metrics.yml deleted file mode 100644 index c2af9004888faa..00000000000000 --- a/.gitlab/pkg_metrics.yml +++ /dev/null @@ -1,344 +0,0 @@ ---- -# pkg_metrics stage -# Contains jobs which send metrics (package size) about the Linus Agent packages to our backend. - -.add_metric_func: - script: - - apt-get install -y jq - # Build up a JSON body in $post_body. - # To send metrics accumulated in $post_body by add_metric, use the .send_metrics script - - 'post_body="{\"series\": []}"' - - currenttime=$(date +%s) - - | - add_metric() { - local metric="${1}" - shift - local value="${1}" - shift - - local tags=[] - while [ -n "${1}" ]; do - tags=$(echo $tags | jq -c ". += [\"${1}\"]") - shift - done - - post_body=$(echo $post_body | jq -c ".series += [{\"metric\":\"$metric\", \"points\":[[$currenttime, $value]],\"tags\":$tags}]") - } - -.send_metrics: - script: - # Send the metrics accumulated by add_metric - - DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key --with-decryption --query "Parameter.Value" --out text) - - 'curl --fail -X POST -H "Content-type: application/json" -d "$post_body" "https://api.datadoghq.com/api/v1/series?api_key=$DD_API_KEY"' - - -send_pkg_size-a6: - allow_failure: true - rules: - !reference [.on_deploy_a6] - stage: pkg_metrics - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: - - agent_deb-x64-a6 - - agent_rpm-x64-a6 - - agent_suse-x64-a6 - - agent_deb-arm64-a6 - - agent_rpm-arm64-a6 - before_script: - # FIXME: tmp while we uppdate the base image - - apt-get install -y wget rpm2cpio cpio - - ls -l $OMNIBUS_PACKAGE_DIR - - ls -l $OMNIBUS_PACKAGE_DIR_SUSE - script: - - source /root/.bashrc - - mkdir -p /tmp/deb/amd64-agent - - mkdir -p /tmp/deb/arm64-agent - - mkdir -p /tmp/rpm/amd64-agent - - mkdir -p /tmp/rpm/arm64-agent - - mkdir -p /tmp/suse/amd64-agent - - # we silence dpkg and cpio output so we don't exceed gitlab log limit - - # debian - - dpkg -x $OMNIBUS_PACKAGE_DIR/datadog-agent_6*_amd64.deb /tmp/deb/amd64-agent > /dev/null - - AMD64_DEB_AGENT_SIZE=$(du -sB1 /tmp/deb/amd64-agent | sed 's/\([0-9]\+\).\+/\1/') - # debian arm64 - - dpkg -x $OMNIBUS_PACKAGE_DIR/datadog-agent_6*_arm64.deb /tmp/deb/arm64-agent > /dev/null - - ARM64_DEB_AGENT_SIZE=$(du -sB1 /tmp/deb/arm64-agent | sed 's/\([0-9]\+\).\+/\1/') - # centos - - cd /tmp/rpm/amd64-agent && rpm2cpio $OMNIBUS_PACKAGE_DIR/datadog-agent-6.*.x86_64.rpm | cpio -idm > /dev/null - - AMD64_RPM_AGENT_SIZE=$(du -sB1 /tmp/rpm/amd64-agent | sed 's/\([0-9]\+\).\+/\1/') - # centos arm64 - - cd /tmp/rpm/arm64-agent && rpm2cpio $OMNIBUS_PACKAGE_DIR/datadog-agent-6.*.aarch64.rpm | cpio -idm > /dev/null - - ARM64_RPM_AGENT_SIZE=$(du -sB1 /tmp/rpm/arm64-agent | sed 's/\([0-9]\+\).\+/\1/') - - # suse - - cd /tmp/suse/amd64-agent && rpm2cpio $OMNIBUS_PACKAGE_DIR_SUSE/datadog-agent-6.*.x86_64.rpm | cpio -idm > /dev/null - - AMD64_SUSE_AGENT_SIZE=$(du -sB1 /tmp/suse/amd64-agent | sed 's/\([0-9]\+\).\+/\1/') - - - currenttime=$(date +%s) - - DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key --with-decryption --query "Parameter.Value" --out text) - - | - curl --fail -X POST -H "Content-type: application/json" \ - -d "{\"series\":[ - {\"metric\":\"datadog.agent.package.size\",\"points\":[[$currenttime, $AMD64_DEB_AGENT_SIZE]], \"tags\":[\"os:debian\", \"package:agent\", \"agent:6\", \"bucket_branch:$BUCKET_BRANCH\", \"arch:amd64\"]}, - {\"metric\":\"datadog.agent.package.size\",\"points\":[[$currenttime, $AMD64_RPM_AGENT_SIZE]], \"tags\":[\"os:centos\", \"package:agent\", \"agent:6\", \"bucket_branch:$BUCKET_BRANCH\", \"arch:amd64\"]}, - {\"metric\":\"datadog.agent.package.size\",\"points\":[[$currenttime, $AMD64_SUSE_AGENT_SIZE]], \"tags\":[\"os:suse\", \"package:agent\", \"agent:6\", \"bucket_branch:$BUCKET_BRANCH\", \"arch:amd64\"]}, - {\"metric\":\"datadog.agent.package.size\",\"points\":[[$currenttime, $ARM64_DEB_AGENT_SIZE]], \"tags\":[\"os:debian\", \"package:agent\", \"agent:6\", \"bucket_branch:$BUCKET_BRANCH\", \"arch:arm64\"]}, - {\"metric\":\"datadog.agent.package.size\",\"points\":[[$currenttime, $ARM64_RPM_AGENT_SIZE]], \"tags\":[\"os:centos\", \"package:agent\", \"agent:6\", \"bucket_branch:$BUCKET_BRANCH\", \"arch:arm64\"]} - ]}" \ - "https://api.datadoghq.com/api/v1/series?api_key=$DD_API_KEY" - -send_pkg_size-a7: - allow_failure: true - rules: - !reference [.on_deploy_a7] - stage: pkg_metrics - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: - - agent_deb-x64-a7 - - agent_deb-arm64-a7 - - iot_agent_deb-x64 - - iot_agent_deb-arm64 - - dogstatsd_deb-x64 - - dogstatsd_deb-arm64 - - agent_heroku_deb-x64-a7 - - agent_rpm-arm64-a7 - - agent_rpm-x64-a7 - - iot_agent_rpm-x64 - - iot_agent_rpm-arm64 - - dogstatsd_rpm-x64 - - agent_suse-x64-a7 - - dogstatsd_suse-x64 - - iot_agent_suse-x64 - before_script: - # FIXME: tmp while we uppdate the base image - - apt-get install -y wget rpm2cpio cpio - - ls -l $OMNIBUS_PACKAGE_DIR - - ls -l $OMNIBUS_PACKAGE_DIR_SUSE - script: - - source /root/.bashrc - - !reference [.add_metric_func, script] - - - source /root/.bashrc - - mkdir -p /tmp/amd64-deb/agent /tmp/amd64-deb/dogstatsd /tmp/amd64-deb/iot-agent /tmp/amd64-deb/heroku-agent - - mkdir -p /tmp/arm64-deb/agent /tmp/arm64-deb/dogstatsd /tmp/arm64-deb/iot-agent - - mkdir -p /tmp/amd64-rpm/agent /tmp/amd64-rpm/dogstatsd /tmp/amd64-rpm/iot-agent - - mkdir -p /tmp/arm64-rpm/agent /tmp/arm64-rpm/iot-agent - - mkdir -p /tmp/amd64-suse/agent /tmp/amd64-suse/dogstatsd /tmp/amd64-suse/iot-agent - - - | - add_metrics() { - local base="${1}" - local os="${2}" - local arch="${3}" - - # record the total uncompressed size of each package - for package in agent dogstatsd iot-agent heroku-agent; do - if [ ! -d "${base}/${package}" ]; then continue; fi - add_metric datadog.agent.package.size $(du -sB1 "${base}/${package}" | sed 's/\([0-9]\+\).\+/\1/') os:${os} package:${package} agent:7 bucket_branch:$BUCKET_BRANCH arch:${arch} - done - - # record the size of each of the important binaries in each package - add_metric datadog.agent.binary.size $(du -sB1 ${base}/agent/opt/datadog-agent/bin/agent/agent | sed 's/\([0-9]\+\).\+/\1/') bin:agent os:${os} package:agent agent:7 bucket_branch:$BUCKET_BRANCH arch:${arch} - add_metric datadog.agent.binary.size $(du -sB1 ${base}/agent/opt/datadog-agent/embedded/bin/trace-agent | sed 's/\([0-9]\+\).\+/\1/') bin:trace-agent os:${os} package:agent agent:7 bucket_branch:$BUCKET_BRANCH arch:${arch} - add_metric datadog.agent.binary.size $(du -sB1 ${base}/agent/opt/datadog-agent/embedded/bin/security-agent | sed 's/\([0-9]\+\).\+/\1/') bin:security-agent os:${os} package:agent agent:7 bucket_branch:$BUCKET_BRANCH arch:${arch} - add_metric datadog.agent.binary.size $(du -sB1 ${base}/agent/opt/datadog-agent/embedded/bin/process-agent | sed 's/\([0-9]\+\).\+/\1/') bin:process-agent os:${os} package:agent agent:7 bucket_branch:$BUCKET_BRANCH arch:${arch} - add_metric datadog.agent.binary.size $(du -sB1 ${base}/agent/opt/datadog-agent/embedded/bin/system-probe | sed 's/\([0-9]\+\).\+/\1/') bin:system-probe os:${os} package:agent agent:7 bucket_branch:$BUCKET_BRANCH arch:${arch} - if [[ "$arch" == "amd64" || "$os" == "debian" ]]; then add_metric datadog.agent.binary.size $(du -sB1 ${base}/dogstatsd/opt/datadog-dogstatsd/bin/dogstatsd | sed 's/\([0-9]\+\).\+/\1/') bin:dogstatsd os:${os} package:dogstatsd agent:7 bucket_branch:$BUCKET_BRANCH arch:${arch}; fi - add_metric datadog.agent.binary.size $(du -sB1 ${base}/iot-agent/opt/datadog-agent/bin/agent/agent | sed 's/\([0-9]\+\).\+/\1/') bin:agent os:${os} package:iot-agent agent:7 bucket_branch:$BUCKET_BRANCH arch:${arch} - if [ -f "${base}/heroku-agent/opt/datadog-agent/bin/agent/agent" ]; then - add_metric datadog.agent.binary.size $(du -sB1 ${base}/heroku-agent/opt/datadog-agent/bin/agent/agent | sed 's/\([0-9]\+\).\+/\1/') bin:agent os:${os} package:heroku-agent agent:7 bucket_branch:$BUCKET_BRANCH arch:${arch}; - fi - } - - # We silence dpkg and cpio output so we don't exceed gitlab log limit - - # debian - - dpkg -x $OMNIBUS_PACKAGE_DIR/datadog-agent_7*_amd64.deb /tmp/amd64-deb/agent > /dev/null - - dpkg -x $OMNIBUS_PACKAGE_DIR/datadog-iot-agent_7*_amd64.deb /tmp/amd64-deb/iot-agent > /dev/null - - dpkg -x $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd_7*_amd64.deb /tmp/amd64-deb/dogstatsd > /dev/null - - dpkg -x $OMNIBUS_PACKAGE_DIR/datadog-heroku-agent_7*_amd64.deb /tmp/amd64-deb/heroku-agent > /dev/null - - add_metrics /tmp/amd64-deb debian amd64 - - # debian arm64 - - dpkg -x $OMNIBUS_PACKAGE_DIR/datadog-agent_7*_arm64.deb /tmp/arm64-deb/agent > /dev/null - - dpkg -x $OMNIBUS_PACKAGE_DIR/datadog-iot-agent_7*_arm64.deb /tmp/arm64-deb/iot-agent > /dev/null - - dpkg -x $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd_7*_arm64.deb /tmp/arm64-deb/dogstatsd > /dev/null - - add_metrics /tmp/arm64-deb debian arm64 - - # centos - - cd /tmp/amd64-rpm/agent && rpm2cpio $OMNIBUS_PACKAGE_DIR/datadog-agent-7.*.x86_64.rpm | cpio -idm > /dev/null - - cd /tmp/amd64-rpm/dogstatsd && rpm2cpio $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd-7.*.x86_64.rpm | cpio -idm > /dev/null - - cd /tmp/amd64-rpm/iot-agent && rpm2cpio $OMNIBUS_PACKAGE_DIR/datadog-iot-agent-7.*.x86_64.rpm | cpio -idm > /dev/null - - add_metrics /tmp/amd64-rpm centos amd64 - - # centos arm64 - - cd /tmp/arm64-rpm/agent && rpm2cpio $OMNIBUS_PACKAGE_DIR/datadog-agent-7.*.aarch64.rpm | cpio -idm > /dev/null - - cd /tmp/arm64-rpm/iot-agent && rpm2cpio $OMNIBUS_PACKAGE_DIR/datadog-iot-agent-7.*.aarch64.rpm | cpio -idm > /dev/null - - add_metrics /tmp/arm64-rpm centos arm64 - - # suse - - cd /tmp/amd64-suse/agent && rpm2cpio $OMNIBUS_PACKAGE_DIR_SUSE/datadog-agent-7.*.x86_64.rpm | cpio -idm > /dev/null - - cd /tmp/amd64-suse/dogstatsd && rpm2cpio $OMNIBUS_PACKAGE_DIR_SUSE/datadog-dogstatsd-7.*.x86_64.rpm | cpio -idm > /dev/null - - cd /tmp/amd64-suse/iot-agent && rpm2cpio $OMNIBUS_PACKAGE_DIR_SUSE/datadog-iot-agent-7.*.x86_64.rpm | cpio -idm > /dev/null - - add_metrics /tmp/amd64-suse suse amd64 - - # Send package and binary size metrics - - !reference [.send_metrics, script] - -.check_pkg_size: - stage: pkg_metrics - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - script: - - source /root/.bashrc - - !reference [.add_metric_func, script] - - - ls -l $OMNIBUS_PACKAGE_DIR - - if [[ "${ARCH}" == "amd64" ]]; then ls -l $OMNIBUS_PACKAGE_DIR_SUSE; fi - - - source /root/.bashrc - - export failures=0 - - export last_stable=$(inv release.get-release-json-value "last_stable::${MAJOR_VERSION}") - # Get stable packages from S3 buckets, send new package sizes & compare stable and new package sizes - # The loop assumes that all flavors start with "da", which is currently the case - # We want to run all package size comparisons before failing, so we set +e while doing the comparisons - # to get the error codes without exiting the shell. - - | - if [[ "${ARCH}" == "amd64" ]]; then ARCH_RPM_EXT="x86_64"; else ARCH_RPM_EXT="aarch64"; fi - for flavor in ${FLAVORS}; do - - if [[ "${ARCH}" == "amd64" && "$flavor" != "datadog-heroku-agent" ]]; then - mkdir -p "/tmp/stable/${flavor}/suse" - curl -sSL "https://s3.amazonaws.com/yum.datadoghq.com/suse/stable/${MAJOR_VERSION}/${ARCH_RPM_EXT}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" -o "/tmp/stable/${flavor}/suse/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" - add_metric datadog.agent.compressed_package.size $(du -sB1 ${OMNIBUS_PACKAGE_DIR_SUSE}/${flavor}-${MAJOR_VERSION}.*.${ARCH_RPM_EXT}.rpm | cut -f -1) os:suse package:${flavor} agent:${MAJOR_VERSION} git_ref:${CI_COMMIT_REF_SLUG} bucket_branch:${BUCKET_BRANCH} arch:${ARCH} - set +e - inv package.compare-size --package-type "${flavor} suse rpm" --last-stable "${last_stable}" --threshold "${max_sizes[${flavor}]}" --new-package "$OMNIBUS_PACKAGE_DIR_SUSE/${flavor}-${MAJOR_VERSION}.*.${ARCH_RPM_EXT}.rpm" --stable-package "/tmp/stable/${flavor}/suse/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" - failures=$((${failures}+$?)) - set -e - fi - - mkdir -p "/tmp/stable/${flavor}" - - curl -sSL "https://s3.amazonaws.com/apt.datadoghq.com/pool/d/da/${flavor}_${last_stable}-1_${ARCH}.deb" -o "/tmp/stable/${flavor}/${flavor}_${last_stable}-1_${ARCH}.deb" - - add_metric datadog.agent.compressed_package.size $(du -sB1 ${OMNIBUS_PACKAGE_DIR}/${flavor}_${MAJOR_VERSION}*_${ARCH}.deb | cut -f -1) os:debian package:${flavor} agent:${MAJOR_VERSION} git_ref:${CI_COMMIT_REF_SLUG} bucket_branch:${BUCKET_BRANCH} arch:${ARCH} - - set +e - inv package.compare-size --package-type "${flavor} deb" --last-stable "${last_stable}" --threshold "${max_sizes[${flavor}]}" --new-package "$OMNIBUS_PACKAGE_DIR/${flavor}_${MAJOR_VERSION}*_${ARCH}.deb" --stable-package "/tmp/stable/${flavor}/${flavor}_${last_stable}-1_${ARCH}.deb" - failures=$((${failures}+$?)) - set -e - - if [[ "$flavor" != "datadog-heroku-agent" && ( "${ARCH}" == "amd64" || "$flavor" != "datadog-dogstatsd") ]]; then - # We don't build RPM packages for the heroku flavor - curl -sSL "https://s3.amazonaws.com/yum.datadoghq.com/stable/${MAJOR_VERSION}/${ARCH_RPM_EXT}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" -o "/tmp/stable/${flavor}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" - add_metric datadog.agent.compressed_package.size $(du -sB1 ${OMNIBUS_PACKAGE_DIR}/${flavor}-${MAJOR_VERSION}.*.${ARCH_RPM_EXT}.rpm | cut -f -1) os:centos package:${flavor} agent:${MAJOR_VERSION} git_ref:${CI_COMMIT_REF_SLUG} bucket_branch:${BUCKET_BRANCH} arch:${ARCH} - set +e - inv package.compare-size --package-type "${flavor} rpm" --last-stable "${last_stable}" --threshold "${max_sizes[${flavor}]}" --new-package "$OMNIBUS_PACKAGE_DIR/${flavor}-${MAJOR_VERSION}.*.${ARCH_RPM_EXT}.rpm" --stable-package "/tmp/stable/${flavor}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" - failures=$((${failures}+$?)) - set -e - fi - done - - # Send package size metrics - - !reference [.send_metrics, script] - - # Make the job fail if at least one package is above threshold - - if [ "${failures}" -ne "0" ]; then false; fi - -check_pkg_size-amd64-a6: - extends: .check_pkg_size - rules: - !reference [.on_a6] - needs: - - agent_deb-x64-a6 - - agent_rpm-x64-a6 - - agent_suse-x64-a6 - variables: - MAJOR_VERSION: 6 - FLAVORS: "datadog-agent" - ARCH: "amd64" - before_script: - # FIXME: ["datadog-agent"]="70000000" should be replaced by "50000000" - # "70000000" is needed as of now because of multiple large additions in 7.45 - - | - declare -Ar max_sizes=( - ["datadog-agent"]="70000000" - ) - -check_pkg_size-arm64-a6: - extends: .check_pkg_size - rules: - !reference [.on_all_builds_a6] - needs: - - agent_deb-arm64-a6 - - agent_rpm-arm64-a6 - variables: - MAJOR_VERSION: 6 - FLAVORS: "datadog-agent" - ARCH: "arm64" - before_script: - # FIXME: ["datadog-agent"]="70000000" should be replaced by "50000000" - # "70000000" is needed as of now because of multiple large additions in 7.45 - - | - declare -Ar max_sizes=( - ["datadog-agent"]="70000000" - ) - -check_pkg_size-amd64-a7: - extends: .check_pkg_size - rules: - !reference [.on_a7] - needs: - - agent_deb-x64-a7 - - iot_agent_deb-x64 - - dogstatsd_deb-x64 - - agent_heroku_deb-x64-a7 - - agent_rpm-x64-a7 - - iot_agent_rpm-x64 - - dogstatsd_rpm-x64 - - agent_suse-x64-a7 - - dogstatsd_suse-x64 - - iot_agent_suse-x64 - variables: - MAJOR_VERSION: 7 - FLAVORS: "datadog-agent datadog-iot-agent datadog-dogstatsd datadog-heroku-agent" - ARCH: "amd64" - before_script: - # FIXME: ["datadog-agent"]="70000000" and ["datadog-heroku-agent"]="70000000" should - # be replaced by "50000000" - # "70000000" is needed as of now because of multiple large additions in 7.45 - - | - declare -Ar max_sizes=( - ["datadog-agent"]="70000000" - ["datadog-iot-agent"]="10000000" - ["datadog-dogstatsd"]="10000000" - ["datadog-heroku-agent"]="70000000" - ) - -check_pkg_size-arm64-a7: - extends: .check_pkg_size - rules: - !reference [.on_all_builds_a7] - needs: - - agent_deb-arm64-a7 - - iot_agent_deb-arm64 - - dogstatsd_deb-arm64 - - agent_rpm-arm64-a7 - - iot_agent_rpm-arm64 - variables: - MAJOR_VERSION: 7 - FLAVORS: "datadog-agent datadog-iot-agent datadog-dogstatsd" - ARCH: "arm64" - before_script: - # FIXME: ["datadog-agent"]="70000000" should be replaced by "70000000" - # "70000000" is needed as of now because of multiple large additions in 7.45 - - | - declare -Ar max_sizes=( - ["datadog-agent"]="70000000" - ["datadog-iot-agent"]="10000000" - ["datadog-dogstatsd"]="10000000" - ) diff --git a/.gitlab/rc_kubernetes_deploy.yml b/.gitlab/rc_kubernetes_deploy.yml deleted file mode 100644 index 636dc4b3191dd3..00000000000000 --- a/.gitlab/rc_kubernetes_deploy.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -# internal_kubernetes_deploy stage -# Contains jobs to trigger a pipeline in our k8s-datadog-agent-ops repo to deploy release candidate build - -rc_kubernetes_deploy: - stage: internal_kubernetes_deploy - rules: - - if: $RC_K8S_DEPLOYMENTS == "true" - when: always - needs: - - job: docker_trigger_internal - artifacts: false - - job: docker_trigger_cluster_agent_internal - artifacts: false - - job: k8s-e2e-main # Currently only require container Argo workflow - artifacts: false - optional: true - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - OPTION_AUTOMATIC_ROLLOUT: "true" - OPTION_PRE_SCRIPT: "patch-cluster-images-operator.sh env=all-staging ${CI_COMMIT_REF_SLUG}-jmx ${CI_COMMIT_REF_SLUG}" - SKIP_PLAN_CHECK: "true" - EXPLICIT_WORKFLOWS: "//workflows:deploy_rc.agents_rc" - script: - - source /root/.bashrc - - set +x - - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text) - - inv pipeline.trigger-child-pipeline --project-name "DataDog/k8s-datadog-agent-ops" --git-ref "main" --variables "OPTION_AUTOMATIC_ROLLOUT,EXPLICIT_WORKFLOWS,OPTION_PRE_SCRIPT,SKIP_PLAN_CHECK" --no-follow diff --git a/.gitlab/setup.yml b/.gitlab/setup.yml deleted file mode 100644 index a6f18b865366bd..00000000000000 --- a/.gitlab/setup.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# setup stage -# Contains job that needs to be done at the start of the CI - -include: - - /.gitlab/setup/setup.yml diff --git a/.gitlab/setup/setup.yml b/.gitlab/setup/setup.yml deleted file mode 100644 index eb2ee0a929328f..00000000000000 --- a/.gitlab/setup/setup.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -setup_agent_version: - stage: setup - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - script: - - source /root/.bashrc - - inv -e agent.version --version-cached - - $S3_CP_CMD $CI_PROJECT_DIR/agent-version.cache $S3_ARTIFACTS_URI/agent-version.cache - needs: [] - diff --git a/.gitlab/shared.yml b/.gitlab/shared.yml deleted file mode 100644 index af06fac5c92cbf..00000000000000 --- a/.gitlab/shared.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- - -.get_artifactory_token_linux: &get_artifactory_token_linux - - ARTIFACTORY_TOKEN=$(aws ssm get-parameter --region us-east-1 --name $ARTIFACTORY_TOKEN_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - -.get_artifactory_token_win: &get_artifactory_token_win - - $ARTIFACTORY_TOKEN=$(aws ssm get-parameter --region us-east-1 --name $ARTIFACTORY_TOKEN_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - -.get_artifactory_bypass_linux: &get_artifactory_bypass_linux - - ARTIFACTORY_BYPASS=$(aws ssm get-parameter --region us-east-1 --name $ARTIFACTORY_BYPASS_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - if [ "${ARTIFACTORY_BYPASS}" = "true" ]; then echo "Bypassing Artifactory"; fi - -.get_artifactory_bypass_win: &get_artifactory_bypass_win - - $ARTIFACTORY_BYPASS=$(aws ssm get-parameter --region us-east-1 --name $ARTIFACTORY_BYPASS_SSM_NAME --with-decryption --query "Parameter.Value" --out text) - - if ($Env:ARTIFACTORY_BYPASS -eq "true") { Write-Host "Bypassing Artifactory" } - -.setup_ruby_mirror_linux: - - *get_artifactory_bypass_linux - - set +x - - *get_artifactory_token_linux - - if [ "${USE_CACHING_PROXY_RUBY}" = "true" ] && [ "${ARTIFACTORY_BYPASS}" = "false" ]; then export BUNDLE_MIRROR__RUBYGEMS__ORG=https://${ARTIFACTORY_USERNAME}:${ARTIFACTORY_TOKEN}@${ARTIFACTORY_URL}/${ARTIFACTORY_GEMS_PATH}; fi - - set -x - -.setup_ruby_mirror_win: - - *get_artifactory_bypass_win - - *get_artifactory_token_win - - if (($Env:USE_CACHING_PROXY_RUBY -eq "true") -and ($Env:ARTIFACTORY_BYPASS -eq "false")) { $BUNDLE_MIRROR__RUBYGEMS__ORG="https://${Env:ARTIFACTORY_USERNAME}:${ARTIFACTORY_TOKEN}@${Env:ARTIFACTORY_URL}/${Env:ARTIFACTORY_GEMS_PATH}" } - -.setup_python_mirror_linux: - - *get_artifactory_bypass_linux - - set +x - - *get_artifactory_token_linux - - if [ "${USE_CACHING_PROXY_PYTHON}" = "true" ] && [ "${ARTIFACTORY_BYPASS}" = "false" ]; then export PIP_INDEX_URL=https://${ARTIFACTORY_USERNAME}:${ARTIFACTORY_TOKEN}@${ARTIFACTORY_URL}/${ARTIFACTORY_PYPI_PATH}; fi - - set -x - -.setup_python_mirror_win: - - *get_artifactory_bypass_win - - *get_artifactory_token_win - - if (($Env:USE_CACHING_PROXY_PYTHON -eq "true") -and ($Env:ARTIFACTORY_BYPASS -eq "false")) { $PIP_INDEX_URL="https://${Env:ARTIFACTORY_USERNAME}:${ARTIFACTORY_TOKEN}@${Env:ARTIFACTORY_URL}/${Env:ARTIFACTORY_PYPI_PATH}" } diff --git a/.gitlab/source_test.yml b/.gitlab/source_test.yml deleted file mode 100644 index c47f8974a23337..00000000000000 --- a/.gitlab/source_test.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -# source_test stage -# Contains jobs which run unit tests of the Agent on various platforms, ebpf tests, -# security scans & go.mod checks. - -include: - - /.gitlab/source_test/ebpf.yml - - /.gitlab/source_test/linux.yml - - /.gitlab/source_test/macos.yml - - /.gitlab/source_test/windows.yml - - /.gitlab/source_test/go_generate_check.yml - - /.gitlab/source_test/slack.yml - - /.gitlab/source_test/golang_deps_diff.yml diff --git a/.gitlab/source_test/ebpf.yml b/.gitlab/source_test/ebpf.yml deleted file mode 100644 index 28dd63b52364bc..00000000000000 --- a/.gitlab/source_test/ebpf.yml +++ /dev/null @@ -1,114 +0,0 @@ ---- -.retrieve_sysprobe_deps: - - mkdir -p $DATADOG_AGENT_EMBEDDED_PATH/bin - - mkdir -p $DATADOG_AGENT_EMBEDDED_PATH/include - - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/clang-$CLANG_LLVM_VER.$ARCH /tmp/clang-bpf - - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/llc-$CLANG_LLVM_VER.$ARCH /tmp/llc-bpf - -.build_sysprobe_artifacts: - # kitchen prepare also builds object files - - inv -e system-probe.kitchen-prepare --ci - -# Run tests for eBPF code -.tests_linux_ebpf: - stage: source_test - needs: ["go_deps", "go_tools_deps"] - variables: - KUBERNETES_MEMORY_REQUEST: "16Gi" - KUBERNETES_MEMORY_LIMIT: "16Gi" - before_script: - - !reference [.retrieve_linux_go_deps] - - !reference [.retrieve_linux_go_tools_deps] - script: - - inv -e install-tools - - inv -e system-probe.object-files - - invoke -e lint-go --build system-probe-unit-tests --cpus 4 --targets ./pkg - - invoke -e security-agent.run-ebpf-unit-tests --verbose - - invoke -e lint-go --targets=./pkg/security/tests --cpus 4 --build-tags="functionaltests stresstests trivy containerd linux_bpf ebpf_bindata" --arch=$TASK_ARCH - -.tests_windows_sysprobe: - stage: source_test - needs: ["go_deps"] - tags: [ "runner:windows-docker", "windowsversion:1809" ] - script: - - $ErrorActionPreference = "Stop" - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - - !reference [.setup_python_mirror_win] - - > - docker run --rm - -m 16384M - -v "$(Get-Location):c:\mnt" - -e AWS_NETWORKING=true - -e CI_PIPELINE_ID=${CI_PIPELINE_ID} - -e CI_PROJECT_NAME=${CI_PROJECT_NAME} - -e SIGN_WINDOWS_DD_WCS=true - -e PY_RUNTIMES="$PYTHON_RUNTIMES" - -e GOMODCACHE="c:\modcache" - -e PIP_INDEX_URL=${PIP_INDEX_URL} - 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\sysprobe.bat - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - artifacts: - when: always - paths: - - $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files - -tests_ebpf_x64: - extends: .tests_linux_ebpf - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:amd64"] - variables: - ARCH: amd64 - TASK_ARCH: x64 - -tests_ebpf_arm64: - extends: .tests_linux_ebpf - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:arm64"] - variables: - ARCH: arm64 - TASK_ARCH: arm64 - -.prepare_ebpf_functional_tests: - stage: source_test - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: ["go_deps"] - artifacts: - when: always - paths: - - $DD_AGENT_TESTING_DIR/site-cookbooks/dd-security-agent-check/files - - $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files - before_script: - - !reference [.retrieve_linux_go_deps] - - inv -e install-tools - - !reference [.retrieve_sysprobe_deps] - script: - - !reference [.build_sysprobe_artifacts] - - invoke -e security-agent.kitchen-prepare --skip-linters - - cp /tmp/clang-bpf $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/clang-bpf - - cp /tmp/llc-bpf $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/llc-bpf - -prepare_ebpf_functional_tests_arm64: - extends: .prepare_ebpf_functional_tests - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:arm64"] - variables: - ARCH: arm64 - -prepare_ebpf_functional_tests_x64: - extends: .prepare_ebpf_functional_tests - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:amd64"] - variables: - ARCH: amd64 - -tests_windows_sysprobe_x64: - extends: .tests_windows_sysprobe - rules: - - !reference [.except_mergequeue] - - when: on_success - variables: - PYTHON_RUNTIMES: 3 - ARCH: "x64" diff --git a/.gitlab/source_test/go_generate_check.yml b/.gitlab/source_test/go_generate_check.yml deleted file mode 100644 index c9b596cc39e5cc..00000000000000 --- a/.gitlab/source_test/go_generate_check.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -# check that go generate has been run in the pkg/security directory -security_go_generate_check: - rules: - - !reference [.except_mergequeue] - - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:amd64"] - stage: source_test - needs: [ "go_deps", "go_tools_deps" ] - before_script: - - !reference [.retrieve_linux_go_deps] - - !reference [.retrieve_linux_go_tools_deps] - - pip3 install -r docs/cloud-workload-security/scripts/requirements-docs.txt - - inv -e install-tools - script: - - inv -e security-agent.go-generate-check diff --git a/.gitlab/source_test/golang_deps_diff.yml b/.gitlab/source_test/golang_deps_diff.yml deleted file mode 100644 index e6729a417e2741..00000000000000 --- a/.gitlab/source_test/golang_deps_diff.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -# golang_deps_diff stage -# Contains the step to generate diff of go imports for each binary/build -golang_deps_diff: - stage: source_test - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - rules: # this should only run on dev branches - - !reference [ .except_main_or_release_branch ] - - when: on_success - needs: ["go_deps"] - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - script: - - inv -e diff.go-deps --report-file=deps-report.md - artifacts: - paths: - - deps-report.md - expire_in: 2 weeks - -golang_deps_commenter: - stage: source_test - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/pr-commenter:2 - tags: ["arch:amd64"] - rules: # this should only run on dev branches - - !reference [ .except_main_or_release_branch ] - - when: on_success - needs: ["golang_deps_diff"] - script: # ignore error message about no PR, because it happens for dev branches without PRs - - echo "${CI_COMMIT_REF_NAME}" - - | - set +e - out=$(pr-commenter --for-pr="${CI_COMMIT_REF_NAME}" --header="Go Package Import Differences" --infile deps-report.md 2>&1) - exitcode=$? - set -e - if [ -n "${out}" ]; then - if [ $exitcode -eq 0 ]; then - echo $out - else - echo $out >&2 - fi - fi - if [ "${out}" != "${out/invalid request: no pr found for this commit}" ]; then - exit 0 - fi - exit $exitcode diff --git a/.gitlab/source_test/golang_deps_generate.yml b/.gitlab/source_test/golang_deps_generate.yml deleted file mode 100644 index db182cbdcf25d7..00000000000000 --- a/.gitlab/source_test/golang_deps_generate.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -# golang_deps_generate stage -# Contains the step to build the golang dependency chain and preserve it -golang_deps_generate: - rules: - !reference [.on_main_or_release_branch] - stage: source_test - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_deps"] - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - script: - - inv agent.build-dep-tree - - tar -czf dependency_tree.tgz dependency_tree.txt - artifacts: - paths: - - dependency_tree.tgz - expire_in: never diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml deleted file mode 100644 index a9f1db150506d9..00000000000000 --- a/.gitlab/source_test/linux.yml +++ /dev/null @@ -1,256 +0,0 @@ ---- -.rtloader_tests: - stage: source_test - before_script: - - source /root/.bashrc && conda activate $CONDA_ENV - - !reference [.retrieve_linux_go_deps] - - inv -e rtloader.make --install-prefix=$CI_PROJECT_DIR/dev --python-runtimes "$PYTHON_RUNTIMES" - - inv -e rtloader.install - - inv -e rtloader.format --raise-if-changed - - inv -e rtloader.test - # Placeholder script, overridden by .linux_tests when running all go tests - script: [ "# Skipping go tests" ] - -.linux_tests: - stage: source_test - variables: - FLAVORS: '--flavors base' - KUBERNETES_CPU_REQUEST: 16 - KUBERNETES_MEMORY_REQUEST: 16Gi - KUBERNETES_MEMORY_LIMIT: 16Gi - TEST_OUTPUT_FILE: test_output.json - script: - - !reference [.retrieve_linux_go_tools_deps] - - inv -e install-tools - - inv -e invoke-unit-tests - - pushd test/kitchen - - inv -e kitchen.invoke-unit-tests - - popd - - inv -e test $FLAVORS --race --profile --rerun-fails=2 --python-runtimes "$PYTHON_RUNTIMES" --cpus $KUBERNETES_CPU_REQUEST $EXTRA_OPTS --save-result-json $TEST_OUTPUT_FILE --junit-tar "junit-${CI_JOB_NAME}.tgz" - artifacts: - expire_in: 2 weeks - when: always - paths: - - $TEST_OUTPUT_FILE - - junit-*.tgz - reports: - junit: "**/junit-out-*.xml" - -.linux_lint: - stage: source_test - variables: - FLAVORS: '--flavors base' - KUBERNETES_CPU_REQUEST: 16 - KUBERNETES_MEMORY_REQUEST: 16Gi - KUBERNETES_MEMORY_LIMIT: 16Gi - script: - - !reference [.retrieve_linux_go_deps] - - !reference [.retrieve_linux_go_tools_deps] - - source /root/.bashrc && conda activate ddpy3 - - inv -e rtloader.make --install-prefix=$CI_PROJECT_DIR/dev --python-runtimes "3" - - inv -e rtloader.install - - inv -e install-tools - - inv -e lint-go --cpus $KUBERNETES_CPU_REQUEST $FLAVORS $EXTRA_OPTS - -.linux_x64: - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - -tests_deb-x64-py2: - extends: - - .rtloader_tests - - .linux_x64 - rules: - !reference [.on_a6] - needs: ["go_deps"] - variables: - PYTHON_RUNTIMES: '2' - CONDA_ENV: ddpy2 - -tests_deb-x64-py3: - extends: - - .rtloader_tests - - .linux_tests - - .linux_x64 - needs: ["go_deps", "go_tools_deps"] - variables: - PYTHON_RUNTIMES: '3' - CONDA_ENV: ddpy3 - -tests_deb-x64-py3-fast: - extends: - - .rtloader_tests - - .linux_tests - - .linux_x64 - needs: ["go_deps", "go_tools_deps"] - variables: - PYTHON_RUNTIMES: '3' - CONDA_ENV: ddpy3 - EXTRA_OPTS: '--only-modified-packages' - TEST_OUTPUT_FILE: test_output_fast.json - allow_failure: true - -lint_linux-x64: - extends: - - .linux_lint - - .linux_x64 - needs: ["go_deps", "go_tools_deps"] - -tests_flavor_iot_deb-x64: - extends: - - .rtloader_tests - - .linux_tests - - .linux_x64 - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: ["go_deps", "go_tools_deps"] - variables: - PYTHON_RUNTIMES: '3' - CONDA_ENV: ddpy3 - FLAVORS: '--flavors iot' - -lint_flavor_iot_linux-x64: - extends: - - .linux_lint - - .linux_x64 - needs: ["go_deps", "go_tools_deps"] - variables: - FLAVORS: '--flavors iot' - -tests_flavor_dogstatsd_deb-x64: - extends: - - .rtloader_tests - - .linux_tests - - .linux_x64 - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: ["go_deps", "go_tools_deps"] - variables: - PYTHON_RUNTIMES: '3' - CONDA_ENV: ddpy3 - FLAVORS: '--flavors dogstatsd' - -lint_flavor_dogstatsd_linux-x64: - extends: - - .linux_lint - - .linux_x64 - needs: ["go_deps", "go_tools_deps"] - variables: - FLAVORS: '--flavors dogstatsd' - -tests_flavor_heroku_deb-x64: - extends: - - .rtloader_tests - - .linux_tests - - .linux_x64 - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: ["go_deps", "go_tools_deps"] - variables: - PYTHON_RUNTIMES: '3' - CONDA_ENV: ddpy3 - FLAVORS: '--flavors heroku' - -lint_flavor_heroku_linux-x64: - extends: - - .linux_lint - - .linux_x64 - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: ["go_deps", "go_tools_deps"] - variables: - FLAVORS: '--flavors heroku' - -# Exclude systemd because it cannot succeed on Centos 6: the image doesn't have the shared object required by -# https://github.com/coreos/go-systemd/blob/c8cc474ba8655dfbdb0ac7fcc09b7faf5b643caf/sdjournal/functions.go#L46 -# This is OK because the test on systemd still runs on the debian image above -tests_rpm-x64-py2: - extends: .rtloader_tests - rules: - !reference [.on_a6] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64_testing$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_deps"] - variables: - PYTHON_RUNTIMES: '2' - CONDA_ENV: ddpy2 - EXTRA_OPTS: '--build-exclude=systemd' - -tests_rpm-x64-py3: - extends: - - .rtloader_tests - - .linux_tests - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64_testing$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_deps", "go_tools_deps"] - variables: - PYTHON_RUNTIMES: '3' - CONDA_ENV: ddpy3 - EXTRA_OPTS: '--build-exclude=systemd' - - -tests_deb-arm64-py2: - extends: .rtloader_tests - rules: - !reference [.on_a6] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_deps"] - variables: - PYTHON_RUNTIMES: '2' - CONDA_ENV: ddpy2 - -tests_deb-arm64-py3: - extends: - - .rtloader_tests - - .linux_tests - needs: ["go_deps", "go_tools_deps"] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:arm64"] - variables: - PYTHON_RUNTIMES: '3' - CONDA_ENV: ddpy3 - -lint_linux-arm64: - extends: - - .linux_lint - needs: ["go_deps", "go_tools_deps"] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:arm64"] - -tests_rpm-arm64-py2: - extends: .rtloader_tests - rules: - !reference [.on_a6] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_deps"] - variables: - PYTHON_RUNTIMES: '2' - CONDA_ENV: ddpy2 - -tests_rpm-arm64-py3: - extends: - - .rtloader_tests - - .linux_tests - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_deps", "go_tools_deps"] - variables: - PYTHON_RUNTIMES: '3' - CONDA_ENV: ddpy3 - -# Check consistency of go.mod file with project imports -go_mod_tidy_check: - stage: source_test - extends: .linux_x64 - needs: ["go_deps"] - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - script: - - inv -e check-mod-tidy diff --git a/.gitlab/source_test/macos.yml b/.gitlab/source_test/macos.yml deleted file mode 100644 index efe673b5da86ea..00000000000000 --- a/.gitlab/source_test/macos.yml +++ /dev/null @@ -1,48 +0,0 @@ - -tests_macos: - stage: source_test - rules: - !reference [.on_a6] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - PYTHON_RUNTIMES: '3' - script: - - source /root/.bashrc - - export GITHUB_KEY_B64=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_key_b64 --with-decryption --query "Parameter.Value" --out text) - - export GITHUB_APP_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_app_id --with-decryption --query "Parameter.Value" --out text) - - export GITHUB_INSTALLATION_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_installation_id --with-decryption --query "Parameter.Value" --out text) - - $S3_CP_CMD $S3_ARTIFACTS_URI/agent-version.cache . - - export VERSION_CACHE_CONTENT=$(cat agent-version.cache | base64 -) - - !reference [.setup_python_mirror_linux] - - python3 -m pip install -r tasks/libs/requirements-github.txt - - inv -e github.trigger-macos-test --datadog-agent-ref "$CI_COMMIT_SHA" --python-runtimes "$PYTHON_RUNTIMES" --version-cache "$VERSION_CACHE_CONTENT" - after_script: - - inv -e junit-macos-repack --infile junit-tests_macos.tgz --outfile junit-tests_macos-repacked.tgz - artifacts: - expire_in: 2 weeks - when: always - paths: - - test_output.json - - junit-*-repacked.tgz - reports: - junit: "**/junit-out-*.xml" - -lint_macos: - stage: source_test - rules: - !reference [.on_a6] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - PYTHON_RUNTIMES: '3' - script: - - source /root/.bashrc - - export GITHUB_KEY_B64=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_key_b64 --with-decryption --query "Parameter.Value" --out text) - - export GITHUB_APP_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_app_id --with-decryption --query "Parameter.Value" --out text) - - export GITHUB_INSTALLATION_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_installation_id --with-decryption --query "Parameter.Value" --out text) - - $S3_CP_CMD $S3_ARTIFACTS_URI/agent-version.cache . - - export VERSION_CACHE_CONTENT=$(cat agent-version.cache | base64 -) - - !reference [.setup_python_mirror_linux] - - python3 -m pip install -r tasks/libs/requirements-github.txt - - inv -e github.trigger-macos-lint --datadog-agent-ref "$CI_COMMIT_SHA" --python-runtimes "$PYTHON_RUNTIMES" --version-cache "$VERSION_CACHE_CONTENT" diff --git a/.gitlab/source_test/slack.yml b/.gitlab/source_test/slack.yml deleted file mode 100644 index f129baca6fc73e..00000000000000 --- a/.gitlab/source_test/slack.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -# Check that each team in CODEOWNERS has its slack notification channel filled in the GITHUB_SLACK_MAP -slack_teams_channels_check: - stage: source_test - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: [] - script: - - source /root/.bashrc - - python3 -m pip install codeowners -c tasks/libs/requirements-notifications.txt - - inv -e pipeline.check-notify-teams - - diff --git a/.gitlab/source_test/windows.yml b/.gitlab/source_test/windows.yml deleted file mode 100644 index 444c9ebe6d06e4..00000000000000 --- a/.gitlab/source_test/windows.yml +++ /dev/null @@ -1,90 +0,0 @@ ---- -.tests_windows_base: - stage: source_test - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: ["go_deps", "go_tools_deps"] - tags: ["runner:windows-docker", "windowsversion:1809"] - script: - - $ErrorActionPreference = "Stop" - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - # we pass in CI_JOB_URL and CI_JOB_NAME so that they can be added to additional tags - # inside JUNIT_TAR and then later used by datadog-ci - - !reference [.setup_python_mirror_win] - - > - docker run --rm - -m 16384M - -v "$(Get-Location):c:\mnt" - -e CI_JOB_URL="${CI_JOB_URL}" - -e CI_JOB_NAME="${CI_JOB_NAME}" - -e CI_PIPELINE_ID=${CI_PIPELINE_ID} - -e CI_PROJECT_NAME=${CI_PROJECT_NAME} - -e AWS_NETWORKING=true - -e SIGN_WINDOWS_DD_WCS=true - -e PY_RUNTIMES="$PYTHON_RUNTIMES" - -e GOMODCACHE="c:\modcache" - -e JUNIT_TAR="c:\mnt\junit-${CI_JOB_NAME}.tgz" - -e PIP_INDEX_URL=${PIP_INDEX_URL} - -e TEST_OUTPUT_FILE=${TEST_OUTPUT_FILE} - -e EXTRA_OPTS="${EXTRA_OPTS}" - 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\unittests.bat - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - variables: - TEST_OUTPUT_FILE: test_output.json - artifacts: - expire_in: 2 weeks - when: always - paths: - - $TEST_OUTPUT_FILE - - junit-*.tgz - reports: - junit: "**/junit-out-*.xml" - -.lint_windows_base: - stage: source_test - needs: ["go_deps", "go_tools_deps"] - tags: ["runner:windows-docker", "windowsversion:1809"] - script: - - $ErrorActionPreference = "Stop" - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - # Previously this job required only 8Gb of memory but since Go 1.20 it requires more to avoid being OOM killed. - # Each Windows VM has 32Gb of memory and contains 3 runners that can run one job at a time each (so a maximum of 3 simultaneous jobs per VM). - # Windows jobs are using either 8Gb or 16Gb of memory so we can limit memory to 16Gb on this job because even if we decided to limit to 10Gb for instance, - # it would leave 6Gb free but we could not fit another job with these 6Gb remaining. - - > - docker run --rm - -m 16384M - -v "$(Get-Location):c:\mnt" - -e AWS_NETWORKING=true - -e CI_PIPELINE_ID=${CI_PIPELINE_ID} - -e CI_PROJECT_NAME=${CI_PROJECT_NAME} - -e PY_RUNTIMES="$PYTHON_RUNTIMES" - -e GOMODCACHE="c:\modcache" - 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:$Env:DATADOG_AGENT_WINBUILDIMAGES c:\mnt\tasks\winbuildscripts\lint.bat - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - -tests_windows-x64: - extends: .tests_windows_base - variables: - PYTHON_RUNTIMES: 3 - ARCH: "x64" - -tests_windows-x64-fast: - extends: .tests_windows_base - variables: - PYTHON_RUNTIMES: 3 - ARCH: "x64" - EXTRA_OPTS: "--only-modified-packages" - TEST_OUTPUT_FILE: test_output_fast.json - allow_failure: true - -lint_windows-x64: - extends: .lint_windows_base - rules: - - !reference [.except_mergequeue] - - when: on_success - variables: - PYTHON_RUNTIMES: 3 - ARCH: "x64" diff --git a/.gitlab/source_test_junit_upload.yml b/.gitlab/source_test_junit_upload.yml deleted file mode 100644 index 465c469c502f4b..00000000000000 --- a/.gitlab/source_test_junit_upload.yml +++ /dev/null @@ -1,16 +0,0 @@ -source_test_junit_upload: - # Uncomment this to only run junit uploads on main/stable branches - # rules: - # !reference [.on_main_or_release_branch] - when: always - stage: source_test_junit_upload - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - allow_failure: true - variables: - DD_ENV: ci - script: - - set +x - - export DATADOG_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text) - - set -x - - for f in junit-*.tgz; do inv -e junit-upload --tgz-path $f; done diff --git a/.gitlab/source_test_stats.yml b/.gitlab/source_test_stats.yml deleted file mode 100644 index 9e50f8f7a9afd7..00000000000000 --- a/.gitlab/source_test_stats.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# source_test stage -# Contains jobs which run stats about unit tests of the Agent on various platforms, ebpf tests, -# security scans & go.mod checks. - -include: - - /.gitlab/source_test_stats/linux.yml - - /.gitlab/source_test_stats/windows.yml diff --git a/.gitlab/source_test_stats/linux.yml b/.gitlab/source_test_stats/linux.yml deleted file mode 100644 index 852c457011c14a..00000000000000 --- a/.gitlab/source_test_stats/linux.yml +++ /dev/null @@ -1,13 +0,0 @@ -stats-fast-tests-deb-x64-py3: - stage: source_test_stats - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["tests_deb-x64-py3-fast", "tests_deb-x64-py3"] - rules: - - !reference [.except_mergequeue] - - when: on_success - script: - - set +x - - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text) - - inv -e send-unit-tests-stats tests_deb-x64-py3 - when: always diff --git a/.gitlab/source_test_stats/windows.yml b/.gitlab/source_test_stats/windows.yml deleted file mode 100644 index 347adcfa8b3591..00000000000000 --- a/.gitlab/source_test_stats/windows.yml +++ /dev/null @@ -1,13 +0,0 @@ -stats-fast-tests-windows-x64: - stage: source_test_stats - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["tests_windows-x64-fast", "tests_windows-x64"] - rules: - - !reference [.except_mergequeue] - - when: on_success - script: - - set +x - - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text) - - inv -e send-unit-tests-stats tests_windows-x64 - when: always diff --git a/.gitlab/trigger_release.yml b/.gitlab/trigger_release.yml deleted file mode 100644 index 672e7a12e8f728..00000000000000 --- a/.gitlab/trigger_release.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -# trigger_release stage -# Contains jobs which trigger release pipelines in the datadog/agent-release-management repository. - -.agent_release_management_trigger: - stage: trigger_release - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - # We don't directly depend/need the package deploy jobs, because - # that would make us deploy even when there are kitchen failures etc - # We only want to allow automatically triggering agent-release-manangement - # pipelines when everything goes well - variables: - ACTION: promote - BUILD_PIPELINE_ID: $CI_PIPELINE_ID - RELEASE_PRODUCT: datadog-agent - TARGET_REPO_BRANCH: $BUCKET_BRANCH - script: - # agent-release-management creates pipeline for both Agent 6 and Agent 7 - # when triggered with major version 7 - - source /root/.bashrc - - export RELEASE_VERSION=$(inv agent.version --major-version 7 --url-safe --omnibus-format)-1 - - inv pipeline.trigger-child-pipeline --no-follow --project-name "DataDog/agent-release-management" --git-ref "main" --variables "ACTION,AUTO_RELEASE,BUILD_PIPELINE_ID,RELEASE_PRODUCT,RELEASE_VERSION,TARGET_REPO,TARGET_REPO_BRANCH" - dependencies: [] - -trigger_auto_staging_release: - extends: .agent_release_management_trigger - variables: - AUTO_RELEASE: "true" - TARGET_REPO: staging - rules: - !reference [.on_deploy] - -trigger_manual_prod_release: - extends: .agent_release_management_trigger - variables: - AUTO_RELEASE: "false" - TARGET_REPO: prod - rules: - !reference [.on_deploy_stable_or_beta_manual_auto_on_stable] diff --git a/pkg/serverless/daemon/routes.go b/pkg/serverless/daemon/routes.go index 599d9cb879ec2a..89de80fbbf3299 100644 --- a/pkg/serverless/daemon/routes.go +++ b/pkg/serverless/daemon/routes.go @@ -74,6 +74,7 @@ func (s *StartInvocation) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set(invocationlifecycle.TraceIDHeader, fmt.Sprintf("%v", s.daemon.InvocationProcessor.GetExecutionInfo().TraceID)) w.Header().Set(invocationlifecycle.SamplingPriorityHeader, fmt.Sprintf("%v", s.daemon.InvocationProcessor.GetExecutionInfo().SamplingPriority)) } + time.Sleep(time.Millisecond) } // EndInvocation is a route that can be called at the end of an invocation to enable