This repository has been archived by the owner on Dec 14, 2021. It is now read-only.
forked from DataDog/datadog-agent
-
Notifications
You must be signed in to change notification settings - Fork 0
/
.gitlab-ci.yml
242 lines (217 loc) · 9.49 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
---
include:
- /.gitlab/maintenance_jobs.yml
- /.gitlab/deps_build.yml
- /.gitlab/deps_fetch.yml
- /.gitlab/source_test.yml
- /.gitlab/binary_build.yml
- /.gitlab/integration_test.yml
- /.gitlab/package_build.yml
- /.gitlab/internal_deploy.yml
- /.gitlab/kitchen_deploy.yml
- /.gitlab/kitchen_testing.yml
- /.gitlab/pkg_metrics.yml
- /.gitlab/image_build.yml
- /.gitlab/image_scan.yml
- /.gitlab/check_deploy.yml
- /.gitlab/image_deploy.yml
- /.gitlab/deploy_6.yml
- /.gitlab/deploy_7.yml
- /.gitlab/choco_build.yml
- /.gitlab/choco_deploy.yml
- /.gitlab/deploy_invalidate.yml
- /.gitlab/trigger_release.yml
- /.gitlab/e2e.yml
- /.gitlab/kitchen_cleanup.yml
- /.gitlab/functional_test.yml
- /.gitlab/functional_test_cleanup.yml
- /.gitlab/notify.yml
# FIXME: our current Gitlab version doesn't support importing a file more than once
# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file
# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987
- /.gitlab/kitchen_common/cleanup.yml
- /.gitlab/kitchen_common/testing.yml
- /.gitlab/docker_common/tag_job_templates.yml
default:
retry:
max: 2
when:
- runner_system_failure
- stuck_or_timeout_failure
- unknown_failure
- api_failure
workflow:
rules:
- if: $CI_COMMIT_TAG =~ /^dca-/ || $CI_COMMIT_TAG == null || $DEPLOY_AGENT == "true"
stages:
- maintenance_jobs
- deps_build
- deps_fetch
- source_test
- binary_build
- integration_test
- package_build
- internal_deploy
- kitchen_deploy
- kitchen_testing
- pkg_metrics
- image_build
- image_scan
- check_deploy
- image_deploy
- deploy6
- deploy7
- choco_build
- choco_deploy
- deploy_invalidate
- trigger_release
- e2e
- kitchen_cleanup
- functional_test
- functional_test_cleanup
- notify
variables:
# The SRC_PATH is in the GOPATH of the builders which
# currently is /go
SRC_PATH: /go/src/github.com/DataDog/datadog-agent
# Directory in which we execute the omnibus build.
# For an unknown reason, it does not go well with
# a ruby dependency if we build directly into $CI_PROJECT_DIR/.omnibus
OMNIBUS_BASE_DIR: /.omnibus
# Directory in which we put the artifacts after the build
# Must be in $CI_PROJECT_DIR
OMNIBUS_PACKAGE_DIR: $CI_PROJECT_DIR/.omnibus/pkg/
# Directory in which we execute the omnibus build for SUSE
# as we want to separate the RPM built for this distro.
OMNIBUS_BASE_DIR_SUSE: /.omnibus/suse
# Directory in which we put the artifacts after the build
# Must be in $CI_PROJECT_DIR
OMNIBUS_PACKAGE_DIR_SUSE: $CI_PROJECT_DIR/.omnibus/suse/pkg
OMNIBUS_BASE_DIR_WIN: c:\omni-base\$CI_RUNNER_ID
OMNIBUS_BASE_DIR_WIN_OMNIBUS: c:/omni-base/$CI_RUNNER_ID
DD_AGENT_TESTING_DIR: $CI_PROJECT_DIR/test/kitchen
STATIC_BINARIES_DIR: bin/static
DOGSTATSD_BINARIES_DIR: bin/dogstatsd
AGENT_BINARIES_DIR: bin/agent
CLUSTER_AGENT_BINARIES_DIR: bin/datadog-cluster-agent
CLUSTER_AGENT_CLOUDFOUNDRY_BINARIES_DIR: bin/datadog-cluster-agent-cloudfoundry
SYSTEM_PROBE_BINARIES_DIR: bin/system-probe
DEB_S3_BUCKET: apt.datad0g.com
RPM_S3_BUCKET: yum.datad0g.com
MACOS_S3_BUCKET: dd-agent-macostesting
WIN_S3_BUCKET: dd-agent-mstesting
PROCESS_S3_BUCKET: datad0g-process-agent
ANDROID_S3_BUCKET: dd-agent-androidtesting
DEB_RPM_BUCKET_BRANCH: nightly # branch of the DEB_S3_BUCKET and RPM_S3_BUCKET repos to release to, 'nightly' or 'beta'
DEB_TESTING_S3_BUCKET: apttesting.datad0g.com
RPM_TESTING_S3_BUCKET: yumtesting.datad0g.com
WINDOWS_TESTING_S3_BUCKET_A6: pipelines/A6/$CI_PIPELINE_ID
WINDOWS_TESTING_S3_BUCKET_A7: pipelines/A7/$CI_PIPELINE_ID
WINDOWS_BUILDS_S3_BUCKET: $WIN_S3_BUCKET/builds
ANDROID_BUILDS_S3_BUCKET: $ANDROID_S3_BUCKET/builds
DEB_RPM_TESTING_BUCKET_BRANCH: testing # branch of the DEB_TESTING_S3_BUCKET and RPM_TESTING_S3_BUCKET repos to release to, 'testing'
DD_REPO_BRANCH_NAME: $CI_COMMIT_REF_NAME
S3_CP_OPTIONS: --only-show-errors --region us-east-1 --sse AES256
S3_CP_CMD: aws s3 cp $S3_CP_OPTIONS
S3_ARTIFACTS_URI: s3://dd-ci-artefacts-build-stable/$CI_PROJECT_NAME/$CI_PIPELINE_ID
S3_PERMANENT_ARTIFACTS_URI: s3://dd-ci-persistent-artefacts-build-stable/$CI_PROJECT_NAME
## comment out both lines below (S3_OMNIBUS_CACHE_BUCKET and USE_S3_CACHING) to allow
## build to succeed with S3 caching disabled.
S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable
USE_S3_CACHING: --omnibus-s3-cache
S3_DSD6_URI: s3://dsd6-staging
RELEASE_VERSION_6: nightly
RELEASE_VERSION_7: nightly-a7
DATADOG_AGENT_BUILDIMAGES: v3686353-1e75137
DATADOG_AGENT_BUILDERS: v3730945-1ae6287
DATADOG_AGENT_WINBUILDIMAGES: v3651799-a21376a
DATADOG_AGENT_ARMBUILDIMAGES: v3507010-02f01e5
DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v3508219-2619f3d
DATADOG_AGENT_LIBBCC_BUILDIMAGES: v2894686-d80c3ce
BCC_VERSION: v0.12.0
SYSTEM_PROBE_GO_VERSION: 1.14.12
DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
# the deb signing key is split in halves, as it exceeds the maximum size of 4096 chars for a single entry
DEB_GPG_KEY_NAME: "Datadog, Inc <[email protected]>"
DEB_GPG_KEY_SSM_NAME_PART1: ci.datadog-agent.deb_signing_private_key_8387EEAF_part1
DEB_GPG_KEY_SSM_NAME_PART2: ci.datadog-agent.deb_signing_private_key_8387EEAF_part2
DEB_SIGNING_PASSPHRASE_SSM_NAME: ci.datadog-agent.deb_signing_key_passphrase_8387EEAF
RPM_GPG_KEY_SSM_NAME: ci.datadog-agent.rpm_signing_private_key_e09422b3
RPM_SIGNING_PASSPHRASE_SSM_NAME: ci.datadog-agent.rpm_signing_key_passphrase_e09422b3
# docker.io authentication
DOCKER_REGISTRY_LOGIN_SSM_KEY: docker_hub_login
DOCKER_REGISTRY_PWD_SSM_KEY: docker_hub_pwd
DOCKER_REGISTRY_URL: docker.io
#
# Condition mixins for simplification of rules
#
# FIXME: currently, the only good way we have to compose rules is by using YAML anchors.
# However, as they are YAML features (and not Gitlab features), they have to be present
# in each file that needs them, which results in a lot of copy-pasting.
# In the future, we may be able to leverage YAML functions (https://gitlab.com/groups/gitlab-org/-/epics/4381)
# and cross-file references (https://gitlab.com/gitlab-org/gitlab/-/issues/266173) to remove this
# currently needed copy-paste.
#
# The list of all available rules is given below. To use a rule in a specific file, copy it.
#
.if_master_branch: &if_master_branch
if: $CI_COMMIT_BRANCH == "master"
.if_not_master_branch: &if_not_master_branch
if: $CI_COMMIT_BRANCH != "master"
.if_tagged_commit: &if_tagged_commit
if: $CI_COMMIT_TAG != null
.if_not_tagged_commit: &if_not_tagged_commit
if: $CI_COMMIT_TAG == null
# Run job only on deploy pipelines
.if_deploy: &if_deploy
if: $DEPLOY_AGENT == "true"
.if_deploy_6: &if_deploy_6
if: $DEPLOY_AGENT == "true" && $RELEASE_VERSION_6 != ""
.if_deploy_7: &if_deploy_7
if: $DEPLOY_AGENT == "true" && $RELEASE_VERSION_7 != ""
.if_not_deploy: &if_not_deploy
if: $DEPLOY_AGENT != "true"
# anchor to trigger test kitchen setup, run, and cleanup (so all stages
# are run if one stage is run). Triggers as defined:
# - master
# - tags (a tagged build)
# - deploys (as above)
# - web (when the build is triggered by a specific build request through the
# web interface. This way, if a kitchen run is desired on a specific branch,
# it can be triggered by requesting a specific build)
# - api (when the build is triggered by an API call)
#
.if_kitchen_deploy: &if_kitchen_deploy
if: $CI_COMMIT_BRANCH == "master" || $CI_COMMIT_TAG != null || $DEPLOY_AGENT == "true" || $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
.if_not_kitchen_deploy: &if_not_kitchen_deploy
if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_TAG == null && $DEPLOY_AGENT != "true" && $CI_PIPELINE_SOURCE != "web" && $CI_PIPELINE_SOURCE != "api"
# true only on deploys and when RELEASE_VERSION_X is NOT "nightly". In this setting
# we are building either a new tagged version of the agent (an RC for example).
# In both cases the artifacts should be uploaded to our staging repository.
.if_deploy_on_tag_6: &if_deploy_on_tag_6
# no RELEASE_VERSION means a nightly build for omnibus
if: $DEPLOY_AGENT == "true" && $RELEASE_VERSION_6 != "nightly" && $RELEASE_VERSION_6 != ""
.if_not_deploy_on_tag_6: &if_not_deploy_on_tag_6
# no RELEASE_VERSION means a nightly build for omnibus
if: $DEPLOY_AGENT != "true" || $RELEASE_VERSION_6 == "nightly" || $RELEASE_VERSION_6 == ""
.if_deploy_on_tag_7: &if_deploy_on_tag_7
# no RELEASE_VERSION means a nightly build for omnibus
if: $DEPLOY_AGENT == "true" && $RELEASE_VERSION_7 != "nightly-a7" && $RELEASE_VERSION_7 != ""
.if_not_deploy_on_tag_7: &if_not_deploy_on_tag_7
# no RELEASE_VERSION means a nightly build for omnibus
if: $DEPLOY_AGENT != "true" || $RELEASE_VERSION_7 == "nightly-a7" || $RELEASE_VERSION_7 == ""
# true only on deploys and when RELEASE_VERSION_X is "nightly". In this setting we
# build from master and update the nightly build for windows, linux and docker.
.if_deploy_on_nightly: &if_deploy_on_nightly
if: $DEPLOY_AGENT == "true" && $RELEASE_VERSION_6 == "nightly" && $RELEASE_VERSION_7 == "nightly-a7"
.if_not_deploy_on_nightly: &if_not_deploy_on_nightly
if: $DEPLOY_AGENT != "true" || $RELEASE_VERSION_6 != "nightly" || $RELEASE_VERSION_7 != "nightly-a7"
# true only when RELEASE_VERSION_X is not set
.if_version_6: &if_version_6
if: $RELEASE_VERSION_6 != ""
.if_not_version_6: &if_not_version_6
if: $RELEASE_VERSION_6 == ""
.if_version_7: &if_version_7
if: $RELEASE_VERSION_7 != ""
.if_not_version_7: &if_not_version_7
if: $RELEASE_VERSION_7 == ""