diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7acbc70212ef..fa48496d385e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,119 +1,153 @@ +# ENV VARS variables: - CI_PROJECT_NAME: "opentelemetry-collector-contrib" - BUILD_STABLE_REGISTRY: "486234852809.dkr.ecr.us-east-1.amazonaws.com" CI_IMAGE: registry.ddbuild.io/ci/opentelemetry-collector-contrib:ci-image-2 - RESTORE_CACHE_ATTEMPTS: 2 BUILD_DD_REGISTRY: registry.ddbuild.io/ci/opentelemetry-collector-contrib BUILD_DEMO_REGISTRY: 172597598159.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib BUILD_SANDBOX_REGISTRY: 601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib - DOCKERFILE: ci/Dockerfile + SANDBOX_CLUSTER_NAME: dd-otel + DEMO_CLUSTER_NAME: prod-eks-otel-a-us-east-1 + SANDBOX_CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" + DEMO_CLUSTER_ARN: "arn:aws:eks:us-east-1:172597598159:cluster/prod-eks-otel-a-us-east-1" + DEPLOY_SCRIPT: "./ci/scripts/ci-deploy.sh" + + +# BUILD STAGES stages: - - manaul-jobs - - build - - push + - build-and-push-ci-image + - build-collector-image + - push-collector-image - staging-deploy - prod-deploy -# ======================================================================= -# Build and deploy the images used for CI -# ======================================================================= -.build-ci-image: &build-ci-image - stage: build +# BUILD IMAGES +build-ci-image: + stage: build-and-push-ci-image + tags: ["runner:docker"] + image: $CI_IMAGE + script: + - TAG=ci-image-2 + - docker build --file ci/Dockerfile.gitlab --tag $BUILD_DD_REGISTRY:$TAG . + - docker push $BUILD_DD_REGISTRY:$TAG + +.build-collector-image: &build-collector-image + stage: build-collector-image tags: ["runner:docker"] image: $CI_IMAGE script: - IMAGE_TAG_PREFIX=otelcolcontrib - TAG="$IMAGE_TAG_PREFIX-v$CI_COMMIT_SHORT_SHA" - - docker build --file $DOCKERFILE --tag $BUILD_REGISTRY:$TAG --label target=staging . + - docker build --file ci/Dockerfile --tag $BUILD_REGISTRY:$TAG --label target=staging . - docker push $BUILD_REGISTRY:$TAG -build-image-main: - !!merge <<: *build-ci-image +build-collector-image-main: + !!merge <<: *build-collector-image variables: BUILD_REGISTRY: $BUILD_DD_REGISTRY -.push-image: &push-image - stage: push + +# PUSH IMAGES +.push-collector-image: &push-collector-image + stage: push-collector-image tags: ["runner:docker"] image: $CI_IMAGE dependencies: - - build-image-main + - build-collector-image-main script: - IMAGE_TAG_PREFIX=otelcolcontrib - TAG="$IMAGE_TAG_PREFIX-v$CI_COMMIT_SHORT_SHA" - docker pull $BUILD_DD_REGISTRY:$TAG - docker tag $BUILD_DD_REGISTRY:$TAG $BUILD_REGISTRY:$TAG - docker push $BUILD_REGISTRY:$TAG -push-image-demo: - !!merge <<: *push-image +push-collector-image-demo: + !!merge <<: *push-collector-image variables: BUILD_REGISTRY: $BUILD_DEMO_REGISTRY - AWS_ACCT_ID: 172597598159 -push-image-staging: - !!merge <<: *push-image +push-collector-image-staging: + !!merge <<: *push-collector-image variables: BUILD_REGISTRY: $BUILD_SANDBOX_REGISTRY - AWS_ACCT_ID: 601427279990 -build-ci-image: - stage: manaul-jobs - when: manual - tags: ["runner:docker"] - image: $CI_IMAGE - script: - - TAG=ci-image-2 - - docker build --file ci/Dockerfile.gitlab --tag $BUILD_DD_REGISTRY:$TAG . - - docker push $BUILD_DD_REGISTRY:$TAG + +# DEPLOY COLLECTOR STAGING .staging-deploy: &staging-deploy stage: staging-deploy tags: ["runner:docker", "size:large"] image: $CI_IMAGE dependencies: - - push-image-staging + - push-collector-image-staging rules: - if: '$CI_COMMIT_REF_NAME =~ /-staging$/' script: - # # For debugging - #- aws sts get-caller-identity - >- TEMP_AWS_ACCESS_KEY_ID=$(aws ssm get-parameter --region us-east-1 --name ci.opentelemetry-collector-contrib.sand-eks-deploy-api-key --with-decryption --query Parameter.Value --out text) - >- TEMP_AWS_SECRET_ACCESS_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.opentelemetry-collector-contrib.sand-eks-deploy-access-key --with-decryption --query Parameter.Value --out text) - export AWS_ACCESS_KEY_ID=$TEMP_AWS_ACCESS_KEY_ID - export AWS_SECRET_ACCESS_KEY=$TEMP_AWS_SECRET_ACCESS_KEY - # # For debugging - # - aws sts get-caller-identity - - bash $SCRIPT $NAMESPACE $VALUES -staging-deploy-demo-eks: + - bash $DEPLOY_SCRIPT +#env:otel-staging +staging-deploy: !!merge <<: *staging-deploy variables: - SCRIPT: ./ci/scripts/ci-deploy-staging.sh + CLUSTER_NAME: $SANDBOX_CLUSTER_NAME + CLUSTER_ARN: $SANDBOX_CLUSTER_ARN NAMESPACE: otel-staging - VALUES: ./ci/values-staging.yaml -staging-deploy-gateway-eks: + NODE_GROUP: ng-3 + MODE: daemonset + REPLICA_COUNT: 1 + CLUSTER_ROLE: opentelemetry-collector + REGISTRY: $BUILD_SANDBOX_REGISTRY + +#env:otel-gateway +staging-deploy-gateway: !!merge <<: *staging-deploy variables: - SCRIPT: ./ci/scripts/ci-deploy-staging.sh + CLUSTER_NAME: $SANDBOX_CLUSTER_NAME + CLUSTER_ARN: $SANDBOX_CLUSTER_ARN NAMESPACE: otel-gateway - VALUES: ./ci/values-gateway.yaml -staging-deploy-ds-gateway-eks: + NODE_GROUP: ng-5 + MODE: deployment + REPLICA_COUNT: 3 + CLUSTER_ROLE: opentelemetry-collector-gateway + REGISTRY: $BUILD_SANDBOX_REGISTRY + +#env:otel-ds-gateway +staging-deploy-ds-gateway: !!merge <<: *staging-deploy variables: - SCRIPT: ./ci/scripts/ci-deploy-ds-gateway.sh -prod-deploy-demo-eks: + CLUSTER_NAME: $SANDBOX_CLUSTER_NAME + CLUSTER_ARN: $SANDBOX_CLUSTER_ARN + NAMESPACE: otel-ds-gateway + NODE_GROUP: ng-6 + MODE: deployment + REPLICA_COUNT: 3 + CLUSTER_ROLE: opentelemetry-collector-ds-gateway + REGISTRY: $BUILD_SANDBOX_REGISTRY + +# DEPLOY COLLECTOR PROD +.prod-deploy: &prod-deploy-demo-eks stage: prod-deploy tags: ["runner:docker", "size:large"] image: $CI_IMAGE dependencies: - - push-image-demo + - push-collector-image-demo rules: + # changing to staging for testing purposes. - if: '$CI_COMMIT_REF_NAME == "prod"' script: - # # For debugging - #- aws sts get-caller-identity - >- TEMP_AWS_ACCESS_KEY_ID=$(aws ssm get-parameter --region us-east-1 --name ci.opentelemetry-collector-contrib.eks_access_key --with-decryption --query Parameter.Value --out text) - >- TEMP_AWS_SECRET_ACCESS_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.opentelemetry-collector-contrib.eks_secret_access_key --with-decryption --query Parameter.Value --out text) - export AWS_ACCESS_KEY_ID=$TEMP_AWS_ACCESS_KEY_ID - export AWS_SECRET_ACCESS_KEY=$TEMP_AWS_SECRET_ACCESS_KEY - # # For debugging - # - aws sts get-caller-identity - - bash ./ci/scripts/ci-deploy-demo.sh + - bash $DEPLOY_SCRIPT +#env:otel +prod-deploy: + !!merge <<: *prod-deploy-demo-eks + variables: + CLUSTER_NAME: $DEMO_CLUSTER_NAME + CLUSTER_ARN: $DEMO_CLUSTER_ARN + NAMESPACE: otel + NODE_GROUP: + MODE: daemonset + REPLICA_COUNT: 1 + CLUSTER_ROLE: opentelemetry-collector + REGISTRY: $BUILD_DEMO_REGISTRY \ No newline at end of file diff --git a/ci/scripts/ci-deploy-demo.sh b/ci/scripts/ci-deploy-demo.sh deleted file mode 100644 index 2b02093b1468..000000000000 --- a/ci/scripts/ci-deploy-demo.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -# This script is used to deploy collector on demo account cluster - -set -euo pipefail -IFS=$'\n\t' -set -x - -install_collector() { - # Set the namespace and release name - release_name="opentelemetry-collector" - release_name_deployment="opentelemetry-collector-deployment" - namespace="otel" - - # if repo already exists, helm 3+ will skip - helm --debug repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts - - # --install will run `helm install` if not already present. - helm --debug upgrade "${release_name}" -n "${namespace}" open-telemetry/opentelemetry-collector --install \ - -f ./ci/values.yaml \ - --set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA" - - # --install collector that fetches jmx metrics. The jmx receiver cannot be used in the daemonset deployment - # as this would lead to duplicate metrics. - helm --debug upgrade "${release_name_deployment}" -n "${namespace}" open-telemetry/opentelemetry-collector --install \ - -f ./ci/values-jmx.yaml \ - --set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA" -} - -########################################################################################################### -clusterName="prod-eks-otel-a-us-east-1" -clusterArn="arn:aws:eks:us-east-1:172597598159:cluster/${clusterName}" - -aws eks --region us-east-1 update-kubeconfig --name "${clusterName}" -kubectl config use-context "${clusterArn}" - -install_collector diff --git a/ci/scripts/ci-deploy-ds-gateway.sh b/ci/scripts/ci-deploy-ds-gateway.sh deleted file mode 100644 index 2f65ea3500b4..000000000000 --- a/ci/scripts/ci-deploy-ds-gateway.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -# This script is used to deploy collector on demo account cluster - -set -euo pipefail -IFS=$'\n\t' -set -x - -install_collector() { - # Set the namespace and release name - release_name="opentelemetry-collector" - release_name_gateway="opentelemetry-collector-gateway" - namespace="otel-ds-gateway" - - # if repo already exists, helm 3+ will skip - helm --debug repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts - - # --install will run `helm install` if not already present. - # gateway with DD exporter - helm --debug upgrade "${release_name_gateway}" -n "${namespace}" open-telemetry/opentelemetry-collector --install \ - -f ./ci/values.yaml \ - -f ./ci/values-ds-gateway.yaml \ - --set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA" \ - --set-string image.repository="601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib" - - # daemonset with otlp exporter - helm --debug upgrade "${release_name}" -n "${namespace}" open-telemetry/opentelemetry-collector --install \ - -f ./ci/values.yaml \ - -f ./ci/values-otlp-col.yaml \ - --set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA" \ - --set-string image.repository="601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib" - -} - -########################################################################################################### -clusterName="dd-otel" -clusterArn="arn:aws:eks:us-east-1:601427279990:cluster/${clusterName}" - -aws eks --region us-east-1 update-kubeconfig --name "${clusterName}" -kubectl config use-context "${clusterArn}" - -install_collector diff --git a/ci/scripts/ci-deploy-staging.sh b/ci/scripts/ci-deploy-staging.sh deleted file mode 100644 index 36749fd85d7c..000000000000 --- a/ci/scripts/ci-deploy-staging.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -# This script is used to deploy collector on demo account cluster - -set -euo pipefail -IFS=$'\n\t' -set -x -namespace=$NAMESPACE -values=$VALUES - -install_collector() { - release_name="opentelemetry-collector" - - # if repo already exists, helm 3+ will skip - helm --debug repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts - helm repo update open-telemetry - # --install will run `helm install` if not already present. - helm --debug upgrade "${release_name}" -n "${namespace}" open-telemetry/opentelemetry-collector --install \ - -f ./ci/values.yaml \ - -f "${values}" \ - --set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA" \ - --set-string image.repository="601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib" - helm list --all-namespaces - - if [ "$namespace" == "otel-staging" ]; then - install_deployment - fi -} - -install_deployment() { - release_name_deployment="opentelemetry-collector-deployment" - - # --install collector that fetches jmx metrics. The jmx receiver cannot be used in the daemonset deployment - # as this would lead to duplicate metrics. - helm --debug upgrade "${release_name_deployment}" -n "${namespace}" open-telemetry/opentelemetry-collector --install \ - -f ./ci/values-jmx.yaml \ - --set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA" \ - --set-string image.repository="601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib" \ - --set nodeSelector.alpha\\.eksctl\\.io/nodegroup-name=ng-3 - -} - -########################################################################################################### -clusterName="dd-otel" -clusterArn="arn:aws:eks:us-east-1:601427279990:cluster/${clusterName}" - -aws eks --region us-east-1 update-kubeconfig --name "${clusterName}" -kubectl config use-context "${clusterArn}" - -install_collector diff --git a/ci/scripts/ci-deploy.sh b/ci/scripts/ci-deploy.sh new file mode 100644 index 000000000000..b78255fb97bf --- /dev/null +++ b/ci/scripts/ci-deploy.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# This script is used to deploy collector on demo account cluster + +set -euo pipefail +IFS=$'\n\t' +set -x + +namespace=$NAMESPACE +nodegroup=$NODE_GROUP +mode=$MODE +replicaCount=$REPLICA_COUNT +clusterRole=$CLUSTER_ROLE +clusterName=$CLUSTER_NAME +clusterArn=$CLUSTER_ARN +registry=$REGISTRY + +install_collector() { + release_name="opentelemetry-collector" + + # Add open-telemetry helm repo (if repo already exists, helm 3+ will skip) + helm --debug repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts + helm repo update open-telemetry + + helm_cmd="helm --debug upgrade ${release_name} -n ${namespace} open-telemetry/opentelemetry-collector --install \ + -f ./ci/values.yaml \ + --set-string image.repository=${registry} \ + --set-string image.tag=otelcolcontrib-v$CI_COMMIT_SHORT_SHA \ + --set clusterRole.name=${clusterRole} \ + --set clusterRole.clusterRoleBinding.name=${clusterRole} \ + --set mode=${mode} \ + --set replicaCount=${replicaCount}" + + if [ -n "$nodegroup" ]; then + helm_cmd+=" --set nodeSelector.\"alpha\\.eksctl\\.io/nodegroup-name\"=${nodegroup}" + fi + + eval $helm_cmd + + # only deploy otlp col for otel-ds-gateway + if [ "$namespace" == "otel-ds-gateway" ]; then + install_ds_otlp + fi +} + +install_ds_otlp() { + release_name="opentelemetry-collector-ds" + + # daemonset with otlp exporter + helm --debug upgrade "${release_name}" -n "${namespace}" open-telemetry/opentelemetry-collector --install \ + -f ./ci/values.yaml \ + -f ./ci/values-otlp-col.yaml \ + --set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA" \ + --set-string image.repository=${registry} +} + +########################################################################################################### +clusterName="${clusterName}" +clusterArn="${clusterArn}" + +aws eks --region us-east-1 update-kubeconfig --name "${clusterName}" +kubectl config use-context "${clusterArn}" + +install_collector \ No newline at end of file diff --git a/ci/values-ds-gateway.yaml b/ci/values-ds-gateway.yaml deleted file mode 100644 index 49a87acd35c2..000000000000 --- a/ci/values-ds-gateway.yaml +++ /dev/null @@ -1,11 +0,0 @@ -mode: deployment -replicaCount: 3 -nodeSelector: - alpha.eksctl.io/nodegroup-name: ng-6 -clusterRole: - name: "opentelemetry-collector-ds-gateway" - clusterRoleBinding: - name: opentelemetry-collector-ds-gateway -presets: - logsCollection: - enabled: true diff --git a/ci/values-gateway.yaml b/ci/values-gateway.yaml deleted file mode 100644 index 342add0e1b16..000000000000 --- a/ci/values-gateway.yaml +++ /dev/null @@ -1,11 +0,0 @@ -mode: deployment -replicaCount: 3 -nodeSelector: - alpha.eksctl.io/nodegroup-name: ng-5 -clusterRole: - name: "opentelemetry-collector-gateway" - clusterRoleBinding: - name: opentelemetry-collector-gateway -presets: - logsCollection: - enabled: true diff --git a/ci/values-jmx.yaml b/ci/values-jmx.yaml deleted file mode 100644 index 2e70080f3667..000000000000 --- a/ci/values-jmx.yaml +++ /dev/null @@ -1,72 +0,0 @@ -mode: deployment -image: - repository: 172597598159.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib - tag: "" -securityContext: - runAsUser: 0 - runAsGroup: 0 -extraEnvs: - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: OTEL_COLLECTOR_NAME - value: $(HOST_IP) - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://$(OTEL_COLLECTOR_NAME):4317 - - name: OTEL_RESOURCE_ATTRIBUTES - value: kafka_source=deploymentcol -config: - receivers: - jmx: - jar_path: /opt/opentelemetry-jmx-metrics.jar - endpoint: "opentelemetry-demo-kafka:1099" - target_system: kafka - log_level: trace - jmx/jvm: - jar_path: /opt/opentelemetry-jmx-metrics.jar - endpoint: "opentelemetry-demo-kafka:1099" - target_system: jvm - log_level: trace - jmx/consumer: - jar_path: /opt/opentelemetry-jmx-metrics.jar - endpoint: "opentelemetry-demo-frauddetectionservice:1098" - target_system: kafka-consumer - log_level: trace - jmx/producer: - jar_path: /opt/opentelemetry-jmx-metrics.jar - endpoint: "opentelemetry-demo-orderproducer:1097" - target_system: kafka-producer - log_level: trace - zookeeper: - endpoint: "opentelemetry-demo-zookeeper:2181" - collection_interval: 20s - initial_delay: 1s - kafkametrics: - brokers: "opentelemetry-demo-kafka:9092" - protocol_version: 2.0.0 - scrapers: - - brokers - - topics - - consumers - exporters: - debug: - verbosity: detailed - logging: null - otlp: - endpoint: ${env:OTEL_EXPORTER_OTLP_ENDPOINT} - tls: - insecure: true - processors: - resourcedetection/env: - detectors: [env] - batch: - send_batch_max_size: 1000 - send_batch_size: 100 - timeout: 10s - service: - pipelines: - metrics: - receivers: [jmx, jmx/jvm, jmx/consumer, jmx/producer, zookeeper, kafkametrics] - processors: [batch, resourcedetection/env] - exporters: [otlp] \ No newline at end of file diff --git a/ci/values-otlp-col.yaml b/ci/values-otlp-col.yaml index 9e0b716e5678..af6275e5fd7e 100644 --- a/ci/values-otlp-col.yaml +++ b/ci/values-otlp-col.yaml @@ -7,16 +7,35 @@ clusterRole: config: exporters: otlp: - endpoint: opentelemetry-collector-gateway:4317 + endpoint: opentelemetry-collector.otel-ds-gateway.svc.cluster.local:4317 tls: insecure: true + processors: + k8sattributes: + passthrough: true + auth_type: "serviceAccount" + receivers: + otlp/2: + protocols: + grpc: + endpoint: 0.0.0.0:1234 service: pipelines: metrics: + receivers: [otlp] + processors: [k8sattributes, batch] exporters: [otlp] traces: + receivers: [otlp] + processors: [k8sattributes, batch] + exporters: [otlp] + # hack to disable this pipeline. otlp/2 does not receive data. + traces/sampled: + receivers: [otlp/2] + processors: [] exporters: [otlp] logs: + processors: [k8sattributes, batch] exporters: [otlp] resources: limits: diff --git a/ci/values-staging.yaml b/ci/values-staging.yaml deleted file mode 100644 index 1d8b6afb0322..000000000000 --- a/ci/values-staging.yaml +++ /dev/null @@ -1,5 +0,0 @@ -nodeSelector: - alpha.eksctl.io/nodegroup-name: ng-3 -presets: - logsCollection: - enabled: true diff --git a/ci/values.yaml b/ci/values.yaml index ba0bccaacc39..ea8b5bfe2f21 100644 --- a/ci/values.yaml +++ b/ci/values.yaml @@ -89,7 +89,7 @@ config: peer_service_aggregation: true trace_buffer: 1000 api: - key: "$DD_API_KEY" + key: "${env:DD_API_KEY}" processors: attributes: actions: @@ -240,7 +240,7 @@ config: traces/sampled: receivers: [datadog/connector] processors: [probabilistic_sampler, batch] - exporters: [datadog] + exporters: [datadog, debug] logs: processors: [memory_limiter, resourcedetection, k8sattributes, attributes, attributes/kafkasrc, transform, batch] exporters: [datadog]