Refactor (#5265)

* Refactor

* remove jmx

* fixes

* update cluster names

* attrs

* standardise

* fix

* fix

* fixes

* fix

* add registry

* fix

* fix

* fix

* use new syntax

* fix

* fix

* change otlp endpoint

* undo change

* debug

* fix otlp col

* fix otlp col

* fix

.gitlab-ci.yml

@@ -1,119 +1,153 @@
+# ENV VARS
 variables:
-  CI_PROJECT_NAME: "opentelemetry-collector-contrib"
-  BUILD_STABLE_REGISTRY: "486234852809.dkr.ecr.us-east-1.amazonaws.com"
   CI_IMAGE: registry.ddbuild.io/ci/opentelemetry-collector-contrib:ci-image-2
-  RESTORE_CACHE_ATTEMPTS: 2
   BUILD_DD_REGISTRY: registry.ddbuild.io/ci/opentelemetry-collector-contrib
   BUILD_DEMO_REGISTRY: 172597598159.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib
   BUILD_SANDBOX_REGISTRY: 601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib
-  DOCKERFILE: ci/Dockerfile
+  SANDBOX_CLUSTER_NAME: dd-otel
+  DEMO_CLUSTER_NAME: prod-eks-otel-a-us-east-1
+  SANDBOX_CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel"
+  DEMO_CLUSTER_ARN: "arn:aws:eks:us-east-1:172597598159:cluster/prod-eks-otel-a-us-east-1"
+  DEPLOY_SCRIPT: "./ci/scripts/ci-deploy.sh"
+
+
+# BUILD STAGES
 stages:
-  - manaul-jobs
-  - build
-  - push
+  - build-and-push-ci-image
+  - build-collector-image
+  - push-collector-image
   - staging-deploy
   - prod-deploy
 
-# =======================================================================
-# Build and deploy the images used for CI
-# =======================================================================
-.build-ci-image: &build-ci-image
-  stage: build
+# BUILD IMAGES
+build-ci-image:
+  stage: build-and-push-ci-image
+  tags: ["runner:docker"]
+  image: $CI_IMAGE
+  script:
+    - TAG=ci-image-2
+    - docker build --file ci/Dockerfile.gitlab --tag $BUILD_DD_REGISTRY:$TAG .
+    - docker push $BUILD_DD_REGISTRY:$TAG
+
+.build-collector-image: &build-collector-image
+  stage: build-collector-image
   tags: ["runner:docker"]
   image: $CI_IMAGE
   script:
     - IMAGE_TAG_PREFIX=otelcolcontrib
     - TAG="$IMAGE_TAG_PREFIX-v$CI_COMMIT_SHORT_SHA"
-    - docker build --file $DOCKERFILE --tag $BUILD_REGISTRY:$TAG --label target=staging .
+    - docker build --file ci/Dockerfile --tag $BUILD_REGISTRY:$TAG --label target=staging .
     - docker push $BUILD_REGISTRY:$TAG
-build-image-main:
-  !!merge <<: *build-ci-image
+build-collector-image-main:
+  !!merge <<: *build-collector-image
   variables:
     BUILD_REGISTRY: $BUILD_DD_REGISTRY
-.push-image: &push-image
-  stage: push
+
+# PUSH IMAGES
+.push-collector-image: &push-collector-image
+  stage: push-collector-image
   tags: ["runner:docker"]
   image: $CI_IMAGE
   dependencies:
-    - build-image-main
+    - build-collector-image-main
   script:
     - IMAGE_TAG_PREFIX=otelcolcontrib
     - TAG="$IMAGE_TAG_PREFIX-v$CI_COMMIT_SHORT_SHA"
     - docker pull $BUILD_DD_REGISTRY:$TAG
     - docker tag $BUILD_DD_REGISTRY:$TAG $BUILD_REGISTRY:$TAG
     - docker push $BUILD_REGISTRY:$TAG
-push-image-demo:
-  !!merge <<: *push-image
+push-collector-image-demo:
+  !!merge <<: *push-collector-image
   variables:
     BUILD_REGISTRY: $BUILD_DEMO_REGISTRY
-    AWS_ACCT_ID: 172597598159
-push-image-staging:
-  !!merge <<: *push-image
+push-collector-image-staging:
+  !!merge <<: *push-collector-image
   variables:
     BUILD_REGISTRY: $BUILD_SANDBOX_REGISTRY
-    AWS_ACCT_ID: 601427279990
-build-ci-image:
-  stage: manaul-jobs
-  when: manual
-  tags: ["runner:docker"]
-  image: $CI_IMAGE
-  script:
-    - TAG=ci-image-2
-    - docker build --file ci/Dockerfile.gitlab --tag $BUILD_DD_REGISTRY:$TAG .
-    - docker push $BUILD_DD_REGISTRY:$TAG
+
+# DEPLOY COLLECTOR STAGING 
 .staging-deploy: &staging-deploy
   stage: staging-deploy
   tags: ["runner:docker", "size:large"]
   image: $CI_IMAGE
   dependencies:
-    - push-image-staging
+    - push-collector-image-staging
   rules:
     - if: '$CI_COMMIT_REF_NAME =~ /-staging$/'
   script:
-    # # For debugging
-    #- aws sts get-caller-identity
     - >-
       TEMP_AWS_ACCESS_KEY_ID=$(aws ssm get-parameter --region us-east-1 --name ci.opentelemetry-collector-contrib.sand-eks-deploy-api-key --with-decryption --query Parameter.Value --out text)
     - >-
       TEMP_AWS_SECRET_ACCESS_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.opentelemetry-collector-contrib.sand-eks-deploy-access-key --with-decryption --query Parameter.Value --out text)
     - export AWS_ACCESS_KEY_ID=$TEMP_AWS_ACCESS_KEY_ID
     - export AWS_SECRET_ACCESS_KEY=$TEMP_AWS_SECRET_ACCESS_KEY
-    # # For debugging
-    # - aws sts get-caller-identity
-    - bash $SCRIPT $NAMESPACE $VALUES
-staging-deploy-demo-eks:
+    - bash $DEPLOY_SCRIPT
+#env:otel-staging
+staging-deploy:
   !!merge <<: *staging-deploy
   variables:
-    SCRIPT: ./ci/scripts/ci-deploy-staging.sh
+    CLUSTER_NAME: $SANDBOX_CLUSTER_NAME
+    CLUSTER_ARN: $SANDBOX_CLUSTER_ARN
     NAMESPACE: otel-staging
-    VALUES: ./ci/values-staging.yaml
-staging-deploy-gateway-eks:
+    NODE_GROUP: ng-3
+    MODE: daemonset
+    REPLICA_COUNT: 1
+    CLUSTER_ROLE: opentelemetry-collector
+    REGISTRY: $BUILD_SANDBOX_REGISTRY
+
+#env:otel-gateway
+staging-deploy-gateway:
   !!merge <<: *staging-deploy
   variables:
-    SCRIPT: ./ci/scripts/ci-deploy-staging.sh
+    CLUSTER_NAME: $SANDBOX_CLUSTER_NAME
+    CLUSTER_ARN: $SANDBOX_CLUSTER_ARN
     NAMESPACE: otel-gateway
-    VALUES: ./ci/values-gateway.yaml
-staging-deploy-ds-gateway-eks:
+    NODE_GROUP: ng-5
+    MODE: deployment
+    REPLICA_COUNT: 3
+    CLUSTER_ROLE: opentelemetry-collector-gateway
+    REGISTRY: $BUILD_SANDBOX_REGISTRY
+
+#env:otel-ds-gateway
+staging-deploy-ds-gateway:
   !!merge <<: *staging-deploy
   variables:
-    SCRIPT: ./ci/scripts/ci-deploy-ds-gateway.sh
-prod-deploy-demo-eks:
+    CLUSTER_NAME: $SANDBOX_CLUSTER_NAME
+    CLUSTER_ARN: $SANDBOX_CLUSTER_ARN
+    NAMESPACE: otel-ds-gateway
+    NODE_GROUP: ng-6
+    MODE: deployment
+    REPLICA_COUNT: 3
+    CLUSTER_ROLE: opentelemetry-collector-ds-gateway
+    REGISTRY: $BUILD_SANDBOX_REGISTRY
+
+# DEPLOY COLLECTOR PROD 
+.prod-deploy: &prod-deploy-demo-eks
   stage: prod-deploy
   tags: ["runner:docker", "size:large"]
   image: $CI_IMAGE
   dependencies:
-    - push-image-demo
+    - push-collector-image-demo
   rules:
+    # changing to staging for testing purposes.
     - if: '$CI_COMMIT_REF_NAME == "prod"'
   script:
-    # # For debugging
-    #- aws sts get-caller-identity
     - >-
       TEMP_AWS_ACCESS_KEY_ID=$(aws ssm get-parameter --region us-east-1 --name ci.opentelemetry-collector-contrib.eks_access_key --with-decryption --query Parameter.Value --out text)
     - >-
       TEMP_AWS_SECRET_ACCESS_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.opentelemetry-collector-contrib.eks_secret_access_key --with-decryption --query Parameter.Value --out text)
     - export AWS_ACCESS_KEY_ID=$TEMP_AWS_ACCESS_KEY_ID
     - export AWS_SECRET_ACCESS_KEY=$TEMP_AWS_SECRET_ACCESS_KEY
-    # # For debugging
-    # - aws sts get-caller-identity
-    - bash ./ci/scripts/ci-deploy-demo.sh
+    - bash $DEPLOY_SCRIPT
+#env:otel
+prod-deploy:
+  !!merge <<: *prod-deploy-demo-eks
+  variables:
+    CLUSTER_NAME: $DEMO_CLUSTER_NAME
+    CLUSTER_ARN: $DEMO_CLUSTER_ARN
+    NAMESPACE: otel
+    NODE_GROUP: 
+    MODE: daemonset
+    REPLICA_COUNT: 1
+    CLUSTER_ROLE: opentelemetry-collector
+    REGISTRY: $BUILD_DEMO_REGISTRY