Merge remote-tracking branch 'upstream/main' into feature/logstash

.buildkite/pipeline-build.yml

@@ -0,0 +1,84 @@
+steps:
+  - group: build
+    steps:
+
+      - label: ":docker: ci image"
+        command: make -C .ci ci-build-image
+        agents:
+          provider: "gcp"
+          image: "family/elastic-buildkite-agent-ubuntu-2004-lts"
+          machineType: "n1-standard-16"
+
+      - wait
+
+      # multiarch dockerx build for !PR branch
+      - label: ":docker: multiarch operator image"
+        key: "operator-image-build"
+        commands:
+          - .buildkite/scripts/build/setenv.sh
+          - make -C .ci TARGET="generate-crds-v1 publish-operator-multiarch-image" ci
+          # save image for e2e-tests
+          - buildkite-agent meta-data set operator-image $$(make print-operator-image)
+        agents:
+          provider: "gcp"
+          image: "family/elastic-buildkite-agent-ubuntu-2004-lts"
+          machineType: "n1-standard-16"
+        artifact_paths:
+          - config/*.yaml
+
+      - label: ":docker: e2e-tests image"
+        key: "e2e-tests-image-build"
+        commands:
+          - .buildkite/scripts/build/setenv.sh
+          - make -C .ci TARGET="publish-e2e-multiarch-image" ci
+          # save image for e2e-tests
+          - buildkite-agent meta-data set e2e-image $$(make print-e2e-image)
+        agents:
+          provider: "gcp"
+          image: "family/elastic-buildkite-agent-ubuntu-2004-lts"
+          machineType: "n1-standard-16"
+
+      # dev-license build for main nightly builds
+      - label: ":docker: dev-license operator image"
+        if: | 
+          ( build.branch == "main" && build.source == "schedule" )
+          || build.message =~ /^buildkite test .*dev-build.*/
+        key: "dev-operator-image-build"
+        env:
+          BUILD_LICENSE_PUBKEY: "dev"
+        commands:
+          - .buildkite/scripts/build/setenv.sh
+          - make -C .ci TARGET="publish-operator-multiarch-image" ci
+        agents:
+          provider: "gcp"
+          image: "family/elastic-buildkite-agent-ubuntu-2004-lts"
+          machineType: "n1-standard-16"
+
+      # fips build for tag builds
+      - label: ":docker: fips operator image"
+        if: |
+          build.tag != null 
+          || build.message =~ /^buildkite test .*release.*/
+        env:
+          ENABLE_FIPS: "true"
+        commands:
+          - .buildkite/scripts/build/setenv.sh
+          - make -C .ci TARGET="publish-operator-multiarch-image" ci
+        agents:
+          provider: "gcp"
+          image: "family/elastic-buildkite-agent-ubuntu-2004-lts"
+          machineType: "n1-standard-16"
+
+      - label: ":go: operatorhub tool"
+        key: "build-operatorhub-tool"
+        commands:
+          - cd hack/operatorhub
+          - make build
+          - buildkite-agent artifact upload bin/operatorhub
+
+      - label: ":go: helm releaser tool"
+        key: "build-helm-releaser-tool"
+        commands:
+          - cd hack/helm/release
+          - make build
+          - buildkite-agent artifact upload bin/releaser

.buildkite/pipeline-nightly-e2e-tests.yml

@@ -1,7 +1,3 @@
-agents:
-  image: docker.elastic.co/ci-agent-images/cloud-k8s-operator/buildkite-agent:latest
-  memory: "4G"
-
 steps:
 
   - group: "aks"
@@ -33,6 +29,8 @@ steps:
     steps:
 
       - label: "eks"
+        agents:
+          memory: "4G"
         commands:
           - .ci/setenvconfig e2e/main
           - .ci/setenvconfig e2e/eks
@@ -73,6 +71,8 @@ steps:
     steps:
 
       - label: "gke 1.21"
+        agents:
+          memory: "4G"
         commands:
           - .ci/setenvconfig e2e/main
           - .ci/setenvconfig e2e/gke-k8s-versions 1.21
@@ -82,6 +82,8 @@ steps:
           - "eck-diagnostic-*.zip"
 
       - label: "gke 1.22"
+        agents:
+          memory: "4G"
         commands:
           - .ci/setenvconfig e2e/main
           - .ci/setenvconfig e2e/gke-k8s-versions 1.22
@@ -91,6 +93,8 @@ steps:
           - "eck-diagnostic-*.zip"
 
       - label: "gke 1.23"
+        agents:
+          memory: "4G"
         commands:
           - .ci/setenvconfig e2e/main
           - .ci/setenvconfig e2e/gke-k8s-versions 1.23
@@ -346,6 +350,8 @@ steps:
     steps:
 
       - label: "e2e/resilience"
+        agents:
+          memory: "4G"
         commands:
           - .ci/setenvconfig e2e/main
           - .ci/setenvconfig e2e/resilience

.buildkite/pipeline-release-helm.yml

@@ -0,0 +1,55 @@
+
+env:
+  DRY_RUN: true
+
+steps:
+  - group: helm-release
+    steps:
+
+      - label: "operator dev helm chart"
+        if: |
+          ( build.branch == "main" && build.source != "schedule" )
+          || build.tag != null
+          || build.message =~ /^buildkite test .*release.*/
+          || build.message == "release eck-operator helm charts"
+        depends_on:
+          - "build-helm-releaser-tool"
+        key: "eck-operator-dev-helm"
+        commands:
+          - buildkite-agent artifact download "bin/releaser" /usr/local/
+          - chmod u+x /usr/local/bin/releaser
+          - releaser --env=dev --charts-dir=deploy/eck-operator --dry-run=\$DRY_RUN
+
+      - label: "eck-resources dev helm charts"
+        if: |
+          ( build.branch == "main" && build.source != "schedule" )
+          || build.tag != null
+          || build.message =~ /^buildkite test .*release.*/
+          || build.message == "release eck-resources helm charts"
+        depends_on:
+          - "build-helm-releaser-tool"
+        key: "eck-resources-dev-helm"
+        commands:
+          - buildkite-agent artifact download "bin/releaser" /usr/local/
+          - chmod u+x /usr/local/bin/releaser
+          - releaser --env=dev --excludes=eck-operator --dry-run=\$DRY_RUN
+
+      - label: "operator prod helm chart"
+        if: build.message == "release eck-operator helm charts"
+        depends_on:
+          - "build-helm-releaser-tool"
+          - "eck-operator-dev-helm"
+        commands:
+          - buildkite-agent artifact download "bin/releaser" /usr/local/
+          - chmod u+x /usr/local/bin/releaser
+          - releaser --env=prod --charts-dir=deploy/eck-operator --dry-run=\$DRY_RUN
+
+      - label: "eck-resources prod helm charts"
+        if: build.message == "release eck-resources helm charts"
+        depends_on:
+          - "build-helm-releaser-tool"
+          - "eck-resources-dev-helm"
+        commands:
+          - buildkite-agent artifact download "bin/releaser" /usr/local/
+          - chmod u+x /usr/local/bin/releaser
+          - releaser --env=prod --excludes=eck-operator --dry-run=\$DRY_RUN

.buildkite/pipeline-release-operatorhub.yml

@@ -1,24 +1,8 @@
-
-agents:
-  image: docker.elastic.co/ci-agent-images/cloud-k8s-operator/buildkite-agent:latest
-  cpu: "4"
-  memory: "2G"
-
 steps:
-
   - group: redhat-release
-
     steps:
 
-      - label: "build operatorhub tool"
-        key: "build-operatorhub-tool"
-        commands:
-          - cd hack/operatorhub
-          - make build
-          - chmod u+x bin/operatorhub
-          - buildkite-agent artifact upload "bin/operator*"
-
-      - label: "Generate CRDs"
+      - label: "generate crds"
         key: "generate-crds"
         depends_on:
           - "build-operatorhub-tool"
@@ -31,20 +15,29 @@ steps:
         key: "redhat-container-push"
         depends_on:
           - "generate-crds"
+          - "build-operatorhub-tool"
         commands:
-          - buildkite-agent artifact download "bin/operator*" /usr/local/
-          - /usr/local/bin/operatorhub container push --dry-run=false
+          - buildkite-agent artifact download "bin/operatorhub" /usr/local/
+          - chmod u+x /usr/local/bin/operatorhub
+          - cd hack/operatorhub
+          - operatorhub container push --dry-run=\${DRY_RUN:-true}
 
       - label: "run container preflight"
         key: "redhat-preflight"
+        if: build.env("DRY_RUN") == "false"
         depends_on:
           - "redhat-container-push"
         commands:
-          - .buildkite/scripts/operatorhub/operations.sh preflight
+          - .buildkite/scripts/release/operatorhub.sh preflight
 
       - label: "run remaining release"
         key: "redhat-release"
         depends_on:
+          - "build-operatorhub-tool"
+          - "redhat-container-push"
           - "redhat-preflight"
         commands:
-          - .buildkite/scripts/operatorhub/operations.sh release
+          - buildkite-agent artifact download "config/*.yaml" .
+          - buildkite-agent artifact download "bin/operatorhub" /usr/local/
+          - chmod u+x /usr/local/bin/operatorhub
+          - .buildkite/scripts/release/operatorhub.sh release

.buildkite/pipeline-release.yml

@@ -0,0 +1,38 @@
+steps:
+  - group: release
+    steps:
+
+      - label: "k8s manifests"
+        if: |
+          ( build.branch == "main" && build.source != "schedule" )
+          || build.tag != null
+          || build.message =~ /^buildkite test .*release.*/
+        depends_on:
+          - "operator-image-build"
+        commands:
+          - buildkite-agent artifact download 'config/*.yaml' config/
+          - .buildkite/scripts/build/setenv.sh
+          - .buildkite/scripts/release/k8s-manifests.sh
+
+      # dry run helm charts release
+      - label: ":buildkite:"
+        if: build.message =~ /^buildkite test .*release.*/
+        command: buildkite-agent pipeline upload .buildkite/pipeline-release-helm.yml
+
+      # live helm charts release
+      - label: ":buildkite:"
+        if: |
+          build.message !~ /^buildkite test .*release.*/
+          && (
+            ( build.branch == "main" && build.source != "schedule" )
+            || build.tag != null
+            || build.message =~ /^release eck/
+          )
+        command: |
+          sed "s|DRY_RUN: true|DRY_RUN: false|" .buildkite/pipeline-release-helm.yml | buildkite-agent pipeline upload 
+
+      - label: ":buildkite:"
+        if: |
+          build.message =~ /^buildkite test .*operatorhub.*/
+          || build.message =~ /^release eck .*operatorhub.*/
+        command: buildkite-agent pipeline upload .buildkite/pipeline-release-operatorhub.yml