diff --git a/.buildkite/pipeline-resource-definitions/kibana-on-merge.yml b/.buildkite/pipeline-resource-definitions/kibana-on-merge.yml new file mode 100644 index 0000000000000..8952951a27a5b --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-on-merge.yml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-on-merge + description: 'Runs for each commit of Kibana, i.e. each time a PR is merged' + links: + - url: 'https://buildkite.com/elastic/kibana-on-merge' + title: Pipeline link +spec: + type: buildkite-pipeline + owner: 'group:kibana-operations' + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / on merge + description: 'Runs for each commit of Kibana, i.e. each time a PR is merged' + spec: + env: + SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts' + GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true' + GITHUB_COMMIT_STATUS_CONTEXT: buildkite/on-merge + REPORT_FAILED_TESTS_TO_GITHUB: 'true' + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + allow_rebuilds: true + branch_configuration: main 7.17 8.13 8.14 + default_branch: main + repository: elastic/kibana + pipeline_file: .buildkite/pipelines/on_merge.yml + skip_intermediate_builds: false + provider_settings: + build_branches: true + build_pull_requests: false + publish_commit_status: false + trigger_mode: code + build_tags: false + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: false + teams: + everyone: + access_level: BUILD_AND_READ + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ diff --git a/.buildkite/pipeline-resource-definitions/locations.yml b/.buildkite/pipeline-resource-definitions/locations.yml index bd7e2c3bb86bd..f9dc312a26582 100644 --- a/.buildkite/pipeline-resource-definitions/locations.yml +++ b/.buildkite/pipeline-resource-definitions/locations.yml @@ -22,6 +22,7 @@ spec: - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-fleet-packages-daily.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-performance-daily.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-performance-data-set-extraction-daily.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-pr.yml diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml index 7060035c41386..ae6c05721ae84 100644 --- a/.buildkite/pipelines/on_merge.yml +++ b/.buildkite/pipelines/on_merge.yml @@ -6,7 +6,10 @@ steps: label: Pre-Build timeout_in_minutes: 10 agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 retry: automatic: - exit_status: '*' @@ -16,7 +19,11 @@ steps: - label: 'Triggering changes-based pipelines' branches: main agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 + # TODO: this can probably be deleted after the migration https://github.com/elastic/kibana-operations/issues/15 plugins: - chronotc/monorepo-diff#v2.0.4: watch: @@ -26,12 +33,19 @@ steps: command: 'ts-node .buildkite/scripts/steps/trigger_pipeline.ts kibana-buildkite-pipelines-deploy main' label: 'Trigger pipeline deploy' agents: - queue: 'kibana-default' + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 - command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh label: Build Kibana Distribution and Plugins agents: - queue: n2-16-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-16 + preemptible: true key: build timeout_in_minutes: 60 retry: @@ -42,7 +56,11 @@ steps: - command: .buildkite/scripts/steps/quick_checks.sh label: 'Quick Checks' agents: - queue: n2-2-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 + preemptible: true key: quick_checks timeout_in_minutes: 60 retry: @@ -56,7 +74,11 @@ steps: label: Check Public API Docs key: public-api-docs agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 80 retry: automatic: @@ -68,7 +90,10 @@ steps: - command: .buildkite/scripts/steps/ci_stats_ready.sh label: Mark CI Stats as ready agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 timeout_in_minutes: 10 depends_on: - build @@ -81,7 +106,10 @@ steps: - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh label: 'Pick Test Group Run Order' agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 timeout_in_minutes: 10 env: JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh' @@ -95,7 +123,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_serverless_entity_analytics.sh label: 'Serverless Entity Analytics - Security Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -109,7 +141,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh label: 'Serverless Explore - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -123,7 +159,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh label: 'Serverless Investigations - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -137,7 +177,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh label: 'Serverless Rule Management - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -151,7 +195,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -165,7 +213,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_rule_management.sh label: 'Rule Management - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -179,7 +231,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh label: 'Rule Management - Prebuilt Rules - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -193,7 +249,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine.sh label: 'Serverless Detection Engine - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -207,7 +267,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh label: 'Serverless Detection Engine - Exceptions - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -221,7 +285,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_detection_engine.sh label: 'Detection Engine - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -235,7 +303,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_detection_engine_exceptions.sh label: 'Detection Engine - Exceptions - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -249,7 +321,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh label: 'Serverless AI Assistant - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -263,7 +339,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_ai_assistant.sh label: 'AI Assistant - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -277,7 +357,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_entity_analytics.sh label: 'Entity Analytics - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -291,7 +375,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_explore.sh label: 'Explore - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -305,7 +393,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh label: 'Investigations - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -319,7 +411,11 @@ steps: - command: .buildkite/scripts/steps/functional/threat_intelligence.sh label: 'Threat Intelligence Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -333,7 +429,11 @@ steps: - command: .buildkite/scripts/steps/functional/osquery_cypress.sh label: 'Osquery Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -347,7 +447,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh label: 'Serverless Osquery Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -361,7 +465,13 @@ steps: - command: .buildkite/scripts/steps/functional/defend_workflows.sh label: 'Defend Workflows Cypress Tests' agents: - queue: n2-4-virt + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + enableNestedVirtualization: true + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-4 depends_on: - build - quick_checks @@ -375,7 +485,13 @@ steps: - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh label: 'Defend Workflows Cypress Tests on Serverless' agents: - queue: n2-4-virt + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + enableNestedVirtualization: true + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-4 depends_on: - build - quick_checks @@ -393,12 +509,19 @@ steps: - build - quick_checks agents: - queue: 'kibana-default' + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 - command: .buildkite/scripts/steps/lint.sh label: 'Linting' agents: - queue: n2-8-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + preemptible: true key: linting timeout_in_minutes: 60 retry: @@ -409,7 +532,11 @@ steps: - command: .buildkite/scripts/steps/lint_with_types.sh label: 'Linting (with types)' agents: - queue: n2-16-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-16 + preemptible: true key: linting_with_types timeout_in_minutes: 90 retry: @@ -420,7 +547,11 @@ steps: - command: .buildkite/scripts/steps/checks.sh label: 'Checks' agents: - queue: n2-2-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 + preemptible: true timeout_in_minutes: 60 retry: automatic: @@ -430,7 +561,11 @@ steps: - command: .buildkite/scripts/steps/check_types.sh label: 'Check Types' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 70 retry: automatic: @@ -440,7 +575,11 @@ steps: - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh label: 'Build Storybooks' agents: - queue: n2-8-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + preemptible: true key: storybooks timeout_in_minutes: 80 retry: @@ -451,7 +590,11 @@ steps: - command: .buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh label: 'Populate local dev bazel cache (Linux)' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 30 retry: automatic: @@ -461,7 +604,11 @@ steps: - command: .buildkite/scripts/steps/archive_so_migration_snapshot.sh target/plugin_so_types_snapshot.json label: 'Extract Saved Object migration plugin types' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true artifact_paths: "target/plugin_so_types_snapshot.json" @@ -472,7 +619,10 @@ steps: label: Post-Build timeout_in_minutes: 10 agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 - wait @@ -480,4 +630,7 @@ steps: label: Trigger container image build timeout_in_minutes: 10 agents: - queue: 'kibana-default' + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2