Merge branch 'master' into keep_field_list_until_field_caps_sees_fields

.ci/Dockerfile

@@ -1,7 +1,7 @@
 # NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
 # If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts
 
-ARG NODE_VERSION=14.15.1
+ARG NODE_VERSION=14.15.2
 
 FROM node:${NODE_VERSION} AS base
 

.ci/teamcity/bootstrap.sh

@@ -7,7 +7,7 @@ source "$(dirname "${0}")/util.sh"
 tc_start_block "Bootstrap"
 
 tc_start_block "yarn install and kbn bootstrap"
-verify_no_git_changes yarn kbn bootstrap --prefer-offline
+verify_no_git_changes yarn kbn bootstrap
 tc_end_block "yarn install and kbn bootstrap"
 
 tc_start_block "build kbn-pm"

.ci/teamcity/checks/bundle_limits.sh

@@ -4,4 +4,5 @@ set -euo pipefail
 
 source "$(dirname "${0}")/../util.sh"
 
-node scripts/build_kibana_platform_plugins --validate-limits
+checks-reporter-with-killswitch "Check Bundle Limits" \
+  node scripts/build_kibana_platform_plugins --validate-limits

.ci/teamcity/checks/commit.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+# Runs pre-commit hook script for the files touched in the last commit.
+# That way we can ensure a set of quick commit checks earlier as we removed
+# the pre-commit hook installation by default.
+# If files are more than 200 we will skip it and just use
+# the further ci steps that already check linting and file casing for the entire repo.
+checks-reporter-with-killswitch "Quick commit checks" \
+  "$(dirname "${0}")/commit_check_runner.sh"

.ci/teamcity/checks/commit_check_runner.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo "!!!!!!!! ATTENTION !!!!!!!!
+That check is intended to provide earlier CI feedback after we remove the automatic install for the local pre-commit hook.
+If you want, you can still manually install the pre-commit hook locally by running 'node scripts/register_git_hook locally'
+!!!!!!!!!!!!!!!!!!!!!!!!!!!
+"
+
+node scripts/precommit_hook.js --ref HEAD~1..HEAD --max-files 200 --verbose

.ci/teamcity/checks/jest_configs.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+checks-reporter-with-killswitch "Check Jest Configs" \
+  node scripts/check_jest_configs

.ci/teamcity/checks/plugins_with_circular_deps.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+checks-reporter-with-killswitch "Check Plugins With Circular Dependencies" \
+  node scripts/find_plugins_with_circular_deps

.ci/teamcity/oss/plugin_functional.sh

@@ -13,6 +13,21 @@ if [[ ! -d "target" ]]; then
 fi
 cd -
 
-./test/scripts/test/plugin_functional.sh
-./test/scripts/test/example_functional.sh
-./test/scripts/test/interpreter_functional.sh
+checks-reporter-with-killswitch "Plugin Functional Tests" \
+  node scripts/functional_tests \
+    --config test/plugin_functional/config.ts \
+    --bail \
+    --debug
+
+checks-reporter-with-killswitch "Example Functional Tests" \
+  node scripts/functional_tests \
+    --config test/examples/config.js \
+    --bail \
+    --debug
+
+checks-reporter-with-killswitch "Interpreter Functional Tests" \
+  node scripts/functional_tests \
+    --config test/interpreter_functional/config.ts \
+    --bail \
+    --debug \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR"

.ci/teamcity/setup_env.sh

@@ -25,12 +25,14 @@ tc_set_env FORCE_COLOR 1
 tc_set_env TEST_BROWSER_HEADLESS 1
 
 tc_set_env ELASTIC_APM_ENVIRONMENT ci
+tc_set_env ELASTIC_APM_TRANSACTION_SAMPLE_RATE 0.1
 
 if [[ "${KIBANA_CI_REPORTER_KEY_BASE64-}" ]]; then
   tc_set_env KIBANA_CI_REPORTER_KEY "$(echo "$KIBANA_CI_REPORTER_KEY_BASE64" | base64 -d)"
 fi
 
 if is_pr; then
+  tc_set_env ELASTIC_APM_ACTIVE false
   tc_set_env CHECKS_REPORTER_ACTIVE true
 
   # These can be removed once we're not supporting Jenkins and TeamCity at the same time
@@ -39,6 +41,7 @@ if is_pr; then
   tc_set_env ghprbActualCommit "$GITHUB_PR_TRIGGERED_SHA"
   tc_set_env BUILD_URL "$TEAMCITY_BUILD_URL"
 else
+  tc_set_env ELASTIC_APM_ACTIVE true
   tc_set_env CHECKS_REPORTER_ACTIVE false
 fi
 

.teamcity/pom.xml

@@ -46,13 +46,25 @@
         <enabled>true</enabled>
       </snapshots>
     </repository>
+    <repository>
+      <id>teamcity</id>
+      <url>https://artifactory.elstc.co/artifactory/teamcity</url>
+      <snapshots>
+        <enabled>true</enabled>
+        <updatePolicy>always</updatePolicy>
+      </snapshots>
+    </repository>
   </repositories>
 
   <pluginRepositories>
     <pluginRepository>
       <id>JetBrains</id>
       <url>https://download.jetbrains.com/teamcity-repository</url>
     </pluginRepository>
+    <pluginRepository>
+      <id>teamcity</id>
+      <url>https://artifactory.elstc.co/artifactory/teamcity</url>
+    </pluginRepository>
   </pluginRepositories>
 
   <build>
@@ -124,5 +136,10 @@
       <artifactId>junit</artifactId>
       <version>4.13</version>
     </dependency>
+    <dependency>
+      <groupId>co.elastic.teamcity</groupId>
+      <artifactId>teamcity-common</artifactId>
+      <version>1.0.0-SNAPSHOT</version>
+    </dependency>
   </dependencies>
 </project>

.teamcity/settings.kts

@@ -2,7 +2,7 @@ import jetbrains.buildServer.configs.kotlin.v2019_2.*
 import projects.Kibana
 import projects.KibanaConfiguration
 
-version = "2020.1"
+version = "2020.2"
 
 val config = KibanaConfiguration {
   agentNetwork = DslContext.getParameter("agentNetwork", "teamcity")

.teamcity/src/Agents.kt

@@ -0,0 +1,28 @@
+import co.elastic.teamcity.common.GoogleCloudAgent
+import co.elastic.teamcity.common.GoogleCloudAgentDiskType
+import co.elastic.teamcity.common.GoogleCloudProfile
+
+private val sizes = listOf("2", "4", "8", "16")
+
+val StandardAgents = sizes.map { size -> size to GoogleCloudAgent {
+  sourceImageFamily = "elastic-kibana-ci-ubuntu-1804-lts"
+  agentPrefix = "kibana-standard-$size-"
+  machineType = "n2-standard-$size"
+  diskSizeGb = 75
+  diskType = GoogleCloudAgentDiskType.SSD
+} }.toMap()
+
+val BuildAgent = GoogleCloudAgent {
+    sourceImageFamily = "elastic-kibana-ci-ubuntu-1804-lts"
+    agentPrefix = "kibana-c2-16-"
+    machineType = "c2-standard-16"
+    diskSizeGb = 250
+    diskType = GoogleCloudAgentDiskType.SSD
+}
+
+val CloudProfile = GoogleCloudProfile {
+  accessKeyId = "447fdd4d-7129-46b7-9822-2e57658c7422"
+
+  agents(StandardAgents)
+  agent(BuildAgent)
+}

.teamcity/src/Extensions.kt

@@ -1,9 +1,7 @@
+import co.elastic.teamcity.common.requireAgent
 import jetbrains.buildServer.configs.kotlin.v2019_2.*
-import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.notifications
 import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.ScriptBuildStep
 import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script
-import jetbrains.buildServer.configs.kotlin.v2019_2.ui.insert
-import projects.kibanaConfiguration
 
 fun BuildFeatures.junit(dirs: String = "target/**/TEST-*.xml") {
   feature {
@@ -13,40 +11,8 @@ fun BuildFeatures.junit(dirs: String = "target/**/TEST-*.xml") {
   }
 }
 
-fun ProjectFeatures.kibanaAgent(init: ProjectFeature.() -> Unit) {
-  feature {
-    type = "CloudImage"
-    param("network", kibanaConfiguration.agentNetwork)
-    param("subnet", kibanaConfiguration.agentSubnet)
-    param("growingId", "true")
-    param("agent_pool_id", "-2")
-    param("preemptible", "false")
-    param("sourceProject", "elastic-images-prod")
-    param("sourceImageFamily", "elastic-kibana-ci-ubuntu-1804-lts")
-    param("zone", "us-central1-a")
-    param("profileId", "kibana")
-    param("diskType", "pd-ssd")
-    param("machineCustom", "false")
-    param("maxInstances", "200")
-    param("imageType", "ImageFamily")
-    param("diskSizeGb", "75") // TODO
-    init()
-  }
-}
-
-fun ProjectFeatures.kibanaAgent(size: String, init: ProjectFeature.() -> Unit = {}) {
-  kibanaAgent {
-    id = "KIBANA_STANDARD_$size"
-    param("source-id", "kibana-standard-$size-")
-    param("machineType", "n2-standard-$size")
-    init()
-  }
-}
-
 fun BuildType.kibanaAgent(size: String) {
-  requirements {
-    startsWith("teamcity.agent.name", "kibana-standard-$size-", "RQ_AGENT_NAME")
-  }
+  requireAgent(StandardAgents[size]!!)
 }
 
 fun BuildType.kibanaAgent(size: Int) {

.teamcity/src/builds/Checks.kt

@@ -11,16 +11,18 @@ object Checks : BuildType({
   kibanaAgent(4)
 
   val checkScripts = mapOf(
+    "Quick Commit Checks" to ".ci/teamcity/checks/commit.sh",
     "Check Telemetry Schema" to ".ci/teamcity/checks/telemetry.sh",
     "Check TypeScript Projects" to ".ci/teamcity/checks/ts_projects.sh",
     "Check File Casing" to ".ci/teamcity/checks/file_casing.sh",
     "Check Licenses" to ".ci/teamcity/checks/licenses.sh",
     "Verify NOTICE" to ".ci/teamcity/checks/verify_notice.sh",
-    "Test Hardening" to ".ci/teamcity/checks/test_hardening.sh",
     "Check Types" to ".ci/teamcity/checks/type_check.sh",
+    "Check Jest Configs" to ".ci/teamcity/checks/jest_configs.sh",
     "Check Doc API Changes" to ".ci/teamcity/checks/doc_api_changes.sh",
     "Check Bundle Limits" to ".ci/teamcity/checks/bundle_limits.sh",
-    "Check i18n" to ".ci/teamcity/checks/i18n.sh"
+    "Check i18n" to ".ci/teamcity/checks/i18n.sh",
+    "Check Plugins With Circular Dependencies" to ".ci/teamcity/checks/plugins_with_circular_deps.sh"
   )
 
   steps {

.teamcity/src/builds/default/DefaultCiGroup.kt

@@ -1,5 +1,7 @@
 package builds.default
 
+import StandardAgents
+import co.elastic.teamcity.common.requireAgent
 import jetbrains.buildServer.configs.kotlin.v2019_2.*
 import runbld
 
@@ -11,5 +13,7 @@ class DefaultCiGroup(val ciGroup: Int = 0, init: BuildType.() -> Unit = {}) : De
     runbld("Default CI Group $ciGroup", "./.ci/teamcity/default/ci_group.sh $ciGroup")
   }
 
+  requireAgent(StandardAgents["4"]!!)
+
   init()
 })

.teamcity/src/builds/default/DefaultCiGroups.kt

@@ -3,7 +3,7 @@ package builds.default
 import dependsOn
 import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
 
-const val DEFAULT_CI_GROUP_COUNT = 10
+const val DEFAULT_CI_GROUP_COUNT = 11
 val defaultCiGroups = (1..DEFAULT_CI_GROUP_COUNT).map { DefaultCiGroup(it) }
 
 object DefaultCiGroups : BuildType({

.teamcity/src/builds/es_snapshots/Verify.kt

@@ -6,7 +6,7 @@ import builds.default.defaultCiGroups
 import builds.oss.OssBuild
 import builds.oss.OssPluginFunctional
 import builds.oss.ossCiGroups
-import builds.test.ApiServerIntegration
+import builds.oss.OssApiServerIntegration
 import builds.test.JestIntegration
 import dependsOn
 import jetbrains.buildServer.configs.kotlin.v2019_2.*
@@ -49,7 +49,7 @@ val defaultBuildsToClone = listOf(
 val defaultCloned = defaultBuildsToClone.map { cloneForVerify(it) }
 
 val integrationsBuildsToClone = listOf(
-  ApiServerIntegration,
+  OssApiServerIntegration,
   JestIntegration
 )
 

.teamcity/src/builds/test/ApiServerIntegration.kt → .teamcity/src/builds/oss/OssApiServerIntegration.kt

@@ -1,17 +1,13 @@
-package builds.test
+package builds.oss
 
-import addTestSettings
-import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
 import runbld
 
-object ApiServerIntegration : BuildType({
+object OssApiServerIntegration : OssFunctionalBase({
   name = "API/Server Integration"
   description = "Executes API and Server Integration Tests"
 
   steps {
     runbld("API Integration", "./.ci/teamcity/oss/api_integration.sh")
     runbld("Server Integration", "./.ci/teamcity/oss/server_integration.sh")
   }
-
-  addTestSettings()
 })

.teamcity/src/builds/test/AllTests.kt

@@ -1,5 +1,6 @@
 package builds.test
 
+import builds.oss.OssApiServerIntegration
 import dependsOn
 import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
 
@@ -8,5 +9,5 @@ object AllTests : BuildType({
   description = "All Non-Functional Tests"
   type = Type.COMPOSITE
 
-  dependsOn(QuickTests, Jest, XPackJest, JestIntegration, ApiServerIntegration)
+  dependsOn(QuickTests, Jest, XPackJest, JestIntegration, OssApiServerIntegration)
 })

.teamcity/src/builds/test/QuickTests.kt

@@ -12,7 +12,7 @@ object QuickTests : BuildType({
   kibanaAgent(2)
 
   val testScripts = mapOf(
-    "Test Hardening" to ".ci/teamcity/checkes/test_hardening.sh",
+    "Test Hardening" to ".ci/teamcity/checks/test_hardening.sh",
     "Test Projects" to ".ci/teamcity/tests/test_projects.sh",
     "Mocha Tests" to ".ci/teamcity/tests/mocha.sh"
   )