[APM] .apm-agent-configuration is not created if Kibana is started wh…

…ile ES is not ready (#61610) * adding retry functionaty when creating index * adding p-retry module to x-pack package.json * addressing pr comments Co-authored-by: Elastic Machine <[email protected]>
elastic · Apr 1, 2020 · a8425e2 · a8425e2
1 parent 36ff703
commit a8425e2
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 20 deletions.
diff --git a/x-pack/package.json b/x-pack/package.json
@@ -289,6 +289,7 @@
     "object-path-immutable": "^3.1.1",
     "oboe": "^2.1.4",
     "oppsy": "^2.0.0",
+    "p-retry": "^4.2.0",
     "papaparse": "^4.6.3",
     "pdfmake": "^0.1.63",
     "pluralize": "3.1.0",

diff --git a/x-pack/plugins/apm/server/lib/helpers/create_or_update_index.ts b/x-pack/plugins/apm/server/lib/helpers/create_or_update_index.ts
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-
+import pRetry from 'p-retry';
 import { IClusterClient, Logger } from 'src/core/server';
 import { CallCluster } from 'src/legacy/core_plugins/elasticsearch';
 
@@ -34,27 +34,39 @@ export async function createOrUpdateIndex({
   logger: Logger;
 }) {
   try {
-    const { callAsInternalUser } = esClient;
-    const indexExists = await callAsInternalUser('indices.exists', { index });
-    const result = indexExists
-      ? await updateExistingIndex({
-          index,
-          callAsInternalUser,
-          mappings
-        })
-      : await createNewIndex({
-          index,
-          callAsInternalUser,
-          mappings
-        });
+    /*
+     * In some cases we could be trying to create an index before ES is ready.
+     * When this happens, we retry creating the index with exponential backoff.
+     * We use retry's default formula, meaning that the first retry happens after 2s,
+     * the 5th after 32s, and the final attempt after around 17m. If the final attempt fails,
+     * the error is logged to the console.
+     * See https://github.com/sindresorhus/p-retry and https://github.com/tim-kos/node-retry.
+     */
+    await pRetry(async () => {
+      const { callAsInternalUser } = esClient;
+      const indexExists = await callAsInternalUser('indices.exists', { index });
+      const result = indexExists
+        ? await updateExistingIndex({
+            index,
+            callAsInternalUser,
+            mappings
+          })
+        : await createNewIndex({
+            index,
+            callAsInternalUser,
+            mappings
+          });
 
-    if (!result.acknowledged) {
-      const resultError =
-        result && result.error && JSON.stringify(result.error);
-      throw new Error(resultError);
-    }
+      if (!result.acknowledged) {
+        const resultError =
+          result && result.error && JSON.stringify(result.error);
+        throw new Error(resultError);
+      }
+    });
   } catch (e) {
-    logger.error(`Could not create APM index: '${index}'. Error: ${e.message}`);
+    logger.error(
+      `Could not create APM index: '${index}'. Error: ${e.message}.`
+    );
   }
 }
 

diff --git a/yarn.lock b/yarn.lock
@@ -4603,6 +4603,11 @@
     "@types/tough-cookie" "*"
     form-data "^2.5.0"
 
+"@types/retry@^0.12.0":
+  version "0.12.0"
+  resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.0.tgz#2b35eccfcee7d38cd72ad99232fbd58bffb3c84d"
+  integrity sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==
+
 "@types/seedrandom@>=2.0.0 <4.0.0":
   version "2.4.28"
   resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.28.tgz#9ce8fa048c1e8c85cb71d7fe4d704e000226036f"
@@ -21924,6 +21929,14 @@ p-retry@^3.0.1:
   dependencies:
     retry "^0.12.0"
 
+p-retry@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-4.2.0.tgz#ea9066c6b44f23cab4cd42f6147cdbbc6604da5d"
+  integrity sha512-jPH38/MRh263KKcq0wBNOGFJbm+U6784RilTmHjB/HM9kH9V8WlCpVUcdOmip9cjXOh6MxZ5yk1z2SjDUJfWmA==
+  dependencies:
+    "@types/retry" "^0.12.0"
+    retry "^0.12.0"
+
 p-some@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/p-some/-/p-some-2.0.1.tgz#65d87c8b154edbcf5221d167778b6d2e150f6f06"