Support saved searches in inliner script

README.md

@@ -1,38 +1,39 @@
 # visualizations integrations tools
 
 This package currently has several functionalities:
-* Analyzes the usage of legacy visualization types in beats and integrations dashboard and indexes them into an Elasticsearch instance.
-* Inlines the visualizations in a given directory as by values panels
-* Tracks the usage of @elastic/charts vs the Lens embeddable in the Kibana code base
+
+- Analyzes the usage of legacy visualization types in the beats and integrations dashboard and indexes them into an Elasticsearch instance.
+- Inlines the visualizations in a given directory as "by values" panels
+- Tracks the usage of @elastic/charts vs the Lens embeddable in the Kibana code base
 
 ## Prep
 
-* Have locally installed node (tested with `16.14.2`)
-* Init submodules using `git submodule update --init --recursive`
-* Install dependencies using `yarn`
+- Have locally installed `node` (tested with `16.14.2`)
+- Init submodules using `git submodule update --init --recursive`
+- Install dependencies using `yarn`
 
 ## Legacy vis analyzer usage
 
-* [Install go](https://go.dev/doc/install)
-* Run `ELASTICSEARCH_URL="<elasticsearch connection string>" go run index.go`
-* Import `dataview.ndjson` to have a bunch of runtime fields analyzing the structure
-
+- [Install Go](https://go.dev/doc/install)
+- Run `ELASTICSEARCH_URL="<elasticsearch connection string>" go run index.go`
+- Import `dataview.ndjson` to have a bunch of runtime fields analyzing the structure
 
 ## Code analyzer usage
 
-* Run `ES="<elasticsearch connection string>" node index_kibana.js`
+- Run `ES="<elasticsearch connection string>" node index_kibana.js`
 
 ## Inliner usage
 
-
 The inliner takes all "by reference" visualizations in a given directory, uses the provided running Kibana instance to migrate them to the latest version, migrates the dashboard saved object as well, then transforms the by-reference visualizations into by-value panels, deletes the visualization json files and updates the dashboard json files.
 
 Important notes:
-* Using the inliner script will make the dashboards incompatible with earlier versions of the stack - e.g. if it has been ran with a stack version 8.2, then the new dashboard json files will only work on version 8.2 and newer
-* For old dashboards (prior to 7.10), some "agg based" visualizations might break if a target version of 7.17 or 8.0 is used. In these cases, please use at least a stack version of 8.1
-
-* Run `KIBANA="<kibana connection string>" node inline.js <path to kibana folder>` (e.g. `./integrations/packages/system/kibana/`)
-  * The kibana connection string has to include the password (for instances with security enabled) and the base path (for instances with configured base path), for example `KIBANA="http://elastic:changeme@localhost:5901/mgp"`
-* Review changes in submodule repo
-  * This review should include loading the dashboard into an instance with data to make sure everything is displayed properly
-* If everything works fine, create PR
+
+- Using the inliner script will make the dashboards incompatible with earlier versions of the stack e.g., if it has been run with a stack version 8.2, then the new dashboard JSON files will only work on version 8.2 and above.
+- For old dashboards (prior to 7.10), some "agg based" visualizations might break if a target version of 7.17 or 8.0 is used. In these cases, please use at least a stack version of 8.1
+
+- Run `KIBANA="<kibana connection string>" node inline.js <path to kibana folder>` (e.g. `./integrations/packages/system/kibana/`)
+  - The Kibana connection string has to include the password (for instances with security enabled) and the base path (for instances with configured base path), for example `KIBANA="http://elastic:changeme@localhost:5601/mgp"`
+  - You may need to run `export NODE_TLS_REJECT_UNAUTHORIZED=0` if you are connecting to Kibana over TLS (https)
+- Review changes in the submodule repo
+  - This review should include loading the dashboard into an instance with data to make sure everything is displayed properly
+- If everything works fine, create a PR

inline.js

@@ -60,6 +60,7 @@ function rehydrateAttributes(attributes) {
   const migratedVisualizations = await migrateSavedObjects("visualization");
   const migratedLens = await migrateSavedObjects("lens");
   const migratedMap = await migrateSavedObjects("map");
+  const migratedSearch = await migrateSavedObjects("search");
 
   const dashboardPath = `${folderPath}/dashboard`;
   const dExists = fs.existsSync(dashboardPath);
@@ -68,6 +69,7 @@ function rehydrateAttributes(attributes) {
   const dashboards = dashboardPaths.map((d) =>
     JSON.parse(fs.readFileSync(`${dashboardPath}/${d}`, { encoding: "utf8" }))
   );
+
   const response3 = await axios.post(
     `${baseUrl}/api/saved_objects/_bulk_create?overwrite=true`,
     dashboards.map(
@@ -97,7 +99,12 @@ function rehydrateAttributes(attributes) {
       },
     }
   );
-  let counter = { visualization: new Set(), lens: new Set(), map: new Set() };
+  let counter = {
+    visualization: new Set(),
+    lens: new Set(),
+    map: new Set(),
+    search: new Set(),
+  };
   const inlinedDashboards = response4.data.saved_objects.map((d) => {
     console.log(`Processing dashboard ${d.attributes.title}`);
     const attributes = d.attributes;
@@ -112,7 +119,7 @@ function rehydrateAttributes(attributes) {
         const visToInline = migratedVisualizations.get(ref.id);
         const visState = JSON.parse(visToInline.attributes.visState);
         p.version = visToInline.migrationVersion.visualization;
-        p.type = 'visualization';
+        p.type = "visualization";
         p.embeddableConfig.savedVis = {
           title: visToInline.attributes.title,
           description: visToInline.attributes.description,
@@ -145,8 +152,11 @@ function rehydrateAttributes(attributes) {
       } else if (ref && migratedLens.has(ref.id)) {
         const visToInline = migratedLens.get(ref.id);
         p.version = visToInline.migrationVersion.lens;
-        p.type = 'lens';
-        p.embeddableConfig.attributes = {...visToInline.attributes, references: visToInline.references };
+        p.type = "lens";
+        p.embeddableConfig.attributes = {
+          ...visToInline.attributes,
+          references: visToInline.references,
+        };
         delete p.panelRefName;
         references.splice(references.indexOf(ref), 1);
         references.push(
@@ -163,7 +173,7 @@ function rehydrateAttributes(attributes) {
       } else if (ref && migratedMap.has(ref.id)) {
         const visToInline = migratedMap.get(ref.id);
         p.version = visToInline.migrationVersion.map;
-        p.type = 'map';
+        p.type = "map";
         p.embeddableConfig.attributes = {
           title: visToInline.attributes.title,
           description: visToInline.attributes.description,
@@ -184,6 +194,27 @@ function rehydrateAttributes(attributes) {
           `Inlined a map, pushed ${visToInline.references.length} inner references`
         );
         counter.map.add(ref.id);
+      } else if (ref && migratedSearch.has(ref.id)) {
+        const searchToInline = migratedSearch.get(ref.id);
+        p.version = searchToInline.migrationVersion.search;
+        p.type = "search";
+        p.embeddableConfig.attributes = {
+          ...searchToInline.attributes,
+          references: searchToInline.references,
+        };
+        delete p.panelRefName;
+        references.splice(references.indexOf(ref), 1);
+        references.push(
+          ...searchToInline.references.map((r) => ({
+            type: r.type,
+            name: `${p.panelIndex}:${r.name}`,
+            id: r.id,
+          }))
+        );
+        console.log(
+          `Inlined a search, pushed ${searchToInline.references.length} inner references`
+        );
+        counter.search.add(ref.id);
       } else {
         if (!ref) {
           if (p.type === undefined) {
@@ -204,9 +235,10 @@ function rehydrateAttributes(attributes) {
   console.log(`Inlined ${counter.visualization.size} visualizations`);
   console.log(`Inlined ${counter.map.size} maps`);
   console.log(`Inlined ${counter.lens.size} lenses`);
+  console.log(`Inlined ${counter.search.size} searches`);
   if (counter.visualization.size !== migratedVisualizations.size) {
     console.log(
-      `Some visualizations did not get inlined! ${counter.visualization.size}/${migratedVisualizations.size}`
+      `Some legacy visualizations did not get inlined! ${counter.visualization.size}/${migratedVisualizations.size}`
     );
     [...migratedVisualizations.values()].map((v) => {
       if (!counter.visualization.has(v.id)) {
@@ -230,6 +262,14 @@ function rehydrateAttributes(attributes) {
       }
     });
   }
+  if (counter.search.size !== migratedSearch.size) {
+    console.log("Some searches did not get inlined!");
+    [...migratedSearch.values()].map((v) => {
+      if (!counter.search.has(v.id)) {
+        console.log(`Did not inline ${v.id} anywhere`);
+      }
+    });
+  }
   if (fs.existsSync(`${folderPath}/visualization`)) {
     console.log("Removing visualization folder");
     fs.rmSync(`${folderPath}/visualization`, { force: true, recursive: true });
@@ -242,6 +282,10 @@ function rehydrateAttributes(attributes) {
     console.log("Removing lens folder");
     fs.rmSync(`${folderPath}/lens`, { force: true, recursive: true });
   }
+  if (fs.existsSync(`${folderPath}/search`)) {
+    console.log("Removing search folder");
+    fs.rmSync(`${folderPath}/search`, { force: true, recursive: true });
+  }
   console.log("Writing back dashboards");
   inlinedDashboards.forEach((d) => {
     fs.writeFileSync(