[kbn/es-archiver] move to a package

.github/CODEOWNERS

@@ -115,7 +115,6 @@
 /src/dev/ @elastic/kibana-operations
 /src/setup_node_env/ @elastic/kibana-operations
 /src/optimize/ @elastic/kibana-operations
-/src/es_archiver/ @elastic/kibana-operations
 /packages/*eslint*/ @elastic/kibana-operations
 /packages/*babel*/ @elastic/kibana-operations
 /packages/kbn-dev-utils*/ @elastic/kibana-operations
@@ -124,6 +123,7 @@
 /packages/kbn-pm/ @elastic/kibana-operations
 /packages/kbn-test/ @elastic/kibana-operations
 /packages/kbn-ui-shared-deps/ @elastic/kibana-operations
+/packages/kbn-es-archiver/ @elastic/kibana-operations
 /src/legacy/server/keystore/ @elastic/kibana-operations
 /src/legacy/server/pid/ @elastic/kibana-operations
 /src/legacy/server/sass/ @elastic/kibana-operations

package.json

@@ -300,6 +300,7 @@
     "@elastic/makelogs": "^6.0.0",
     "@kbn/dev-utils": "1.0.0",
     "@kbn/es": "1.0.0",
+    "@kbn/es-archiver": "1.0.0",
     "@kbn/eslint-import-resolver-kibana": "2.0.0",
     "@kbn/eslint-plugin-eslint": "1.0.0",
     "@kbn/expect": "1.0.0",

packages/kbn-es-archiver/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@kbn/es-archiver",
+  "version": "1.0.0",
+  "license": "Apache-2.0",
+  "main": "target/index.js",
+  "scripts": {
+    "kbn:bootstrap": "tsc",
+    "kbn:watch": "tsc --watch"
+  },
+  "dependencies": {
+    "@kbn/dev-utils": "1.0.0",
+    "elasticsearch": "^16.7.0"
+  },
+  "devDependencies": {
+    "@types/elasticsearch": "^5.0.33"
+  }
+}

src/es_archiver/actions/edit.ts → packages/kbn-es-archiver/src/actions/edit.ts

@@ -24,7 +24,7 @@ import { promisify } from 'util';
 import globby from 'globby';
 import { ToolingLog } from '@kbn/dev-utils';
 
-import { createPromiseFromStreams } from '../../legacy/utils';
+import { createPromiseFromStreams } from '../lib/streams';
 
 const unlinkAsync = promisify(Fs.unlink);
 

src/es_archiver/actions/load.ts → packages/kbn-es-archiver/src/actions/load.ts

@@ -23,7 +23,7 @@ import { Readable } from 'stream';
 import { ToolingLog, KbnClient } from '@kbn/dev-utils';
 import { Client } from 'elasticsearch';
 
-import { createPromiseFromStreams, concatStreamProviders } from '../../legacy/utils';
+import { createPromiseFromStreams, concatStreamProviders } from '../lib/streams';
 
 import {
   isGzip,

src/es_archiver/actions/rebuild_all.ts → packages/kbn-es-archiver/src/actions/rebuild_all.ts

@@ -23,7 +23,7 @@ import { Readable, Writable } from 'stream';
 import { fromNode } from 'bluebird';
 import { ToolingLog } from '@kbn/dev-utils';
 
-import { createPromiseFromStreams } from '../../legacy/utils';
+import { createPromiseFromStreams } from '../lib/streams';
 import {
   prioritizeMappings,
   readDirectory,

src/es_archiver/actions/save.ts → packages/kbn-es-archiver/src/actions/save.ts

@@ -23,7 +23,7 @@ import { Readable, Writable } from 'stream';
 import { Client } from 'elasticsearch';
 import { ToolingLog } from '@kbn/dev-utils';
 
-import { createListStream, createPromiseFromStreams } from '../../legacy/utils';
+import { createListStream, createPromiseFromStreams } from '../lib/streams';
 import {
   createStats,
   createGenerateIndexRecordsStream,

src/es_archiver/actions/unload.ts → packages/kbn-es-archiver/src/actions/unload.ts

@@ -23,7 +23,7 @@ import { Readable, Writable } from 'stream';
 import { Client } from 'elasticsearch';
 import { ToolingLog, KbnClient } from '@kbn/dev-utils';
 
-import { createPromiseFromStreams } from '../../legacy/utils';
+import { createPromiseFromStreams } from '../lib/streams';
 import {
   isGzip,
   createStats,

packages/kbn-es-archiver/src/cli.ts

@@ -0,0 +1,244 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/** ***********************************************************
+ *
+ *  Run `node scripts/es_archiver --help` for usage information
+ *
+ *************************************************************/
+
+import Path from 'path';
+import Url from 'url';
+import readline from 'readline';
+
+import { RunWithCommands, createFlagError } from '@kbn/dev-utils';
+import { readConfigFile } from '@kbn/test';
+import legacyElasticsearch from 'elasticsearch';
+
+import { EsArchiver } from './es_archiver';
+
+const resolveConfigPath = (v: string) => Path.resolve(process.cwd(), v);
+const defaultConfigPath = resolveConfigPath('test/functional/config.js');
+
+export function runCli() {
+  new RunWithCommands({
+    description: 'CLI to manage archiving/restoring data in elasticsearch',
+    globalFlags: {
+      string: ['es-url', 'kibana-url', 'dir', 'config'],
+      help: `
+        --config           path to an FTR config file that sets --es-url, --kibana-url, and --dir
+                             default: ${defaultConfigPath}
+        --es-url           url for Elasticsearch, prefer the --config flag
+        --kibana-url       url for Kibana, prefer the --config flag
+        --dir              where arechives are stored, prefer the --config flag
+      `,
+    },
+    async extendContext({ log, flags, addCleanupTask }) {
+      const configPath = flags.config || defaultConfigPath;
+      if (typeof configPath !== 'string') {
+        throw createFlagError('--config must be a string');
+      }
+      const config = await readConfigFile(log, Path.resolve(configPath));
+
+      let esUrl = flags['es-url'];
+      if (esUrl && typeof esUrl !== 'string') {
+        throw createFlagError('--es-url must be a string');
+      }
+      if (!esUrl && config) {
+        esUrl = Url.format(config.get('servers.elasticsearch'));
+      }
+      if (!esUrl) {
+        throw createFlagError('--es-url or --config must be defined');
+      }
+
+      let kibanaUrl = flags['kibana-url'];
+      if (kibanaUrl && typeof kibanaUrl !== 'string') {
+        throw createFlagError('--kibana-url must be a string');
+      }
+      if (!kibanaUrl && config) {
+        kibanaUrl = Url.format(config.get('servers.kibana'));
+      }
+      if (!kibanaUrl) {
+        throw createFlagError('--kibana-url or --config must be defined');
+      }
+
+      let dir = flags.dir;
+      if (dir && typeof dir !== 'string') {
+        throw createFlagError('--dir must be a string');
+      }
+      if (!dir && config) {
+        dir = Path.resolve(config.get('esArchiver.directory'));
+      }
+      if (!dir) {
+        throw createFlagError('--dir or --config must be defined');
+      }
+
+      const client = new legacyElasticsearch.Client({
+        host: esUrl,
+        log: flags.verbose ? 'trace' : [],
+      });
+      addCleanupTask(() => client.close());
+
+      const esArchiver = new EsArchiver({
+        log,
+        client,
+        dataDir: dir,
+        kibanaUrl,
+      });
+
+      return {
+        esArchiver,
+      };
+    },
+  })
+    .command({
+      name: 'save',
+      usage: 'save [name] [...indices]',
+      description: `
+        archive the [indices ...] into the --dir with [name]
+
+        Example:
+          Save all [logstash-*] indices from http://localhost:9200 to [snapshots/my_test_data] directory
+
+          WARNING: If the [my_test_data] snapshot exists it will be deleted!
+
+          $ node scripts/es_archiver save my_test_data logstash-* --dir snapshots
+      `,
+      flags: {
+        boolean: ['raw'],
+        help: `
+          --raw              don't gzip the archives
+        `,
+      },
+      async run({ flags, esArchiver }) {
+        const [name, ...indices] = flags._;
+        if (!name) {
+          throw createFlagError('missing [name] argument');
+        }
+        if (!indices.length) {
+          throw createFlagError('missing [...indices] arguments');
+        }
+
+        const raw = flags.raw;
+        if (typeof raw !== 'boolean') {
+          throw createFlagError('--raw does not take a value');
+        }
+
+        await esArchiver.save(name, indices, { raw });
+      },
+    })
+    .command({
+      name: 'load',
+      usage: 'load [name]',
+      description: `
+        load the archive in --dir with [name]
+
+        Example:
+          Load the [my_test_data] snapshot from the archive directory and elasticsearch instance defined
+          in the [test/functional/config.js] config file
+
+          WARNING: If the indices exist already they will be deleted!
+
+          $ node scripts/es_archiver load my_test_data --config test/functional/config.js
+      `,
+      flags: {
+        boolean: ['use-create'],
+        help: `
+          --use-create       use create instead of index for loading documents
+        `,
+      },
+      async run({ flags, esArchiver }) {
+        const [name] = flags._;
+        if (!name) {
+          throw createFlagError('missing [name] argument');
+        }
+        if (flags._.length > 1) {
+          throw createFlagError(`unknown extra arguments: [${flags._.slice(1).join(', ')}]`);
+        }
+
+        const useCreate = flags['use-create'];
+        if (typeof useCreate !== 'boolean') {
+          throw createFlagError('--use-create does not take a value');
+        }
+
+        await esArchiver.load(name, { useCreate });
+      },
+    })
+    .command({
+      name: 'unload',
+      usage: 'unload [name]',
+      description: 'remove indices created by the archive in --dir with [name]',
+      async run({ flags, esArchiver }) {
+        const [name] = flags._;
+        if (!name) {
+          throw createFlagError('missing [name] argument');
+        }
+        if (flags._.length > 1) {
+          throw createFlagError(`unknown extra arguments: [${flags._.slice(1).join(', ')}]`);
+        }
+
+        await esArchiver.unload(name);
+      },
+    })
+    .command({
+      name: 'edit',
+      usage: 'edit [prefix]',
+      description:
+        'extract the archives under the prefix, wait for edits to be completed, and then recompress the archives',
+      async run({ flags, esArchiver }) {
+        const [prefix] = flags._;
+        if (!prefix) {
+          throw createFlagError('missing [prefix] argument');
+        }
+        if (flags._.length > 1) {
+          throw createFlagError(`unknown extra arguments: [${flags._.slice(1).join(', ')}]`);
+        }
+
+        await esArchiver.edit(prefix, async () => {
+          const rl = readline.createInterface({
+            input: process.stdin,
+            output: process.stdout,
+          });
+
+          await new Promise((resolveInput) => {
+            rl.question(`Press enter when you're done`, () => {
+              rl.close();
+              resolveInput();
+            });
+          });
+        });
+      },
+    })
+    .command({
+      name: 'empty-kibana-index',
+      description:
+        '[internal] Delete any Kibana indices, and initialize the Kibana index as Kibana would do on startup.',
+      async run({ esArchiver }) {
+        await esArchiver.emptyKibanaIndex();
+      },
+    })
+    .command({
+      name: 'rebuild-all',
+      description: '[internal] read and write all archives in --dir to remove any inconsistencies',
+      async run({ esArchiver }) {
+        await esArchiver.rebuildAll();
+      },
+    })
+    .execute();
+}

src/es_archiver/index.ts → packages/kbn-es-archiver/src/index.ts

@@ -18,3 +18,4 @@
  */
 
 export { EsArchiver } from './es_archiver';
+export * from './cli';

src/es_archiver/lib/archives/__tests__/format.ts → packages/kbn-es-archiver/src/lib/archives/__tests__/format.ts

@@ -22,11 +22,7 @@ import { createGunzip } from 'zlib';
 
 import expect from '@kbn/expect';
 
-import {
-  createListStream,
-  createPromiseFromStreams,
-  createConcatStream,
-} from '../../../../legacy/utils';
+import { createListStream, createPromiseFromStreams, createConcatStream } from '../../streams';
 
 import { createFormatArchiveStreams } from '../format';
 

src/es_archiver/lib/archives/__tests__/parse.ts → packages/kbn-es-archiver/src/lib/archives/__tests__/parse.ts

@@ -22,11 +22,7 @@ import { createGzip } from 'zlib';
 
 import expect from '@kbn/expect';
 
-import {
-  createConcatStream,
-  createListStream,
-  createPromiseFromStreams,
-} from '../../../../legacy/utils';
+import { createConcatStream, createListStream, createPromiseFromStreams } from '../../streams';
 
 import { createParseArchiveStreams } from '../parse';
 
@@ -109,7 +105,7 @@ describe('esArchiver createParseArchiveStreams', () => {
               Buffer.from('{"a": 2}\n\n'),
             ]),
             ...createParseArchiveStreams({ gzip: false }),
-            createConcatStream(),
+            createConcatStream([]),
           ] as [Readable, ...Writable[]]);
           throw new Error('should have failed');
         } catch (err) {
@@ -172,7 +168,7 @@ describe('esArchiver createParseArchiveStreams', () => {
           await createPromiseFromStreams([
             createListStream([Buffer.from('{"a": 1}')]),
             ...createParseArchiveStreams({ gzip: true }),
-            createConcatStream(),
+            createConcatStream([]),
           ] as [Readable, ...Writable[]]);
           throw new Error('should have failed');
         } catch (err) {

src/es_archiver/lib/archives/format.ts → packages/kbn-es-archiver/src/lib/archives/format.ts

@@ -21,7 +21,7 @@ import { createGzip, Z_BEST_COMPRESSION } from 'zlib';
 import { PassThrough } from 'stream';
 import stringify from 'json-stable-stringify';
 
-import { createMapStream, createIntersperseStream } from '../../../legacy/utils';
+import { createMapStream, createIntersperseStream } from '../streams';
 import { RECORD_SEPARATOR } from './constants';
 
 export function createFormatArchiveStreams({ gzip = false }: { gzip?: boolean } = {}) {