Using re2 for Timelion regular expressions (#67416) (#68795)

* Revert "Revert "Using re2 for Timelion regular expressions (#55208)""

This reverts commit c90293d.

* Updating re2 to 1.14.0. Still need to update build patching

* Extract the gzip to the destination, supporting multiple extract methods

* Adding 'node' to jest's moduleFileExtensions

'node' is in the defaults, not sure why we aren't using the defaults...
https://jestjs.io/docs/en/configuration#modulefileextensions-arraystring

Co-authored-by: Elastic Machine <[email protected]>

Co-authored-by: Elastic Machine <[email protected]>

.gitignore

@@ -4,6 +4,7 @@
 /.es
 .DS_Store
 .node_binaries
+.native_modules
 node_modules
 !/src/dev/npm/__tests__/fixtures/fixture1/node_modules
 !/src/dev/notice/__fixtures__/node_modules

package.json

@@ -213,6 +213,7 @@
     "pug": "^2.0.3",
     "querystring-browser": "1.0.4",
     "raw-loader": "0.5.1",
+    "re2": "1.14.0",
     "react": "^16.6.0",
     "react-addons-shallow-compare": "15.6.2",
     "react-anything-sortable": "^1.7.4",

src/dev/build/build_distributables.js

@@ -44,6 +44,7 @@ import {
   ExtractNodeBuildsTask,
   InstallDependenciesTask,
   OptimizeBuildTask,
+  PatchNativeModulesTask,
   RemovePackageJsonDepsTask,
   RemoveWorkspacesTask,
   TranspileBabelTask,
@@ -127,6 +128,7 @@ export async function buildDistributables(options) {
    * directories and perform platform-specific steps
    */
   await run(CreateArchivesSourcesTask);
+  await run(PatchNativeModulesTask);
   await run(CleanExtraBinScriptsTask);
   await run(CleanExtraBrowsersTask);
   await run(CleanNodeBuildsTask);

src/dev/build/lib/__tests__/fs.js

@@ -23,11 +23,12 @@ import { chmodSync, statSync } from 'fs';
 import del from 'del';
 import expect from 'expect.js';
 
-import { mkdirp, write, read, getChildPaths, copy, copyAll, getFileHash, untar } from '../fs';
+import { mkdirp, write, read, getChildPaths, copy, copyAll, getFileHash, untar, gunzip } from '../fs';
 
 const TMP = resolve(__dirname, '__tmp__');
 const FIXTURES = resolve(__dirname, 'fixtures');
 const FOO_TAR_PATH = resolve(FIXTURES, 'foo_dir.tar.gz');
+const FOO_GZIP_PATH = resolve(FIXTURES, 'foo.txt.gz');
 const BAR_TXT_PATH = resolve(FIXTURES, 'foo_dir/bar.txt');
 const WORLD_EXECUTABLE = resolve(FIXTURES, 'bin/world_executable');
 
@@ -362,4 +363,39 @@ describe('dev/build/lib/fs', () => {
       expect(await read(resolve(destination, 'foo/foo.txt'))).to.be('foo\n');
     });
   });
+
+  describe('gunzip()', () => {
+    it('rejects if source path is not absolute', async () => {
+      try {
+        await gunzip('foo/bar', '**/*', __dirname);
+        throw new Error('Expected gunzip() to reject');
+      } catch (error) {
+        assertNonAbsoluteError(error);
+      }
+    });
+
+    it('rejects if destination path is not absolute', async () => {
+      try {
+        await gunzip(__dirname, '**/*', 'foo/bar');
+        throw new Error('Expected gunzip() to reject');
+      } catch (error) {
+        assertNonAbsoluteError(error);
+      }
+    });
+
+    it('rejects if neither path is not absolute', async () => {
+      try {
+        await gunzip('foo/bar', '**/*', 'foo/bar');
+        throw new Error('Expected gunzip() to reject');
+      } catch (error) {
+        assertNonAbsoluteError(error);
+      }
+    });
+
+    it('extracts gzip from source into destination, creating destination if necessary', async () => {
+      const destination = resolve(TMP, 'z/y/x/v/u/t/foo.txt');
+      await gunzip(FOO_GZIP_PATH, destination);
+      expect(await read(resolve(destination))).to.be('foo\n');
+    });
+  });
 });

src/dev/build/lib/fs.js

@@ -205,6 +205,19 @@ export async function untar(source, destination, extractOptions = {}) {
   ]);
 }
 
+export async function gunzip(source, destination) {
+  assertAbsolute(source);
+  assertAbsolute(destination);
+
+  await mkdirp(dirname(destination));
+
+  await createPromiseFromStreams([
+    fs.createReadStream(source),
+    createGunzip(),
+    fs.createWriteStream(destination),
+  ]);
+}
+
 export async function compress(type, options = {}, source, destination) {
   const output = fs.createWriteStream(destination);
   const archive = archiver(type, options.archiverOptions);

src/dev/build/lib/index.js

@@ -29,9 +29,11 @@ export {
   copyAll,
   getFileHash,
   untar,
+  gunzip,
   deleteAll,
   deleteEmptyFolders,
   compress,
 } from './fs';
+export { download } from './download';
 export { scanDelete } from './scan_delete';
 export { scanCopy } from './scan_copy';

src/dev/build/tasks/index.js

@@ -32,6 +32,7 @@ export * from './nodejs_modules';
 export * from './notice_file_task';
 export * from './optimize_task';
 export * from './os_packages';
+export * from './patch_native_modules_task';
 export * from './transpile_babel_task';
 export * from './transpile_typescript_task';
 export * from './transpile_scss_task';

src/dev/build/tasks/nodejs/__tests__/download_node_builds_task.js

@@ -22,7 +22,7 @@ import expect from 'expect.js';
 
 import * as NodeShasumsNS from '../node_shasums';
 import * as NodeDownloadInfoNS from '../node_download_info';
-import * as DownloadNS from '../download';
+import * as DownloadNS from '../../../lib/download'; // sinon can't stub '../../../lib' properly
 import { DownloadNodeBuildsTask } from '../download_node_builds_task';
 
 describe('src/dev/build/tasks/nodejs/download_node_builds_task', () => {

src/dev/build/tasks/nodejs/download_node_builds_task.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { download } from './download';
+import { download } from '../../lib';
 import { getNodeShasums } from './node_shasums';
 import { getNodeDownloadInfo } from './node_download_info';
 

src/dev/build/tasks/patch_native_modules_task.js

@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import fs from 'fs';
+import path from 'path';
+import util from 'util';
+import { deleteAll, download, gunzip, untar } from '../lib';
+
+const DOWNLOAD_DIRECTORY = '.native_modules';
+
+const packages = [
+  {
+    name: 're2',
+    version: '1.14.0',
+    destinationPath: 'node_modules/re2/build/Release/re2.node',
+    extractMethod: 'gunzip',
+    archives: {
+      darwin: {
+        url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/darwin-x64-64.gz',
+        sha256: '54c8386cb7cd53895cf379522114bfe82378e300e127e58d392ddd40a77e396f',
+      },
+      linux: {
+        url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/linux-x64-64.gz',
+        sha256: 'f54f059035e71a7ccb3fa201080e260c41d228d13a8247974b4bb157691b6757',
+      },
+      windows: {
+        url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/win32-x64-64.gz',
+        sha256: 'de708446a8b802f4634c2cfef097c2625a2811fdcd8133dfd7b7c485f966caa9',
+      },
+    },
+  },
+];
+
+async function getInstalledVersion(config, packageName) {
+  const packageJSONPath = config.resolveFromRepo(
+    path.join('node_modules', packageName, 'package.json')
+  );
+  const buffer = await util.promisify(fs.readFile)(packageJSONPath);
+  const packageJSON = JSON.parse(buffer);
+  return packageJSON.version;
+}
+
+async function patchModule(config, log, build, platform, pkg) {
+  const installedVersion = await getInstalledVersion(config, pkg.name);
+  if (installedVersion !== pkg.version) {
+    throw new Error(
+      `Can't patch ${pkg.name}'s native module, we were expecting version ${pkg.version} and found ${installedVersion}`
+    );
+  }
+  const platformName = platform.getName();
+  const archive = pkg.archives[platformName];
+  const archiveName = path.basename(archive.url);
+  const downloadPath = config.resolveFromRepo(DOWNLOAD_DIRECTORY, pkg.name, archiveName);
+  const extractPath = build.resolvePathForPlatform(platform, pkg.destinationPath);
+  log.debug(`Patching ${pkg.name} binaries from ${archive.url} to ${extractPath}`);
+
+  await deleteAll([extractPath], log);
+  await download({
+    log,
+    url: archive.url,
+    destination: downloadPath,
+    sha256: archive.sha256,
+    retries: 3,
+  });
+  switch (pkg.extractMethod) {
+    case 'gunzip':
+      await gunzip(downloadPath, extractPath);
+      break;
+    case 'untar':
+      await untar(downloadPath, extractPath);
+      break;
+    default:
+      throw new Error(`Extract method of ${pkg.extractMethod} is not supported`);
+  }
+}
+
+export const PatchNativeModulesTask = {
+  description: 'Patching platform-specific native modules',
+  async run(config, log, build) {
+    for (const pkg of packages) {
+      await Promise.all(
+        config.getTargetPlatforms().map(async (platform) => {
+          await patchModule(config, log, build, platform, pkg);
+        })
+      );
+    }
+  },
+};

src/dev/jest/config.js

@@ -68,6 +68,7 @@ export default {
     'json',
     'ts',
     'tsx',
+    'node',
   ],
   modulePathIgnorePatterns: [
     '__fixtures__/',

src/legacy/core_plugins/timelion/server/series_functions/label.js

@@ -25,7 +25,7 @@ export default new Chainable('label', {
   args: [
     {
       name: 'inputSeries',
-      types: ['seriesList']
+      types: ['seriesList'],
     },
     {
       name: 'label',
@@ -42,7 +42,7 @@ export default new Chainable('label', {
       help: i18n.translate('timelion.help.functions.label.args.regexHelpText', {
         defaultMessage: 'A regex with capture group support',
       }),
-    }
+    },
   ],
   help: i18n.translate('timelion.help.functions.labelHelpText', {
     defaultMessage: 'Change the label of the series. Use %s to reference the existing label',
@@ -51,12 +51,15 @@ export default new Chainable('label', {
     const config = args.byName;
     return alter(args, function (eachSeries) {
       if (config.regex) {
-        eachSeries.label = eachSeries.label.replace(new RegExp(config.regex), config.label);
+        // not using a standard `import` so that if there's an issue with the re2 native module
+        // that it doesn't prevent Kibana from starting up and we only have an issue using Timelion labels
+        const RE2 = require('re2');
+        eachSeries.label = eachSeries.label.replace(new RE2(config.regex), config.label);
       } else {
         eachSeries.label = config.label;
       }
 
       return eachSeries;
     });
-  }
+  },
 });

x-pack/dev-tools/jest/create_jest_config.js

@@ -19,6 +19,7 @@ export function createJestConfig({
       "json",
       "ts",
       "tsx",
+      "node",
     ],
     moduleNameMapper: {
       "^ui/(.*)": `${kibanaDirectory}/src/ui/public/$1`,

x-pack/test_utils/jest/config.js

@@ -42,6 +42,7 @@ export default {
     'json',
     'ts',
     'tsx',
+    'node',
   ],
   modulePathIgnorePatterns: [
     '__fixtures__/',