From 34c397c555392cba3b790693ac3d8cc88c4dc80d Mon Sep 17 00:00:00 2001 From: Miki Date: Tue, 1 Nov 2022 10:26:28 -0700 Subject: [PATCH] Adds @osd/cross-platform (#2703) * Adds helper functions, @osd/cross-platform, to work around the differences of platforms Signed-off-by: Miki --- .../src/errors/schema_error.test.ts | 4 +- packages/osd-cross-platform/README.md | 27 ++- .../src/__snapshots__/path.test.ts.snap | 13 ++ packages/osd-cross-platform/src/index.ts | 1 + packages/osd-cross-platform/src/path.test.ts | 208 ++++++++++++++++++ packages/osd-cross-platform/src/path.ts | 144 +++++++++++- packages/osd-cross-platform/src/process.ts | 28 ++- .../src/repo_root.ts | 51 +++-- .../serializers/absolute_path_serializer.ts | 27 ++- .../rules/no_restricted_paths.js | 2 +- packages/osd-optimizer/src/node/cache.ts | 23 +- .../src/node/node_auto_tranpilation.ts | 13 +- .../src/optimizer/cache_keys.test.ts | 41 +--- .../osd-optimizer/src/optimizer/cache_keys.ts | 12 +- packages/osd-plugin-generator/src/cli.ts | 4 +- .../integration_tests/generate_plugin.test.ts | 12 +- packages/osd-plugin-helpers/package.json | 2 +- .../src/integration_tests/build.test.ts | 22 +- .../src/tasks/create_archive.ts | 3 +- packages/osd-pm/src/utils/projects.test.ts | 6 +- packages/osd-utils/src/index.ts | 2 +- packages/osd-utils/src/package_json/index.ts | 2 +- packages/osd-utils/src/path/index.ts | 2 +- src/cli_plugin/install/download.test.js | 13 +- .../install/opensearch_dashboards.test.js | 12 +- src/cli_plugin/install/pack.test.js | 11 +- src/cli_plugin/list/list.test.js | 12 +- src/cli_plugin/remove/remove.test.js | 23 +- .../discovery/plugins_discovery.test.ts | 74 +++---- .../server/plugins/plugins_service.test.ts | 8 +- src/dev/build/lib/config.test.ts | 6 +- .../build/lib/integration_tests/fs.test.ts | 5 +- .../lib/integration_tests/scan_copy.test.ts | 3 +- src/dev/build/lib/scan_delete.test.ts | 12 +- src/dev/file.ts | 3 +- src/dev/i18n/integrate_locale_files.test.ts | 16 +- src/dev/i18n/utils/utils.js | 3 +- src/dev/jest/config.js | 8 +- .../integration_tests/junit_reporter.test.js | 3 +- 39 files changed, 639 insertions(+), 222 deletions(-) create mode 100644 packages/osd-cross-platform/src/__snapshots__/path.test.ts.snap create mode 100644 packages/osd-cross-platform/src/path.test.ts rename packages/{osd-utils => osd-cross-platform}/src/repo_root.ts (62%) diff --git a/packages/osd-config-schema/src/errors/schema_error.test.ts b/packages/osd-config-schema/src/errors/schema_error.test.ts index 345304c955e5..c134c888d510 100644 --- a/packages/osd-config-schema/src/errors/schema_error.test.ts +++ b/packages/osd-config-schema/src/errors/schema_error.test.ts @@ -31,7 +31,7 @@ import { relative, sep } from 'path'; import { SchemaError } from '.'; -import { standardize, PROCESS_WORKING_DIR } from '@osd/cross-platform'; +import { standardize, getRepoRoot } from '@osd/cross-platform'; /** * Make all paths in stacktrace relative. @@ -48,7 +48,7 @@ export const cleanStack = (stack: string) => } const path = parts[1]; - const relativePath = standardize(relative(PROCESS_WORKING_DIR, path)); + const relativePath = standardize(relative(getRepoRoot(path) || '.', path)); return line.replace(path, relativePath); }) diff --git a/packages/osd-cross-platform/README.md b/packages/osd-cross-platform/README.md index a61a2f184f02..0583b295c5f1 100644 --- a/packages/osd-cross-platform/README.md +++ b/packages/osd-cross-platform/README.md @@ -1,3 +1,28 @@ # `@osd/cross-platform` — OpenSearch Dashboards cross-platform helpers -This package contains the helper functions to work around the differences of platforms +This package contains the helpers to work around the differences across platforms, such as the difference in the path segment separator and the possibility of referencing a path using the short 8.3 name (SFN), a long name, and a long UNC on Windows. + +Some helpers are functions that `standardize` the reference to a path or help `getRepoRoot`, and some are constants referencing the `PROCESS_WORKING_DIR` or `REPO_ROOT`. + +### Example + +When the relative reference of `path` to the working directory is needed, using the code below would produce different results on Linux that it would on Windows and if the process was started in a Windows shell that used short paths, the results differ from a Windows shell that used long paths. +```js +import { relative } from 'path'; + +const relativePath = relative(process.cwd(), path); + +// Output on Linux: relative-path/to/a/file +// Windows: relative-path\to\a\file +// Windows SFN: RELATI~1\to\a\file +``` + +To avoid those differences, helper functions and constants can be used: +```js +import { relative } from 'path'; +import { standardize, PROCESS_WORKING_DIR } from '@osd/cross-platform'; + +const relativePath = standardize(relative(PROCESS_WORKING_DIR, path)); + +// Output: relative-path/to/a/file +``` \ No newline at end of file diff --git a/packages/osd-cross-platform/src/__snapshots__/path.test.ts.snap b/packages/osd-cross-platform/src/__snapshots__/path.test.ts.snap new file mode 100644 index 000000000000..20c44ec1c2d6 --- /dev/null +++ b/packages/osd-cross-platform/src/__snapshots__/path.test.ts.snap @@ -0,0 +1,13 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Cross Platform path standardize on POSIX-compatible platforms ignores additional parameters 1`] = `"/a/b/c"`; + +exports[`Cross Platform path standardize on POSIX-compatible platforms produces a path in POSIX format 1`] = `"/a/b/c"`; + +exports[`Cross Platform path standardize on Windows produces a path in POSIX format 1`] = `"C:/a/b/c"`; + +exports[`Cross Platform path standardize on Windows produces a path in native format 1`] = `"C:\\\\a\\\\b\\\\c"`; + +exports[`Cross Platform path standardize on Windows produces a path in native format even for POSIX input 1`] = `"C:\\\\a\\\\b\\\\c"`; + +exports[`Cross Platform path standardize on Windows produces a path in native format with escaped backslashes 1`] = `"C:\\\\\\\\a\\\\\\\\b\\\\\\\\c"`; diff --git a/packages/osd-cross-platform/src/index.ts b/packages/osd-cross-platform/src/index.ts index 343d7e9257a4..bc05aa9a955d 100644 --- a/packages/osd-cross-platform/src/index.ts +++ b/packages/osd-cross-platform/src/index.ts @@ -5,3 +5,4 @@ export * from './path'; export * from './process'; +export * from './repo_root'; diff --git a/packages/osd-cross-platform/src/path.test.ts b/packages/osd-cross-platform/src/path.test.ts new file mode 100644 index 000000000000..ed405298ad13 --- /dev/null +++ b/packages/osd-cross-platform/src/path.test.ts @@ -0,0 +1,208 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import path from 'path'; +import fs from 'fs'; +import { access, rmdir, mkdir, writeFile, symlink } from 'fs/promises'; + +import { + resolveToFullNameSync, + resolveToFullPathSync, + resolveToShortNameSync, + resolveToShortPathSync, + shortNamesSupportedSync, + realPathSync, + realShortPathSync, + standardize, +} from './path'; + +const tmpTestFolder = './__test_artifacts__'; +const longFolderName = '.long-folder-name'; +const longFileName = '.long-file-name.txt'; +const longSymlinkName = '.sym.link'; +const shortFolderName = 'LONG-F~1'; +const shortFileName = 'LONG-F~1.TXT'; +const dummyWindowsPath = 'C:\\a\\b\\c'; +const dummyWindowsPOSIXPath = 'C:/a/b/c'; +const dummyPOSIXPath = '/a/b/c'; + +const onWindows = process.platform === 'win32' ? describe : xdescribe; +const onWindowsWithShortNames = shortNamesSupportedSync() ? describe : xdescribe; + +// Save the real process.platform +const realPlatform = Object.getOwnPropertyDescriptor(process, 'platform')!; + +describe('Cross Platform', () => { + describe('path', () => { + onWindows('on Windows', () => { + onWindowsWithShortNames('when 8.3 is supported', () => { + beforeAll(async () => { + // Cleanup + try { + // If leftover artifacts were found, get rid of them + await access(tmpTestFolder); + await rmdir(tmpTestFolder, { recursive: true }); + } catch (ex) { + // Do nothing; if `rmdir` failed, let the `mkdir` below throw the error + } + + await mkdir(tmpTestFolder); + await mkdir(path.resolve(tmpTestFolder, longFolderName)); + await writeFile(path.resolve(tmpTestFolder, longFolderName, longFileName), ''); + await symlink( + path.resolve(tmpTestFolder, longFolderName), + path.resolve(tmpTestFolder, longSymlinkName), + 'junction' + ); + }); + + afterAll(async () => { + try { + await rmdir(tmpTestFolder, { recursive: true }); + } catch (ex) { + // Do nothing + } + }); + + it('can synchronously extract full name of a folder', () => { + const name = path.basename( + resolveToFullPathSync(path.resolve(tmpTestFolder, shortFolderName)) + ); + expect(name).toBe(longFolderName); + }); + + it('can synchronously extract full name of a file', () => { + const name = path.basename( + resolveToFullNameSync(path.resolve(tmpTestFolder, shortFolderName, shortFileName)) + ); + expect(name).toBe(longFileName); + }); + + it('can synchronously extract short name of a folder', () => { + const name = path.basename( + resolveToShortPathSync(path.resolve(tmpTestFolder, longFolderName)) + ); + expect(name).toBe(shortFolderName); + }); + + it('can synchronously extract short name of a file', () => { + const name = path.basename( + resolveToShortNameSync(path.resolve(tmpTestFolder, longFolderName, longFileName)) + ); + expect(name).toBe(shortFileName); + }); + + it('can synchronously extract full name of a symbolic link', () => { + const name = path.basename(realPathSync(path.resolve(tmpTestFolder, longSymlinkName))); + expect(name).toBe(longFolderName); + }); + + it('can synchronously extract short name of a symbolic link', () => { + const name = path.basename( + realShortPathSync(path.resolve(tmpTestFolder, longSymlinkName)) + ); + expect(name).toBe(shortFolderName); + }); + }); + }); + + describe('on platforms other than Windows', () => { + let mockPathNormalize: jest.SpyInstance; + let mockPathResolve: jest.SpyInstance; + let mockFSRealPathSync: jest.SpyInstance; + + beforeAll(() => { + Object.defineProperty(process, 'platform', { + ...Object.getOwnPropertyDescriptor(process, 'property'), + value: 'linux', + }); + + mockPathNormalize = jest.spyOn(path, 'normalize').mockReturnValue(dummyPOSIXPath); + mockPathResolve = jest.spyOn(path, 'resolve').mockReturnValue(dummyPOSIXPath); + mockFSRealPathSync = jest + .spyOn(fs, 'realpathSync') + .mockReturnValue(dummyPOSIXPath) as jest.SpyInstance; + }); + + afterAll(() => { + // Restore the real property value after each test + Object.defineProperty(process, 'platform', realPlatform); + mockPathNormalize.mockRestore(); + mockPathResolve.mockRestore(); + mockFSRealPathSync.mockRestore(); + }); + + it('all short and full name methods return just the normalized paths', () => { + expect(shortNamesSupportedSync()).toBe(false); + expect(resolveToFullPathSync(dummyPOSIXPath)).toBe(dummyPOSIXPath); + expect(resolveToShortPathSync(dummyPOSIXPath)).toBe(dummyPOSIXPath); + }); + }); + + describe('standardize', () => { + describe('on Windows', () => { + let mockPathNormalize: jest.SpyInstance; + + beforeAll(() => { + Object.defineProperty(process, 'platform', { + ...Object.getOwnPropertyDescriptor(process, 'property'), + value: 'win32', + }); + + mockPathNormalize = jest.spyOn(path, 'normalize').mockReturnValue(dummyWindowsPath); + }); + + afterAll(() => { + // Restore the real property value after each test + Object.defineProperty(process, 'platform', realPlatform); + mockPathNormalize.mockRestore(); + }); + + it('produces a path in native format', () => { + expect(standardize(dummyWindowsPath, false, false)).toMatchSnapshot(); + }); + + it('produces a path in native format even for POSIX input', () => { + expect(standardize(dummyWindowsPOSIXPath, false, false)).toMatchSnapshot(); + }); + + it('produces a path in native format with escaped backslashes', () => { + expect(standardize(dummyWindowsPath, false, true)).toMatchSnapshot(); + }); + + it('produces a path in POSIX format', () => { + expect(standardize(dummyWindowsPath)).toMatchSnapshot(); + }); + }); + + describe('on POSIX-compatible platforms', () => { + let mockPathNormalize: jest.SpyInstance; + + beforeAll(() => { + Object.defineProperty(process, 'platform', { + ...Object.getOwnPropertyDescriptor(process, 'property'), + value: 'linux', + }); + + mockPathNormalize = jest.spyOn(path, 'normalize').mockReturnValue(dummyPOSIXPath); + }); + + afterAll(() => { + // Restore the real property value after each test + Object.defineProperty(process, 'platform', realPlatform); + mockPathNormalize.mockRestore(); + }); + + it('produces a path in POSIX format', () => { + expect(standardize(dummyPOSIXPath)).toMatchSnapshot(); + }); + + it('ignores additional parameters', () => { + expect(standardize(dummyPOSIXPath, false, true)).toMatchSnapshot(); + }); + }); + }); + }); +}); diff --git a/packages/osd-cross-platform/src/path.ts b/packages/osd-cross-platform/src/path.ts index 7e37443f1bca..d5b2befb56c1 100644 --- a/packages/osd-cross-platform/src/path.ts +++ b/packages/osd-cross-platform/src/path.ts @@ -3,28 +3,158 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { normalize } from 'path'; +import { execSync } from 'child_process'; +import { basename, normalize, resolve } from 'path'; +import { + realpathSync as nativeRealpathSync, + openSync, + closeSync, + existsSync, + unlinkSync, +} from 'fs'; + +export const NAMESPACE_PREFIX = process.platform === 'win32' ? '\\\\?\\' : ''; /** * Get a standardized reference to a path * @param {string} path - the path to standardize * @param {boolean} [usePosix=true] - produce a posix reference * @param {boolean} [escapedBackslashes=true] - on Windows, double-backslash the reference - * @internal + * @param {boolean} [returnUNC=false] - produce an extended reference */ export const standardize = ( path: string, usePosix: boolean = true, - escapedBackslashes: boolean = true + escapedBackslashes: boolean = true, + returnUNC: boolean = false ) => { - /* Force os-dependant separators - * path.posix.normalize doesn't convert backslashes to slashes on Windows so we manually force it afterwards - */ + // Force os-dependant separators const normal = normalize(path); // Filter out in-browser executions as well as non-windows ones if (process?.platform !== 'win32') return normal; if (usePosix) return normal.replace(/\\/g, '/'); - return escapedBackslashes ? normal.replace(/\\/g, '\\\\') : normal; + else if (escapedBackslashes) return normal.replace(/\\/g, '\\\\'); + else if (returnUNC) return '\\\\?\\' + normal; + return normal; +}; + +/** + * Windows-only function that uses PowerShell to calculate the full path + * @param {string} path + * @private + */ +const getFullPathSync = (path: string) => { + if (process.platform !== 'win32') return path; + + try { + const fullName = execSync(`powershell "(Get-Item -LiteralPath '${path}').FullName"`, { + encoding: 'utf8', + })?.trim?.(); + + // Make sure we got something back + if (fullName?.length > 2) return fullName; + } catch (ex) { + // Do nothing + } + + return path; +}; + +/** + * Windows-only function that uses PowerShell and Com Object to calculate the 8.3 path + * @param {string} path + * @private + */ +const getShortPathSync = (path: string) => { + if (process.platform !== 'win32') return path; + + try { + const shortPath = execSync( + `powershell "$FSO = New-Object -ComObject Scripting.FileSystemObject; $O = (Get-Item -LiteralPath '${path}'); if ($O.PSIsContainer) { $FSO.GetFolder($O.FullName).ShortPath } else { $FSO.GetFile($O.FullName).ShortPath }"`, + { + encoding: 'utf8', + } + )?.trim?.(); + + // Make sure we got something back + if (shortPath?.length > 2) return shortPath; + } catch (ex) { + // Do nothing + } + + return path; }; + +/** + * Checks if Windows 8.3 short names are supported on the volume of the given path + * @param {string} [path='.'] - the path to examine + */ +export const shortNamesSupportedSync = (path: string = '.') => { + if (process.platform !== 'win32') return false; + + const testFileName = '.___osd-cross-platform-test.file'; + const file = resolve(path, testFileName); + + // Create a test file if it doesn't exist + if (!existsSync(file)) closeSync(openSync(file, 'w')); + + // If the returned value's basename is not the same as the requested file name, it must be a short name + const foundShortName = basename(getShortPathSync(file)) !== testFileName; + + // Cleanup + unlinkSync(file); + + return foundShortName; +}; + +/** + * @borrows shortNamesSupportedSync + */ +export const shortNameSupportedSync = shortNamesSupportedSync; + +/** + * Get the full pathname + * @param {string} path - the path to resolve + */ +export const resolveToFullPathSync = (path: string) => getFullPathSync(resolve(path)); + +/** + * @borrows resolveToFullPathSync + */ +export const resolveToFullNameSync = resolveToFullPathSync; + +/** + * Get the short pathname + * @param {string} path - the path to resolve + */ +export const resolveToShortPathSync = (path: string) => getShortPathSync(resolve(path)); + +/** + * @borrows resolveToShortPathSync + */ +export const resolveToShortNameSync = resolveToShortPathSync; + +/** + * Get the canonical pathname + * @param {string} path - the path to resolve + */ +export const realPathSync = (path: string) => getFullPathSync(nativeRealpathSync(path, 'utf8')); + +/** + * @borrows realPathSync + */ +export const realpathSync = realPathSync; + +/** + * Get the canonical pathname + * @param {string} path - the path to resolve + */ +export const realShortPathSync = (path: string) => + getShortPathSync(nativeRealpathSync(path, 'utf8')); + +/** + * @borrows realShortPathSync + */ +export const realshortpathSync = realShortPathSync; diff --git a/packages/osd-cross-platform/src/process.ts b/packages/osd-cross-platform/src/process.ts index fa593c943687..ade19cca5615 100644 --- a/packages/osd-cross-platform/src/process.ts +++ b/packages/osd-cross-platform/src/process.ts @@ -3,20 +3,18 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { execSync } from 'child_process'; +import { resolveToFullPathSync, standardize } from './path'; -let workingDir = process.cwd(); - -if (process.platform === 'win32') { - try { - const pathFullName = execSync('powershell "(Get-Item -LiteralPath $pwd).FullName"', { - cwd: workingDir, - encoding: 'utf8', - })?.trim?.(); - if (pathFullName?.length > 2) workingDir = pathFullName; - } catch (ex) { - // Do nothing - } -} +/** + * The full pathname of the working directory of the process + * @constant + * @type {string} + */ +export const PROCESS_WORKING_DIR: string = resolveToFullPathSync(process.cwd()); -export const PROCESS_WORKING_DIR = workingDir; +/** + * The full pathname of the working directory of the process, in POSIX format + * @constant + * @type {string} + */ +export const PROCESS_POSIX_WORKING_DIR: string = standardize(PROCESS_WORKING_DIR); diff --git a/packages/osd-utils/src/repo_root.ts b/packages/osd-cross-platform/src/repo_root.ts similarity index 62% rename from packages/osd-utils/src/repo_root.ts rename to packages/osd-cross-platform/src/repo_root.ts index c12004442f3c..a7ffc19a7f7c 100644 --- a/packages/osd-utils/src/repo_root.ts +++ b/packages/osd-cross-platform/src/repo_root.ts @@ -4,6 +4,9 @@ * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. */ /* @@ -25,26 +28,16 @@ * under the License. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -import Path from 'path'; -import Fs from 'fs'; +import { resolve, parse, dirname, isAbsolute, relative } from 'path'; import loadJsonFile from 'load-json-file'; +import { resolveToFullPathSync, resolveToShortPathSync, realPathSync } from './path'; const readOpenSearchDashboardsPkgJson = (dir: string) => { try { - const path = Path.resolve(dir, 'package.json'); - const json = loadJsonFile.sync(path); - if ( - json && - typeof json === 'object' && - 'name' in json && - json.name === 'opensearch-dashboards' - ) { + const path = resolve(dir, 'package.json'); + const json = loadJsonFile.sync(path) as { [key: string]: any }; + if (json?.name === 'opensearch-dashboards') { return json; } } catch (error) { @@ -60,8 +53,8 @@ const findOpenSearchDashboardsPackageJson = () => { // search for the opensearch-dashboards directory, since this file is moved around it might // not be where we think but should always be a relatively close parent // of this directory - const startDir = Fs.realpathSync(__dirname); - const { root: rootDir } = Path.parse(startDir); + const startDir = realPathSync(__dirname); + const { root: rootDir } = parse(startDir); let cursor = startDir; while (true) { const opensearchDashboardsPkgJson = readOpenSearchDashboardsPkgJson(cursor); @@ -75,7 +68,7 @@ const findOpenSearchDashboardsPackageJson = () => { }; } - const parent = Path.dirname(cursor); + const parent = dirname(cursor); if (parent === rootDir) { throw new Error(`unable to find opensearch-dashboards directory from ${startDir}`); } @@ -88,5 +81,25 @@ const { opensearchDashboardsPkgJson, } = findOpenSearchDashboardsPackageJson(); -export const REPO_ROOT = opensearchDashboardsDir; +export const REPO_ROOT = resolveToFullPathSync(opensearchDashboardsDir); +export const REPO_ROOT_8_3 = resolveToShortPathSync(opensearchDashboardsDir); export const UPSTREAM_BRANCH = opensearchDashboardsPkgJson.branch; + +export const getMatchingRoot = (path: string, rootPaths: string | string[]) => { + const rootPathsArray = Array.isArray(rootPaths) ? rootPaths : [rootPaths]; + + // We can only find the appropriate root if an absolute path was given + if (path && isAbsolute(path)) { + // Return the matching root if one is found or return `undefined` + return rootPathsArray.find((root) => path.startsWith(root)); + } + + return undefined; +}; + +export const getRepoRoot = (path: string) => getMatchingRoot(path, [REPO_ROOT, REPO_ROOT_8_3]); + +export const relativeToRepoRoot = (path: string) => { + const repoRoot = getRepoRoot(path); + return repoRoot ? relative(repoRoot, path) : null; +}; diff --git a/packages/osd-dev-utils/src/serializers/absolute_path_serializer.ts b/packages/osd-dev-utils/src/serializers/absolute_path_serializer.ts index 05172f5ac330..939b6e9924df 100644 --- a/packages/osd-dev-utils/src/serializers/absolute_path_serializer.ts +++ b/packages/osd-dev-utils/src/serializers/absolute_path_serializer.ts @@ -4,6 +4,9 @@ * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. */ /* @@ -25,19 +28,25 @@ * under the License. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -import { REPO_ROOT } from '@osd/utils'; +// Not importing from @osd/cross-platform to allow some complicated tests to run: suite_tracker.test.ts +import { REPO_ROOT, REPO_ROOT_8_3 } from '@osd/utils'; export function createAbsolutePathSerializer( - rootPath: string = REPO_ROOT, + rootPath: string | string[] = [REPO_ROOT, REPO_ROOT_8_3], replacement = '' ) { + const rootPaths = Array.isArray(rootPath) ? rootPath : [rootPath]; + return { - test: (value: any) => typeof value === 'string' && value.startsWith(rootPath), - serialize: (value: string) => value.replace(rootPath, replacement).replace(/\\/g, '/'), + test: (value: any) => + typeof value === 'string' && rootPaths.some((path) => value.startsWith(path)), + serialize: (value: string) => + rootPaths + // Replace all instances of `rootPaths` found at the beginning of the `value` + .reduce( + (result, path) => (result.startsWith(path) ? result.replace(path, replacement) : result), + value + ) + .replace(/\\/g, '/'), }; } diff --git a/packages/osd-eslint-plugin-eslint/rules/no_restricted_paths.js b/packages/osd-eslint-plugin-eslint/rules/no_restricted_paths.js index d3ac19fa6d19..e6139d29d592 100644 --- a/packages/osd-eslint-plugin-eslint/rules/no_restricted_paths.js +++ b/packages/osd-eslint-plugin-eslint/rules/no_restricted_paths.js @@ -47,7 +47,7 @@ function traverseToTopFolder(src, pattern) { const srcIdx = src.lastIndexOf(path.sep); src = src.slice(0, srcIdx); } - return src; + return src.replace(/\\/g, '/'); } function isSameFolderOrDescendent(src, imported, pattern) { diff --git a/packages/osd-optimizer/src/node/cache.ts b/packages/osd-optimizer/src/node/cache.ts index ff81613382ea..aab60ad4da48 100644 --- a/packages/osd-optimizer/src/node/cache.ts +++ b/packages/osd-optimizer/src/node/cache.ts @@ -4,6 +4,9 @@ * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. */ /* @@ -25,16 +28,12 @@ * under the License. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - import Path from 'path'; // @ts-expect-error no types available import * as LmdbStore from 'lmdb-store'; import { REPO_ROOT, UPSTREAM_BRANCH } from '@osd/dev-utils'; +import { getMatchingRoot } from '@osd/cross-platform'; // This is to enable parallel jobs on CI. const CACHE_DIR = process.env.CACHE_DIR @@ -139,7 +138,19 @@ export class Cache { } private getKey(path: string) { - return `${this.prefix}${path}`; + const resolvedPath = Path.resolve(path); + /* Try to find the root that is the parent to `path` so we can make a nimble + * and unique key based on the relative path. If A root was not found, just + * use any of the roots; the key would just be long. + */ + const pathRoot = getMatchingRoot(resolvedPath, this.pathRoots) || this.pathRoots[0]; + + const normalizedPath = + Path.sep !== '/' + ? Path.relative(pathRoot, resolvedPath).split(Path.sep).join('/') + : Path.relative(pathRoot, resolvedPath); + + return `${this.prefix}${normalizedPath}`; } private async pruneOldKeys() { diff --git a/packages/osd-optimizer/src/node/node_auto_tranpilation.ts b/packages/osd-optimizer/src/node/node_auto_tranpilation.ts index 1e3ef8503780..962bbd12ee80 100644 --- a/packages/osd-optimizer/src/node/node_auto_tranpilation.ts +++ b/packages/osd-optimizer/src/node/node_auto_tranpilation.ts @@ -1,5 +1,16 @@ /* eslint-disable @osd/eslint/require-license-header */ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + /** * This module is based on @babel/register @ 9808d25, modified to use * a more efficient caching implementation which writes to disk as @@ -39,7 +50,7 @@ import Crypto from 'crypto'; import * as babel from '@babel/core'; import { addHook } from 'pirates'; -import { REPO_ROOT } from '@osd/dev-utils'; +import { REPO_ROOT, REPO_ROOT_8_3, UPSTREAM_BRANCH } from '@osd/dev-utils'; import sourceMapSupport from 'source-map-support'; import { Cache } from './cache'; diff --git a/packages/osd-optimizer/src/optimizer/cache_keys.test.ts b/packages/osd-optimizer/src/optimizer/cache_keys.test.ts index 0fa4006a41d9..b01228a09574 100644 --- a/packages/osd-optimizer/src/optimizer/cache_keys.test.ts +++ b/packages/osd-optimizer/src/optimizer/cache_keys.test.ts @@ -4,6 +4,9 @@ * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. */ /* @@ -25,14 +28,8 @@ * under the License. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -import Path from 'path'; - -import jestDiff from 'jest-diff'; +import fs from 'fs/promises'; +import { diff } from 'jest-diff'; import { REPO_ROOT } from '@osd/utils'; import { createAbsolutePathSerializer } from '@osd/dev-utils'; @@ -54,14 +51,6 @@ jest.mock('./get_mtimes.ts', () => ({ jest.mock('execa'); -jest.mock('fs', () => { - const realFs = jest.requireActual('fs'); - return { - ...realFs, - readFile: jest.fn(realFs.readFile), - }; -}); - expect.addSnapshotSerializer(createAbsolutePathSerializer()); jest.requireMock('execa').mockImplementation(async (cmd: string, args: string[], opts: object) => { @@ -85,17 +74,9 @@ jest.requireMock('execa').mockImplementation(async (cmd: string, args: string[], describe('getOptimizerCacheKey()', () => { it('uses latest commit, bootstrap cache, and changed files to create unique value', async () => { - jest - .requireMock('fs') - .readFile.mockImplementation( - (path: string, enc: string, cb: (err: null, file: string) => void) => { - expect(path).toBe( - Path.resolve(REPO_ROOT, 'packages/osd-optimizer/target/.bootstrap-cache') - ); - expect(enc).toBe('utf8'); - cb(null, ''); - } - ); + const mockFSReadFileAsync = jest + .spyOn(fs, 'readFile') + .mockReturnValue(Promise.resolve('')); const config = OptimizerConfig.create({ repoRoot: REPO_ROOT, @@ -124,6 +105,8 @@ describe('getOptimizerCacheKey()', () => { }, } `); + + mockFSReadFileAsync.mockRestore(); }); }); @@ -187,7 +170,7 @@ describe('diffCacheKey()', () => { describe('reformatJestDiff()', () => { it('reformats large jestDiff output to focus on the changed lines', () => { - const diff = jestDiff( + const jestDiff = diff( { a: ['1', '1', '1', '1', '1', '1', '1', '2', '1', '1', '1', '1', '1', '1', '1', '1', '1'], }, @@ -196,7 +179,7 @@ describe('reformatJestDiff()', () => { } ); - expect(reformatJestDiff(diff)).toMatchInlineSnapshot(` + expect(reformatJestDiff(jestDiff)).toMatchInlineSnapshot(` "- Expected + Received diff --git a/packages/osd-optimizer/src/optimizer/cache_keys.ts b/packages/osd-optimizer/src/optimizer/cache_keys.ts index 1edc7c3c9c12..dc9e433bedc0 100644 --- a/packages/osd-optimizer/src/optimizer/cache_keys.ts +++ b/packages/osd-optimizer/src/optimizer/cache_keys.ts @@ -4,6 +4,9 @@ * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. */ /* @@ -25,18 +28,13 @@ * under the License. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - import Path from 'path'; import Fs from 'fs'; import { promisify } from 'util'; import Chalk from 'chalk'; import execa from 'execa'; -import { REPO_ROOT } from '@osd/utils'; +import { relativeToRepoRoot, REPO_ROOT } from '@osd/cross-platform'; import stripAnsi from 'strip-ansi'; import jestDiff from 'jest-diff'; @@ -48,7 +46,7 @@ import { getChanges } from './get_changes'; import { OptimizerConfig } from './optimizer_config'; const OPTIMIZER_DIR = Path.dirname(require.resolve('../../package.json')); -const RELATIVE_DIR = Path.relative(REPO_ROOT, OPTIMIZER_DIR); +const RELATIVE_DIR = relativeToRepoRoot(OPTIMIZER_DIR)!; export function diffCacheKey(expected?: unknown, actual?: unknown) { const expectedJson = jsonStable(expected, { diff --git a/packages/osd-plugin-generator/src/cli.ts b/packages/osd-plugin-generator/src/cli.ts index 51b6ddc7ca20..e0be64922e2e 100644 --- a/packages/osd-plugin-generator/src/cli.ts +++ b/packages/osd-plugin-generator/src/cli.ts @@ -34,7 +34,7 @@ import Path from 'path'; import Fs from 'fs'; import execa from 'execa'; -import { REPO_ROOT } from '@osd/utils'; +import { PROCESS_WORKING_DIR, REPO_ROOT } from '@osd/cross-platform'; import { run, createFailError, createFlagError } from '@osd/dev-utils'; import { snakeCase } from './casing'; @@ -80,7 +80,7 @@ export function runCli() { } log.success( - `🎉\n\nYour plugin has been created in ${Path.relative(process.cwd(), outputDir)}\n` + `🎉\n\nYour plugin has been created in ${Path.relative(PROCESS_WORKING_DIR, outputDir)}\n` ); }, { diff --git a/packages/osd-plugin-generator/src/integration_tests/generate_plugin.test.ts b/packages/osd-plugin-generator/src/integration_tests/generate_plugin.test.ts index 3ed08083ec4e..38ad0496c176 100644 --- a/packages/osd-plugin-generator/src/integration_tests/generate_plugin.test.ts +++ b/packages/osd-plugin-generator/src/integration_tests/generate_plugin.test.ts @@ -4,6 +4,9 @@ * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. */ /* @@ -25,17 +28,12 @@ * under the License. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - import Path from 'path'; import del from 'del'; import execa from 'execa'; -import { standardize } from '@osd/cross-platform'; -import { REPO_ROOT, createAbsolutePathSerializer } from '@osd/dev-utils'; +import { standardize, REPO_ROOT } from '@osd/cross-platform'; +import { createAbsolutePathSerializer } from '@osd/dev-utils'; import globby from 'globby'; // Has to be a posix reference because it is used to generate glob patterns diff --git a/packages/osd-plugin-helpers/package.json b/packages/osd-plugin-helpers/package.json index 55e5e803fe47..6868d0ae6137 100644 --- a/packages/osd-plugin-helpers/package.json +++ b/packages/osd-plugin-helpers/package.json @@ -12,7 +12,7 @@ "plugin-helpers": "bin/plugin-helpers.js" }, "scripts": { - "osd:bootstrap": "node ../../scripts/remove.js && tsc", + "osd:bootstrap": "node ../../scripts/remove.js target && tsc", "osd:watch": "tsc --watch" }, "dependencies": { diff --git a/packages/osd-plugin-helpers/src/integration_tests/build.test.ts b/packages/osd-plugin-helpers/src/integration_tests/build.test.ts index 1d70db3d18a2..780682d27658 100644 --- a/packages/osd-plugin-helpers/src/integration_tests/build.test.ts +++ b/packages/osd-plugin-helpers/src/integration_tests/build.test.ts @@ -4,6 +4,9 @@ * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. */ /* @@ -25,17 +28,12 @@ * under the License. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - import Path from 'path'; import Fs from 'fs'; import execa from 'execa'; -import { standardize } from '@osd/cross-platform'; -import { REPO_ROOT, createStripAnsiSerializer, createReplaceSerializer } from '@osd/dev-utils'; +import { REPO_ROOT, standardize } from '@osd/cross-platform'; +import { createStripAnsiSerializer, createReplaceSerializer } from '@osd/dev-utils'; import extract from 'extract-zip'; import del from 'del'; import globby from 'globby'; @@ -103,13 +101,12 @@ it('builds a generated plugin into a viable archive', async () => { info running @osd/optimizer │ info initialized, 0 bundles cached │ info starting worker [1 bundle] - │ warn worker stderr Browserslist: caniuse-lite is outdated. Please run: - │ warn worker stderr npx browserslist@latest --update-db │ succ 1 bundles compiled successfully after