[Backport 2.x] Enhance darwin-arm64 release

.github/workflows/build_and_test_workflow.yml

@@ -281,7 +281,7 @@ jobs:
       - name: Build plugins
         run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 10 --scan-dir "./test/plugin_functional/plugins"
 
-      - name: Run functional plugin tests 
+      - name: Run functional plugin tests
         id: plugin-ftr-tests
         run: node scripts/functional_tests.js --config test/plugin_functional/config.ts
 
@@ -308,6 +308,16 @@ jobs:
             ext: tar.gz
             suffix: linux-arm64
             script: build-platform --linux-arm --skip-os-packages
+          - os: macos-latest
+            name: macOS x64
+            ext: tar.gz
+            suffix: darwin-x64
+            script: build-platform --darwin --skip-os-packages
+          - os: macos-latest
+            name: macOS ARM64
+            ext: tar.gz
+            suffix: darwin-arm64
+            script: build-platform --darwin-arm --skip-os-packages
           - os: windows-latest
             name: Windows x64
             ext: zip

src/dev/build/args.test.ts

@@ -60,6 +60,7 @@ it('build dist for current platform, without packages, by default', () => {
         "targetAllPlatforms": false,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": false,
@@ -89,6 +90,7 @@ it('build dist for linux x64 platform, without packages, if --linux is passed',
         "targetAllPlatforms": false,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": true,
           "linuxArm": false,
           "windows": false,
@@ -118,6 +120,7 @@ it('build dist for linux arm64 platform, without packages, if --linux-arm is pas
         "targetAllPlatforms": false,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": true,
           "windows": false,
@@ -147,6 +150,7 @@ it('build dist for darwin x64 platform, without packages, if --darwin is passed'
         "targetAllPlatforms": false,
         "targetPlatforms": Object {
           "darwin": true,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": false,
@@ -176,6 +180,7 @@ it('build dist for windows x64 platform, without packages, if --windows is passe
         "targetAllPlatforms": false,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": true,
@@ -205,6 +210,7 @@ it('builds packages if --all-platforms is passed', () => {
         "targetAllPlatforms": true,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": false,
@@ -234,6 +240,7 @@ it('limits packages if --rpm passed with --all-platforms', () => {
         "targetAllPlatforms": true,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": false,
@@ -263,6 +270,7 @@ it('limits packages if --deb passed with --all-platforms', () => {
         "targetAllPlatforms": true,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": false,
@@ -293,6 +301,7 @@ it('limits packages if --docker passed with --all-platforms', () => {
         "targetAllPlatforms": true,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": false,
@@ -323,6 +332,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform
         "targetAllPlatforms": true,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": false,

src/dev/build/args.ts

@@ -52,6 +52,7 @@ export function readCliArgs(argv: string[]) {
       'all-platforms',
       'windows',
       'darwin',
+      'darwin-arm',
       'linux',
       'linux-arm',
       'verbose',
@@ -133,6 +134,7 @@ export function readCliArgs(argv: string[]) {
     targetPlatforms: {
       windows: Boolean(flags.windows),
       darwin: Boolean(flags.darwin),
+      darwinArm: Boolean(flags['darwin-arm']),
       linux: Boolean(flags.linux),
       linuxArm: Boolean(flags['linux-arm']),
     },

src/dev/build/cli.ts

@@ -61,6 +61,7 @@ if (showHelp) {
         --linux                 {dim Produce archives only for linux x64 platform}
         --linux-arm             {dim Produce archives only for linux arm64 platform}
         --darwin                {dim Produce archives only for darwin x64 platform}
+        --darwin-arm            {dim Produce archives only for darwin arm64 platform}
         --windows               {dim Produce archives only for windows x64 platform}
         --rpm                   {dim Only build the rpm package}
         --deb                   {dim Only build the deb package}

src/dev/build/lib/fs.ts

@@ -29,6 +29,7 @@
  */
 
 import fs from 'fs';
+import { rm } from 'fs/promises';
 import { createHash } from 'crypto';
 import { pipeline, Writable } from 'stream';
 import { resolve, dirname, isAbsolute, sep } from 'path';
@@ -138,8 +139,7 @@ export async function deleteEmptyFolders(
   );
   assertAbsolute(rootFolderPath.startsWith('!') ? rootFolderPath.slice(1) : rootFolderPath);
 
-  // Delete empty is used to gather all the empty folders and
-  // then we use del to actually delete them
+  // `deleteEmpty` is used to gather all the empty folders then `rm` is used to actually delete them
   const emptyFoldersList = await deleteEmpty(rootFolderPath, {
     // @ts-expect-error DT package has incorrect types https://github.com/jonschlinkert/delete-empty/blob/6ae34547663e6845c3c98b184c606fa90ef79c0a/index.js#L160
     dryRun: true,
@@ -148,12 +148,11 @@ export async function deleteEmptyFolders(
   const foldersToDelete = emptyFoldersList.filter((folderToDelete) => {
     return !foldersToKeep.some((folderToKeep) => folderToDelete.includes(folderToKeep));
   });
-  const deletedEmptyFolders = await del(foldersToDelete, {
-    concurrency: 4,
-  });
 
-  log.debug('Deleted %d empty folders', deletedEmptyFolders.length);
-  log.verbose('Deleted:', longInspect(deletedEmptyFolders));
+  await Promise.all(foldersToDelete.map((folder) => rm(folder, { force: true, recursive: true })));
+
+  log.debug('Deleted %d empty folders', foldersToDelete.length);
+  log.verbose('Deleted:', longInspect(foldersToDelete));
 }
 
 interface CopyOptions {

src/dev/build/tasks/create_archives_sources_task.ts

@@ -56,17 +56,21 @@ export const CreateArchivesSources: Task = {
 
         // ToDo [NODE14]: Remove this Node.js 14 fallback download
         // Copy the Node.js 14 binaries into node/fallback to be used by `use_node`
-        await scanCopy({
-          source: (
-            await getNodeVersionDownloadInfo(
-              NODE14_FALLBACK_VERSION,
-              platform.getNodeArch(),
-              platform.isWindows(),
-              config.resolveFromRepo()
-            )
-          ).extractDir,
-          destination: build.resolvePathForPlatform(platform, 'node', 'fallback'),
-        });
+        if (platform.getBuildName() === 'darwin-arm64') {
+          log.warning(`There are no fallback Node.js versions released for darwin-arm64.`);
+        } else {
+          await scanCopy({
+            source: (
+              await getNodeVersionDownloadInfo(
+                NODE14_FALLBACK_VERSION,
+                platform.getNodeArch(),
+                platform.isWindows(),
+                config.resolveFromRepo()
+              )
+            ).extractDir,
+            destination: build.resolvePathForPlatform(platform, 'node', 'fallback'),
+          });
+        }
 
         log.debug('Node.js copied into', platform.getNodeArch(), 'specific build directory');
       })

src/dev/build/tasks/nodejs/download_node_builds_task.test.ts

@@ -36,13 +36,15 @@ import {
 
 import { Config, Platform } from '../../lib';
 import { DownloadNodeBuilds } from './download_node_builds_task';
+import { stripAnsiSnapshotSerializer } from '../../../../core/test_helpers/strip_ansi_snapshot_serializer';
 
 jest.mock('./node_shasums');
 jest.mock('./node_download_info');
 jest.mock('../../lib/download');
 jest.mock('../../lib/get_build_number');
 
 expect.addSnapshotSerializer(createAnyInstanceSerializer(ToolingLog));
+expect.addSnapshotSerializer(stripAnsiSnapshotSerializer);
 
 const { getNodeDownloadInfo, getNodeVersionDownloadInfo } = jest.requireMock(
   './node_download_info'
@@ -67,6 +69,8 @@ async function setup({ failOnUrl }: { failOnUrl?: string } = {}) {
       linux: false,
       linuxArm: false,
       darwin: false,
+      darwinArm: false,
+      windows: false,
     },
   });
 
@@ -78,19 +82,23 @@ async function setup({ failOnUrl }: { failOnUrl?: string } = {}) {
     };
   });
 
-  getNodeVersionDownloadInfo.mockImplementation((version, architecture, isWindows, repoRoot) => {
-    return {
-      url: `https://mirrors.nodejs.org/dist/v${version}/node-v${version}-${architecture}.tar.gz`,
-      downloadName: `node-v${version}-${architecture}.tar.gz`,
-      downloadPath: `/mocked/path/.node_binaries/${version}/node-v${version}-${architecture}.tar.gz`,
-      extractDir: `/mocked/path/.node_binaries/${version}/${architecture}`,
-      version,
-    };
-  });
+  getNodeVersionDownloadInfo.mockImplementation(
+    (version: string, architecture: string, isWindows: boolean, repoRoot: string) => {
+      return {
+        url: `https://mirrors.nodejs.org/dist/v${version}/node-v${version}-${architecture}.tar.gz`,
+        downloadName: `node-v${version}-${architecture}.tar.gz`,
+        downloadPath: `/mocked/path/.node_binaries/${version}/node-v${version}-${architecture}.tar.gz`,
+        extractDir: `/mocked/path/.node_binaries/${version}/${architecture}`,
+        version,
+      };
+    }
+  );
 
   getNodeShasums.mockReturnValue({
     'linux:downloadName': 'linux:sha256',
+    'linux-arm64:downloadName': 'linux-arm64:sha256',
     'darwin:downloadName': 'darwin:sha256',
+    'darwin-arm64:downloadName': 'darwin-arm64:sha256',
     'win32:downloadName': 'win32:sha256',
   });
 
@@ -202,7 +210,10 @@ it('downloads node builds for each platform', async () => {
       ],
     ]
   `);
-  expect(testWriter.messages).toMatchInlineSnapshot(`Array []`);
+  /* ToDo [NODE14]: Replace when Node.js 14 support is removed
+   * expect(testWriter.messages).toMatchInlineSnapshot(`Array []`);
+   */
+  expect(testWriter.messages).toMatchSnapshot();
 });
 
 it('rejects if any download fails', async () => {
@@ -211,5 +222,8 @@ it('rejects if any download fails', async () => {
   await expect(DownloadNodeBuilds.run(config, log, [])).rejects.toMatchInlineSnapshot(
     `[Error: Download failed for reasons]`
   );
-  expect(testWriter.messages).toMatchInlineSnapshot(`Array []`);
+  /* ToDo [NODE14]: Replace when Node.js 14 support is removed
+   * expect(testWriter.messages).toMatchInlineSnapshot(`Array []`);
+   */
+  expect(testWriter.messages).toMatchSnapshot();
 });

src/dev/build/tasks/nodejs/download_node_builds_task.ts

@@ -60,6 +60,10 @@ export const DownloadNodeBuilds: GlobalTask = {
       }),
       // ToDo [NODE14]: Remove this Node.js 14 fallback download
       ...config.getTargetPlatforms().map(async (platform) => {
+        if (platform.getBuildName() === 'darwin-arm64') {
+          log.warning(`There are no fallback Node.js versions released for darwin-arm64.`);
+          return;
+        }
         const { url, downloadPath, downloadName } = await getNodeVersionDownloadInfo(
           NODE14_FALLBACK_VERSION,
           platform.getNodeArch(),

src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts

@@ -58,6 +58,7 @@ async function setup() {
       linux: false,
       linuxArm: false,
       darwin: false,
+      darwinArm: false,
       windows: false,
     },
   });

src/dev/build/tasks/nodejs/extract_node_builds_task.ts

@@ -38,7 +38,7 @@ import {
 export const ExtractNodeBuilds: GlobalTask = {
   global: true,
   description: 'Extracting node.js builds for all platforms',
-  async run(config) {
+  async run(config, log) {
     await Promise.all([
       ...config.getTargetPlatforms().map(async (platform) => {
         const { downloadPath, extractDir } = await getNodeDownloadInfo(config, platform);
@@ -50,6 +50,10 @@ export const ExtractNodeBuilds: GlobalTask = {
       }),
       // ToDo [NODE14]: Remove this Node.js 14 fallback download
       ...config.getTargetPlatforms().map(async (platform) => {
+        if (platform.getBuildName() === 'darwin-arm64') {
+          log.warning(`There are no fallback Node.js versions released for darwin-arm64.`);
+          return;
+        }
         const { downloadPath, extractDir } = await getNodeVersionDownloadInfo(
           NODE14_FALLBACK_VERSION,
           platform.getNodeArch(),