feat(module-federation): use proxy servers to proxy to single file se…

…rver for static remotes (#26782)

<!-- Please make sure you have read the submission guidelines before
posting an PR -->
<!--
https://github.com/nrwl/nx/blob/master/CONTRIBUTING.md#-submitting-a-pr
-->

<!-- Please make sure that your commit message follows our format -->
<!-- Example: `fix(nx): must begin with lowercase` -->

<!-- If this is a particularly complex change or feature addition, you
can request a dedicated Nx release for this pull request branch. Mention
someone from the Nx team or the `@nrwl/nx-pipelines-reviewers` and they
will confirm if the PR warrants its own release for testing purposes,
and generate it for you if appropriate. -->

## Current Behavior
<!-- This is the behavior we have today -->
Remotes that are served for a host are usually served from a single file
server running on a single port.
We perform some mapping logic during the build of the host application
to update the locations the remotes can be found at to point to the
single file server.

This works, but it's also wrong, as it breaks the flow that users
expect.
It also breaks dynamic remotes.


## Expected Behavior
<!-- This is the behavior we should expect with the changes in this PR
-->
Continue to serve the remotes from a single file server, as this helps
reduce the amount of resources used on developers machines.
Use express to create proxy servers that will proxy requests from the
original remote location to the single file server.

This allows applications to continue to work without us having to
interfere and map any remote locations.
It also solves the issue with dynamic remotes.


## Related Issue(s)
<!-- Please link the issue being fixed so it gets closed when this is
merged. -->

Fixes #26318

e2e/react/src/react-module-federation.test.ts

@@ -20,13 +20,13 @@ import { join } from 'path';
 import { createTreeWithEmptyWorkspace } from 'nx/src/devkit-testing-exports';
 
 describe('React Module Federation', () => {
-  beforeAll(() => {
-    newProject({ packages: ['@nx/react'] });
-  });
+  describe('Default Configuration', () => {
+    beforeAll(() => {
+      newProject({ packages: ['@nx/react'] });
+    });
 
-  afterAll(() => cleanupProject());
+    afterAll(() => cleanupProject());
 
-  describe('Default Configuration', () => {
     it.each`
       js
       ${false}
@@ -39,9 +39,6 @@ describe('React Module Federation', () => {
         const remote2 = uniq('remote2');
         const remote3 = uniq('remote3');
 
-        // Since we are using a single-file server for the remotes
-        const defaultRemotePort = 4201;
-
         runCLI(
           `generate @nx/react:host ${shell} --remotes=${remote1},${remote2},${remote3} --e2eTestRunner=cypress --style=css --no-interactive --skipFormat --js=${js}`
         );
@@ -65,54 +62,6 @@ describe('React Module Federation', () => {
           ),
         });
 
-        if (js) {
-          updateFile(
-            `apps/${shell}/webpack.config.js`,
-            stripIndents`
-        const { composePlugins, withNx } = require('@nx/webpack');
-        const { withReact } = require('@nx/react');
-        const { withModuleFederation } = require('@nx/react/module-federation');
-        
-        const baseConfig = require('./module-federation.config');
-        
-        const config = {
-          ...baseConfig,
-              remotes: [
-                '${remote1}',
-                ['${remote2}', 'http://localhost:${defaultRemotePort}/${remote2}/remoteEntry.js'],
-                ['${remote3}', 'http://localhost:${defaultRemotePort}/${remote3}/remoteEntry.js'],
-              ],
-        };
-
-        // Nx plugins for webpack to build config object from Nx options and context.
-        module.exports = composePlugins(withNx(), withReact(), withModuleFederation(config));
-      `
-          );
-        } else {
-          updateFile(
-            `apps/${shell}/webpack.config.ts`,
-            stripIndents`
-        import { composePlugins, withNx } from '@nx/webpack';
-        import { withReact } from '@nx/react';
-        import { withModuleFederation } from '@nx/react/module-federation';
-        
-        import baseConfig from './module-federation.config';
-        
-        const config = {
-          ...baseConfig,
-              remotes: [
-                '${remote1}',
-                ['${remote2}', 'http://localhost:${defaultRemotePort}/${remote2}/remoteEntry.js'],
-                ['${remote3}', 'http://localhost:${defaultRemotePort}/${remote3}/remoteEntry.js'],
-              ],
-        };
-
-        // Nx plugins for webpack to build config object from Nx options and context.
-        export default composePlugins(withNx(), withReact(), withModuleFederation(config));
-      `
-          );
-        }
-
         updateFile(
           `apps/${shell}-e2e/src/integration/app.spec.${js ? 'js' : 'ts'}`,
           stripIndents`
@@ -153,23 +102,15 @@ describe('React Module Federation', () => {
           output.includes(`http://localhost:${readPort(shell)}`)
         );
 
-        await killProcessAndPorts(
-          serveResult.pid,
-          readPort(shell),
-          defaultRemotePort
-        );
+        await killProcessAndPorts(serveResult.pid, readPort(shell));
 
         if (runE2ETests()) {
           const e2eResultsSwc = await runCommandUntil(
             `e2e ${shell}-e2e --no-watch --verbose`,
             (output) => output.includes('All specs passed!')
           );
 
-          await killProcessAndPorts(
-            e2eResultsSwc.pid,
-            readPort(shell),
-            defaultRemotePort
-          );
+          await killProcessAndPorts(e2eResultsSwc.pid, readPort(shell));
 
           const e2eResultsTsNode = await runCommandUntil(
             `e2e ${shell}-e2e --no-watch --verbose`,
@@ -179,11 +120,7 @@ describe('React Module Federation', () => {
               env: { NX_PREFER_TS_NODE: 'true' },
             }
           );
-          await killProcessAndPorts(
-            e2eResultsTsNode.pid,
-            readPort(shell),
-            defaultRemotePort
-          );
+          await killProcessAndPorts(e2eResultsTsNode.pid, readPort(shell));
         }
       },
       500_000
@@ -924,14 +861,20 @@ describe('React Module Federation', () => {
     });
 
     afterAll(() => cleanupProject());
-    it('should load remote dynamic module', async () => {
+    it('ttt should load remote dynamic module', async () => {
       const shell = uniq('shell');
       const remote = uniq('remote');
+      const remotePort = 4205;
 
       runCLI(
         `generate @nx/react:host ${shell} --remotes=${remote} --e2eTestRunner=cypress --dynamic=true --project-name-and-root-format=as-provided --no-interactive --skipFormat`
       );
 
+      updateJson(`${remote}/project.json`, (project) => {
+        project.targets.serve.options.port = remotePort;
+        return project;
+      });
+
       // Webpack prod config should not exists when loading dynamic modules
       expect(
         fileExists(`${tmpProjPath()}/${shell}/webpack.config.prod.ts`)
@@ -942,12 +885,20 @@ describe('React Module Federation', () => {
         )
       ).toBeTruthy();
 
+      updateJson(
+        `${shell}/src/assets/module-federation.manifest.json`,
+        (json) => {
+          return {
+            [remote]: `http://localhost:${remotePort}`,
+          };
+        }
+      );
+
       const manifest = readJson(
         `${shell}/src/assets/module-federation.manifest.json`
       );
-
       expect(manifest[remote]).toBeDefined();
-      expect(manifest[remote]).toEqual('http://localhost:4201');
+      expect(manifest[remote]).toEqual('http://localhost:4205');
 
       // update e2e
       updateFile(
@@ -981,7 +932,6 @@ describe('React Module Federation', () => {
       expect(remoteOutput).toContain('Successfully ran target build');
 
       const shellPort = readPort(shell);
-      const remotePort = readPort(remote);
 
       if (runE2ETests()) {
         // Serve Remote since it is dynamic and won't be started with the host

package.json

@@ -180,7 +180,7 @@
     "eslint-plugin-react": "7.32.2",
     "eslint-plugin-react-hooks": "4.6.0",
     "eslint-plugin-storybook": "^0.6.12",
-    "express": "^4.18.1",
+    "express": "^4.19.2",
     "fast-xml-parser": "^4.2.7",
     "figures": "3.2.0",
     "file-type": "^16.2.0",
@@ -192,6 +192,7 @@
     "gpt3-tokenizer": "^1.1.5",
     "handlebars": "4.7.7",
     "html-webpack-plugin": "5.5.0",
+    "http-proxy-middleware": "^3.0.0",
     "http-server": "14.1.0",
     "husky": "^8.0.1",
     "identity-obj-proxy": "3.0.0",

packages/angular/ng-package.json

@@ -21,6 +21,9 @@
     "ts-node",
     "tsconfig-paths",
     "semver",
+    "webpack",
+    "express",
+    "http-proxy-middleware",
     "http-server",
     "magic-string",
     "enquirer",

packages/angular/src/executors/module-federation-dev-server/lib/build-static-remotes.ts

@@ -1,10 +1,10 @@
 import { type Schema } from '../schema';
 import { type ExecutorContext, logger } from '@nx/devkit';
-import type { StaticRemotesConfig } from './parse-static-remotes-config';
 import { workspaceDataDirectory } from 'nx/src/utils/cache-directory';
 import { fork } from 'node:child_process';
 import { join } from 'node:path';
 import { createWriteStream } from 'node:fs';
+import type { StaticRemotesConfig } from '@nx/webpack/src/utils/module-federation/parse-static-remotes-config';
 
 export async function buildStaticRemotes(
   staticRemotesConfig: StaticRemotesConfig,
@@ -23,9 +23,6 @@ export async function buildStaticRemotes(
       staticRemotesConfig.config[app].urlSegment
     }`;
   }
-  process.env.NX_MF_DEV_SERVER_STATIC_REMOTES = JSON.stringify(
-    mappedLocationOfRemotes
-  );
 
   await new Promise<void>((res) => {
     logger.info(
@@ -81,4 +78,6 @@ export async function buildStaticRemotes(
     process.on('SIGTERM', () => staticProcess.kill('SIGTERM'));
     process.on('exit', () => staticProcess.kill('SIGTERM'));
   });
+
+  return mappedLocationOfRemotes;
 }

packages/angular/src/executors/module-federation-dev-server/lib/index.ts

@@ -1,5 +1,4 @@
 export * from './build-static-remotes';
 export * from './normalize-options';
-export * from './parse-static-remotes-config';
 export * from './start-dev-remotes';
 export * from './start-static-remotes-file-server';

packages/angular/src/executors/module-federation-dev-server/lib/start-static-remotes-file-server.ts

@@ -3,13 +3,19 @@ import { type Schema } from '../schema';
 import fileServerExecutor from '@nx/web/src/executors/file-server/file-server.impl';
 import { join } from 'path';
 import { cpSync } from 'fs';
-import type { StaticRemotesConfig } from './parse-static-remotes-config';
+import type { StaticRemotesConfig } from '@nx/webpack/src/utils/module-federation/parse-static-remotes-config';
 
 export function startStaticRemotesFileServer(
   staticRemotesConfig: StaticRemotesConfig,
   context: ExecutorContext,
   options: Schema
 ) {
+  if (
+    !staticRemotesConfig.remotes ||
+    staticRemotesConfig.remotes.length === 0
+  ) {
+    return;
+  }
   let shouldMoveToCommonLocation = false;
   let commonOutputDirectory: string;
   for (const app of staticRemotesConfig.remotes) {

packages/angular/src/executors/module-federation-dev-server/module-federation-dev-server.impl.ts

@@ -7,7 +7,6 @@ import { type Schema } from './schema';
 import {
   buildStaticRemotes,
   normalizeOptions,
-  parseStaticRemotesConfig,
   startRemotes,
   startStaticRemotesFileServer,
 } from './lib';
@@ -31,6 +30,8 @@ import {
 } from '../../builders/utilities/module-federation';
 import { extname, join } from 'path';
 import { existsSync } from 'fs';
+import { startRemoteProxies } from '@nx/webpack/src/utils/module-federation/start-remote-proxies';
+import { parseStaticRemotesConfig } from '@nx/webpack/src/utils/module-federation/parse-static-remotes-config';
 
 export async function* moduleFederationDevServerExecutor(
   schema: Schema,
@@ -39,7 +40,6 @@ export async function* moduleFederationDevServerExecutor(
   // Force Node to resolve to look for the nx binary that is inside node_modules
   const nxBin = require.resolve('nx/bin/nx');
   const options = normalizeOptions(schema);
-  options.staticRemotesPort ??= options.port + 1;
 
   const { projects: workspaceProjects } =
     readProjectsConfigurationFromProjectGraph(context.projectGraph);
@@ -123,30 +123,23 @@ export async function* moduleFederationDevServerExecutor(
     pathToManifestFile
   );
 
+  options.staticRemotesPort ??= remotes.staticRemotePort;
+
   // Set NX_MF_DEV_REMOTES for the Nx Runtime Library Control Plugin
   process.env.NX_MF_DEV_REMOTES = JSON.stringify(
     remotes.devRemotes.map((r) => (typeof r === 'string' ? r : r.remoteName))
   );
 
-  if (remotes.devRemotes.length > 0 && !schema.staticRemotesPort) {
-    options.staticRemotesPort = options.devRemotes.reduce((portToUse, r) => {
-      const remoteName = typeof r === 'string' ? r : r.remoteName;
-      const remotePort =
-        context.projectGraph.nodes[remoteName].data.targets['serve'].options
-          .port;
-      if (remotePort >= portToUse) {
-        return remotePort + 1;
-      } else {
-        return portToUse;
-      }
-    }, options.staticRemotesPort);
-  }
-
   const staticRemotesConfig = parseStaticRemotesConfig(
-    remotes.staticRemotes,
+    [...remotes.staticRemotes, ...remotes.dynamicRemotes],
     context
   );
-  await buildStaticRemotes(staticRemotesConfig, nxBin, context, options);
+  const mappedLocationsOfStaticRemotes = await buildStaticRemotes(
+    staticRemotesConfig,
+    nxBin,
+    context,
+    options
+  );
 
   const devRemoteIters = await startRemotes(
     remotes.devRemotes,
@@ -156,18 +149,13 @@ export async function* moduleFederationDevServerExecutor(
     'serve'
   );
 
-  const dynamicRemoteIters = await startRemotes(
-    remotes.dynamicRemotes,
-    workspaceProjects,
-    options,
+  const staticRemotesIter = startStaticRemotesFileServer(
+    staticRemotesConfig,
     context,
-    'serve-static'
+    options
   );
 
-  const staticRemotesIter =
-    remotes.staticRemotes.length > 0
-      ? startStaticRemotesFileServer(staticRemotesConfig, context, options)
-      : undefined;
+  startRemoteProxies(staticRemotesConfig, mappedLocationsOfStaticRemotes);
 
   const removeBaseUrlEmission = (iter: AsyncIterable<unknown>) =>
     mapAsyncIterable(iter, (v) => ({
@@ -178,7 +166,6 @@ export async function* moduleFederationDevServerExecutor(
   return yield* combineAsyncIterables(
     removeBaseUrlEmission(currIter),
     ...devRemoteIters.map(removeBaseUrlEmission),
-    ...dynamicRemoteIters.map(removeBaseUrlEmission),
     ...(staticRemotesIter ? [removeBaseUrlEmission(staticRemotesIter)] : []),
     createAsyncIterable<{ success: true; baseUrl: string }>(
       async ({ next, done }) => {