Skip to content

Commit

Permalink
feat(cli): add frontend dynamic plugins base build config (#747)
Browse files Browse the repository at this point in the history
* feat(cli): build dynamic frontend plugins

chore(ci): bump test node version (#831)

feat(cli): add frontend dynamic plugins base build config

fix(cli): resolve post dynamic build stats errors

fix(cli): remove eager consumption settings from shared modules

Eager packages will always end up within the container entry JS files
which significantly increases the network bandwidth even if packages the
shared packages have been already initialized in the shared scope via
different container.

fix(cli): remove the requirement for root tsconfig.json

chore(dynamic-plugins): add sample dynamic plugin package

chore(app-next): add sample app with dynamic plugins

fix(cli): align versions of @backstage/backend-common package

feat(app-next): dynamically load entire backstage plugins

chore(app-next): add readme

chore(app-next): fix eslint errors

fix(cli): do not treat warnings as errors on CI

chore(dependencies): align dependency versions

* chore(cli): cleanup and enable quay dynamic build

Signed-off-by: Tomas Coufal <[email protected]>

---------

Signed-off-by: Tomas Coufal <[email protected]>
Co-authored-by: Tomas Coufal <[email protected]>
  • Loading branch information
Hyperkid123 and tumido authored Oct 23, 2023
1 parent 04b2d07 commit 91e06da
Show file tree
Hide file tree
Showing 18 changed files with 717 additions and 161 deletions.
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ node_modules/
# Build output
dist
dist-types
dist-scalprum

# Temporary change files created by Vim
*.swp
Expand All @@ -49,3 +50,6 @@ site

# turbo
.turbo

# build cache
.webpack-cache
6 changes: 3 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@
"preinstall": "npx only-allow yarn",
"start": "turbo run start",
"start:backstage": "turbo run start --filter=...{./packages/app} --filter=...{./packages/backend}",
"start:plugins": "turbo run start --filter=...{./plugins/*}",
"start:plugins": "turbo run start --filter=@janus-idp/*",
"build": "turbo run build",
"build:backstage": "turbo run build --filter=...{./packages/*}",
"build:plugins": "turbo run build --filter=...{./plugins/*}",
"build:backstage": "turbo run build --filter=...{./packages/app} --filter=...{./packages/backend}",
"build:plugins": "turbo run build --filter=@janus-idp/*",
"tsc": "turbo run tsc",
"clean": "turbo run clean",
"test": "turbo run test",
Expand Down
6 changes: 6 additions & 0 deletions packages/cli/bin/janus-cli
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,12 @@ const isLocal = require('fs').existsSync(path.resolve(__dirname, '../src'));
if (!isLocal || process.env.BACKSTAGE_E2E_CLI_TEST) {
require('..');
} else {
/**
* TODO: Figure out of we need to find `project` path leading to a local plugin
* tsconfig.json.
* This will become relevant once we start migration plugins to the Janus cli that have
* different tsconfig.json specifications.
* */
require('ts-node').register({
transpileOnly: true,
/* eslint-disable-next-line no-restricted-syntax */
Expand Down
5 changes: 3 additions & 2 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
"@backstage/integration": "^1.7.1",
"@backstage/release-manifests": "^0.0.10",
"@backstage/types": "^1.1.1",
"@openshift/dynamic-plugin-sdk-webpack": "^3.0.0",
"@esbuild-kit/cjs-loader": "^2.4.4",
"@esbuild-kit/esm-loader": "^2.6.5",
"@manypkg/get-packages": "^1.1.3",
Expand Down Expand Up @@ -119,7 +120,7 @@
"react-refresh": "^0.14.0",
"recursive-readdir": "^2.2.2",
"replace-in-file": "^6.0.0",
"rollup": "^2.60.2",
"rollup": "^2.78.0",
"rollup-plugin-dts": "^4.0.1",
"rollup-plugin-esbuild": "^4.7.2",
"rollup-plugin-postcss": "^4.0.0",
Expand All @@ -137,7 +138,7 @@
"webpack-node-externals": "^3.0.0",
"yaml": "^2.3.3",
"yml-loader": "^2.1.0",
"yn": "^5.0.0",
"yn": "^4.0.0",
"zod": "^3.22.4"
},
"devDependencies": {
Expand Down
5 changes: 4 additions & 1 deletion packages/cli/src/commands/build/buildFrontend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
* limitations under the License.
*/

import { PluginBuildMetadata } from '@openshift/dynamic-plugin-sdk-webpack';
import fs from 'fs-extra';

import { resolve as resolvePath } from 'path';
Expand All @@ -26,16 +27,18 @@ interface BuildAppOptions {
targetDir: string;
writeStats: boolean;
configPaths: string[];
pluginMetadata?: PluginBuildMetadata;
}

export async function buildFrontend(options: BuildAppOptions) {
const { targetDir, writeStats, configPaths } = options;
const { targetDir, writeStats, configPaths, pluginMetadata } = options;
const { name } = await fs.readJson(resolvePath(targetDir, 'package.json'));
await buildBundle({
targetDir,
entry: 'src/index',
parallelism: getEnvironmentParallelism(),
statsJsonEnabled: writeStats,
pluginMetadata,
...(await loadCliConfig({
args: configPaths,
fromPackage: name,
Expand Down
26 changes: 25 additions & 1 deletion packages/cli/src/commands/export-dynamic-plugin/frontend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,34 @@
import { PackageRoleInfo } from '@backstage/cli-node';

import { OptionValues } from 'commander';
import fs from 'fs-extra';

import { buildScalprumPlugin } from '../../lib/builder/buildScalprumPlugin';
import { paths } from '../../lib/paths';

export async function frontend(
_: PackageRoleInfo,
__: OptionValues,
): Promise<void> {
throw new Error('frontend not yet implemented');
const { name, version, scalprum } = await fs.readJson(
paths.resolveTarget('package.json'),
);
if (scalprum === undefined) {
throw new Error(
`Package doesn't seem to support dynamic loading. It should have a 'scalprum' key in 'package.json' containing the dynamic loading entrypoints.`,
);
}

await fs.remove(paths.resolveTarget('dist-scalprum'));

await buildScalprumPlugin({
writeStats: false,
configPaths: [],
targetDir: paths.targetDir,
pluginMetadata: {
...scalprum,
version,
},
fromPackage: name,
});
}
27 changes: 27 additions & 0 deletions packages/cli/src/lib/builder/buildScalprumPlugin.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import { PluginBuildMetadata } from '@openshift/dynamic-plugin-sdk-webpack';

import { buildScalprumBundle } from '../bundler/bundlePlugin';
import { loadCliConfig } from '../config';
import { getEnvironmentParallelism } from '../parallel';

interface BuildScalprumPluginOptions {
targetDir: string;
writeStats: boolean;
configPaths: string[];
pluginMetadata: PluginBuildMetadata;
fromPackage: string;
}

export async function buildScalprumPlugin(options: BuildScalprumPluginOptions) {
const { targetDir, pluginMetadata, fromPackage } = options;
await buildScalprumBundle({
targetDir,
entry: 'src/index',
parallelism: getEnvironmentParallelism(),
pluginMetadata,
...(await loadCliConfig({
args: [],
fromPackage,
})),
});
}
123 changes: 123 additions & 0 deletions packages/cli/src/lib/bundler/bundlePlugin.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
import {
measureFileSizesBeforeBuild,
printFileSizesAfterBuild,
} from 'react-dev-utils/FileSizeReporter';
import formatWebpackMessages from 'react-dev-utils/formatWebpackMessages';

import chalk from 'chalk';
import fs from 'fs-extra';
import webpack from 'webpack';
import yn from 'yn';

import { resolveBaseUrl } from './config';
import { BundlingPathsOptions, resolveBundlingPaths } from './paths';
import { createScalprumConfig } from './scalprumConfig';
import { DynamicPluginOptions } from './types';

// TODO(Rugvip): Limits from CRA, we might want to tweak these though.
const WARN_AFTER_BUNDLE_GZIP_SIZE = 512 * 1024;
const WARN_AFTER_CHUNK_GZIP_SIZE = 1024 * 1024;

function applyContextToError(error: string, moduleName: string): string {
return `Failed to compile '${moduleName}':\n ${error}`;
}

export async function buildScalprumBundle(
options: BundlingPathsOptions & DynamicPluginOptions,
) {
const paths = resolveBundlingPaths(options);
const config = await createScalprumConfig(paths, {
...options,
checksEnabled: false,
isDev: false,
baseUrl: resolveBaseUrl(options.frontendConfig),
});

const isCi = yn(process.env.CI, { default: false });

const previousFileSizes = await measureFileSizesBeforeBuild(
paths.targetScalprumDist,
);
await fs.emptyDir(paths.targetScalprumDist);

if (paths.targetPublic) {
await fs.copy(paths.targetPublic, paths.targetDist, {
dereference: true,
filter: file => file !== paths.targetHtml,
});
}

const { stats } = await build(config, isCi);

if (!stats) {
throw new Error('No stats returned');
}

printFileSizesAfterBuild(
stats,
previousFileSizes,
paths.targetScalprumDist,
WARN_AFTER_BUNDLE_GZIP_SIZE,
WARN_AFTER_CHUNK_GZIP_SIZE,
);
}

async function build(config: webpack.Configuration, isCi: boolean) {
const stats = await new Promise<webpack.Stats | undefined>(
(resolve, reject) => {
webpack(config, (err, buildStats) => {
if (err) {
if (err.message) {
const { errors } = formatWebpackMessages({
errors: [err.message],
warnings: new Array<string>(),
_showErrors: true,
_showWarnings: true,
});

throw new Error(errors[0]);
} else {
reject(err);
}
} else {
resolve(buildStats);
}
});
},
);

if (!stats) {
throw new Error('Failed to compile: No stats provided');
}

const serializedStats = stats.toJson({
all: false,
warnings: true,
errors: true,
});
const { errors, warnings } = formatWebpackMessages({
errors: serializedStats.errors,
warnings: serializedStats.warnings,
});

if (errors.length) {
// Only keep the first error. Others are often indicative
// of the same problem, but confuse the reader with noise.
const errorWithContext = applyContextToError(
errors[0],
serializedStats.errors?.[0]?.moduleName ?? '',
);
throw new Error(errorWithContext);
}
if (isCi && warnings.length) {
const warningsWithContext = warnings.map((warning, i) => {
return applyContextToError(
warning,
serializedStats.warnings?.[i]?.moduleName ?? '',
);
});
console.log(chalk.yellow(warningsWithContext.join('\n\n')));
}

return { stats };
}
34 changes: 32 additions & 2 deletions packages/cli/src/lib/bundler/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,15 @@ import fs from 'fs-extra';
import HtmlWebpackPlugin from 'html-webpack-plugin';
import pickBy from 'lodash/pickBy';
import { RunScriptWebpackPlugin } from 'run-script-webpack-plugin';
import webpack, { ProvidePlugin } from 'webpack';
import webpack, { container, ProvidePlugin } from 'webpack';
import nodeExternals from 'webpack-node-externals';
import yn from 'yn';

import { posix as posixPath, resolve as resolvePath } from 'path';
import {
join as joinPath,
posix as posixPath,
resolve as resolvePath,
} from 'path';

import { paths as cliPaths } from '../../lib/paths';
import { version } from '../../lib/version';
Expand All @@ -40,9 +44,18 @@ import { runPlain } from '../run';
import { LinkedPackageResolvePlugin } from './LinkedPackageResolvePlugin';
import { optimization } from './optimization';
import { BundlingPaths } from './paths';
import { sharedModules } from './scalprumConfig';
import { transforms } from './transforms';
import { BackendBundlingOptions, BundlingOptions } from './types';

const { ModuleFederationPlugin } = container;

const scalprumPlugin = new ModuleFederationPlugin({
name: 'backstageHost',
filename: 'backstageHost.[fullhash].js',
shared: [sharedModules],
});

const BUILD_CACHE_ENV_VAR = 'BACKSTAGE_CLI_EXPERIMENTAL_BUILD_CACHE';

export function resolveBaseUrl(config: Config): URL {
Expand Down Expand Up @@ -142,6 +155,8 @@ export async function createConfig(
}),
);

plugins.push(scalprumPlugin);

// These files are required by the transpiled code when using React Refresh.
// They need to be excluded to the module scope plugin which ensures that files
// that exist in the package are required.
Expand All @@ -156,6 +171,11 @@ export async function createConfig(
const withCache = yn(process.env[BUILD_CACHE_ENV_VAR], { default: false });

return {
cache: {
type: 'filesystem',
allowCollectingMemory: true,
cacheDirectory: joinPath(process.cwd(), '.webpack-cache'),
},
mode: isDev ? 'development' : 'production',
profile: false,
optimization: optimization(options),
Expand All @@ -167,6 +187,16 @@ export async function createConfig(
context: paths.targetPath,
entry: [...(options.additionalEntryPoints ?? []), paths.targetEntry],
resolve: {
alias: {
'@backstage/frontend-app-api/src': joinPath(
process.cwd(),
'src',
'overrides',
'@backstage',
'frontend-app-api',
'src',
),
},
extensions: ['.ts', '.tsx', '.mjs', '.js', '.jsx', '.json', '.wasm'],
mainFields: ['browser', 'module', 'main'],
fallback: {
Expand Down
1 change: 1 addition & 0 deletions packages/cli/src/lib/bundler/paths.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ export function resolveBundlingPaths(options: BundlingPathsOptions) {
targetPath: resolvePath(targetDir, '.'),
targetRunFile: runFileExists ? targetRunFile : undefined,
targetDist: resolvePath(targetDir, 'dist'),
targetScalprumDist: resolvePath(targetDir, 'dist-scalprum'),
targetAssets: resolvePath(targetDir, 'assets'),
targetSrc: resolvePath(targetDir, 'src'),
targetDev: resolvePath(targetDir, 'dev'),
Expand Down
Loading

0 comments on commit 91e06da

Please sign in to comment.